def add_blob(body_dict):
date_format = '%Y-%m-%d %H:%M:%S.%f'
b0 = datetime.strptime(body_dict["upload_date"], date_format)
b1 = datetime.strptime(body_dict["file_last_update"], date_format)
b=db_session.query(Blob).get(body_dict["file_id"])
if b == None:
print "Blob did not exist"
b = Blob(b0, b1, b1)
db_session.add(b)
db_session.commit()
b.id = body_dict["file_id"]
b.upload_date = b0
b.last_change = b1
b.last_sync = b1
db_session.commit()
print "add_blob"
bs=db_session.query(Blob).order_by(Blob.id)
for b_ in bs:
print "blob: ",b_.id
data = {"message_id":body_dict["message_id"],
"type":body_dict["type"],
"node_id":body_dict["node_id"],
"node_ip":body_dict["node_ip"],
"node_port":body_dict["node_port"],
"file_id":body_dict["file_id"],
"upload_date":body_dict["upload_date"],
"file_last_update":body_dict["file_last_update"],
"file_last_sync":body_dict["file_last_update"]}
emit_manager(json.dumps(data))
def show_node(node_id):
node=db_session.query(Node).get(node_id)
if request.method == 'GET':
return render_template('show_node.html', node=node)
elif request.method == 'DELETE':
#print "Delete node"
#print "processid: %s" % node.process_id
r = requests.get(r'http://jsonip.com')
my_ip= r.json()['ip']
my_ip2 = "10.10.10.15"
if node.ip==my_ip or node.ip==my_ip2:
stop_docker_process(node.process_id)
else:
stop_docker_process_openstack(node.ip, node.process_id)
db_session.delete(node)
db_session.commit()
nodes=db_session.query(Node).all()
if len(nodes) == 0:
blobs=db_session.query(Blob).order_by(Blob.id)
for blob in blobs:
db_session.delete(blob)
db_session.commit()
flash('All nodes gone and files removed')
else:
flash('Node is removed.')
return redirect(url_for('manager'))
elif request.method == 'POST':
# This should totally never ever happen.. but it needs to support it, don't judge.
flash('I think I broke something, call my mummy..')
return render_template('show_nodes.html', node=node)
def add_blob(body_dict):
date_format = '%Y-%m-%d %H:%M:%S.%f'
b0 = datetime.strptime(body_dict["upload_date"], date_format)
b1 = datetime.strptime(body_dict["file_last_update"], date_format)
b = db_session.query(Blob).get(body_dict["file_id"])
if b == None:
print "Blob did not exist"
b = Blob(b0, b1, b1)
db_session.add(b)
db_session.commit()
b.id = body_dict["file_id"]
b.upload_date = b0
b.last_change = b1
b.last_sync = b1
db_session.commit()
print "add_blob"
bs = db_session.query(Blob).order_by(Blob.id)
for b_ in bs:
print "blob: ", b_.id
data = {
"message_id": body_dict["message_id"],
"type": body_dict["type"],
"node_id": body_dict["node_id"],
"node_ip": body_dict["node_ip"],
"node_port": body_dict["node_port"],
"file_id": body_dict["file_id"],
"upload_date": body_dict["upload_date"],
"file_last_update": body_dict["file_last_update"],
"file_last_sync": body_dict["file_last_update"]
}
emit_manager(json.dumps(data))
def update_blob(body_dict):
date_format = '%Y-%m-%d %H:%M:%S.%f'
b0 = datetime.strptime(body_dict["upload_date"], date_format)
b1 = datetime.strptime(body_dict["file_last_update"], date_format)
b=db_session.query(Blob).get(body_dict["file_id"])
if b == None:
b = Blob(b0, b1, b1)
db_session.add(b)
db_session.commit()
b.id = body_dict["file_id"]
b.upload_date = b0
b.last_change = b1
b.last_sync = b1
db_session.commit()
else:
date_format = '%Y-%m-%d %H:%M:%S.%f'
b.upload_date = b0
b.last_change = b1
b.last_sync = b1
db_session.commit()
data = {"message_id":body_dict["message_id"],
"type":body_dict["type"],
"node_id":body_dict["node_id"],
"node_ip":body_dict["node_ip"],
"node_port":body_dict["node_port"],
"file_id":body_dict["file_id"],
"upload_date":body_dict["upload_date"],
"file_last_update":body_dict["file_last_update"],
"file_last_sync":body_dict["file_last_update"]}
emit_manager(json.dumps(data))
def update_blob(body_dict):
date_format = '%Y-%m-%d %H:%M:%S.%f'
b0 = datetime.strptime(body_dict["upload_date"], date_format)
b1 = datetime.strptime(body_dict["file_last_update"], date_format)
b = db_session.query(Blob).get(body_dict["file_id"])
if b == None:
b = Blob(b0, b1, b1)
db_session.add(b)
db_session.commit()
b.id = body_dict["file_id"]
b.upload_date = b0
b.last_change = b1
b.last_sync = b1
db_session.commit()
else:
date_format = '%Y-%m-%d %H:%M:%S.%f'
b.upload_date = b0
b.last_change = b1
b.last_sync = b1
db_session.commit()
data = {
"message_id": body_dict["message_id"],
"type": body_dict["type"],
"node_id": body_dict["node_id"],
"node_ip": body_dict["node_ip"],
"node_port": body_dict["node_port"],
"file_id": body_dict["file_id"],
"upload_date": body_dict["upload_date"],
"file_last_update": body_dict["file_last_update"],
"file_last_sync": body_dict["file_last_update"]
}
emit_manager(json.dumps(data))
def manager():
#data = {"message_id":(uuid.uuid4().int & (1<<63)-1),
#"type":"STATUS",
#"node_id":0}
#rabbitmq.emit_manager(json.dumps(data))
if request.method == 'GET':
r = requests.get(r'http://jsonip.com')
ip= r.json()['ip']
#vms = [("publicvagrant",ip),("localvagrant","10.10.10.15")]
vms = [ip, "10.10.10.15"]
vms_db = db_session.query(VM).order_by(VM.id)
for vm in vms_db:
vms.append(vm.ip)
#vms = vms+[(vm.instance, vm.ip)]
print vms
return render_template('show_nodes.html', nodes=db_session.query(Node).order_by(Node.id), blobs=db_session.query(Blob).order_by(Blob.id), vms=vms)
elif request.method == 'POST':
r = requests.get(r'http://jsonip.com')
my_ip= r.json()['ip']
my_ip2 = "10.10.10.15"
ip = request.form['vms']
port = request.form['port']
if is_valid_port(port):
n = Node(ip,port)
db_session.add(n)
db_session.commit()
if ip==my_ip or ip==my_ip2:
n.process_id = run_server_on_container("websync", port, ip, n.id)
print "Processid: ", n.process_id
db_session.commit()
flash('Node created.')
else:
n.process_id = run_server_on_openstack("websync", port, ip, n.id)
print "Processid: ", n.process_id
db_session.commit()
flash('Node created.')
return redirect(url_for('manager'))
def show_vm(vm_id):
vm=db_session.query(VM).get(vm_id)
if request.method == 'GET':
return redirect(url_for('machines'))
#return render_template('show_node.html', node=node)
elif request.method == 'DELETE':
#send delete-thingy
remove_instance(vm.instance,vm.ip)
db_session.delete(vm)
db_session.commit()
flash('Instance is removed.')
return redirect(url_for('machines'))
elif request.method == 'POST':
# This should totally never ever happen.. but it needs to support it, don't judge.
flash('I think I broke something, call my mummy..')
return render_template('show_nodes.html', node=node)
def machines():
if request.method == 'GET':
r = requests.get(r'http://jsonip.com')
ip= r.json()['ip']
vms=db_session.query(VM).order_by(VM.id)
return render_template('show_vms.html', vms=vms)
elif request.method == 'POST':
instance = (str(add_instance()).split(":"))[1]
#instance = str(add_instance())
print "instance: ", instance
ip = get_floating()
assign_floating(instance, ip)
vm = VM(ip, instance)
db_session.add(vm)
db_session.commit()
flash('Instance created @',ip)
return redirect(url_for('machines'))
def delete_blob(body_dict):
b=db_session.query(Blob).get(body_dict["file_id"])
if not b == None:
db_session.delete(b)
db_session.commit()
def sync_blobs(body_dict):
nodes=db_session.query(Node).order_by(Node.id)
sync_node = db_session.query(Node).get(body_dict["node_id"])
source_node = sync_node
for node in nodes:
if not node.id == source_node.id:
source_node = node
break
#if not source_node == sync_node:
blobs_manager=db_session.query(Blob).order_by(Blob.id)
for blob_manager in blobs_manager:
blobs_node=body_dict["blobs"]
blob_node=None
for b in blobs_node:
b_id, b_last_change, b_last_sync = b
if b_id == blob_manager.id:
blob_node=b
break
if not blob_node == None:
blob_id, blob_last_change, blob_last_sync = blob_node
if not (blob_last_sync == str(blob_manager.last_sync)):
#Update has changed on cloud when node was offline
if not (blob_last_change == blob_last_sync):
#AMAGAAAD CONFLICT
print "Manager and node has file, conflict detected"
data = {"message_id":(uuid.uuid4().int & (1<<63)-1),
"type":"CONFLICT",
"node_id":sync_node.id,
"file_id":blob_id}
emit_manager(json.dumps(data))
print "Pushing manager-version to all nodes"
data = {"message_id":(uuid.uuid4().int & (1<<63)-1),
"type":"POST",
"node_id":source_node.id,
"node_ip":source_node.ip,
"node_port":source_node.port,
"file_id":blob_manager.id,
"upload_date":str(blob_manager.upload_date),
"file_last_update":str(blob_manager.last_change),
"file_last_sync":str(blob_manager.last_sync)}
emit_manager(json.dumps(data))
else:
if not (blob_last_change== str(blob_manager.last_change)):
#This node has changed file when offline.
date_format = '%Y-%m-%d %H:%M:%S.%f'
blob_manager.last_change = datetime.strptime(blob_last_change, date_format)
blob_manager.last_sync = blob_manager.last_change
db_session.commit()
print "Node changed file but hadn't synced it. Manager tells everyone to pull from node"
data = {"message_id":(uuid.uuid4().int & (1<<63)-1),
"type":"PUT",
"node_id":sync_node.id,
"node_ip":sync_node.ip,
"node_port":sync_node.port,
"file_id":blob_manager.id,
"upload_date":str(blob_manager.upload_date),
"file_last_update":str(blob_manager.last_change),
"file_last_sync":str(blob_manager.last_sync)}
emit_manager(json.dumps(data))
else:
print "Node does not have file, tells node to get it from a source-node"
data = {"message_id":(uuid.uuid4().int & (1<<63)-1),
"type":"POST",
"node_id":source_node.id,
"node_ip":source_node.ip,
"node_port":source_node.port,
"file_id":blob_manager.id,
"upload_date":str(blob_manager.upload_date),
"file_last_update":str(blob_manager.last_change),
"file_last_sync":str(blob_manager.last_sync)}
emit_manager(json.dumps(data))
#else:
blobs_node=body_dict["blobs"]
blobs_manager=db_session.query(Blob).order_by(Blob.id)
for blob_node in blobs_node:
blob_node_id, blob_node_last_change, blob_node_last_sync = blob_node
blob_manager = None
for b in blobs_manager:
if b.id == blob_node_id:
blob_manager=b
break
if not blob_manager == None:
if not (blob_node_last_sync == str(blob_manager.last_sync)):
pass
# #Update has changed on cloud when node was offline
# if not (blob_node_last_change == blob_node_last_sync):
# #AMAGAAAD CONFLICT
# print "Conflict found in second for-loop.. weird"
# data = {"message_id":(uuid.uuid4().int & (1<<63)-1),
# "type":"CONFLICT",
# "node_id":sync_node.id,
# "file_id":blob_node_id}
# emit_manager(json.dumps(data))
# print "Second loop.. sending file to node"
# data = {"message_id":(uuid.uuid4().int & (1<<63)-1),
# "type":"POST",
# "node_id":source_node.id,
# "node_ip":source_node.ip,
# "node_port":source_node.port,
# "file_id":blob_manager.id,
# "upload_date":str(blob_manager.upload_date),
# "file_last_update":str(blob_manager.last_change),
# "file_last_sync":str(blob_manager.last_sync)}
# emit_manager(json.dumps(data))
else:
if not (blob_node_last_change== str(blob_manager.last_change)):
#This node has changed file when offline.
date_format = '%Y-%m-%d %H:%M:%S.%f'
blob_manager.last_change = datetime.strptime(blob_node_last_change, date_format)
blob_manager.last_sync = blob_manager.last_change
db_session.commit()
print "Node has changed file. Tells all other nodes to pull from it"
data = {"message_id":(uuid.uuid4().int & (1<<63)-1),
"type":"PUT",
"node_id":sync_node.id,
"node_ip":sync_node.ip,
"node_port":sync_node.port,
"file_id":blob_manager.id,
"upload_date":str(blob_manager.upload_date),
"file_last_update":str(blob_manager.last_change),
"file_last_sync":str(blob_manager.last_sync)}
emit_manager(json.dumps(data))
else:
print "Node has added file when offline, asks it to push it again"
data = {"message_id":(uuid.uuid4().int & (1<<63)-1),
"type":"REUPLOAD",
"node_id":sync_node.id,
"file_id":blob_node_id}
emit_manager(json.dumps(data))
def delete_blob(body_dict):
b = db_session.query(Blob).get(body_dict["file_id"])
if not b == None:
db_session.delete(b)
db_session.commit()
def sync_blobs(body_dict):
nodes = db_session.query(Node).order_by(Node.id)
sync_node = db_session.query(Node).get(body_dict["node_id"])
source_node = sync_node
for node in nodes:
if not node.id == source_node.id:
source_node = node
break
#if not source_node == sync_node:
blobs_manager = db_session.query(Blob).order_by(Blob.id)
for blob_manager in blobs_manager:
blobs_node = body_dict["blobs"]
blob_node = None
for b in blobs_node:
b_id, b_last_change, b_last_sync = b
if b_id == blob_manager.id:
blob_node = b
break
if not blob_node == None:
blob_id, blob_last_change, blob_last_sync = blob_node
if not (blob_last_sync == str(blob_manager.last_sync)):
#Update has changed on cloud when node was offline
if not (blob_last_change == blob_last_sync):
#AMAGAAAD CONFLICT
print "Manager and node has file, conflict detected"
data = {
"message_id": (uuid.uuid4().int & (1 << 63) - 1),
"type": "CONFLICT",
"node_id": sync_node.id,
"file_id": blob_id
}
emit_manager(json.dumps(data))
print "Pushing manager-version to all nodes"
data = {
"message_id": (uuid.uuid4().int & (1 << 63) - 1),
"type": "POST",
"node_id": source_node.id,
"node_ip": source_node.ip,
"node_port": source_node.port,
"file_id": blob_manager.id,
"upload_date": str(blob_manager.upload_date),
"file_last_update": str(blob_manager.last_change),
"file_last_sync": str(blob_manager.last_sync)
}
emit_manager(json.dumps(data))
else:
if not (blob_last_change == str(blob_manager.last_change)):
#This node has changed file when offline.
date_format = '%Y-%m-%d %H:%M:%S.%f'
blob_manager.last_change = datetime.strptime(
blob_last_change, date_format)
blob_manager.last_sync = blob_manager.last_change
db_session.commit()
print "Node changed file but hadn't synced it. Manager tells everyone to pull from node"
data = {
"message_id": (uuid.uuid4().int & (1 << 63) - 1),
"type": "PUT",
"node_id": sync_node.id,
"node_ip": sync_node.ip,
"node_port": sync_node.port,
"file_id": blob_manager.id,
"upload_date": str(blob_manager.upload_date),
"file_last_update": str(blob_manager.last_change),
"file_last_sync": str(blob_manager.last_sync)
}
emit_manager(json.dumps(data))
else:
print "Node does not have file, tells node to get it from a source-node"
data = {
"message_id": (uuid.uuid4().int & (1 << 63) - 1),
"type": "POST",
"node_id": source_node.id,
"node_ip": source_node.ip,
"node_port": source_node.port,
"file_id": blob_manager.id,
"upload_date": str(blob_manager.upload_date),
"file_last_update": str(blob_manager.last_change),
"file_last_sync": str(blob_manager.last_sync)
}
emit_manager(json.dumps(data))
#else:
blobs_node = body_dict["blobs"]
blobs_manager = db_session.query(Blob).order_by(Blob.id)
for blob_node in blobs_node:
blob_node_id, blob_node_last_change, blob_node_last_sync = blob_node
blob_manager = None
for b in blobs_manager:
if b.id == blob_node_id:
blob_manager = b
break
if not blob_manager == None:
if not (blob_node_last_sync == str(blob_manager.last_sync)):
pass
# #Update has changed on cloud when node was offline
# if not (blob_node_last_change == blob_node_last_sync):
# #AMAGAAAD CONFLICT
# print "Conflict found in second for-loop.. weird"
# data = {"message_id":(uuid.uuid4().int & (1<<63)-1),
# "type":"CONFLICT",
# "node_id":sync_node.id,
# "file_id":blob_node_id}
# emit_manager(json.dumps(data))
# print "Second loop.. sending file to node"
# data = {"message_id":(uuid.uuid4().int & (1<<63)-1),
# "type":"POST",
# "node_id":source_node.id,
# "node_ip":source_node.ip,
# "node_port":source_node.port,
# "file_id":blob_manager.id,
# "upload_date":str(blob_manager.upload_date),
# "file_last_update":str(blob_manager.last_change),
# "file_last_sync":str(blob_manager.last_sync)}
# emit_manager(json.dumps(data))
else:
if not (blob_node_last_change == str(
blob_manager.last_change)):
#This node has changed file when offline.
date_format = '%Y-%m-%d %H:%M:%S.%f'
blob_manager.last_change = datetime.strptime(
blob_node_last_change, date_format)
blob_manager.last_sync = blob_manager.last_change
db_session.commit()
print "Node has changed file. Tells all other nodes to pull from it"
data = {
"message_id": (uuid.uuid4().int & (1 << 63) - 1),
"type": "PUT",
"node_id": sync_node.id,
"node_ip": sync_node.ip,
"node_port": sync_node.port,
"file_id": blob_manager.id,
"upload_date": str(blob_manager.upload_date),
"file_last_update": str(blob_manager.last_change),
"file_last_sync": str(blob_manager.last_sync)
}
emit_manager(json.dumps(data))
else:
print "Node has added file when offline, asks it to push it again"
data = {
"message_id": (uuid.uuid4().int & (1 << 63) - 1),
"type": "REUPLOAD",
"node_id": sync_node.id,
"file_id": blob_node_id
}
emit_manager(json.dumps(data))
