def add_blob(body_dict): date_format = '%Y-%m-%d %H:%M:%S.%f' b0 = datetime.strptime(body_dict["upload_date"], date_format) b1 = datetime.strptime(body_dict["file_last_update"], date_format) b=db_session.query(Blob).get(body_dict["file_id"]) if b == None: print "Blob did not exist" b = Blob(b0, b1, b1) db_session.add(b) db_session.commit() b.id = body_dict["file_id"] b.upload_date = b0 b.last_change = b1 b.last_sync = b1 db_session.commit() print "add_blob" bs=db_session.query(Blob).order_by(Blob.id) for b_ in bs: print "blob: ",b_.id data = {"message_id":body_dict["message_id"], "type":body_dict["type"], "node_id":body_dict["node_id"], "node_ip":body_dict["node_ip"], "node_port":body_dict["node_port"], "file_id":body_dict["file_id"], "upload_date":body_dict["upload_date"], "file_last_update":body_dict["file_last_update"], "file_last_sync":body_dict["file_last_update"]} emit_manager(json.dumps(data))
def show_node(node_id): node=db_session.query(Node).get(node_id) if request.method == 'GET': return render_template('show_node.html', node=node) elif request.method == 'DELETE': #print "Delete node" #print "processid: %s" % node.process_id r = requests.get(r'http://jsonip.com') my_ip= r.json()['ip'] my_ip2 = "10.10.10.15" if node.ip==my_ip or node.ip==my_ip2: stop_docker_process(node.process_id) else: stop_docker_process_openstack(node.ip, node.process_id) db_session.delete(node) db_session.commit() nodes=db_session.query(Node).all() if len(nodes) == 0: blobs=db_session.query(Blob).order_by(Blob.id) for blob in blobs: db_session.delete(blob) db_session.commit() flash('All nodes gone and files removed') else: flash('Node is removed.') return redirect(url_for('manager')) elif request.method == 'POST': # This should totally never ever happen.. but it needs to support it, don't judge. flash('I think I broke something, call my mummy..') return render_template('show_nodes.html', node=node)
def add_blob(body_dict): date_format = '%Y-%m-%d %H:%M:%S.%f' b0 = datetime.strptime(body_dict["upload_date"], date_format) b1 = datetime.strptime(body_dict["file_last_update"], date_format) b = db_session.query(Blob).get(body_dict["file_id"]) if b == None: print "Blob did not exist" b = Blob(b0, b1, b1) db_session.add(b) db_session.commit() b.id = body_dict["file_id"] b.upload_date = b0 b.last_change = b1 b.last_sync = b1 db_session.commit() print "add_blob" bs = db_session.query(Blob).order_by(Blob.id) for b_ in bs: print "blob: ", b_.id data = { "message_id": body_dict["message_id"], "type": body_dict["type"], "node_id": body_dict["node_id"], "node_ip": body_dict["node_ip"], "node_port": body_dict["node_port"], "file_id": body_dict["file_id"], "upload_date": body_dict["upload_date"], "file_last_update": body_dict["file_last_update"], "file_last_sync": body_dict["file_last_update"] } emit_manager(json.dumps(data))
def show_vm(vm_id): vm=db_session.query(VM).get(vm_id) if request.method == 'GET': return redirect(url_for('machines')) #return render_template('show_node.html', node=node) elif request.method == 'DELETE': #send delete-thingy remove_instance(vm.instance,vm.ip) db_session.delete(vm) db_session.commit() flash('Instance is removed.') return redirect(url_for('machines')) elif request.method == 'POST': # This should totally never ever happen.. but it needs to support it, don't judge. flash('I think I broke something, call my mummy..') return render_template('show_nodes.html', node=node)
def machines(): if request.method == 'GET': r = requests.get(r'http://jsonip.com') ip= r.json()['ip'] vms=db_session.query(VM).order_by(VM.id) return render_template('show_vms.html', vms=vms) elif request.method == 'POST': instance = (str(add_instance()).split(":"))[1] #instance = str(add_instance()) print "instance: ", instance ip = get_floating() assign_floating(instance, ip) vm = VM(ip, instance) db_session.add(vm) db_session.commit() flash('Instance created @',ip) return redirect(url_for('machines'))
def update_blob(body_dict): date_format = '%Y-%m-%d %H:%M:%S.%f' b0 = datetime.strptime(body_dict["upload_date"], date_format) b1 = datetime.strptime(body_dict["file_last_update"], date_format) b=db_session.query(Blob).get(body_dict["file_id"]) if b == None: b = Blob(b0, b1, b1) db_session.add(b) db_session.commit() b.id = body_dict["file_id"] b.upload_date = b0 b.last_change = b1 b.last_sync = b1 db_session.commit() else: date_format = '%Y-%m-%d %H:%M:%S.%f' b.upload_date = b0 b.last_change = b1 b.last_sync = b1 db_session.commit() data = {"message_id":body_dict["message_id"], "type":body_dict["type"], "node_id":body_dict["node_id"], "node_ip":body_dict["node_ip"], "node_port":body_dict["node_port"], "file_id":body_dict["file_id"], "upload_date":body_dict["upload_date"], "file_last_update":body_dict["file_last_update"], "file_last_sync":body_dict["file_last_update"]} emit_manager(json.dumps(data))
def update_blob(body_dict): date_format = '%Y-%m-%d %H:%M:%S.%f' b0 = datetime.strptime(body_dict["upload_date"], date_format) b1 = datetime.strptime(body_dict["file_last_update"], date_format) b = db_session.query(Blob).get(body_dict["file_id"]) if b == None: b = Blob(b0, b1, b1) db_session.add(b) db_session.commit() b.id = body_dict["file_id"] b.upload_date = b0 b.last_change = b1 b.last_sync = b1 db_session.commit() else: date_format = '%Y-%m-%d %H:%M:%S.%f' b.upload_date = b0 b.last_change = b1 b.last_sync = b1 db_session.commit() data = { "message_id": body_dict["message_id"], "type": body_dict["type"], "node_id": body_dict["node_id"], "node_ip": body_dict["node_ip"], "node_port": body_dict["node_port"], "file_id": body_dict["file_id"], "upload_date": body_dict["upload_date"], "file_last_update": body_dict["file_last_update"], "file_last_sync": body_dict["file_last_update"] } emit_manager(json.dumps(data))
def manager(): #data = {"message_id":(uuid.uuid4().int & (1<<63)-1), #"type":"STATUS", #"node_id":0} #rabbitmq.emit_manager(json.dumps(data)) if request.method == 'GET': r = requests.get(r'http://jsonip.com') ip= r.json()['ip'] #vms = [("publicvagrant",ip),("localvagrant","10.10.10.15")] vms = [ip, "10.10.10.15"] vms_db = db_session.query(VM).order_by(VM.id) for vm in vms_db: vms.append(vm.ip) #vms = vms+[(vm.instance, vm.ip)] print vms return render_template('show_nodes.html', nodes=db_session.query(Node).order_by(Node.id), blobs=db_session.query(Blob).order_by(Blob.id), vms=vms) elif request.method == 'POST': r = requests.get(r'http://jsonip.com') my_ip= r.json()['ip'] my_ip2 = "10.10.10.15" ip = request.form['vms'] port = request.form['port'] if is_valid_port(port): n = Node(ip,port) db_session.add(n) db_session.commit() if ip==my_ip or ip==my_ip2: n.process_id = run_server_on_container("websync", port, ip, n.id) print "Processid: ", n.process_id db_session.commit() flash('Node created.') else: n.process_id = run_server_on_openstack("websync", port, ip, n.id) print "Processid: ", n.process_id db_session.commit() flash('Node created.') return redirect(url_for('manager'))
def delete_blob(body_dict): b=db_session.query(Blob).get(body_dict["file_id"]) if not b == None: db_session.delete(b) db_session.commit()
def sync_blobs(body_dict): nodes=db_session.query(Node).order_by(Node.id) sync_node = db_session.query(Node).get(body_dict["node_id"]) source_node = sync_node for node in nodes: if not node.id == source_node.id: source_node = node break #if not source_node == sync_node: blobs_manager=db_session.query(Blob).order_by(Blob.id) for blob_manager in blobs_manager: blobs_node=body_dict["blobs"] blob_node=None for b in blobs_node: b_id, b_last_change, b_last_sync = b if b_id == blob_manager.id: blob_node=b break if not blob_node == None: blob_id, blob_last_change, blob_last_sync = blob_node if not (blob_last_sync == str(blob_manager.last_sync)): #Update has changed on cloud when node was offline if not (blob_last_change == blob_last_sync): #AMAGAAAD CONFLICT print "Manager and node has file, conflict detected" data = {"message_id":(uuid.uuid4().int & (1<<63)-1), "type":"CONFLICT", "node_id":sync_node.id, "file_id":blob_id} emit_manager(json.dumps(data)) print "Pushing manager-version to all nodes" data = {"message_id":(uuid.uuid4().int & (1<<63)-1), "type":"POST", "node_id":source_node.id, "node_ip":source_node.ip, "node_port":source_node.port, "file_id":blob_manager.id, "upload_date":str(blob_manager.upload_date), "file_last_update":str(blob_manager.last_change), "file_last_sync":str(blob_manager.last_sync)} emit_manager(json.dumps(data)) else: if not (blob_last_change== str(blob_manager.last_change)): #This node has changed file when offline. date_format = '%Y-%m-%d %H:%M:%S.%f' blob_manager.last_change = datetime.strptime(blob_last_change, date_format) blob_manager.last_sync = blob_manager.last_change db_session.commit() print "Node changed file but hadn't synced it. Manager tells everyone to pull from node" data = {"message_id":(uuid.uuid4().int & (1<<63)-1), "type":"PUT", "node_id":sync_node.id, "node_ip":sync_node.ip, "node_port":sync_node.port, "file_id":blob_manager.id, "upload_date":str(blob_manager.upload_date), "file_last_update":str(blob_manager.last_change), "file_last_sync":str(blob_manager.last_sync)} emit_manager(json.dumps(data)) else: print "Node does not have file, tells node to get it from a source-node" data = {"message_id":(uuid.uuid4().int & (1<<63)-1), "type":"POST", "node_id":source_node.id, "node_ip":source_node.ip, "node_port":source_node.port, "file_id":blob_manager.id, "upload_date":str(blob_manager.upload_date), "file_last_update":str(blob_manager.last_change), "file_last_sync":str(blob_manager.last_sync)} emit_manager(json.dumps(data)) #else: blobs_node=body_dict["blobs"] blobs_manager=db_session.query(Blob).order_by(Blob.id) for blob_node in blobs_node: blob_node_id, blob_node_last_change, blob_node_last_sync = blob_node blob_manager = None for b in blobs_manager: if b.id == blob_node_id: blob_manager=b break if not blob_manager == None: if not (blob_node_last_sync == str(blob_manager.last_sync)): pass # #Update has changed on cloud when node was offline # if not (blob_node_last_change == blob_node_last_sync): # #AMAGAAAD CONFLICT # print "Conflict found in second for-loop.. weird" # data = {"message_id":(uuid.uuid4().int & (1<<63)-1), # "type":"CONFLICT", # "node_id":sync_node.id, # "file_id":blob_node_id} # emit_manager(json.dumps(data)) # print "Second loop.. sending file to node" # data = {"message_id":(uuid.uuid4().int & (1<<63)-1), # "type":"POST", # "node_id":source_node.id, # "node_ip":source_node.ip, # "node_port":source_node.port, # "file_id":blob_manager.id, # "upload_date":str(blob_manager.upload_date), # "file_last_update":str(blob_manager.last_change), # "file_last_sync":str(blob_manager.last_sync)} # emit_manager(json.dumps(data)) else: if not (blob_node_last_change== str(blob_manager.last_change)): #This node has changed file when offline. date_format = '%Y-%m-%d %H:%M:%S.%f' blob_manager.last_change = datetime.strptime(blob_node_last_change, date_format) blob_manager.last_sync = blob_manager.last_change db_session.commit() print "Node has changed file. Tells all other nodes to pull from it" data = {"message_id":(uuid.uuid4().int & (1<<63)-1), "type":"PUT", "node_id":sync_node.id, "node_ip":sync_node.ip, "node_port":sync_node.port, "file_id":blob_manager.id, "upload_date":str(blob_manager.upload_date), "file_last_update":str(blob_manager.last_change), "file_last_sync":str(blob_manager.last_sync)} emit_manager(json.dumps(data)) else: print "Node has added file when offline, asks it to push it again" data = {"message_id":(uuid.uuid4().int & (1<<63)-1), "type":"REUPLOAD", "node_id":sync_node.id, "file_id":blob_node_id} emit_manager(json.dumps(data))
def delete_blob(body_dict): b = db_session.query(Blob).get(body_dict["file_id"]) if not b == None: db_session.delete(b) db_session.commit()
def sync_blobs(body_dict): nodes = db_session.query(Node).order_by(Node.id) sync_node = db_session.query(Node).get(body_dict["node_id"]) source_node = sync_node for node in nodes: if not node.id == source_node.id: source_node = node break #if not source_node == sync_node: blobs_manager = db_session.query(Blob).order_by(Blob.id) for blob_manager in blobs_manager: blobs_node = body_dict["blobs"] blob_node = None for b in blobs_node: b_id, b_last_change, b_last_sync = b if b_id == blob_manager.id: blob_node = b break if not blob_node == None: blob_id, blob_last_change, blob_last_sync = blob_node if not (blob_last_sync == str(blob_manager.last_sync)): #Update has changed on cloud when node was offline if not (blob_last_change == blob_last_sync): #AMAGAAAD CONFLICT print "Manager and node has file, conflict detected" data = { "message_id": (uuid.uuid4().int & (1 << 63) - 1), "type": "CONFLICT", "node_id": sync_node.id, "file_id": blob_id } emit_manager(json.dumps(data)) print "Pushing manager-version to all nodes" data = { "message_id": (uuid.uuid4().int & (1 << 63) - 1), "type": "POST", "node_id": source_node.id, "node_ip": source_node.ip, "node_port": source_node.port, "file_id": blob_manager.id, "upload_date": str(blob_manager.upload_date), "file_last_update": str(blob_manager.last_change), "file_last_sync": str(blob_manager.last_sync) } emit_manager(json.dumps(data)) else: if not (blob_last_change == str(blob_manager.last_change)): #This node has changed file when offline. date_format = '%Y-%m-%d %H:%M:%S.%f' blob_manager.last_change = datetime.strptime( blob_last_change, date_format) blob_manager.last_sync = blob_manager.last_change db_session.commit() print "Node changed file but hadn't synced it. Manager tells everyone to pull from node" data = { "message_id": (uuid.uuid4().int & (1 << 63) - 1), "type": "PUT", "node_id": sync_node.id, "node_ip": sync_node.ip, "node_port": sync_node.port, "file_id": blob_manager.id, "upload_date": str(blob_manager.upload_date), "file_last_update": str(blob_manager.last_change), "file_last_sync": str(blob_manager.last_sync) } emit_manager(json.dumps(data)) else: print "Node does not have file, tells node to get it from a source-node" data = { "message_id": (uuid.uuid4().int & (1 << 63) - 1), "type": "POST", "node_id": source_node.id, "node_ip": source_node.ip, "node_port": source_node.port, "file_id": blob_manager.id, "upload_date": str(blob_manager.upload_date), "file_last_update": str(blob_manager.last_change), "file_last_sync": str(blob_manager.last_sync) } emit_manager(json.dumps(data)) #else: blobs_node = body_dict["blobs"] blobs_manager = db_session.query(Blob).order_by(Blob.id) for blob_node in blobs_node: blob_node_id, blob_node_last_change, blob_node_last_sync = blob_node blob_manager = None for b in blobs_manager: if b.id == blob_node_id: blob_manager = b break if not blob_manager == None: if not (blob_node_last_sync == str(blob_manager.last_sync)): pass # #Update has changed on cloud when node was offline # if not (blob_node_last_change == blob_node_last_sync): # #AMAGAAAD CONFLICT # print "Conflict found in second for-loop.. weird" # data = {"message_id":(uuid.uuid4().int & (1<<63)-1), # "type":"CONFLICT", # "node_id":sync_node.id, # "file_id":blob_node_id} # emit_manager(json.dumps(data)) # print "Second loop.. sending file to node" # data = {"message_id":(uuid.uuid4().int & (1<<63)-1), # "type":"POST", # "node_id":source_node.id, # "node_ip":source_node.ip, # "node_port":source_node.port, # "file_id":blob_manager.id, # "upload_date":str(blob_manager.upload_date), # "file_last_update":str(blob_manager.last_change), # "file_last_sync":str(blob_manager.last_sync)} # emit_manager(json.dumps(data)) else: if not (blob_node_last_change == str( blob_manager.last_change)): #This node has changed file when offline. date_format = '%Y-%m-%d %H:%M:%S.%f' blob_manager.last_change = datetime.strptime( blob_node_last_change, date_format) blob_manager.last_sync = blob_manager.last_change db_session.commit() print "Node has changed file. Tells all other nodes to pull from it" data = { "message_id": (uuid.uuid4().int & (1 << 63) - 1), "type": "PUT", "node_id": sync_node.id, "node_ip": sync_node.ip, "node_port": sync_node.port, "file_id": blob_manager.id, "upload_date": str(blob_manager.upload_date), "file_last_update": str(blob_manager.last_change), "file_last_sync": str(blob_manager.last_sync) } emit_manager(json.dumps(data)) else: print "Node has added file when offline, asks it to push it again" data = { "message_id": (uuid.uuid4().int & (1 << 63) - 1), "type": "REUPLOAD", "node_id": sync_node.id, "file_id": blob_node_id } emit_manager(json.dumps(data))