def __init__(self): self.port_ip = uwsgi.opt["http"].split(":") self.data_path =\ "/opt/gfdenis/metadata/" + self.port_ip[0] + "/" + self.port_ip[1] self.storage_nodes = [] self.proxy_nodes = [] kwargs = {'node_id': uwsgi.opt['node_id'], 'address': (uwsgi.opt["broker_ip"], uwsgi.opt['broker_port']), 'node_type': uwsgi.opt["node_type"]} self.group = GroupMessaging(kwargs=kwargs) self.group.start() kwargs = {'node_id': "PROXY_" + self.port_ip[0] + "_" + self.port_ip[1], 'address': (uwsgi.opt["broker_ip"], uwsgi.opt['broker_port']), 'node_type': "NAMENODESTORAGE"} self.group_proxy_storage = GroupMessaging(kwargs=kwargs) self.group_proxy_storage.start() self.http = uwsgi.opt['http'] self.node_id = uwsgi.opt['node_id'] logging.basicConfig(filename="/var/log/denisfs/proxy_" +\ self.node_id+ "_" + self.http + '.log',\ level=logging.DEBUG)
class Application(ApplicationBase): def __init__(self): self.port_ip = uwsgi.opt["http"].split(":") self.data_path = "/opt/gfdenis/data/" +\ self.port_ip[0] + "/" + self.port_ip[1] + "/" self.node_id = uwsgi.opt['node_id'] self.http = uwsgi.opt['http'] logging.basicConfig(filename="/var/log/denisfs/storage_" +\ self.node_id + "_" + self.http + '.log',\ level=logging.DEBUG) kwargs = {'node_id': uwsgi.opt['node_id'], 'address': (uwsgi.opt["broker_ip"], uwsgi.opt['broker_port']), 'node_type': uwsgi.opt["node_type"]} self.group = GroupMessaging(kwargs=kwargs) self.group.start() kwargs = {'node_id': "DATA_" + self.port_ip[0] + "_" + self.port_ip[1], 'address': (uwsgi.opt["broker_ip"], uwsgi.opt['broker_port']), 'node_type': "NAMENODESTORAGE"} self.group_proxy_storage = GroupMessaging(kwargs=kwargs) self.group_proxy_storage.start() def _unique_file_dir(self, filename): """ Generate a unique path for a filename """ return self.data_path + filename + "/" def _post_request(self, filename, host, headers=None): response = {"headers": [], "status": "200 OK", "body": ""} try: r = requests.post("http://"+host[0] + ":"+ host[1] + "/" + filename, headers=headers) response['status'] = \ utils.http_code[str(r.status_code)] response['body'] = str(r.content) except requests.ConnectionError: response['status'] = utils.http_code['500'] response['body'] = "No file created. Unknow error" return response def PUT(self, request): """ Create path if needed and write data to it """ logging.debug("storage.put") #request.showVariables() response = {"headers": [], "status": "201 OK", "body": ""} if not request.file_name: response['status'] = http_code['304'] return response replication = request.metadata.get("HTTP_REPLICATION", None) if replication == "1": logging.debug("REPLICATION") fm = FileManager(self.data_path + "/" + request.file_name,\ request.file_name) if not fm.write_file(request.body): response['status'] = http_code['500'] response['body'] = "Could not replicate" logging.debug("Could not replicate") return response client_id = request.metadata.get("HTTP_CLIENT_ID", "test:test") logging.debug("HTTP_CLIENT_ID: %s" % client_id) replicas_count = request.metadata.get("HTTP_REPLICAS_COUNT", 3) fm = FileManager(self.data_path + "/" + request.file_name\ , request.file_name) if not fm.write_file(request.body): response['status'] = http_code['500'] logging.debug("Starting to save data in disk") storage_replicas = make_tuple(request.metadata["HTTP_STORAGE_LIST"]) for replica in storage_replicas: if replica[0] == self.port_ip[0] and\ replica[1] == self.port_ip[1]: continue headers = {} headers['file_name'] = request.file_name headers['data_length'] = request.content_length headers['etag'] = request.file_name headers['replicas_count'] = replicas_count headers['client_id'] = "%s:%s" % (request.remote_addr, request.remote_port) headers["replication"] = "1" response = self._put_request(request.file_name, replica, data=request.body, headers=headers) logging.debug("response: %s" % response) if not "201" in response["status"]: response['status'] = http_code['500'] response['body'] = "Could reach proper number of replicas" return response logging.debug("Starting to send metadata confirmation for proxy") proxy_list = make_tuple(request.metadata["HTTP_PROXY_LIST"]) logging.debug(proxy_list) for proxy in proxy_list: logging.debug(proxy) headers = {} headers['data_length'] = request.content_length headers['etag'] = request.file_name headers['replicas_count'] = 3 headers['client_id'] = client_id response = self._post_request(request.file_name, proxy, headers=headers) logging.debug("response: %s" % response) if "201" in response["status"]: break return response def GET(self, request): """ Read data from file """ logging.debug("storage.get") #request.showVariables() response = {"headers": [], "status": "200 OK", "body": ""} if not request.file_name: response['status'] = http_code['304'] return response path = self._unique_file_dir(request.file_name) #try: #path = self._unique_file_dir(request.file_name) #logging.debug("path: %s" % path) #with open(path + "tombstone", "rb") as f: #response['status'] = http_code['404'] #response['body'] = "File in tombstone mode" #return response #except Exception as e: #print(e) try: with open(path + request.file_name, "rb") as f: response['body'] = f.read() except Exception as e: print(e) response['status'] = http_code['500'] #logging.debug("response: %s" % response["body"]) return response def DELETE(self, request): """ Create a tombstone file to register the file as deleted """ print(request.showVariables()) response = {"headers": [], "status": "200 OK", "body": ""} try: path = self._unique_file_dir(request.file_name) if not os.path.exists(path): os.makedirs(path) with open(path + "tombstone", "wb") as f: f.write(request.body) except Exception as e: print(e) response['status'] = http_code['500'] return response
class Application(ApplicationBase): # WSGI Server Application designed to act as a distributed metadata manager. # Consistency is guarantee by election a leader to play a central self # for mutual exclusion role. def __init__(self): self.port_ip = uwsgi.opt["http"].split(":") self.data_path =\ "/opt/gfdenis/metadata/" + self.port_ip[0] + "/" + self.port_ip[1] self.storage_nodes = [] self.proxy_nodes = [] kwargs = {'node_id': uwsgi.opt['node_id'], 'address': (uwsgi.opt["broker_ip"], uwsgi.opt['broker_port']), 'node_type': uwsgi.opt["node_type"]} self.group = GroupMessaging(kwargs=kwargs) self.group.start() kwargs = {'node_id': "PROXY_" + self.port_ip[0] + "_" + self.port_ip[1], 'address': (uwsgi.opt["broker_ip"], uwsgi.opt['broker_port']), 'node_type': "NAMENODESTORAGE"} self.group_proxy_storage = GroupMessaging(kwargs=kwargs) self.group_proxy_storage.start() self.http = uwsgi.opt['http'] self.node_id = uwsgi.opt['node_id'] logging.basicConfig(filename="/var/log/denisfs/proxy_" +\ self.node_id+ "_" + self.http + '.log',\ level=logging.DEBUG) def get_nodes(self, proxy_nodes, storage_nodes): for key, value in self.group_proxy_storage.workers.iteritems(): #print("key: %s" % key) #print("value: %s" % value) for node_type in value: temp_list = node_type.split("_") #print("temp_list: %s" % temp_list) if "DATA" in temp_list[1]: #print("value: %s" % value) ip = temp_list[2] port = temp_list[3] storage_nodes.append((ip, port)) if "PROXY" in temp_list[1]: #print("value: %s" % value) ip = temp_list[2] port = temp_list[3] proxy_nodes.append((ip, port)) # It validates the file and a possible storage; # It saves the file_data entry in the proxy datebase # # Return to the client the storage node chosen for that file def PUT(self, request): logging.debug("proxy.put") proxy_nodes = [] storage_nodes = [] self.get_nodes(proxy_nodes, storage_nodes) print("storage_node: %s" % storage_nodes) print("proxy_node: %s" % proxy_nodes) #request.showVariables() print("Am I leader? %s" % self.group.is_leader) response = {"headers": [], "status": "200 OK", "body": ""} replication = request.metadata.get("HTTP_REPLICATION", None) if replication == "1": logging.debug("REPLICATION") fm = FileManager(self.data_path + "/" + request.file_name,\ request.file_name) if not fm.write_file(request.body): response['status'] = http_code['500'] response['body'] = "Could not replicate" logging.debug("Could not replicate") return response if len(proxy_nodes) < 3: response['status'] = http_code['304'] response['body'] = "Deny of service due server is down" print("Deny of service due server is down") logging.debug("Deny of service due server is down") return response if not self.group.is_leader: response['status'] = http_code['304'] response['body'] = "No leader. Try to find the leader" logging.debug("I'm no leader. Try to find the leader") return response if not request.file_name: response['status'] = http_code['400'] response['body'] = "No valid file name found" logging.debug("No valid file name found") return response client_id = request.metadata.get("HTTP_CLIENT_ID", "test:test") #logging.debug("HTTP_CLIENT_ID: %s" % client_id) metadata = MetadataManager(self.data_path + "/" + request.file_name, request.file_name) file_exist = metadata.restore() if len(metadata.lock_list) > 0: response['status'] = http_code['403'] response['body'] = "File is locked" logging.debug("file is locked") return response metadata.size = request.metadata["HTTP_DATA_LENGTH"] metadata.etag = request.metadata["HTTP_ETAG"] # Create File in any nodes if not file_exist: write_balancer = WriteLoadBalancer(self.data_path, storage_nodes) balanced_storage_list = write_balancer.balance() if balanced_storage_list: storage_list = balanced_storage_list else: storage_list = storage_nodes logging.debug("storage_list: %s" % storage_list) replicas_count = request.metadata.get("HTTP_REPLICAS_COUNT", 3) replica_list = storage_list[:int(replicas_count)] logging.debug("replicas_list: %s" % replica_list) metadata.storage_list = replica_list metadata.replicas_count = replicas_count # Update File in the same nodes else: replica_list = metadata.storage_list available_replicas = [] for replica in metadata.storage_list: for node in storage_nodes: if replica[0] == node[0] and replica[1] == node[1]: available_replicas.append((replica[0], replica[1])) if len(available_replicas) != len(metadata.storage_list): response['status'] = http_code['304'] response['body'] = "Deny of service due server is down" print("Deny of service due server is down") logging.debug("Deny of service due server is down") return response #if len(replica_list) < 3: #response['status'] = http_code['304'] #response['body'] = "Deny of service due server is down" #return response #fl = FileLock(self.data_path + "/" + request.file_name) #while True: #if not fl.is_locked(): #break #else: #logging.debug("file is locked. Someone is updating") #time.sleep(1) #response['status'] = http_code['403'] #response['body'] = "File is locked. Someone is updating." #return response # Cuts off not needed replicas metadata.add_lock_item(client_id, "LOCK_WRITE") metadata.save() #fl.lock() body = {} body["STORAGE_LIST"] = replica_list response['body'] = json.dumps(body) response['status'] = http_code['201'] logging.debug("response: %s" % response) return response def POST(self, request): logging.debug("proxy.post") proxy_nodes = [] storage_nodes = [] self.get_nodes(proxy_nodes, storage_nodes) print("storage_node: %s" % storage_nodes) print("proxy_node: %s" % proxy_nodes) #request.showVariables() response = {"headers": [], "status": "201 OK", "body": ""} if not request.file_name: response['status'] = http_code['400'] return response if not self.group.is_leader: response['status'] = http_code['304'] response['body'] = "No leader. Try to find the leader" logging.debug("I'm no leader. Try to find the leader") return response #client_id = request.metadata.get("HTTP_CLIENT_ID", "test:test") #logging.debug("HTTP_CLIENT_ID: %s" % client_id) replicas_count = request.metadata["HTTP_REPLICAS_COUNT"] metadata = MetadataManager(self.data_path + "/" + request.file_name, request.file_name, request.metadata["HTTP_DATA_LENGTH"], replicas_count, request.metadata["HTTP_ETAG"]) if not metadata.restore(): response['status'] = http_code['304'] response['body'] = "File does not exist" logging.debug("File does not exist") return response logging.debug(self.data_path + "/" + request.file_name) fl = FileLock(self.data_path + "/" + request.file_name) if metadata.lock_list: metadata.lock_list.pop() metadata.save() fl.unlock() logging.debug("self.proxy_nodes: %s" % self.proxy_nodes) for proxy_replicas in proxy_nodes: if proxy_replicas[0] == self.port_ip[0] and\ proxy_replicas[1] == self.port_ip[1]: continue headers = {} headers['file_name'] = request.file_name headers['data_length'] = request.content_length headers['etag'] = request.file_name headers['replicas_count'] = replicas_count headers['client_id'] = "%s:%s" % (request.remote_addr, request.remote_port) headers["replication"] = "1" metadata_copy = json.dumps(metadata.resource_dict) response = self._put_request(request.file_name, proxy_replicas, data=metadata_copy, headers=headers) logging.debug("response: %s" % response) return response def GET(self, request): logging.debug("proxy.get") response = {"headers": [], "status": "200 OK", "body": ""} print("Am I leader? %s" % self.group.is_leader) proxy_nodes = [] storage_nodes = [] self.get_nodes(proxy_nodes, storage_nodes) print("storage_node: %s" % storage_nodes) print("proxy_node: %s" % proxy_nodes) #request.showVariables() if not request.file_name: response['status'] = http_code['400'] return response metadata = MetadataManager(self.data_path + "/" + request.file_name, request.file_name) file_exist = metadata.restore() # File is locked if metadata.lock_list: # File is not commited yet if len(metadata.storage_list) == 0: response['status'] = http_code['404'] response['body'] = "File is not fully created yet" print("file is not fully created yet") return response if not file_exist: response['status'] = http_code['404'] response['body'] = "File does not exist" print("file does not exist") return response available_replicas = [] for replica in metadata.storage_list: for node in storage_nodes: if replica[0] == node[0] and replica[1] == node[1]: available_replicas.append((replica[0], replica[1])) response['body'] = json.dumps(available_replicas) return response def DELETE(self, request): logging.debug("proxy.delete") response = {"headers": [], "status": "200 OK", "body": ""} # Deletion happens when leader wants to sync data deletion to # other nodes deletion = request.metadata.get("HTTP_DELETION", None) if deletion == "1": logging.debug("DELETION") metadata = MetadataManager(self.data_path + "/" + request.file_name,\ request.file_name) if not metadata.delete(): response['status'] = http_code['304'] response['body'] = "Could not delete" logging.debug("Could not delete") return response if not self.group.is_leader: response['status'] = http_code['304'] response['body'] = "No leader. Try to find the leader" logging.debug("I'm no leader. Try to find the leader") return response proxy_nodes = [] storage_nodes = [] self.get_nodes(proxy_nodes, storage_nodes) print("storage_node: %s" % storage_nodes) print("proxy_node: %s" % proxy_nodes) if not request.file_name: response['status'] = http_code['400'] return response metadata = MetadataManager(self.data_path + "/" + request.file_name, request.file_name) file_exist = metadata.restore() # File is locked if metadata.lock_list: # File is not commited yet if len(metadata.storage_list) == 0: response['status'] = http_code['304'] response['body'] = "File is not fully created yet" print("file is not fully created yet") logging.debug("file is not fully created yet") return response else: response['status'] = http_code['304'] response['body'] = "File is locked" print("file is locked") return response if not file_exist: response['status'] = http_code['404'] response['body'] = "File does not exist" print("file does not exist") return response #metadata.delete() logging.debug("self.proxy_nodes: %s" % proxy_nodes) if len(proxy_nodes) < 3: response['status'] = http_code['304'] response['body'] = "Deny of service due server is down" print("Deny of service due replica down") logging.debug("Deny of service due replica down") return response metadata.delete() for proxy_replicas in proxy_nodes: if proxy_replicas[0] == self.port_ip[0] and\ proxy_replicas[1] == self.port_ip[1]: continue headers = {} headers['file_name'] = request.file_name headers['data_length'] = request.content_length headers['etag'] = request.file_name headers['client_id'] = "%s:%s" % (request.remote_addr, request.remote_port) headers["deletion"] = "1" response = self._delete_request(request.file_name, proxy_replicas, headers=headers) logging.debug("response: %s" % response) return response