Example #1
0
    def GET(self, request):
        logging.debug("proxy.get")
        response = {"headers": [], "status": "200 OK", "body": ""}

        print("Am I leader? %s" % self.group.is_leader)

        proxy_nodes = []
        storage_nodes = []
        self.get_nodes(proxy_nodes, storage_nodes)
        print("storage_node: %s" % storage_nodes)
        print("proxy_node: %s" % proxy_nodes)

        #request.showVariables()

        if not request.file_name:
            response['status'] = http_code['400']
            return response

        metadata = MetadataManager(self.data_path + "/" + request.file_name,
                                   request.file_name)

        file_exist = metadata.restore()
        # File is locked
        if metadata.lock_list:
            # File is not commited yet
            if len(metadata.storage_list) == 0:
                response['status'] = http_code['404']
                response['body'] = "File is not fully created yet"
                print("file is not fully created yet")
                return response

        if not file_exist:
            response['status'] = http_code['404']
            response['body'] = "File does not exist"
            print("file does not exist")
            return response

        available_replicas = []
        for replica in metadata.storage_list:
            for node in storage_nodes:
                if replica[0] == node[0] and replica[1] == node[1]:
                    available_replicas.append((replica[0], replica[1]))

        response['body'] = json.dumps(available_replicas)

        return response
Example #2
0
    def PUT(self, request):
        logging.debug("proxy.put")

        proxy_nodes = []
        storage_nodes = []
        self.get_nodes(proxy_nodes, storage_nodes)
        print("storage_node: %s" % storage_nodes)
        print("proxy_node: %s" % proxy_nodes)

        #request.showVariables()
        print("Am I leader? %s" % self.group.is_leader)
        response = {"headers": [], "status": "200 OK", "body": ""}

        replication = request.metadata.get("HTTP_REPLICATION", None)
        if replication == "1":
            logging.debug("REPLICATION")
            fm = FileManager(self.data_path + "/" + request.file_name,\
                             request.file_name)
            if not fm.write_file(request.body):
                response['status'] = http_code['500']
                response['body'] = "Could not replicate"
                logging.debug("Could not replicate")
            return response

        if len(proxy_nodes) < 3:
            response['status'] = http_code['304']
            response['body'] = "Deny of service due server is down"
            print("Deny of service due server is down")
            logging.debug("Deny of service due server is down")
            return response

        if not self.group.is_leader:
            response['status'] = http_code['304']
            response['body'] = "No leader. Try to find the leader"
            logging.debug("I'm no leader. Try to find the leader")
            return response

        if not request.file_name:
            response['status'] = http_code['400']
            response['body'] = "No valid file name found"
            logging.debug("No valid file name found")
            return response

        client_id = request.metadata.get("HTTP_CLIENT_ID", "test:test")
        #logging.debug("HTTP_CLIENT_ID: %s" % client_id)


        metadata = MetadataManager(self.data_path + "/" + request.file_name,
                                   request.file_name)

        file_exist = metadata.restore()
        if len(metadata.lock_list) > 0:
            response['status'] = http_code['403']
            response['body'] = "File is locked"
            logging.debug("file is locked")
            return response

        metadata.size = request.metadata["HTTP_DATA_LENGTH"]
        metadata.etag = request.metadata["HTTP_ETAG"]

        # Create File in any nodes
        if not file_exist:
            write_balancer = WriteLoadBalancer(self.data_path, storage_nodes)
            balanced_storage_list = write_balancer.balance()
            if balanced_storage_list:
                storage_list = balanced_storage_list
            else:
                storage_list = storage_nodes

            logging.debug("storage_list: %s" % storage_list)
            replicas_count = request.metadata.get("HTTP_REPLICAS_COUNT", 3)
            replica_list = storage_list[:int(replicas_count)]
            logging.debug("replicas_list: %s" % replica_list)
            metadata.storage_list = replica_list
            metadata.replicas_count = replicas_count

        # Update File in the same nodes
        else:
            replica_list = metadata.storage_list

        available_replicas = []
        for replica in metadata.storage_list:
            for node in storage_nodes:
                if replica[0] == node[0] and replica[1] == node[1]:
                    available_replicas.append((replica[0], replica[1]))

        if len(available_replicas) != len(metadata.storage_list):
            response['status'] = http_code['304']
            response['body'] = "Deny of service due server is down"
            print("Deny of service due server is down")
            logging.debug("Deny of service due server is down")
            return response

        #if  len(replica_list) < 3:
            #response['status'] = http_code['304']
            #response['body'] = "Deny of service due server is down"
            #return response

        #fl = FileLock(self.data_path + "/" + request.file_name)
        #while True:
            #if not fl.is_locked():
                #break
            #else:
                #logging.debug("file is locked. Someone is updating")
                #time.sleep(1)
                #response['status'] = http_code['403']
                #response['body'] = "File is locked. Someone is updating."
                #return response


        # Cuts off not needed replicas
        metadata.add_lock_item(client_id, "LOCK_WRITE")
        metadata.save()
        #fl.lock()

        body = {}
        body["STORAGE_LIST"] = replica_list
        response['body'] = json.dumps(body)

        response['status'] = http_code['201']
        logging.debug("response: %s" % response)
        return response
Example #3
0
    def DELETE(self, request):
        logging.debug("proxy.delete")
        response = {"headers": [], "status": "200 OK", "body": ""}

        # Deletion happens when leader wants to sync data deletion to
        # other nodes
        deletion = request.metadata.get("HTTP_DELETION", None)
        if deletion == "1":
            logging.debug("DELETION")
            metadata = MetadataManager(self.data_path + "/" + request.file_name,\
                             request.file_name)
            if not metadata.delete():
                response['status'] = http_code['304']
                response['body'] = "Could not delete"
                logging.debug("Could not delete")
            return response

        if not self.group.is_leader:
            response['status'] = http_code['304']
            response['body'] = "No leader. Try to find the leader"
            logging.debug("I'm no leader. Try to find the leader")
            return response

        proxy_nodes = []
        storage_nodes = []
        self.get_nodes(proxy_nodes, storage_nodes)
        print("storage_node: %s" % storage_nodes)
        print("proxy_node: %s" % proxy_nodes)


        if not request.file_name:
            response['status'] = http_code['400']
            return response

        metadata = MetadataManager(self.data_path + "/" + request.file_name,
                                   request.file_name)

        file_exist = metadata.restore()
        # File is locked
        if metadata.lock_list:
            # File is not commited yet
            if len(metadata.storage_list) == 0:
                response['status'] = http_code['304']
                response['body'] = "File is not fully created yet"
                print("file is not fully created yet")
                logging.debug("file is not fully created yet")
                return response
            else:
                response['status'] = http_code['304']
                response['body'] = "File is locked"
                print("file is locked")
                return response


        if not file_exist:
            response['status'] = http_code['404']
            response['body'] = "File does not exist"
            print("file does not exist")
            return response

        #metadata.delete()
        logging.debug("self.proxy_nodes: %s" % proxy_nodes)
        if len(proxy_nodes) < 3:
            response['status'] = http_code['304']
            response['body'] = "Deny of service due server is down"
            print("Deny of service due replica down")
            logging.debug("Deny of service due replica down")
            return response

        metadata.delete()
        for proxy_replicas in proxy_nodes:
            if proxy_replicas[0] == self.port_ip[0] and\
                proxy_replicas[1] == self.port_ip[1]:
                    continue

            headers = {}
            headers['file_name'] = request.file_name
            headers['data_length'] = request.content_length
            headers['etag'] = request.file_name
            headers['client_id'] = "%s:%s" % (request.remote_addr,
                                            request.remote_port)

            headers["deletion"] = "1"
            response = self._delete_request(request.file_name,
                                        proxy_replicas,
                                        headers=headers)
            logging.debug("response: %s" % response)

        return response
Example #4
0
    def POST(self, request):
        logging.debug("proxy.post")

        proxy_nodes = []
        storage_nodes = []
        self.get_nodes(proxy_nodes, storage_nodes)
        print("storage_node: %s" % storage_nodes)
        print("proxy_node: %s" % proxy_nodes)

        #request.showVariables()
        response = {"headers": [], "status": "201 OK", "body": ""}

        if not request.file_name:
            response['status'] = http_code['400']
            return response

        if not self.group.is_leader:
            response['status'] = http_code['304']
            response['body'] = "No leader. Try to find the leader"
            logging.debug("I'm no leader. Try to find the leader")
            return response

        #client_id = request.metadata.get("HTTP_CLIENT_ID", "test:test")
        #logging.debug("HTTP_CLIENT_ID: %s" % client_id)

        replicas_count = request.metadata["HTTP_REPLICAS_COUNT"]
        metadata = MetadataManager(self.data_path + "/" + request.file_name,
                                   request.file_name,
                                   request.metadata["HTTP_DATA_LENGTH"],
                                   replicas_count,
                                   request.metadata["HTTP_ETAG"])


        if not metadata.restore():
            response['status'] = http_code['304']
            response['body'] = "File does not exist"
            logging.debug("File does not exist")
            return response

        logging.debug(self.data_path + "/" + request.file_name)
        fl = FileLock(self.data_path + "/" + request.file_name)
        if metadata.lock_list:
            metadata.lock_list.pop()
        metadata.save()
        fl.unlock()

        logging.debug("self.proxy_nodes: %s" % self.proxy_nodes)
        for proxy_replicas in proxy_nodes:
            if proxy_replicas[0] == self.port_ip[0] and\
                proxy_replicas[1] == self.port_ip[1]:
                    continue

            headers = {}
            headers['file_name'] = request.file_name
            headers['data_length'] = request.content_length
            headers['etag'] = request.file_name
            headers['replicas_count'] = replicas_count
            headers['client_id'] = "%s:%s" % (request.remote_addr,
                                              request.remote_port)

            headers["replication"] = "1"
            metadata_copy = json.dumps(metadata.resource_dict)
            response = self._put_request(request.file_name,
                                         proxy_replicas,
                                         data=metadata_copy,
                                         headers=headers)
            logging.debug("response: %s" % response)

        return response