def clientapp_get_block_lists_to_write(directory, file_name, size): """ Returns the necessary block information from name_node to write files to SUFS""" route = connect_to_host("name", default_env) + '/file/' response = r.post(route, json={ 'file_name': file_name, 'directory': directory, 'size': size }) if response: response = response.json() if "error" in response: print("NameNode {0}".format(response['error'])) return 0 else: file_size = get_file_size(file_name) block_count = int(response['block_count']) block_size = int(response['block_size']) blocks = response['blocks_and_dns'] block_data = {} block_data['file_name'] = file_name block_data['file_size'] = file_size block_data['block_count'] = int(block_count) block_data['block_size'] = int(block_size) block_data['data_blocks'] = blocks return block_data else: print("ERROR: NAMENODE IS OFFLINE. ENVIRONMENT IS {0}.".format( default_env))
def forward_data(file_name, block_id, copy_node, block_body): """ Forward data to the copy node""" route = connect_to_host(copy_node, default_env) + '/forward_block/' print( r.post(route + file_name + "/" + block_id, json={ 'block_body': block_body }).json())
def remove_directory(directory): """ Removes the specified directory, if it is empty """ route = connect_to_host("name", default_env) + '/directories/' response = r.delete(route, json={'directory': directory}) if response: response = response.json() if "error" in response: print("Namenode {0}".format(response['error'])) else: print(response['message'])
def clientapp_get_block_names(file_name, directory): """This function returns a list of block_ids where blocks of data associated with the given file_name can be passed to the read functions""" route = connect_to_host("name", default_env) + '/file/' response = r.get(route + directory + "/" + file_name) if response: return response.json() else: print("ERROR: NAMENODE IS OFFLINE. ENVIRONMENT IS {0}.".format( default_env))
def datanode_send_heartbeat(datanode_id): """ Posts a heartbeat to the name node :param name: datanode_id :param type: str The name of the datanode that is posting the heartbeat """ print("\nStarting a hearbeat") route = connect_to_host("name", default_env) + '/hearbeat/' print(r.put(route, json={'id': datanode_id}).text)
def clientapp_makedir(directory_name): route = connect_to_host( "name", default_env) + '/directories/' + directory_name + '/' response = r.get(route) if response: response = response.json() if "error" in response: print("Namenode {0}".format(response['error'])) else: print(response['message']) else: print("ERROR: NAMENODE IS OFFLINE. ENVIRONMENT IS {0}.".format( default_env))
def list_all_in_directory(directory): """ Lists all the files of the specified directory """ route = connect_to_host("name", default_env) + '/directories/' response = r.post(route, json={'directory': directory}) if response: response = response.json() if "error" in response: print("Namenode {0}".format(response['error'])) else: print(response['message']) else: print("ERROR: NAMENODE IS OFFLINE. ENVIRONMENT IS {0}.".format( default_env))
def client_delete_file_from_namenode(directory, file_name): """This function deletes file from nn block list and file dir and returns a list of block_ids where blocks of data associated with the given file_name can be passed to the delete functions for the datanodes """ route = connect_to_host("name", default_env) + '/file/' response = r.delete(route + directory + "/" + file_name) if response: return response.json() else: print("ERROR: NAMENODE IS OFFLINE. ENVIRONMENT IS {0}.".format( default_env)) return 0
def datanode_send_blockreport(datanode_id, block_data): """ Posts a block report to the name node :param name: datanode_id :param type: str The name of the datanode that is posting the request :param name: block_data :param type: list The list of blocks on the datanode """ print("\nStarting a block report") route = connect_to_host("name", default_env) + '/block_report/' payload = {'id': datanode_id, 'block_data': block_data} print(r.put(route, json=payload).text)
def write_request_to_dn(cur_data_node, file_name, block_id, copy_node, block_body): route = connect_to_host(cur_data_node, default_env) + '/block/' response = r.post(route + file_name + "/" + str(block_id), json={ 'block_body': block_body, 'copy_node': copy_node }) if response: response = response.json() if "error" in response: print("DataNode {0}".format(response['error'])) else: print(response['message']) else: print("ERROR: DATANODE {0} IS OFFLINE. ENVIRONMENT IS {1}.".format( cur_data_node, default_env))
def remove_subdir(parent, sub_dir): """Removes a child driectory from a parent directory""" route = connect_to_host("name", default_env) + '/directories/sub_dir/' response = r.delete(route, json={ 'parent': parent, 'sub_directory': sub_dir }) if response: response = response.json() if "error" in response: print("Namenode {0}".format(response['error'])) else: print(response['message']) else: print("ERROR: NAMENODE IS OFFLINE. ENVIRONMENT IS {0}.".format( default_env))
def forward_data(dn_with_block, copy_node, block_name): """ Connects to dn and asks it to make a copy to send to the copy_node """ route = connect_to_host(dn_with_block, default_env) + '/copy_block_data/' response = r.post(route, json={ 'copy_node': copy_node, 'block_name': block_name }) if response: response = response.json() if 'error' in response: print(response['error']) else: print(response['message']) else: print("ERROR: DATANODE {0} IS OFFLINE. ENVIRONMENT IS {1}".format( copy_node, default_env))
def clientapp_delete_file(blocks_dns): """ The delete function reads a list of block_ids from the name node after supplying the file we'd like to delete. the list is copied to a dict which is then used to send a delete request to the data_nodes. """ blocks_and_dns = blocks_dns['blocks_and_dns'] for block_id, data_node_list in blocks_and_dns.items(): for dn in data_node_list: route = connect_to_host(dn, default_env) + '/block/' response = r.delete(route + block_id + "/") if response: response = response.json() if "error" in response: print("Datanode {0}: {1}".format(dn, response['error'])) else: print(response['message']) else: print("ERROR: DATANODE {0} IS OFFLINE. ENVIRONMENT IS {1}.". format(dn, default_env))
def clientapp_read_file(file_name, blocks): """ The read function takes in a file_name and a list of datablocks supplied by the name_node which then sends requests to the corresponding data_nodes to access block data. """ new_file_name = "local_" + file_name f = open(new_file_name, "w") b = blocks['message'] for block_id, nodes in b.items(): block_id = int(block_id) for block_id, nodes in sorted(b.items()): dn = random.choice(nodes) route = connect_to_host(dn, default_env) + '/block/' variables = r.get(route + str(block_id) + '/').json() text = str(variables['block_body']) f.write(text) f.close()