def clientapp_get_block_lists_to_write(directory, file_name, size):
""" Returns the necessary block information from name_node to write files to SUFS"""
route = connect_to_host("name", default_env) + '/file/'
response = r.post(route,
json={
'file_name': file_name,
'directory': directory,
'size': size
})
if response:
response = response.json()
if "error" in response:
print("NameNode {0}".format(response['error']))
return 0
else:
file_size = get_file_size(file_name)
block_count = int(response['block_count'])
block_size = int(response['block_size'])
blocks = response['blocks_and_dns']
block_data = {}
block_data['file_name'] = file_name
block_data['file_size'] = file_size
block_data['block_count'] = int(block_count)
block_data['block_size'] = int(block_size)
block_data['data_blocks'] = blocks
return block_data
else:
print("ERROR: NAMENODE IS OFFLINE. ENVIRONMENT IS {0}.".format(
default_env))
def forward_data(file_name, block_id, copy_node, block_body):
""" Forward data to the copy node"""
route = connect_to_host(copy_node, default_env) + '/forward_block/'
print(
r.post(route + file_name + "/" + block_id,
json={
'block_body': block_body
}).json())
def remove_directory(directory):
""" Removes the specified directory, if it is empty """
route = connect_to_host("name", default_env) + '/directories/'
response = r.delete(route, json={'directory': directory})
if response:
response = response.json()
if "error" in response:
print("Namenode {0}".format(response['error']))
else:
print(response['message'])
def clientapp_get_block_names(file_name, directory):
"""This function returns a list of block_ids where blocks of
data associated with the given file_name can be passed
to the read functions"""
route = connect_to_host("name", default_env) + '/file/'
response = r.get(route + directory + "/" + file_name)
if response:
return response.json()
else:
print("ERROR: NAMENODE IS OFFLINE. ENVIRONMENT IS {0}.".format(
default_env))
def datanode_send_heartbeat(datanode_id):
"""
Posts a heartbeat to the name node
:param name: datanode_id
:param type: str
The name of the datanode that is posting the heartbeat
"""
print("\nStarting a hearbeat")
route = connect_to_host("name", default_env) + '/hearbeat/'
print(r.put(route, json={'id': datanode_id}).text)
def clientapp_makedir(directory_name):
route = connect_to_host(
"name", default_env) + '/directories/' + directory_name + '/'
response = r.get(route)
if response:
response = response.json()
if "error" in response:
print("Namenode {0}".format(response['error']))
else:
print(response['message'])
else:
print("ERROR: NAMENODE IS OFFLINE. ENVIRONMENT IS {0}.".format(
default_env))
def list_all_in_directory(directory):
""" Lists all the files of the specified directory """
route = connect_to_host("name", default_env) + '/directories/'
response = r.post(route, json={'directory': directory})
if response:
response = response.json()
if "error" in response:
print("Namenode {0}".format(response['error']))
else:
print(response['message'])
else:
print("ERROR: NAMENODE IS OFFLINE. ENVIRONMENT IS {0}.".format(
default_env))
def client_delete_file_from_namenode(directory, file_name):
"""This function deletes file from nn block list and file dir
and returns a list of block_ids where blocks of
data associated with the given file_name can be passed
to the delete functions for the datanodes """
route = connect_to_host("name", default_env) + '/file/'
response = r.delete(route + directory + "/" + file_name)
if response:
return response.json()
else:
print("ERROR: NAMENODE IS OFFLINE. ENVIRONMENT IS {0}.".format(
default_env))
return 0
def datanode_send_blockreport(datanode_id, block_data):
"""
Posts a block report to the name node
:param name: datanode_id
:param type: str
The name of the datanode that is posting the request
:param name: block_data
:param type: list
The list of blocks on the datanode
"""
print("\nStarting a block report")
route = connect_to_host("name", default_env) + '/block_report/'
payload = {'id': datanode_id, 'block_data': block_data}
print(r.put(route, json=payload).text)
def write_request_to_dn(cur_data_node, file_name, block_id, copy_node,
block_body):
route = connect_to_host(cur_data_node, default_env) + '/block/'
response = r.post(route + file_name + "/" + str(block_id),
json={
'block_body': block_body,
'copy_node': copy_node
})
if response:
response = response.json()
if "error" in response:
print("DataNode {0}".format(response['error']))
else:
print(response['message'])
else:
print("ERROR: DATANODE {0} IS OFFLINE. ENVIRONMENT IS {1}.".format(
cur_data_node, default_env))
def remove_subdir(parent, sub_dir):
"""Removes a child driectory from a parent directory"""
route = connect_to_host("name", default_env) + '/directories/sub_dir/'
response = r.delete(route,
json={
'parent': parent,
'sub_directory': sub_dir
})
if response:
response = response.json()
if "error" in response:
print("Namenode {0}".format(response['error']))
else:
print(response['message'])
else:
print("ERROR: NAMENODE IS OFFLINE. ENVIRONMENT IS {0}.".format(
default_env))
def forward_data(dn_with_block, copy_node, block_name):
""" Connects to dn and asks it to make a copy to send to the copy_node """
route = connect_to_host(dn_with_block, default_env) + '/copy_block_data/'
response = r.post(route,
json={
'copy_node': copy_node,
'block_name': block_name
})
if response:
response = response.json()
if 'error' in response:
print(response['error'])
else:
print(response['message'])
else:
print("ERROR: DATANODE {0} IS OFFLINE. ENVIRONMENT IS {1}".format(
copy_node, default_env))
def clientapp_delete_file(blocks_dns):
"""
The delete function reads a list of block_ids from the name node
after supplying the file we'd like to delete. the list is copied to a dict
which is then used to send a delete request to the data_nodes.
"""
blocks_and_dns = blocks_dns['blocks_and_dns']
for block_id, data_node_list in blocks_and_dns.items():
for dn in data_node_list:
route = connect_to_host(dn, default_env) + '/block/'
response = r.delete(route + block_id + "/")
if response:
response = response.json()
if "error" in response:
print("Datanode {0}: {1}".format(dn, response['error']))
else:
print(response['message'])
else:
print("ERROR: DATANODE {0} IS OFFLINE. ENVIRONMENT IS {1}.".
format(dn, default_env))
def clientapp_read_file(file_name, blocks):
"""
The read function takes in a file_name and a list of
datablocks supplied by the name_node which then sends
requests to the corresponding data_nodes to access block
data.
"""
new_file_name = "local_" + file_name
f = open(new_file_name, "w")
b = blocks['message']
for block_id, nodes in b.items():
block_id = int(block_id)
for block_id, nodes in sorted(b.items()):
dn = random.choice(nodes)
route = connect_to_host(dn, default_env) + '/block/'
variables = r.get(route + str(block_id) + '/').json()
text = str(variables['block_body'])
f.write(text)
f.close()
