def tell_naming_node_im_born():
debug_log("Wait 7 sec...")
sleep(7)
pub_ip = requests.request(
"GET", "http://169.254.169.254/latest/meta-data/public-ipv4").text
debug_log(f"Going to tell that I'm born, pub_ip = {pub_ip}")
request_node(NAMENODE_IP, '/new_node', {'pub': pub_ip})
def fread_command(params):
if len(params) < 1:
print("Please specify a file to read")
return
token = read_token()
filename = os.path.basename(params[0])
# getting right path
params[0], result = _get_path(params[0])
if not result:
print("Wrong path")
return
res = request_node(NAMENODE_IP, "/fread", {TOKEN_KEY: token, PATH_KEY: params[0]})
if res.status == HTTPStatus.OK:
storage_node_ip = get_dict_from_response(res)[NODE_IP_KEY]
full_path = get_dict_from_response(res)[FULL_PATH_KEY]
storage_res = request_node(storage_node_ip, "/fread", {FULL_PATH_KEY: full_path})
if storage_res.status == HTTPStatus.OK:
file_data = get_dict_from_response(storage_res)[FILE_BYTES]
with open(filename, "wb") as f:
f.write(base64.decodebytes(file_data.encode()))
print("Success!")
else:
print("Unsuccessful")
else:
msg = get_dict_from_response(res)[MESSAGE_KEY]
print(msg)
def flask_replicate():
full_file_path = flask.request.form.get(key=FULL_PATH_KEY,
default=None,
type=str)
target_node_ip = flask.request.form.get(key=NODE_IP_KEY,
default=None,
type=str)
if not full_file_path or not target_node_ip:
data = {
MESSAGE_KEY:
f"Missing required parameters: `{FULL_PATH_KEY}`, `{NODE_IP_KEY}`"
}
return flask.make_response(flask.jsonify(data),
HTTPStatus.UNPROCESSABLE_ENTITY)
file_path = get_path(full_file_path)
try:
with open(file_path, "rb") as f:
file_data = str(base64.encodebytes(f.read()), 'utf-8')
except OSError as e:
debug_log(e.strerror)
data = {MESSAGE_KEY: "Error on storage server"}
return flask.make_response(flask.jsonify(data),
HTTPStatus.INTERNAL_SERVER_ERROR)
data = {FULL_PATH_KEY: full_file_path, FILE_BYTES: file_data}
request_node(target_node_ip, '/save_replication', data)
return flask.make_response(flask.jsonify({}), HTTPStatus.OK)
def flask_fdelete():
token = flask.request.form.get(key=TOKEN_KEY, default=None, type=str)
file_path = flask.request.form.get(key=PATH_KEY, default=None, type=str)
if not token or not file_path:
data = {MESSAGE_KEY: f"Missing required parameters: `{TOKEN_KEY}`, `{PATH_KEY}`"}
return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY)
login = decode_auth_token(token)
if not login or (type(login) == str and login not in db_auth.keys()):
data = {MESSAGE_KEY: "The token is invalid or has expired"}
return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN)
full_file_path = os.path.join(login, file_path)
exists = full_file_path in db_user2files.lrange(login, 0, -1)
if not exists:
data = {MESSAGE_KEY: "The file for deleting does not exist"}
return flask.make_response(flask.jsonify(data), HTTPStatus.NOT_FOUND)
for node_ip in db_file2nodes.lrange(full_file_path, 0, -1):
res = request_node(node_ip, '/fdelete', {FULL_PATH_KEY: full_file_path})
res = get_dict_from_response(res)
if res is None:
debug_log(f"Node {node_ip} did not response on /fdelete")
db_node2files.lrem(node_ip, 0, full_file_path)
db_user2files.lrem(login, 0, full_file_path)
db_file2nodes.delete(full_file_path)
db_file2size.delete(full_file_path)
data = {MESSAGE_KEY: f"Successfully deleted the file"}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def finfo_command(params):
if len(params) < 1:
print("Please specify a file")
return
# getting right path
params[0], result = _get_path(params[0])
if not result:
print("Wrong path")
return
token = read_token()
params.append(token)
params = _get_dict(params, [PATH_KEY, TOKEN_KEY])
res = request_node(NAMENODE_IP, "/finfo", params)
if res.status == HTTPStatus.OK:
file_size = get_dict_from_response(res)[FILE_SIZE_KEY]
nodes_ip = get_dict_from_response(res)[NODE_IP_KEY]
print(f"Size of `{params[PATH_KEY]}` = {file_size} bytes\n"
f"It's stored on {len(nodes_ip)} nodes: {nodes_ip}")
else:
msg = get_dict_from_response(res)[MESSAGE_KEY]
print(msg)
def flask_uploaded():
full_file_path = flask.request.form.get(key=FULL_PATH_KEY, default=None, type=str)
file_size = flask.request.form.get(key=FILE_SIZE_KEY, default=None, type=str)
if not full_file_path or not file_size:
data = {MESSAGE_KEY: f"Missing required parameters: `{FULL_PATH_KEY}`, `{FILE_SIZE_KEY}`"}
return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY)
login = full_file_path.split('/')[0]
db_file2size.set(full_file_path, file_size)
if full_file_path not in db_user2files.lrange(login, 0, -1):
db_user2files.lpush(login, full_file_path)
source_node_ip = flask.request.environ.get('HTTP_X_REAL_IP', flask.request.remote_addr)
nodes_with_obsolete_files = db_file2nodes.lrange(full_file_path, 0, -1)
if source_node_ip in nodes_with_obsolete_files:
nodes_with_obsolete_files.remove(source_node_ip)
else:
db_node2files.lpush(source_node_ip, full_file_path)
db_file2nodes.lpush(full_file_path, source_node_ip)
for node_ip in nodes_with_obsolete_files:
res = request_node(node_ip, '/fdelete', {FULL_PATH_KEY: full_file_path})
res = get_dict_from_response(res)
if res is None:
debug_log(f"Node {node_ip} did not response on /fdelete")
db_node2files.lrem(node_ip, 0, full_file_path)
db_file2nodes.lrem(full_file_path, 0, node_ip)
replicate(full_file_path=full_file_path)
data = {MESSAGE_KEY: 'OK, uploaded.'}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def rdir_command(params):
if len(params) < 1:
print("Please specify a directory to read")
return
# getting right path
params[0], result = _get_path(params[0])
if not result:
print("Wrong path")
return
token = read_token()
params.append(token)
params = _get_dict(params, [PATH_KEY, TOKEN_KEY])
res = request_node(NAMENODE_IP, "/rdir", params)
if res.status == HTTPStatus.OK:
dir_list = get_dict_from_response(res)[DIR_LIST_KEY]
for l in dir_list:
print(f" -- {l}")
else:
msg = get_dict_from_response(res)[MESSAGE_KEY]
print(msg)
def _auth(params, action):
# login and registration with getting token
if len(params) < 2:
print("Specify login and password please.")
return
login = params[0]
password = params[1]
if login == "" or password == "":
print("Please give me non-empty login and password")
return
encrypted_pass = hashlib.md5(password.encode('utf-8')).hexdigest()
data = {LOGIN_KEY: login, ENCRYPTED_PASS_KEY: encrypted_pass}
res = request_node(NAMENODE_IP, f"/{action}", data)
if res.status == HTTPStatus.OK:
token = get_dict_from_response(res)[TOKEN_KEY]
save_token(token)
_update_current_path(START_PATH)
print("Success!\nYou were logged in.")
else:
msg = get_dict_from_response(res)[MESSAGE_KEY]
print(msg)
def flask_ddir():
token = flask.request.form.get(key=TOKEN_KEY, default=None, type=str)
dir_path = flask.request.form.get(key=PATH_KEY, default=None, type=str)
force = flask.request.form.get(key=FORCE_KEY, default=None, type=str)
if not token or dir_path is None or force is None:
data = {MESSAGE_KEY: f"Missing required parameters: `{TOKEN_KEY}`, `{PATH_KEY}`, `{FORCE_KEY}`"}
return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY)
login = decode_auth_token(token)
if not login or (type(login) == str and login not in db_auth.keys()):
data = {MESSAGE_KEY: "The token is invalid or has expired"}
return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN)
dir_path = dir_path[:-1] if dir_path and dir_path[-1] == '/' else dir_path
if not dir_path:
data = {MESSAGE_KEY: "Can't delete the root directory"}
return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN)
full_dir_path = os.path.join(login, dir_path)
if full_dir_path not in db_user2folders.lrange(login, 0, -1):
data = {MESSAGE_KEY: "The folder for deleting does not exist"}
return flask.make_response(flask.jsonify(data), HTTPStatus.NOT_FOUND)
inner_files_path_list = [full_path for full_path in db_user2files.lrange(login, 0, -1) if
full_path.startswith(full_dir_path + '/')]
inner_folders_path_list = [full_path for full_path in db_user2folders.lrange(login, 0, -1)
if full_path.startswith(full_dir_path + '/')]
if force == 'False' and (inner_files_path_list or inner_folders_path_list):
data = {MESSAGE_KEY: 'The directory contains files. `force=true` to delete'}
return flask.make_response(flask.jsonify(data), HTTPStatus.NOT_ACCEPTABLE)
db_user2folders.lrem(login, 0, full_dir_path)
for inner_dir in inner_folders_path_list:
db_user2folders.lrem(login, 0, inner_dir)
nodes_ip = set()
for inner_file in inner_files_path_list:
db_file2size.delete(inner_file)
db_user2files.lrem(login, 0, inner_file)
for node_ip in db_file2nodes.lrange(inner_file, 0, -1):
db_file2nodes.lrem(inner_file, 0, node_ip)
db_node2files.lrem(node_ip, 0, inner_file)
nodes_ip.add(node_ip)
for node_ip in list(nodes_ip):
res = request_node(node_ip, '/ddir', {FULL_PATH_KEY: full_dir_path})
res = get_dict_from_response(res)
if res is None:
debug_log(f"Node {node_ip} did not response on /ddir ({full_dir_path})")
data = {MESSAGE_KEY: 'Successfully deleted the directory'}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def _dir_exists(path):
token = read_token()
data = {TOKEN_KEY: token, PATH_KEY: path}
res = request_node(NAMENODE_IP, "/dir_exists", data)
if res.status == HTTPStatus.OK:
return True, get_dict_from_response(res)[MESSAGE_KEY]
else:
return False, get_dict_from_response(res)[MESSAGE_KEY]
def fwrite_command(params):
if len(params) < 1:
print("Please specify a file to write to")
return
token = read_token()
filename = params[0]
if not os.path.exists(filename):
print("There is no such file")
return
# getting right path
params[0], result = _get_path(os.path.basename(params[0]))
if not result:
print("Wrong path")
return
res = request_node(NAMENODE_IP, "/fwrite", {TOKEN_KEY: token, PATH_KEY: params[0]})
if res.status == HTTPStatus.OK:
storage_node_ip = get_dict_from_response(res)[NODE_IP_KEY]
with open(filename, 'rb') as f:
file_data = f.read()
storage_res = request_node(storage_node_ip, "/fwrite",
{FULL_PATH_KEY: get_dict_from_response(res)[FULL_PATH_KEY],
FILE_BYTES: str(base64.encodebytes(file_data), 'utf-8')})
if storage_res.status == HTTPStatus.OK:
print("Success!")
else:
msg = get_dict_from_response(storage_res)[MESSAGE_KEY]
print(msg)
else:
msg = get_dict_from_response(res)[MESSAGE_KEY]
print(msg)
def flask_fmove():
token = flask.request.form.get(key=TOKEN_KEY, default=None, type=str)
file_path = flask.request.form.get(key=PATH_KEY, default=None, type=str)
file_destination_path = flask.request.form.get(key=PATH_DESTINATION_KEY, default=None, type=str)
if not token or not file_path or file_destination_path is None:
data = {MESSAGE_KEY: f"Missing required parameters: `{TOKEN_KEY}`, `{PATH_KEY}`, `{PATH_DESTINATION_KEY}`"}
return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY)
login = decode_auth_token(token)
if not login or (type(login) == str and login not in db_auth.keys()):
data = {MESSAGE_KEY: "The token is invalid or has expired"}
return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN)
if not os.path.basename(file_destination_path):
data = {MESSAGE_KEY: "The destination file name is empty"}
return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN)
full_file_path = os.path.join(login, file_path)
full_file_destination_path = os.path.join(login, file_destination_path)
if full_file_path not in db_user2files.lrange(login, 0, -1):
data = {MESSAGE_KEY: "The source file doesn't exist"}
return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN)
if os.path.dirname(full_file_destination_path) not in db_user2folders.lrange(login, 0, -1):
data = {MESSAGE_KEY: "Can't move the file. Destination folder doesn't exist"}
return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN)
for node_ip in db_file2nodes.lrange(full_file_path, 0, -1):
res = request_node(node_ip, '/fmove', {FULL_PATH_KEY: full_file_path,
FULL_PATH_DESTINATION_KEY: full_file_destination_path})
res = get_dict_from_response(res)
if res is None:
debug_log(f"Node {node_ip} did not response on /fmove")
else:
db_node2files.lrem(node_ip, 0, full_file_path)
db_node2files.lpush(node_ip, full_file_destination_path)
db_file2nodes.lrem(full_file_path, 0, node_ip)
db_file2nodes.lpush(full_file_destination_path, node_ip)
db_user2files.lrem(login, 0, full_file_path)
db_user2files.lpush(login, full_file_destination_path)
db_file2size.set(full_file_destination_path, db_file2size.get(full_file_path))
db_file2size.delete(full_file_path)
data = {MESSAGE_KEY: f"Successfully moved the file"}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def init(login):
data_for_node = {LOGIN_KEY: login}
for node_ip in db_congestion.keys():
res = request_node(node_ip, '/init', data_for_node)
res = get_dict_from_response(res)
debug_log(f"/init - node {node_ip} responded wih {res}")
full_file_paths = db_user2files.lrange(login, 0, -1)
was_initialised_before = db_user2files.delete(login)
for full_file_path in full_file_paths:
db_file2size.delete(full_file_path)
nodes_containing_file = db_file2nodes.lrange(full_file_path, 0, -1)
db_file2nodes.delete(full_file_path)
db_user2files.delete(full_file_path)
for node in nodes_containing_file:
db_node2files.lrem(node, 1, full_file_path)
res = request_node(node, '/fdelete', {FULL_PATH_KEY: full_file_path})
res = get_dict_from_response(res)
if res is None:
debug_log(f"Node {node_ip} did not response on /fdelete")
db_user2folders.delete(login)
db_user2folders.lpush(login, login)
return was_initialised_before
def ddir_command(params):
if len(params) < 1:
print("Please specify a directory to delete")
return
# getting right path
params[0], result = _get_path(params[0])
if not result:
print("Wrong path")
return
if _get_current_path() == params[0] or params[0] == '':
print("Denied")
return
token = read_token()
res = request_node(NAMENODE_IP, "/ddir", {TOKEN_KEY: token, PATH_KEY: params[0], FORCE_KEY: False})
if res.status == HTTPStatus.OK:
print("Success!")
elif res.status == HTTPStatus.NOT_ACCEPTABLE:
msg = get_dict_from_response(res)[MESSAGE_KEY]
print(msg)
answer = input("Delete? [Y/n]: ")
if answer.lower() == "y":
res = request_node(NAMENODE_IP, "/ddir", {TOKEN_KEY: token, PATH_KEY: params[0], FORCE_KEY: True})
if res.status == HTTPStatus.OK:
print("Success!")
else:
msg = get_dict_from_response(res)[MESSAGE_KEY]
print(msg)
else:
msg = get_dict_from_response(res)[MESSAGE_KEY]
print(msg)
def ping_nodes():
for node_ip in db_congestion.keys():
if node_ip not in non_responsive_count:
non_responsive_count[node_ip] = 0
res = request_node(node_ip, '/ping', {})
res = get_dict_from_response(res)
if res is None:
non_responsive_count[node_ip] += 1
debug_log(f"Node {node_ip} did not response {non_responsive_count[node_ip]} times")
if non_responsive_count[node_ip] == MAX_PING_TRIES:
remove_node(node_ip)
del non_responsive_count[node_ip]
else:
debug_log(f"Pinging node {node_ip} - {res}")
db_congestion.set(node_ip, res[CONGESTION_KEY])
def _command(params, keys, action):
# executing command with token
token = read_token()
params.append(token)
keys.append(TOKEN_KEY)
params = _get_dict(params, keys)
res = request_node(NAMENODE_IP, f"/{action}", params)
if res.status == HTTPStatus.OK:
print("Success!")
else:
msg = get_dict_from_response(res)[MESSAGE_KEY]
print(msg)
def flask_fwrite():
full_file_path = flask.request.form.get(key=FULL_PATH_KEY,
default=None,
type=str)
file_bytes = flask.request.form.get(key=FILE_BYTES, default=None, type=str)
if not full_file_path or file_bytes is None:
data = {
MESSAGE_KEY:
f"Missing required parameters: `{FULL_PATH_KEY}`, `{FILE_BYTES}`"
}
return flask.make_response(flask.jsonify(data),
HTTPStatus.UNPROCESSABLE_ENTITY)
file_path = get_path(full_file_path)
file_bytes = base64.decodebytes(file_bytes.encode())
try:
all_dirs = full_file_path.split("/")[:-1]
dir_path = ROOT
for directory in all_dirs:
dir_path = os.path.join(dir_path, directory)
if not os.path.exists(dir_path):
os.mkdir(dir_path)
# writeing to file
with open(file_path, 'wb') as f:
f.write(file_bytes)
except OSError as e:
debug_log(e.strerror)
data = {MESSAGE_KEY: "Error on the server"}
return flask.make_response(flask.jsonify(data),
HTTPStatus.INTERNAL_SERVER_ERROR)
data = {FULL_PATH_KEY: full_file_path, FILE_SIZE_KEY: len(file_bytes)}
res = request_node(NAMENODE_IP, '/uploaded', data)
if res.status != HTTPStatus.OK:
data = {MESSAGE_KEY: get_dict_from_response(res)[MESSAGE_KEY]}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
data = {MESSAGE_KEY: "OK"}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)
def replicate(full_file_path: str):
source_nodes_ip = db_file2nodes.lrange(full_file_path, 0, -1)
assert source_nodes_ip
target_nodes_ip = db_congestion.keys()
target_nodes_ip = list(set(target_nodes_ip) - set(source_nodes_ip))
congestions = [(node_ip, db_congestion.get(node_ip)) for node_ip in target_nodes_ip]
congestions = sorted(congestions, key=lambda x: x[1])
target_nodes_ip = [congestion[0] for congestion in congestions[:REPLICATION_FACTOR - len(source_nodes_ip)]]
debug_log(f"Going to replicate {full_file_path} from {source_nodes_ip} "
f"to nodes with lowest congestion: {target_nodes_ip}")
for target_node_ip in target_nodes_ip:
res = request_node(source_nodes_ip[0], '/replicate', {FULL_PATH_KEY: full_file_path,
NODE_IP_KEY: target_node_ip})
res = get_dict_from_response(res)
if res is None:
debug_log(f"Node {source_nodes_ip[0]} did not response on /replicate")
else:
db_file2nodes.lpush(full_file_path, target_node_ip)
db_node2files.lpush(target_node_ip, full_file_path)
def flask_fcreate():
token = flask.request.form.get(key=TOKEN_KEY, default=None, type=str)
file_path = flask.request.form.get(key=PATH_KEY, default=None, type=str)
if not token or not file_path:
data = {MESSAGE_KEY: f"Missing required parameters: `{TOKEN_KEY}`, `{PATH_KEY}`"}
return flask.make_response(flask.jsonify(data), HTTPStatus.UNPROCESSABLE_ENTITY)
login = decode_auth_token(token)
if not login or (type(login) == str and login not in db_auth.keys()):
data = {MESSAGE_KEY: "The token is invalid or has expired"}
return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN)
full_file_path = os.path.join(login, file_path)
if os.path.dirname(full_file_path) not in db_user2folders.lrange(login, 0, -1):
data = {MESSAGE_KEY: "Can't create the file. Folder doesn't exist"}
return flask.make_response(flask.jsonify(data), HTTPStatus.FORBIDDEN)
congestions = [(node_ip, db_congestion.get(node_ip)) for node_ip in db_congestion.keys()]
congestions = sorted(congestions, key=lambda x: x[1])
nodes = [congestion[0] for congestion in congestions[:REPLICATION_FACTOR]]
debug_log(f"/fcreate - nodes with lowest congestion: {nodes}")
data_for_node = {FULL_PATH_KEY: full_file_path}
for node_ip in nodes:
res = request_node(node_ip, '/fcreate', data_for_node)
res = get_dict_from_response(res)
debug_log(f"/fcreate - storage node {node_ip} response: {res}")
db_node2files.lpush(node_ip, full_file_path)
db_user2files.lpush(login, full_file_path)
db_file2nodes.lpush(full_file_path, *nodes)
db_file2size.set(full_file_path, 0)
data = {MESSAGE_KEY: f"Successfully created the file"}
return flask.make_response(flask.jsonify(data), HTTPStatus.OK)