def validate_path(path):
if not path or path == parameters.sep:
# root
return '/'
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.connect((Constants.NAMENODE_IP, Constants.CLIENT_TO_NAMENODE))
sock.send(str(Codes.validate_path).encode('utf-8'))
sock.recv(1024)
sock.send(path.encode('utf-8'))
is_valid = (sock.recv(1024).decode('utf-8'))
try:
is_valid = int(is_valid)
except:
logger.print_debug_info("Couldn't cast to int")
return False
if is_valid:
sock.send('ok'.encode('utf-8'))
true_path = sock.recv(1024).decode('utf-8')
else:
true_path = None
return true_path
def run(self):
code = int(self.sock.recv(1024).decode('utf-8'))
self.sock.send('ok'.encode('utf-8'))
logger.print_debug_info(code)
if code == Codes.print:
full_path = self.sock.recv(1024).decode('utf-8')
CommandHandler.handle_print_to(self.sock, full_path)
elif code == Codes.upload:
full_path = self.sock.recv(1024).decode('utf-8')
self.sock.send('ok'.encode('utf-8'))
CommandHandler.handle_upload_from(self.sock, full_path)
elif code == Codes.download_all:
CommandHandler.handle_download_all(self.address[0])
elif code == Codes.make_dir:
full_path = self.sock.recv(1024).decode('utf-8')
CommandHandler.handle_mkdir(full_path)
else:
print("StorageListener: no command correspond to code", code)
# regarding shutdown:
# Shut down one or both halves of the connection.
# If how is SHUT_RD, further receives are disallowed.
# If how is SHUT_WR, further sends are disallowed.
# If how is SHUT_RDWR, further sends and receives are disallowed.
self.sock.close()
def handle_make_file(full_path: str):
# to properly join
full_path = full_path.strip(os.sep)
with open(os.path.join(Constants.STORAGE_PATH, full_path), "wb+"):
logger.print_debug_info(
os.path.join(Constants.STORAGE_PATH, full_path))
return
def recreate_storage_dirs():
if os.path.exists(Constants.STORAGE_PATH):
try:
# os.removedirs(Constants.STORAGE_PATH)
os.system("rm -rf {}/*".format(Constants.STORAGE_PATH))
except:
logger.print_debug_info("removedirs fail")
# then create this dir empty
logger.print_debug_info(Constants.STORAGE_PATH, "(re)created")
def handle_print_to(socket: socket.socket, full_path: str):
# to properly join
full_path = full_path.strip(os.sep)
# sending file to a client
logger.print_debug_info("Sending", full_path)
with open(os.path.join(Constants.STORAGE_PATH, full_path),
'rb') as file:
data = file.read(1024)
while data:
socket.send(data)
data = file.read(1024)
def handle_upload_from(socket: socket.socket, full_path: str):
# to properly join
full_path = full_path.strip(os.sep)
# receiving file from a client
logger.print_debug_info("Receiving", full_path)
with open(os.path.join(Constants.STORAGE_PATH, full_path),
'wb+') as file:
data = socket.recv(1024)
while data:
if data:
file.write(data)
else:
return
data = socket.recv(1024)
def ping_thread():
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.settimeout(Constants.PING_TIMEOUT)
while True:
time.sleep(10)
for ip in clean_nodes.nodes.copy():
try:
sock.sendto('ping'.encode('utf-8'), (ip, Constants.STORAGE_PING))
sock.recvfrom(1024)
except:
# socket.timeout or socket.gaierror:
with clean_nodes.lock:
clean_nodes.nodes.discard(ip)
logger.print_debug_info('timeout storagenode', ip)
def handle_download_all(ip: tuple):
"""address is (host, port) tuple
We do separate request for each make_dir & upload request
for simplicity of code.
"""
@logger.log
def ask(code, full_path: str):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((ip, Constants.STORAGE_TO_STORAGE))
sock.send(str(code).encode('utf-8'))
sock.recv(1024) # ack
if code == Codes.upload:
# we need off distribution on upload
sock.send(full_path.encode('utf-8'))
sock.recv(1024)
CommandHandler.handle_print_to(sock, full_path)
else:
sock.send(full_path.encode('utf-8'))
sock.close()
home_path_length = len(Constants.STORAGE_PATH) + 1
# print("home_path_length", home_path_length)
# for all files
for dir_name, subdir_list, file_list in os.walk(
Constants.STORAGE_PATH):
# print("not processed", dir_name)
dir_name = dir_name[home_path_length:]
ask(Codes.make_dir, dir_name)
# print("processed", dir_name)
if len(dir_name) > 0:
# dir_name = os.path.join(*dir_name)
logger.print_debug_info("replicate dir {}".format(dir_name))
# print("dir_name is", dir_name)
ask(Codes.make_dir, dir_name)
else:
dir_name = ''
for file in file_list:
logger.print_debug_info("downloading all: current file is",
file)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((ip, Constants.STORAGE_TO_STORAGE))
full_path = os.path.join(dir_name, file)
# print("fullpath is ", full_path)
ask(Codes.upload, full_path)
def run(self):
code = int(self.sock.recv(4).decode('utf-8'))
self.sock.send('ok'.encode('utf-8'))
logger.print_debug_info(code)
if code == Codes.make_file:
full_path = self.sock.recv(1024).decode('utf-8')
CommandHandler.handle_make_file(full_path)
elif code == Codes.rm:
full_path = self.sock.recv(1024).decode('utf-8')
CommandHandler.handle_rm(full_path)
elif code == Codes.copy:
source = self.sock.recv(1024).decode('utf-8')
self.sock.send('ok'.encode('utf-8'))
destination = self.sock.recv(1024).decode('utf-8')
CommandHandler.handle_copy(source, destination)
elif code == Codes.move:
source = self.sock.recv(1024).decode('utf-8')
self.sock.send('ok'.encode('utf-8'))
destination = self.sock.recv(1024).decode('utf-8')
CommandHandler.handle_move(source, destination)
elif code == Codes.make_dir:
full_path = self.sock.recv(1024).decode('utf-8')
CommandHandler.handle_mkdir(full_path)
elif code == Codes.rmdir:
full_path = self.sock.recv(1024).decode('utf-8')
CommandHandler.handle_rmdir(full_path)
elif code == Codes.init:
if os.path.exists(Constants.STORAGE_PATH):
try:
# os.removedirs(Constants.STORAGE_PATH)
os.system("rm -rf {}/*".format(Constants.STORAGE_PATH))
except:
logger.print_debug_info("rmdirs failed")
# then create this dir empty
elif code == Codes.upload:
logger.print_debug_info("hey")
full_path = self.sock.recv(1024).decode('utf-8')
self.sock.send('ok'.encode('utf-8'))
CommandHandler.handle_upload_from(self.sock, full_path)
logger.print_debug_info("hey2")
CommandHandler.distribute(full_path)
logger.log("he3")
elif code == Codes.print:
full_path = self.sock.recv(1024).decode('utf-8')
CommandHandler.handle_print_to(self.sock, full_path)
else:
print("NamenodeListener: no command correspond to code", code)
self.sock.close()
def distribute(full_path: str):
"""Distributing a file which we just uploaded from a client"""
# real_path = os.path.join(Constants.STORAGE_PATH, full_path)
# ask namenode for all storage's ips
storages_ip = CommandHandler._get_all_storages_ip()
# send file to everybody
if storages_ip is None:
return
for ip in storages_ip:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((ip, Constants.STORAGE_TO_STORAGE))
# Check if another storage doesn't have a file
sock.send(str(Codes.upload).encode('utf-8'))
sock.recv(1024)
if not CommandHandler._has_file(sock, full_path):
logger.print_debug_info("Distributing", full_path)
sock.send(full_path.encode('utf-8'))
sock.recv(1024)
CommandHandler.handle_print_to(sock, full_path)
sock.close()
def insert(self, path, size=-1, ip_address_pool=None):
path = path.split('/')
path = list(filter(lambda path: path != '', path))
curr = self.__root
for node_name in path[:-1]:
child = curr.get_child(node_name)
if not child:
child = self.DirNode(node_name, curr)
curr = child
if path[-1] in curr.children.keys():
logger.print_debug_info("Item already exists: ignoring")
if size < 0:
# Creating dir
child = self.DirNode(path[-1], curr)
else:
child = self.FileNode(path[-1], curr, size, ip_address_pool)
return child
def init_sync():
# First we delete everything we have
# because we don't resurrect old nodes.
recreate_storage_dirs()
storage_ip = get_sync_storage_ip()
logger.print_debug_info("storage_ip", storage_ip)
if storage_ip == '-1':
return
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# connect and ask for all files
sock.connect((storage_ip, Constants.STORAGE_TO_STORAGE))
sock.send(str(Codes.download_all).encode('utf-8'))
# From this point we are going to receive a lot mkdir, and upload requests
# from another storage and in this socket we are just waiting
# When we receive ack then we are consistent with storage with 'storage_ip'
sock.recv(1024)
# Notify we are clear
logger.print_debug_info("Sending i_clear")
notify_i_clear()
sock.close()
def waiter(sock: socket.socket, is_namenode):
logger.print_debug_info("Wait on accept.", "is_namenode = ", is_namenode)
while True:
socket, addr = sock.accept()
if is_namenode:
logger.print_debug_info("This is Namenode connection", addr)
NamenodeListener(socket).start()
else:
logger.print_debug_info("This is Storage connection", addr)
StorageListener(socket, addr).start()
def new_nodes_listener_thread():
soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
soc.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
soc.bind(('', Constants.NEW_NODES_PORT))
soc.listen()
while True:
# get_all_storage_ips = 12
# init_new_storage = 13
# i_clear = 15
con, addr = soc.accept()
code = int(con.recv(1024).decode('utf-8'))
# ack
con.send('ack'.encode('utf-8'))
storagename = con.recv(1024).decode('utf-8')
if code == Codes.i_clear:
logger.print_debug_info("Node {} says it's clean".format(storagename))
dirty_nodes.nodes.discard(storagename)
clean_nodes.nodes.add(storagename)
elif code == Codes.init_new_storage:
if len(clean_nodes.nodes) > 0:
dirty_nodes.nodes.add(storagename)
copy_from = random_address()
con.send(copy_from.encode('utf-8'))
logger.print_debug_info("New storage node {} connected "
"and will copy from {}".format(storagename, copy_from))
else:
# sanity check
assert len(dirty_nodes.nodes) == 0
con.send('-1'.encode('utf-8'))
clean_nodes.nodes.add(storagename)
logger.print_debug_info("New storage node {} connected, system init".format(storagename))
elif code == Codes.get_all_storage_ips:
if len(clean_nodes.nodes) > 0:
res = ';'.join(clean_nodes.nodes)
else:
res = '-1'
con.send(res.encode('utf-8'))
logger.print_debug_info("Node {} requested ip list. Sending {}".format(storagename, res))
con.close()
def multicast(cmd, arg1='', arg2=''):
"""
threaded multicast to all storage servers
:param cmd:
:param arg1:
:param arg2:
:return:
"""
logger.print_debug_info("code", cmd)
logger.print_debug_info("dirty_nodes:", dirty_nodes)
logger.print_debug_info("clean_nodes:", clean_nodes)
# wait for all the dirty nodes to become clean
while len(dirty_nodes.nodes) > 0:
pass
for ip in clean_nodes.nodes:
thread = Thread(target=send_args, args=[ip, Constants.NAMENODE_TO_STORAGE, cmd, arg1, arg2])
thread.start()
new_nodes_listener.start()
new_nodes_listener()
ping()
tree = FSTree()
# Main thread
while True:
soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
soc.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
soc.bind(('', Constants.CLIENT_TO_NAMENODE))
soc.listen()
while True:
logger.print_debug_info("Waiting for a new client connection")
con, addr = soc.accept() # addr is a tuple
logger.print_debug_info('New client connection')
code = int(con.recv(1024).decode('utf-8'))
logger.print_debug_info('Received code ' + str(code))
con.send('ok'.encode('utf-8'))
if code == Codes.init: # init
del tree
tree = FSTree()
multicast(Codes.init)
logger.print_debug_info('init success')
elif code == Codes.make_file: # make_file (not dir)
