def handle_read_req(conn, payload):
"""
Handles a read_req
"""
res_pkt = b''
try:
path = join(LOCAL_ROOT, payload['filename'])
offset = payload['offset']
cnt = payload['cnt']
f = open(path, 'rb')
except FileNotFoundError:
res_pkt = protocol.construct_packet(protocol.Verbs.READ_RES,
protocol.Status.ENOENT, {})
except Exception as ex:
logging.exception(ex)
res_pkt = protocol.construct_packet(protocol.Verbs.READ_RES,
protocol.Status.ENOENT, {})
else:
f.seek(offset)
buf = f.read(cnt)
res_pkt = protocol.construct_packet(protocol.Verbs.READ_RES,
protocol.Status.ERROR, {})
read_res = list(buf)
res_payload = {'bytes': read_res, 'cnt': len(read_res)}
res_pkt = protocol.construct_packet(protocol.Verbs.READ_RES,
protocol.Status.OK,
res_payload)
f.close()
finally:
conn.send(res_pkt)
def handle_write_req(conn, payload):
"""
Handle write request
"""
res_pkt = b''
try:
path = join(LOCAL_ROOT, payload['filename'])
offset = payload['offset']
data = bytes(payload['bytes'])
fd = os.open(path, os.O_WRONLY)
except FileNotFoundError:
res_pkt = protocol.construct_packet(protocol.Verbs.WRITE_RES,
protocol.Status.ENOENT, {})
except Exception as ex:
logging.exception(ex)
res_pkt = protocol.construct_packet(protocol.Verbs.WRITE_RES,
protocol.Status.ERROR, {})
else:
os.lseek(fd, offset, os.SEEK_SET)
writen = os.write(fd, data)
res_payload = {'cnt': writen}
res_pkt = protocol.construct_packet(protocol.Verbs.WRITE_RES,
protocol.Status.OK,
res_payload)
os.close(fd)
finally:
conn.send(res_pkt)
def handle_create(conn, payload):
"""
Handles CREATE: creates a file in the local filesystem
:param: conn: open connection
:param: payload: contains the filename of the file to send
:return: None
"""
try:
name = payload['filename']
except KeyError as ex:
logging.exception(ex)
return
pkt = b''
try:
open(join(LOCAL_ROOT, name), 'x').close()
except FileExistsError:
pkt = protocol.construct_packet(protocol.Verbs.ERROR,
protocol.Status.EEXIST, {})
except PermissionError:
pkt = protocol.construct_packet(protocol.Verbs.ERROR,
protocol.Status.EACCES, {})
except Exception as ex:
pkt = protocol.construct_packet(protocol.Verbs.ERROR,
protocol.Status.ERROR, {})
logging.exception(ex)
else:
pkt = protocol.construct_packet(protocol.Verbs.OK,
protocol.Status.OK, {})
finally:
conn.send(pkt)
def handle_stat_req(conn, payload):
"""
Handles a stat_req
:conn: connection to requesting node
:payload: contains the filename to stat
"""
res_pkt = b''
try:
stat = os.stat(join(LOCAL_ROOT, payload['filename']))
except FileNotFoundError as ex:
logging.exception(ex)
res_pkt = protocol.construct_packet(protocol.Verbs.STAT_RES,
protocol.Status.ENOENT, {})
except PermissionError as ex:
logging.exception(ex)
res_pkt = protocol.construct_packet(protocol.Verbs.STAT_RES,
protocol.Status.EACCES, {})
except Exception:
res_pkt = protocol.construct_packet(protocol.Verbs.STAT_RES,
protocol.Status.ERROR, {})
else:
res_payload = {'stat': stat}
res_pkt = protocol.construct_packet(protocol.Verbs.STAT_RES,
protocol.Status.OK,
res_payload)
finally:
conn.send(res_pkt)
def work_routine(bootstrap):
"""
Routine for worker threads. gets messages from queue and calls handler
"""
while True:
conn, addr = bootstrap.queue.get()
try:
buf = conn.recv(protocol.HEADER.sizeof())
req = protocol.HEADER.parse(buf)
payload = conn.recv(req.payload_size)
logging.info(f'received: {req}\n{payload}')
# JOIN
if req.verb == protocol.Verbs.JOIN.name:
handle_join(bootstrap, conn, addr, json.loads(payload))
# EXIT
elif req.verb == protocol.Verbs.EXIT.name:
handle_exit(bootstrap, conn, addr, json.loads(payload))
# Unsupported/invalid request
else:
logging.info(f'unsupported operation {req.verb}')
err_pkt = protocol.construct_packet(protocol.Verbs.ERROR,
protocol.Status.ERROR,
{'msg': 'unsupported operation'})
conn.send(err_pkt)
except ConstructError as ex:
logging.error(f'bad packet {req} from connection {addr}: {ex}')
finally:
conn.close()
def read(self, path, size, offset, fh):
"""
Read
"""
path = path[1:]
if path in os.listdir(LOCAL_ROOT):
with open(join(LOCAL_ROOT, path), 'rb') as f:
f.seek(offset)
return f.read(size)
try:
host_ip = protocol.lookup(self.bootstrap_ip, path)
req_payload = {'filename': path, 'cnt': size, 'offset': offset}
req_pkt = protocol.construct_packet(protocol.Verbs.READ_REQ,
protocol.Status.OK,
req_payload)
header, payload = protocol.sock_send_recv(host_ip, req_pkt)
if header.status != protocol.Status.OK.name:
if header.status == protocol.Status.ENOENT.name:
raise FileNotFoundError
elif header.status == protocol.Status.EACCES.name:
raise PermissionError
return bytes(payload['bytes'])
except ProtocolError as ex:
logging.exception(ex)
raise FuseOSError(errno.EIO)
except FileNotFoundError:
raise FuseOSError(errno.ENOENT)
except PermissionError:
raise FuseOSError(errno.EACCES)
def unlink(self, path):
"""
unlink: lookup the location of the file and send request to host. if local, remove.
"""
path = path[1:]
# check if local, if local send unlink req and return
if path in os.listdir(LOCAL_ROOT):
os.remove(join(LOCAL_ROOT, path))
return
req_payload = {'filename': path}
req_pkt = protocol.construct_packet(protocol.Verbs.UNLINK_REQ,
protocol.Status.OK, req_payload)
# not local, lookup
host_ip = protocol.lookup(self.bootstrap_ip, path)
try:
header, _ = protocol.sock_send_recv(host_ip, req_pkt)
if header.status != protocol.Status.OK.name:
if header.status == protocol.Status.ENOENT.name:
raise FuseOSError(errno.ENOENT)
elif header.status == protocol.Status.EACCES.name:
raise FuseOSError(errno.EACCES)
else:
raise FuseOSError(errno.EIO)
except IOError:
raise FuseOSError(errno.EIO)
def __init__(self, bootstrap_ip, fuse_mount):
"""
init the node server, join the cluster, construct the listening socket
"""
self.fuse_mount = fuse_mount
self.bootstrap_ip = bootstrap_ip
# establish server socket
self.sock = socket.socket()
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind((HOST, protocol.PORT))
self.sock.listen(5)
# Send JOIN to the bootstrap and initialize the Hashing data structures
req_pkt = protocol.construct_packet(protocol.Verbs.JOIN,
protocol.Status.OK, {})
try:
# send JOIN and receive IP_TABLE
header, payload = protocol.sock_send_recv(bootstrap_ip, req_pkt,
protocol.BOOTSTRAP_PORT)
logging.debug(f'received {header}')
if header.status != protocol.Status.OK.name:
raise IOError('Did not received status: ok')
# init the hashing object using the Bootstrap's data
ip_table = Hashing.deserializable_ip_table(payload['ip_table'])
hash_unit = sorted(ip_table.keys())
id_hash = Hashing.difuse_hash(payload['ip'])
hashing.HASHING_OBJ = Hashing(hash_unit, ip_table, id_hash)
logging.info(f'ID: {id_hash}')
# if files need to be relocated, relocate them
relocate_files()
except IOError as ex:
logging.exception(ex)
def write(self, path, data, offset, fh):
"""
Write
"""
path = path[1:]
if path in os.listdir(LOCAL_ROOT):
path = join(LOCAL_ROOT, path)
fd = os.open(path, os.O_WRONLY)
os.lseek(fd, offset, os.SEEK_SET)
writen = os.write(fd, data)
os.close(fd)
return writen
try:
host_ip = protocol.lookup(self.bootstrap_ip, path)
req_payload = {
'filename': path,
'offset': offset,
'bytes': list(data)
}
req_pkt = protocol.construct_packet(protocol.Verbs.WRITE_REQ,
protocol.Status.OK,
req_payload)
header, payload = protocol.sock_send_recv(host_ip, req_pkt)
logging.debug(header)
if header.status != protocol.Status.OK.name:
if header.status == protocol.Status.EACCES.name:
raise FuseOSError(errno.EACCES)
elif header.status == protocol.Status.ENOENT.name:
raise FuseOSError(errno.ENOENT)
else:
raise FuseOSError(errno.EIO)
return payload['cnt']
except (ProtocolError, KeyError):
raise FuseOSError(errno.EIO)
def destroy(self, path):
"""
Destroy: send EXIT to bootstrap, XFER files to the successor of this node and destroy the FUSE mount
"""
# notify bootstrap that we left
try:
req_pkt = protocol.construct_packet(protocol.Verbs.EXIT,
protocol.Status.OK, {})
protocol.sock_send_recv(self.bootstrap_ip, req_pkt,
protocol.BOOTSTRAP_PORT)
except ProtocolError:
pass
except IOError as ex:
logging.exception(ex)
pass
# remove our self so relocate_files will not calculate our self as the succ for any files
try:
hashing.HASHING_OBJ.remove_node(hashing.HASHING_OBJ.id_hash)
except ZeroDivisionError: # thrown if last node is exiting
pass
relocate_files(True)
# destroy
return super(Difuse, self).destroy(path)
def truncate(self, path, length, fh=None):
"""
Truncate
"""
path = path[1:]
if path in os.listdir(LOCAL_ROOT):
os.truncate(join(LOCAL_ROOT, path), length)
try:
host_ip = protocol.lookup(self.bootstrap_ip, path)
except FileNotFoundError:
raise FuseOSError(errno.ENOENT)
try:
req_payload = {'filename': path, 'len': length}
req_pkt = protocol.construct_packet(protocol.Verbs.TRUNC_REQ,
protocol.Status.OK,
req_payload)
header, payload = protocol.sock_send_recv(host_ip, req_pkt)
if header.status != protocol.Status.OK.name:
if header.status == protocol.Status.EACCES.name:
raise FuseOSError(errno.EACCES)
elif header.status == protocol.Status.ENOENT.name:
raise FuseOSError(errno.ENOENT)
else:
raise FuseOSError(errno.EIO)
except ProtocolError:
raise FuseOSError(errno.EIO)
def handle_trunc_req(conn, payload):
"""
handles trunc requet
"""
res_pkt = b''
try:
path = join(LOCAL_ROOT, payload['filename'])
size = payload['len']
os.truncate(path, size)
except Exception:
res_pkt = protocol.construct_packet(protocol.Verbs.TRUNC_RES,
protocol.Status.ERROR, {})
else:
res_pkt = protocol.construct_packet(protocol.Verbs.TRUNC_RES,
protocol.Status.OK, {})
finally:
conn.send(res_pkt)
def handle_readdir(conn, _payload):
'''
handles READDIR, responding to the connection with a list of the files on the local root
:param conn: open connection
:param payload: payload, empty
:return: None
'''
pkt = b''
try:
entries = os.listdir(LOCAL_ROOT)
except Exception:
pkt = protocol.construct_packet(protocol.Verbs.READDIR_RES,
protocol.Status.ERROR, {})
else:
pkt = protocol.construct_packet(protocol.Verbs.READDIR_RES,
protocol.Status.OK,
{'entries': entries})
finally:
conn.send(pkt)
def serve(self):
"""
Serves requests from other nodes
"""
while True:
conn, addr = self.sock.accept()
try:
req = conn.recv(protocol.HEADER.sizeof())
req = protocol.HEADER.parse(req)
payload = conn.recv(req.payload_size)
logging.info(f'received: {req}\n{payload}')
# STAT_REQ
if req.verb == protocol.Verbs.STAT_REQ.name:
self.handle_stat_req(conn, json.loads(payload))
# READ_REQ
elif req.verb == protocol.Verbs.READ_REQ.name:
self.handle_read_req(conn, json.loads(payload))
# WRITE_REQ
elif req.verb == protocol.Verbs.WRITE_REQ.name:
self.handle_write_req(conn, json.loads(payload))
# TRUNC
elif req.verb == protocol.Verbs.TRUNC_REQ.name:
self.handle_trunc_req(conn, json.loads(payload))
# UNLINK
elif req.verb == protocol.Verbs.UNLINK_REQ.name:
self.handle_unlink_req(conn, json.loads(payload))
# XFER
elif req.verb == protocol.Verbs.XFER.name:
self.handle_xfer_res(conn, json.loads(payload))
# NEW_NODE
elif req.verb == protocol.Verbs.NEW_NODE.name:
self.handle_new_node(conn, json.loads(payload))
# CREATE
elif req.verb == protocol.Verbs.CREATE.name:
self.handle_create(conn, json.loads(payload))
# LEFT_NODE
elif req.verb == protocol.Verbs.LEFT_NODE.name:
self.handle_left_node(conn, json.loads(payload))
# READDIR_REQ
elif req.verb == protocol.Verbs.READDIR_REQ.name:
self.handle_readdir(conn, json.loads(payload))
# Unsupported/Unexpected request
else:
err_pkt = protocol.construct_packet(
protocol.Verbs.ERROR, protocol.Status.ERROR,
{'msg': 'unsupported operation'})
conn.send(err_pkt)
except ConstructError:
logging.error(f'bad packet from {addr}')
finally:
conn.close()
def handle_unlink_req(self, conn, payload):
"""
Handles unlink request, removes file and responds to the conn
"""
res_pkt = b''
try:
os.unlink(join(LOCAL_ROOT, payload['filename']))
except FileNotFoundError as ex:
logging.exception(ex)
res_pkt = protocol.construct_packet(protocol.Verbs.UNLINK_RES,
protocol.Status.ENOENT, {})
except PermissionError as ex:
logging.exception(ex)
res_pkt = protocol.construct_packet(protocol.Verbs.UNLINK_RES,
protocol.Status.EACCES, {})
except Exception as ex:
logging.exception(ex)
res_pkt = protocol.construct_packet(protocol.Verbs.UNLINK_RES,
protocol.Status.ERROR, {})
else:
res_pkt = protocol.construct_packet(protocol.Verbs.UNLINK_RES,
protocol.Status.OK, {})
finally:
conn.send(res_pkt)
def handle_xfer_res(self, conn, payload):
"""
Handles the XFER_RES, copying all the files in the payload to the local root
:param conn: connection
:param payload: payload: list of files
:return: None
"""
files_list = payload['files']
for file_dict in files_list:
try:
with open(join(LOCAL_ROOT, file_dict['filename']), 'wb') as f:
f.write(bytes(file_dict['bytes']))
except Exception as ex:
logging.exception(ex)
pkt = protocol.construct_packet(protocol.Verbs.OK, protocol.Status.OK,
{})
conn.send(pkt)
def relocate_files(relocate_all=False):
"""
Scans the LOCAL_ROOT directory for files that need to be relocated and sends XFER_RES to each node where files
should be transferred.
:relocate_all: Default false. If true, all files on local root will be transferred to the successor
:return: None
"""
# table that maps an ID hash to a list of dictionary to send
xfer_res_table = {id_hash: [] for id_hash in hashing.HASHING_OBJ.hash_unit}
# build xfer res table for each file in the local root
for filename in os.listdir(LOCAL_ROOT):
file_hash = Hashing.difuse_hash(filename)
try:
file_succ = Hashing.succ(file_hash)
except ZeroDivisionError: # thrown if last node exiting
return
logging.debug(f'filename: {filename} | file_hash: {file_hash}')
if relocate_all or file_succ != hashing.HASHING_OBJ.id_hash:
try:
with open(join(LOCAL_ROOT, filename), 'rb') as f:
file_bytes = f.read()
xfer_res_table[file_succ].append({
'filename': filename,
'bytes': list(file_bytes)
})
except Exception as ex:
logging.exception(ex)
pass
# send the XFER_RES
for id_hash, xfer_list in xfer_res_table.items():
# no files to transfer, do not send anything
if not xfer_list:
continue
conn_ip = hashing.HASHING_OBJ.ip_table[id_hash]
logging.info(f'relocating {xfer_list} to {conn_ip}')
xfer_res_pkt = protocol.construct_packet(protocol.Verbs.XFER, protocol.Status.OK, {'files': xfer_list})
try:
protocol.sock_send_recv(conn_ip, xfer_res_pkt)
except Exception as ex:
logging.exception(ex)
else:
for xfer_file in xfer_list:
os.remove(join(LOCAL_ROOT, xfer_file['filename']))
def create(self, path, mode, fi=None):
"""
:param path: filename
:param mode: mode to creat
:param fi: no idea
:return:
"""
path = path[1:]
hash_id = hashing.HASHING_OBJ.difuse_hash(path)
succ = hashing.HASHING_OBJ.succ(hash_id)
host_ip = hashing.HASHING_OBJ.ip_table[succ]
# if the successor is local id, create the file here
if succ == hashing.HASHING_OBJ.id_hash:
try:
open(join(LOCAL_ROOT, path), 'xb').close()
except OSError as ex:
raise FuseOSError(ex.errno)
else:
self.fd += 1
return self.fd
# non local creation necessary
logging.info(f'sending create {path} to {host_ip}')
payload = {'filename': path}
pkt = protocol.construct_packet(protocol.Verbs.CREATE,
protocol.Status.OK, payload)
try:
header, payload = protocol.sock_send_recv(host_ip, pkt)
if header.status == protocol.Status.EEXIST.name:
raise FileExistsError
if header.status == protocol.Status.EACCES.name:
raise PermissionError
if header.status != protocol.Status.OK.name:
raise FuseOSError(errno.EIO)
except FileExistsError:
raise FuseOSError(errno.EEXIST)
except PermissionError:
raise FuseOSError(errno.EACCES)
except ProtocolError:
raise FuseOSError(errno.EIO)
else:
self.fd += 1
return self.fd
def handle_exit(_bootstrap, conn, addr, _payload):
"""
Handles EXIT verb.
Removes the entry from the IP Table and broadcasts LEFT_NODE to all nodes in the cluster
"""
# remove node
hashing.HASHING_OBJ.remove_node(Hashing.difuse_hash(addr[0]))
# send ack
try:
res_pkt = protocol.construct_packet(protocol.Verbs.OK,
protocol.Status.OK,
{})
conn.send(res_pkt)
except IOError as ex:
logging.exception(ex)
# bcast LEFT_NODE
protocol.broadcast_no_recv(protocol.Verbs.LEFT_NODE, {'ip': addr[0]})
def handle_join(_bootstrap, conn, addr, _payload):
"""
Handles the JOIN protocol verb
"""
logging.info(f'JOIN request from {addr}')
# add new node to IP Table
hashing.HASHING_OBJ.add_node(addr[0])
# send NEW_NODE to cluster
protocol.broadcast_no_recv(protocol.Verbs.NEW_NODE, {'ip': addr[0]})
# send ip table to joining conn
ip_table_payload = {
'ip': addr[0],
'ip_table': hashing.HASHING_OBJ.serializable_ip_table()
}
logging.info(f'IP_TABLE: {hashing.HASHING_OBJ.ip_table}')
ip_table_pkt = protocol.construct_packet(protocol.Verbs.IP_TABLE,
protocol.Status.OK,
ip_table_payload)
conn.send(ip_table_pkt)
def readdir(self, _path, fh):
"""
readdir: broadcasts a READDIR_REQ to every node in the cluster and collects the results of all the READDIR_RES
it receives
"""
# create list of sockets to use for connection
conns = [
socket() for _ in range(len(hashing.HASHING_OBJ.ip_table.values()))
]
# broadcast READ DIR
for conn, host_ip in zip(conns, hashing.HASHING_OBJ.ip_table.values()):
try:
conn.connect((host_ip, protocol.PORT))
pkt = protocol.construct_packet(protocol.Verbs.READDIR_REQ,
protocol.Status.OK, {})
conn.send(pkt)
except Exception as ex:
logging.debug(f'error connecting to {host_ip}')
logging.exception(ex)
continue
entries = []
# collect the READDIR_RES'
for conn in conns:
try:
header = protocol.HEADER.parse(
conn.recv(protocol.HEADER.sizeof()))
payload = json.loads(conn.recv(header.payload_size))
if header.status != protocol.Status.OK.name:
continue
entries += payload['entries']
except (ConstructError, OSError, StreamError) as ex:
logging.exception(ex)
continue
finally:
conn.close()
return ['.', '..'] + sorted(entries)
def getattr(self, path, fh=None):
"""
stat/getattr syscall
"""
if path == '/':
return self._stat_dict(os.stat(LOCAL_ROOT))
path = path[1:]
if path in os.listdir(LOCAL_ROOT):
stat = os.stat(join(LOCAL_ROOT, path))
stat_dict = self._stat_dict(stat)
stat_dict['st_uid'] = os.getuid()
stat_dict['st_gid'] = os.getgid()
return stat_dict
host_ip = protocol.lookup(self.bootstrap_ip, path)
try:
pkt = protocol.construct_packet(protocol.Verbs.STAT_REQ,
protocol.Status.OK,
{'filename': path})
header, payload = protocol.sock_send_recv(host_ip, pkt)
logging.debug(f'getattr received {header}')
if header.verb != protocol.Verbs.STAT_RES.name:
raise FuseOSError(errno.EIO)
if header.status == protocol.Status.ENOENT.name:
raise FuseOSError(errno.ENOENT)
stat = os.stat_result(payload['stat'])
stat_dict = self._stat_dict(stat)
stat_dict['st_uid'] = os.getuid()
stat_dict['st_gid'] = os.getgid()
return stat_dict
except FuseOSError:
raise
except (ProtocolError, Exception) as ex:
logging.exception(ex)
raise FuseOSError(errno.EIO) from ex
