def main():
CNP.SetLogger(Logger)
NSP.SetLogger(Logger)
Storages.SetLogger(Logger)
LoadActual()
SetLocalNet()
Storages.StartFinder()
# Thread(daemon=True, target=incoming).start()
incoming()
def flush(sock: socket) -> ResultType:
Logger.addHost(*sock.getpeername(), 'attempts to flush storage')
space = 0
for ip in Storages.GetAliveServers():
rcvd = CallNSP(ip, NSP.flush)
if rcvd: space += rcvd
space //= ReplicationFactor
# All OK, flush actual
Actual.flush()
Storages.Flush()
# Add log and response
Logger.addHost(*sock.getpeername(), 'has flushed storage')
SendULong(sock, space)
return Result_Success
def copy(sock: socket) -> ResultType:
what = RecvStr(sock)
to = RecvStr(sock)
t = (what, to)
Logger.addHost(*sock.getpeername(), 'attempts to copy \'%s\' to \'%s\'' % t)
# Check if can be copied
if Actual.cantBeCopied(what, to):
SendResponse(sock, '\'%s\' cannot be copied to \'%s\' on remote' % t + Mes_UpdateLocal)
return Result_Denied
# Check if file
if _is_dir(sock, what):
return Result_Denied
# Can be copied
count = 0
for ip in Storages.GetASWithPath(what):
if CallNSP(ip, NSP.copy, what, to):
count += 1
# No server copied - no copy on actual
if count == 0:
SendResponse(sock, '\'%s\' cannot be copied to \'%s\' on remote' % t + Mes_UpdateLocal)
return Result_Denied
# All OK, copy on actual
Actual.copy(what, to)
# Add log and response
Logger.addHost(*sock.getpeername(), 'has copied \'%s\' to \'%s\'' % t)
SendResponse(sock, SUCCESS)
return Result_Success
def rename(sock: socket) -> ResultType:
path = RecvStr(sock)
name = RecvStr(sock)
t = (path, name)
Logger.addHost(*sock.getpeername(), 'attempts to rename \'%s\' to \'%s\'' % t)
# Check if can be renamed
if Actual.cantBeRenamed(path, name):
SendResponse(sock, '\'%s\' cannot be renamed to \'%s\' on remote' % t + Mes_UpdateLocal)
return Result_Denied
# Check if path is file
if _is_dir(sock, path):
return Result_Denied
# Can be renamed
count = 0
for ip in Storages.GetASWithPath(path):
if CallNSP(ip, NSP.rename, path, name):
count += 1
# No server renamed - no rename on actual
if count == 0:
SendResponse(sock, '\'%s\' cannot be renamed to \'%s\' on remote' % t + Mes_UpdateLocal)
return Result_Denied
# All OK, rename on actual
Actual.rename(path, name)
# Add log and response
Logger.addHost(*sock.getpeername(), 'has renamed \'%s\' to \'%s\'' % t)
SendResponse(sock, SUCCESS)
return Result_Success
def _replicate_from_one(loader: str, paths: list):
servers = {}
# Get dict of servers to replicate
for path in paths:
ips = Storages.GetASNoPath(path)
for ip in ips:
if ip in servers:
servers[ip].append(path)
else:
servers[ip] = [path]
# Try to replicate
for ip, pts in servers.items():
# Remove paths already replicated
pts = [p for p in pts if p in paths]
if not pts: continue
job = Jobs.new()
try:
if do_replicate(ip, loader, job, pts):
# Success, update storage data
fs = Storages.GetStorage(ip)
for p in pts:
if p in fs:
fs.remove(p)
fs.add(p, False)
# Remove replicated paths
paths = [path for path in paths if path not in pts]
# Break if all replicated
if not paths: break
except NSPException as e:
Logger.addError(
'A protocol error occurred during connecting to storage %s' %
ip, e)
except SocketError as e:
Logger.addError(
'A socket error occurred during connecting to storage %s' % ip,
e)
except VFSException as e:
Logger.addError(
'A VFS error occurred during connecting to storage %s' % ip, e)
except Exception as e:
Logger.addError(
'An unknown error occurred during connecting to storage %s' %
ip, e)
Jobs.complete(job)
return paths
def remove(sock: socket) -> ResultType:
path = RecvStr(sock)
Logger.addHost(*sock.getpeername(), 'attempts to remove \'%s\'' % path)
# Check if path can be deleted
if Actual.cantBeRemoved(path):
SendResponse(sock, '\'%s\' was already removed on remote' % path)
return Result_Denied
# Path can be deleted
for ip in Storages.GetASWithPath(path):
CallNSP(ip, NSP.remove, path)
# Remove on actual
Actual.remove(path)
# Remove path on all storages
Storages.RemovePath(path)
# Add log and response
Logger.addHost(*sock.getpeername(), 'has removed \'%s\'' % path)
SendResponse(sock, SUCCESS)
return Result_Success
def mkfile(sock: socket, log: str, path: str) -> bool:
SendStr(sock, path)
if LogResponse(sock, log):
ip = sock.getpeername()[0]
fs = Storages.GetStorage(ip)
if path in fs:
fs.remove(path)
fs.add(path, True)
return True
return False
def move(sock: socket, log: str, what: str, to: str) -> bool:
SendStr(sock, what)
SendStr(sock, to)
if LogResponse(sock, log):
ip = sock.getpeername()[0]
fs = Storages.GetStorage(ip)
name = fs.nodeAt(what).name
newpath = Join(to, name)
if newpath in fs:
fs.remove(newpath)
fs.move(what, to)
return True
return False
def upload(sock: socket) -> ResultType:
path = RecvStr(sock)
if _cant_add_node(sock, path):
return Result_Denied
# Create Job
job = Jobs.new(sock)
# Select server
loader = ''
for ip in Storages.GetAliveServers():
if CallNSP(ip, NSP.upload, job, path):
loader = ip
break
if not loader:
SendResponse(sock, '\'%s\' cannot be uploaded' % path)
Jobs.complete(job)
return Result_Denied
# Response client
loaderfs = Storages.GetStorage(loader)
SendResponse(sock, SUCCESS)
SendJob(sock, job)
SendStr(sock, loaderfs.pubip)
# Get answer from client
re = RecvResponse(sock)
# Complete job
Jobs.complete(job)
# Check answer
if re != SUCCESS:
return Result_Fail
# All OK, add on actual
Actual.add(path, False)
# Add to storage
if path in loaderfs:
loaderfs.remove(path)
loaderfs.add(path, False)
# Log
Logger.addHost(*sock.getpeername(), 'has added \'%s\'' % path)
# Init replication
NSP.ReplicateFromOne(loader, [path])
return Result_Success
def download(sock: socket) -> ResultType:
path = RecvStr(sock)
Logger.addHost(*sock.getpeername(), 'attempts to download \'%s\'' % path)
# Check if exists
if path not in Actual:
SendResponse(sock, '\'%s\' does not exist on remote' % path + Mes_UpdateLocal)
return Result_Denied
# Check if file
if _is_dir(sock, path):
return Result_Denied
# Can be downloaded
# Create Job
job = Jobs.new(sock)
# Select server
loader = ''
for ip in Storages.GetASWithPath(path):
if CallNSP(ip, NSP.download, job, path):
loader = ip
break
# No server has file - cannot be downloaded
if not loader:
SendResponse(sock, '\'%s\' cannot be downloaded' % path + Mes_UpdateLocal)
Jobs.complete(job)
return Result_Denied
# Response client
SendResponse(sock, SUCCESS)
SendJob(sock, job)
SendStr(sock, Storages.GetStorage(loader).pubip)
# Get answer from client
re = RecvResponse(sock)
# Complete job
Jobs.complete(job)
# Check answer
if re != SUCCESS:
return Result_Fail
# Log
Logger.addHost(*sock.getpeername(), 'has downloaded \'%s\'' % path)
return Result_Success
def rename(sock: socket, log: str, path: str, name: str) -> bool:
SendStr(sock, path)
SendStr(sock, name)
if LogResponse(sock, log):
ip = sock.getpeername()[0]
fs = Storages.GetStorage(ip)
names = fs.parsePath(path)[1]
names[-1] = name
newpath = Join(*names)
if newpath in fs:
fs.remove(newpath)
fs.rename(path, name)
return True
return False
def mkfile(sock: socket) -> ResultType:
path = RecvStr(sock)
if _cant_add_node(sock, path):
return Result_Denied
# Add file on ReplicationFactor servers
count = 0
for ip in Storages.GetAliveServers():
if CallNSP(ip, NSP.mkfile, path):
count += 1
if count == ReplicationFactor: break
# No server added - no update on actual
if count == 0:
SendResponse(sock, '\'%s\' cannot be created on remote' % path)
return Result_Denied
# Response
Actual.add(path, False)
return _node_added(sock, path)
def info(sock: socket) -> ResultType:
path = RecvStr(sock)
Logger.addHost(*sock.getpeername(), 'attempts to get stats of \'%s\'' % path)
if path not in Actual:
SendResponse(sock, '\'%s\' does not exist on remote' % path + Mes_UpdateLocal)
return Result_Denied
# Check if file
if _is_dir(sock, path):
return Result_Denied
# Exists
stats = None
for ip in Storages.GetASWithPath(path):
stats = CallNSP(ip, NSP.info, path)
if stats: break
if stats is None:
SendResponse(sock, '\'%s\' does not have stats' % path)
return Result_Denied
# All OK, response
Logger.addHost(*sock.getpeername(), 'has got stats of \'%s\'' % path)
SendResponse(sock, SUCCESS)
SendStr(sock, stats)
return Result_Success
def _replicate(paths: list):
servers = {}
# Get dict of servers where that paths exists
for path in paths:
ips = Storages.GetASWithPath(path)
for ip in ips:
if ip in servers:
servers[ip].append(path)
else:
servers[ip] = [path]
# Replicate until all replicated
for ip, pts in servers.items():
# Remove paths already replicated
pts = [p for p in pts if p in paths]
# Continue if no paths
if not pts: continue
# Replicate from current
remaining = _replicate_from_one(ip, pts)
# Remove replicated paths
paths = [
path for path in paths if (path not in pts) or (path in remaining)
]
# Break if all replicated
if not paths: break
def SetLocalNet():
net = GetNet(NetPath, 'Input network address where storages are situated')
Storages.SetNet(net)
ClientSocket = BindAndListen('', NameServerClientPort)
print('Server started')
while True:
sock, addr = Accept(ClientSocket)
Thread(daemon=True, target=serve, args=(sock, addr)).start()
def main():
CNP.SetLogger(Logger)
NSP.SetLogger(Logger)
Storages.SetLogger(Logger)
LoadActual()
SetLocalNet()
Storages.StartFinder()
# Thread(daemon=True, target=incoming).start()
incoming()
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
Logger.add('Application was interrupted')
except Exception as e:
Logger.addError('An error occurred', e)
finally:
SaveActual()
Storages.SaveData()
if isinstance(ClientSocket, socket):
ClientSocket.close()