class SocketCommunicator(): def __init__(self, maxUnacceptConnections=5): self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_address = ('', 10000) self.sock.bind(server_address) self.sock.listen(maxUnacceptConnections) self.theData = [] self.theDataLock = Lock() self.shutdown_event = Event() self.theThread = None self.connections = LRU(10) def startServer(self): if self.theThread is not None: return def start(): while not self.shutdown_event.is_set(): print('waiting for connection...') connection, client_address = self.sock.accept() print('accepted connection from: {}'.format(client_address)) data = connection.recv(1) if data: data = data.decode() print(data) print() if self.connections.get(client_address) is None: self.connections[client_address] = data with self.theDataLock: self.theData.append(data) self.sock.close() servThread = Thread(target=start, daemon=True) servThread.start() self.theThread = servThread def sendMessage(self, addr, port, data): connection = socket.create_connection((addr, port)) connection.sendall(data) connection.close() def stop(self): self.shutdown_event.set() if self.theThread is not None: self.theThread.join() def getTheData(self): with self.theDataLock: tmp = self.theData.copy() self.theData = [] return tmp def whoSent(self, data): items = self.connections.items() for addr, msg in items: if msg==data: return addr return None
class Drawings: def __init__(self, max_active): self.drawings = LRU(max_active) def get_drawing(self, room): for (k, v) in self.drawings.items(): logger.debug('{}: {} (len {})'.format(id(self), k, len(v))) try: d = self.drawings[room] except KeyError: d = self.drawings[room] = [] return d
class Drawings: def __init__(self, max_active): self.drawings = LRU(max_active) def get_drawing(self, room): for (k, v) in self.drawings.items(): logger.debug('{}: {} (len {})'.format(id(self), k , len(v))) try: d = self.drawings[room] except KeyError: d = self.drawings[room] = [ ] return d
class FileServer(fileService_pb2_grpc.FileserviceServicer): def __init__(self, hostname, server_port, activeNodesChecker, shardingHandler, superNodeAddress): self.serverPort = server_port self.serverAddress = hostname + ":" + server_port self.activeNodesChecker = activeNodesChecker self.shardingHandler = shardingHandler self.hostname = hostname self.lru = LRU(5) self.superNodeAddress = superNodeAddress # # This service gets invoked when user uploads a new file. # def UploadFile(self, request_iterator, context): print("Inside Server method ---------- UploadFile") data = bytes("", 'utf-8') username, filename = "", "" totalDataSize = 0 active_ip_channel_dict = self.activeNodesChecker.getActiveChannels() # list to store the info related to file location. metaData = [] # If the node is the leader of the cluster. if (int(db.get("primaryStatus")) == 1): print("Inside primary upload") currDataSize = 0 currDataBytes = bytes("", 'utf-8') seqNo = 1 # Step 1: # Get 2 least loaded nodes based on the CPU stats. # 'Node' is where the actual data goes and 'node_replica' is where replica will go. node, node_replica = self.getLeastLoadedNode() if (node == -1): return fileService_pb2.ack( success=False, message="Error Saving File. No active nodes.") # Step 2: # Check whether file already exists, if yes then return with message 'File already exists'. for request in request_iterator: username, filename = request.username, request.filename print("Key is-----------------", username + "_" + filename) if (self.fileExists(username, filename) == 1): print("sending neg ack") return fileService_pb2.ack( success=False, message= "File already exists for this user. Please rename or delete file first." ) break # Step 3: # Make chunks of size 'UPLOAD_SHARD_SIZE' and start sending the data to the least utilized node trough gRPC streaming. currDataSize += sys.getsizeof(request.data) currDataBytes += request.data for request in request_iterator: if ((currDataSize + sys.getsizeof(request.data)) > UPLOAD_SHARD_SIZE): response = self.sendDataToDestination( currDataBytes, node, node_replica, username, filename, seqNo, active_ip_channel_dict[node]) metaData.append([node, seqNo, node_replica]) currDataBytes = request.data currDataSize = sys.getsizeof(request.data) seqNo += 1 node, node_replica = self.getLeastLoadedNode() else: currDataSize += sys.getsizeof(request.data) currDataBytes += request.data if (currDataSize > 0): response = self.sendDataToDestination( currDataBytes, node, node_replica, username, filename, seqNo, active_ip_channel_dict[node]) metaData.append([node, seqNo, node_replica]) # Step 4: # Save the metadata on the primary node after the completion of sharding. if (response.success): db.saveMetaData(username, filename, metaData) db.saveUserFile(username, filename) # Step 5: # Make a gRPC call to replicate the matadata on all the other nodes. self.saveMetadataOnAllNodes(username, filename, metaData) return fileService_pb2.ack(success=True, message="Saved") # If the node is not the leader. else: print("Saving the data on my local db") sequenceNumberOfChunk = 0 dataToBeSaved = bytes("", 'utf-8') # Gather all the data from gRPC stream for request in request_iterator: username, filename, sequenceNumberOfChunk = request.username, request.filename, request.seqNo dataToBeSaved += request.data key = username + "_" + filename + "_" + str(sequenceNumberOfChunk) # Save the data in local DB. db.setData(key, dataToBeSaved) # After saving the chunk in the local DB, make a gRPC call to save the replica of the chunk on different # node only if the replicaNode is present. if (request.replicaNode != ""): print("Sending replication to ", request.replicaNode) replica_channel = active_ip_channel_dict[request.replicaNode] t1 = Thread(target=self.replicateChunkData, args=( replica_channel, dataToBeSaved, username, filename, sequenceNumberOfChunk, )) t1.start() # stub = fileService_pb2_grpc.FileserviceStub(replica_channel) # response = stub.UploadFile(self.sendDataInStream(dataToBeSaved, username, filename, sequenceNumberOfChunk, "")) return fileService_pb2.ack(success=True, message="Saved") def replicateChunkData(self, replica_channel, dataToBeSaved, username, filename, sequenceNumberOfChunk): stub = fileService_pb2_grpc.FileserviceStub(replica_channel) response = stub.UploadFile( self.sendDataInStream(dataToBeSaved, username, filename, sequenceNumberOfChunk, "")) # This helper method is responsible for sending the data to destination node through gRPC stream. def sendDataToDestination(self, currDataBytes, node, nodeReplica, username, filename, seqNo, channel): if (node == self.serverAddress): key = username + "_" + filename + "_" + str(seqNo) db.setData(key, currDataBytes) if (nodeReplica != ""): print("Sending replication to ", nodeReplica) active_ip_channel_dict = self.activeNodesChecker.getActiveChannels( ) replica_channel = active_ip_channel_dict[nodeReplica] stub = fileService_pb2_grpc.FileserviceStub(replica_channel) response = stub.UploadFile( self.sendDataInStream(currDataBytes, username, filename, seqNo, "")) return response else: print("Sending the UPLOAD_SHARD_SIZE to node :", node) stub = fileService_pb2_grpc.FileserviceStub(channel) response = stub.UploadFile( self.sendDataInStream(currDataBytes, username, filename, seqNo, nodeReplica)) print("Response from uploadFile: ", response.message) return response # This helper method actually makes chunks of less than 4MB and streams them through gRPC. # 4 MB is the max data packet size in gRPC while sending. That's why it is necessary. def sendDataInStream(self, dataBytes, username, filename, seqNo, replicaNode): chunk_size = 4000000 start, end = 0, chunk_size while (True): chunk = dataBytes[start:end] if (len(chunk) == 0): break start = end end += chunk_size yield fileService_pb2.FileData(username=username, filename=filename, data=chunk, seqNo=seqNo, replicaNode=replicaNode) # # This service gets invoked when user requests an uploaded file. # def DownloadFile(self, request, context): print("Inside Download") # If the node is the leader of the cluster. if (int(db.get("primaryStatus")) == 1): print("Inside primary download") # Check if file exists if (self.fileExists(request.username, request.filename) == 0): print("File does not exist") yield fileService_pb2.FileData(username=request.username, filename=request.filename, data=bytes("", 'utf-8'), seqNo=0) return # If the file is present in cache then just fetch it and return. No need to go to individual node. if (self.lru.has_key(request.username + "_" + request.filename)): print("Fetching data from Cache") CHUNK_SIZE = 4000000 fileName = request.username + "_" + request.filename filePath = self.lru[fileName] outfile = os.path.join(filePath, fileName) with open(outfile, 'rb') as infile: while True: chunk = infile.read(CHUNK_SIZE) if not chunk: break yield fileService_pb2.FileData( username=request.username, filename=request.filename, data=chunk, seqNo=1) # If the file is not present in the cache, then fetch it from the individual node. else: print("Fetching the metadata") # Step 1: get metadata i.e. the location of chunks. metaData = db.parseMetaData(request.username, request.filename) print(metaData) #Step 2: make gRPC calls and get the fileData from all the nodes. downloadHelper = DownloadHelper(self.hostname, self.serverPort, self.activeNodesChecker) data = downloadHelper.getDataFromNodes(request.username, request.filename, metaData) print("Sending the data to client") #Step 3: send the file to supernode using gRPC streaming. chunk_size = 4000000 start, end = 0, chunk_size while (True): chunk = data[start:end] if (len(chunk) == 0): break start = end end += chunk_size yield fileService_pb2.FileData(username=request.username, filename=request.filename, data=chunk, seqNo=request.seqNo) # Step 4: update the cache based on LRU(least recently used) algorithm. self.saveInCache(request.username, request.filename, data) # If the node is not the leader, then just fetch the fileChunk from the local db and stream it back to leader. else: key = request.username + "_" + request.filename + "_" + str( request.seqNo) print(key) data = db.getFileData(key) chunk_size = 4000000 start, end = 0, chunk_size while (True): chunk = data[start:end] if (len(chunk) == 0): break start = end end += chunk_size yield fileService_pb2.FileData(username=request.username, filename=request.filename, data=chunk, seqNo=request.seqNo) # This service is responsible fetching all the files. def FileList(self, request, context): print("File List Called") userFiles = db.getUserFiles(request.username) return fileService_pb2.FileListResponse(Filenames=str(userFiles)) # This helper method checks whether the file is present in db or not. def fileExists(self, username, filename): print("isFile Present", db.keyExists(username + "_" + filename)) return db.keyExists(username + "_" + filename) # This helper method returns 2 least loaded nodes from the cluster. def getLeastLoadedNode(self): print("Ready to enter sharding handler") node, node_replica = self.shardingHandler.leastUtilizedNode() print("Least loaded node is :", node) print("Replica node - ", node_replica) return node, node_replica # This helper method replicates the metadata on all nodes. def saveMetadataOnAllNodes(self, username, filename, metadata): print("saveMetadataOnAllNodes") active_ip_channel_dict = self.activeNodesChecker.getActiveChannels() uniqueFileName = username + "_" + filename for ip, channel in active_ip_channel_dict.items(): if (self.isChannelAlive(channel)): stub = fileService_pb2_grpc.FileserviceStub(channel) response = stub.MetaDataInfo( fileService_pb2.MetaData( filename=uniqueFileName, seqValues=str(metadata).encode('utf-8'))) print(response.message) # This service is responsible for saving the metadata on local db. def MetaDataInfo(self, request, context): print("Inside Metadatainfo") fileName = request.filename seqValues = request.seqValues db.saveMetaDataOnOtherNodes(fileName, seqValues) ack_message = "Successfully saved the metadata on " + self.serverAddress return fileService_pb2.ack(success=True, message=ack_message) # This helper method checks whethere created channel is alive or not def isChannelAlive(self, channel): try: grpc.channel_ready_future(channel).result(timeout=1) except grpc.FutureTimeoutError: #print("Connection timeout. Unable to connect to port ") return False return True # This helper method is responsible for updating the cache for faster lookup. def saveInCache(self, username, filename, data): if (len(self.lru.items()) >= self.lru.get_size()): fileToDel, path = self.lru.peek_last_item() os.remove(path + "/" + fileToDel) self.lru[username + "_" + filename] = "cache" filePath = os.path.join('cache', username + "_" + filename) saveFile = open(filePath, 'wb') saveFile.write(data) saveFile.close() # This service is responsible for sending the whole cluster stats to superNode def getClusterStats(self, request, context): print("Inside getClusterStats") active_ip_channel_dict = self.activeNodesChecker.getActiveChannels() total_cpu_usage, total_disk_space, total_used_mem = 0.0, 0.0, 0.0 total_nodes = 0 for ip, channel in active_ip_channel_dict.items(): if (self.isChannelAlive(channel)): stub = heartbeat_pb2_grpc.HearBeatStub(channel) stats = stub.isAlive(heartbeat_pb2.NodeInfo(ip="", port="")) total_cpu_usage = float(stats.cpu_usage) total_disk_space = float(stats.disk_space) total_used_mem = float(stats.used_mem) total_nodes += 1 if (total_nodes == 0): return fileService_pb2.ClusterStats(cpu_usage=str(100.00), disk_space=str(100.00), used_mem=str(100.00)) return fileService_pb2.ClusterStats( cpu_usage=str(total_cpu_usage / total_nodes), disk_space=str(total_disk_space / total_nodes), used_mem=str(total_used_mem / total_nodes)) # This service is responsible for sending the leader info to superNode as soon as leader changes. def getLeaderInfo(self, request, context): channel = grpc.insecure_channel('{}'.format(self.superNodeAddress)) stub = fileService_pb2_grpc.FileserviceStub(channel) response = stub.getLeaderInfo( fileService_pb2.ClusterInfo(ip=self.hostname, port=self.serverPort, clusterName="team1")) print(response.message) # # This service gets invoked when user deletes a file. # def FileDelete(self, request, data): username = request.username filename = request.filename if (int(db.get("primaryStatus")) == 1): if (self.fileExists(username, filename) == 0): print("File does not exist") return fileService_pb2.ack(success=False, message="File does not exist") print("Fetching metadata from leader") metadata = db.parseMetaData(request.username, request.filename) print("Successfully retrieved metadata from leader") deleteHelper = DeleteHelper(self.hostname, self.serverPort, self.activeNodesChecker) deleteHelper.deleteFileChunksAndMetaFromNodes( username, filename, metadata) return fileService_pb2.ack( success=True, message="Successfully deleted file from the cluster") else: seqNo = -1 try: seqNo = request.seqNo except: return fileService_pb2.ack(success=False, message="Internal Error") metaDataKey = username + "_" + filename dataChunkKey = username + "_" + filename + "_" + str(seqNo) if (db.keyExists(metaDataKey) == 1): print("FileDelete: Deleting the metadataEntry from local db :") db.deleteEntry(metaDataKey) if (db.keyExists(dataChunkKey)): print("FileDelete: Deleting the data chunk from local db: ") db.deleteEntry(dataChunkKey) return fileService_pb2.ack( success=True, message="Successfully deleted file from the cluster") # # This service gets invoked when user wants to check if the file is present. # def FileSearch(self, request, data): username, filename = request.username, request.filename if (self.fileExists(username, filename) == 1): return fileService_pb2.ack(success=True, message="File exists in the cluster.") else: return fileService_pb2.ack( success=False, message="File does not exist in the cluster.") # # This service gets invoked when user wants to update a file. # def UpdateFile(self, request_iterator, context): username, filename = "", "" fileData = bytes("", 'utf-8') for request in request_iterator: fileData += request.data username, filename = request.username, request.filename def getFileChunks(fileData): # Maximum chunk size that can be sent CHUNK_SIZE = 4000000 outfile = os.path.join('files', fileName) sTime = time.time() while True: chunk = fileData.read(CHUNK_SIZE) if not chunk: break yield fileService_pb2.FileData(username=username, filename=fileName, data=chunk, seqNo=1) print("Time for upload= ", time.time() - sTime) if (int(db.get("primaryStatus")) == 1): channel = grpc.insecure_channel('{}'.format(self.serverAddress)) stub = fileService_pb2_grpc.FileserviceStub(channel) response1 = stub.FileDelete( fileService_pb2.FileInfo(username=userName, filename=fileName)) if (response1.success): response2 = stub.UploadFile(getFileChunks(fileData)) if (response2.success): return fileService_pb2.ack( success=True, message="File suceessfully updated.") else: return fileService_pb2.ack(success=False, message="Internal error.") else: return fileService_pb2.ack(success=False, message="Internal error.")
fileName="CertificateLabSafetyRustemCanAygunUCLA.pdf"; #fileSize=os.path.getsize(fileName) digest=hashlib.md5(fileName).hexdigest() key=me+digest #record1=CacheData(key,None) #dictionary #cacheDictionary = {record1.key: record1} #print(cacheDictionary[record1.key].key) #print(key) #fileSize=os.path.getsize(fileName) ## LRU l = LRU(5) for i in range(5): l[i] = str(i) print (l.items())
class Cache: # Replacement policies LRU = "LRU" FIFO = 'FIFO' def __init__(self, name, size, policy): self.name = name self.size = size self.free_space = size self.policy = policy # Eviction policy self.hashmap = {} # Mapping <objname,objsize> if (self.policy == Cache.LRU): self.cache = LRU(self.size) elif (self.policy == Cache.FIFO): self.cache = queue.Queue(maxsize=self.size) # Statistics self.hit_count = 0 self.miss_count = 0 def has_key(self, key): if key in self.hashmap.keys(): return True else: return False def update(self, key, size): self.hashmap[key] = size self.hit_count += 1 if (self.policy == Cache.LRU): self.cache.update(key=size) elif (self.policy == Cache.FIFO): self.cache.put(key) def insert(self, key, size, directory): if (self.policy == Cache.LRU): self.insertLRU(key, size, directory) elif (self.policy == Cache.FIFO): self.insertFIFO(key, size, directory) def evictLRU(self, directory): oid = self.cache.peek_last_item()[0] directory.removeBlock(oid, self.name) del [oid] del self.hashmap[oid] self.free_space += int(self.hashmap[oid]) def evictFIFO(self, directory): oid = self.cache.get() directory.removeBlock(oid, self.name) self.free_space += int(self.hashmap[oid]) del self.hashmap[oid] def insertLRU(self, key, size, directory): while (int(size) >= self.free_space): self.evictLRU(directory) self.cache[key] = size self.hashmap[key] = size self.free_space += size self.miss_count += 1 def insertFIFO(self, key, size, directory): while (int(size) >= self.free_space): self.evictFIFO(directory) self.cache.put(key) self.hashmap[key] = size self.free_space += size self.miss_count += 1 def put(self, key, size, directory): if self.has_key(key): self.update(key, size) else: self.insert(key, size, directory) def print(self): if (self.policy == Cache.LRU): print(self.name, "LRU", self.hashmap, self.cache.items()) elif (self.policy == Cache.FIFO): print(self.name, "LRU", self.hashmap, list(self.cache.queue)) def remove(self, key): del self.hashmap[key] if (self.policy == Cache.LRU): del self.cache[key] elif (self.policy == Cache.FIFO): a = 5
from lru import LRU l = LRU(5) # Create an LRU container that can hold 5 items print(l.peek_first_item(), l.peek_last_item()) #return the MRU key and LRU key # Would print None None for i in range(5): l[i] = str(i) print(l.items()) # Prints items in MRU order # Would print [(4, '4'), (3, '3'), (2, '2'), (1, '1'), (0, '0')] print(l.peek_first_item(), l.peek_last_item()) #return the MRU key and LRU key # Would print (4, '4') (0, '0') l[5] = '5' # Inserting one more item should evict the old item print(l.items()) # Would print [(5, '5'), (4, '4'), (3, '3'), (2, '2'), (1, '1')] l[3] # Accessing an item would make it MRU print(l.items()) # Would print [(3, '3'), (5, '5'), (4, '4'), (2, '2'), (1, '1')] # Now 3 is in front l.keys() # Can get keys alone in MRU order # Would print [3, 5, 4, 2, 1] del l[4] # Delete an item print(l.items()) # Would print [(3, '3'), (5, '5'), (2, '2'), (1, '1')] print(l.get_size())