def copyFromDFS(address, fname, path): """ Contact the metadata server to ask for the file blocks of the file fname. Get the data blocks from the data nodes. Saves the data in path. """ # Contact the metadata server to ask for information of fname sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(address) # Fill code p = Packet() get_packet = p.BuildGetPacket(fname) sock.sendall(p.getEncodedPacket().encode()) # If there is no error response Retreive the data blocks msg = sock.recv(1024).decode('utf-8') # danger msg=eval(msg) # Decode the packet received sock.close() # Fill code if msg != 'DUP': n = len(msg) with open(path, 'w') as f: for i in range(n): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) data_node = msg[i] ip=data_node[0] port=data_node[1] chunkid=data_node[2] sock.connect((ip,port)) p.BuildGetDataBlockPacket(chunkid) sock.sendall(p.getEncodedPacket().encode()) data = sock.recv(1024).decode() sock.close() f.write(data)
def copyFromDFS(address, fname, path): """ Contact the metadata server to ask for the file blocks of the file fname. Get the data blocks from the data nodes. Saves the data in path. """ print "copy from dfs" # Contact the metadata server to ask for information of fname print "meta address ", address print "fname of file i want to copy ", fname print "path of new file (block destiny) ", path p = Packet() p.BuildGetPacket(fname) # request metadata fname's inode sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((address[0], address[1])) sock.sendall(p.getEncodedPacket()) metadata = sock.recv(1024) # recieve file = open(path, 'wb') # Ifs there is no error response Retreive the data blocks print "estamoredi ", metadata p.DecodePacket(metadata) servers = p.getDataNodes() print "servers ", servers for dserver in servers: print "dserver ", dserver pack = Packet() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((dserver[0], dserver[1])) pack.BuildGetDataBlockPacket(dserver[2]) s.sendall(pack.getEncodedPacket()) dblock_size = s.recv(1024) print "dblock-size ", dblock_size s.sendall("OK") # send ok to copy after recieving dblock dblock = "" # string to store the info to be written to the new file while len(dblock) < int(dblock_size): dblock_chunk = s.recv(1024) print "len of recieved dblock chunk ", len(dblock_chunk) print "dblock chunk recieved" dblock = dblock + dblock_chunk # print "dblock after chunk appended ", dblock print "dblock len after chunk appended ", len(dblock) s.sendall("OK") # Save the file file.write(dblock) s.close() file.close()
def copyFromDFS(address, fname, path): """ Contact the metadata server to ask for the file blocks of the file fname. Get the data blocks from the data nodes. Saves the data in path. """ data = fname.split('.') file_extension = data[1] # Contact the metadata server to ask for information of fname sock = socket.socket() sock.connect(address) p = Packet() p.BuildGetPacket(fname) sock.sendall(p.getEncodedPacket()) # Fill code # If there is no error response Retreive the data blocks response = sock.recv(1024) # from meta data server sock.close() sock = None p.DecodePacket(response) blocks = p.packet["servers"] chunks_toSave = [] # chunks that are going to be saved in file 'path' for host, port, chunk_id in blocks: print(host, port, chunk_id) sock = socket.socket() sock.connect((host, port)) p.BuildGetDataBlockPacket(chunk_id + '.' + file_extension) sock.sendall(p.getEncodedPacket()) data_block = sock.recv(1024) try: data_block = data_block.decode() except: print(data_block) chunks_toSave.append(data_block) print(data_block, 'me le devuelve el data node') sock.close() sock = None try: fd = open(path, "w") for chunk in chunks_toSave: fd.write(chunk) fd.close() except: fd = open(path, "wb") for chunk in chunks_toSave: fd.write(chunk) fd.close()
def copyFromDFS(address, fname, path): """ Contact the metadata server to ask for the file blocks of the file fname. Get the data blocks from the data nodes. Saves the data in path. """ # Contact the metadata server to ask for information of fname sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) #connecting to meta-data server. sock.connect(address) #building packet with the fname and the command get. p = Packet() p.BuildGetPacket(fname) sock.sendall(p.getEncodedPacket()) #print("Received") #Receiving meta-data server response respond = sock.recv(4096) #print("you've got mail:", respond) #If the file is in the inode table in the database, then it look for the #Datanode information. if respond != "NFOUND": packetres = Packet() packetres.DecodePacket(respond) print("Datablocks information decoded") DataNodes = packetres.getDataNodes() with open(path, "a+b") as file: for n in DataNodes: soky = socket.socket(socket.AF_INET, socket.SOCK_STREAM) print(n[:2]) n, m = tuple(n[:2]), n[2] #print("server:", n, "chunk_ids:",m) soky.connect(n) packety = Packet() packety.BuildGetDataBlockPacket(m) soky.sendall(packety.getEncodedPacket()) print("Packet request sent") msg = soky.recv(2**16) print("Packet received") # print(msg) soky.close() file.write(msg) print("writing to file") print("Writing DONE!!!!") else: print(fname, ": File not found!")
def copyFromDFS(address, from_path, to_path): """ Contact the metadata server to ask for the file blocks, and then get the data blocks from the data nodes. Saves the data in 'to_path'. """ # Contact the MDS to ask for information of 'to_path' print "Connecting to MDS..." sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(address) print "Connected!" # Create `get` packet with the remote file name, and send it to MDS p1 = Packet() p1.BuildGetPacket(from_path) sock.send(p1.getEncodedPacket()) print "Sent `get` request to MDS for remote file '%s'!" % from_path # If there is no error, retrieve the data blocks msg = sock.recv(CHUNKLIST_BUFFER) sock.close() if msg == "NFOUND": print "File '%s' doesn't exist in DFS server! Exiting..." % from_path sys.exit(0) p2 = Packet() p2.DecodePacket(msg) metalist = p2.getDataNodes() fsize = p2.getFileSize() print "Received list of %d chunks! Closing connection to MDS..." % len(metalist) # Save the file in local machine (in 'to_path') chunk, total = 1, 0 fd = open(to_path, "wb") for ip, prt, cid in metalist: # Connect to data node and send data block `get` request sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock2.connect((ip, prt)) p3 = Packet() p3.BuildGetDataBlockPacket(cid) sock2.send(p3.getEncodedPacket()) print "\n\t- Sent `get` request to data node at %s:%s!" % (ip, prt) # Receive data little by little while True: data = sock2.recv(SUBCHUNK_BUFFER) if data == "DONE": break fd.write(data) total += len(data) # Blocking dummy variable so that messages don't corrupt themselves in data node sock2.send("MORE") # Disconnect from data node print "\t- Received chunk #%d!" % chunk chunk += 1 sock2.close() print "\t- Disconnecting from data node..." # Close local file fd.close() # Check if file was recieved completely if total == fsize: print "\nWhole file received!!!" else: print "File wasn't received completely!"
def copyFromDFS(address, fname, path): """ Contact the metadata server to ask for the file blocks of the file fname. Get the data blocks from the data nodes. Saves the data in path. """ # Contact the metadata server to ask for information of fname socke = socket.socket(socket.AF_INET, socket.SOCK_STREAM) socke.connect(address) packt = Packet() packt.BuildGetPacket(fname) socke.sendall(packt.getEncodedPacket()) # Fill code md_response = socke.recv(4096) if md_response == "NFOUND": print("NO, DALE REWIND!") else: # print "tamo aki" print md_response packt.DecodePacket(md_response) dnodes = packt.getDataNodes() fsize = packt.packet["fsize"] # print(dnodes) file_to_append = open(path, 'ab') blocks_string = "" for i in dnodes: print("address:", i[0]) print("port:", i[1]) print("blockid:", i[-1]) address1 = i[0] port1 = i[1] block_id = i[-1] sok = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sok.connect((address1, port1)) packert = Packet() packert.BuildGetDataBlockPacket(block_id) sok.sendall(packert.getEncodedPacket()) block_size = (fsize / len(dnodes)) + 1 # print (block_size) last_data_block = block_size - (fsize / block_size - 1) # print (last_data_block) bytes_atm = 0 while bytes_atm < block_size: block = sok.recv(4096) # print (block) blocks_string += block bytes_atm += sys.getsizeof(block) # print bytes_atm if bytes_atm == last_data_block: break file_to_append.write(blocks_string) file_to_append.close()
print ( "Connection to the server failed. Error: " + str(e) ) sys.exit(1) print ( "Connection to the data server was completed with no errors!" ) # Contact the metadata server to ask for information of fname # Fill code # try will create a packet and a socket. EncodedPacket is sent throught the socket. We get a reply. # Decode the packet # Get the data nodes from the packet try: created_packet = Packet() created_packet.BuildGetPacket(fname) create_socket.sendall(created_packet.getEncodedPacket()) reply = create_socket.recv(1024) print("Reply is:") print(reply) created_packet.DecodePacket(reply) print("after decoding:") print(reply) data_nodes = created_packet.getDataNodes() if reply == "NFOUND":