def copyFromDFS(address, fname, path):
""" Contact the metadata server to ask for the file blocks of the file fname. Get the data blocks from the data nodes. Saves the data in path.
"""
# Contact the metadata server to ask for information of fname
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(address)
# Fill code
p = Packet()
get_packet = p.BuildGetPacket(fname)
sock.sendall(p.getEncodedPacket().encode())
# If there is no error response Retreive the data blocks
msg = sock.recv(1024).decode('utf-8')
# danger
msg=eval(msg)
# Decode the packet received
sock.close()
# Fill code
if msg != 'DUP':
n = len(msg)
with open(path, 'w') as f:
for i in range(n):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
data_node = msg[i]
ip=data_node[0]
port=data_node[1]
chunkid=data_node[2]
sock.connect((ip,port))
p.BuildGetDataBlockPacket(chunkid)
sock.sendall(p.getEncodedPacket().encode())
data = sock.recv(1024).decode()
sock.close()
f.write(data)
def copyFromDFS(address, fname, path):
""" Contact the metadata server to ask for the file blocks of
the file fname. Get the data blocks from the data nodes.
Saves the data in path.
"""
print "copy from dfs"
# Contact the metadata server to ask for information of fname
print "meta address ", address
print "fname of file i want to copy ", fname
print "path of new file (block destiny) ", path
p = Packet()
p.BuildGetPacket(fname) # request metadata fname's inode
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((address[0], address[1]))
sock.sendall(p.getEncodedPacket())
metadata = sock.recv(1024) # recieve
file = open(path, 'wb')
# Ifs there is no error response Retreive the data blocks
print "estamoredi ", metadata
p.DecodePacket(metadata)
servers = p.getDataNodes()
print "servers ", servers
for dserver in servers:
print "dserver ", dserver
pack = Packet()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((dserver[0], dserver[1]))
pack.BuildGetDataBlockPacket(dserver[2])
s.sendall(pack.getEncodedPacket())
dblock_size = s.recv(1024)
print "dblock-size ", dblock_size
s.sendall("OK") # send ok to copy after recieving dblock
dblock = "" # string to store the info to be written to the new file
while len(dblock) < int(dblock_size):
dblock_chunk = s.recv(1024)
print "len of recieved dblock chunk ", len(dblock_chunk)
print "dblock chunk recieved"
dblock = dblock + dblock_chunk
# print "dblock after chunk appended ", dblock
print "dblock len after chunk appended ", len(dblock)
s.sendall("OK")
# Save the file
file.write(dblock)
s.close()
file.close()
def copyFromDFS(address, fname, path):
""" Contact the metadata server to ask for the file blocks of
the file fname. Get the data blocks from the data nodes.
Saves the data in path.
"""
data = fname.split('.')
file_extension = data[1]
# Contact the metadata server to ask for information of fname
sock = socket.socket()
sock.connect(address)
p = Packet()
p.BuildGetPacket(fname)
sock.sendall(p.getEncodedPacket())
# Fill code
# If there is no error response Retreive the data blocks
response = sock.recv(1024) # from meta data server
sock.close()
sock = None
p.DecodePacket(response)
blocks = p.packet["servers"]
chunks_toSave = [] # chunks that are going to be saved in file 'path'
for host, port, chunk_id in blocks:
print(host, port, chunk_id)
sock = socket.socket()
sock.connect((host, port))
p.BuildGetDataBlockPacket(chunk_id + '.' + file_extension)
sock.sendall(p.getEncodedPacket())
data_block = sock.recv(1024)
try:
data_block = data_block.decode()
except:
print(data_block)
chunks_toSave.append(data_block)
print(data_block, 'me le devuelve el data node')
sock.close()
sock = None
try:
fd = open(path, "w")
for chunk in chunks_toSave:
fd.write(chunk)
fd.close()
except:
fd = open(path, "wb")
for chunk in chunks_toSave:
fd.write(chunk)
fd.close()
def copyFromDFS(address, fname, path):
""" Contact the metadata server to ask for the file blocks of
the file fname. Get the data blocks from the data nodes.
Saves the data in path.
"""
# Contact the metadata server to ask for information of fname
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#connecting to meta-data server.
sock.connect(address)
#building packet with the fname and the command get.
p = Packet()
p.BuildGetPacket(fname)
sock.sendall(p.getEncodedPacket())
#print("Received")
#Receiving meta-data server response
respond = sock.recv(4096)
#print("you've got mail:", respond)
#If the file is in the inode table in the database, then it look for the
#Datanode information.
if respond != "NFOUND":
packetres = Packet()
packetres.DecodePacket(respond)
print("Datablocks information decoded")
DataNodes = packetres.getDataNodes()
with open(path, "a+b") as file:
for n in DataNodes:
soky = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print(n[:2])
n, m = tuple(n[:2]), n[2]
#print("server:", n, "chunk_ids:",m)
soky.connect(n)
packety = Packet()
packety.BuildGetDataBlockPacket(m)
soky.sendall(packety.getEncodedPacket())
print("Packet request sent")
msg = soky.recv(2**16)
print("Packet received")
# print(msg)
soky.close()
file.write(msg)
print("writing to file")
print("Writing DONE!!!!")
else:
print(fname, ": File not found!")
def copyFromDFS(address, from_path, to_path):
"""
Contact the metadata server to ask for the file blocks,
and then get the data blocks from the data nodes.
Saves the data in 'to_path'.
"""
# Contact the MDS to ask for information of 'to_path'
print "Connecting to MDS..."
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(address)
print "Connected!"
# Create `get` packet with the remote file name, and send it to MDS
p1 = Packet()
p1.BuildGetPacket(from_path)
sock.send(p1.getEncodedPacket())
print "Sent `get` request to MDS for remote file '%s'!" % from_path
# If there is no error, retrieve the data blocks
msg = sock.recv(CHUNKLIST_BUFFER)
sock.close()
if msg == "NFOUND":
print "File '%s' doesn't exist in DFS server! Exiting..." % from_path
sys.exit(0)
p2 = Packet()
p2.DecodePacket(msg)
metalist = p2.getDataNodes()
fsize = p2.getFileSize()
print "Received list of %d chunks! Closing connection to MDS..." % len(metalist)
# Save the file in local machine (in 'to_path')
chunk, total = 1, 0
fd = open(to_path, "wb")
for ip, prt, cid in metalist:
# Connect to data node and send data block `get` request
sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock2.connect((ip, prt))
p3 = Packet()
p3.BuildGetDataBlockPacket(cid)
sock2.send(p3.getEncodedPacket())
print "\n\t- Sent `get` request to data node at %s:%s!" % (ip, prt)
# Receive data little by little
while True:
data = sock2.recv(SUBCHUNK_BUFFER)
if data == "DONE":
break
fd.write(data)
total += len(data)
# Blocking dummy variable so that messages don't corrupt themselves in data node
sock2.send("MORE")
# Disconnect from data node
print "\t- Received chunk #%d!" % chunk
chunk += 1
sock2.close()
print "\t- Disconnecting from data node..."
# Close local file
fd.close()
# Check if file was recieved completely
if total == fsize:
print "\nWhole file received!!!"
else:
print "File wasn't received completely!"
def copyFromDFS(address, fname, path):
""" Contact the metadata server to ask for the file blocks of
the file fname. Get the data blocks from the data nodes.
Saves the data in path.
"""
# Contact the metadata server to ask for information of fname
socke = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socke.connect(address)
packt = Packet()
packt.BuildGetPacket(fname)
socke.sendall(packt.getEncodedPacket())
# Fill code
md_response = socke.recv(4096)
if md_response == "NFOUND":
print("NO, DALE REWIND!")
else:
# print "tamo aki"
print md_response
packt.DecodePacket(md_response)
dnodes = packt.getDataNodes()
fsize = packt.packet["fsize"]
# print(dnodes)
file_to_append = open(path, 'ab')
blocks_string = ""
for i in dnodes:
print("address:", i[0])
print("port:", i[1])
print("blockid:", i[-1])
address1 = i[0]
port1 = i[1]
block_id = i[-1]
sok = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sok.connect((address1, port1))
packert = Packet()
packert.BuildGetDataBlockPacket(block_id)
sok.sendall(packert.getEncodedPacket())
block_size = (fsize / len(dnodes)) + 1
# print (block_size)
last_data_block = block_size - (fsize / block_size - 1)
# print (last_data_block)
bytes_atm = 0
while bytes_atm < block_size:
block = sok.recv(4096)
# print (block)
blocks_string += block
bytes_atm += sys.getsizeof(block)
# print bytes_atm
if bytes_atm == last_data_block:
break
file_to_append.write(blocks_string)
file_to_append.close()
print ( "Connection to the server failed. Error: " + str(e) )
sys.exit(1)
print ( "Connection to the data server was completed with no errors!" )
# Contact the metadata server to ask for information of fname
# Fill code
# try will create a packet and a socket. EncodedPacket is sent throught the socket. We get a reply.
# Decode the packet
# Get the data nodes from the packet
try:
created_packet = Packet()
created_packet.BuildGetPacket(fname)
create_socket.sendall(created_packet.getEncodedPacket())
reply = create_socket.recv(1024)
print("Reply is:")
print(reply)
created_packet.DecodePacket(reply)
print("after decoding:")
print(reply)
data_nodes = created_packet.getDataNodes()
if reply == "NFOUND":