def __receive_file_data( self ):
"""
If this protocol is to accumulate files straight to disk, then receive files.
"""
if self.ift_job == None and (self.ready_to_receive == False or self.transmit_state != TRANSMIT_STATE_CHUNKS):
return (0, E_BAD_STATE) # nothing to do
# attempt to get the files
files_list = None
recv_rc = 0
try:
# if we can be assured that we'll get the chunks we want, go ahead and receive the next unreceived chunks
recv_rc = 0
chunk_table = None
file_names = self.__next_files()
if len(file_names) == 0:
# they're all reserved...
#print self.name + ": all reserved..."
return (PROTO_MSG_NONE, E_TRY_AGAIN)
# attempt to receive the files
stime = time.time()
recv_rc, files_list = self.__recv_files( file_names, self.ift_job.get_attr( iftfile.JOB_ATTR_DEST_CHUNK_DIR ) )
etime = time.time()
# did we get the whole file back?
if len(files_list) > 0 and files_list[0][0] == -1:
whole_file_path = files_list[0][1]
iftlog.log(3, self.name + ": got whole file back, saved in " + whole_file_path)
try:
shutil.move( whole_file_path, self.iftfile_ref.path )
iftfile.apply_dir_permissions( self.iftfile_ref.path )
self.iftfile_ref.mark_complete()
return self.__recv_cleanup( TRANSMIT_STATE_SUCCESS )
except Exception, inst:
if self.iftfile_ref.path == None: # someone beat us...
iftlog.log(3, self.name + ": file is already received")
os.remove( whole_file_path )
else:
iftlog.exception( self.name + ": exception while receiving whole file", inst )
return self.__recv_cleanup( TRANSMIT_STATE_FAILURE )
# did we only get one file (i.e. no chunking even from IFTD)?
elif len(files_list) > 0:
if recv_rc != 0:
iftlog.log(5, self.name + ".__receive_file_data: WARNING! got rc=" + str(recv_rc) )
if self.ift_job.get_attr( iftfile.JOB_ATTR_REMOTE_IFTD ) == False and self.get_chunking_mode() == PROTO_NO_CHUNKING:
# got the file directly, so we can bypass the iftfile instance and move the file in place directly
fname = files_list[0][1]
try:
shutil.move( fname, self.iftfile_ref.path )
iftfile.apply_dir_permissions( self.iftfile_ref.path )
self.__recv_cleanup( TRANSMIT_STATE_SUCCESS )
return (PROTO_MSG_END, 0)
except Exception, inst:
iftlog.exception(self.name + ".__receive_file_data: could not receive file " + str(fname))
return (PROTO_MSG_ERROR_FATAL, E_UNHANDLED_EXCEPTION)
def __receive_chunk_data( self ):
"""
If this protocol is to accumulate chunks as byte strings, then receive chunks of data.
"""
if self.ift_job == None and (self.ready_to_receive == False or self.transmit_state != TRANSMIT_STATE_CHUNKS):
return (0, E_BAD_STATE) # nothing to do
# try to receive a chunk
chunk_table = None
recv_rc = 0
try:
# if we can be assured that we'll get the chunks we want, go ahead and receive the next unreceived chunks
recv_rc = 0
chunk_table = None
desired_chunk_ids = self.__next_chunks()
if len(desired_chunk_ids) == 0:
# they're all reserved...
#print self.name + ": all reserved..."
return (PROTO_MSG_NONE, E_TRY_AGAIN)
# attempt to receive
stime = time.time()
recv_rc, chunk_table = self.__recv_chunks( self.ift_job.get_attr( iftfile.JOB_ATTR_SRC_CHUNK_DIR ), desired_chunk_ids )
etime = time.time()
status = {}
# if by some miracle we got the whole file at once, then check the file and be done with it.
whole_file_path = chunk_table.get("whole_file")
if whole_file_path != None:
iftlog.log(3, self.name + ": got whole file back, saved in " + whole_file_path)
shutil.move( whole_file_path, self.iftfile_ref.path )
iftfile.apply_dir_permissions( self.iftfile_ref.path )
self.iftfile_ref.mark_complete()
return self.__recv_cleanup( TRANSMIT_STATE_SUCCESS )
# validate the chunk table otherwise.
elif chunk_table and len(chunk_table.keys()) > 0:
# verify chunk hashes if we need to.
# record each chunk if that's what we got
if self.ift_job.get_attr( iftfile.JOB_ATTR_CHUNK_HASHES ) != None:
# verify each hash
chunk_hashes = self.ift_job.get_attr( iftfile.JOB_ATTR_CHUNK_HASHES )
for k in chunk_table.keys():
if len(chunk_hashes) > k:
m = hashlib.sha1()
m.update( chunk_table[k] )
if chunk_hashes[k] != m.hexdigest():
iftlog.log(5, self.name + ": chunk " + str(k) + "'s hash is incorrect!")
status[k] = False
else:
status[k] = True
# log chunk data
for k in chunk_table.keys():
if status == {}:
if recv_rc == 0:
iftstats.log_chunk( self.ift_job, self.name, True, stime, etime, self.ift_job.get_attr( iftfile.JOB_ATTR_CHUNKSIZE ) ) # no way to verify chunk correctness
else:
iftstats.log_chunk( self.ift_job, self.name, False, stime, etime, self.ift_job.get_attr( iftfile.JOB_ATTR_CHUNKSIZE ) )
else:
iftstats.log_chunk( self.ift_job, self.name, status[k], stime, etime, self.ift_job.get_attr( iftfile.JOB_ATTR_CHUNKSIZE ) )
# if we had a non-zero RC, we should warn the user
if recv_rc != 0:
# got some data, but still encountered an error so we need to emit a warning
# are we missing any chunks?
not_received = []
for recved_id in chunk_table.keys():
if desired_chunk_ids.count(recved_id) == 0:
not_received.append( recved_id )
iftlog.log(5, "WARNING: " + self.name + " receive RC = " + str(recv_rc))
if len(not_received) != 0:
iftlog.log(5, "WARNING: " + self.name + " did not receive chunks " + str(not_received))
# store chunks
for chunk_id in chunk_table.keys():
msg, rc = self.__store_chunk( chunk_table[chunk_id], chunk_id )
if rc != E_DUPLICATE and (msg == PROTO_MSG_ERROR or msg == PROTO_MSG_ERROR_FATAL):
if msg == PROTO_MSG_ERROR_FATAL:
self.__recv_cleanup( TRANSMIT_STATE_FAILURE )
return (msg, rc)
# are we done?
if self.iftfile_ref.is_complete():
return self.__recv_cleanup( TRANSMIT_STATE_SUCCESS )
# are we finished receiving, as indicated by the protocol?
if self.recv_finish:
return self.__recv_cleanup( self.recv_status )
return (0, 0) # get moar!
elif recv_rc != 0:
# negative RC and no data given back
if recv_rc == E_EOF:
# nothing left for us to receive--save the file (if we handle file I/O) and exit
return self.__recv_cleanup( TRANSMIT_STATE_SUCCESS )
else:
# are we finished receiving, as indicated by the protocol?
if self.recv_finish:
return self.__recv_cleanup( self.recv_status )
self.__recv_cleanup( TRANSMIT_STATE_FAILURE )
return (PROTO_MSG_ERROR_FATAL, recv_rc)
else:
# recv_rc == 0 and no chunks given
# are we finished receiving, as indicated by the protocol?
if self.recv_finish:
return self.__recv_cleanup( self.recv_status )
return (0, 0) # just try again...
except Exception, inst:
iftlog.exception( self.name + ": could not receive chunk", inst )
self.close_connection( TRANSMIT_STATE_FAILURE )
t = time.time()
iftstats.log_chunk( self.ift_job, self.name, False, t, t, 0 )
self.__recv_cleanup( TRANSMIT_STATE_FAILURE )
return (PROTO_MSG_ERROR_FATAL, recv_rc)