def __send_file_chunks( self ):
"""
Call repeatedly in the ifttransmit main loop to send chunks.
This will be called once prepare_transmit and possibly send_job have been called
Return an event to be handled by ifttransmit
"""
if self.ready_to_send == False or self.transmit_state != TRANSMIT_STATE_CHUNKS:
return (0,E_BAD_STATE)
# can we do anything?
if self.ift_job == None and self.ready_to_send == True:
iftlog.log(5, self.name + ": No job to process! Use my assign_job() method and resume me")
self.ready_to_send = False
return (0,E_BAD_STATE)
chunk = None
chunk_id = -1
rc = 0
chunk, chunk_id, chunk_path, remote_chunk_path = self.__next_chunk()
try:
rc = self.send_one_chunk( chunk, chunk_id, chunk_path, remote_chunk_path )
except Exception, inst:
iftlog.exception( self.name + ": could not send data", inst )
self.close_connection( TRANSMIT_STATE_FAILURE )
t = time.time()
iftstats.log_chunk( self.ift_job, self.name, False, t, t, 0 )
return (PROTO_MSG_ERROR_FATAL, E_NO_DATA)
def send_one_chunk( self, chunk, chunk_id, chunk_path, remote_chunk_path, job = None ):
"""
Send a single chunk.
@arg chunk
The chunk data to send. None is interpreted as "send the entire file"
@arg chunk_id
The numerical id of the chunk. 0 if chunk == None
@arg chunk_path
The path to the chunk on disk, or None if chunk != None
@arg remote_chunk_path
The path to which to send the chunk on the remote host.
"""
stime = time.time()
rc = self.send_chunk( chunk, chunk_id, chunk_path, remote_chunk_path )
etime = time.time()
status = False
chunk_len = None
if rc >= 0:
status = True
if chunk:
chunk_len = max( len(chunk), rc )
else:
chunk_len = iftfile.get_filesize( self.ift_job.get_attr( iftfile.JOB_ATTR_SRC_NAME ) )
if job == None:
job = self.ift_job
iftstats.log_chunk( job, self.name, status, stime, etime, chunk_len )
return rc
def __receive_chunk_data( self ):
"""
If this protocol is to accumulate chunks as byte strings, then receive chunks of data.
"""
if self.ift_job == None and (self.ready_to_receive == False or self.transmit_state != TRANSMIT_STATE_CHUNKS):
return (0, E_BAD_STATE) # nothing to do
# try to receive a chunk
chunk_table = None
recv_rc = 0
try:
# if we can be assured that we'll get the chunks we want, go ahead and receive the next unreceived chunks
recv_rc = 0
chunk_table = None
desired_chunk_ids = self.__next_chunks()
if len(desired_chunk_ids) == 0:
# they're all reserved...
#print self.name + ": all reserved..."
return (PROTO_MSG_NONE, E_TRY_AGAIN)
# attempt to receive
stime = time.time()
recv_rc, chunk_table = self.__recv_chunks( self.ift_job.get_attr( iftfile.JOB_ATTR_SRC_CHUNK_DIR ), desired_chunk_ids )
etime = time.time()
status = {}
# if by some miracle we got the whole file at once, then check the file and be done with it.
whole_file_path = chunk_table.get("whole_file")
if whole_file_path != None:
iftlog.log(3, self.name + ": got whole file back, saved in " + whole_file_path)
shutil.move( whole_file_path, self.iftfile_ref.path )
iftfile.apply_dir_permissions( self.iftfile_ref.path )
self.iftfile_ref.mark_complete()
return self.__recv_cleanup( TRANSMIT_STATE_SUCCESS )
# validate the chunk table otherwise.
elif chunk_table and len(chunk_table.keys()) > 0:
# verify chunk hashes if we need to.
# record each chunk if that's what we got
if self.ift_job.get_attr( iftfile.JOB_ATTR_CHUNK_HASHES ) != None:
# verify each hash
chunk_hashes = self.ift_job.get_attr( iftfile.JOB_ATTR_CHUNK_HASHES )
for k in chunk_table.keys():
if len(chunk_hashes) > k:
m = hashlib.sha1()
m.update( chunk_table[k] )
if chunk_hashes[k] != m.hexdigest():
iftlog.log(5, self.name + ": chunk " + str(k) + "'s hash is incorrect!")
status[k] = False
else:
status[k] = True
# log chunk data
for k in chunk_table.keys():
if status == {}:
if recv_rc == 0:
iftstats.log_chunk( self.ift_job, self.name, True, stime, etime, self.ift_job.get_attr( iftfile.JOB_ATTR_CHUNKSIZE ) ) # no way to verify chunk correctness
else:
iftstats.log_chunk( self.ift_job, self.name, False, stime, etime, self.ift_job.get_attr( iftfile.JOB_ATTR_CHUNKSIZE ) )
else:
iftstats.log_chunk( self.ift_job, self.name, status[k], stime, etime, self.ift_job.get_attr( iftfile.JOB_ATTR_CHUNKSIZE ) )
# if we had a non-zero RC, we should warn the user
if recv_rc != 0:
# got some data, but still encountered an error so we need to emit a warning
# are we missing any chunks?
not_received = []
for recved_id in chunk_table.keys():
if desired_chunk_ids.count(recved_id) == 0:
not_received.append( recved_id )
iftlog.log(5, "WARNING: " + self.name + " receive RC = " + str(recv_rc))
if len(not_received) != 0:
iftlog.log(5, "WARNING: " + self.name + " did not receive chunks " + str(not_received))
# store chunks
for chunk_id in chunk_table.keys():
msg, rc = self.__store_chunk( chunk_table[chunk_id], chunk_id )
if rc != E_DUPLICATE and (msg == PROTO_MSG_ERROR or msg == PROTO_MSG_ERROR_FATAL):
if msg == PROTO_MSG_ERROR_FATAL:
self.__recv_cleanup( TRANSMIT_STATE_FAILURE )
return (msg, rc)
# are we done?
if self.iftfile_ref.is_complete():
return self.__recv_cleanup( TRANSMIT_STATE_SUCCESS )
# are we finished receiving, as indicated by the protocol?
if self.recv_finish:
return self.__recv_cleanup( self.recv_status )
return (0, 0) # get moar!
elif recv_rc != 0:
# negative RC and no data given back
if recv_rc == E_EOF:
# nothing left for us to receive--save the file (if we handle file I/O) and exit
return self.__recv_cleanup( TRANSMIT_STATE_SUCCESS )
else:
# are we finished receiving, as indicated by the protocol?
if self.recv_finish:
return self.__recv_cleanup( self.recv_status )
self.__recv_cleanup( TRANSMIT_STATE_FAILURE )
return (PROTO_MSG_ERROR_FATAL, recv_rc)
else:
# recv_rc == 0 and no chunks given
# are we finished receiving, as indicated by the protocol?
if self.recv_finish:
return self.__recv_cleanup( self.recv_status )
return (0, 0) # just try again...
except Exception, inst:
iftlog.exception( self.name + ": could not receive chunk", inst )
self.close_connection( TRANSMIT_STATE_FAILURE )
t = time.time()
iftstats.log_chunk( self.ift_job, self.name, False, t, t, 0 )
self.__recv_cleanup( TRANSMIT_STATE_FAILURE )
return (PROTO_MSG_ERROR_FATAL, recv_rc)