Example #1
0
   def __receive_file_data( self ):
      """
      If this protocol is to accumulate files straight to disk, then receive files.
      """
      
      if self.ift_job == None and (self.ready_to_receive == False or self.transmit_state != TRANSMIT_STATE_CHUNKS):
         return (0, E_BAD_STATE)     # nothing to do

      
      # attempt to get the files
      files_list = None
      recv_rc = 0
      
      try:
         # if we can be assured that we'll get the chunks we want, go ahead and receive the next unreceived chunks
         
         recv_rc = 0
         chunk_table = None

         file_names = self.__next_files()
         if len(file_names) == 0:
            # they're all reserved...
            #print self.name + ": all reserved..."
            return (PROTO_MSG_NONE, E_TRY_AGAIN)


         # attempt to receive the files
         stime = time.time()
         recv_rc, files_list = self.__recv_files( file_names, self.ift_job.get_attr( iftfile.JOB_ATTR_DEST_CHUNK_DIR ) )
         etime = time.time()
         
         # did we get the whole file back?
         if len(files_list) > 0 and files_list[0][0] == -1:
            whole_file_path = files_list[0][1]
            iftlog.log(3, self.name + ": got whole file back, saved in " + whole_file_path)
            try:
               shutil.move( whole_file_path, self.iftfile_ref.path )
               iftfile.apply_dir_permissions( self.iftfile_ref.path )
               self.iftfile_ref.mark_complete()
               return self.__recv_cleanup( TRANSMIT_STATE_SUCCESS )
            except Exception, inst:
               if self.iftfile_ref.path == None:      # someone beat us...
                  iftlog.log(3, self.name + ": file is already received")
                  os.remove( whole_file_path )
               else:
                  iftlog.exception( self.name + ": exception while receiving whole file", inst )
               
               return self.__recv_cleanup( TRANSMIT_STATE_FAILURE )
            
            
         # did we only get one file (i.e. no chunking even from IFTD)?
         elif len(files_list) > 0:
            if recv_rc != 0:
               iftlog.log(5, self.name + ".__receive_file_data: WARNING! got rc=" + str(recv_rc) )
            
            if self.ift_job.get_attr( iftfile.JOB_ATTR_REMOTE_IFTD ) == False and self.get_chunking_mode() == PROTO_NO_CHUNKING:
               # got the file directly, so we can bypass the iftfile instance and move the file in place directly
               fname = files_list[0][1]
               try:
                  shutil.move( fname, self.iftfile_ref.path )
                  iftfile.apply_dir_permissions( self.iftfile_ref.path )
                  self.__recv_cleanup( TRANSMIT_STATE_SUCCESS )
                  return (PROTO_MSG_END, 0)
               except Exception, inst:
                  iftlog.exception(self.name + ".__receive_file_data: could not receive file " + str(fname))
                  return (PROTO_MSG_ERROR_FATAL, E_UNHANDLED_EXCEPTION)
Example #2
0
   def __receive_chunk_data( self ):
      """
      If this protocol is to accumulate chunks as byte strings, then receive chunks of data.
      """
      
      if self.ift_job == None and (self.ready_to_receive == False or self.transmit_state != TRANSMIT_STATE_CHUNKS):
         return (0, E_BAD_STATE)     # nothing to do
      
      # try to receive a chunk
      chunk_table = None
      recv_rc = 0
      
      try:
         # if we can be assured that we'll get the chunks we want, go ahead and receive the next unreceived chunks
         
         recv_rc = 0
         chunk_table = None

         desired_chunk_ids = self.__next_chunks()
         if len(desired_chunk_ids) == 0:
            # they're all reserved...
            #print self.name + ": all reserved..."
            return (PROTO_MSG_NONE, E_TRY_AGAIN)

         # attempt to receive
         stime = time.time()
         recv_rc, chunk_table = self.__recv_chunks( self.ift_job.get_attr( iftfile.JOB_ATTR_SRC_CHUNK_DIR ), desired_chunk_ids )
         etime = time.time()
         
         status = {}
         
         # if by some miracle we got the whole file at once, then check the file and be done with it.
         whole_file_path = chunk_table.get("whole_file")
         if whole_file_path != None:
            iftlog.log(3, self.name + ": got whole file back, saved in " + whole_file_path)
            shutil.move( whole_file_path, self.iftfile_ref.path )
            iftfile.apply_dir_permissions( self.iftfile_ref.path )
            self.iftfile_ref.mark_complete()
            return self.__recv_cleanup( TRANSMIT_STATE_SUCCESS )

         # validate the chunk table otherwise.
         elif chunk_table and len(chunk_table.keys()) > 0:     
         
            # verify chunk hashes if we need to.
            # record each chunk if that's what we got
            if self.ift_job.get_attr( iftfile.JOB_ATTR_CHUNK_HASHES ) != None:
               # verify each hash
               chunk_hashes = self.ift_job.get_attr( iftfile.JOB_ATTR_CHUNK_HASHES )
               for k in chunk_table.keys():
                  if len(chunk_hashes) > k:
                     m = hashlib.sha1()
                     m.update( chunk_table[k] )
                     if chunk_hashes[k] != m.hexdigest():
                        iftlog.log(5, self.name + ": chunk " + str(k) + "'s hash is incorrect!")
                        status[k] = False
                     else:
                        status[k] = True
         
         
            # log chunk data
            for k in chunk_table.keys():
               if status == {}:
                  if recv_rc == 0:
                     iftstats.log_chunk( self.ift_job, self.name, True, stime, etime, self.ift_job.get_attr( iftfile.JOB_ATTR_CHUNKSIZE ) )    # no way to verify chunk correctness
                  else:
                     iftstats.log_chunk( self.ift_job, self.name, False, stime, etime, self.ift_job.get_attr( iftfile.JOB_ATTR_CHUNKSIZE ) )
               
               else:
                  iftstats.log_chunk( self.ift_job, self.name, status[k], stime, etime, self.ift_job.get_attr( iftfile.JOB_ATTR_CHUNKSIZE ) )
               
            
            # if we had a non-zero RC, we should warn the user
            if recv_rc != 0:
               # got some data, but still encountered an error so we need to emit a warning
               # are we missing any chunks?
               not_received = []
               for recved_id in chunk_table.keys():
                  if desired_chunk_ids.count(recved_id) == 0:
                     not_received.append( recved_id )
               
               iftlog.log(5, "WARNING: " + self.name + " receive RC = " + str(recv_rc))
               if len(not_received) != 0:
                  iftlog.log(5, "WARNING: " + self.name + " did not receive chunks " + str(not_received))

            # store chunks
            for chunk_id in chunk_table.keys():
               msg, rc = self.__store_chunk( chunk_table[chunk_id], chunk_id )
               if rc != E_DUPLICATE and (msg == PROTO_MSG_ERROR or msg == PROTO_MSG_ERROR_FATAL):
                  if msg == PROTO_MSG_ERROR_FATAL:
                     self.__recv_cleanup( TRANSMIT_STATE_FAILURE )
                  return (msg, rc)
                  
            # are we done?
            if self.iftfile_ref.is_complete():
               return self.__recv_cleanup( TRANSMIT_STATE_SUCCESS )

            
            # are we finished receiving, as indicated by the protocol?
            if self.recv_finish:
               return self.__recv_cleanup( self.recv_status )
               
            return (0, 0)     # get moar!
            
         
         elif recv_rc != 0:
            # negative RC and no data given back
            if recv_rc == E_EOF:
               # nothing left for us to receive--save the file (if we handle file I/O) and exit
               return self.__recv_cleanup( TRANSMIT_STATE_SUCCESS )
            
            else:
               # are we finished receiving, as indicated by the protocol?
               if self.recv_finish:
                  return self.__recv_cleanup( self.recv_status )

               self.__recv_cleanup( TRANSMIT_STATE_FAILURE )
               return (PROTO_MSG_ERROR_FATAL, recv_rc)

         
         else:
            # recv_rc == 0 and no chunks given
            # are we finished receiving, as indicated by the protocol?
            if self.recv_finish:
               return self.__recv_cleanup( self.recv_status )

            return (0, 0)     # just try again...
        
            
      except Exception, inst:
         iftlog.exception( self.name + ": could not receive chunk", inst )
         self.close_connection( TRANSMIT_STATE_FAILURE )
         
         t = time.time()
         iftstats.log_chunk( self.ift_job, self.name, False, t, t, 0 )
         
         self.__recv_cleanup( TRANSMIT_STATE_FAILURE )
         return (PROTO_MSG_ERROR_FATAL, recv_rc)