def download_seeds( the_file_object, the_file_basename, the_output_dir, the_number_threads, the_printing_depth ) : an_is_download_ok = False while not the_file_object.sealed() or not an_is_download_ok : a_worker_pool = WorkerPool( the_number_threads ) try: for a_seed_object in the_file_object : a_hex_md5 = a_seed_object.hex_md5() a_seed_path = os.path.join( the_output_dir, a_seed_object.basename() ) if os.path.exists( a_seed_path ) : a_file_pointer = open( a_seed_path, 'rb' ) a_md5 = compute_md5( a_file_pointer )[ 0 ] if a_hex_md5 == a_md5 : continue os.remove( a_seed_path ) pass print_d( "a_seed_path = '%s'\n" % a_seed_path, the_printing_depth ) a_worker_pool.charge( download_seed, ( a_seed_object, a_seed_path, the_printing_depth + 1 ) ) except: from cloudflu.common import print_traceback print_traceback( the_printing_depth ) pass a_worker_pool.shutdown() an_is_download_ok = a_worker_pool.is_all_right() print_d( "'%s'.uploaded() == %s\n" % ( the_file_object.located_file(), the_file_object.sealed() ), the_printing_depth ) pass return True
def upload_seeds( the_file_object, the_working_dir, the_number_threads, the_printing_depth ) : "Uploading file items" while True : a_dir_contents = os.listdir( the_working_dir ) a_number_threads = max( min( the_number_threads, len( a_dir_contents ) ), 1 ) print_d( "a_number_threads = %d\n" % a_number_threads, the_printing_depth ) a_worker_pool = WorkerPool( a_number_threads ) a_dir_contents.sort() a_dir_contents.reverse() a_seed_names = [ a_seed_object.basename() for a_seed_object in the_file_object ] for a_seed_name in a_dir_contents : a_seed_path = os.path.join( the_working_dir, a_seed_name ) print_d( "'%s'\n" % a_seed_path, the_printing_depth + 1 ) if a_seed_name in a_seed_names : os.remove( a_seed_path ) continue a_worker_pool.charge( upload_seed, ( the_file_object, a_seed_name, a_seed_path, the_printing_depth + 2 ) ) pass a_worker_pool.shutdown() an_is_all_right = a_worker_pool.is_all_right() if an_is_all_right : mark_finished( the_file_object, the_working_dir, the_printing_depth ) break pass return True