def file_entry_point( the_study_object, the_output_dir, the_located_file, the_number_threads, the_wait, the_remove, the_fresh, the_callback ) : if the_wait == False or the_study_object.sealed() : try: a_file_object = TFileObject.get( the_study_object, the_located_file ) download_file( a_file_object, the_output_dir, the_number_threads, the_remove, the_fresh, the_callback ) except: from cloudflu.common import print_traceback print_traceback( 0 ) pass pass else: print_d( "waiting " ) while True : an_sealed = the_study_object.sealed() try: a_file_object = TFileObject.get( the_study_object, the_located_file ) print_d( "\n" ) download_file( a_file_object, the_output_dir, the_number_threads, the_remove, the_fresh, the_callback ) break except: if an_sealed : break print_d( "." ) pass pass print_d( " compleated\n" ) pass pass
def entry_point( the_study_object, the_located_files, the_number_threads ) : a_spent_time = Timer() if the_located_files == None : upload_files( the_study_object, the_number_threads, 0 ) pass else : a_worker_pool = WorkerPool( the_number_threads ) for a_located_file in the_located_files: a_file_object = TFileObject.get( the_study_object, a_located_file ) a_worker_pool.charge( upload_file, ( a_file_object, the_number_threads, 0 ) ) pass a_worker_pool.shutdown() a_worker_pool.join() pass print_d( "a_spent_time = %s, sec\n" % a_spent_time ) return the_study_object
def main() : #----------------------- Defining utility command-line interface ------------------------- an_usage_description = "%prog" from rm_options import usage_description as usage_description_options an_usage_description += usage_description_options() from cloudflu import VERSION a_version = "%s" % VERSION from optparse import IndentedHelpFormatter a_help_formatter = IndentedHelpFormatter( width = 127 ) from optparse import OptionParser an_option_parser = OptionParser( usage = an_usage_description, version = a_version, formatter = a_help_formatter ) #----------------------- Definition of the command line arguments ------------------------ from rm_options import add as add_options add_options( an_option_parser ) amazon.security_options.add( an_option_parser ) common.concurrency_options.add( an_option_parser ) common.communication_options.add( an_option_parser ) common.options.add( an_option_parser ) #------------------ Extracting and verifying command-line arguments ---------------------- an_options, an_args = an_option_parser.parse_args() common.options.extract( an_option_parser ) common.communication_options.extract( an_option_parser ) a_number_threads = common.concurrency_options.extract( an_option_parser ) AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY = amazon.security_options.extract( an_option_parser ) from rm_options import extract as extract_options a_study_name, a_located_files = extract_options( an_option_parser ) print_i( "--------------------------- Looking for study object ----------------------------\n" ) a_root_object = TRootObject.get( AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY ) print_d( "a_root_object = %s\n" % a_root_object ) a_study_object = TStudyObject.get( a_root_object, a_study_name ) print_d( "a_study_object = %s\n" % a_study_object ) print_i( "-------------------------- Removing study files -----------------------------\n" ) a_worker_pool = WorkerPool( len( a_located_files ) ) a_deleter = lambda the_object, the_number_threads, the_printing_depth : \ the_object.delete( the_number_threads, the_printing_depth ) for a_located_file in a_located_files : print_d( "a_located_file = %s\n" % a_located_file ) a_file_object = TFileObject.get( a_study_object, a_located_file ) print_d( "a_file_object = %s\n" % a_file_object, 1 ) a_worker_pool.charge( a_deleter, ( a_file_object, a_number_threads, 2 ) ) pass a_worker_pool.shutdown() a_worker_pool.join() print_i( "------------------- Printing succussive pipeline arguments ----------------------\n" ) print a_study_name print_i( "-------------------------------------- OK ---------------------------------------\n" ) pass