def download_seeds( the_file_object, the_file_basename, the_output_dir, the_number_threads, the_printing_depth ) : an_is_download_ok = False while not the_file_object.sealed() or not an_is_download_ok : a_worker_pool = WorkerPool( the_number_threads ) try: for a_seed_object in the_file_object : a_hex_md5 = a_seed_object.hex_md5() a_seed_path = os.path.join( the_output_dir, a_seed_object.basename() ) if os.path.exists( a_seed_path ) : a_file_pointer = open( a_seed_path, 'rb' ) a_md5 = compute_md5( a_file_pointer )[ 0 ] if a_hex_md5 == a_md5 : continue os.remove( a_seed_path ) pass print_d( "a_seed_path = '%s'\n" % a_seed_path, the_printing_depth ) a_worker_pool.charge( download_seed, ( a_seed_object, a_seed_path, the_printing_depth + 1 ) ) except: from cloudflu.common import print_traceback print_traceback( the_printing_depth ) pass a_worker_pool.shutdown() an_is_download_ok = a_worker_pool.is_all_right() print_d( "'%s'.uploaded() == %s\n" % ( the_file_object.located_file(), the_file_object.sealed() ), the_printing_depth ) pass return True
def file_entry_point( the_study_object, the_output_dir, the_located_file, the_number_threads, the_wait, the_remove, the_fresh, the_callback ) : if the_wait == False or the_study_object.sealed() : try: a_file_object = TFileObject.get( the_study_object, the_located_file ) download_file( a_file_object, the_output_dir, the_number_threads, the_remove, the_fresh, the_callback ) except: from cloudflu.common import print_traceback print_traceback( 0 ) pass pass else: print_d( "waiting " ) while True : an_sealed = the_study_object.sealed() try: a_file_object = TFileObject.get( the_study_object, the_located_file ) print_d( "\n" ) download_file( a_file_object, the_output_dir, the_number_threads, the_remove, the_fresh, the_callback ) break except: if an_sealed : break print_d( "." ) pass pass print_d( " compleated\n" ) pass pass
def mark_finished( the_file_object, the_working_dir, the_printing_depth ) : try : the_file_object.seal( the_working_dir ) return True except : from cloudflu.common import print_traceback print_traceback( the_printing_depth ) pass return False
def download_seed( the_seed_object, the_file_path, the_printing_depth ) : try : the_seed_object.download( the_file_path ) print_d( "the_seed_object = %s\n" % the_seed_object, the_printing_depth ) return True except : from cloudflu.common import print_traceback print_traceback( the_printing_depth + 1 ) pass return False
def __iter__( self ) : "Iterates through study files" for a_file_key in self.iterator() : try: yield TFileObject._get( self, get_key_name( a_file_key ) ) except: from cloudflu.common import print_traceback print_traceback() pass pass pass
def upload_seed( the_file_object, the_seed_name, the_seed_path, the_printing_depth ) : "Uploading file item" try : a_seed_object = TSeedObject.create( the_file_object, the_seed_name, the_seed_path ) print_d( "%s\n" % a_seed_object, the_printing_depth ) return True except : from cloudflu.common import print_traceback print_traceback( the_printing_depth ) pass return False
def _get( the_root_object, the_study_key_name ) : a_study_key = get_key( the_root_object._bucket, the_study_key_name ) an_api_version = None try: an_api_version = a_study_key.get_contents_as_string() except: from cloudflu.common import print_traceback print_traceback() raise NameError( "There is no '%s' study" % the_study_name ) pass a_location = None a_study_name = get_study_name( the_study_key_name ) a_study_props_key_name = study_props_key_name( a_study_name, an_api_version ) a_study_props_key = get_key( the_root_object._bucket, a_study_props_key_name ) try: a_study_props = a_study_props_key.get_contents_as_string() a_location = read_study_props( a_study_props, an_api_version ) except: from cloudflu.common import print_traceback print_traceback() pass a_bucket_name = create_region_bucket_name( the_root_object._bucket.name, a_location ) a_bucket = None try: # in case of broken entities a_bucket = the_root_object._connection.get_bucket( a_bucket_name ) except : import sys, traceback traceback.print_exc( file = sys.stderr ) pass return TStudyObject( the_root_object, a_study_key, a_study_props_key, a_bucket, an_api_version )
#-------------------------------------------------------------------------------------- def execute( the_ec2_conn, the_cluster_id ) : try: a_reservation = ec2.use.get_reservation( the_ec2_conn, the_cluster_id ) print_d( '< %r > : %s\n' % ( a_reservation, a_reservation.instances ) ) except Exception, exc: print_e( '%s\n' % exc, False ) return a_security_group = None try: a_security_group = ec2.use.get_security_group( the_ec2_conn, a_reservation ) print_d( "< %r > : %s\n" % ( a_security_group, a_security_group.rules ) ) except: from cloudflu.common import print_traceback print_traceback() pass an_instance = a_reservation.instances[ 0 ] an_identity_file = ec2.run.get_identity_filepath( an_instance.key_name ) for an_instance in a_reservation.instances: an_instance.terminate() pass try: the_ec2_conn.delete_key_pair( an_instance.key_name ) import os; os.remove( an_identity_file ) except: from cloudflu.common import print_traceback print_traceback()