def download_seeds( the_file_object, the_file_basename, the_output_dir, the_number_threads, the_printing_depth ) :
an_is_download_ok = False
while not the_file_object.sealed() or not an_is_download_ok :
a_worker_pool = WorkerPool( the_number_threads )
try:
for a_seed_object in the_file_object :
a_hex_md5 = a_seed_object.hex_md5()
a_seed_path = os.path.join( the_output_dir, a_seed_object.basename() )
if os.path.exists( a_seed_path ) :
a_file_pointer = open( a_seed_path, 'rb' )
a_md5 = compute_md5( a_file_pointer )[ 0 ]
if a_hex_md5 == a_md5 :
continue
os.remove( a_seed_path )
pass
print_d( "a_seed_path = '%s'\n" % a_seed_path, the_printing_depth )
a_worker_pool.charge( download_seed, ( a_seed_object, a_seed_path, the_printing_depth + 1 ) )
except:
from cloudflu.common import print_traceback
print_traceback( the_printing_depth )
pass
a_worker_pool.shutdown()
an_is_download_ok = a_worker_pool.is_all_right()
print_d( "'%s'.uploaded() == %s\n" % ( the_file_object.located_file(), the_file_object.sealed() ), the_printing_depth )
pass
return True
def file_entry_point( the_study_object, the_output_dir, the_located_file,
the_number_threads, the_wait, the_remove, the_fresh, the_callback ) :
if the_wait == False or the_study_object.sealed() :
try:
a_file_object = TFileObject.get( the_study_object, the_located_file )
download_file( a_file_object, the_output_dir, the_number_threads, the_remove, the_fresh, the_callback )
except:
from cloudflu.common import print_traceback
print_traceback( 0 )
pass
pass
else:
print_d( "waiting " )
while True :
an_sealed = the_study_object.sealed()
try:
a_file_object = TFileObject.get( the_study_object, the_located_file )
print_d( "\n" )
download_file( a_file_object, the_output_dir, the_number_threads, the_remove, the_fresh, the_callback )
break
except:
if an_sealed :
break
print_d( "." )
pass
pass
print_d( " compleated\n" )
pass
pass
def mark_finished( the_file_object, the_working_dir, the_printing_depth ) :
try :
the_file_object.seal( the_working_dir )
return True
except :
from cloudflu.common import print_traceback
print_traceback( the_printing_depth )
pass
return False
def download_seed( the_seed_object, the_file_path, the_printing_depth ) :
try :
the_seed_object.download( the_file_path )
print_d( "the_seed_object = %s\n" % the_seed_object, the_printing_depth )
return True
except :
from cloudflu.common import print_traceback
print_traceback( the_printing_depth + 1 )
pass
return False
def __iter__( self ) :
"Iterates through study files"
for a_file_key in self.iterator() :
try:
yield TFileObject._get( self, get_key_name( a_file_key ) )
except:
from cloudflu.common import print_traceback
print_traceback()
pass
pass
pass
def upload_seed( the_file_object, the_seed_name, the_seed_path, the_printing_depth ) :
"Uploading file item"
try :
a_seed_object = TSeedObject.create( the_file_object, the_seed_name, the_seed_path )
print_d( "%s\n" % a_seed_object, the_printing_depth )
return True
except :
from cloudflu.common import print_traceback
print_traceback( the_printing_depth )
pass
return False
def _get( the_root_object, the_study_key_name ) :
a_study_key = get_key( the_root_object._bucket, the_study_key_name )
an_api_version = None
try:
an_api_version = a_study_key.get_contents_as_string()
except:
from cloudflu.common import print_traceback
print_traceback()
raise NameError( "There is no '%s' study" % the_study_name )
pass
a_location = None
a_study_name = get_study_name( the_study_key_name )
a_study_props_key_name = study_props_key_name( a_study_name, an_api_version )
a_study_props_key = get_key( the_root_object._bucket, a_study_props_key_name )
try:
a_study_props = a_study_props_key.get_contents_as_string()
a_location = read_study_props( a_study_props, an_api_version )
except:
from cloudflu.common import print_traceback
print_traceback()
pass
a_bucket_name = create_region_bucket_name( the_root_object._bucket.name, a_location )
a_bucket = None
try: # in case of broken entities
a_bucket = the_root_object._connection.get_bucket( a_bucket_name )
except :
import sys, traceback
traceback.print_exc( file = sys.stderr )
pass
return TStudyObject( the_root_object, a_study_key, a_study_props_key, a_bucket, an_api_version )
#--------------------------------------------------------------------------------------
def execute( the_ec2_conn, the_cluster_id ) :
try:
a_reservation = ec2.use.get_reservation( the_ec2_conn, the_cluster_id )
print_d( '< %r > : %s\n' % ( a_reservation, a_reservation.instances ) )
except Exception, exc:
print_e( '%s\n' % exc, False )
return
a_security_group = None
try:
a_security_group = ec2.use.get_security_group( the_ec2_conn, a_reservation )
print_d( "< %r > : %s\n" % ( a_security_group, a_security_group.rules ) )
except:
from cloudflu.common import print_traceback
print_traceback()
pass
an_instance = a_reservation.instances[ 0 ]
an_identity_file = ec2.run.get_identity_filepath( an_instance.key_name )
for an_instance in a_reservation.instances:
an_instance.terminate()
pass
try:
the_ec2_conn.delete_key_pair( an_instance.key_name )
import os; os.remove( an_identity_file )
except:
from cloudflu.common import print_traceback
print_traceback()