def main() :
#----------------------- Defining utility command-line interface -------------------------
an_usage_description = "%prog"
from cloudflu import VERSION
a_version = "%s" % VERSION
from optparse import IndentedHelpFormatter
a_help_formatter = IndentedHelpFormatter( width = 127 )
from optparse import OptionParser
an_option_parser = OptionParser( usage = an_usage_description, version = a_version, formatter = a_help_formatter )
#----------------------- Definition of the command line arguments ------------------------
amazon.security_options.add( an_option_parser )
common.options.add( an_option_parser )
#------------------ Extracting and verifying command-line arguments ----------------------
an_options, an_args = an_option_parser.parse_args()
common.options.extract( an_option_parser )
AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY = amazon.security_options.extract( an_option_parser )
print_i( "-------------------------- Running actual functionality -------------------------\n" )
import boto.ec2
for a_region in boto.ec2.regions( aws_access_key_id = AWS_ACCESS_KEY_ID, aws_secret_access_key = AWS_SECRET_ACCESS_KEY ) :
an_ec2_conn = a_region.connect()
print_d( "region - '%s'" % a_region.name )
print_i( "------------------------------- Delete EC2 instances ----------------------------\n" )
for a_reservation in an_ec2_conn.get_all_instances() :
for an_instance in a_reservation.instances :
terminate_instance( an_instance )
pass
pass
print_i( "------------------------------- Delete EC2 key pairs ----------------------------\n" )
for a_key_pair in an_ec2_conn.get_all_key_pairs() :
delete_key_pair( a_key_pair )
pass
print_i( "---------------------------- Delete EC2 security groups -------------------------\n" )
for a_security_group in an_ec2_conn.get_all_security_groups() :
delete_security_group( an_ec2_conn, a_security_group )
pass
pass
print_i( "-------------------------------------- OK ---------------------------------------\n" )
pass
def entry_point( the_study_objects, the_number_threads, the_printing_depth = 0 ) :
a_spent_time = Timer()
a_worker_pool = WorkerPool( len( the_study_objects ) )
a_deleter = lambda the_object, the_number_threads, the_printing_depth : \
the_object.delete( the_number_threads, the_printing_depth )
for a_study_object in the_study_objects :
print_i( "------------------------------- Removing study ----------------------------------\n" )
print_d( "a_study_object = %s\n" % a_study_object, the_printing_depth )
a_worker_pool.charge( a_deleter, ( a_study_object, the_number_threads, the_printing_depth ) )
pass
a_worker_pool.shutdown()
a_worker_pool.join()
print_d( "a_spent_time = %s, sec\n" % a_spent_time, the_printing_depth )
return
def main() :
#----------------------- Defining utility command-line interface -------------------------
an_usage_description = "%prog"
from ls_options import usage_description as usage_description_options
an_usage_description += usage_description_options()
from cloudflu import VERSION
a_version = "%s" % VERSION
from optparse import IndentedHelpFormatter
a_help_formatter = IndentedHelpFormatter( width = 127 )
from optparse import OptionParser
an_option_parser = OptionParser( usage = an_usage_description, version = a_version, formatter = a_help_formatter )
# Definition of the command line arguments
from ls_options import add as add_options
add_options( an_option_parser )
amazon.security_options.add( an_option_parser )
common.options.add( an_option_parser )
#------------------ Extracting and verifying command-line arguments ----------------------
an_options, an_args = an_option_parser.parse_args()
common.options.extract( an_option_parser )
AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY = amazon.security_options.extract( an_option_parser )
from ls_options import extract as extract_options
a_study_name = extract_options( an_option_parser )
print_i( "--------------------------- Looking for study root ------------------------------\n" )
a_root_object = TRootObject.get( AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY )
print_d( "a_root_object = %s\n" % a_root_object )
if a_study_name == None :
print_i( "---------------------------- Reading the studies --------------------------------\n" )
read_studies( a_root_object, 0 )
pass
else :
a_study_object = None
try:
a_study_object = TStudyObject.get( a_root_object, a_study_name )
except Exception, exc:
print_e( str( exc ) )
pass
print_d( "a_study_object = %s\n" % a_study_object )
print_i( "---------------------------- Reading the study files ----------------------------\n" )
read_files( a_study_object, 0 )
pass
def main() :
#----------------------- Defining utility command-line interface -------------------------
import study_book_options
an_usage_description = "%prog"
an_usage_description += study_book_options.usage_description()
from cloudflu import VERSION
a_version = "%s" % VERSION
from optparse import IndentedHelpFormatter
a_help_formatter = IndentedHelpFormatter( width = 127 )
from optparse import OptionParser
an_option_parser = OptionParser( usage = an_usage_description, version = a_version, formatter = a_help_formatter )
#----------------------- Definition of the command line arguments ------------------------
study_book_options.add( an_option_parser )
common.communication_options.add( an_option_parser )
amazon.security_options.add( an_option_parser )
common.options.add( an_option_parser )
#------------------ Extracting and verifying command-line arguments ----------------------
an_options, an_args = an_option_parser.parse_args()
common.options.extract( an_option_parser )
common.communication_options.extract( an_option_parser )
AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY = amazon.security_options.extract( an_option_parser )
a_study_name = study_book_options.extract( an_option_parser )
print_d( "\n--------------------------- Canonical substitution ------------------------\n" )
import sys; an_engine = sys.argv[ 0 ]
a_call = "%s %s" % ( an_engine, study_book_options.compose( a_study_name ) )
print_d( a_call + '\n' )
print_i( "--------------------------- Defining the study object ---------------------------\n" )
a_root_object = TRootObject.get( AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY )
print_d( "a_root_object = %s\n" % a_root_object )
print_i( "-------------------------- Running actual functionality -------------------------\n" )
a_study_object = entry_point( a_root_object, a_study_name )
print_i( "-------------------- Printing succussive pipeline arguments ---------------------\n" )
print a_study_object.name()
print_d( "\n--------------------------- Canonical substitution ------------------------\n" )
print_d( a_call + '\n' )
print_i( "-------------------------------------- OK ---------------------------------------\n" )
pass
def main() :
#----------------------- Defining utility command-line interface -------------------------
an_usage_description = "%prog"
from study_rm_options import usage_description as usage_description_options
an_usage_description += usage_description_options()
from cloudflu import VERSION
a_version = "%s" % VERSION
from optparse import IndentedHelpFormatter
a_help_formatter = IndentedHelpFormatter( width = 127 )
from optparse import OptionParser
an_option_parser = OptionParser( usage = an_usage_description, version = a_version, formatter = a_help_formatter )
#----------------------- Definition of the command line arguments ------------------------
amazon.security_options.add( an_option_parser )
common.concurrency_options.add( an_option_parser )
common.communication_options.add( an_option_parser )
common.options.add( an_option_parser )
#------------------ Extracting and verifying command-line arguments ----------------------
an_options, an_args = an_option_parser.parse_args()
common.options.extract( an_option_parser )
common.communication_options.extract( an_option_parser )
a_number_threads = common.concurrency_options.extract( an_option_parser )
AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY = amazon.security_options.extract( an_option_parser )
from study_rm_options import extract as extract_options
a_study_names = extract_options( an_option_parser )
print_d( "\n--------------------------- Canonical substitution ------------------------\n" )
import sys; an_engine = sys.argv[ 0 ]
from study_rm_options import compose as compose_options
a_call = "%s %s" % ( an_engine, compose_options( a_study_names ) )
print_d( a_call + '\n' )
print_i( "--------------------------- Looking for study object ----------------------------\n" )
a_root_object = TRootObject.get( AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY )
print_d( "a_root_object = %s\n" % a_root_object )
a_study_objects = []
for a_study_name in a_study_names :
if a_study_name == None :
continue
a_study_object = None
try:
a_study_object = TStudyObject.get( a_root_object, a_study_name )
except Exception, exc:
print_e( '%s\n' % exc, False )
continue
a_study_objects.append( a_study_object )
pass
continue
a_study_object = None
try:
a_study_object = TStudyObject.get( a_root_object, a_study_name )
except Exception, exc:
print_e( '%s\n' % exc, False )
continue
a_study_objects.append( a_study_object )
pass
entry_point( a_study_objects, a_number_threads, 1 )
print_d( "\n--------------------------- Canonical substitution ------------------------\n" )
print_d( a_call + '\n' )
print_i( "-------------------------------------- OK ---------------------------------------\n" )
pass
#------------------------------------------------------------------------------------------
if __name__ == '__main__' :
main()
pass
#------------------------------------------------------------------------------------------
def main() :
#----------------------- Defining utility command-line interface -------------------------
import data_upload_options
import study_book_options
import upload_start_options
import data_seeding_options
an_usage_description = "%prog"
an_usage_description += upload_start_options.usage_description()
an_usage_description += data_upload_options.usage_description()
from cloudflu import VERSION
a_version = "%s" % VERSION
from optparse import IndentedHelpFormatter
a_help_formatter = IndentedHelpFormatter( width = 127 )
from optparse import OptionParser
an_option_parser = OptionParser( usage = an_usage_description, version = a_version, formatter = a_help_formatter )
#----------------------- Definition of the command line arguments ------------------------
data_upload_options.add( an_option_parser )
upload_start_options.add( an_option_parser )
study_book_options.add( an_option_parser )
data_seeding_options.add( an_option_parser )
common.concurrency_options.add( an_option_parser )
common.communication_options.add( an_option_parser )
amazon.security_options.add( an_option_parser )
common.options.add( an_option_parser )
#------------------ Extracting and verifying command-line arguments ----------------------
an_options, an_args = an_option_parser.parse_args()
common.options.extract( an_option_parser )
AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY = amazon.security_options.extract( an_option_parser )
common.communication_options.extract( an_option_parser )
a_number_threads = common.concurrency_options.extract( an_option_parser )
an_upload_seed_size = data_seeding_options.extract( an_option_parser )
a_file2locations = upload_start_options.extract( an_option_parser )
a_booked = data_upload_options.extract( an_option_parser )
a_study_name = study_book_options.extract( an_option_parser )
print_i( "--------------------------- Defining the study object ---------------------------\n" )
from cloudflu.amazon.s3 import TRootObject
a_root_object = TRootObject.get( AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY )
print_d( "a_root_object = %s\n" % a_root_object )
print_i( "-------------------------- Running actual functionality -------------------------\n" )
a_study_object = entry_point( a_root_object, a_file2locations, an_upload_seed_size, a_number_threads, a_study_name, a_booked )
print_i( "-------------------- Printing succussive pipeline arguments ---------------------\n" )
print a_study_object.name()
print_i( "-------------------------------------- OK ---------------------------------------\n" )
pass
def main() :
#----------------------- Defining utility command-line interface -------------------------
import data_transfer_options
import download_options
an_usage_description = "%prog"
an_usage_description += data_transfer_options.usage_description()
an_usage_description += download_options.usage_description()
from cloudflu import VERSION
a_version = "%s" % VERSION
from optparse import IndentedHelpFormatter
a_help_formatter = IndentedHelpFormatter( width = 127 )
from optparse import OptionParser
an_option_parser = OptionParser( usage = an_usage_description, version = a_version, formatter = a_help_formatter )
#----------------------- Definition of the command line arguments ------------------------
data_transfer_options.add( an_option_parser )
download_options.add( an_option_parser )
amazon.security_options.add( an_option_parser )
common.concurrency_options.add( an_option_parser )
common.communication_options.add( an_option_parser )
common.options.add( an_option_parser )
#------------------ Extracting and verifying command-line arguments ----------------------
an_options, an_args = an_option_parser.parse_args()
common.options.extract( an_option_parser )
common.communication_options.extract( an_option_parser )
a_number_threads = common.concurrency_options.extract( an_option_parser )
AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY = amazon.security_options.extract( an_option_parser )
a_located_files, an_output_dir, a_fresh, a_wait, a_remove = download_options.extract( an_option_parser )
a_study_name = data_transfer_options.extract( an_option_parser )
print_d( "\n--------------------------- Canonical substitution ------------------------\n" )
import sys; an_engine = sys.argv[ 0 ]
from download_options import compose as compose_options
a_call = "%s %s %s" % ( an_engine,
data_transfer_options.compose( a_study_name ),
download_options.compose( a_located_files, an_output_dir, a_fresh, a_wait, a_remove ) )
print_d( a_call + '\n' )
print_i( "--------------------------- Looking for study object ----------------------------\n" )
a_root_object = TRootObject.get( AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY )
print_d( "a_root_object = %s\n" % a_root_object )
a_study_object = None
try:
a_study_object = TStudyObject.get( a_root_object, a_study_name )
except Exception, exc:
print_e( '%s\n' % exc )
pass
print_i( "--------------------------- Looking for study object ----------------------------\n" )
a_root_object = TRootObject.get( AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY )
print_d( "a_root_object = %s\n" % a_root_object )
a_study_object = None
try:
a_study_object = TStudyObject.get( a_root_object, a_study_name )
except Exception, exc:
print_e( '%s\n' % exc )
pass
print_d( "a_study_object = %s\n" % a_study_object )
print_i( "-------------------------- Running actual functionality -------------------------\n" )
a_study_object = entry_point( a_study_object, an_output_dir, a_located_files,
a_number_threads, a_wait, a_remove, a_fresh )
print_i( "------------------- Printing succussive pipeline arguments ----------------------\n" )
print a_study_name
print_d( "\n--------------------------- Canonical substitution ------------------------\n" )
print_d( a_call + '\n' )
print_i( "-------------------------------------- OK ---------------------------------------\n" )
pass
def main() :
#----------------------- Defining utility command-line interface -------------------------
an_usage_description = "%prog"
from study_seal_options import usage_description as usage_description_options
an_usage_description += usage_description_options()
from cloudflu import VERSION
a_version = "%s" % VERSION
from optparse import IndentedHelpFormatter
a_help_formatter = IndentedHelpFormatter( width = 127 )
from optparse import OptionParser
an_option_parser = OptionParser( usage = an_usage_description, version = a_version, formatter = a_help_formatter )
#----------------------- Definition of the command line arguments ------------------------
from study_seal_options import add as add_options
add_options( an_option_parser )
amazon.security_options.add( an_option_parser )
common.concurrency_options.add( an_option_parser )
common.communication_options.add( an_option_parser )
common.options.add( an_option_parser )
#------------------ Extracting and verifying command-line arguments ----------------------
an_options, an_args = an_option_parser.parse_args()
common.options.extract( an_option_parser )
common.communication_options.extract( an_option_parser )
a_number_threads = common.concurrency_options.extract( an_option_parser )
AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY = amazon.security_options.extract( an_option_parser )
from study_seal_options import extract as extract_options
a_study_name = extract_options( an_option_parser )
print_d( "\n--------------------------- Canonical substitution ------------------------\n" )
import sys; an_engine = sys.argv[ 0 ]
from study_seal_options import compose as compose_options
a_call = "%s %s" % ( an_engine, compose_options( a_study_name ) )
print_d( a_call + '\n' )
print_i( "--------------------------- Defining the study object ---------------------------\n" )
a_spent_time = Timer()
a_root_object = TRootObject.get( AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY )
print_d( "a_root_object = %s\n" % a_root_object )
a_study_object = TStudyObject.get( a_root_object, a_study_name )
print_d( "a_study_object = %s\n" % a_study_object )
a_study_object.seal()
print_d( "is sealed : %r\n" % a_study_object.sealed() )
print_d( "a_spent_time = %s, sec\n" % a_spent_time )
print_i( "-------------------- Printing succussive pipeline arguments ---------------------\n" )
print a_study_name
print_d( "\n--------------------------- Canonical substitution ------------------------\n" )
print_d( a_call + '\n' )
print_i( "-------------------------------------- OK ---------------------------------------\n" )
pass
def main() :
#----------------------- Defining utility command-line interface -------------------------
an_usage_description = "%prog"
from rm_options import usage_description as usage_description_options
an_usage_description += usage_description_options()
from cloudflu import VERSION
a_version = "%s" % VERSION
from optparse import IndentedHelpFormatter
a_help_formatter = IndentedHelpFormatter( width = 127 )
from optparse import OptionParser
an_option_parser = OptionParser( usage = an_usage_description, version = a_version, formatter = a_help_formatter )
#----------------------- Definition of the command line arguments ------------------------
from rm_options import add as add_options
add_options( an_option_parser )
amazon.security_options.add( an_option_parser )
common.concurrency_options.add( an_option_parser )
common.communication_options.add( an_option_parser )
common.options.add( an_option_parser )
#------------------ Extracting and verifying command-line arguments ----------------------
an_options, an_args = an_option_parser.parse_args()
common.options.extract( an_option_parser )
common.communication_options.extract( an_option_parser )
a_number_threads = common.concurrency_options.extract( an_option_parser )
AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY = amazon.security_options.extract( an_option_parser )
from rm_options import extract as extract_options
a_study_name, a_located_files = extract_options( an_option_parser )
print_i( "--------------------------- Looking for study object ----------------------------\n" )
a_root_object = TRootObject.get( AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY )
print_d( "a_root_object = %s\n" % a_root_object )
a_study_object = TStudyObject.get( a_root_object, a_study_name )
print_d( "a_study_object = %s\n" % a_study_object )
print_i( "-------------------------- Removing study files -----------------------------\n" )
a_worker_pool = WorkerPool( len( a_located_files ) )
a_deleter = lambda the_object, the_number_threads, the_printing_depth : \
the_object.delete( the_number_threads, the_printing_depth )
for a_located_file in a_located_files :
print_d( "a_located_file = %s\n" % a_located_file )
a_file_object = TFileObject.get( a_study_object, a_located_file )
print_d( "a_file_object = %s\n" % a_file_object, 1 )
a_worker_pool.charge( a_deleter, ( a_file_object, a_number_threads, 2 ) )
pass
a_worker_pool.shutdown()
a_worker_pool.join()
print_i( "------------------- Printing succussive pipeline arguments ----------------------\n" )
print a_study_name
print_i( "-------------------------------------- OK ---------------------------------------\n" )
pass
def main() :
#----------------------- Defining utility command-line interface -------------------------
an_usage_description = "%prog"
an_usage_description += common.concurrency_options.usage_description()
from cloudflu import VERSION
a_version = "%s" % VERSION
from optparse import IndentedHelpFormatter
a_help_formatter = IndentedHelpFormatter( width = 127 )
from optparse import OptionParser
an_option_parser = OptionParser( usage = an_usage_description, version = a_version, formatter = a_help_formatter )
#----------------------- Definition of the command line arguments ------------------------
common.concurrency_options.add( an_option_parser )
common.communication_options.add( an_option_parser )
amazon.security_options.add( an_option_parser )
common.options.add( an_option_parser )
#------------------ Extracting and verifying command-line arguments ----------------------
an_options, an_args = an_option_parser.parse_args()
common.options.extract( an_option_parser )
common.communication_options.extract( an_option_parser )
a_number_threads = common.concurrency_options.extract( an_option_parser )
AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY = amazon.security_options.extract( an_option_parser )
print_i( "-------------------------- Running actual functionality -------------------------\n" )
import boto; a_s3_conn = boto.connect_s3( AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY )
a_worker_pool = WorkerPool( a_number_threads )
print_i( "------------------------ Remove all the bucket keys, first ----------------------\n" )
for a_bucket in a_s3_conn.get_all_buckets() :
try :
a_s3_bucket_keys = a_bucket.get_all_keys()
print_d( "'%s' : %d\n" % ( a_bucket.name, len( a_s3_bucket_keys ) ) )
for a_s3_bucket_key in a_bucket.list() :
print_d( "\t'%s'\n" % ( a_s3_bucket_key.name ) )
a_worker_pool.charge( lambda the_s3_bucket_key : the_s3_bucket_key.delete(), [ a_s3_bucket_key ] )
pass
except :
pass
pass
a_worker_pool.join()
print_i( "--------------------------- Remove the buckets itself ---------------------------\n" )
for a_bucket in a_s3_conn.get_all_buckets() :
print_d( "'%s'\n" % ( a_bucket.name ) )
a_worker_pool.charge( lambda the_s3_bucket : the_s3_bucket.delete(), [ a_bucket ] )
pass
a_worker_pool.shutdown()
a_worker_pool.join()
print_i( "-------------------------------------- OK ---------------------------------------\n" )
pass
