def extract( the_option_parser ) :
    from cloudflu.common import print_d, print_i, print_e
    import os, os.path
    
    an_options, an_args = the_option_parser.parse_args()

    a_located_files = an_options.located_files
    if a_located_files != None :
        from cloudflu import common
        a_located_files = a_located_files.split( common.arg_list_separator() )
        pass

    print_d( "a_located_files = %s\n" % a_located_files )

    an_output_dir = an_options.output_dir
    print_d( "an_output_dir = '%s'\n" % an_output_dir )
    
    a_fresh = an_options.fresh
    print_d( "a_fresh = %s\n" % a_fresh )

    a_wait = an_options.wait
    print_d( "a_wait = %s\n" % a_wait )

    a_remove = an_options.remove
    print_d( "a_remove = %s\n" % a_remove )

    return a_located_files, an_output_dir, a_fresh, a_wait, a_remove
def delete_security_group( the_ec2_conn, the_security_group ) :
    if the_security_group.name != 'default' :
        print_d( the_security_group.name )

        the_ec2_conn.delete_security_group( the_security_group.name )
        pass
    pass
def extract(the_option_parser, the_case_dir=None):
    from cloudflu.common import print_d

    an_options, an_args = the_option_parser.parse_args()

    an_output_dir = an_options.output_dir
    if an_output_dir == None:
        if the_case_dir != None:
            import time

            an_output_dir = "%s %s" % (the_case_dir, time.strftime("%Y-%m-%d %H:%M"))
        else:
            the_option_parser.error("--output-dir='%s' is not defined\n" % an_output_dir)
            pass
        pass

    import os.path

    an_output_dir = os.path.abspath(os.path.expanduser(an_output_dir))
    print_d("an_output_dir = '%s'\n" % an_output_dir)

    a_before_hook = an_options.before_hook
    a_time_hook = an_options.time_hook
    an_after_hook = an_options.after_hook

    return an_output_dir, a_before_hook, a_time_hook, an_after_hook
def extract( the_option_parser ) :
    from cloudflu.common import print_d, print_i, print_e
    
    an_options, an_args = the_option_parser.parse_args()

    if len( an_args ) != 0 :
        a_solver_run = ' '.join( an_args )
    else:
        a_solver_run = 'cat ./log.interFoam.cloudflu'
        pass
    print_d( "a_solver_run = '%s'\n" % a_solver_run )

    import os.path; a_case_dir = os.path.abspath( os.path.expanduser( an_options.case_dir ) )
    if not os.path.isdir( a_case_dir ) :
        the_option_parser.error( "--case-dir='%s' should be a folder\n" % a_case_dir )
        pass

    a_solver_log = an_options.solver_log

    a_watched_keyword = an_options.watched_keyword
    print_d( "a_watched_keyword = '%s'\n" % a_watched_keyword )

    a_time_log = an_options.time_log

    return a_solver_run, a_case_dir, a_solver_log, a_watched_keyword, a_time_log
def execute( the_ec2_conn, the_cluster_id ) :
    try:
        a_reservation = ec2.use.get_reservation( the_ec2_conn, the_cluster_id )
        print_d( '< %r > : %s\n' % ( a_reservation, a_reservation.instances ) )
    except Exception, exc:
        print_e( '%s\n' % exc, False )
        return
Beispiel #6
0
def entry_point( the_study_object, the_output_dir, the_located_files = None, 
                 the_number_threads = 3, the_wait = True, the_remove = False, the_fresh = False, the_callback = None ) :
    a_spent_time = Timer()
    
    if the_output_dir == None :
        the_output_dir = os.path.join( os.curdir, the_study_object.name() )
        pass

    if the_located_files == None :
        download_files( the_study_object, the_output_dir,
                        the_number_threads, the_wait, the_remove, the_fresh, the_callback )
    else :
        a_worker_pool = TaggedWorkerPool( the_number_threads )
        
        for a_located_file in the_located_files :
            a_worker_pool.charge( a_located_file, file_entry_point, 
                                  ( the_study_object, the_output_dir, a_located_file, 
                                    the_number_threads, the_wait, the_remove, the_fresh, the_callback ) )
            pass
    
        a_worker_pool.shutdown()
        a_worker_pool.join()
        pass
    
    print_d( "a_spent_time = %s, sec\n" % a_spent_time )

    return the_study_object
Beispiel #7
0
def download_seeds( the_file_object, the_file_basename, the_output_dir, the_number_threads, the_printing_depth ) :
    an_is_download_ok = False
    while not the_file_object.sealed() or not an_is_download_ok :
        a_worker_pool = WorkerPool( the_number_threads )
        try:
            for a_seed_object in the_file_object :
                a_hex_md5 = a_seed_object.hex_md5()
                a_seed_path = os.path.join( the_output_dir, a_seed_object.basename() )

                if os.path.exists( a_seed_path ) :
                    a_file_pointer = open( a_seed_path, 'rb' )
                    a_md5 = compute_md5( a_file_pointer )[ 0 ]
                    
                    if a_hex_md5 == a_md5 :
                        continue
                    
                    os.remove( a_seed_path )

                    pass

                print_d( "a_seed_path = '%s'\n" % a_seed_path, the_printing_depth )

                a_worker_pool.charge( download_seed, ( a_seed_object, a_seed_path, the_printing_depth + 1 ) )
        except:
            from cloudflu.common import print_traceback
            print_traceback( the_printing_depth )
            pass

        a_worker_pool.shutdown()
        an_is_download_ok = a_worker_pool.is_all_right()

        print_d( "'%s'.uploaded() == %s\n" % ( the_file_object.located_file(), the_file_object.sealed() ), the_printing_depth )
        pass
    
    return True
def entry_point( the_study_object, the_file2locations, the_upload_seed_size, the_number_threads ) :
    a_spent_time = Timer()
    
    upload_files( the_study_object, the_file2locations, the_upload_seed_size, the_number_threads, 0 )
    
    print_d( "a_spent_time = %s, sec\n" % a_spent_time )

    return the_study_object
def extract( the_option_parser ) :
    an_options, an_args = the_option_parser.parse_args()

    a_hostfile = an_options.hostfile
    from cloudflu.common import print_d
    print_d( "a_hostfile = '%s'\n" % a_hostfile )

    return a_hostfile
def upload_file( the_file_object, the_number_threads, the_printing_depth ) :
    a_working_dir = generate_uploading_dir( the_file_object.file_path() )
    if not os.path.exists( a_working_dir ) :
        return True

    print_d( "a_working_dir = '%s'\n" % a_working_dir, the_printing_depth )

    return upload_seeds( the_file_object, a_working_dir, the_number_threads, the_printing_depth + 1 )
def extract( the_option_parser ) :
    from cloudflu.common import print_d
    an_options, an_args = the_option_parser.parse_args()

    a_booked = an_options.booked
    print_d( "a_booked = %s\n" % a_booked )

    return a_booked
def main() :
    #----------------------- Defining utility command-line interface -------------------------
    an_usage_description = "%prog"
    
    from cloudflu import VERSION
    a_version = "%s" % VERSION
    
    from optparse import IndentedHelpFormatter
    a_help_formatter = IndentedHelpFormatter( width = 127 )
    
    from optparse import OptionParser
    an_option_parser = OptionParser( usage = an_usage_description, version = a_version, formatter = a_help_formatter )
    

    #----------------------- Definition of the command line arguments ------------------------
    amazon.security_options.add( an_option_parser )

    common.options.add( an_option_parser )

    
    #------------------ Extracting and verifying command-line arguments ----------------------
    an_options, an_args = an_option_parser.parse_args()

    common.options.extract( an_option_parser )
    
    AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY = amazon.security_options.extract( an_option_parser )


    print_i( "-------------------------- Running actual functionality -------------------------\n" )
    import boto.ec2
    for a_region in boto.ec2.regions( aws_access_key_id = AWS_ACCESS_KEY_ID, aws_secret_access_key = AWS_SECRET_ACCESS_KEY ) :
        an_ec2_conn = a_region.connect()
        print_d( "region - '%s'" % a_region.name )


        print_i( "------------------------------- Delete EC2 instances ----------------------------\n" )
        for a_reservation in an_ec2_conn.get_all_instances() :
            for an_instance in a_reservation.instances :
                terminate_instance( an_instance )
                pass
            pass

        print_i( "------------------------------- Delete EC2 key pairs ----------------------------\n" )
        for a_key_pair in an_ec2_conn.get_all_key_pairs() :
            delete_key_pair( a_key_pair )
            pass


        print_i( "---------------------------- Delete EC2 security groups -------------------------\n" )
        for a_security_group in an_ec2_conn.get_all_security_groups() :
            delete_security_group( an_ec2_conn, a_security_group )
            pass

        pass


    print_i( "-------------------------------------- OK ---------------------------------------\n" )
    pass
Beispiel #13
0
def read_files( the_study_object, the_printing_depth ) :
    "Reading the study files"
    for a_file_object in the_study_object :
        print_d( "a_file_object = %s\n" % a_file_object, the_printing_depth )
        print a_file_object.located_file()
        
        pass

    pass
Beispiel #14
0
def main() :
    #----------------------- Defining utility command-line interface -------------------------    
    an_usage_description = "%prog"

    from ls_options import usage_description as usage_description_options
    an_usage_description += usage_description_options()
    
    from cloudflu import VERSION
    a_version = "%s" % VERSION

    from optparse import IndentedHelpFormatter
    a_help_formatter = IndentedHelpFormatter( width = 127 )

    from optparse import OptionParser
    an_option_parser = OptionParser( usage = an_usage_description, version = a_version, formatter = a_help_formatter )

    # Definition of the command line arguments
    from ls_options import add as add_options
    add_options( an_option_parser )

    amazon.security_options.add( an_option_parser )

    common.options.add( an_option_parser )


    #------------------ Extracting and verifying command-line arguments ----------------------
    an_options, an_args = an_option_parser.parse_args()

    common.options.extract( an_option_parser )

    AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY = amazon.security_options.extract( an_option_parser )

    from ls_options import extract as extract_options
    a_study_name = extract_options( an_option_parser )

    print_i( "--------------------------- Looking for study root ------------------------------\n" )
    a_root_object = TRootObject.get( AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY )
    print_d( "a_root_object = %s\n" % a_root_object )


    if a_study_name == None :
        print_i( "---------------------------- Reading the studies --------------------------------\n" )
        read_studies( a_root_object, 0 )
        pass
    else :
        a_study_object = None
        try:
            a_study_object = TStudyObject.get( a_root_object, a_study_name )
        except Exception, exc:
            print_e( str( exc ) )
            pass
        print_d( "a_study_object = %s\n" % a_study_object )

        print_i( "---------------------------- Reading the study files ----------------------------\n" )
        read_files( a_study_object, 0 )
        
        pass
Beispiel #15
0
def read_studies( the_root_object, the_printing_depth ) :
    "Reading the studies"
    for a_study_object in the_root_object :
        print_d( "a_study_object = %s\n" % a_study_object, the_printing_depth )
        print a_study_object.name()

        pass

    pass
def extract( the_option_parser ) :
    from cloudflu.common import print_d, print_i, print_e
    
    an_options, an_args = the_option_parser.parse_args()

    an_upload_seed_size = an_options.upload_seed_size
    print_d( "an_upload_seed_size = %d, bytes\n" % an_upload_seed_size )

    return an_upload_seed_size
def terminate_instance( the_instance ) :
    a_status = the_instance.update()
    if a_status != 'terminated' :
        print_d( "%s : %s : '%s'\n" % ( the_instance, a_status, the_instance.dns_name ) )
        the_instance.terminate()

        pass

    pass
Beispiel #18
0
 def _next( self ) :
     for a_study_key in self._bucket.list( prefix = _decorate_key_name( study_name_prefix() ) ) :
         try :
             yield TStudyObject._get( self, get_key_name( a_study_key ) )
         except :
             print_d( "study '%s' has no corresponding bucket\n" % get_key_name( a_study_key ) )
             pass
         
         pass
     
     pass
Beispiel #19
0
def download_seed( the_seed_object, the_file_path, the_printing_depth ) :
    try :
        the_seed_object.download( the_file_path )
        print_d( "the_seed_object = %s\n" % the_seed_object, the_printing_depth )

        return True
    except :
        from cloudflu.common import print_traceback
        print_traceback( the_printing_depth + 1 )
        pass

    return False
def upload_files( the_study_object, the_number_threads, the_printing_depth ) :
    a_worker_pool = WorkerPool( the_number_threads )

    for a_file_object in the_study_object :
        print_d( "a_file_object = %s\n" % a_file_object, the_printing_depth )

        a_worker_pool.charge( upload_file, ( a_file_object, the_number_threads, the_printing_depth + 1 ) )                    
        pass

    a_worker_pool.shutdown()
    a_worker_pool.join()
    pass
def extract(the_option_parser):
    from cloudflu.common import print_d, print_i, print_e

    an_options, an_args = the_option_parser.parse_args()

    a_study_name = an_options.study_name

    if a_study_name != None:
        print_d("a_study_name = '%s'\n" % a_study_name)
        pass

    return a_study_name
def upload_seed( the_file_object, the_seed_name, the_seed_path, the_printing_depth ) :
    "Uploading file item"
    try :
        a_seed_object = TSeedObject.create( the_file_object, the_seed_name, the_seed_path )
        print_d( "%s\n" % a_seed_object, the_printing_depth )
        
        return True
    except :
        from cloudflu.common import print_traceback
        print_traceback( the_printing_depth )
        pass

    return False
def extract( the_option_parser ) :
    from cloudflu.common import print_d
    an_options, an_args = the_option_parser.parse_args()

    a_located_files = an_options.located_files
    if a_located_files != None :
        from cloudflu import common
        a_located_files = a_located_files.split( common.arg_list_separator() )
        pass

    print_d( "a_located_files = %s\n" % a_located_files )

    return a_located_files
def extract( the_option_parser ) :
    from cloudflu.common import print_d, print_i, print_e
    
    an_options, an_args = the_option_parser.parse_args()

    a_study_name = an_options.study_name
    if a_study_name == None :
        from cloudflu.preferences import get_input
        a_study_name, an_args = get_input( an_args )
        pass
    
    print_d( "a_study_name = '%s'\n" % a_study_name )

    return a_study_name
Beispiel #25
0
def file_entry_point( the_study_object, the_output_dir, the_located_file, 
                      the_number_threads, the_wait, the_remove, the_fresh, the_callback ) :
    if the_wait == False or the_study_object.sealed() :
        try:
            a_file_object = TFileObject.get( the_study_object, the_located_file )
            download_file( a_file_object, the_output_dir, the_number_threads, the_remove, the_fresh, the_callback )
        except:
            from cloudflu.common import print_traceback
            print_traceback( 0 )
            pass
        pass
    else:
        print_d( "waiting " )
        
        while True :
            an_sealed = the_study_object.sealed()
            
            try:
                a_file_object = TFileObject.get( the_study_object, the_located_file )
                print_d( "\n" )
                download_file( a_file_object, the_output_dir, the_number_threads, the_remove, the_fresh, the_callback )
                break
            except:
                if an_sealed :
                    break
                print_d( "." )
                pass
            
            
            pass

        print_d( " compleated\n" )
        pass
    
    pass
def delete_key_pair( the_key_pair ) :
    print_d( the_key_pair.name )

    # an_ec2_conn.delete_key_pair( the_key_pair ) # Does not work (bug)
    the_key_pair.delete()

    import os, os.path
    a_key_pair_dir = os.path.expanduser( "~/.ssh")
    a_key_pair_file = os.path.join( a_key_pair_dir, the_key_pair.name ) + os.path.extsep + "pem"

    if os.path.isfile( a_key_pair_file ) :
        os.remove( a_key_pair_file )
        pass
    
    pass
Beispiel #27
0
def echo(the_password, the_identity_file, the_host_port, the_login_name, the_host_name):
    from cloudflu.common import print_d

    if the_password != None:
        print_d(
            'sshpass -p %s ssh -o "StrictHostKeyChecking no" -p %d %s@%s\n'
            % (the_password, the_host_port, the_login_name, the_host_name)
        )
    else:
        print_d(
            'ssh -o "StrictHostKeyChecking no" -i %s -p %d %s@%s\n'
            % (the_identity_file, the_host_port, the_login_name, the_host_name)
        )
        pass

    pass
Beispiel #28
0
def wait( the_ssh_connect, the_ssh_client, the_command ) :
    from cloudflu.common import print_d
    print_d( "ssh'ing " )
    
    while True :
        try :
            print_d( '.' )
            the_ssh_connect()
            command( the_ssh_client, the_command )
            break
        except :
            # import sys, traceback
            # traceback.print_exc( file = sys.stderr )
            continue
        pass

    pass
Beispiel #29
0
def extract( the_option_parser ) :
    from cloudflu.common import print_d, print_i, print_e

    an_options, an_args = the_option_parser.parse_args()

    a_study_name = an_options.study_name
    if a_study_name == None :
        the_option_parser.error( "Use --study-name option to define proper value\n" )
        pass
    
    print_d( "a_study_name = '%s'\n" % a_study_name )

    from cloudflu.preferences import get_inputs
    a_located_files = get_inputs( an_args )

    print_d( "a_located_files = %r\n" % a_located_files )

    return a_study_name, a_located_files
Beispiel #30
0
    def delete( self, the_number_threads, the_printing_depth ) :
        print_d( "deleting - %s\n" % self, the_printing_depth )

        a_worker_pool = WorkerPool( the_number_threads )

        a_deleter = lambda the_object, the_printing_depth : \
            the_object.delete( the_printing_depth )

        for a_seed_object in self :
            a_worker_pool.charge( a_deleter, ( a_seed_object, the_printing_depth + 1 ) )
            pass

        TSealingObject.delete( self )
        
        a_worker_pool.shutdown()
        a_worker_pool.join()

        self._key.delete()
        pass