def extract( the_option_parser ) :
from cloudflu.common import print_d, print_i, print_e
import os, os.path
an_options, an_args = the_option_parser.parse_args()
a_located_files = an_options.located_files
if a_located_files != None :
from cloudflu import common
a_located_files = a_located_files.split( common.arg_list_separator() )
pass
print_d( "a_located_files = %s\n" % a_located_files )
an_output_dir = an_options.output_dir
print_d( "an_output_dir = '%s'\n" % an_output_dir )
a_fresh = an_options.fresh
print_d( "a_fresh = %s\n" % a_fresh )
a_wait = an_options.wait
print_d( "a_wait = %s\n" % a_wait )
a_remove = an_options.remove
print_d( "a_remove = %s\n" % a_remove )
return a_located_files, an_output_dir, a_fresh, a_wait, a_remove
def delete_security_group( the_ec2_conn, the_security_group ) :
if the_security_group.name != 'default' :
print_d( the_security_group.name )
the_ec2_conn.delete_security_group( the_security_group.name )
pass
pass
def extract(the_option_parser, the_case_dir=None):
from cloudflu.common import print_d
an_options, an_args = the_option_parser.parse_args()
an_output_dir = an_options.output_dir
if an_output_dir == None:
if the_case_dir != None:
import time
an_output_dir = "%s %s" % (the_case_dir, time.strftime("%Y-%m-%d %H:%M"))
else:
the_option_parser.error("--output-dir='%s' is not defined\n" % an_output_dir)
pass
pass
import os.path
an_output_dir = os.path.abspath(os.path.expanduser(an_output_dir))
print_d("an_output_dir = '%s'\n" % an_output_dir)
a_before_hook = an_options.before_hook
a_time_hook = an_options.time_hook
an_after_hook = an_options.after_hook
return an_output_dir, a_before_hook, a_time_hook, an_after_hook
def extract( the_option_parser ) :
from cloudflu.common import print_d, print_i, print_e
an_options, an_args = the_option_parser.parse_args()
if len( an_args ) != 0 :
a_solver_run = ' '.join( an_args )
else:
a_solver_run = 'cat ./log.interFoam.cloudflu'
pass
print_d( "a_solver_run = '%s'\n" % a_solver_run )
import os.path; a_case_dir = os.path.abspath( os.path.expanduser( an_options.case_dir ) )
if not os.path.isdir( a_case_dir ) :
the_option_parser.error( "--case-dir='%s' should be a folder\n" % a_case_dir )
pass
a_solver_log = an_options.solver_log
a_watched_keyword = an_options.watched_keyword
print_d( "a_watched_keyword = '%s'\n" % a_watched_keyword )
a_time_log = an_options.time_log
return a_solver_run, a_case_dir, a_solver_log, a_watched_keyword, a_time_log
def execute( the_ec2_conn, the_cluster_id ) :
try:
a_reservation = ec2.use.get_reservation( the_ec2_conn, the_cluster_id )
print_d( '< %r > : %s\n' % ( a_reservation, a_reservation.instances ) )
except Exception, exc:
print_e( '%s\n' % exc, False )
return
def entry_point( the_study_object, the_output_dir, the_located_files = None,
the_number_threads = 3, the_wait = True, the_remove = False, the_fresh = False, the_callback = None ) :
a_spent_time = Timer()
if the_output_dir == None :
the_output_dir = os.path.join( os.curdir, the_study_object.name() )
pass
if the_located_files == None :
download_files( the_study_object, the_output_dir,
the_number_threads, the_wait, the_remove, the_fresh, the_callback )
else :
a_worker_pool = TaggedWorkerPool( the_number_threads )
for a_located_file in the_located_files :
a_worker_pool.charge( a_located_file, file_entry_point,
( the_study_object, the_output_dir, a_located_file,
the_number_threads, the_wait, the_remove, the_fresh, the_callback ) )
pass
a_worker_pool.shutdown()
a_worker_pool.join()
pass
print_d( "a_spent_time = %s, sec\n" % a_spent_time )
return the_study_object
def download_seeds( the_file_object, the_file_basename, the_output_dir, the_number_threads, the_printing_depth ) :
an_is_download_ok = False
while not the_file_object.sealed() or not an_is_download_ok :
a_worker_pool = WorkerPool( the_number_threads )
try:
for a_seed_object in the_file_object :
a_hex_md5 = a_seed_object.hex_md5()
a_seed_path = os.path.join( the_output_dir, a_seed_object.basename() )
if os.path.exists( a_seed_path ) :
a_file_pointer = open( a_seed_path, 'rb' )
a_md5 = compute_md5( a_file_pointer )[ 0 ]
if a_hex_md5 == a_md5 :
continue
os.remove( a_seed_path )
pass
print_d( "a_seed_path = '%s'\n" % a_seed_path, the_printing_depth )
a_worker_pool.charge( download_seed, ( a_seed_object, a_seed_path, the_printing_depth + 1 ) )
except:
from cloudflu.common import print_traceback
print_traceback( the_printing_depth )
pass
a_worker_pool.shutdown()
an_is_download_ok = a_worker_pool.is_all_right()
print_d( "'%s'.uploaded() == %s\n" % ( the_file_object.located_file(), the_file_object.sealed() ), the_printing_depth )
pass
return True
def entry_point( the_study_object, the_file2locations, the_upload_seed_size, the_number_threads ) :
a_spent_time = Timer()
upload_files( the_study_object, the_file2locations, the_upload_seed_size, the_number_threads, 0 )
print_d( "a_spent_time = %s, sec\n" % a_spent_time )
return the_study_object
def extract( the_option_parser ) :
an_options, an_args = the_option_parser.parse_args()
a_hostfile = an_options.hostfile
from cloudflu.common import print_d
print_d( "a_hostfile = '%s'\n" % a_hostfile )
return a_hostfile
def upload_file( the_file_object, the_number_threads, the_printing_depth ) :
a_working_dir = generate_uploading_dir( the_file_object.file_path() )
if not os.path.exists( a_working_dir ) :
return True
print_d( "a_working_dir = '%s'\n" % a_working_dir, the_printing_depth )
return upload_seeds( the_file_object, a_working_dir, the_number_threads, the_printing_depth + 1 )
def extract( the_option_parser ) :
from cloudflu.common import print_d
an_options, an_args = the_option_parser.parse_args()
a_booked = an_options.booked
print_d( "a_booked = %s\n" % a_booked )
return a_booked
def main() :
#----------------------- Defining utility command-line interface -------------------------
an_usage_description = "%prog"
from cloudflu import VERSION
a_version = "%s" % VERSION
from optparse import IndentedHelpFormatter
a_help_formatter = IndentedHelpFormatter( width = 127 )
from optparse import OptionParser
an_option_parser = OptionParser( usage = an_usage_description, version = a_version, formatter = a_help_formatter )
#----------------------- Definition of the command line arguments ------------------------
amazon.security_options.add( an_option_parser )
common.options.add( an_option_parser )
#------------------ Extracting and verifying command-line arguments ----------------------
an_options, an_args = an_option_parser.parse_args()
common.options.extract( an_option_parser )
AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY = amazon.security_options.extract( an_option_parser )
print_i( "-------------------------- Running actual functionality -------------------------\n" )
import boto.ec2
for a_region in boto.ec2.regions( aws_access_key_id = AWS_ACCESS_KEY_ID, aws_secret_access_key = AWS_SECRET_ACCESS_KEY ) :
an_ec2_conn = a_region.connect()
print_d( "region - '%s'" % a_region.name )
print_i( "------------------------------- Delete EC2 instances ----------------------------\n" )
for a_reservation in an_ec2_conn.get_all_instances() :
for an_instance in a_reservation.instances :
terminate_instance( an_instance )
pass
pass
print_i( "------------------------------- Delete EC2 key pairs ----------------------------\n" )
for a_key_pair in an_ec2_conn.get_all_key_pairs() :
delete_key_pair( a_key_pair )
pass
print_i( "---------------------------- Delete EC2 security groups -------------------------\n" )
for a_security_group in an_ec2_conn.get_all_security_groups() :
delete_security_group( an_ec2_conn, a_security_group )
pass
pass
print_i( "-------------------------------------- OK ---------------------------------------\n" )
pass
def read_files( the_study_object, the_printing_depth ) :
"Reading the study files"
for a_file_object in the_study_object :
print_d( "a_file_object = %s\n" % a_file_object, the_printing_depth )
print a_file_object.located_file()
pass
pass
def main() :
#----------------------- Defining utility command-line interface -------------------------
an_usage_description = "%prog"
from ls_options import usage_description as usage_description_options
an_usage_description += usage_description_options()
from cloudflu import VERSION
a_version = "%s" % VERSION
from optparse import IndentedHelpFormatter
a_help_formatter = IndentedHelpFormatter( width = 127 )
from optparse import OptionParser
an_option_parser = OptionParser( usage = an_usage_description, version = a_version, formatter = a_help_formatter )
# Definition of the command line arguments
from ls_options import add as add_options
add_options( an_option_parser )
amazon.security_options.add( an_option_parser )
common.options.add( an_option_parser )
#------------------ Extracting and verifying command-line arguments ----------------------
an_options, an_args = an_option_parser.parse_args()
common.options.extract( an_option_parser )
AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY = amazon.security_options.extract( an_option_parser )
from ls_options import extract as extract_options
a_study_name = extract_options( an_option_parser )
print_i( "--------------------------- Looking for study root ------------------------------\n" )
a_root_object = TRootObject.get( AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY )
print_d( "a_root_object = %s\n" % a_root_object )
if a_study_name == None :
print_i( "---------------------------- Reading the studies --------------------------------\n" )
read_studies( a_root_object, 0 )
pass
else :
a_study_object = None
try:
a_study_object = TStudyObject.get( a_root_object, a_study_name )
except Exception, exc:
print_e( str( exc ) )
pass
print_d( "a_study_object = %s\n" % a_study_object )
print_i( "---------------------------- Reading the study files ----------------------------\n" )
read_files( a_study_object, 0 )
pass
def read_studies( the_root_object, the_printing_depth ) :
"Reading the studies"
for a_study_object in the_root_object :
print_d( "a_study_object = %s\n" % a_study_object, the_printing_depth )
print a_study_object.name()
pass
pass
def extract( the_option_parser ) :
from cloudflu.common import print_d, print_i, print_e
an_options, an_args = the_option_parser.parse_args()
an_upload_seed_size = an_options.upload_seed_size
print_d( "an_upload_seed_size = %d, bytes\n" % an_upload_seed_size )
return an_upload_seed_size
def terminate_instance( the_instance ) :
a_status = the_instance.update()
if a_status != 'terminated' :
print_d( "%s : %s : '%s'\n" % ( the_instance, a_status, the_instance.dns_name ) )
the_instance.terminate()
pass
pass
def _next( self ) :
for a_study_key in self._bucket.list( prefix = _decorate_key_name( study_name_prefix() ) ) :
try :
yield TStudyObject._get( self, get_key_name( a_study_key ) )
except :
print_d( "study '%s' has no corresponding bucket\n" % get_key_name( a_study_key ) )
pass
pass
pass
def download_seed( the_seed_object, the_file_path, the_printing_depth ) :
try :
the_seed_object.download( the_file_path )
print_d( "the_seed_object = %s\n" % the_seed_object, the_printing_depth )
return True
except :
from cloudflu.common import print_traceback
print_traceback( the_printing_depth + 1 )
pass
return False
def upload_files( the_study_object, the_number_threads, the_printing_depth ) :
a_worker_pool = WorkerPool( the_number_threads )
for a_file_object in the_study_object :
print_d( "a_file_object = %s\n" % a_file_object, the_printing_depth )
a_worker_pool.charge( upload_file, ( a_file_object, the_number_threads, the_printing_depth + 1 ) )
pass
a_worker_pool.shutdown()
a_worker_pool.join()
pass
def extract(the_option_parser):
from cloudflu.common import print_d, print_i, print_e
an_options, an_args = the_option_parser.parse_args()
a_study_name = an_options.study_name
if a_study_name != None:
print_d("a_study_name = '%s'\n" % a_study_name)
pass
return a_study_name
def upload_seed( the_file_object, the_seed_name, the_seed_path, the_printing_depth ) :
"Uploading file item"
try :
a_seed_object = TSeedObject.create( the_file_object, the_seed_name, the_seed_path )
print_d( "%s\n" % a_seed_object, the_printing_depth )
return True
except :
from cloudflu.common import print_traceback
print_traceback( the_printing_depth )
pass
return False
def extract( the_option_parser ) :
from cloudflu.common import print_d
an_options, an_args = the_option_parser.parse_args()
a_located_files = an_options.located_files
if a_located_files != None :
from cloudflu import common
a_located_files = a_located_files.split( common.arg_list_separator() )
pass
print_d( "a_located_files = %s\n" % a_located_files )
return a_located_files
def extract( the_option_parser ) :
from cloudflu.common import print_d, print_i, print_e
an_options, an_args = the_option_parser.parse_args()
a_study_name = an_options.study_name
if a_study_name == None :
from cloudflu.preferences import get_input
a_study_name, an_args = get_input( an_args )
pass
print_d( "a_study_name = '%s'\n" % a_study_name )
return a_study_name
def file_entry_point( the_study_object, the_output_dir, the_located_file,
the_number_threads, the_wait, the_remove, the_fresh, the_callback ) :
if the_wait == False or the_study_object.sealed() :
try:
a_file_object = TFileObject.get( the_study_object, the_located_file )
download_file( a_file_object, the_output_dir, the_number_threads, the_remove, the_fresh, the_callback )
except:
from cloudflu.common import print_traceback
print_traceback( 0 )
pass
pass
else:
print_d( "waiting " )
while True :
an_sealed = the_study_object.sealed()
try:
a_file_object = TFileObject.get( the_study_object, the_located_file )
print_d( "\n" )
download_file( a_file_object, the_output_dir, the_number_threads, the_remove, the_fresh, the_callback )
break
except:
if an_sealed :
break
print_d( "." )
pass
pass
print_d( " compleated\n" )
pass
pass
def delete_key_pair( the_key_pair ) :
print_d( the_key_pair.name )
# an_ec2_conn.delete_key_pair( the_key_pair ) # Does not work (bug)
the_key_pair.delete()
import os, os.path
a_key_pair_dir = os.path.expanduser( "~/.ssh")
a_key_pair_file = os.path.join( a_key_pair_dir, the_key_pair.name ) + os.path.extsep + "pem"
if os.path.isfile( a_key_pair_file ) :
os.remove( a_key_pair_file )
pass
pass
def echo(the_password, the_identity_file, the_host_port, the_login_name, the_host_name):
from cloudflu.common import print_d
if the_password != None:
print_d(
'sshpass -p %s ssh -o "StrictHostKeyChecking no" -p %d %s@%s\n'
% (the_password, the_host_port, the_login_name, the_host_name)
)
else:
print_d(
'ssh -o "StrictHostKeyChecking no" -i %s -p %d %s@%s\n'
% (the_identity_file, the_host_port, the_login_name, the_host_name)
)
pass
pass
def wait( the_ssh_connect, the_ssh_client, the_command ) :
from cloudflu.common import print_d
print_d( "ssh'ing " )
while True :
try :
print_d( '.' )
the_ssh_connect()
command( the_ssh_client, the_command )
break
except :
# import sys, traceback
# traceback.print_exc( file = sys.stderr )
continue
pass
pass
def extract( the_option_parser ) :
from cloudflu.common import print_d, print_i, print_e
an_options, an_args = the_option_parser.parse_args()
a_study_name = an_options.study_name
if a_study_name == None :
the_option_parser.error( "Use --study-name option to define proper value\n" )
pass
print_d( "a_study_name = '%s'\n" % a_study_name )
from cloudflu.preferences import get_inputs
a_located_files = get_inputs( an_args )
print_d( "a_located_files = %r\n" % a_located_files )
return a_study_name, a_located_files
def delete( self, the_number_threads, the_printing_depth ) :
print_d( "deleting - %s\n" % self, the_printing_depth )
a_worker_pool = WorkerPool( the_number_threads )
a_deleter = lambda the_object, the_printing_depth : \
the_object.delete( the_printing_depth )
for a_seed_object in self :
a_worker_pool.charge( a_deleter, ( a_seed_object, the_printing_depth + 1 ) )
pass
TSealingObject.delete( self )
a_worker_pool.shutdown()
a_worker_pool.join()
self._key.delete()
pass