def file_entry_point( the_study_object, the_output_dir, the_located_file,
the_number_threads, the_wait, the_remove, the_fresh, the_callback ) :
if the_wait == False or the_study_object.sealed() :
try:
a_file_object = TFileObject.get( the_study_object, the_located_file )
download_file( a_file_object, the_output_dir, the_number_threads, the_remove, the_fresh, the_callback )
except:
from cloudflu.common import print_traceback
print_traceback( 0 )
pass
pass
else:
print_d( "waiting " )
while True :
an_sealed = the_study_object.sealed()
try:
a_file_object = TFileObject.get( the_study_object, the_located_file )
print_d( "\n" )
download_file( a_file_object, the_output_dir, the_number_threads, the_remove, the_fresh, the_callback )
break
except:
if an_sealed :
break
print_d( "." )
pass
pass
print_d( " compleated\n" )
pass
pass
def upload_file( the_worker_pool, the_file_path, the_file_location, the_study_object, the_upload_seed_size, the_printing_depth ) :
a_working_dir = generate_uploading_dir( the_file_path )
import shutil
shutil.rmtree( a_working_dir, True )
os.makedirs( a_working_dir )
print_d( "a_working_dir = '%s'\n" % a_working_dir, the_printing_depth )
a_file_dirname = os.path.dirname( the_file_path )
a_file_basename = os.path.basename( the_file_path )
import tempfile
a_tmp_file = tempfile.mkstemp( dir = a_working_dir )[ 1 ]
# a_tmp_file = tempfile.mkstemp()[ 1 ] # use this work arround for FAT file systems
sh_command( "cd '%s' && tar -czf %s '%s'" %
( a_file_dirname, a_tmp_file, a_file_basename ), the_printing_depth )
a_statinfo = os.stat( a_tmp_file )
print_d( "a_statinfo.st_size = %d, bytes\n" % a_statinfo.st_size, the_printing_depth )
import math
a_suffix_length = math.log10( float( a_statinfo.st_size ) / the_upload_seed_size )
if a_suffix_length > 0 :
a_suffix_length = int( a_suffix_length + 1.0 )
else:
a_suffix_length = 0
pass
print_d( "a_suffix_length = %d, digits\n" % a_suffix_length, the_printing_depth )
a_file_seed_target = os.path.join( a_working_dir, a_file_basename )
sh_command( "cat '%s' | split --bytes=%d --numeric-suffixes --suffix-length=%d - %s.tgz-" %
( a_tmp_file, the_upload_seed_size, a_suffix_length, a_file_seed_target ), the_printing_depth )
a_file_pointer = open( a_tmp_file, 'rb' )
a_md5 = compute_md5( a_file_pointer )
a_hex_md5, a_base64md5 = a_md5
a_file_pointer.close()
os.remove( a_tmp_file )
a_file_object = TFileObject.create( the_study_object, the_file_path, the_file_location, a_hex_md5 )
print_d( "a_file_object = %s\n" % a_file_object, the_printing_depth )
pass
def entry_point( the_study_object, the_located_files, the_number_threads ) :
a_spent_time = Timer()
if the_located_files == None :
upload_files( the_study_object, the_number_threads, 0 )
pass
else :
a_worker_pool = WorkerPool( the_number_threads )
for a_located_file in the_located_files:
a_file_object = TFileObject.get( the_study_object, a_located_file )
a_worker_pool.charge( upload_file, ( a_file_object, the_number_threads, 0 ) )
pass
a_worker_pool.shutdown()
a_worker_pool.join()
pass
print_d( "a_spent_time = %s, sec\n" % a_spent_time )
return the_study_object
def main() :
#----------------------- Defining utility command-line interface -------------------------
an_usage_description = "%prog"
from rm_options import usage_description as usage_description_options
an_usage_description += usage_description_options()
from cloudflu import VERSION
a_version = "%s" % VERSION
from optparse import IndentedHelpFormatter
a_help_formatter = IndentedHelpFormatter( width = 127 )
from optparse import OptionParser
an_option_parser = OptionParser( usage = an_usage_description, version = a_version, formatter = a_help_formatter )
#----------------------- Definition of the command line arguments ------------------------
from rm_options import add as add_options
add_options( an_option_parser )
amazon.security_options.add( an_option_parser )
common.concurrency_options.add( an_option_parser )
common.communication_options.add( an_option_parser )
common.options.add( an_option_parser )
#------------------ Extracting and verifying command-line arguments ----------------------
an_options, an_args = an_option_parser.parse_args()
common.options.extract( an_option_parser )
common.communication_options.extract( an_option_parser )
a_number_threads = common.concurrency_options.extract( an_option_parser )
AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY = amazon.security_options.extract( an_option_parser )
from rm_options import extract as extract_options
a_study_name, a_located_files = extract_options( an_option_parser )
print_i( "--------------------------- Looking for study object ----------------------------\n" )
a_root_object = TRootObject.get( AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY )
print_d( "a_root_object = %s\n" % a_root_object )
a_study_object = TStudyObject.get( a_root_object, a_study_name )
print_d( "a_study_object = %s\n" % a_study_object )
print_i( "-------------------------- Removing study files -----------------------------\n" )
a_worker_pool = WorkerPool( len( a_located_files ) )
a_deleter = lambda the_object, the_number_threads, the_printing_depth : \
the_object.delete( the_number_threads, the_printing_depth )
for a_located_file in a_located_files :
print_d( "a_located_file = %s\n" % a_located_file )
a_file_object = TFileObject.get( a_study_object, a_located_file )
print_d( "a_file_object = %s\n" % a_file_object, 1 )
a_worker_pool.charge( a_deleter, ( a_file_object, a_number_threads, 2 ) )
pass
a_worker_pool.shutdown()
a_worker_pool.join()
print_i( "------------------- Printing succussive pipeline arguments ----------------------\n" )
print a_study_name
print_i( "-------------------------------------- OK ---------------------------------------\n" )
pass
