def report_percent_complete(self):
"""
update the task state with the progress of the bundle
"""
meta_str = "Bundling percent complete: " + str(int(self.percent_complete))
print meta_str
task_state('PROGRESS', meta_str)
def progress(_download_t, _download_d, upload_t, upload_d):
"""
gets the progress of the current pycurl upload
"""
if upload_t > 0:
try:
percent = 100.0 * float(upload_d) / float(upload_t)
if percent - TrackPercent.percent > 5:
meta_dict={'Status': "upload percent complete: " + str(int(percent))}
task_state("PROGRESS", meta_dict)
TrackPercent.percent = percent
except Exception, e:
raise task_error("Error during callback: " + e.message)
def upload_files(ingest_server = '',
bundle_name='',
file_list=None,
bundle_size=0,
meta_list=None,
tartar = False):
"""
task created on a separate Celery process to bundle and upload in the background
status and errors are pushed by celery to the main server through RabbitMQ
"""
target_dir = os.path.dirname(bundle_name)
if not os.path.isdir(target_dir):
current_task.update_state(state='ERROR', meta={'Status': 'Bundle directory does not exist'})
task_state("PROGRESS", "Cleaning previous uploads")
##clean tar directory
#if CLEAN_TAR:
# err_str = clean_target_directory(target_dir)
# if err_str:
# task_state('PROGRESS', err_str)
# initial state pushed through celery
task_state("PROGRESS", "Starting Bundle/Upload Process")
bundle(bundle_name=bundle_name,
file_list=file_list,
meta_list= meta_list,
bundle_size=bundle_size)
task_state("PROGRESS", "Completed Bundling")
if tartar:
# create the file tuple list of 1 file
dir = os.path.dirname(bundle_name)
fname = os.path.basename(bundle_name)
file_tuples=[]
file_tuples.append((bundle_name, fname))
bundle_size = os.path.getsize(bundle_name)
# dual extension indicates tartar
bundle_name += '.tar'
bundle(bundle_name=bundle_name,
file_list=file_tuples,
meta_list= meta_list,
bundle_size=bundle_size)
task_state("PROGRESS", "Starting Upload")
result = upload(bundle_name=bundle_name, ingest_server = ingest_server)
if not result:
task_state('FAILURE', "Uploader dieded. We don't know why it did")
try:
status = json.loads(result)
except Exception, e:
task_state('FAILURE', e.message)
return 'Upload Failed'
def ping():
"""
check to see if the celery task process is started.
"""
print "Pinged!"
task_state('PING', "Background process is alive")
bundle(bundle_name=bundle_name,
file_list=file_tuples,
meta_list= meta_list,
bundle_size=bundle_size)
task_state("PROGRESS", "Starting Upload")
result = upload(bundle_name=bundle_name, ingest_server = ingest_server)
if not result:
task_state('FAILURE', "Uploader dieded. We don't know why it did")
try:
status = json.loads(result)
except Exception, e:
task_state('FAILURE', e.message)
return 'Upload Failed'
if status['state'] != 'OK':
task_state('FAILURE', result)
return 'Upload Failed'
try:
print "rename"
tar_man.rename_tar_file(target_dir, bundle_name, status['job_id'])
task_state('DONE', result)
return result
except Exception, e:
task_state('FAILURE', 'Unable to rename ' + bundle_name)
return 'Rename Failed'
def bundle(bundle_name='',
instrument_name='',
proposal='',
file_list=None,
groups=None,
bundle_size=0):
"""
Bundles a list of files into a single aggregated bundle file
:Parameters:
bundle_name
The target bundle file in which to aggregate the file list
instrument_name
The name of the instrument that produced the data files that will be bundled
groups
The a hash of type/name groups to attach to
tarfile
If true, tarfile format is used to bundle. Otherwise zipfile format is used
proposal
An optional proposal ID to attach to the bundle
file_list
The list of files to bundle
"""
# validate parameters
if bundle_name == None or bundle_name == '':
task_error("Missing bundle name")
if instrument_name == None or instrument_name == '':
task_error("Missing instrument name")
if proposal == None or proposal == '':
task_error("Missing proposal")
if file_list == None or len(file_list) == 0:
task_error("Missing file list")
if groups == None or groups == '':
task_error("Missing groups")
#print >> sys.stderr, "Start bundling %s" % bundle_name
# Set up the bundle file
bundle_path = os.path.abspath(bundle_name)
#print >> sys.stderr, "Bundle file set to %s" % bundle_path
# Set up the bundler object
bundler = None
# dfh note we are setting the instrument name and ID to the same thing,
# which is being
# sent in as the instrument name but is actually the instrument ID. Fix
# this.
bundler = TarBundler(bundle_path, proposal_ID=proposal,
instrument_name=instrument_name,
instrument_ID=instrument_name,
groups=groups)
bundler.bundle_file(file_list, bundle_size)
bundler.bundle_metadata()
#print >> sys.stderr, "Finished bundling"
task_state('PROGRESS', "Bundling complete")
