def report_percent_complete(self): """ update the task state with the progress of the bundle """ meta_str = "Bundling percent complete: " + str(int(self.percent_complete)) print meta_str task_state('PROGRESS', meta_str)
def progress(_download_t, _download_d, upload_t, upload_d): """ gets the progress of the current pycurl upload """ if upload_t > 0: try: percent = 100.0 * float(upload_d) / float(upload_t) if percent - TrackPercent.percent > 5: meta_dict={'Status': "upload percent complete: " + str(int(percent))} task_state("PROGRESS", meta_dict) TrackPercent.percent = percent except Exception, e: raise task_error("Error during callback: " + e.message)
def upload_files(ingest_server = '', bundle_name='', file_list=None, bundle_size=0, meta_list=None, tartar = False): """ task created on a separate Celery process to bundle and upload in the background status and errors are pushed by celery to the main server through RabbitMQ """ target_dir = os.path.dirname(bundle_name) if not os.path.isdir(target_dir): current_task.update_state(state='ERROR', meta={'Status': 'Bundle directory does not exist'}) task_state("PROGRESS", "Cleaning previous uploads") ##clean tar directory #if CLEAN_TAR: # err_str = clean_target_directory(target_dir) # if err_str: # task_state('PROGRESS', err_str) # initial state pushed through celery task_state("PROGRESS", "Starting Bundle/Upload Process") bundle(bundle_name=bundle_name, file_list=file_list, meta_list= meta_list, bundle_size=bundle_size) task_state("PROGRESS", "Completed Bundling") if tartar: # create the file tuple list of 1 file dir = os.path.dirname(bundle_name) fname = os.path.basename(bundle_name) file_tuples=[] file_tuples.append((bundle_name, fname)) bundle_size = os.path.getsize(bundle_name) # dual extension indicates tartar bundle_name += '.tar' bundle(bundle_name=bundle_name, file_list=file_tuples, meta_list= meta_list, bundle_size=bundle_size) task_state("PROGRESS", "Starting Upload") result = upload(bundle_name=bundle_name, ingest_server = ingest_server) if not result: task_state('FAILURE', "Uploader dieded. We don't know why it did") try: status = json.loads(result) except Exception, e: task_state('FAILURE', e.message) return 'Upload Failed'
def ping(): """ check to see if the celery task process is started. """ print "Pinged!" task_state('PING', "Background process is alive")
bundle(bundle_name=bundle_name, file_list=file_tuples, meta_list= meta_list, bundle_size=bundle_size) task_state("PROGRESS", "Starting Upload") result = upload(bundle_name=bundle_name, ingest_server = ingest_server) if not result: task_state('FAILURE', "Uploader dieded. We don't know why it did") try: status = json.loads(result) except Exception, e: task_state('FAILURE', e.message) return 'Upload Failed' if status['state'] != 'OK': task_state('FAILURE', result) return 'Upload Failed' try: print "rename" tar_man.rename_tar_file(target_dir, bundle_name, status['job_id']) task_state('DONE', result) return result except Exception, e: task_state('FAILURE', 'Unable to rename ' + bundle_name) return 'Rename Failed'
def bundle(bundle_name='', instrument_name='', proposal='', file_list=None, groups=None, bundle_size=0): """ Bundles a list of files into a single aggregated bundle file :Parameters: bundle_name The target bundle file in which to aggregate the file list instrument_name The name of the instrument that produced the data files that will be bundled groups The a hash of type/name groups to attach to tarfile If true, tarfile format is used to bundle. Otherwise zipfile format is used proposal An optional proposal ID to attach to the bundle file_list The list of files to bundle """ # validate parameters if bundle_name == None or bundle_name == '': task_error("Missing bundle name") if instrument_name == None or instrument_name == '': task_error("Missing instrument name") if proposal == None or proposal == '': task_error("Missing proposal") if file_list == None or len(file_list) == 0: task_error("Missing file list") if groups == None or groups == '': task_error("Missing groups") #print >> sys.stderr, "Start bundling %s" % bundle_name # Set up the bundle file bundle_path = os.path.abspath(bundle_name) #print >> sys.stderr, "Bundle file set to %s" % bundle_path # Set up the bundler object bundler = None # dfh note we are setting the instrument name and ID to the same thing, # which is being # sent in as the instrument name but is actually the instrument ID. Fix # this. bundler = TarBundler(bundle_path, proposal_ID=proposal, instrument_name=instrument_name, instrument_ID=instrument_name, groups=groups) bundler.bundle_file(file_list, bundle_size) bundler.bundle_metadata() #print >> sys.stderr, "Finished bundling" task_state('PROGRESS', "Bundling complete")