def _stageOutApi(self, src, fspec): from rucio.client.uploadclient import UploadClient # rucio logger init. rucio_logger = logging.getLogger('rucio_mover') upload_client = UploadClient(logger=rucio_logger) # File existence verification faileds are turned off if hasattr(upload_client, 'tracing'): upload_client.tracing = self.tracing # file specifications before the upload f = {} f['path'] = fspec.pfn if fspec.pfn else fspec.lfn f['rse'] = fspec.ddmendpoint f['did_scope'] = fspec.scope f['no_register'] = True #if fspec.filesize: # f['transfer_timeout'] = self.getTimeOut(fspec.filesize) # too harsh, max 3 hours if fspec.storageId and int(fspec.storageId) > 0: if not self.isDeterministic(fspec.ddmendpoint): f['pfn'] = fspec.turl elif fspec.lfn and '.root' in fspec.lfn: f['guid'] = fspec.guid # process the upload tolog('_stageOutApi: %s' % str(f)) upload_client.upload([f]) return {'ddmendpoint': fspec.ddmendpoint, 'surl': fspec.surl, 'pfn': fspec.lfn}
def _stage_out_api(fspec, summary_file_path, trace_report, trace_report_out, transfer_timeout): # init. download client from rucio.client.uploadclient import UploadClient upload_client = UploadClient(logger=logger) # traces are turned off if hasattr(upload_client, 'tracing'): upload_client.tracing = tracing_rucio if tracing_rucio: upload_client.trace = trace_report # file specifications before the upload f = {} f['path'] = fspec.surl or getattr(fspec, 'pfn', None) or os.path.join(fspec.workdir, fspec.lfn) f['rse'] = fspec.ddmendpoint f['did_scope'] = fspec.scope f['no_register'] = True if transfer_timeout: f['transfer_timeout'] = transfer_timeout # if fspec.storageId and int(fspec.storageId) > 0: # if fspec.turl and fspec.is_nondeterministic: # f['pfn'] = fspec.turl # elif fspec.lfn and '.root' in fspec.lfn: # f['guid'] = fspec.guid if fspec.lfn and '.root' in fspec.lfn: f['guid'] = fspec.guid # process with the upload logger.info('_stage_out_api: %s' % str(f)) result = None # upload client raises an exception if any file failed try: result = upload_client.upload([f], summary_file_path=summary_file_path, traces_copy_out=trace_report_out) except UnboundLocalError: logger.warning('rucio still needs a bug fix of the summary in the uploadclient') logger.debug('Rucio upload client returned %s' % result) try: file_exists = verify_stage_out(fspec) logger.info('File exists at the storage: %s' % str(file_exists)) if not file_exists: raise StageOutFailure('stageOut: Physical check after upload failed.') except Exception as e: msg = 'stageOut: File existence verification failed with: %s' % str(e) logger.info(msg) raise StageOutFailure(msg) return trace_report_out
def _stage_out_api(fspec, summary_file_path, trace_report): # init. download client from rucio.client.uploadclient import UploadClient upload_client = UploadClient() # traces are turned off if hasattr(upload_client, 'tracing'): upload_client.tracing = tracing_rucio if tracing_rucio: upload_client.trace = trace_report # file specifications before the upload f = {} f['path'] = fspec.surl or getattr(fspec, 'pfn', None) or os.path.join( fspec.workdir, fspec.lfn) f['rse'] = fspec.ddmendpoint f['did_scope'] = fspec.scope f['no_register'] = True if fspec.filesize: f['transfer_timeout'] = get_timeout(fspec.filesize) # if fspec.storageId and int(fspec.storageId) > 0: # if fspec.turl and fspec.is_nondeterministic: # f['pfn'] = fspec.turl # elif fspec.lfn and '.root' in fspec.lfn: # f['guid'] = fspec.guid if fspec.lfn and '.root' in fspec.lfn: f['guid'] = fspec.guid # process with the upload logger.info('_stage_out_api: %s' % str(f)) result = None try: result = upload_client.upload([f], summary_file_path) except UnboundLocalError: logger.warning( 'rucio still needs a bug fix of the summary in the uploadclient') result = 0 client_state = 'FAILED' if result == 0: client_state = 'DONE' return client_state
def _stageOutApi(self, src, fspec): from rucio.client.uploadclient import UploadClient # rucio logger init. rucio_logger = logging.getLogger('rucio_mover') upload_client = UploadClient(logger=rucio_logger) # File existence verification faileds are turned off if hasattr(upload_client, 'tracing'): upload_client.tracing = self.tracing # file specifications before the upload f = {} f['path'] = fspec.pfn if fspec.pfn else fspec.lfn f['rse'] = fspec.ddmendpoint f['did_scope'] = fspec.scope f['no_register'] = True #if fspec.filesize: # f['transfer_timeout'] = self.getTimeOut(fspec.filesize) # too harsh, max 3 hours if fspec.storageId and int(fspec.storageId) > 0: if not self.isDeterministic(fspec.ddmendpoint): f['pfn'] = fspec.turl elif fspec.lfn and '.root' in fspec.lfn: f['guid'] = fspec.guid # process the upload tolog('_stageOutApi: %s' % str(f)) upload_client.upload([f]) return { 'ddmendpoint': fspec.ddmendpoint, 'surl': fspec.surl, 'pfn': fspec.lfn }
def _stage_out_api(fspec, summary_file_path, trace_report, trace_report_out, transfer_timeout): ec = 0 # init. download client from rucio.client.uploadclient import UploadClient upload_client = UploadClient(logger=logger) # traces are turned off if hasattr(upload_client, 'tracing'): upload_client.tracing = tracing_rucio if tracing_rucio: upload_client.trace = trace_report # file specifications before the upload f = {} f['path'] = fspec.surl or getattr(fspec, 'pfn', None) or os.path.join( fspec.workdir, fspec.lfn) f['rse'] = fspec.ddmendpoint f['did_scope'] = fspec.scope f['no_register'] = True if transfer_timeout: f['transfer_timeout'] = transfer_timeout f['connection_timeout'] = 60 * 60 # if fspec.storageId and int(fspec.storageId) > 0: # if fspec.turl and fspec.is_nondeterministic: # f['pfn'] = fspec.turl # elif fspec.lfn and '.root' in fspec.lfn: # f['guid'] = fspec.guid if fspec.lfn and '.root' in fspec.lfn: f['guid'] = fspec.guid logger.info('rucio API stage-out dictionary: %s' % f) # upload client raises an exception if any file failed try: logger.info('*** rucio API uploading file (taking over logging) ***') logger.debug('summary_file_path=%s' % summary_file_path) logger.debug('trace_report_out=%s' % trace_report_out) result = upload_client.upload([f], summary_file_path=summary_file_path, traces_copy_out=trace_report_out) except Exception as e: logger.warning('*** rucio API upload client failed ***') logger.warning('caught exception: %s' % e) import traceback logger.error(traceback.format_exc()) logger.debug('trace_report_out=%s' % trace_report_out) if not trace_report_out: raise e if not trace_report_out[0].get('stateReason'): raise e ec = -1 except UnboundLocalError: logger.warning('*** rucio API upload client failed ***') logger.warning( 'rucio still needs a bug fix of the summary in the uploadclient') else: logger.warning('*** rucio API upload client finished ***') logger.debug('client returned %s' % result) try: file_exists = verify_stage_out(fspec) logger.info('file exists at the storage: %s' % str(file_exists)) if not file_exists: raise StageOutFailure('physical check after upload failed') except Exception as e: msg = 'file existence verification failed with: %s' % e logger.info(msg) raise StageOutFailure(msg) return ec, trace_report_out