def _stage_out_api(fspec, summary_file_path, trace_report, trace_report_out, transfer_timeout):
# init. download client
from rucio.client.uploadclient import UploadClient
upload_client = UploadClient(logger=logger)
# traces are turned off
if hasattr(upload_client, 'tracing'):
upload_client.tracing = tracing_rucio
if tracing_rucio:
upload_client.trace = trace_report
# file specifications before the upload
f = {}
f['path'] = fspec.surl or getattr(fspec, 'pfn', None) or os.path.join(fspec.workdir, fspec.lfn)
f['rse'] = fspec.ddmendpoint
f['did_scope'] = fspec.scope
f['no_register'] = True
if transfer_timeout:
f['transfer_timeout'] = transfer_timeout
# if fspec.storageId and int(fspec.storageId) > 0:
# if fspec.turl and fspec.is_nondeterministic:
# f['pfn'] = fspec.turl
# elif fspec.lfn and '.root' in fspec.lfn:
# f['guid'] = fspec.guid
if fspec.lfn and '.root' in fspec.lfn:
f['guid'] = fspec.guid
# process with the upload
logger.info('_stage_out_api: %s' % str(f))
result = None
# upload client raises an exception if any file failed
try:
result = upload_client.upload([f], summary_file_path=summary_file_path, traces_copy_out=trace_report_out)
except UnboundLocalError:
logger.warning('rucio still needs a bug fix of the summary in the uploadclient')
logger.debug('Rucio upload client returned %s' % result)
try:
file_exists = verify_stage_out(fspec)
logger.info('File exists at the storage: %s' % str(file_exists))
if not file_exists:
raise StageOutFailure('stageOut: Physical check after upload failed.')
except Exception as e:
msg = 'stageOut: File existence verification failed with: %s' % str(e)
logger.info(msg)
raise StageOutFailure(msg)
return trace_report_out
def _stage_out_api(fspec, summary_file_path, trace_report):
# init. download client
from rucio.client.uploadclient import UploadClient
upload_client = UploadClient()
# traces are turned off
if hasattr(upload_client, 'tracing'):
upload_client.tracing = tracing_rucio
if tracing_rucio:
upload_client.trace = trace_report
# file specifications before the upload
f = {}
f['path'] = fspec.surl or getattr(fspec, 'pfn', None) or os.path.join(
fspec.workdir, fspec.lfn)
f['rse'] = fspec.ddmendpoint
f['did_scope'] = fspec.scope
f['no_register'] = True
if fspec.filesize:
f['transfer_timeout'] = get_timeout(fspec.filesize)
# if fspec.storageId and int(fspec.storageId) > 0:
# if fspec.turl and fspec.is_nondeterministic:
# f['pfn'] = fspec.turl
# elif fspec.lfn and '.root' in fspec.lfn:
# f['guid'] = fspec.guid
if fspec.lfn and '.root' in fspec.lfn:
f['guid'] = fspec.guid
# process with the upload
logger.info('_stage_out_api: %s' % str(f))
result = None
try:
result = upload_client.upload([f], summary_file_path)
except UnboundLocalError:
logger.warning(
'rucio still needs a bug fix of the summary in the uploadclient')
result = 0
client_state = 'FAILED'
if result == 0:
client_state = 'DONE'
return client_state
def _stage_out_api(fspec, summary_file_path, trace_report, trace_report_out,
transfer_timeout):
ec = 0
# init. download client
from rucio.client.uploadclient import UploadClient
upload_client = UploadClient(logger=logger)
# traces are turned off
if hasattr(upload_client, 'tracing'):
upload_client.tracing = tracing_rucio
if tracing_rucio:
upload_client.trace = trace_report
# file specifications before the upload
f = {}
f['path'] = fspec.surl or getattr(fspec, 'pfn', None) or os.path.join(
fspec.workdir, fspec.lfn)
f['rse'] = fspec.ddmendpoint
f['did_scope'] = fspec.scope
f['no_register'] = True
if transfer_timeout:
f['transfer_timeout'] = transfer_timeout
f['connection_timeout'] = 60 * 60
# if fspec.storageId and int(fspec.storageId) > 0:
# if fspec.turl and fspec.is_nondeterministic:
# f['pfn'] = fspec.turl
# elif fspec.lfn and '.root' in fspec.lfn:
# f['guid'] = fspec.guid
if fspec.lfn and '.root' in fspec.lfn:
f['guid'] = fspec.guid
logger.info('rucio API stage-out dictionary: %s' % f)
# upload client raises an exception if any file failed
try:
logger.info('*** rucio API uploading file (taking over logging) ***')
logger.debug('summary_file_path=%s' % summary_file_path)
logger.debug('trace_report_out=%s' % trace_report_out)
result = upload_client.upload([f],
summary_file_path=summary_file_path,
traces_copy_out=trace_report_out)
except Exception as e:
logger.warning('*** rucio API upload client failed ***')
logger.warning('caught exception: %s' % e)
import traceback
logger.error(traceback.format_exc())
logger.debug('trace_report_out=%s' % trace_report_out)
if not trace_report_out:
raise e
if not trace_report_out[0].get('stateReason'):
raise e
ec = -1
except UnboundLocalError:
logger.warning('*** rucio API upload client failed ***')
logger.warning(
'rucio still needs a bug fix of the summary in the uploadclient')
else:
logger.warning('*** rucio API upload client finished ***')
logger.debug('client returned %s' % result)
try:
file_exists = verify_stage_out(fspec)
logger.info('file exists at the storage: %s' % str(file_exists))
if not file_exists:
raise StageOutFailure('physical check after upload failed')
except Exception as e:
msg = 'file existence verification failed with: %s' % e
logger.info(msg)
raise StageOutFailure(msg)
return ec, trace_report_out
