def _stage_in_bulk(dst,
files,
trace_report_out=None,
trace_common_fields=None):
"""
Stage-in files in bulk using the Rucio API.
:param dst: destination (string).
:param files: list of fspec objects.
:param trace_report:
:param trace_report_out:
:return:
"""
# init. download client
from rucio.client.downloadclient import DownloadClient
download_client = DownloadClient(logger=logger)
# traces are switched off
if hasattr(download_client, 'tracing'):
download_client.tracing = tracing_rucio
# build the list of file dictionaries before calling the download function
file_list = []
for fspec in files:
fspec.status_code = 0
# file specifications before the actual download
f = {}
f['did_scope'] = fspec.scope
f['did_name'] = fspec.lfn
f['did'] = '%s:%s' % (fspec.scope, fspec.lfn)
f['rse'] = fspec.ddmendpoint
f['base_dir'] = fspec.workdir or dst
f['no_subdir'] = True
if fspec.turl:
f['pfn'] = fspec.turl
else:
logger.warning(
'cannot perform bulk download since fspec.turl is not set (required by download_pfns()'
)
# fail somehow
if fspec.filesize:
f['transfer_timeout'] = get_timeout(fspec.filesize)
file_list.append(f)
# proceed with the download
trace_pattern = trace_common_fields if trace_common_fields else {}
# download client raises an exception if any file failed
num_threads = len(file_list)
result = download_client.download_pfns(file_list,
num_threads,
trace_custom_fields=trace_pattern,
traces_copy_out=trace_report_out)
logger.debug('Rucio download client returned %s' % result)
def _stageInApi(self, dst, fspec):
from rucio.client.downloadclient import DownloadClient
# rucio logger init.
rucio_logger = logging.getLogger('rucio_mover')
download_client = DownloadClient(logger=rucio_logger)
# traces are switched off
if hasattr(download_client, 'tracing'):
download_client.tracing = self.tracing
# file specifications before the actual download
f = {}
f['did_scope'] = fspec.scope
f['did_name'] = fspec.lfn
f['did'] = '%s:%s' % (fspec.scope, fspec.lfn)
f['rse'] = fspec.ddmendpoint
f['base_dir'] = dirname(dst)
if fspec.turl:
f['pfn'] = fspec.turl
#if fspec.filesize:
# f['transfer_timeout'] = self.getTimeOut(fspec.filesize) # too harsh, max 3 hours
# proceed with the download
tolog('_stageInApi file: %s' % str(f))
trace_pattern = {}
if self.trace_report:
trace_pattern = self.trace_report
result = []
if fspec.turl:
result = download_client.download_pfns(
[f], 1, trace_custom_fields=trace_pattern)
else:
result = download_client.download_dids(
[f], trace_custom_fields=trace_pattern)
clientState = 'FAILED'
if result:
clientState = result[0].get('clientState', 'FAILED')
return clientState
def _stage_in_api(dst, fspec, trace_report):
# init. download client
from rucio.client.downloadclient import DownloadClient
download_client = DownloadClient()
# traces are switched off
if hasattr(download_client, 'tracing'):
download_client.tracing = tracing_rucio
# file specifications before the actual download
f = {}
f['did_scope'] = fspec.scope
f['did_name'] = fspec.lfn
f['did'] = '%s:%s' % (fspec.scope, fspec.lfn)
f['rse'] = fspec.ddmendpoint
f['base_dir'] = dst
f['no_subdir'] = True
if fspec.turl:
f['pfn'] = fspec.turl
if fspec.filesize:
f['transfer_timeout'] = get_timeout(fspec.filesize)
# proceed with the download
logger.info('_stage_in_api file: %s' % str(f))
trace_pattern = {}
if trace_report:
trace_pattern = trace_report
result = []
if fspec.turl:
result = download_client.download_pfns(
[f], 1, trace_custom_fields=trace_pattern)
else:
result = download_client.download_dids(
[f], trace_custom_fields=trace_pattern)
client_state = 'FAILED'
if result:
client_state = result[0].get('clientState', 'FAILED')
return client_state
def _stageInApi(self, dst, fspec):
from rucio.client.downloadclient import DownloadClient
# rucio logger init.
rucio_logger = logging.getLogger('rucio_mover')
download_client = DownloadClient(logger=rucio_logger)
# traces are switched off
if hasattr(download_client, 'tracing'):
download_client.tracing = self.tracing
# file specifications before the actual download
f = {}
f['did_scope'] = fspec.scope
f['did_name'] = fspec.lfn
f['did'] = '%s:%s' % (fspec.scope, fspec.lfn)
f['rse'] = fspec.ddmendpoint
f['base_dir'] = dirname(dst)
if fspec.turl:
f['pfn'] = fspec.turl
#if fspec.filesize:
# f['transfer_timeout'] = self.getTimeOut(fspec.filesize) # too harsh, max 3 hours
# proceed with the download
tolog('_stageInApi file: %s' % str(f))
trace_pattern = {}
if self.trace_report:
trace_pattern = self.trace_report
result = []
if fspec.turl:
result = download_client.download_pfns([f], 1, trace_custom_fields=trace_pattern)
else:
result = download_client.download_dids([f], trace_custom_fields=trace_pattern)
clientState = 'FAILED'
if result:
clientState = result[0].get('clientState', 'FAILED')
return clientState
def _stage_in_api(dst, fspec, trace_report, trace_report_out, transfer_timeout):
# init. download client
from rucio.client.downloadclient import DownloadClient
download_client = DownloadClient(logger=logger)
# traces are switched off
if hasattr(download_client, 'tracing'):
download_client.tracing = tracing_rucio
# file specifications before the actual download
f = {}
f['did_scope'] = fspec.scope
f['did_name'] = fspec.lfn
f['did'] = '%s:%s' % (fspec.scope, fspec.lfn)
f['rse'] = fspec.ddmendpoint
f['base_dir'] = dst
f['no_subdir'] = True
if fspec.turl:
f['pfn'] = fspec.turl
if transfer_timeout:
f['transfer_timeout'] = transfer_timeout
# proceed with the download
logger.info('_stage_in_api file: %s' % str(f))
trace_pattern = {}
if trace_report:
trace_pattern = trace_report
# download client raises an exception if any file failed
if fspec.turl:
result = download_client.download_pfns([f], 1, trace_custom_fields=trace_pattern, traces_copy_out=trace_report_out)
else:
result = download_client.download_dids([f], trace_custom_fields=trace_pattern, traces_copy_out=trace_report_out)
logger.debug('Rucio download client returned %s' % result)
return trace_report_out
def _stage_in_api(dst, fspec, trace_report, trace_report_out, transfer_timeout,
use_pcache):
ec = 0
# init. download client
from rucio.client.downloadclient import DownloadClient
download_client = DownloadClient(logger=logger)
if use_pcache:
download_client.check_pcache = True
# traces are switched off
if hasattr(download_client, 'tracing'):
download_client.tracing = tracing_rucio
# file specifications before the actual download
f = {}
f['did_scope'] = fspec.scope
f['did_name'] = fspec.lfn
f['did'] = '%s:%s' % (fspec.scope, fspec.lfn)
f['rse'] = fspec.ddmendpoint
f['base_dir'] = dst
f['no_subdir'] = True
if fspec.turl:
f['pfn'] = fspec.turl
if transfer_timeout:
f['transfer_timeout'] = transfer_timeout
f['connection_timeout'] = 60 * 60
# proceed with the download
logger.info('rucio API stage-in dictionary: %s' % f)
trace_pattern = {}
if trace_report:
trace_pattern = trace_report
# download client raises an exception if any file failed
try:
logger.info('*** rucio API downloading file (taking over logging) ***')
if fspec.turl:
result = download_client.download_pfns(
[f],
1,
trace_custom_fields=trace_pattern,
traces_copy_out=trace_report_out)
else:
result = download_client.download_dids(
[f],
trace_custom_fields=trace_pattern,
traces_copy_out=trace_report_out)
except Exception as e:
logger.warning('*** rucio API download client failed ***')
logger.warning('caught exception: %s' % e)
logger.debug('trace_report_out=%s' % trace_report_out)
# only raise an exception if the error info cannot be extracted
if not trace_report_out:
raise e
if not trace_report_out[0].get('stateReason'):
raise e
ec = -1
else:
logger.info('*** rucio API download client finished ***')
logger.debug('client returned %s' % result)
logger.debug('trace_report_out=%s' % trace_report_out)
return ec, trace_report_out
