def _stage_in_bulk(dst, files, trace_report_out=None, trace_common_fields=None): """ Stage-in files in bulk using the Rucio API. :param dst: destination (string). :param files: list of fspec objects. :param trace_report: :param trace_report_out: :return: """ # init. download client from rucio.client.downloadclient import DownloadClient download_client = DownloadClient(logger=logger) # traces are switched off if hasattr(download_client, 'tracing'): download_client.tracing = tracing_rucio # build the list of file dictionaries before calling the download function file_list = [] for fspec in files: fspec.status_code = 0 # file specifications before the actual download f = {} f['did_scope'] = fspec.scope f['did_name'] = fspec.lfn f['did'] = '%s:%s' % (fspec.scope, fspec.lfn) f['rse'] = fspec.ddmendpoint f['base_dir'] = fspec.workdir or dst f['no_subdir'] = True if fspec.turl: f['pfn'] = fspec.turl else: logger.warning( 'cannot perform bulk download since fspec.turl is not set (required by download_pfns()' ) # fail somehow if fspec.filesize: f['transfer_timeout'] = get_timeout(fspec.filesize) file_list.append(f) # proceed with the download trace_pattern = trace_common_fields if trace_common_fields else {} # download client raises an exception if any file failed num_threads = len(file_list) result = download_client.download_pfns(file_list, num_threads, trace_custom_fields=trace_pattern, traces_copy_out=trace_report_out) logger.debug('Rucio download client returned %s' % result)
def _stageInApi(self, dst, fspec): from rucio.client.downloadclient import DownloadClient # rucio logger init. rucio_logger = logging.getLogger('rucio_mover') download_client = DownloadClient(logger=rucio_logger) # traces are switched off if hasattr(download_client, 'tracing'): download_client.tracing = self.tracing # file specifications before the actual download f = {} f['did_scope'] = fspec.scope f['did_name'] = fspec.lfn f['did'] = '%s:%s' % (fspec.scope, fspec.lfn) f['rse'] = fspec.ddmendpoint f['base_dir'] = dirname(dst) if fspec.turl: f['pfn'] = fspec.turl #if fspec.filesize: # f['transfer_timeout'] = self.getTimeOut(fspec.filesize) # too harsh, max 3 hours # proceed with the download tolog('_stageInApi file: %s' % str(f)) trace_pattern = {} if self.trace_report: trace_pattern = self.trace_report result = [] if fspec.turl: result = download_client.download_pfns( [f], 1, trace_custom_fields=trace_pattern) else: result = download_client.download_dids( [f], trace_custom_fields=trace_pattern) clientState = 'FAILED' if result: clientState = result[0].get('clientState', 'FAILED') return clientState
def _stage_in_api(dst, fspec, trace_report): # init. download client from rucio.client.downloadclient import DownloadClient download_client = DownloadClient() # traces are switched off if hasattr(download_client, 'tracing'): download_client.tracing = tracing_rucio # file specifications before the actual download f = {} f['did_scope'] = fspec.scope f['did_name'] = fspec.lfn f['did'] = '%s:%s' % (fspec.scope, fspec.lfn) f['rse'] = fspec.ddmendpoint f['base_dir'] = dst f['no_subdir'] = True if fspec.turl: f['pfn'] = fspec.turl if fspec.filesize: f['transfer_timeout'] = get_timeout(fspec.filesize) # proceed with the download logger.info('_stage_in_api file: %s' % str(f)) trace_pattern = {} if trace_report: trace_pattern = trace_report result = [] if fspec.turl: result = download_client.download_pfns( [f], 1, trace_custom_fields=trace_pattern) else: result = download_client.download_dids( [f], trace_custom_fields=trace_pattern) client_state = 'FAILED' if result: client_state = result[0].get('clientState', 'FAILED') return client_state
def _stageInApi(self, dst, fspec): from rucio.client.downloadclient import DownloadClient # rucio logger init. rucio_logger = logging.getLogger('rucio_mover') download_client = DownloadClient(logger=rucio_logger) # traces are switched off if hasattr(download_client, 'tracing'): download_client.tracing = self.tracing # file specifications before the actual download f = {} f['did_scope'] = fspec.scope f['did_name'] = fspec.lfn f['did'] = '%s:%s' % (fspec.scope, fspec.lfn) f['rse'] = fspec.ddmendpoint f['base_dir'] = dirname(dst) if fspec.turl: f['pfn'] = fspec.turl #if fspec.filesize: # f['transfer_timeout'] = self.getTimeOut(fspec.filesize) # too harsh, max 3 hours # proceed with the download tolog('_stageInApi file: %s' % str(f)) trace_pattern = {} if self.trace_report: trace_pattern = self.trace_report result = [] if fspec.turl: result = download_client.download_pfns([f], 1, trace_custom_fields=trace_pattern) else: result = download_client.download_dids([f], trace_custom_fields=trace_pattern) clientState = 'FAILED' if result: clientState = result[0].get('clientState', 'FAILED') return clientState
def _stage_in_api(dst, fspec, trace_report, trace_report_out, transfer_timeout): # init. download client from rucio.client.downloadclient import DownloadClient download_client = DownloadClient(logger=logger) # traces are switched off if hasattr(download_client, 'tracing'): download_client.tracing = tracing_rucio # file specifications before the actual download f = {} f['did_scope'] = fspec.scope f['did_name'] = fspec.lfn f['did'] = '%s:%s' % (fspec.scope, fspec.lfn) f['rse'] = fspec.ddmendpoint f['base_dir'] = dst f['no_subdir'] = True if fspec.turl: f['pfn'] = fspec.turl if transfer_timeout: f['transfer_timeout'] = transfer_timeout # proceed with the download logger.info('_stage_in_api file: %s' % str(f)) trace_pattern = {} if trace_report: trace_pattern = trace_report # download client raises an exception if any file failed if fspec.turl: result = download_client.download_pfns([f], 1, trace_custom_fields=trace_pattern, traces_copy_out=trace_report_out) else: result = download_client.download_dids([f], trace_custom_fields=trace_pattern, traces_copy_out=trace_report_out) logger.debug('Rucio download client returned %s' % result) return trace_report_out
def _stage_in_api(dst, fspec, trace_report, trace_report_out, transfer_timeout, use_pcache): ec = 0 # init. download client from rucio.client.downloadclient import DownloadClient download_client = DownloadClient(logger=logger) if use_pcache: download_client.check_pcache = True # traces are switched off if hasattr(download_client, 'tracing'): download_client.tracing = tracing_rucio # file specifications before the actual download f = {} f['did_scope'] = fspec.scope f['did_name'] = fspec.lfn f['did'] = '%s:%s' % (fspec.scope, fspec.lfn) f['rse'] = fspec.ddmendpoint f['base_dir'] = dst f['no_subdir'] = True if fspec.turl: f['pfn'] = fspec.turl if transfer_timeout: f['transfer_timeout'] = transfer_timeout f['connection_timeout'] = 60 * 60 # proceed with the download logger.info('rucio API stage-in dictionary: %s' % f) trace_pattern = {} if trace_report: trace_pattern = trace_report # download client raises an exception if any file failed try: logger.info('*** rucio API downloading file (taking over logging) ***') if fspec.turl: result = download_client.download_pfns( [f], 1, trace_custom_fields=trace_pattern, traces_copy_out=trace_report_out) else: result = download_client.download_dids( [f], trace_custom_fields=trace_pattern, traces_copy_out=trace_report_out) except Exception as e: logger.warning('*** rucio API download client failed ***') logger.warning('caught exception: %s' % e) logger.debug('trace_report_out=%s' % trace_report_out) # only raise an exception if the error info cannot be extracted if not trace_report_out: raise e if not trace_report_out[0].get('stateReason'): raise e ec = -1 else: logger.info('*** rucio API download client finished ***') logger.debug('client returned %s' % result) logger.debug('trace_report_out=%s' % trace_report_out) return ec, trace_report_out