def __update_atime(self): """ Bulk update atime. """ replicas = [] for report in self.__reports: try: # check if scope in report. if not skip this one. if 'scope' not in report: record_counter('daemons.tracer.kronos.missing_scope') continue else: record_counter('daemons.tracer.kronos.with_scope') # for the moment only report with eventType get* are handled. if not report['eventType'].startswith('get'): continue record_counter('daemons.tracer.kronos.total_get') if report['eventType'] == 'get': record_counter('daemons.tracer.kronos.dq2clients') elif report['eventType'] == 'get_sm': record_counter('daemons.tracer.kronos.panda_production') elif report['eventType'] == 'get_sm_a': record_counter('daemons.tracer.kronos.panda_analysis') else: record_counter('daemons.tracer.kronos.other_get') # check if the report has the right state. if report['clientState'] in self.__excluded_states: continue if report['usrdn'] in self.__excluded_usrdns: continue if not report['remoteSite']: continue replicas.append({'name': report['filename'], 'scope': report['scope'], 'rse': report['remoteSite'], 'accessed_at': datetime.utcfromtimestamp(report['traceTimeentryUnix'])}) except (KeyError, AttributeError): logging.error(format_exc()) record_counter('daemons.tracer.kronos.report_error') continue for did in list_parent_dids(report['scope'], report['filename']): if did['type'] != DIDType.DATASET: continue # do not update _dis datasets if did['scope'] == 'panda' and '_dis' in did['name']: continue self.__dataset_queue.put({'scope': did['scope'], 'name': did['name'], 'did_type': did['type'], 'rse': report['remoteSite'], 'accessed_at': datetime.utcfromtimestamp(report['traceTimeentryUnix'])}) logging.info(replicas) try: ts = time() touch_replicas(replicas) record_timer('daemons.tracer.kronos.update_atime', (time() - ts) * 1000) except: logging.error(format_exc()) record_counter('daemons.tracer.kronos.update_error') logging.info('(kronos_file) updated %d replicas' % len(replicas))
def list_parent_dids(scope, name): """ List parent datasets and containers of a did. :param scope: The scope. :param name: The name. """ return did.list_parent_dids(scope=scope, name=name)
def list_parent_dids(scope, name): """ List parent datasets and containers of a did. :param scope: The scope. :param name: The name. """ scope = InternalScope(scope) dids = did.list_parent_dids(scope=scope, name=name) for d in dids: yield api_update_return_dict(d)
def list_parent_dids(scope, name, vo='def', session=None): """ List parent datasets and containers of a did. :param scope: The scope. :param name: The name. :param vo: The VO to act on. :param session: The database session in use. """ scope = InternalScope(scope, vo=vo) dids = did.list_parent_dids(scope=scope, name=name, session=session) for d in dids: yield api_update_return_dict(d, session=session)
def __update_atime(self): """ Bulk update atime. """ replicas = [] rses = [] for report in self.__reports: if 'vo' not in report: report['vo'] = 'def' try: # Identify suspicious files try: if self.__bad_files_patterns and report['eventType'] in [ 'get_sm', 'get_sm_a', 'get' ] and 'clientState' in report and report[ 'clientState'] not in [ 'DONE', 'FOUND_ROOT', 'ALREADY_DONE' ]: for pattern in self.__bad_files_patterns: if 'stateReason' in report and report[ 'stateReason'] and isinstance( report['stateReason'], str) and pattern.match( report['stateReason']): reason = report['stateReason'][:255] if 'url' not in report or not report['url']: self.__logger( logging.ERROR, 'Missing url in the following trace : ' + str(report)) else: try: surl = report['url'] declare_bad_file_replicas( [ surl, ], reason=reason, issuer=InternalAccount( 'root', vo=report['vo']), status=BadFilesStatus.SUSPICIOUS) self.__logger( logging.INFO, 'Declare suspicious file %s with reason %s' % (report['url'], reason)) except Exception as error: self.__logger( logging.ERROR, 'Failed to declare suspicious file' + str(error)) except Exception as error: self.__logger( logging.ERROR, 'Problem with bad trace : %s . Error %s' % (str(report), str(error))) # check if scope in report. if not skip this one. if 'scope' not in report: record_counter('daemons.tracer.kronos.missing_scope') if report['eventType'] != 'touch': continue else: record_counter('daemons.tracer.kronos.with_scope') report['scope'] = InternalScope(report['scope'], report['vo']) # handle all events starting with get* and download and touch events. if not report['eventType'].startswith('get') and not report[ 'eventType'].startswith('sm_get') and not report[ 'eventType'] == 'download' and not report[ 'eventType'] == 'touch': continue if report['eventType'].endswith('_es'): continue record_counter('daemons.tracer.kronos.total_get') if report['eventType'] == 'get': record_counter('daemons.tracer.kronos.dq2clients') elif report['eventType'] == 'get_sm' or report[ 'eventType'] == 'sm_get': if report['eventVersion'] == 'aCT': record_counter( 'daemons.tracer.kronos.panda_production_act') else: record_counter( 'daemons.tracer.kronos.panda_production') elif report['eventType'] == 'get_sm_a' or report[ 'eventType'] == 'sm_get_a': if report['eventVersion'] == 'aCT': record_counter( 'daemons.tracer.kronos.panda_analysis_act') else: record_counter('daemons.tracer.kronos.panda_analysis') elif report['eventType'] == 'download': record_counter('daemons.tracer.kronos.rucio_download') elif report['eventType'] == 'touch': record_counter('daemons.tracer.kronos.rucio_touch') else: record_counter('daemons.tracer.kronos.other_get') if report['eventType'] == 'download' or report[ 'eventType'] == 'touch': report['usrdn'] = report['account'] if report['usrdn'] in self.__excluded_usrdns: continue # handle touch and non-touch traces differently if report['eventType'] != 'touch': # check if the report has the right state. if 'eventVersion' in report: if report['eventVersion'] != 'aCT': if report['clientState'] in self.__excluded_states: continue if 'remoteSite' not in report: continue if not report['remoteSite']: continue if 'filename' not in report: if 'name' in report: report['filename'] = report['name'] rses = report['remoteSite'].strip().split(',') for rse in rses: try: rse_id = get_rse_id(rse=rse, vo=report['vo']) except RSENotFound: self.__logger( logging.WARNING, "Cannot lookup rse_id for %s. Will skip this report.", rse) record_counter( 'daemons.tracer.kronos.rse_not_found') continue replicas.append({ 'name': report['filename'], 'scope': report['scope'], 'rse': rse, 'rse_id': rse_id, 'accessed_at': datetime.utcfromtimestamp( report['traceTimeentryUnix']), 'traceTimeentryUnix': report['traceTimeentryUnix'], 'eventVersion': report['eventVersion'] }) else: # if touch event and if datasetScope is in the report then it means # that there is no file scope/name and therefore only the dataset is # put in the queue to be updated and the rest is skipped. rse_id = None rse = None if 'remoteSite' in report: rse = report['remoteSite'] try: rse_id = get_rse_id(rse=rse, vo=report['vo']) except RSENotFound: self.__logger(logging.WARNING, "Cannot lookup rse_id for %s.", rse) record_counter( 'daemons.tracer.kronos.rse_not_found') if 'datasetScope' in report: self.__dataset_queue.put({ 'scope': InternalScope(report['datasetScope'], vo=report['vo']), 'name': report['dataset'], 'rse_id': rse_id, 'accessed_at': datetime.utcfromtimestamp( report['traceTimeentryUnix']) }) continue else: if 'remoteSite' not in report: continue replicas.append({ 'name': report['filename'], 'scope': report['scope'], 'rse': rse, 'rse_id': rse_id, 'accessed_at': datetime.utcfromtimestamp( report['traceTimeentryUnix']) }) except (KeyError, AttributeError): self.__logger(logging.ERROR, "Cannot handle report.", exc_info=True) record_counter('daemons.tracer.kronos.report_error') continue except Exception: self.__logger(logging.ERROR, "Exception", exc_info=True) continue for did in list_parent_dids(report['scope'], report['filename']): if did['type'] != DIDType.DATASET: continue # do not update _dis datasets if did['scope'].external == 'panda' and '_dis' in did['name']: continue for rse in rses: try: rse_id = get_rse_id(rse=rse, vo=report['vo']) except RSENotFound: self.__logger( logging.WARNING, "Cannot lookup rse_id for %s. Will skip this report.", rse) record_counter('daemons.tracer.kronos.rse_not_found') continue self.__dataset_queue.put({ 'scope': did['scope'], 'name': did['name'], 'did_type': did['type'], 'rse_id': rse_id, 'accessed_at': datetime.utcfromtimestamp(report['traceTimeentryUnix']) }) if not len(replicas): return self.__logger(logging.DEBUG, "trying to update replicas: %s", replicas) try: start_time = time() for replica in replicas: # if touch replica hits a locked row put the trace back into queue for later retry if not touch_replica(replica): resubmit = { 'filename': replica['name'], 'scope': replica['scope'].external, 'remoteSite': replica['rse'], 'traceTimeentryUnix': replica['traceTimeentryUnix'], 'eventType': 'get', 'usrdn': 'someuser', 'clientState': 'DONE', 'eventVersion': replica['eventVersion'] } if replica['scope'].vo != 'def': resubmit['vo'] = replica['scope'].vo self.__conn.send(body=jdumps(resubmit), destination=self.__queue, headers={ 'appversion': 'rucio', 'resubmitted': '1' }) record_counter('daemons.tracer.kronos.sent_resubmitted') self.__logger(logging.WARNING, 'hit locked row, resubmitted to queue') record_timer('daemons.tracer.kronos.update_atime', (time() - start_time) * 1000) except Exception: self.__logger(logging.ERROR, "Cannot update replicas.", exc_info=True) record_counter('daemons.tracer.kronos.update_error') self.__logger(logging.INFO, 'updated %d replica(s)' % len(replicas))
def __update_atime(self): """ Bulk update atime. """ replicas = [] rses = [] for report in self.__reports: try: # check if scope in report. if not skip this one. if 'scope' not in report: record_counter('daemons.tracer.kronos.missing_scope') if report['eventType'] != 'touch': continue else: record_counter('daemons.tracer.kronos.with_scope') # handle all events starting with get* and download and touch events. if not report['eventType'].startswith('get') and not report[ 'eventType'].startswith('sm_get') and not report[ 'eventType'] == 'download' and not report[ 'eventType'] == 'touch': continue if report['eventType'].endswith('_es'): continue record_counter('daemons.tracer.kronos.total_get') if report['eventType'] == 'get': record_counter('daemons.tracer.kronos.dq2clients') elif report['eventType'] == 'get_sm' or report[ 'eventType'] == 'sm_get': if report['eventVersion'] == 'aCT': record_counter( 'daemons.tracer.kronos.panda_production_act') else: record_counter( 'daemons.tracer.kronos.panda_production') elif report['eventType'] == 'get_sm_a' or report[ 'eventType'] == 'sm_get_a': if report['eventVersion'] == 'aCT': record_counter( 'daemons.tracer.kronos.panda_analysis_act') else: record_counter('daemons.tracer.kronos.panda_analysis') elif report['eventType'] == 'download': record_counter('daemons.tracer.kronos.rucio_download') elif report['eventType'] == 'touch': record_counter('daemons.tracer.kronos.rucio_touch') else: record_counter('daemons.tracer.kronos.other_get') if report['eventType'] == 'download' or report[ 'eventType'] == 'touch': report['usrdn'] = report['account'] if report['usrdn'] in self.__excluded_usrdns: continue # handle touch and non-touch traces differently if report['eventType'] != 'touch': # check if the report has the right state. if 'eventVersion' in report: if report['eventVersion'] != 'aCT': if report['clientState'] in self.__excluded_states: continue if 'remoteSite' not in report: continue if not report['remoteSite']: continue if 'filename' not in report: if 'name' in report: report['filename'] = report['name'] rses = report['remoteSite'].strip().split(',') for rse in rses: replicas.append({ 'name': report['filename'], 'scope': report['scope'], 'rse': rse, 'accessed_at': datetime.utcfromtimestamp( report['traceTimeentryUnix']), 'traceTimeentryUnix': report['traceTimeentryUnix'], 'eventVersion': report['eventVersion'] }) else: # if touch event and if datasetScope is in the report then it means # that there is no file scope/name and therefore only the dataset is # put in the queue to be updated and the rest is skipped. if 'datasetScope' in report: rse = None if 'remoteSite' in report: rse = report['remoteSite'] self.__dataset_queue.put({ 'scope': report['datasetScope'], 'name': report['dataset'], 'rse': rse, 'accessed_at': datetime.utcfromtimestamp( report['traceTimeentryUnix']) }) continue else: if 'remoteSite' not in report: continue replicas.append({ 'name': report['filename'], 'scope': report['scope'], 'rse': report['remoteSite'], 'accessed_at': datetime.utcfromtimestamp( report['traceTimeentryUnix']) }) except (KeyError, AttributeError): logging.error(format_exc()) record_counter('daemons.tracer.kronos.report_error') continue for did in list_parent_dids(report['scope'], report['filename']): if did['type'] != DIDType.DATASET: continue # do not update _dis datasets if did['scope'] == 'panda' and '_dis' in did['name']: continue for rse in rses: self.__dataset_queue.put({ 'scope': did['scope'], 'name': did['name'], 'did_type': did['type'], 'rse': rse, 'accessed_at': datetime.utcfromtimestamp(report['traceTimeentryUnix']) }) logging.debug(replicas) try: ts = time() for replica in replicas: # if touch replica hits a locked row put the trace back into queue for later retry if not touch_replica(replica): resubmit = { 'filename': replica['name'], 'scope': replica['scope'], 'remoteSite': replica['rse'], 'traceTimeentryUnix': replica['traceTimeentryUnix'], 'eventType': 'get', 'usrdn': 'someuser', 'clientState': 'DONE', 'eventVersion': replica['eventVersion'] } self.__conn.send(body=jdumps(resubmit), destination=self.__queue, headers={ 'appversion': 'rucio', 'resubmitted': '1' }) record_counter('daemons.tracer.kronos.sent_resubmitted') logging.warning( '(kronos_file) hit locked row, resubmitted to queue') record_timer('daemons.tracer.kronos.update_atime', (time() - ts) * 1000) except: logging.error(format_exc()) record_counter('daemons.tracer.kronos.update_error') logging.info('(kronos_file) updated %d replicas' % len(replicas))
def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0, limit=None, activity=None, older_than=None, rses=None, schemes=None, bring_online=43200, retry_other_fts=False, failover_schemes=None, session=None): """ Get transfer requests and the associated source replicas :param total_workers: Number of total workers. :param worker_number: Id of the executing worker. :param limit: Limit. :param activity: Activity. :param older_than: Get transfers older than. :param rses: Include RSES. :param schemes: Include schemes. :param bring_online: Bring online timeout. :parm retry_other_fts: Retry other fts servers. :param failover_schemes: Failover schemes. :session: The database session in use. :returns: transfers, reqs_no_source, reqs_scheme_mismatch, reqs_only_tape_source """ req_sources = __list_transfer_requests_and_source_replicas(total_workers=total_workers, worker_number=worker_number, limit=limit, activity=activity, older_than=older_than, rses=rses, session=session) unavailable_read_rse_ids = __get_unavailable_read_rse_ids(session=session) bring_online_local = bring_online transfers, rses_info, protocols, rse_attrs, reqs_no_source, reqs_only_tape_source, reqs_scheme_mismatch = {}, {}, {}, {}, [], [], [] for req_id, rule_id, scope, name, md5, adler32, bytes, activity, attributes, previous_attempt_id, dest_rse_id, source_rse_id, rse, deterministic, rse_type, path, retry_count, src_url, ranking, link_ranking in req_sources: transfer_src_type = "DISK" transfer_dst_type = "DISK" allow_tape_source = True try: if rses and dest_rse_id not in rses: continue current_schemes = schemes if previous_attempt_id and failover_schemes: current_schemes = failover_schemes if req_id not in transfers: if req_id not in reqs_no_source: reqs_no_source.append(req_id) # source_rse_id will be None if no source replicas # rse will be None if rse is staging area if source_rse_id is None or rse is None: continue if link_ranking is None: logging.debug("Request %s: no link from %s to %s" % (req_id, source_rse_id, dest_rse_id)) continue if source_rse_id in unavailable_read_rse_ids: continue # Get destination rse information if dest_rse_id not in rses_info: dest_rse = get_rse_name(rse_id=dest_rse_id, session=session) rses_info[dest_rse_id] = rsemgr.get_rse_info(dest_rse, session=session) if dest_rse_id not in rse_attrs: rse_attrs[dest_rse_id] = get_rse_attributes(dest_rse_id, session=session) # Get the source rse information if source_rse_id not in rses_info: source_rse = get_rse_name(rse_id=source_rse_id, session=session) rses_info[source_rse_id] = rsemgr.get_rse_info(source_rse, session=session) if source_rse_id not in rse_attrs: rse_attrs[source_rse_id] = get_rse_attributes(source_rse_id, session=session) attr = None if attributes: if type(attributes) is dict: attr = json.loads(json.dumps(attributes)) else: attr = json.loads(str(attributes)) # parse source expression source_replica_expression = attr["source_replica_expression"] if (attr and "source_replica_expression" in attr) else None if source_replica_expression: try: parsed_rses = parse_expression(source_replica_expression, session=session) except InvalidRSEExpression as error: logging.error("Invalid RSE exception %s: %s" % (source_replica_expression, error)) continue else: allowed_rses = [x['rse'] for x in parsed_rses] if rse not in allowed_rses: continue # parse allow tape source expression, not finally version. # allow_tape_source = attr["allow_tape_source"] if (attr and "allow_tape_source" in attr) else True allow_tape_source = True # Find matching scheme between destination and source try: matching_scheme = rsemgr.find_matching_scheme(rse_settings_dest=rses_info[dest_rse_id], rse_settings_src=rses_info[source_rse_id], operation_src='third_party_copy', operation_dest='third_party_copy', domain='wan', scheme=current_schemes) except RSEProtocolNotSupported: logging.error('No matching schemes in %s for operation "third_party_copy" between %s and %s' % (current_schemes, rses_info[source_rse_id]['rse'], rses_info[dest_rse_id]['rse'])) if req_id in reqs_no_source: reqs_no_source.remove(req_id) if req_id not in reqs_scheme_mismatch: reqs_scheme_mismatch.append(req_id) continue # Get destination protocol dest_rse_id_key = '%s_%s' % (dest_rse_id, matching_scheme[0]) if dest_rse_id_key not in protocols: try: protocols[dest_rse_id_key] = rsemgr.create_protocol(rses_info[dest_rse_id], 'third_party_copy', matching_scheme[0]) except RSEProtocolNotSupported: logging.error('Operation "third_party_copy" not supported by dest_rse %s with schemes %s' % (rses_info[dest_rse_id]['rse'], current_schemes)) if req_id in reqs_no_source: reqs_no_source.remove(req_id) if req_id not in reqs_scheme_mismatch: reqs_scheme_mismatch.append(req_id) continue # get dest space token dest_spacetoken = None if protocols[dest_rse_id_key].attributes and \ 'extended_attributes' in protocols[dest_rse_id_key].attributes and \ protocols[dest_rse_id_key].attributes['extended_attributes'] and \ 'space_token' in protocols[dest_rse_id_key].attributes['extended_attributes']: dest_spacetoken = protocols[dest_rse_id_key].attributes['extended_attributes']['space_token'] # Compute the destination url if rses_info[dest_rse_id]['deterministic']: dest_url = list(protocols[dest_rse_id_key].lfns2pfns(lfns={'scope': scope, 'name': name}).values())[0] else: # compute dest url in case of non deterministic # naming convention, etc. dsn = 'other' if attr and 'ds_name' in attr: dsn = attr["ds_name"] else: # select a containing dataset for parent in did.list_parent_dids(scope, name): if parent['type'] == DIDType.DATASET: dsn = parent['name'] break # DQ2 path always starts with /, but prefix might not end with / naming_convention = rse_attrs[dest_rse_id].get('naming_convention', None) dest_path = construct_surl(dsn, name, naming_convention) if rses_info[dest_rse_id]['rse_type'] == RSEType.TAPE or rses_info[dest_rse_id]['rse_type'] == 'TAPE': if retry_count or activity == 'Recovery': dest_path = '%s_%i' % (dest_path, int(time.time())) dest_url = list(protocols[dest_rse_id_key].lfns2pfns(lfns={'scope': scope, 'name': name, 'path': dest_path}).values())[0] # Get source protocol source_rse_id_key = '%s_%s' % (source_rse_id, '_'.join([matching_scheme[0], matching_scheme[1]])) if source_rse_id_key not in protocols: try: protocols[source_rse_id_key] = rsemgr.create_protocol(rses_info[source_rse_id], 'third_party_copy', matching_scheme[1]) except RSEProtocolNotSupported: logging.error('Operation "third_party_copy" not supported by source_rse %s with schemes %s' % (rses_info[source_rse_id]['rse'], matching_scheme[1])) if req_id in reqs_no_source: reqs_no_source.remove(req_id) if req_id not in reqs_scheme_mismatch: reqs_scheme_mismatch.append(req_id) continue source_url = list(protocols[source_rse_id_key].lfns2pfns(lfns={'scope': scope, 'name': name, 'path': path}).values())[0] # Extend the metadata dictionary with request attributes overwrite, bring_online = True, None if rses_info[source_rse_id]['rse_type'] == RSEType.TAPE or rses_info[source_rse_id]['rse_type'] == 'TAPE': bring_online = bring_online_local transfer_src_type = "TAPE" if not allow_tape_source: if req_id not in reqs_only_tape_source: reqs_only_tape_source.append(req_id) if req_id in reqs_no_source: reqs_no_source.remove(req_id) continue if rses_info[dest_rse_id]['rse_type'] == RSEType.TAPE or rses_info[dest_rse_id]['rse_type'] == 'TAPE': overwrite = False transfer_dst_type = "TAPE" # get external_host fts_hosts = rse_attrs[dest_rse_id].get('fts', None) if not fts_hosts: logging.error('Destination RSE %s FTS attribute not defined - SKIP REQUEST %s' % (dest_rse, req_id)) continue if retry_count is None: retry_count = 0 fts_list = fts_hosts.split(",") verify_checksum = 'both' if not rse_attrs[dest_rse_id].get('verify_checksum', True): if not rse_attrs[source_rse_id].get('verify_checksum', True): verify_checksum = 'none' else: verify_checksum = 'source' else: if not rse_attrs[source_rse_id].get('verify_checksum', True): verify_checksum = 'destination' else: verify_checksum = 'both' external_host = fts_list[0] if retry_other_fts: external_host = fts_list[retry_count % len(fts_list)] file_metadata = {'request_id': req_id, 'scope': scope, 'name': name, 'activity': activity, 'request_type': str(RequestType.TRANSFER).lower(), 'src_type': transfer_src_type, 'dst_type': transfer_dst_type, 'src_rse': rse, 'dst_rse': rses_info[dest_rse_id]['rse'], 'src_rse_id': source_rse_id, 'dest_rse_id': dest_rse_id, 'filesize': bytes, 'md5': md5, 'adler32': adler32, 'verify_checksum': verify_checksum} if previous_attempt_id: file_metadata['previous_attempt_id'] = previous_attempt_id transfers[req_id] = {'request_id': req_id, 'schemes': __add_compatible_schemes(schemes=[matching_scheme[0]], allowed_schemes=current_schemes), # 'src_urls': [source_url], 'sources': [(rse, source_url, source_rse_id, ranking if ranking is not None else 0, link_ranking)], 'dest_urls': [dest_url], 'src_spacetoken': None, 'dest_spacetoken': dest_spacetoken, 'overwrite': overwrite, 'bring_online': bring_online, 'copy_pin_lifetime': attr.get('lifetime', -1), 'external_host': external_host, 'selection_strategy': 'auto', 'rule_id': rule_id, 'file_metadata': file_metadata} else: current_schemes = transfers[req_id]['schemes'] # source_rse_id will be None if no source replicas # rse will be None if rse is staging area if source_rse_id is None or rse is None: continue if link_ranking is None: logging.debug("Request %s: no link from %s to %s" % (req_id, source_rse_id, dest_rse_id)) continue if source_rse_id in unavailable_read_rse_ids: continue attr = None if attributes: if type(attributes) is dict: attr = json.loads(json.dumps(attributes)) else: attr = json.loads(str(attributes)) # parse source expression source_replica_expression = attr["source_replica_expression"] if (attr and "source_replica_expression" in attr) else None if source_replica_expression: try: parsed_rses = parse_expression(source_replica_expression, session=session) except InvalidRSEExpression as error: logging.error("Invalid RSE exception %s: %s" % (source_replica_expression, error)) continue else: allowed_rses = [x['rse'] for x in parsed_rses] if rse not in allowed_rses: continue # parse allow tape source expression, not finally version. allow_tape_source = attr["allow_tape_source"] if (attr and "allow_tape_source" in attr) else True # Compute the source rse information if source_rse_id not in rses_info: source_rse = get_rse_name(rse_id=source_rse_id, session=session) rses_info[source_rse_id] = rsemgr.get_rse_info(source_rse, session=session) # Get protocol source_rse_id_key = '%s_%s' % (source_rse_id, '_'.join(current_schemes)) if source_rse_id_key not in protocols: try: protocols[source_rse_id_key] = rsemgr.create_protocol(rses_info[source_rse_id], 'third_party_copy', current_schemes) except RSEProtocolNotSupported: logging.error('Operation "third_party_copy" not supported by %s with schemes %s' % (rses_info[source_rse_id]['rse'], current_schemes)) continue source_url = list(protocols[source_rse_id_key].lfns2pfns(lfns={'scope': scope, 'name': name, 'path': path}).values())[0] if ranking is None: ranking = 0 # TAPE should not mixed with Disk and should not use as first try # If there is a source whose ranking is no less than the Tape ranking, Tape will not be used. if rses_info[source_rse_id]['rse_type'] == RSEType.TAPE or rses_info[source_rse_id]['rse_type'] == 'TAPE': # current src_rse is Tape if not allow_tape_source: continue if not transfers[req_id]['bring_online']: # the sources already founded are disks. avail_top_ranking = None founded_sources = transfers[req_id]['sources'] for founded_source in founded_sources: if avail_top_ranking is None: avail_top_ranking = founded_source[3] continue if founded_source[3] is not None and founded_source[3] > avail_top_ranking: avail_top_ranking = founded_source[3] if avail_top_ranking >= ranking: # current Tape source is not the highest ranking, will use disk sources continue else: transfers[req_id]['sources'] = [] transfers[req_id]['bring_online'] = bring_online_local transfer_src_type = "TAPE" transfers[req_id]['file_metadata']['src_type'] = transfer_src_type transfers[req_id]['file_metadata']['src_rse'] = rse else: # the sources already founded is Tape too. # multiple Tape source replicas are not allowed in FTS3. if transfers[req_id]['sources'][0][3] > ranking or (transfers[req_id]['sources'][0][3] == ranking and transfers[req_id]['sources'][0][4] <= link_ranking): continue else: transfers[req_id]['sources'] = [] transfers[req_id]['bring_online'] = bring_online_local transfers[req_id]['file_metadata']['src_rse'] = rse else: # current src_rse is Disk if transfers[req_id]['bring_online']: # the founded sources are Tape avail_top_ranking = None founded_sources = transfers[req_id]['sources'] for founded_source in founded_sources: if avail_top_ranking is None: avail_top_ranking = founded_source[3] continue if founded_source[3] is not None and founded_source[3] > avail_top_ranking: avail_top_ranking = founded_source[3] if ranking >= avail_top_ranking: # current disk replica has higher ranking than founded sources # remove founded Tape sources transfers[req_id]['sources'] = [] transfers[req_id]['bring_online'] = None transfer_src_type = "DISK" transfers[req_id]['file_metadata']['src_type'] = transfer_src_type transfers[req_id]['file_metadata']['src_rse'] = rse else: continue # transfers[id]['src_urls'].append((source_rse_id, source_url)) transfers[req_id]['sources'].append((rse, source_url, source_rse_id, ranking, link_ranking)) except Exception: logging.critical("Exception happened when trying to get transfer for request %s: %s" % (req_id, traceback.format_exc())) break for req_id in transfers: if req_id in reqs_no_source: reqs_no_source.remove(req_id) if req_id in reqs_only_tape_source: reqs_only_tape_source.remove(req_id) if req_id in reqs_scheme_mismatch: reqs_scheme_mismatch.remove(req_id) return transfers, reqs_no_source, reqs_scheme_mismatch, reqs_only_tape_source
def get_destinations(rse_info, scheme, req, sources): dsn = 'other' pfn = {} paths = {} if not rse_info['deterministic']: ts = time.time() # get rule scope and name if req['attributes']: if type(req['attributes']) is dict: req_attributes = json.loads(json.dumps(req['attributes'])) else: req_attributes = json.loads(str(req['attributes'])) if 'ds_name' in req_attributes: dsn = req_attributes["ds_name"] if dsn == 'other': # select a containing dataset for parent in did.list_parent_dids(req['scope'], req['name']): if parent['type'] == DIDType.DATASET: dsn = parent['name'] break record_timer('daemons.conveyor.submitter.list_parent_dids', (time.time() - ts) * 1000) # always use SRM ts = time.time() nondet = rsemgr.create_protocol(rse_info, 'write', scheme='srm') record_timer('daemons.conveyor.submitter.create_protocol', (time.time() - ts) * 1000) # if there exists a prefix for SRM, use it prefix = '' for s in rse_info['protocols']: if s['scheme'] == 'srm': prefix = s['prefix'] # DQ2 path always starts with /, but prefix might not end with / path = construct_surl_DQ2(dsn, req['name']) # retrial transfers to tape need a new filename - add timestamp if req['request_type'] == RequestType.TRANSFER\ and 'previous_attempt_id' in req\ and req['previous_attempt_id']\ and rse_info['rse_type'] == 'TAPE': # TODO: RUCIO-809 - rsemanager: get_rse_info -> rse_type is string instead of RSEType path = '%s_%i' % (path, int(time.time())) logging.debug('Retrial transfer request %s DID %s:%s to tape %s renamed to %s' % (req['request_id'], req['scope'], req['name'], rse_info['rse'], path)) tmp_path = '%s%s' % (prefix[:-1], path) if prefix[-1] != '/': tmp_path = '%s%s' % (prefix, path) paths[req['scope'], req['name']] = path # add the hostname pfn['%s:%s' % (req['scope'], req['name'])] = nondet.path2pfn(tmp_path) if req['request_type'] == RequestType.STAGEIN: if len(sources) == 1: pfn['%s:%s' % (req['scope'], req['name'])] = sources[0][1] else: # TODO: need to check return None, None # we must set the destination path for nondeterministic replicas explicitly replica.update_replicas_paths([{'scope': req['scope'], 'name': req['name'], 'rse_id': req['dest_rse_id'], 'path': path}]) else: ts = time.time() try: pfn = rsemgr.lfns2pfns(rse_info, lfns=[{'scope': req['scope'], 'name': req['name']}], scheme=scheme) except RSEProtocolNotSupported: logging.warn('%s not supported by %s' % (scheme, rse_info['rse'])) return None, None record_timer('daemons.conveyor.submitter.lfns2pfns', (time.time() - ts) * 1000) destinations = [] for k in pfn: if isinstance(pfn[k], (str, unicode)): destinations.append(pfn[k]) elif isinstance(pfn[k], (tuple, list)): for url in pfn[k]: destinations.append(pfn[k][url]) protocols = None try: protocols = rsemgr.select_protocol(rse_info, 'write', scheme=scheme) except RSEProtocolNotSupported: logging.warn('%s not supported by %s' % (scheme, rse_info['rse'])) return None, None # we need to set the spacetoken if we use SRM dest_spacetoken = None if scheme == 'srm': dest_spacetoken = protocols['extended_attributes']['space_token'] return destinations, dest_spacetoken