def __update_atime(self):
"""
Bulk update atime.
"""
replicas = []
for report in self.__reports:
try:
# check if scope in report. if not skip this one.
if 'scope' not in report:
record_counter('daemons.tracer.kronos.missing_scope')
continue
else:
record_counter('daemons.tracer.kronos.with_scope')
# for the moment only report with eventType get* are handled.
if not report['eventType'].startswith('get'):
continue
record_counter('daemons.tracer.kronos.total_get')
if report['eventType'] == 'get':
record_counter('daemons.tracer.kronos.dq2clients')
elif report['eventType'] == 'get_sm':
record_counter('daemons.tracer.kronos.panda_production')
elif report['eventType'] == 'get_sm_a':
record_counter('daemons.tracer.kronos.panda_analysis')
else:
record_counter('daemons.tracer.kronos.other_get')
# check if the report has the right state.
if report['clientState'] in self.__excluded_states:
continue
if report['usrdn'] in self.__excluded_usrdns:
continue
if not report['remoteSite']:
continue
replicas.append({'name': report['filename'], 'scope': report['scope'], 'rse': report['remoteSite'], 'accessed_at': datetime.utcfromtimestamp(report['traceTimeentryUnix'])})
except (KeyError, AttributeError):
logging.error(format_exc())
record_counter('daemons.tracer.kronos.report_error')
continue
for did in list_parent_dids(report['scope'], report['filename']):
if did['type'] != DIDType.DATASET:
continue
# do not update _dis datasets
if did['scope'] == 'panda' and '_dis' in did['name']:
continue
self.__dataset_queue.put({'scope': did['scope'], 'name': did['name'], 'did_type': did['type'], 'rse': report['remoteSite'], 'accessed_at': datetime.utcfromtimestamp(report['traceTimeentryUnix'])})
logging.info(replicas)
try:
ts = time()
touch_replicas(replicas)
record_timer('daemons.tracer.kronos.update_atime', (time() - ts) * 1000)
except:
logging.error(format_exc())
record_counter('daemons.tracer.kronos.update_error')
logging.info('(kronos_file) updated %d replicas' % len(replicas))
def list_parent_dids(scope, name):
"""
List parent datasets and containers of a did.
:param scope: The scope.
:param name: The name.
"""
return did.list_parent_dids(scope=scope, name=name)
def list_parent_dids(scope, name):
"""
List parent datasets and containers of a did.
:param scope: The scope.
:param name: The name.
"""
return did.list_parent_dids(scope=scope, name=name)
def list_parent_dids(scope, name):
"""
List parent datasets and containers of a did.
:param scope: The scope.
:param name: The name.
"""
scope = InternalScope(scope)
dids = did.list_parent_dids(scope=scope, name=name)
for d in dids:
yield api_update_return_dict(d)
def list_parent_dids(scope, name, vo='def', session=None):
"""
List parent datasets and containers of a did.
:param scope: The scope.
:param name: The name.
:param vo: The VO to act on.
:param session: The database session in use.
"""
scope = InternalScope(scope, vo=vo)
dids = did.list_parent_dids(scope=scope, name=name, session=session)
for d in dids:
yield api_update_return_dict(d, session=session)
def __update_atime(self):
"""
Bulk update atime.
"""
replicas = []
rses = []
for report in self.__reports:
if 'vo' not in report:
report['vo'] = 'def'
try:
# Identify suspicious files
try:
if self.__bad_files_patterns and report['eventType'] in [
'get_sm', 'get_sm_a', 'get'
] and 'clientState' in report and report[
'clientState'] not in [
'DONE', 'FOUND_ROOT', 'ALREADY_DONE'
]:
for pattern in self.__bad_files_patterns:
if 'stateReason' in report and report[
'stateReason'] and isinstance(
report['stateReason'],
str) and pattern.match(
report['stateReason']):
reason = report['stateReason'][:255]
if 'url' not in report or not report['url']:
self.__logger(
logging.ERROR,
'Missing url in the following trace : '
+ str(report))
else:
try:
surl = report['url']
declare_bad_file_replicas(
[
surl,
],
reason=reason,
issuer=InternalAccount(
'root', vo=report['vo']),
status=BadFilesStatus.SUSPICIOUS)
self.__logger(
logging.INFO,
'Declare suspicious file %s with reason %s'
% (report['url'], reason))
except Exception as error:
self.__logger(
logging.ERROR,
'Failed to declare suspicious file'
+ str(error))
except Exception as error:
self.__logger(
logging.ERROR,
'Problem with bad trace : %s . Error %s' %
(str(report), str(error)))
# check if scope in report. if not skip this one.
if 'scope' not in report:
record_counter('daemons.tracer.kronos.missing_scope')
if report['eventType'] != 'touch':
continue
else:
record_counter('daemons.tracer.kronos.with_scope')
report['scope'] = InternalScope(report['scope'],
report['vo'])
# handle all events starting with get* and download and touch events.
if not report['eventType'].startswith('get') and not report[
'eventType'].startswith('sm_get') and not report[
'eventType'] == 'download' and not report[
'eventType'] == 'touch':
continue
if report['eventType'].endswith('_es'):
continue
record_counter('daemons.tracer.kronos.total_get')
if report['eventType'] == 'get':
record_counter('daemons.tracer.kronos.dq2clients')
elif report['eventType'] == 'get_sm' or report[
'eventType'] == 'sm_get':
if report['eventVersion'] == 'aCT':
record_counter(
'daemons.tracer.kronos.panda_production_act')
else:
record_counter(
'daemons.tracer.kronos.panda_production')
elif report['eventType'] == 'get_sm_a' or report[
'eventType'] == 'sm_get_a':
if report['eventVersion'] == 'aCT':
record_counter(
'daemons.tracer.kronos.panda_analysis_act')
else:
record_counter('daemons.tracer.kronos.panda_analysis')
elif report['eventType'] == 'download':
record_counter('daemons.tracer.kronos.rucio_download')
elif report['eventType'] == 'touch':
record_counter('daemons.tracer.kronos.rucio_touch')
else:
record_counter('daemons.tracer.kronos.other_get')
if report['eventType'] == 'download' or report[
'eventType'] == 'touch':
report['usrdn'] = report['account']
if report['usrdn'] in self.__excluded_usrdns:
continue
# handle touch and non-touch traces differently
if report['eventType'] != 'touch':
# check if the report has the right state.
if 'eventVersion' in report:
if report['eventVersion'] != 'aCT':
if report['clientState'] in self.__excluded_states:
continue
if 'remoteSite' not in report:
continue
if not report['remoteSite']:
continue
if 'filename' not in report:
if 'name' in report:
report['filename'] = report['name']
rses = report['remoteSite'].strip().split(',')
for rse in rses:
try:
rse_id = get_rse_id(rse=rse, vo=report['vo'])
except RSENotFound:
self.__logger(
logging.WARNING,
"Cannot lookup rse_id for %s. Will skip this report.",
rse)
record_counter(
'daemons.tracer.kronos.rse_not_found')
continue
replicas.append({
'name':
report['filename'],
'scope':
report['scope'],
'rse':
rse,
'rse_id':
rse_id,
'accessed_at':
datetime.utcfromtimestamp(
report['traceTimeentryUnix']),
'traceTimeentryUnix':
report['traceTimeentryUnix'],
'eventVersion':
report['eventVersion']
})
else:
# if touch event and if datasetScope is in the report then it means
# that there is no file scope/name and therefore only the dataset is
# put in the queue to be updated and the rest is skipped.
rse_id = None
rse = None
if 'remoteSite' in report:
rse = report['remoteSite']
try:
rse_id = get_rse_id(rse=rse, vo=report['vo'])
except RSENotFound:
self.__logger(logging.WARNING,
"Cannot lookup rse_id for %s.", rse)
record_counter(
'daemons.tracer.kronos.rse_not_found')
if 'datasetScope' in report:
self.__dataset_queue.put({
'scope':
InternalScope(report['datasetScope'],
vo=report['vo']),
'name':
report['dataset'],
'rse_id':
rse_id,
'accessed_at':
datetime.utcfromtimestamp(
report['traceTimeentryUnix'])
})
continue
else:
if 'remoteSite' not in report:
continue
replicas.append({
'name':
report['filename'],
'scope':
report['scope'],
'rse':
rse,
'rse_id':
rse_id,
'accessed_at':
datetime.utcfromtimestamp(
report['traceTimeentryUnix'])
})
except (KeyError, AttributeError):
self.__logger(logging.ERROR,
"Cannot handle report.",
exc_info=True)
record_counter('daemons.tracer.kronos.report_error')
continue
except Exception:
self.__logger(logging.ERROR, "Exception", exc_info=True)
continue
for did in list_parent_dids(report['scope'], report['filename']):
if did['type'] != DIDType.DATASET:
continue
# do not update _dis datasets
if did['scope'].external == 'panda' and '_dis' in did['name']:
continue
for rse in rses:
try:
rse_id = get_rse_id(rse=rse, vo=report['vo'])
except RSENotFound:
self.__logger(
logging.WARNING,
"Cannot lookup rse_id for %s. Will skip this report.",
rse)
record_counter('daemons.tracer.kronos.rse_not_found')
continue
self.__dataset_queue.put({
'scope':
did['scope'],
'name':
did['name'],
'did_type':
did['type'],
'rse_id':
rse_id,
'accessed_at':
datetime.utcfromtimestamp(report['traceTimeentryUnix'])
})
if not len(replicas):
return
self.__logger(logging.DEBUG, "trying to update replicas: %s", replicas)
try:
start_time = time()
for replica in replicas:
# if touch replica hits a locked row put the trace back into queue for later retry
if not touch_replica(replica):
resubmit = {
'filename': replica['name'],
'scope': replica['scope'].external,
'remoteSite': replica['rse'],
'traceTimeentryUnix': replica['traceTimeentryUnix'],
'eventType': 'get',
'usrdn': 'someuser',
'clientState': 'DONE',
'eventVersion': replica['eventVersion']
}
if replica['scope'].vo != 'def':
resubmit['vo'] = replica['scope'].vo
self.__conn.send(body=jdumps(resubmit),
destination=self.__queue,
headers={
'appversion': 'rucio',
'resubmitted': '1'
})
record_counter('daemons.tracer.kronos.sent_resubmitted')
self.__logger(logging.WARNING,
'hit locked row, resubmitted to queue')
record_timer('daemons.tracer.kronos.update_atime',
(time() - start_time) * 1000)
except Exception:
self.__logger(logging.ERROR,
"Cannot update replicas.",
exc_info=True)
record_counter('daemons.tracer.kronos.update_error')
self.__logger(logging.INFO, 'updated %d replica(s)' % len(replicas))
def __update_atime(self):
"""
Bulk update atime.
"""
replicas = []
rses = []
for report in self.__reports:
try:
# check if scope in report. if not skip this one.
if 'scope' not in report:
record_counter('daemons.tracer.kronos.missing_scope')
if report['eventType'] != 'touch':
continue
else:
record_counter('daemons.tracer.kronos.with_scope')
# handle all events starting with get* and download and touch events.
if not report['eventType'].startswith('get') and not report[
'eventType'].startswith('sm_get') and not report[
'eventType'] == 'download' and not report[
'eventType'] == 'touch':
continue
if report['eventType'].endswith('_es'):
continue
record_counter('daemons.tracer.kronos.total_get')
if report['eventType'] == 'get':
record_counter('daemons.tracer.kronos.dq2clients')
elif report['eventType'] == 'get_sm' or report[
'eventType'] == 'sm_get':
if report['eventVersion'] == 'aCT':
record_counter(
'daemons.tracer.kronos.panda_production_act')
else:
record_counter(
'daemons.tracer.kronos.panda_production')
elif report['eventType'] == 'get_sm_a' or report[
'eventType'] == 'sm_get_a':
if report['eventVersion'] == 'aCT':
record_counter(
'daemons.tracer.kronos.panda_analysis_act')
else:
record_counter('daemons.tracer.kronos.panda_analysis')
elif report['eventType'] == 'download':
record_counter('daemons.tracer.kronos.rucio_download')
elif report['eventType'] == 'touch':
record_counter('daemons.tracer.kronos.rucio_touch')
else:
record_counter('daemons.tracer.kronos.other_get')
if report['eventType'] == 'download' or report[
'eventType'] == 'touch':
report['usrdn'] = report['account']
if report['usrdn'] in self.__excluded_usrdns:
continue
# handle touch and non-touch traces differently
if report['eventType'] != 'touch':
# check if the report has the right state.
if 'eventVersion' in report:
if report['eventVersion'] != 'aCT':
if report['clientState'] in self.__excluded_states:
continue
if 'remoteSite' not in report:
continue
if not report['remoteSite']:
continue
if 'filename' not in report:
if 'name' in report:
report['filename'] = report['name']
rses = report['remoteSite'].strip().split(',')
for rse in rses:
replicas.append({
'name':
report['filename'],
'scope':
report['scope'],
'rse':
rse,
'accessed_at':
datetime.utcfromtimestamp(
report['traceTimeentryUnix']),
'traceTimeentryUnix':
report['traceTimeentryUnix'],
'eventVersion':
report['eventVersion']
})
else:
# if touch event and if datasetScope is in the report then it means
# that there is no file scope/name and therefore only the dataset is
# put in the queue to be updated and the rest is skipped.
if 'datasetScope' in report:
rse = None
if 'remoteSite' in report:
rse = report['remoteSite']
self.__dataset_queue.put({
'scope':
report['datasetScope'],
'name':
report['dataset'],
'rse':
rse,
'accessed_at':
datetime.utcfromtimestamp(
report['traceTimeentryUnix'])
})
continue
else:
if 'remoteSite' not in report:
continue
replicas.append({
'name':
report['filename'],
'scope':
report['scope'],
'rse':
report['remoteSite'],
'accessed_at':
datetime.utcfromtimestamp(
report['traceTimeentryUnix'])
})
except (KeyError, AttributeError):
logging.error(format_exc())
record_counter('daemons.tracer.kronos.report_error')
continue
for did in list_parent_dids(report['scope'], report['filename']):
if did['type'] != DIDType.DATASET:
continue
# do not update _dis datasets
if did['scope'] == 'panda' and '_dis' in did['name']:
continue
for rse in rses:
self.__dataset_queue.put({
'scope':
did['scope'],
'name':
did['name'],
'did_type':
did['type'],
'rse':
rse,
'accessed_at':
datetime.utcfromtimestamp(report['traceTimeentryUnix'])
})
logging.debug(replicas)
try:
ts = time()
for replica in replicas:
# if touch replica hits a locked row put the trace back into queue for later retry
if not touch_replica(replica):
resubmit = {
'filename': replica['name'],
'scope': replica['scope'],
'remoteSite': replica['rse'],
'traceTimeentryUnix': replica['traceTimeentryUnix'],
'eventType': 'get',
'usrdn': 'someuser',
'clientState': 'DONE',
'eventVersion': replica['eventVersion']
}
self.__conn.send(body=jdumps(resubmit),
destination=self.__queue,
headers={
'appversion': 'rucio',
'resubmitted': '1'
})
record_counter('daemons.tracer.kronos.sent_resubmitted')
logging.warning(
'(kronos_file) hit locked row, resubmitted to queue')
record_timer('daemons.tracer.kronos.update_atime',
(time() - ts) * 1000)
except:
logging.error(format_exc())
record_counter('daemons.tracer.kronos.update_error')
logging.info('(kronos_file) updated %d replicas' % len(replicas))
def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0, limit=None, activity=None, older_than=None, rses=None, schemes=None,
bring_online=43200, retry_other_fts=False, failover_schemes=None, session=None):
"""
Get transfer requests and the associated source replicas
:param total_workers: Number of total workers.
:param worker_number: Id of the executing worker.
:param limit: Limit.
:param activity: Activity.
:param older_than: Get transfers older than.
:param rses: Include RSES.
:param schemes: Include schemes.
:param bring_online: Bring online timeout.
:parm retry_other_fts: Retry other fts servers.
:param failover_schemes: Failover schemes.
:session: The database session in use.
:returns: transfers, reqs_no_source, reqs_scheme_mismatch, reqs_only_tape_source
"""
req_sources = __list_transfer_requests_and_source_replicas(total_workers=total_workers,
worker_number=worker_number,
limit=limit,
activity=activity,
older_than=older_than,
rses=rses,
session=session)
unavailable_read_rse_ids = __get_unavailable_read_rse_ids(session=session)
bring_online_local = bring_online
transfers, rses_info, protocols, rse_attrs, reqs_no_source, reqs_only_tape_source, reqs_scheme_mismatch = {}, {}, {}, {}, [], [], []
for req_id, rule_id, scope, name, md5, adler32, bytes, activity, attributes, previous_attempt_id, dest_rse_id, source_rse_id, rse, deterministic, rse_type, path, retry_count, src_url, ranking, link_ranking in req_sources:
transfer_src_type = "DISK"
transfer_dst_type = "DISK"
allow_tape_source = True
try:
if rses and dest_rse_id not in rses:
continue
current_schemes = schemes
if previous_attempt_id and failover_schemes:
current_schemes = failover_schemes
if req_id not in transfers:
if req_id not in reqs_no_source:
reqs_no_source.append(req_id)
# source_rse_id will be None if no source replicas
# rse will be None if rse is staging area
if source_rse_id is None or rse is None:
continue
if link_ranking is None:
logging.debug("Request %s: no link from %s to %s" % (req_id, source_rse_id, dest_rse_id))
continue
if source_rse_id in unavailable_read_rse_ids:
continue
# Get destination rse information
if dest_rse_id not in rses_info:
dest_rse = get_rse_name(rse_id=dest_rse_id, session=session)
rses_info[dest_rse_id] = rsemgr.get_rse_info(dest_rse, session=session)
if dest_rse_id not in rse_attrs:
rse_attrs[dest_rse_id] = get_rse_attributes(dest_rse_id, session=session)
# Get the source rse information
if source_rse_id not in rses_info:
source_rse = get_rse_name(rse_id=source_rse_id, session=session)
rses_info[source_rse_id] = rsemgr.get_rse_info(source_rse, session=session)
if source_rse_id not in rse_attrs:
rse_attrs[source_rse_id] = get_rse_attributes(source_rse_id, session=session)
attr = None
if attributes:
if type(attributes) is dict:
attr = json.loads(json.dumps(attributes))
else:
attr = json.loads(str(attributes))
# parse source expression
source_replica_expression = attr["source_replica_expression"] if (attr and "source_replica_expression" in attr) else None
if source_replica_expression:
try:
parsed_rses = parse_expression(source_replica_expression, session=session)
except InvalidRSEExpression as error:
logging.error("Invalid RSE exception %s: %s" % (source_replica_expression, error))
continue
else:
allowed_rses = [x['rse'] for x in parsed_rses]
if rse not in allowed_rses:
continue
# parse allow tape source expression, not finally version.
# allow_tape_source = attr["allow_tape_source"] if (attr and "allow_tape_source" in attr) else True
allow_tape_source = True
# Find matching scheme between destination and source
try:
matching_scheme = rsemgr.find_matching_scheme(rse_settings_dest=rses_info[dest_rse_id],
rse_settings_src=rses_info[source_rse_id],
operation_src='third_party_copy',
operation_dest='third_party_copy',
domain='wan',
scheme=current_schemes)
except RSEProtocolNotSupported:
logging.error('No matching schemes in %s for operation "third_party_copy" between %s and %s' % (current_schemes, rses_info[source_rse_id]['rse'], rses_info[dest_rse_id]['rse']))
if req_id in reqs_no_source:
reqs_no_source.remove(req_id)
if req_id not in reqs_scheme_mismatch:
reqs_scheme_mismatch.append(req_id)
continue
# Get destination protocol
dest_rse_id_key = '%s_%s' % (dest_rse_id, matching_scheme[0])
if dest_rse_id_key not in protocols:
try:
protocols[dest_rse_id_key] = rsemgr.create_protocol(rses_info[dest_rse_id], 'third_party_copy', matching_scheme[0])
except RSEProtocolNotSupported:
logging.error('Operation "third_party_copy" not supported by dest_rse %s with schemes %s' % (rses_info[dest_rse_id]['rse'], current_schemes))
if req_id in reqs_no_source:
reqs_no_source.remove(req_id)
if req_id not in reqs_scheme_mismatch:
reqs_scheme_mismatch.append(req_id)
continue
# get dest space token
dest_spacetoken = None
if protocols[dest_rse_id_key].attributes and \
'extended_attributes' in protocols[dest_rse_id_key].attributes and \
protocols[dest_rse_id_key].attributes['extended_attributes'] and \
'space_token' in protocols[dest_rse_id_key].attributes['extended_attributes']:
dest_spacetoken = protocols[dest_rse_id_key].attributes['extended_attributes']['space_token']
# Compute the destination url
if rses_info[dest_rse_id]['deterministic']:
dest_url = list(protocols[dest_rse_id_key].lfns2pfns(lfns={'scope': scope, 'name': name}).values())[0]
else:
# compute dest url in case of non deterministic
# naming convention, etc.
dsn = 'other'
if attr and 'ds_name' in attr:
dsn = attr["ds_name"]
else:
# select a containing dataset
for parent in did.list_parent_dids(scope, name):
if parent['type'] == DIDType.DATASET:
dsn = parent['name']
break
# DQ2 path always starts with /, but prefix might not end with /
naming_convention = rse_attrs[dest_rse_id].get('naming_convention', None)
dest_path = construct_surl(dsn, name, naming_convention)
if rses_info[dest_rse_id]['rse_type'] == RSEType.TAPE or rses_info[dest_rse_id]['rse_type'] == 'TAPE':
if retry_count or activity == 'Recovery':
dest_path = '%s_%i' % (dest_path, int(time.time()))
dest_url = list(protocols[dest_rse_id_key].lfns2pfns(lfns={'scope': scope, 'name': name, 'path': dest_path}).values())[0]
# Get source protocol
source_rse_id_key = '%s_%s' % (source_rse_id, '_'.join([matching_scheme[0], matching_scheme[1]]))
if source_rse_id_key not in protocols:
try:
protocols[source_rse_id_key] = rsemgr.create_protocol(rses_info[source_rse_id], 'third_party_copy', matching_scheme[1])
except RSEProtocolNotSupported:
logging.error('Operation "third_party_copy" not supported by source_rse %s with schemes %s' % (rses_info[source_rse_id]['rse'], matching_scheme[1]))
if req_id in reqs_no_source:
reqs_no_source.remove(req_id)
if req_id not in reqs_scheme_mismatch:
reqs_scheme_mismatch.append(req_id)
continue
source_url = list(protocols[source_rse_id_key].lfns2pfns(lfns={'scope': scope, 'name': name, 'path': path}).values())[0]
# Extend the metadata dictionary with request attributes
overwrite, bring_online = True, None
if rses_info[source_rse_id]['rse_type'] == RSEType.TAPE or rses_info[source_rse_id]['rse_type'] == 'TAPE':
bring_online = bring_online_local
transfer_src_type = "TAPE"
if not allow_tape_source:
if req_id not in reqs_only_tape_source:
reqs_only_tape_source.append(req_id)
if req_id in reqs_no_source:
reqs_no_source.remove(req_id)
continue
if rses_info[dest_rse_id]['rse_type'] == RSEType.TAPE or rses_info[dest_rse_id]['rse_type'] == 'TAPE':
overwrite = False
transfer_dst_type = "TAPE"
# get external_host
fts_hosts = rse_attrs[dest_rse_id].get('fts', None)
if not fts_hosts:
logging.error('Destination RSE %s FTS attribute not defined - SKIP REQUEST %s' % (dest_rse, req_id))
continue
if retry_count is None:
retry_count = 0
fts_list = fts_hosts.split(",")
verify_checksum = 'both'
if not rse_attrs[dest_rse_id].get('verify_checksum', True):
if not rse_attrs[source_rse_id].get('verify_checksum', True):
verify_checksum = 'none'
else:
verify_checksum = 'source'
else:
if not rse_attrs[source_rse_id].get('verify_checksum', True):
verify_checksum = 'destination'
else:
verify_checksum = 'both'
external_host = fts_list[0]
if retry_other_fts:
external_host = fts_list[retry_count % len(fts_list)]
file_metadata = {'request_id': req_id,
'scope': scope,
'name': name,
'activity': activity,
'request_type': str(RequestType.TRANSFER).lower(),
'src_type': transfer_src_type,
'dst_type': transfer_dst_type,
'src_rse': rse,
'dst_rse': rses_info[dest_rse_id]['rse'],
'src_rse_id': source_rse_id,
'dest_rse_id': dest_rse_id,
'filesize': bytes,
'md5': md5,
'adler32': adler32,
'verify_checksum': verify_checksum}
if previous_attempt_id:
file_metadata['previous_attempt_id'] = previous_attempt_id
transfers[req_id] = {'request_id': req_id,
'schemes': __add_compatible_schemes(schemes=[matching_scheme[0]], allowed_schemes=current_schemes),
# 'src_urls': [source_url],
'sources': [(rse, source_url, source_rse_id, ranking if ranking is not None else 0, link_ranking)],
'dest_urls': [dest_url],
'src_spacetoken': None,
'dest_spacetoken': dest_spacetoken,
'overwrite': overwrite,
'bring_online': bring_online,
'copy_pin_lifetime': attr.get('lifetime', -1),
'external_host': external_host,
'selection_strategy': 'auto',
'rule_id': rule_id,
'file_metadata': file_metadata}
else:
current_schemes = transfers[req_id]['schemes']
# source_rse_id will be None if no source replicas
# rse will be None if rse is staging area
if source_rse_id is None or rse is None:
continue
if link_ranking is None:
logging.debug("Request %s: no link from %s to %s" % (req_id, source_rse_id, dest_rse_id))
continue
if source_rse_id in unavailable_read_rse_ids:
continue
attr = None
if attributes:
if type(attributes) is dict:
attr = json.loads(json.dumps(attributes))
else:
attr = json.loads(str(attributes))
# parse source expression
source_replica_expression = attr["source_replica_expression"] if (attr and "source_replica_expression" in attr) else None
if source_replica_expression:
try:
parsed_rses = parse_expression(source_replica_expression, session=session)
except InvalidRSEExpression as error:
logging.error("Invalid RSE exception %s: %s" % (source_replica_expression, error))
continue
else:
allowed_rses = [x['rse'] for x in parsed_rses]
if rse not in allowed_rses:
continue
# parse allow tape source expression, not finally version.
allow_tape_source = attr["allow_tape_source"] if (attr and "allow_tape_source" in attr) else True
# Compute the source rse information
if source_rse_id not in rses_info:
source_rse = get_rse_name(rse_id=source_rse_id, session=session)
rses_info[source_rse_id] = rsemgr.get_rse_info(source_rse, session=session)
# Get protocol
source_rse_id_key = '%s_%s' % (source_rse_id, '_'.join(current_schemes))
if source_rse_id_key not in protocols:
try:
protocols[source_rse_id_key] = rsemgr.create_protocol(rses_info[source_rse_id], 'third_party_copy', current_schemes)
except RSEProtocolNotSupported:
logging.error('Operation "third_party_copy" not supported by %s with schemes %s' % (rses_info[source_rse_id]['rse'], current_schemes))
continue
source_url = list(protocols[source_rse_id_key].lfns2pfns(lfns={'scope': scope, 'name': name, 'path': path}).values())[0]
if ranking is None:
ranking = 0
# TAPE should not mixed with Disk and should not use as first try
# If there is a source whose ranking is no less than the Tape ranking, Tape will not be used.
if rses_info[source_rse_id]['rse_type'] == RSEType.TAPE or rses_info[source_rse_id]['rse_type'] == 'TAPE':
# current src_rse is Tape
if not allow_tape_source:
continue
if not transfers[req_id]['bring_online']:
# the sources already founded are disks.
avail_top_ranking = None
founded_sources = transfers[req_id]['sources']
for founded_source in founded_sources:
if avail_top_ranking is None:
avail_top_ranking = founded_source[3]
continue
if founded_source[3] is not None and founded_source[3] > avail_top_ranking:
avail_top_ranking = founded_source[3]
if avail_top_ranking >= ranking:
# current Tape source is not the highest ranking, will use disk sources
continue
else:
transfers[req_id]['sources'] = []
transfers[req_id]['bring_online'] = bring_online_local
transfer_src_type = "TAPE"
transfers[req_id]['file_metadata']['src_type'] = transfer_src_type
transfers[req_id]['file_metadata']['src_rse'] = rse
else:
# the sources already founded is Tape too.
# multiple Tape source replicas are not allowed in FTS3.
if transfers[req_id]['sources'][0][3] > ranking or (transfers[req_id]['sources'][0][3] == ranking and transfers[req_id]['sources'][0][4] <= link_ranking):
continue
else:
transfers[req_id]['sources'] = []
transfers[req_id]['bring_online'] = bring_online_local
transfers[req_id]['file_metadata']['src_rse'] = rse
else:
# current src_rse is Disk
if transfers[req_id]['bring_online']:
# the founded sources are Tape
avail_top_ranking = None
founded_sources = transfers[req_id]['sources']
for founded_source in founded_sources:
if avail_top_ranking is None:
avail_top_ranking = founded_source[3]
continue
if founded_source[3] is not None and founded_source[3] > avail_top_ranking:
avail_top_ranking = founded_source[3]
if ranking >= avail_top_ranking:
# current disk replica has higher ranking than founded sources
# remove founded Tape sources
transfers[req_id]['sources'] = []
transfers[req_id]['bring_online'] = None
transfer_src_type = "DISK"
transfers[req_id]['file_metadata']['src_type'] = transfer_src_type
transfers[req_id]['file_metadata']['src_rse'] = rse
else:
continue
# transfers[id]['src_urls'].append((source_rse_id, source_url))
transfers[req_id]['sources'].append((rse, source_url, source_rse_id, ranking, link_ranking))
except Exception:
logging.critical("Exception happened when trying to get transfer for request %s: %s" % (req_id, traceback.format_exc()))
break
for req_id in transfers:
if req_id in reqs_no_source:
reqs_no_source.remove(req_id)
if req_id in reqs_only_tape_source:
reqs_only_tape_source.remove(req_id)
if req_id in reqs_scheme_mismatch:
reqs_scheme_mismatch.remove(req_id)
return transfers, reqs_no_source, reqs_scheme_mismatch, reqs_only_tape_source
def get_destinations(rse_info, scheme, req, sources):
dsn = 'other'
pfn = {}
paths = {}
if not rse_info['deterministic']:
ts = time.time()
# get rule scope and name
if req['attributes']:
if type(req['attributes']) is dict:
req_attributes = json.loads(json.dumps(req['attributes']))
else:
req_attributes = json.loads(str(req['attributes']))
if 'ds_name' in req_attributes:
dsn = req_attributes["ds_name"]
if dsn == 'other':
# select a containing dataset
for parent in did.list_parent_dids(req['scope'], req['name']):
if parent['type'] == DIDType.DATASET:
dsn = parent['name']
break
record_timer('daemons.conveyor.submitter.list_parent_dids', (time.time() - ts) * 1000)
# always use SRM
ts = time.time()
nondet = rsemgr.create_protocol(rse_info, 'write', scheme='srm')
record_timer('daemons.conveyor.submitter.create_protocol', (time.time() - ts) * 1000)
# if there exists a prefix for SRM, use it
prefix = ''
for s in rse_info['protocols']:
if s['scheme'] == 'srm':
prefix = s['prefix']
# DQ2 path always starts with /, but prefix might not end with /
path = construct_surl_DQ2(dsn, req['name'])
# retrial transfers to tape need a new filename - add timestamp
if req['request_type'] == RequestType.TRANSFER\
and 'previous_attempt_id' in req\
and req['previous_attempt_id']\
and rse_info['rse_type'] == 'TAPE': # TODO: RUCIO-809 - rsemanager: get_rse_info -> rse_type is string instead of RSEType
path = '%s_%i' % (path, int(time.time()))
logging.debug('Retrial transfer request %s DID %s:%s to tape %s renamed to %s' % (req['request_id'],
req['scope'],
req['name'],
rse_info['rse'],
path))
tmp_path = '%s%s' % (prefix[:-1], path)
if prefix[-1] != '/':
tmp_path = '%s%s' % (prefix, path)
paths[req['scope'], req['name']] = path
# add the hostname
pfn['%s:%s' % (req['scope'], req['name'])] = nondet.path2pfn(tmp_path)
if req['request_type'] == RequestType.STAGEIN:
if len(sources) == 1:
pfn['%s:%s' % (req['scope'], req['name'])] = sources[0][1]
else:
# TODO: need to check
return None, None
# we must set the destination path for nondeterministic replicas explicitly
replica.update_replicas_paths([{'scope': req['scope'],
'name': req['name'],
'rse_id': req['dest_rse_id'],
'path': path}])
else:
ts = time.time()
try:
pfn = rsemgr.lfns2pfns(rse_info,
lfns=[{'scope': req['scope'],
'name': req['name']}],
scheme=scheme)
except RSEProtocolNotSupported:
logging.warn('%s not supported by %s' % (scheme, rse_info['rse']))
return None, None
record_timer('daemons.conveyor.submitter.lfns2pfns', (time.time() - ts) * 1000)
destinations = []
for k in pfn:
if isinstance(pfn[k], (str, unicode)):
destinations.append(pfn[k])
elif isinstance(pfn[k], (tuple, list)):
for url in pfn[k]:
destinations.append(pfn[k][url])
protocols = None
try:
protocols = rsemgr.select_protocol(rse_info, 'write', scheme=scheme)
except RSEProtocolNotSupported:
logging.warn('%s not supported by %s' % (scheme, rse_info['rse']))
return None, None
# we need to set the spacetoken if we use SRM
dest_spacetoken = None
if scheme == 'srm':
dest_spacetoken = protocols['extended_attributes']['space_token']
return destinations, dest_spacetoken
