def get_stagein_requests_and_source_replicas(total_workers=0, worker_number=0, failover_schemes=None, limit=None, activity=None, older_than=None,
rses=None, mock=False, schemes=None, bring_online=43200, retry_other_fts=False, logging_prepend_str='', session=None):
"""
Get staging requests and the associated source replicas
:param total_workers: Number of total workers.
:param worker_number: Id of the executing worker.
:param failover_schemes: Failover schemes.
:param limit: Limit.
:param activity: Activity.
:param older_than: Get transfers older than.
:param rses: Include RSES.
:param mock: Mock testing.
:param schemes: Include schemes.
:param bring_online: Bring online timeout.
:parm retry_other_fts: Retry other fts servers.
:param logging_prepend_str: String to prepend to the logging
:session: The database session in use.
:returns: transfers, reqs_no_source, reqs_scheme_mismatch, reqs_only_tape_source
"""
prepend_str = ''
if logging_prepend_str:
prepend_str = logging_prepend_str
req_sources = request.list_stagein_requests_and_source_replicas(total_workers=total_workers,
worker_number=worker_number,
limit=limit,
activity=activity,
older_than=older_than,
rses=rses,
session=session)
transfers, rses_info, protocols, rse_attrs, reqs_no_source = {}, {}, {}, {}, []
for req_id, rule_id, scope, name, md5, adler32, bytes, activity, attributes, dest_rse_id, source_rse_id, rse, deterministic, rse_type, path, staging_buffer, retry_count, previous_attempt_id, src_url, ranking in req_sources:
try:
if rses and dest_rse_id not in rses:
continue
current_schemes = schemes
if previous_attempt_id and failover_schemes:
current_schemes = failover_schemes
if req_id not in transfers:
if req_id not in reqs_no_source:
reqs_no_source.append(req_id)
if not src_url:
# source_rse_id will be None if no source replicas
# rse will be None if rse is staging area
# staging_buffer will be None if rse has no key 'staging_buffer'
if source_rse_id is None or rse is None or staging_buffer is None:
continue
# Get destination rse information and protocol
dest_rse_name = get_rse_name(rse_id=dest_rse_id, session=session)
if dest_rse_id not in rses_info:
rses_info[dest_rse_id] = rsemgr.get_rse_info(rse_id=dest_rse_id, session=session)
if staging_buffer != dest_rse_id:
continue
attr = None
if attributes:
if isinstance(attributes, dict):
attr = json.loads(json.dumps(attributes))
else:
attr = json.loads(str(attributes))
source_replica_expression = attr["source_replica_expression"] if "source_replica_expression" in attr else None
if source_replica_expression:
try:
parsed_rses = parse_expression(source_replica_expression, filter={'vo': scope.vo}, session=session)
except InvalidRSEExpression as error:
logging.error(prepend_str + "Invalid RSE exception %s: %s" % (source_replica_expression, error))
continue
else:
allowed_rses = [x['id'] for x in parsed_rses]
if source_rse_id not in allowed_rses:
continue
source_rse_name = get_rse_name(rse_id=source_rse_id, session=session)
if source_rse_id not in rses_info:
rses_info[source_rse_id] = rsemgr.get_rse_info(rse_id=source_rse_id, session=session)
if source_rse_id not in rse_attrs:
rse_attrs[source_rse_id] = get_rse_attributes(rse_id=source_rse_id, session=session)
if source_rse_id not in protocols:
protocols[source_rse_id] = rsemgr.create_protocol(rses_info[source_rse_id], 'write', current_schemes)
# we need to set the spacetoken if we use SRM
dest_spacetoken = None
if protocols[source_rse_id].attributes and \
'extended_attributes' in protocols[source_rse_id].attributes and \
protocols[source_rse_id].attributes['extended_attributes'] and \
'space_token' in protocols[source_rse_id].attributes['extended_attributes']:
dest_spacetoken = protocols[source_rse_id].attributes['extended_attributes']['space_token']
source_url = protocols[source_rse_id].lfns2pfns(lfns={'scope': scope.external, 'name': name, 'path': path}).values()[0]
else:
# source_rse_id will be None if no source replicas
# rse will be None if rse is staging area
# staging_buffer will be None if rse has no key 'staging_buffer'
if source_rse_id is None or rse is None or staging_buffer is None:
continue
attr = None
if attributes:
if type(attributes) is dict:
attr = json.loads(json.dumps(attributes))
else:
attr = json.loads(str(attributes))
# to get space token and fts attribute
source_rse_name = get_rse_name(rse_id=source_rse_id, session=session)
if source_rse_id not in rses_info:
rses_info[source_rse_id] = rsemgr.get_rse_info(rse_id=source_rse_id, session=session)
if source_rse_id not in rse_attrs:
rse_attrs[source_rse_id] = get_rse_attributes(rse_id=source_rse_id, session=session)
if source_rse_id not in protocols:
protocols[source_rse_id] = rsemgr.create_protocol(rses_info[source_rse_id], 'write', current_schemes)
# we need to set the spacetoken if we use SRM
dest_spacetoken = None
if protocols[source_rse_id].attributes and \
'extended_attributes' in protocols[source_rse_id].attributes and \
protocols[source_rse_id].attributes['extended_attributes'] and \
'space_token' in protocols[source_rse_id].attributes['extended_attributes']:
dest_spacetoken = protocols[source_rse_id].attributes['extended_attributes']['space_token']
source_url = src_url
fts_hosts = rse_attrs[source_rse_id].get('fts', None)
if not fts_hosts:
logging.error(prepend_str + 'Source RSE %s FTS attribute not defined - SKIP REQUEST %s' % (rse, req_id))
continue
if not retry_count:
retry_count = 0
fts_list = fts_hosts.split(",")
external_host = fts_list[0]
if retry_other_fts:
external_host = fts_list[retry_count % len(fts_list)]
if req_id in reqs_no_source:
reqs_no_source.remove(req_id)
file_metadata = {'request_id': req_id,
'scope': scope,
'name': name,
'activity': activity,
'request_type': str(RequestType.STAGEIN).lower(),
'src_type': "TAPE",
'dst_type': "DISK",
'src_rse': source_rse_name,
'dst_rse': dest_rse_name,
'src_rse_id': source_rse_id,
'dest_rse_id': dest_rse_id,
'filesize': bytes,
'md5': md5,
'adler32': adler32}
if previous_attempt_id:
file_metadata['previous_attempt_id'] = previous_attempt_id
transfers[req_id] = {'request_id': req_id,
# 'src_urls': [source_url],
'sources': [(rse, source_url, source_rse_id, ranking)],
'dest_urls': [source_url],
'src_spacetoken': None,
'dest_spacetoken': dest_spacetoken,
'overwrite': False,
'bring_online': bring_online,
'copy_pin_lifetime': attr.get('lifetime', -1) if attr else -1,
'external_host': external_host,
'selection_strategy': 'auto',
'rule_id': rule_id,
'file_metadata': file_metadata}
logging.debug(prepend_str + "Transfer for request(%s): %s" % (req_id, transfers[req_id]))
except Exception:
logging.critical(prepend_str + "Exception happened when trying to get transfer for request %s: %s" % (req_id, traceback.format_exc()))
break
return transfers, reqs_no_source
def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0, limit=None, activity=None, older_than=None, rses=None, schemes=None,
bring_online=43200, retry_other_fts=False, failover_schemes=None, session=None):
"""
Get transfer requests and the associated source replicas
:param total_workers: Number of total workers.
:param worker_number: Id of the executing worker.
:param limit: Limit.
:param activity: Activity.
:param older_than: Get transfers older than.
:param rses: Include RSES.
:param schemes: Include schemes.
:param bring_online: Bring online timeout.
:parm retry_other_fts: Retry other fts servers.
:param failover_schemes: Failover schemes.
:session: The database session in use.
:returns: transfers, reqs_no_source, reqs_scheme_mismatch, reqs_only_tape_source
"""
req_sources = __list_transfer_requests_and_source_replicas(total_workers=total_workers,
worker_number=worker_number,
limit=limit,
activity=activity,
older_than=older_than,
rses=rses,
session=session)
unavailable_read_rse_ids = __get_unavailable_read_rse_ids(session=session)
bring_online_local = bring_online
transfers, rses_info, protocols, rse_attrs, reqs_no_source, reqs_only_tape_source, reqs_scheme_mismatch = {}, {}, {}, {}, [], [], []
for req_id, rule_id, scope, name, md5, adler32, bytes, activity, attributes, previous_attempt_id, dest_rse_id, source_rse_id, rse, deterministic, rse_type, path, retry_count, src_url, ranking, link_ranking in req_sources:
transfer_src_type = "DISK"
transfer_dst_type = "DISK"
allow_tape_source = True
try:
if rses and dest_rse_id not in rses:
continue
current_schemes = schemes
if previous_attempt_id and failover_schemes:
current_schemes = failover_schemes
if req_id not in transfers:
if req_id not in reqs_no_source:
reqs_no_source.append(req_id)
# source_rse_id will be None if no source replicas
# rse will be None if rse is staging area
if source_rse_id is None or rse is None:
continue
if link_ranking is None:
logging.debug("Request %s: no link from %s to %s" % (req_id, source_rse_id, dest_rse_id))
continue
if source_rse_id in unavailable_read_rse_ids:
continue
# Get destination rse information
if dest_rse_id not in rses_info:
dest_rse = get_rse_name(rse_id=dest_rse_id, session=session)
rses_info[dest_rse_id] = rsemgr.get_rse_info(dest_rse, session=session)
if dest_rse_id not in rse_attrs:
rse_attrs[dest_rse_id] = get_rse_attributes(dest_rse_id, session=session)
# Get the source rse information
if source_rse_id not in rses_info:
source_rse = get_rse_name(rse_id=source_rse_id, session=session)
rses_info[source_rse_id] = rsemgr.get_rse_info(source_rse, session=session)
if source_rse_id not in rse_attrs:
rse_attrs[source_rse_id] = get_rse_attributes(source_rse_id, session=session)
attr = None
if attributes:
if type(attributes) is dict:
attr = json.loads(json.dumps(attributes))
else:
attr = json.loads(str(attributes))
# parse source expression
source_replica_expression = attr["source_replica_expression"] if (attr and "source_replica_expression" in attr) else None
if source_replica_expression:
try:
parsed_rses = parse_expression(source_replica_expression, session=session)
except InvalidRSEExpression as error:
logging.error("Invalid RSE exception %s: %s" % (source_replica_expression, error))
continue
else:
allowed_rses = [x['rse'] for x in parsed_rses]
if rse not in allowed_rses:
continue
# parse allow tape source expression, not finally version.
# allow_tape_source = attr["allow_tape_source"] if (attr and "allow_tape_source" in attr) else True
allow_tape_source = True
# Find matching scheme between destination and source
try:
matching_scheme = rsemgr.find_matching_scheme(rse_settings_dest=rses_info[dest_rse_id],
rse_settings_src=rses_info[source_rse_id],
operation_src='third_party_copy',
operation_dest='third_party_copy',
domain='wan',
scheme=current_schemes)
except RSEProtocolNotSupported:
logging.error('No matching schemes in %s for operation "third_party_copy" between %s and %s' % (current_schemes, rses_info[source_rse_id]['rse'], rses_info[dest_rse_id]['rse']))
if req_id in reqs_no_source:
reqs_no_source.remove(req_id)
if req_id not in reqs_scheme_mismatch:
reqs_scheme_mismatch.append(req_id)
continue
# Get destination protocol
dest_rse_id_key = '%s_%s' % (dest_rse_id, matching_scheme[0])
if dest_rse_id_key not in protocols:
try:
protocols[dest_rse_id_key] = rsemgr.create_protocol(rses_info[dest_rse_id], 'third_party_copy', matching_scheme[0])
except RSEProtocolNotSupported:
logging.error('Operation "third_party_copy" not supported by dest_rse %s with schemes %s' % (rses_info[dest_rse_id]['rse'], current_schemes))
if req_id in reqs_no_source:
reqs_no_source.remove(req_id)
if req_id not in reqs_scheme_mismatch:
reqs_scheme_mismatch.append(req_id)
continue
# get dest space token
dest_spacetoken = None
if protocols[dest_rse_id_key].attributes and \
'extended_attributes' in protocols[dest_rse_id_key].attributes and \
protocols[dest_rse_id_key].attributes['extended_attributes'] and \
'space_token' in protocols[dest_rse_id_key].attributes['extended_attributes']:
dest_spacetoken = protocols[dest_rse_id_key].attributes['extended_attributes']['space_token']
# Compute the destination url
if rses_info[dest_rse_id]['deterministic']:
dest_url = list(protocols[dest_rse_id_key].lfns2pfns(lfns={'scope': scope, 'name': name}).values())[0]
else:
# compute dest url in case of non deterministic
# naming convention, etc.
dsn = 'other'
if attr and 'ds_name' in attr:
dsn = attr["ds_name"]
else:
# select a containing dataset
for parent in did.list_parent_dids(scope, name):
if parent['type'] == DIDType.DATASET:
dsn = parent['name']
break
# DQ2 path always starts with /, but prefix might not end with /
naming_convention = rse_attrs[dest_rse_id].get('naming_convention', None)
dest_path = construct_surl(dsn, name, naming_convention)
if rses_info[dest_rse_id]['rse_type'] == RSEType.TAPE or rses_info[dest_rse_id]['rse_type'] == 'TAPE':
if retry_count or activity == 'Recovery':
dest_path = '%s_%i' % (dest_path, int(time.time()))
dest_url = list(protocols[dest_rse_id_key].lfns2pfns(lfns={'scope': scope, 'name': name, 'path': dest_path}).values())[0]
# Get source protocol
source_rse_id_key = '%s_%s' % (source_rse_id, '_'.join([matching_scheme[0], matching_scheme[1]]))
if source_rse_id_key not in protocols:
try:
protocols[source_rse_id_key] = rsemgr.create_protocol(rses_info[source_rse_id], 'third_party_copy', matching_scheme[1])
except RSEProtocolNotSupported:
logging.error('Operation "third_party_copy" not supported by source_rse %s with schemes %s' % (rses_info[source_rse_id]['rse'], matching_scheme[1]))
if req_id in reqs_no_source:
reqs_no_source.remove(req_id)
if req_id not in reqs_scheme_mismatch:
reqs_scheme_mismatch.append(req_id)
continue
source_url = list(protocols[source_rse_id_key].lfns2pfns(lfns={'scope': scope, 'name': name, 'path': path}).values())[0]
# Extend the metadata dictionary with request attributes
overwrite, bring_online = True, None
if rses_info[source_rse_id]['rse_type'] == RSEType.TAPE or rses_info[source_rse_id]['rse_type'] == 'TAPE':
bring_online = bring_online_local
transfer_src_type = "TAPE"
if not allow_tape_source:
if req_id not in reqs_only_tape_source:
reqs_only_tape_source.append(req_id)
if req_id in reqs_no_source:
reqs_no_source.remove(req_id)
continue
if rses_info[dest_rse_id]['rse_type'] == RSEType.TAPE or rses_info[dest_rse_id]['rse_type'] == 'TAPE':
overwrite = False
transfer_dst_type = "TAPE"
# get external_host
fts_hosts = rse_attrs[dest_rse_id].get('fts', None)
if not fts_hosts:
logging.error('Destination RSE %s FTS attribute not defined - SKIP REQUEST %s' % (dest_rse, req_id))
continue
if retry_count is None:
retry_count = 0
fts_list = fts_hosts.split(",")
verify_checksum = 'both'
if not rse_attrs[dest_rse_id].get('verify_checksum', True):
if not rse_attrs[source_rse_id].get('verify_checksum', True):
verify_checksum = 'none'
else:
verify_checksum = 'source'
else:
if not rse_attrs[source_rse_id].get('verify_checksum', True):
verify_checksum = 'destination'
else:
verify_checksum = 'both'
external_host = fts_list[0]
if retry_other_fts:
external_host = fts_list[retry_count % len(fts_list)]
file_metadata = {'request_id': req_id,
'scope': scope,
'name': name,
'activity': activity,
'request_type': str(RequestType.TRANSFER).lower(),
'src_type': transfer_src_type,
'dst_type': transfer_dst_type,
'src_rse': rse,
'dst_rse': rses_info[dest_rse_id]['rse'],
'src_rse_id': source_rse_id,
'dest_rse_id': dest_rse_id,
'filesize': bytes,
'md5': md5,
'adler32': adler32,
'verify_checksum': verify_checksum}
if previous_attempt_id:
file_metadata['previous_attempt_id'] = previous_attempt_id
transfers[req_id] = {'request_id': req_id,
'schemes': __add_compatible_schemes(schemes=[matching_scheme[0]], allowed_schemes=current_schemes),
# 'src_urls': [source_url],
'sources': [(rse, source_url, source_rse_id, ranking if ranking is not None else 0, link_ranking)],
'dest_urls': [dest_url],
'src_spacetoken': None,
'dest_spacetoken': dest_spacetoken,
'overwrite': overwrite,
'bring_online': bring_online,
'copy_pin_lifetime': attr.get('lifetime', -1),
'external_host': external_host,
'selection_strategy': 'auto',
'rule_id': rule_id,
'file_metadata': file_metadata}
else:
current_schemes = transfers[req_id]['schemes']
# source_rse_id will be None if no source replicas
# rse will be None if rse is staging area
if source_rse_id is None or rse is None:
continue
if link_ranking is None:
logging.debug("Request %s: no link from %s to %s" % (req_id, source_rse_id, dest_rse_id))
continue
if source_rse_id in unavailable_read_rse_ids:
continue
attr = None
if attributes:
if type(attributes) is dict:
attr = json.loads(json.dumps(attributes))
else:
attr = json.loads(str(attributes))
# parse source expression
source_replica_expression = attr["source_replica_expression"] if (attr and "source_replica_expression" in attr) else None
if source_replica_expression:
try:
parsed_rses = parse_expression(source_replica_expression, session=session)
except InvalidRSEExpression as error:
logging.error("Invalid RSE exception %s: %s" % (source_replica_expression, error))
continue
else:
allowed_rses = [x['rse'] for x in parsed_rses]
if rse not in allowed_rses:
continue
# parse allow tape source expression, not finally version.
allow_tape_source = attr["allow_tape_source"] if (attr and "allow_tape_source" in attr) else True
# Compute the source rse information
if source_rse_id not in rses_info:
source_rse = get_rse_name(rse_id=source_rse_id, session=session)
rses_info[source_rse_id] = rsemgr.get_rse_info(source_rse, session=session)
# Get protocol
source_rse_id_key = '%s_%s' % (source_rse_id, '_'.join(current_schemes))
if source_rse_id_key not in protocols:
try:
protocols[source_rse_id_key] = rsemgr.create_protocol(rses_info[source_rse_id], 'third_party_copy', current_schemes)
except RSEProtocolNotSupported:
logging.error('Operation "third_party_copy" not supported by %s with schemes %s' % (rses_info[source_rse_id]['rse'], current_schemes))
continue
source_url = list(protocols[source_rse_id_key].lfns2pfns(lfns={'scope': scope, 'name': name, 'path': path}).values())[0]
if ranking is None:
ranking = 0
# TAPE should not mixed with Disk and should not use as first try
# If there is a source whose ranking is no less than the Tape ranking, Tape will not be used.
if rses_info[source_rse_id]['rse_type'] == RSEType.TAPE or rses_info[source_rse_id]['rse_type'] == 'TAPE':
# current src_rse is Tape
if not allow_tape_source:
continue
if not transfers[req_id]['bring_online']:
# the sources already founded are disks.
avail_top_ranking = None
founded_sources = transfers[req_id]['sources']
for founded_source in founded_sources:
if avail_top_ranking is None:
avail_top_ranking = founded_source[3]
continue
if founded_source[3] is not None and founded_source[3] > avail_top_ranking:
avail_top_ranking = founded_source[3]
if avail_top_ranking >= ranking:
# current Tape source is not the highest ranking, will use disk sources
continue
else:
transfers[req_id]['sources'] = []
transfers[req_id]['bring_online'] = bring_online_local
transfer_src_type = "TAPE"
transfers[req_id]['file_metadata']['src_type'] = transfer_src_type
transfers[req_id]['file_metadata']['src_rse'] = rse
else:
# the sources already founded is Tape too.
# multiple Tape source replicas are not allowed in FTS3.
if transfers[req_id]['sources'][0][3] > ranking or (transfers[req_id]['sources'][0][3] == ranking and transfers[req_id]['sources'][0][4] <= link_ranking):
continue
else:
transfers[req_id]['sources'] = []
transfers[req_id]['bring_online'] = bring_online_local
transfers[req_id]['file_metadata']['src_rse'] = rse
else:
# current src_rse is Disk
if transfers[req_id]['bring_online']:
# the founded sources are Tape
avail_top_ranking = None
founded_sources = transfers[req_id]['sources']
for founded_source in founded_sources:
if avail_top_ranking is None:
avail_top_ranking = founded_source[3]
continue
if founded_source[3] is not None and founded_source[3] > avail_top_ranking:
avail_top_ranking = founded_source[3]
if ranking >= avail_top_ranking:
# current disk replica has higher ranking than founded sources
# remove founded Tape sources
transfers[req_id]['sources'] = []
transfers[req_id]['bring_online'] = None
transfer_src_type = "DISK"
transfers[req_id]['file_metadata']['src_type'] = transfer_src_type
transfers[req_id]['file_metadata']['src_rse'] = rse
else:
continue
# transfers[id]['src_urls'].append((source_rse_id, source_url))
transfers[req_id]['sources'].append((rse, source_url, source_rse_id, ranking, link_ranking))
except Exception:
logging.critical("Exception happened when trying to get transfer for request %s: %s" % (req_id, traceback.format_exc()))
break
for req_id in transfers:
if req_id in reqs_no_source:
reqs_no_source.remove(req_id)
if req_id in reqs_only_tape_source:
reqs_only_tape_source.remove(req_id)
if req_id in reqs_scheme_mismatch:
reqs_scheme_mismatch.remove(req_id)
return transfers, reqs_no_source, reqs_scheme_mismatch, reqs_only_tape_source