def test_request_submitted_in_order(rse_factory, did_factory, root_account):
src_rses = [rse_factory.make_posix_rse() for _ in range(2)]
dst_rses = [rse_factory.make_posix_rse() for _ in range(3)]
for _, src_rse_id in src_rses:
for _, dst_rse_id in dst_rses:
distance_core.add_distance(src_rse_id=src_rse_id, dest_rse_id=dst_rse_id, ranking=10)
distance_core.add_distance(src_rse_id=dst_rse_id, dest_rse_id=src_rse_id, ranking=10)
# Create a certain number of files on source RSEs with replication rules towards random destination RSEs
nb_files = 15
dids = []
requests = []
src_rses_iterator = itertools.cycle(src_rses)
dst_rses_iterator = itertools.cycle(dst_rses)
for _ in range(nb_files):
src_rse_name, src_rse_id = next(src_rses_iterator)
dst_rse_name, dst_rse_id = next(dst_rses_iterator)
did = did_factory.upload_test_file(rse_name=src_rse_name)
rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
requests.append(request_core.get_request_by_did(rse_id=dst_rse_id, **did))
dids.append(did)
# Forge request creation time to a random moment in the past hour
@transactional_session
def _forge_requests_creation_time(session=None):
base_time = datetime.utcnow().replace(microsecond=0, minute=0) - timedelta(hours=1)
assigned_times = set()
for request in requests:
request_creation_time = None
while not request_creation_time or request_creation_time in assigned_times:
# Ensure uniqueness to avoid multiple valid submission orders and make tests deterministic with simple sorting techniques
request_creation_time = base_time + timedelta(minutes=randint(0, 3600))
assigned_times.add(request_creation_time)
session.query(Request).filter(Request.id == request['id']).update({'created_at': request_creation_time})
request['created_at'] = request_creation_time
_forge_requests_creation_time()
requests = sorted(requests, key=lambda r: r['created_at'])
for request in requests:
assert request_core.get_request(request_id=request['id'])['state'] == RequestState.QUEUED
requests_id_in_submission_order = []
with patch('rucio.transfertool.mock.MockTransfertool.submit') as mock_transfertool_submit:
# Record the order of requests passed to MockTranfertool.submit()
mock_transfertool_submit.side_effect = lambda jobs, _: requests_id_in_submission_order.extend([j['metadata']['request_id'] for j in jobs])
submitter(once=True, rses=[{'id': rse_id} for _, rse_id in dst_rses], partition_wait_time=None, transfertool='mock', transfertype='single', filter_transfertool=None)
for request in requests:
assert request_core.get_request(request_id=request['id'])['state'] == RequestState.SUBMITTED
# Requests must be submitted in the order of their creation
assert requests_id_in_submission_order == [r['id'] for r in requests]
def request(self, session):
"""
Fetch the request by request_id if needed.
"""
if not self.__request:
self.__request = get_request(self.request_id, session=session)
return self.__request
def update_request_state(response, session=None):
"""
Used by poller and consumer to update the internal state of requests,
after the response by the external transfertool.
:param response: The transfertool response dictionary, retrieved via request.query_request().
:param session: The database session to use.
:returns commit_or_rollback: Boolean.
"""
try:
if not response['new_state']:
request_core.touch_request(response['request_id'], session=session)
return False
else:
request = request_core.get_request(response['request_id'], session=session)
if request and request['external_id'] == response['transfer_id'] and request['state'] != response['new_state']:
response['submitted_at'] = request.get('submitted_at', None)
response['external_host'] = request['external_host']
transfer_id = response['transfer_id'] if 'transfer_id' in response else None
logging.info('UPDATING REQUEST %s FOR TRANSFER %s STATE %s' % (str(response['request_id']), transfer_id, str(response['new_state'])))
job_m_replica = response.get('job_m_replica', None)
src_url = response.get('src_url', None)
src_rse = response.get('src_rse', None)
src_rse_id = response.get('src_rse_id', None)
started_at = response.get('started_at', None)
transferred_at = response.get('transferred_at', None)
scope = response.get('scope', None)
name = response.get('name', None)
if job_m_replica and (str(job_m_replica).lower() == str('true')) and src_url:
try:
src_rse_name, src_rse_id = get_source_rse(response['request_id'], scope, name, src_url, session=session)
except:
logging.warn('Cannot get correct RSE for source url: %s(%s)' % (src_url, traceback.format_exc()))
src_rse_name = None
if src_rse_name and src_rse_name != src_rse:
response['src_rse'] = src_rse_name
response['src_rse_id'] = src_rse_id
logging.debug('Correct RSE: %s for source surl: %s' % (src_rse_name, src_url))
err_msg = get_transfer_error(response['new_state'], response['reason'] if 'reason' in response else None)
request_core.set_request_state(response['request_id'],
response['new_state'],
transfer_id=transfer_id,
started_at=started_at,
transferred_at=transferred_at,
src_rse_id=src_rse_id,
err_msg=err_msg,
session=session)
add_monitor_message(request, response, session=session)
return True
elif not request:
logging.debug("Request %s doesn't exist, will not update" % (response['request_id']))
return False
elif request['external_id'] != response['transfer_id']:
logging.warning("Reponse %s with transfer id %s is different from the request transfer id %s, will not update" % (response['request_id'], response['transfer_id'], request['external_id']))
return False
else:
logging.debug("Request %s is already in %s state, will not update" % (response['request_id'], response['new_state']))
return False
except UnsupportedOperation as error:
logging.warning("Request %s doesn't exist - Error: %s" % (response['request_id'], str(error).replace('\n', '')))
return False
except:
logging.critical(traceback.format_exc())
elif response['new_state'] and 'job_state' in response and response['job_state']:
logging.debug('UPDATING REQUEST %s FOR TRANSFER(%s) STATE %s' % (str(response['request_id']), str(response['transfer_id']), str(response['job_state'])))
else:
return False
if response['new_state']:
request.set_request_state(response['request_id'],
response['new_state'],
session=session)
if response['new_state'] == RequestState.DONE:
rse_name = response['dst_rse']
rse_update_name = rse_name
req = request.get_request(response['request_id'], session=session)
if req['request_type'] == RequestType.STAGEIN:
rse_update_name = rse_core.list_rse_attributes(response['dst_rse'], session=session)['staging_buffer']
logging.debug('OVERRIDE REPLICA DID %s:%s RSE %s TO %s' % (response['scope'], response['name'], response['dst_rse'], rse_update_name))
try:
tss = time.time()
logging.debug('UPDATE REPLICA STATE DID %s:%s RSE %s' % (response['scope'], response['name'], rse_update_name))
# make sure we do not leave the transaction
try:
# try quickly
replica.update_replicas_states([{'rse': rse_update_name,
'scope': response['scope'],
'name': response['name'],
'state': ReplicaState.AVAILABLE}],