def test_request_submitted_in_order(rse_factory, did_factory, root_account): src_rses = [rse_factory.make_posix_rse() for _ in range(2)] dst_rses = [rse_factory.make_posix_rse() for _ in range(3)] for _, src_rse_id in src_rses: for _, dst_rse_id in dst_rses: distance_core.add_distance(src_rse_id=src_rse_id, dest_rse_id=dst_rse_id, ranking=10) distance_core.add_distance(src_rse_id=dst_rse_id, dest_rse_id=src_rse_id, ranking=10) # Create a certain number of files on source RSEs with replication rules towards random destination RSEs nb_files = 15 dids = [] requests = [] src_rses_iterator = itertools.cycle(src_rses) dst_rses_iterator = itertools.cycle(dst_rses) for _ in range(nb_files): src_rse_name, src_rse_id = next(src_rses_iterator) dst_rse_name, dst_rse_id = next(dst_rses_iterator) did = did_factory.upload_test_file(rse_name=src_rse_name) rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse_name, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) requests.append(request_core.get_request_by_did(rse_id=dst_rse_id, **did)) dids.append(did) # Forge request creation time to a random moment in the past hour @transactional_session def _forge_requests_creation_time(session=None): base_time = datetime.utcnow().replace(microsecond=0, minute=0) - timedelta(hours=1) assigned_times = set() for request in requests: request_creation_time = None while not request_creation_time or request_creation_time in assigned_times: # Ensure uniqueness to avoid multiple valid submission orders and make tests deterministic with simple sorting techniques request_creation_time = base_time + timedelta(minutes=randint(0, 3600)) assigned_times.add(request_creation_time) session.query(Request).filter(Request.id == request['id']).update({'created_at': request_creation_time}) request['created_at'] = request_creation_time _forge_requests_creation_time() requests = sorted(requests, key=lambda r: r['created_at']) for request in requests: assert request_core.get_request(request_id=request['id'])['state'] == RequestState.QUEUED requests_id_in_submission_order = [] with patch('rucio.transfertool.mock.MockTransfertool.submit') as mock_transfertool_submit: # Record the order of requests passed to MockTranfertool.submit() mock_transfertool_submit.side_effect = lambda jobs, _: requests_id_in_submission_order.extend([j['metadata']['request_id'] for j in jobs]) submitter(once=True, rses=[{'id': rse_id} for _, rse_id in dst_rses], partition_wait_time=None, transfertool='mock', transfertype='single', filter_transfertool=None) for request in requests: assert request_core.get_request(request_id=request['id'])['state'] == RequestState.SUBMITTED # Requests must be submitted in the order of their creation assert requests_id_in_submission_order == [r['id'] for r in requests]
def request(self, session): """ Fetch the request by request_id if needed. """ if not self.__request: self.__request = get_request(self.request_id, session=session) return self.__request
def update_request_state(response, session=None): """ Used by poller and consumer to update the internal state of requests, after the response by the external transfertool. :param response: The transfertool response dictionary, retrieved via request.query_request(). :param session: The database session to use. :returns commit_or_rollback: Boolean. """ try: if not response['new_state']: request_core.touch_request(response['request_id'], session=session) return False else: request = request_core.get_request(response['request_id'], session=session) if request and request['external_id'] == response['transfer_id'] and request['state'] != response['new_state']: response['submitted_at'] = request.get('submitted_at', None) response['external_host'] = request['external_host'] transfer_id = response['transfer_id'] if 'transfer_id' in response else None logging.info('UPDATING REQUEST %s FOR TRANSFER %s STATE %s' % (str(response['request_id']), transfer_id, str(response['new_state']))) job_m_replica = response.get('job_m_replica', None) src_url = response.get('src_url', None) src_rse = response.get('src_rse', None) src_rse_id = response.get('src_rse_id', None) started_at = response.get('started_at', None) transferred_at = response.get('transferred_at', None) scope = response.get('scope', None) name = response.get('name', None) if job_m_replica and (str(job_m_replica).lower() == str('true')) and src_url: try: src_rse_name, src_rse_id = get_source_rse(response['request_id'], scope, name, src_url, session=session) except: logging.warn('Cannot get correct RSE for source url: %s(%s)' % (src_url, traceback.format_exc())) src_rse_name = None if src_rse_name and src_rse_name != src_rse: response['src_rse'] = src_rse_name response['src_rse_id'] = src_rse_id logging.debug('Correct RSE: %s for source surl: %s' % (src_rse_name, src_url)) err_msg = get_transfer_error(response['new_state'], response['reason'] if 'reason' in response else None) request_core.set_request_state(response['request_id'], response['new_state'], transfer_id=transfer_id, started_at=started_at, transferred_at=transferred_at, src_rse_id=src_rse_id, err_msg=err_msg, session=session) add_monitor_message(request, response, session=session) return True elif not request: logging.debug("Request %s doesn't exist, will not update" % (response['request_id'])) return False elif request['external_id'] != response['transfer_id']: logging.warning("Reponse %s with transfer id %s is different from the request transfer id %s, will not update" % (response['request_id'], response['transfer_id'], request['external_id'])) return False else: logging.debug("Request %s is already in %s state, will not update" % (response['request_id'], response['new_state'])) return False except UnsupportedOperation as error: logging.warning("Request %s doesn't exist - Error: %s" % (response['request_id'], str(error).replace('\n', ''))) return False except: logging.critical(traceback.format_exc())
elif response['new_state'] and 'job_state' in response and response['job_state']: logging.debug('UPDATING REQUEST %s FOR TRANSFER(%s) STATE %s' % (str(response['request_id']), str(response['transfer_id']), str(response['job_state']))) else: return False if response['new_state']: request.set_request_state(response['request_id'], response['new_state'], session=session) if response['new_state'] == RequestState.DONE: rse_name = response['dst_rse'] rse_update_name = rse_name req = request.get_request(response['request_id'], session=session) if req['request_type'] == RequestType.STAGEIN: rse_update_name = rse_core.list_rse_attributes(response['dst_rse'], session=session)['staging_buffer'] logging.debug('OVERRIDE REPLICA DID %s:%s RSE %s TO %s' % (response['scope'], response['name'], response['dst_rse'], rse_update_name)) try: tss = time.time() logging.debug('UPDATE REPLICA STATE DID %s:%s RSE %s' % (response['scope'], response['name'], rse_update_name)) # make sure we do not leave the transaction try: # try quickly replica.update_replicas_states([{'rse': rse_update_name, 'scope': response['scope'], 'name': response['name'], 'state': ReplicaState.AVAILABLE}],