def on_message(self, frame): record_counter('daemons.conveyor.receiver.message_all') msg = json.loads(frame.body) if not self.__all_vos: if 'vo' not in msg or msg['vo'] != get_policy(): return if 'job_metadata' in msg.keys() \ and isinstance(msg['job_metadata'], dict) \ and 'issuer' in msg['job_metadata'].keys() \ and str(msg['job_metadata']['issuer']) == str('rucio'): if 'job_state' in msg.keys() and str( msg['job_state']) != str('ACTIVE'): record_counter('daemons.conveyor.receiver.message_rucio') self._perform_request_update(msg)
class Receiver(object): def __init__(self, broker, id, total_threads, full_mode=False): self.__broker = broker self.__id = id self.__total_threads = total_threads self.__full_mode = full_mode def on_error(self, headers, message): record_counter('daemons.conveyor.receiver.error') logging.error('[%s] %s' % (self.__broker, message)) def on_message(self, headers, message): record_counter('daemons.conveyor.receiver.message_all') try: msg = json.loads(message) except Exception, e: msg = json.loads( message[:-1] ) # Note: I am not sure if this is needed anymore, this was due to an unparsable EOT character if 'vo' not in msg or msg['vo'] != get_policy(): return if 'job_metadata' in msg.keys() \ and isinstance(msg['job_metadata'], dict) \ and 'issuer' in msg['job_metadata'].keys() \ and str(msg['job_metadata']['issuer']) == str('rucio'): if 'job_m_replica' in msg.keys() and 'job_state' in msg.keys() \ and (str(msg['job_m_replica']).lower() == str('false') or (str(msg['job_m_replica']).lower() == str('true') and str(msg['job_state']) != str('ACTIVE'))): if 'request_id' in msg['job_metadata']: # submitted by old submitter response = { 'new_state': None, 'transfer_id': msg.get('tr_id').split("__")[-1], 'job_state': msg.get('t_final_transfer_state', None), 'src_url': msg.get('src_url', None), 'dst_url': msg.get('dst_url', None), 'transferred_at': datetime.datetime.utcfromtimestamp( float(msg.get('tr_timestamp_complete', 0)) / 1000), 'duration': (float(msg.get('tr_timestamp_complete', 0)) - float(msg.get('tr_timestamp_start', 0))) / 1000, 'reason': msg.get('t__error_message', None), 'scope': msg['job_metadata'].get('scope', None), 'name': msg['job_metadata'].get('name', None), 'src_rse': msg['job_metadata'].get('src_rse', None), 'dst_rse': msg['job_metadata'].get('dst_rse', None), 'request_id': msg['job_metadata'].get('request_id', None), 'activity': msg['job_metadata'].get('activity', None), 'dest_rse_id': msg['job_metadata'].get('dest_rse_id', None), 'previous_attempt_id': msg['job_metadata'].get('previous_attempt_id', None), 'adler32': msg['job_metadata'].get('adler32', None), 'md5': msg['job_metadata'].get('md5', None), 'filesize': msg['job_metadata'].get('filesize', None), 'external_host': msg.get('endpnt', None), 'job_m_replica': msg.get('job_m_replica', None), 'details': { 'files': msg['job_metadata'] } } else: # for new submitter, file_metadata replace the job_metadata response = { 'new_state': None, 'transfer_id': msg.get('tr_id').split("__")[-1], 'job_state': msg.get('t_final_transfer_state', None), 'src_url': msg.get('src_url', None), 'dst_url': msg.get('dst_url', None), 'started_at': datetime.datetime.utcfromtimestamp( float(msg.get('tr_timestamp_start', 0)) / 1000), 'transferred_at': datetime.datetime.utcfromtimestamp( float(msg.get('tr_timestamp_complete', 0)) / 1000), 'duration': (float(msg.get('tr_timestamp_complete', 0)) - float(msg.get('tr_timestamp_start', 0))) / 1000, 'reason': msg.get('t__error_message', None), 'scope': msg['file_metadata'].get('scope', None), 'name': msg['file_metadata'].get('name', None), 'src_type': msg['file_metadata'].get('src_type', None), 'dst_type': msg['file_metadata'].get('dst_type', None), 'src_rse': msg['file_metadata'].get('src_rse', None), 'dst_rse': msg['file_metadata'].get('dst_rse', None), 'request_id': msg['file_metadata'].get('request_id', None), 'activity': msg['file_metadata'].get('activity', None), 'src_rse_id': msg['file_metadata'].get('src_rse_id', None), 'dest_rse_id': msg['file_metadata'].get('dest_rse_id', None), 'previous_attempt_id': msg['file_metadata'].get('previous_attempt_id', None), 'adler32': msg['file_metadata'].get('adler32', None), 'md5': msg['file_metadata'].get('md5', None), 'filesize': msg['file_metadata'].get('filesize', None), 'external_host': msg.get('endpnt', None), 'job_m_replica': msg.get('job_m_replica', None), 'details': { 'files': msg['file_metadata'] } } record_counter('daemons.conveyor.receiver.message_rucio') if str(msg['t_final_transfer_state']) == str( FTSCompleteState.OK): response['new_state'] = RequestState.DONE elif str(msg['t_final_transfer_state']) == str( FTSCompleteState.ERROR): response['new_state'] = RequestState.FAILED try: if response['new_state']: logging.info( 'RECEIVED DID %s:%s FROM %s TO %s REQUEST %s TRANSFER_ID %s STATE %s' % (response['scope'], response['name'], response['src_rse'], response['dst_rse'], response['request_id'], response['transfer_id'], response['new_state'])) if self.__full_mode: ret = request.update_request_state(response) record_counter( 'daemons.conveyor.receiver.update_request_state.%s' % ret) else: try: logging.debug("Update request %s update time" % response['request_id']) set_transfer_update_time( response['external_host'], response['transfer_id'], datetime.datetime.utcnow() - datetime.timedelta(hours=24)) record_counter( 'daemons.conveyor.receiver.set_transfer_update_time' ) except Exception, e: logging.debug( "Failed to update transfer's update time: %s" % str(e)) except: logging.critical(traceback.format_exc())
def test_atlas_archival_policy(self): """ UNDERTAKER (CORE): Test the atlas archival policy. """ if get_policy() != 'atlas': LOG.info("Skipping atlas-specific test") return tmp_scope = InternalScope('mock', **self.vo) jdoe = InternalAccount('jdoe', **self.vo) root = InternalAccount('root', **self.vo) nbdatasets = 5 nbfiles = 5 rse = 'LOCALGROUPDISK_%s' % rse_name_generator() rse_id = add_rse(rse, **self.vo) set_local_account_limit(jdoe, rse_id, -1) dsns2 = [{ 'name': 'dsn_%s' % generate_uuid(), 'scope': tmp_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{ 'account': jdoe, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET' }] } for i in range(nbdatasets)] add_dids(dids=dsns2, account=root) replicas = list() for dsn in dsns2: files = [{ 'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': { 'events': 10 } } for i in range(nbfiles)] attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account=root) replicas += files undertaker(worker_number=1, total_workers=1, once=True) for replica in replicas: assert (get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'] is None) for dsn in dsns2: assert (get_did(scope=InternalScope('archive', **self.vo), name=dsn['name'])['name'] == dsn['name']) assert (len([ x for x in list_rules( filters={ 'scope': InternalScope('archive', **self.vo), 'name': dsn['name'] }) ]) == 1)
def on_message(self, frame): record_counter('daemons.conveyor.receiver.message_all') msg = json.loads(frame.body) if not self.__all_vos: if 'vo' not in msg or msg['vo'] != get_policy(): return if 'job_metadata' in msg.keys() \ and isinstance(msg['job_metadata'], dict) \ and 'issuer' in msg['job_metadata'].keys() \ and str(msg['job_metadata']['issuer']) == str('rucio'): if 'job_state' in msg.keys() and ( str(msg['job_state']) != str('ACTIVE') or str(msg['job_state']) == str('ACTIVE') and 'job_m_replica' in msg.keys() and (str(msg['job_m_replica']).lower() == str('true'))): response = { 'new_state': None, 'transfer_id': msg.get('tr_id').split("__")[-1], 'job_state': msg.get('t_final_transfer_state', None), 'src_url': msg.get('src_url', None), 'dst_url': msg.get('dst_url', None), 'started_at': datetime.datetime.utcfromtimestamp( float(msg.get('tr_timestamp_start', 0)) / 1000), 'transferred_at': datetime.datetime.utcfromtimestamp( float(msg.get('tr_timestamp_complete', 0)) / 1000), 'duration': (float(msg.get('tr_timestamp_complete', 0)) - float(msg.get('tr_timestamp_start', 0))) / 1000, 'reason': msg.get('t__error_message', None), 'scope': msg['file_metadata'].get('scope', None), 'name': msg['file_metadata'].get('name', None), 'src_type': msg['file_metadata'].get('src_type', None), 'dst_type': msg['file_metadata'].get('dst_type', None), 'src_rse': msg['file_metadata'].get('src_rse', None), 'dst_rse': msg['file_metadata'].get('dst_rse', None), 'request_id': msg['file_metadata'].get('request_id', None), 'activity': msg['file_metadata'].get('activity', None), 'src_rse_id': msg['file_metadata'].get('src_rse_id', None), 'dest_rse_id': msg['file_metadata'].get('dest_rse_id', None), 'previous_attempt_id': msg['file_metadata'].get('previous_attempt_id', None), 'adler32': msg['file_metadata'].get('adler32', None), 'md5': msg['file_metadata'].get('md5', None), 'filesize': msg['file_metadata'].get('filesize', None), 'external_host': msg.get('endpnt', None), 'job_m_replica': msg.get('job_m_replica', None), 'details': { 'files': msg['file_metadata'] } } record_counter('daemons.conveyor.receiver.message_rucio') if str(msg['t_final_transfer_state']) == FTS_COMPLETE_STATE.OK: # pylint:disable=no-member response['new_state'] = RequestState.DONE elif str(msg['t_final_transfer_state'] ) == FTS_COMPLETE_STATE.ERROR: # pylint:disable=no-member response['new_state'] = RequestState.FAILED try: if response['new_state']: logging.info( 'RECEIVED DID %s:%s FROM %s TO %s REQUEST %s TRANSFER_ID %s STATE %s' % (response['scope'], response['name'], response['src_rse'], response['dst_rse'], response['request_id'], response['transfer_id'], response['new_state'])) if self.__full_mode: ret = request.update_request_state(response) record_counter( 'daemons.conveyor.receiver.update_request_state.%s' % ret) else: try: logging.debug("Update request %s update time" % response['request_id']) set_transfer_update_time( response['external_host'], response['transfer_id'], datetime.datetime.utcnow() - datetime.timedelta(hours=24)) record_counter( 'daemons.conveyor.receiver.set_transfer_update_time' ) except Exception as error: logging.debug( "Failed to update transfer's update time: %s" % str(error)) except Exception: logging.critical(traceback.format_exc())