def on_message(self, frame):
record_counter('daemons.conveyor.receiver.message_all')
msg = json.loads(frame.body)
if not self.__all_vos:
if 'vo' not in msg or msg['vo'] != get_policy():
return
if 'job_metadata' in msg.keys() \
and isinstance(msg['job_metadata'], dict) \
and 'issuer' in msg['job_metadata'].keys() \
and str(msg['job_metadata']['issuer']) == str('rucio'):
if 'job_state' in msg.keys() and str(
msg['job_state']) != str('ACTIVE'):
record_counter('daemons.conveyor.receiver.message_rucio')
self._perform_request_update(msg)
class Receiver(object):
def __init__(self, broker, id, total_threads, full_mode=False):
self.__broker = broker
self.__id = id
self.__total_threads = total_threads
self.__full_mode = full_mode
def on_error(self, headers, message):
record_counter('daemons.conveyor.receiver.error')
logging.error('[%s] %s' % (self.__broker, message))
def on_message(self, headers, message):
record_counter('daemons.conveyor.receiver.message_all')
try:
msg = json.loads(message)
except Exception, e:
msg = json.loads(
message[:-1]
) # Note: I am not sure if this is needed anymore, this was due to an unparsable EOT character
if 'vo' not in msg or msg['vo'] != get_policy():
return
if 'job_metadata' in msg.keys() \
and isinstance(msg['job_metadata'], dict) \
and 'issuer' in msg['job_metadata'].keys() \
and str(msg['job_metadata']['issuer']) == str('rucio'):
if 'job_m_replica' in msg.keys() and 'job_state' in msg.keys() \
and (str(msg['job_m_replica']).lower() == str('false') or (str(msg['job_m_replica']).lower() == str('true') and str(msg['job_state']) != str('ACTIVE'))):
if 'request_id' in msg['job_metadata']:
# submitted by old submitter
response = {
'new_state':
None,
'transfer_id':
msg.get('tr_id').split("__")[-1],
'job_state':
msg.get('t_final_transfer_state', None),
'src_url':
msg.get('src_url', None),
'dst_url':
msg.get('dst_url', None),
'transferred_at':
datetime.datetime.utcfromtimestamp(
float(msg.get('tr_timestamp_complete', 0)) / 1000),
'duration':
(float(msg.get('tr_timestamp_complete', 0)) -
float(msg.get('tr_timestamp_start', 0))) / 1000,
'reason':
msg.get('t__error_message', None),
'scope':
msg['job_metadata'].get('scope', None),
'name':
msg['job_metadata'].get('name', None),
'src_rse':
msg['job_metadata'].get('src_rse', None),
'dst_rse':
msg['job_metadata'].get('dst_rse', None),
'request_id':
msg['job_metadata'].get('request_id', None),
'activity':
msg['job_metadata'].get('activity', None),
'dest_rse_id':
msg['job_metadata'].get('dest_rse_id', None),
'previous_attempt_id':
msg['job_metadata'].get('previous_attempt_id', None),
'adler32':
msg['job_metadata'].get('adler32', None),
'md5':
msg['job_metadata'].get('md5', None),
'filesize':
msg['job_metadata'].get('filesize', None),
'external_host':
msg.get('endpnt', None),
'job_m_replica':
msg.get('job_m_replica', None),
'details': {
'files': msg['job_metadata']
}
}
else:
# for new submitter, file_metadata replace the job_metadata
response = {
'new_state':
None,
'transfer_id':
msg.get('tr_id').split("__")[-1],
'job_state':
msg.get('t_final_transfer_state', None),
'src_url':
msg.get('src_url', None),
'dst_url':
msg.get('dst_url', None),
'started_at':
datetime.datetime.utcfromtimestamp(
float(msg.get('tr_timestamp_start', 0)) / 1000),
'transferred_at':
datetime.datetime.utcfromtimestamp(
float(msg.get('tr_timestamp_complete', 0)) / 1000),
'duration':
(float(msg.get('tr_timestamp_complete', 0)) -
float(msg.get('tr_timestamp_start', 0))) / 1000,
'reason':
msg.get('t__error_message', None),
'scope':
msg['file_metadata'].get('scope', None),
'name':
msg['file_metadata'].get('name', None),
'src_type':
msg['file_metadata'].get('src_type', None),
'dst_type':
msg['file_metadata'].get('dst_type', None),
'src_rse':
msg['file_metadata'].get('src_rse', None),
'dst_rse':
msg['file_metadata'].get('dst_rse', None),
'request_id':
msg['file_metadata'].get('request_id', None),
'activity':
msg['file_metadata'].get('activity', None),
'src_rse_id':
msg['file_metadata'].get('src_rse_id', None),
'dest_rse_id':
msg['file_metadata'].get('dest_rse_id', None),
'previous_attempt_id':
msg['file_metadata'].get('previous_attempt_id', None),
'adler32':
msg['file_metadata'].get('adler32', None),
'md5':
msg['file_metadata'].get('md5', None),
'filesize':
msg['file_metadata'].get('filesize', None),
'external_host':
msg.get('endpnt', None),
'job_m_replica':
msg.get('job_m_replica', None),
'details': {
'files': msg['file_metadata']
}
}
record_counter('daemons.conveyor.receiver.message_rucio')
if str(msg['t_final_transfer_state']) == str(
FTSCompleteState.OK):
response['new_state'] = RequestState.DONE
elif str(msg['t_final_transfer_state']) == str(
FTSCompleteState.ERROR):
response['new_state'] = RequestState.FAILED
try:
if response['new_state']:
logging.info(
'RECEIVED DID %s:%s FROM %s TO %s REQUEST %s TRANSFER_ID %s STATE %s'
% (response['scope'], response['name'],
response['src_rse'], response['dst_rse'],
response['request_id'], response['transfer_id'],
response['new_state']))
if self.__full_mode:
ret = request.update_request_state(response)
record_counter(
'daemons.conveyor.receiver.update_request_state.%s'
% ret)
else:
try:
logging.debug("Update request %s update time" %
response['request_id'])
set_transfer_update_time(
response['external_host'],
response['transfer_id'],
datetime.datetime.utcnow() -
datetime.timedelta(hours=24))
record_counter(
'daemons.conveyor.receiver.set_transfer_update_time'
)
except Exception, e:
logging.debug(
"Failed to update transfer's update time: %s"
% str(e))
except:
logging.critical(traceback.format_exc())
def test_atlas_archival_policy(self):
""" UNDERTAKER (CORE): Test the atlas archival policy. """
if get_policy() != 'atlas':
LOG.info("Skipping atlas-specific test")
return
tmp_scope = InternalScope('mock', **self.vo)
jdoe = InternalAccount('jdoe', **self.vo)
root = InternalAccount('root', **self.vo)
nbdatasets = 5
nbfiles = 5
rse = 'LOCALGROUPDISK_%s' % rse_name_generator()
rse_id = add_rse(rse, **self.vo)
set_local_account_limit(jdoe, rse_id, -1)
dsns2 = [{
'name':
'dsn_%s' % generate_uuid(),
'scope':
tmp_scope,
'type':
'DATASET',
'lifetime':
-1,
'rules': [{
'account': jdoe,
'copies': 1,
'rse_expression': rse,
'grouping': 'DATASET'
}]
} for i in range(nbdatasets)]
add_dids(dids=dsns2, account=root)
replicas = list()
for dsn in dsns2:
files = [{
'scope': tmp_scope,
'name': 'file_%s' % generate_uuid(),
'bytes': 1,
'adler32': '0cc737eb',
'tombstone': datetime.utcnow() + timedelta(weeks=2),
'meta': {
'events': 10
}
} for i in range(nbfiles)]
attach_dids(scope=tmp_scope,
name=dsn['name'],
rse_id=rse_id,
dids=files,
account=root)
replicas += files
undertaker(worker_number=1, total_workers=1, once=True)
for replica in replicas:
assert (get_replica(scope=replica['scope'],
name=replica['name'],
rse_id=rse_id)['tombstone'] is None)
for dsn in dsns2:
assert (get_did(scope=InternalScope('archive', **self.vo),
name=dsn['name'])['name'] == dsn['name'])
assert (len([
x for x in list_rules(
filters={
'scope': InternalScope('archive', **self.vo),
'name': dsn['name']
})
]) == 1)
def on_message(self, frame):
record_counter('daemons.conveyor.receiver.message_all')
msg = json.loads(frame.body)
if not self.__all_vos:
if 'vo' not in msg or msg['vo'] != get_policy():
return
if 'job_metadata' in msg.keys() \
and isinstance(msg['job_metadata'], dict) \
and 'issuer' in msg['job_metadata'].keys() \
and str(msg['job_metadata']['issuer']) == str('rucio'):
if 'job_state' in msg.keys() and (
str(msg['job_state']) != str('ACTIVE')
or str(msg['job_state']) == str('ACTIVE')
and 'job_m_replica' in msg.keys() and
(str(msg['job_m_replica']).lower() == str('true'))):
response = {
'new_state':
None,
'transfer_id':
msg.get('tr_id').split("__")[-1],
'job_state':
msg.get('t_final_transfer_state', None),
'src_url':
msg.get('src_url', None),
'dst_url':
msg.get('dst_url', None),
'started_at':
datetime.datetime.utcfromtimestamp(
float(msg.get('tr_timestamp_start', 0)) / 1000),
'transferred_at':
datetime.datetime.utcfromtimestamp(
float(msg.get('tr_timestamp_complete', 0)) / 1000),
'duration':
(float(msg.get('tr_timestamp_complete', 0)) -
float(msg.get('tr_timestamp_start', 0))) / 1000,
'reason':
msg.get('t__error_message', None),
'scope':
msg['file_metadata'].get('scope', None),
'name':
msg['file_metadata'].get('name', None),
'src_type':
msg['file_metadata'].get('src_type', None),
'dst_type':
msg['file_metadata'].get('dst_type', None),
'src_rse':
msg['file_metadata'].get('src_rse', None),
'dst_rse':
msg['file_metadata'].get('dst_rse', None),
'request_id':
msg['file_metadata'].get('request_id', None),
'activity':
msg['file_metadata'].get('activity', None),
'src_rse_id':
msg['file_metadata'].get('src_rse_id', None),
'dest_rse_id':
msg['file_metadata'].get('dest_rse_id', None),
'previous_attempt_id':
msg['file_metadata'].get('previous_attempt_id', None),
'adler32':
msg['file_metadata'].get('adler32', None),
'md5':
msg['file_metadata'].get('md5', None),
'filesize':
msg['file_metadata'].get('filesize', None),
'external_host':
msg.get('endpnt', None),
'job_m_replica':
msg.get('job_m_replica', None),
'details': {
'files': msg['file_metadata']
}
}
record_counter('daemons.conveyor.receiver.message_rucio')
if str(msg['t_final_transfer_state']) == FTS_COMPLETE_STATE.OK: # pylint:disable=no-member
response['new_state'] = RequestState.DONE
elif str(msg['t_final_transfer_state']
) == FTS_COMPLETE_STATE.ERROR: # pylint:disable=no-member
response['new_state'] = RequestState.FAILED
try:
if response['new_state']:
logging.info(
'RECEIVED DID %s:%s FROM %s TO %s REQUEST %s TRANSFER_ID %s STATE %s'
% (response['scope'], response['name'],
response['src_rse'], response['dst_rse'],
response['request_id'], response['transfer_id'],
response['new_state']))
if self.__full_mode:
ret = request.update_request_state(response)
record_counter(
'daemons.conveyor.receiver.update_request_state.%s'
% ret)
else:
try:
logging.debug("Update request %s update time" %
response['request_id'])
set_transfer_update_time(
response['external_host'],
response['transfer_id'],
datetime.datetime.utcnow() -
datetime.timedelta(hours=24))
record_counter(
'daemons.conveyor.receiver.set_transfer_update_time'
)
except Exception as error:
logging.debug(
"Failed to update transfer's update time: %s"
% str(error))
except Exception:
logging.critical(traceback.format_exc())