def _perform_request_update(self, msg, session=None, logger=logging.log):
external_host = msg.get('endpnt', None)
request_id = msg['file_metadata'].get('request_id', None)
try:
tt_status_report = FTS3CompletionMessageTransferStatusReport(
external_host, request_id=request_id, fts_message=msg)
if tt_status_report.get_db_fields_to_update(session=session,
logger=logger):
logging.info('RECEIVED %s', tt_status_report)
if self.__full_mode:
ret = request_core.update_request_state(tt_status_report,
session=session,
logger=logger)
record_counter(
'daemons.conveyor.receiver.update_request_state.{updated}',
labels={'updated': ret})
else:
try:
logging.debug("Update request %s update time" %
request_id)
set_transfer_update_time(external_host,
tt_status_report.external_id,
datetime.datetime.utcnow() -
datetime.timedelta(hours=24),
session=session)
record_counter(
'daemons.conveyor.receiver.set_transfer_update_time'
)
except Exception as error:
logging.debug(
"Failed to update transfer's update time: %s" %
str(error))
except Exception:
logging.critical(traceback.format_exc())
class Receiver(object):
def __init__(self, broker, id, total_threads, full_mode=False):
self.__broker = broker
self.__id = id
self.__total_threads = total_threads
self.__full_mode = full_mode
def on_error(self, headers, message):
record_counter('daemons.conveyor.receiver.error')
logging.error('[%s] %s' % (self.__broker, message))
def on_message(self, headers, message):
record_counter('daemons.conveyor.receiver.message_all')
try:
msg = json.loads(message)
except Exception, e:
msg = json.loads(
message[:-1]
) # Note: I am not sure if this is needed anymore, this was due to an unparsable EOT character
if 'vo' not in msg or msg['vo'] != get_policy():
return
if 'job_metadata' in msg.keys() \
and isinstance(msg['job_metadata'], dict) \
and 'issuer' in msg['job_metadata'].keys() \
and str(msg['job_metadata']['issuer']) == str('rucio'):
if 'job_m_replica' in msg.keys() and 'job_state' in msg.keys() \
and (str(msg['job_m_replica']).lower() == str('false') or (str(msg['job_m_replica']).lower() == str('true') and str(msg['job_state']) != str('ACTIVE'))):
if 'request_id' in msg['job_metadata']:
# submitted by old submitter
response = {
'new_state':
None,
'transfer_id':
msg.get('tr_id').split("__")[-1],
'job_state':
msg.get('t_final_transfer_state', None),
'src_url':
msg.get('src_url', None),
'dst_url':
msg.get('dst_url', None),
'transferred_at':
datetime.datetime.utcfromtimestamp(
float(msg.get('tr_timestamp_complete', 0)) / 1000),
'duration':
(float(msg.get('tr_timestamp_complete', 0)) -
float(msg.get('tr_timestamp_start', 0))) / 1000,
'reason':
msg.get('t__error_message', None),
'scope':
msg['job_metadata'].get('scope', None),
'name':
msg['job_metadata'].get('name', None),
'src_rse':
msg['job_metadata'].get('src_rse', None),
'dst_rse':
msg['job_metadata'].get('dst_rse', None),
'request_id':
msg['job_metadata'].get('request_id', None),
'activity':
msg['job_metadata'].get('activity', None),
'dest_rse_id':
msg['job_metadata'].get('dest_rse_id', None),
'previous_attempt_id':
msg['job_metadata'].get('previous_attempt_id', None),
'adler32':
msg['job_metadata'].get('adler32', None),
'md5':
msg['job_metadata'].get('md5', None),
'filesize':
msg['job_metadata'].get('filesize', None),
'external_host':
msg.get('endpnt', None),
'job_m_replica':
msg.get('job_m_replica', None),
'details': {
'files': msg['job_metadata']
}
}
else:
# for new submitter, file_metadata replace the job_metadata
response = {
'new_state':
None,
'transfer_id':
msg.get('tr_id').split("__")[-1],
'job_state':
msg.get('t_final_transfer_state', None),
'src_url':
msg.get('src_url', None),
'dst_url':
msg.get('dst_url', None),
'started_at':
datetime.datetime.utcfromtimestamp(
float(msg.get('tr_timestamp_start', 0)) / 1000),
'transferred_at':
datetime.datetime.utcfromtimestamp(
float(msg.get('tr_timestamp_complete', 0)) / 1000),
'duration':
(float(msg.get('tr_timestamp_complete', 0)) -
float(msg.get('tr_timestamp_start', 0))) / 1000,
'reason':
msg.get('t__error_message', None),
'scope':
msg['file_metadata'].get('scope', None),
'name':
msg['file_metadata'].get('name', None),
'src_type':
msg['file_metadata'].get('src_type', None),
'dst_type':
msg['file_metadata'].get('dst_type', None),
'src_rse':
msg['file_metadata'].get('src_rse', None),
'dst_rse':
msg['file_metadata'].get('dst_rse', None),
'request_id':
msg['file_metadata'].get('request_id', None),
'activity':
msg['file_metadata'].get('activity', None),
'src_rse_id':
msg['file_metadata'].get('src_rse_id', None),
'dest_rse_id':
msg['file_metadata'].get('dest_rse_id', None),
'previous_attempt_id':
msg['file_metadata'].get('previous_attempt_id', None),
'adler32':
msg['file_metadata'].get('adler32', None),
'md5':
msg['file_metadata'].get('md5', None),
'filesize':
msg['file_metadata'].get('filesize', None),
'external_host':
msg.get('endpnt', None),
'job_m_replica':
msg.get('job_m_replica', None),
'details': {
'files': msg['file_metadata']
}
}
record_counter('daemons.conveyor.receiver.message_rucio')
if str(msg['t_final_transfer_state']) == str(
FTSCompleteState.OK):
response['new_state'] = RequestState.DONE
elif str(msg['t_final_transfer_state']) == str(
FTSCompleteState.ERROR):
response['new_state'] = RequestState.FAILED
try:
if response['new_state']:
logging.info(
'RECEIVED DID %s:%s FROM %s TO %s REQUEST %s TRANSFER_ID %s STATE %s'
% (response['scope'], response['name'],
response['src_rse'], response['dst_rse'],
response['request_id'], response['transfer_id'],
response['new_state']))
if self.__full_mode:
ret = request.update_request_state(response)
record_counter(
'daemons.conveyor.receiver.update_request_state.%s'
% ret)
else:
try:
logging.debug("Update request %s update time" %
response['request_id'])
set_transfer_update_time(
response['external_host'],
response['transfer_id'],
datetime.datetime.utcnow() -
datetime.timedelta(hours=24))
record_counter(
'daemons.conveyor.receiver.set_transfer_update_time'
)
except Exception, e:
logging.debug(
"Failed to update transfer's update time: %s"
% str(e))
except:
logging.critical(traceback.format_exc())
def on_message(self, frame):
record_counter('daemons.conveyor.receiver.message_all')
msg = json.loads(frame.body)
if not self.__all_vos:
if 'vo' not in msg or msg['vo'] != get_policy():
return
if 'job_metadata' in msg.keys() \
and isinstance(msg['job_metadata'], dict) \
and 'issuer' in msg['job_metadata'].keys() \
and str(msg['job_metadata']['issuer']) == str('rucio'):
if 'job_state' in msg.keys() and (
str(msg['job_state']) != str('ACTIVE')
or str(msg['job_state']) == str('ACTIVE')
and 'job_m_replica' in msg.keys() and
(str(msg['job_m_replica']).lower() == str('true'))):
response = {
'new_state':
None,
'transfer_id':
msg.get('tr_id').split("__")[-1],
'job_state':
msg.get('t_final_transfer_state', None),
'src_url':
msg.get('src_url', None),
'dst_url':
msg.get('dst_url', None),
'started_at':
datetime.datetime.utcfromtimestamp(
float(msg.get('tr_timestamp_start', 0)) / 1000),
'transferred_at':
datetime.datetime.utcfromtimestamp(
float(msg.get('tr_timestamp_complete', 0)) / 1000),
'duration':
(float(msg.get('tr_timestamp_complete', 0)) -
float(msg.get('tr_timestamp_start', 0))) / 1000,
'reason':
msg.get('t__error_message', None),
'scope':
msg['file_metadata'].get('scope', None),
'name':
msg['file_metadata'].get('name', None),
'src_type':
msg['file_metadata'].get('src_type', None),
'dst_type':
msg['file_metadata'].get('dst_type', None),
'src_rse':
msg['file_metadata'].get('src_rse', None),
'dst_rse':
msg['file_metadata'].get('dst_rse', None),
'request_id':
msg['file_metadata'].get('request_id', None),
'activity':
msg['file_metadata'].get('activity', None),
'src_rse_id':
msg['file_metadata'].get('src_rse_id', None),
'dest_rse_id':
msg['file_metadata'].get('dest_rse_id', None),
'previous_attempt_id':
msg['file_metadata'].get('previous_attempt_id', None),
'adler32':
msg['file_metadata'].get('adler32', None),
'md5':
msg['file_metadata'].get('md5', None),
'filesize':
msg['file_metadata'].get('filesize', None),
'external_host':
msg.get('endpnt', None),
'job_m_replica':
msg.get('job_m_replica', None),
'details': {
'files': msg['file_metadata']
}
}
record_counter('daemons.conveyor.receiver.message_rucio')
if str(msg['t_final_transfer_state']) == FTS_COMPLETE_STATE.OK: # pylint:disable=no-member
response['new_state'] = RequestState.DONE
elif str(msg['t_final_transfer_state']
) == FTS_COMPLETE_STATE.ERROR: # pylint:disable=no-member
response['new_state'] = RequestState.FAILED
try:
if response['new_state']:
logging.info(
'RECEIVED DID %s:%s FROM %s TO %s REQUEST %s TRANSFER_ID %s STATE %s'
% (response['scope'], response['name'],
response['src_rse'], response['dst_rse'],
response['request_id'], response['transfer_id'],
response['new_state']))
if self.__full_mode:
ret = request.update_request_state(response)
record_counter(
'daemons.conveyor.receiver.update_request_state.%s'
% ret)
else:
try:
logging.debug("Update request %s update time" %
response['request_id'])
set_transfer_update_time(
response['external_host'],
response['transfer_id'],
datetime.datetime.utcnow() -
datetime.timedelta(hours=24))
record_counter(
'daemons.conveyor.receiver.set_transfer_update_time'
)
except Exception as error:
logging.debug(
"Failed to update transfer's update time: %s"
% str(error))
except Exception:
logging.critical(traceback.format_exc())
