def __init__(self):
self.next_timed_ack_id = 46
self.jscbd = JobScoreboard()
self.ascbd = AckScoreboard()
self.fill_job_scoreboard()
def setup_scoreboards(self):
try:
# Create Redis Forwarder table with Forwarder info
self.FWD_SCBD = ForwarderScoreboard('PP_FWD_SCBD',
self._scbd_dict['PP_FWD_SCBD'],
self._forwarder_dict)
self.JOB_SCBD = JobScoreboard('PP_JOB_SCBD',
self._scbd_dict['PP_JOB_SCBD'])
self.ACK_SCBD = AckScoreboard('PP_ACK_SCBD',
self._scbd_dict['PP_ACK_SCBD'])
except L1RabbitConnectionError as e:
LOGGER.error(
"PP_Device unable to complete setup_scoreboards-No Rabbit Connect: %s"
% e.arg)
print(
"PP_Device unable to complete setup_scoreboards - No Rabbit Connection: %s"
% e.arg)
sys.exit(self.ErrorCodePrefix + 11)
except L1RedisError as e:
LOGGER.error(
"PP_Device unable to complete setup_scoreboards - no Redis connect: %s"
% e.arg)
print(
"PP_Device unable to complete setup_scoreboards - no Redis connection: %s"
% e.arg)
sys.exit(self.ErrorCodePrefix + 12)
except Exception as e:
LOGGER.error(
"PP_Device init unable to complete setup_scoreboards: %s" %
e.arg)
print("PP_Device unable to complete setup_scoreboards: %s" % e.arg)
sys.exit(self.ErrorCodePrefix + 10)
def __init__(self, filename=None):
toolsmod.singleton(self)
cdm = self.loadConfigFile(filename)
try:
self._msg_name = cdm[ROOT][
PFM_BROKER_NAME] # Message broker user & passwd
self._msg_passwd = cdm[ROOT][PFM_BROKER_PASSWD]
self._ncsa_name = cdm[ROOT][NCSA_BROKER_NAME]
self._ncsa_passwd = cdm[ROOT][NCSA_BROKER_PASSWD]
self._base_broker_addr = cdm[ROOT][BASE_BROKER_ADDR]
self._ncsa_broker_addr = cdm[ROOT][NCSA_BROKER_ADDR]
forwarder_dict = cdm[ROOT][XFER_COMPONENTS][FORWARDERS]
except KeyError as e:
print("Dictionary error")
print("Bailing out...")
sys.exit(99)
#if 'QUEUE_PURGES' in cdm[ROOT]:
# self.purge_broker(cdm['ROOT']['QUEUE_PURGES'])
self._base_msg_format = self.YAML
self._ncsa_msg_format = self.YAML
if 'BASE_MSG_FORMAT' in cdm[ROOT]:
self._base_msg_format = cdm[ROOT][BASE_MSG_FORMAT]
if 'NCSA_MSG_FORMAT' in cdm[ROOT]:
self._ncsa_msg_format = cdm[ROOT][NCSA_MSG_FORMAT]
self._base_broker_url = 'amqp_url'
self._ncsa_broker_url = 'amqp_url'
self._next_timed_ack_id = 0
# Create Redis Forwarder table with Forwarder info
self.FWD_SCBD = ForwarderScoreboard(forwarder_dict)
self.JOB_SCBD = JobScoreboard()
self.ACK_SCBD = AckScoreboard()
self._msg_actions = {
'NEW_JOB': self.process_dmcs_new_job,
'READOUT': self.process_dmcs_readout,
'NCSA_RESOURCE_QUERY_ACK': self.process_ack,
'NCSA_STANDBY_ACK': self.process_ack,
'NCSA_READOUT_ACK': self.process_ack,
'FORWARDER_HEALTH_ACK': self.process_ack,
'FORWARDER_JOB_PARAMS_ACK': self.process_ack,
'FORWARDER_READOUT_ACK': self.process_ack,
'NEW_JOB_ACK': self.process_ack
}
self._base_broker_url = "amqp://" + self._msg_name + ":" + self._msg_passwd + "@" + str(
self._base_broker_addr)
self._ncsa_broker_url = "amqp://" + self._ncsa_name + ":" + self._ncsa_passwd + "@" + str(
self._ncsa_broker_addr)
LOGGER.info('Building _base_broker_url. Result is %s',
self._base_broker_url)
LOGGER.info('Building _ncsa_broker_url. Result is %s',
self._ncsa_broker_url)
def setup_scoreboards(self):
# Create Redis Distributor table with Distributor info
self.DIST_SCBD = DistributorScoreboard('NCSA_DIST_SCBD', \
self._scbd_dict['NCSA_DIST_SCBD'], \
self.distributor_dict)
self.JOB_SCBD = JobScoreboard('NCSA_JOB_SCBD',
self._scbd_dict['NCSA_JOB_SCBD'])
self.ACK_SCBD = AckScoreboard('NCSA_ACK_SCBD',
self._scbd_dict['NCSA_ACK_SCBD'])
def setup_scoreboards(self):
""" Create Redis Forwarder table with Forwarder info. Create Job and Ack Scoreboard
objects with values retrieved from configuration file.
:params: None.
:return: None.
"""
# Create Redis Forwarder table with Forwarder info
self.FWD_SCBD = ForwarderScoreboard('AR_FWD_SCBD',
self._scbd_dict['AR_FWD_SCBD'],
self._forwarder_dict)
self.JOB_SCBD = JobScoreboard('AR_JOB_SCBD',
self._scbd_dict['AR_JOB_SCBD'])
self.ACK_SCBD = AckScoreboard('AR_ACK_SCBD',
self._scbd_dict['AR_ACK_SCBD'])
class BaseForeman:
FWD_SCBD = None
JOB_SCBD = None
ACK_SCBD = None
ACK_PUBLISH = "ack_publish"
YAML = 'YAML'
def __init__(self, filename=None):
toolsmod.singleton(self)
self._config_file = 'ForemanCfg.yaml'
if filename != None:
self._config_file = filename
cdm = toolsmod.intake_yaml_file(self._config_file)
try:
self._base_name = cdm[ROOT][
BASE_BROKER_NAME] # Message broker user & passwd
self._base_passwd = cdm[ROOT][BASE_BROKER_PASSWD]
self._ncsa_name = cdm[ROOT][NCSA_BROKER_NAME]
self._ncsa_passwd = cdm[ROOT][NCSA_BROKER_PASSWD]
self._base_broker_addr = cdm[ROOT][BASE_BROKER_ADDR]
self._ncsa_broker_addr = cdm[ROOT][NCSA_BROKER_ADDR]
forwarder_dict = cdm[ROOT][XFER_COMPONENTS][FORWARDERS]
except KeyError as e:
print("Dictionary error")
print("Bailing out...")
sys.exit(99)
if 'QUEUE_PURGES' in cdm[ROOT]:
self.purge_broker(cdm['ROOT']['QUEUE_PURGES'])
self._base_msg_format = self.YAML
self._ncsa_msg_format = self.YAML
if 'BASE_MSG_FORMAT' in cdm[ROOT]:
self._base_msg_format = cdm[ROOT][BASE_MSG_FORMAT]
if 'NCSA_MSG_FORMAT' in cdm[ROOT]:
self._ncsa_msg_format = cdm[ROOT][NCSA_MSG_FORMAT]
self._base_broker_url = 'amqp_url'
self._ncsa_broker_url = 'amqp_url'
self._next_timed_ack_id = 0
# Create Redis Forwarder table with Forwarder info
self.FWD_SCBD = ForwarderScoreboard(forwarder_dict)
self.JOB_SCBD = JobScoreboard()
self.ACK_SCBD = AckScoreboard()
self._msg_actions = {
'NEW_JOB': self.process_dmcs_new_job,
'READOUT': self.process_dmcs_readout,
'NCSA_RESOURCE_QUERY_ACK': self.process_ack,
'NCSA_STANDBY_ACK': self.process_ack,
'NCSA_READOUT_ACK': self.process_ack,
'FORWARDER_HEALTH_ACK': self.process_ack,
'FORWARDER_JOB_PARAMS_ACK': self.process_ack,
'FORWARDER_READOUT_ACK': self.process_ack,
'NEW_JOB_ACK': self.process_ack
}
self._base_broker_url = "amqp://" + self._base_name + ":" + self._base_passwd + "@" + str(
self._base_broker_addr)
self._ncsa_broker_url = "amqp://" + self._ncsa_name + ":" + self._ncsa_passwd + "@" + str(
self._ncsa_broker_addr)
LOGGER.info('Building _base_broker_url. Result is %s',
self._base_broker_url)
LOGGER.info('Building _ncsa_broker_url. Result is %s',
self._ncsa_broker_url)
self.setup_publishers()
self.setup_consumers()
#self._ncsa_broker_url = ""
#self.setup_federated_exchange()
def setup_consumers(self):
"""This method sets up a message listener from each entity
with which the BaseForeman has contact here. These
listeners are instanced in this class, but their run
methods are each called as a separate thread. While
pika does not claim to be thread safe, the manner in which
the listeners are invoked below is a safe implementation
that provides non-blocking, fully asynchronous messaging
to the BaseForeman.
The code in this file expects message bodies to arrive as
YAML'd python dicts, while in fact, message bodies are sent
on the wire as XML; this way message format can be validated,
versioned, and specified in just one place. To make this work,
there is an object that translates the params dict to XML, and
visa versa. The translation object is instantiated by the consumer
and acts as a filter before sending messages on to the registered
callback for processing.
"""
LOGGER.info('Setting up consumers on %s', self._base_broker_url)
LOGGER.info('Running start_new_thread on all consumer methods')
self._dmcs_consumer = Consumer(self._base_broker_url,
self.DMCS_PUBLISH,
self._base_msg_format)
try:
_thread.start_new_thread(self.run_dmcs_consumer, (
"thread-dmcs-consumer",
2,
))
except:
LOGGER.critical('Cannot start DMCS consumer thread, exiting...')
sys.exit(99)
self._forwarder_consumer = Consumer(self._base_broker_url,
self.FORWARDER_PUBLISH,
self._base_msg_format)
try:
_thread.start_new_thread(self.run_forwarder_consumer, (
"thread-forwarder-consumer",
2,
))
except:
LOGGER.critical(
'Cannot start FORWARDERS consumer thread, exiting...')
sys.exit(100)
self._ncsa_consumer = Consumer(self._base_broker_url,
self.NCSA_PUBLISH,
self._base_msg_format)
try:
_thread.start_new_thread(self.run_ncsa_consumer, (
"thread-ncsa-consumer",
2,
))
except:
LOGGER.critical('Cannot start NCSA consumer thread, exiting...')
sys.exit(101)
self._ack_consumer = Consumer(self._base_broker_url, self.ACK_PUBLISH,
self._base_msg_format)
try:
_thread.start_new_thread(self.run_ack_consumer, (
"thread-ack-consumer",
2,
))
except:
LOGGER.critical('Cannot start ACK consumer thread, exiting...')
sys.exit(102)
LOGGER.info('Finished starting all three consumer threads')
def run_dmcs_consumer(self, threadname, delay):
self._dmcs_consumer.run(self.on_dmcs_message)
def run_forwarder_consumer(self, threadname, delay):
self._forwarder_consumer.run(self.on_forwarder_message)
def run_ncsa_consumer(self, threadname, delay):
self._ncsa_consumer.run(self.on_ncsa_message)
def run_ack_consumer(self, threadname, delay):
self._ack_consumer.run(self.on_ack_message)
def setup_publishers(self):
LOGGER.info('Setting up Base publisher on %s using %s',
self._base_broker_url, self._base_msg_format)
LOGGER.info('Setting up NCSA publisher on %s using %s',
self._ncsa_broker_url, self._ncsa_msg_format)
self._base_publisher = SimplePublisher(self._base_broker_url,
self._base_msg_format)
self._ncsa_publisher = SimplePublisher(self._ncsa_broker_url,
self._ncsa_msg_format)
# def setup_federated_exchange(self):
# # Set up connection URL for NCSA Broker here.
# self._ncsa_broker_url = "amqp://" + self._name + ":" + self._passwd + "@" + str(self._ncsa_broker_addr)
# LOGGER.info('Building _ncsa_broker_url. Result is %s', self._ncsa_broker_url)
# pass
def on_dmcs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
#msg_dict = yaml.load(body)
msg_dict = body
LOGGER.info('In DMCS message callback')
LOGGER.debug('Thread in DMCS callback is %s', _thread.get_ident())
LOGGER.info('Message from DMCS callback message body is: %s',
str(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def on_forwarder_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
LOGGER.info('In Forwarder message callback, thread is %s',
_thread.get_ident())
LOGGER.info('forwarder callback msg body is: %s', str(body))
pass
def on_ncsa_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
LOGGER.info('In ncsa message callback, thread is %s',
_thread.get_ident())
#msg_dict = yaml.load(body)
msg_dict = body
LOGGER.info('ncsa msg callback body is: %s', str(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def on_ack_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
msg_dict = body
LOGGER.info('In ACK message callback')
LOGGER.debug('Thread in ACK callback is %s', _thread.get_ident())
LOGGER.info('Message from ACK callback message body is: %s',
str(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def process_dmcs_new_job(self, params):
input_params = params
needed_workers = len(input_params[RAFTS])
ack_id = self.forwarder_health_check(input_params)
self.ack_timer(
7
) # This is a HUGE num seconds for now..final setting will be milliseconds
healthy_forwarders = self.ACK_SCBD.get_components_for_timed_ack(
timed_ack)
num_healthy_forwarders = len(healthy_forwarders)
if needed_workers > num_healthy_forwarders:
result = self.insufficient_base_resources(input_params,
healthy_forwarders)
return result
else:
healthy_status = {
"STATUS": "HEALTHY",
"STATE": "READY_WITHOUT_PARAMS"
}
self.FWD_SCBD.set_forwarder_params(healthy_forwarders,
healthy_status)
ack_id = self.ncsa_resources_query(input_params,
healthy_forwarders)
self.ack_timer(3)
#Check ACK scoreboard for response from NCSA
ncsa_response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if ncsa_response:
pairs = {}
ack_bool = None
try:
ack_bool = ncsa_response[ACK_BOOL]
if ack_bool == True:
pairs = ncsa_response[PAIRS]
except KeyError as e:
pass
# Distribute job params and tell DMCS I'm ready.
if ack_bool == TRUE:
fwd_ack_id = self.distribute_job_params(
input_params, pairs)
self.ack_timer(3)
fwd_params_response = self.ACK_SCBD.get_components_for_timed_ack(
fwd_ack_id)
if fwd_params_response and (len(fwd_params_response)
== len(fwders)):
self.JOB_SCBD.set_value_for_job(
job_num, "STATE", "BASE_TASK_PARAMS_SENT")
self.JOB_SCBD.set_value_for_job(
job_num, "TIME_BASE_TASK_PARAMS_SENT",
get_timestamp())
in_ready_state = {'STATE': 'READY_WITH_PARAMS'}
self.FWD_SCBD.set_forwarder_params(
fwders, in_ready_state)
# Tell DMCS we are ready
result = self.accept_job(job_num)
else:
#not enough ncsa resources to do job - Notify DMCS
idle_param = {'STATE': 'IDLE'}
self.FWD_SCBD.set_forwarder_params(healthy_forwarders,
idle_params)
result = self.insufficient_ncsa_resources(ncsa_response)
return result
else:
result = self.ncsa_no_response(input_params)
idle_param = {'STATE': 'IDLE'}
self.FWD_SCBD.set_forwarder_params(
list(forwarder_candidate_dict.keys()), idle_params)
return result
def forwarder_health_check(self, params):
job_num = str(params[JOB_NUM])
raft_list = params['RAFTS']
needed_workers = len(raft_list)
self.JOB_SCBD.add_job(job_num, needed_workers)
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ADDED",
get_timestamp())
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ADDED_E",
get_epoch_timestamp())
LOGGER.info('Received new job %s. Needed workers is %s', job_num,
needed_workers)
# run forwarder health check
# get timed_ack_id
timed_ack = self.get_next_timed_ack_id("FORWARDER_HEALTH_CHECK_ACK")
forwarders = self.FWD_SCBD.return_available_forwarders_list()
# Mark all healthy Forwarders Unknown
state_status = {"STATE": "HEALTH_CHECK", "STATUS": "UNKNOWN"}
self.FWD_SCBD.set_forwarder_params(forwarders, state_status)
# send health check messages
ack_params = {}
ack_params[MSG_TYPE] = FORWARDER_HEALTH_CHECK
ack_params["ACK_ID"] = timed_ack
ack_params[JOB_NUM] = job_num
self.JOB_SCBD.set_value_for_job(job_num, "STATE",
"BASE_RESOURCE_QUERY")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_BASE_RESOURCE_QUERY",
get_timestamp())
audit_params = {}
audit_params['DATA_TYPE'] = 'FOREMAN_ACK_REQUEST'
audit_params['SUB_TYPE'] = 'FORWARDER_HEALTH_CHECK_ACK'
audit_params['ACK_ID'] = timed_ack
audit_parsms['COMPONENT_NAME'] = 'BASE_FOREMAN'
audit_params['TIME'] = get_epoch_timestamp()
for forwarder in forwarders:
self._base_publisher.publish_message(
self.FWD_SCBD.get_value_for_forwarder(forwarder,
"CONSUME_QUEUE"),
ack_params)
return timed_ack
def insufficient_base_resources(self, params, healthy_forwarders):
# send response msg to dmcs refusing job
job_num = str(params[JOB_NUM])
raft_list = params[RAFTS]
ack_id = params['ACK_ID']
needed_workers = len(raft_list)
LOGGER.info(
'Reporting to DMCS that there are insufficient healthy forwarders for job #%s',
job_num)
dmcs_params = {}
fail_dict = {}
dmcs_params[MSG_TYPE] = NEW_JOB_ACK
dmcs_params[JOB_NUM] = job_num
dmcs_params[ACK_BOOL] = False
dmcs_params[ACK_ID] = ack_id
### NOTE FOR DMCS ACK PROCESSING:
### if ACK_BOOL == True, there will NOT be a FAIL_DETAILS section
### If ACK_BOOL == False, there will always be a FAIL_DICT to examine AND there will always be a
### BASE_RESOURCES inside the FAIL_DICT
### If ACK_BOOL == False, and the BASE_RESOURCES inside FAIL_DETAILS == 0,
### there will be only NEEDED and AVAILABLE Forwarder params - nothing more
### If ACK_BOOL == False and BASE_RESOURCES inside FAIL_DETAILS == 1, there will always be a
### NCSA_RESOURCES inside FAIL_DETAILS set to either 0 or 'NO_RESPONSE'
### if NCSA_RESPONSE == 0, there will be NEEDED and AVAILABLE Distributor params
### if NCSA_RESOURCES == 'NO_RESPONSE' there will be nothing else
fail_dict['BASE_RESOURCES'] = '0'
fail_dict[NEEDED_FORWARDERS] = str(needed_workers)
fail_dict[AVAILABLE_FORWARDERS] = str(len(healthy_forwarders))
dmcs_params['FAIL_DETAILS'] = fail_dict
self._base_publisher.publish_message("dmcs_consume", dmcs_params)
# mark job refused, and leave Forwarders in Idle state
self.JOB_SCBD.set_value_for_job(job_num, "STATE", "JOB_ABORTED")
self.JOB_SCBD.set_value_for_job(job_num,
"TIME_JOB_ABORTED_BASE_RESOURCES",
get_timestamp())
idle_state = {"STATE": "IDLE"}
self.FWD_SCBD.set_forwarder_params(healthy_forwarders, idle_state)
return False
def ncsa_resources_query(self, params, healthy_forwarders):
job_num = str(params[JOB_NUM])
raft_list = params[RAFTS]
needed_workers = len(raft_list)
LOGGER.info('Sufficient forwarders have been found. Checking NCSA')
self._pairs_dict = {}
forwarder_candidate_dict = {}
for i in range(0, needed_workers):
forwarder_candidate_dict[healthy_forwarders[i]] = raft_list[i]
self.FWD_SCBD.set_forwarder_status(healthy_forwarders[i],
NCSA_RESOURCES_QUERY)
# Call this method for testing...
# There should be a message sent to NCSA here asking for available resources
timed_ack_id = self.get_next_timed_ack_id("NCSA_Ack")
ncsa_params = {}
ncsa_params[MSG_TYPE] = "NCSA_RESOURCES_QUERY"
ncsa_params[JOB_NUM] = job_num
#ncsa_params[RAFT_NUM] = needed_workers
ncsa_params[ACK_ID] = timed_ack_id
ncsa_params["FORWARDERS"] = forwarder_candidate_dict
self.JOB_SCBD.set_value_for_job(job_num, "STATE",
"NCSA_RESOURCES_QUERY_SENT")
self.JOB_SCBD.set_value_for_job(job_num,
"TIME_NCSA_RESOURCES_QUERY_SENT",
get_timestamp())
self._ncsa_publisher.publish_message(self.NCSA_CONSUME, ncsa_params)
LOGGER.info(
'The following forwarders have been sent to NCSA for pairing:')
LOGGER.info(forwarder_candidate_dict)
return timed_ack_id
def distribute_job_params(self, params, pairs):
#ncsa has enough resources...
job_num = str(params[JOB_NUM])
self.JOB_SCBD.set_pairs_for_job(job_num, pairs)
self.JOB_SCBD.set_value_for_job(job_num, "TIME_PAIRS_ADDED",
get_timestamp())
LOGGER.info('The following pairs will be used for Job #%s: %s',
job_num, pairs)
fwd_ack_id = self.get_next_timed_ack_id("FWD_PARAMS_ACK")
fwders = list(pairs.keys())
fwd_params = {}
fwd_params[MSG_TYPE] = "FORWARDER_JOB_PARAMS"
fwd_params[JOB_NUM] = job_num
fwd_params[ACK_ID] = fwd_ack_id
for fwder in fwders:
fwd_params["TRANSFER_PARAMS"] = pairs[fwder]
route_key = self.FWD_SCBD.get_value_for_forwarder(
fwder, "CONSUME_QUEUE")
self._base_publisher.publish_message(route_key, fwd_params)
return fwd_ack_id
def accept_job(self, job_num):
dmcs_message = {}
dmcs_message[JOB_NUM] = job_num
dmcs_message[MSG_TYPE] = NEW_JOB_ACK
dmcs_message[ACK_BOOL] = True
self.JOB_SCBD.set_value_for_job(job_num, STATE, "JOB_ACCEPTED")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ACCEPTED",
get_timestamp())
self._base_publisher.publish_message("dmcs_consume", dmcs_message)
return True
def insufficient_ncsa_resources(self, ncsa_response):
dmcs_params = {}
dmcs_params[MSG_TYPE] = "NEW_JOB_ACK"
dmcs_params[JOB_NUM] = job_num
dmcs_params[ACK_BOOL] = False
dmcs_params[BASE_RESOURCES] = '1'
dmcs_params[NCSA_RESOURCES] = '0'
dmcs_params[NEEDED_DISTRIBUTORS] = ncsa_response[NEEDED_DISTRIBUTORS]
dmcs_params[AVAILABLE_DISTRIBUTORS] = ncsa_response[
AVAILABLE_DISTRIBUTORS]
#try: FIXME - catch exception
self._base_publisher.publish_message("dmcs_consume", dmcs_params)
#except L1MessageError e:
# return False
return True
def ncsa_no_response(self, params):
#No answer from NCSA...
job_num = str(params[JOB_NUM])
raft_list = params[RAFTS]
needed_workers = len(raft_list)
dmcs_params = {}
dmcs_params[MSG_TYPE] = "NEW_JOB_ACK"
dmcs_params[JOB_NUM] = job_num
dmcs_params[ACK_BOOL] = False
dmcs_params[BASE_RESOURCES] = '1'
dmcs_params[NCSA_RESOURCES] = 'NO_RESPONSE'
self._base_publisher.publish_message("dmcs_consume", dmcs_params)
def process_dmcs_readout(self, params):
job_number = params[JOB_NUM]
pairs = self.JOB_SCBD.get_pairs_for_job(job_number)
date - get_timestamp()
self.JOB_SCBD.set_value_for_job(job_number, TIME_START_READOUT, date)
# The following line extracts the distributor FQNs from pairs dict using
# list comprehension values; faster than for loops
distributors = [v['FQN'] for v in list(pairs.values())]
forwarders = list(pairs.keys())
ack_id = self.get_next_timed_ack_id('NCSA_READOUT')
### Send READOUT to NCSA with ACK_ID
ncsa_params = {}
ncsa_params[MSG_TYPE] = 'NCSA_READOUT'
ncsa_params[ACK_ID] = ack_id
self._ncsa_publisher.publish_message(NCSA_CONSUME,
yaml.dump(ncsa_params))
self.ack_timer(4)
ncsa_response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if ncsa_response:
if ncsa_response['ACK_BOOL'] == True:
#inform forwarders
fwd_ack_id = self.get_next_timed_ack_id('FORWARDER_READOUT')
for forwarder in forwarders:
name = self.FWD_SCBD.get_value_for_forwarder(
forwarder, NAME)
routing_key = self.FWD_SCBD.get_routing_key(forwarder)
msg_params = {}
msg_params[MSG_TYPE] = 'FORWARDER_READOUT'
msg_params[JOB_NUM] = job_number
msg_params['ACK_ID'] = fwd_ack_id
self.FWD_SCBD.set_forwarder_state(forwarder, START_READOUT)
self._publisher.publish_message(routing_key,
yaml.dump(msg_params))
self.ack_timer(4)
forwarder_responses = self.ACK_SCBD.get_components_for_timed_ack(
fwd_ack_id)
if len(forwarder_responses) == len(forwarders):
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['ACK_BOOL'] = True
dmcs_params[
'COMMENT'] = "Readout begun at %s" % get_timestamp()
self._publisher.publish_message('dmcs_consume',
yaml.dump(dmcs_params))
else:
#send problem with ncsa to DMCS
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['ACK_BOOL'] = False
dmcs_params[
'COMMENT'] = 'Readout Failed: Problem at NCSA - Expected Distributor Acks is %s, Number of Distributor Acks received is %s' % (
ncsa_response['EXPECTED_DISTRIBUTOR_ACKS'],
ncsa_response['RECEIVED_DISTRIBUTOR_ACKS'])
self._base_publisher.publish_message('dmcs_consume',
yaml.dump(dmcs_params))
else:
#send 'no response from ncsa' to DMCS )
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['ACK_BOOL'] = False
dmcs_params['COMMENT'] = "Readout Failed: No Response from NCSA"
self._base_publisher.publish_message('dmcs_consume',
yaml.dump(dmcs_params))
def process_ack(self, params):
self.ACK_SCBD.add_timed_ack(params)
def get_next_timed_ack_id(self, ack_type):
self._next_timed_ack_id = self._next_timed_ack_id + 1
retval = ack_type + "_" + str(self._next_timed_ack_id).zfill(6)
return retval
def ack_timer(self, seconds):
sleep(seconds)
return True
def purge_broker(self, queues):
for q in queues:
cmd = "rabbitmqctl -p /tester purge_queue " + q
os.system(cmd)
def __init__(self, filename=None):
toolsmod.singleton(self)
self._config_file = 'ForemanCfg.yaml'
if filename != None:
self._config_file = filename
cdm = toolsmod.intake_yaml_file(self._config_file)
try:
self._base_name = cdm[ROOT][BASE_BROKER_NAME] # Message broker user & passwd
self._base_passwd = cdm[ROOT][BASE_BROKER_PASSWD]
self._ncsa_name = cdm[ROOT][NCSA_BROKER_NAME]
self._ncsa_passwd = cdm[ROOT][NCSA_BROKER_PASSWD]
self._base_broker_addr = cdm[ROOT][BASE_BROKER_ADDR]
self._ncsa_broker_addr = cdm[ROOT][NCSA_BROKER_ADDR]
forwarder_dict = cdm[ROOT][XFER_COMPONENTS][FORWARDERS]
except KeyError as e:
print("Dictionary error")
print("Bailing out...")
sys.exit(99)
if 'QUEUE_PURGES' in cdm[ROOT]:
self.purge_broker(cdm['ROOT']['QUEUE_PURGES'])
self._base_msg_format = self.YAML
self._ncsa_msg_format = self.YAML
if 'BASE_MSG_FORMAT' in cdm[ROOT]:
self._base_msg_format = cdm[ROOT][BASE_MSG_FORMAT]
if 'NCSA_MSG_FORMAT' in cdm[ROOT]:
self._ncsa_msg_format = cdm[ROOT][NCSA_MSG_FORMAT]
self._base_broker_url = 'amqp_url'
self._ncsa_broker_url = 'amqp_url'
self._next_timed_ack_id = 0
# Create Redis Forwarder table with Forwarder info
self.FWD_SCBD = ForwarderScoreboard(forwarder_dict)
self.JOB_SCBD = JobScoreboard()
self.ACK_SCBD = AckScoreboard()
self._msg_actions = { 'NEW_JOB': self.process_dmcs_new_job,
'READOUT': self.process_dmcs_readout,
'NCSA_RESOURCE_QUERY_ACK': self.process_ack,
'NCSA_STANDBY_ACK': self.process_ack,
'NCSA_READOUT_ACK': self.process_ack,
'FORWARDER_HEALTH_ACK': self.process_ack,
'FORWARDER_JOB_PARAMS_ACK': self.process_ack,
'FORWARDER_READOUT_ACK': self.process_ack,
'NEW_JOB_ACK': self.process_ack }
self._base_broker_url = "amqp://" + self._base_name + ":" + self._base_passwd + "@" + str(self._base_broker_addr)
self._ncsa_broker_url = "amqp://" + self._ncsa_name + ":" + self._ncsa_passwd + "@" + str(self._ncsa_broker_addr)
LOGGER.info('Building _base_broker_url. Result is %s', self._base_broker_url)
LOGGER.info('Building _ncsa_broker_url. Result is %s', self._ncsa_broker_url)
self.setup_publishers()
self.setup_consumers()
class BaseForeman:
FWD_SCBD = None
JOB_SCBD = None
ACK_SCBD = None
ACK_PUBLISH = "ack_publish"
YAML = 'YAML'
def __init__(self, filename=None):
toolsmod.singleton(self)
self._config_file = 'ForemanCfg.yaml'
if filename != None:
self._config_file = filename
cdm = toolsmod.intake_yaml_file(self._config_file)
try:
self._base_name = cdm[ROOT][BASE_BROKER_NAME] # Message broker user & passwd
self._base_passwd = cdm[ROOT][BASE_BROKER_PASSWD]
self._ncsa_name = cdm[ROOT][NCSA_BROKER_NAME]
self._ncsa_passwd = cdm[ROOT][NCSA_BROKER_PASSWD]
self._base_broker_addr = cdm[ROOT][BASE_BROKER_ADDR]
self._ncsa_broker_addr = cdm[ROOT][NCSA_BROKER_ADDR]
forwarder_dict = cdm[ROOT][XFER_COMPONENTS][FORWARDERS]
except KeyError as e:
print("Dictionary error")
print("Bailing out...")
sys.exit(99)
if 'QUEUE_PURGES' in cdm[ROOT]:
self.purge_broker(cdm['ROOT']['QUEUE_PURGES'])
self._base_msg_format = self.YAML
self._ncsa_msg_format = self.YAML
if 'BASE_MSG_FORMAT' in cdm[ROOT]:
self._base_msg_format = cdm[ROOT][BASE_MSG_FORMAT]
if 'NCSA_MSG_FORMAT' in cdm[ROOT]:
self._ncsa_msg_format = cdm[ROOT][NCSA_MSG_FORMAT]
self._base_broker_url = 'amqp_url'
self._ncsa_broker_url = 'amqp_url'
self._next_timed_ack_id = 0
# Create Redis Forwarder table with Forwarder info
self.FWD_SCBD = ForwarderScoreboard(forwarder_dict)
self.JOB_SCBD = JobScoreboard()
self.ACK_SCBD = AckScoreboard()
self._msg_actions = { 'NEW_JOB': self.process_dmcs_new_job,
'READOUT': self.process_dmcs_readout,
'NCSA_RESOURCE_QUERY_ACK': self.process_ack,
'NCSA_STANDBY_ACK': self.process_ack,
'NCSA_READOUT_ACK': self.process_ack,
'FORWARDER_HEALTH_ACK': self.process_ack,
'FORWARDER_JOB_PARAMS_ACK': self.process_ack,
'FORWARDER_READOUT_ACK': self.process_ack,
'NEW_JOB_ACK': self.process_ack }
self._base_broker_url = "amqp://" + self._base_name + ":" + self._base_passwd + "@" + str(self._base_broker_addr)
self._ncsa_broker_url = "amqp://" + self._ncsa_name + ":" + self._ncsa_passwd + "@" + str(self._ncsa_broker_addr)
LOGGER.info('Building _base_broker_url. Result is %s', self._base_broker_url)
LOGGER.info('Building _ncsa_broker_url. Result is %s', self._ncsa_broker_url)
self.setup_publishers()
self.setup_consumers()
#self._ncsa_broker_url = ""
#self.setup_federated_exchange()
def setup_consumers(self):
"""This method sets up a message listener from each entity
with which the BaseForeman has contact here. These
listeners are instanced in this class, but their run
methods are each called as a separate thread. While
pika does not claim to be thread safe, the manner in which
the listeners are invoked below is a safe implementation
that provides non-blocking, fully asynchronous messaging
to the BaseForeman.
The code in this file expects message bodies to arrive as
YAML'd python dicts, while in fact, message bodies are sent
on the wire as XML; this way message format can be validated,
versioned, and specified in just one place. To make this work,
there is an object that translates the params dict to XML, and
visa versa. The translation object is instantiated by the consumer
and acts as a filter before sending messages on to the registered
callback for processing.
"""
LOGGER.info('Setting up consumers on %s', self._base_broker_url)
LOGGER.info('Running start_new_thread on all consumer methods')
self._dmcs_consumer = Consumer(self._base_broker_url, self.DMCS_PUBLISH, self._base_msg_format)
try:
_thread.start_new_thread( self.run_dmcs_consumer, ("thread-dmcs-consumer", 2,) )
except:
LOGGER.critical('Cannot start DMCS consumer thread, exiting...')
sys.exit(99)
self._forwarder_consumer = Consumer(self._base_broker_url, self.FORWARDER_PUBLISH, self._base_msg_format)
try:
_thread.start_new_thread( self.run_forwarder_consumer, ("thread-forwarder-consumer", 2,) )
except:
LOGGER.critical('Cannot start FORWARDERS consumer thread, exiting...')
sys.exit(100)
self._ncsa_consumer = Consumer(self._base_broker_url, self.NCSA_PUBLISH, self._base_msg_format)
try:
_thread.start_new_thread( self.run_ncsa_consumer, ("thread-ncsa-consumer", 2,) )
except:
LOGGER.critical('Cannot start NCSA consumer thread, exiting...')
sys.exit(101)
self._ack_consumer = Consumer(self._base_broker_url, self.ACK_PUBLISH, self._base_msg_format)
try:
_thread.start_new_thread( self.run_ack_consumer, ("thread-ack-consumer", 2,) )
except:
LOGGER.critical('Cannot start ACK consumer thread, exiting...')
sys.exit(102)
LOGGER.info('Finished starting all three consumer threads')
def run_dmcs_consumer(self, threadname, delay):
self._dmcs_consumer.run(self.on_dmcs_message)
def run_forwarder_consumer(self, threadname, delay):
self._forwarder_consumer.run(self.on_forwarder_message)
def run_ncsa_consumer(self, threadname, delay):
self._ncsa_consumer.run(self.on_ncsa_message)
def run_ack_consumer(self, threadname, delay):
self._ack_consumer.run(self.on_ack_message)
def setup_publishers(self):
LOGGER.info('Setting up Base publisher on %s using %s', self._base_broker_url, self._base_msg_format)
LOGGER.info('Setting up NCSA publisher on %s using %s', self._ncsa_broker_url, self._ncsa_msg_format)
self._base_publisher = SimplePublisher(self._base_broker_url, self._base_msg_format)
self._ncsa_publisher = SimplePublisher(self._ncsa_broker_url, self._ncsa_msg_format)
# def setup_federated_exchange(self):
# # Set up connection URL for NCSA Broker here.
# self._ncsa_broker_url = "amqp://" + self._name + ":" + self._passwd + "@" + str(self._ncsa_broker_addr)
# LOGGER.info('Building _ncsa_broker_url. Result is %s', self._ncsa_broker_url)
# pass
def on_dmcs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
#msg_dict = yaml.load(body)
msg_dict = body
LOGGER.info('In DMCS message callback')
LOGGER.debug('Thread in DMCS callback is %s', _thread.get_ident())
LOGGER.info('Message from DMCS callback message body is: %s', str(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def on_forwarder_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
LOGGER.info('In Forwarder message callback, thread is %s', _thread.get_ident())
LOGGER.info('forwarder callback msg body is: %s', str(body))
pass
def on_ncsa_message(self,ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
LOGGER.info('In ncsa message callback, thread is %s', _thread.get_ident())
#msg_dict = yaml.load(body)
msg_dict = body
LOGGER.info('ncsa msg callback body is: %s', str(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def on_ack_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
msg_dict = body
LOGGER.info('In ACK message callback')
LOGGER.debug('Thread in ACK callback is %s', _thread.get_ident())
LOGGER.info('Message from ACK callback message body is: %s', str(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def process_dmcs_new_job(self, params):
input_params = params
needed_workers = len(input_params[RAFTS])
ack_id = self.forwarder_health_check(input_params)
self.ack_timer(7) # This is a HUGE num seconds for now..final setting will be milliseconds
healthy_forwarders = self.ACK_SCBD.get_components_for_timed_ack(timed_ack)
num_healthy_forwarders = len(healthy_forwarders)
if needed_workers > num_healthy_forwarders:
result = self.insufficient_base_resources(input_params, healthy_forwarders)
return result
else:
healthy_status = {"STATUS": "HEALTHY", "STATE":"READY_WITHOUT_PARAMS"}
self.FWD_SCBD.set_forwarder_params(healthy_forwarders, healthy_status)
ack_id = self.ncsa_resources_query(input_params, healthy_forwarders)
self.ack_timer(3)
#Check ACK scoreboard for response from NCSA
ncsa_response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if ncsa_response:
pairs = {}
ack_bool = None
try:
ack_bool = ncsa_response[ACK_BOOL]
if ack_bool == True:
pairs = ncsa_response[PAIRS]
except KeyError as e:
pass
# Distribute job params and tell DMCS I'm ready.
if ack_bool == TRUE:
fwd_ack_id = self.distribute_job_params(input_params, pairs)
self.ack_timer(3)
fwd_params_response = self.ACK_SCBD.get_components_for_timed_ack(fwd_ack_id)
if fwd_params_response and (len(fwd_params_response) == len(fwders)):
self.JOB_SCBD.set_value_for_job(job_num, "STATE", "BASE_TASK_PARAMS_SENT")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_BASE_TASK_PARAMS_SENT", get_timestamp())
in_ready_state = {'STATE':'READY_WITH_PARAMS'}
self.FWD_SCBD.set_forwarder_params(fwders, in_ready_state)
# Tell DMCS we are ready
result = self.accept_job(job_num)
else:
#not enough ncsa resources to do job - Notify DMCS
idle_param = {'STATE': 'IDLE'}
self.FWD_SCBD.set_forwarder_params(healthy_forwarders, idle_params)
result = self.insufficient_ncsa_resources(ncsa_response)
return result
else:
result = self.ncsa_no_response(input_params)
idle_param = {'STATE': 'IDLE'}
self.FWD_SCBD.set_forwarder_params(list(forwarder_candidate_dict.keys()), idle_params)
return result
def forwarder_health_check(self, params):
job_num = str(params[JOB_NUM])
raft_list = params['RAFTS']
needed_workers = len(raft_list)
self.JOB_SCBD.add_job(job_num, needed_workers)
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ADDED", get_timestamp())
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ADDED_E", get_epoch_timestamp())
LOGGER.info('Received new job %s. Needed workers is %s', job_num, needed_workers)
# run forwarder health check
# get timed_ack_id
timed_ack = self.get_next_timed_ack_id("FORWARDER_HEALTH_CHECK_ACK")
forwarders = self.FWD_SCBD.return_available_forwarders_list()
# Mark all healthy Forwarders Unknown
state_status = {"STATE": "HEALTH_CHECK", "STATUS": "UNKNOWN"}
self.FWD_SCBD.set_forwarder_params(forwarders, state_status)
# send health check messages
ack_params = {}
ack_params[MSG_TYPE] = FORWARDER_HEALTH_CHECK
ack_params["ACK_ID"] = timed_ack
ack_params[JOB_NUM] = job_num
self.JOB_SCBD.set_value_for_job(job_num, "STATE", "BASE_RESOURCE_QUERY")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_BASE_RESOURCE_QUERY", get_timestamp())
audit_params = {}
audit_params['DATA_TYPE'] = 'FOREMAN_ACK_REQUEST'
audit_params['SUB_TYPE'] = 'FORWARDER_HEALTH_CHECK_ACK'
audit_params['ACK_ID'] = timed_ack
audit_parsms['COMPONENT_NAME'] = 'BASE_FOREMAN'
audit_params['TIME'] = get_epoch_timestamp()
for forwarder in forwarders:
self._base_publisher.publish_message(self.FWD_SCBD.get_value_for_forwarder(forwarder,"CONSUME_QUEUE"),
ack_params)
return timed_ack
def insufficient_base_resources(self, params, healthy_forwarders):
# send response msg to dmcs refusing job
job_num = str(params[JOB_NUM])
raft_list = params[RAFTS]
ack_id = params['ACK_ID']
needed_workers = len(raft_list)
LOGGER.info('Reporting to DMCS that there are insufficient healthy forwarders for job #%s', job_num)
dmcs_params = {}
fail_dict = {}
dmcs_params[MSG_TYPE] = NEW_JOB_ACK
dmcs_params[JOB_NUM] = job_num
dmcs_params[ACK_BOOL] = False
dmcs_params[ACK_ID] = ack_id
### NOTE FOR DMCS ACK PROCESSING:
### if ACK_BOOL == True, there will NOT be a FAIL_DETAILS section
### If ACK_BOOL == False, there will always be a FAIL_DICT to examine AND there will always be a
### BASE_RESOURCES inside the FAIL_DICT
### If ACK_BOOL == False, and the BASE_RESOURCES inside FAIL_DETAILS == 0,
### there will be only NEEDED and AVAILABLE Forwarder params - nothing more
### If ACK_BOOL == False and BASE_RESOURCES inside FAIL_DETAILS == 1, there will always be a
### NCSA_RESOURCES inside FAIL_DETAILS set to either 0 or 'NO_RESPONSE'
### if NCSA_RESPONSE == 0, there will be NEEDED and AVAILABLE Distributor params
### if NCSA_RESOURCES == 'NO_RESPONSE' there will be nothing else
fail_dict['BASE_RESOURCES'] = '0'
fail_dict[NEEDED_FORWARDERS] = str(needed_workers)
fail_dict[AVAILABLE_FORWARDERS] = str(len(healthy_forwarders))
dmcs_params['FAIL_DETAILS'] = fail_dict
self._base_publisher.publish_message("dmcs_consume", dmcs_params)
# mark job refused, and leave Forwarders in Idle state
self.JOB_SCBD.set_value_for_job(job_num, "STATE", "JOB_ABORTED")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ABORTED_BASE_RESOURCES", get_timestamp())
idle_state = {"STATE": "IDLE"}
self.FWD_SCBD.set_forwarder_params(healthy_forwarders, idle_state)
return False
def ncsa_resources_query(self, params, healthy_forwarders):
job_num = str(params[JOB_NUM])
raft_list = params[RAFTS]
needed_workers = len(raft_list)
LOGGER.info('Sufficient forwarders have been found. Checking NCSA')
self._pairs_dict = {}
forwarder_candidate_dict = {}
for i in range (0, needed_workers):
forwarder_candidate_dict[healthy_forwarders[i]] = raft_list[i]
self.FWD_SCBD.set_forwarder_status(healthy_forwarders[i], NCSA_RESOURCES_QUERY)
# Call this method for testing...
# There should be a message sent to NCSA here asking for available resources
timed_ack_id = self.get_next_timed_ack_id("NCSA_Ack")
ncsa_params = {}
ncsa_params[MSG_TYPE] = "NCSA_RESOURCES_QUERY"
ncsa_params[JOB_NUM] = job_num
#ncsa_params[RAFT_NUM] = needed_workers
ncsa_params[ACK_ID] = timed_ack_id
ncsa_params["FORWARDERS"] = forwarder_candidate_dict
self.JOB_SCBD.set_value_for_job(job_num, "STATE", "NCSA_RESOURCES_QUERY_SENT")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_NCSA_RESOURCES_QUERY_SENT", get_timestamp())
self._ncsa_publisher.publish_message(self.NCSA_CONSUME, ncsa_params)
LOGGER.info('The following forwarders have been sent to NCSA for pairing:')
LOGGER.info(forwarder_candidate_dict)
return timed_ack_id
def distribute_job_params(self, params, pairs):
#ncsa has enough resources...
job_num = str(params[JOB_NUM])
self.JOB_SCBD.set_pairs_for_job(job_num, pairs)
self.JOB_SCBD.set_value_for_job(job_num, "TIME_PAIRS_ADDED", get_timestamp())
LOGGER.info('The following pairs will be used for Job #%s: %s',
job_num, pairs)
fwd_ack_id = self.get_next_timed_ack_id("FWD_PARAMS_ACK")
fwders = list(pairs.keys())
fwd_params = {}
fwd_params[MSG_TYPE] = "FORWARDER_JOB_PARAMS"
fwd_params[JOB_NUM] = job_num
fwd_params[ACK_ID] = fwd_ack_id
for fwder in fwders:
fwd_params["TRANSFER_PARAMS"] = pairs[fwder]
route_key = self.FWD_SCBD.get_value_for_forwarder(fwder, "CONSUME_QUEUE")
self._base_publisher.publish_message(route_key, fwd_params)
return fwd_ack_id
def accept_job(self, job_num):
dmcs_message = {}
dmcs_message[JOB_NUM] = job_num
dmcs_message[MSG_TYPE] = NEW_JOB_ACK
dmcs_message[ACK_BOOL] = True
self.JOB_SCBD.set_value_for_job(job_num, STATE, "JOB_ACCEPTED")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ACCEPTED", get_timestamp())
self._base_publisher.publish_message("dmcs_consume", dmcs_message)
return True
def insufficient_ncsa_resources(self, ncsa_response):
dmcs_params = {}
dmcs_params[MSG_TYPE] = "NEW_JOB_ACK"
dmcs_params[JOB_NUM] = job_num
dmcs_params[ACK_BOOL] = False
dmcs_params[BASE_RESOURCES] = '1'
dmcs_params[NCSA_RESOURCES] = '0'
dmcs_params[NEEDED_DISTRIBUTORS] = ncsa_response[NEEDED_DISTRIBUTORS]
dmcs_params[AVAILABLE_DISTRIBUTORS] = ncsa_response[AVAILABLE_DISTRIBUTORS]
#try: FIXME - catch exception
self._base_publisher.publish_message("dmcs_consume", dmcs_params )
#except L1MessageError e:
# return False
return True
def ncsa_no_response(self,params):
#No answer from NCSA...
job_num = str(params[JOB_NUM])
raft_list = params[RAFTS]
needed_workers = len(raft_list)
dmcs_params = {}
dmcs_params[MSG_TYPE] = "NEW_JOB_ACK"
dmcs_params[JOB_NUM] = job_num
dmcs_params[ACK_BOOL] = False
dmcs_params[BASE_RESOURCES] = '1'
dmcs_params[NCSA_RESOURCES] = 'NO_RESPONSE'
self._base_publisher.publish_message("dmcs_consume", dmcs_params )
def process_dmcs_readout(self, params):
job_number = params[JOB_NUM]
pairs = self.JOB_SCBD.get_pairs_for_job(job_number)
date - get_timestamp()
self.JOB_SCBD.set_value_for_job(job_number, TIME_START_READOUT, date)
# The following line extracts the distributor FQNs from pairs dict using
# list comprehension values; faster than for loops
distributors = [v['FQN'] for v in list(pairs.values())]
forwarders = list(pairs.keys())
ack_id = self.get_next_timed_ack_id('NCSA_READOUT')
### Send READOUT to NCSA with ACK_ID
ncsa_params = {}
ncsa_params[MSG_TYPE] = 'NCSA_READOUT'
ncsa_params[ACK_ID] = ack_id
self._ncsa_publisher.publish_message(NCSA_CONSUME, yaml.dump(ncsa_params))
self.ack_timer(4)
ncsa_response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if ncsa_response:
if ncsa_response['ACK_BOOL'] == True:
#inform forwarders
fwd_ack_id = self.get_next_timed_ack_id('FORWARDER_READOUT')
for forwarder in forwarders:
name = self.FWD_SCBD.get_value_for_forwarder(forwarder, NAME)
routing_key = self.FWD_SCBD.get_routing_key(forwarder)
msg_params = {}
msg_params[MSG_TYPE] = 'FORWARDER_READOUT'
msg_params[JOB_NUM] = job_number
msg_params['ACK_ID'] = fwd_ack_id
self.FWD_SCBD.set_forwarder_state(forwarder, START_READOUT)
self._publisher.publish_message(routing_key, yaml.dump(msg_params))
self.ack_timer(4)
forwarder_responses = self.ACK_SCBD.get_components_for_timed_ack(fwd_ack_id)
if len(forwarder_responses) == len(forwarders):
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['ACK_BOOL'] = True
dmcs_params['COMMENT'] = "Readout begun at %s" % get_timestamp()
self._publisher.publish_message('dmcs_consume', yaml.dump(dmcs_params))
else:
#send problem with ncsa to DMCS
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['ACK_BOOL'] = False
dmcs_params['COMMENT'] = 'Readout Failed: Problem at NCSA - Expected Distributor Acks is %s, Number of Distributor Acks received is %s' % (ncsa_response['EXPECTED_DISTRIBUTOR_ACKS'], ncsa_response['RECEIVED_DISTRIBUTOR_ACKS'])
self._base_publisher.publish_message('dmcs_consume', yaml.dump(dmcs_params))
else:
#send 'no response from ncsa' to DMCS )
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['ACK_BOOL'] = False
dmcs_params['COMMENT'] = "Readout Failed: No Response from NCSA"
self._base_publisher.publish_message('dmcs_consume', yaml.dump(dmcs_params))
def process_ack(self, params):
self.ACK_SCBD.add_timed_ack(params)
def get_next_timed_ack_id(self, ack_type):
self._next_timed_ack_id = self._next_timed_ack_id + 1
retval = ack_type + "_" + str(self._next_timed_ack_id).zfill(6)
return retval
def ack_timer(self, seconds):
sleep(seconds)
return True
def purge_broker(self, queues):
for q in queues:
cmd = "rabbitmqctl -p /tester purge_queue " + q
os.system(cmd)
def __init__(self):
self.jscbd = JobScoreboard()
self.fill_job_scoreboard()
class SimJobs:
def __init__(self):
self.jscbd = JobScoreboard()
self.fill_job_scoreboard()
def fill_job_scoreboard(self):
# S1
self.jscbd.set_session("SU_7004")
sleep(0.1)
# V1
self.jscbd.set_visit("V_1001")
sleep(0.2)
# J1
self.jscbd.add_job("PP_14006", "IM4109", 161)
sleep(0.4)
self.jscbd.set_job_params("PP_14006",
{'STATE': 'BASE_RESOURCES_QUERY'})
sleep(0.1)
self.jscbd.set_job_params("PP_14006",
{'STATE': 'NCSA_RESOURCES_QUERY'})
sleep(0.2)
self.jscbd.set_job_params("PP_14006",
{'STATE': 'BASE_EVENT_PARAMS_SENT'})
sleep(0.2)
self.jscbd.set_job_params("PP_14006", {'STATE': 'READY_FOR_EVENT'})
sleep(0.1)
self.jscbd.set_job_params("PP_14006", {'STATE': 'READOUT'})
sleep(0.1)
self.jscbd.set_job_params("PP_14006", {'STATE': 'READOUT_COMPLETE'})
sleep(1.1)
self.jscbd.set_job_params("PP_14006", {'STATE': 'DELIVERY_COMPLETE'})
sleep(2.1)
self.jscbd.set_job_params("PP_14006", {'STATE': '_COMPLETE'})
sleep(0.1)
self.jscbd.set_job_status("PP_14006", 'COMPLETE')
# V2
self.jscbd.set_visit("V_1002")
sleep(0.2)
# J2
self.jscbd.add_job("PP_14007", "IM_4110", 189)
sleep(0.4)
self.jscbd.set_job_params("PP_14007",
{'STATE': 'BASE_RESOURCES_QUERY'})
sleep(0.1)
self.jscbd.set_job_params("PP_14007",
{'STATE': 'NCSA_RESOURCES_QUERY'})
sleep(0.1)
self.jscbd.set_job_params("PP_14007",
{'STATE': 'BASE_EVENT_PARAMS_SENT'})
sleep(0.3)
self.jscbd.set_job_params("PP_14007", {'STATE': 'READY_FOR_EVENT'})
sleep(0.1)
self.jscbd.set_job_params("PP_14007", {'STATE': 'READOUT'})
sleep(2.1)
self.jscbd.set_job_params("PP_14007", {'STATE': 'READOUT_COMPLETE'})
sleep(3.4)
self.jscbd.set_job_params("PP_14007", {'STATE': 'DELIVERY_COMPLETE'})
sleep(0.1)
self.jscbd.set_job_params("PP_14007", {'STATE': '_COMPLETE'})
sleep(0.1)
self.jscbd.set_job_status("PP_14007", 'COMPLETE')
# J3
self.jscbd.add_job("PP_14008", "IM_4111", 189)
sleep(0.4)
self.jscbd.set_job_params("PP_14008",
{'STATE': 'BASE_RESOURCES_QUERY'})
sleep(0.1)
self.jscbd.set_job_params("PP_14008",
{'STATE': 'NCSA_RESOURCES_QUERY'})
sleep(0.1)
self.jscbd.set_job_params("PP_14008", {'STATE': 'SCRUBBED'})
self.jscbd.set_job_status("PP_14008", 'TERMINATED')
# S2
self.jscbd.set_session("SU_7005")
sleep(0.1)
# V3
self.jscbd.set_visit("V_1003")
sleep(0.2)
self.jscbd.add_job("PP_14009", "IM_4112", 161)
sleep(0.4)
self.jscbd.set_job_params("PP_14009",
{'STATE': 'BASE_RESOURCES_QUERY'})
sleep(0.3)
self.jscbd.set_job_params("PP_14009",
{'STATE': 'NCSA_RESOURCES_QUERY'})
sleep(0.4)
self.jscbd.set_job_params("PP_14009",
{'STATE': 'BASE_EVENT_PARAMS_SENT'})
sleep(0.2)
self.jscbd.set_job_params("PP_14009", {'STATE': 'READY_FOR_EVENT'})
sleep(0.1)
self.jscbd.set_job_params("PP_14009", {'STATE': 'READOUT'})
sleep(1.1)
self.jscbd.set_job_params("PP_14009", {'STATE': 'READOUT_COMPLETE'})
sleep(4.1)
self.jscbd.set_job_params("PP_14009", {'STATE': 'DELIVERY_COMPLETE'})
sleep(2.8)
self.jscbd.set_job_params("PP_14009", {'STATE': '_COMPLETE'})
self.jscbd.set_job_status("PP_14009", 'COMPLETE')
print("Job SCBD Load done")
class SimJobs:
def __init__(self):
self.next_timed_ack_id = 46
self.jscbd = JobScoreboard()
self.ascbd = AckScoreboard()
self.fill_job_scoreboard()
def fill_job_scoreboard(self):
# S1
self.jscbd.set_session("SU_7004")
sleep(0.1)
# V1
self.jscbd.set_visit("V_1001")
sleep(0.2)
# J1
self.jscbd.add_job("PP_14006", "IM4109", 161)
sleep(0.4)
self.jscbd.set_job_params("PP_14006", {'STATE': 'BASE_RESOURCES_QUERY'})
sleep(0.1)
for i in range(1, 22):
sleep(0.1)
params = {}
params['ACK_ID'] = self.get_next_timed_ack('forwarder_health_response')
params['MSG_TYPE'] = 'FORWARDER_HEALTH_ACK'
params['COMPONENT_NAME'] = "FORWARDER_" + str(i)
params['JOB_NUM'] = 'PP_14006'
params['ACK_BOOL'] = True
params['IMAGE_ID'] = 'IM4109'
self.ascbd.add_timed_ack(params)
self.jscbd.set_job_params("PP_14006", {'STATE': 'NCSA_RESOURCES_QUERY'})
sleep(0.2)
params = {}
params['ACK_ID'] = self.get_next_timed_ack('ncsa_resource_response')
params['MSG_TYPE'] = 'NCSA_RESOURCE_QUERY_ACK'
params['COMPONENT_NAME'] = "NCSA_FOREMAN"
params['JOB_NUM'] = 'PP_14006'
params['ACK_BOOL'] = True
params['IMAGE_ID'] = 'IM4109'
self.ascbd.add_timed_ack(params)
self.jscbd.set_job_params("PP_14006", {'STATE': 'SENDING_BASE_EVENT_PARAMS'})
sleep(0.2)
for i in range(1, 22):
sleep(0.1)
params = {}
params['ACK_ID'] = self.get_next_timed_ack('forwarder_job_params_response')
params['MSG_TYPE'] = 'FORWARDER_JOB_PARAMS_ACK'
params['COMPONENT_NAME'] = "FORWARDER_" + str(i)
params['JOB_NUM'] = 'PP_14006'
params['ACK_BOOL'] = True
params['IMAGE_ID'] = 'IM4109'
self.ascbd.add_timed_ack(params)
self.jscbd.set_job_params("PP_14006", {'STATE': 'BASE_EVENT_PARAMS_SENT'})
sleep(0.2)
self.jscbd.set_job_params("PP_14006", {'STATE': 'READY_FOR_EVENT'})
sleep(0.1)
self.jscbd.set_job_params("PP_14006", {'STATE': 'READOUT'})
sleep(0.1)
params = {}
params['ACK_ID'] = self.get_next_timed_ack('ncsa_readout_response')
params['MSG_TYPE'] = 'NCSA_READOUT_ACK'
params['COMPONENT_NAME'] = "NCSA_FOREMAN"
params['JOB_NUM'] = 'PP_14006'
params['ACK_BOOL'] = True
params['IMAGE_ID'] = 'IM4109'
self.ascbd.add_timed_ack(params)
for i in range(1, 22):
sleep(0.1)
params = {}
params['ACK_ID'] = self.get_next_timed_ack('readout_response')
params['MSG_TYPE'] = 'FORWARDER_READOUT_ACK'
params['COMPONENT_NAME'] = "FORWARDER_" + str(i)
params['JOB_NUM'] = 'PP_14006'
params['ACK_BOOL'] = True
params['IMAGE_ID'] = 'IM4109'
self.ascbd.add_timed_ack(params)
self.jscbd.set_job_params("PP_14006", {'STATE': 'READOUT_COMPLETE'})
sleep(1.1)
self.jscbd.set_job_params("PP_14006", {'STATE': 'DELIVERY_COMPLETE'})
sleep(2.1)
self.jscbd.set_job_params("PP_14006", {'STATE': '_COMPLETE'})
sleep(0.1)
self.jscbd.set_job_status("PP_14006", 'COMPLETE')
# V2
self.jscbd.set_visit("V_1002")
sleep(0.2)
# J2
self.jscbd.add_job("PP_14007", "IM_4110", 189)
sleep(0.4)
self.jscbd.set_job_params("PP_14007", {'STATE': 'BASE_RESOURCES_QUERY'})
sleep(0.1)
self.jscbd.set_job_params("PP_14007", {'STATE': 'NCSA_RESOURCES_QUERY'})
sleep(0.1)
self.jscbd.set_job_params("PP_14007", {'STATE': 'BASE_EVENT_PARAMS_SENT'})
sleep(0.3)
self.jscbd.set_job_params("PP_14007", {'STATE': 'READY_FOR_EVENT'})
sleep(0.1)
self.jscbd.set_job_params("PP_14007", {'STATE': 'READOUT'})
sleep(2.1)
self.jscbd.set_job_params("PP_14007", {'STATE': 'READOUT_COMPLETE'})
sleep(3.4)
self.jscbd.set_job_params("PP_14007", {'STATE': 'DELIVERY_COMPLETE'})
sleep(0.1)
self.jscbd.set_job_params("PP_14007", {'STATE': '_COMPLETE'})
sleep(0.1)
self.jscbd.set_job_status("PP_14007", 'COMPLETE')
# J3
self.jscbd.add_job("PP_14008", "IM_4111", 189)
sleep(0.4)
self.jscbd.set_job_params("PP_14008", {'STATE': 'BASE_RESOURCES_QUERY'})
sleep(0.1)
for i in range(1, 22):
sleep(0.1)
params = {}
params['ACK_ID'] = self.get_next_timed_ack('forwarder_health_response')
params['MSG_TYPE'] = 'FORWARDER_HEALTH_ACK'
params['COMPONENT_NAME'] = "FORWARDER_" + str(i)
params['JOB_NUM'] = 'PP_14008'
params['ACK_BOOL'] = True
params['IMAGE_ID'] = 'IM4111'
self.ascbd.add_timed_ack(params)
self.jscbd.set_job_params("PP_14008", {'STATE': 'NCSA_RESOURCES_QUERY'})
sleep(0.1)
params = {}
params['ACK_ID'] = self.get_next_timed_ack('ncsa_resource_response')
params['MSG_TYPE'] = 'NCSA_RESOURCE_QUERY_ACK'
params['COMPONENT_NAME'] = "NCSA_FOREMAN"
params['JOB_NUM'] = 'PP_14008'
params['ACK_BOOL'] = False
params['IMAGE_ID'] = 'IM4111'
self.ascbd.add_timed_ack(params)
self.jscbd.set_job_params("PP_14008", {'STATE': 'SCRUBBED'})
self.jscbd.set_job_status("PP_14008", 'TERMINATED')
# S2
self.jscbd.set_session("SU_7005")
sleep(0.1)
# V3
self.jscbd.set_visit("V_1003")
sleep(0.2)
self.jscbd.add_job("PP_14009", "IM_4112", 161)
sleep(0.4)
self.jscbd.set_job_params("PP_14009", {'STATE': 'BASE_RESOURCES_QUERY'})
sleep(0.3)
self.jscbd.set_job_params("PP_14009", {'STATE': 'NCSA_RESOURCES_QUERY'})
sleep(0.4)
self.jscbd.set_job_params("PP_14009", {'STATE': 'BASE_EVENT_PARAMS_SENT'})
sleep(0.2)
self.jscbd.set_job_params("PP_14009", {'STATE': 'READY_FOR_EVENT'})
sleep(0.1)
self.jscbd.set_job_params("PP_14009", {'STATE': 'READOUT'})
sleep(1.1)
self.jscbd.set_job_params("PP_14009", {'STATE': 'READOUT_COMPLETE'})
sleep(4.1)
self.jscbd.set_job_params("PP_14009", {'STATE': 'DELIVERY_COMPLETE'})
sleep(2.8)
self.jscbd.set_job_params("PP_14009", {'STATE': '_COMPLETE'})
self.jscbd.set_job_status("PP_14009", 'COMPLETE')
print("Job SCBD Load done")
def get_next_timed_ack(self, ack_type):
self.next_timed_ack_id = self.next_timed_ack_id + 1
retval = ack_type + "_" + str(self.next_timed_ack_id).zfill(6)
return retval