def main(args):
vehicle = Vehicle("Opel", "Corsa")
vehicle.engine_on()
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
print('Press Ctrl+C to exit')
ThreadManager.waitForFinish()
signal.pause()
return 0
def setup_consumer_threads(self):
""" Create ThreadManager object with base broker url and kwargs to setup consumers.
:params: None.
:return: None.
"""
base_broker_url = "amqp://" + self._msg_name + ":" + \
self._msg_passwd + "@" + \
str(self._base_broker_addr)
LOGGER.info('Building _base_broker_url. Result is %s', base_broker_url)
self.shutdown_event = threading.Event()
self.shutdown_event.clear()
# Set up kwargs that describe consumers to be started
# The Archive Device needs three message consumers
kws = {}
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-aux_foreman_consume'
md['queue'] = 'at_foreman_consume'
md['callback'] = self.on_aux_foreman_message
md['format'] = "YAML"
md['test_val'] = None
kws[md['name']] = md
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-at_foreman_ack_publish'
md['queue'] = 'at_foreman_ack_publish'
md['callback'] = self.on_ack_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-archive_ctrl_publish'
md['queue'] = 'archive_ctrl_publish'
md['callback'] = self.on_archive_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
self.thread_manager = ThreadManager('thread-manager', kws,
self.shutdown_event)
self.thread_manager.start()
class Main(object):
def __init__(self):
self.module_manager = ModuleManager(self)
self.thread_manager = ThreadManager()
self.active = True
def start(self):
self.module_manager.start_modules()
while self.active:
self.thread_manager.check_scheduled()
if not self.active:
self.__exit()
def __exit(self):
self.thread_manager.stop_all()
# Hacky way to get a clean exit when exiting through the display exit button:
os._exit(0)
def setup_consumer(self):
LOGGER.info('Setting up archive consumers on %s',
self._base_broker_url)
LOGGER.info('Running start_new_thread for archive consumer')
self.shutdown_event = threading.Event()
self.shutdown_event.clear()
kws = {}
md = {}
md['amqp_url'] = self._base_broker_url
md['name'] = 'Thread-ar_ctrl_consume'
md['queue'] = self.ARCHIVE_CTRL_CONSUME
md['callback'] = self.on_archive_message
md['format'] = "YAML"
md['test_val'] = None
kws[md['name']] = md
self.thread_manager = ThreadManager('thread-manager', kws,
self.shutdown_event)
self.thread_manager.start()
def setup_consumer_threads(self):
LOGGER.info('Building _base_broker_url')
base_broker_url = "amqp://" + self._sub_base_name + ":" + \
self._sub_base_passwd + "@" + \
str(self._base_broker_addr)
ncsa_broker_url = "amqp://" + self._sub_ncsa_name + ":" + \
self._sub_ncsa_passwd + "@" + \
str(self._ncsa_broker_addr)
self.shutdown_event = threading.Event()
# Set up kwargs that describe consumers to be started
# The Archive Device needs three message consumers
kws = {}
md = {}
md['amqp_url'] = ncsa_broker_url
md['name'] = 'Thread-ncsa_foreman_ack_publish'
md['queue'] = 'ncsa_foreman_ack_publish'
md['callback'] = self.on_ack_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-ncsa_consume'
md['queue'] = 'ncsa_consume'
md['callback'] = self.on_pp_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
self.thread_manager = ThreadManager('thread-manager', kws,
self.shutdown_event)
self.thread_manager.start()
def signal_handler(sig, frame): print "Signal received, stopping" ThreadManager.terminate() sys.exit(42)
class NcsaForeman:
NCSA_CONSUME = "ncsa_consume"
NCSA_PUBLISH = "ncsa_publish"
COMPONENT_NAME = 'NCSA_FOREMAN'
DISTRIBUTOR_PUBLISH = "distributor_publish"
ACK_PUBLISH = "ack_publish"
CFG_FILE = 'L1SystemCfg.yaml'
prp = toolsmod.prp
def __init__(self, filename=None):
toolsmod.singleton(self)
self._config_file = self.CFG_FILE
if filename != None:
self._config_file = filename
#self._pairing_dict = {}
LOGGER.info('Extracting values from Config dictionary')
self.extract_config_values()
self._msg_actions = {
'NCSA_NEXT_VISIT': self.set_visit,
'NCSA_NEW_SESSION': self.set_session,
'NCSA_START_INTEGRATION': self.process_start_integration,
'NCSA_READOUT': self.process_readout,
'DISTRIBUTOR_HEALTH_CHECK_ACK': self.process_ack,
'DISTRIBUTOR_XFER_PARAMS_ACK': self.process_ack,
'DISTRIBUTOR_READOUT_ACK': self.process_ack
}
self._next_timed_ack_id = 10000
self.setup_publishers()
self.setup_scoreboards()
self.setup_publishers()
self.setup_consumer_threads()
LOGGER.info('Ncsa foreman consumer setup')
self.thread_manager = None
self.setup_consumer_threads()
LOGGER.info('Ncsa Foreman Init complete')
def setup_publishers(self):
self._pub_base_broker_url = "amqp://" + self._pub_base_name + ":" + \
self._pub_base_passwd + "@" + \
str(self._base_broker_addr)
self._pub_ncsa_broker_url = "amqp://" + self._pub_ncsa_name + ":" + \
self._pub_ncsa_passwd + "@" + \
str(self._ncsa_broker_addr)
LOGGER.info('Setting up Base publisher on %s using %s', \
self._pub_base_broker_url, self._base_msg_format)
self._base_publisher = SimplePublisher(self._pub_base_broker_url,
self._base_msg_format)
LOGGER.info('Setting up NCSA publisher on %s using %s', \
self._pub_ncsa_broker_url, self._ncsa_msg_format)
self._ncsa_publisher = SimplePublisher(self._pub_ncsa_broker_url,
self._ncsa_msg_format)
def on_pp_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
msg_dict = body
LOGGER.debug('Message from PP callback message body is: %s',
self.prp.pformat(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def on_ack_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
msg_dict = body
LOGGER.info('In ACK message callback')
LOGGER.debug('Message from ACK callback message body is: %s',
self.prp.pformat(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def set_visit(self, params):
bore_sight = params['BORE_SIGHT']
visit_id = params['VISIT_ID']
self.JOB_SCBD.set_visit_id(visit_id, bore_sight)
ack_id = params['ACK_ID']
msg = {}
###
### Send Boresight to Someone here...
###
msg['MSG_TYPE'] = 'NCSA_NEXT_VISIT_ACK'
msg['COMPONENT'] = self.COMPONENT_NAME
msg['ACK_ID'] = ack_id
msg['ACK_BOOL'] = True
route_key = params['REPLY_QUEUE']
self._base_publisher.publish_message(route_key, msg)
def set_session(self, params):
self.JOB_SCBD.set_session(params['SESSION_ID'])
ack_id = params['ACK_ID']
msg = {}
msg['MSG_TYPE'] = 'NCSA_NEW_SESSION_ACK'
msg['COMPONENT'] = self.COMPONENT_NAME
msg['ACK_ID'] = ack_id
msg['ACK_BOOL'] = True
route_key = params['REPLY_QUEUE']
self._base_publisher.publish_message(route_key, msg)
def process_start_integration(self, params):
job_num = str(params[JOB_NUM])
image_id = params['IMAGE_ID']
visit_id = params['VISIT_ID']
response_timed_ack_id = params["ACK_ID"]
LOGGER.info('NCSA received Start Integration message from Base')
LOGGER.debug('NCSA Start Integration incoming message: %s' % params)
forwarders_list = params['FORWARDERS']['FORWARDER_LIST']
ccd_list = params['FORWARDERS']['CCD_LIST'] # A list of lists...
len_forwarders_list = len(forwarders_list)
self.JOB_SCBD.add_job(job_num, image_id, visit_id, ccd_list)
LOGGER.info('Received new job %s. Needed workers is %s', job_num,
str(len_forwarders_list))
# run distributor health check
# get timed_ack_id
timed_ack = self.get_next_timed_ack_id("DISTRIBUTOR_HEALTH_CHECK_ACK")
distributors = self.DIST_SCBD.return_distributors_list()
# Mark all healthy distributors Unknown
state_unknown = {"STATE": "HEALTH_CHECK", "STATUS": "UNKNOWN"}
self.DIST_SCBD.set_distributor_params(distributors, state_unknown)
# send health check messages
ack_params = {}
ack_params[MSG_TYPE] = "DISTRIBUTOR_HEALTH_CHECK"
ack_params['REPLY_QUEUE'] = 'ncsa_foreman_ack_publish'
ack_params["ACK_ID"] = timed_ack
ack_params[JOB_NUM] = job_num
for distributor in distributors:
self._ncsa_publisher.publish_message(
self.DIST_SCBD.get_value_for_distributor(
distributor, "CONSUME_QUEUE"), ack_params)
# start timers
self.ack_timer(2)
# at end of timer, get list of distributors
dicts_of_distributors = self.ACK_SCBD.get_components_for_timed_ack(
timed_ack)
healthy_distributors = list(dicts_of_distributors.keys())
# update distributor scoreboard with healthy distributors
healthy_status = {"STATUS": "HEALTHY"}
self.DIST_SCBD.set_distributor_params(healthy_distributors,
healthy_status)
num_healthy_distributors = len(healthy_distributors)
if len_forwarders_list > num_healthy_distributors:
print("Cannot Do Job - more fwdrs than dists")
# send response msg to base refusing job
LOGGER.info(
'Reporting to base insufficient healthy distributors for job #%s',
job_num)
ncsa_params = {}
ncsa_params[MSG_TYPE] = "NCSA_RESOURCES_QUERY_ACK"
ncsa_params[JOB_NUM] = job_num
ncsa_params["ACK_BOOL"] = False
ncsa_params["ACK_ID"] = response_timed_ack_id
self._base_publisher.publish_message(NCSA_PUBLISH,
yaml.dump(ncsa_params))
# delete job and leave distributors in Idle state
self.JOB_SCBD.delete_job(job_num)
idle_state = {"STATE": "IDLE"}
self.DIST_SCBD.set_distributor_params(healthy_distributors,
idle_state)
else:
Pairs = self.assemble_pairs(forwarders_list, ccd_list,
healthy_distributors)
self.JOB_SCBD.set_pairs_for_job(job_num, Pairs)
# send pair info to each distributor
job_params_ack = self.get_next_timed_ack_id(
'DISTRIBUTOR_XFER_PARAMS_ACK')
for j in range(0, len(Pairs)):
tmp_msg = {}
tmp_msg[MSG_TYPE] = 'DISTRIBUTOR_XFER_PARAMS'
tmp_msg['XFER_PARAMS'] = Pairs[j]
tmp_msg[JOB_NUM] = job_num
tmp_msg[ACK_ID] = job_params_ack
tmp_msg['REPLY_QUEUE'] = 'ncsa_foreman_ack_publish'
tmp_msg['VISIT_ID'] = visit_id
tmp_msg['IMAGE_ID'] = image_id
fqn = Pairs[j]['DISTRIBUTOR']['FQN']
route_key = self.DIST_SCBD.get_value_for_distributor(
fqn, 'CONSUME_QUEUE')
self._ncsa_publisher.publish_message(route_key, tmp_msg)
self.DIST_SCBD.set_distributor_params(healthy_distributors,
{STATE: IN_READY_STATE})
dist_params_response = self.progressive_ack_timer(
job_params_ack, num_healthy_distributors, 2.0)
if dist_params_response == None:
print(
"RECEIVED NO ACK RESPONSES FROM DISTRIBUTORS AFTER SENDING XFER PARAMS"
)
pass #Do something such as raise a system wide exception
# Now inform PP Foreman that all is in ready state
ncsa_params = {}
ncsa_params[MSG_TYPE] = "NCSA_START_INTEGRATION_ACK"
ncsa_params[JOB_NUM] = job_num
ncsa_params['IMAGE_ID'] = image_id
ncsa_params['VISIT_ID'] = visit_id
ncsa_params['SESSION_ID'] = params['SESSION_ID']
ncsa_params['COMPONENT'] = 'NCSA_FOREMAN'
ncsa_params[ACK_BOOL] = True
ncsa_params["ACK_ID"] = response_timed_ack_id
ncsa_params["PAIRS"] = Pairs
self._base_publisher.publish_message(params['REPLY_QUEUE'],
ncsa_params)
LOGGER.info(
'Sufficient distributors and workers are available. Informing Base'
)
LOGGER.debug('NCSA Start Integration incoming message: %s' %
ncsa_params)
LOGGER.info(
'The following pairings have been sent to the Base for job %s:'
% job_num)
LOGGER.info(Pairs)
def assemble_pairs(self, forwarders_list, ccd_list, healthy_distributors):
#build dict...
PAIRS = []
for i in range(0, len(forwarders_list)):
tmp_dict = {}
sub_dict = {}
tmp_dict['FORWARDER'] = forwarders_list[i]
tmp_dict['CCD_LIST'] = ccd_list[i]
tmp_dict['DISTRIBUTOR'] = {}
distributor = healthy_distributors[i]
sub_dict['FQN'] = distributor
sub_dict['HOSTNAME'] = self.DIST_SCBD.get_value_for_distributor(
distributor, HOSTNAME)
sub_dict['NAME'] = self.DIST_SCBD.get_value_for_distributor(
distributor, NAME)
sub_dict['IP_ADDR'] = self.DIST_SCBD.get_value_for_distributor(
distributor, IP_ADDR)
sub_dict['TARGET_DIR'] = self.DIST_SCBD.get_value_for_distributor(
distributor, TARGET_DIR)
tmp_dict['DISTRIBUTOR'] = sub_dict
PAIRS.append(tmp_dict)
return PAIRS
def process_readout(self, params):
job_number = params[JOB_NUM]
response_ack_id = params[ACK_ID]
pairs = self.JOB_SCBD.get_pairs_for_job(job_number)
sleep(3)
len_pairs = len(pairs)
ack_id = self.get_next_timed_ack_id(DISTRIBUTOR_READOUT_ACK)
# The following line extracts the distributor FQNs from pairs dict using
# list comprehension values; faster than for loops
# distributors = [v['FQN'] for v in list(pairs.values())]
for i in range(0, len_pairs): # Pairs is a list of dictionaries
distributor = pairs[i]['DISTRIBUTOR']['FQN']
msg_params = {}
msg_params[MSG_TYPE] = DISTRIBUTOR_READOUT
msg_params[JOB_NUM] = job_number
msg_params['REPLY_QUEUE'] = 'ncsa_foreman_ack_publish'
msg_params[ACK_ID] = ack_id
routing_key = self.DIST_SCBD.get_routing_key(distributor)
self.DIST_SCBD.set_distributor_state(distributor, 'START_READOUT')
self._ncsa_publisher.publish_message(routing_key, msg_params)
distributor_responses = self.progressive_ack_timer(
ack_id, len_pairs, 24)
if distributor_responses != None:
RESULT_LIST = {}
CCD_LIST = []
RECEIPT_LIST = []
ncsa_params = {}
ncsa_params[MSG_TYPE] = NCSA_READOUT_ACK
ncsa_params[JOB_NUM] = job_number
ncsa_params['IMAGE_ID'] = params['IMAGE_ID']
ncsa_params['VISIT_ID'] = params['VISIT_ID']
ncsa_params['SESSION_ID'] = params['SESSION_ID']
ncsa_params['COMPONENT'] = 'NCSA_FOREMAN'
ncsa_params[ACK_ID] = response_ack_id
ncsa_params[ACK_BOOL] = True
distributors = list(distributor_responses.keys())
for dist in distributors:
ccd_list = distributor_responses[dist]['RESULT_LIST'][
'CCD_LIST']
receipt_list = distributor_responses[dist]['RESULT_LIST'][
'RECEIPT_LIST']
for i in range(0, len(ccd_list)):
CCD_LIST.append(ccd_list[i])
RECEIPT_LIST.append(receipt_list[i])
RESULT_LIST['CCD_LIST'] = CCD_LIST
RESULT_LIST['RECEIPT_LIST'] = RECEIPT_LIST
ncsa_params['RESULT_LIST'] = RESULT_LIST
self._base_publisher.publish_message(params['REPLY_QUEUE'],
msg_params)
else:
ncsa_params = {}
ncsa_params[MSG_TYPE] = NCSA_READOUT_ACK
ncsa_params[JOB_NUM] = job_number
ncsa_params['COMPONENT_NAME'] = NCSA_FOREMAN
ncsa_params['IMAGE_ID'] = params['IMAGE_ID']
ncsa_params['VISIT_ID'] = params['VISIT_ID']
ncsa_params['SESSION_ID'] = params['SESSION_ID']
ncsa_params[ACK_ID] = response_ack_id
ncsa_params[ACK_BOOL] = FALSE
ncsa_params['RESULT_LIST'] = {}
ncsa_params['RESULT_LIST']['CCD_LIST'] = None
ncsa_params['RESULT_LIST']['RECEIPT_LIST'] = None
self._base_publisher.publish_message(params['REPLY_QUEUE'],
msg_params)
def process_ack(self, params):
self.ACK_SCBD.add_timed_ack(params)
def get_next_timed_ack_id(self, ack_type):
self._next_timed_ack_id = self._next_timed_ack_id + 1
retval = ack_type + "_" + str(self._next_timed_ack_id).zfill(6)
return retval
def ack_timer(self, seconds):
sleep(seconds)
return True
def progressive_ack_timer(self, ack_id, expected_replies, seconds):
counter = 0.0
while (counter < seconds):
counter = counter + 0.5
sleep(0.5)
response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if response == None:
continue
if len(list(response.keys())) == expected_replies:
return response
## Try one final time
response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if response == None:
return None
elif len(list(response.keys())) == expected_replies:
return response
else:
return None
def extract_config_values(self):
try:
cdm = toolsmod.intake_yaml_file(self._config_file)
except IOError as e:
LOGGER.critical("Unable to find CFG Yaml file %s\n" %
self._config_file)
sys.exit(101)
try:
self._base_broker_addr = cdm[ROOT][BASE_BROKER_ADDR]
self._ncsa_broker_addr = cdm[ROOT][NCSA_BROKER_ADDR]
self._sub_base_name = cdm[ROOT][
'NFM_BASE_BROKER_NAME'] # Message broker user & passwd
self._sub_base_passwd = cdm[ROOT]['NFM_BASE_BROKER_PASSWD']
self._sub_ncsa_name = cdm[ROOT][
'NFM_NCSA_BROKER_NAME'] # Message broker user & passwd
self._sub_ncsa_passwd = cdm[ROOT]['NFM_NCSA_BROKER_PASSWD']
self._pub_base_name = cdm[ROOT]['BASE_BROKER_PUB_NAME']
self._pub_base_passwd = cdm[ROOT]['BASE_BROKER_PUB_PASSWD']
self._pub_ncsa_name = cdm[ROOT]['NCSA_BROKER_PUB_NAME']
self._pub_ncsa_passwd = cdm[ROOT]['NCSA_BROKER_PUB_PASSWD']
self._scbd_dict = cdm[ROOT]['SCOREBOARDS']
self.distributor_dict = cdm[ROOT][XFER_COMPONENTS][DISTRIBUTORS]
except KeyError as e:
LOGGER.critical("CDM Dictionary error - missing Key")
LOGGER.critical("Offending Key is %s", str(e))
LOGGER.critical("Bailing Out...")
sys.exit(99)
self._base_msg_format = 'YAML'
self._ncsa_msg_format = 'YAML'
if 'BASE_MSG_FORMAT' in cdm[ROOT]:
self._base_msg_format = cdm[ROOT][BASE_MSG_FORMAT]
if 'NCSA_MSG_FORMAT' in cdm[ROOT]:
self._ncsa_msg_format = cdm[ROOT][NCSA_MSG_FORMAT]
def setup_consumer_threads(self):
LOGGER.info('Building _base_broker_url')
base_broker_url = "amqp://" + self._sub_base_name + ":" + \
self._sub_base_passwd + "@" + \
str(self._base_broker_addr)
ncsa_broker_url = "amqp://" + self._sub_ncsa_name + ":" + \
self._sub_ncsa_passwd + "@" + \
str(self._ncsa_broker_addr)
self.shutdown_event = threading.Event()
# Set up kwargs that describe consumers to be started
# The Archive Device needs three message consumers
kws = {}
md = {}
md['amqp_url'] = ncsa_broker_url
md['name'] = 'Thread-ncsa_foreman_ack_publish'
md['queue'] = 'ncsa_foreman_ack_publish'
md['callback'] = self.on_ack_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-ncsa_consume'
md['queue'] = 'ncsa_consume'
md['callback'] = self.on_pp_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
self.thread_manager = ThreadManager('thread-manager', kws,
self.shutdown_event)
self.thread_manager.start()
def setup_scoreboards(self):
# Create Redis Distributor table with Distributor info
self.DIST_SCBD = DistributorScoreboard('NCSA_DIST_SCBD', \
self._scbd_dict['NCSA_DIST_SCBD'], \
self.distributor_dict)
self.JOB_SCBD = JobScoreboard('NCSA_JOB_SCBD',
self._scbd_dict['NCSA_JOB_SCBD'])
self.ACK_SCBD = AckScoreboard('NCSA_ACK_SCBD',
self._scbd_dict['NCSA_ACK_SCBD'])
def shutdown(self):
LOGGER.debug("NCSA Foreman: Shutting down Consumer threads.")
self.shutdown_event.set()
LOGGER.debug("Thread Manager shutting down and app exiting...")
print("\n")
os._exit(0)
def defaultDispatcher_logic(operation): if operation == "exit": ThreadManager.stopAllThreads() return "ok" else: return "unknown"
class AuxDevice:
""" The Spec Device is a commandable device which coordinates the ingest of
images from the telescope camera and then the transfer of those images to
the base site archive storage.
It receives jobs and divides and assigns the work to forwarders, records state and
status change of forwarders, and sends messages accordingly.
"""
COMPONENT_NAME = 'AUX_FOREMAN'
AT_FOREMAN_CONSUME = "at_foreman_consume"
ARCHIVE_CTRL_PUBLISH = "archive_ctrl_publish"
ARCHIVE_CTRL_CONSUME = "archive_ctrl_consume"
AT_FOREMAN_ACK_PUBLISH = "at_foreman_ack_publish"
START_INTEGRATION_XFER_PARAMS = {}
ACK_QUEUE = []
CFG_FILE = 'L1SystemCfg.yaml'
prp = toolsmod.prp
DP = toolsmod.DP
RAFT_LIST = []
RAFT_CCD_LIST = ['00']
def __init__(self, filename=None):
""" Create a new instance of the Spectrograph Device class.
Instantiate the instance, raise assertion error if already instantiated.
Extract config values from yaml file.
Store handler methods for each message type.
Set up base broker url, publishers, and scoreboards. Consumer threads
are started within a Thread Manager object so that they can be monitored
for health and shutdown/joined cleanly when the app exits.
:params filename: Deflaut 'L1SystemCfg.yaml'. Can be assigned by user.
:return: None.
"""
toolsmod.singleton(self)
self._config_file = self.CFG_FILE
if filename != None:
self._config_file = filename
LOGGER.info('Extracting values from Config dictionary')
self.extract_config_values()
#self.purge_broker(cdm['ROOT']['QUEUE_PURGES'])
self._msg_actions = {
'AT_START_INTEGRATION': self.process_at_start_integration,
'AT_NEW_SESSION': self.set_session,
#'AR_READOUT': self.process_dmcs_readout,
'AUX_FWDR_HEALTH_CHECK_ACK': self.process_ack,
'AUX_FWDR_XFER_PARAMS_ACK': self.process_ack,
'AR_FWDR_READOUT_ACK': self.process_ack,
'AR_ITEMS_XFERD_ACK': self.process_ack,
'AT_HEADER_READY': self.process_header_ready_event,
'NEW_ARCHIVE_ITEM_ACK': self.process_ack,
#'AUX_TAKE_IMAGES': self.take_images,
'AT_END_READOUT': self.process_at_end_readout
}
self._next_timed_ack_id = 0
self.setup_publishers()
LOGGER.info('ar foreman consumer setup')
self.thread_manager = None
self.setup_consumer_threads()
LOGGER.info('Archive Foreman Init complete')
def setup_publishers(self):
""" Set up base publisher with pub_base_broker_url by creating a new instance
of SimplePublisher class with yaml format
:params: None.
:return: None.
"""
self.pub_base_broker_url = "amqp://" + self._msg_pub_name + ":" + \
self._msg_pub_passwd + "@" + \
str(self._base_broker_addr)
LOGGER.info('Setting up Base publisher on %s using %s',
self.pub_base_broker_url, self._base_msg_format)
self._publisher = SimplePublisher(self.pub_base_broker_url,
self._base_msg_format)
def on_aux_foreman_message(self, ch, method, properties, body):
""" Calls the appropriate AR message action handler according to message type.
:params ch: Channel to message broker, unused unless testing.
:params method: Delivery method from Pika, unused unless testing.
:params properties: Properties from DMCS to AR Foreman callback message
body, unused unless testing.
:params body: A dictionary that stores the message body.
:return: None.
"""
#msg_dict = yaml.load(body)
ch.basic_ack(method.delivery_tag)
msg_dict = body
LOGGER.info('In AUX Foreman message callback')
LOGGER.info(
'Message from DMCS to AUX Foreman callback message body is: %s',
str(msg_dict))
print("Incoming AUX msg is: %s" % msg_dict)
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def on_archive_message(self, ch, method, properties, body):
""" Calls the appropriate AR message action handler according to message type.
:params ch: Channel to message broker, unused unless testing.
:params method: Delivery method from Pika, unused unless testing.
:params properties: Properties from AR CTRL callback message body,
unused unless testing.
:params body: A dictionary that stores the message body.
:return: None.
"""
ch.basic_ack(method.delivery_tag)
LOGGER.info('AR CTRL callback msg body is: %s', str(body))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def on_ack_message(self, ch, method, properties, body):
""" Calls the appropriate AR message action handler according to message type.
:params ch: Channel to message broker, unused unless testing.
:params method: Delivery method from Pika, unused unless testing.
:params properties: Properties from ACK callback message body, unused
unless testing.
:params body: A dictionary that stores the message body.
:return: None.
"""
ch.basic_ack(method.delivery_tag)
msg_dict = body
print("")
print("")
print("")
print("RECEIVING ack MESSAGE:")
print(msg_dict)
print("")
print("")
print("")
# XXX FIX Ignoring all log messages
return
LOGGER.info('In ACK message callback')
LOGGER.info('Message from ACK callback message body is: %s',
str(msg_dict))
# XXX FIX Ignoring all log messages
return
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def process_at_start_integration(self, params):
# When this method is invoked, the following must happen:
# 1) Health check all forwarders
# 2) Divide work and generate dict of forwarders and which rafts/ccds they are fetching
# 3) Inform each forwarder which rafts they are responsible for
# receive new job_number and image_id; session and visit are current
# and deep copy it with some additions such as session and visit
# These next three lines must have WFS and Guide sensor info added
start_int_ack_id = params[ACK_ID]
print("Incoming AUX AT_Start Int msg")
# next, run health check
self.ACK_QUEUE = {}
health_check_ack_id = self.get_next_timed_ack_id('AUX_FWDR_HEALTH_ACK')
num_fwdrs_checked = self.fwdr_health_check(health_check_ack_id)
# Add job scbd entry
self.ack_timer(1.4)
#healthy_fwdrs = self.ACK_QUEUE.get_components_for_timed_ack(health_check_ack_id)
#if healthy_fwdrs == None:
# self.refuse_job(params, "No forwarders available")
# ### FIX send error code for this...
# return
fwdr_names = list(self._forwarder_dict.keys())
self._current_fwdr = self._forwarder_dict[fwdr_names[0]]
# Add archive check when necessary...
# send new_archive_item msg to archive controller
#start_int_params = {}
#ac_timed_ack = self.get_next_timed_ack_id('AUX_CTRL_NEW_ITEM')
#start_int_params[MSG_TYPE] = 'NEW_ARCHIVE_ITEM'
#start_int_params['ACK_ID'] = ac_timed_ack
#start_int_params['JOB_NUM'] = job_number
#start_int_params['SESSION_ID'] = session_id
#start_int_params['VISIT_ID'] = visit_id
#start_int_params['IMAGE_ID'] = image_id
#start_int_params['REPLY_QUEUE'] = self.AUX_FOREMAN_ACK_PUBLISH
#self.JOB_SCBD.set_job_state(job_number, 'AR_NEW_ITEM_QUERY')
#self._publisher.publish_message(self.ARCHIVE_CTRL_CONSUME, start_int_params)
#ar_response = self.progressive_ack_timer(ac_timed_ack, 1, 2.0)
#if ar_response == None:
# FIXME raise L1 exception and bail out
# print("B-B-BAD Trouble; no ar_response")
#target_dir = ar_response['ARCHIVE_CTRL']['TARGET_DIR']
target_dir = self.archive_xfer_root
#self.JOB_SCBD.set_job_params(job_number, {'STATE':'AR_NEW_ITEM_RESPONSE', 'TARGET_DIR': dir})
# divide image fetch across forwarders
#list_of_fwdrs = list(healthy_fwdrs.keys())
#work_schedule = self.divide_work(list_of_fwdrs, raft_list, raft_ccd_list)
# send target dir, and job, session,visit and work to do to healthy forwarders
#self.JOB_SCBD.set_value_for_job(job_number, 'STATE','SENDING_XFER_PARAMS')
#set_sched_result = self.JOB_SCBD.set_work_schedule_for_job(job_number, work_schedule)
#if set_sched_result == False:
# FIXME Raise L1 exception and bail
# print("BIG PROBLEM - CANNOT SET WORK SCHED IN SCBD")
xfer_params_ack_id = self.get_next_timed_ack_id("AT_FWDR_PARAMS_ACK")
fwdr_new_target_params = {}
fwdr_new_target_params['XFER_PARAMS'] = {}
fwdr_new_target_params[MSG_TYPE] = 'AT_FWDR_XFER_PARAMS'
#fwdr_new_target_params[SESSION_ID] = session_id
fwdr_new_target_params[IMAGE_ID] = params[IMAGE_ID]
fwdr_new_target_params['IMAGE_INDEX'] = params['IMAGE_INDEX']
#fwdr_new_target_params[VISIT_ID] = visit_id
#fwdr_new_target_params[JOB_NUM] = job_number
fwdr_new_target_params[ACK_ID] = xfer_params_ack_id
fwdr_new_target_params[REPLY_QUEUE] = self.AT_FOREMAN_ACK_PUBLISH
target_location = self.archive_name + "@" + self.archive_ip + ":" + target_dir
fwdr_new_target_params['TARGET_LOCATION'] = target_location
xfer_params_dict = {}
xfer_params_dict['RAFT_LIST'] = self._wfs_raft
#xfer_params_dict['RAFT_LIST'] = self.RAFT_LIST
#xfer_params_dict['RAFT_LIST'].append(self.RAFT_LIST)
#xfer_params_dict['RAFT_CCD_LIST'] = []
#xfer_params_dict['RAFT_CCD_LIST'].append(self.RAFT_CCD_LIST)
xfer_params_dict['AT_FWDR'] = self._current_fwdr
fwdr_new_target_params['XFER_PARAMS'] = xfer_params_dict
route_key = self._current_fwdr["CONSUME_QUEUE"]
self._publisher.publish_message(route_key, fwdr_new_target_params)
"""
# receive ack back from forwarders that they have job params
params_acks = self.progressive_ack_timer(xfer_params_ack_id, len_fwdrs_list, 3.0)
### FIX
# if params_acks == None:
# raise L1Exception and bail
self.JOB_SCBD.set_value_for_job(job_number,'STATE','XFER_PARAMS_SENT')
# accept job by Ach'ing True
st_int_params_ack = {}
st_int_params_ack['MSG_TYPE'] = 'AR_START_INTEGRATION_ACK'
st_int_params_ack['ACK_ID'] = start_int_ack_id
st_int_params_ack['ACK_BOOL'] = True
st_int_params_ack['JOB_NUM'] = job_number
st_int_params_ack['SESSION_ID'] = session_id
st_int_params_ack['IMAGE_ID'] = image_id
st_int_params_ack['VISIT_ID'] = visit_id
st_int_params_ack['COMPONENT'] = self.COMPONENT_NAME
self.accept_job(st_int_params_ack)
self.JOB_SCBD.set_value_for_job(job_number, STATE, "JOB_ACCEPTED")
fscbd_params = {'STATE':'AWAITING_READOUT'}
self.FWD_SCBD.set_forwarder_params(healthy_fwdrs, fscbd_params)
"""
def fwdr_health_check(self, ack_id):
""" Send AR_FWDR_HEALTH_CHECK message to ar_foreman_ack_publish queue.
Retrieve available forwarders from ForwarderScoreboard, set their state to
HEALTH_CHECK, status to UNKNOWN, and publish the message.
:params ack_id: Ack id for AR forwarder health check.
:return: Number of health checks sent.
"""
msg_params = {}
msg_params[MSG_TYPE] = 'AT_FWDR_HEALTH_CHECK'
msg_params[ACK_ID] = ack_id
msg_params[REPLY_QUEUE] = self.AT_FOREMAN_ACK_PUBLISH
forwarders = list(self._forwarder_dict.keys())
for x in range(0, len(forwarders)):
route_key = self._forwarder_dict[forwarders[x]]["CONSUME_QUEUE"]
self._publisher.publish_message(route_key, msg_params)
return len(forwarders)
def divide_work(self, fwdrs_list, raft_list, raft_ccd_list):
""" Divide work (ccds) among forwarders.
If only one forwarder available, give it all the work.
If have less or equal ccds then forwarders, give the first few forwarders one
ccd each.
Else, evenly distribute ccds among forwarders, and give extras to the first
forwarder, make sure that ccd list for each forwarder is continuous.
:params fwdrs_list: List of available forwarders for the job.
:params ccd_list: List of ccds to be distributed.
:return schedule: Distribution of ccds among forwarders.
"""
num_fwdrs = len(fwdrs_list)
num_rafts = len(raft_list)
schedule = {}
schedule['FORWARDER_LIST'] = []
schedule['CCD_LIST'] = [
] # A list of ccd lists; index of main list matches same forwarder list index
FORWARDER_LIST = []
RAFT_LIST = [] # This is a 'list of lists'
RAFT_CCD_LIST = [] # This is a 'list of lists'
if num_fwdrs == 1:
FORWARDER_LIST.append(fwdrs_list[0])
RAFT_LIST = deepcopy(raft_list)
RAFT_CCD_LIST = deepcopy(raft_ccd_list)
schedule['FORWARDER_LIST'] = FORWARDER_LIST
schedule['RAFT_LIST'] = RAFT_LIST
schedule['RAFT_CCD_LIST'] = RAFT_CCD_LIST
return schedule
if num_rafts <= num_fwdrs:
for k in range(0, num_rafts):
FORWARDER_LIST.append(fwdrs_list[k])
#little_list.append(ccd_list[k])
RAFT_LIST.append(raft_list[k]) # Need a copy here...
RAFT_CCD_LIST.append = deepcopy(raft_ccd_list[k])
schedule['FORWARDER_LIST'] = FORWARDER_LIST
schedule['RAFT_LIST'] = RAFT_LIST
schedule['RAFT_CCD_LIST'] = RAFT_CCD_LIST
else:
rafts_per_fwdr = len(raft_list) // num_fwdrs
remainder_rafts = len(raft_list) % num_fwdrs
offset = 0
for i in range(0, num_fwdrs):
tmp_list = []
tmp_raft_list = []
for j in range(offset, (rafts_per_fwdr + offset)):
if (j) >= num_rafts:
break
tmp_list.append(raft_list[j])
tmp_raft_list.append(deepcopy(raft_ccd_list[j]))
offset = offset + rafts_per_fwdr
# If num_fwdrs divided into num_rafts equally, we are done...else, deal with remainder
if remainder_rafts != 0 and i == 0:
for k in range(offset, offset + remainder_rafts):
tmp_list.append(raft_list[k])
tmp_raft_list.append(deepcopy(raft_ccd_list[k]))
offset = offset + remainder_rafts
FORWARDER_LIST.append(fwdrs_list[i])
RAFT_LIST.append(list(tmp_list))
RAFT_CCD_LIST.append(list(tmp_raft_list))
schedule['FORWARDER_LIST'] = FORWARDER_LIST
schedule['RAFT_LIST'] = RAFT_LIST
schedule['RAFT_CCD_LIST'] = RAFT_CCD_LIST
return schedule
def accept_job(self, dmcs_message):
""" Send AR_START_INTEGRATION_ACK message with ack_bool equals True (job accepted)
and other job specs to dmcs_ack_consume queue.
:params dmcs_message: A dictionary that stores info of a job.
:return: None.
"""
self._publisher.publish_message("dmcs_ack_consume", dmcs_message)
def refuse_job(self, params, fail_details):
""" Send AR_START_INTEGRATION_ACK message with ack_bool equals False (job refused)
and other job specs to dmcs_ack_consume queue.
Set job state as JOB_REFUSED in JobScoreboard.
:params parmas: A dictionary that stores info of a job.
:params fail_details: A string that describes what went wrong, not used for now.
:return: None.
"""
dmcs_message = {}
dmcs_message[JOB_NUM] = params[JOB_NUM]
dmcs_message[MSG_TYPE] = 'AR_START_INTEGRATION_ACK'
dmcs_message['ACK_ID'] = params['ACK_ID']
dmcs_message['SESSION_ID'] = params['SESSION_ID']
dmcs_message['VISIT_ID'] = params['VISIT_ID']
dmcs_message['IMAGE_ID'] = params['IMAGE_ID']
dmcs_message[ACK_BOOL] = False
dmcs_message['COMPONENT'] = self.COMPONENT_NAME
self.JOB_SCBD.set_value_for_job(params[JOB_NUM], STATE, "JOB_REFUSED")
self._publisher.publish_message("dmcs_ack_consume", dmcs_message)
def process_at_end_readout(self, params):
""" Set job state as PREPARE_READOUT in JobScoreboard.
Send readout to forwarders.
Set job state as READOUT_STARTED in JobScoreboard.
Wait to retrieve and process readout responses.
:params parmas: A dictionary that stores info of a job.
:return: None.
"""
print("Incoming AUX AT_END_READOUT msg")
reply_queue = params['REPLY_QUEUE']
readout_ack_id = params[ACK_ID]
#job_number = params[JOB_NUM]
image_id = params[IMAGE_ID]
# send readout to forwarders
#self.JOB_SCBD.set_value_for_job(job_number, 'STATE', 'READOUT')
fwdr_readout_ack = self.get_next_timed_ack_id("AR_FWDR_READOUT_ACK")
#work_schedule = self.JOB_SCBD.get_work_schedule_for_job(job_number)
current_fwdr = self._current_fwdr
msg = {}
msg[MSG_TYPE] = 'AT_FWDR_END_READOUT'
#msg[JOB_NUM] = job_number
msg[IMAGE_ID] = image_id
msg['IMAGE_INDEX'] = params['IMAGE_INDEX']
route_key = self._current_fwdr['CONSUME_QUEUE']
self._publisher.publish_message(route_key, msg)
#readout_responses = self.progressive_ack_timer(fwdr_readout_ack, len(fwdrs), 4.0)
# if readout_responses == None:
# raise L1 exception
#self.process_readout_responses(readout_ack_id, reply_queue, image_id, readout_responses)
def process_readout_responses(self, readout_ack_id, reply_queue, image_id,
readout_responses):
""" From readout_responses param, retrieve image_id and job_number, and create list of
ccd, filename, and checksum from all forwarders. Store into xfer_list_msg and
send to archive to confirm each file made it intact.
Send AR_READOUT_ACK message with results and ack_bool equals True to
dmcs_ack_comsume queue.
:params readout_ack_id: Ack id for AR_READOUT_ACK message.
:params image_id:
:params readout_responses: Readout responses from AckScoreboard.
:return: None.
"""
job_number = None
image_id = None
confirm_ack = self.get_next_timed_ack_id('AR_ITEMS_XFERD_ACK')
fwdrs = list(readout_responses.keys())
CCD_LIST = []
FILENAME_LIST = []
CHECKSUM_LIST = []
for fwdr in fwdrs:
ccds = readout_responses[fwdr]['RESULT_LIST']['CCD_LIST']
num_ccds = len(ccds)
fnames = readout_responses[fwdr]['RESULT_LIST']['FILENAME_LIST']
csums = readout_responses[fwdr]['RESULT_LIST']['CHECKSUM_LIST']
for i in range(0, num_ccds):
msg = {}
CCD_LIST.append(ccds[i])
FILENAME_LIST.append(fnames[i])
CHECKSUM_LIST.append(csums[i])
job_number = readout_responses[fwdr][JOB_NUM]
image_id = readout_responses[fwdr]['IMAGE_ID']
xfer_list_msg = {}
xfer_list_msg[MSG_TYPE] = 'AR_ITEMS_XFERD'
xfer_list_msg[ACK_ID] = confirm_ack
xfer_list_msg['IMAGE_ID'] = image_id
xfer_list_msg['REPLY_QUEUE'] = self.AR_FOREMAN_ACK_PUBLISH
xfer_list_msg['RESULT_LIST'] = {}
xfer_list_msg['RESULT_LIST']['CCD_LIST'] = CCD_LIST
xfer_list_msg['RESULT_LIST']['FILENAME_LIST'] = FILENAME_LIST
xfer_list_msg['RESULT_LIST']['CHECKSUM_LIST'] = CHECKSUM_LIST
self._publisher.publish_message(self.ARCHIVE_CTRL_CONSUME,
xfer_list_msg)
xfer_check_responses = self.progressive_ack_timer(confirm_ack, 1, 4.0)
# if xfer_check_responses == None:
# raise L1 exception and bail
results = xfer_check_responses['ARCHIVE_CTRL']['RESULT_LIST']
ack_msg = {}
ack_msg['MSG_TYPE'] = 'AR_READOUT_ACK'
ack_msg['JOB_NUM'] = job_number
ack_msg['COMPONENT'] = self.COMPONENT_NAME
ack_msg['ACK_ID'] = readout_ack_id
ack_msg['ACK_BOOL'] = True
ack_msg['RESULT_LIST'] = results
self._publisher.publish_message(reply_queue, ack_msg)
### FIXME Set state as complete for Job
def send_readout(self, params, fwdrs, readout_ack):
""" Send AR_FWDR_READOUT message to each forwarder working on the job with
ar_foreman_ack_publish queue as reply queue.
:params params: A dictionary that stores info of a job.
:params readout_ack: Ack id for AR_FWDR_READOUT message.
:return: None.
"""
ro_params = {}
job_number = params['JOB_NUM']
ro_params['MSG_TYPE'] = 'AR_FWDR_READOUT'
ro_params['JOB_NUM'] = job_number
ro_params['SESSION_ID'] = self.get_current_session()
ro_params['VISIT_ID'] = self.get_current_visit()
ro_params['IMAGE_ID'] = params['IMAGE_ID']
ro_params['ACK_ID'] = readout_ack
ro_params['REPLY_QUEUE'] = self.AR_FOREMAN_ACK_PUBLISH
for fwdr in fwdrs:
route_key = self.FWD_SCBD.get_value_for_forwarder(
fwdr, "CONSUME_QUEUE")
self._publisher.publish_message(route_key, ro_params)
def process_header_ready_event(self, params):
fname = params['FILENAME']
image_id = params['IMAGE_ID']
msg = {}
msg['MSG_TYPE'] = 'AT_FWDR_HEADER_READY'
msg['FILENAME'] = fname
msg['IMAGE_ID'] = image_id
#XXX FIX remove hard code queue
#route_key = self._current_fwdr['CONSUME_QUEUE']
route_key = "f99_consume"
self._publisher.publish_message(route_key, msg)
def take_images_done(self, params):
reply_queue = params['REPLY_QUEUE']
readout_ack_id = params[ACK_ID]
job_number = params[JOB_NUM]
self.JOB_SCBD.set_value_for_job(job_number, 'STATE',
'TAKE_IMAGES_DONE')
fwdr_readout_ack = self.get_next_timed_ack_id(
"AR_FWDR_TAKE_IMAGES_DONE_ACK")
work_schedule = self.JOB_SCBD.get_work_schedule_for_job(job_number)
fwdrs = work_schedule['FORWARDER_LIST']
len_fwdrs = len(fwdrs)
msg = {}
msg[MSG_TYPE] = 'AR_FWDR_TAKE_IMAGES_DONE'
msg[JOB_NUM] = job_number
msg[ACK_ID] = fwdr_readout_ack
for i in range(0, len_fwdrs):
route_key = self.FWDR_SCBD.get_value_for_forwarder(
fwdrs[i], 'CONSUME_QUEUE')
self._publisher.publish_message(route_key, msg)
### FIX Add Final Response to DMCS
def process_ack(self, params):
""" Add new ACKS for a particular ACK_ID to the Ack Scoreboards
where they are collated.
:params: New ack to be checked in.
:return: None.
"""
pass
#self.ACK_SCBD.add_timed_ack(params)
def get_next_timed_ack_id(self, ack_type):
""" Increment ack id by 1, and store it.
Return ack id with ack type as a string.
:params ack_type: Informational string to prepend Ack ID.
:return retval: String with ack type followed by next ack id.
"""
self._next_timed_ack_id = self._next_timed_ack_id + 1
return (ack_type + "_" + str(self._next_timed_ack_id).zfill(6))
def set_session(self, params):
pass
""" Record new session in JobScoreboard.
Send AR_NEW_SESSION_ACK message with ack_bool equals True to specified reply queue.
:params params: Dictionary with info about new session.
:return: None.
self.JOB_SCBD.set_session(params['SESSION_ID'])
ack_id = params['ACK_ID']
msg = {}
msg['MSG_TYPE'] = 'AR_NEW_SESSION_ACK'
msg['COMPONENT'] = self.COMPONENT_NAME
msg['ACK_ID'] = ack_id
msg['ACK_BOOL'] = True
route_key = params['REPLY_QUEUE']
self._publisher.publish_message(route_key, msg)
"""
def get_current_session(self):
""" Retreive current session from JobSocreboard.
:params: None.
:return: Current session returned by JobSocreboard.
"""
return self.JOB_SCBD.get_current_session()
def set_visit(self, params):
""" Set current visit_id in JobScoreboard.
Send AR_NEXT_VISIT_ACK message with ack_bool equals True to specified reply queue.
:params params: Message dictionary with info about new visit.
:return: None.
"""
bore_sight = params['BORE_SIGHT']
self.JOB_SCBD.set_visit_id(params['VISIT_ID'], bore_sight)
ack_id = params['ACK_ID']
msg = {}
## XXX FIXME Do something with the bore sight in params['BORE_SIGHT']
msg['MSG_TYPE'] = 'AR_NEXT_VISIT_ACK'
msg['COMPONENT'] = self.COMPONENT_NAME
msg['ACK_ID'] = ack_id
msg['ACK_BOOL'] = True
route_key = params['REPLY_QUEUE']
self._publisher.publish_message(route_key, msg)
def get_current_visit(self):
""" Retrieve current visit from JobSocreboard.
:params: None.
:return: Current visit returned by JobSocreboard.
"""
return self.JOB_SCBD.get_current_visit()
def ack_timer(self, seconds):
""" Sleeps for user-defined seconds.
:params seconds: Time to sleep in seconds.
:return: True.
"""
sleep(seconds)
return True
def progressive_ack_timer(self, ack_id, expected_replies, seconds):
""" Sleeps for user-defined seconds, or less if everyone has reported back in.
:params ack_id: Ack ID to wait for.
:params expected_replies: Number of components expected to ack..
:params seconds: Maximum time to wait in seconds.
:return: The dictionary that represents the responses from the components ack'ing.
Note: If only one component will ack, this method breaks out of its
loop after the one ack shows up - effectively beating the maximum
wait time.
"""
counter = 0.0
while (counter < seconds):
counter = counter + 0.5
sleep(0.5)
response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if response == None:
continue
if len(list(response.keys())) == expected_replies:
return response
## Try one final time
response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if response == None:
return None
elif len(list(response.keys())) == expected_replies:
return response
else:
return None
def extract_config_values(self):
""" Parse system config yaml file.
Throw error messages if Yaml file or key not found.
:params: None.
:return: True.
"""
LOGGER.info('Reading YAML Config file %s' % self._config_file)
try:
cdm = toolsmod.intake_yaml_file(self._config_file)
except IOError as e:
LOGGER.critical("Unable to find CFG Yaml file %s\n" %
self._config_file)
sys.exit(101)
try:
self._msg_name = cdm[ROOT][
'AUX_BROKER_NAME'] # Message broker user & passwd
self._msg_passwd = cdm[ROOT]['AUX_BROKER_PASSWD']
self._msg_pub_name = cdm[ROOT][
'AUX_BROKER_PUB_NAME'] # Message broker user & passwd
self._msg_pub_passwd = cdm[ROOT]['AUX_BROKER_PUB_PASSWD']
self._base_broker_addr = cdm[ROOT][BASE_BROKER_ADDR]
self._forwarder_dict = cdm[ROOT][XFER_COMPONENTS]['AUX_FORWARDERS']
self._wfs_raft = cdm[ROOT]['ATS']['WFS_RAFT']
# Placeholder until eventually worked out by Data Backbone team
self.archive_fqn = cdm[ROOT]['ARCHIVE']['ARCHIVE_NAME']
self.archive_name = cdm[ROOT]['ARCHIVE']['ARCHIVE_LOGIN']
self.archive_ip = cdm[ROOT]['ARCHIVE']['ARCHIVE_IP']
self.archive_xfer_root = cdm[ROOT]['ARCHIVE']['ARCHIVE_XFER_ROOT']
except KeyError as e:
print("Dictionary error")
print("Bailing out...")
sys.exit(99)
self._base_msg_format = 'YAML'
if 'BASE_MSG_FORMAT' in cdm[ROOT]:
self._base_msg_format = cdm[ROOT]['BASE_MSG_FORMAT']
def setup_consumer_threads(self):
""" Create ThreadManager object with base broker url and kwargs to setup consumers.
:params: None.
:return: None.
"""
base_broker_url = "amqp://" + self._msg_name + ":" + \
self._msg_passwd + "@" + \
str(self._base_broker_addr)
LOGGER.info('Building _base_broker_url. Result is %s', base_broker_url)
self.shutdown_event = threading.Event()
self.shutdown_event.clear()
# Set up kwargs that describe consumers to be started
# The Archive Device needs three message consumers
kws = {}
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-aux_foreman_consume'
md['queue'] = 'at_foreman_consume'
md['callback'] = self.on_aux_foreman_message
md['format'] = "YAML"
md['test_val'] = None
kws[md['name']] = md
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-at_foreman_ack_publish'
md['queue'] = 'at_foreman_ack_publish'
md['callback'] = self.on_ack_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-archive_ctrl_publish'
md['queue'] = 'archive_ctrl_publish'
md['callback'] = self.on_archive_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
self.thread_manager = ThreadManager('thread-manager', kws,
self.shutdown_event)
self.thread_manager.start()
def shutdown(self):
LOGGER.info("Shutting down Consumer threads.")
self.shutdown_event.set()
LOGGER.debug("Thread Manager shutting down and app exiting...")
print("\n")
os._exit(0)
class ArchiveController:
ARCHIVE_CTRL_PUBLISH = "archive_ctrl_publish"
ARCHIVE_CTRL_CONSUME = "archive_ctrl_consume"
ACK_PUBLISH = "ar_foreman_ack_publish"
AUDIT_CONSUME = "audit_consume"
YAML = 'YAML'
RECEIPT_FILE = "/var/archive/archive_controller_receipt"
def __init__(self, filename=None):
self._session_id = None
self._name = "ARCHIVE_CTRL"
self._config_file = 'L1SystemCfg.yaml'
if filename != None:
self._config_file = filename
cdm = toolsmod.intake_yaml_file(self._config_file)
try:
self._archive_name = cdm[ROOT][
'ARCHIVE_BROKER_NAME'] # Message broker user/passwd for component
self._archive_passwd = cdm[ROOT]['ARCHIVE_BROKER_PASSWD']
self._base_broker_addr = cdm[ROOT][BASE_BROKER_ADDR]
self._archive_xfer_root = cdm[ROOT]['ARCHIVE']['ARCHIVE_XFER_ROOT']
if cdm[ROOT]['ARCHIVE']['CHECKSUM_ENABLED'] == 'yes':
self.CHECKSUM_ENABLED = True
else:
self.CHECKSUM_ENABLED = False
except KeyError as e:
raise L1Error(e)
self._base_msg_format = self.YAML
if 'BASE_MSG_FORMAT' in cdm[ROOT]:
self._base_msg_format = cdm[ROOT][BASE_MSG_FORMAT]
self._base_broker_url = "amqp://" + self._archive_name + ":" + self._archive_passwd + "@" + str(
self._base_broker_addr)
LOGGER.info(
'Building _base_broker_url connection string for Archive Controller. Result is %s',
self._base_broker_url)
self._msg_actions = {
'ARCHIVE_HEALTH_CHECK': self.process_health_check,
'NEW_ARCHIVE_ITEM': self.process_new_archive_item,
'AR_ITEMS_XFERD': self.process_transfer_complete
}
self.setup_consumer()
self.setup_publisher()
def setup_consumer(self):
LOGGER.info('Setting up archive consumers on %s',
self._base_broker_url)
LOGGER.info('Running start_new_thread for archive consumer')
self.shutdown_event = threading.Event()
self.shutdown_event.clear()
kws = {}
md = {}
md['amqp_url'] = self._base_broker_url
md['name'] = 'Thread-ar_ctrl_consume'
md['queue'] = self.ARCHIVE_CTRL_CONSUME
md['callback'] = self.on_archive_message
md['format'] = "YAML"
md['test_val'] = None
kws[md['name']] = md
self.thread_manager = ThreadManager('thread-manager', kws,
self.shutdown_event)
self.thread_manager.start()
def setup_publisher(self):
LOGGER.info('Setting up Archive publisher on %s using %s',
self._base_broker_url, self._base_msg_format)
self._archive_publisher = SimplePublisher(self._base_broker_url,
self._base_msg_format)
#self._audit_publisher = SimplePublisher(self._base_broker_url, self._base_msg_format)
def on_archive_message(self, ch, method, properties, msg_dict):
LOGGER.info('Message from Archive callback message body is: %s',
str(msg_dict))
ch.basic_ack(method.delivery_tag)
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def process_health_check(self, params):
"""Input 'params' for this method is a dict made up of:
:param str 'MESSAGE_TYPE' value is 'ARCHIVE_HEALTH_CHECK'
:param str 'ACK_ID' value is an alphanumeric string, with
the numeric part a momotonically increasing sequence.
This value is passed back to the foreman and used to keep
track of acknowledgement time.
:param str 'SESSION_ID' Might be useful for the controller to
generate a target location for new items to be archived?
"""
self.send_audit_message("received_", params)
self.send_health_ack_response("ARCHIVE_HEALTH_CHECK_ACK", params)
def process_new_archive_item(self, params):
self.send_audit_message("received_", params)
target_dir = self.construct_send_target_dir(params)
self.send_new_item_ack(target_dir, params)
def process_transfer_complete(self, params):
transfer_results = {}
ccds = params['RESULT_LIST']['CCD_LIST']
fnames = params['RESULT_LIST']['FILENAME_LIST']
csums = params['RESULT_LIST']['CHECKSUM_LIST']
num_ccds = len(ccds)
transfer_results = {}
RECEIPT_LIST = []
for i in range(0, num_ccds):
ccd = ccds[i]
pathway = fnames[i]
csum = csums[i]
transfer_result = self.check_transferred_file(pathway, csum)
if transfer_result == None:
RECEIPT_LIST.append('0')
else:
RECEIPT_LIST.append(transfer_result)
transfer_results['CCD_LIST'] = ccds
transfer_results['RECEIPT_LIST'] = RECEIPT_LIST
self.send_transfer_complete_ack(transfer_results, params)
def check_transferred_file(self, pathway, csum):
if not os.path.isfile(pathway):
return ('-1')
if self.CHECKSUM_ENABLED:
with open(pathway) as file_to_calc:
data = file_to_calc.read()
resulting_md5 = hashlib.md5(data).hexdigest()
if resulting_md5 != csum:
return ('0')
return self.next_receipt_number()
def next_receipt_number(self):
last_receipt = toolsmod.intake_yaml_file(self.RECEIPT_FILE)
current_receipt = int(last_receipt['RECEIPT_ID']) + 1
session_dict = {}
session_dict['RECEIPT_ID'] = current_receipt
toolsmod.export_yaml_file(self.RECEIPT_FILE, session_dict)
return current_receipt
def send_health_ack_response(self, type, params):
try:
ack_id = params.get("ACK_ID")
self._current_session_id = params.get("SESSION_ID")
except:
if ack_id == None:
LOGGER.info('%s failed, missing ACK_ID field', type)
raise L1MessageError(
"Missing ACK_ID message param needed for send_ack_response"
)
else:
LOGGER.info('%s failed, missing SESSION_ID field', type)
raise L1MessageError(
"Missing SESSION_ID param needed for send_ack_response")
msg_params = {}
msg_params[MSG_TYPE] = type
msg_params[COMPONENT] = self._name
msg_params[ACK_BOOL] = "TRUE"
msg_params['ACK_ID'] = ack_id
LOGGER.info('%s sent for ACK ID: %s', type, ack_id)
self._archive_publisher.publish_message(self.ACK_PUBLISH, msg_params)
def send_audit_message(self, prefix, params):
audit_params = {}
audit_params['SUB_TYPE'] = str(prefix) + str(
params['MSG_TYPE']) + "_msg"
audit_params['DATA_TYPE'] = self._name
audit_params['TIME'] = get_epoch_timestamp()
self._archive_publisher.publish_message(self.AUDIT_CONSUME,
audit_params)
def construct_send_target_dir(self, params):
#session = params['SESSION_ID']
visit = params['VISIT_ID']
image = params['IMAGE_ID']
ack_id = params['ACK_ID']
#target_dir = self._archive_xfer_root + "_" + str(image_type) + "_" + str(session) + "_" + str(visit) + "_" + str(image)
target_dir_visit = self._archive_xfer_root + visit + "/"
target_dir_image = self._archive_xfer_root + visit + "/" + str(
image) + "/"
if os.path.isdir(target_dir_visit):
pass
else:
os.mkdir(target_dir_visit, 0o766)
if os.path.isdir(target_dir_image):
pass
else:
os.mkdir(target_dir_image, 0o766)
return target_dir_image
def send_new_item_ack(self, target_dir, params):
ack_params = {}
ack_params[MSG_TYPE] = 'NEW_ARCHIVE_ITEM_ACK'
ack_params['TARGET_DIR'] = target_dir
ack_params['ACK_ID'] = params['ACK_ID']
ack_params['JOB_NUM'] = params['JOB_NUM']
ack_params['IMAGE_ID'] = params['IMAGE_ID']
ack_params['COMPONENT'] = self._name
ack_params['ACK_BOOL'] = True
self._archive_publisher.publish_message(self.ACK_PUBLISH, ack_params)
def send_transfer_complete_ack(self, transfer_results, params):
ack_params = {}
keez = list(params.keys())
for kee in keez:
if kee == 'MSG_TYPE' or kee == 'CCD_LIST':
continue
### XXX FIXME Dump loop and just pull the correct values from the input params
ack_params[kee] = params[kee]
ack_params['MSG_TYPE'] = 'AR_ITEMS_XFERD_ACK'
ack_params['COMPONENT'] = self._name
ack_params['ACK_ID'] = params['ACK_ID']
ack_params['ACK_BOOL'] = True
ack_params['RESULTS'] = transfer_results
self._archive_publisher.publish_message(self.ACK_PUBLISH, ack_params)
class PromptProcessDevice:
PP_JOB_SCBD = None
PP_FWD_SCBD = None
PP_ACK_SCBD = None
COMPONENT_NAME = 'PROMPT_PROCESS_FOREMAN'
PP_FOREMAN_CONSUME = "pp_foreman_consume"
PP_FOREMAN_ACK_PUBLISH = "pp_foreman_ack_publish"
PP_START_INTEGRATION_ACK = "PP_START_INTEGRATION_ACK"
NCSA_PUBLISH = "ncsa_publish"
NCSA_CONSUME = "ncsa_consume"
NCSA_NO_RESPONSE = 5705
FORWARDER_NO_RESPONSE = 5605
FORWARDER_PUBLISH = "forwarder_publish"
CFG_FILE = 'L1SystemCfg.yaml'
ERROR_CODE_PREFIX = 5500
prp = toolsmod.prp
def __init__(self, filename=None):
toolsmod.singleton(self)
self._config_file = self.CFG_FILE
if filename != None:
self._config_file = filename
LOGGER.info('Extracting values from Config dictionary')
try:
self.extract_config_values()
except Exception as e:
LOGGER.error("PP_Device problem configuring with file %s: %s" %
(self._config_file, e.arg))
print("PP_Device unable to read Config file %s: %s" %
(self._config_file, e.arg))
sys.exit(self.ErrorCodePrefix + 20)
#self.purge_broker(cdm['ROOT']['QUEUE_PURGES'])
self._msg_actions = {
'PP_NEW_SESSION': self.set_session,
'PP_NEXT_VISIT': self.set_visit,
'PP_START_INTEGRATION': self.process_start_integration,
'PP_READOUT': self.process_dmcs_readout,
'NCSA_RESOURCE_QUERY_ACK': self.process_ack,
'NCSA_START_INTEGRATION_ACK': self.process_ack,
'NCSA_READOUT_ACK': self.process_ack,
'PP_FWDR_HEALTH_CHECK_ACK': self.process_ack,
'PP_FWDR_XFER_PARAMS_ACK': self.process_ack,
'PP_FWDR_READOUT_ACK': self.process_ack,
'PENDING_ACK': self.process_pending_ack,
'NCSA_NEXT_VISIT_ACK': self.process_ack
}
self._next_timed_ack_id = 0
try:
self.setup_publishers()
except L1PublisherError as e:
LOGGER.error("PP_Device unable to start Publishers: %s" % e.arg)
print("PP_Device unable to start Publishers: %s" % e.arg)
sys.exit(self.ErrorCodePrefix + 31)
self.setup_scoreboards()
LOGGER.info('pp foreman consumer setup')
self.thread_manager = None
try:
self.setup_consumer_threads()
except L1Exception as e:
LOGGER.error("PP_Device unable to launch ThreadManager: %s" %
e.arg)
print("PP_Device unable to launch ThreadManager: %s" % e.arg)
sys.exit(self.ErrorCodePrefix + 1)
LOGGER.info('Prompt Process Foreman Init complete')
def setup_publishers(self):
self._pub_base_broker_url = "amqp://" + self._pub_name + ":" + \
self._pub_passwd + "@" + \
str(self._base_broker_addr)
self._pub_ncsa_broker_url = "amqp://" + self._pub_ncsa_name + ":" + \
self._pub_ncsa_passwd + "@" + \
str(self._ncsa_broker_addr)
try:
LOGGER.info('Setting up Base publisher on %s using %s', \
self._pub_base_broker_url, self._base_msg_format)
self._base_publisher = SimplePublisher(self._pub_base_broker_url,
self._base_msg_format)
LOGGER.info('Setting up NCSA publisher on %s using %s', \
self._pub_ncsa_broker_url, self._ncsa_msg_format)
self._ncsa_publisher = SimplePublisher(self._pub_ncsa_broker_url,
self._ncsa_msg_format)
except Exception as e:
LOGGER.error("PP_Device unable to start Publishers: %s" % e.arg)
print("PP_Device unable to start Publishers: %s" % e.arg)
raise L1PublisherError(
"Critical Error: Unable to create Publishers: %s" % e.arg)
def on_dmcs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
#msg_dict = yaml.load(body)
msg_dict = body
LOGGER.info('In DMCS message callback')
LOGGER.info('Message from DMCS callback message body is: %s',
str(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def on_forwarder_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
LOGGER.info('In Forwarder message callback, thread is %s',
_thread.get_ident())
LOGGER.info('forwarder callback msg body is: %s', str(body))
pass
def on_ncsa_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
msg_dict = body
LOGGER.info('ncsa msg callback body is: %s', str(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def on_ack_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
msg_dict = body
LOGGER.info('In ACK message callback')
LOGGER.info('Message from ACK callback message body is: %s',
str(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def set_session(self, params):
self.JOB_SCBD.set_session(params['SESSION_ID'])
ack_id = params['ACK_ID']
msg = {}
msg['MSG_TYPE'] = 'PP_NEW_SESSION_ACK'
msg['COMPONENT'] = self.COMPONENT_NAME
msg['ACK_ID'] = ack_id
msg['ACK_BOOL'] = True
route_key = params['REPLY_QUEUE']
self._base_publisher.publish_message(route_key, msg)
def set_visit(self, params):
bore_sight = params['BORE_SIGHT']
visit_id = params['VISIT_ID']
self.JOB_SCBD.set_visit_id(visit_id, bore_sight)
ack_id = params['ACK_ID']
msg = {}
ncsa_result = self.send_visit_boresight_to_ncsa(visit_id, bore_sight)
msg['MSG_TYPE'] = 'PP_NEXT_VISIT_ACK'
msg['COMPONENT'] = self.COMPONENT_NAME
msg['ACK_ID'] = ack_id
msg['ACK_BOOL'] = True
route_key = params['REPLY_QUEUE']
self._base_publisher.publish_message(route_key, msg)
def send_visit_boresight_to_ncsa(self, visit_id, bore_sight):
msg = {}
msg['MSG_TYPE'] = 'NCSA_NEXT_VISIT'
msg['VISIT_ID'] = visit_id
msg['BORE_SIGHT'] = bore_sight
msg['SESSION_ID'] = self.JOB_SCBD.get_current_session()
ack_id = self.get_next_timed_ack_id('NCSA_NEXT_VISIT_ACK')
msg['ACK_ID'] = ack_id
msg['REPLY_QUEUE'] = self.PP_FOREMAN_ACK_PUBLISH
self._ncsa_publisher.publish_message(self.NCSA_CONSUME, msg)
wait_time = 4
acks = []
acks.append(ack_id)
self.set_pending_nonblock_acks(acks, wait_time)
def process_start_integration(self, input_params):
"""
Add job to Job Scoreboard
Check forwarder health
Check Policy, bail if necessary
Mark Forwarder scoreboard as a result of above
Divide work and assemble as a forwarder dictionary for NCSA
Send work division to NCSA
Check Policy, bail if necessary
Persist pairings to Job Scoreboard
Send params to Forwarders
Confirm Forwarder Acks
Send confirm to DMCS
"""
ccd_list = input_params['CCD_LIST']
job_num = str(input_params[JOB_NUM])
visit_id = input_params['VISIT_ID']
image_id = input_params['IMAGE_ID']
self.JOB_SCBD.add_job(job_num, image_id, visit_id, ccd_list)
unknown_status = {"STATUS": "UNKNOWN", "STATE": "UNRESPONSIVE"}
self.FWD_SCBD.setall_forwarder_params(unknown_status)
ack_id = self.forwarder_health_check(input_params)
self.ack_timer(2.5)
healthy_forwarders = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if healthy_forwarders == None:
self.JOB_SCBD.set_job_state(job_number, 'SCRUBBED')
self.JOB_SCBD.set_job_status(job_number, 'INACTIVE')
self.send_fault("No Response From Forwarders",
self.FORWARDER_NO_RESPONSE, job_num,
self.COMPONENT_NAME)
raise L1ForwarderError(
"No response from any Forwarder when sending job params")
healthy_forwarders_list = list(healthy_forwarders.keys())
for forwarder in healthy_forwarders_list:
self.FWD_SCBD.set_forwarder_state(forwarder, 'BUSY')
self.FWD_SCBD.set_forwarder_status(forwarder, 'HEALTHY')
num_healthy_forwarders = len(healthy_forwarders_list)
ready_status = {"STATUS": "HEALTHY", "STATE": "READY_WITHOUT_PARAMS"}
self.FWD_SCBD.set_forwarder_params(healthy_forwarders_list,
ready_status)
work_schedule = self.divide_work(healthy_forwarders_list, ccd_list)
ack_id = self.ncsa_resources_query(input_params, work_schedule)
ncsa_response = self.progressive_ack_timer(ack_id, 1, 2.0)
#Check ACK scoreboard for response from NCSA
if ncsa_response:
pairs = []
pairs = ncsa_response['NCSA_FOREMAN']['PAIRS']
# Distribute job params and tell DMCS we are ready.
fwd_ack_id = self.distribute_job_params(input_params, pairs)
num_fwdrs = len(pairs)
fwdr_params_response = self.progressive_ack_timer(
fwd_ack_id, num_fwdrs, 3.0)
if fwdr_params_response:
self.JOB_SCBD.set_value_for_job(job_num, "STATE",
"FWDR_PARAMS_RECEIVED")
in_ready_state = {'STATE': 'READY_WITH_PARAMS'}
self.FWD_SCBD.set_forwarder_params(healthy_forwarders_list,
in_ready_state)
# Tell DMCS we are ready
result = self.accept_job(input_params['ACK_ID'], job_num)
else:
idle_params = {'STATE': 'IDLE'}
self.FWD_SCBD.set_forwarder_params(needed_forwarders,
idle_params)
self.send_fault("No RESPONSE FROM NCSA FOREMAN",
self.NCSA_NO_RESPONSE, job_num,
self.COMPONENT_NAME)
raise L1NcsaForemanError("No Response From NCSA Foreman")
else:
result = self.ncsa_no_response(input_params)
idle_params = {'STATE': 'IDLE'}
self.FWD_SCBD.set_forwarder_params(needed_forwarders, idle_params)
return result
def forwarder_health_check(self, params):
# get timed_ack_id
timed_ack = self.get_next_timed_ack_id("PP_FWDR_HEALTH_CHECK_ACK")
forwarders = self.FWD_SCBD.return_forwarders_list()
job_num = params[JOB_NUM]
# send health check messages
msg_params = {}
msg_params[MSG_TYPE] = 'PP_FWDR_HEALTH_CHECK'
msg_params['ACK_ID'] = timed_ack
msg_params['REPLY_QUEUE'] = self.PP_FOREMAN_ACK_PUBLISH
msg_params[JOB_NUM] = job_num
self.JOB_SCBD.set_value_for_job(job_num, "STATE", "HEALTH_CHECK")
for forwarder in forwarders:
self._base_publisher.publish_message(
self.FWD_SCBD.get_routing_key(forwarder), msg_params)
return timed_ack
def divide_work(self, fwdrs_list, ccd_list):
num_fwdrs = len(fwdrs_list)
num_ccds = len(ccd_list)
schedule = {}
schedule['FORWARDER_LIST'] = []
schedule['CCD_LIST'] = [
] # A list of ccd lists; index of main list matches same forwarder list index
FORWARDER_LIST = []
CCD_LIST = [] # This is a 'list of lists'
if num_fwdrs == 1:
FORWARDER_LIST.append(fwdrs_list[0])
CCD_LIST.append(ccd_list)
schedule['FORWARDERS_LIST'] = FORWARDER_LIST
schedule['CCD_LIST'] = CCD_LIST
return schedule
if num_ccds <= num_fwdrs:
for k in range(0, num_ccds):
little_list = []
FORWARDER_LIST.append(fwdrs_list[k])
little_list.append(ccd_list[k])
CCD_LIST.append(list(little_list)) # Need a copy here...
schedule['FORWARDER_LIST'] = FORWARDER_LIST
schedule['CCD_LIST'] = CCD_LIST
else:
ccds_per_fwdr = len(ccd_list) // num_fwdrs
remainder_ccds = len(ccd_list) % num_fwdrs
offset = 0
for i in range(0, num_fwdrs):
tmp_list = []
for j in range(offset, (ccds_per_fwdr + offset)):
if (j) >= num_ccds:
break
tmp_list.append(ccd_list[j])
# CCD_LIST.append(ccd_list[j])
offset = offset + ccds_per_fwdr
if remainder_ccds != 0 and i == 0:
for k in range(offset, offset + remainder_ccds):
tmp_list.append(ccd_list[k])
offset = offset + remainder_ccds
FORWARDER_LIST.append(fwdrs_list[i])
CCD_LIST.append(list(tmp_list))
#schedule[fwdrs_list[i]] = {}
#schedule[fwdrs_list[i]]['CCD_LIST'] = tmp_list
schedule['FORWARDER_LIST'] = FORWARDER_LIST
schedule['CCD_LIST'] = CCD_LIST
return schedule
def ncsa_resources_query(self, params, work_schedule):
job_num = str(params[JOB_NUM])
timed_ack_id = self.get_next_timed_ack_id("NCSA_START_INTEGRATION_ACK")
ncsa_params = {}
ncsa_params[MSG_TYPE] = "NCSA_START_INTEGRATION"
ncsa_params[JOB_NUM] = job_num
ncsa_params['VISIT_ID'] = params['VISIT_ID']
ncsa_params['IMAGE_ID'] = params['IMAGE_ID']
ncsa_params['SESSION_ID'] = params['SESSION_ID']
ncsa_params['REPLY_QUEUE'] = self.PP_FOREMAN_ACK_PUBLISH
ncsa_params[ACK_ID] = timed_ack_id
ncsa_params["FORWARDERS"] = work_schedule
self.JOB_SCBD.set_value_for_job(job_num, "STATE",
"NCSA_START_INT_SENT")
self._ncsa_publisher.publish_message(self.NCSA_CONSUME, ncsa_params)
LOGGER.info(
'The following forwarders schedule has been sent to NCSA for pairing:'
)
LOGGER.info(work_schedule)
return timed_ack_id
def distribute_job_params(self, params, pairs):
""" pairs param is a list of dicts. (look at messages.yaml, search for 'PAIR' key, and copy here
"""
#ncsa has enough resources...
job_num = str(params[JOB_NUM])
self.JOB_SCBD.set_pairs_for_job(job_num, pairs)
LOGGER.info('The following pairs will be used for Job #%s: %s',
job_num, pairs)
fwd_ack_id = self.get_next_timed_ack_id("FWD_PARAMS_ACK")
fwd_params = {}
fwd_params[MSG_TYPE] = "PP_FWDR_XFER_PARAMS"
fwd_params[JOB_NUM] = job_num
fwd_params['IMAGE_ID'] = params['IMAGE_ID']
fwd_params['VISIT_ID'] = params['VISIT_ID']
fwd_params['REPLY_QUEUE'] = self.PP_FOREMAN_ACK_PUBLISH
fwd_params[ACK_ID] = fwd_ack_id
fwd_params['XFER_PARAMS'] = {}
for i in range(0, len(pairs)):
ddict = {}
ddict = pairs[i]
fwdr = ddict['FORWARDER']
fwd_params['XFER_PARAMS']['CCD_LIST'] = ddict['CCD_LIST']
fwd_params['XFER_PARAMS']['DISTRIBUTOR'] = ddict['DISTRIBUTOR']
route_key = self.FWD_SCBD.get_value_for_forwarder(
fwdr, "CONSUME_QUEUE")
self._base_publisher.publish_message(route_key, fwd_params)
return fwd_ack_id
def accept_job(self, ack_id, job_num):
dmcs_message = {}
dmcs_message[JOB_NUM] = job_num
dmcs_message[MSG_TYPE] = self.PP_START_INTEGRATION_ACK
dmcs_message['COMPONENT'] = self.COMPONENT_NAME
dmcs_message[ACK_BOOL] = True
dmcs_message['ACK_ID'] = ack_id
self.JOB_SCBD.set_value_for_job(job_num, STATE, "JOB_ACCEPTED")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ACCEPTED",
get_timestamp())
self._base_publisher.publish_message("dmcs_ack_consume", dmcs_message)
return True
def process_dmcs_readout(self, params):
job_number = params[JOB_NUM]
pairs = self.JOB_SCBD.get_pairs_for_job(job_number)
### Send READOUT to NCSA with ACK_ID
ack_id = self.get_next_timed_ack_id('NCSA_READOUT_ACK')
ncsa_params = {}
ncsa_params[MSG_TYPE] = 'NCSA_READOUT'
ncsa_params['JOB_NUM'] = job_number
ncsa_params['VISIT_ID'] = params['VISIT_ID']
ncsa_params['SESSION_ID'] = params['SESSION_ID']
ncsa_params['IMAGE_ID'] = params['IMAGE_ID']
ncsa_params['REPLY_QUEUE'] = 'pp_foreman_ack_publish'
ncsa_params[ACK_ID] = ack_id
self._ncsa_publisher.publish_message(self.NCSA_CONSUME, ncsa_params)
ncsa_response = self.progressive_ack_timer(ack_id, 1, 3.0)
if ncsa_response:
if ncsa_response['NCSA_FOREMAN']['ACK_BOOL'] == True:
#inform forwarders
fwd_ack_id = self.get_next_timed_ack_id('PP_FWDR_READOUT_ACK')
len_pairs = len(pairs)
for i in range(0, len_pairs):
forwarder = pairs[i]['FORWARDER']
routing_key = self.FWD_SCBD.get_routing_key(forwarder)
msg_params = {}
msg_params[MSG_TYPE] = 'PP_FWDR_READOUT'
msg_params[JOB_NUM] = job_number
msg_params['REPLY_QUEUE'] = 'pp_foreman_ack_publish'
msg_params['ACK_ID'] = fwd_ack_id
self.FWD_SCBD.set_forwarder_state(forwarder,
'START_READOUT')
self._base_publisher.publish_message(
routing_key, msg_params)
forwarder_responses = self.progressive_ack_timer(
fwd_ack_id, len_pairs, 4.0)
if forwarder_responses:
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'PP_READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['COMPONENT'] = self.COMPONENT_NAME
dmcs_params['ACK_BOOL'] = True
dmcs_params['ACK_ID'] = params['ACK_ID']
self._base_publisher.publish_message(
params['REPLY_QUEUE'], dmcs_params)
else:
#send problem with ncsa to DMCS
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'PP_READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['COMPONENT'] = self.COMPONENT_NAME
dmcs_params['ACK_BOOL'] = False
dmcs_params['ACK_ID'] = params['ACK_ID']
self._base_publisher.publish_message('dmcs_ack_consume',
dmcs_params)
else:
#send 'no response from ncsa' to DMCS )
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'PP_READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['COMPONENT'] = self.COMPONENT_NAME
dmcs_params['ACK_BOOL'] = False
dmcs_params['ACK_ID'] = params['ACK_ID']
self._base_publisher.publish_message(params['REPLY_QUEUE'],
dmcs_params)
def process_ack(self, params):
self.ACK_SCBD.add_timed_ack(params)
def get_next_timed_ack_id(self, ack_type):
self._next_timed_ack_id = self._next_timed_ack_id + 1
return (ack_type + "_" + str(self._next_timed_ack_id).zfill(6))
def ack_timer(self, seconds):
sleep(seconds)
return True
def progressive_ack_timer(self, ack_id, expected_replies, seconds):
counter = 0.0
while (counter < seconds):
counter = counter + 0.5
sleep(0.5)
response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if response == None:
continue
if len(list(response.keys())) == expected_replies:
return response
## Try one final time
response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if response == None:
return None
elif len(list(response.keys())) == expected_replies:
return response
else:
return None
def set_pending_nonblock_acks(self, acks, wait_time):
start_time = datetime.datetime.now().time()
expiry_time = self.add_seconds(start_time, wait_time)
ack_msg = {}
ack_msg[MSG_TYPE] = 'PENDING_ACK'
ack_msg['EXPIRY_TIME'] = expiry_time
for ack in acks:
ack_msg[ACK_ID] = ack
self._base_publisher.publish_message(self.PP_FOREMAN_ACK_PUBLISH,
ack_msg)
def process_pending_ack(self, params):
self.ACK_SCBD.add_pending_nonblock_ack(params)
def add_seconds(self, intime, secs):
basetime = datetime.datetime(100, 1, 1, intime.hour, intime.minute,
intime.second)
newtime = basetime + datetime.timedelta(seconds=secs)
return newtime.time()
def extract_config_values(self):
LOGGER.info('Reading YAML Config file %s' % self._config_file)
try:
cdm = toolsmod.intake_yaml_file(self._config_file)
except IOError as e:
LOGGER.critical("Unable to find CFG Yaml file %s\n" %
self._config_file)
print("Unable to find CFG Yaml file %s\n" % self._config_file)
raise L1ConfigIOError("Trouble opening CFG Yaml file %s: %s" %
(self._config_file, e.arg))
try:
self._sub_name = cdm[ROOT][
PFM_BROKER_NAME] # Message broker user & passwd
self._sub_passwd = cdm[ROOT][PFM_BROKER_PASSWD]
self._pub_name = cdm[ROOT][
'PFM_BROKER_PUB_NAME'] # Message broker user & passwd
self._pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD']
self._sub_ncsa_name = cdm[ROOT]['PFM_NCSA_BROKER_NAME']
self._sub_ncsa_passwd = cdm[ROOT]['PFM_NCSA_BROKER_PASSWD']
self._pub_ncsa_name = cdm[ROOT]['PFM_NCSA_BROKER_PUB_NAME']
self._pub_ncsa_passwd = cdm[ROOT]['PFM_NCSA_BROKER_PUB_PASSWD']
self._base_broker_addr = cdm[ROOT][BASE_BROKER_ADDR]
self._ncsa_broker_addr = cdm[ROOT][NCSA_BROKER_ADDR]
self._forwarder_dict = cdm[ROOT][XFER_COMPONENTS]['PP_FORWARDERS']
self._scbd_dict = cdm[ROOT]['SCOREBOARDS']
self.DMCS_FAULT_QUEUE = cdm[ROOT]['DMCS_FAULT_QUEUE']
self._policy_max_ccds_per_fwdr = int(
cdm[ROOT]['POLICY']['MAX_CCDS_PER_FWDR'])
except KeyError as e:
LOGGER.critical("CDM Dictionary Key error")
LOGGER.critical("Offending Key is %s", str(e))
LOGGER.critical("Bailing out...")
print("KeyError when reading CFG file. Check logs...exiting...")
raise L1ConfigKeyError("Key Error when reading config file: %s" %
e.arg)
self._base_msg_format = 'YAML'
self._ncsa_msg_format = 'YAML'
if 'BASE_MSG_FORMAT' in cdm[ROOT]:
self._base_msg_format = cdm[ROOT][BASE_MSG_FORMAT]
if 'NCSA_MSG_FORMAT' in cdm[ROOT]:
self._ncsa_msg_format = cdm[ROOT][NCSA_MSG_FORMAT]
def setup_consumer_threads(self):
LOGGER.info('Building _base_broker_url')
base_broker_url = "amqp://" + self._sub_name + ":" + \
self._sub_passwd + "@" + \
str(self._base_broker_addr)
ncsa_broker_url = "amqp://" + self._sub_ncsa_name + ":" + \
self._sub_ncsa_passwd + "@" + \
str(self._ncsa_broker_addr)
self.shutdown_event = threading.Event()
self.shutdown_event.clear()
# Set up kwargs that describe consumers to be started
# The Archive Device needs three message consumers
kws = {}
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-pp_foreman_consume'
md['queue'] = 'pp_foreman_consume'
md['callback'] = self.on_dmcs_message
md['format'] = "YAML"
md['test_val'] = None
kws[md['name']] = md
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-pp_foreman_ack_publish'
md['queue'] = 'pp_foreman_ack_publish'
md['callback'] = self.on_ack_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
md = {}
md['amqp_url'] = ncsa_broker_url
md['name'] = 'Thread-ncsa_publish'
md['queue'] = 'ncsa_publish'
md['callback'] = self.on_ncsa_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
try:
self.thread_manager = ThreadManager('thread-manager', kws,
self.shutdown_event)
self.thread_manager.start()
except ThreadError as e:
LOGGER.error(
"PP_Device unable to launch Consumers - Thread Error: %s" %
e.arg)
print("PP_Device unable to launch Consumers - Thread Error: %s" %
e.arg)
raise L1ConsumerError(
"Thread problem preventing Consumer launch: %s" % e.arg)
except Exception as e:
LOGGER.error("PP_Device unable to launch Consumers: %s" % e.arg)
print("PP_Device unable to launch Consumers: %s" % e.arg)
raise L1Error(
"PP_Device unable to launch Consumers - Rabbit Problem?: %s" %
e.arg)
def setup_scoreboards(self):
try:
# Create Redis Forwarder table with Forwarder info
self.FWD_SCBD = ForwarderScoreboard('PP_FWD_SCBD',
self._scbd_dict['PP_FWD_SCBD'],
self._forwarder_dict)
self.JOB_SCBD = JobScoreboard('PP_JOB_SCBD',
self._scbd_dict['PP_JOB_SCBD'])
self.ACK_SCBD = AckScoreboard('PP_ACK_SCBD',
self._scbd_dict['PP_ACK_SCBD'])
except L1RabbitConnectionError as e:
LOGGER.error(
"PP_Device unable to complete setup_scoreboards-No Rabbit Connect: %s"
% e.arg)
print(
"PP_Device unable to complete setup_scoreboards - No Rabbit Connection: %s"
% e.arg)
sys.exit(self.ErrorCodePrefix + 11)
except L1RedisError as e:
LOGGER.error(
"PP_Device unable to complete setup_scoreboards - no Redis connect: %s"
% e.arg)
print(
"PP_Device unable to complete setup_scoreboards - no Redis connection: %s"
% e.arg)
sys.exit(self.ErrorCodePrefix + 12)
except Exception as e:
LOGGER.error(
"PP_Device init unable to complete setup_scoreboards: %s" %
e.arg)
print("PP_Device unable to complete setup_scoreboards: %s" % e.arg)
sys.exit(self.ErrorCodePrefix + 10)
def send_fault(error_string, error_code, job_num, component_name):
msg = {}
msg['MSG_TYPE'] = 'FAULT'
msg['COMPONENT'] = component_name
msg['JOB_NUM'] = job_num
msg['ERROR_CODE'] = str(error_code)
msg["DESCRIPTION"] = error_string
self._base_publisher.publish_message(self.DMCS_FAULT_QUEUE, msg)
def purge_broker(self, queues):
for q in queues:
cmd = "rabbitmqctl -p /tester purge_queue " + q
os.system(cmd)
def shutdown(self):
LOGGER.debug("PromptProcessDevice: Shutting down Consumer threads.")
self.shutdown_event.set()
LOGGER.debug("Thread Manager shutting down and app exiting...")
print("\n")
os._exit(0)
def engine_off(self): ThreadManager.unsubscribe(self.eventGenerator) self.running = 0
def engine_on(self): self.running = 1 #self.eventGenerator = threading.Thread(target=generateEvents, args=(self,)) self.eventGenerator = EventGenerator(self) ThreadManager.subscribe(self.eventGenerator) self.eventGenerator.start()
from TriatgeWS import TriatgeWS
import logging
import sys
# LOG_FILENAME = '/var/mail/pysimplesoap.server'
# LOG_FILENAME = 'C:/Users/greusr/Desktop/borrame.txt'
if len(sys.argv) > 1:
Constants.HOME_DIR = sys.argv[1]
LOG_FILENAME = Constants.HOME_DIR + '/log/log.log'
logging.basicConfig(filename=LOG_FILENAME, level=logging.DEBUG)
print("Init")
defaultConfig = DefaultDispatcher.defaultDispatcher_init()
threadManager = ThreadManager()
ThreadManager.setDefaultThread(defaultConfig)
#Add your WS here
triatgeConfig = TriatgeWS.triatgeDispatcher_init()
ThreadManager.addThread(triatgeConfig)
#Init all Threads
ThreadManager.startAllThreads()
print("Finish")
def __init__(self):
self.module_manager = ModuleManager(self)
self.thread_manager = ThreadManager()
self.active = True
def setup_consumer_threads(self):
LOGGER.info('Building _base_broker_url')
base_broker_url = "amqp://" + self._sub_name + ":" + \
self._sub_passwd + "@" + \
str(self._base_broker_addr)
ncsa_broker_url = "amqp://" + self._sub_ncsa_name + ":" + \
self._sub_ncsa_passwd + "@" + \
str(self._ncsa_broker_addr)
self.shutdown_event = threading.Event()
self.shutdown_event.clear()
# Set up kwargs that describe consumers to be started
# The Archive Device needs three message consumers
kws = {}
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-pp_foreman_consume'
md['queue'] = 'pp_foreman_consume'
md['callback'] = self.on_dmcs_message
md['format'] = "YAML"
md['test_val'] = None
kws[md['name']] = md
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-pp_foreman_ack_publish'
md['queue'] = 'pp_foreman_ack_publish'
md['callback'] = self.on_ack_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
md = {}
md['amqp_url'] = ncsa_broker_url
md['name'] = 'Thread-ncsa_publish'
md['queue'] = 'ncsa_publish'
md['callback'] = self.on_ncsa_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
try:
self.thread_manager = ThreadManager('thread-manager', kws,
self.shutdown_event)
self.thread_manager.start()
except ThreadError as e:
LOGGER.error(
"PP_Device unable to launch Consumers - Thread Error: %s" %
e.arg)
print("PP_Device unable to launch Consumers - Thread Error: %s" %
e.arg)
raise L1ConsumerError(
"Thread problem preventing Consumer launch: %s" % e.arg)
except Exception as e:
LOGGER.error("PP_Device unable to launch Consumers: %s" % e.arg)
print("PP_Device unable to launch Consumers: %s" % e.arg)
raise L1Error(
"PP_Device unable to launch Consumers - Rabbit Problem?: %s" %
e.arg)
from ThreadManager import ThreadManager
class RandomSource:
def __init__(self):
pass
def printRandom(self, amount, sleep):
for i in range(amount):
print("[{}]: {}".format(i, self.getRandomNumber(0, 10)))
time.sleep(sleep)
def getRandomNumber(self, min, max):
return random.randint(min, max)
source = RandomSource()
manager = ThreadManager.getInstance()
print("Creating first thread.")
manager.createThread("source1", source.printRandom, 5, 0.2)
manager.getThread("source1").waitOn()
print("Creating second thread.")
manager.createThread("source2", source.printRandom, 10, 0.2)
print("Creating third thread.")
manager.createThread("source3", source.printRandom, 5, 1)
manager.waitForThreads()
import sys
sys.path.append("/home/pi/MMNE/Threading")
from WorkerThread import WorkerThread
from ThreadManager import ThreadManager
from APListener import APListener
import subprocess
from Queue import Queue
import time
if __name__ == '__main__':
queueIn = Queue()
queueOut = Queue()
queueTuple = (queueIn, queueOut)
ListenerID = "AP_Status"
qDict = {ListenerID: queueTuple}
APdict = {ListenerID: APListener}
threadMgr = ThreadManager(APdict, qDict)
threadMgr.startThread(ListenerID)
while True:
time.sleep(1)
iwData, arpData = threadMgr.getFrom(ListenerID)
print 'iw Data: '
print iwData
print 'arp Data: '
print arpData