def __init__(self):
logging.basicConfig()
#os.system('rabbitmqctl -p /tester purge_queue firehose')
#os.system('rabbitmqctl -p /tester purge_queue ack_publish')
self.sp2 = SimplePublisher(
'amqp://*****:*****@141.142.238.160:5672/%2Fbunny', "YAML")
time.sleep(3)
broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Ftester'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#self._cons = FirehoseConsumer(broker_url, 'firehose', "YAML")
self._cons = Consumer(broker_url, 'f1_consume', "YAML")
#self._cons = Consumer(broker_url, 'pp_foreman_consume', "YAML")
self._cons2 = Consumer(broker_url, 'ncsa_consume', "YAML")
try:
_thread.start_new_thread(self.do_it, (
"thread-1",
2,
))
except e:
print("Cannot start thread")
print(e)
try:
_thread.start_new_thread(self.do_it2, (
"thread-2",
2,
))
except e:
print("Cannot start thread")
print(e)
def __init__(self, file=None):
if file == None:
file = 'L1SystemCfg.yaml'
try:
f = open(file)
except IOError:
print("Can't open %s" % file)
raise L1Error
self.cdm = yaml.safe_load(f)
broker_address = self.cdm['ROOT']['BASE_BROKER_ADDR']
name = self.cdm['ROOT']['BASE_BROKER_NAME']
passwd = self.cdm['ROOT']['BASE_BROKER_PASSWD']
self.broker_url = "amqp://" + name + ":" + passwd + "@" + str(
broker_address)
self.audit_format = "YAML"
if 'AUDIT_MSG_FORMAT' in self.cdm['ROOT']:
self.audit_format = self.cdm['ROOT']['AUDIT_MSG_FORMAT']
try:
self.audit_publisher = SimplePublisher(self.broker_url,
self.audit_format)
except L1RabbitConnectionError as e:
LOGGER.error(
"Scoreboard Parent Class cannot create SimplePublisher: ",
e.arg)
print("No Publisher for YOU")
raise L1Error('Cant create SimplePublisher'.e.arg)
def main():
premium = Premium()
sp1 = SimplePublisher('amqp://*****:*****@141.142.238.160:5672/%2Fbunny',
"YAML")
# while 1:
msg = {}
msg['MSG_TYPE'] = "STANDBY"
msg['DEVICE'] = 'AR'
time.sleep(29)
sp1.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "DISABLE"
msg['DEVICE'] = 'AR'
time.sleep(45)
sp1.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "ENABLE"
msg['DEVICE'] = 'AR'
time.sleep(74)
sp1.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "WORKING"
msg['DEVICE'] = 'AR'
time.sleep(150)
sp1.publish_message("ocs_dmcs_consume", msg)
time.sleep(5)
print("Sender done")
def test_ocs_acksubscriber(self):
try:
cdm = toolsmod.intake_yaml_file(
"../../tests/yaml/L1SystemCfg_Test_ocs_bridge.yaml")
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to fine CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]["BASE_BROKER_ADDR"]
# dmcs publisher
dmcs_pub_name = cdm[ROOT]["DMCS_BROKER_PUB_NAME"]
dmcs_pub_pwd = cdm[ROOT]["DMCS_BROKER_PUB_PASSWD"]
dmcs_broker_pub_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_pwd + "@" + \
broker_addr
self.dmcs_publisher = SimplePublisher(dmcs_broker_pub_url, "YAML")
# dmcs consumer
dmcs_name = cdm[ROOT]["DMCS_BROKER_NAME"]
dmcs_pwd = cdm[ROOT]["DMCS_BROKER_PASSWD"]
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_pwd + "@" + \
broker_addr
self.dmcs_consumer = Consumer(dmcs_broker_url, "ocs_dmcs_consume",
"thread-dmcs-consume",
self.on_ocs_message, "YAML")
self.dmcs_consumer.start()
# ocs consumer from DMCS
ocs_name = cdm[ROOT]["OCS_BROKER_NAME"]
ocs_pwd = cdm[ROOT]["OCS_BROKER_PASSWD"]
# FIXME: New OCS account for consumer test_dmcs_ocs_publish
ocs_broker_url = "amqp://" + "AFM" + ":" +\
"AFM" + "@" +\
broker_addr
self.ocs_consumer = Consumer(ocs_broker_url, "test_dmcs_ocs_publish",
"thread-ocs-consume",
self.on_dmcs_message, "YAML")
self.ocs_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority("../../messages.yaml")
self.send_messages()
sleep(10)
os.killpg(os.getpgid(self.cmdListener.pid), signal.SIGTERM)
os.killpg(os.getpgid(self.ackSubscriber.pid), signal.SIGTERM)
print("MY OCS MESSAGES: %s" % self.ocs_consumer_msg_list)
self.verify_ocs_messages()
print("Finished with CommandListener tests.")
def setup_publishers(self):
LOGGER.info('Setting up Base publisher on %s using %s',
self._base_broker_url, self._base_msg_format)
LOGGER.info('Setting up NCSA publisher on %s using %s',
self._ncsa_broker_url, self._ncsa_msg_format)
self._base_publisher = SimplePublisher(self._base_broker_url,
self._base_msg_format)
self._ncsa_publisher = SimplePublisher(self._ncsa_broker_url,
self._ncsa_msg_format)
def setup_publishers(self):
""" Set up base publisher with pub_base_broker_url by creating a new instance
of SimplePublisher class with yaml format
:params: None.
:return: None.
"""
self.pub_base_broker_url = "amqp://" + self._msg_pub_name + ":" + \
self._msg_pub_passwd + "@" + \
str(self._base_broker_addr)
LOGGER.info('Setting up Base publisher on %s using %s',
self.pub_base_broker_url, self._base_msg_format)
self._publisher = SimplePublisher(self.pub_base_broker_url,
self._base_msg_format)
def __init__(self):
logging.basicConfig()
self.ack_test = True
#os.system('rabbitmqctl -p /tester purge_queue firehose')
#os.system('rabbitmqctl -p /tester purge_queue ack_publish')
broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Ftester'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#self._cons = FirehoseConsumer(broker_url, 'firehose', "YAML")
self.sp1 = SimplePublisher(
'amqp://*****:*****@141.142.208.191:5672/%2Fbunny')
cdm = toolsmod.intake_yaml_file('ForemanCfg.yaml')
self.fdict = cdm[ROOT]['XFER_COMPONENTS']['ARCHIVE_FORWARDERS']
self.fwdrs = list(self.fdict.keys())
self._cons = Consumer(broker_url, 'ar_foreman_ack_publish', "YAML")
#self._cons = Consumer(broker_url, 'ar_foreman_consume', "YAML")
try:
_thread.start_new_thread(self.do_it, (
"thread-1",
2,
))
except e:
print("Cannot start thread")
print(e)
def parent_init(self, db_num, prog_name, type):
custom_print.define_new_name(self.PROGRAM_NAME)
# Create machine scoreboard
self._sb_mach = Scoreboard(db_num, prog_name, type)
# Messaging URL (rabbitmq server IP)
self._broker_url = "amqp://" + AMQP_BF_USER + ":" + AMQP_BF_PSWD + "@" + AMQP_BROKER_ADDR + ":" + AMQP_BROKER_PORT + "/" + AMQP_BROKER_VHOST
# Publisher object for sending messages to rabbit
printc("Creating publisher...")
self._publisher = SimplePublisher(self._broker_url)
# Machine messages
self._msg_actions_mach = {
'TRANSFER_DONE': self.process_transfer_done, # Machine done with the current job
'REGISTER': self.process_register, # New machine wants to join
'DEREGISTER': self.process_deregister, # Machine is leaving
'STATE_UPDATE': self.process_state_update # Machine updating us on its state
}
# Machines register with us and let us know how they are doing
printc("Creating machine consumer...")
self._mach_consumer = Consumer(self._broker_url, self._machine_publish_q)
try:
thread.start_new_thread(self.run_mach_consumer, ())
except:
printc("Thread run_mach_consumer failed, quitting...")
sys.exit()
return
def __init__(self):
printc("Setting up DMCS...")
self._options = "\
1 - (READY) Send Job Information\n\
2 - (SET) Send Standby Message\n\
3 - (GO) Send Readout Message\n\
4 - (RESET) Cancel a Job\n\
0 - (EXIT) Quit DMCS Simulator\n"
self._broker_url = 'amqp://' + AMQP_DMCS_USER + ':' + AMQP_DMCS_PSWD + '@' + AMQP_BROKER_ADDR + ':' + AMQP_BROKER_PORT + '/' + AMQP_BROKER_VHOST
printc('Using broker url: %s' % self._broker_url)
printc("Declaring and binding exchanges...")
printc("Attempting to create a consumer for the '%s' queue." % (Q_DMCS_CONSUME))
self._dmcs_consumer = Consumer(self._broker_url, Q_DMCS_CONSUME)
try:
printc("Attempting to start the consumer thread...")
thread.start_new_thread(self.run_dmcs_consumer, ())
except:
printc("Failed to start consumer thread, quitting...")
sys.exit()
printc("Done setting up consumer thread.")
printc("Setting up publisher...")
self._publisher = SimplePublisher(self._broker_url)
printc("Done creating publisher.")
self._job_msg = {}
self._job_msg['MSG_TYPE'] = 'JOB'
self._job_msg['JOB_NUM'] = 0
self._job_msg['RAFT_NUM'] = 1
self._standby_msg = {}
self._standby_msg['MSG_TYPE'] = 'STANDBY'
self._readout_msg = {}
self._readout_msg['MSG_TYPE'] = 'READOUT'
self._stop_msg = {}
self._stop_msg['MSG_TYPE'] = 'CANCEL'
self._shutdown_msg = {}
self._shutdown_msg['MSG_TYPE'] = 'SHUTDOWN'
class Scoreboard:
"""This is the parent class of the three scoreboard classes.
It, and they, form an interface for the Redis in-memory DB
that continually stores state information about components and jobs.
"""
AUDIT_QUEUE = 'audit_consume'
def __init__(self, file=None):
if file == None:
file = 'L1SystemCfg.yaml'
try:
f = open(file)
except IOError:
print("Can't open %s" % file)
raise L1Error
self.cdm = yaml.safe_load(f)
broker_address = self.cdm['ROOT']['BASE_BROKER_ADDR']
name = self.cdm['ROOT']['BASE_BROKER_NAME']
passwd = self.cdm['ROOT']['BASE_BROKER_PASSWD']
self.broker_url = "amqp://" + name + ":" + passwd + "@" + str(
broker_address)
self.audit_format = "YAML"
if 'AUDIT_MSG_FORMAT' in self.cdm['ROOT']:
self.audit_format = self.cdm['ROOT']['AUDIT_MSG_FORMAT']
try:
self.audit_publisher = SimplePublisher(self.broker_url,
self.audit_format)
except L1RabbitConnectionError as e:
LOGGER.error(
"Scoreboard Parent Class cannot create SimplePublisher: ",
e.arg)
print("No Publisher for YOU")
raise L1Error('Cant create SimplePublisher'.e.arg)
def persist(self, data):
self.audit_publisher.publish_message(self.AUDIT_QUEUE, data)
def persist_snapshot(self, connection, filename):
pass
"""
def setup_publishers(self):
self._pub_base_broker_url = "amqp://" + self._pub_base_name + ":" + \
self._pub_base_passwd + "@" + \
str(self._base_broker_addr)
self._pub_ncsa_broker_url = "amqp://" + self._pub_ncsa_name + ":" + \
self._pub_ncsa_passwd + "@" + \
str(self._ncsa_broker_addr)
LOGGER.info('Setting up Base publisher on %s using %s', \
self._pub_base_broker_url, self._base_msg_format)
self._base_publisher = SimplePublisher(self._pub_base_broker_url,
self._base_msg_format)
LOGGER.info('Setting up NCSA publisher on %s using %s', \
self._pub_ncsa_broker_url, self._ncsa_msg_format)
self._ncsa_publisher = SimplePublisher(self._pub_ncsa_broker_url,
self._ncsa_msg_format)
def test_ar_ctrl(self):
try:
cdm = toolsmod.intake_yaml_file(
'/home/centos/src/git/ctrl_iip/python/lsst/iip/tests/yaml/L1SystemCfg_Test_ar-ctrl.yaml'
)
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
afm_name = cdm[ROOT]["AFM_BROKER_NAME"]
afm_pwd = cdm[ROOT]["AFM_BROKER_PASSWD"]
afm_pub_name = cdm[ROOT]["AFM_BROKER_PUB_NAME"]
afm_pub_pwd = cdm[ROOT]["AFM_BROKER_PUB_PASSWD"]
self.xfer_root = cdm[ROOT]["ARCHIVE"]["ARCHIVE_XFER_ROOT"]
ar_dev_pub_broker_url = "amqp://" + afm_pub_name + ":" +\
afm_pub_pwd + "@" +\
broker_addr
self.ar_dev_publisher = SimplePublisher(ar_dev_pub_broker_url, "YAML")
ar_dev_broker_url = "amqp://" + afm_name + ":" +\
afm_pwd + "@" +\
broker_addr
self.ar_dev_consumer = Consumer(ar_dev_broker_url, self.ACK_FROM_CTRL,
"thread-ar-dev", self.on_ar_message,
"YAML", None)
self.ar_dev_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority(
'/home/centos/src/git/ctrl_iip/python/lsst/iip/messages.yaml')
self.send_messages()
sleep(10)
self.verify_ar_messages()
self.verify_path_files()
print("Finished with AR CTRL tests.")
def test_ocs_acksubscriber(self):
try:
cdm = toolsmod.intake_yaml_file("../../tests/yaml/L1SystemCfg_Test_ocs_bridge.yaml")
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to fine CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]["BASE_BROKER_ADDR"]
# dmcs publisher
dmcs_pub_name = cdm[ROOT]["DMCS_BROKER_PUB_NAME"]
dmcs_pub_pwd = cdm[ROOT]["DMCS_BROKER_PUB_PASSWD"]
dmcs_broker_pub_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_pwd + "@" + \
broker_addr
self.dmcs_publisher = SimplePublisher(dmcs_broker_pub_url, "YAML")
# dmcs consumer
dmcs_name = cdm[ROOT]["DMCS_BROKER_NAME"]
dmcs_pwd = cdm[ROOT]["DMCS_BROKER_PASSWD"]
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_pwd + "@" + \
broker_addr
self.dmcs_consumer = Consumer(dmcs_broker_url, "ocs_dmcs_consume", "thread-dmcs-consume",
self.on_ocs_message, "YAML")
self.dmcs_consumer.start()
# ocs consumer from DMCS
ocs_name = cdm[ROOT]["OCS_BROKER_NAME"]
ocs_pwd = cdm[ROOT]["OCS_BROKER_PASSWD"]
# FIXME: New OCS account for consumer test_dmcs_ocs_publish
ocs_broker_url = "amqp://" + "AFM" + ":" +\
"AFM" + "@" +\
broker_addr
self.ocs_consumer = Consumer(ocs_broker_url, "test_dmcs_ocs_publish", "thread-ocs-consume",
self.on_dmcs_message, "YAML")
self.ocs_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority("../../messages.yaml")
self.send_messages()
sleep(10)
os.killpg(os.getpgid(self.cmdListener.pid), signal.SIGTERM)
os.killpg(os.getpgid(self.ackSubscriber.pid), signal.SIGTERM)
print("MY OCS MESSAGES: %s" % self.ocs_consumer_msg_list)
self.verify_ocs_messages()
print("Finished with CommandListener tests.")
def __init__(self):
try:
# Create a lock for critical sections (used when canceling jobs)
self._lock = thread.allocate_lock()
self.child_init()
# Create a temporary name until unique name is assigned
c_name = ''.join(random.choice(string.ascii_letters) for x in range(NAME_LENGTH))
c_name = self._type + "_" + c_name
custom_print.define_new_name(c_name)
printc("Starting...")
# Register the machine with foreman before doing anything else
self._register = Registration(self._publish_queue,
self._consume_queue)
# Block until a unique name is received
self._name = self._register.request_name()
except:
return
try:
# Debug print setup
custom_print.define_new_name(self._name)
# Internal variable of the current state
self._current_state = IDLE
# Messaging setup
self._broker_url = "amqp://" + AMQP_MACH_USER + ":" + AMQP_MACH_PSWD + "@" + AMQP_BROKER_ADDR + ":" + AMQP_BROKER_PORT + "/" + AMQP_BROKER_VHOST
self._consume_queue = self._name + "_consume"
self._home_dir = XFER_DIR
# Messages foreman can send to us
self._msg_actions = {
JOB: self.process_foreman_job,
STANDBY: self.process_foreman_standby,
READOUT: self.process_foreman_readout,
CANCEL: self.process_foreman_cancel
}
# Publisher to send messages to foreman
printc("Setting up publisher...")
self._publisher = SimplePublisher(self._broker_url)
# Consumer for getting messages from foreman
printc("Setting up consumer...")
self._consumer = Consumer(self._broker_url, self._consume_queue)
except:
pass
# Run blocking consumer
try:
self.run_consumer()
except:
pass
# Alert foreman this machine is shutting down
self.deregister()
return
class Registration:
# Registration class that is used to register a machine as a forwarder or
# distributor along with assigning a unique name.
def __init__(self, publish, consume):
printc("Sending registration request")
self._name = "UNKNOWN"
# Messaging setup
self._broker_url = "amqp://" + AMQP_MACH_USER + ":" + AMQP_MACH_PSWD + "@" + AMQP_BROKER_ADDR + ":" + AMQP_BROKER_PORT + "/" + AMQP_BROKER_VHOST
self._publisher = SimplePublisher(self._broker_url)
self._register_msg = {}
self._register_msg[MSG_TYPE] = 'REGISTER'
self._register_msg['IP_ADDR'] = DETECTED_IP_ADDRESS
self._request_publish = publish
self._request_consume = consume
self._connection = pika.BlockingConnection(pika.URLParameters(self._broker_url))
self._channel = self._connection.channel()
self._channel.queue_declare(queue=self._request_consume)
return
def request_name(self):
# Send name request message and wait for a name
printc("Requesting name...")
self._publisher.publish_message(self._request_publish,
yaml.dump(self._register_msg))
method_frame, header_frame, body = self._channel.basic_get(queue=self._request_consume)
while None == method_frame:
method_frame, header_frame, body = self._channel.basic_get(queue=self._request_consume)
# The else clause is only executed when your while condition becomes
# false. If you break out of the loop, or if an exception is raised,
# it won't be executed.
else:
self._channel.basic_ack(delivery_tag=method_frame.delivery_tag)
self._connection.close()
msg_dict = yaml.load(body)
self._name = msg_dict['NAME']
printc("Got name %s" % self._name)
return self._name
def setup_publishers(self):
self._pub_base_broker_url = "amqp://" + self._pub_name + ":" + \
self._pub_passwd + "@" + \
str(self._base_broker_addr)
self._pub_ncsa_broker_url = "amqp://" + self._pub_ncsa_name + ":" + \
self._pub_ncsa_passwd + "@" + \
str(self._ncsa_broker_addr)
try:
LOGGER.info('Setting up Base publisher on %s using %s', \
self._pub_base_broker_url, self._base_msg_format)
self._base_publisher = SimplePublisher(self._pub_base_broker_url,
self._base_msg_format)
LOGGER.info('Setting up NCSA publisher on %s using %s', \
self._pub_ncsa_broker_url, self._ncsa_msg_format)
self._ncsa_publisher = SimplePublisher(self._pub_ncsa_broker_url,
self._ncsa_msg_format)
except Exception as e:
LOGGER.error("PP_Device unable to start Publishers: %s" % e.arg)
print("PP_Device unable to start Publishers: %s" % e.arg)
raise L1PublisherError(
"Critical Error: Unable to create Publishers: %s" % e.arg)
def __init__(self, publish, consume):
printc("Sending registration request")
self._name = "UNKNOWN"
# Messaging setup
self._broker_url = "amqp://" + AMQP_MACH_USER + ":" + AMQP_MACH_PSWD + "@" + AMQP_BROKER_ADDR + ":" + AMQP_BROKER_PORT + "/" + AMQP_BROKER_VHOST
self._publisher = SimplePublisher(self._broker_url)
self._register_msg = {}
self._register_msg[MSG_TYPE] = 'REGISTER'
self._register_msg['IP_ADDR'] = DETECTED_IP_ADDRESS
self._request_publish = publish
self._request_consume = consume
self._connection = pika.BlockingConnection(pika.URLParameters(self._broker_url))
self._channel = self._connection.channel()
self._channel.queue_declare(queue=self._request_consume)
return
def test_ar_ctrl(self):
try:
cdm = toolsmod.intake_yaml_file('/home/centos/src/git/ctrl_iip/python/lsst/iip/tests/yaml/L1SystemCfg_Test_ar-ctrl.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
afm_name = cdm[ROOT]["AFM_BROKER_NAME"]
afm_pwd = cdm[ROOT]["AFM_BROKER_PASSWD"]
afm_pub_name = cdm[ROOT]["AFM_BROKER_PUB_NAME"]
afm_pub_pwd = cdm[ROOT]["AFM_BROKER_PUB_PASSWD"]
self.xfer_root = cdm[ROOT]["ARCHIVE"]["ARCHIVE_XFER_ROOT"]
ar_dev_pub_broker_url = "amqp://" + afm_pub_name + ":" +\
afm_pub_pwd + "@" +\
broker_addr
self.ar_dev_publisher = SimplePublisher(ar_dev_pub_broker_url, "YAML")
ar_dev_broker_url = "amqp://" + afm_name + ":" +\
afm_pwd + "@" +\
broker_addr
self.ar_dev_consumer = Consumer(ar_dev_broker_url, self.ACK_FROM_CTRL, "thread-ar-dev",
self.on_ar_message, "YAML", None)
self.ar_dev_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority('/home/centos/src/git/ctrl_iip/python/lsst/iip/messages.yaml')
self.send_messages()
sleep(10)
self.verify_ar_messages()
self.verify_path_files()
print("Finished with AR CTRL tests.")
class TestNcsa:
pp_pub_broker_url = None
pp_publisher = None
pp_consumer = None
pp_consumer_msg_list = []
D1_pub_broker_url = None
D1_publisher = None
D1_consumer = None
d1_consumer_msg_list = []
D2_pub_broker_url = None
D2_publisher = None
D2_consumer = None
d2_consumer_msg_list = []
EXPECTED_PP_MESSAGES = 1
EXPECTED_D1_MESSAGES = 1
EXPECTED_D2_MESSAGES = 1
ccd_list = [14, 17, 21, 86]
prp = toolsmod.prp
DP = toolsmod.DP
def test_ncsa(self, Ncsa):
self.ncsa = Ncsa
try:
cdm = toolsmod.intake_yaml_file(
'./tests/yaml/L1SystemCfg_Test_ncsa.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
pp_name = cdm[ROOT]['PFM_BROKER_NAME']
pp_passwd = cdm[ROOT]['PFM_BROKER_PASSWD']
pp_pub_name = cdm[ROOT]['PFM_BROKER_PUB_NAME']
pp_pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD']
pp_broker_url = "amqp://" + pp_name + ":" + \
pp_passwd + "@" + \
broker_addr
pp_pub_broker_url = "amqp://" + pp_pub_name + ":" + \
pp_pub_passwd + "@" + \
broker_addr
self.pp_publisher = SimplePublisher(pp_pub_broker_url, "YAML")
D1_name = 'D1'
D1_passwd = 'D1'
D1_pub_name = 'D1_PUB'
D1_pub_passwd = 'D1_PUB'
D1_broker_url = "amqp://" + D1_name + ":" + \
D1_passwd + "@" + \
broker_addr
D1_pub_broker_url = "amqp://" + D1_pub_name + ":" + \
D1_pub_passwd + "@" + \
broker_addr
self.d1_publisher = SimplePublisher(D1_pub_broker_url, "YAML")
D2_name = 'D2'
D2_passwd = 'D2'
D2_pub_name = 'D2_PUB'
D2_pub_passwd = 'D2_PUB'
D2_broker_url = "amqp://" + D2_name + ":" + \
D2_passwd + "@" + \
broker_addr
D2_pub_broker_url = "amqp://" + D2_pub_name + ":" + \
D2_pub_passwd + "@" + \
broker_addr
self.d2_publisher = SimplePublisher(D2_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.pp_consumer = Consumer(pp_broker_url, 'pp_foreman_ack_publish',
'thread-pp', self.on_pp_message, 'YAML')
self.pp_consumer.start()
self.D1_consumer = Consumer(D1_broker_url, 'd1_consume', 'thread-d1',
self.on_d1_message, 'YAML')
self.D1_consumer.start()
self.D2_consumer = Consumer(D2_broker_url, 'd2_consume', 'thread-d2',
self.on_d2_message, 'YAML')
self.D2_consumer.start()
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(7)
self.verify_pp_messages()
self.verify_D2_messages()
self.verify_D1_messages()
sleep(2)
self.pp_consumer.stop()
self.pp_consumer.join()
self.D1_consumer.stop()
self.D1_consumer.join()
self.D2_consumer.stop()
self.D2_consumer.join()
if self.DP:
print("Finished with NCSA tests.")
def send_messages(self):
print("Starting send_messages")
# Tests only an AR device
# self.clear_message_lists()
self.EXPECTED_PP_MESSAGES = 4
self.EXPECTED_D1_MESSAGES = 3
self.EXPECTED_D2_MESSAGES = 3
msg = {}
msg['MSG_TYPE'] = "NCSA_NEW_SESSION"
msg['SESSION_ID'] = 'SI_469976'
msg['ACK_ID'] = 'NCSA_NEW_SESSION_ACK_44221'
msg['REPLY_QUEUE'] = 'pp_foreman_ack_publish'
time.sleep(1)
if self.DP:
print("New Session Message")
self.pp_publisher.publish_message("ncsa_consume", msg)
msg = {}
msg['MSG_TYPE'] = "NCSA_NEXT_VISIT"
msg['VISIT_ID'] = 'XX_28272'
msg['SESSION_ID'] = 'NNV_469976'
msg['REPLY_QUEUE'] = 'pp_foreman_ack_publish'
msg['ACK_ID'] = 'NCSA_NEW_VISIT_ACK_76'
msg['BORE_SIGHT'] = "231,123786456342, -45.3457156906, FK5"
time.sleep(1)
if self.DP:
print("Next Visit Message")
self.pp_publisher.publish_message("ncsa_consume", msg)
msg = {}
msg['MSG_TYPE'] = "NCSA_START_INTEGRATION"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444244'
msg['VISIT_ID'] = 'V14494'
msg['SESSION_ID'] = '4_14_7211511'
msg['REPLY_QUEUE'] = 'pp_foreman_ack_publish'
msg['ACK_ID'] = 'PP_ACK_94671'
msg['CCD_LIST'] = [4, 14, 16, 17, 29, 35, 36]
time.sleep(2)
msg['FORWARDERS'] = {}
forwarder_list = ['FORWARDER_2', 'FORWARDER_1']
ccd_list = [[17, 18, 111, 126], [128, 131, 132]]
msg['FORWARDERS']['FORWARDER_LIST'] = forwarder_list
msg['FORWARDERS']['CCD_LIST'] = ccd_list
if self.DP:
print("NCSA START INTEGRATION Message")
self.pp_publisher.publish_message("ncsa_consume", msg)
time.sleep(7)
msg = {}
msg['MSG_TYPE'] = "NCSA_READOUT"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444244'
msg['VISIT_ID'] = 'V14494'
msg['SESSION_ID'] = '4_14_7211511'
msg['REPLY_QUEUE'] = 'pp_foreman_ack_publish'
msg['ACK_ID'] = 'NCSA_READOUT_ACK_44221'
time.sleep(2)
if self.DP:
print("NCSA READOUT Message")
self.pp_publisher.publish_message("ncsa_consume", msg)
time.sleep(2)
print("Message Sender done")
def verify_pp_messages(self):
len_list = len(self.pp_consumer_msg_list)
if len_list != self.EXPECTED_PP_MESSAGES:
pytest.fail('PP simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_PP_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.pp_consumer_msg_list[i]
if msg['MSG_TYPE'] == 'NCSA_START_INTEGRATION_ACK':
result = self.check_start_int_ack(msg)
else:
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail(
"The following message to the PP Foreman failed when compared with the sovereign example: %s"
% msg)
if self.DP:
print("Messages to the PP Foreman pass verification.")
def check_start_int_ack(self, msg):
"""the PAIRS param in the message is a list. Every item in the list is a dictionary.
because it is not known how many entried will be on the list, the dictionaries
are deepcopied and checked against the MessageAuthority and check one at a time.
In the meanwhile, the shape of the incoming message without the dictionaries and the
PAIRS list set to None is checked against the MessageAuthority.
BTW, because the individual dictionaries do not have a MSG_TYPE, one is
added to each dictionary to be checked so the dict to compare to can be
located in the messages.yaml file.
"""
c_msg = deepcopy(msg)
pairs = deepcopy(c_msg['PAIRS'])
c_msg['PAIRS'] = None
overall_shape = self._msg_auth.check_message_shape(c_msg)
if overall_shape == False:
return False
for i in range(0, len(pairs)):
pairs[i]['MSG_TYPE'] = 'PAIR'
result = self._msg_auth.check_message_shape(pairs[i])
if result == False:
return False
return True
def verify_D1_messages(self):
len_list = len(self.d1_consumer_msg_list)
if len_list != self.EXPECTED_D1_MESSAGES:
if self.DP:
print("Messages received by verify_D1_messages:")
self.prp.pprint(self.f1_consumer_msg_list)
pytest.fail('F1 simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_D1_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.d1_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail(
"The following message to D1 failed when compared with the sovereign example: %s"
% msg)
if self.DP:
print("Messages to D1 pass verification.")
def verify_D2_messages(self):
len_list = len(self.d2_consumer_msg_list)
if len_list != self.EXPECTED_D2_MESSAGES:
if self.DP:
print("Messages received by verify_D2_messages:")
self.prp.pprint(self.d2_consumer_msg_list)
pytest.fail('D2 simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_D2_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.d2_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail(
"The following message to D2 failed when compared with the sovereign example: %s"
% msg)
if self.DP:
print("Messages to D2 pass verification.")
def on_pp_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_ncsa - incoming on_pp_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.pp_consumer_msg_list.append(body)
def on_d1_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_ncsa - incoming on_D1_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.d1_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'DISTRIBUTOR_HEALTH_CHECK':
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_HEALTH_CHECK_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_1'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.d1_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'DISTRIBUTOR_XFER_PARAMS':
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_XFER_PARAMS_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_1'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.d1_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'DISTRIBUTOR_READOUT':
# Find message in message list for xfer_params
xfer_msg = None
for msg in self.d1_consumer_msg_list:
if msg['MSG_TYPE'] == 'DISTRIBUTOR_XFER_PARAMS':
xfer_msg = msg
break
if xfer_msg == None:
pytest.fail(
"The DISTRIBUTOR_XFER_PARAMS message was not received before DISTRIBUTOE_READOUT in D1"
)
# use message to build response
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_READOUT_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_1'
msg['JOB_NUM'] = xfer_msg['JOB_NUM']
msg['IMAGE_ID'] = xfer_msg['IMAGE_ID']
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
msg['RESULT_LIST'] = {}
msg['RESULT_LIST']['CCD_LIST'] = []
msg['RESULT_LIST']['RECEIPT_LIST'] = []
ccd_list = xfer_msg['XFER_PARAMS']['CCD_LIST']
receipt_list = []
for i in range(0, len(ccd_list)):
receipt_list.append('F1_Rec_x477_' + str(i))
msg['RESULT_LIST']['RECEIPT_LIST'] = receipt_list
msg['RESULT_LIST']['CCD_LIST'] = list(ccd_list)
self.d1_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail(
"The following unknown message was received by D1: %s" % body)
def on_d2_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_ncsa - incoming on_D2_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.d2_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'DISTRIBUTOR_HEALTH_CHECK':
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_HEALTH_CHECK_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_2'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.d2_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'DISTRIBUTOR_XFER_PARAMS':
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_XFER_PARAMS_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_2'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.d2_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'DISTRIBUTOR_READOUT':
# Find message in message list for xfer_params
xfer_msg = None
for msg in self.d2_consumer_msg_list:
if msg['MSG_TYPE'] == 'DISTRIBUTOR_XFER_PARAMS':
xfer_msg = msg
break
if xfer_msg == None:
pytest.fail(
"The DISTRIBUTOR_XFER_PARAMS message was not received before DISTRIBUTOR_READOUT in D2"
)
# use message to build response
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_READOUT_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_2'
msg['JOB_NUM'] = xfer_msg['JOB_NUM']
msg['IMAGE_ID'] = xfer_msg['IMAGE_ID']
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
msg['RESULT_LIST'] = {}
msg['RESULT_LIST']['CCD_LIST'] = []
msg['RESULT_LIST']['RECEIPT_LIST'] = []
ccd_list = xfer_msg['XFER_PARAMS']['CCD_LIST']
receipt_list = []
for i in range(0, len(ccd_list)):
receipt_list.append('F2_Rec_x447_' + str(i))
msg['RESULT_LIST']['RECEIPT_LIST'] = receipt_list
msg['RESULT_LIST']['CCD_LIST'] = list(ccd_list)
self.d2_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail(
"The following unknown message was received by D2: %s" % body)
def test_ardev(self, Ardev):
self.ardev = Ardev
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test_ar.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
raise
raft_dict = cdm[ROOT]['DEFAULT_RAFT_CONFIGURATION']
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
dmcs_name = cdm[ROOT]['DMCS_BROKER_NAME']
dmcs_passwd = cdm[ROOT]['DMCS_BROKER_PASSWD']
dmcs_pub_name = cdm[ROOT]['DMCS_BROKER_PUB_NAME']
dmcs_pub_passwd = cdm[ROOT]['DMCS_BROKER_PUB_PASSWD']
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_passwd + "@" + \
broker_addr
dmcs_pub_broker_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % dmcs_pub_broker_url)
self.dmcs_publisher = SimplePublisher(dmcs_pub_broker_url, "YAML")
ar_ctrl_name = cdm[ROOT]['ARCHIVE_BROKER_NAME']
ar_ctrl_passwd = cdm[ROOT]['ARCHIVE_BROKER_PASSWD']
ar_ctrl_pub_name = cdm[ROOT]['ARCHIVE_BROKER_PUB_NAME']
ar_ctrl_pub_passwd = cdm[ROOT]['ARCHIVE_BROKER_PUB_PASSWD']
ar_ctrl_broker_url = "amqp://" + ar_ctrl_name + ":" + \
ar_ctrl_passwd + "@" + \
broker_addr
ar_ctrl_pub_broker_url = "amqp://" + ar_ctrl_pub_name + ":" + \
ar_ctrl_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % ar_ctrl_pub_broker_url)
self.ar_ctrl_publisher = SimplePublisher(ar_ctrl_pub_broker_url, "YAML")
F1_name = 'F1'
F1_passwd = 'F1'
F1_pub_name = 'F1_PUB'
F1_pub_passwd = 'F1_PUB'
F1_broker_url = "amqp://" + F1_name + ":" + \
F1_passwd + "@" + \
broker_addr
F1_pub_broker_url = "amqp://" + F1_pub_name + ":" + \
F1_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % F1_pub_broker_url)
self.F1_publisher = SimplePublisher(F1_pub_broker_url, "YAML")
F2_name = 'F2'
F2_passwd = 'F2'
F2_pub_name = 'F2_PUB'
F2_pub_passwd = 'F2_PUB'
F2_broker_url = "amqp://" + F2_name + ":" + \
F2_passwd + "@" + \
broker_addr
F2_pub_broker_url = "amqp://" + F2_pub_name + ":" + \
F2_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % F2_pub_broker_url)
self.F2_publisher = SimplePublisher(F2_pub_broker_url, "YAML")
print("All publishers are running...")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
print("MessageAuthority running...")
self.dmcs_consumer = Consumer(dmcs_broker_url,'dmcs_ack_consume', 'thread-dmcs',
self.on_dmcs_message,'YAML')
self.dmcs_consumer.start()
print("DMCS Consumer running...")
self.ar_ctrl_consumer = Consumer(ar_ctrl_broker_url,'archive_ctrl_consume', 'thread-ar-ctrl',
self.on_ar_ctrl_message,'YAML')
self.ar_ctrl_consumer.start()
print("ar_ctrl Consumer running...")
self.F1_consumer = Consumer(F1_broker_url,'f1_consume', 'thread-f1',
self.on_f1_message,'YAML')
self.F1_consumer.start()
print("F1 Consumer running...")
self.F2_consumer = Consumer(F2_broker_url,'f2_consume', 'thread-f2',
self.on_f2_message,'YAML')
self.F2_consumer.start()
print("F2 Consumer running...")
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(8)
self.verify_dmcs_messages()
self.verify_ar_ctrl_messages()
self.verify_F1_messages()
self.verify_F2_messages()
sleep(3)
# Shut down consumer threads nicely
self.dmcs_consumer.stop()
self.dmcs_consumer.join()
self.ar_ctrl_consumer.stop()
self.ar_ctrl_consumer.join()
self.F1_consumer.stop()
self.F1_consumer.join()
self.F2_consumer.stop()
self.F2_consumer.join()
if self.DP:
print("Finished with AR tests.")
class Forwarder:
"""Presents a vanilla L1 Forwarder personality. In
nightly operation, at least 21 of these
components will be available at any time (one for each raft).
"""
def __init__(self):
self._registered = False
f = open('ForwarderCfg.yaml')
#cfg data map...
cdm = yaml.safe_load(f)
try:
self._name = cdm[NAME]
self._passwd = cdm[PASSWD]
self._fqn = cdm[FQN]
self._base_broker_addr = cdm[BASE_BROKER_ADDR]
self._consume_queue = cdm[CONSUME_QUEUE]
#self._publish_queue = cdm[PUBLISH_QUEUE]
self._hostname = cdm[HOSTNAME]
self._ip_addr = cdm[IP_ADDR]
self._DAQ_PATH = cdm['DAQ_PATH']
## XXX FIX: Put in config file
self.CHECKSUM_ENABLED = False
except KeyError as e:
print("Missing base keywords in yaml file... Bailing out...")
sys.exit(99)
#self._DAQ_PATH = "/home/F1/xfer_dir/"
self._home_dir = "/home/" + self._name + "/"
self._base_broker_url = "amqp://" + self._name + ":" + self._passwd + "@" + str(
self._base_broker_addr)
self._msg_actions = {
FORWARDER_HEALTH_CHECK: self.process_health_check,
FORWARDER_JOB_PARAMS: self.process_job_params,
'AR_FWDR_XFER_PARAMS':
self.process_job_params, # Here if AR case needs different handler
'AR_FWDR_READOUT': self.process_foreman_readout,
FORWARDER_READOUT: self.process_foreman_readout
}
self.setup_publishers()
self.setup_consumers()
self._job_scratchpad = Scratchpad(self._base_broker_url)
def setup_publishers(self):
self._publisher = SimplePublisher(self._base_broker_url)
def setup_consumers(self):
threadname = "thread-" + self._consume_queue
print("Threadname is %s" % threadname)
self._consumer = Consumer(self._base_broker_url, self._consume_queue)
try:
_thread.start_new_thread(self.run_consumer, (
threadname,
2,
))
print("Started Consumer Thread")
except:
sys.exit(99)
def run_consumer(self, threadname, delay):
self._consumer.run(self.on_message)
def on_message(self, ch, method, properties, body):
ch.basic_ack(delivery_tag)
print("INcoming PARAMS, body is:\n%s" % body)
msg_dict = body
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def process_health_check(self, params):
self.send_ack_response("FORWARDER_HEALTH_CHECK_ACK", params)
def process_job_params(self, params):
""" The structure of the incoming job params is identical to the way job params
are sent to prompt processing forwarders:
MSG_TYPE: AR_FWDR_XFER_PARAMS
JOB_NUM: .....
ACK_ID: x1
REPLY_QUEUE: .....
FITS: FITS metadata someday?
TRANSFER_PARAMS:
DISTRIBUTOR:
FQN: Name of entity receivine file
NAME: login name for receiving entity
HOSTNAME: Full host name for receiving entity
IP_ADDR: ip addr of archive
TARGET_DIR: Where to put file
## Below might be better as 'xfer_unit_list' for ccds or rafts, or other
CCD_LIST: for example...[1,2,3,7,10,14]
XFER_UNIT: CCD
FITS: FITS metadata someday?
After the xfer params arrive, and ack is returned, we set up some short cut helpers, such as:
1) Make a filename stub for job that leaves out all but the CCD number
2) Put together the scp/bbftp string with login name and ip addr, plus target dir
"""
job_params = copy.deepcopy(params)
xfer_params = job_params['TRANSFER_PARAMS']
# Also RM fits files in xfer_dir
cmd = "rm " + self._DAQ_PATH + "*.fits"
os.system(cmd)
filename_stub = str(job_params['JOB_NUM']) + "_" + str(
job_params['VISIT_ID']) + "_" + str(job_params['IMAGE_ID']) + "_"
login_str = str(xfer_params['DISTRIBUTOR']['NAME']) + "@" + str(
xfer_params['DISTRIBUTOR']['IP_ADDR']) + ":"
target_dir = str(xfer_params['DISTRIBUTOR']['TARGET_DIR'])
#xfer_params = transfer_params['XFER_PARAMS']
s_params = {}
s_params['CCD_LIST'] = xfer_params['CCD_LIST']
s_params['LOGIN_STR'] = login_str
s_params['TARGET_DIR'] = target_dir
s_params['FILENAME_STUB'] = filename_stub
print("S_params are: %s" % s_params)
# Now, s_params should have all we need for job. Place as value for job_num key
self._job_scratchpad.set_job_transfer_params(params[JOB_NUM], s_params)
self._job_scratchpad.set_job_state(params['JOB_NUM'],
"READY_WITH_PARAMS")
self.send_ack_response('FORWARDER_JOB_PARAMS_ACK', params)
def process_foreman_readout(self, params):
# self.send_ack_response("FORWARDER_READOUT_ACK", params)
reply_queue = params['REPLY_QUEUE']
job_number = params[JOB_NUM]
# Check and see if scratchpad has this job_num
if job_number not in list(self._job_scratchpad.keys()):
# Raise holy hell...
pass
# raw_files_dict is of the form { ccd: filename} like { 2: /home/F1/xfer_dir/ccd_2.data
raw_files_dict = self.fetch(job_number)
final_filenames = self.format(job_number, raw_files_dict)
results = self.forward(job_number, final_filenames)
msg = {}
msg['MSG_TYPE'] = 'AR_ITEMS_XFERD_ACK'
msg['JOB_NUM'] = job_number
msg['IMAGE_ID'] = params['IMAGE_ID']
msg['COMPONENT'] = self._fqn
msg['ACK_ID'] = params['ACK_ID']
msg['ACK_BOOL'] = True # See if num keys of results == len(ccd_list) from orig msg params
msg['RESULT_LIST'] = results
self._publisher.publish_message(reply_queue, msg)
def fetch(self, job_num):
raw_files_dict = {}
ccd_list = self._job_scratchpad.get_job_value(job_num, 'CCD_LIST')
for ccd in ccd_list:
filename = "ccd_" + str(ccd) + ".data"
raw_files_dict[ccd] = filename
print("In Forwarder Fetch method, raw_files_dict is: \n%s" %
raw_files_dict)
return raw_files_dict
"""
format raw files to fits file with header data
:param file_list: dictionary of file_name against raw file name
:param mdata: primary meta data stream fetched from camera daq
"""
def format(self, file_list, mdata):
final_filenames = []
for ccd_id, raw_file_name in file_list.items():
image_array = np.fromfile(raw_file_name, dtype=np.int32)
header_data = mdata[ccd_id]["primary_metadata_chunk"]
secondary_data = mdata[ccd_id]["secondary_metadata_chunk"]
header_data.update(secondary_data)
primary_header = pyfits.Header()
for key, value in header_data.items():
primary_header[key] = value
fits_file = pyfits.PrimaryHDU(header=primary_header,
data=image_array)
fits_file.writeto(ccd_id + ".fits")
final_filenames.append(ccd_id + ".fits")
return final_filenames
def format(self, job_num, raw_files_dict):
keez = list(raw_files_dict.keys())
filename_stub = self._job_scratchpad.get_job_value(
job_num, 'FILENAME_STUB')
final_filenames = {}
for kee in keez:
final_filename = filename_stub + "_" + kee + ".fits"
target = self._DAQ_PATH + final_filename
print("Final filename is %s" % final_filename)
print("target is %s" % target)
cmd1 = 'cat ' + self._DAQ_PATH + "ccd.header" + " >> " + target
cmd2 = 'cat ' + self._DAQ_PATH + raw_files_dict[
kee] + " >> " + target
dte = get_epoch_timestamp()
print("DTE IS %s" % dte)
cmd3 = 'echo ' + str(dte) + " >> " + target
print("cmd1 is %s" % cmd1)
print("cmd2 is %s" % cmd2)
os.system(cmd1)
os.system(cmd2)
os.system(cmd3)
final_filenames[kee] = final_filename
print("Done in format()...file list is: %s" % final_filenames)
print("In format method, final_filenames are:\n%s" % final_filenames)
return final_filenames
def forward(self, job_num, final_filenames):
print("Start Time of READOUT IS: %s" % get_timestamp())
login_str = self._job_scratchpad.get_job_value(job_num, 'LOGIN_STR')
target_dir = self._job_scratchpad.get_job_value(job_num, 'TARGET_DIR')
results = {}
CCD_LIST = []
FILENAME_LIST = []
CHECKSUM_LIST = []
ccds = list(final_filenames.keys())
for ccd in ccds:
final_file = final_filenames[ccd]
pathway = self._DAQ_PATH + final_file
with open(pathway) as file_to_calc:
if self.CHECKSUM_ENABLED:
data = file_to_calc.read()
resulting_md5 = hashlib.md5(data).hexdigest()
else:
resulting_md5 = '0'
minidict = {}
CCD_LIST.append(ccd)
CHECKSUM_LIST.append(resulting_md5)
FILENAME_LIST.append(target_dir + final_file)
cmd = 'scp ' + pathway + " " + login_str + target_dir + final_file
print("Finish Time of SCP'ing %s IS: %s" %
(pathway, get_timestamp()))
print("In forward() method, cmd is %s" % cmd)
os.system(cmd)
results['CCD_LIST'] = CCD_LIST
results['FILENAME_LIST'] = FILENAME_LIST
results['CHECKSUM_LIST'] = CHECKSUM_LIST
print("END Time of READOUT XFER IS: %s" % get_timestamp())
print("In forward method, results are: \n%s" % results)
return results
#cmd = 'cd ~/xfer_dir && scp -r $(ls -t)' + ' ' + str(self._xfer_login) + ':xfer_dir'
#pass
def send_ack_response(self, type, params):
timed_ack = params.get("ACK_ID")
job_num = params.get(JOB_NUM)
response_queue = params['RESPONSE_QUEUE']
msg_params = {}
msg_params[MSG_TYPE] = type
msg_params[JOB_NUM] = job_num
msg_params['COMPONENT'] = self._fqn
msg_params[ACK_BOOL] = "TRUE"
msg_params[ACK_ID] = timed_ack
self._publisher.publish_message(response_queue, msg_params)
def register(self):
pass
class Machine:
# Parent class for forwarders and distributors for their shared attributes.
def __init__(self):
try:
# Create a lock for critical sections (used when canceling jobs)
self._lock = thread.allocate_lock()
self.child_init()
# Create a temporary name until unique name is assigned
c_name = ''.join(random.choice(string.ascii_letters) for x in range(NAME_LENGTH))
c_name = self._type + "_" + c_name
custom_print.define_new_name(c_name)
printc("Starting...")
# Register the machine with foreman before doing anything else
self._register = Registration(self._publish_queue,
self._consume_queue)
# Block until a unique name is received
self._name = self._register.request_name()
except:
return
try:
# Debug print setup
custom_print.define_new_name(self._name)
# Internal variable of the current state
self._current_state = IDLE
# Messaging setup
self._broker_url = "amqp://" + AMQP_MACH_USER + ":" + AMQP_MACH_PSWD + "@" + AMQP_BROKER_ADDR + ":" + AMQP_BROKER_PORT + "/" + AMQP_BROKER_VHOST
self._consume_queue = self._name + "_consume"
self._home_dir = XFER_DIR
# Messages foreman can send to us
self._msg_actions = {
JOB: self.process_foreman_job,
STANDBY: self.process_foreman_standby,
READOUT: self.process_foreman_readout,
CANCEL: self.process_foreman_cancel
}
# Publisher to send messages to foreman
printc("Setting up publisher...")
self._publisher = SimplePublisher(self._broker_url)
# Consumer for getting messages from foreman
printc("Setting up consumer...")
self._consumer = Consumer(self._broker_url, self._consume_queue)
except:
pass
# Run blocking consumer
try:
self.run_consumer()
except:
pass
# Alert foreman this machine is shutting down
self.deregister()
return
def run_consumer(self):
# Consumer blocking function
self._consumer.run(self.on_message)
return
def child_init(self):
# Function that child can call override to add variables during init
_type = 'UNSET'
return
def deregister(self):
# Send message to foreman to deregister machine
msg = {}
msg[MSG_TYPE] = 'DEREGISTER'
msg[NAME] = self._name
self._publisher.publish_message(self._publish_queue, yaml.dump(msg))
return
def state_update(self, key, field, value):
# Send message to foreman about a state update
msg = {}
msg[MSG_TYPE] = 'STATE_UPDATE'
msg['KEY'] = key
msg['FIELD'] = field
msg['VALUE'] = value
self._publisher.publish_message(self._publish_queue, yaml.dump(msg))
return
# Foreman messaging
def on_message(self, ch, method, properties, body):
# Consumer callback function
printc("Processing message...")
msg_dict = yaml.load(body)
try:
af_handler = self._msg_actions.get(msg_dict[MSG_TYPE])
except KeyError:
printc("Invalid message received, cannot process.")
ch.basic_ack(delivery_tag=method.delivery_tag)
return
af_handler(msg_dict)
ch.basic_ack(delivery_tag=method.delivery_tag)
# If there was an ACK_ID in this message, they want a response
# Initial idea is ACK to be sent after the callback has been called
if 'ACK_ID' in msg_dict:
self.send_ack(msg_dict['ACK_ID'], msg_dict['ACK_TYPE'])
return
def send_ack(self, session_id, type = None):
if session_id is None:
return
if type is None:
type = 'UNDEFINED'
ack_msg = {}
ack_msg['MSG_TYPE'] = 'ACK_RECEIVED'
ack_msg['ACK_ID'] = str(session_id)
ack_msg['ACK_NAME'] = str(self._name)
ack_msg['ACK_TYPE'] = str(type)
ack_msg['ACK_BOOL'] = 'TRUE'
self._publisher.publish_message(Q_ACK_PUBLISH, yaml.dump(ack_msg))
return
def process_foreman_job(self, msg_params):
return
def process_foreman_standby(self, msg_params):
return
def process_foreman_readout(self, msg_params):
return
def process_foreman_cancel(self, msg_params):
return
def setup_publishers(self):
LOGGER.info('Setting up publisher for Distributor on %s',
self._ncsa_broker_url)
self._publisher = SimplePublisher(self._ncsa_broker_url)
def test_ncsa(self, Ncsa):
self.ncsa = Ncsa
try:
cdm = toolsmod.intake_yaml_file('./tests/yaml/L1SystemCfg_Test_ncsa.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
pp_name = cdm[ROOT]['PFM_BROKER_NAME']
pp_passwd = cdm[ROOT]['PFM_BROKER_PASSWD']
pp_pub_name = cdm[ROOT]['PFM_BROKER_PUB_NAME']
pp_pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD']
pp_broker_url = "amqp://" + pp_name + ":" + \
pp_passwd + "@" + \
broker_addr
pp_pub_broker_url = "amqp://" + pp_pub_name + ":" + \
pp_pub_passwd + "@" + \
broker_addr
self.pp_publisher = SimplePublisher(pp_pub_broker_url, "YAML")
D1_name = 'D1'
D1_passwd = 'D1'
D1_pub_name = 'D1_PUB'
D1_pub_passwd = 'D1_PUB'
D1_broker_url = "amqp://" + D1_name + ":" + \
D1_passwd + "@" + \
broker_addr
D1_pub_broker_url = "amqp://" + D1_pub_name + ":" + \
D1_pub_passwd + "@" + \
broker_addr
self.d1_publisher = SimplePublisher(D1_pub_broker_url, "YAML")
D2_name = 'D2'
D2_passwd = 'D2'
D2_pub_name = 'D2_PUB'
D2_pub_passwd = 'D2_PUB'
D2_broker_url = "amqp://" + D2_name + ":" + \
D2_passwd + "@" + \
broker_addr
D2_pub_broker_url = "amqp://" + D2_pub_name + ":" + \
D2_pub_passwd + "@" + \
broker_addr
self.d2_publisher = SimplePublisher(D2_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.pp_consumer = Consumer(pp_broker_url,'pp_foreman_ack_publish', 'thread-pp',
self.on_pp_message,'YAML')
self.pp_consumer.start()
self.D1_consumer = Consumer(D1_broker_url,'d1_consume', 'thread-d1',
self.on_d1_message,'YAML')
self.D1_consumer.start()
self.D2_consumer = Consumer(D2_broker_url,'d2_consume', 'thread-d2',
self.on_d2_message,'YAML')
self.D2_consumer.start()
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(7)
self.verify_pp_messages()
self.verify_D2_messages()
self.verify_D1_messages()
sleep(2)
self.pp_consumer.stop()
self.pp_consumer.join()
self.D1_consumer.stop()
self.D1_consumer.join()
self.D2_consumer.stop()
self.D2_consumer.join()
if self.DP:
print("Finished with NCSA tests.")
class TestDMCS_AR_PP:
ocs_pub_broker_url = None
ocs_publisher = None
ocs_consumer = None
ocs_consumer_msg_list = []
ar_pub_broker_url = None
ar_publisher = None
ar_consumer = None
ar_consumer_msg_list = []
pp_pub_broker_url = None
pp_publisher = None
pp_consumer = None
pp_consumer_msg_list = []
EXPECTED_AR_MESSAGES = 1
EXPECTED_PP_MESSAGES = 1
EXPECTED_OCS_MESSAGES = 1
ccd_list = [14,17,21.86]
prp = toolsmod.prp
DP = toolsmod.DP # Debug Print
def test_dmcs(self, Dmcs):
self.dmcs = Dmcs
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
ocs_name = cdm[ROOT]['OCS_BROKER_NAME']
ocs_passwd = cdm[ROOT]['OCS_BROKER_PASSWD']
ocs_pub_name = cdm[ROOT]['OCS_BROKER_PUB_NAME']
ocs_pub_passwd = cdm[ROOT]['OCS_BROKER_PUB_PASSWD']
ocs_broker_url = "amqp://" + ocs_name + ":" + \
ocs_passwd + "@" + \
broker_addr
self. ocs_pub_broker_url = "amqp://" + ocs_pub_name + ":" + \
ocs_pub_passwd + "@" + \
broker_addr
self.ocs_publisher = SimplePublisher(self.ocs_pub_broker_url, "YAML")
ar_name = cdm[ROOT]['AFM_BROKER_NAME']
ar_passwd = cdm[ROOT]['AFM_BROKER_PASSWD']
ar_pub_name = cdm[ROOT]['AFM_BROKER_PUB_NAME']
ar_pub_passwd = cdm[ROOT]['AFM_BROKER_PUB_PASSWD']
ar_broker_url = "amqp://" + ar_name + ":" + \
ar_passwd + "@" + \
broker_addr
self.ar_pub_broker_url = "amqp://" + ar_pub_name + ":" + \
ar_pub_passwd + "@" + \
broker_addr
self.ar_publisher = SimplePublisher(self.ar_pub_broker_url, "YAML")
pp_name = cdm[ROOT]['PFM_BROKER_NAME']
pp_passwd = cdm[ROOT]['PFM_BROKER_PASSWD']
pp_pub_name = cdm[ROOT]['PFM_BROKER_PUB_NAME']
pp_pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD']
pp_broker_url = "amqp://" + pp_name + ":" + \
pp_passwd + "@" + \
broker_addr
self.pp_pub_broker_url = "amqp://" + pp_pub_name + ":" + \
pp_pub_passwd + "@" + \
broker_addr
self.pp_publisher = SimplePublisher(self.pp_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.ocs_consumer = Consumer(ocs_broker_url,'dmcs_ocs_publish', 'thread-ocs',
self.on_ocs_message,'YAML')
self.ocs_consumer.start()
self.ar_consumer = Consumer(ar_broker_url,'ar_foreman_consume', 'thread-ar',
self.on_ar_message,'YAML')
self.ar_consumer.start()
self.pp_consumer = Consumer(pp_broker_url,'pp_foreman_consume', 'thread-pp',
self.on_pp_message,'YAML')
self.pp_consumer.start()
### call message sender and pass in ocs_publisher
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(3)
self.verify_ocs_messages()
sleep(3)
self.verify_ar_messages()
sleep(3)
self.verify_pp_messages()
sleep(2)
self.ocs_consumer.stop()
self.ocs_consumer.join()
self.ar_consumer.stop()
self.ar_consumer.join()
self.pp_consumer.stop()
self.pp_consumer.join()
print("Finished with DMCS AR and PP tests.")
def send_messages(self):
print("Starting send_messages")
# Tests both AR and PP devices
self.EXPECTED_OCS_MESSAGES = 12
self.EXPECTED_AR_MESSAGES = 7
self.EXPECTED_PP_MESSAGES = 7
self.clear_message_lists()
msg = {}
msg['MSG_TYPE'] = "STANDBY"
msg['DEVICE'] = 'AR'
msg['CMD_ID'] = '16729948'
msg['CFG_KEY'] = "2C16"
msg['ACK_ID'] = 'AR_4'
msg['ACK_DELAY'] = 2
time.sleep(1)
print("AR STANDBY")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "STANDBY"
msg['DEVICE'] = 'PP'
msg['CMD_ID'] = '16729948'
msg['CFG_KEY'] = "2C16"
msg['ACK_ID'] = 'PP_7'
msg['ACK_DELAY'] = 2
time.sleep(5)
print("PP STANDBY")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
#msg = {}
#msg['MSG_TYPE'] = "NEW_SESSION"
#msg['SESSION_ID'] = 'SI_469976'
#msg['ACK_ID'] = 'NEW_SESSION_ACK_44221'
#msg['RESPONSE_QUEUE'] = "dmcs_ack_consume"
##time.sleep(1)
##self.ocs_publisher.publish_message("ar_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "DISABLE"
msg['DEVICE'] = 'AR'
msg['CMD_ID'] = '16729948'
msg['ACK_ID'] = 'AR_6'
msg['ACK_DELAY'] = 2
time.sleep(5)
print("AR DISABLE")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "DISABLE"
msg['DEVICE'] = 'PP'
msg['CMD_ID'] = '16729948'
msg['ACK_ID'] = 'PP_8'
msg['ACK_DELAY'] = 2
time.sleep(5)
print("PP DISABLE")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
# Make certain DMCS is doing proper bookkeeping
sleep(6.5)
assert self.dmcs.STATE_SCBD.get_archive_state() == 'DISABLE'
assert self.dmcs.STATE_SCBD.get_prompt_process_state() == 'DISABLE'
msg = {}
msg['MSG_TYPE'] = "ENABLE"
msg['DEVICE'] = 'AR'
msg['CMD_ID'] = '16729948'
msg['ACK_ID'] = 'AR_11'
msg['ACK_DELAY'] = 2
time.sleep(5)
print("AR ENABLE")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "ENABLE"
msg['DEVICE'] = 'PP'
msg['CMD_ID'] = '16729948'
msg['ACK_ID'] = 'PP_12'
msg['ACK_DELAY'] = 2
time.sleep(5)
print("PP ENABLE")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
sleep(6.5)
assert self.dmcs.STATE_SCBD.get_archive_state() == 'ENABLE'
assert self.dmcs.STATE_SCBD.get_prompt_process_state() == 'ENABLE'
msg = {}
msg['MSG_TYPE'] = "NEXT_VISIT"
msg['VISIT_ID'] = 'V_1443'
msg['RESPONSE_QUEUE'] = "dmcs_ack_consume"
msg['ACK_ID'] = 'NEW_VISIT_ACK_76'
msg['BORE_SIGHT'] = "231,123786456342, -45.3457156906, FK5"
time.sleep(5)
print("Next Visit Message")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
sleep(3.5)
assert self.dmcs.STATE_SCBD.get_current_visit() == 'V_1443'
msg = {}
msg['MSG_TYPE'] = "START_INTEGRATION"
msg['IMAGE_ID'] = 'IMG_4276'
msg['VISIT_ID'] = 'V_1443'
msg['ACK_ID'] = 'START_INT_ACK_76'
msg['RESPONSE_QUEUE'] = "dmcs_ack_consume"
msg['CCD_LIST'] = self.ccd_list
time.sleep(5)
print("Start Integration Message")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "READOUT"
msg['VISIT_ID'] = 'V_1443'
msg['IMAGE_ID'] = 'IMG_4276'
msg['IMAGE_SRC'] = 'MAIN'
msg['RESPONSE_QUEUE'] = "dmcs_ack_consume"
msg['ACK_ID'] = 'READOUT_ACK_77'
time.sleep(5)
print("READOUT Message")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "START_INTEGRATION"
msg['IMAGE_ID'] = 'IMG_4277'
msg['IMAGE_SRC'] = 'MAIN'
msg['VISIT_ID'] = 'V_1443'
msg['ACK_ID'] = 'START_INT_ACK_78'
msg['RESPONSE_QUEUE'] = "dmcs_ack_consume"
msg['CCD_LIST'] = self.ccd_list
time.sleep(5)
print("Start Integration Message")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "READOUT"
msg['VISIT_ID'] = 'V_1443'
msg['IMAGE_ID'] = 'IMG_4277'
msg['IMAGE_SRC'] = 'MAIN'
msg['RESPONSE_QUEUE'] = "dmcs_ack_consume"
msg['ACK_ID'] = 'READOUT_ACK_79'
time.sleep(5)
print("READOUT Message")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
time.sleep(2)
print("Message Sender done")
def clear_message_lists(self):
self.ocs_consumer_msg_list = []
self.ar_consumer_msg_list = []
self.pp_consumer_msg_list = []
def verify_ocs_messages(self):
print("Messages received by verify_ocs_messages:")
self.prp.pprint(self.ocs_consumer_msg_list)
len_list = len(self.ocs_consumer_msg_list)
if len_list != self.EXPECTED_OCS_MESSAGES:
print("Messages received by verify_ocs_messages:")
self.prp.pprint(self.ocs_consumer_msg_list)
pytest.fail('OCS simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_OCS_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.ocs_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following OCS Bridge response message failed when compared with the sovereign example: %s" % msg)
print("Responses to OCS Bridge pass verification.")
def verify_ar_messages(self):
print("Messages received by verify_ar_messages:")
self.prp.pprint(self.ar_consumer_msg_list)
len_list = len(self.ar_consumer_msg_list)
if len_list != self.EXPECTED_AR_MESSAGES:
print("Messages received by verify_ar_messages:")
self.prp.pprint(self.ar_consumer_msg_list)
pytest.fail('AR simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_AR_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.ar_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following message to the AR failed when compared with the sovereign example: %s" % msg)
print("Messages to the AR pass verification.")
def verify_pp_messages(self):
print("Messages received by verify_pp_messages:")
self.prp.pprint(self.pp_consumer_msg_list)
len_list = len(self.pp_consumer_msg_list)
if len_list != self.EXPECTED_PP_MESSAGES:
print("Messages received by verify_pp_messages:")
self.prp.pprint(self.pp_consumer_list)
pytest.fail('PP simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_PP_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.pp_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following message to the PP device failed when compared with the sovereign example: %s" % msg)
print("Messages to the PP device pass verification.")
def on_ocs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.ocs_consumer_msg_list.append(body)
def on_ar_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.ar_consumer_msg_list.append(body)
def on_pp_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.pp_consumer_msg_list.append(body)
class TestAR_Ctrl:
arctrl = ArchiveController(
"/home/centos/src/git/ctrl_iip/python/lsst/iip/tests/yaml/L1SystemCfg_Test_ar-ctrl.yaml"
)
ar_consumer_msg_list = []
EXPECTED_AR_DEV_MESSAGES = 2
VISIT_ID = "V_1213"
IMAGE_ID = "IMG_13241"
SEND_TO_AR = "archive_ctrl_consume"
ACK_FROM_CTRL = "ar_foreman_ack_publish"
def test_ar_ctrl(self):
try:
cdm = toolsmod.intake_yaml_file(
'/home/centos/src/git/ctrl_iip/python/lsst/iip/tests/yaml/L1SystemCfg_Test_ar-ctrl.yaml'
)
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
afm_name = cdm[ROOT]["AFM_BROKER_NAME"]
afm_pwd = cdm[ROOT]["AFM_BROKER_PASSWD"]
afm_pub_name = cdm[ROOT]["AFM_BROKER_PUB_NAME"]
afm_pub_pwd = cdm[ROOT]["AFM_BROKER_PUB_PASSWD"]
self.xfer_root = cdm[ROOT]["ARCHIVE"]["ARCHIVE_XFER_ROOT"]
ar_dev_pub_broker_url = "amqp://" + afm_pub_name + ":" +\
afm_pub_pwd + "@" +\
broker_addr
self.ar_dev_publisher = SimplePublisher(ar_dev_pub_broker_url, "YAML")
ar_dev_broker_url = "amqp://" + afm_name + ":" +\
afm_pwd + "@" +\
broker_addr
self.ar_dev_consumer = Consumer(ar_dev_broker_url, self.ACK_FROM_CTRL,
"thread-ar-dev", self.on_ar_message,
"YAML", None)
self.ar_dev_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority(
'/home/centos/src/git/ctrl_iip/python/lsst/iip/messages.yaml')
self.send_messages()
sleep(10)
self.verify_ar_messages()
self.verify_path_files()
print("Finished with AR CTRL tests.")
def send_messages(self):
# msg = {}
# msg["MSG_TYPE"] = "ARCHIVE_HEALTH_CHECK"
# msg["SESSION_ID"] = "SI_314161"
# msg["ACK_ID"] = "AR_ACK_10001"
# self.ar_dev_publisher.publish_message(self.SEND_TO_AR, msg)
# print("ARCHIVE_HEALTH_CHECK Message")
# sleep(2)
msg = {}
msg["MSG_TYPE"] = "NEW_ARCHIVE_ITEM"
msg["ACK_ID"] = "NEW_AR_10002"
msg["JOB_NUM"] = "job_101"
msg["SESSION_ID"] = "SI_314162"
msg["VISIT_ID"] = self.VISIT_ID
msg["IMAGE_ID"] = self.IMAGE_ID
msg["REPLY_QUEUE"] = "ar_foreman_ack_publish"
self.ar_dev_publisher.publish_message(self.SEND_TO_AR, msg)
print("NEW_ARCHIVE_ITEM Message")
sleep(2)
msg = {}
msg["MSG_TYPE"] = "AR_ITEMS_XFERD"
msg["ACK_ID"] = "AR_ACK_10003"
msg["IMAGE_ID"] = "IMG_103"
msg["REPLY_QUEUE"] = "ar_foreman_ack_publish"
msg["RESULT_LIST"] = {}
msg["RESULT_LIST"]["CCD_LIST"] = ["101", "102"]
msg["RESULT_LIST"]["FILENAME_LIST"] = ["img1_ccd101", "img1_ccd102"]
msg["RESULT_LIST"]["CHECKSUM_LIST"] = [
"F8DC17E983616D492A41344D736AD5D9",
"99158C7DFD8351C9DEBC124029421993"
]
self.ar_dev_publisher.publish_message(self.SEND_TO_AR, msg)
print("AR_ITEMS_XFERD Message")
sleep(2)
def clear_message_lists(self):
self.ar_consumer_msg_list = []
def verify_ar_messages(self):
len_list = len(self.ar_consumer_msg_list)
if len_list != self.EXPECTED_AR_DEV_MESSAGES:
pytest.fail(
"ArchiveController simulator received incorrect number of messages.\n"
+ "Expected %s but received %s" %
(self.EXPECTED_AR_DEV_MESSAGES, len_list))
for i in range(0, len_list):
msg = self.ar_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if not result:
pytest.fail(
"The following message to Archive Device failed when\
compared with the sovereign example: %s" % msg)
print("Message to ArchiveDevice pass verification")
def verify_path_files(self):
path = self.xfer_root + self.VISIT_ID + "/" + self.IMAGE_ID
if not os.path.isdir(path):
pytest.fail(
"The following directory was not created by Archive Controller \
during the NEW_ARCHIVE_ITEM phase: %s" % path)
print("Path creation during NEW_ARCHIVE_ITEM phase pass verification")
def on_ar_message(self, ch, method, properties, body):
# ch.basic_ack(method.delivery_tag)
self.ar_consumer_msg_list.append(body)
class TestPpDev:
dmcs_pub_broker_url = None
dmcs_publisher = None
dmcs_consumer = None
dmcs_consumer_msg_list = []
ncsa_pub_broker_url = None
ncsa_publisher = None
ncsa_ctrl_consumer = None
ncsa_consumer_msg_list = []
F1_pub_broker_url = None
F1_publisher = None
F1_consumer = None
f1_consumer_msg_list = []
F2_pub_broker_url = None
F2_publisher = None
F2_consumer = None
f2_consumer_msg_list = []
EXPECTED_NCSA_MESSAGES = 1
EXPECTED_DMCS_MESSAGES = 1
EXPECTED_F1_MESSAGES = 1
EXPECTED_F2_MESSAGES = 1
ccd_list = [14,17,21,86]
prp = toolsmod.prp # pretty printing
DP = toolsmod.DP # Debug printing either True of False...set here to override this file only
def test_ppdev(self, Ppdev):
self.ppdev = Ppdev
try:
cdm = toolsmod.intake_yaml_file('./tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
dmcs_name = cdm[ROOT]['DMCS_BROKER_NAME']
dmcs_passwd = cdm[ROOT]['DMCS_BROKER_PASSWD']
dmcs_pub_name = cdm[ROOT]['DMCS_BROKER_PUB_NAME']
dmcs_pub_passwd = cdm[ROOT]['DMCS_BROKER_PUB_PASSWD']
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_passwd + "@" + \
broker_addr
dmcs_pub_broker_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_passwd + "@" + \
broker_addr
self.dmcs_publisher = SimplePublisher(dmcs_pub_broker_url, "YAML")
ncsa_name = cdm[ROOT]['NCSA_BROKER_NAME']
ncsa_passwd = cdm[ROOT]['NCSA_BROKER_PASSWD']
ncsa_pub_name = cdm[ROOT]['NCSA_BROKER_PUB_NAME']
ncsa_pub_passwd = cdm[ROOT]['NCSA_BROKER_PUB_PASSWD']
ncsa_broker_url = "amqp://" + ncsa_name + ":" + \
ncsa_passwd + "@" + \
broker_addr
ncsa_pub_broker_url = "amqp://" + ncsa_pub_name + ":" + \
ncsa_pub_passwd + "@" + \
broker_addr
self.ncsa_publisher = SimplePublisher(ncsa_pub_broker_url, "YAML")
F1_name = 'F1'
F1_passwd = 'F1'
F1_pub_name = 'F1_PUB'
F1_pub_passwd = 'F1_PUB'
F1_broker_url = "amqp://" + F1_name + ":" + \
F1_passwd + "@" + \
broker_addr
F1_pub_broker_url = "amqp://" + F1_pub_name + ":" + \
F1_pub_passwd + "@" + \
broker_addr
self.f1_publisher = SimplePublisher(F1_pub_broker_url, "YAML")
F2_name = 'F2'
F2_passwd = 'F2'
F2_pub_name = 'F2_PUB'
F2_pub_passwd = 'F2_PUB'
F2_broker_url = "amqp://" + F2_name + ":" + \
F2_passwd + "@" + \
broker_addr
F2_pub_broker_url = "amqp://" + F2_pub_name + ":" + \
F2_pub_passwd + "@" + \
broker_addr
self.f2_publisher = SimplePublisher(F2_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.dmcs_consumer = Consumer(dmcs_broker_url,'dmcs_ack_consume', 'thread-dmcs',
self.on_dmcs_message,'YAML')
self.dmcs_consumer.start()
self.ncsa_consumer = Consumer(ncsa_broker_url,'ncsa_consume', 'thread-ncsa',
self.on_ncsa_message,'YAML')
self.ncsa_consumer.start()
self.F1_consumer = Consumer(F1_broker_url,'f1_consume', 'thread-f1',
self.on_f1_message,'YAML')
self.F1_consumer.start()
self.F2_consumer = Consumer(F2_broker_url,'f2_consume', 'thread-f2',
self.on_f2_message,'YAML')
self.F2_consumer.start()
sleep(3)
if self.DP:
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(6)
self.verify_ncsa_messages()
self.verify_F2_messages()
self.verify_F1_messages()
self.verify_dmcs_messages()
sleep(2)
self.dmcs_consumer.stop()
self.dmcs_consumer.join()
self.ncsa_consumer.stop()
self.ncsa_consumer.join()
self.F1_consumer.stop()
self.F1_consumer.join()
self.F2_consumer.stop()
self.F2_consumer.join()
if self.DP:
print("Finished with PP tests.")
def send_messages(self):
if self.DP:
print("Starting send_messages")
# Tests only an AR device
# self.clear_message_lists()
self.EXPECTED_NCSA_MESSAGES = 3
self.EXPECTED_DMCS_MESSAGES = 4
self.EXPECTED_F1_MESSAGES = 3
self.EXPECTED_F2_MESSAGES = 3
msg = {}
msg['MSG_TYPE'] = "PP_NEW_SESSION"
msg['SESSION_ID'] = 'SI_469976'
msg['ACK_ID'] = 'NEW_SESSION_ACK_44221'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
time.sleep(3)
if self.DP:
print("New Session Message")
self.dmcs_publisher.publish_message("pp_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "PP_NEXT_VISIT"
msg['VISIT_ID'] = 'XX_28272'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'NEW_VISIT_ACK_76'
msg['BORE_SIGHT'] = "231,123786456342, -45.3457156906, FK5"
time.sleep(2)
if self.DP:
print("Next Visit Message")
self.dmcs_publisher.publish_message("pp_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "PP_START_INTEGRATION"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444244'
msg['VISIT_ID'] = 'V14494'
msg['SESSION_ID'] = '4_14_7211511'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'PP_ACK_94671'
msg['CCD_LIST'] = [4,14,16,17,29,35,36]
time.sleep(4)
if self.DP:
print("PP_START_INTEGRATION Message")
self.dmcs_publisher.publish_message("pp_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "PP_READOUT"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444244'
msg['VISIT_ID'] = 'V14494'
msg['SESSION_ID'] = '4_14_7211511'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'PP_READOUT_ACK_44221'
time.sleep(4)
if self.DP:
print("PP_READOUT Message")
self.dmcs_publisher.publish_message("pp_foreman_consume", msg)
time.sleep(2)
if self.DP:
print("Message Sender done")
def verify_dmcs_messages(self):
if self.DP:
print("Messages received by verify_dmcs_messages:")
self.prp.pprint(self.dmcs_consumer_msg_list)
len_list = len(self.dmcs_consumer_msg_list)
if len_list != self.EXPECTED_DMCS_MESSAGES:
pytest.fail('DMCS simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_DMCS_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.dmcs_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following DMCS Bridge response message failed when compared with the sovereign example: %s" % msg)
print("Responses to DMCS Bridge pass verification.")
def verify_ncsa_messages(self):
if self.DP:
print("Messages received by verify_ncsa_messages:")
self.prp.pprint(self.ncsa_consumer_msg_list)
len_list = len(self.ncsa_consumer_msg_list)
if len_list != self.EXPECTED_NCSA_MESSAGES:
pytest.fail('NCSA simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_NCSA_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.ncsa_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following message to the NCSA Foreman failed when compared with the sovereign example: %s" % msg)
print("Messages to the NCSA Foreman pass verification.")
def verify_F1_messages(self):
if self.DP:
print("Messages received by verify_F1_messages:")
self.prp.pprint(self.f1_consumer_msg_list)
len_list = len(self.f1_consumer_msg_list)
if len_list != self.EXPECTED_F1_MESSAGES:
pytest.fail('F1 simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_F1_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.f1_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following message to F1 failed when compared with the sovereign example: %s" % msg)
else:
print("Messages to F1 pass verification.")
def verify_F2_messages(self):
if self.DP:
print("Messages received by verify_F2_messages:")
self.prp.pprint(self.f2_consumer_msg_list)
len_list = len(self.f2_consumer_msg_list)
if len_list != self.EXPECTED_F2_MESSAGES:
pytest.fail('F2 simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_F2_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.f2_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following message to F2 failed when compared with the sovereign example: %s" % msg)
else:
print("Messages to F2 pass verification.")
def on_dmcs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_pp_dev - incoming on_dmcs_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.dmcs_consumer_msg_list.append(body)
def on_ncsa_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_pp_dev - incoming on_ncsa_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.ncsa_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'NCSA_NEW_SESSION':
msg = {}
msg['MSG_TYPE'] = 'NCSA_NEW_SESSION_ACK'
msg['COMPONENT'] = 'NCSA_FOREMAN'
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
self.ncsa_publisher.publish_message(body['REPLY_QUEUE'], msg)
return
if body['MSG_TYPE'] == 'NCSA_NEXT_VISIT':
msg = {}
msg['MSG_TYPE'] = 'NCSA_NEXT_VISIT_ACK'
msg['COMPONENT'] = 'NCSA_FOREMAN'
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
self.ncsa_publisher.publish_message(body['REPLY_QUEUE'], msg)
return
if body['MSG_TYPE'] == 'NCSA_START_INTEGRATION':
msg = {}
msg['ACK_ID'] = body['ACK_ID']
msg['MSG_TYPE'] = 'NCSA_START_INTEGRATION_ACK'
msg['COMPONENT'] = 'NCSA_FOREMAN'
fwdrs = deepcopy(body['FORWARDERS'])
fwdr_list = fwdrs['FORWARDER_LIST']
ccd_list = fwdrs['CCD_LIST']
i = 1
msg['PAIRS'] = [] # This will be a list of dictionaries
for i in range(0,len(fwdr_list)):
fwdr = fwdr_list[i]
dist = {}
pair = {}
dist['FQN'] = "Distributor_" + str(i)
dist['NAME'] = "D" + str(i)
dist['HOSTNAME'] = "D" + str(i)
dist['TARGET_DIR'] = "/dev/null"
dist['IP_ADDR'] = "141.142.237.16" + str(i)
pair['FORWARDER'] = fwdr_list[i]
pair['CCD_LIST'] = ccd_list[i] #Get the list at index position i in ccd_list
pair['DISTRIBUTOR'] = dist
msg['PAIRS'].append(deepcopy(pair))
msg['ACK_BOOL'] = True
msg['JOB_NUM'] = body['JOB_NUM']
msg['IMAGE_ID'] = body['IMAGE_ID']
msg['VISIT_ID'] = body['VISIT_ID']
msg['SESSION_ID'] = body['SESSION_ID']
self.ncsa_publisher.publish_message(body['REPLY_QUEUE'], msg)
return
if body['MSG_TYPE'] == 'NCSA_READOUT':
# Find earlier Start Int message
st_int_msg = None
for msg in self.ncsa_consumer_msg_list:
if msg['MSG_TYPE'] == 'NCSA_START_INTEGRATION':
st_int_msg = msg
break
if st_int_msg == None:
pytest.fail("The NCSA_START_INTEGRATION message wasn't received before NCSA_READOUT in on_ncsa_msg")
# Now build response with previous message
msg = {}
msg['MSG_TYPE'] = 'NCSA_READOUT_ACK'
msg['JOB_NUM'] = body['JOB_NUM']
msg['IMAGE_ID'] = body['IMAGE_ID']
msg['VISIT_ID'] = body['VISIT_ID']
msg['SESSION_ID'] = body['SESSION_ID']
msg['COMPONENT'] = 'NCSA_FOREMAN'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
#msg['RESULT_LIST']['FORWARDER_LIST'] = st_int_msg['FORWARDERS']['FORWARDER_LIST']
ccd_list = st_int_msg['FORWARDERS']['CCD_LIST']
receipt_list = []
for i in range(0, len(ccd_list)):
receipt_list.append('Rec_x447_' + str(i))
msg['RESULT_LIST'] = {}
msg['RESULT_LIST']['RECEIPT_LIST'] = receipt_list
msg['RESULT_LIST']['CCD_LIST'] = list(ccd_list)
#sleep(2) #Give FWDRs time to respond with ack first
self.ncsa_publisher.publish_message(body['REPLY_QUEUE'], msg)
def on_f1_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_pp_dev - incoming on_f1_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.f1_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'PP_FWDR_HEALTH_CHECK':
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_HEALTH_CHECK_ACK'
msg['COMPONENT'] = 'FORWARDER_1'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.f1_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'PP_FWDR_XFER_PARAMS':
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_XFER_PARAMS_ACK'
msg['COMPONENT'] = 'FORWARDER_1'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.f1_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'PP_FWDR_READOUT':
# Find message in message list for xfer_params
xfer_msg = None
for msg in self.f1_consumer_msg_list:
if msg['MSG_TYPE'] == 'PP_FWDR_XFER_PARAMS':
xfer_msg = msg
break
if xfer_msg == None:
pytest.fail("The PP_FWDR_XFER_PARAMS message was not received before PP_FWDR_READOUT in F1")
# use message to build response
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_READOUT_ACK'
msg['COMPONENT'] = 'FORWARDER_1'
msg['JOB_NUM'] = xfer_msg['JOB_NUM']
msg['IMAGE_ID'] = xfer_msg['IMAGE_ID']
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
msg['RESULT_LIST'] = {}
msg['RESULT_LIST']['CCD_LIST'] = []
msg['RESULT_LIST']['RECEIPT_LIST'] = []
ccd_list = xfer_msg['XFER_PARAMS']['CCD_LIST']
receipt_list = []
for i in range(0, len(ccd_list)):
receipt_list.append('F1_Rec_x477_' + str(i))
msg['RESULT_LIST']['RECEIPT_LIST'] = receipt_list
msg['RESULT_LIST']['CCD_LIST'] = list(ccd_list)
self.f1_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail("The following unknown message was received by FWDR F1: %s" % body)
def on_f2_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_pp_dev - incoming on_f2_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.f2_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'PP_FWDR_HEALTH_CHECK':
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_HEALTH_CHECK_ACK'
msg['COMPONENT'] = 'FORWARDER_2'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.f2_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'PP_FWDR_XFER_PARAMS':
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_XFER_PARAMS_ACK'
msg['COMPONENT'] = 'FORWARDER_2'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.f2_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'PP_FWDR_READOUT':
# Find message in message list for xfer_params
xfer_msg = None
for msg in self.f2_consumer_msg_list:
if msg['MSG_TYPE'] == 'PP_FWDR_XFER_PARAMS':
xfer_msg = msg
break
if xfer_msg == None:
pytest.fail("The PP_FWDR_XFER_PARAMS message was not received before AR_FWDR_READOUT in F2")
# use message to build response
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_READOUT_ACK'
msg['COMPONENT'] = 'FORWARDER_2'
msg['JOB_NUM'] = xfer_msg['JOB_NUM']
msg['IMAGE_ID'] = xfer_msg['IMAGE_ID']
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
msg['RESULT_LIST'] = {}
msg['RESULT_LIST']['CCD_LIST'] = []
msg['RESULT_LIST']['RECEIPT_LIST'] = []
ccd_list = xfer_msg['XFER_PARAMS']['CCD_LIST']
receipt_list = []
for i in range(0, len(ccd_list)):
receipt_list.append('F2_Rec_x447_' + str(i))
msg['RESULT_LIST']['RECEIPT_LIST'] = receipt_list
msg['RESULT_LIST']['CCD_LIST'] = list(ccd_list)
self.f2_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail("The following unknown message was received by FWDR F2: %s" % body)
class TestPpDev:
dmcs_pub_broker_url = None
dmcs_publisher = None
dmcs_consumer = None
dmcs_consumer_msg_list = []
ncsa_pub_broker_url = None
ncsa_publisher = None
ncsa_ctrl_consumer = None
ncsa_consumer_msg_list = []
F1_pub_broker_url = None
F1_publisher = None
F1_consumer = None
f1_consumer_msg_list = []
F2_pub_broker_url = None
F2_publisher = None
F2_consumer = None
f2_consumer_msg_list = []
EXPECTED_NCSA_MESSAGES = 1
EXPECTED_DMCS_MESSAGES = 1
EXPECTED_F1_MESSAGES = 1
EXPECTED_F2_MESSAGES = 1
ccd_list = [14, 17, 21, 86]
prp = toolsmod.prp # pretty printing
DP = toolsmod.DP # Debug printing either True of False...set here to override this file only
def test_ppdev(self, Ppdev):
self.ppdev = Ppdev
try:
cdm = toolsmod.intake_yaml_file(
'./tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
dmcs_name = cdm[ROOT]['DMCS_BROKER_NAME']
dmcs_passwd = cdm[ROOT]['DMCS_BROKER_PASSWD']
dmcs_pub_name = cdm[ROOT]['DMCS_BROKER_PUB_NAME']
dmcs_pub_passwd = cdm[ROOT]['DMCS_BROKER_PUB_PASSWD']
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_passwd + "@" + \
broker_addr
dmcs_pub_broker_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_passwd + "@" + \
broker_addr
self.dmcs_publisher = SimplePublisher(dmcs_pub_broker_url, "YAML")
ncsa_name = cdm[ROOT]['NCSA_BROKER_NAME']
ncsa_passwd = cdm[ROOT]['NCSA_BROKER_PASSWD']
ncsa_pub_name = cdm[ROOT]['NCSA_BROKER_PUB_NAME']
ncsa_pub_passwd = cdm[ROOT]['NCSA_BROKER_PUB_PASSWD']
ncsa_broker_url = "amqp://" + ncsa_name + ":" + \
ncsa_passwd + "@" + \
broker_addr
ncsa_pub_broker_url = "amqp://" + ncsa_pub_name + ":" + \
ncsa_pub_passwd + "@" + \
broker_addr
self.ncsa_publisher = SimplePublisher(ncsa_pub_broker_url, "YAML")
F1_name = 'F1'
F1_passwd = 'F1'
F1_pub_name = 'F1_PUB'
F1_pub_passwd = 'F1_PUB'
F1_broker_url = "amqp://" + F1_name + ":" + \
F1_passwd + "@" + \
broker_addr
F1_pub_broker_url = "amqp://" + F1_pub_name + ":" + \
F1_pub_passwd + "@" + \
broker_addr
self.f1_publisher = SimplePublisher(F1_pub_broker_url, "YAML")
F2_name = 'F2'
F2_passwd = 'F2'
F2_pub_name = 'F2_PUB'
F2_pub_passwd = 'F2_PUB'
F2_broker_url = "amqp://" + F2_name + ":" + \
F2_passwd + "@" + \
broker_addr
F2_pub_broker_url = "amqp://" + F2_pub_name + ":" + \
F2_pub_passwd + "@" + \
broker_addr
self.f2_publisher = SimplePublisher(F2_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.dmcs_consumer = Consumer(dmcs_broker_url, 'dmcs_ack_consume',
'thread-dmcs', self.on_dmcs_message,
'YAML')
self.dmcs_consumer.start()
self.ncsa_consumer = Consumer(ncsa_broker_url, 'ncsa_consume',
'thread-ncsa', self.on_ncsa_message,
'YAML')
self.ncsa_consumer.start()
self.F1_consumer = Consumer(F1_broker_url, 'f1_consume', 'thread-f1',
self.on_f1_message, 'YAML')
self.F1_consumer.start()
self.F2_consumer = Consumer(F2_broker_url, 'f2_consume', 'thread-f2',
self.on_f2_message, 'YAML')
self.F2_consumer.start()
sleep(3)
if self.DP:
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(6)
self.verify_ncsa_messages()
self.verify_F2_messages()
self.verify_F1_messages()
self.verify_dmcs_messages()
sleep(2)
self.dmcs_consumer.stop()
self.dmcs_consumer.join()
self.ncsa_consumer.stop()
self.ncsa_consumer.join()
self.F1_consumer.stop()
self.F1_consumer.join()
self.F2_consumer.stop()
self.F2_consumer.join()
if self.DP:
print("Finished with PP tests.")
def send_messages(self):
if self.DP:
print("Starting send_messages")
# Tests only an AR device
# self.clear_message_lists()
self.EXPECTED_NCSA_MESSAGES = 3
self.EXPECTED_DMCS_MESSAGES = 4
self.EXPECTED_F1_MESSAGES = 3
self.EXPECTED_F2_MESSAGES = 3
msg = {}
msg['MSG_TYPE'] = "PP_NEW_SESSION"
msg['SESSION_ID'] = 'SI_469976'
msg['ACK_ID'] = 'NEW_SESSION_ACK_44221'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
time.sleep(3)
if self.DP:
print("New Session Message")
self.dmcs_publisher.publish_message("pp_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "PP_NEXT_VISIT"
msg['VISIT_ID'] = 'XX_28272'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'NEW_VISIT_ACK_76'
msg['BORE_SIGHT'] = "231,123786456342, -45.3457156906, FK5"
time.sleep(2)
if self.DP:
print("Next Visit Message")
self.dmcs_publisher.publish_message("pp_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "PP_START_INTEGRATION"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444244'
msg['VISIT_ID'] = 'V14494'
msg['SESSION_ID'] = '4_14_7211511'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'PP_ACK_94671'
msg['CCD_LIST'] = [4, 14, 16, 17, 29, 35, 36]
time.sleep(4)
if self.DP:
print("PP_START_INTEGRATION Message")
self.dmcs_publisher.publish_message("pp_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "PP_READOUT"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444244'
msg['VISIT_ID'] = 'V14494'
msg['SESSION_ID'] = '4_14_7211511'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'PP_READOUT_ACK_44221'
time.sleep(4)
if self.DP:
print("PP_READOUT Message")
self.dmcs_publisher.publish_message("pp_foreman_consume", msg)
time.sleep(2)
if self.DP:
print("Message Sender done")
def verify_dmcs_messages(self):
if self.DP:
print("Messages received by verify_dmcs_messages:")
self.prp.pprint(self.dmcs_consumer_msg_list)
len_list = len(self.dmcs_consumer_msg_list)
if len_list != self.EXPECTED_DMCS_MESSAGES:
pytest.fail('DMCS simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_DMCS_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.dmcs_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail(
"The following DMCS Bridge response message failed when compared with the sovereign example: %s"
% msg)
print("Responses to DMCS Bridge pass verification.")
def verify_ncsa_messages(self):
if self.DP:
print("Messages received by verify_ncsa_messages:")
self.prp.pprint(self.ncsa_consumer_msg_list)
len_list = len(self.ncsa_consumer_msg_list)
if len_list != self.EXPECTED_NCSA_MESSAGES:
pytest.fail('NCSA simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_NCSA_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.ncsa_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail(
"The following message to the NCSA Foreman failed when compared with the sovereign example: %s"
% msg)
print("Messages to the NCSA Foreman pass verification.")
def verify_F1_messages(self):
if self.DP:
print("Messages received by verify_F1_messages:")
self.prp.pprint(self.f1_consumer_msg_list)
len_list = len(self.f1_consumer_msg_list)
if len_list != self.EXPECTED_F1_MESSAGES:
pytest.fail('F1 simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_F1_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.f1_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail(
"The following message to F1 failed when compared with the sovereign example: %s"
% msg)
else:
print("Messages to F1 pass verification.")
def verify_F2_messages(self):
if self.DP:
print("Messages received by verify_F2_messages:")
self.prp.pprint(self.f2_consumer_msg_list)
len_list = len(self.f2_consumer_msg_list)
if len_list != self.EXPECTED_F2_MESSAGES:
pytest.fail('F2 simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_F2_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.f2_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail(
"The following message to F2 failed when compared with the sovereign example: %s"
% msg)
else:
print("Messages to F2 pass verification.")
def on_dmcs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_pp_dev - incoming on_dmcs_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.dmcs_consumer_msg_list.append(body)
def on_ncsa_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_pp_dev - incoming on_ncsa_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.ncsa_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'NCSA_NEW_SESSION':
msg = {}
msg['MSG_TYPE'] = 'NCSA_NEW_SESSION_ACK'
msg['COMPONENT'] = 'NCSA_FOREMAN'
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
self.ncsa_publisher.publish_message(body['REPLY_QUEUE'], msg)
return
if body['MSG_TYPE'] == 'NCSA_NEXT_VISIT':
msg = {}
msg['MSG_TYPE'] = 'NCSA_NEXT_VISIT_ACK'
msg['COMPONENT'] = 'NCSA_FOREMAN'
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
self.ncsa_publisher.publish_message(body['REPLY_QUEUE'], msg)
return
if body['MSG_TYPE'] == 'NCSA_START_INTEGRATION':
msg = {}
msg['ACK_ID'] = body['ACK_ID']
msg['MSG_TYPE'] = 'NCSA_START_INTEGRATION_ACK'
msg['COMPONENT'] = 'NCSA_FOREMAN'
fwdrs = deepcopy(body['FORWARDERS'])
fwdr_list = fwdrs['FORWARDER_LIST']
ccd_list = fwdrs['CCD_LIST']
i = 1
msg['PAIRS'] = [] # This will be a list of dictionaries
for i in range(0, len(fwdr_list)):
fwdr = fwdr_list[i]
dist = {}
pair = {}
dist['FQN'] = "Distributor_" + str(i)
dist['NAME'] = "D" + str(i)
dist['HOSTNAME'] = "D" + str(i)
dist['TARGET_DIR'] = "/dev/null"
dist['IP_ADDR'] = "141.142.237.16" + str(i)
pair['FORWARDER'] = fwdr_list[i]
pair['CCD_LIST'] = ccd_list[
i] #Get the list at index position i in ccd_list
pair['DISTRIBUTOR'] = dist
msg['PAIRS'].append(deepcopy(pair))
msg['ACK_BOOL'] = True
msg['JOB_NUM'] = body['JOB_NUM']
msg['IMAGE_ID'] = body['IMAGE_ID']
msg['VISIT_ID'] = body['VISIT_ID']
msg['SESSION_ID'] = body['SESSION_ID']
self.ncsa_publisher.publish_message(body['REPLY_QUEUE'], msg)
return
if body['MSG_TYPE'] == 'NCSA_READOUT':
# Find earlier Start Int message
st_int_msg = None
for msg in self.ncsa_consumer_msg_list:
if msg['MSG_TYPE'] == 'NCSA_START_INTEGRATION':
st_int_msg = msg
break
if st_int_msg == None:
pytest.fail(
"The NCSA_START_INTEGRATION message wasn't received before NCSA_READOUT in on_ncsa_msg"
)
# Now build response with previous message
msg = {}
msg['MSG_TYPE'] = 'NCSA_READOUT_ACK'
msg['JOB_NUM'] = body['JOB_NUM']
msg['IMAGE_ID'] = body['IMAGE_ID']
msg['VISIT_ID'] = body['VISIT_ID']
msg['SESSION_ID'] = body['SESSION_ID']
msg['COMPONENT'] = 'NCSA_FOREMAN'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
#msg['RESULT_LIST']['FORWARDER_LIST'] = st_int_msg['FORWARDERS']['FORWARDER_LIST']
ccd_list = st_int_msg['FORWARDERS']['CCD_LIST']
receipt_list = []
for i in range(0, len(ccd_list)):
receipt_list.append('Rec_x447_' + str(i))
msg['RESULT_LIST'] = {}
msg['RESULT_LIST']['RECEIPT_LIST'] = receipt_list
msg['RESULT_LIST']['CCD_LIST'] = list(ccd_list)
#sleep(2) #Give FWDRs time to respond with ack first
self.ncsa_publisher.publish_message(body['REPLY_QUEUE'], msg)
def on_f1_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_pp_dev - incoming on_f1_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.f1_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'PP_FWDR_HEALTH_CHECK':
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_HEALTH_CHECK_ACK'
msg['COMPONENT'] = 'FORWARDER_1'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.f1_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'PP_FWDR_XFER_PARAMS':
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_XFER_PARAMS_ACK'
msg['COMPONENT'] = 'FORWARDER_1'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.f1_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'PP_FWDR_READOUT':
# Find message in message list for xfer_params
xfer_msg = None
for msg in self.f1_consumer_msg_list:
if msg['MSG_TYPE'] == 'PP_FWDR_XFER_PARAMS':
xfer_msg = msg
break
if xfer_msg == None:
pytest.fail(
"The PP_FWDR_XFER_PARAMS message was not received before PP_FWDR_READOUT in F1"
)
# use message to build response
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_READOUT_ACK'
msg['COMPONENT'] = 'FORWARDER_1'
msg['JOB_NUM'] = xfer_msg['JOB_NUM']
msg['IMAGE_ID'] = xfer_msg['IMAGE_ID']
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
msg['RESULT_LIST'] = {}
msg['RESULT_LIST']['CCD_LIST'] = []
msg['RESULT_LIST']['RECEIPT_LIST'] = []
ccd_list = xfer_msg['XFER_PARAMS']['CCD_LIST']
receipt_list = []
for i in range(0, len(ccd_list)):
receipt_list.append('F1_Rec_x477_' + str(i))
msg['RESULT_LIST']['RECEIPT_LIST'] = receipt_list
msg['RESULT_LIST']['CCD_LIST'] = list(ccd_list)
self.f1_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail(
"The following unknown message was received by FWDR F1: %s" %
body)
def on_f2_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_pp_dev - incoming on_f2_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.f2_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'PP_FWDR_HEALTH_CHECK':
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_HEALTH_CHECK_ACK'
msg['COMPONENT'] = 'FORWARDER_2'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.f2_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'PP_FWDR_XFER_PARAMS':
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_XFER_PARAMS_ACK'
msg['COMPONENT'] = 'FORWARDER_2'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.f2_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'PP_FWDR_READOUT':
# Find message in message list for xfer_params
xfer_msg = None
for msg in self.f2_consumer_msg_list:
if msg['MSG_TYPE'] == 'PP_FWDR_XFER_PARAMS':
xfer_msg = msg
break
if xfer_msg == None:
pytest.fail(
"The PP_FWDR_XFER_PARAMS message was not received before AR_FWDR_READOUT in F2"
)
# use message to build response
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_READOUT_ACK'
msg['COMPONENT'] = 'FORWARDER_2'
msg['JOB_NUM'] = xfer_msg['JOB_NUM']
msg['IMAGE_ID'] = xfer_msg['IMAGE_ID']
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
msg['RESULT_LIST'] = {}
msg['RESULT_LIST']['CCD_LIST'] = []
msg['RESULT_LIST']['RECEIPT_LIST'] = []
ccd_list = xfer_msg['XFER_PARAMS']['CCD_LIST']
receipt_list = []
for i in range(0, len(ccd_list)):
receipt_list.append('F2_Rec_x447_' + str(i))
msg['RESULT_LIST']['RECEIPT_LIST'] = receipt_list
msg['RESULT_LIST']['CCD_LIST'] = list(ccd_list)
self.f2_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail(
"The following unknown message was received by FWDR F2: %s" %
body)
def test_ppdev(self, Ppdev):
self.ppdev = Ppdev
try:
cdm = toolsmod.intake_yaml_file(
'./tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
dmcs_name = cdm[ROOT]['DMCS_BROKER_NAME']
dmcs_passwd = cdm[ROOT]['DMCS_BROKER_PASSWD']
dmcs_pub_name = cdm[ROOT]['DMCS_BROKER_PUB_NAME']
dmcs_pub_passwd = cdm[ROOT]['DMCS_BROKER_PUB_PASSWD']
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_passwd + "@" + \
broker_addr
dmcs_pub_broker_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_passwd + "@" + \
broker_addr
self.dmcs_publisher = SimplePublisher(dmcs_pub_broker_url, "YAML")
ncsa_name = cdm[ROOT]['NCSA_BROKER_NAME']
ncsa_passwd = cdm[ROOT]['NCSA_BROKER_PASSWD']
ncsa_pub_name = cdm[ROOT]['NCSA_BROKER_PUB_NAME']
ncsa_pub_passwd = cdm[ROOT]['NCSA_BROKER_PUB_PASSWD']
ncsa_broker_url = "amqp://" + ncsa_name + ":" + \
ncsa_passwd + "@" + \
broker_addr
ncsa_pub_broker_url = "amqp://" + ncsa_pub_name + ":" + \
ncsa_pub_passwd + "@" + \
broker_addr
self.ncsa_publisher = SimplePublisher(ncsa_pub_broker_url, "YAML")
F1_name = 'F1'
F1_passwd = 'F1'
F1_pub_name = 'F1_PUB'
F1_pub_passwd = 'F1_PUB'
F1_broker_url = "amqp://" + F1_name + ":" + \
F1_passwd + "@" + \
broker_addr
F1_pub_broker_url = "amqp://" + F1_pub_name + ":" + \
F1_pub_passwd + "@" + \
broker_addr
self.f1_publisher = SimplePublisher(F1_pub_broker_url, "YAML")
F2_name = 'F2'
F2_passwd = 'F2'
F2_pub_name = 'F2_PUB'
F2_pub_passwd = 'F2_PUB'
F2_broker_url = "amqp://" + F2_name + ":" + \
F2_passwd + "@" + \
broker_addr
F2_pub_broker_url = "amqp://" + F2_pub_name + ":" + \
F2_pub_passwd + "@" + \
broker_addr
self.f2_publisher = SimplePublisher(F2_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.dmcs_consumer = Consumer(dmcs_broker_url, 'dmcs_ack_consume',
'thread-dmcs', self.on_dmcs_message,
'YAML')
self.dmcs_consumer.start()
self.ncsa_consumer = Consumer(ncsa_broker_url, 'ncsa_consume',
'thread-ncsa', self.on_ncsa_message,
'YAML')
self.ncsa_consumer.start()
self.F1_consumer = Consumer(F1_broker_url, 'f1_consume', 'thread-f1',
self.on_f1_message, 'YAML')
self.F1_consumer.start()
self.F2_consumer = Consumer(F2_broker_url, 'f2_consume', 'thread-f2',
self.on_f2_message, 'YAML')
self.F2_consumer.start()
sleep(3)
if self.DP:
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(6)
self.verify_ncsa_messages()
self.verify_F2_messages()
self.verify_F1_messages()
self.verify_dmcs_messages()
sleep(2)
self.dmcs_consumer.stop()
self.dmcs_consumer.join()
self.ncsa_consumer.stop()
self.ncsa_consumer.join()
self.F1_consumer.stop()
self.F1_consumer.join()
self.F2_consumer.stop()
self.F2_consumer.join()
if self.DP:
print("Finished with PP tests.")
def __init__(self, broker_url):
self._name = 'scratchpad'
self._passwd = 'scratchpad'
self._pad = {}
self._broker_url = broker_url
self._publisher = SimplePublisher(self._broker_url)
class TestAtDev:
dmcs_pub_broker_url = None
dmcs_publisher = None
dmcs_consumer = None
dmcs_consumer_msg_list = []
at_ctrl_pub_broker_url = None
at_ctrl_publisher = None
at_ctrl_consumer = None
at_ctrl_consumer_msg_list = []
F99_pub_broker_url = None
F99_publisher = None
F99_consumer = None
f99_consumer_msg_list = []
EXPECTED_AR_CTRL_MESSAGES = 1
EXPECTED_DMCS_MESSAGES = 1
EXPECTED_F99_MESSAGES = 1
ccd_list = [14,17,21.86]
prp = toolsmod.prp
DP = toolsmod.DP # Debug Printing either True or False...override for this file only...
def test_atdev(self, Atdev):
self.atdev = Atdev
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test_at.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
raise
raft_dict = cdm[ROOT]['DEFAULT_RAFT_CONFIGURATION']
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
dmcs_name = cdm[ROOT]['DMCS_BROKER_NAME']
dmcs_passwd = cdm[ROOT]['DMCS_BROKER_PASSWD']
dmcs_pub_name = cdm[ROOT]['DMCS_BROKER_PUB_NAME']
dmcs_pub_passwd = cdm[ROOT]['DMCS_BROKER_PUB_PASSWD']
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_passwd + "@" + \
broker_addr
dmcs_pub_broker_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % dmcs_pub_broker_url)
self.dmcs_publisher = SimplePublisher(dmcs_pub_broker_url, "YAML")
"""
ar_ctrl_name = cdm[ROOT]['ARCHIVE_BROKER_NAME']
ar_ctrl_passwd = cdm[ROOT]['ARCHIVE_BROKER_PASSWD']
ar_ctrl_pub_name = cdm[ROOT]['ARCHIVE_BROKER_PUB_NAME']
ar_ctrl_pub_passwd = cdm[ROOT]['ARCHIVE_BROKER_PUB_PASSWD']
ar_ctrl_broker_url = "amqp://" + ar_ctrl_name + ":" + \
ar_ctrl_passwd + "@" + \
broker_addr
ar_ctrl_pub_broker_url = "amqp://" + ar_ctrl_pub_name + ":" + \
ar_ctrl_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % ar_ctrl_pub_broker_url)
self.ar_ctrl_publisher = SimplePublisher(ar_ctrl_pub_broker_url, "YAML")
"""
F99_name = 'F99'
F99_passwd = 'F99'
F99_pub_name = 'F99_PUB'
F99_pub_passwd = 'F99_PUB'
F99_broker_url = "amqp://" + F99_name + ":" + \
F99_passwd + "@" + \
broker_addr
F99_pub_broker_url = "amqp://" + F99_pub_name + ":" + \
F99_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % F99_pub_broker_url)
self.F99_publisher = SimplePublisher(F99_pub_broker_url, "YAML")
print("All publishers are running...")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
print("MessageAuthority running...")
self.dmcs_consumer = Consumer(dmcs_broker_url,'dmcs_ack_consume', 'thread-dmcs',
self.on_dmcs_message,'YAML')
self.dmcs_consumer.start()
print("DMCS Consumer running...")
"""
self.ar_ctrl_consumer = Consumer(ar_ctrl_broker_url,'archive_ctrl_consume', 'thread-ar-ctrl',
self.on_ar_ctrl_message,'YAML')
self.ar_ctrl_consumer.start()
print("ar_ctrl Consumer running...")
"""
self.F99_consumer = Consumer(F99_broker_url,'f99_consume', 'thread-f99',
self.on_f99_message,'YAML')
self.F99_consumer.start()
print("F99 Consumer running...")
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(8)
self.verify_dmcs_messages()
self.verify_F99_messages()
#self.verify_ar_ctrl_messages()
sleep(3)
# Shut down consumer threads nicely
self.dmcs_consumer.stop()
self.dmcs_consumer.join()
#self.ar_ctrl_consumer.stop()
#self.ar_ctrl_consumer.join()
self.F99_consumer.stop()
self.F99_consumer.join()
if self.DP:
print("Finished with AT tests.")
def send_messages(self):
LOGGER.warning('In on_f99_message handler')
print("Starting send_messages")
# Tests only an AT device
self.EXPECTED_AR_CTRL_MESSAGES = 2
self.EXPECTED_DMCS_MESSAGES = 2
self.EXPECTED_F99_MESSAGES = 4
"""
msg = {}
msg['MSG_TYPE'] = "AT_NEW_SESSION"
msg['SESSION_ID'] = 'SI_469976'
msg['ACK_ID'] = 'AT_NEW_SESSION_ACK_44221'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
#time.sleep(2)
time.sleep(12)
print("New Session Message")
self.dmcs_publisher.publish_message("at_foreman_consume", msg)
"""
LOGGER.warning('About to send start_int')
msg = {}
msg['MSG_TYPE'] = "AT_START_INTEGRATION"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444245'
msg['IMAGE_INDEX'] = 2
msg['IMAGE_SEQUENCE_NAME'] = 'XX_seq'
msg['IMAGES_IN_SEQUENCE'] = 8
msg['SESSION_ID'] = 'SI_469976'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'NEW_VISIT_ACK_76'
msg['RAFT_LIST'] = ['WFS_RAFT']
msg['RAFT_CCD_LIST'] = [['WFS_CCD']]
time.sleep(12)
LOGGER.warning('Here comes start_int message')
print("Start Integration Message")
self.dmcs_publisher.publish_message("at_foreman_consume", msg)
LOGGER.warning('Just after publishing start_int message')
#time.sleep(3)
time.sleep(8)
msg = {}
msg['MSG_TYPE'] = "AT_END_READOUT"
msg['JOB_NUM'] = '4xx72'
msg['SESSION_ID'] = 'SI_469976'
msg['IMAGE_ID'] = 'IMG_444245'
msg['IMAGE_INDEX'] = 2
msg['IMAGE_SEQUENCE_NAME'] = 'XX_seq'
msg['IMAGES_IN_SEQUENCE'] = 8
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'AT_ACK_94671'
#time.sleep(11)
#time.sleep(3)
print("AT END READOUT Message")
self.dmcs_publisher.publish_message("at_foreman_consume", msg)
time.sleep(5)
msg = {}
msg['MSG_TYPE'] = "AT_HEADER_READY"
msg['IMAGE_ID'] = 'IMG_444245'
msg['FILENAME'] = '/mnt/headers'
msg['ACK_ID'] = 'AT_ACK_1444'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
print("AT HEADER_READY Message")
self.dmcs_publisher.publish_message("at_foreman_consume", msg)
time.sleep(2)
print("Message Sender done")
def verify_dmcs_messages(self):
len_list = len(self.dmcs_consumer_msg_list)
print("DMCS RECEIVED %s MESSAGES" % len_list)
if len_list != self.EXPECTED_DMCS_MESSAGES:
print("Incorrect number of DMCS messages received")
pytest.fail('DMCS simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_DMCS_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.dmcs_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
print("This DMCS message failed verification: ")
self.prp.pprint(msg)
pytest.fail("The following DMCS Bridge response message failed when compared with the sovereign example: %s" % msg)
print("Responses to DMCS Bridge pass verification.")
def verify_ar_ctrl_messages(self):
len_list = len(self.ar_ctrl_consumer_msg_list)
print("AR_CTRL RECEIVED %s MESSAGES" % len_list)
if len_list != self.EXPECTED_AR_CTRL_MESSAGES:
print("Incorrect number of AR_CTRL messages received")
pytest.fail('AR CTRL simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_AR_CTRL_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.ar_ctrl_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
print("This AR_CTRL message failed verification: ")
self.prp.pprint(msg)
pytest.fail("The following message to the AR CTRL failed when compared with the sovereign example: %s" % msg)
print("Messages to the AR CTRL pass verification.")
def verify_F99_messages(self):
len_list = len(self.f99_consumer_msg_list)
print("Number of messages is: %s" % len_list)
if len_list != self.EXPECTED_F99_MESSAGES:
print("Incorrect number of F99 messages received")
pytest.fail('F99 simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_F99_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.f99_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
print("This F99 message failed verification: ")
self.prp.pprint(msg)
pytest.fail("The following message to F99 failed when compared with the sovereign example: %s" % msg)
print("Messages to F99 pass verification.")
def on_dmcs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.dmcs_consumer_msg_list.append(body)
def on_ar_ctrl_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.ar_ctrl_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'NEW_ARCHIVE_ITEM':
msg = {}
msg['MSG_TYPE'] = 'NEW_ARCHIVE_ITEM_ACK'
msg['COMPONENT'] = 'ARCHIVE_CTRL'
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
msg['TARGET_LOCATION'] = '/tmp'
self.ar_ctrl_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'AR_ITEMS_XFERD':
msg = {}
msg['MSG_TYPE'] = 'AR_ITEMS_XFERD_ACK'
msg['COMPONENT'] = 'ARCHIVE_CTRL'
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
filename_list = body['RESULT_SET']['FILENAME_LIST']
msg['RESULT_SET'] = {}
msg['RESULT_SET']['IMAGE_ID_LIST'] = body['RESULT_SET']['IMAGE_ID_LIST']
msg['RESULT_SET']['RECEIPT_LIST'] = []
msg['RESULT_SET']['FILENAME_LIST'] = filename_list
RECEIPT_LIST = []
for filename in filename_list:
RECEIPT_LIST.append('x14_' + str(filename))
msg['RESULT_LIST']['RECEIPT_LIST'] = RECEIPT_LIST
self.ar_ctrl_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail("The following unknown message was received by the Archive CTRL: %s" % body)
def on_f99_message(self, ch, method, properties, body):
LOGGER.debug("Inside on_f99_message......")
print("Inside on_f99_message test handler........")
ch.basic_ack(method.delivery_tag)
self.f99_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'AT_FWDR_HEALTH_CHECK':
LOGGER.debug('Message received by health check test code. body is: %s', pformat(str(body)))
msg = {}
msg['MSG_TYPE'] = 'AT_FWDR_HEALTH_CHECK_ACK'
msg['COMPONENT'] = 'FORWARDER_99'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
if self.DP:
print("In test - on_f99_message health check - sending ack: %s" % msg)
self.F99_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'AT_FWDR_XFER_PARAMS':
msg = {}
msg['MSG_TYPE'] = 'AT_FWDR_XFER_PARAMS_ACK'
msg['COMPONENT'] = 'FORWARDER_99'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.F99_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'AT_FWDR_HEADER_READY':
# No need to ACK
pass
elif body['MSG_TYPE'] == 'AT_FWDR_END_READOUT':
LOGGER.warning("Inside on_f99_message AT_FWDR_END_READOUT case...")
xfer_msg = None
image_id = body['IMAGE_ID']
for msg in self.f99_consumer_msg_list:
if msg['MSG_TYPE'] == 'AT_FWDR_XFER_PARAMS':
xfer_msg = msg
if xfer_msg == None:
LOGGER.warning("The AT_FWDR_XFER_PARAMS message was not received before AT_FWDR_READOUT in F99")
fail_msg = "The AT_FWDR_XFER_PARAMS message was not received before AT_FWDR_READOUT in F99"
pytest.fail("\n".join(fail_msg), pytrace=True)
# use message to build response
msg = {}
msg['MSG_TYPE'] = 'AT_FWDR_END_READOUT_ACK'
msg['COMPONENT'] = 'FORWARDER_99'
msg['JOB_NUM'] = body['JOB_NUM']
msg['SESSION_ID'] = body['SESSION_ID']
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
msg['RESULT_SET'] = {}
msg['RESULT_SET']['FILENAME_LIST'] = []
msg['RESULT_SET']['CHECKSUM_LIST'] = []
RAFT_PLUS_CCD_LIST = []
FILENAME_LIST = []
CHECKSUM_LIST = []
target_location = xfer_msg['TARGET_LOCATION']
FILENAME_LIST.append(target_location + "/" + image_id + ".fits")
CHECKSUM_LIST.append('XXXXFFFF4444$$$$')
msg['RESULT_SET']['FILENAME_LIST'] = FILENAME_LIST
msg['RESULT_SET']['CHECKSUM_LIST'] = CHECKSUM_LIST
self.F99_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail("The following unknown message was received by FWDR F99: %s" % body)
def convert_raftdict_to_name_list(self, rdict):
raft_list = list(rdict.keys())
num_rafts = len(raft_list)
integrated_names_list = []
for i in range(0,num_rafts):
current_raft = raft_list[i]
ccd_list = []
ccd_list = rdict[current_raft]
if ccd_list[0] == 'ALL':
ccd_list = ['00','10','20','01','11','21','02','12','22']
num_current_ccds = len(ccd_list)
for j in range(0,num_current_ccds):
tmp_str = current_raft + '-' + ccd_list[j]
integrated_names_list.append(tmp_str)
return integrated_names_list
def convert_raft_and_ccd_list_to_name_list(self, raft_list, raft_ccd_list):
#raft_list = list(rdict.keys())
num_rafts = len(raft_list)
integrated_names_list = []
for i in range(0,num_rafts):
current_raft = raft_list[i]
ccd_list = []
ccd_list = raft_ccd_list[i]
if ccd_list[0] == 'ALL':
ccd_list = ['00','10','20','01','11','21','02','12','22']
num_current_ccds = len(ccd_list)
for j in range(0,num_current_ccds):
tmp_str = current_raft + '-' + ccd_list[j]
integrated_names_list.append(tmp_str)
return integrated_names_list
msg['FILE_NAME'] = formatted_filename
msg['IMAGE_ID'] = id
if (self.Readout_Image_IDs is None) and (self.Take_Images_Done_Message is None):
return
if (self.Readout_Image_IDs is None) and (self.Take_Images_Done_Message is not None):
message = {}
message = self.Take_Images_Done_Message
message['MSG_TYPE'] = "FORWARD_TAKE_IMAGES_DONE"
print("Outgoing FORWARD_TAKE_IMAGES_DONE message is: ")
print(message)
# publish message
if __name__ == "__main__":
premium = Premium()
sp1 = SimplePublisher('amqp://*****:*****@141.142.238.10:5672/%2Fbunny', "YAML")
time.sleep(3)
msg = {}
msg['MSG_TYPE'] = "FORMAT_END_READOUT"
msg['IMAGE_ID'] = "Img_X14001"
time.sleep(2)
sp1.publish_message("format_consume_from_f1", msg)
msg = {}
msg['MSG_TYPE'] = "FORMAT_END_READOUT"
msg['IMAGE_ID'] = 'Img_X14022'
time.sleep(4)
sp1.publish_message("format_consume_from_f1", msg)
msg = {}
class DMCS:
# This DMCS simulator is how we interact with the test system.
# Commands can be sent manually to simulate how to DMCS might work.
def __init__(self):
printc("Setting up DMCS...")
self._options = "\
1 - (READY) Send Job Information\n\
2 - (SET) Send Standby Message\n\
3 - (GO) Send Readout Message\n\
4 - (RESET) Cancel a Job\n\
0 - (EXIT) Quit DMCS Simulator\n"
self._broker_url = 'amqp://' + AMQP_DMCS_USER + ':' + AMQP_DMCS_PSWD + '@' + AMQP_BROKER_ADDR + ':' + AMQP_BROKER_PORT + '/' + AMQP_BROKER_VHOST
printc('Using broker url: %s' % self._broker_url)
printc("Declaring and binding exchanges...")
printc("Attempting to create a consumer for the '%s' queue." % (Q_DMCS_CONSUME))
self._dmcs_consumer = Consumer(self._broker_url, Q_DMCS_CONSUME)
try:
printc("Attempting to start the consumer thread...")
thread.start_new_thread(self.run_dmcs_consumer, ())
except:
printc("Failed to start consumer thread, quitting...")
sys.exit()
printc("Done setting up consumer thread.")
printc("Setting up publisher...")
self._publisher = SimplePublisher(self._broker_url)
printc("Done creating publisher.")
self._job_msg = {}
self._job_msg['MSG_TYPE'] = 'JOB'
self._job_msg['JOB_NUM'] = 0
self._job_msg['RAFT_NUM'] = 1
self._standby_msg = {}
self._standby_msg['MSG_TYPE'] = 'STANDBY'
self._readout_msg = {}
self._readout_msg['MSG_TYPE'] = 'READOUT'
self._stop_msg = {}
self._stop_msg['MSG_TYPE'] = 'CANCEL'
self._shutdown_msg = {}
self._shutdown_msg['MSG_TYPE'] = 'SHUTDOWN'
def on_dmcs_messages(self, ch, method, properties, body):
msg_dict = yaml.load(body)
printc("Received: %r" % msg_dict)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
def run_dmcs_consumer(self):
self._dmcs_consumer.run(self.on_dmcs_messages)
return
def run(self):
keep_running = True
while keep_running:
try:
user_input = int(raw_input(self._options))
except:
user_input = -1
# New Job
if 1 == user_input:
good_input = True
try:
new_raft_num = int(raw_input("How many pairs? : "))
except:
good_input = False
printc("Bad input...")
if good_input:
self._job_msg['JOB_NUM'] = self._job_msg['JOB_NUM'] + 1
self._job_msg['RAFT_NUM'] = new_raft_num
self._publisher.publish_message(Q_DMCS_PUBLISH, yaml.dump(self._job_msg))
pass
# Standby
elif 2 == user_input:
good_input = True
try:
new_job_num = int(raw_input("STANDBY which job? : "))
except:
good_input = False
printc("Bad input...")
if good_input:
self._standby_msg['JOB_NUM'] = str(new_job_num)
self._publisher.publish_message(Q_DMCS_PUBLISH, yaml.dump(self._standby_msg))
pass
# Readout
elif 3 == user_input:
good_input = True
try:
new_job_num = int(raw_input("READOUT which job? : "))
except:
good_input = False
printc("Bad input...")
if good_input:
self._readout_msg['JOB_NUM'] = str(new_job_num)
self._publisher.publish_message(Q_DMCS_PUBLISH, yaml.dump(self._readout_msg))
pass
# Cancel
elif 4 == user_input:
good_input = True
try:
job_cancel = int(raw_input("Cancel which job? : "))
except:
good_input = False
printc("Bad input...")
if good_input:
self._stop_msg['JOB_NUM'] = job_cancel
self._publisher.publish_message(Q_DMCS_PUBLISH, yaml.dump(self._stop_msg))
pass
# Exit
elif 0 == user_input:
keep_running = False
else:
printc("Invalid input...\n")
return
def main():
premium = Premium()
sp1 = SimplePublisher('amqp://*****:*****@141.142.208.191:5672/%2Fbunny', "YAML")
#sp2 = SimplePublisher('amqp://*****:*****@141.142.208.191:5672/%2Ftester')
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#cons = Consumer(broker_url, 'F8_consume')
#try:
# thread.start_new_thread( do_it, ("thread-1", 2,) )
#except:
# print "Cannot start thread"
# while 1:
msg = {}
msg['MSG_TYPE'] = "FORWARDER_HEALTH_CHECK"
msg['ACK_ID'] = 'AR_FWD_HEALTH_ACK_1662'
msg['REPLY_QUEUE'] = 'ar_foreman_ack_publish'
time.sleep(3)
sp1.publish_message("f1_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_XFER_PARAMS"
msg['JOB_NUM'] = "j_1412_z"
msg['VISIT_ID'] = 'V229'
msg['IMAGE_ID'] = 'IMG_226XT_46'
msg['ACK_ID'] = 'AR_XFER_PARAMS_ACK_112'
msg['TARGET_DIR'] = "/mnt/xfer_dir/"
msg['REPLY_QUEUE'] = 'ar_foreman_ack_publish'
m = {}
m['CCD_LIST'] = ['1','6','2','3','9','4','7','8']
m['XFER_UNIT'] = ['CCD']
m['FQN'] = 'ARCHIVE'
m['NAME'] = "archive"
m['HOSTNAME'] = 'blarg'
m['IP_ADDR'] = '141.142.210.202'
m['IMG_SRC'] = "blah" # For future use...
msg['XFER_PARAMS'] = m
time.sleep(3)
sp1.publish_message("f1_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_READOUT"
msg['JOB_NUM'] = "j_1412_z"
msg['IMAGE_ID'] = 'IMG_226XT_46'
msg['ACK_ID'] = 'AR_READOUT_ACK_92112'
msg['REPLY_QUEUE'] = 'ar_foreman_ack_publish'
time.sleep(4)
sp1.publish_message("f1_consume", msg)
"""
msg = {}
msg['MSG_TYPE'] = "NEXT_VISIT"
msg['VISIT_ID'] = 'XX_28272'
msg['BORE_SIGHT'] = 'A LITTLE TO THE LEFT'
time.sleep(4)
sp1.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "START_INTEGRATION"
msg['IMAGE_ID'] = 'IMG_444244'
msg['DEVICE'] = 'AR'
time.sleep(4)
sp1.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "READOUT"
msg['IMAGE_ID'] = 'IMG_444244'
msg['DEVICE'] = 'AR'
time.sleep(4)
sp1.publish_message("ocs_dmcs_consume", msg)
"""
print("Sender done")
def test_dmcs(self, Dmcs):
self.dmcs = Dmcs
#logging.warning("Logging is Working!")
LOGGER.critical("LOGGING is Working!")
#self.LOGGER.info("self Logging is Working!")
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
ocs_name = cdm[ROOT]['OCS_BROKER_NAME']
ocs_passwd = cdm[ROOT]['OCS_BROKER_PASSWD']
ocs_pub_name = cdm[ROOT]['OCS_BROKER_PUB_NAME']
ocs_pub_passwd = cdm[ROOT]['OCS_BROKER_PUB_PASSWD']
ocs_broker_url = "amqp://" + ocs_name + ":" + \
ocs_passwd + "@" + \
broker_addr
self.ocs_pub_broker_url = "amqp://" + ocs_pub_name + ":" + \
ocs_pub_passwd + "@" + \
broker_addr
self.ocs_publisher = SimplePublisher(self.ocs_pub_broker_url, "YAML")
at_name = cdm[ROOT]['AUX_BROKER_NAME']
at_passwd = cdm[ROOT]['AUX_BROKER_PASSWD']
at_pub_name = cdm[ROOT]['AUX_BROKER_PUB_NAME']
at_pub_passwd = cdm[ROOT]['AUX_BROKER_PUB_PASSWD']
at_broker_url = "amqp://" + at_name + ":" + \
at_passwd + "@" + \
broker_addr
self.at_pub_broker_url = "amqp://" + at_pub_name + ":" + \
at_pub_passwd + "@" + \
broker_addr
self.at_publisher = SimplePublisher(self.at_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.ocs_consumer = Consumer(ocs_broker_url,'dmcs_ocs_publish', 'thread-ocs',
self.on_ocs_message,'YAML')
self.ocs_consumer.start()
self.at_consumer = Consumer(at_broker_url,'at_foreman_consume', 'thread-at',
self.on_at_message,'YAML')
self.at_consumer.start()
sleep(3)
if self.DP:
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(3)
self.verify_ocs_messages()
sleep(3)
self.verify_at_messages()
sleep(2)
self.ocs_consumer.stop()
self.ocs_consumer.join()
self.at_consumer.stop()
self.at_consumer.join()
if self.DP:
print("Finished with DMCS AT tests.")
class BaseForeman:
FWD_SCBD = None
JOB_SCBD = None
ACK_SCBD = None
ACK_PUBLISH = "ack_publish"
YAML = 'YAML'
def __init__(self, filename=None):
toolsmod.singleton(self)
self._config_file = 'ForemanCfg.yaml'
if filename != None:
self._config_file = filename
cdm = toolsmod.intake_yaml_file(self._config_file)
try:
self._base_name = cdm[ROOT][BASE_BROKER_NAME] # Message broker user & passwd
self._base_passwd = cdm[ROOT][BASE_BROKER_PASSWD]
self._ncsa_name = cdm[ROOT][NCSA_BROKER_NAME]
self._ncsa_passwd = cdm[ROOT][NCSA_BROKER_PASSWD]
self._base_broker_addr = cdm[ROOT][BASE_BROKER_ADDR]
self._ncsa_broker_addr = cdm[ROOT][NCSA_BROKER_ADDR]
forwarder_dict = cdm[ROOT][XFER_COMPONENTS][FORWARDERS]
except KeyError as e:
print("Dictionary error")
print("Bailing out...")
sys.exit(99)
if 'QUEUE_PURGES' in cdm[ROOT]:
self.purge_broker(cdm['ROOT']['QUEUE_PURGES'])
self._base_msg_format = self.YAML
self._ncsa_msg_format = self.YAML
if 'BASE_MSG_FORMAT' in cdm[ROOT]:
self._base_msg_format = cdm[ROOT][BASE_MSG_FORMAT]
if 'NCSA_MSG_FORMAT' in cdm[ROOT]:
self._ncsa_msg_format = cdm[ROOT][NCSA_MSG_FORMAT]
self._base_broker_url = 'amqp_url'
self._ncsa_broker_url = 'amqp_url'
self._next_timed_ack_id = 0
# Create Redis Forwarder table with Forwarder info
self.FWD_SCBD = ForwarderScoreboard(forwarder_dict)
self.JOB_SCBD = JobScoreboard()
self.ACK_SCBD = AckScoreboard()
self._msg_actions = { 'NEW_JOB': self.process_dmcs_new_job,
'READOUT': self.process_dmcs_readout,
'NCSA_RESOURCE_QUERY_ACK': self.process_ack,
'NCSA_STANDBY_ACK': self.process_ack,
'NCSA_READOUT_ACK': self.process_ack,
'FORWARDER_HEALTH_ACK': self.process_ack,
'FORWARDER_JOB_PARAMS_ACK': self.process_ack,
'FORWARDER_READOUT_ACK': self.process_ack,
'NEW_JOB_ACK': self.process_ack }
self._base_broker_url = "amqp://" + self._base_name + ":" + self._base_passwd + "@" + str(self._base_broker_addr)
self._ncsa_broker_url = "amqp://" + self._ncsa_name + ":" + self._ncsa_passwd + "@" + str(self._ncsa_broker_addr)
LOGGER.info('Building _base_broker_url. Result is %s', self._base_broker_url)
LOGGER.info('Building _ncsa_broker_url. Result is %s', self._ncsa_broker_url)
self.setup_publishers()
self.setup_consumers()
#self._ncsa_broker_url = ""
#self.setup_federated_exchange()
def setup_consumers(self):
"""This method sets up a message listener from each entity
with which the BaseForeman has contact here. These
listeners are instanced in this class, but their run
methods are each called as a separate thread. While
pika does not claim to be thread safe, the manner in which
the listeners are invoked below is a safe implementation
that provides non-blocking, fully asynchronous messaging
to the BaseForeman.
The code in this file expects message bodies to arrive as
YAML'd python dicts, while in fact, message bodies are sent
on the wire as XML; this way message format can be validated,
versioned, and specified in just one place. To make this work,
there is an object that translates the params dict to XML, and
visa versa. The translation object is instantiated by the consumer
and acts as a filter before sending messages on to the registered
callback for processing.
"""
LOGGER.info('Setting up consumers on %s', self._base_broker_url)
LOGGER.info('Running start_new_thread on all consumer methods')
self._dmcs_consumer = Consumer(self._base_broker_url, self.DMCS_PUBLISH, self._base_msg_format)
try:
_thread.start_new_thread( self.run_dmcs_consumer, ("thread-dmcs-consumer", 2,) )
except:
LOGGER.critical('Cannot start DMCS consumer thread, exiting...')
sys.exit(99)
self._forwarder_consumer = Consumer(self._base_broker_url, self.FORWARDER_PUBLISH, self._base_msg_format)
try:
_thread.start_new_thread( self.run_forwarder_consumer, ("thread-forwarder-consumer", 2,) )
except:
LOGGER.critical('Cannot start FORWARDERS consumer thread, exiting...')
sys.exit(100)
self._ncsa_consumer = Consumer(self._base_broker_url, self.NCSA_PUBLISH, self._base_msg_format)
try:
_thread.start_new_thread( self.run_ncsa_consumer, ("thread-ncsa-consumer", 2,) )
except:
LOGGER.critical('Cannot start NCSA consumer thread, exiting...')
sys.exit(101)
self._ack_consumer = Consumer(self._base_broker_url, self.ACK_PUBLISH, self._base_msg_format)
try:
_thread.start_new_thread( self.run_ack_consumer, ("thread-ack-consumer", 2,) )
except:
LOGGER.critical('Cannot start ACK consumer thread, exiting...')
sys.exit(102)
LOGGER.info('Finished starting all three consumer threads')
def run_dmcs_consumer(self, threadname, delay):
self._dmcs_consumer.run(self.on_dmcs_message)
def run_forwarder_consumer(self, threadname, delay):
self._forwarder_consumer.run(self.on_forwarder_message)
def run_ncsa_consumer(self, threadname, delay):
self._ncsa_consumer.run(self.on_ncsa_message)
def run_ack_consumer(self, threadname, delay):
self._ack_consumer.run(self.on_ack_message)
def setup_publishers(self):
LOGGER.info('Setting up Base publisher on %s using %s', self._base_broker_url, self._base_msg_format)
LOGGER.info('Setting up NCSA publisher on %s using %s', self._ncsa_broker_url, self._ncsa_msg_format)
self._base_publisher = SimplePublisher(self._base_broker_url, self._base_msg_format)
self._ncsa_publisher = SimplePublisher(self._ncsa_broker_url, self._ncsa_msg_format)
# def setup_federated_exchange(self):
# # Set up connection URL for NCSA Broker here.
# self._ncsa_broker_url = "amqp://" + self._name + ":" + self._passwd + "@" + str(self._ncsa_broker_addr)
# LOGGER.info('Building _ncsa_broker_url. Result is %s', self._ncsa_broker_url)
# pass
def on_dmcs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
#msg_dict = yaml.load(body)
msg_dict = body
LOGGER.info('In DMCS message callback')
LOGGER.debug('Thread in DMCS callback is %s', _thread.get_ident())
LOGGER.info('Message from DMCS callback message body is: %s', str(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def on_forwarder_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
LOGGER.info('In Forwarder message callback, thread is %s', _thread.get_ident())
LOGGER.info('forwarder callback msg body is: %s', str(body))
pass
def on_ncsa_message(self,ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
LOGGER.info('In ncsa message callback, thread is %s', _thread.get_ident())
#msg_dict = yaml.load(body)
msg_dict = body
LOGGER.info('ncsa msg callback body is: %s', str(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def on_ack_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
msg_dict = body
LOGGER.info('In ACK message callback')
LOGGER.debug('Thread in ACK callback is %s', _thread.get_ident())
LOGGER.info('Message from ACK callback message body is: %s', str(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def process_dmcs_new_job(self, params):
input_params = params
needed_workers = len(input_params[RAFTS])
ack_id = self.forwarder_health_check(input_params)
self.ack_timer(7) # This is a HUGE num seconds for now..final setting will be milliseconds
healthy_forwarders = self.ACK_SCBD.get_components_for_timed_ack(timed_ack)
num_healthy_forwarders = len(healthy_forwarders)
if needed_workers > num_healthy_forwarders:
result = self.insufficient_base_resources(input_params, healthy_forwarders)
return result
else:
healthy_status = {"STATUS": "HEALTHY", "STATE":"READY_WITHOUT_PARAMS"}
self.FWD_SCBD.set_forwarder_params(healthy_forwarders, healthy_status)
ack_id = self.ncsa_resources_query(input_params, healthy_forwarders)
self.ack_timer(3)
#Check ACK scoreboard for response from NCSA
ncsa_response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if ncsa_response:
pairs = {}
ack_bool = None
try:
ack_bool = ncsa_response[ACK_BOOL]
if ack_bool == True:
pairs = ncsa_response[PAIRS]
except KeyError as e:
pass
# Distribute job params and tell DMCS I'm ready.
if ack_bool == TRUE:
fwd_ack_id = self.distribute_job_params(input_params, pairs)
self.ack_timer(3)
fwd_params_response = self.ACK_SCBD.get_components_for_timed_ack(fwd_ack_id)
if fwd_params_response and (len(fwd_params_response) == len(fwders)):
self.JOB_SCBD.set_value_for_job(job_num, "STATE", "BASE_TASK_PARAMS_SENT")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_BASE_TASK_PARAMS_SENT", get_timestamp())
in_ready_state = {'STATE':'READY_WITH_PARAMS'}
self.FWD_SCBD.set_forwarder_params(fwders, in_ready_state)
# Tell DMCS we are ready
result = self.accept_job(job_num)
else:
#not enough ncsa resources to do job - Notify DMCS
idle_param = {'STATE': 'IDLE'}
self.FWD_SCBD.set_forwarder_params(healthy_forwarders, idle_params)
result = self.insufficient_ncsa_resources(ncsa_response)
return result
else:
result = self.ncsa_no_response(input_params)
idle_param = {'STATE': 'IDLE'}
self.FWD_SCBD.set_forwarder_params(list(forwarder_candidate_dict.keys()), idle_params)
return result
def forwarder_health_check(self, params):
job_num = str(params[JOB_NUM])
raft_list = params['RAFTS']
needed_workers = len(raft_list)
self.JOB_SCBD.add_job(job_num, needed_workers)
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ADDED", get_timestamp())
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ADDED_E", get_epoch_timestamp())
LOGGER.info('Received new job %s. Needed workers is %s', job_num, needed_workers)
# run forwarder health check
# get timed_ack_id
timed_ack = self.get_next_timed_ack_id("FORWARDER_HEALTH_CHECK_ACK")
forwarders = self.FWD_SCBD.return_available_forwarders_list()
# Mark all healthy Forwarders Unknown
state_status = {"STATE": "HEALTH_CHECK", "STATUS": "UNKNOWN"}
self.FWD_SCBD.set_forwarder_params(forwarders, state_status)
# send health check messages
ack_params = {}
ack_params[MSG_TYPE] = FORWARDER_HEALTH_CHECK
ack_params["ACK_ID"] = timed_ack
ack_params[JOB_NUM] = job_num
self.JOB_SCBD.set_value_for_job(job_num, "STATE", "BASE_RESOURCE_QUERY")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_BASE_RESOURCE_QUERY", get_timestamp())
audit_params = {}
audit_params['DATA_TYPE'] = 'FOREMAN_ACK_REQUEST'
audit_params['SUB_TYPE'] = 'FORWARDER_HEALTH_CHECK_ACK'
audit_params['ACK_ID'] = timed_ack
audit_parsms['COMPONENT_NAME'] = 'BASE_FOREMAN'
audit_params['TIME'] = get_epoch_timestamp()
for forwarder in forwarders:
self._base_publisher.publish_message(self.FWD_SCBD.get_value_for_forwarder(forwarder,"CONSUME_QUEUE"),
ack_params)
return timed_ack
def insufficient_base_resources(self, params, healthy_forwarders):
# send response msg to dmcs refusing job
job_num = str(params[JOB_NUM])
raft_list = params[RAFTS]
ack_id = params['ACK_ID']
needed_workers = len(raft_list)
LOGGER.info('Reporting to DMCS that there are insufficient healthy forwarders for job #%s', job_num)
dmcs_params = {}
fail_dict = {}
dmcs_params[MSG_TYPE] = NEW_JOB_ACK
dmcs_params[JOB_NUM] = job_num
dmcs_params[ACK_BOOL] = False
dmcs_params[ACK_ID] = ack_id
### NOTE FOR DMCS ACK PROCESSING:
### if ACK_BOOL == True, there will NOT be a FAIL_DETAILS section
### If ACK_BOOL == False, there will always be a FAIL_DICT to examine AND there will always be a
### BASE_RESOURCES inside the FAIL_DICT
### If ACK_BOOL == False, and the BASE_RESOURCES inside FAIL_DETAILS == 0,
### there will be only NEEDED and AVAILABLE Forwarder params - nothing more
### If ACK_BOOL == False and BASE_RESOURCES inside FAIL_DETAILS == 1, there will always be a
### NCSA_RESOURCES inside FAIL_DETAILS set to either 0 or 'NO_RESPONSE'
### if NCSA_RESPONSE == 0, there will be NEEDED and AVAILABLE Distributor params
### if NCSA_RESOURCES == 'NO_RESPONSE' there will be nothing else
fail_dict['BASE_RESOURCES'] = '0'
fail_dict[NEEDED_FORWARDERS] = str(needed_workers)
fail_dict[AVAILABLE_FORWARDERS] = str(len(healthy_forwarders))
dmcs_params['FAIL_DETAILS'] = fail_dict
self._base_publisher.publish_message("dmcs_consume", dmcs_params)
# mark job refused, and leave Forwarders in Idle state
self.JOB_SCBD.set_value_for_job(job_num, "STATE", "JOB_ABORTED")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ABORTED_BASE_RESOURCES", get_timestamp())
idle_state = {"STATE": "IDLE"}
self.FWD_SCBD.set_forwarder_params(healthy_forwarders, idle_state)
return False
def ncsa_resources_query(self, params, healthy_forwarders):
job_num = str(params[JOB_NUM])
raft_list = params[RAFTS]
needed_workers = len(raft_list)
LOGGER.info('Sufficient forwarders have been found. Checking NCSA')
self._pairs_dict = {}
forwarder_candidate_dict = {}
for i in range (0, needed_workers):
forwarder_candidate_dict[healthy_forwarders[i]] = raft_list[i]
self.FWD_SCBD.set_forwarder_status(healthy_forwarders[i], NCSA_RESOURCES_QUERY)
# Call this method for testing...
# There should be a message sent to NCSA here asking for available resources
timed_ack_id = self.get_next_timed_ack_id("NCSA_Ack")
ncsa_params = {}
ncsa_params[MSG_TYPE] = "NCSA_RESOURCES_QUERY"
ncsa_params[JOB_NUM] = job_num
#ncsa_params[RAFT_NUM] = needed_workers
ncsa_params[ACK_ID] = timed_ack_id
ncsa_params["FORWARDERS"] = forwarder_candidate_dict
self.JOB_SCBD.set_value_for_job(job_num, "STATE", "NCSA_RESOURCES_QUERY_SENT")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_NCSA_RESOURCES_QUERY_SENT", get_timestamp())
self._ncsa_publisher.publish_message(self.NCSA_CONSUME, ncsa_params)
LOGGER.info('The following forwarders have been sent to NCSA for pairing:')
LOGGER.info(forwarder_candidate_dict)
return timed_ack_id
def distribute_job_params(self, params, pairs):
#ncsa has enough resources...
job_num = str(params[JOB_NUM])
self.JOB_SCBD.set_pairs_for_job(job_num, pairs)
self.JOB_SCBD.set_value_for_job(job_num, "TIME_PAIRS_ADDED", get_timestamp())
LOGGER.info('The following pairs will be used for Job #%s: %s',
job_num, pairs)
fwd_ack_id = self.get_next_timed_ack_id("FWD_PARAMS_ACK")
fwders = list(pairs.keys())
fwd_params = {}
fwd_params[MSG_TYPE] = "FORWARDER_JOB_PARAMS"
fwd_params[JOB_NUM] = job_num
fwd_params[ACK_ID] = fwd_ack_id
for fwder in fwders:
fwd_params["TRANSFER_PARAMS"] = pairs[fwder]
route_key = self.FWD_SCBD.get_value_for_forwarder(fwder, "CONSUME_QUEUE")
self._base_publisher.publish_message(route_key, fwd_params)
return fwd_ack_id
def accept_job(self, job_num):
dmcs_message = {}
dmcs_message[JOB_NUM] = job_num
dmcs_message[MSG_TYPE] = NEW_JOB_ACK
dmcs_message[ACK_BOOL] = True
self.JOB_SCBD.set_value_for_job(job_num, STATE, "JOB_ACCEPTED")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ACCEPTED", get_timestamp())
self._base_publisher.publish_message("dmcs_consume", dmcs_message)
return True
def insufficient_ncsa_resources(self, ncsa_response):
dmcs_params = {}
dmcs_params[MSG_TYPE] = "NEW_JOB_ACK"
dmcs_params[JOB_NUM] = job_num
dmcs_params[ACK_BOOL] = False
dmcs_params[BASE_RESOURCES] = '1'
dmcs_params[NCSA_RESOURCES] = '0'
dmcs_params[NEEDED_DISTRIBUTORS] = ncsa_response[NEEDED_DISTRIBUTORS]
dmcs_params[AVAILABLE_DISTRIBUTORS] = ncsa_response[AVAILABLE_DISTRIBUTORS]
#try: FIXME - catch exception
self._base_publisher.publish_message("dmcs_consume", dmcs_params )
#except L1MessageError e:
# return False
return True
def ncsa_no_response(self,params):
#No answer from NCSA...
job_num = str(params[JOB_NUM])
raft_list = params[RAFTS]
needed_workers = len(raft_list)
dmcs_params = {}
dmcs_params[MSG_TYPE] = "NEW_JOB_ACK"
dmcs_params[JOB_NUM] = job_num
dmcs_params[ACK_BOOL] = False
dmcs_params[BASE_RESOURCES] = '1'
dmcs_params[NCSA_RESOURCES] = 'NO_RESPONSE'
self._base_publisher.publish_message("dmcs_consume", dmcs_params )
def process_dmcs_readout(self, params):
job_number = params[JOB_NUM]
pairs = self.JOB_SCBD.get_pairs_for_job(job_number)
date - get_timestamp()
self.JOB_SCBD.set_value_for_job(job_number, TIME_START_READOUT, date)
# The following line extracts the distributor FQNs from pairs dict using
# list comprehension values; faster than for loops
distributors = [v['FQN'] for v in list(pairs.values())]
forwarders = list(pairs.keys())
ack_id = self.get_next_timed_ack_id('NCSA_READOUT')
### Send READOUT to NCSA with ACK_ID
ncsa_params = {}
ncsa_params[MSG_TYPE] = 'NCSA_READOUT'
ncsa_params[ACK_ID] = ack_id
self._ncsa_publisher.publish_message(NCSA_CONSUME, yaml.dump(ncsa_params))
self.ack_timer(4)
ncsa_response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if ncsa_response:
if ncsa_response['ACK_BOOL'] == True:
#inform forwarders
fwd_ack_id = self.get_next_timed_ack_id('FORWARDER_READOUT')
for forwarder in forwarders:
name = self.FWD_SCBD.get_value_for_forwarder(forwarder, NAME)
routing_key = self.FWD_SCBD.get_routing_key(forwarder)
msg_params = {}
msg_params[MSG_TYPE] = 'FORWARDER_READOUT'
msg_params[JOB_NUM] = job_number
msg_params['ACK_ID'] = fwd_ack_id
self.FWD_SCBD.set_forwarder_state(forwarder, START_READOUT)
self._publisher.publish_message(routing_key, yaml.dump(msg_params))
self.ack_timer(4)
forwarder_responses = self.ACK_SCBD.get_components_for_timed_ack(fwd_ack_id)
if len(forwarder_responses) == len(forwarders):
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['ACK_BOOL'] = True
dmcs_params['COMMENT'] = "Readout begun at %s" % get_timestamp()
self._publisher.publish_message('dmcs_consume', yaml.dump(dmcs_params))
else:
#send problem with ncsa to DMCS
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['ACK_BOOL'] = False
dmcs_params['COMMENT'] = 'Readout Failed: Problem at NCSA - Expected Distributor Acks is %s, Number of Distributor Acks received is %s' % (ncsa_response['EXPECTED_DISTRIBUTOR_ACKS'], ncsa_response['RECEIVED_DISTRIBUTOR_ACKS'])
self._base_publisher.publish_message('dmcs_consume', yaml.dump(dmcs_params))
else:
#send 'no response from ncsa' to DMCS )
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['ACK_BOOL'] = False
dmcs_params['COMMENT'] = "Readout Failed: No Response from NCSA"
self._base_publisher.publish_message('dmcs_consume', yaml.dump(dmcs_params))
def process_ack(self, params):
self.ACK_SCBD.add_timed_ack(params)
def get_next_timed_ack_id(self, ack_type):
self._next_timed_ack_id = self._next_timed_ack_id + 1
retval = ack_type + "_" + str(self._next_timed_ack_id).zfill(6)
return retval
def ack_timer(self, seconds):
sleep(seconds)
return True
def purge_broker(self, queues):
for q in queues:
cmd = "rabbitmqctl -p /tester purge_queue " + q
os.system(cmd)
class Foreman:
# Parent class for Base and NCSA Foremen.
# Contains set up for forwarder and distributor machines
# along with code that was common between them.
def parent_init(self, db_num, prog_name, type):
custom_print.define_new_name(self.PROGRAM_NAME)
# Create machine scoreboard
self._sb_mach = Scoreboard(db_num, prog_name, type)
# Messaging URL (rabbitmq server IP)
self._broker_url = "amqp://" + AMQP_BF_USER + ":" + AMQP_BF_PSWD + "@" + AMQP_BROKER_ADDR + ":" + AMQP_BROKER_PORT + "/" + AMQP_BROKER_VHOST
# Publisher object for sending messages to rabbit
printc("Creating publisher...")
self._publisher = SimplePublisher(self._broker_url)
# Machine messages
self._msg_actions_mach = {
'TRANSFER_DONE': self.process_transfer_done, # Machine done with the current job
'REGISTER': self.process_register, # New machine wants to join
'DEREGISTER': self.process_deregister, # Machine is leaving
'STATE_UPDATE': self.process_state_update # Machine updating us on its state
}
# Machines register with us and let us know how they are doing
printc("Creating machine consumer...")
self._mach_consumer = Consumer(self._broker_url, self._machine_publish_q)
try:
thread.start_new_thread(self.run_mach_consumer, ())
except:
printc("Thread run_mach_consumer failed, quitting...")
sys.exit()
return
def run_mach_consumer(self):
# Consume messages continuously
printc("Machine message consumer is running...")
self._mach_consumer.run(self.on_mach_message)
return
def on_mach_message(self, ch, method, properties, body):
# Callback from consumer to process machine messages
# Load the message which came in yaml format
msg_dict = yaml.load(body)
# Determine which function needs to be called for this message type
try:
af_handler = self._msg_actions_mach.get(msg_dict[MSG_TYPE])
except:
printc("Bad machine message received...")
ch.basic_ack(delivery_tag=method.delivery_tag)
return
# Call that function and provide it with the message
af_handler(msg_dict)
# Acknowledge that we processed the message so rabbit can remove it from the queue
ch.basic_ack(delivery_tag=method.delivery_tag)
return
def process_register(self, msg_params):
# Process a request for a machine that is registering with us
printc("Processing name request...")
tmp_name = self.pick_name()
while False == self._sb_mach.register_machine(tmp_name):
tmp_name = self.pick_name()
self._sb_mach._redis.hset(tmp_name, 'IP_ADDR', msg_params['IP_ADDR'])
printc("%s has registered. (%s)" % (tmp_name, msg_params['IP_ADDR']))
msg = {}
msg[MSG_TYPE] = 'REGISTRATION'
msg[NAME] = tmp_name
self._publisher.publish_message(self._machine_consume_q, yaml.dump(msg))
return
def pick_name(self):
# Name creation
tmp_name = ''.join(random.choice(string.ascii_letters) for x in range(NAME_LENGTH))
return self._machine_prefix + tmp_name
def process_deregister(self, msg_params):
# Machine is deregistering with us
printc("%s has deregistered." % msg_params[NAME])
self._sb_mach.machine_deregister(msg_params[NAME])
return
def process_state_update(self, msg_params):
# Machine is updating us on something, report it in the Scoreboard
self._sb_mach.machine_update(msg_params['KEY'], msg_params['FIELD'], msg_params['VALUE'])
return
def process_transfer_done(self, msg_params):
return
class TestOCS_AckSubscriber:
os.chdir("ocs/src")
ackSubscriber = subprocess.Popen("./AckSubscriber&",
shell=True,
preexec_fn=os.setsid)
print("Preparing ackSubscriber ...")
sleep(120)
cmdListener = subprocess.Popen("./CommandListener&",
shell=True,
preexec_fn=os.setsid)
print("Preparing cmdListener ...")
sleep(10)
EXPECTED_OCS_MESSAGES = 48
ocs_consumer_msg_list = []
def test_ocs_acksubscriber(self):
try:
cdm = toolsmod.intake_yaml_file(
"../../tests/yaml/L1SystemCfg_Test_ocs_bridge.yaml")
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to fine CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]["BASE_BROKER_ADDR"]
# dmcs publisher
dmcs_pub_name = cdm[ROOT]["DMCS_BROKER_PUB_NAME"]
dmcs_pub_pwd = cdm[ROOT]["DMCS_BROKER_PUB_PASSWD"]
dmcs_broker_pub_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_pwd + "@" + \
broker_addr
self.dmcs_publisher = SimplePublisher(dmcs_broker_pub_url, "YAML")
# dmcs consumer
dmcs_name = cdm[ROOT]["DMCS_BROKER_NAME"]
dmcs_pwd = cdm[ROOT]["DMCS_BROKER_PASSWD"]
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_pwd + "@" + \
broker_addr
self.dmcs_consumer = Consumer(dmcs_broker_url, "ocs_dmcs_consume",
"thread-dmcs-consume",
self.on_ocs_message, "YAML")
self.dmcs_consumer.start()
# ocs consumer from DMCS
ocs_name = cdm[ROOT]["OCS_BROKER_NAME"]
ocs_pwd = cdm[ROOT]["OCS_BROKER_PASSWD"]
# FIXME: New OCS account for consumer test_dmcs_ocs_publish
ocs_broker_url = "amqp://" + "AFM" + ":" +\
"AFM" + "@" +\
broker_addr
self.ocs_consumer = Consumer(ocs_broker_url, "test_dmcs_ocs_publish",
"thread-ocs-consume",
self.on_dmcs_message, "YAML")
self.ocs_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority("../../messages.yaml")
self.send_messages()
sleep(10)
os.killpg(os.getpgid(self.cmdListener.pid), signal.SIGTERM)
os.killpg(os.getpgid(self.ackSubscriber.pid), signal.SIGTERM)
print("MY OCS MESSAGES: %s" % self.ocs_consumer_msg_list)
self.verify_ocs_messages()
print("Finished with CommandListener tests.")
def send_messages(self):
os.chdir("../commands/")
commands = [
"start", "stop", "enable", "disable", "enterControl",
"exitControl", "standby", "abort"
]
devices = [
"archiver", "catchuparchiver", "processingcluster", "atArchiver"
]
for device in devices:
for command in commands:
cmd = None
if command == "start":
cmd = "./sacpp_" + device + "_" + command + "_commander Normal"
else:
cmd = "./sacpp_" + device + "_" + command + "_commander 0"
p = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid)
print("=== " + device.upper() + " " + command.upper() +
" Message")
sleep(
10
) # this is not random. startup .sacpp_ thing takes about 7 seconds.
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
device_sh = ["AR", "CU", "PP", "AT"]
for device in device_sh:
my_dev = None
if device == "AR":
my_dev = "archiver"
elif device == "CU":
my_dev = "catchuparchiver"
elif device == "PP":
my_dev = "processingcluster"
elif device == "AT":
my_dev = "atArchiver"
cmd = "./sacpp_" + my_dev + "_SummaryState_log"
run = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid)
sleep(10)
msg = {}
msg["MSG_TYPE"] = "SUMMARY_STATE_EVENT"
msg["DEVICE"] = device
msg["CURRENT_STATE"] = 0
self.dmcs_publisher.publish_message("dmcs_ocs_publish", msg)
print("=== " + device.upper() + " SummaryState Event Message")
sleep(10)
os.killpg(os.getpgid(run.pid), signal.SIGTERM)
cmd1 = "./sacpp_" + my_dev + "_SettingVersions_log"
run1 = subprocess.Popen(cmd1, shell=True, preexec_fn=os.setsid)
sleep(10)
msg1 = {}
msg1["MSG_TYPE"] = "RECOMMENDED_SETTINGS_VERSION_EVENT"
msg1["DEVICE"] = device
msg1["CFG_KEY"] = "Normal"
self.dmcs_publisher.publish_message("dmcs_ocs_publish", msg1)
print("=== " + device.upper() +
" RecommendSettingsVersion Event Message")
sleep(10)
os.killpg(os.getpgid(run1.pid), signal.SIGTERM)
cmd2 = "./sacpp_" + my_dev + "_AppliedSettingsMatchStart_log"
run2 = subprocess.Popen(cmd2, shell=True, preexec_fn=os.setsid)
sleep(10)
msg2 = {}
msg2["MSG_TYPE"] = "APPLIED_SETTINGS_MATCH_START_EVENT"
msg2["DEVICE"] = device
msg2["SETTING"] = "Normal"
msg2["APPLIED"] = True
self.dmcs_publisher.publish_message("dmcs_ocs_publish", msg2)
print("=== " + device.upper() +
" AppliedSettingsMatchStart Event Message")
sleep(10)
os.killpg(os.getpgid(run2.pid), signal.SIGTERM)
cmd3 = "./sacpp_" + my_dev + "_ErrorCode_log"
run3 = subprocess.Popen(cmd3, shell=True, preexec_fn=os.setsid)
sleep(10)
msg3 = {}
msg3["MSG_TYPE"] = "ERROR_CODE_EVENT"
msg3["DEVICE"] = device
msg3["ERROR_CODE"] = 0
self.dmcs_publisher.publish_message("dmcs_ocs_publish", msg3)
print("=== " + device.upper() + " ErrorCode Event Message")
sleep(10)
os.killpg(os.getpgid(run3.pid), signal.SIGTERM)
sleep(20)
print("Message Sender Done.")
def verify_ocs_messages(self):
len_list = len(self.ocs_consumer_msg_list)
if len_list != self.EXPECTED_OCS_MESSAGES:
pytest.fail("OCS simulator received incorrect number of messages.\n Expected %s but received %s" \
% (self.EXPECTED_OCS_MESSAGES, len_list))
for i in range(0, len_list):
msg = self.ocs_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail(
"The following AckSubscriber response message failed when compared with the sovereign\
example: %s" % msg)
print("Responses to OCS pass verification")
def on_ocs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
msg = deepcopy(body)
msg_type = msg["MSG_TYPE"]
if msg_type == "START":
msg.pop("CFG_KEY", None) # pop CFG_KEY
msg["MSG_TYPE"] = msg_type + "_ACK"
msg["ACK_BOOL"] = True
msg["ACK_STATEMENT"] = "test"
self.dmcs_publisher.publish_message("dmcs_ocs_publish", msg)
def on_dmcs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.ocs_consumer_msg_list.append(body)
class NcsaForeman:
NCSA_CONSUME = "ncsa_consume"
NCSA_PUBLISH = "ncsa_publish"
COMPONENT_NAME = 'NCSA_FOREMAN'
DISTRIBUTOR_PUBLISH = "distributor_publish"
ACK_PUBLISH = "ack_publish"
CFG_FILE = 'L1SystemCfg.yaml'
prp = toolsmod.prp
def __init__(self, filename=None):
toolsmod.singleton(self)
self._config_file = self.CFG_FILE
if filename != None:
self._config_file = filename
#self._pairing_dict = {}
LOGGER.info('Extracting values from Config dictionary')
self.extract_config_values()
self._msg_actions = {
'NCSA_NEXT_VISIT': self.set_visit,
'NCSA_NEW_SESSION': self.set_session,
'NCSA_START_INTEGRATION': self.process_start_integration,
'NCSA_READOUT': self.process_readout,
'DISTRIBUTOR_HEALTH_CHECK_ACK': self.process_ack,
'DISTRIBUTOR_XFER_PARAMS_ACK': self.process_ack,
'DISTRIBUTOR_READOUT_ACK': self.process_ack
}
self._next_timed_ack_id = 10000
self.setup_publishers()
self.setup_scoreboards()
self.setup_publishers()
self.setup_consumer_threads()
LOGGER.info('Ncsa foreman consumer setup')
self.thread_manager = None
self.setup_consumer_threads()
LOGGER.info('Ncsa Foreman Init complete')
def setup_publishers(self):
self._pub_base_broker_url = "amqp://" + self._pub_base_name + ":" + \
self._pub_base_passwd + "@" + \
str(self._base_broker_addr)
self._pub_ncsa_broker_url = "amqp://" + self._pub_ncsa_name + ":" + \
self._pub_ncsa_passwd + "@" + \
str(self._ncsa_broker_addr)
LOGGER.info('Setting up Base publisher on %s using %s', \
self._pub_base_broker_url, self._base_msg_format)
self._base_publisher = SimplePublisher(self._pub_base_broker_url,
self._base_msg_format)
LOGGER.info('Setting up NCSA publisher on %s using %s', \
self._pub_ncsa_broker_url, self._ncsa_msg_format)
self._ncsa_publisher = SimplePublisher(self._pub_ncsa_broker_url,
self._ncsa_msg_format)
def on_pp_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
msg_dict = body
LOGGER.debug('Message from PP callback message body is: %s',
self.prp.pformat(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def on_ack_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
msg_dict = body
LOGGER.info('In ACK message callback')
LOGGER.debug('Message from ACK callback message body is: %s',
self.prp.pformat(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def set_visit(self, params):
bore_sight = params['BORE_SIGHT']
visit_id = params['VISIT_ID']
self.JOB_SCBD.set_visit_id(visit_id, bore_sight)
ack_id = params['ACK_ID']
msg = {}
###
### Send Boresight to Someone here...
###
msg['MSG_TYPE'] = 'NCSA_NEXT_VISIT_ACK'
msg['COMPONENT'] = self.COMPONENT_NAME
msg['ACK_ID'] = ack_id
msg['ACK_BOOL'] = True
route_key = params['REPLY_QUEUE']
self._base_publisher.publish_message(route_key, msg)
def set_session(self, params):
self.JOB_SCBD.set_session(params['SESSION_ID'])
ack_id = params['ACK_ID']
msg = {}
msg['MSG_TYPE'] = 'NCSA_NEW_SESSION_ACK'
msg['COMPONENT'] = self.COMPONENT_NAME
msg['ACK_ID'] = ack_id
msg['ACK_BOOL'] = True
route_key = params['REPLY_QUEUE']
self._base_publisher.publish_message(route_key, msg)
def process_start_integration(self, params):
job_num = str(params[JOB_NUM])
image_id = params['IMAGE_ID']
visit_id = params['VISIT_ID']
response_timed_ack_id = params["ACK_ID"]
LOGGER.info('NCSA received Start Integration message from Base')
LOGGER.debug('NCSA Start Integration incoming message: %s' % params)
forwarders_list = params['FORWARDERS']['FORWARDER_LIST']
ccd_list = params['FORWARDERS']['CCD_LIST'] # A list of lists...
len_forwarders_list = len(forwarders_list)
self.JOB_SCBD.add_job(job_num, image_id, visit_id, ccd_list)
LOGGER.info('Received new job %s. Needed workers is %s', job_num,
str(len_forwarders_list))
# run distributor health check
# get timed_ack_id
timed_ack = self.get_next_timed_ack_id("DISTRIBUTOR_HEALTH_CHECK_ACK")
distributors = self.DIST_SCBD.return_distributors_list()
# Mark all healthy distributors Unknown
state_unknown = {"STATE": "HEALTH_CHECK", "STATUS": "UNKNOWN"}
self.DIST_SCBD.set_distributor_params(distributors, state_unknown)
# send health check messages
ack_params = {}
ack_params[MSG_TYPE] = "DISTRIBUTOR_HEALTH_CHECK"
ack_params['REPLY_QUEUE'] = 'ncsa_foreman_ack_publish'
ack_params["ACK_ID"] = timed_ack
ack_params[JOB_NUM] = job_num
for distributor in distributors:
self._ncsa_publisher.publish_message(
self.DIST_SCBD.get_value_for_distributor(
distributor, "CONSUME_QUEUE"), ack_params)
# start timers
self.ack_timer(2)
# at end of timer, get list of distributors
dicts_of_distributors = self.ACK_SCBD.get_components_for_timed_ack(
timed_ack)
healthy_distributors = list(dicts_of_distributors.keys())
# update distributor scoreboard with healthy distributors
healthy_status = {"STATUS": "HEALTHY"}
self.DIST_SCBD.set_distributor_params(healthy_distributors,
healthy_status)
num_healthy_distributors = len(healthy_distributors)
if len_forwarders_list > num_healthy_distributors:
print("Cannot Do Job - more fwdrs than dists")
# send response msg to base refusing job
LOGGER.info(
'Reporting to base insufficient healthy distributors for job #%s',
job_num)
ncsa_params = {}
ncsa_params[MSG_TYPE] = "NCSA_RESOURCES_QUERY_ACK"
ncsa_params[JOB_NUM] = job_num
ncsa_params["ACK_BOOL"] = False
ncsa_params["ACK_ID"] = response_timed_ack_id
self._base_publisher.publish_message(NCSA_PUBLISH,
yaml.dump(ncsa_params))
# delete job and leave distributors in Idle state
self.JOB_SCBD.delete_job(job_num)
idle_state = {"STATE": "IDLE"}
self.DIST_SCBD.set_distributor_params(healthy_distributors,
idle_state)
else:
Pairs = self.assemble_pairs(forwarders_list, ccd_list,
healthy_distributors)
self.JOB_SCBD.set_pairs_for_job(job_num, Pairs)
# send pair info to each distributor
job_params_ack = self.get_next_timed_ack_id(
'DISTRIBUTOR_XFER_PARAMS_ACK')
for j in range(0, len(Pairs)):
tmp_msg = {}
tmp_msg[MSG_TYPE] = 'DISTRIBUTOR_XFER_PARAMS'
tmp_msg['XFER_PARAMS'] = Pairs[j]
tmp_msg[JOB_NUM] = job_num
tmp_msg[ACK_ID] = job_params_ack
tmp_msg['REPLY_QUEUE'] = 'ncsa_foreman_ack_publish'
tmp_msg['VISIT_ID'] = visit_id
tmp_msg['IMAGE_ID'] = image_id
fqn = Pairs[j]['DISTRIBUTOR']['FQN']
route_key = self.DIST_SCBD.get_value_for_distributor(
fqn, 'CONSUME_QUEUE')
self._ncsa_publisher.publish_message(route_key, tmp_msg)
self.DIST_SCBD.set_distributor_params(healthy_distributors,
{STATE: IN_READY_STATE})
dist_params_response = self.progressive_ack_timer(
job_params_ack, num_healthy_distributors, 2.0)
if dist_params_response == None:
print(
"RECEIVED NO ACK RESPONSES FROM DISTRIBUTORS AFTER SENDING XFER PARAMS"
)
pass #Do something such as raise a system wide exception
# Now inform PP Foreman that all is in ready state
ncsa_params = {}
ncsa_params[MSG_TYPE] = "NCSA_START_INTEGRATION_ACK"
ncsa_params[JOB_NUM] = job_num
ncsa_params['IMAGE_ID'] = image_id
ncsa_params['VISIT_ID'] = visit_id
ncsa_params['SESSION_ID'] = params['SESSION_ID']
ncsa_params['COMPONENT'] = 'NCSA_FOREMAN'
ncsa_params[ACK_BOOL] = True
ncsa_params["ACK_ID"] = response_timed_ack_id
ncsa_params["PAIRS"] = Pairs
self._base_publisher.publish_message(params['REPLY_QUEUE'],
ncsa_params)
LOGGER.info(
'Sufficient distributors and workers are available. Informing Base'
)
LOGGER.debug('NCSA Start Integration incoming message: %s' %
ncsa_params)
LOGGER.info(
'The following pairings have been sent to the Base for job %s:'
% job_num)
LOGGER.info(Pairs)
def assemble_pairs(self, forwarders_list, ccd_list, healthy_distributors):
#build dict...
PAIRS = []
for i in range(0, len(forwarders_list)):
tmp_dict = {}
sub_dict = {}
tmp_dict['FORWARDER'] = forwarders_list[i]
tmp_dict['CCD_LIST'] = ccd_list[i]
tmp_dict['DISTRIBUTOR'] = {}
distributor = healthy_distributors[i]
sub_dict['FQN'] = distributor
sub_dict['HOSTNAME'] = self.DIST_SCBD.get_value_for_distributor(
distributor, HOSTNAME)
sub_dict['NAME'] = self.DIST_SCBD.get_value_for_distributor(
distributor, NAME)
sub_dict['IP_ADDR'] = self.DIST_SCBD.get_value_for_distributor(
distributor, IP_ADDR)
sub_dict['TARGET_DIR'] = self.DIST_SCBD.get_value_for_distributor(
distributor, TARGET_DIR)
tmp_dict['DISTRIBUTOR'] = sub_dict
PAIRS.append(tmp_dict)
return PAIRS
def process_readout(self, params):
job_number = params[JOB_NUM]
response_ack_id = params[ACK_ID]
pairs = self.JOB_SCBD.get_pairs_for_job(job_number)
sleep(3)
len_pairs = len(pairs)
ack_id = self.get_next_timed_ack_id(DISTRIBUTOR_READOUT_ACK)
# The following line extracts the distributor FQNs from pairs dict using
# list comprehension values; faster than for loops
# distributors = [v['FQN'] for v in list(pairs.values())]
for i in range(0, len_pairs): # Pairs is a list of dictionaries
distributor = pairs[i]['DISTRIBUTOR']['FQN']
msg_params = {}
msg_params[MSG_TYPE] = DISTRIBUTOR_READOUT
msg_params[JOB_NUM] = job_number
msg_params['REPLY_QUEUE'] = 'ncsa_foreman_ack_publish'
msg_params[ACK_ID] = ack_id
routing_key = self.DIST_SCBD.get_routing_key(distributor)
self.DIST_SCBD.set_distributor_state(distributor, 'START_READOUT')
self._ncsa_publisher.publish_message(routing_key, msg_params)
distributor_responses = self.progressive_ack_timer(
ack_id, len_pairs, 24)
if distributor_responses != None:
RESULT_LIST = {}
CCD_LIST = []
RECEIPT_LIST = []
ncsa_params = {}
ncsa_params[MSG_TYPE] = NCSA_READOUT_ACK
ncsa_params[JOB_NUM] = job_number
ncsa_params['IMAGE_ID'] = params['IMAGE_ID']
ncsa_params['VISIT_ID'] = params['VISIT_ID']
ncsa_params['SESSION_ID'] = params['SESSION_ID']
ncsa_params['COMPONENT'] = 'NCSA_FOREMAN'
ncsa_params[ACK_ID] = response_ack_id
ncsa_params[ACK_BOOL] = True
distributors = list(distributor_responses.keys())
for dist in distributors:
ccd_list = distributor_responses[dist]['RESULT_LIST'][
'CCD_LIST']
receipt_list = distributor_responses[dist]['RESULT_LIST'][
'RECEIPT_LIST']
for i in range(0, len(ccd_list)):
CCD_LIST.append(ccd_list[i])
RECEIPT_LIST.append(receipt_list[i])
RESULT_LIST['CCD_LIST'] = CCD_LIST
RESULT_LIST['RECEIPT_LIST'] = RECEIPT_LIST
ncsa_params['RESULT_LIST'] = RESULT_LIST
self._base_publisher.publish_message(params['REPLY_QUEUE'],
msg_params)
else:
ncsa_params = {}
ncsa_params[MSG_TYPE] = NCSA_READOUT_ACK
ncsa_params[JOB_NUM] = job_number
ncsa_params['COMPONENT_NAME'] = NCSA_FOREMAN
ncsa_params['IMAGE_ID'] = params['IMAGE_ID']
ncsa_params['VISIT_ID'] = params['VISIT_ID']
ncsa_params['SESSION_ID'] = params['SESSION_ID']
ncsa_params[ACK_ID] = response_ack_id
ncsa_params[ACK_BOOL] = FALSE
ncsa_params['RESULT_LIST'] = {}
ncsa_params['RESULT_LIST']['CCD_LIST'] = None
ncsa_params['RESULT_LIST']['RECEIPT_LIST'] = None
self._base_publisher.publish_message(params['REPLY_QUEUE'],
msg_params)
def process_ack(self, params):
self.ACK_SCBD.add_timed_ack(params)
def get_next_timed_ack_id(self, ack_type):
self._next_timed_ack_id = self._next_timed_ack_id + 1
retval = ack_type + "_" + str(self._next_timed_ack_id).zfill(6)
return retval
def ack_timer(self, seconds):
sleep(seconds)
return True
def progressive_ack_timer(self, ack_id, expected_replies, seconds):
counter = 0.0
while (counter < seconds):
counter = counter + 0.5
sleep(0.5)
response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if response == None:
continue
if len(list(response.keys())) == expected_replies:
return response
## Try one final time
response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if response == None:
return None
elif len(list(response.keys())) == expected_replies:
return response
else:
return None
def extract_config_values(self):
try:
cdm = toolsmod.intake_yaml_file(self._config_file)
except IOError as e:
LOGGER.critical("Unable to find CFG Yaml file %s\n" %
self._config_file)
sys.exit(101)
try:
self._base_broker_addr = cdm[ROOT][BASE_BROKER_ADDR]
self._ncsa_broker_addr = cdm[ROOT][NCSA_BROKER_ADDR]
self._sub_base_name = cdm[ROOT][
'NFM_BASE_BROKER_NAME'] # Message broker user & passwd
self._sub_base_passwd = cdm[ROOT]['NFM_BASE_BROKER_PASSWD']
self._sub_ncsa_name = cdm[ROOT][
'NFM_NCSA_BROKER_NAME'] # Message broker user & passwd
self._sub_ncsa_passwd = cdm[ROOT]['NFM_NCSA_BROKER_PASSWD']
self._pub_base_name = cdm[ROOT]['BASE_BROKER_PUB_NAME']
self._pub_base_passwd = cdm[ROOT]['BASE_BROKER_PUB_PASSWD']
self._pub_ncsa_name = cdm[ROOT]['NCSA_BROKER_PUB_NAME']
self._pub_ncsa_passwd = cdm[ROOT]['NCSA_BROKER_PUB_PASSWD']
self._scbd_dict = cdm[ROOT]['SCOREBOARDS']
self.distributor_dict = cdm[ROOT][XFER_COMPONENTS][DISTRIBUTORS]
except KeyError as e:
LOGGER.critical("CDM Dictionary error - missing Key")
LOGGER.critical("Offending Key is %s", str(e))
LOGGER.critical("Bailing Out...")
sys.exit(99)
self._base_msg_format = 'YAML'
self._ncsa_msg_format = 'YAML'
if 'BASE_MSG_FORMAT' in cdm[ROOT]:
self._base_msg_format = cdm[ROOT][BASE_MSG_FORMAT]
if 'NCSA_MSG_FORMAT' in cdm[ROOT]:
self._ncsa_msg_format = cdm[ROOT][NCSA_MSG_FORMAT]
def setup_consumer_threads(self):
LOGGER.info('Building _base_broker_url')
base_broker_url = "amqp://" + self._sub_base_name + ":" + \
self._sub_base_passwd + "@" + \
str(self._base_broker_addr)
ncsa_broker_url = "amqp://" + self._sub_ncsa_name + ":" + \
self._sub_ncsa_passwd + "@" + \
str(self._ncsa_broker_addr)
self.shutdown_event = threading.Event()
# Set up kwargs that describe consumers to be started
# The Archive Device needs three message consumers
kws = {}
md = {}
md['amqp_url'] = ncsa_broker_url
md['name'] = 'Thread-ncsa_foreman_ack_publish'
md['queue'] = 'ncsa_foreman_ack_publish'
md['callback'] = self.on_ack_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-ncsa_consume'
md['queue'] = 'ncsa_consume'
md['callback'] = self.on_pp_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
self.thread_manager = ThreadManager('thread-manager', kws,
self.shutdown_event)
self.thread_manager.start()
def setup_scoreboards(self):
# Create Redis Distributor table with Distributor info
self.DIST_SCBD = DistributorScoreboard('NCSA_DIST_SCBD', \
self._scbd_dict['NCSA_DIST_SCBD'], \
self.distributor_dict)
self.JOB_SCBD = JobScoreboard('NCSA_JOB_SCBD',
self._scbd_dict['NCSA_JOB_SCBD'])
self.ACK_SCBD = AckScoreboard('NCSA_ACK_SCBD',
self._scbd_dict['NCSA_ACK_SCBD'])
def shutdown(self):
LOGGER.debug("NCSA Foreman: Shutting down Consumer threads.")
self.shutdown_event.set()
LOGGER.debug("Thread Manager shutting down and app exiting...")
print("\n")
os._exit(0)
class TestAR_Ctrl:
arctrl = ArchiveController("/home/centos/src/git/ctrl_iip/python/lsst/iip/tests/yaml/L1SystemCfg_Test_ar-ctrl.yaml")
ar_consumer_msg_list = []
EXPECTED_AR_DEV_MESSAGES = 2
VISIT_ID = "V_1213"
IMAGE_ID = "IMG_13241"
SEND_TO_AR = "archive_ctrl_consume"
ACK_FROM_CTRL = "ar_foreman_ack_publish"
def test_ar_ctrl(self):
try:
cdm = toolsmod.intake_yaml_file('/home/centos/src/git/ctrl_iip/python/lsst/iip/tests/yaml/L1SystemCfg_Test_ar-ctrl.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
afm_name = cdm[ROOT]["AFM_BROKER_NAME"]
afm_pwd = cdm[ROOT]["AFM_BROKER_PASSWD"]
afm_pub_name = cdm[ROOT]["AFM_BROKER_PUB_NAME"]
afm_pub_pwd = cdm[ROOT]["AFM_BROKER_PUB_PASSWD"]
self.xfer_root = cdm[ROOT]["ARCHIVE"]["ARCHIVE_XFER_ROOT"]
ar_dev_pub_broker_url = "amqp://" + afm_pub_name + ":" +\
afm_pub_pwd + "@" +\
broker_addr
self.ar_dev_publisher = SimplePublisher(ar_dev_pub_broker_url, "YAML")
ar_dev_broker_url = "amqp://" + afm_name + ":" +\
afm_pwd + "@" +\
broker_addr
self.ar_dev_consumer = Consumer(ar_dev_broker_url, self.ACK_FROM_CTRL, "thread-ar-dev",
self.on_ar_message, "YAML", None)
self.ar_dev_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority('/home/centos/src/git/ctrl_iip/python/lsst/iip/messages.yaml')
self.send_messages()
sleep(10)
self.verify_ar_messages()
self.verify_path_files()
print("Finished with AR CTRL tests.")
def send_messages(self):
# msg = {}
# msg["MSG_TYPE"] = "ARCHIVE_HEALTH_CHECK"
# msg["SESSION_ID"] = "SI_314161"
# msg["ACK_ID"] = "AR_ACK_10001"
# self.ar_dev_publisher.publish_message(self.SEND_TO_AR, msg)
# print("ARCHIVE_HEALTH_CHECK Message")
# sleep(2)
msg = {}
msg["MSG_TYPE"] = "NEW_ARCHIVE_ITEM"
msg["ACK_ID"] = "NEW_AR_10002"
msg["JOB_NUM"] = "job_101"
msg["SESSION_ID"] = "SI_314162"
msg["VISIT_ID"] = self.VISIT_ID
msg["IMAGE_ID"] = self.IMAGE_ID
msg["REPLY_QUEUE"] = "ar_foreman_ack_publish"
self.ar_dev_publisher.publish_message(self.SEND_TO_AR, msg)
print("NEW_ARCHIVE_ITEM Message")
sleep(2)
msg = {}
msg["MSG_TYPE"] = "AR_ITEMS_XFERD"
msg["ACK_ID"] = "AR_ACK_10003"
msg["IMAGE_ID"] = "IMG_103"
msg["REPLY_QUEUE"] = "ar_foreman_ack_publish"
msg["RESULT_LIST"] = {}
msg["RESULT_LIST"]["CCD_LIST"] = ["101", "102"]
msg["RESULT_LIST"]["FILENAME_LIST"] = ["img1_ccd101", "img1_ccd102"]
msg["RESULT_LIST"]["CHECKSUM_LIST"] = ["F8DC17E983616D492A41344D736AD5D9", "99158C7DFD8351C9DEBC124029421993"]
self.ar_dev_publisher.publish_message(self.SEND_TO_AR, msg)
print("AR_ITEMS_XFERD Message")
sleep(2)
def clear_message_lists(self):
self.ar_consumer_msg_list = []
def verify_ar_messages(self):
len_list = len(self.ar_consumer_msg_list)
if len_list != self.EXPECTED_AR_DEV_MESSAGES:
pytest.fail("ArchiveController simulator received incorrect number of messages.\n" +
"Expected %s but received %s" % (self.EXPECTED_AR_DEV_MESSAGES, len_list))
for i in range(0, len_list):
msg = self.ar_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if not result:
pytest.fail("The following message to Archive Device failed when\
compared with the sovereign example: %s" % msg)
print("Message to ArchiveDevice pass verification")
def verify_path_files(self):
path = self.xfer_root + self.VISIT_ID + "/" + self.IMAGE_ID
if not os.path.isdir(path):
pytest.fail("The following directory was not created by Archive Controller \
during the NEW_ARCHIVE_ITEM phase: %s" % path)
print("Path creation during NEW_ARCHIVE_ITEM phase pass verification")
def on_ar_message(self, ch, method, properties, body):
# ch.basic_ack(method.delivery_tag)
self.ar_consumer_msg_list.append(body)
class Distributor:
"""This is a basic Distributor class. The cadence of the file
is very similar to its workmate the Forwarder class and begins to
viollate the DRY rule. It may be that this class and the
Forwarder class are eventually combined into single class so that
a personality can be chosen at the time of initialization. Or a
parent class for both may be a better approach... but at this
point, they are separate classes until it is certain that
individual classes are definetely not necessary.
"""
def __init__(self):
LOGGER.info("Initializing Distributor object")
self._registered = False
f = open('DistributorCfg.yaml')
# data map
cdm = yaml.safe_load(f)
try:
self._name = cdm[NAME]
self._passwd = cdm[PASSWD]
self._fqn_name = cdm[FQN]
self._ncsa_broker_addr = cdm[NCSA_BROKER_ADDR]
self._consume_queue = cdm[CONSUME_QUEUE]
self._publish_queue = cdm[PUBLISH_QUEUE]
self._hostname = cdm[HOSTNAME]
self._ip_addr = cdm[IP_ADDR]
self._target_dir = cdm[TARGET_DIR]
self._sentinel_file = cdm[SENTINEL_FILE]
except KeyError as e:
LOGGER.critical(e)
print("Key error reading cfg file.")
print("Bailing out...")
sys.exit(99)
self._home_dir = "/home/" + self._name + "/"
self._ncsa_broker_url = "amqp://" + self._name + ":" + self._passwd + "@" + str(
self._ncsa_broker_addr)
self._msg_actions = {
DISTRIBUTOR_HEALTH_CHECK: self.process_health_check,
DISTRIBUTOR_JOB_PARAMS: self.process_job_params,
DISTRIBUTOR_READOUT: self.process_foreman_readout
}
self.setup_publishers()
self.setup_consumers()
self._job_scratchpad = Scratchpad(self._ncsa_broker_url)
def setup_publishers(self):
LOGGER.info('Setting up publisher for Distributor on %s',
self._ncsa_broker_url)
self._publisher = SimplePublisher(self._ncsa_broker_url)
def setup_consumers(self):
LOGGER.info('Distributor %s setting up consumer on %s', self._name,
self._ncsa_broker_url)
LOGGER.info('Starting new thread on consumer method')
threadname = "thread-" + self._consume_queue
self._consumer = Consumer(self._ncsa_broker_url, self._consume_queue)
try:
_thread.start_new_thread(self.run_consumer, (
threadname,
2,
))
LOGGER.info('Started distributor consumer thread %s', threadname)
except:
LOGGER.critical(
'Cannot start Distributor consumer thread, exiting...')
sys.exit(107)
def run_consumer(self, threadname, delay):
self._consumer.run(self.on_message)
def on_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
msg_dict = yaml.load(body)
LOGGER.info('In %s message callback', self._name)
LOGGER.debug('Thread in %s callback is %s', self._name,
_thread.get_ident())
LOGGER.debug('%s callback message body is: %s', self._name,
str(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def process_health_check(self, params):
job_number = params[JOB_NUM]
self._job_scratchpad.set_job_value(job_number, "STATE", "ADD_JOB")
self._job_scratchpad.set_job_value(job_number, "ADD_JOB_TIME",
get_timestamp())
self.send_ack_response("DISTRIBUTOR_HEALTH_ACK", params)
def process_job_params(self, params):
transfer_params = params[TRANSFER_PARAMS]
self._job_scratchpad.set_job_transfer_params(params[JOB_NUM],
transfer_params)
self._job_scratchpad.set_job_value(job_number, "STATE",
"READY_WITH_PARAMS")
self._job_scratchpad.set_job_value(job_number,
"READY_WITH_PARAMS_TIME",
get_timestamp())
self.send_ack_response(DISTRIBUTOR_JOB_PARAMS_ACK, params)
def process_foreman_readout(self, params):
LOGGER.info('At Top of Distributor readout')
job_number = params[JOB_NUM]
cmd = self._target_dir + "check_sentinel.sh"
result = subprocess.check_output(cmd, shell=True)
LOGGER.info('check_sentinel test is complete')
# xfer complete
#xfer_time = ""
"""
###########XXXXXXXXXXXXXXX###############
#### Checking for and processing image file goes here
"""
command = "cat " + self._target_dir + "rcv_logg.test"
cat_result = subprocess.check_output(command, shell=True)
#filename = self._target_dir + "rcv_logg.test"
#f = open(filename, 'r')
#for line in f:
# xfer_time = xfer_time + line + "\n"
msg = {}
msg[MSG_TYPE] = 'XFER_TIME'
msg[NAME] = self._name
msg[JOB_NUM] = job_number
msg['COMPONENT'] = "DISTRIBUTOR"
msg['COMMENT1'] = "Result from xfer command is: %s" % result
msg['COMMENT2'] = "cat_result is --> %s" % cat_result
msg['COMMENT3'] = "Command used to call check_sentinel.sh is %s" % cmd
self._publisher.publish_message("reports", yaml.dump(msg))
readout_dict = {}
readout_dict[MSG_TYPE] = "DISTRIBUTOR_READOUT_ACK"
readout_dict[JOB_NUM] = params[JOB_NUM]
readout_dict["COMPONENT"] = self._fqn_name
readout_dict["ACK_BOOL"] = True
readout_dict["ACK_ID"] = params["TIMED_ACK_ID"]
self._publisher.publish_message(self._publish_queue,
yaml.dump(readout_dict))
def send_ack_response(self, type, params):
timed_ack = params.get("TIMED_ACK_ID")
job_num = params.get(JOB_NUM)
if timed_ack is None:
LOGGER.info('%s failed, missing TIMED_ACK_ID', type)
elif job_num is None:
LOGGER.info('%s failed, missing JOB_NUM for ACK ID: %s', type)
else:
msg_params = {}
msg_params[MSG_TYPE] = type
msg_params[JOB_NUM] = job_num
msg_params[NAME] = "DISTRIBUTOR_" + self._name
msg_params[ACK_BOOL] = "TRUE"
msg_params[TIMED_ACK] = timed_ack
self._publisher.publish_message("reports", yaml.dump(msg_params))
LOGGER.info('%s sent for ACK ID: %s and JOB_NUM: %s', type,
timed_ack, job_num)
def register(self):
pass
def test_dmcs(self, Dmcs):
self.dmcs = Dmcs
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
ocs_name = cdm[ROOT]['OCS_BROKER_NAME']
ocs_passwd = cdm[ROOT]['OCS_BROKER_PASSWD']
ocs_pub_name = cdm[ROOT]['OCS_BROKER_PUB_NAME']
ocs_pub_passwd = cdm[ROOT]['OCS_BROKER_PUB_PASSWD']
ocs_broker_url = "amqp://" + ocs_name + ":" + \
ocs_passwd + "@" + \
broker_addr
self. ocs_pub_broker_url = "amqp://" + ocs_pub_name + ":" + \
ocs_pub_passwd + "@" + \
broker_addr
self.ocs_publisher = SimplePublisher(self.ocs_pub_broker_url, "YAML")
ar_name = cdm[ROOT]['AFM_BROKER_NAME']
ar_passwd = cdm[ROOT]['AFM_BROKER_PASSWD']
ar_pub_name = cdm[ROOT]['AFM_BROKER_PUB_NAME']
ar_pub_passwd = cdm[ROOT]['AFM_BROKER_PUB_PASSWD']
ar_broker_url = "amqp://" + ar_name + ":" + \
ar_passwd + "@" + \
broker_addr
self.ar_pub_broker_url = "amqp://" + ar_pub_name + ":" + \
ar_pub_passwd + "@" + \
broker_addr
self.ar_publisher = SimplePublisher(self.ar_pub_broker_url, "YAML")
pp_name = cdm[ROOT]['PFM_BROKER_NAME']
pp_passwd = cdm[ROOT]['PFM_BROKER_PASSWD']
pp_pub_name = cdm[ROOT]['PFM_BROKER_PUB_NAME']
pp_pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD']
pp_broker_url = "amqp://" + pp_name + ":" + \
pp_passwd + "@" + \
broker_addr
self.pp_pub_broker_url = "amqp://" + pp_pub_name + ":" + \
pp_pub_passwd + "@" + \
broker_addr
self.pp_publisher = SimplePublisher(self.pp_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.ocs_consumer = Consumer(ocs_broker_url,'dmcs_ocs_publish', 'thread-ocs',
self.on_ocs_message,'YAML')
self.ocs_consumer.start()
self.ar_consumer = Consumer(ar_broker_url,'ar_foreman_consume', 'thread-ar',
self.on_ar_message,'YAML')
self.ar_consumer.start()
self.pp_consumer = Consumer(pp_broker_url,'pp_foreman_consume', 'thread-pp',
self.on_pp_message,'YAML')
self.pp_consumer.start()
### call message sender and pass in ocs_publisher
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(3)
self.verify_ocs_messages()
sleep(3)
self.verify_ar_messages()
sleep(3)
self.verify_pp_messages()
sleep(2)
self.ocs_consumer.stop()
self.ocs_consumer.join()
self.ar_consumer.stop()
self.ar_consumer.join()
self.pp_consumer.stop()
self.pp_consumer.join()
print("Finished with DMCS AR and PP tests.")
class TestArDev:
dmcs_pub_broker_url = None
dmcs_publisher = None
dmcs_consumer = None
dmcs_consumer_msg_list = []
ar_ctrl_pub_broker_url = None
ar_ctrl_publisher = None
ar_ctrl_consumer = None
ar_ctrl_consumer_msg_list = []
F1_pub_broker_url = None
F1_publisher = None
F1_consumer = None
f1_consumer_msg_list = []
F2_pub_broker_url = None
F2_publisher = None
F2_consumer = None
f2_consumer_msg_list = []
EXPECTED_AR_CTRL_MESSAGES = 1
EXPECTED_DMCS_MESSAGES = 1
EXPECTED_F1_MESSAGES = 1
EXPECTED_F2_MESSAGES = 1
NUM_READOUTS = 0
ccd_list = [14,17,21.86]
prp = toolsmod.prp
DP = toolsmod.DP # Debug Printing either True or False...override for this file only...
def test_ardev(self, Ardev):
self.ardev = Ardev
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test_ar.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
raise
raft_dict = cdm[ROOT]['DEFAULT_RAFT_CONFIGURATION']
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
dmcs_name = cdm[ROOT]['DMCS_BROKER_NAME']
dmcs_passwd = cdm[ROOT]['DMCS_BROKER_PASSWD']
dmcs_pub_name = cdm[ROOT]['DMCS_BROKER_PUB_NAME']
dmcs_pub_passwd = cdm[ROOT]['DMCS_BROKER_PUB_PASSWD']
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_passwd + "@" + \
broker_addr
dmcs_pub_broker_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % dmcs_pub_broker_url)
self.dmcs_publisher = SimplePublisher(dmcs_pub_broker_url, "YAML")
ar_ctrl_name = cdm[ROOT]['ARCHIVE_BROKER_NAME']
ar_ctrl_passwd = cdm[ROOT]['ARCHIVE_BROKER_PASSWD']
ar_ctrl_pub_name = cdm[ROOT]['ARCHIVE_BROKER_PUB_NAME']
ar_ctrl_pub_passwd = cdm[ROOT]['ARCHIVE_BROKER_PUB_PASSWD']
ar_ctrl_broker_url = "amqp://" + ar_ctrl_name + ":" + \
ar_ctrl_passwd + "@" + \
broker_addr
ar_ctrl_pub_broker_url = "amqp://" + ar_ctrl_pub_name + ":" + \
ar_ctrl_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % ar_ctrl_pub_broker_url)
self.ar_ctrl_publisher = SimplePublisher(ar_ctrl_pub_broker_url, "YAML")
F1_name = 'F1'
F1_passwd = 'F1'
F1_pub_name = 'F1_PUB'
F1_pub_passwd = 'F1_PUB'
F1_broker_url = "amqp://" + F1_name + ":" + \
F1_passwd + "@" + \
broker_addr
F1_pub_broker_url = "amqp://" + F1_pub_name + ":" + \
F1_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % F1_pub_broker_url)
self.F1_publisher = SimplePublisher(F1_pub_broker_url, "YAML")
F2_name = 'F2'
F2_passwd = 'F2'
F2_pub_name = 'F2_PUB'
F2_pub_passwd = 'F2_PUB'
F2_broker_url = "amqp://" + F2_name + ":" + \
F2_passwd + "@" + \
broker_addr
F2_pub_broker_url = "amqp://" + F2_pub_name + ":" + \
F2_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % F2_pub_broker_url)
self.F2_publisher = SimplePublisher(F2_pub_broker_url, "YAML")
print("All publishers are running...")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
print("MessageAuthority running...")
self.dmcs_consumer = Consumer(dmcs_broker_url,'dmcs_ack_consume', 'thread-dmcs',
self.on_dmcs_message,'YAML')
self.dmcs_consumer.start()
print("DMCS Consumer running...")
self.ar_ctrl_consumer = Consumer(ar_ctrl_broker_url,'archive_ctrl_consume', 'thread-ar-ctrl',
self.on_ar_ctrl_message,'YAML')
self.ar_ctrl_consumer.start()
print("ar_ctrl Consumer running...")
self.F1_consumer = Consumer(F1_broker_url,'f1_consume', 'thread-f1',
self.on_f1_message,'YAML')
self.F1_consumer.start()
print("F1 Consumer running...")
self.F2_consumer = Consumer(F2_broker_url,'f2_consume', 'thread-f2',
self.on_f2_message,'YAML')
self.F2_consumer.start()
print("F2 Consumer running...")
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(8)
self.verify_dmcs_messages()
self.verify_ar_ctrl_messages()
self.verify_F1_messages()
self.verify_F2_messages()
sleep(3)
# Shut down consumer threads nicely
self.dmcs_consumer.stop()
self.dmcs_consumer.join()
self.ar_ctrl_consumer.stop()
self.ar_ctrl_consumer.join()
self.F1_consumer.stop()
self.F1_consumer.join()
self.F2_consumer.stop()
self.F2_consumer.join()
if self.DP:
print("Finished with AR tests.")
def send_messages(self):
print("Starting send_messages")
# Tests only an AR device
self.EXPECTED_AR_CTRL_MESSAGES = 2
self.EXPECTED_DMCS_MESSAGES = 2
self.EXPECTED_F1_MESSAGES = 3
self.EXPECTED_F2_MESSAGES = 3
msg = {}
msg['MSG_TYPE'] = "AR_NEW_SESSION"
msg['SESSION_ID'] = 'SI_469976'
msg['ACK_ID'] = 'NEW_SESSION_ACK_44221'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
time.sleep(2)
print("New Session Message")
self.dmcs_publisher.publish_message("ar_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_NEXT_VISIT"
msg['VISIT_ID'] = 'XX_28272'
msg['JOB_NUM'] = '4xx72'
msg['SESSION_ID'] = 'SI_469976'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'NEW_VISIT_ACK_76'
msg['RA'] = "231.221"
msg['DEC'] = "-45.34"
msg['ANGLE'] = "120.0"
msg['RAFT_LIST'] = ['10','32','41','42','43']
msg['RAFT_CCD_LIST'] = [['ALL'],['02','11','12'],['00','02'],['02','12','11','22','00'],['ALL']]
time.sleep(2)
print("Next Visit Message")
self.dmcs_publisher.publish_message("ar_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_TAKE_IMAGES"
msg['JOB_NUM'] = '4xx72'
msg['NUM_IMAGES'] = '4'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'ACK_44221'
time.sleep(2)
print("AR Take Images Message")
self.dmcs_publisher.publish_message("ar_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_END_READOUT"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444245'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'AR_ACK_94671'
time.sleep(2)
print("AR END READOUT Message")
self.dmcs_publisher.publish_message("ar_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_END_READOUT"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444246'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'AR_ACK_94673'
time.sleep(2)
print("AR END READOUT Message")
self.dmcs_publisher.publish_message("ar_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_END_READOUT"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444247'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'AR_ACK_94676'
time.sleep(2)
print("AR END READOUT Message")
self.dmcs_publisher.publish_message("ar_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_END_READOUT"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444248'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'AR_ACK_94677'
time.sleep(2)
print("AR END READOUT Message")
self.dmcs_publisher.publish_message("ar_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_TAKE_IMAGES_DONE"
msg['JOB_NUM'] = '4xx72'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'ACK_44221'
time.sleep(2)
print("AR Take Images Done Message")
self.dmcs_publisher.publish_message("ar_foreman_consume", msg)
time.sleep(9)
print("Message Sender done")
def verify_dmcs_messages(self):
len_list = len(self.dmcs_consumer_msg_list)
print("DMCS RECEIVED %s MESSAGES" % len_list)
if len_list != self.EXPECTED_DMCS_MESSAGES:
print("Incorrect number of DMCS messages received")
pytest.fail('DMCS simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_DMCS_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.dmcs_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
print("This DMCS message failed verification: ")
self.prp.pprint(msg)
pytest.fail("The following DMCS Bridge response message failed when compared with the sovereign example: %s" % msg)
print("Responses to DMCS Bridge pass verification.")
def verify_ar_ctrl_messages(self):
len_list = len(self.ar_ctrl_consumer_msg_list)
print("AR_CTRL RECEIVED %s MESSAGES" % len_list)
if len_list != self.EXPECTED_AR_CTRL_MESSAGES:
print("Incorrect number of AR_CTRL messages received")
pytest.fail('AR CTRL simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_AR_CTRL_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.ar_ctrl_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
print("This AR_CTRL message failed verification: ")
self.prp.pprint(msg)
pytest.fail("The following message to the AR CTRL failed when compared with the sovereign example: %s" % msg)
print("Messages to the AR CTRL pass verification.")
def verify_F1_messages(self):
len_list = len(self.f1_consumer_msg_list)
print("F1 RECEIVED %s MESSAGES" % len_list)
if len_list != self.EXPECTED_F1_MESSAGES:
print("Incorrect number of F1 messages received")
pytest.fail('F1 simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_F1_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.f1_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
print("This F1 message failed verification: ")
self.prp.pprint(msg)
pytest.fail("The following message to F1 failed when compared with the sovereign example: %s" % msg)
print("Messages to F1 pass verification.")
def verify_F2_messages(self):
len_list = len(self.f2_consumer_msg_list)
print("F2 RECEIVED %s MESSAGES" % len_list)
if len_list != self.EXPECTED_F2_MESSAGES:
print("Incorrect number of F2 messages received")
pytest.fail('F2 simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_F2_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.f2_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
print("This F2 message failed verification: ")
self.prp.pprint(msg)
pytest.fail("The following message to F2 failed when compared with the sovereign example: %s" % msg)
print("Messages to F2 pass verification.")
def on_dmcs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.dmcs_consumer_msg_list.append(body)
def on_ar_ctrl_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.ar_ctrl_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'NEW_ARCHIVE_ITEM':
msg = {}
msg['MSG_TYPE'] = 'NEW_ARCHIVE_ITEM_ACK'
msg['COMPONENT'] = 'ARCHIVE_CTRL'
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
msg['TARGET_LOCATION'] = '/tmp'
self.ar_ctrl_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'AR_ITEMS_XFERD':
msg = {}
msg['MSG_TYPE'] = 'AR_ITEMS_XFERD_ACK'
msg['COMPONENT'] = 'ARCHIVE_CTRL'
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
filename_list = body['RESULT_SET']['FILENAME_LIST']
msg['RESULT_SET'] = {}
msg['RESULT_SET']['IMAGE_ID_LIST'] = body['RESULT_SET']['IMAGE_ID_LIST']
msg['RESULT_SET']['RECEIPT_LIST'] = []
msg['RESULT_SET']['FILENAME_LIST'] = filename_list
RECEIPT_LIST = []
for filename in filename_list:
RECEIPT_LIST.append('x14_' + str(filename))
msg['RESULT_LIST']['RECEIPT_LIST'] = RECEIPT_LIST
self.ar_ctrl_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail("The following unknown message was received by the Archive CTRL: %s" % body)
def on_f1_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.f1_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'AR_FWDR_HEALTH_CHECK':
msg = {}
msg['MSG_TYPE'] = 'AR_FWDR_HEALTH_CHECK_ACK'
msg['COMPONENT'] = 'FORWARDER_1'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.F1_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'AR_FWDR_XFER_PARAMS':
msg = {}
msg['MSG_TYPE'] = 'AR_FWDR_XFER_PARAMS_ACK'
msg['COMPONENT'] = 'FORWARDER_1'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.F1_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'AR_FWDR_TAKE_IMAGES':
# This message handler is not necessary as it does nothing
# But it is explanatory in nature for understanding/maintaining the file.
#
# No ack necessary - but NUM_IMAGES param will be
# needed in AR_FWDR_TAKE_IMAGES_DONE message handler below
pass
elif body['MSG_TYPE'] == 'AR_FWDR_END_READOUT':
self.NUM_READOUTS = self.NUM_READOUTS + 1
elif body['MSG_TYPE'] == 'AR_FWDR_TAKE_IMAGES_DONE':
# Find message in message list for xfer_params
xfer_msg = None
image_id_list = []
num_images = 0
for msg in self.f1_consumer_msg_list:
if msg['MSG_TYPE'] == 'AR_FWDR_END_READOUT':
image_id_list.append(msg['IMAGE_ID'])
if msg['MSG_TYPE'] == 'AR_FWDR_XFER_PARAMS':
xfer_msg = msg
if msg['MSG_TYPE'] == 'AR_FWDR_TAKE_IMAGES':
num_images = int(msg['NUM_IMAGES'])
if xfer_msg == None:
pytest.fail("The AR_FWDR_XFER_PARAMS message was not received before AR_FWDR_READOUT in F1")
# use message to build response
msg = {}
msg['MSG_TYPE'] = 'AR_FWDR_TAKE_IMAGES_DONE_ACK'
msg['COMPONENT'] = 'FORWARDER_1'
msg['JOB_NUM'] = xfer_msg['JOB_NUM']
msg['ACK_ID'] = body['ACK_ID']
raft_list = xfer_msg['XFER_PARAMS']['RAFT_LIST']
raft_ccd_list = xfer_msg['XFER_PARAMS']['RAFT_CCD_LIST']
msg['RESULT_SET'] = {}
msg['RESULT_SET']['RAFT_LIST'] = raft_list
msg['RESULT_SET']['RAFT_CCD_LIST'] = raft_ccd_list
msg['RESULT_SET']['RAFT_PLUS_CCD_LIST'] = []
msg['RESULT_SET']['FILENAME_LIST'] = []
msg['RESULT_SET']['CHECKSUM_LIST'] = []
RAFT_PLUS_CCD_LIST = []
FILENAME_LIST = []
CHECKSUM_LIST = []
target_location = xfer_msg['TARGET_LOCATION']
raft_plus_ccd_list = self.convert_raft_and_ccd_list_to_name_list(raft_list, raft_ccd_list)
for ccd in raft_plus_ccd_list:
RAFT_PLUS_CCD_LIST.append(ccd)
### XXX ADD IMAGE_ID from IMAGE_ID_LIST to target_dir and ccd name
FILENAME_LIST.append(target_location + "/" + str(ccd))
CHECKSUM_LIST.append('XXXXFFFF4444$$$$')
msg['RESULT_SET']['RAFT_PLUS_CCD_LIST'] = RAFT_PLUS_CCD_LIST
msg['RESULT_SET']['FILENAME_LIST'] = FILENAME_LIST
msg['RESULT_SET']['CHECKSUM_LIST'] = CHECKSUM_LIST
if num_images == self.NUM_READOUTS:
msg['ACK_BOOL'] = True
else:
msg['ACK_BOOL'] = False
self.F1_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail("The following unknown message was received by FWDR F1: %s" % body)
def on_f2_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.f2_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'AR_FWDR_HEALTH_CHECK':
msg = {}
msg['MSG_TYPE'] = 'AR_FWDR_HEALTH_CHECK_ACK'
msg['COMPONENT'] = 'FORWARDER_2'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.F2_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'AR_FWDR_XFER_PARAMS':
msg = {}
msg['MSG_TYPE'] = 'AR_FWDR_XFER_PARAMS_ACK'
msg['COMPONENT'] = 'FORWARDER_2'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.F2_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'AR_FWDR_TAKE_IMAGES':
# This message handler is not necessary as it does nothing
# But it is explanatory in nature for understanding/maintaining the file.
#
# No ack necessary - but NUM_IMAGES param will be
# needed in AR_FWDR_TAKE_IMAGES_DONE message handler below
pass
elif body['MSG_TYPE'] == 'AR_FWDR_END_READOUT':
pass
elif body['MSG_TYPE'] == 'AR_FWDR_TAKE_IMAGES_DONE':
# Find message in message list for xfer_params
xfer_msg = None
image_id_list = []
num_images = 0;
for msg in self.f2_consumer_msg_list:
if msg['MSG_TYPE'] == 'AR_FWDR_END_READOUT':
image_id_list.append(msg['IMAGE_ID'])
if msg['MSG_TYPE'] == 'AR_FWDR_XFER_PARAMS':
xfer_msg = msg
if msg['MSG_TYPE'] == 'AR_FWDR_TAKE_IMAGES':
num_images = msg['NUM_IMAGES']
if xfer_msg == None:
pytest.fail("The AR_FWDR_XFER_PARAMS message was not received before AR_FWDR_READOUT in F1")
# use message to build response
msg = {}
msg['MSG_TYPE'] = 'AR_FWDR_TAKE_IMAGES_DONE_ACK'
msg['COMPONENT'] = 'FORWARDER_2'
msg['JOB_NUM'] = xfer_msg['JOB_NUM']
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
raft_list = xfer_msg['XFER_PARAMS']['RAFT_LIST']
raft_ccd_list = xfer_msg['XFER_PARAMS']['RAFT_CCD_LIST']
msg['RESULT_SET'] = {}
msg['RESULT_SET']['RAFT_LIST'] = raft_list
msg['RESULT_SET']['RAFT_CCD_LIST'] = raft_ccd_list
msg['RESULT_SET']['RAFT_PLUS_CCD_LIST'] = []
msg['RESULT_SET']['FILENAME_LIST'] = []
msg['RESULT_SET']['CHECKSUM_LIST'] = []
RAFT_PLUS_CCD_LIST = []
FILENAME_LIST = []
CHECKSUM_LIST = []
target_location = xfer_msg['TARGET_LOCATION']
raft_plus_ccd_list = self.convert_raft_and_ccd_list_to_name_list(raft_list, raft_ccd_list)
for ccd in raft_plus_ccd_list:
RAFT_PLUS_CCD_LIST.append(ccd)
### XXX ADD IMAGE_ID from IMAGE_ID_LIST to target_location and ccd name
FILENAME_LIST.append(target_location + "/" + str(ccd))
CHECKSUM_LIST.append('XXXXFFFF4444$$$$')
msg['RESULT_SET']['RAFT_PLUS_CCD_LIST'] = RAFT_PLUS_CCD_LIST
msg['RESULT_SET']['FILENAME_LIST'] = FILENAME_LIST
msg['RESULT_SET']['CHECKSUM_LIST'] = CHECKSUM_LIST
self.F2_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail("The following unknown message was received by FWDR F2: %s" % body)
def convert_raftdict_to_name_list(self, rdict):
raft_list = list(rdict.keys())
num_rafts = len(raft_list)
integrated_names_list = []
for i in range(0,num_rafts):
current_raft = raft_list[i]
ccd_list = []
ccd_list = rdict[current_raft]
if ccd_list[0] == 'ALL':
ccd_list = ['00','10','20','01','11','21','02','12','22']
num_current_ccds = len(ccd_list)
for j in range(0,num_current_ccds):
tmp_str = current_raft + '-' + ccd_list[j]
integrated_names_list.append(tmp_str)
return integrated_names_list
def convert_raft_and_ccd_list_to_name_list(self, raft_list, raft_ccd_list):
#raft_list = list(rdict.keys())
num_rafts = len(raft_list)
integrated_names_list = []
for i in range(0,num_rafts):
current_raft = raft_list[i]
ccd_list = []
ccd_list = raft_ccd_list[i]
if ccd_list[0] == 'ALL':
ccd_list = ['00','10','20','01','11','21','02','12','22']
num_current_ccds = len(ccd_list)
for j in range(0,num_current_ccds):
tmp_str = current_raft + '-' + ccd_list[j]
integrated_names_list.append(tmp_str)
return integrated_names_list
def main():
premium = Premium()
#sp1 = SimplePublisher('amqp://*****:*****@141.142.238.160:5672/%2Fbunny?heartbeat=300', "YAML")
sp1 = SimplePublisher('amqp://*****:*****@141.142.238.160:5672/%2Fbunny', "YAML")
#sp2 = SimplePublisher('amqp://*****:*****@141.142.208.191:5672/%2Ftester')
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#cons = Consumer(broker_url, 'F8_consume')
#try:
# thread.start_new_thread( do_it, ("thread-1", 2,) )
#except:
# print "Cannot start thread"
# while 1:
msg = {}
msg['MSG_TYPE'] = "STANDBY"
msg['DEVICE'] = 'AR'
time.sleep(29)
sp1.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "DISABLE"
msg['DEVICE'] = 'AR'
time.sleep(45)
sp1.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "ENABLE"
msg['DEVICE'] = 'AR'
time.sleep(74)
sp1.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "WORKING"
msg['DEVICE'] = 'AR'
time.sleep(150)
sp1.publish_message("ocs_dmcs_consume", msg)
time.sleep(5)
"""
msg = {}
msg['MSG_TYPE'] = "NEXT_VISIT"
msg['VISIT_ID'] = 'XX_28272'
msg['BORE_SIGHT'] = 'A LITTLE TO THE LEFT'
time.sleep(4)
sp1.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "START_INTEGRATION"
msg['IMAGE_ID'] = 'IMG_444244'
msg['DEVICE'] = 'AR'
time.sleep(4)
sp1.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "READOUT"
msg['IMAGE_ID'] = 'IMG_444244'
msg['DEVICE'] = 'AR'
time.sleep(4)
sp1.publish_message("ocs_dmcs_consume", msg)
"""
print("Sender done")
import copy
import sys
sys.path.append("../ctrl_iip/python/lsst/iip/")
from SimplePublisher import SimplePublisher
from Consumer import Consumer
pub = SimplePublisher(
"amqp://*****:*****@140.252.32.128:5672/%2ftest_at")
def on_message(ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
msg = copy.deepcopy(body)
msg_type = body["MSG_TYPE"] + "_ACK"
msg["MSG_TYPE"] = msg_type
msg["ACK_BOOL"] = 1
msg["ACK_STATEMENT"] = "everything went well"
pub.publish_message("dmcs_ocs_publish", msg)
sub = Consumer("amqp://*****:*****@140.252.32.128:5672/%2ftest_at",
"ocs_dmcs_consume", "hello", on_message, "YAML")
sub.run()
class AuxDevice:
""" The Spec Device is a commandable device which coordinates the ingest of
images from the telescope camera and then the transfer of those images to
the base site archive storage.
It receives jobs and divides and assigns the work to forwarders, records state and
status change of forwarders, and sends messages accordingly.
"""
COMPONENT_NAME = 'AUX_FOREMAN'
AT_FOREMAN_CONSUME = "at_foreman_consume"
ARCHIVE_CTRL_PUBLISH = "archive_ctrl_publish"
ARCHIVE_CTRL_CONSUME = "archive_ctrl_consume"
AT_FOREMAN_ACK_PUBLISH = "at_foreman_ack_publish"
START_INTEGRATION_XFER_PARAMS = {}
ACK_QUEUE = []
CFG_FILE = 'L1SystemCfg.yaml'
prp = toolsmod.prp
DP = toolsmod.DP
RAFT_LIST = []
RAFT_CCD_LIST = ['00']
def __init__(self, filename=None):
""" Create a new instance of the Spectrograph Device class.
Instantiate the instance, raise assertion error if already instantiated.
Extract config values from yaml file.
Store handler methods for each message type.
Set up base broker url, publishers, and scoreboards. Consumer threads
are started within a Thread Manager object so that they can be monitored
for health and shutdown/joined cleanly when the app exits.
:params filename: Deflaut 'L1SystemCfg.yaml'. Can be assigned by user.
:return: None.
"""
toolsmod.singleton(self)
self._config_file = self.CFG_FILE
if filename != None:
self._config_file = filename
LOGGER.info('Extracting values from Config dictionary')
self.extract_config_values()
#self.purge_broker(cdm['ROOT']['QUEUE_PURGES'])
self._msg_actions = {
'AT_START_INTEGRATION': self.process_at_start_integration,
'AT_NEW_SESSION': self.set_session,
#'AR_READOUT': self.process_dmcs_readout,
'AUX_FWDR_HEALTH_CHECK_ACK': self.process_ack,
'AUX_FWDR_XFER_PARAMS_ACK': self.process_ack,
'AR_FWDR_READOUT_ACK': self.process_ack,
'AR_ITEMS_XFERD_ACK': self.process_ack,
'AT_HEADER_READY': self.process_header_ready_event,
'NEW_ARCHIVE_ITEM_ACK': self.process_ack,
#'AUX_TAKE_IMAGES': self.take_images,
'AT_END_READOUT': self.process_at_end_readout
}
self._next_timed_ack_id = 0
self.setup_publishers()
LOGGER.info('ar foreman consumer setup')
self.thread_manager = None
self.setup_consumer_threads()
LOGGER.info('Archive Foreman Init complete')
def setup_publishers(self):
""" Set up base publisher with pub_base_broker_url by creating a new instance
of SimplePublisher class with yaml format
:params: None.
:return: None.
"""
self.pub_base_broker_url = "amqp://" + self._msg_pub_name + ":" + \
self._msg_pub_passwd + "@" + \
str(self._base_broker_addr)
LOGGER.info('Setting up Base publisher on %s using %s',
self.pub_base_broker_url, self._base_msg_format)
self._publisher = SimplePublisher(self.pub_base_broker_url,
self._base_msg_format)
def on_aux_foreman_message(self, ch, method, properties, body):
""" Calls the appropriate AR message action handler according to message type.
:params ch: Channel to message broker, unused unless testing.
:params method: Delivery method from Pika, unused unless testing.
:params properties: Properties from DMCS to AR Foreman callback message
body, unused unless testing.
:params body: A dictionary that stores the message body.
:return: None.
"""
#msg_dict = yaml.load(body)
ch.basic_ack(method.delivery_tag)
msg_dict = body
LOGGER.info('In AUX Foreman message callback')
LOGGER.info(
'Message from DMCS to AUX Foreman callback message body is: %s',
str(msg_dict))
print("Incoming AUX msg is: %s" % msg_dict)
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def on_archive_message(self, ch, method, properties, body):
""" Calls the appropriate AR message action handler according to message type.
:params ch: Channel to message broker, unused unless testing.
:params method: Delivery method from Pika, unused unless testing.
:params properties: Properties from AR CTRL callback message body,
unused unless testing.
:params body: A dictionary that stores the message body.
:return: None.
"""
ch.basic_ack(method.delivery_tag)
LOGGER.info('AR CTRL callback msg body is: %s', str(body))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def on_ack_message(self, ch, method, properties, body):
""" Calls the appropriate AR message action handler according to message type.
:params ch: Channel to message broker, unused unless testing.
:params method: Delivery method from Pika, unused unless testing.
:params properties: Properties from ACK callback message body, unused
unless testing.
:params body: A dictionary that stores the message body.
:return: None.
"""
ch.basic_ack(method.delivery_tag)
msg_dict = body
print("")
print("")
print("")
print("RECEIVING ack MESSAGE:")
print(msg_dict)
print("")
print("")
print("")
# XXX FIX Ignoring all log messages
return
LOGGER.info('In ACK message callback')
LOGGER.info('Message from ACK callback message body is: %s',
str(msg_dict))
# XXX FIX Ignoring all log messages
return
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def process_at_start_integration(self, params):
# When this method is invoked, the following must happen:
# 1) Health check all forwarders
# 2) Divide work and generate dict of forwarders and which rafts/ccds they are fetching
# 3) Inform each forwarder which rafts they are responsible for
# receive new job_number and image_id; session and visit are current
# and deep copy it with some additions such as session and visit
# These next three lines must have WFS and Guide sensor info added
start_int_ack_id = params[ACK_ID]
print("Incoming AUX AT_Start Int msg")
# next, run health check
self.ACK_QUEUE = {}
health_check_ack_id = self.get_next_timed_ack_id('AUX_FWDR_HEALTH_ACK')
num_fwdrs_checked = self.fwdr_health_check(health_check_ack_id)
# Add job scbd entry
self.ack_timer(1.4)
#healthy_fwdrs = self.ACK_QUEUE.get_components_for_timed_ack(health_check_ack_id)
#if healthy_fwdrs == None:
# self.refuse_job(params, "No forwarders available")
# ### FIX send error code for this...
# return
fwdr_names = list(self._forwarder_dict.keys())
self._current_fwdr = self._forwarder_dict[fwdr_names[0]]
# Add archive check when necessary...
# send new_archive_item msg to archive controller
#start_int_params = {}
#ac_timed_ack = self.get_next_timed_ack_id('AUX_CTRL_NEW_ITEM')
#start_int_params[MSG_TYPE] = 'NEW_ARCHIVE_ITEM'
#start_int_params['ACK_ID'] = ac_timed_ack
#start_int_params['JOB_NUM'] = job_number
#start_int_params['SESSION_ID'] = session_id
#start_int_params['VISIT_ID'] = visit_id
#start_int_params['IMAGE_ID'] = image_id
#start_int_params['REPLY_QUEUE'] = self.AUX_FOREMAN_ACK_PUBLISH
#self.JOB_SCBD.set_job_state(job_number, 'AR_NEW_ITEM_QUERY')
#self._publisher.publish_message(self.ARCHIVE_CTRL_CONSUME, start_int_params)
#ar_response = self.progressive_ack_timer(ac_timed_ack, 1, 2.0)
#if ar_response == None:
# FIXME raise L1 exception and bail out
# print("B-B-BAD Trouble; no ar_response")
#target_dir = ar_response['ARCHIVE_CTRL']['TARGET_DIR']
target_dir = self.archive_xfer_root
#self.JOB_SCBD.set_job_params(job_number, {'STATE':'AR_NEW_ITEM_RESPONSE', 'TARGET_DIR': dir})
# divide image fetch across forwarders
#list_of_fwdrs = list(healthy_fwdrs.keys())
#work_schedule = self.divide_work(list_of_fwdrs, raft_list, raft_ccd_list)
# send target dir, and job, session,visit and work to do to healthy forwarders
#self.JOB_SCBD.set_value_for_job(job_number, 'STATE','SENDING_XFER_PARAMS')
#set_sched_result = self.JOB_SCBD.set_work_schedule_for_job(job_number, work_schedule)
#if set_sched_result == False:
# FIXME Raise L1 exception and bail
# print("BIG PROBLEM - CANNOT SET WORK SCHED IN SCBD")
xfer_params_ack_id = self.get_next_timed_ack_id("AT_FWDR_PARAMS_ACK")
fwdr_new_target_params = {}
fwdr_new_target_params['XFER_PARAMS'] = {}
fwdr_new_target_params[MSG_TYPE] = 'AT_FWDR_XFER_PARAMS'
#fwdr_new_target_params[SESSION_ID] = session_id
fwdr_new_target_params[IMAGE_ID] = params[IMAGE_ID]
fwdr_new_target_params['IMAGE_INDEX'] = params['IMAGE_INDEX']
#fwdr_new_target_params[VISIT_ID] = visit_id
#fwdr_new_target_params[JOB_NUM] = job_number
fwdr_new_target_params[ACK_ID] = xfer_params_ack_id
fwdr_new_target_params[REPLY_QUEUE] = self.AT_FOREMAN_ACK_PUBLISH
target_location = self.archive_name + "@" + self.archive_ip + ":" + target_dir
fwdr_new_target_params['TARGET_LOCATION'] = target_location
xfer_params_dict = {}
xfer_params_dict['RAFT_LIST'] = self._wfs_raft
#xfer_params_dict['RAFT_LIST'] = self.RAFT_LIST
#xfer_params_dict['RAFT_LIST'].append(self.RAFT_LIST)
#xfer_params_dict['RAFT_CCD_LIST'] = []
#xfer_params_dict['RAFT_CCD_LIST'].append(self.RAFT_CCD_LIST)
xfer_params_dict['AT_FWDR'] = self._current_fwdr
fwdr_new_target_params['XFER_PARAMS'] = xfer_params_dict
route_key = self._current_fwdr["CONSUME_QUEUE"]
self._publisher.publish_message(route_key, fwdr_new_target_params)
"""
# receive ack back from forwarders that they have job params
params_acks = self.progressive_ack_timer(xfer_params_ack_id, len_fwdrs_list, 3.0)
### FIX
# if params_acks == None:
# raise L1Exception and bail
self.JOB_SCBD.set_value_for_job(job_number,'STATE','XFER_PARAMS_SENT')
# accept job by Ach'ing True
st_int_params_ack = {}
st_int_params_ack['MSG_TYPE'] = 'AR_START_INTEGRATION_ACK'
st_int_params_ack['ACK_ID'] = start_int_ack_id
st_int_params_ack['ACK_BOOL'] = True
st_int_params_ack['JOB_NUM'] = job_number
st_int_params_ack['SESSION_ID'] = session_id
st_int_params_ack['IMAGE_ID'] = image_id
st_int_params_ack['VISIT_ID'] = visit_id
st_int_params_ack['COMPONENT'] = self.COMPONENT_NAME
self.accept_job(st_int_params_ack)
self.JOB_SCBD.set_value_for_job(job_number, STATE, "JOB_ACCEPTED")
fscbd_params = {'STATE':'AWAITING_READOUT'}
self.FWD_SCBD.set_forwarder_params(healthy_fwdrs, fscbd_params)
"""
def fwdr_health_check(self, ack_id):
""" Send AR_FWDR_HEALTH_CHECK message to ar_foreman_ack_publish queue.
Retrieve available forwarders from ForwarderScoreboard, set their state to
HEALTH_CHECK, status to UNKNOWN, and publish the message.
:params ack_id: Ack id for AR forwarder health check.
:return: Number of health checks sent.
"""
msg_params = {}
msg_params[MSG_TYPE] = 'AT_FWDR_HEALTH_CHECK'
msg_params[ACK_ID] = ack_id
msg_params[REPLY_QUEUE] = self.AT_FOREMAN_ACK_PUBLISH
forwarders = list(self._forwarder_dict.keys())
for x in range(0, len(forwarders)):
route_key = self._forwarder_dict[forwarders[x]]["CONSUME_QUEUE"]
self._publisher.publish_message(route_key, msg_params)
return len(forwarders)
def divide_work(self, fwdrs_list, raft_list, raft_ccd_list):
""" Divide work (ccds) among forwarders.
If only one forwarder available, give it all the work.
If have less or equal ccds then forwarders, give the first few forwarders one
ccd each.
Else, evenly distribute ccds among forwarders, and give extras to the first
forwarder, make sure that ccd list for each forwarder is continuous.
:params fwdrs_list: List of available forwarders for the job.
:params ccd_list: List of ccds to be distributed.
:return schedule: Distribution of ccds among forwarders.
"""
num_fwdrs = len(fwdrs_list)
num_rafts = len(raft_list)
schedule = {}
schedule['FORWARDER_LIST'] = []
schedule['CCD_LIST'] = [
] # A list of ccd lists; index of main list matches same forwarder list index
FORWARDER_LIST = []
RAFT_LIST = [] # This is a 'list of lists'
RAFT_CCD_LIST = [] # This is a 'list of lists'
if num_fwdrs == 1:
FORWARDER_LIST.append(fwdrs_list[0])
RAFT_LIST = deepcopy(raft_list)
RAFT_CCD_LIST = deepcopy(raft_ccd_list)
schedule['FORWARDER_LIST'] = FORWARDER_LIST
schedule['RAFT_LIST'] = RAFT_LIST
schedule['RAFT_CCD_LIST'] = RAFT_CCD_LIST
return schedule
if num_rafts <= num_fwdrs:
for k in range(0, num_rafts):
FORWARDER_LIST.append(fwdrs_list[k])
#little_list.append(ccd_list[k])
RAFT_LIST.append(raft_list[k]) # Need a copy here...
RAFT_CCD_LIST.append = deepcopy(raft_ccd_list[k])
schedule['FORWARDER_LIST'] = FORWARDER_LIST
schedule['RAFT_LIST'] = RAFT_LIST
schedule['RAFT_CCD_LIST'] = RAFT_CCD_LIST
else:
rafts_per_fwdr = len(raft_list) // num_fwdrs
remainder_rafts = len(raft_list) % num_fwdrs
offset = 0
for i in range(0, num_fwdrs):
tmp_list = []
tmp_raft_list = []
for j in range(offset, (rafts_per_fwdr + offset)):
if (j) >= num_rafts:
break
tmp_list.append(raft_list[j])
tmp_raft_list.append(deepcopy(raft_ccd_list[j]))
offset = offset + rafts_per_fwdr
# If num_fwdrs divided into num_rafts equally, we are done...else, deal with remainder
if remainder_rafts != 0 and i == 0:
for k in range(offset, offset + remainder_rafts):
tmp_list.append(raft_list[k])
tmp_raft_list.append(deepcopy(raft_ccd_list[k]))
offset = offset + remainder_rafts
FORWARDER_LIST.append(fwdrs_list[i])
RAFT_LIST.append(list(tmp_list))
RAFT_CCD_LIST.append(list(tmp_raft_list))
schedule['FORWARDER_LIST'] = FORWARDER_LIST
schedule['RAFT_LIST'] = RAFT_LIST
schedule['RAFT_CCD_LIST'] = RAFT_CCD_LIST
return schedule
def accept_job(self, dmcs_message):
""" Send AR_START_INTEGRATION_ACK message with ack_bool equals True (job accepted)
and other job specs to dmcs_ack_consume queue.
:params dmcs_message: A dictionary that stores info of a job.
:return: None.
"""
self._publisher.publish_message("dmcs_ack_consume", dmcs_message)
def refuse_job(self, params, fail_details):
""" Send AR_START_INTEGRATION_ACK message with ack_bool equals False (job refused)
and other job specs to dmcs_ack_consume queue.
Set job state as JOB_REFUSED in JobScoreboard.
:params parmas: A dictionary that stores info of a job.
:params fail_details: A string that describes what went wrong, not used for now.
:return: None.
"""
dmcs_message = {}
dmcs_message[JOB_NUM] = params[JOB_NUM]
dmcs_message[MSG_TYPE] = 'AR_START_INTEGRATION_ACK'
dmcs_message['ACK_ID'] = params['ACK_ID']
dmcs_message['SESSION_ID'] = params['SESSION_ID']
dmcs_message['VISIT_ID'] = params['VISIT_ID']
dmcs_message['IMAGE_ID'] = params['IMAGE_ID']
dmcs_message[ACK_BOOL] = False
dmcs_message['COMPONENT'] = self.COMPONENT_NAME
self.JOB_SCBD.set_value_for_job(params[JOB_NUM], STATE, "JOB_REFUSED")
self._publisher.publish_message("dmcs_ack_consume", dmcs_message)
def process_at_end_readout(self, params):
""" Set job state as PREPARE_READOUT in JobScoreboard.
Send readout to forwarders.
Set job state as READOUT_STARTED in JobScoreboard.
Wait to retrieve and process readout responses.
:params parmas: A dictionary that stores info of a job.
:return: None.
"""
print("Incoming AUX AT_END_READOUT msg")
reply_queue = params['REPLY_QUEUE']
readout_ack_id = params[ACK_ID]
#job_number = params[JOB_NUM]
image_id = params[IMAGE_ID]
# send readout to forwarders
#self.JOB_SCBD.set_value_for_job(job_number, 'STATE', 'READOUT')
fwdr_readout_ack = self.get_next_timed_ack_id("AR_FWDR_READOUT_ACK")
#work_schedule = self.JOB_SCBD.get_work_schedule_for_job(job_number)
current_fwdr = self._current_fwdr
msg = {}
msg[MSG_TYPE] = 'AT_FWDR_END_READOUT'
#msg[JOB_NUM] = job_number
msg[IMAGE_ID] = image_id
msg['IMAGE_INDEX'] = params['IMAGE_INDEX']
route_key = self._current_fwdr['CONSUME_QUEUE']
self._publisher.publish_message(route_key, msg)
#readout_responses = self.progressive_ack_timer(fwdr_readout_ack, len(fwdrs), 4.0)
# if readout_responses == None:
# raise L1 exception
#self.process_readout_responses(readout_ack_id, reply_queue, image_id, readout_responses)
def process_readout_responses(self, readout_ack_id, reply_queue, image_id,
readout_responses):
""" From readout_responses param, retrieve image_id and job_number, and create list of
ccd, filename, and checksum from all forwarders. Store into xfer_list_msg and
send to archive to confirm each file made it intact.
Send AR_READOUT_ACK message with results and ack_bool equals True to
dmcs_ack_comsume queue.
:params readout_ack_id: Ack id for AR_READOUT_ACK message.
:params image_id:
:params readout_responses: Readout responses from AckScoreboard.
:return: None.
"""
job_number = None
image_id = None
confirm_ack = self.get_next_timed_ack_id('AR_ITEMS_XFERD_ACK')
fwdrs = list(readout_responses.keys())
CCD_LIST = []
FILENAME_LIST = []
CHECKSUM_LIST = []
for fwdr in fwdrs:
ccds = readout_responses[fwdr]['RESULT_LIST']['CCD_LIST']
num_ccds = len(ccds)
fnames = readout_responses[fwdr]['RESULT_LIST']['FILENAME_LIST']
csums = readout_responses[fwdr]['RESULT_LIST']['CHECKSUM_LIST']
for i in range(0, num_ccds):
msg = {}
CCD_LIST.append(ccds[i])
FILENAME_LIST.append(fnames[i])
CHECKSUM_LIST.append(csums[i])
job_number = readout_responses[fwdr][JOB_NUM]
image_id = readout_responses[fwdr]['IMAGE_ID']
xfer_list_msg = {}
xfer_list_msg[MSG_TYPE] = 'AR_ITEMS_XFERD'
xfer_list_msg[ACK_ID] = confirm_ack
xfer_list_msg['IMAGE_ID'] = image_id
xfer_list_msg['REPLY_QUEUE'] = self.AR_FOREMAN_ACK_PUBLISH
xfer_list_msg['RESULT_LIST'] = {}
xfer_list_msg['RESULT_LIST']['CCD_LIST'] = CCD_LIST
xfer_list_msg['RESULT_LIST']['FILENAME_LIST'] = FILENAME_LIST
xfer_list_msg['RESULT_LIST']['CHECKSUM_LIST'] = CHECKSUM_LIST
self._publisher.publish_message(self.ARCHIVE_CTRL_CONSUME,
xfer_list_msg)
xfer_check_responses = self.progressive_ack_timer(confirm_ack, 1, 4.0)
# if xfer_check_responses == None:
# raise L1 exception and bail
results = xfer_check_responses['ARCHIVE_CTRL']['RESULT_LIST']
ack_msg = {}
ack_msg['MSG_TYPE'] = 'AR_READOUT_ACK'
ack_msg['JOB_NUM'] = job_number
ack_msg['COMPONENT'] = self.COMPONENT_NAME
ack_msg['ACK_ID'] = readout_ack_id
ack_msg['ACK_BOOL'] = True
ack_msg['RESULT_LIST'] = results
self._publisher.publish_message(reply_queue, ack_msg)
### FIXME Set state as complete for Job
def send_readout(self, params, fwdrs, readout_ack):
""" Send AR_FWDR_READOUT message to each forwarder working on the job with
ar_foreman_ack_publish queue as reply queue.
:params params: A dictionary that stores info of a job.
:params readout_ack: Ack id for AR_FWDR_READOUT message.
:return: None.
"""
ro_params = {}
job_number = params['JOB_NUM']
ro_params['MSG_TYPE'] = 'AR_FWDR_READOUT'
ro_params['JOB_NUM'] = job_number
ro_params['SESSION_ID'] = self.get_current_session()
ro_params['VISIT_ID'] = self.get_current_visit()
ro_params['IMAGE_ID'] = params['IMAGE_ID']
ro_params['ACK_ID'] = readout_ack
ro_params['REPLY_QUEUE'] = self.AR_FOREMAN_ACK_PUBLISH
for fwdr in fwdrs:
route_key = self.FWD_SCBD.get_value_for_forwarder(
fwdr, "CONSUME_QUEUE")
self._publisher.publish_message(route_key, ro_params)
def process_header_ready_event(self, params):
fname = params['FILENAME']
image_id = params['IMAGE_ID']
msg = {}
msg['MSG_TYPE'] = 'AT_FWDR_HEADER_READY'
msg['FILENAME'] = fname
msg['IMAGE_ID'] = image_id
#XXX FIX remove hard code queue
#route_key = self._current_fwdr['CONSUME_QUEUE']
route_key = "f99_consume"
self._publisher.publish_message(route_key, msg)
def take_images_done(self, params):
reply_queue = params['REPLY_QUEUE']
readout_ack_id = params[ACK_ID]
job_number = params[JOB_NUM]
self.JOB_SCBD.set_value_for_job(job_number, 'STATE',
'TAKE_IMAGES_DONE')
fwdr_readout_ack = self.get_next_timed_ack_id(
"AR_FWDR_TAKE_IMAGES_DONE_ACK")
work_schedule = self.JOB_SCBD.get_work_schedule_for_job(job_number)
fwdrs = work_schedule['FORWARDER_LIST']
len_fwdrs = len(fwdrs)
msg = {}
msg[MSG_TYPE] = 'AR_FWDR_TAKE_IMAGES_DONE'
msg[JOB_NUM] = job_number
msg[ACK_ID] = fwdr_readout_ack
for i in range(0, len_fwdrs):
route_key = self.FWDR_SCBD.get_value_for_forwarder(
fwdrs[i], 'CONSUME_QUEUE')
self._publisher.publish_message(route_key, msg)
### FIX Add Final Response to DMCS
def process_ack(self, params):
""" Add new ACKS for a particular ACK_ID to the Ack Scoreboards
where they are collated.
:params: New ack to be checked in.
:return: None.
"""
pass
#self.ACK_SCBD.add_timed_ack(params)
def get_next_timed_ack_id(self, ack_type):
""" Increment ack id by 1, and store it.
Return ack id with ack type as a string.
:params ack_type: Informational string to prepend Ack ID.
:return retval: String with ack type followed by next ack id.
"""
self._next_timed_ack_id = self._next_timed_ack_id + 1
return (ack_type + "_" + str(self._next_timed_ack_id).zfill(6))
def set_session(self, params):
pass
""" Record new session in JobScoreboard.
Send AR_NEW_SESSION_ACK message with ack_bool equals True to specified reply queue.
:params params: Dictionary with info about new session.
:return: None.
self.JOB_SCBD.set_session(params['SESSION_ID'])
ack_id = params['ACK_ID']
msg = {}
msg['MSG_TYPE'] = 'AR_NEW_SESSION_ACK'
msg['COMPONENT'] = self.COMPONENT_NAME
msg['ACK_ID'] = ack_id
msg['ACK_BOOL'] = True
route_key = params['REPLY_QUEUE']
self._publisher.publish_message(route_key, msg)
"""
def get_current_session(self):
""" Retreive current session from JobSocreboard.
:params: None.
:return: Current session returned by JobSocreboard.
"""
return self.JOB_SCBD.get_current_session()
def set_visit(self, params):
""" Set current visit_id in JobScoreboard.
Send AR_NEXT_VISIT_ACK message with ack_bool equals True to specified reply queue.
:params params: Message dictionary with info about new visit.
:return: None.
"""
bore_sight = params['BORE_SIGHT']
self.JOB_SCBD.set_visit_id(params['VISIT_ID'], bore_sight)
ack_id = params['ACK_ID']
msg = {}
## XXX FIXME Do something with the bore sight in params['BORE_SIGHT']
msg['MSG_TYPE'] = 'AR_NEXT_VISIT_ACK'
msg['COMPONENT'] = self.COMPONENT_NAME
msg['ACK_ID'] = ack_id
msg['ACK_BOOL'] = True
route_key = params['REPLY_QUEUE']
self._publisher.publish_message(route_key, msg)
def get_current_visit(self):
""" Retrieve current visit from JobSocreboard.
:params: None.
:return: Current visit returned by JobSocreboard.
"""
return self.JOB_SCBD.get_current_visit()
def ack_timer(self, seconds):
""" Sleeps for user-defined seconds.
:params seconds: Time to sleep in seconds.
:return: True.
"""
sleep(seconds)
return True
def progressive_ack_timer(self, ack_id, expected_replies, seconds):
""" Sleeps for user-defined seconds, or less if everyone has reported back in.
:params ack_id: Ack ID to wait for.
:params expected_replies: Number of components expected to ack..
:params seconds: Maximum time to wait in seconds.
:return: The dictionary that represents the responses from the components ack'ing.
Note: If only one component will ack, this method breaks out of its
loop after the one ack shows up - effectively beating the maximum
wait time.
"""
counter = 0.0
while (counter < seconds):
counter = counter + 0.5
sleep(0.5)
response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if response == None:
continue
if len(list(response.keys())) == expected_replies:
return response
## Try one final time
response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if response == None:
return None
elif len(list(response.keys())) == expected_replies:
return response
else:
return None
def extract_config_values(self):
""" Parse system config yaml file.
Throw error messages if Yaml file or key not found.
:params: None.
:return: True.
"""
LOGGER.info('Reading YAML Config file %s' % self._config_file)
try:
cdm = toolsmod.intake_yaml_file(self._config_file)
except IOError as e:
LOGGER.critical("Unable to find CFG Yaml file %s\n" %
self._config_file)
sys.exit(101)
try:
self._msg_name = cdm[ROOT][
'AUX_BROKER_NAME'] # Message broker user & passwd
self._msg_passwd = cdm[ROOT]['AUX_BROKER_PASSWD']
self._msg_pub_name = cdm[ROOT][
'AUX_BROKER_PUB_NAME'] # Message broker user & passwd
self._msg_pub_passwd = cdm[ROOT]['AUX_BROKER_PUB_PASSWD']
self._base_broker_addr = cdm[ROOT][BASE_BROKER_ADDR]
self._forwarder_dict = cdm[ROOT][XFER_COMPONENTS]['AUX_FORWARDERS']
self._wfs_raft = cdm[ROOT]['ATS']['WFS_RAFT']
# Placeholder until eventually worked out by Data Backbone team
self.archive_fqn = cdm[ROOT]['ARCHIVE']['ARCHIVE_NAME']
self.archive_name = cdm[ROOT]['ARCHIVE']['ARCHIVE_LOGIN']
self.archive_ip = cdm[ROOT]['ARCHIVE']['ARCHIVE_IP']
self.archive_xfer_root = cdm[ROOT]['ARCHIVE']['ARCHIVE_XFER_ROOT']
except KeyError as e:
print("Dictionary error")
print("Bailing out...")
sys.exit(99)
self._base_msg_format = 'YAML'
if 'BASE_MSG_FORMAT' in cdm[ROOT]:
self._base_msg_format = cdm[ROOT]['BASE_MSG_FORMAT']
def setup_consumer_threads(self):
""" Create ThreadManager object with base broker url and kwargs to setup consumers.
:params: None.
:return: None.
"""
base_broker_url = "amqp://" + self._msg_name + ":" + \
self._msg_passwd + "@" + \
str(self._base_broker_addr)
LOGGER.info('Building _base_broker_url. Result is %s', base_broker_url)
self.shutdown_event = threading.Event()
self.shutdown_event.clear()
# Set up kwargs that describe consumers to be started
# The Archive Device needs three message consumers
kws = {}
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-aux_foreman_consume'
md['queue'] = 'at_foreman_consume'
md['callback'] = self.on_aux_foreman_message
md['format'] = "YAML"
md['test_val'] = None
kws[md['name']] = md
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-at_foreman_ack_publish'
md['queue'] = 'at_foreman_ack_publish'
md['callback'] = self.on_ack_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-archive_ctrl_publish'
md['queue'] = 'archive_ctrl_publish'
md['callback'] = self.on_archive_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
self.thread_manager = ThreadManager('thread-manager', kws,
self.shutdown_event)
self.thread_manager.start()
def shutdown(self):
LOGGER.info("Shutting down Consumer threads.")
self.shutdown_event.set()
LOGGER.debug("Thread Manager shutting down and app exiting...")
print("\n")
os._exit(0)
def setup_publishers(self):
self._publisher = SimplePublisher(self._base_broker_url)
class TestOCS_AckSubscriber:
os.chdir("ocs/src")
ackSubscriber = subprocess.Popen("./AckSubscriber&", shell=True, preexec_fn=os.setsid)
print("Preparing ackSubscriber ...")
sleep(120)
cmdListener = subprocess.Popen("./CommandListener&", shell=True, preexec_fn=os.setsid)
print("Preparing cmdListener ...")
sleep(10)
EXPECTED_OCS_MESSAGES = 48
ocs_consumer_msg_list = []
def test_ocs_acksubscriber(self):
try:
cdm = toolsmod.intake_yaml_file("../../tests/yaml/L1SystemCfg_Test_ocs_bridge.yaml")
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to fine CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]["BASE_BROKER_ADDR"]
# dmcs publisher
dmcs_pub_name = cdm[ROOT]["DMCS_BROKER_PUB_NAME"]
dmcs_pub_pwd = cdm[ROOT]["DMCS_BROKER_PUB_PASSWD"]
dmcs_broker_pub_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_pwd + "@" + \
broker_addr
self.dmcs_publisher = SimplePublisher(dmcs_broker_pub_url, "YAML")
# dmcs consumer
dmcs_name = cdm[ROOT]["DMCS_BROKER_NAME"]
dmcs_pwd = cdm[ROOT]["DMCS_BROKER_PASSWD"]
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_pwd + "@" + \
broker_addr
self.dmcs_consumer = Consumer(dmcs_broker_url, "ocs_dmcs_consume", "thread-dmcs-consume",
self.on_ocs_message, "YAML")
self.dmcs_consumer.start()
# ocs consumer from DMCS
ocs_name = cdm[ROOT]["OCS_BROKER_NAME"]
ocs_pwd = cdm[ROOT]["OCS_BROKER_PASSWD"]
# FIXME: New OCS account for consumer test_dmcs_ocs_publish
ocs_broker_url = "amqp://" + "AFM" + ":" +\
"AFM" + "@" +\
broker_addr
self.ocs_consumer = Consumer(ocs_broker_url, "test_dmcs_ocs_publish", "thread-ocs-consume",
self.on_dmcs_message, "YAML")
self.ocs_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority("../../messages.yaml")
self.send_messages()
sleep(10)
os.killpg(os.getpgid(self.cmdListener.pid), signal.SIGTERM)
os.killpg(os.getpgid(self.ackSubscriber.pid), signal.SIGTERM)
print("MY OCS MESSAGES: %s" % self.ocs_consumer_msg_list)
self.verify_ocs_messages()
print("Finished with CommandListener tests.")
def send_messages(self):
os.chdir("../commands/")
commands = ["start", "stop", "enable", "disable", "enterControl", "exitControl", "standby", "abort"]
devices = ["archiver" , "catchuparchiver", "processingcluster", "atArchiver"]
for device in devices:
for command in commands:
cmd = None
if command == "start":
cmd = "./sacpp_" + device + "_" + command + "_commander Normal"
else:
cmd = "./sacpp_" + device + "_" + command + "_commander 0"
p = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid)
print("=== " + device.upper() + " " + command.upper() + " Message")
sleep(10) # this is not random. startup .sacpp_ thing takes about 7 seconds.
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
device_sh = ["AR", "CU", "PP", "AT"]
for device in device_sh:
my_dev = None
if device == "AR":
my_dev = "archiver"
elif device == "CU":
my_dev = "catchuparchiver"
elif device == "PP":
my_dev = "processingcluster"
elif device == "AT":
my_dev = "atArchiver"
cmd = "./sacpp_" + my_dev + "_SummaryState_log"
run = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid)
sleep(10)
msg = {}
msg["MSG_TYPE"] = "SUMMARY_STATE_EVENT"
msg["DEVICE"] = device
msg["CURRENT_STATE"] = 0
self.dmcs_publisher.publish_message("dmcs_ocs_publish", msg)
print("=== " + device.upper() + " SummaryState Event Message")
sleep(10)
os.killpg(os.getpgid(run.pid), signal.SIGTERM)
cmd1 = "./sacpp_" + my_dev + "_SettingVersions_log"
run1 = subprocess.Popen(cmd1, shell=True, preexec_fn=os.setsid)
sleep(10)
msg1 = {}
msg1["MSG_TYPE"] = "RECOMMENDED_SETTINGS_VERSION_EVENT"
msg1["DEVICE"] = device
msg1["CFG_KEY"] = "Normal"
self.dmcs_publisher.publish_message("dmcs_ocs_publish", msg1)
print("=== " + device.upper() + " RecommendSettingsVersion Event Message")
sleep(10)
os.killpg(os.getpgid(run1.pid), signal.SIGTERM)
cmd2 = "./sacpp_" + my_dev + "_AppliedSettingsMatchStart_log"
run2 = subprocess.Popen(cmd2, shell=True, preexec_fn=os.setsid)
sleep(10)
msg2 = {}
msg2["MSG_TYPE"] = "APPLIED_SETTINGS_MATCH_START_EVENT"
msg2["DEVICE"] = device
msg2["SETTING"] = "Normal"
msg2["APPLIED"] = True
self.dmcs_publisher.publish_message("dmcs_ocs_publish", msg2)
print("=== " + device.upper() + " AppliedSettingsMatchStart Event Message")
sleep(10)
os.killpg(os.getpgid(run2.pid), signal.SIGTERM)
cmd3 = "./sacpp_" + my_dev + "_ErrorCode_log"
run3 = subprocess.Popen(cmd3, shell=True, preexec_fn=os.setsid)
sleep(10)
msg3 = {}
msg3["MSG_TYPE"] = "ERROR_CODE_EVENT"
msg3["DEVICE"] = device
msg3["ERROR_CODE"] = 0
self.dmcs_publisher.publish_message("dmcs_ocs_publish", msg3)
print("=== " + device.upper() + " ErrorCode Event Message")
sleep(10)
os.killpg(os.getpgid(run3.pid), signal.SIGTERM)
sleep(20)
print("Message Sender Done.")
def verify_ocs_messages(self):
len_list = len(self.ocs_consumer_msg_list)
if len_list != self.EXPECTED_OCS_MESSAGES:
pytest.fail("OCS simulator received incorrect number of messages.\n Expected %s but received %s" \
% (self.EXPECTED_OCS_MESSAGES, len_list))
for i in range(0, len_list):
msg = self.ocs_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following AckSubscriber response message failed when compared with the sovereign\
example: %s" % msg)
print("Responses to OCS pass verification")
def on_ocs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
msg = deepcopy(body)
msg_type = msg["MSG_TYPE"]
if msg_type == "START":
msg.pop("CFG_KEY", None) # pop CFG_KEY
msg["MSG_TYPE"] = msg_type + "_ACK"
msg["ACK_BOOL"] = True
msg["ACK_STATEMENT"] = "test"
self.dmcs_publisher.publish_message("dmcs_ocs_publish", msg)
def on_dmcs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.ocs_consumer_msg_list.append(body)
def test_ncsa(self, Ncsa):
self.ncsa = Ncsa
try:
cdm = toolsmod.intake_yaml_file(
'./tests/yaml/L1SystemCfg_Test_ncsa.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
pp_name = cdm[ROOT]['PFM_BROKER_NAME']
pp_passwd = cdm[ROOT]['PFM_BROKER_PASSWD']
pp_pub_name = cdm[ROOT]['PFM_BROKER_PUB_NAME']
pp_pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD']
pp_broker_url = "amqp://" + pp_name + ":" + \
pp_passwd + "@" + \
broker_addr
pp_pub_broker_url = "amqp://" + pp_pub_name + ":" + \
pp_pub_passwd + "@" + \
broker_addr
self.pp_publisher = SimplePublisher(pp_pub_broker_url, "YAML")
D1_name = 'D1'
D1_passwd = 'D1'
D1_pub_name = 'D1_PUB'
D1_pub_passwd = 'D1_PUB'
D1_broker_url = "amqp://" + D1_name + ":" + \
D1_passwd + "@" + \
broker_addr
D1_pub_broker_url = "amqp://" + D1_pub_name + ":" + \
D1_pub_passwd + "@" + \
broker_addr
self.d1_publisher = SimplePublisher(D1_pub_broker_url, "YAML")
D2_name = 'D2'
D2_passwd = 'D2'
D2_pub_name = 'D2_PUB'
D2_pub_passwd = 'D2_PUB'
D2_broker_url = "amqp://" + D2_name + ":" + \
D2_passwd + "@" + \
broker_addr
D2_pub_broker_url = "amqp://" + D2_pub_name + ":" + \
D2_pub_passwd + "@" + \
broker_addr
self.d2_publisher = SimplePublisher(D2_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.pp_consumer = Consumer(pp_broker_url, 'pp_foreman_ack_publish',
'thread-pp', self.on_pp_message, 'YAML')
self.pp_consumer.start()
self.D1_consumer = Consumer(D1_broker_url, 'd1_consume', 'thread-d1',
self.on_d1_message, 'YAML')
self.D1_consumer.start()
self.D2_consumer = Consumer(D2_broker_url, 'd2_consume', 'thread-d2',
self.on_d2_message, 'YAML')
self.D2_consumer.start()
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(7)
self.verify_pp_messages()
self.verify_D2_messages()
self.verify_D1_messages()
sleep(2)
self.pp_consumer.stop()
self.pp_consumer.join()
self.D1_consumer.stop()
self.D1_consumer.join()
self.D2_consumer.stop()
self.D2_consumer.join()
if self.DP:
print("Finished with NCSA tests.")
class TestNcsa:
pp_pub_broker_url = None
pp_publisher = None
pp_consumer = None
pp_consumer_msg_list = []
D1_pub_broker_url = None
D1_publisher = None
D1_consumer = None
d1_consumer_msg_list = []
D2_pub_broker_url = None
D2_publisher = None
D2_consumer = None
d2_consumer_msg_list = []
EXPECTED_PP_MESSAGES = 1
EXPECTED_D1_MESSAGES = 1
EXPECTED_D2_MESSAGES = 1
ccd_list = [14,17,21,86]
prp = toolsmod.prp
DP = toolsmod.DP
def test_ncsa(self, Ncsa):
self.ncsa = Ncsa
try:
cdm = toolsmod.intake_yaml_file('./tests/yaml/L1SystemCfg_Test_ncsa.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
pp_name = cdm[ROOT]['PFM_BROKER_NAME']
pp_passwd = cdm[ROOT]['PFM_BROKER_PASSWD']
pp_pub_name = cdm[ROOT]['PFM_BROKER_PUB_NAME']
pp_pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD']
pp_broker_url = "amqp://" + pp_name + ":" + \
pp_passwd + "@" + \
broker_addr
pp_pub_broker_url = "amqp://" + pp_pub_name + ":" + \
pp_pub_passwd + "@" + \
broker_addr
self.pp_publisher = SimplePublisher(pp_pub_broker_url, "YAML")
D1_name = 'D1'
D1_passwd = 'D1'
D1_pub_name = 'D1_PUB'
D1_pub_passwd = 'D1_PUB'
D1_broker_url = "amqp://" + D1_name + ":" + \
D1_passwd + "@" + \
broker_addr
D1_pub_broker_url = "amqp://" + D1_pub_name + ":" + \
D1_pub_passwd + "@" + \
broker_addr
self.d1_publisher = SimplePublisher(D1_pub_broker_url, "YAML")
D2_name = 'D2'
D2_passwd = 'D2'
D2_pub_name = 'D2_PUB'
D2_pub_passwd = 'D2_PUB'
D2_broker_url = "amqp://" + D2_name + ":" + \
D2_passwd + "@" + \
broker_addr
D2_pub_broker_url = "amqp://" + D2_pub_name + ":" + \
D2_pub_passwd + "@" + \
broker_addr
self.d2_publisher = SimplePublisher(D2_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.pp_consumer = Consumer(pp_broker_url,'pp_foreman_ack_publish', 'thread-pp',
self.on_pp_message,'YAML')
self.pp_consumer.start()
self.D1_consumer = Consumer(D1_broker_url,'d1_consume', 'thread-d1',
self.on_d1_message,'YAML')
self.D1_consumer.start()
self.D2_consumer = Consumer(D2_broker_url,'d2_consume', 'thread-d2',
self.on_d2_message,'YAML')
self.D2_consumer.start()
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(7)
self.verify_pp_messages()
self.verify_D2_messages()
self.verify_D1_messages()
sleep(2)
self.pp_consumer.stop()
self.pp_consumer.join()
self.D1_consumer.stop()
self.D1_consumer.join()
self.D2_consumer.stop()
self.D2_consumer.join()
if self.DP:
print("Finished with NCSA tests.")
def send_messages(self):
print("Starting send_messages")
# Tests only an AR device
# self.clear_message_lists()
self.EXPECTED_PP_MESSAGES = 4
self.EXPECTED_D1_MESSAGES = 3
self.EXPECTED_D2_MESSAGES = 3
msg = {}
msg['MSG_TYPE'] = "NCSA_NEW_SESSION"
msg['SESSION_ID'] = 'SI_469976'
msg['ACK_ID'] = 'NCSA_NEW_SESSION_ACK_44221'
msg['REPLY_QUEUE'] = 'pp_foreman_ack_publish'
time.sleep(1)
if self.DP:
print("New Session Message")
self.pp_publisher.publish_message("ncsa_consume", msg)
msg = {}
msg['MSG_TYPE'] = "NCSA_NEXT_VISIT"
msg['VISIT_ID'] = 'XX_28272'
msg['SESSION_ID'] = 'NNV_469976'
msg['REPLY_QUEUE'] = 'pp_foreman_ack_publish'
msg['ACK_ID'] = 'NCSA_NEW_VISIT_ACK_76'
msg['BORE_SIGHT'] = "231,123786456342, -45.3457156906, FK5"
time.sleep(1)
if self.DP:
print("Next Visit Message")
self.pp_publisher.publish_message("ncsa_consume", msg)
msg = {}
msg['MSG_TYPE'] = "NCSA_START_INTEGRATION"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444244'
msg['VISIT_ID'] = 'V14494'
msg['SESSION_ID'] = '4_14_7211511'
msg['REPLY_QUEUE'] = 'pp_foreman_ack_publish'
msg['ACK_ID'] = 'PP_ACK_94671'
msg['CCD_LIST'] = [4,14,16,17,29,35,36]
time.sleep(2)
msg['FORWARDERS'] = {}
forwarder_list = ['FORWARDER_2', 'FORWARDER_1']
ccd_list = [[17,18,111,126],[128,131,132]]
msg['FORWARDERS']['FORWARDER_LIST'] = forwarder_list
msg['FORWARDERS']['CCD_LIST'] = ccd_list
if self.DP:
print("NCSA START INTEGRATION Message")
self.pp_publisher.publish_message("ncsa_consume", msg)
time.sleep(7)
msg = {}
msg['MSG_TYPE'] = "NCSA_READOUT"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444244'
msg['VISIT_ID'] = 'V14494'
msg['SESSION_ID'] = '4_14_7211511'
msg['REPLY_QUEUE'] = 'pp_foreman_ack_publish'
msg['ACK_ID'] = 'NCSA_READOUT_ACK_44221'
time.sleep(2)
if self.DP:
print("NCSA READOUT Message")
self.pp_publisher.publish_message("ncsa_consume", msg)
time.sleep(2)
print("Message Sender done")
def verify_pp_messages(self):
len_list = len(self.pp_consumer_msg_list)
if len_list != self.EXPECTED_PP_MESSAGES:
pytest.fail('PP simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_PP_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.pp_consumer_msg_list[i]
if msg['MSG_TYPE'] == 'NCSA_START_INTEGRATION_ACK':
result = self.check_start_int_ack(msg)
else:
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following message to the PP Foreman failed when compared with the sovereign example: %s" % msg)
if self.DP:
print("Messages to the PP Foreman pass verification.")
def check_start_int_ack(self, msg):
"""the PAIRS param in the message is a list. Every item in the list is a dictionary.
because it is not known how many entried will be on the list, the dictionaries
are deepcopied and checked against the MessageAuthority and check one at a time.
In the meanwhile, the shape of the incoming message without the dictionaries and the
PAIRS list set to None is checked against the MessageAuthority.
BTW, because the individual dictionaries do not have a MSG_TYPE, one is
added to each dictionary to be checked so the dict to compare to can be
located in the messages.yaml file.
"""
c_msg = deepcopy(msg)
pairs = deepcopy(c_msg['PAIRS'])
c_msg['PAIRS'] = None
overall_shape = self._msg_auth.check_message_shape(c_msg)
if overall_shape == False:
return False
for i in range (0, len(pairs)):
pairs[i]['MSG_TYPE'] = 'PAIR'
result = self._msg_auth.check_message_shape(pairs[i])
if result == False:
return False
return True
def verify_D1_messages(self):
len_list = len(self.d1_consumer_msg_list)
if len_list != self.EXPECTED_D1_MESSAGES:
if self.DP:
print("Messages received by verify_D1_messages:")
self.prp.pprint(self.f1_consumer_msg_list)
pytest.fail('F1 simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_D1_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.d1_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following message to D1 failed when compared with the sovereign example: %s" % msg)
if self.DP:
print("Messages to D1 pass verification.")
def verify_D2_messages(self):
len_list = len(self.d2_consumer_msg_list)
if len_list != self.EXPECTED_D2_MESSAGES:
if self.DP:
print("Messages received by verify_D2_messages:")
self.prp.pprint(self.d2_consumer_msg_list)
pytest.fail('D2 simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_D2_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.d2_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following message to D2 failed when compared with the sovereign example: %s" % msg)
if self.DP:
print("Messages to D2 pass verification.")
def on_pp_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_ncsa - incoming on_pp_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.pp_consumer_msg_list.append(body)
def on_d1_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_ncsa - incoming on_D1_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.d1_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'DISTRIBUTOR_HEALTH_CHECK':
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_HEALTH_CHECK_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_1'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.d1_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'DISTRIBUTOR_XFER_PARAMS':
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_XFER_PARAMS_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_1'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.d1_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'DISTRIBUTOR_READOUT':
# Find message in message list for xfer_params
xfer_msg = None
for msg in self.d1_consumer_msg_list:
if msg['MSG_TYPE'] == 'DISTRIBUTOR_XFER_PARAMS':
xfer_msg = msg
break
if xfer_msg == None:
pytest.fail("The DISTRIBUTOR_XFER_PARAMS message was not received before DISTRIBUTOE_READOUT in D1")
# use message to build response
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_READOUT_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_1'
msg['JOB_NUM'] = xfer_msg['JOB_NUM']
msg['IMAGE_ID'] = xfer_msg['IMAGE_ID']
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
msg['RESULT_LIST'] = {}
msg['RESULT_LIST']['CCD_LIST'] = []
msg['RESULT_LIST']['RECEIPT_LIST'] = []
ccd_list = xfer_msg['XFER_PARAMS']['CCD_LIST']
receipt_list = []
for i in range(0, len(ccd_list)):
receipt_list.append('F1_Rec_x477_' + str(i))
msg['RESULT_LIST']['RECEIPT_LIST'] = receipt_list
msg['RESULT_LIST']['CCD_LIST'] = list(ccd_list)
self.d1_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail("The following unknown message was received by D1: %s" % body)
def on_d2_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_ncsa - incoming on_D2_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.d2_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'DISTRIBUTOR_HEALTH_CHECK':
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_HEALTH_CHECK_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_2'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.d2_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'DISTRIBUTOR_XFER_PARAMS':
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_XFER_PARAMS_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_2'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.d2_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'DISTRIBUTOR_READOUT':
# Find message in message list for xfer_params
xfer_msg = None
for msg in self.d2_consumer_msg_list:
if msg['MSG_TYPE'] == 'DISTRIBUTOR_XFER_PARAMS':
xfer_msg = msg
break
if xfer_msg == None:
pytest.fail("The DISTRIBUTOR_XFER_PARAMS message was not received before DISTRIBUTOR_READOUT in D2")
# use message to build response
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_READOUT_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_2'
msg['JOB_NUM'] = xfer_msg['JOB_NUM']
msg['IMAGE_ID'] = xfer_msg['IMAGE_ID']
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
msg['RESULT_LIST'] = {}
msg['RESULT_LIST']['CCD_LIST'] = []
msg['RESULT_LIST']['RECEIPT_LIST'] = []
ccd_list = xfer_msg['XFER_PARAMS']['CCD_LIST']
receipt_list = []
for i in range(0, len(ccd_list)):
receipt_list.append('F2_Rec_x447_' + str(i))
msg['RESULT_LIST']['RECEIPT_LIST'] = receipt_list
msg['RESULT_LIST']['CCD_LIST'] = list(ccd_list)
self.d2_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail("The following unknown message was received by D2: %s" % body)
class PromptProcessDevice:
PP_JOB_SCBD = None
PP_FWD_SCBD = None
PP_ACK_SCBD = None
COMPONENT_NAME = 'PROMPT_PROCESS_FOREMAN'
PP_FOREMAN_CONSUME = "pp_foreman_consume"
PP_FOREMAN_ACK_PUBLISH = "pp_foreman_ack_publish"
PP_START_INTEGRATION_ACK = "PP_START_INTEGRATION_ACK"
NCSA_PUBLISH = "ncsa_publish"
NCSA_CONSUME = "ncsa_consume"
NCSA_NO_RESPONSE = 5705
FORWARDER_NO_RESPONSE = 5605
FORWARDER_PUBLISH = "forwarder_publish"
CFG_FILE = 'L1SystemCfg.yaml'
ERROR_CODE_PREFIX = 5500
prp = toolsmod.prp
def __init__(self, filename=None):
toolsmod.singleton(self)
self._config_file = self.CFG_FILE
if filename != None:
self._config_file = filename
LOGGER.info('Extracting values from Config dictionary')
try:
self.extract_config_values()
except Exception as e:
LOGGER.error("PP_Device problem configuring with file %s: %s" %
(self._config_file, e.arg))
print("PP_Device unable to read Config file %s: %s" %
(self._config_file, e.arg))
sys.exit(self.ErrorCodePrefix + 20)
#self.purge_broker(cdm['ROOT']['QUEUE_PURGES'])
self._msg_actions = {
'PP_NEW_SESSION': self.set_session,
'PP_NEXT_VISIT': self.set_visit,
'PP_START_INTEGRATION': self.process_start_integration,
'PP_READOUT': self.process_dmcs_readout,
'NCSA_RESOURCE_QUERY_ACK': self.process_ack,
'NCSA_START_INTEGRATION_ACK': self.process_ack,
'NCSA_READOUT_ACK': self.process_ack,
'PP_FWDR_HEALTH_CHECK_ACK': self.process_ack,
'PP_FWDR_XFER_PARAMS_ACK': self.process_ack,
'PP_FWDR_READOUT_ACK': self.process_ack,
'PENDING_ACK': self.process_pending_ack,
'NCSA_NEXT_VISIT_ACK': self.process_ack
}
self._next_timed_ack_id = 0
try:
self.setup_publishers()
except L1PublisherError as e:
LOGGER.error("PP_Device unable to start Publishers: %s" % e.arg)
print("PP_Device unable to start Publishers: %s" % e.arg)
sys.exit(self.ErrorCodePrefix + 31)
self.setup_scoreboards()
LOGGER.info('pp foreman consumer setup')
self.thread_manager = None
try:
self.setup_consumer_threads()
except L1Exception as e:
LOGGER.error("PP_Device unable to launch ThreadManager: %s" %
e.arg)
print("PP_Device unable to launch ThreadManager: %s" % e.arg)
sys.exit(self.ErrorCodePrefix + 1)
LOGGER.info('Prompt Process Foreman Init complete')
def setup_publishers(self):
self._pub_base_broker_url = "amqp://" + self._pub_name + ":" + \
self._pub_passwd + "@" + \
str(self._base_broker_addr)
self._pub_ncsa_broker_url = "amqp://" + self._pub_ncsa_name + ":" + \
self._pub_ncsa_passwd + "@" + \
str(self._ncsa_broker_addr)
try:
LOGGER.info('Setting up Base publisher on %s using %s', \
self._pub_base_broker_url, self._base_msg_format)
self._base_publisher = SimplePublisher(self._pub_base_broker_url,
self._base_msg_format)
LOGGER.info('Setting up NCSA publisher on %s using %s', \
self._pub_ncsa_broker_url, self._ncsa_msg_format)
self._ncsa_publisher = SimplePublisher(self._pub_ncsa_broker_url,
self._ncsa_msg_format)
except Exception as e:
LOGGER.error("PP_Device unable to start Publishers: %s" % e.arg)
print("PP_Device unable to start Publishers: %s" % e.arg)
raise L1PublisherError(
"Critical Error: Unable to create Publishers: %s" % e.arg)
def on_dmcs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
#msg_dict = yaml.load(body)
msg_dict = body
LOGGER.info('In DMCS message callback')
LOGGER.info('Message from DMCS callback message body is: %s',
str(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def on_forwarder_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
LOGGER.info('In Forwarder message callback, thread is %s',
_thread.get_ident())
LOGGER.info('forwarder callback msg body is: %s', str(body))
pass
def on_ncsa_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
msg_dict = body
LOGGER.info('ncsa msg callback body is: %s', str(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def on_ack_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
msg_dict = body
LOGGER.info('In ACK message callback')
LOGGER.info('Message from ACK callback message body is: %s',
str(msg_dict))
handler = self._msg_actions.get(msg_dict[MSG_TYPE])
result = handler(msg_dict)
def set_session(self, params):
self.JOB_SCBD.set_session(params['SESSION_ID'])
ack_id = params['ACK_ID']
msg = {}
msg['MSG_TYPE'] = 'PP_NEW_SESSION_ACK'
msg['COMPONENT'] = self.COMPONENT_NAME
msg['ACK_ID'] = ack_id
msg['ACK_BOOL'] = True
route_key = params['REPLY_QUEUE']
self._base_publisher.publish_message(route_key, msg)
def set_visit(self, params):
bore_sight = params['BORE_SIGHT']
visit_id = params['VISIT_ID']
self.JOB_SCBD.set_visit_id(visit_id, bore_sight)
ack_id = params['ACK_ID']
msg = {}
ncsa_result = self.send_visit_boresight_to_ncsa(visit_id, bore_sight)
msg['MSG_TYPE'] = 'PP_NEXT_VISIT_ACK'
msg['COMPONENT'] = self.COMPONENT_NAME
msg['ACK_ID'] = ack_id
msg['ACK_BOOL'] = True
route_key = params['REPLY_QUEUE']
self._base_publisher.publish_message(route_key, msg)
def send_visit_boresight_to_ncsa(self, visit_id, bore_sight):
msg = {}
msg['MSG_TYPE'] = 'NCSA_NEXT_VISIT'
msg['VISIT_ID'] = visit_id
msg['BORE_SIGHT'] = bore_sight
msg['SESSION_ID'] = self.JOB_SCBD.get_current_session()
ack_id = self.get_next_timed_ack_id('NCSA_NEXT_VISIT_ACK')
msg['ACK_ID'] = ack_id
msg['REPLY_QUEUE'] = self.PP_FOREMAN_ACK_PUBLISH
self._ncsa_publisher.publish_message(self.NCSA_CONSUME, msg)
wait_time = 4
acks = []
acks.append(ack_id)
self.set_pending_nonblock_acks(acks, wait_time)
def process_start_integration(self, input_params):
"""
Add job to Job Scoreboard
Check forwarder health
Check Policy, bail if necessary
Mark Forwarder scoreboard as a result of above
Divide work and assemble as a forwarder dictionary for NCSA
Send work division to NCSA
Check Policy, bail if necessary
Persist pairings to Job Scoreboard
Send params to Forwarders
Confirm Forwarder Acks
Send confirm to DMCS
"""
ccd_list = input_params['CCD_LIST']
job_num = str(input_params[JOB_NUM])
visit_id = input_params['VISIT_ID']
image_id = input_params['IMAGE_ID']
self.JOB_SCBD.add_job(job_num, image_id, visit_id, ccd_list)
unknown_status = {"STATUS": "UNKNOWN", "STATE": "UNRESPONSIVE"}
self.FWD_SCBD.setall_forwarder_params(unknown_status)
ack_id = self.forwarder_health_check(input_params)
self.ack_timer(2.5)
healthy_forwarders = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if healthy_forwarders == None:
self.JOB_SCBD.set_job_state(job_number, 'SCRUBBED')
self.JOB_SCBD.set_job_status(job_number, 'INACTIVE')
self.send_fault("No Response From Forwarders",
self.FORWARDER_NO_RESPONSE, job_num,
self.COMPONENT_NAME)
raise L1ForwarderError(
"No response from any Forwarder when sending job params")
healthy_forwarders_list = list(healthy_forwarders.keys())
for forwarder in healthy_forwarders_list:
self.FWD_SCBD.set_forwarder_state(forwarder, 'BUSY')
self.FWD_SCBD.set_forwarder_status(forwarder, 'HEALTHY')
num_healthy_forwarders = len(healthy_forwarders_list)
ready_status = {"STATUS": "HEALTHY", "STATE": "READY_WITHOUT_PARAMS"}
self.FWD_SCBD.set_forwarder_params(healthy_forwarders_list,
ready_status)
work_schedule = self.divide_work(healthy_forwarders_list, ccd_list)
ack_id = self.ncsa_resources_query(input_params, work_schedule)
ncsa_response = self.progressive_ack_timer(ack_id, 1, 2.0)
#Check ACK scoreboard for response from NCSA
if ncsa_response:
pairs = []
pairs = ncsa_response['NCSA_FOREMAN']['PAIRS']
# Distribute job params and tell DMCS we are ready.
fwd_ack_id = self.distribute_job_params(input_params, pairs)
num_fwdrs = len(pairs)
fwdr_params_response = self.progressive_ack_timer(
fwd_ack_id, num_fwdrs, 3.0)
if fwdr_params_response:
self.JOB_SCBD.set_value_for_job(job_num, "STATE",
"FWDR_PARAMS_RECEIVED")
in_ready_state = {'STATE': 'READY_WITH_PARAMS'}
self.FWD_SCBD.set_forwarder_params(healthy_forwarders_list,
in_ready_state)
# Tell DMCS we are ready
result = self.accept_job(input_params['ACK_ID'], job_num)
else:
idle_params = {'STATE': 'IDLE'}
self.FWD_SCBD.set_forwarder_params(needed_forwarders,
idle_params)
self.send_fault("No RESPONSE FROM NCSA FOREMAN",
self.NCSA_NO_RESPONSE, job_num,
self.COMPONENT_NAME)
raise L1NcsaForemanError("No Response From NCSA Foreman")
else:
result = self.ncsa_no_response(input_params)
idle_params = {'STATE': 'IDLE'}
self.FWD_SCBD.set_forwarder_params(needed_forwarders, idle_params)
return result
def forwarder_health_check(self, params):
# get timed_ack_id
timed_ack = self.get_next_timed_ack_id("PP_FWDR_HEALTH_CHECK_ACK")
forwarders = self.FWD_SCBD.return_forwarders_list()
job_num = params[JOB_NUM]
# send health check messages
msg_params = {}
msg_params[MSG_TYPE] = 'PP_FWDR_HEALTH_CHECK'
msg_params['ACK_ID'] = timed_ack
msg_params['REPLY_QUEUE'] = self.PP_FOREMAN_ACK_PUBLISH
msg_params[JOB_NUM] = job_num
self.JOB_SCBD.set_value_for_job(job_num, "STATE", "HEALTH_CHECK")
for forwarder in forwarders:
self._base_publisher.publish_message(
self.FWD_SCBD.get_routing_key(forwarder), msg_params)
return timed_ack
def divide_work(self, fwdrs_list, ccd_list):
num_fwdrs = len(fwdrs_list)
num_ccds = len(ccd_list)
schedule = {}
schedule['FORWARDER_LIST'] = []
schedule['CCD_LIST'] = [
] # A list of ccd lists; index of main list matches same forwarder list index
FORWARDER_LIST = []
CCD_LIST = [] # This is a 'list of lists'
if num_fwdrs == 1:
FORWARDER_LIST.append(fwdrs_list[0])
CCD_LIST.append(ccd_list)
schedule['FORWARDERS_LIST'] = FORWARDER_LIST
schedule['CCD_LIST'] = CCD_LIST
return schedule
if num_ccds <= num_fwdrs:
for k in range(0, num_ccds):
little_list = []
FORWARDER_LIST.append(fwdrs_list[k])
little_list.append(ccd_list[k])
CCD_LIST.append(list(little_list)) # Need a copy here...
schedule['FORWARDER_LIST'] = FORWARDER_LIST
schedule['CCD_LIST'] = CCD_LIST
else:
ccds_per_fwdr = len(ccd_list) // num_fwdrs
remainder_ccds = len(ccd_list) % num_fwdrs
offset = 0
for i in range(0, num_fwdrs):
tmp_list = []
for j in range(offset, (ccds_per_fwdr + offset)):
if (j) >= num_ccds:
break
tmp_list.append(ccd_list[j])
# CCD_LIST.append(ccd_list[j])
offset = offset + ccds_per_fwdr
if remainder_ccds != 0 and i == 0:
for k in range(offset, offset + remainder_ccds):
tmp_list.append(ccd_list[k])
offset = offset + remainder_ccds
FORWARDER_LIST.append(fwdrs_list[i])
CCD_LIST.append(list(tmp_list))
#schedule[fwdrs_list[i]] = {}
#schedule[fwdrs_list[i]]['CCD_LIST'] = tmp_list
schedule['FORWARDER_LIST'] = FORWARDER_LIST
schedule['CCD_LIST'] = CCD_LIST
return schedule
def ncsa_resources_query(self, params, work_schedule):
job_num = str(params[JOB_NUM])
timed_ack_id = self.get_next_timed_ack_id("NCSA_START_INTEGRATION_ACK")
ncsa_params = {}
ncsa_params[MSG_TYPE] = "NCSA_START_INTEGRATION"
ncsa_params[JOB_NUM] = job_num
ncsa_params['VISIT_ID'] = params['VISIT_ID']
ncsa_params['IMAGE_ID'] = params['IMAGE_ID']
ncsa_params['SESSION_ID'] = params['SESSION_ID']
ncsa_params['REPLY_QUEUE'] = self.PP_FOREMAN_ACK_PUBLISH
ncsa_params[ACK_ID] = timed_ack_id
ncsa_params["FORWARDERS"] = work_schedule
self.JOB_SCBD.set_value_for_job(job_num, "STATE",
"NCSA_START_INT_SENT")
self._ncsa_publisher.publish_message(self.NCSA_CONSUME, ncsa_params)
LOGGER.info(
'The following forwarders schedule has been sent to NCSA for pairing:'
)
LOGGER.info(work_schedule)
return timed_ack_id
def distribute_job_params(self, params, pairs):
""" pairs param is a list of dicts. (look at messages.yaml, search for 'PAIR' key, and copy here
"""
#ncsa has enough resources...
job_num = str(params[JOB_NUM])
self.JOB_SCBD.set_pairs_for_job(job_num, pairs)
LOGGER.info('The following pairs will be used for Job #%s: %s',
job_num, pairs)
fwd_ack_id = self.get_next_timed_ack_id("FWD_PARAMS_ACK")
fwd_params = {}
fwd_params[MSG_TYPE] = "PP_FWDR_XFER_PARAMS"
fwd_params[JOB_NUM] = job_num
fwd_params['IMAGE_ID'] = params['IMAGE_ID']
fwd_params['VISIT_ID'] = params['VISIT_ID']
fwd_params['REPLY_QUEUE'] = self.PP_FOREMAN_ACK_PUBLISH
fwd_params[ACK_ID] = fwd_ack_id
fwd_params['XFER_PARAMS'] = {}
for i in range(0, len(pairs)):
ddict = {}
ddict = pairs[i]
fwdr = ddict['FORWARDER']
fwd_params['XFER_PARAMS']['CCD_LIST'] = ddict['CCD_LIST']
fwd_params['XFER_PARAMS']['DISTRIBUTOR'] = ddict['DISTRIBUTOR']
route_key = self.FWD_SCBD.get_value_for_forwarder(
fwdr, "CONSUME_QUEUE")
self._base_publisher.publish_message(route_key, fwd_params)
return fwd_ack_id
def accept_job(self, ack_id, job_num):
dmcs_message = {}
dmcs_message[JOB_NUM] = job_num
dmcs_message[MSG_TYPE] = self.PP_START_INTEGRATION_ACK
dmcs_message['COMPONENT'] = self.COMPONENT_NAME
dmcs_message[ACK_BOOL] = True
dmcs_message['ACK_ID'] = ack_id
self.JOB_SCBD.set_value_for_job(job_num, STATE, "JOB_ACCEPTED")
self.JOB_SCBD.set_value_for_job(job_num, "TIME_JOB_ACCEPTED",
get_timestamp())
self._base_publisher.publish_message("dmcs_ack_consume", dmcs_message)
return True
def process_dmcs_readout(self, params):
job_number = params[JOB_NUM]
pairs = self.JOB_SCBD.get_pairs_for_job(job_number)
### Send READOUT to NCSA with ACK_ID
ack_id = self.get_next_timed_ack_id('NCSA_READOUT_ACK')
ncsa_params = {}
ncsa_params[MSG_TYPE] = 'NCSA_READOUT'
ncsa_params['JOB_NUM'] = job_number
ncsa_params['VISIT_ID'] = params['VISIT_ID']
ncsa_params['SESSION_ID'] = params['SESSION_ID']
ncsa_params['IMAGE_ID'] = params['IMAGE_ID']
ncsa_params['REPLY_QUEUE'] = 'pp_foreman_ack_publish'
ncsa_params[ACK_ID] = ack_id
self._ncsa_publisher.publish_message(self.NCSA_CONSUME, ncsa_params)
ncsa_response = self.progressive_ack_timer(ack_id, 1, 3.0)
if ncsa_response:
if ncsa_response['NCSA_FOREMAN']['ACK_BOOL'] == True:
#inform forwarders
fwd_ack_id = self.get_next_timed_ack_id('PP_FWDR_READOUT_ACK')
len_pairs = len(pairs)
for i in range(0, len_pairs):
forwarder = pairs[i]['FORWARDER']
routing_key = self.FWD_SCBD.get_routing_key(forwarder)
msg_params = {}
msg_params[MSG_TYPE] = 'PP_FWDR_READOUT'
msg_params[JOB_NUM] = job_number
msg_params['REPLY_QUEUE'] = 'pp_foreman_ack_publish'
msg_params['ACK_ID'] = fwd_ack_id
self.FWD_SCBD.set_forwarder_state(forwarder,
'START_READOUT')
self._base_publisher.publish_message(
routing_key, msg_params)
forwarder_responses = self.progressive_ack_timer(
fwd_ack_id, len_pairs, 4.0)
if forwarder_responses:
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'PP_READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['COMPONENT'] = self.COMPONENT_NAME
dmcs_params['ACK_BOOL'] = True
dmcs_params['ACK_ID'] = params['ACK_ID']
self._base_publisher.publish_message(
params['REPLY_QUEUE'], dmcs_params)
else:
#send problem with ncsa to DMCS
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'PP_READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['COMPONENT'] = self.COMPONENT_NAME
dmcs_params['ACK_BOOL'] = False
dmcs_params['ACK_ID'] = params['ACK_ID']
self._base_publisher.publish_message('dmcs_ack_consume',
dmcs_params)
else:
#send 'no response from ncsa' to DMCS )
dmcs_params = {}
dmcs_params[MSG_TYPE] = 'PP_READOUT_ACK'
dmcs_params[JOB_NUM] = job_number
dmcs_params['COMPONENT'] = self.COMPONENT_NAME
dmcs_params['ACK_BOOL'] = False
dmcs_params['ACK_ID'] = params['ACK_ID']
self._base_publisher.publish_message(params['REPLY_QUEUE'],
dmcs_params)
def process_ack(self, params):
self.ACK_SCBD.add_timed_ack(params)
def get_next_timed_ack_id(self, ack_type):
self._next_timed_ack_id = self._next_timed_ack_id + 1
return (ack_type + "_" + str(self._next_timed_ack_id).zfill(6))
def ack_timer(self, seconds):
sleep(seconds)
return True
def progressive_ack_timer(self, ack_id, expected_replies, seconds):
counter = 0.0
while (counter < seconds):
counter = counter + 0.5
sleep(0.5)
response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if response == None:
continue
if len(list(response.keys())) == expected_replies:
return response
## Try one final time
response = self.ACK_SCBD.get_components_for_timed_ack(ack_id)
if response == None:
return None
elif len(list(response.keys())) == expected_replies:
return response
else:
return None
def set_pending_nonblock_acks(self, acks, wait_time):
start_time = datetime.datetime.now().time()
expiry_time = self.add_seconds(start_time, wait_time)
ack_msg = {}
ack_msg[MSG_TYPE] = 'PENDING_ACK'
ack_msg['EXPIRY_TIME'] = expiry_time
for ack in acks:
ack_msg[ACK_ID] = ack
self._base_publisher.publish_message(self.PP_FOREMAN_ACK_PUBLISH,
ack_msg)
def process_pending_ack(self, params):
self.ACK_SCBD.add_pending_nonblock_ack(params)
def add_seconds(self, intime, secs):
basetime = datetime.datetime(100, 1, 1, intime.hour, intime.minute,
intime.second)
newtime = basetime + datetime.timedelta(seconds=secs)
return newtime.time()
def extract_config_values(self):
LOGGER.info('Reading YAML Config file %s' % self._config_file)
try:
cdm = toolsmod.intake_yaml_file(self._config_file)
except IOError as e:
LOGGER.critical("Unable to find CFG Yaml file %s\n" %
self._config_file)
print("Unable to find CFG Yaml file %s\n" % self._config_file)
raise L1ConfigIOError("Trouble opening CFG Yaml file %s: %s" %
(self._config_file, e.arg))
try:
self._sub_name = cdm[ROOT][
PFM_BROKER_NAME] # Message broker user & passwd
self._sub_passwd = cdm[ROOT][PFM_BROKER_PASSWD]
self._pub_name = cdm[ROOT][
'PFM_BROKER_PUB_NAME'] # Message broker user & passwd
self._pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD']
self._sub_ncsa_name = cdm[ROOT]['PFM_NCSA_BROKER_NAME']
self._sub_ncsa_passwd = cdm[ROOT]['PFM_NCSA_BROKER_PASSWD']
self._pub_ncsa_name = cdm[ROOT]['PFM_NCSA_BROKER_PUB_NAME']
self._pub_ncsa_passwd = cdm[ROOT]['PFM_NCSA_BROKER_PUB_PASSWD']
self._base_broker_addr = cdm[ROOT][BASE_BROKER_ADDR]
self._ncsa_broker_addr = cdm[ROOT][NCSA_BROKER_ADDR]
self._forwarder_dict = cdm[ROOT][XFER_COMPONENTS]['PP_FORWARDERS']
self._scbd_dict = cdm[ROOT]['SCOREBOARDS']
self.DMCS_FAULT_QUEUE = cdm[ROOT]['DMCS_FAULT_QUEUE']
self._policy_max_ccds_per_fwdr = int(
cdm[ROOT]['POLICY']['MAX_CCDS_PER_FWDR'])
except KeyError as e:
LOGGER.critical("CDM Dictionary Key error")
LOGGER.critical("Offending Key is %s", str(e))
LOGGER.critical("Bailing out...")
print("KeyError when reading CFG file. Check logs...exiting...")
raise L1ConfigKeyError("Key Error when reading config file: %s" %
e.arg)
self._base_msg_format = 'YAML'
self._ncsa_msg_format = 'YAML'
if 'BASE_MSG_FORMAT' in cdm[ROOT]:
self._base_msg_format = cdm[ROOT][BASE_MSG_FORMAT]
if 'NCSA_MSG_FORMAT' in cdm[ROOT]:
self._ncsa_msg_format = cdm[ROOT][NCSA_MSG_FORMAT]
def setup_consumer_threads(self):
LOGGER.info('Building _base_broker_url')
base_broker_url = "amqp://" + self._sub_name + ":" + \
self._sub_passwd + "@" + \
str(self._base_broker_addr)
ncsa_broker_url = "amqp://" + self._sub_ncsa_name + ":" + \
self._sub_ncsa_passwd + "@" + \
str(self._ncsa_broker_addr)
self.shutdown_event = threading.Event()
self.shutdown_event.clear()
# Set up kwargs that describe consumers to be started
# The Archive Device needs three message consumers
kws = {}
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-pp_foreman_consume'
md['queue'] = 'pp_foreman_consume'
md['callback'] = self.on_dmcs_message
md['format'] = "YAML"
md['test_val'] = None
kws[md['name']] = md
md = {}
md['amqp_url'] = base_broker_url
md['name'] = 'Thread-pp_foreman_ack_publish'
md['queue'] = 'pp_foreman_ack_publish'
md['callback'] = self.on_ack_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
md = {}
md['amqp_url'] = ncsa_broker_url
md['name'] = 'Thread-ncsa_publish'
md['queue'] = 'ncsa_publish'
md['callback'] = self.on_ncsa_message
md['format'] = "YAML"
md['test_val'] = 'test_it'
kws[md['name']] = md
try:
self.thread_manager = ThreadManager('thread-manager', kws,
self.shutdown_event)
self.thread_manager.start()
except ThreadError as e:
LOGGER.error(
"PP_Device unable to launch Consumers - Thread Error: %s" %
e.arg)
print("PP_Device unable to launch Consumers - Thread Error: %s" %
e.arg)
raise L1ConsumerError(
"Thread problem preventing Consumer launch: %s" % e.arg)
except Exception as e:
LOGGER.error("PP_Device unable to launch Consumers: %s" % e.arg)
print("PP_Device unable to launch Consumers: %s" % e.arg)
raise L1Error(
"PP_Device unable to launch Consumers - Rabbit Problem?: %s" %
e.arg)
def setup_scoreboards(self):
try:
# Create Redis Forwarder table with Forwarder info
self.FWD_SCBD = ForwarderScoreboard('PP_FWD_SCBD',
self._scbd_dict['PP_FWD_SCBD'],
self._forwarder_dict)
self.JOB_SCBD = JobScoreboard('PP_JOB_SCBD',
self._scbd_dict['PP_JOB_SCBD'])
self.ACK_SCBD = AckScoreboard('PP_ACK_SCBD',
self._scbd_dict['PP_ACK_SCBD'])
except L1RabbitConnectionError as e:
LOGGER.error(
"PP_Device unable to complete setup_scoreboards-No Rabbit Connect: %s"
% e.arg)
print(
"PP_Device unable to complete setup_scoreboards - No Rabbit Connection: %s"
% e.arg)
sys.exit(self.ErrorCodePrefix + 11)
except L1RedisError as e:
LOGGER.error(
"PP_Device unable to complete setup_scoreboards - no Redis connect: %s"
% e.arg)
print(
"PP_Device unable to complete setup_scoreboards - no Redis connection: %s"
% e.arg)
sys.exit(self.ErrorCodePrefix + 12)
except Exception as e:
LOGGER.error(
"PP_Device init unable to complete setup_scoreboards: %s" %
e.arg)
print("PP_Device unable to complete setup_scoreboards: %s" % e.arg)
sys.exit(self.ErrorCodePrefix + 10)
def send_fault(error_string, error_code, job_num, component_name):
msg = {}
msg['MSG_TYPE'] = 'FAULT'
msg['COMPONENT'] = component_name
msg['JOB_NUM'] = job_num
msg['ERROR_CODE'] = str(error_code)
msg["DESCRIPTION"] = error_string
self._base_publisher.publish_message(self.DMCS_FAULT_QUEUE, msg)
def purge_broker(self, queues):
for q in queues:
cmd = "rabbitmqctl -p /tester purge_queue " + q
os.system(cmd)
def shutdown(self):
LOGGER.debug("PromptProcessDevice: Shutting down Consumer threads.")
self.shutdown_event.set()
LOGGER.debug("Thread Manager shutting down and app exiting...")
print("\n")
os._exit(0)
class TestDMCS_AT:
ocs_pub_broker_url = None
ocs_publisher = None
ocs_consumer = None
ocs_consumer_msg_list = []
at_pub_broker_url = None
at_publisher = None
at_consumer = None
at_consumer_msg_list = []
EXPECTED_AT_MESSAGES = 1
EXPECTED_OCS_MESSAGES = 1
prp = toolsmod.prp
#DP = toolsmod.DP #Debug print
DP = True #Debug print
def test_dmcs(self, Dmcs):
self.dmcs = Dmcs
#logging.warning("Logging is Working!")
LOGGER.critical("LOGGING is Working!")
#self.LOGGER.info("self Logging is Working!")
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
ocs_name = cdm[ROOT]['OCS_BROKER_NAME']
ocs_passwd = cdm[ROOT]['OCS_BROKER_PASSWD']
ocs_pub_name = cdm[ROOT]['OCS_BROKER_PUB_NAME']
ocs_pub_passwd = cdm[ROOT]['OCS_BROKER_PUB_PASSWD']
ocs_broker_url = "amqp://" + ocs_name + ":" + \
ocs_passwd + "@" + \
broker_addr
self.ocs_pub_broker_url = "amqp://" + ocs_pub_name + ":" + \
ocs_pub_passwd + "@" + \
broker_addr
self.ocs_publisher = SimplePublisher(self.ocs_pub_broker_url, "YAML")
at_name = cdm[ROOT]['AUX_BROKER_NAME']
at_passwd = cdm[ROOT]['AUX_BROKER_PASSWD']
at_pub_name = cdm[ROOT]['AUX_BROKER_PUB_NAME']
at_pub_passwd = cdm[ROOT]['AUX_BROKER_PUB_PASSWD']
at_broker_url = "amqp://" + at_name + ":" + \
at_passwd + "@" + \
broker_addr
self.at_pub_broker_url = "amqp://" + at_pub_name + ":" + \
at_pub_passwd + "@" + \
broker_addr
self.at_publisher = SimplePublisher(self.at_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.ocs_consumer = Consumer(ocs_broker_url,'dmcs_ocs_publish', 'thread-ocs',
self.on_ocs_message,'YAML')
self.ocs_consumer.start()
self.at_consumer = Consumer(at_broker_url,'at_foreman_consume', 'thread-at',
self.on_at_message,'YAML')
self.at_consumer.start()
sleep(3)
if self.DP:
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(3)
self.verify_ocs_messages()
sleep(3)
self.verify_at_messages()
sleep(2)
self.ocs_consumer.stop()
self.ocs_consumer.join()
self.at_consumer.stop()
self.at_consumer.join()
if self.DP:
print("Finished with DMCS AT tests.")
def send_messages(self):
if self.DP:
print("Starting send_messages")
# Tests only an AT device
self.clear_message_lists()
self.EXPECTED_OCS_MESSAGES = 6
self.EXPECTED_AT_MESSAGES = 4
msg = {}
msg['MSG_TYPE'] = "STANDBY"
msg['DEVICE'] = 'AT'
msg['CMD_ID'] = '17718411'
msg['CFG_KEY'] = "2C16"
msg['ACK_ID'] = 'AT_4'
msg['ACK_DELAY'] = 2
time.sleep(2)
if self.DP:
print("Sending AT STANDBY")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "DISABLE"
msg['DEVICE'] = 'AT'
msg['CMD_ID'] = '17718411'
msg['ACK_ID'] = 'AT_6'
msg['ACK_DELAY'] = 2
time.sleep(2)
if self.DP:
print("Sending AT DISABLE")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
sleep(2.0)
# Make sertain scoreboard values are being set
if self.DP:
print("Checking State Scoreboard entries.")
#assert self.dmcs.STATE_SCBD.get_auxdev_state() == 'DISABLE'
assert self.dmcs.STATE_SCBD.get_auxtel_state() == 'DISABLE'
msg = {}
msg['MSG_TYPE'] = "ENABLE"
msg['DEVICE'] = 'AT'
msg['CMD_ID'] = '17718411'
msg['ACK_ID'] = 'AT_11'
msg['ACK_DELAY'] = 2
time.sleep(4)
if self.DP:
print("Sending AT ENABLE")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
sleep(2.0)
#assert self.dmcs.STATE_SCBD.get_auxdev_state() == 'ENABLE'
assert self.dmcs.STATE_SCBD.get_auxtel_state() == 'ENABLE'
msg = {}
msg['MSG_TYPE'] = "DMCS_AT_START_INTEGRATION"
msg['IMAGE_ID'] = 'IMG_4280'
msg['IMAGE_INDEX'] = 2
msg['IMAGE_SEQUENCE_NAME'] = 'XX_seq'
msg['IMAGES_IN_SEQUENCE'] = 8
time.sleep(5)
if self.DP:
print("Sending Start Integration Message")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "DMCS_AT_END_READOUT"
msg['IMAGE_ID'] = 'IMG_4280'
msg['IMAGE_INDEX'] = 2
msg['IMAGE_SEQUENCE_NAME'] = 'XX_seq'
msg['IMAGES_IN_SEQUENCE'] = 8
time.sleep(5)
if self.DP:
print("Sending READOUT Message")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "DMCS_AT_HEADER_READY"
msg['IMAGE_ID'] = 'IMG_4280'
msg['FILENAME'] = "/mnt/headers/IMG_4277.header"
msg['ACK_ID'] = 'DMCS_END_READOUT_ACK_82'
time.sleep(5)
if self.DP:
print("Sending READOUT Message")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
time.sleep(5)
if self.DP:
print("Message Sender done")
def clear_message_lists(self):
self.ocs_consumer_msg_list = []
self.at_consumer_msg_list = []
def verify_ocs_messages(self):
if self.DP:
print("Messages received by verify_ocs_messages:")
self.prp.pprint(self.ocs_consumer_msg_list)
len_list = len(self.ocs_consumer_msg_list)
if len_list != self.EXPECTED_OCS_MESSAGES:
if self.DP:
print("Incorrect number of messages received by OCS ACK Consumer.")
print("Messages received by verify_ocs_messages:")
self.prp.pprint(self.ocs_consumer_msg_list)
pytest.fail('OCS simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_OCS_MESSAGES, len_list))
# Now check num keys in each message and check for key errors
for i in range(0, len_list):
msg = self.ocs_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following OCS Bridge response message failed when compared with the sovereign example: %s" % msg)
if self.DP:
print("Responses to OCS Bridge pass verification.")
def verify_at_messages(self):
if self.DP:
print("Messages received by verify_at_messages:")
self.prp.pprint(self.at_consumer_msg_list)
len_list = len(self.at_consumer_msg_list)
if self.DP:
print("The number of messages the AT received is %s" % len_list)
if len_list != self.EXPECTED_AT_MESSAGES:
if self.DP:
print("Messages received by verify_at_messages:")
self.prp.pprint(self.at_consumer_msg_list)
pytest.fail('AT simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_AT_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.at_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
if self.DP:
print("The following message to the AT failed when compared with " \
"the sovereign example: %s" % msg)
pytest.fail("The following message to the AT failed when compared with " \
"the sovereign example: %s" % msg)
if self.DP:
print("Messages to the AT pass verification.")
def on_ocs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_dmcs-at - incoming on_ocs_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.ocs_consumer_msg_list.append(body)
def on_at_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_dmcs-at - incoming on_at_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.at_consumer_msg_list.append(body)
def test_ppdev(self, Ppdev):
self.ppdev = Ppdev
try:
cdm = toolsmod.intake_yaml_file('./tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
dmcs_name = cdm[ROOT]['DMCS_BROKER_NAME']
dmcs_passwd = cdm[ROOT]['DMCS_BROKER_PASSWD']
dmcs_pub_name = cdm[ROOT]['DMCS_BROKER_PUB_NAME']
dmcs_pub_passwd = cdm[ROOT]['DMCS_BROKER_PUB_PASSWD']
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_passwd + "@" + \
broker_addr
dmcs_pub_broker_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_passwd + "@" + \
broker_addr
self.dmcs_publisher = SimplePublisher(dmcs_pub_broker_url, "YAML")
ncsa_name = cdm[ROOT]['NCSA_BROKER_NAME']
ncsa_passwd = cdm[ROOT]['NCSA_BROKER_PASSWD']
ncsa_pub_name = cdm[ROOT]['NCSA_BROKER_PUB_NAME']
ncsa_pub_passwd = cdm[ROOT]['NCSA_BROKER_PUB_PASSWD']
ncsa_broker_url = "amqp://" + ncsa_name + ":" + \
ncsa_passwd + "@" + \
broker_addr
ncsa_pub_broker_url = "amqp://" + ncsa_pub_name + ":" + \
ncsa_pub_passwd + "@" + \
broker_addr
self.ncsa_publisher = SimplePublisher(ncsa_pub_broker_url, "YAML")
F1_name = 'F1'
F1_passwd = 'F1'
F1_pub_name = 'F1_PUB'
F1_pub_passwd = 'F1_PUB'
F1_broker_url = "amqp://" + F1_name + ":" + \
F1_passwd + "@" + \
broker_addr
F1_pub_broker_url = "amqp://" + F1_pub_name + ":" + \
F1_pub_passwd + "@" + \
broker_addr
self.f1_publisher = SimplePublisher(F1_pub_broker_url, "YAML")
F2_name = 'F2'
F2_passwd = 'F2'
F2_pub_name = 'F2_PUB'
F2_pub_passwd = 'F2_PUB'
F2_broker_url = "amqp://" + F2_name + ":" + \
F2_passwd + "@" + \
broker_addr
F2_pub_broker_url = "amqp://" + F2_pub_name + ":" + \
F2_pub_passwd + "@" + \
broker_addr
self.f2_publisher = SimplePublisher(F2_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.dmcs_consumer = Consumer(dmcs_broker_url,'dmcs_ack_consume', 'thread-dmcs',
self.on_dmcs_message,'YAML')
self.dmcs_consumer.start()
self.ncsa_consumer = Consumer(ncsa_broker_url,'ncsa_consume', 'thread-ncsa',
self.on_ncsa_message,'YAML')
self.ncsa_consumer.start()
self.F1_consumer = Consumer(F1_broker_url,'f1_consume', 'thread-f1',
self.on_f1_message,'YAML')
self.F1_consumer.start()
self.F2_consumer = Consumer(F2_broker_url,'f2_consume', 'thread-f2',
self.on_f2_message,'YAML')
self.F2_consumer.start()
sleep(3)
if self.DP:
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(6)
self.verify_ncsa_messages()
self.verify_F2_messages()
self.verify_F1_messages()
self.verify_dmcs_messages()
sleep(2)
self.dmcs_consumer.stop()
self.dmcs_consumer.join()
self.ncsa_consumer.stop()
self.ncsa_consumer.join()
self.F1_consumer.stop()
self.F1_consumer.join()
self.F2_consumer.stop()
self.F2_consumer.join()
if self.DP:
print("Finished with PP tests.")
def main():
premium = Premium()
sp1 = SimplePublisher('amqp://*****:*****@141.142.208.191:5672/%2Fbunny',
"YAML")
#sp2 = SimplePublisher('amqp://*****:*****@141.142.208.191:5672/%2Ftester')
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#cons = Consumer(broker_url, 'F8_consume')
#try:
# thread.start_new_thread( do_it, ("thread-1", 2,) )
#except:
# print "Cannot start thread"
# while 1:
msg = {}
msg['MSG_TYPE'] = "FORWARDER_HEALTH_CHECK"
msg['ACK_ID'] = 'AR_FWD_HEALTH_ACK_1662'
msg['REPLY_QUEUE'] = 'ar_foreman_ack_publish'
time.sleep(3)
sp1.publish_message("f1_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_XFER_PARAMS"
msg['JOB_NUM'] = "j_1412_z"
msg['VISIT_ID'] = 'V229'
msg['IMAGE_ID'] = 'IMG_226XT_46'
msg['ACK_ID'] = 'AR_XFER_PARAMS_ACK_112'
msg['TARGET_DIR'] = "/mnt/xfer_dir/"
msg['REPLY_QUEUE'] = 'ar_foreman_ack_publish'
m = {}
m['CCD_LIST'] = ['1', '6', '2', '3', '9', '4', '7', '8']
m['XFER_UNIT'] = ['CCD']
m['FQN'] = 'ARCHIVE'
m['NAME'] = "archive"
m['HOSTNAME'] = 'blarg'
m['IP_ADDR'] = '141.142.210.202'
m['IMG_SRC'] = "blah" # For future use...
msg['XFER_PARAMS'] = m
time.sleep(3)
sp1.publish_message("f1_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_READOUT"
msg['JOB_NUM'] = "j_1412_z"
msg['IMAGE_ID'] = 'IMG_226XT_46'
msg['ACK_ID'] = 'AR_READOUT_ACK_92112'
msg['REPLY_QUEUE'] = 'ar_foreman_ack_publish'
time.sleep(4)
sp1.publish_message("f1_consume", msg)
"""
msg = {}
msg['MSG_TYPE'] = "NEXT_VISIT"
msg['VISIT_ID'] = 'XX_28272'
msg['BORE_SIGHT'] = 'A LITTLE TO THE LEFT'
time.sleep(4)
sp1.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "START_INTEGRATION"
msg['IMAGE_ID'] = 'IMG_444244'
msg['DEVICE'] = 'AR'
time.sleep(4)
sp1.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "READOUT"
msg['IMAGE_ID'] = 'IMG_444244'
msg['DEVICE'] = 'AR'
time.sleep(4)
sp1.publish_message("ocs_dmcs_consume", msg)
"""
print("Sender done")