def home():
print("----------------Welcome to Consumer APP----------------")
print("You can select one of these options:")
while (True):
print("1. Login\n2. Sign in\n3. Exit")
print("-------------------------------------------------------")
option = input()
if option == '1':
email = input("Please insert your e-mail: ")
password = input("Please insert your password: "******"2":
email = input("Please insert your e-mail: ")
password = input("Please insert your password: "******"Please insert your name: ")
consumer = Consumer(email, password, name)
consumer.sign_in()
elif option == "3":
print("Kind Regards!")
break
else:
print("Please insert a valid option.")
def test_ocs_commandlistener(self):
try:
cdm = toolsmod.intake_yaml_file(
"/home/hwin16/src/git/ctrl_iip/python/lsst/iip/tests/yaml/L1SystemCfg_Test_ocs_bridge.yaml"
)
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to fine CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]["BASE_BROKER_ADDR"]
dmcs_name = cdm[ROOT]["DMCS_BROKER_NAME"]
dmcs_pwd = cdm[ROOT]["DMCS_BROKER_PASSWD"]
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_pwd + "@" + \
broker_addr
self.dmcs_consumer = Consumer(dmcs_broker_url, "ocs_dmcs_consume",
"thread-dmcs-consume",
self.on_ocs_message, "YAML")
self.dmcs_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority(
"/home/hwin16/src/git/ctrl_iip/python/lsst/iip/messages.yaml")
self.send_messages()
sleep(10)
os.killpg(os.getpgid(self.cmd.pid), signal.SIGTERM)
self.verify_ocs_messages()
print("Finished with CommandListener tests.")
class Premium:
def __init__(self):
logging.basicConfig()
#os.system('rabbitmqctl -p /tester purge_queue firehose')
#os.system('rabbitmqctl -p /tester purge_queue ack_publish')
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny?heartbeat=300'
broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny?autoAck=true'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Ftester'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#self._cons = FirehoseConsumer(broker_url, 'firehose', "YAML")
self._cons = Consumer(broker_url, 'ocs_dmcs_consume', "YAML")
try:
_thread.start_new_thread( self.do_it, ("thread-1", 2,) )
except e:
print("Cannot start thread")
print(e)
time.sleep(420)
def mycallback(self, ch, methon, properties, body):
print(" ")
print(">>>>>>>>>>>>>>><<<<<<<<<<<<<<<<")
print((" [x] method Received %r" % methon))
print((" [y] properties Received %r" % properties))
print((" [z] body Received %r" % body))
print("Message done")
print("Still listening...")
def do_it(self, threadname, delay):
#example = ExampleConsumer('amqp://*****:*****@141.142.208.191:5672/%2Fbunny')
print("Before run call")
self._cons.run(self.mycallback)
print("After run call - not blocking")
def __init__(self):
logging.basicConfig()
self.ack_test = True
#os.system('rabbitmqctl -p /tester purge_queue firehose')
#os.system('rabbitmqctl -p /tester purge_queue ack_publish')
broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Ftester'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#self._cons = FirehoseConsumer(broker_url, 'firehose', "YAML")
self.sp1 = SimplePublisher(
'amqp://*****:*****@141.142.208.191:5672/%2Fbunny')
cdm = toolsmod.intake_yaml_file('ForemanCfg.yaml')
self.fdict = cdm[ROOT]['XFER_COMPONENTS']['ARCHIVE_FORWARDERS']
self.fwdrs = list(self.fdict.keys())
self._cons = Consumer(broker_url, 'ar_foreman_ack_publish', "YAML")
#self._cons = Consumer(broker_url, 'ar_foreman_consume', "YAML")
try:
_thread.start_new_thread(self.do_it, (
"thread-1",
2,
))
except e:
print("Cannot start thread")
print(e)
def main():
stack = []
n = Semaphore(0)
s = Semaphore(1)
# Create Producer and Consumer threads
producer1 = Producer(stack, n, s, 1)
producer2 = Producer(stack, n, s, 2)
producer3 = Producer(stack, n, s, 3)
producer4 = Producer(stack, n, s, 4)
producer5 = Producer(stack, n, s, 5)
consumer = Consumer(stack, n, s)
# Start all the threads
producer1.start()
producer2.start()
producer3.start()
producer4.start()
producer5.start()
consumer.start()
# Wait for threads to finish
producer1.join()
producer2.join()
producer3.join()
producer4.join()
producer5.join()
consumer.join()
print("Done.")
def main():
# handle user input
from_date_string = str(
raw_input("Please specify a Start Date with format ddmmYYYY: "))
# raw input in python2 is the same as input in python3 (returns a string instead of a python expression)
to_date_string = str(
raw_input("Please specify a End Date with format ddmmYYYY: "))
locale.setlocale(locale.LC_ALL, '')
from_date = datetime.datetime.strptime(from_date_string,
'%d%m%Y').isoformat()
to_date = datetime.datetime.strptime(to_date_string, '%d%m%Y')
tmp_date = to_date + datetime.timedelta(1)
to_date = tmp_date.isoformat()
threads = list()
stopper = threading.Event()
consumer = Consumer()
producer = Producer(from_date=from_date, to_date=to_date)
threads.append(consumer)
threads.append(producer)
handler = SignalHandler(stopper, threads)
signal.signal(signal.SIGINT, handler)
producer.start()
consumer.start()
def __init__(self):
logging.basicConfig()
#os.system('rabbitmqctl -p /tester purge_queue firehose')
#os.system('rabbitmqctl -p /tester purge_queue ack_publish')
#broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny?heartbeat=300'
broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny'
#pub_broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Ftester'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#self._cons = FirehoseConsumer(broker_url, 'firehose', "YAML")
self._cons = Consumer(broker_url, 'ar_foreman_consume', "YAML")
try:
_thread.start_new_thread( self.do_it, ("thread-1", 2,) )
except e:
print("Cannot start thread")
print(e)
self._cons2 = Consumer(broker_url, 'pp_foreman_consume', "YAML")
try:
_thread.start_new_thread( self.do_it2, ("thread-3", 2,) )
except e:
print("Cannot start thread")
print(e)
cdm = toolsmod.intake_yaml_file("L1SystemCfg.yaml")
self.ccd_list = cdm['ROOT']['CCD_LIST']
def __init__(self):
logging.basicConfig()
#os.system('rabbitmqctl -p /tester purge_queue firehose')
#os.system('rabbitmqctl -p /tester purge_queue ack_publish')
self.sp2 = SimplePublisher(
'amqp://*****:*****@141.142.238.160:5672/%2Fbunny', "YAML")
time.sleep(3)
broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Ftester'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#self._cons = FirehoseConsumer(broker_url, 'firehose', "YAML")
self._cons = Consumer(broker_url, 'f1_consume', "YAML")
#self._cons = Consumer(broker_url, 'pp_foreman_consume', "YAML")
self._cons2 = Consumer(broker_url, 'ncsa_consume', "YAML")
try:
_thread.start_new_thread(self.do_it, (
"thread-1",
2,
))
except e:
print("Cannot start thread")
print(e)
try:
_thread.start_new_thread(self.do_it2, (
"thread-2",
2,
))
except e:
print("Cannot start thread")
print(e)
class Premium:
def __init__(self):
logging.basicConfig()
broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny?autoAck=true'
self._cons = Consumer(broker_url, 'ocs_dmcs_consume', "YAML")
try:
_thread.start_new_thread(self.do_it, (
"thread-1",
2,
))
except e:
print("Cannot start thread")
print(e)
def mycallback(self, ch, methon, properties, body):
print(" ")
print(">>>>>>>>>>>>>>><<<<<<<<<<<<<<<<")
print((" [x] method Received %r" % methon))
print((" [y] properties Received %r" % properties))
print((" [z] body Received %r" % body))
print("Message done")
print("Still listening...")
def do_it(self, threadname, delay):
print("Before run call")
self._cons.run(self.mycallback)
print("After run call - not blocking")
class Premium:
def __init__(self):
logging.basicConfig()
#os.system('rabbitmqctl -p /tester purge_queue firehose')
#os.system('rabbitmqctl -p /tester purge_queue ack_publish')
#broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny?heartbeat=300'
broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny'
#pub_broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Ftester'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#self._cons = FirehoseConsumer(broker_url, 'firehose', "YAML")
self._cons = Consumer(broker_url, 'ar_foreman_consume', "YAML")
try:
_thread.start_new_thread( self.do_it, ("thread-1", 2,) )
except e:
print("Cannot start thread")
print(e)
self._cons2 = Consumer(broker_url, 'pp_foreman_consume', "YAML")
try:
_thread.start_new_thread( self.do_it2, ("thread-3", 2,) )
except e:
print("Cannot start thread")
print(e)
cdm = toolsmod.intake_yaml_file("L1SystemCfg.yaml")
self.ccd_list = cdm['ROOT']['CCD_LIST']
def mycallback(self, ch, methon, properties, body):
print(" ")
print(">>>>>>>>>>>>>>><<<<<<<<<<<<<<<<")
print((" [z] body Received %r" % body))
print(">>>>>>>>>>>>>>><<<<<<<<<<<<<<<<")
#print("Message done")
#print("Still listening...")
def mycallback2(self, ch, methon, properties, body):
print(" ")
print("++++++++++++++-----------+++++++++++++++")
print((" [z] body Received %r" % body))
print("++++++++++++++-----------+++++++++++++++")
#print("Message done")
#print("Still listening...")
def do_it(self, threadname, delay):
#example = ExampleConsumer('amqp://*****:*****@141.142.208.191:5672/%2Fbunny')
print("Before run call")
self._cons.run(self.mycallback)
print("After run call - not blocking")
def do_it2(self, threadname, delay):
#example = ExampleConsumer('amqp://*****:*****@141.142.208.191:5672/%2Fbunny')
print("Before run call")
self._cons2.run(self.mycallback2)
print("After run call - not blocking")
def test_ocs_acksubscriber(self):
try:
cdm = toolsmod.intake_yaml_file(
"../../tests/yaml/L1SystemCfg_Test_ocs_bridge.yaml")
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to fine CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]["BASE_BROKER_ADDR"]
# dmcs publisher
dmcs_pub_name = cdm[ROOT]["DMCS_BROKER_PUB_NAME"]
dmcs_pub_pwd = cdm[ROOT]["DMCS_BROKER_PUB_PASSWD"]
dmcs_broker_pub_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_pwd + "@" + \
broker_addr
self.dmcs_publisher = SimplePublisher(dmcs_broker_pub_url, "YAML")
# dmcs consumer
dmcs_name = cdm[ROOT]["DMCS_BROKER_NAME"]
dmcs_pwd = cdm[ROOT]["DMCS_BROKER_PASSWD"]
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_pwd + "@" + \
broker_addr
self.dmcs_consumer = Consumer(dmcs_broker_url, "ocs_dmcs_consume",
"thread-dmcs-consume",
self.on_ocs_message, "YAML")
self.dmcs_consumer.start()
# ocs consumer from DMCS
ocs_name = cdm[ROOT]["OCS_BROKER_NAME"]
ocs_pwd = cdm[ROOT]["OCS_BROKER_PASSWD"]
# FIXME: New OCS account for consumer test_dmcs_ocs_publish
ocs_broker_url = "amqp://" + "AFM" + ":" +\
"AFM" + "@" +\
broker_addr
self.ocs_consumer = Consumer(ocs_broker_url, "test_dmcs_ocs_publish",
"thread-ocs-consume",
self.on_dmcs_message, "YAML")
self.ocs_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority("../../messages.yaml")
self.send_messages()
sleep(10)
os.killpg(os.getpgid(self.cmdListener.pid), signal.SIGTERM)
os.killpg(os.getpgid(self.ackSubscriber.pid), signal.SIGTERM)
print("MY OCS MESSAGES: %s" % self.ocs_consumer_msg_list)
self.verify_ocs_messages()
print("Finished with CommandListener tests.")
def __start_race(self):
self.buffer = __buffer_sensor__()
self.producer_race = Producer(self.buffer, self.sensor, self.race,
'race')
self.consumer_race = Consumer(self.buffer, self.race, self.publisher,
'race')
self.producer_race.start()
self.consumer_race.start()
return ''
def __start_qualification(self):
self.buffer = __buffer_sensor__()
self.producer_qualification = Producer(self.buffer, self.sensor,
self.race, 'qualification')
self.consumer_qualification = Consumer(self.buffer, self.race,
self.publisher, 'qualification')
self.producer_qualification.start()
self.consumer_qualification.start()
return ''
def Welfare(self, output_list):
# Initiate an instance of Consumer
C = Consumer(self.c, self.g, self.f, self.A, self.B, self.T)
# Generate the utility list of the consumer instance
utility_list = C.Utility(output_list)
'''
An utilitarian welfare function
'''
welfare = -1 * np.sum(utility_list)
return welfare
def test_ocs_acksubscriber(self):
try:
cdm = toolsmod.intake_yaml_file("../../tests/yaml/L1SystemCfg_Test_ocs_bridge.yaml")
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to fine CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]["BASE_BROKER_ADDR"]
# dmcs publisher
dmcs_pub_name = cdm[ROOT]["DMCS_BROKER_PUB_NAME"]
dmcs_pub_pwd = cdm[ROOT]["DMCS_BROKER_PUB_PASSWD"]
dmcs_broker_pub_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_pwd + "@" + \
broker_addr
self.dmcs_publisher = SimplePublisher(dmcs_broker_pub_url, "YAML")
# dmcs consumer
dmcs_name = cdm[ROOT]["DMCS_BROKER_NAME"]
dmcs_pwd = cdm[ROOT]["DMCS_BROKER_PASSWD"]
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_pwd + "@" + \
broker_addr
self.dmcs_consumer = Consumer(dmcs_broker_url, "ocs_dmcs_consume", "thread-dmcs-consume",
self.on_ocs_message, "YAML")
self.dmcs_consumer.start()
# ocs consumer from DMCS
ocs_name = cdm[ROOT]["OCS_BROKER_NAME"]
ocs_pwd = cdm[ROOT]["OCS_BROKER_PASSWD"]
# FIXME: New OCS account for consumer test_dmcs_ocs_publish
ocs_broker_url = "amqp://" + "AFM" + ":" +\
"AFM" + "@" +\
broker_addr
self.ocs_consumer = Consumer(ocs_broker_url, "test_dmcs_ocs_publish", "thread-ocs-consume",
self.on_dmcs_message, "YAML")
self.ocs_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority("../../messages.yaml")
self.send_messages()
sleep(10)
os.killpg(os.getpgid(self.cmdListener.pid), signal.SIGTERM)
os.killpg(os.getpgid(self.ackSubscriber.pid), signal.SIGTERM)
print("MY OCS MESSAGES: %s" % self.ocs_consumer_msg_list)
self.verify_ocs_messages()
print("Finished with CommandListener tests.")
def setup_consumer_thread(self, consumer_params):
url = consumer_params['amqp_url']
q = consumer_params['queue']
threadname = consumer_params['name']
callback = consumer_params['callback']
format = consumer_params['format']
new_thread = Consumer(url, q, threadname, callback, format)
new_thread.start()
sleep(1)
return new_thread
def __init__(self):
logging.basicConfig()
broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny?autoAck=true'
self._cons = Consumer(broker_url, 'ocs_dmcs_consume', "YAML")
try:
_thread.start_new_thread(self.do_it, (
"thread-1",
2,
))
except e:
print("Cannot start thread")
print(e)
def start_consumer(self, broker_url, format):
self.influx_consumer = Consumer(self.broker_url, "audit_consume",
format)
try:
_thread.start_new_thread(self.run_influx_consumer, (
"thread-influx-consumer",
2,
))
except:
LOGGER.critical('Cannot start influx consumer thread, exiting...')
sys.exit(99)
def setup_consumers(self):
threadname = "thread-" + self._consume_queue
print("Threadname is %s" % threadname)
self._consumer = Consumer(self._base_broker_url, self._consume_queue)
try:
_thread.start_new_thread(self.run_consumer, (
threadname,
2,
))
print("Started Consumer Thread")
except:
sys.exit(99)
def run(self):
roomName = input('Please enter the name of the room: \n')
clientName = input('Please enter the name of the client: \n')
privateKey = bytearray(123)
res = self.enterRoom(roomName, clientName, privateKey)
if (res):
print('Entered')
self.consumer = Consumer(clientName, self.channel, self.queue_name)
self.consumer.start()
self.producer = Producer(clientName, self.channel, roomName,
privateKey)
self.producer.start()
def index_files(self):
thread_add_to_dic = threading.Thread(target=self.add_to_dicts2())
threads = []
for name in self.name_of_files:
thread = Consumer(self.name_of_files[name], name, self.file_type,
self.main_dictionary, 1, self.temp_position_dic)
threads.append(thread)
thread.start()
thread_add_to_dic.start()
thread_add_to_dic.join()
for i in range(0, len(threads)):
threads[i].stop_thread()
for i in range(0, len(threads)):
threads[i].join()
def __init__(self):
printc("Starting...")
custom_print.define_new_name(self.PROGRAM_NAME)
self._sb_mach = Scoreboard
self._machine_prefix = 'D:'
self._machine_publish_q = Q_DIST_PUBLISH
self._machine_consume_q = Q_DIST_CONSUME
# Messages we can recieve from BaseForeman
self._msg_actions_bf = {
JOB_REQUEST: self.process_bf_job_request,
DISTRIBUTOR_REQUEST: self.process_bf_distributor_request,
STANDBY: self.process_bf_standby,
READOUT: self.process_bf_readout,
CANCEL: self.process_bf_cancel,
TRANSFER_DONE: self.process_bf_transfer_done
}
# Run parent init, starts distributor scoreboard and consumer
self.parent_init(SCOREBOARD_DB_DIST, PROGRAM_NAME, REGISTER_DISTRIBUTOR)
# BaseForeman consumer
printc("Creating BaseForeman consumer...")
self._bf_consumer = Consumer(self._broker_url, Q_NCSA_CONSUME)
try:
thread.start_new_thread(self.run_bf_consumer, ())
except:
printc("Thread run_bf_consumer failed, quitting...")
sys.exit()
return
def parent_init(self, db_num, prog_name, type):
custom_print.define_new_name(self.PROGRAM_NAME)
# Create machine scoreboard
self._sb_mach = Scoreboard(db_num, prog_name, type)
# Messaging URL (rabbitmq server IP)
self._broker_url = "amqp://" + AMQP_BF_USER + ":" + AMQP_BF_PSWD + "@" + AMQP_BROKER_ADDR + ":" + AMQP_BROKER_PORT + "/" + AMQP_BROKER_VHOST
# Publisher object for sending messages to rabbit
printc("Creating publisher...")
self._publisher = SimplePublisher(self._broker_url)
# Machine messages
self._msg_actions_mach = {
'TRANSFER_DONE': self.process_transfer_done, # Machine done with the current job
'REGISTER': self.process_register, # New machine wants to join
'DEREGISTER': self.process_deregister, # Machine is leaving
'STATE_UPDATE': self.process_state_update # Machine updating us on its state
}
# Machines register with us and let us know how they are doing
printc("Creating machine consumer...")
self._mach_consumer = Consumer(self._broker_url, self._machine_publish_q)
try:
thread.start_new_thread(self.run_mach_consumer, ())
except:
printc("Thread run_mach_consumer failed, quitting...")
sys.exit()
return
def test_ocs_eventsubscriber(self):
try:
cdm = toolsmod.intake_yaml_file("../../tests/yaml/L1SystemCfg_Test_ocs_bridge.yaml")
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to fine CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]["BASE_BROKER_ADDR"]
dmcs_name = cdm[ROOT]["DMCS_BROKER_NAME"]
dmcs_pwd = cdm[ROOT]["DMCS_BROKER_PASSWD"]
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_pwd + "@" + \
broker_addr
self.dmcs_consumer = Consumer(dmcs_broker_url, "ocs_dmcs_consume", "thread-dmcs-consume",
self.on_ocs_message, "YAML")
self.dmcs_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority("../../messages.yaml")
self.send_messages()
sleep(10)
os.killpg(os.getpgid(self.cmd.pid), signal.SIGTERM)
self.verify_ocs_messages()
print("Finished with EventSubscriber tests.")
def __init__(self):
logging.basicConfig()
broker_url = 'amqp://*****:*****@141.142.238.10:5672/%2Fbunny'
self._cons = Consumer(broker_url, 'fetch_consume_from_f1', "Threadthread", self.mycallback, "YAML")
Readout_Image_IDs = []
Header_Info_Dict = {}
Take_Images_Done_Message = {}
def __init__(self):
printc("Setting up DMCS...")
self._options = "\
1 - (READY) Send Job Information\n\
2 - (SET) Send Standby Message\n\
3 - (GO) Send Readout Message\n\
4 - (RESET) Cancel a Job\n\
0 - (EXIT) Quit DMCS Simulator\n"
self._broker_url = 'amqp://' + AMQP_DMCS_USER + ':' + AMQP_DMCS_PSWD + '@' + AMQP_BROKER_ADDR + ':' + AMQP_BROKER_PORT + '/' + AMQP_BROKER_VHOST
printc('Using broker url: %s' % self._broker_url)
printc("Declaring and binding exchanges...")
printc("Attempting to create a consumer for the '%s' queue." % (Q_DMCS_CONSUME))
self._dmcs_consumer = Consumer(self._broker_url, Q_DMCS_CONSUME)
try:
printc("Attempting to start the consumer thread...")
thread.start_new_thread(self.run_dmcs_consumer, ())
except:
printc("Failed to start consumer thread, quitting...")
sys.exit()
printc("Done setting up consumer thread.")
printc("Setting up publisher...")
self._publisher = SimplePublisher(self._broker_url)
printc("Done creating publisher.")
self._job_msg = {}
self._job_msg['MSG_TYPE'] = 'JOB'
self._job_msg['JOB_NUM'] = 0
self._job_msg['RAFT_NUM'] = 1
self._standby_msg = {}
self._standby_msg['MSG_TYPE'] = 'STANDBY'
self._readout_msg = {}
self._readout_msg['MSG_TYPE'] = 'READOUT'
self._stop_msg = {}
self._stop_msg['MSG_TYPE'] = 'CANCEL'
self._shutdown_msg = {}
self._shutdown_msg['MSG_TYPE'] = 'SHUTDOWN'
def __init__(self, securities):
self.consumer_group = []
for security in securities:
self.consumer_group.append(Consumer(security, self))
self.producer = Producer(self.consumer_group, self)
self.flag = AtomicNumber()
self.i = 0
def setup_consumers(self):
LOGGER.info('Distributor %s setting up consumer on %s', self._name,
self._ncsa_broker_url)
LOGGER.info('Starting new thread on consumer method')
threadname = "thread-" + self._consume_queue
self._consumer = Consumer(self._ncsa_broker_url, self._consume_queue)
try:
_thread.start_new_thread(self.run_consumer, (
threadname,
2,
))
LOGGER.info('Started distributor consumer thread %s', threadname)
except:
LOGGER.critical(
'Cannot start Distributor consumer thread, exiting...')
sys.exit(107)
class QueryRunner():
def __init__(self, host,port,srcFile):
file=open(srcFile)
data=file.read()
self.consumer = Consumer(data+"@"+host,port, self,runOnlyOnce=True)
def run(self):
while(True):
try:
self.consumer.waitData()
data = self.consumer.getData()
if data != None:
print str(data)
except Consumer.ConsumerException,e:
print e.msg
return
def __init__(self):
logging.basicConfig()
#os.system('rabbitmqctl -p /tester purge_queue firehose')
#os.system('rabbitmqctl -p /tester purge_queue ack_publish')
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny?heartbeat=300'
broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny?autoAck=true'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Ftester'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#self._cons = FirehoseConsumer(broker_url, 'firehose', "YAML")
self._cons = Consumer(broker_url, 'ocs_dmcs_consume', "YAML")
try:
_thread.start_new_thread( self.do_it, ("thread-1", 2,) )
except e:
print("Cannot start thread")
print(e)
time.sleep(420)
class Premium:
def __init__(self):
logging.basicConfig()
self.ack_test = True
#os.system('rabbitmqctl -p /tester purge_queue firehose')
#os.system('rabbitmqctl -p /tester purge_queue ack_publish')
broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Ftester'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#self._cons = FirehoseConsumer(broker_url, 'firehose', "YAML")
self.sp1 = SimplePublisher(
'amqp://*****:*****@141.142.208.191:5672/%2Fbunny')
cdm = toolsmod.intake_yaml_file('ForemanCfg.yaml')
self.fdict = cdm[ROOT]['XFER_COMPONENTS']['ARCHIVE_FORWARDERS']
self.fwdrs = list(self.fdict.keys())
self._cons = Consumer(broker_url, 'ar_foreman_ack_publish', "YAML")
#self._cons = Consumer(broker_url, 'ar_foreman_consume', "YAML")
try:
_thread.start_new_thread(self.do_it, (
"thread-1",
2,
))
except e:
print("Cannot start thread")
print(e)
def mycallback(self, ch, methon, properties, body):
print(" ")
print(">>>>>>>>>>>>>>><<<<<<<<<<<<<<<<")
print((" [x] method Received %r" % methon))
print((" [y] properties Received %r" % properties))
print((" [z] body Received %r" % body))
print("Message done")
print("Still listening...")
def do_it(self, threadname, delay):
#example = ExampleConsumer('amqp://*****:*****@141.142.208.191:5672/%2Fbunny')
print("Before run call")
self._cons.run(self.mycallback)
print("After run call - not blocking")
def Welfare(self,output_list):
# Initiate an instance of Consumer
C = Consumer(self.c,self.g,self.f,self.A,self.B,self.T)
# Generate the utility list of the consumer instance
utility_list = C.Utility(output_list)
# Generate the parameters for the welfare function
# Obtain overall weight for the welfare function
overall_weight = float(1)
# Obtain individual utility weights for each consumer for the welfare function
utility_weights = Loop(self.c)
# Calculate the welfare
# The welfare function is defined as the (sum of U^(2x)) ^ y, where x are individual weights and y is the overall weight
utility_list_square = np.power(np.e,utility_list)
welfare = (np.sum(np.power(utility_list_square , utility_weights)))
# Obtain the negative of the welfare for minimization
neg_welfare = welfare * -1
return neg_welfare
def createConsumer(self, connect, stats, routingKey):
channel = connect.channel()
if self.consumerTxSize > 0:
channel.tx_select()
self.configureQueue(connect, routingKey)
return Consumer(channel, routingKey, self.queueName,
self.consumerRateLimit, self.consumerTxSize,
self.autoAck, self.multiAckEvery, stats,
self.consumerMsgCount, self.timeLimit)
def test_ar_ctrl(self):
try:
cdm = toolsmod.intake_yaml_file(
'/home/centos/src/git/ctrl_iip/python/lsst/iip/tests/yaml/L1SystemCfg_Test_ar-ctrl.yaml'
)
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
afm_name = cdm[ROOT]["AFM_BROKER_NAME"]
afm_pwd = cdm[ROOT]["AFM_BROKER_PASSWD"]
afm_pub_name = cdm[ROOT]["AFM_BROKER_PUB_NAME"]
afm_pub_pwd = cdm[ROOT]["AFM_BROKER_PUB_PASSWD"]
self.xfer_root = cdm[ROOT]["ARCHIVE"]["ARCHIVE_XFER_ROOT"]
ar_dev_pub_broker_url = "amqp://" + afm_pub_name + ":" +\
afm_pub_pwd + "@" +\
broker_addr
self.ar_dev_publisher = SimplePublisher(ar_dev_pub_broker_url, "YAML")
ar_dev_broker_url = "amqp://" + afm_name + ":" +\
afm_pwd + "@" +\
broker_addr
self.ar_dev_consumer = Consumer(ar_dev_broker_url, self.ACK_FROM_CTRL,
"thread-ar-dev", self.on_ar_message,
"YAML", None)
self.ar_dev_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority(
'/home/centos/src/git/ctrl_iip/python/lsst/iip/messages.yaml')
self.send_messages()
sleep(10)
self.verify_ar_messages()
self.verify_path_files()
print("Finished with AR CTRL tests.")
def setup_consumers(self):
"""This method sets up a message listener from each entity
with which the BaseForeman has contact here. These
listeners are instanced in this class, but their run
methods are each called as a separate thread. While
pika does not claim to be thread safe, the manner in which
the listeners are invoked below is a safe implementation
that provides non-blocking, fully asynchronous messaging
to the BaseForeman.
The code in this file expects message bodies to arrive as
YAML'd python dicts, while in fact, message bodies are sent
on the wire as XML; this way message format can be validated,
versioned, and specified in just one place. To make this work,
there is an object that translates the params dict to XML, and
visa versa. The translation object is instantiated by the consumer
and acts as a filter before sending messages on to the registered
callback for processing.
"""
LOGGER.info('Setting up consumers on %s', self._base_broker_url)
LOGGER.info('Running start_new_thread on all consumer methods')
self._dmcs_consumer = Consumer(self._base_broker_url, self.DMCS_PUBLISH, self._base_msg_format)
try:
_thread.start_new_thread( self.run_dmcs_consumer, ("thread-dmcs-consumer", 2,) )
except:
LOGGER.critical('Cannot start DMCS consumer thread, exiting...')
sys.exit(99)
self._forwarder_consumer = Consumer(self._base_broker_url, self.FORWARDER_PUBLISH, self._base_msg_format)
try:
_thread.start_new_thread( self.run_forwarder_consumer, ("thread-forwarder-consumer", 2,) )
except:
LOGGER.critical('Cannot start FORWARDERS consumer thread, exiting...')
sys.exit(100)
self._ncsa_consumer = Consumer(self._base_broker_url, self.NCSA_PUBLISH, self._base_msg_format)
try:
_thread.start_new_thread( self.run_ncsa_consumer, ("thread-ncsa-consumer", 2,) )
except:
LOGGER.critical('Cannot start NCSA consumer thread, exiting...')
sys.exit(101)
self._ack_consumer = Consumer(self._base_broker_url, self.ACK_PUBLISH, self._base_msg_format)
try:
_thread.start_new_thread( self.run_ack_consumer, ("thread-ack-consumer", 2,) )
except:
LOGGER.critical('Cannot start ACK consumer thread, exiting...')
sys.exit(102)
LOGGER.info('Finished starting all three consumer threads')
def start(self):
if not os.path.exists('resources/'):
os.mkdir('resources/')
if not os.path.exists('jobs/'):
os.mkdir('jobs/')
self.job_queue = JoinableQueue()
self.consumers = [
Consumer(self.job_queue, self.manifest)
for _ in range(self.num_processes)
]
for consumer in self.consumers:
consumer.start()
def __init__(self):
self._broker_url = 'amqp://' + AMQP_DMCS_USER + ':' + AMQP_DMCS_PSWD + '@' + AMQP_BROKER_ADDR + ':' + AMQP_BROKER_PORT + '/' + AMQP_BROKER_VHOST
connection = pika.BlockingConnection(pika.URLParameters(self._broker_url))
channel = connection.channel()
channel.exchange_declare(exchange=EXCHANGE_NAME, exchange_type=EXCHANGE_TYPE)
channel.queue_declare(queue='forwarder_publish')
channel.queue_declare(queue='forwarder_consume')
channel.queue_declare(queue='distributor_publish')
channel.queue_declare(queue='distributor_consume')
channel.queue_declare(queue='dmcs_consume')
channel.queue_declare(queue='dmcs_publish')
channel.queue_declare(queue='ncsa_consume')
channel.queue_declare(queue='ncsa_publish')
channel.queue_declare(queue='ack_consume')
channel.queue_declare(queue='ack_publish')
channel.queue_declare(queue='reports')
channel.queue_bind(exchange=EXCHANGE_NAME, queue=Q_DMCS_CONSUME, routing_key=Q_DMCS_CONSUME)
channel.queue_bind(exchange=EXCHANGE_NAME, queue=Q_DMCS_PUBLISH, routing_key=Q_DMCS_PUBLISH)
connection.close()
consumer = Consumer(self._broker_url, Q_FORW_CONSUME)
consumer.run_2(None)
consumer = Consumer(self._broker_url, Q_DIST_CONSUME)
consumer.run_2(None)
return
class Client():
def __init__(self, address):
self.connection = pika.BlockingConnection(
pika.ConnectionParameters(address))
self.channel = self.connection.channel()
def run(self):
roomName = input('Please enter the name of the room: \n')
clientName = input('Please enter the name of the client: \n')
privateKey = bytearray(123)
res = self.enterRoom(roomName, clientName, privateKey)
if (res):
print('Entered')
self.consumer = Consumer(clientName, self.channel, self.queue_name)
self.consumer.start()
self.producer = Producer(clientName, self.channel, roomName,
privateKey)
self.producer.start()
def enterRoom(self, room_name, name, privateKey):
# Hash the room name and client name with private key
room_name = hmac.new(privateKey, room_name.encode(),
hashlib.sha256).hexdigest()
name = hmac.new(privateKey, name.encode(), hashlib.sha256).hexdigest()
# Generate a random unique key for the queue name
unique_key = ''.join(
random.SystemRandom().choice(string.ascii_uppercase +
string.digits) for _ in range(64))
# set queue names
self.queue_name = room_name + ":" + name
self.exchange = self.channel.exchange_declare(exchange=room_name,
exchange_type='fanout')
self.queue = self.channel.queue_declare(queue=self.queue_name)
self.channel.queue_bind(exchange=room_name, queue=self.queue_name)
return True
def __init__(self):
try:
# Create a lock for critical sections (used when canceling jobs)
self._lock = thread.allocate_lock()
self.child_init()
# Create a temporary name until unique name is assigned
c_name = ''.join(random.choice(string.ascii_letters) for x in range(NAME_LENGTH))
c_name = self._type + "_" + c_name
custom_print.define_new_name(c_name)
printc("Starting...")
# Register the machine with foreman before doing anything else
self._register = Registration(self._publish_queue,
self._consume_queue)
# Block until a unique name is received
self._name = self._register.request_name()
except:
return
try:
# Debug print setup
custom_print.define_new_name(self._name)
# Internal variable of the current state
self._current_state = IDLE
# Messaging setup
self._broker_url = "amqp://" + AMQP_MACH_USER + ":" + AMQP_MACH_PSWD + "@" + AMQP_BROKER_ADDR + ":" + AMQP_BROKER_PORT + "/" + AMQP_BROKER_VHOST
self._consume_queue = self._name + "_consume"
self._home_dir = XFER_DIR
# Messages foreman can send to us
self._msg_actions = {
JOB: self.process_foreman_job,
STANDBY: self.process_foreman_standby,
READOUT: self.process_foreman_readout,
CANCEL: self.process_foreman_cancel
}
# Publisher to send messages to foreman
printc("Setting up publisher...")
self._publisher = SimplePublisher(self._broker_url)
# Consumer for getting messages from foreman
printc("Setting up consumer...")
self._consumer = Consumer(self._broker_url, self._consume_queue)
except:
pass
# Run blocking consumer
try:
self.run_consumer()
except:
pass
# Alert foreman this machine is shutting down
self.deregister()
return
def test_ar_ctrl(self):
try:
cdm = toolsmod.intake_yaml_file('/home/centos/src/git/ctrl_iip/python/lsst/iip/tests/yaml/L1SystemCfg_Test_ar-ctrl.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
afm_name = cdm[ROOT]["AFM_BROKER_NAME"]
afm_pwd = cdm[ROOT]["AFM_BROKER_PASSWD"]
afm_pub_name = cdm[ROOT]["AFM_BROKER_PUB_NAME"]
afm_pub_pwd = cdm[ROOT]["AFM_BROKER_PUB_PASSWD"]
self.xfer_root = cdm[ROOT]["ARCHIVE"]["ARCHIVE_XFER_ROOT"]
ar_dev_pub_broker_url = "amqp://" + afm_pub_name + ":" +\
afm_pub_pwd + "@" +\
broker_addr
self.ar_dev_publisher = SimplePublisher(ar_dev_pub_broker_url, "YAML")
ar_dev_broker_url = "amqp://" + afm_name + ":" +\
afm_pwd + "@" +\
broker_addr
self.ar_dev_consumer = Consumer(ar_dev_broker_url, self.ACK_FROM_CTRL, "thread-ar-dev",
self.on_ar_message, "YAML", None)
self.ar_dev_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority('/home/centos/src/git/ctrl_iip/python/lsst/iip/messages.yaml')
self.send_messages()
sleep(10)
self.verify_ar_messages()
self.verify_path_files()
print("Finished with AR CTRL tests.")
def main():
stack = Manager().list()
n = Semaphore(0)
s = Semaphore(1)
# Create Producer and Consumer threads
producer1 = Producer(stack, n, s, 1)
#producer2 = Producer(stack, n, s, 2)
#producer3 = Producer(stack, n, s, 3)
#producer4 = Producer(stack, n, s, 4)
#producer5 = Producer(stack, n, s, 5)
consumer = Consumer(stack, n, s)
# Start all the threads
producer1.start()
#producer2.start()
#producer3.start()
#producer4.start()
#producer5.start()
consumer.start()
time.sleep(2)
consumer.stop()
time.sleep(0.1)
producer1.stop()
# Wait for threads to finish
producer1.join()
#producer2.join()
#producer3.join()
#producer4.join()
#producer5.join()
consumer.join()
print("Done.")
class TestDMCS_AT:
ocs_pub_broker_url = None
ocs_publisher = None
ocs_consumer = None
ocs_consumer_msg_list = []
at_pub_broker_url = None
at_publisher = None
at_consumer = None
at_consumer_msg_list = []
EXPECTED_AT_MESSAGES = 1
EXPECTED_OCS_MESSAGES = 1
prp = toolsmod.prp
#DP = toolsmod.DP #Debug print
DP = True #Debug print
def test_dmcs(self, Dmcs):
self.dmcs = Dmcs
#logging.warning("Logging is Working!")
LOGGER.critical("LOGGING is Working!")
#self.LOGGER.info("self Logging is Working!")
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
ocs_name = cdm[ROOT]['OCS_BROKER_NAME']
ocs_passwd = cdm[ROOT]['OCS_BROKER_PASSWD']
ocs_pub_name = cdm[ROOT]['OCS_BROKER_PUB_NAME']
ocs_pub_passwd = cdm[ROOT]['OCS_BROKER_PUB_PASSWD']
ocs_broker_url = "amqp://" + ocs_name + ":" + \
ocs_passwd + "@" + \
broker_addr
self.ocs_pub_broker_url = "amqp://" + ocs_pub_name + ":" + \
ocs_pub_passwd + "@" + \
broker_addr
self.ocs_publisher = SimplePublisher(self.ocs_pub_broker_url, "YAML")
at_name = cdm[ROOT]['AUX_BROKER_NAME']
at_passwd = cdm[ROOT]['AUX_BROKER_PASSWD']
at_pub_name = cdm[ROOT]['AUX_BROKER_PUB_NAME']
at_pub_passwd = cdm[ROOT]['AUX_BROKER_PUB_PASSWD']
at_broker_url = "amqp://" + at_name + ":" + \
at_passwd + "@" + \
broker_addr
self.at_pub_broker_url = "amqp://" + at_pub_name + ":" + \
at_pub_passwd + "@" + \
broker_addr
self.at_publisher = SimplePublisher(self.at_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.ocs_consumer = Consumer(ocs_broker_url,'dmcs_ocs_publish', 'thread-ocs',
self.on_ocs_message,'YAML')
self.ocs_consumer.start()
self.at_consumer = Consumer(at_broker_url,'at_foreman_consume', 'thread-at',
self.on_at_message,'YAML')
self.at_consumer.start()
sleep(3)
if self.DP:
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(3)
self.verify_ocs_messages()
sleep(3)
self.verify_at_messages()
sleep(2)
self.ocs_consumer.stop()
self.ocs_consumer.join()
self.at_consumer.stop()
self.at_consumer.join()
if self.DP:
print("Finished with DMCS AT tests.")
def send_messages(self):
if self.DP:
print("Starting send_messages")
# Tests only an AT device
self.clear_message_lists()
self.EXPECTED_OCS_MESSAGES = 6
self.EXPECTED_AT_MESSAGES = 4
msg = {}
msg['MSG_TYPE'] = "STANDBY"
msg['DEVICE'] = 'AT'
msg['CMD_ID'] = '17718411'
msg['CFG_KEY'] = "2C16"
msg['ACK_ID'] = 'AT_4'
msg['ACK_DELAY'] = 2
time.sleep(2)
if self.DP:
print("Sending AT STANDBY")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "DISABLE"
msg['DEVICE'] = 'AT'
msg['CMD_ID'] = '17718411'
msg['ACK_ID'] = 'AT_6'
msg['ACK_DELAY'] = 2
time.sleep(2)
if self.DP:
print("Sending AT DISABLE")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
sleep(2.0)
# Make sertain scoreboard values are being set
if self.DP:
print("Checking State Scoreboard entries.")
#assert self.dmcs.STATE_SCBD.get_auxdev_state() == 'DISABLE'
assert self.dmcs.STATE_SCBD.get_auxtel_state() == 'DISABLE'
msg = {}
msg['MSG_TYPE'] = "ENABLE"
msg['DEVICE'] = 'AT'
msg['CMD_ID'] = '17718411'
msg['ACK_ID'] = 'AT_11'
msg['ACK_DELAY'] = 2
time.sleep(4)
if self.DP:
print("Sending AT ENABLE")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
sleep(2.0)
#assert self.dmcs.STATE_SCBD.get_auxdev_state() == 'ENABLE'
assert self.dmcs.STATE_SCBD.get_auxtel_state() == 'ENABLE'
msg = {}
msg['MSG_TYPE'] = "DMCS_AT_START_INTEGRATION"
msg['IMAGE_ID'] = 'IMG_4280'
msg['IMAGE_INDEX'] = 2
msg['IMAGE_SEQUENCE_NAME'] = 'XX_seq'
msg['IMAGES_IN_SEQUENCE'] = 8
time.sleep(5)
if self.DP:
print("Sending Start Integration Message")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "DMCS_AT_END_READOUT"
msg['IMAGE_ID'] = 'IMG_4280'
msg['IMAGE_INDEX'] = 2
msg['IMAGE_SEQUENCE_NAME'] = 'XX_seq'
msg['IMAGES_IN_SEQUENCE'] = 8
time.sleep(5)
if self.DP:
print("Sending READOUT Message")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "DMCS_AT_HEADER_READY"
msg['IMAGE_ID'] = 'IMG_4280'
msg['FILENAME'] = "/mnt/headers/IMG_4277.header"
msg['ACK_ID'] = 'DMCS_END_READOUT_ACK_82'
time.sleep(5)
if self.DP:
print("Sending READOUT Message")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
time.sleep(5)
if self.DP:
print("Message Sender done")
def clear_message_lists(self):
self.ocs_consumer_msg_list = []
self.at_consumer_msg_list = []
def verify_ocs_messages(self):
if self.DP:
print("Messages received by verify_ocs_messages:")
self.prp.pprint(self.ocs_consumer_msg_list)
len_list = len(self.ocs_consumer_msg_list)
if len_list != self.EXPECTED_OCS_MESSAGES:
if self.DP:
print("Incorrect number of messages received by OCS ACK Consumer.")
print("Messages received by verify_ocs_messages:")
self.prp.pprint(self.ocs_consumer_msg_list)
pytest.fail('OCS simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_OCS_MESSAGES, len_list))
# Now check num keys in each message and check for key errors
for i in range(0, len_list):
msg = self.ocs_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following OCS Bridge response message failed when compared with the sovereign example: %s" % msg)
if self.DP:
print("Responses to OCS Bridge pass verification.")
def verify_at_messages(self):
if self.DP:
print("Messages received by verify_at_messages:")
self.prp.pprint(self.at_consumer_msg_list)
len_list = len(self.at_consumer_msg_list)
if self.DP:
print("The number of messages the AT received is %s" % len_list)
if len_list != self.EXPECTED_AT_MESSAGES:
if self.DP:
print("Messages received by verify_at_messages:")
self.prp.pprint(self.at_consumer_msg_list)
pytest.fail('AT simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_AT_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.at_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
if self.DP:
print("The following message to the AT failed when compared with " \
"the sovereign example: %s" % msg)
pytest.fail("The following message to the AT failed when compared with " \
"the sovereign example: %s" % msg)
if self.DP:
print("Messages to the AT pass verification.")
def on_ocs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_dmcs-at - incoming on_ocs_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.ocs_consumer_msg_list.append(body)
def on_at_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_dmcs-at - incoming on_at_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.at_consumer_msg_list.append(body)
class TestPpDev:
dmcs_pub_broker_url = None
dmcs_publisher = None
dmcs_consumer = None
dmcs_consumer_msg_list = []
ncsa_pub_broker_url = None
ncsa_publisher = None
ncsa_ctrl_consumer = None
ncsa_consumer_msg_list = []
F1_pub_broker_url = None
F1_publisher = None
F1_consumer = None
f1_consumer_msg_list = []
F2_pub_broker_url = None
F2_publisher = None
F2_consumer = None
f2_consumer_msg_list = []
EXPECTED_NCSA_MESSAGES = 1
EXPECTED_DMCS_MESSAGES = 1
EXPECTED_F1_MESSAGES = 1
EXPECTED_F2_MESSAGES = 1
ccd_list = [14,17,21,86]
prp = toolsmod.prp # pretty printing
DP = toolsmod.DP # Debug printing either True of False...set here to override this file only
def test_ppdev(self, Ppdev):
self.ppdev = Ppdev
try:
cdm = toolsmod.intake_yaml_file('./tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
dmcs_name = cdm[ROOT]['DMCS_BROKER_NAME']
dmcs_passwd = cdm[ROOT]['DMCS_BROKER_PASSWD']
dmcs_pub_name = cdm[ROOT]['DMCS_BROKER_PUB_NAME']
dmcs_pub_passwd = cdm[ROOT]['DMCS_BROKER_PUB_PASSWD']
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_passwd + "@" + \
broker_addr
dmcs_pub_broker_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_passwd + "@" + \
broker_addr
self.dmcs_publisher = SimplePublisher(dmcs_pub_broker_url, "YAML")
ncsa_name = cdm[ROOT]['NCSA_BROKER_NAME']
ncsa_passwd = cdm[ROOT]['NCSA_BROKER_PASSWD']
ncsa_pub_name = cdm[ROOT]['NCSA_BROKER_PUB_NAME']
ncsa_pub_passwd = cdm[ROOT]['NCSA_BROKER_PUB_PASSWD']
ncsa_broker_url = "amqp://" + ncsa_name + ":" + \
ncsa_passwd + "@" + \
broker_addr
ncsa_pub_broker_url = "amqp://" + ncsa_pub_name + ":" + \
ncsa_pub_passwd + "@" + \
broker_addr
self.ncsa_publisher = SimplePublisher(ncsa_pub_broker_url, "YAML")
F1_name = 'F1'
F1_passwd = 'F1'
F1_pub_name = 'F1_PUB'
F1_pub_passwd = 'F1_PUB'
F1_broker_url = "amqp://" + F1_name + ":" + \
F1_passwd + "@" + \
broker_addr
F1_pub_broker_url = "amqp://" + F1_pub_name + ":" + \
F1_pub_passwd + "@" + \
broker_addr
self.f1_publisher = SimplePublisher(F1_pub_broker_url, "YAML")
F2_name = 'F2'
F2_passwd = 'F2'
F2_pub_name = 'F2_PUB'
F2_pub_passwd = 'F2_PUB'
F2_broker_url = "amqp://" + F2_name + ":" + \
F2_passwd + "@" + \
broker_addr
F2_pub_broker_url = "amqp://" + F2_pub_name + ":" + \
F2_pub_passwd + "@" + \
broker_addr
self.f2_publisher = SimplePublisher(F2_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.dmcs_consumer = Consumer(dmcs_broker_url,'dmcs_ack_consume', 'thread-dmcs',
self.on_dmcs_message,'YAML')
self.dmcs_consumer.start()
self.ncsa_consumer = Consumer(ncsa_broker_url,'ncsa_consume', 'thread-ncsa',
self.on_ncsa_message,'YAML')
self.ncsa_consumer.start()
self.F1_consumer = Consumer(F1_broker_url,'f1_consume', 'thread-f1',
self.on_f1_message,'YAML')
self.F1_consumer.start()
self.F2_consumer = Consumer(F2_broker_url,'f2_consume', 'thread-f2',
self.on_f2_message,'YAML')
self.F2_consumer.start()
sleep(3)
if self.DP:
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(6)
self.verify_ncsa_messages()
self.verify_F2_messages()
self.verify_F1_messages()
self.verify_dmcs_messages()
sleep(2)
self.dmcs_consumer.stop()
self.dmcs_consumer.join()
self.ncsa_consumer.stop()
self.ncsa_consumer.join()
self.F1_consumer.stop()
self.F1_consumer.join()
self.F2_consumer.stop()
self.F2_consumer.join()
if self.DP:
print("Finished with PP tests.")
def send_messages(self):
if self.DP:
print("Starting send_messages")
# Tests only an AR device
# self.clear_message_lists()
self.EXPECTED_NCSA_MESSAGES = 3
self.EXPECTED_DMCS_MESSAGES = 4
self.EXPECTED_F1_MESSAGES = 3
self.EXPECTED_F2_MESSAGES = 3
msg = {}
msg['MSG_TYPE'] = "PP_NEW_SESSION"
msg['SESSION_ID'] = 'SI_469976'
msg['ACK_ID'] = 'NEW_SESSION_ACK_44221'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
time.sleep(3)
if self.DP:
print("New Session Message")
self.dmcs_publisher.publish_message("pp_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "PP_NEXT_VISIT"
msg['VISIT_ID'] = 'XX_28272'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'NEW_VISIT_ACK_76'
msg['BORE_SIGHT'] = "231,123786456342, -45.3457156906, FK5"
time.sleep(2)
if self.DP:
print("Next Visit Message")
self.dmcs_publisher.publish_message("pp_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "PP_START_INTEGRATION"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444244'
msg['VISIT_ID'] = 'V14494'
msg['SESSION_ID'] = '4_14_7211511'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'PP_ACK_94671'
msg['CCD_LIST'] = [4,14,16,17,29,35,36]
time.sleep(4)
if self.DP:
print("PP_START_INTEGRATION Message")
self.dmcs_publisher.publish_message("pp_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "PP_READOUT"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444244'
msg['VISIT_ID'] = 'V14494'
msg['SESSION_ID'] = '4_14_7211511'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'PP_READOUT_ACK_44221'
time.sleep(4)
if self.DP:
print("PP_READOUT Message")
self.dmcs_publisher.publish_message("pp_foreman_consume", msg)
time.sleep(2)
if self.DP:
print("Message Sender done")
def verify_dmcs_messages(self):
if self.DP:
print("Messages received by verify_dmcs_messages:")
self.prp.pprint(self.dmcs_consumer_msg_list)
len_list = len(self.dmcs_consumer_msg_list)
if len_list != self.EXPECTED_DMCS_MESSAGES:
pytest.fail('DMCS simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_DMCS_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.dmcs_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following DMCS Bridge response message failed when compared with the sovereign example: %s" % msg)
print("Responses to DMCS Bridge pass verification.")
def verify_ncsa_messages(self):
if self.DP:
print("Messages received by verify_ncsa_messages:")
self.prp.pprint(self.ncsa_consumer_msg_list)
len_list = len(self.ncsa_consumer_msg_list)
if len_list != self.EXPECTED_NCSA_MESSAGES:
pytest.fail('NCSA simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_NCSA_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.ncsa_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following message to the NCSA Foreman failed when compared with the sovereign example: %s" % msg)
print("Messages to the NCSA Foreman pass verification.")
def verify_F1_messages(self):
if self.DP:
print("Messages received by verify_F1_messages:")
self.prp.pprint(self.f1_consumer_msg_list)
len_list = len(self.f1_consumer_msg_list)
if len_list != self.EXPECTED_F1_MESSAGES:
pytest.fail('F1 simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_F1_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.f1_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following message to F1 failed when compared with the sovereign example: %s" % msg)
else:
print("Messages to F1 pass verification.")
def verify_F2_messages(self):
if self.DP:
print("Messages received by verify_F2_messages:")
self.prp.pprint(self.f2_consumer_msg_list)
len_list = len(self.f2_consumer_msg_list)
if len_list != self.EXPECTED_F2_MESSAGES:
pytest.fail('F2 simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_F2_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.f2_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following message to F2 failed when compared with the sovereign example: %s" % msg)
else:
print("Messages to F2 pass verification.")
def on_dmcs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_pp_dev - incoming on_dmcs_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.dmcs_consumer_msg_list.append(body)
def on_ncsa_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_pp_dev - incoming on_ncsa_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.ncsa_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'NCSA_NEW_SESSION':
msg = {}
msg['MSG_TYPE'] = 'NCSA_NEW_SESSION_ACK'
msg['COMPONENT'] = 'NCSA_FOREMAN'
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
self.ncsa_publisher.publish_message(body['REPLY_QUEUE'], msg)
return
if body['MSG_TYPE'] == 'NCSA_NEXT_VISIT':
msg = {}
msg['MSG_TYPE'] = 'NCSA_NEXT_VISIT_ACK'
msg['COMPONENT'] = 'NCSA_FOREMAN'
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
self.ncsa_publisher.publish_message(body['REPLY_QUEUE'], msg)
return
if body['MSG_TYPE'] == 'NCSA_START_INTEGRATION':
msg = {}
msg['ACK_ID'] = body['ACK_ID']
msg['MSG_TYPE'] = 'NCSA_START_INTEGRATION_ACK'
msg['COMPONENT'] = 'NCSA_FOREMAN'
fwdrs = deepcopy(body['FORWARDERS'])
fwdr_list = fwdrs['FORWARDER_LIST']
ccd_list = fwdrs['CCD_LIST']
i = 1
msg['PAIRS'] = [] # This will be a list of dictionaries
for i in range(0,len(fwdr_list)):
fwdr = fwdr_list[i]
dist = {}
pair = {}
dist['FQN'] = "Distributor_" + str(i)
dist['NAME'] = "D" + str(i)
dist['HOSTNAME'] = "D" + str(i)
dist['TARGET_DIR'] = "/dev/null"
dist['IP_ADDR'] = "141.142.237.16" + str(i)
pair['FORWARDER'] = fwdr_list[i]
pair['CCD_LIST'] = ccd_list[i] #Get the list at index position i in ccd_list
pair['DISTRIBUTOR'] = dist
msg['PAIRS'].append(deepcopy(pair))
msg['ACK_BOOL'] = True
msg['JOB_NUM'] = body['JOB_NUM']
msg['IMAGE_ID'] = body['IMAGE_ID']
msg['VISIT_ID'] = body['VISIT_ID']
msg['SESSION_ID'] = body['SESSION_ID']
self.ncsa_publisher.publish_message(body['REPLY_QUEUE'], msg)
return
if body['MSG_TYPE'] == 'NCSA_READOUT':
# Find earlier Start Int message
st_int_msg = None
for msg in self.ncsa_consumer_msg_list:
if msg['MSG_TYPE'] == 'NCSA_START_INTEGRATION':
st_int_msg = msg
break
if st_int_msg == None:
pytest.fail("The NCSA_START_INTEGRATION message wasn't received before NCSA_READOUT in on_ncsa_msg")
# Now build response with previous message
msg = {}
msg['MSG_TYPE'] = 'NCSA_READOUT_ACK'
msg['JOB_NUM'] = body['JOB_NUM']
msg['IMAGE_ID'] = body['IMAGE_ID']
msg['VISIT_ID'] = body['VISIT_ID']
msg['SESSION_ID'] = body['SESSION_ID']
msg['COMPONENT'] = 'NCSA_FOREMAN'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
#msg['RESULT_LIST']['FORWARDER_LIST'] = st_int_msg['FORWARDERS']['FORWARDER_LIST']
ccd_list = st_int_msg['FORWARDERS']['CCD_LIST']
receipt_list = []
for i in range(0, len(ccd_list)):
receipt_list.append('Rec_x447_' + str(i))
msg['RESULT_LIST'] = {}
msg['RESULT_LIST']['RECEIPT_LIST'] = receipt_list
msg['RESULT_LIST']['CCD_LIST'] = list(ccd_list)
#sleep(2) #Give FWDRs time to respond with ack first
self.ncsa_publisher.publish_message(body['REPLY_QUEUE'], msg)
def on_f1_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_pp_dev - incoming on_f1_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.f1_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'PP_FWDR_HEALTH_CHECK':
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_HEALTH_CHECK_ACK'
msg['COMPONENT'] = 'FORWARDER_1'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.f1_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'PP_FWDR_XFER_PARAMS':
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_XFER_PARAMS_ACK'
msg['COMPONENT'] = 'FORWARDER_1'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.f1_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'PP_FWDR_READOUT':
# Find message in message list for xfer_params
xfer_msg = None
for msg in self.f1_consumer_msg_list:
if msg['MSG_TYPE'] == 'PP_FWDR_XFER_PARAMS':
xfer_msg = msg
break
if xfer_msg == None:
pytest.fail("The PP_FWDR_XFER_PARAMS message was not received before PP_FWDR_READOUT in F1")
# use message to build response
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_READOUT_ACK'
msg['COMPONENT'] = 'FORWARDER_1'
msg['JOB_NUM'] = xfer_msg['JOB_NUM']
msg['IMAGE_ID'] = xfer_msg['IMAGE_ID']
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
msg['RESULT_LIST'] = {}
msg['RESULT_LIST']['CCD_LIST'] = []
msg['RESULT_LIST']['RECEIPT_LIST'] = []
ccd_list = xfer_msg['XFER_PARAMS']['CCD_LIST']
receipt_list = []
for i in range(0, len(ccd_list)):
receipt_list.append('F1_Rec_x477_' + str(i))
msg['RESULT_LIST']['RECEIPT_LIST'] = receipt_list
msg['RESULT_LIST']['CCD_LIST'] = list(ccd_list)
self.f1_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail("The following unknown message was received by FWDR F1: %s" % body)
def on_f2_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_pp_dev - incoming on_f2_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.f2_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'PP_FWDR_HEALTH_CHECK':
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_HEALTH_CHECK_ACK'
msg['COMPONENT'] = 'FORWARDER_2'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.f2_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'PP_FWDR_XFER_PARAMS':
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_XFER_PARAMS_ACK'
msg['COMPONENT'] = 'FORWARDER_2'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.f2_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'PP_FWDR_READOUT':
# Find message in message list for xfer_params
xfer_msg = None
for msg in self.f2_consumer_msg_list:
if msg['MSG_TYPE'] == 'PP_FWDR_XFER_PARAMS':
xfer_msg = msg
break
if xfer_msg == None:
pytest.fail("The PP_FWDR_XFER_PARAMS message was not received before AR_FWDR_READOUT in F2")
# use message to build response
msg = {}
msg['MSG_TYPE'] = 'PP_FWDR_READOUT_ACK'
msg['COMPONENT'] = 'FORWARDER_2'
msg['JOB_NUM'] = xfer_msg['JOB_NUM']
msg['IMAGE_ID'] = xfer_msg['IMAGE_ID']
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
msg['RESULT_LIST'] = {}
msg['RESULT_LIST']['CCD_LIST'] = []
msg['RESULT_LIST']['RECEIPT_LIST'] = []
ccd_list = xfer_msg['XFER_PARAMS']['CCD_LIST']
receipt_list = []
for i in range(0, len(ccd_list)):
receipt_list.append('F2_Rec_x447_' + str(i))
msg['RESULT_LIST']['RECEIPT_LIST'] = receipt_list
msg['RESULT_LIST']['CCD_LIST'] = list(ccd_list)
self.f2_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail("The following unknown message was received by FWDR F2: %s" % body)
class DMCS:
# This DMCS simulator is how we interact with the test system.
# Commands can be sent manually to simulate how to DMCS might work.
def __init__(self):
printc("Setting up DMCS...")
self._options = "\
1 - (READY) Send Job Information\n\
2 - (SET) Send Standby Message\n\
3 - (GO) Send Readout Message\n\
4 - (RESET) Cancel a Job\n\
0 - (EXIT) Quit DMCS Simulator\n"
self._broker_url = 'amqp://' + AMQP_DMCS_USER + ':' + AMQP_DMCS_PSWD + '@' + AMQP_BROKER_ADDR + ':' + AMQP_BROKER_PORT + '/' + AMQP_BROKER_VHOST
printc('Using broker url: %s' % self._broker_url)
printc("Declaring and binding exchanges...")
printc("Attempting to create a consumer for the '%s' queue." % (Q_DMCS_CONSUME))
self._dmcs_consumer = Consumer(self._broker_url, Q_DMCS_CONSUME)
try:
printc("Attempting to start the consumer thread...")
thread.start_new_thread(self.run_dmcs_consumer, ())
except:
printc("Failed to start consumer thread, quitting...")
sys.exit()
printc("Done setting up consumer thread.")
printc("Setting up publisher...")
self._publisher = SimplePublisher(self._broker_url)
printc("Done creating publisher.")
self._job_msg = {}
self._job_msg['MSG_TYPE'] = 'JOB'
self._job_msg['JOB_NUM'] = 0
self._job_msg['RAFT_NUM'] = 1
self._standby_msg = {}
self._standby_msg['MSG_TYPE'] = 'STANDBY'
self._readout_msg = {}
self._readout_msg['MSG_TYPE'] = 'READOUT'
self._stop_msg = {}
self._stop_msg['MSG_TYPE'] = 'CANCEL'
self._shutdown_msg = {}
self._shutdown_msg['MSG_TYPE'] = 'SHUTDOWN'
def on_dmcs_messages(self, ch, method, properties, body):
msg_dict = yaml.load(body)
printc("Received: %r" % msg_dict)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
def run_dmcs_consumer(self):
self._dmcs_consumer.run(self.on_dmcs_messages)
return
def run(self):
keep_running = True
while keep_running:
try:
user_input = int(raw_input(self._options))
except:
user_input = -1
# New Job
if 1 == user_input:
good_input = True
try:
new_raft_num = int(raw_input("How many pairs? : "))
except:
good_input = False
printc("Bad input...")
if good_input:
self._job_msg['JOB_NUM'] = self._job_msg['JOB_NUM'] + 1
self._job_msg['RAFT_NUM'] = new_raft_num
self._publisher.publish_message(Q_DMCS_PUBLISH, yaml.dump(self._job_msg))
pass
# Standby
elif 2 == user_input:
good_input = True
try:
new_job_num = int(raw_input("STANDBY which job? : "))
except:
good_input = False
printc("Bad input...")
if good_input:
self._standby_msg['JOB_NUM'] = str(new_job_num)
self._publisher.publish_message(Q_DMCS_PUBLISH, yaml.dump(self._standby_msg))
pass
# Readout
elif 3 == user_input:
good_input = True
try:
new_job_num = int(raw_input("READOUT which job? : "))
except:
good_input = False
printc("Bad input...")
if good_input:
self._readout_msg['JOB_NUM'] = str(new_job_num)
self._publisher.publish_message(Q_DMCS_PUBLISH, yaml.dump(self._readout_msg))
pass
# Cancel
elif 4 == user_input:
good_input = True
try:
job_cancel = int(raw_input("Cancel which job? : "))
except:
good_input = False
printc("Bad input...")
if good_input:
self._stop_msg['JOB_NUM'] = job_cancel
self._publisher.publish_message(Q_DMCS_PUBLISH, yaml.dump(self._stop_msg))
pass
# Exit
elif 0 == user_input:
keep_running = False
else:
printc("Invalid input...\n")
return
def test_dmcs(self, Dmcs):
self.dmcs = Dmcs
#logging.warning("Logging is Working!")
LOGGER.critical("LOGGING is Working!")
#self.LOGGER.info("self Logging is Working!")
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
ocs_name = cdm[ROOT]['OCS_BROKER_NAME']
ocs_passwd = cdm[ROOT]['OCS_BROKER_PASSWD']
ocs_pub_name = cdm[ROOT]['OCS_BROKER_PUB_NAME']
ocs_pub_passwd = cdm[ROOT]['OCS_BROKER_PUB_PASSWD']
ocs_broker_url = "amqp://" + ocs_name + ":" + \
ocs_passwd + "@" + \
broker_addr
self.ocs_pub_broker_url = "amqp://" + ocs_pub_name + ":" + \
ocs_pub_passwd + "@" + \
broker_addr
self.ocs_publisher = SimplePublisher(self.ocs_pub_broker_url, "YAML")
at_name = cdm[ROOT]['AUX_BROKER_NAME']
at_passwd = cdm[ROOT]['AUX_BROKER_PASSWD']
at_pub_name = cdm[ROOT]['AUX_BROKER_PUB_NAME']
at_pub_passwd = cdm[ROOT]['AUX_BROKER_PUB_PASSWD']
at_broker_url = "amqp://" + at_name + ":" + \
at_passwd + "@" + \
broker_addr
self.at_pub_broker_url = "amqp://" + at_pub_name + ":" + \
at_pub_passwd + "@" + \
broker_addr
self.at_publisher = SimplePublisher(self.at_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.ocs_consumer = Consumer(ocs_broker_url,'dmcs_ocs_publish', 'thread-ocs',
self.on_ocs_message,'YAML')
self.ocs_consumer.start()
self.at_consumer = Consumer(at_broker_url,'at_foreman_consume', 'thread-at',
self.on_at_message,'YAML')
self.at_consumer.start()
sleep(3)
if self.DP:
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(3)
self.verify_ocs_messages()
sleep(3)
self.verify_at_messages()
sleep(2)
self.ocs_consumer.stop()
self.ocs_consumer.join()
self.at_consumer.stop()
self.at_consumer.join()
if self.DP:
print("Finished with DMCS AT tests.")
def test_ardev(self, Ardev):
self.ardev = Ardev
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test_ar.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
raise
raft_dict = cdm[ROOT]['DEFAULT_RAFT_CONFIGURATION']
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
dmcs_name = cdm[ROOT]['DMCS_BROKER_NAME']
dmcs_passwd = cdm[ROOT]['DMCS_BROKER_PASSWD']
dmcs_pub_name = cdm[ROOT]['DMCS_BROKER_PUB_NAME']
dmcs_pub_passwd = cdm[ROOT]['DMCS_BROKER_PUB_PASSWD']
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_passwd + "@" + \
broker_addr
dmcs_pub_broker_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % dmcs_pub_broker_url)
self.dmcs_publisher = SimplePublisher(dmcs_pub_broker_url, "YAML")
ar_ctrl_name = cdm[ROOT]['ARCHIVE_BROKER_NAME']
ar_ctrl_passwd = cdm[ROOT]['ARCHIVE_BROKER_PASSWD']
ar_ctrl_pub_name = cdm[ROOT]['ARCHIVE_BROKER_PUB_NAME']
ar_ctrl_pub_passwd = cdm[ROOT]['ARCHIVE_BROKER_PUB_PASSWD']
ar_ctrl_broker_url = "amqp://" + ar_ctrl_name + ":" + \
ar_ctrl_passwd + "@" + \
broker_addr
ar_ctrl_pub_broker_url = "amqp://" + ar_ctrl_pub_name + ":" + \
ar_ctrl_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % ar_ctrl_pub_broker_url)
self.ar_ctrl_publisher = SimplePublisher(ar_ctrl_pub_broker_url, "YAML")
F1_name = 'F1'
F1_passwd = 'F1'
F1_pub_name = 'F1_PUB'
F1_pub_passwd = 'F1_PUB'
F1_broker_url = "amqp://" + F1_name + ":" + \
F1_passwd + "@" + \
broker_addr
F1_pub_broker_url = "amqp://" + F1_pub_name + ":" + \
F1_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % F1_pub_broker_url)
self.F1_publisher = SimplePublisher(F1_pub_broker_url, "YAML")
F2_name = 'F2'
F2_passwd = 'F2'
F2_pub_name = 'F2_PUB'
F2_pub_passwd = 'F2_PUB'
F2_broker_url = "amqp://" + F2_name + ":" + \
F2_passwd + "@" + \
broker_addr
F2_pub_broker_url = "amqp://" + F2_pub_name + ":" + \
F2_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % F2_pub_broker_url)
self.F2_publisher = SimplePublisher(F2_pub_broker_url, "YAML")
print("All publishers are running...")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
print("MessageAuthority running...")
self.dmcs_consumer = Consumer(dmcs_broker_url,'dmcs_ack_consume', 'thread-dmcs',
self.on_dmcs_message,'YAML')
self.dmcs_consumer.start()
print("DMCS Consumer running...")
self.ar_ctrl_consumer = Consumer(ar_ctrl_broker_url,'archive_ctrl_consume', 'thread-ar-ctrl',
self.on_ar_ctrl_message,'YAML')
self.ar_ctrl_consumer.start()
print("ar_ctrl Consumer running...")
self.F1_consumer = Consumer(F1_broker_url,'f1_consume', 'thread-f1',
self.on_f1_message,'YAML')
self.F1_consumer.start()
print("F1 Consumer running...")
self.F2_consumer = Consumer(F2_broker_url,'f2_consume', 'thread-f2',
self.on_f2_message,'YAML')
self.F2_consumer.start()
print("F2 Consumer running...")
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(8)
self.verify_dmcs_messages()
self.verify_ar_ctrl_messages()
self.verify_F1_messages()
self.verify_F2_messages()
sleep(3)
# Shut down consumer threads nicely
self.dmcs_consumer.stop()
self.dmcs_consumer.join()
self.ar_ctrl_consumer.stop()
self.ar_ctrl_consumer.join()
self.F1_consumer.stop()
self.F1_consumer.join()
self.F2_consumer.stop()
self.F2_consumer.join()
if self.DP:
print("Finished with AR tests.")
def test_dmcs(self, Dmcs):
self.dmcs = Dmcs
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
ocs_name = cdm[ROOT]['OCS_BROKER_NAME']
ocs_passwd = cdm[ROOT]['OCS_BROKER_PASSWD']
ocs_pub_name = cdm[ROOT]['OCS_BROKER_PUB_NAME']
ocs_pub_passwd = cdm[ROOT]['OCS_BROKER_PUB_PASSWD']
ocs_broker_url = "amqp://" + ocs_name + ":" + \
ocs_passwd + "@" + \
broker_addr
self. ocs_pub_broker_url = "amqp://" + ocs_pub_name + ":" + \
ocs_pub_passwd + "@" + \
broker_addr
self.ocs_publisher = SimplePublisher(self.ocs_pub_broker_url, "YAML")
ar_name = cdm[ROOT]['AFM_BROKER_NAME']
ar_passwd = cdm[ROOT]['AFM_BROKER_PASSWD']
ar_pub_name = cdm[ROOT]['AFM_BROKER_PUB_NAME']
ar_pub_passwd = cdm[ROOT]['AFM_BROKER_PUB_PASSWD']
ar_broker_url = "amqp://" + ar_name + ":" + \
ar_passwd + "@" + \
broker_addr
self.ar_pub_broker_url = "amqp://" + ar_pub_name + ":" + \
ar_pub_passwd + "@" + \
broker_addr
self.ar_publisher = SimplePublisher(self.ar_pub_broker_url, "YAML")
pp_name = cdm[ROOT]['PFM_BROKER_NAME']
pp_passwd = cdm[ROOT]['PFM_BROKER_PASSWD']
pp_pub_name = cdm[ROOT]['PFM_BROKER_PUB_NAME']
pp_pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD']
pp_broker_url = "amqp://" + pp_name + ":" + \
pp_passwd + "@" + \
broker_addr
self.pp_pub_broker_url = "amqp://" + pp_pub_name + ":" + \
pp_pub_passwd + "@" + \
broker_addr
self.pp_publisher = SimplePublisher(self.pp_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.ocs_consumer = Consumer(ocs_broker_url,'dmcs_ocs_publish', 'thread-ocs',
self.on_ocs_message,'YAML')
self.ocs_consumer.start()
self.ar_consumer = Consumer(ar_broker_url,'ar_foreman_consume', 'thread-ar',
self.on_ar_message,'YAML')
self.ar_consumer.start()
self.pp_consumer = Consumer(pp_broker_url,'pp_foreman_consume', 'thread-pp',
self.on_pp_message,'YAML')
self.pp_consumer.start()
### call message sender and pass in ocs_publisher
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(3)
self.verify_ocs_messages()
sleep(3)
self.verify_ar_messages()
sleep(3)
self.verify_pp_messages()
sleep(2)
self.ocs_consumer.stop()
self.ocs_consumer.join()
self.ar_consumer.stop()
self.ar_consumer.join()
self.pp_consumer.stop()
self.pp_consumer.join()
print("Finished with DMCS AR and PP tests.")
def __init__(self, host,port,srcFile):
file=open(srcFile)
data=file.read()
self.consumer = Consumer(data+"@"+host,port, self,runOnlyOnce=True)
class TestNcsa:
pp_pub_broker_url = None
pp_publisher = None
pp_consumer = None
pp_consumer_msg_list = []
D1_pub_broker_url = None
D1_publisher = None
D1_consumer = None
d1_consumer_msg_list = []
D2_pub_broker_url = None
D2_publisher = None
D2_consumer = None
d2_consumer_msg_list = []
EXPECTED_PP_MESSAGES = 1
EXPECTED_D1_MESSAGES = 1
EXPECTED_D2_MESSAGES = 1
ccd_list = [14,17,21,86]
prp = toolsmod.prp
DP = toolsmod.DP
def test_ncsa(self, Ncsa):
self.ncsa = Ncsa
try:
cdm = toolsmod.intake_yaml_file('./tests/yaml/L1SystemCfg_Test_ncsa.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
pp_name = cdm[ROOT]['PFM_BROKER_NAME']
pp_passwd = cdm[ROOT]['PFM_BROKER_PASSWD']
pp_pub_name = cdm[ROOT]['PFM_BROKER_PUB_NAME']
pp_pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD']
pp_broker_url = "amqp://" + pp_name + ":" + \
pp_passwd + "@" + \
broker_addr
pp_pub_broker_url = "amqp://" + pp_pub_name + ":" + \
pp_pub_passwd + "@" + \
broker_addr
self.pp_publisher = SimplePublisher(pp_pub_broker_url, "YAML")
D1_name = 'D1'
D1_passwd = 'D1'
D1_pub_name = 'D1_PUB'
D1_pub_passwd = 'D1_PUB'
D1_broker_url = "amqp://" + D1_name + ":" + \
D1_passwd + "@" + \
broker_addr
D1_pub_broker_url = "amqp://" + D1_pub_name + ":" + \
D1_pub_passwd + "@" + \
broker_addr
self.d1_publisher = SimplePublisher(D1_pub_broker_url, "YAML")
D2_name = 'D2'
D2_passwd = 'D2'
D2_pub_name = 'D2_PUB'
D2_pub_passwd = 'D2_PUB'
D2_broker_url = "amqp://" + D2_name + ":" + \
D2_passwd + "@" + \
broker_addr
D2_pub_broker_url = "amqp://" + D2_pub_name + ":" + \
D2_pub_passwd + "@" + \
broker_addr
self.d2_publisher = SimplePublisher(D2_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.pp_consumer = Consumer(pp_broker_url,'pp_foreman_ack_publish', 'thread-pp',
self.on_pp_message,'YAML')
self.pp_consumer.start()
self.D1_consumer = Consumer(D1_broker_url,'d1_consume', 'thread-d1',
self.on_d1_message,'YAML')
self.D1_consumer.start()
self.D2_consumer = Consumer(D2_broker_url,'d2_consume', 'thread-d2',
self.on_d2_message,'YAML')
self.D2_consumer.start()
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(7)
self.verify_pp_messages()
self.verify_D2_messages()
self.verify_D1_messages()
sleep(2)
self.pp_consumer.stop()
self.pp_consumer.join()
self.D1_consumer.stop()
self.D1_consumer.join()
self.D2_consumer.stop()
self.D2_consumer.join()
if self.DP:
print("Finished with NCSA tests.")
def send_messages(self):
print("Starting send_messages")
# Tests only an AR device
# self.clear_message_lists()
self.EXPECTED_PP_MESSAGES = 4
self.EXPECTED_D1_MESSAGES = 3
self.EXPECTED_D2_MESSAGES = 3
msg = {}
msg['MSG_TYPE'] = "NCSA_NEW_SESSION"
msg['SESSION_ID'] = 'SI_469976'
msg['ACK_ID'] = 'NCSA_NEW_SESSION_ACK_44221'
msg['REPLY_QUEUE'] = 'pp_foreman_ack_publish'
time.sleep(1)
if self.DP:
print("New Session Message")
self.pp_publisher.publish_message("ncsa_consume", msg)
msg = {}
msg['MSG_TYPE'] = "NCSA_NEXT_VISIT"
msg['VISIT_ID'] = 'XX_28272'
msg['SESSION_ID'] = 'NNV_469976'
msg['REPLY_QUEUE'] = 'pp_foreman_ack_publish'
msg['ACK_ID'] = 'NCSA_NEW_VISIT_ACK_76'
msg['BORE_SIGHT'] = "231,123786456342, -45.3457156906, FK5"
time.sleep(1)
if self.DP:
print("Next Visit Message")
self.pp_publisher.publish_message("ncsa_consume", msg)
msg = {}
msg['MSG_TYPE'] = "NCSA_START_INTEGRATION"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444244'
msg['VISIT_ID'] = 'V14494'
msg['SESSION_ID'] = '4_14_7211511'
msg['REPLY_QUEUE'] = 'pp_foreman_ack_publish'
msg['ACK_ID'] = 'PP_ACK_94671'
msg['CCD_LIST'] = [4,14,16,17,29,35,36]
time.sleep(2)
msg['FORWARDERS'] = {}
forwarder_list = ['FORWARDER_2', 'FORWARDER_1']
ccd_list = [[17,18,111,126],[128,131,132]]
msg['FORWARDERS']['FORWARDER_LIST'] = forwarder_list
msg['FORWARDERS']['CCD_LIST'] = ccd_list
if self.DP:
print("NCSA START INTEGRATION Message")
self.pp_publisher.publish_message("ncsa_consume", msg)
time.sleep(7)
msg = {}
msg['MSG_TYPE'] = "NCSA_READOUT"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444244'
msg['VISIT_ID'] = 'V14494'
msg['SESSION_ID'] = '4_14_7211511'
msg['REPLY_QUEUE'] = 'pp_foreman_ack_publish'
msg['ACK_ID'] = 'NCSA_READOUT_ACK_44221'
time.sleep(2)
if self.DP:
print("NCSA READOUT Message")
self.pp_publisher.publish_message("ncsa_consume", msg)
time.sleep(2)
print("Message Sender done")
def verify_pp_messages(self):
len_list = len(self.pp_consumer_msg_list)
if len_list != self.EXPECTED_PP_MESSAGES:
pytest.fail('PP simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_PP_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.pp_consumer_msg_list[i]
if msg['MSG_TYPE'] == 'NCSA_START_INTEGRATION_ACK':
result = self.check_start_int_ack(msg)
else:
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following message to the PP Foreman failed when compared with the sovereign example: %s" % msg)
if self.DP:
print("Messages to the PP Foreman pass verification.")
def check_start_int_ack(self, msg):
"""the PAIRS param in the message is a list. Every item in the list is a dictionary.
because it is not known how many entried will be on the list, the dictionaries
are deepcopied and checked against the MessageAuthority and check one at a time.
In the meanwhile, the shape of the incoming message without the dictionaries and the
PAIRS list set to None is checked against the MessageAuthority.
BTW, because the individual dictionaries do not have a MSG_TYPE, one is
added to each dictionary to be checked so the dict to compare to can be
located in the messages.yaml file.
"""
c_msg = deepcopy(msg)
pairs = deepcopy(c_msg['PAIRS'])
c_msg['PAIRS'] = None
overall_shape = self._msg_auth.check_message_shape(c_msg)
if overall_shape == False:
return False
for i in range (0, len(pairs)):
pairs[i]['MSG_TYPE'] = 'PAIR'
result = self._msg_auth.check_message_shape(pairs[i])
if result == False:
return False
return True
def verify_D1_messages(self):
len_list = len(self.d1_consumer_msg_list)
if len_list != self.EXPECTED_D1_MESSAGES:
if self.DP:
print("Messages received by verify_D1_messages:")
self.prp.pprint(self.f1_consumer_msg_list)
pytest.fail('F1 simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_D1_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.d1_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following message to D1 failed when compared with the sovereign example: %s" % msg)
if self.DP:
print("Messages to D1 pass verification.")
def verify_D2_messages(self):
len_list = len(self.d2_consumer_msg_list)
if len_list != self.EXPECTED_D2_MESSAGES:
if self.DP:
print("Messages received by verify_D2_messages:")
self.prp.pprint(self.d2_consumer_msg_list)
pytest.fail('D2 simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_D2_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.d2_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following message to D2 failed when compared with the sovereign example: %s" % msg)
if self.DP:
print("Messages to D2 pass verification.")
def on_pp_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_ncsa - incoming on_pp_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.pp_consumer_msg_list.append(body)
def on_d1_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_ncsa - incoming on_D1_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.d1_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'DISTRIBUTOR_HEALTH_CHECK':
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_HEALTH_CHECK_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_1'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.d1_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'DISTRIBUTOR_XFER_PARAMS':
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_XFER_PARAMS_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_1'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.d1_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'DISTRIBUTOR_READOUT':
# Find message in message list for xfer_params
xfer_msg = None
for msg in self.d1_consumer_msg_list:
if msg['MSG_TYPE'] == 'DISTRIBUTOR_XFER_PARAMS':
xfer_msg = msg
break
if xfer_msg == None:
pytest.fail("The DISTRIBUTOR_XFER_PARAMS message was not received before DISTRIBUTOE_READOUT in D1")
# use message to build response
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_READOUT_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_1'
msg['JOB_NUM'] = xfer_msg['JOB_NUM']
msg['IMAGE_ID'] = xfer_msg['IMAGE_ID']
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
msg['RESULT_LIST'] = {}
msg['RESULT_LIST']['CCD_LIST'] = []
msg['RESULT_LIST']['RECEIPT_LIST'] = []
ccd_list = xfer_msg['XFER_PARAMS']['CCD_LIST']
receipt_list = []
for i in range(0, len(ccd_list)):
receipt_list.append('F1_Rec_x477_' + str(i))
msg['RESULT_LIST']['RECEIPT_LIST'] = receipt_list
msg['RESULT_LIST']['CCD_LIST'] = list(ccd_list)
self.d1_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail("The following unknown message was received by D1: %s" % body)
def on_d2_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
if self.DP:
print("In test_ncsa - incoming on_D2_message")
self.prp.pprint(body)
print("\n----------------------\n\n")
self.d2_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'DISTRIBUTOR_HEALTH_CHECK':
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_HEALTH_CHECK_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_2'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.d2_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'DISTRIBUTOR_XFER_PARAMS':
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_XFER_PARAMS_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_2'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.d2_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'DISTRIBUTOR_READOUT':
# Find message in message list for xfer_params
xfer_msg = None
for msg in self.d2_consumer_msg_list:
if msg['MSG_TYPE'] == 'DISTRIBUTOR_XFER_PARAMS':
xfer_msg = msg
break
if xfer_msg == None:
pytest.fail("The DISTRIBUTOR_XFER_PARAMS message was not received before DISTRIBUTOR_READOUT in D2")
# use message to build response
msg = {}
msg['MSG_TYPE'] = 'DISTRIBUTOR_READOUT_ACK'
msg['COMPONENT'] = 'DISTRIBUTOR_2'
msg['JOB_NUM'] = xfer_msg['JOB_NUM']
msg['IMAGE_ID'] = xfer_msg['IMAGE_ID']
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
msg['RESULT_LIST'] = {}
msg['RESULT_LIST']['CCD_LIST'] = []
msg['RESULT_LIST']['RECEIPT_LIST'] = []
ccd_list = xfer_msg['XFER_PARAMS']['CCD_LIST']
receipt_list = []
for i in range(0, len(ccd_list)):
receipt_list.append('F2_Rec_x447_' + str(i))
msg['RESULT_LIST']['RECEIPT_LIST'] = receipt_list
msg['RESULT_LIST']['CCD_LIST'] = list(ccd_list)
self.d2_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail("The following unknown message was received by D2: %s" % body)
def __init__(self, *args, **kwargs):
Consumer.__init__(self, *args, **kwargs)
class TestDMCS_AR_PP:
ocs_pub_broker_url = None
ocs_publisher = None
ocs_consumer = None
ocs_consumer_msg_list = []
ar_pub_broker_url = None
ar_publisher = None
ar_consumer = None
ar_consumer_msg_list = []
pp_pub_broker_url = None
pp_publisher = None
pp_consumer = None
pp_consumer_msg_list = []
EXPECTED_AR_MESSAGES = 1
EXPECTED_PP_MESSAGES = 1
EXPECTED_OCS_MESSAGES = 1
ccd_list = [14,17,21.86]
prp = toolsmod.prp
DP = toolsmod.DP # Debug Print
def test_dmcs(self, Dmcs):
self.dmcs = Dmcs
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
ocs_name = cdm[ROOT]['OCS_BROKER_NAME']
ocs_passwd = cdm[ROOT]['OCS_BROKER_PASSWD']
ocs_pub_name = cdm[ROOT]['OCS_BROKER_PUB_NAME']
ocs_pub_passwd = cdm[ROOT]['OCS_BROKER_PUB_PASSWD']
ocs_broker_url = "amqp://" + ocs_name + ":" + \
ocs_passwd + "@" + \
broker_addr
self. ocs_pub_broker_url = "amqp://" + ocs_pub_name + ":" + \
ocs_pub_passwd + "@" + \
broker_addr
self.ocs_publisher = SimplePublisher(self.ocs_pub_broker_url, "YAML")
ar_name = cdm[ROOT]['AFM_BROKER_NAME']
ar_passwd = cdm[ROOT]['AFM_BROKER_PASSWD']
ar_pub_name = cdm[ROOT]['AFM_BROKER_PUB_NAME']
ar_pub_passwd = cdm[ROOT]['AFM_BROKER_PUB_PASSWD']
ar_broker_url = "amqp://" + ar_name + ":" + \
ar_passwd + "@" + \
broker_addr
self.ar_pub_broker_url = "amqp://" + ar_pub_name + ":" + \
ar_pub_passwd + "@" + \
broker_addr
self.ar_publisher = SimplePublisher(self.ar_pub_broker_url, "YAML")
pp_name = cdm[ROOT]['PFM_BROKER_NAME']
pp_passwd = cdm[ROOT]['PFM_BROKER_PASSWD']
pp_pub_name = cdm[ROOT]['PFM_BROKER_PUB_NAME']
pp_pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD']
pp_broker_url = "amqp://" + pp_name + ":" + \
pp_passwd + "@" + \
broker_addr
self.pp_pub_broker_url = "amqp://" + pp_pub_name + ":" + \
pp_pub_passwd + "@" + \
broker_addr
self.pp_publisher = SimplePublisher(self.pp_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.ocs_consumer = Consumer(ocs_broker_url,'dmcs_ocs_publish', 'thread-ocs',
self.on_ocs_message,'YAML')
self.ocs_consumer.start()
self.ar_consumer = Consumer(ar_broker_url,'ar_foreman_consume', 'thread-ar',
self.on_ar_message,'YAML')
self.ar_consumer.start()
self.pp_consumer = Consumer(pp_broker_url,'pp_foreman_consume', 'thread-pp',
self.on_pp_message,'YAML')
self.pp_consumer.start()
### call message sender and pass in ocs_publisher
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(3)
self.verify_ocs_messages()
sleep(3)
self.verify_ar_messages()
sleep(3)
self.verify_pp_messages()
sleep(2)
self.ocs_consumer.stop()
self.ocs_consumer.join()
self.ar_consumer.stop()
self.ar_consumer.join()
self.pp_consumer.stop()
self.pp_consumer.join()
print("Finished with DMCS AR and PP tests.")
def send_messages(self):
print("Starting send_messages")
# Tests both AR and PP devices
self.EXPECTED_OCS_MESSAGES = 12
self.EXPECTED_AR_MESSAGES = 7
self.EXPECTED_PP_MESSAGES = 7
self.clear_message_lists()
msg = {}
msg['MSG_TYPE'] = "STANDBY"
msg['DEVICE'] = 'AR'
msg['CMD_ID'] = '16729948'
msg['CFG_KEY'] = "2C16"
msg['ACK_ID'] = 'AR_4'
msg['ACK_DELAY'] = 2
time.sleep(1)
print("AR STANDBY")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "STANDBY"
msg['DEVICE'] = 'PP'
msg['CMD_ID'] = '16729948'
msg['CFG_KEY'] = "2C16"
msg['ACK_ID'] = 'PP_7'
msg['ACK_DELAY'] = 2
time.sleep(5)
print("PP STANDBY")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
#msg = {}
#msg['MSG_TYPE'] = "NEW_SESSION"
#msg['SESSION_ID'] = 'SI_469976'
#msg['ACK_ID'] = 'NEW_SESSION_ACK_44221'
#msg['RESPONSE_QUEUE'] = "dmcs_ack_consume"
##time.sleep(1)
##self.ocs_publisher.publish_message("ar_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "DISABLE"
msg['DEVICE'] = 'AR'
msg['CMD_ID'] = '16729948'
msg['ACK_ID'] = 'AR_6'
msg['ACK_DELAY'] = 2
time.sleep(5)
print("AR DISABLE")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "DISABLE"
msg['DEVICE'] = 'PP'
msg['CMD_ID'] = '16729948'
msg['ACK_ID'] = 'PP_8'
msg['ACK_DELAY'] = 2
time.sleep(5)
print("PP DISABLE")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
# Make certain DMCS is doing proper bookkeeping
sleep(6.5)
assert self.dmcs.STATE_SCBD.get_archive_state() == 'DISABLE'
assert self.dmcs.STATE_SCBD.get_prompt_process_state() == 'DISABLE'
msg = {}
msg['MSG_TYPE'] = "ENABLE"
msg['DEVICE'] = 'AR'
msg['CMD_ID'] = '16729948'
msg['ACK_ID'] = 'AR_11'
msg['ACK_DELAY'] = 2
time.sleep(5)
print("AR ENABLE")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "ENABLE"
msg['DEVICE'] = 'PP'
msg['CMD_ID'] = '16729948'
msg['ACK_ID'] = 'PP_12'
msg['ACK_DELAY'] = 2
time.sleep(5)
print("PP ENABLE")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
sleep(6.5)
assert self.dmcs.STATE_SCBD.get_archive_state() == 'ENABLE'
assert self.dmcs.STATE_SCBD.get_prompt_process_state() == 'ENABLE'
msg = {}
msg['MSG_TYPE'] = "NEXT_VISIT"
msg['VISIT_ID'] = 'V_1443'
msg['RESPONSE_QUEUE'] = "dmcs_ack_consume"
msg['ACK_ID'] = 'NEW_VISIT_ACK_76'
msg['BORE_SIGHT'] = "231,123786456342, -45.3457156906, FK5"
time.sleep(5)
print("Next Visit Message")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
sleep(3.5)
assert self.dmcs.STATE_SCBD.get_current_visit() == 'V_1443'
msg = {}
msg['MSG_TYPE'] = "START_INTEGRATION"
msg['IMAGE_ID'] = 'IMG_4276'
msg['VISIT_ID'] = 'V_1443'
msg['ACK_ID'] = 'START_INT_ACK_76'
msg['RESPONSE_QUEUE'] = "dmcs_ack_consume"
msg['CCD_LIST'] = self.ccd_list
time.sleep(5)
print("Start Integration Message")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "READOUT"
msg['VISIT_ID'] = 'V_1443'
msg['IMAGE_ID'] = 'IMG_4276'
msg['IMAGE_SRC'] = 'MAIN'
msg['RESPONSE_QUEUE'] = "dmcs_ack_consume"
msg['ACK_ID'] = 'READOUT_ACK_77'
time.sleep(5)
print("READOUT Message")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "START_INTEGRATION"
msg['IMAGE_ID'] = 'IMG_4277'
msg['IMAGE_SRC'] = 'MAIN'
msg['VISIT_ID'] = 'V_1443'
msg['ACK_ID'] = 'START_INT_ACK_78'
msg['RESPONSE_QUEUE'] = "dmcs_ack_consume"
msg['CCD_LIST'] = self.ccd_list
time.sleep(5)
print("Start Integration Message")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
msg = {}
msg['MSG_TYPE'] = "READOUT"
msg['VISIT_ID'] = 'V_1443'
msg['IMAGE_ID'] = 'IMG_4277'
msg['IMAGE_SRC'] = 'MAIN'
msg['RESPONSE_QUEUE'] = "dmcs_ack_consume"
msg['ACK_ID'] = 'READOUT_ACK_79'
time.sleep(5)
print("READOUT Message")
self.ocs_publisher.publish_message("ocs_dmcs_consume", msg)
time.sleep(2)
print("Message Sender done")
def clear_message_lists(self):
self.ocs_consumer_msg_list = []
self.ar_consumer_msg_list = []
self.pp_consumer_msg_list = []
def verify_ocs_messages(self):
print("Messages received by verify_ocs_messages:")
self.prp.pprint(self.ocs_consumer_msg_list)
len_list = len(self.ocs_consumer_msg_list)
if len_list != self.EXPECTED_OCS_MESSAGES:
print("Messages received by verify_ocs_messages:")
self.prp.pprint(self.ocs_consumer_msg_list)
pytest.fail('OCS simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_OCS_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.ocs_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following OCS Bridge response message failed when compared with the sovereign example: %s" % msg)
print("Responses to OCS Bridge pass verification.")
def verify_ar_messages(self):
print("Messages received by verify_ar_messages:")
self.prp.pprint(self.ar_consumer_msg_list)
len_list = len(self.ar_consumer_msg_list)
if len_list != self.EXPECTED_AR_MESSAGES:
print("Messages received by verify_ar_messages:")
self.prp.pprint(self.ar_consumer_msg_list)
pytest.fail('AR simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_AR_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.ar_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following message to the AR failed when compared with the sovereign example: %s" % msg)
print("Messages to the AR pass verification.")
def verify_pp_messages(self):
print("Messages received by verify_pp_messages:")
self.prp.pprint(self.pp_consumer_msg_list)
len_list = len(self.pp_consumer_msg_list)
if len_list != self.EXPECTED_PP_MESSAGES:
print("Messages received by verify_pp_messages:")
self.prp.pprint(self.pp_consumer_list)
pytest.fail('PP simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_PP_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.pp_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following message to the PP device failed when compared with the sovereign example: %s" % msg)
print("Messages to the PP device pass verification.")
def on_ocs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.ocs_consumer_msg_list.append(body)
def on_ar_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.ar_consumer_msg_list.append(body)
def on_pp_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.pp_consumer_msg_list.append(body)
class NCSAForeman(Foreman):
# NCSAForeman receives messages from BaseForeman
# and coordinates with distributors
PROGRAM_NAME = "NCSA"
def __init__(self):
printc("Starting...")
custom_print.define_new_name(self.PROGRAM_NAME)
self._sb_mach = Scoreboard
self._machine_prefix = 'D:'
self._machine_publish_q = Q_DIST_PUBLISH
self._machine_consume_q = Q_DIST_CONSUME
# Messages we can recieve from BaseForeman
self._msg_actions_bf = {
JOB_REQUEST: self.process_bf_job_request,
DISTRIBUTOR_REQUEST: self.process_bf_distributor_request,
STANDBY: self.process_bf_standby,
READOUT: self.process_bf_readout,
CANCEL: self.process_bf_cancel,
TRANSFER_DONE: self.process_bf_transfer_done
}
# Run parent init, starts distributor scoreboard and consumer
self.parent_init(SCOREBOARD_DB_DIST, PROGRAM_NAME, REGISTER_DISTRIBUTOR)
# BaseForeman consumer
printc("Creating BaseForeman consumer...")
self._bf_consumer = Consumer(self._broker_url, Q_NCSA_CONSUME)
try:
thread.start_new_thread(self.run_bf_consumer, ())
except:
printc("Thread run_bf_consumer failed, quitting...")
sys.exit()
return
def run_bf_consumer(self):
printc("BaseForeman message consumer is running...")
self._bf_consumer.run(self.on_bf_message)
return
# BaseForeman messaging
def on_bf_message(self, ch, method, properties, body):
# BaseForeman message consumer
msg_dict = yaml.load(body)
try:
af_handler = self._msg_actions_bf.get(msg_dict[MSG_TYPE])
except:
printc("Bad message received...")
ch.basic_ack(delivery_tag=method.delivery_tag)
return
af_handler(msg_dict)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
def process_bf_job_request(self, msg_params):
# NCSA would perform any clean up needed as this point to prepare
# for a new job.
printc("NCSA Foreman is online and ready for the Job")
return
def process_bf_distributor_request(self, msg_params):
# BaseForeman wants some distributors for a job
# Get the amount of distributors requested
dist_needed = int(msg_params[DIST_NEEDED])
# Check how many we have available
num_healthy_distributors = self._sb_mach.count_idle(LIST_DISTRIBUTORS)
printc("%d pairs available, %d were requested." % (num_healthy_distributors, dist_needed))
# If we don't have enough in IDLE, report back that we don't have enough resources
if dist_needed > num_healthy_distributors:
printc("Not enough distributors.")
insuff_dist_msg = {}
insuff_dist_msg[MSG_TYPE] = ACK_RECEIVED
insuff_dist_msg[JOB_NUM] = msg_params[JOB_NUM]
insuff_dist_msg[PAIRS] = None
insuff_dist_msg[ACK_ID] = msg_params[ACK_ID]
insuff_dist_msg[ACK_NAME] = PAIRING
insuff_dist_msg[ACK_BOOL] = False
self._publisher.publish_message(Q_ACK_PUBLISH, yaml.dump(insuff_dist_msg))
# Otherwise, take the list of forwarders sent to us and pair each one
# with an IDLE distributor and then send that pairing list back
else:
# Get a list that is the size we need of IDLE distributors
self._dist_list = self._sb_mach.get_idle_list(LIST_DISTRIBUTORS,
dist_needed,
msg_params[JOB_NUM])
printc("Generating pairs list...")
forw_list = msg_params['FORW_LIST']
current_pairs = {}
counter = 0
for each in forw_list:
current_pairs[each] = self._dist_list[counter]
counter = counter + 1
# Send the pairing list back
pair_msg = {}
pair_msg[MSG_TYPE] = ACK_RECEIVED
pair_msg[JOB_NUM] = msg_params[JOB_NUM]
pair_msg[PAIRS] = current_pairs
pair_msg[ACK_ID] = msg_params[ACK_ID]
pair_msg[ACK_NAME] = PAIRING
pair_msg[ACK_BOOL] = True
self._publisher.publish_message(Q_ACK_PUBLISH, yaml.dump(pair_msg))
return
def process_bf_standby(self, msg_params):
# A job has moved to STANDBY
job_num_tmp = str(msg_params[JOB_NUM])
# Get the name of the file the distributors will be receiving
xfer_file_main = msg_params[XFER_FILE]
# Confirm distributors are working on this job
if self._sb_mach.machine_find_all_m_check(LIST_DISTRIBUTORS, job_num_tmp) > 0:
# Make a list of all the distributors for this job
distributors = self._sb_mach.machine_find_all_m(LIST_DISTRIBUTORS, job_num_tmp)
for distributor in distributors:
ds_msg = {}
ds_msg[MSG_TYPE] = STANDBY
ds_msg[XFER_FILE] = string.replace(xfer_file_main + '_' + distributor + '.raw', "D:", "")
routing_key = distributor + "_consume"
self._publisher.publish_message(routing_key, yaml.dump(ds_msg))
printc("Distributors have been sent the STANDBY message.")
# Would probably wait and after getting ACK's from Distributors
printc("Sending the STANDBY ACK...")
ack_msg = {}
ack_msg[ACK_ID] = msg_params[ACK_ID]
ack_msg[ACK_NAME] = 'NCSA_STANDBY'
ack_msg[MSG_TYPE] = ACK_RECEIVED
ack_msg[JOB_NUM] = msg_params[JOB_NUM]
self._publisher.publish_message(Q_ACK_PUBLISH, yaml.dump(ack_msg))
else:
printc("No distributors are assigned to job %s, no STANDBY sent." % job_num_tmp)
return
def process_bf_readout(self, msg_params):
# A job has moved to READOUT
job_num_tmp = str(msg_params[JOB_NUM])
# Confirm there are distributors working on this job and alert them
if self._sb_mach.machine_find_all_m_check(LIST_DISTRIBUTORS, job_num_tmp) > 0:
distributors = self._sb_mach.machine_find_all_m(LIST_DISTRIBUTORS, job_num_tmp)
for distributor in distributors:
dist_start = {}
dist_start[MSG_TYPE] = READOUT
dist_start[JOB_NUM] = msg_params[JOB_NUM]
routing_key = distributor + "_consume"
self._publisher.publish_message(routing_key, yaml.dump(dist_start))
printc("Distributors have been sent the READOUT message.")
printc("Sending the READOUT ACK...")
ack_msg = {}
ack_msg[ACK_ID] = msg_params[ACK_ID]
ack_msg[ACK_NAME] = READOUT
ack_msg[MSG_TYPE] = ACK_RECEIVED
ack_msg[JOB_NUM] = msg_params[JOB_NUM]
self._publisher.publish_message(Q_ACK_PUBLISH, yaml.dump(ack_msg))
else:
printc("No distributors are assigned to job %s, no READOUT sent." % job_num_tmp)
return
def process_bf_cancel(self, msg_params):
# A job was canceled
# Get the job number being canceled
job_to_stop = int(msg_params[JOB_NUM])
printc("Telling distributors to cancel job %d..." % job_to_stop)
# Tell distributors to stop
stop_msg = {}
stop_msg[MSG_TYPE] = 'CANCEL'
stop_msg[JOB_NUM] = str(job_to_stop)
# Find the distributors on that job
list_of_q = self._sb_mach.machine_find_job(LIST_DISTRIBUTORS, job_to_stop)
for q in list_of_q:
self._publisher.publish_message(q, yaml.dump(stop_msg))
return
def process_bf_transfer_done(self, msg_params):
# Distributors should know when they are done and tell us,
# probably no need for Base Foreman to do it
return
class Foreman:
# Parent class for Base and NCSA Foremen.
# Contains set up for forwarder and distributor machines
# along with code that was common between them.
def parent_init(self, db_num, prog_name, type):
custom_print.define_new_name(self.PROGRAM_NAME)
# Create machine scoreboard
self._sb_mach = Scoreboard(db_num, prog_name, type)
# Messaging URL (rabbitmq server IP)
self._broker_url = "amqp://" + AMQP_BF_USER + ":" + AMQP_BF_PSWD + "@" + AMQP_BROKER_ADDR + ":" + AMQP_BROKER_PORT + "/" + AMQP_BROKER_VHOST
# Publisher object for sending messages to rabbit
printc("Creating publisher...")
self._publisher = SimplePublisher(self._broker_url)
# Machine messages
self._msg_actions_mach = {
'TRANSFER_DONE': self.process_transfer_done, # Machine done with the current job
'REGISTER': self.process_register, # New machine wants to join
'DEREGISTER': self.process_deregister, # Machine is leaving
'STATE_UPDATE': self.process_state_update # Machine updating us on its state
}
# Machines register with us and let us know how they are doing
printc("Creating machine consumer...")
self._mach_consumer = Consumer(self._broker_url, self._machine_publish_q)
try:
thread.start_new_thread(self.run_mach_consumer, ())
except:
printc("Thread run_mach_consumer failed, quitting...")
sys.exit()
return
def run_mach_consumer(self):
# Consume messages continuously
printc("Machine message consumer is running...")
self._mach_consumer.run(self.on_mach_message)
return
def on_mach_message(self, ch, method, properties, body):
# Callback from consumer to process machine messages
# Load the message which came in yaml format
msg_dict = yaml.load(body)
# Determine which function needs to be called for this message type
try:
af_handler = self._msg_actions_mach.get(msg_dict[MSG_TYPE])
except:
printc("Bad machine message received...")
ch.basic_ack(delivery_tag=method.delivery_tag)
return
# Call that function and provide it with the message
af_handler(msg_dict)
# Acknowledge that we processed the message so rabbit can remove it from the queue
ch.basic_ack(delivery_tag=method.delivery_tag)
return
def process_register(self, msg_params):
# Process a request for a machine that is registering with us
printc("Processing name request...")
tmp_name = self.pick_name()
while False == self._sb_mach.register_machine(tmp_name):
tmp_name = self.pick_name()
self._sb_mach._redis.hset(tmp_name, 'IP_ADDR', msg_params['IP_ADDR'])
printc("%s has registered. (%s)" % (tmp_name, msg_params['IP_ADDR']))
msg = {}
msg[MSG_TYPE] = 'REGISTRATION'
msg[NAME] = tmp_name
self._publisher.publish_message(self._machine_consume_q, yaml.dump(msg))
return
def pick_name(self):
# Name creation
tmp_name = ''.join(random.choice(string.ascii_letters) for x in range(NAME_LENGTH))
return self._machine_prefix + tmp_name
def process_deregister(self, msg_params):
# Machine is deregistering with us
printc("%s has deregistered." % msg_params[NAME])
self._sb_mach.machine_deregister(msg_params[NAME])
return
def process_state_update(self, msg_params):
# Machine is updating us on something, report it in the Scoreboard
self._sb_mach.machine_update(msg_params['KEY'], msg_params['FIELD'], msg_params['VALUE'])
return
def process_transfer_done(self, msg_params):
return
def test_ncsa(self, Ncsa):
self.ncsa = Ncsa
try:
cdm = toolsmod.intake_yaml_file('./tests/yaml/L1SystemCfg_Test_ncsa.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
pp_name = cdm[ROOT]['PFM_BROKER_NAME']
pp_passwd = cdm[ROOT]['PFM_BROKER_PASSWD']
pp_pub_name = cdm[ROOT]['PFM_BROKER_PUB_NAME']
pp_pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD']
pp_broker_url = "amqp://" + pp_name + ":" + \
pp_passwd + "@" + \
broker_addr
pp_pub_broker_url = "amqp://" + pp_pub_name + ":" + \
pp_pub_passwd + "@" + \
broker_addr
self.pp_publisher = SimplePublisher(pp_pub_broker_url, "YAML")
D1_name = 'D1'
D1_passwd = 'D1'
D1_pub_name = 'D1_PUB'
D1_pub_passwd = 'D1_PUB'
D1_broker_url = "amqp://" + D1_name + ":" + \
D1_passwd + "@" + \
broker_addr
D1_pub_broker_url = "amqp://" + D1_pub_name + ":" + \
D1_pub_passwd + "@" + \
broker_addr
self.d1_publisher = SimplePublisher(D1_pub_broker_url, "YAML")
D2_name = 'D2'
D2_passwd = 'D2'
D2_pub_name = 'D2_PUB'
D2_pub_passwd = 'D2_PUB'
D2_broker_url = "amqp://" + D2_name + ":" + \
D2_passwd + "@" + \
broker_addr
D2_pub_broker_url = "amqp://" + D2_pub_name + ":" + \
D2_pub_passwd + "@" + \
broker_addr
self.d2_publisher = SimplePublisher(D2_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.pp_consumer = Consumer(pp_broker_url,'pp_foreman_ack_publish', 'thread-pp',
self.on_pp_message,'YAML')
self.pp_consumer.start()
self.D1_consumer = Consumer(D1_broker_url,'d1_consume', 'thread-d1',
self.on_d1_message,'YAML')
self.D1_consumer.start()
self.D2_consumer = Consumer(D2_broker_url,'d2_consume', 'thread-d2',
self.on_d2_message,'YAML')
self.D2_consumer.start()
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(7)
self.verify_pp_messages()
self.verify_D2_messages()
self.verify_D1_messages()
sleep(2)
self.pp_consumer.stop()
self.pp_consumer.join()
self.D1_consumer.stop()
self.D1_consumer.join()
self.D2_consumer.stop()
self.D2_consumer.join()
if self.DP:
print("Finished with NCSA tests.")
def test_ppdev(self, Ppdev):
self.ppdev = Ppdev
try:
cdm = toolsmod.intake_yaml_file('./tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
dmcs_name = cdm[ROOT]['DMCS_BROKER_NAME']
dmcs_passwd = cdm[ROOT]['DMCS_BROKER_PASSWD']
dmcs_pub_name = cdm[ROOT]['DMCS_BROKER_PUB_NAME']
dmcs_pub_passwd = cdm[ROOT]['DMCS_BROKER_PUB_PASSWD']
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_passwd + "@" + \
broker_addr
dmcs_pub_broker_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_passwd + "@" + \
broker_addr
self.dmcs_publisher = SimplePublisher(dmcs_pub_broker_url, "YAML")
ncsa_name = cdm[ROOT]['NCSA_BROKER_NAME']
ncsa_passwd = cdm[ROOT]['NCSA_BROKER_PASSWD']
ncsa_pub_name = cdm[ROOT]['NCSA_BROKER_PUB_NAME']
ncsa_pub_passwd = cdm[ROOT]['NCSA_BROKER_PUB_PASSWD']
ncsa_broker_url = "amqp://" + ncsa_name + ":" + \
ncsa_passwd + "@" + \
broker_addr
ncsa_pub_broker_url = "amqp://" + ncsa_pub_name + ":" + \
ncsa_pub_passwd + "@" + \
broker_addr
self.ncsa_publisher = SimplePublisher(ncsa_pub_broker_url, "YAML")
F1_name = 'F1'
F1_passwd = 'F1'
F1_pub_name = 'F1_PUB'
F1_pub_passwd = 'F1_PUB'
F1_broker_url = "amqp://" + F1_name + ":" + \
F1_passwd + "@" + \
broker_addr
F1_pub_broker_url = "amqp://" + F1_pub_name + ":" + \
F1_pub_passwd + "@" + \
broker_addr
self.f1_publisher = SimplePublisher(F1_pub_broker_url, "YAML")
F2_name = 'F2'
F2_passwd = 'F2'
F2_pub_name = 'F2_PUB'
F2_pub_passwd = 'F2_PUB'
F2_broker_url = "amqp://" + F2_name + ":" + \
F2_passwd + "@" + \
broker_addr
F2_pub_broker_url = "amqp://" + F2_pub_name + ":" + \
F2_pub_passwd + "@" + \
broker_addr
self.f2_publisher = SimplePublisher(F2_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.dmcs_consumer = Consumer(dmcs_broker_url,'dmcs_ack_consume', 'thread-dmcs',
self.on_dmcs_message,'YAML')
self.dmcs_consumer.start()
self.ncsa_consumer = Consumer(ncsa_broker_url,'ncsa_consume', 'thread-ncsa',
self.on_ncsa_message,'YAML')
self.ncsa_consumer.start()
self.F1_consumer = Consumer(F1_broker_url,'f1_consume', 'thread-f1',
self.on_f1_message,'YAML')
self.F1_consumer.start()
self.F2_consumer = Consumer(F2_broker_url,'f2_consume', 'thread-f2',
self.on_f2_message,'YAML')
self.F2_consumer.start()
sleep(3)
if self.DP:
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(6)
self.verify_ncsa_messages()
self.verify_F2_messages()
self.verify_F1_messages()
self.verify_dmcs_messages()
sleep(2)
self.dmcs_consumer.stop()
self.dmcs_consumer.join()
self.ncsa_consumer.stop()
self.ncsa_consumer.join()
self.F1_consumer.stop()
self.F1_consumer.join()
self.F2_consumer.stop()
self.F2_consumer.join()
if self.DP:
print("Finished with PP tests.")
class TestArDev:
dmcs_pub_broker_url = None
dmcs_publisher = None
dmcs_consumer = None
dmcs_consumer_msg_list = []
ar_ctrl_pub_broker_url = None
ar_ctrl_publisher = None
ar_ctrl_consumer = None
ar_ctrl_consumer_msg_list = []
F1_pub_broker_url = None
F1_publisher = None
F1_consumer = None
f1_consumer_msg_list = []
F2_pub_broker_url = None
F2_publisher = None
F2_consumer = None
f2_consumer_msg_list = []
EXPECTED_AR_CTRL_MESSAGES = 1
EXPECTED_DMCS_MESSAGES = 1
EXPECTED_F1_MESSAGES = 1
EXPECTED_F2_MESSAGES = 1
NUM_READOUTS = 0
ccd_list = [14,17,21.86]
prp = toolsmod.prp
DP = toolsmod.DP # Debug Printing either True or False...override for this file only...
def test_ardev(self, Ardev):
self.ardev = Ardev
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test_ar.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
raise
raft_dict = cdm[ROOT]['DEFAULT_RAFT_CONFIGURATION']
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
dmcs_name = cdm[ROOT]['DMCS_BROKER_NAME']
dmcs_passwd = cdm[ROOT]['DMCS_BROKER_PASSWD']
dmcs_pub_name = cdm[ROOT]['DMCS_BROKER_PUB_NAME']
dmcs_pub_passwd = cdm[ROOT]['DMCS_BROKER_PUB_PASSWD']
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_passwd + "@" + \
broker_addr
dmcs_pub_broker_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % dmcs_pub_broker_url)
self.dmcs_publisher = SimplePublisher(dmcs_pub_broker_url, "YAML")
ar_ctrl_name = cdm[ROOT]['ARCHIVE_BROKER_NAME']
ar_ctrl_passwd = cdm[ROOT]['ARCHIVE_BROKER_PASSWD']
ar_ctrl_pub_name = cdm[ROOT]['ARCHIVE_BROKER_PUB_NAME']
ar_ctrl_pub_passwd = cdm[ROOT]['ARCHIVE_BROKER_PUB_PASSWD']
ar_ctrl_broker_url = "amqp://" + ar_ctrl_name + ":" + \
ar_ctrl_passwd + "@" + \
broker_addr
ar_ctrl_pub_broker_url = "amqp://" + ar_ctrl_pub_name + ":" + \
ar_ctrl_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % ar_ctrl_pub_broker_url)
self.ar_ctrl_publisher = SimplePublisher(ar_ctrl_pub_broker_url, "YAML")
F1_name = 'F1'
F1_passwd = 'F1'
F1_pub_name = 'F1_PUB'
F1_pub_passwd = 'F1_PUB'
F1_broker_url = "amqp://" + F1_name + ":" + \
F1_passwd + "@" + \
broker_addr
F1_pub_broker_url = "amqp://" + F1_pub_name + ":" + \
F1_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % F1_pub_broker_url)
self.F1_publisher = SimplePublisher(F1_pub_broker_url, "YAML")
F2_name = 'F2'
F2_passwd = 'F2'
F2_pub_name = 'F2_PUB'
F2_pub_passwd = 'F2_PUB'
F2_broker_url = "amqp://" + F2_name + ":" + \
F2_passwd + "@" + \
broker_addr
F2_pub_broker_url = "amqp://" + F2_pub_name + ":" + \
F2_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % F2_pub_broker_url)
self.F2_publisher = SimplePublisher(F2_pub_broker_url, "YAML")
print("All publishers are running...")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
print("MessageAuthority running...")
self.dmcs_consumer = Consumer(dmcs_broker_url,'dmcs_ack_consume', 'thread-dmcs',
self.on_dmcs_message,'YAML')
self.dmcs_consumer.start()
print("DMCS Consumer running...")
self.ar_ctrl_consumer = Consumer(ar_ctrl_broker_url,'archive_ctrl_consume', 'thread-ar-ctrl',
self.on_ar_ctrl_message,'YAML')
self.ar_ctrl_consumer.start()
print("ar_ctrl Consumer running...")
self.F1_consumer = Consumer(F1_broker_url,'f1_consume', 'thread-f1',
self.on_f1_message,'YAML')
self.F1_consumer.start()
print("F1 Consumer running...")
self.F2_consumer = Consumer(F2_broker_url,'f2_consume', 'thread-f2',
self.on_f2_message,'YAML')
self.F2_consumer.start()
print("F2 Consumer running...")
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(8)
self.verify_dmcs_messages()
self.verify_ar_ctrl_messages()
self.verify_F1_messages()
self.verify_F2_messages()
sleep(3)
# Shut down consumer threads nicely
self.dmcs_consumer.stop()
self.dmcs_consumer.join()
self.ar_ctrl_consumer.stop()
self.ar_ctrl_consumer.join()
self.F1_consumer.stop()
self.F1_consumer.join()
self.F2_consumer.stop()
self.F2_consumer.join()
if self.DP:
print("Finished with AR tests.")
def send_messages(self):
print("Starting send_messages")
# Tests only an AR device
self.EXPECTED_AR_CTRL_MESSAGES = 2
self.EXPECTED_DMCS_MESSAGES = 2
self.EXPECTED_F1_MESSAGES = 3
self.EXPECTED_F2_MESSAGES = 3
msg = {}
msg['MSG_TYPE'] = "AR_NEW_SESSION"
msg['SESSION_ID'] = 'SI_469976'
msg['ACK_ID'] = 'NEW_SESSION_ACK_44221'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
time.sleep(2)
print("New Session Message")
self.dmcs_publisher.publish_message("ar_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_NEXT_VISIT"
msg['VISIT_ID'] = 'XX_28272'
msg['JOB_NUM'] = '4xx72'
msg['SESSION_ID'] = 'SI_469976'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'NEW_VISIT_ACK_76'
msg['RA'] = "231.221"
msg['DEC'] = "-45.34"
msg['ANGLE'] = "120.0"
msg['RAFT_LIST'] = ['10','32','41','42','43']
msg['RAFT_CCD_LIST'] = [['ALL'],['02','11','12'],['00','02'],['02','12','11','22','00'],['ALL']]
time.sleep(2)
print("Next Visit Message")
self.dmcs_publisher.publish_message("ar_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_TAKE_IMAGES"
msg['JOB_NUM'] = '4xx72'
msg['NUM_IMAGES'] = '4'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'ACK_44221'
time.sleep(2)
print("AR Take Images Message")
self.dmcs_publisher.publish_message("ar_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_END_READOUT"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444245'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'AR_ACK_94671'
time.sleep(2)
print("AR END READOUT Message")
self.dmcs_publisher.publish_message("ar_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_END_READOUT"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444246'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'AR_ACK_94673'
time.sleep(2)
print("AR END READOUT Message")
self.dmcs_publisher.publish_message("ar_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_END_READOUT"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444247'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'AR_ACK_94676'
time.sleep(2)
print("AR END READOUT Message")
self.dmcs_publisher.publish_message("ar_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_END_READOUT"
msg['JOB_NUM'] = '4xx72'
msg['IMAGE_ID'] = 'IMG_444248'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'AR_ACK_94677'
time.sleep(2)
print("AR END READOUT Message")
self.dmcs_publisher.publish_message("ar_foreman_consume", msg)
msg = {}
msg['MSG_TYPE'] = "AR_TAKE_IMAGES_DONE"
msg['JOB_NUM'] = '4xx72'
msg['REPLY_QUEUE'] = 'dmcs_ack_consume'
msg['ACK_ID'] = 'ACK_44221'
time.sleep(2)
print("AR Take Images Done Message")
self.dmcs_publisher.publish_message("ar_foreman_consume", msg)
time.sleep(9)
print("Message Sender done")
def verify_dmcs_messages(self):
len_list = len(self.dmcs_consumer_msg_list)
print("DMCS RECEIVED %s MESSAGES" % len_list)
if len_list != self.EXPECTED_DMCS_MESSAGES:
print("Incorrect number of DMCS messages received")
pytest.fail('DMCS simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_DMCS_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.dmcs_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
print("This DMCS message failed verification: ")
self.prp.pprint(msg)
pytest.fail("The following DMCS Bridge response message failed when compared with the sovereign example: %s" % msg)
print("Responses to DMCS Bridge pass verification.")
def verify_ar_ctrl_messages(self):
len_list = len(self.ar_ctrl_consumer_msg_list)
print("AR_CTRL RECEIVED %s MESSAGES" % len_list)
if len_list != self.EXPECTED_AR_CTRL_MESSAGES:
print("Incorrect number of AR_CTRL messages received")
pytest.fail('AR CTRL simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_AR_CTRL_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.ar_ctrl_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
print("This AR_CTRL message failed verification: ")
self.prp.pprint(msg)
pytest.fail("The following message to the AR CTRL failed when compared with the sovereign example: %s" % msg)
print("Messages to the AR CTRL pass verification.")
def verify_F1_messages(self):
len_list = len(self.f1_consumer_msg_list)
print("F1 RECEIVED %s MESSAGES" % len_list)
if len_list != self.EXPECTED_F1_MESSAGES:
print("Incorrect number of F1 messages received")
pytest.fail('F1 simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_F1_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.f1_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
print("This F1 message failed verification: ")
self.prp.pprint(msg)
pytest.fail("The following message to F1 failed when compared with the sovereign example: %s" % msg)
print("Messages to F1 pass verification.")
def verify_F2_messages(self):
len_list = len(self.f2_consumer_msg_list)
print("F2 RECEIVED %s MESSAGES" % len_list)
if len_list != self.EXPECTED_F2_MESSAGES:
print("Incorrect number of F2 messages received")
pytest.fail('F2 simulator received incorrect number of messages.\nExpected %s but received %s'\
% (self.EXPECTED_F2_MESSAGES, len_list))
# Now check num keys in each message first, then check for key errors
for i in range(0, len_list):
msg = self.f2_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
print("This F2 message failed verification: ")
self.prp.pprint(msg)
pytest.fail("The following message to F2 failed when compared with the sovereign example: %s" % msg)
print("Messages to F2 pass verification.")
def on_dmcs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.dmcs_consumer_msg_list.append(body)
def on_ar_ctrl_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.ar_ctrl_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'NEW_ARCHIVE_ITEM':
msg = {}
msg['MSG_TYPE'] = 'NEW_ARCHIVE_ITEM_ACK'
msg['COMPONENT'] = 'ARCHIVE_CTRL'
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
msg['TARGET_LOCATION'] = '/tmp'
self.ar_ctrl_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'AR_ITEMS_XFERD':
msg = {}
msg['MSG_TYPE'] = 'AR_ITEMS_XFERD_ACK'
msg['COMPONENT'] = 'ARCHIVE_CTRL'
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
filename_list = body['RESULT_SET']['FILENAME_LIST']
msg['RESULT_SET'] = {}
msg['RESULT_SET']['IMAGE_ID_LIST'] = body['RESULT_SET']['IMAGE_ID_LIST']
msg['RESULT_SET']['RECEIPT_LIST'] = []
msg['RESULT_SET']['FILENAME_LIST'] = filename_list
RECEIPT_LIST = []
for filename in filename_list:
RECEIPT_LIST.append('x14_' + str(filename))
msg['RESULT_LIST']['RECEIPT_LIST'] = RECEIPT_LIST
self.ar_ctrl_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail("The following unknown message was received by the Archive CTRL: %s" % body)
def on_f1_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.f1_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'AR_FWDR_HEALTH_CHECK':
msg = {}
msg['MSG_TYPE'] = 'AR_FWDR_HEALTH_CHECK_ACK'
msg['COMPONENT'] = 'FORWARDER_1'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.F1_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'AR_FWDR_XFER_PARAMS':
msg = {}
msg['MSG_TYPE'] = 'AR_FWDR_XFER_PARAMS_ACK'
msg['COMPONENT'] = 'FORWARDER_1'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.F1_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'AR_FWDR_TAKE_IMAGES':
# This message handler is not necessary as it does nothing
# But it is explanatory in nature for understanding/maintaining the file.
#
# No ack necessary - but NUM_IMAGES param will be
# needed in AR_FWDR_TAKE_IMAGES_DONE message handler below
pass
elif body['MSG_TYPE'] == 'AR_FWDR_END_READOUT':
self.NUM_READOUTS = self.NUM_READOUTS + 1
elif body['MSG_TYPE'] == 'AR_FWDR_TAKE_IMAGES_DONE':
# Find message in message list for xfer_params
xfer_msg = None
image_id_list = []
num_images = 0
for msg in self.f1_consumer_msg_list:
if msg['MSG_TYPE'] == 'AR_FWDR_END_READOUT':
image_id_list.append(msg['IMAGE_ID'])
if msg['MSG_TYPE'] == 'AR_FWDR_XFER_PARAMS':
xfer_msg = msg
if msg['MSG_TYPE'] == 'AR_FWDR_TAKE_IMAGES':
num_images = int(msg['NUM_IMAGES'])
if xfer_msg == None:
pytest.fail("The AR_FWDR_XFER_PARAMS message was not received before AR_FWDR_READOUT in F1")
# use message to build response
msg = {}
msg['MSG_TYPE'] = 'AR_FWDR_TAKE_IMAGES_DONE_ACK'
msg['COMPONENT'] = 'FORWARDER_1'
msg['JOB_NUM'] = xfer_msg['JOB_NUM']
msg['ACK_ID'] = body['ACK_ID']
raft_list = xfer_msg['XFER_PARAMS']['RAFT_LIST']
raft_ccd_list = xfer_msg['XFER_PARAMS']['RAFT_CCD_LIST']
msg['RESULT_SET'] = {}
msg['RESULT_SET']['RAFT_LIST'] = raft_list
msg['RESULT_SET']['RAFT_CCD_LIST'] = raft_ccd_list
msg['RESULT_SET']['RAFT_PLUS_CCD_LIST'] = []
msg['RESULT_SET']['FILENAME_LIST'] = []
msg['RESULT_SET']['CHECKSUM_LIST'] = []
RAFT_PLUS_CCD_LIST = []
FILENAME_LIST = []
CHECKSUM_LIST = []
target_location = xfer_msg['TARGET_LOCATION']
raft_plus_ccd_list = self.convert_raft_and_ccd_list_to_name_list(raft_list, raft_ccd_list)
for ccd in raft_plus_ccd_list:
RAFT_PLUS_CCD_LIST.append(ccd)
### XXX ADD IMAGE_ID from IMAGE_ID_LIST to target_dir and ccd name
FILENAME_LIST.append(target_location + "/" + str(ccd))
CHECKSUM_LIST.append('XXXXFFFF4444$$$$')
msg['RESULT_SET']['RAFT_PLUS_CCD_LIST'] = RAFT_PLUS_CCD_LIST
msg['RESULT_SET']['FILENAME_LIST'] = FILENAME_LIST
msg['RESULT_SET']['CHECKSUM_LIST'] = CHECKSUM_LIST
if num_images == self.NUM_READOUTS:
msg['ACK_BOOL'] = True
else:
msg['ACK_BOOL'] = False
self.F1_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail("The following unknown message was received by FWDR F1: %s" % body)
def on_f2_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.f2_consumer_msg_list.append(body)
if body['MSG_TYPE'] == 'AR_FWDR_HEALTH_CHECK':
msg = {}
msg['MSG_TYPE'] = 'AR_FWDR_HEALTH_CHECK_ACK'
msg['COMPONENT'] = 'FORWARDER_2'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.F2_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'AR_FWDR_XFER_PARAMS':
msg = {}
msg['MSG_TYPE'] = 'AR_FWDR_XFER_PARAMS_ACK'
msg['COMPONENT'] = 'FORWARDER_2'
msg['ACK_BOOL'] = True
msg['ACK_ID'] = body['ACK_ID']
self.F2_publisher.publish_message(body['REPLY_QUEUE'], msg)
elif body['MSG_TYPE'] == 'AR_FWDR_TAKE_IMAGES':
# This message handler is not necessary as it does nothing
# But it is explanatory in nature for understanding/maintaining the file.
#
# No ack necessary - but NUM_IMAGES param will be
# needed in AR_FWDR_TAKE_IMAGES_DONE message handler below
pass
elif body['MSG_TYPE'] == 'AR_FWDR_END_READOUT':
pass
elif body['MSG_TYPE'] == 'AR_FWDR_TAKE_IMAGES_DONE':
# Find message in message list for xfer_params
xfer_msg = None
image_id_list = []
num_images = 0;
for msg in self.f2_consumer_msg_list:
if msg['MSG_TYPE'] == 'AR_FWDR_END_READOUT':
image_id_list.append(msg['IMAGE_ID'])
if msg['MSG_TYPE'] == 'AR_FWDR_XFER_PARAMS':
xfer_msg = msg
if msg['MSG_TYPE'] == 'AR_FWDR_TAKE_IMAGES':
num_images = msg['NUM_IMAGES']
if xfer_msg == None:
pytest.fail("The AR_FWDR_XFER_PARAMS message was not received before AR_FWDR_READOUT in F1")
# use message to build response
msg = {}
msg['MSG_TYPE'] = 'AR_FWDR_TAKE_IMAGES_DONE_ACK'
msg['COMPONENT'] = 'FORWARDER_2'
msg['JOB_NUM'] = xfer_msg['JOB_NUM']
msg['ACK_ID'] = body['ACK_ID']
msg['ACK_BOOL'] = True
raft_list = xfer_msg['XFER_PARAMS']['RAFT_LIST']
raft_ccd_list = xfer_msg['XFER_PARAMS']['RAFT_CCD_LIST']
msg['RESULT_SET'] = {}
msg['RESULT_SET']['RAFT_LIST'] = raft_list
msg['RESULT_SET']['RAFT_CCD_LIST'] = raft_ccd_list
msg['RESULT_SET']['RAFT_PLUS_CCD_LIST'] = []
msg['RESULT_SET']['FILENAME_LIST'] = []
msg['RESULT_SET']['CHECKSUM_LIST'] = []
RAFT_PLUS_CCD_LIST = []
FILENAME_LIST = []
CHECKSUM_LIST = []
target_location = xfer_msg['TARGET_LOCATION']
raft_plus_ccd_list = self.convert_raft_and_ccd_list_to_name_list(raft_list, raft_ccd_list)
for ccd in raft_plus_ccd_list:
RAFT_PLUS_CCD_LIST.append(ccd)
### XXX ADD IMAGE_ID from IMAGE_ID_LIST to target_location and ccd name
FILENAME_LIST.append(target_location + "/" + str(ccd))
CHECKSUM_LIST.append('XXXXFFFF4444$$$$')
msg['RESULT_SET']['RAFT_PLUS_CCD_LIST'] = RAFT_PLUS_CCD_LIST
msg['RESULT_SET']['FILENAME_LIST'] = FILENAME_LIST
msg['RESULT_SET']['CHECKSUM_LIST'] = CHECKSUM_LIST
self.F2_publisher.publish_message(body['REPLY_QUEUE'], msg)
else:
pytest.fail("The following unknown message was received by FWDR F2: %s" % body)
def convert_raftdict_to_name_list(self, rdict):
raft_list = list(rdict.keys())
num_rafts = len(raft_list)
integrated_names_list = []
for i in range(0,num_rafts):
current_raft = raft_list[i]
ccd_list = []
ccd_list = rdict[current_raft]
if ccd_list[0] == 'ALL':
ccd_list = ['00','10','20','01','11','21','02','12','22']
num_current_ccds = len(ccd_list)
for j in range(0,num_current_ccds):
tmp_str = current_raft + '-' + ccd_list[j]
integrated_names_list.append(tmp_str)
return integrated_names_list
def convert_raft_and_ccd_list_to_name_list(self, raft_list, raft_ccd_list):
#raft_list = list(rdict.keys())
num_rafts = len(raft_list)
integrated_names_list = []
for i in range(0,num_rafts):
current_raft = raft_list[i]
ccd_list = []
ccd_list = raft_ccd_list[i]
if ccd_list[0] == 'ALL':
ccd_list = ['00','10','20','01','11','21','02','12','22']
num_current_ccds = len(ccd_list)
for j in range(0,num_current_ccds):
tmp_str = current_raft + '-' + ccd_list[j]
integrated_names_list.append(tmp_str)
return integrated_names_list
class TestOCS_CommandListener:
os.chdir("ocs/src")
cmd = subprocess.Popen("./CommandListener&", shell=True, preexec_fn=os.setsid)
sleep(10)
EXPECTED_DMCS_MESSAGES = 7
dmcs_consumer = None
dmcs_consumer_msg_list = []
def test_ocs_commandlistener(self):
try:
cdm = toolsmod.intake_yaml_file("../../tests/yaml/L1SystemCfg_Test_ocs_bridge.yaml")
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to fine CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]["BASE_BROKER_ADDR"]
dmcs_name = cdm[ROOT]["DMCS_BROKER_NAME"]
dmcs_pwd = cdm[ROOT]["DMCS_BROKER_PASSWD"]
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_pwd + "@" + \
broker_addr
self.dmcs_consumer = Consumer(dmcs_broker_url, "ocs_dmcs_consume", "thread-dmcs-consume",
self.on_ocs_message, "YAML")
self.dmcs_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority("../../messages.yaml")
self.send_messages()
sleep(10)
os.killpg(os.getpgid(self.cmd.pid), signal.SIGTERM)
self.verify_ocs_messages()
print("Finished with CommandListener tests.")
def send_messages(self):
os.chdir("../commands/")
devices = ["archiver", "catchuparchiver", "processingcluster", "atArchiver"]
commands = ["start", "enable", "disable", "enterControl", "exitControl", "standby", "abort"]
for device in devices:
for command in commands:
cmd = None
if command == "start":
cmd = "./sacpp_" + device + "_" + command + "_commander Normal"
else:
cmd = "./sacpp_" + device + "_" + command + "_commander 0"
p = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid)
print("=== " + device.upper() + " " + command.upper() + " Message")
sleep(10) # this is not random. startup .sacpp_ thing takes about 7 seconds.
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
print("Message Sender Done.")
def verify_ocs_messages(self):
len_list = len(self.dmcs_consumer_msg_list)
if len_list != self.EXPECTED_DMCS_MESSAGES:
pytest.fail("DMCS simulator received incorrect number of messages.\n Expected %s but received %s" \
% (self.EXPECTED_DMCS_MESSAGES, len_list))
for i in range(0, len_list):
msg = self.dmcs_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following OCS Bridge response message failed when compared with the sovereign\
example: %s" % msg)
print("Responses to DMCS pass verification")
def clear_message_lists(self):
self.dmcs_consumer_msg_list = []
def on_ocs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.dmcs_consumer_msg_list.append(body)
class TestOCS_EventSubscriber:
os.chdir("ocs/src")
cmd = subprocess.Popen("./EventSubscriber&", shell=True, preexec_fn=os.setsid)
sleep(10)
EXPECTED_DMCS_MESSAGES = 3
dmcs_consumer = None
dmcs_consumer_msg_list = []
def test_ocs_eventsubscriber(self):
try:
cdm = toolsmod.intake_yaml_file("../../tests/yaml/L1SystemCfg_Test_ocs_bridge.yaml")
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to fine CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]["BASE_BROKER_ADDR"]
dmcs_name = cdm[ROOT]["DMCS_BROKER_NAME"]
dmcs_pwd = cdm[ROOT]["DMCS_BROKER_PASSWD"]
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_pwd + "@" + \
broker_addr
self.dmcs_consumer = Consumer(dmcs_broker_url, "ocs_dmcs_consume", "thread-dmcs-consume",
self.on_ocs_message, "YAML")
self.dmcs_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority("../../messages.yaml")
self.send_messages()
sleep(10)
os.killpg(os.getpgid(self.cmd.pid), signal.SIGTERM)
self.verify_ocs_messages()
print("Finished with EventSubscriber tests.")
def send_messages(self):
os.chdir("../events/")
#startIntegration = subprocess.Popen("./startIntegration testImage 0", shell=True, preexec_fn=os.setsid)
#print("=== START_INTEGRATION Message")
#sleep(10)
#startReadout = subprocess.Popen("./startReadout 0", shell=True, preexec_fn=os.setsid)
#print("=== START_READOUT Message")
#sleep(10)
#endReadout = subprocess.Popen("./sacpp_archiver_endReadout_send IMG1 0", shell=True, preexec_fn=os.setsid)
#print("=== END_READOUT Message")
#sleep(10)
#startShutterOpen = subprocess.Popen("./startShutterOpen 0", shell=True, preexec_fn=os.setsid)
#print("=== START_SHUTTER_OPEN Message")
#sleep(10)
#startShutterClose = subprocess.Popen("./startShutterClose 0", shell=True, preexec_fn=os.setsid)
#print("=== START_SHUTTER_CLOSE Message")
#sleep(10)
#endShutterOpen = subprocess.Popen("./endShutterOpen 0", shell=True, preexec_fn=os.setsid)
#print("=== END_SHUTTER_OPEN Message")
#sleep(10)
#endShutterClose = subprocess.Popen("./endShutterClose 0", shell=True, preexec_fn=os.setsid)
#print("=== END_SHUTTER_CLOSE Message")
#sleep(10)
# completed ok gives problem
#takeImages = subprocess.Popen("./takeImages 1 1 1 1 1 1 testing", shell=True, preexec_fn=os.setsid)
#print("=== TAKE_IMAGES Message")
#sleep(10)
#os.killpg(os.getpgid(takeImages.pid), signal.SIGTERM)
#tcsTarget = subprocess.Popen("./tcsTarget 1 1 1 1 1 1 1 1 1 1 1", shell=True, preexec_fn=os.setsid)
#print("=== TCS_TARGET Message")
#sleep(10)
#os.killpg(os.getpgid(tcsTarget.pid), signal.SIGTERM)
#######################################################################
# AT EVENTS
#######################################################################
at_startIntegration = subprocess.Popen("./sacpp_atcamera_startIntegration_send seq1 1 image_1 1 0 0 0", shell=True, preexec_fn=os.setsid)
print("=== AT_START_INTEGRATION Message")
sleep(10)
os.killpg(os.getpgid(at_startIntegration.pid), signal.SIGTERM)
at_endReadout = subprocess.Popen("./sacpp_atcamera_endReadout_send seq1 1 image_1 1 0 0 0", shell=True, preexec_fn=os.setsid)
print("=== AT_END_READOUT Message")
sleep(10)
os.killpg(os.getpgid(at_endReadout.pid), signal.SIGTERM)
efd_lfoa = subprocess.Popen("./sacpp_efd_LargeFileObjectAvailable_send 0 0 AT msg https://141.142.238.15//tmp/header/img_1.header 0 img_1 0", shell=True, preexec_fn=os.setsid)
print("=== EFD_HEADER_READY Message")
sleep(10)
os.killpg(os.getpgid(efd_lfoa.pid), signal.SIGTERM)
print("Message Sender Done.")
def verify_ocs_messages(self):
len_list = len(self.dmcs_consumer_msg_list)
if len_list != self.EXPECTED_DMCS_MESSAGES:
pytest.fail("DMCS simulator received incorrect number of messages.\n Expected %s but received %s" \
% (self.EXPECTED_DMCS_MESSAGES, len_list))
for i in range(0, len_list):
msg = self.dmcs_consumer_msg_list[i]
result = self._msg_auth.check_message_shape(msg)
if result == False:
pytest.fail("The following OCS Bridge response message failed when compared with the sovereign\
example: %s" % msg)
print("Responses to DMCS pass verification")
def clear_message_lists(self):
self.dmcs_consumer_msg_list = []
def on_ocs_message(self, ch, method, properties, body):
ch.basic_ack(method.delivery_tag)
self.dmcs_consumer_msg_list.append(body)