def __init__(self):
logging.basicConfig()
#os.system('rabbitmqctl -p /tester purge_queue firehose')
#os.system('rabbitmqctl -p /tester purge_queue ack_publish')
#broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny?heartbeat=300'
broker_url = 'amqp://*****:*****@141.142.238.10:5672/%2Ftest'
#pub_broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Ftester'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#self._cons = FirehoseConsumer(broker_url, 'firehose', "YAML")
"""
self._cons = Consumer(broker_url, 'ar_foreman_consume', "YAML")
try:
_thread.start_new_thread( self.do_it, ("thread-1", 2,) )
except e:
print("Cannot start thread")
print(e)
self._cons2 = Consumer(broker_url, 'pp_foreman_consume', "YAML")
try:
_thread.start_new_thread( self.do_it2, ("thread-3", 2,) )
except e:
print("Cannot start thread")
print(e)
"""
cdm = toolsmod.intake_yaml_file("L1SystemCfg.yaml")
self.ccd_list = cdm['ROOT']['CCD_LIST']
def next_receipt_number(self):
last_receipt = toolsmod.intake_yaml_file(self.RECEIPT_FILE)
current_receipt = int(last_receipt['RECEIPT_ID']) + 1
session_dict = {}
session_dict['RECEIPT_ID'] = current_receipt
toolsmod.export_yaml_file(self.RECEIPT_FILE, session_dict)
return current_receipt
def test_ocs_commandlistener(self):
try:
cdm = toolsmod.intake_yaml_file(
"/home/hwin16/src/git/ctrl_iip/python/lsst/iip/tests/yaml/L1SystemCfg_Test_ocs_bridge.yaml"
)
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to fine CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]["BASE_BROKER_ADDR"]
dmcs_name = cdm[ROOT]["DMCS_BROKER_NAME"]
dmcs_pwd = cdm[ROOT]["DMCS_BROKER_PASSWD"]
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_pwd + "@" + \
broker_addr
self.dmcs_consumer = Consumer(dmcs_broker_url, "ocs_dmcs_consume",
"thread-dmcs-consume",
self.on_ocs_message, "YAML")
self.dmcs_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority(
"/home/hwin16/src/git/ctrl_iip/python/lsst/iip/messages.yaml")
self.send_messages()
sleep(10)
os.killpg(os.getpgid(self.cmd.pid), signal.SIGTERM)
self.verify_ocs_messages()
print("Finished with CommandListener tests.")
def __init__(self):
logging.basicConfig()
self.ack_test = True
#os.system('rabbitmqctl -p /tester purge_queue firehose')
#os.system('rabbitmqctl -p /tester purge_queue ack_publish')
broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Ftester'
#broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
#self._cons = FirehoseConsumer(broker_url, 'firehose', "YAML")
self.sp1 = SimplePublisher(
'amqp://*****:*****@141.142.208.191:5672/%2Fbunny')
cdm = toolsmod.intake_yaml_file('ForemanCfg.yaml')
self.fdict = cdm[ROOT]['XFER_COMPONENTS']['ARCHIVE_FORWARDERS']
self.fwdrs = list(self.fdict.keys())
self._cons = Consumer(broker_url, 'ar_foreman_ack_publish', "YAML")
#self._cons = Consumer(broker_url, 'ar_foreman_consume', "YAML")
try:
_thread.start_new_thread(self.do_it, (
"thread-1",
2,
))
except e:
print("Cannot start thread")
print(e)
def test_ocs_eventsubscriber(self):
try:
cdm = toolsmod.intake_yaml_file("../../tests/yaml/L1SystemCfg_Test_ocs_bridge.yaml")
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to fine CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]["BASE_BROKER_ADDR"]
dmcs_name = cdm[ROOT]["DMCS_BROKER_NAME"]
dmcs_pwd = cdm[ROOT]["DMCS_BROKER_PASSWD"]
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_pwd + "@" + \
broker_addr
self.dmcs_consumer = Consumer(dmcs_broker_url, "ocs_dmcs_consume", "thread-dmcs-consume",
self.on_ocs_message, "YAML")
self.dmcs_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority("../../messages.yaml")
self.send_messages()
sleep(10)
os.killpg(os.getpgid(self.cmd.pid), signal.SIGTERM)
self.verify_ocs_messages()
print("Finished with EventSubscriber tests.")
def setUp(self):
cred = Credentials("iip_cred.yaml")
user = cred.getUser()
passwd = cred.getPasswd()
# read CFG file
self._cfg_root = toolsmod.intake_yaml_file(self.CFG_FILE)['ROOT']
# setup forwarder publisher
self._fwdr_cfg = self._cfg_root['XFER_COMPONENTS']['AUX_FORWARDERS'][
self.FORWARDER]
self._fwdr_amqp = 'amqp://%s:%s@%s' % (
user, passwd, self._cfg_root['BASE_BROKER_ADDR'])
self._fwdr_publisher = SimplePublisher(self._fwdr_amqp, 'YAML')
# setup dmcs publisher
self._dmcs_amqp = 'amqp://%s:%s@%s' % (
user, passwd, self._cfg_root['BASE_BROKER_ADDR'])
self._dmcs_publisher = SimplePublisher(self._dmcs_amqp, 'YAML')
# setup archiveController publisher
self._at_ctrl_amqp = 'amqp://%s:%s@%s' % (
user, passwd, self._cfg_root['BASE_BROKER_ADDR'])
self._at_ctrl_publisher = SimplePublisher(self._at_ctrl_amqp, 'YAML')
def test_ocs_acksubscriber(self):
try:
cdm = toolsmod.intake_yaml_file(
"../../tests/yaml/L1SystemCfg_Test_ocs_bridge.yaml")
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to fine CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]["BASE_BROKER_ADDR"]
# dmcs publisher
dmcs_pub_name = cdm[ROOT]["DMCS_BROKER_PUB_NAME"]
dmcs_pub_pwd = cdm[ROOT]["DMCS_BROKER_PUB_PASSWD"]
dmcs_broker_pub_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_pwd + "@" + \
broker_addr
self.dmcs_publisher = SimplePublisher(dmcs_broker_pub_url, "YAML")
# dmcs consumer
dmcs_name = cdm[ROOT]["DMCS_BROKER_NAME"]
dmcs_pwd = cdm[ROOT]["DMCS_BROKER_PASSWD"]
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_pwd + "@" + \
broker_addr
self.dmcs_consumer = Consumer(dmcs_broker_url, "ocs_dmcs_consume",
"thread-dmcs-consume",
self.on_ocs_message, "YAML")
self.dmcs_consumer.start()
# ocs consumer from DMCS
ocs_name = cdm[ROOT]["OCS_BROKER_NAME"]
ocs_pwd = cdm[ROOT]["OCS_BROKER_PASSWD"]
# FIXME: New OCS account for consumer test_dmcs_ocs_publish
ocs_broker_url = "amqp://" + "AFM" + ":" +\
"AFM" + "@" +\
broker_addr
self.ocs_consumer = Consumer(ocs_broker_url, "test_dmcs_ocs_publish",
"thread-ocs-consume",
self.on_dmcs_message, "YAML")
self.ocs_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority("../../messages.yaml")
self.send_messages()
sleep(10)
os.killpg(os.getpgid(self.cmdListener.pid), signal.SIGTERM)
os.killpg(os.getpgid(self.ackSubscriber.pid), signal.SIGTERM)
print("MY OCS MESSAGES: %s" % self.ocs_consumer_msg_list)
self.verify_ocs_messages()
print("Finished with CommandListener tests.")
def test_ocs_acksubscriber(self):
try:
cdm = toolsmod.intake_yaml_file("../../tests/yaml/L1SystemCfg_Test_ocs_bridge.yaml")
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to fine CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]["BASE_BROKER_ADDR"]
# dmcs publisher
dmcs_pub_name = cdm[ROOT]["DMCS_BROKER_PUB_NAME"]
dmcs_pub_pwd = cdm[ROOT]["DMCS_BROKER_PUB_PASSWD"]
dmcs_broker_pub_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_pwd + "@" + \
broker_addr
self.dmcs_publisher = SimplePublisher(dmcs_broker_pub_url, "YAML")
# dmcs consumer
dmcs_name = cdm[ROOT]["DMCS_BROKER_NAME"]
dmcs_pwd = cdm[ROOT]["DMCS_BROKER_PASSWD"]
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_pwd + "@" + \
broker_addr
self.dmcs_consumer = Consumer(dmcs_broker_url, "ocs_dmcs_consume", "thread-dmcs-consume",
self.on_ocs_message, "YAML")
self.dmcs_consumer.start()
# ocs consumer from DMCS
ocs_name = cdm[ROOT]["OCS_BROKER_NAME"]
ocs_pwd = cdm[ROOT]["OCS_BROKER_PASSWD"]
# FIXME: New OCS account for consumer test_dmcs_ocs_publish
ocs_broker_url = "amqp://" + "AFM" + ":" +\
"AFM" + "@" +\
broker_addr
self.ocs_consumer = Consumer(ocs_broker_url, "test_dmcs_ocs_publish", "thread-ocs-consume",
self.on_dmcs_message, "YAML")
self.ocs_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority("../../messages.yaml")
self.send_messages()
sleep(10)
os.killpg(os.getpgid(self.cmdListener.pid), signal.SIGTERM)
os.killpg(os.getpgid(self.ackSubscriber.pid), signal.SIGTERM)
print("MY OCS MESSAGES: %s" % self.ocs_consumer_msg_list)
self.verify_ocs_messages()
print("Finished with CommandListener tests.")
def __init__(self):
logging.basicConfig()
self.ack_test = True
broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
self.sp1 = SimplePublisher('amqp://*****:*****@141.142.208.191:5672/%2Fbunny')
cdm = toolsmod.intake_yaml_file('ForemanCfg.yaml')
self.fdict = cdm[ROOT]['XFER_COMPONENTS']['ARCHIVE_FORWARDERS']
self.fwdrs = list(self.fdict.keys())
self._cons = Consumer(broker_url, 'ar_foreman_ack_publish', "YAML")
try:
_thread.start_new_thread(self.do_it, ("thread-1", 2,))
except Exception as e:
print("Cannot start thread")
print(e)
def __init__(self, filename=None):
self.prp = pprint.PrettyPrinter(indent=4)
self._message_dictionary_file = './messages.yaml'
if filename != None:
self._message_dictionary_file = filename
LOGGER.info('Reading YAML message dictionary file %s' %
self._message_dictionary_file)
try:
self.MSG_DICT = toolsmod.intake_yaml_file(
self._message_dictionary_file)
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find Message Dictionary Yaml file %s\n" % self._message_dictionary_file
LOGGER.critical(emsg + trace)
sys.exit(101)
def extract_config_values(self):
LOGGER.info('Reading YAML Config file %s' % self._config_file)
try:
cdm = toolsmod.intake_yaml_file(self._config_file)
except IOError as e:
LOGGER.critical("Unable to find CFG Yaml file %s\n" %
self._config_file)
print("Unable to find CFG Yaml file %s\n" % self._config_file)
raise L1ConfigIOError("Trouble opening CFG Yaml file %s: %s" %
(self._config_file, e.arg))
try:
self._sub_name = cdm[ROOT][
PFM_BROKER_NAME] # Message broker user & passwd
self._sub_passwd = cdm[ROOT][PFM_BROKER_PASSWD]
self._pub_name = cdm[ROOT][
'PFM_BROKER_PUB_NAME'] # Message broker user & passwd
self._pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD']
self._sub_ncsa_name = cdm[ROOT]['PFM_NCSA_BROKER_NAME']
self._sub_ncsa_passwd = cdm[ROOT]['PFM_NCSA_BROKER_PASSWD']
self._pub_ncsa_name = cdm[ROOT]['PFM_NCSA_BROKER_PUB_NAME']
self._pub_ncsa_passwd = cdm[ROOT]['PFM_NCSA_BROKER_PUB_PASSWD']
self._base_broker_addr = cdm[ROOT][BASE_BROKER_ADDR]
self._ncsa_broker_addr = cdm[ROOT][NCSA_BROKER_ADDR]
self._forwarder_dict = cdm[ROOT][XFER_COMPONENTS]['PP_FORWARDERS']
self._scbd_dict = cdm[ROOT]['SCOREBOARDS']
self.DMCS_FAULT_QUEUE = cdm[ROOT]['DMCS_FAULT_QUEUE']
self._policy_max_ccds_per_fwdr = int(
cdm[ROOT]['POLICY']['MAX_CCDS_PER_FWDR'])
except KeyError as e:
LOGGER.critical("CDM Dictionary Key error")
LOGGER.critical("Offending Key is %s", str(e))
LOGGER.critical("Bailing out...")
print("KeyError when reading CFG file. Check logs...exiting...")
raise L1ConfigKeyError("Key Error when reading config file: %s" %
e.arg)
self._base_msg_format = 'YAML'
self._ncsa_msg_format = 'YAML'
if 'BASE_MSG_FORMAT' in cdm[ROOT]:
self._base_msg_format = cdm[ROOT][BASE_MSG_FORMAT]
if 'NCSA_MSG_FORMAT' in cdm[ROOT]:
self._ncsa_msg_format = cdm[ROOT][NCSA_MSG_FORMAT]
def extract_config_values(self):
""" Parse system config yaml file.
Throw error messages if Yaml file or key not found.
:params: None.
:return: True.
"""
LOGGER.info('Reading YAML Config file %s' % self._config_file)
try:
cdm = toolsmod.intake_yaml_file(self._config_file)
except IOError as e:
LOGGER.critical("Unable to find CFG Yaml file %s\n" %
self._config_file)
sys.exit(101)
try:
self._msg_name = cdm[ROOT][
AFM_BROKER_NAME] # Message broker user & passwd
self._msg_passwd = cdm[ROOT][AFM_BROKER_PASSWD]
self._msg_pub_name = cdm[ROOT][
'AFM_BROKER_PUB_NAME'] # Message broker user & passwd
self._msg_pub_passwd = cdm[ROOT]['AFM_BROKER_PUB_PASSWD']
self._ncsa_name = cdm[ROOT][NCSA_BROKER_NAME]
self._ncsa_passwd = cdm[ROOT][NCSA_BROKER_PASSWD]
self._base_broker_addr = cdm[ROOT][BASE_BROKER_ADDR]
self._ncsa_broker_addr = cdm[ROOT][NCSA_BROKER_ADDR]
self._forwarder_dict = cdm[ROOT][XFER_COMPONENTS][
'ARCHIVE_FORWARDERS']
self._scbd_dict = cdm[ROOT]['SCOREBOARDS']
# Placeholder until eventually worked out by Data Backbone team
self.archive_fqn = cdm[ROOT]['ARCHIVE']['ARCHIVE_NAME']
self.archive_name = cdm[ROOT]['ARCHIVE']['ARCHIVE_LOGIN']
self.archive_ip = cdm[ROOT]['ARCHIVE']['ARCHIVE_IP']
except KeyError as e:
print("Dictionary error")
print("Bailing out...")
sys.exit(99)
self._base_msg_format = 'YAML'
if 'BASE_MSG_FORMAT' in cdm[ROOT]:
self._base_msg_format = cdm[ROOT]['BASE_MSG_FORMAT']
def test_ar_ctrl(self):
try:
cdm = toolsmod.intake_yaml_file(
'/home/centos/src/git/ctrl_iip/python/lsst/iip/tests/yaml/L1SystemCfg_Test_ar-ctrl.yaml'
)
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
afm_name = cdm[ROOT]["AFM_BROKER_NAME"]
afm_pwd = cdm[ROOT]["AFM_BROKER_PASSWD"]
afm_pub_name = cdm[ROOT]["AFM_BROKER_PUB_NAME"]
afm_pub_pwd = cdm[ROOT]["AFM_BROKER_PUB_PASSWD"]
self.xfer_root = cdm[ROOT]["ARCHIVE"]["ARCHIVE_XFER_ROOT"]
ar_dev_pub_broker_url = "amqp://" + afm_pub_name + ":" +\
afm_pub_pwd + "@" +\
broker_addr
self.ar_dev_publisher = SimplePublisher(ar_dev_pub_broker_url, "YAML")
ar_dev_broker_url = "amqp://" + afm_name + ":" +\
afm_pwd + "@" +\
broker_addr
self.ar_dev_consumer = Consumer(ar_dev_broker_url, self.ACK_FROM_CTRL,
"thread-ar-dev", self.on_ar_message,
"YAML", None)
self.ar_dev_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority(
'/home/centos/src/git/ctrl_iip/python/lsst/iip/messages.yaml')
self.send_messages()
sleep(10)
self.verify_ar_messages()
self.verify_path_files()
print("Finished with AR CTRL tests.")
def __init__(self, filename=None):
self._session_id = None
self._name = "ARCHIVE_CTRL"
self._config_file = 'L1SystemCfg.yaml'
if filename != None:
self._config_file = filename
cdm = toolsmod.intake_yaml_file(self._config_file)
try:
self._archive_name = cdm[ROOT][
'ARCHIVE_BROKER_NAME'] # Message broker user/passwd for component
self._archive_passwd = cdm[ROOT]['ARCHIVE_BROKER_PASSWD']
self._base_broker_addr = cdm[ROOT][BASE_BROKER_ADDR]
self._archive_xfer_root = cdm[ROOT]['ARCHIVE']['ARCHIVE_XFER_ROOT']
if cdm[ROOT]['ARCHIVE']['CHECKSUM_ENABLED'] == 'yes':
self.CHECKSUM_ENABLED = True
else:
self.CHECKSUM_ENABLED = False
except KeyError as e:
raise L1Error(e)
self._base_msg_format = self.YAML
if 'BASE_MSG_FORMAT' in cdm[ROOT]:
self._base_msg_format = cdm[ROOT][BASE_MSG_FORMAT]
self._base_broker_url = "amqp://" + self._archive_name + ":" + self._archive_passwd + "@" + str(
self._base_broker_addr)
LOGGER.info(
'Building _base_broker_url connection string for Archive Controller. Result is %s',
self._base_broker_url)
self._msg_actions = {
'ARCHIVE_HEALTH_CHECK': self.process_health_check,
'NEW_ARCHIVE_ITEM': self.process_new_archive_item,
'AR_ITEMS_XFERD': self.process_transfer_complete
}
self.setup_consumer()
self.setup_publisher()
def __init__(self):
logging.basicConfig()
self.ack_test = True
broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny'
self.sp1 = SimplePublisher(
'amqp://*****:*****@141.142.208.191:5672/%2Fbunny')
cdm = toolsmod.intake_yaml_file('ForemanCfg.yaml')
self.fdict = cdm[ROOT]['XFER_COMPONENTS']['ARCHIVE_FORWARDERS']
self.fwdrs = list(self.fdict.keys())
self._cons = Consumer(broker_url, 'ar_foreman_ack_publish', "YAML")
try:
_thread.start_new_thread(self.do_it, (
"thread-1",
2,
))
except Exception as e:
print("Cannot start thread")
print(e)
def extract_config_values(self):
try:
cdm = toolsmod.intake_yaml_file(self._config_file)
except IOError as e:
LOGGER.critical("Unable to find CFG Yaml file %s\n" %
self._config_file)
sys.exit(101)
try:
self._base_broker_addr = cdm[ROOT][BASE_BROKER_ADDR]
self._ncsa_broker_addr = cdm[ROOT][NCSA_BROKER_ADDR]
self._sub_base_name = cdm[ROOT][
'NFM_BASE_BROKER_NAME'] # Message broker user & passwd
self._sub_base_passwd = cdm[ROOT]['NFM_BASE_BROKER_PASSWD']
self._sub_ncsa_name = cdm[ROOT][
'NFM_NCSA_BROKER_NAME'] # Message broker user & passwd
self._sub_ncsa_passwd = cdm[ROOT]['NFM_NCSA_BROKER_PASSWD']
self._pub_base_name = cdm[ROOT]['BASE_BROKER_PUB_NAME']
self._pub_base_passwd = cdm[ROOT]['BASE_BROKER_PUB_PASSWD']
self._pub_ncsa_name = cdm[ROOT]['NCSA_BROKER_PUB_NAME']
self._pub_ncsa_passwd = cdm[ROOT]['NCSA_BROKER_PUB_PASSWD']
self._scbd_dict = cdm[ROOT]['SCOREBOARDS']
self.distributor_dict = cdm[ROOT][XFER_COMPONENTS][DISTRIBUTORS]
except KeyError as e:
LOGGER.critical("CDM Dictionary error - missing Key")
LOGGER.critical("Offending Key is %s", str(e))
LOGGER.critical("Bailing Out...")
sys.exit(99)
self._base_msg_format = 'YAML'
self._ncsa_msg_format = 'YAML'
if 'BASE_MSG_FORMAT' in cdm[ROOT]:
self._base_msg_format = cdm[ROOT][BASE_MSG_FORMAT]
if 'NCSA_MSG_FORMAT' in cdm[ROOT]:
self._ncsa_msg_format = cdm[ROOT][NCSA_MSG_FORMAT]
def test_ar_ctrl(self):
try:
cdm = toolsmod.intake_yaml_file('/home/centos/src/git/ctrl_iip/python/lsst/iip/tests/yaml/L1SystemCfg_Test_ar-ctrl.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
afm_name = cdm[ROOT]["AFM_BROKER_NAME"]
afm_pwd = cdm[ROOT]["AFM_BROKER_PASSWD"]
afm_pub_name = cdm[ROOT]["AFM_BROKER_PUB_NAME"]
afm_pub_pwd = cdm[ROOT]["AFM_BROKER_PUB_PASSWD"]
self.xfer_root = cdm[ROOT]["ARCHIVE"]["ARCHIVE_XFER_ROOT"]
ar_dev_pub_broker_url = "amqp://" + afm_pub_name + ":" +\
afm_pub_pwd + "@" +\
broker_addr
self.ar_dev_publisher = SimplePublisher(ar_dev_pub_broker_url, "YAML")
ar_dev_broker_url = "amqp://" + afm_name + ":" +\
afm_pwd + "@" +\
broker_addr
self.ar_dev_consumer = Consumer(ar_dev_broker_url, self.ACK_FROM_CTRL, "thread-ar-dev",
self.on_ar_message, "YAML", None)
self.ar_dev_consumer.start()
print("Test setup Complete. Commencing Messages...")
self._msg_auth = MessageAuthority('/home/centos/src/git/ctrl_iip/python/lsst/iip/messages.yaml')
self.send_messages()
sleep(10)
self.verify_ar_messages()
self.verify_path_files()
print("Finished with AR CTRL tests.")
def test_ardev(self, Ardev):
self.ardev = Ardev
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test_ar.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
raise
raft_dict = cdm[ROOT]['DEFAULT_RAFT_CONFIGURATION']
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
dmcs_name = cdm[ROOT]['DMCS_BROKER_NAME']
dmcs_passwd = cdm[ROOT]['DMCS_BROKER_PASSWD']
dmcs_pub_name = cdm[ROOT]['DMCS_BROKER_PUB_NAME']
dmcs_pub_passwd = cdm[ROOT]['DMCS_BROKER_PUB_PASSWD']
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_passwd + "@" + \
broker_addr
dmcs_pub_broker_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % dmcs_pub_broker_url)
self.dmcs_publisher = SimplePublisher(dmcs_pub_broker_url, "YAML")
ar_ctrl_name = cdm[ROOT]['ARCHIVE_BROKER_NAME']
ar_ctrl_passwd = cdm[ROOT]['ARCHIVE_BROKER_PASSWD']
ar_ctrl_pub_name = cdm[ROOT]['ARCHIVE_BROKER_PUB_NAME']
ar_ctrl_pub_passwd = cdm[ROOT]['ARCHIVE_BROKER_PUB_PASSWD']
ar_ctrl_broker_url = "amqp://" + ar_ctrl_name + ":" + \
ar_ctrl_passwd + "@" + \
broker_addr
ar_ctrl_pub_broker_url = "amqp://" + ar_ctrl_pub_name + ":" + \
ar_ctrl_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % ar_ctrl_pub_broker_url)
self.ar_ctrl_publisher = SimplePublisher(ar_ctrl_pub_broker_url, "YAML")
F1_name = 'F1'
F1_passwd = 'F1'
F1_pub_name = 'F1_PUB'
F1_pub_passwd = 'F1_PUB'
F1_broker_url = "amqp://" + F1_name + ":" + \
F1_passwd + "@" + \
broker_addr
F1_pub_broker_url = "amqp://" + F1_pub_name + ":" + \
F1_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % F1_pub_broker_url)
self.F1_publisher = SimplePublisher(F1_pub_broker_url, "YAML")
F2_name = 'F2'
F2_passwd = 'F2'
F2_pub_name = 'F2_PUB'
F2_pub_passwd = 'F2_PUB'
F2_broker_url = "amqp://" + F2_name + ":" + \
F2_passwd + "@" + \
broker_addr
F2_pub_broker_url = "amqp://" + F2_pub_name + ":" + \
F2_pub_passwd + "@" + \
broker_addr
print("Opening publisher with this URL string: %s" % F2_pub_broker_url)
self.F2_publisher = SimplePublisher(F2_pub_broker_url, "YAML")
print("All publishers are running...")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
print("MessageAuthority running...")
self.dmcs_consumer = Consumer(dmcs_broker_url,'dmcs_ack_consume', 'thread-dmcs',
self.on_dmcs_message,'YAML')
self.dmcs_consumer.start()
print("DMCS Consumer running...")
self.ar_ctrl_consumer = Consumer(ar_ctrl_broker_url,'archive_ctrl_consume', 'thread-ar-ctrl',
self.on_ar_ctrl_message,'YAML')
self.ar_ctrl_consumer.start()
print("ar_ctrl Consumer running...")
self.F1_consumer = Consumer(F1_broker_url,'f1_consume', 'thread-f1',
self.on_f1_message,'YAML')
self.F1_consumer.start()
print("F1 Consumer running...")
self.F2_consumer = Consumer(F2_broker_url,'f2_consume', 'thread-f2',
self.on_f2_message,'YAML')
self.F2_consumer.start()
print("F2 Consumer running...")
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(8)
self.verify_dmcs_messages()
self.verify_ar_ctrl_messages()
self.verify_F1_messages()
self.verify_F2_messages()
sleep(3)
# Shut down consumer threads nicely
self.dmcs_consumer.stop()
self.dmcs_consumer.join()
self.ar_ctrl_consumer.stop()
self.ar_ctrl_consumer.join()
self.F1_consumer.stop()
self.F1_consumer.join()
self.F2_consumer.stop()
self.F2_consumer.join()
if self.DP:
print("Finished with AR tests.")
def test_ncsa(self, Ncsa):
self.ncsa = Ncsa
try:
cdm = toolsmod.intake_yaml_file('./tests/yaml/L1SystemCfg_Test_ncsa.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
pp_name = cdm[ROOT]['PFM_BROKER_NAME']
pp_passwd = cdm[ROOT]['PFM_BROKER_PASSWD']
pp_pub_name = cdm[ROOT]['PFM_BROKER_PUB_NAME']
pp_pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD']
pp_broker_url = "amqp://" + pp_name + ":" + \
pp_passwd + "@" + \
broker_addr
pp_pub_broker_url = "amqp://" + pp_pub_name + ":" + \
pp_pub_passwd + "@" + \
broker_addr
self.pp_publisher = SimplePublisher(pp_pub_broker_url, "YAML")
D1_name = 'D1'
D1_passwd = 'D1'
D1_pub_name = 'D1_PUB'
D1_pub_passwd = 'D1_PUB'
D1_broker_url = "amqp://" + D1_name + ":" + \
D1_passwd + "@" + \
broker_addr
D1_pub_broker_url = "amqp://" + D1_pub_name + ":" + \
D1_pub_passwd + "@" + \
broker_addr
self.d1_publisher = SimplePublisher(D1_pub_broker_url, "YAML")
D2_name = 'D2'
D2_passwd = 'D2'
D2_pub_name = 'D2_PUB'
D2_pub_passwd = 'D2_PUB'
D2_broker_url = "amqp://" + D2_name + ":" + \
D2_passwd + "@" + \
broker_addr
D2_pub_broker_url = "amqp://" + D2_pub_name + ":" + \
D2_pub_passwd + "@" + \
broker_addr
self.d2_publisher = SimplePublisher(D2_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.pp_consumer = Consumer(pp_broker_url,'pp_foreman_ack_publish', 'thread-pp',
self.on_pp_message,'YAML')
self.pp_consumer.start()
self.D1_consumer = Consumer(D1_broker_url,'d1_consume', 'thread-d1',
self.on_d1_message,'YAML')
self.D1_consumer.start()
self.D2_consumer = Consumer(D2_broker_url,'d2_consume', 'thread-d2',
self.on_d2_message,'YAML')
self.D2_consumer.start()
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(7)
self.verify_pp_messages()
self.verify_D2_messages()
self.verify_D1_messages()
sleep(2)
self.pp_consumer.stop()
self.pp_consumer.join()
self.D1_consumer.stop()
self.D1_consumer.join()
self.D2_consumer.stop()
self.D2_consumer.join()
if self.DP:
print("Finished with NCSA tests.")
def test_dmcs(self, Dmcs):
self.dmcs = Dmcs
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
ocs_name = cdm[ROOT]['OCS_BROKER_NAME']
ocs_passwd = cdm[ROOT]['OCS_BROKER_PASSWD']
ocs_pub_name = cdm[ROOT]['OCS_BROKER_PUB_NAME']
ocs_pub_passwd = cdm[ROOT]['OCS_BROKER_PUB_PASSWD']
ocs_broker_url = "amqp://" + ocs_name + ":" + \
ocs_passwd + "@" + \
broker_addr
self. ocs_pub_broker_url = "amqp://" + ocs_pub_name + ":" + \
ocs_pub_passwd + "@" + \
broker_addr
self.ocs_publisher = SimplePublisher(self.ocs_pub_broker_url, "YAML")
ar_name = cdm[ROOT]['AFM_BROKER_NAME']
ar_passwd = cdm[ROOT]['AFM_BROKER_PASSWD']
ar_pub_name = cdm[ROOT]['AFM_BROKER_PUB_NAME']
ar_pub_passwd = cdm[ROOT]['AFM_BROKER_PUB_PASSWD']
ar_broker_url = "amqp://" + ar_name + ":" + \
ar_passwd + "@" + \
broker_addr
self.ar_pub_broker_url = "amqp://" + ar_pub_name + ":" + \
ar_pub_passwd + "@" + \
broker_addr
self.ar_publisher = SimplePublisher(self.ar_pub_broker_url, "YAML")
pp_name = cdm[ROOT]['PFM_BROKER_NAME']
pp_passwd = cdm[ROOT]['PFM_BROKER_PASSWD']
pp_pub_name = cdm[ROOT]['PFM_BROKER_PUB_NAME']
pp_pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD']
pp_broker_url = "amqp://" + pp_name + ":" + \
pp_passwd + "@" + \
broker_addr
self.pp_pub_broker_url = "amqp://" + pp_pub_name + ":" + \
pp_pub_passwd + "@" + \
broker_addr
self.pp_publisher = SimplePublisher(self.pp_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.ocs_consumer = Consumer(ocs_broker_url, 'dmcs_ocs_publish',
'thread-ocs', self.on_ocs_message, 'YAML')
self.ocs_consumer.start()
self.ar_consumer = Consumer(ar_broker_url, 'ar_foreman_consume',
'thread-ar', self.on_ar_message, 'YAML')
self.ar_consumer.start()
self.pp_consumer = Consumer(pp_broker_url, 'pp_foreman_consume',
'thread-pp', self.on_pp_message, 'YAML')
self.pp_consumer.start()
### call message sender and pass in ocs_publisher
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(3)
self.verify_ocs_messages()
sleep(3)
self.verify_ar_messages()
sleep(3)
self.verify_pp_messages()
sleep(2)
self.ocs_consumer.stop()
self.ocs_consumer.join()
self.ar_consumer.stop()
self.ar_consumer.join()
self.pp_consumer.stop()
self.pp_consumer.join()
print("Finished with DMCS AR and PP tests.")
def test_dmcs(self, Dmcs):
self.dmcs = Dmcs
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
ocs_name = cdm[ROOT]['OCS_BROKER_NAME']
ocs_passwd = cdm[ROOT]['OCS_BROKER_PASSWD']
ocs_pub_name = cdm[ROOT]['OCS_BROKER_PUB_NAME']
ocs_pub_passwd = cdm[ROOT]['OCS_BROKER_PUB_PASSWD']
ocs_broker_url = "amqp://" + ocs_name + ":" + \
ocs_passwd + "@" + \
broker_addr
self. ocs_pub_broker_url = "amqp://" + ocs_pub_name + ":" + \
ocs_pub_passwd + "@" + \
broker_addr
self.ocs_publisher = SimplePublisher(self.ocs_pub_broker_url, "YAML")
ar_name = cdm[ROOT]['AFM_BROKER_NAME']
ar_passwd = cdm[ROOT]['AFM_BROKER_PASSWD']
ar_pub_name = cdm[ROOT]['AFM_BROKER_PUB_NAME']
ar_pub_passwd = cdm[ROOT]['AFM_BROKER_PUB_PASSWD']
ar_broker_url = "amqp://" + ar_name + ":" + \
ar_passwd + "@" + \
broker_addr
self.ar_pub_broker_url = "amqp://" + ar_pub_name + ":" + \
ar_pub_passwd + "@" + \
broker_addr
self.ar_publisher = SimplePublisher(self.ar_pub_broker_url, "YAML")
pp_name = cdm[ROOT]['PFM_BROKER_NAME']
pp_passwd = cdm[ROOT]['PFM_BROKER_PASSWD']
pp_pub_name = cdm[ROOT]['PFM_BROKER_PUB_NAME']
pp_pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD']
pp_broker_url = "amqp://" + pp_name + ":" + \
pp_passwd + "@" + \
broker_addr
self.pp_pub_broker_url = "amqp://" + pp_pub_name + ":" + \
pp_pub_passwd + "@" + \
broker_addr
self.pp_publisher = SimplePublisher(self.pp_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.ocs_consumer = Consumer(ocs_broker_url,'dmcs_ocs_publish', 'thread-ocs',
self.on_ocs_message,'YAML')
self.ocs_consumer.start()
self.ar_consumer = Consumer(ar_broker_url,'ar_foreman_consume', 'thread-ar',
self.on_ar_message,'YAML')
self.ar_consumer.start()
self.pp_consumer = Consumer(pp_broker_url,'pp_foreman_consume', 'thread-pp',
self.on_pp_message,'YAML')
self.pp_consumer.start()
### call message sender and pass in ocs_publisher
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(3)
self.verify_ocs_messages()
sleep(3)
self.verify_ar_messages()
sleep(3)
self.verify_pp_messages()
sleep(2)
self.ocs_consumer.stop()
self.ocs_consumer.join()
self.ar_consumer.stop()
self.ar_consumer.join()
self.pp_consumer.stop()
self.pp_consumer.join()
print("Finished with DMCS AR and PP tests.")
def __init__(self, filename=None):
toolsmod.singleton(self)
self._config_file = 'ForemanCfg.yaml'
if filename != None:
self._config_file = filename
cdm = toolsmod.intake_yaml_file(self._config_file)
try:
self._base_name = cdm[ROOT][BASE_BROKER_NAME] # Message broker user & passwd
self._base_passwd = cdm[ROOT][BASE_BROKER_PASSWD]
self._ncsa_name = cdm[ROOT][NCSA_BROKER_NAME]
self._ncsa_passwd = cdm[ROOT][NCSA_BROKER_PASSWD]
self._base_broker_addr = cdm[ROOT][BASE_BROKER_ADDR]
self._ncsa_broker_addr = cdm[ROOT][NCSA_BROKER_ADDR]
forwarder_dict = cdm[ROOT][XFER_COMPONENTS][FORWARDERS]
except KeyError as e:
print("Dictionary error")
print("Bailing out...")
sys.exit(99)
if 'QUEUE_PURGES' in cdm[ROOT]:
self.purge_broker(cdm['ROOT']['QUEUE_PURGES'])
self._base_msg_format = self.YAML
self._ncsa_msg_format = self.YAML
if 'BASE_MSG_FORMAT' in cdm[ROOT]:
self._base_msg_format = cdm[ROOT][BASE_MSG_FORMAT]
if 'NCSA_MSG_FORMAT' in cdm[ROOT]:
self._ncsa_msg_format = cdm[ROOT][NCSA_MSG_FORMAT]
self._base_broker_url = 'amqp_url'
self._ncsa_broker_url = 'amqp_url'
self._next_timed_ack_id = 0
# Create Redis Forwarder table with Forwarder info
self.FWD_SCBD = ForwarderScoreboard(forwarder_dict)
self.JOB_SCBD = JobScoreboard()
self.ACK_SCBD = AckScoreboard()
self._msg_actions = { 'NEW_JOB': self.process_dmcs_new_job,
'READOUT': self.process_dmcs_readout,
'NCSA_RESOURCE_QUERY_ACK': self.process_ack,
'NCSA_STANDBY_ACK': self.process_ack,
'NCSA_READOUT_ACK': self.process_ack,
'FORWARDER_HEALTH_ACK': self.process_ack,
'FORWARDER_JOB_PARAMS_ACK': self.process_ack,
'FORWARDER_READOUT_ACK': self.process_ack,
'NEW_JOB_ACK': self.process_ack }
self._base_broker_url = "amqp://" + self._base_name + ":" + self._base_passwd + "@" + str(self._base_broker_addr)
self._ncsa_broker_url = "amqp://" + self._ncsa_name + ":" + self._ncsa_passwd + "@" + str(self._ncsa_broker_addr)
LOGGER.info('Building _base_broker_url. Result is %s', self._base_broker_url)
LOGGER.info('Building _ncsa_broker_url. Result is %s', self._ncsa_broker_url)
self.setup_publishers()
self.setup_consumers()
def test_dmcs(self, Dmcs):
self.dmcs = Dmcs
#logging.warning("Logging is Working!")
LOGGER.critical("LOGGING is Working!")
#self.LOGGER.info("self Logging is Working!")
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
ocs_name = cdm[ROOT]['OCS_BROKER_NAME']
ocs_passwd = cdm[ROOT]['OCS_BROKER_PASSWD']
ocs_pub_name = cdm[ROOT]['OCS_BROKER_PUB_NAME']
ocs_pub_passwd = cdm[ROOT]['OCS_BROKER_PUB_PASSWD']
ocs_broker_url = "amqp://" + ocs_name + ":" + \
ocs_passwd + "@" + \
broker_addr
self.ocs_pub_broker_url = "amqp://" + ocs_pub_name + ":" + \
ocs_pub_passwd + "@" + \
broker_addr
self.ocs_publisher = SimplePublisher(self.ocs_pub_broker_url, "YAML")
at_name = cdm[ROOT]['AUX_BROKER_NAME']
at_passwd = cdm[ROOT]['AUX_BROKER_PASSWD']
at_pub_name = cdm[ROOT]['AUX_BROKER_PUB_NAME']
at_pub_passwd = cdm[ROOT]['AUX_BROKER_PUB_PASSWD']
at_broker_url = "amqp://" + at_name + ":" + \
at_passwd + "@" + \
broker_addr
self.at_pub_broker_url = "amqp://" + at_pub_name + ":" + \
at_pub_passwd + "@" + \
broker_addr
self.at_publisher = SimplePublisher(self.at_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.ocs_consumer = Consumer(ocs_broker_url,'dmcs_ocs_publish', 'thread-ocs',
self.on_ocs_message,'YAML')
self.ocs_consumer.start()
self.at_consumer = Consumer(at_broker_url,'at_foreman_consume', 'thread-at',
self.on_at_message,'YAML')
self.at_consumer.start()
sleep(3)
if self.DP:
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(3)
self.verify_ocs_messages()
sleep(3)
self.verify_at_messages()
sleep(2)
self.ocs_consumer.stop()
self.ocs_consumer.join()
self.at_consumer.stop()
self.at_consumer.join()
if self.DP:
print("Finished with DMCS AT tests.")
def test_dmcs(self, Dmcs):
self.dmcs = Dmcs
#logging.warning("Logging is Working!")
LOGGER.critical("LOGGING is Working!")
#self.LOGGER.info("self Logging is Working!")
try:
cdm = toolsmod.intake_yaml_file('tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
ocs_name = cdm[ROOT]['OCS_BROKER_NAME']
ocs_passwd = cdm[ROOT]['OCS_BROKER_PASSWD']
ocs_pub_name = cdm[ROOT]['OCS_BROKER_PUB_NAME']
ocs_pub_passwd = cdm[ROOT]['OCS_BROKER_PUB_PASSWD']
ocs_broker_url = "amqp://" + ocs_name + ":" + \
ocs_passwd + "@" + \
broker_addr
self.ocs_pub_broker_url = "amqp://" + ocs_pub_name + ":" + \
ocs_pub_passwd + "@" + \
broker_addr
self.ocs_publisher = SimplePublisher(self.ocs_pub_broker_url, "YAML")
at_name = cdm[ROOT]['AUX_BROKER_NAME']
at_passwd = cdm[ROOT]['AUX_BROKER_PASSWD']
at_pub_name = cdm[ROOT]['AUX_BROKER_PUB_NAME']
at_pub_passwd = cdm[ROOT]['AUX_BROKER_PUB_PASSWD']
at_broker_url = "amqp://" + at_name + ":" + \
at_passwd + "@" + \
broker_addr
self.at_pub_broker_url = "amqp://" + at_pub_name + ":" + \
at_pub_passwd + "@" + \
broker_addr
self.at_publisher = SimplePublisher(self.at_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.ocs_consumer = Consumer(ocs_broker_url, 'dmcs_ocs_publish',
'thread-ocs', self.on_ocs_message, 'YAML')
self.ocs_consumer.start()
self.at_consumer = Consumer(at_broker_url, 'at_foreman_consume',
'thread-at', self.on_at_message, 'YAML')
self.at_consumer.start()
sleep(3)
if self.DP:
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(3)
self.verify_ocs_messages()
sleep(3)
self.verify_at_messages()
sleep(2)
self.ocs_consumer.stop()
self.ocs_consumer.join()
self.at_consumer.stop()
self.at_consumer.join()
if self.DP:
print("Finished with DMCS AT tests.")
def test_ncsa(self, Ncsa):
self.ncsa = Ncsa
try:
cdm = toolsmod.intake_yaml_file(
'./tests/yaml/L1SystemCfg_Test_ncsa.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
pp_name = cdm[ROOT]['PFM_BROKER_NAME']
pp_passwd = cdm[ROOT]['PFM_BROKER_PASSWD']
pp_pub_name = cdm[ROOT]['PFM_BROKER_PUB_NAME']
pp_pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD']
pp_broker_url = "amqp://" + pp_name + ":" + \
pp_passwd + "@" + \
broker_addr
pp_pub_broker_url = "amqp://" + pp_pub_name + ":" + \
pp_pub_passwd + "@" + \
broker_addr
self.pp_publisher = SimplePublisher(pp_pub_broker_url, "YAML")
D1_name = 'D1'
D1_passwd = 'D1'
D1_pub_name = 'D1_PUB'
D1_pub_passwd = 'D1_PUB'
D1_broker_url = "amqp://" + D1_name + ":" + \
D1_passwd + "@" + \
broker_addr
D1_pub_broker_url = "amqp://" + D1_pub_name + ":" + \
D1_pub_passwd + "@" + \
broker_addr
self.d1_publisher = SimplePublisher(D1_pub_broker_url, "YAML")
D2_name = 'D2'
D2_passwd = 'D2'
D2_pub_name = 'D2_PUB'
D2_pub_passwd = 'D2_PUB'
D2_broker_url = "amqp://" + D2_name + ":" + \
D2_passwd + "@" + \
broker_addr
D2_pub_broker_url = "amqp://" + D2_pub_name + ":" + \
D2_pub_passwd + "@" + \
broker_addr
self.d2_publisher = SimplePublisher(D2_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.pp_consumer = Consumer(pp_broker_url, 'pp_foreman_ack_publish',
'thread-pp', self.on_pp_message, 'YAML')
self.pp_consumer.start()
self.D1_consumer = Consumer(D1_broker_url, 'd1_consume', 'thread-d1',
self.on_d1_message, 'YAML')
self.D1_consumer.start()
self.D2_consumer = Consumer(D2_broker_url, 'd2_consume', 'thread-d2',
self.on_d2_message, 'YAML')
self.D2_consumer.start()
sleep(3)
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(7)
self.verify_pp_messages()
self.verify_D2_messages()
self.verify_D1_messages()
sleep(2)
self.pp_consumer.stop()
self.pp_consumer.join()
self.D1_consumer.stop()
self.D1_consumer.join()
self.D2_consumer.stop()
self.D2_consumer.join()
if self.DP:
print("Finished with NCSA tests.")
def test_ppdev(self, Ppdev):
self.ppdev = Ppdev
try:
cdm = toolsmod.intake_yaml_file('./tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
dmcs_name = cdm[ROOT]['DMCS_BROKER_NAME']
dmcs_passwd = cdm[ROOT]['DMCS_BROKER_PASSWD']
dmcs_pub_name = cdm[ROOT]['DMCS_BROKER_PUB_NAME']
dmcs_pub_passwd = cdm[ROOT]['DMCS_BROKER_PUB_PASSWD']
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_passwd + "@" + \
broker_addr
dmcs_pub_broker_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_passwd + "@" + \
broker_addr
self.dmcs_publisher = SimplePublisher(dmcs_pub_broker_url, "YAML")
ncsa_name = cdm[ROOT]['NCSA_BROKER_NAME']
ncsa_passwd = cdm[ROOT]['NCSA_BROKER_PASSWD']
ncsa_pub_name = cdm[ROOT]['NCSA_BROKER_PUB_NAME']
ncsa_pub_passwd = cdm[ROOT]['NCSA_BROKER_PUB_PASSWD']
ncsa_broker_url = "amqp://" + ncsa_name + ":" + \
ncsa_passwd + "@" + \
broker_addr
ncsa_pub_broker_url = "amqp://" + ncsa_pub_name + ":" + \
ncsa_pub_passwd + "@" + \
broker_addr
self.ncsa_publisher = SimplePublisher(ncsa_pub_broker_url, "YAML")
F1_name = 'F1'
F1_passwd = 'F1'
F1_pub_name = 'F1_PUB'
F1_pub_passwd = 'F1_PUB'
F1_broker_url = "amqp://" + F1_name + ":" + \
F1_passwd + "@" + \
broker_addr
F1_pub_broker_url = "amqp://" + F1_pub_name + ":" + \
F1_pub_passwd + "@" + \
broker_addr
self.f1_publisher = SimplePublisher(F1_pub_broker_url, "YAML")
F2_name = 'F2'
F2_passwd = 'F2'
F2_pub_name = 'F2_PUB'
F2_pub_passwd = 'F2_PUB'
F2_broker_url = "amqp://" + F2_name + ":" + \
F2_passwd + "@" + \
broker_addr
F2_pub_broker_url = "amqp://" + F2_pub_name + ":" + \
F2_pub_passwd + "@" + \
broker_addr
self.f2_publisher = SimplePublisher(F2_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.dmcs_consumer = Consumer(dmcs_broker_url,'dmcs_ack_consume', 'thread-dmcs',
self.on_dmcs_message,'YAML')
self.dmcs_consumer.start()
self.ncsa_consumer = Consumer(ncsa_broker_url,'ncsa_consume', 'thread-ncsa',
self.on_ncsa_message,'YAML')
self.ncsa_consumer.start()
self.F1_consumer = Consumer(F1_broker_url,'f1_consume', 'thread-f1',
self.on_f1_message,'YAML')
self.F1_consumer.start()
self.F2_consumer = Consumer(F2_broker_url,'f2_consume', 'thread-f2',
self.on_f2_message,'YAML')
self.F2_consumer.start()
sleep(3)
if self.DP:
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(6)
self.verify_ncsa_messages()
self.verify_F2_messages()
self.verify_F1_messages()
self.verify_dmcs_messages()
sleep(2)
self.dmcs_consumer.stop()
self.dmcs_consumer.join()
self.ncsa_consumer.stop()
self.ncsa_consumer.join()
self.F1_consumer.stop()
self.F1_consumer.join()
self.F2_consumer.stop()
self.F2_consumer.join()
if self.DP:
print("Finished with PP tests.")
def test_ppdev(self, Ppdev):
self.ppdev = Ppdev
try:
cdm = toolsmod.intake_yaml_file(
'./tests/yaml/L1SystemCfg_Test.yaml')
except IOError as e:
trace = traceback.print_exc()
emsg = "Unable to find CFG Yaml file %s\n" % self._config_file
print(emsg + trace)
sys.exit(101)
broker_addr = cdm[ROOT]['BASE_BROKER_ADDR']
dmcs_name = cdm[ROOT]['DMCS_BROKER_NAME']
dmcs_passwd = cdm[ROOT]['DMCS_BROKER_PASSWD']
dmcs_pub_name = cdm[ROOT]['DMCS_BROKER_PUB_NAME']
dmcs_pub_passwd = cdm[ROOT]['DMCS_BROKER_PUB_PASSWD']
dmcs_broker_url = "amqp://" + dmcs_name + ":" + \
dmcs_passwd + "@" + \
broker_addr
dmcs_pub_broker_url = "amqp://" + dmcs_pub_name + ":" + \
dmcs_pub_passwd + "@" + \
broker_addr
self.dmcs_publisher = SimplePublisher(dmcs_pub_broker_url, "YAML")
ncsa_name = cdm[ROOT]['NCSA_BROKER_NAME']
ncsa_passwd = cdm[ROOT]['NCSA_BROKER_PASSWD']
ncsa_pub_name = cdm[ROOT]['NCSA_BROKER_PUB_NAME']
ncsa_pub_passwd = cdm[ROOT]['NCSA_BROKER_PUB_PASSWD']
ncsa_broker_url = "amqp://" + ncsa_name + ":" + \
ncsa_passwd + "@" + \
broker_addr
ncsa_pub_broker_url = "amqp://" + ncsa_pub_name + ":" + \
ncsa_pub_passwd + "@" + \
broker_addr
self.ncsa_publisher = SimplePublisher(ncsa_pub_broker_url, "YAML")
F1_name = 'F1'
F1_passwd = 'F1'
F1_pub_name = 'F1_PUB'
F1_pub_passwd = 'F1_PUB'
F1_broker_url = "amqp://" + F1_name + ":" + \
F1_passwd + "@" + \
broker_addr
F1_pub_broker_url = "amqp://" + F1_pub_name + ":" + \
F1_pub_passwd + "@" + \
broker_addr
self.f1_publisher = SimplePublisher(F1_pub_broker_url, "YAML")
F2_name = 'F2'
F2_passwd = 'F2'
F2_pub_name = 'F2_PUB'
F2_pub_passwd = 'F2_PUB'
F2_broker_url = "amqp://" + F2_name + ":" + \
F2_passwd + "@" + \
broker_addr
F2_pub_broker_url = "amqp://" + F2_pub_name + ":" + \
F2_pub_passwd + "@" + \
broker_addr
self.f2_publisher = SimplePublisher(F2_pub_broker_url, "YAML")
# Must be done before consumer threads are started
# This is used for verifying message structure
self._msg_auth = MessageAuthority()
self.dmcs_consumer = Consumer(dmcs_broker_url, 'dmcs_ack_consume',
'thread-dmcs', self.on_dmcs_message,
'YAML')
self.dmcs_consumer.start()
self.ncsa_consumer = Consumer(ncsa_broker_url, 'ncsa_consume',
'thread-ncsa', self.on_ncsa_message,
'YAML')
self.ncsa_consumer.start()
self.F1_consumer = Consumer(F1_broker_url, 'f1_consume', 'thread-f1',
self.on_f1_message, 'YAML')
self.F1_consumer.start()
self.F2_consumer = Consumer(F2_broker_url, 'f2_consume', 'thread-f2',
self.on_f2_message, 'YAML')
self.F2_consumer.start()
sleep(3)
if self.DP:
print("Test Setup Complete. Commencing Messages...")
self.send_messages()
sleep(6)
self.verify_ncsa_messages()
self.verify_F2_messages()
self.verify_F1_messages()
self.verify_dmcs_messages()
sleep(2)
self.dmcs_consumer.stop()
self.dmcs_consumer.join()
self.ncsa_consumer.stop()
self.ncsa_consumer.join()
self.F1_consumer.stop()
self.F1_consumer.join()
self.F2_consumer.stop()
self.F2_consumer.join()
if self.DP:
print("Finished with PP tests.")
