def home(): print("----------------Welcome to Consumer APP----------------") print("You can select one of these options:") while (True): print("1. Login\n2. Sign in\n3. Exit") print("-------------------------------------------------------") option = input() if option == '1': email = input("Please insert your e-mail: ") password = input("Please insert your password: "******"2": email = input("Please insert your e-mail: ") password = input("Please insert your password: "******"Please insert your name: ") consumer = Consumer(email, password, name) consumer.sign_in() elif option == "3": print("Kind Regards!") break else: print("Please insert a valid option.")
def __init__(self): logging.basicConfig() #os.system('rabbitmqctl -p /tester purge_queue firehose') #os.system('rabbitmqctl -p /tester purge_queue ack_publish') self.sp2 = SimplePublisher( 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny', "YAML") time.sleep(3) broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny' #broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Ftester' #broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny' #self._cons = FirehoseConsumer(broker_url, 'firehose', "YAML") self._cons = Consumer(broker_url, 'f1_consume', "YAML") #self._cons = Consumer(broker_url, 'pp_foreman_consume', "YAML") self._cons2 = Consumer(broker_url, 'ncsa_consume', "YAML") try: _thread.start_new_thread(self.do_it, ( "thread-1", 2, )) except e: print("Cannot start thread") print(e) try: _thread.start_new_thread(self.do_it2, ( "thread-2", 2, )) except e: print("Cannot start thread") print(e)
def __init__(self): logging.basicConfig() #os.system('rabbitmqctl -p /tester purge_queue firehose') #os.system('rabbitmqctl -p /tester purge_queue ack_publish') #broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny?heartbeat=300' broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny' #pub_broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny' #broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Ftester' #broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny' #self._cons = FirehoseConsumer(broker_url, 'firehose', "YAML") self._cons = Consumer(broker_url, 'ar_foreman_consume', "YAML") try: _thread.start_new_thread( self.do_it, ("thread-1", 2,) ) except e: print("Cannot start thread") print(e) self._cons2 = Consumer(broker_url, 'pp_foreman_consume', "YAML") try: _thread.start_new_thread( self.do_it2, ("thread-3", 2,) ) except e: print("Cannot start thread") print(e) cdm = toolsmod.intake_yaml_file("L1SystemCfg.yaml") self.ccd_list = cdm['ROOT']['CCD_LIST']
def test_ocs_acksubscriber(self): try: cdm = toolsmod.intake_yaml_file( "../../tests/yaml/L1SystemCfg_Test_ocs_bridge.yaml") except IOError as e: trace = traceback.print_exc() emsg = "Unable to fine CFG Yaml file %s\n" % self._config_file print(emsg + trace) sys.exit(101) broker_addr = cdm[ROOT]["BASE_BROKER_ADDR"] # dmcs publisher dmcs_pub_name = cdm[ROOT]["DMCS_BROKER_PUB_NAME"] dmcs_pub_pwd = cdm[ROOT]["DMCS_BROKER_PUB_PASSWD"] dmcs_broker_pub_url = "amqp://" + dmcs_pub_name + ":" + \ dmcs_pub_pwd + "@" + \ broker_addr self.dmcs_publisher = SimplePublisher(dmcs_broker_pub_url, "YAML") # dmcs consumer dmcs_name = cdm[ROOT]["DMCS_BROKER_NAME"] dmcs_pwd = cdm[ROOT]["DMCS_BROKER_PASSWD"] dmcs_broker_url = "amqp://" + dmcs_name + ":" + \ dmcs_pwd + "@" + \ broker_addr self.dmcs_consumer = Consumer(dmcs_broker_url, "ocs_dmcs_consume", "thread-dmcs-consume", self.on_ocs_message, "YAML") self.dmcs_consumer.start() # ocs consumer from DMCS ocs_name = cdm[ROOT]["OCS_BROKER_NAME"] ocs_pwd = cdm[ROOT]["OCS_BROKER_PASSWD"] # FIXME: New OCS account for consumer test_dmcs_ocs_publish ocs_broker_url = "amqp://" + "AFM" + ":" +\ "AFM" + "@" +\ broker_addr self.ocs_consumer = Consumer(ocs_broker_url, "test_dmcs_ocs_publish", "thread-ocs-consume", self.on_dmcs_message, "YAML") self.ocs_consumer.start() print("Test setup Complete. Commencing Messages...") self._msg_auth = MessageAuthority("../../messages.yaml") self.send_messages() sleep(10) os.killpg(os.getpgid(self.cmdListener.pid), signal.SIGTERM) os.killpg(os.getpgid(self.ackSubscriber.pid), signal.SIGTERM) print("MY OCS MESSAGES: %s" % self.ocs_consumer_msg_list) self.verify_ocs_messages() print("Finished with CommandListener tests.")
def setup_consumers(self): """This method sets up a message listener from each entity with which the BaseForeman has contact here. These listeners are instanced in this class, but their run methods are each called as a separate thread. While pika does not claim to be thread safe, the manner in which the listeners are invoked below is a safe implementation that provides non-blocking, fully asynchronous messaging to the BaseForeman. The code in this file expects message bodies to arrive as YAML'd python dicts, while in fact, message bodies are sent on the wire as XML; this way message format can be validated, versioned, and specified in just one place. To make this work, there is an object that translates the params dict to XML, and visa versa. The translation object is instantiated by the consumer and acts as a filter before sending messages on to the registered callback for processing. """ LOGGER.info('Setting up consumers on %s', self._base_broker_url) LOGGER.info('Running start_new_thread on all consumer methods') self._dmcs_consumer = Consumer(self._base_broker_url, self.DMCS_PUBLISH, self._base_msg_format) try: _thread.start_new_thread( self.run_dmcs_consumer, ("thread-dmcs-consumer", 2,) ) except: LOGGER.critical('Cannot start DMCS consumer thread, exiting...') sys.exit(99) self._forwarder_consumer = Consumer(self._base_broker_url, self.FORWARDER_PUBLISH, self._base_msg_format) try: _thread.start_new_thread( self.run_forwarder_consumer, ("thread-forwarder-consumer", 2,) ) except: LOGGER.critical('Cannot start FORWARDERS consumer thread, exiting...') sys.exit(100) self._ncsa_consumer = Consumer(self._base_broker_url, self.NCSA_PUBLISH, self._base_msg_format) try: _thread.start_new_thread( self.run_ncsa_consumer, ("thread-ncsa-consumer", 2,) ) except: LOGGER.critical('Cannot start NCSA consumer thread, exiting...') sys.exit(101) self._ack_consumer = Consumer(self._base_broker_url, self.ACK_PUBLISH, self._base_msg_format) try: _thread.start_new_thread( self.run_ack_consumer, ("thread-ack-consumer", 2,) ) except: LOGGER.critical('Cannot start ACK consumer thread, exiting...') sys.exit(102) LOGGER.info('Finished starting all three consumer threads')
def main(): # handle user input from_date_string = str( raw_input("Please specify a Start Date with format ddmmYYYY: ")) # raw input in python2 is the same as input in python3 (returns a string instead of a python expression) to_date_string = str( raw_input("Please specify a End Date with format ddmmYYYY: ")) locale.setlocale(locale.LC_ALL, '') from_date = datetime.datetime.strptime(from_date_string, '%d%m%Y').isoformat() to_date = datetime.datetime.strptime(to_date_string, '%d%m%Y') tmp_date = to_date + datetime.timedelta(1) to_date = tmp_date.isoformat() threads = list() stopper = threading.Event() consumer = Consumer() producer = Producer(from_date=from_date, to_date=to_date) threads.append(consumer) threads.append(producer) handler = SignalHandler(stopper, threads) signal.signal(signal.SIGINT, handler) producer.start() consumer.start()
def test_ocs_commandlistener(self): try: cdm = toolsmod.intake_yaml_file( "/home/hwin16/src/git/ctrl_iip/python/lsst/iip/tests/yaml/L1SystemCfg_Test_ocs_bridge.yaml" ) except IOError as e: trace = traceback.print_exc() emsg = "Unable to fine CFG Yaml file %s\n" % self._config_file print(emsg + trace) sys.exit(101) broker_addr = cdm[ROOT]["BASE_BROKER_ADDR"] dmcs_name = cdm[ROOT]["DMCS_BROKER_NAME"] dmcs_pwd = cdm[ROOT]["DMCS_BROKER_PASSWD"] dmcs_broker_url = "amqp://" + dmcs_name + ":" + \ dmcs_pwd + "@" + \ broker_addr self.dmcs_consumer = Consumer(dmcs_broker_url, "ocs_dmcs_consume", "thread-dmcs-consume", self.on_ocs_message, "YAML") self.dmcs_consumer.start() print("Test setup Complete. Commencing Messages...") self._msg_auth = MessageAuthority( "/home/hwin16/src/git/ctrl_iip/python/lsst/iip/messages.yaml") self.send_messages() sleep(10) os.killpg(os.getpgid(self.cmd.pid), signal.SIGTERM) self.verify_ocs_messages() print("Finished with CommandListener tests.")
def __init__(self): logging.basicConfig() self.ack_test = True #os.system('rabbitmqctl -p /tester purge_queue firehose') #os.system('rabbitmqctl -p /tester purge_queue ack_publish') broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny' #broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Ftester' #broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny' #self._cons = FirehoseConsumer(broker_url, 'firehose', "YAML") self.sp1 = SimplePublisher( 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny') cdm = toolsmod.intake_yaml_file('ForemanCfg.yaml') self.fdict = cdm[ROOT]['XFER_COMPONENTS']['ARCHIVE_FORWARDERS'] self.fwdrs = list(self.fdict.keys()) self._cons = Consumer(broker_url, 'ar_foreman_ack_publish', "YAML") #self._cons = Consumer(broker_url, 'ar_foreman_consume', "YAML") try: _thread.start_new_thread(self.do_it, ( "thread-1", 2, )) except e: print("Cannot start thread") print(e)
def __init__(self): logging.basicConfig() broker_url = 'amqp://*****:*****@141.142.238.10:5672/%2Fbunny' self._cons = Consumer(broker_url, 'fetch_consume_from_f1', "Threadthread", self.mycallback, "YAML") Readout_Image_IDs = [] Header_Info_Dict = {} Take_Images_Done_Message = {}
def __init__(self, securities): self.consumer_group = [] for security in securities: self.consumer_group.append(Consumer(security, self)) self.producer = Producer(self.consumer_group, self) self.flag = AtomicNumber() self.i = 0
def __start_race(self): self.buffer = __buffer_sensor__() self.producer_race = Producer(self.buffer, self.sensor, self.race, 'race') self.consumer_race = Consumer(self.buffer, self.race, self.publisher, 'race') self.producer_race.start() self.consumer_race.start() return ''
def __start_qualification(self): self.buffer = __buffer_sensor__() self.producer_qualification = Producer(self.buffer, self.sensor, self.race, 'qualification') self.consumer_qualification = Consumer(self.buffer, self.race, self.publisher, 'qualification') self.producer_qualification.start() self.consumer_qualification.start() return ''
def createConsumer(self, connect, stats, routingKey): channel = connect.channel() if self.consumerTxSize > 0: channel.tx_select() self.configureQueue(connect, routingKey) return Consumer(channel, routingKey, self.queueName, self.consumerRateLimit, self.consumerTxSize, self.autoAck, self.multiAckEvery, stats, self.consumerMsgCount, self.timeLimit)
def Welfare(self, output_list): # Initiate an instance of Consumer C = Consumer(self.c, self.g, self.f, self.A, self.B, self.T) # Generate the utility list of the consumer instance utility_list = C.Utility(output_list) ''' An utilitarian welfare function ''' welfare = -1 * np.sum(utility_list) return welfare
def run(self): consumer = Consumer() # consumer.setPort(9092) # consumer.setIP("10.0.0.63") consumer.startConnection() self.points.consumerTime = time.time() while True: # self.semaphore.acquire() consumer.getMessage(self.points)
def setup_consumer_thread(self, consumer_params): url = consumer_params['amqp_url'] q = consumer_params['queue'] threadname = consumer_params['name'] callback = consumer_params['callback'] format = consumer_params['format'] new_thread = Consumer(url, q, threadname, callback, format) new_thread.start() sleep(1) return new_thread
def __init__(self): logging.basicConfig() broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny?autoAck=true' self._cons = Consumer(broker_url, 'ocs_dmcs_consume', "YAML") try: _thread.start_new_thread(self.do_it, ( "thread-1", 2, )) except e: print("Cannot start thread") print(e)
def start(self): if not os.path.exists('resources/'): os.mkdir('resources/') if not os.path.exists('jobs/'): os.mkdir('jobs/') self.job_queue = JoinableQueue() self.consumers = [ Consumer(self.job_queue, self.manifest) for _ in range(self.num_processes) ] for consumer in self.consumers: consumer.start()
def start_consumer(self, broker_url, format): self.influx_consumer = Consumer(self.broker_url, "audit_consume", format) try: _thread.start_new_thread(self.run_influx_consumer, ( "thread-influx-consumer", 2, )) except: LOGGER.critical('Cannot start influx consumer thread, exiting...') sys.exit(99)
def run(self): roomName = input('Please enter the name of the room: \n') clientName = input('Please enter the name of the client: \n') privateKey = bytearray(123) res = self.enterRoom(roomName, clientName, privateKey) if (res): print('Entered') self.consumer = Consumer(clientName, self.channel, self.queue_name) self.consumer.start() self.producer = Producer(clientName, self.channel, roomName, privateKey) self.producer.start()
def setup_consumers(self): threadname = "thread-" + self._consume_queue print("Threadname is %s" % threadname) self._consumer = Consumer(self._base_broker_url, self._consume_queue) try: _thread.start_new_thread(self.run_consumer, ( threadname, 2, )) print("Started Consumer Thread") except: sys.exit(99)
def index_files(self): thread_add_to_dic = threading.Thread(target=self.add_to_dicts2()) threads = [] for name in self.name_of_files: thread = Consumer(self.name_of_files[name], name, self.file_type, self.main_dictionary, 1, self.temp_position_dic) threads.append(thread) thread.start() thread_add_to_dic.start() thread_add_to_dic.join() for i in range(0, len(threads)): threads[i].stop_thread() for i in range(0, len(threads)): threads[i].join()
def setup_consumers(self): LOGGER.info('Distributor %s setting up consumer on %s', self._name, self._ncsa_broker_url) LOGGER.info('Starting new thread on consumer method') threadname = "thread-" + self._consume_queue self._consumer = Consumer(self._ncsa_broker_url, self._consume_queue) try: _thread.start_new_thread(self.run_consumer, ( threadname, 2, )) LOGGER.info('Started distributor consumer thread %s', threadname) except: LOGGER.critical( 'Cannot start Distributor consumer thread, exiting...') sys.exit(107)
def loadConsumers(): global consumers, orderedConsumers, factoryEvents consumers = np.empty(shape=[0], dtype=Consumer) consumersFile = open('res\\data\\consumers.txt') consumersData = consumersFile.read().split('\n') for i in range(0, len(consumersData)): consumerDataStr = consumersData[i].split() consumerStartTime = int(consumerDataStr[0]) consumerRequirement = np.zeros(NUM_OF_PRODUCT) for j in range(1, len(consumerDataStr)): consumerRequirement[j - 1] = int(consumerDataStr[j]) consumers = np.append(consumers, Consumer(consumerStartTime, consumerRequirement)) print(consumers[i]) addNewEvent(consumerStartTime, 0, i) consumersFile.close() orderedConsumers = np.empty(shape=[0], dtype=int)
def __init__(self): logging.basicConfig() #os.system('rabbitmqctl -p /tester purge_queue firehose') #os.system('rabbitmqctl -p /tester purge_queue ack_publish') #broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny' #broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny?heartbeat=300' broker_url = 'amqp://*****:*****@141.142.238.160:5672/%2Fbunny?autoAck=true' #broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Ftester' #broker_url = 'amqp://*****:*****@141.142.208.191:5672/%2Fbunny' #self._cons = FirehoseConsumer(broker_url, 'firehose', "YAML") self._cons = Consumer(broker_url, 'ocs_dmcs_consume', "YAML") try: _thread.start_new_thread( self.do_it, ("thread-1", 2,) ) except e: print("Cannot start thread") print(e) time.sleep(420)
def Welfare(self,output_list): # Initiate an instance of Consumer C = Consumer(self.c,self.g,self.f,self.A,self.B,self.T) # Generate the utility list of the consumer instance utility_list = C.Utility(output_list) # Generate the parameters for the welfare function # Obtain overall weight for the welfare function overall_weight = float(1) # Obtain individual utility weights for each consumer for the welfare function utility_weights = Loop(self.c) # Calculate the welfare # The welfare function is defined as the (sum of U^(2x)) ^ y, where x are individual weights and y is the overall weight utility_list_square = np.power(np.e,utility_list) welfare = (np.sum(np.power(utility_list_square , utility_weights))) # Obtain the negative of the welfare for minimization neg_welfare = welfare * -1 return neg_welfare
def test_ar_ctrl(self): try: cdm = toolsmod.intake_yaml_file( '/home/centos/src/git/ctrl_iip/python/lsst/iip/tests/yaml/L1SystemCfg_Test_ar-ctrl.yaml' ) except IOError as e: trace = traceback.print_exc() emsg = "Unable to find CFG Yaml file %s\n" % self._config_file print(emsg + trace) sys.exit(101) broker_addr = cdm[ROOT]['BASE_BROKER_ADDR'] afm_name = cdm[ROOT]["AFM_BROKER_NAME"] afm_pwd = cdm[ROOT]["AFM_BROKER_PASSWD"] afm_pub_name = cdm[ROOT]["AFM_BROKER_PUB_NAME"] afm_pub_pwd = cdm[ROOT]["AFM_BROKER_PUB_PASSWD"] self.xfer_root = cdm[ROOT]["ARCHIVE"]["ARCHIVE_XFER_ROOT"] ar_dev_pub_broker_url = "amqp://" + afm_pub_name + ":" +\ afm_pub_pwd + "@" +\ broker_addr self.ar_dev_publisher = SimplePublisher(ar_dev_pub_broker_url, "YAML") ar_dev_broker_url = "amqp://" + afm_name + ":" +\ afm_pwd + "@" +\ broker_addr self.ar_dev_consumer = Consumer(ar_dev_broker_url, self.ACK_FROM_CTRL, "thread-ar-dev", self.on_ar_message, "YAML", None) self.ar_dev_consumer.start() print("Test setup Complete. Commencing Messages...") self._msg_auth = MessageAuthority( '/home/centos/src/git/ctrl_iip/python/lsst/iip/messages.yaml') self.send_messages() sleep(10) self.verify_ar_messages() self.verify_path_files() print("Finished with AR CTRL tests.")
def __init__(self, module, section, cfgfile): self.module = module self.topic = section self.cfgfile = cfgfile self.idx_file = None self.dump_dir = None self.delimiter = None self.start_delimiter = "START" self.end_delimiter = "END" self.consumer_cfg = { "bootstrap.servers": "localhost:9092", "group.id": "group", "session.timeout.ms": 6000, "default.topic.config": { "auto.commit.enable": True, "auto.offset.reset": "smallest", "auto.commit.interval.ms": 1000 } } self.get_parser(module, cfgfile) self.set_config() self.set_logger() self.Consumer = Consumer(self.topic, self.consumer_cfg)
def test_ncsa(self, Ncsa): self.ncsa = Ncsa try: cdm = toolsmod.intake_yaml_file( './tests/yaml/L1SystemCfg_Test_ncsa.yaml') except IOError as e: trace = traceback.print_exc() emsg = "Unable to find CFG Yaml file %s\n" % self._config_file print(emsg + trace) sys.exit(101) broker_addr = cdm[ROOT]['BASE_BROKER_ADDR'] pp_name = cdm[ROOT]['PFM_BROKER_NAME'] pp_passwd = cdm[ROOT]['PFM_BROKER_PASSWD'] pp_pub_name = cdm[ROOT]['PFM_BROKER_PUB_NAME'] pp_pub_passwd = cdm[ROOT]['PFM_BROKER_PUB_PASSWD'] pp_broker_url = "amqp://" + pp_name + ":" + \ pp_passwd + "@" + \ broker_addr pp_pub_broker_url = "amqp://" + pp_pub_name + ":" + \ pp_pub_passwd + "@" + \ broker_addr self.pp_publisher = SimplePublisher(pp_pub_broker_url, "YAML") D1_name = 'D1' D1_passwd = 'D1' D1_pub_name = 'D1_PUB' D1_pub_passwd = 'D1_PUB' D1_broker_url = "amqp://" + D1_name + ":" + \ D1_passwd + "@" + \ broker_addr D1_pub_broker_url = "amqp://" + D1_pub_name + ":" + \ D1_pub_passwd + "@" + \ broker_addr self.d1_publisher = SimplePublisher(D1_pub_broker_url, "YAML") D2_name = 'D2' D2_passwd = 'D2' D2_pub_name = 'D2_PUB' D2_pub_passwd = 'D2_PUB' D2_broker_url = "amqp://" + D2_name + ":" + \ D2_passwd + "@" + \ broker_addr D2_pub_broker_url = "amqp://" + D2_pub_name + ":" + \ D2_pub_passwd + "@" + \ broker_addr self.d2_publisher = SimplePublisher(D2_pub_broker_url, "YAML") # Must be done before consumer threads are started # This is used for verifying message structure self._msg_auth = MessageAuthority() self.pp_consumer = Consumer(pp_broker_url, 'pp_foreman_ack_publish', 'thread-pp', self.on_pp_message, 'YAML') self.pp_consumer.start() self.D1_consumer = Consumer(D1_broker_url, 'd1_consume', 'thread-d1', self.on_d1_message, 'YAML') self.D1_consumer.start() self.D2_consumer = Consumer(D2_broker_url, 'd2_consume', 'thread-d2', self.on_d2_message, 'YAML') self.D2_consumer.start() sleep(3) print("Test Setup Complete. Commencing Messages...") self.send_messages() sleep(7) self.verify_pp_messages() self.verify_D2_messages() self.verify_D1_messages() sleep(2) self.pp_consumer.stop() self.pp_consumer.join() self.D1_consumer.stop() self.D1_consumer.join() self.D2_consumer.stop() self.D2_consumer.join() if self.DP: print("Finished with NCSA tests.")
import copy import sys sys.path.append("../ctrl_iip/python/lsst/iip/") from SimplePublisher import SimplePublisher from Consumer import Consumer pub = SimplePublisher( "amqp://*****:*****@140.252.32.128:5672/%2ftest_at") def on_message(ch, method, properties, body): ch.basic_ack(method.delivery_tag) msg = copy.deepcopy(body) msg_type = body["MSG_TYPE"] + "_ACK" msg["MSG_TYPE"] = msg_type msg["ACK_BOOL"] = 1 msg["ACK_STATEMENT"] = "everything went well" pub.publish_message("dmcs_ocs_publish", msg) sub = Consumer("amqp://*****:*****@140.252.32.128:5672/%2ftest_at", "ocs_dmcs_consume", "hello", on_message, "YAML") sub.run()