def main(): rospy.init_node('iot_consumer', anonymous=True, disable_signals=True) params = wrap_namespace(rospy.get_param('~')) control_pub = rospy.Publisher(params.topic.control_cmd, Control, queue_size=10) mission_pub = rospy.Publisher(params.topic.mission_cmd, Mission, queue_size=10) cmdexe_pub = rospy.Publisher(params.topic.navi_cmdexe, String, queue_size=10) naviCommand = NaviCommand(control_pub, mission_pub, cmdexe_pub) dispatcher = Dispatcher(naviCommand) consumer = Consumer(dispatcher.dispatch_cb) def handler(signum, frame): rospy.loginfo('shutting down...') consumer.shutdown() signal.signal(signal.SIGINT, handler) Container(consumer).run() rospy.signal_shutdown('finish')
def run(self): self.logger.info("Running Kafka Consumer") consumer_thread = Consumer(self.kafka_broker, self.topic) consumer_thread.register_kafka_listener()
def publish_and_get_response(self, message, payload, timeout_secs=5): sent_correlation = str(uuid.uuid1()) consumer_ready = Event() def on_consumer_ready(): consumer_ready.set() consumer = Consumer(self.broker_url, self._queue_prefix, self.exchange) consumer.on_connection_setup_finished = on_consumer_ready response = {} response_received = Event() def response_callback(response_payload, **kwargs): if not sent_correlation == kwargs['properties'].correlation_id: return response['payload'] = response_payload response_received.set() def wait_for_response(): consumer.subscribe(message + '.answered', response_callback, transient_queue=True) consumer.start() thread = Thread(target=wait_for_response) thread.daemon = True thread.start() consumer_ready.wait(2) self._publish(message, payload, correlation_id=sent_correlation) timed_out = not response_received.wait(timeout_secs) if timed_out: raise MessageBusTimeoutError() consumer.stop() return response.get('payload')
def __init__(self, interval, processes, topic, partition, **consumer): self._logger = logging.getLogger('SHIELD.SIMPLE.WORKER') self._logger.info('Initializing Simple Worker process...') self._interval = interval self._isalive = True self._processes = processes # .............................init Kafka Consumer self.Consumer = Consumer(**consumer) self.Consumer.assign(topic, [int(partition)]) # .............................set up local staging area self._tmpdir = tempfile.mkdtemp(prefix='_SW.', dir=tempfile.gettempdir()) self._logger.info('Use directory "{0}" as local staging area.'.format( self._tmpdir)) # .............................define a process pool object self._pool = Pool(self._processes, init_child) self._logger.info( 'Master Collector will use {0} parallel processes.'.format( self._processes)) signal.signal(signal.SIGUSR1, self.kill) self._logger.info('Initialization completed successfully!')
def event_loop(): "this is the main event loop where everything happens" # this isn't being called during sys.exit :/ atexit.register(config.STOP_EVENT.set) tailer_threads = [] # initiate threads to tail from files for fdict in config.FILES: for tailer in path_tailers(fdict): tailer.start() tailer_threads.append(tailer) # initiate threads to consume logs pushed into queue consumer_threads = [] for i in range(config.NUM_CONSUMERS): consumer = Consumer(config.LOG_QUEUE, config.STOP_EVENT, poll_interval=config.POLL_INTERVAL, name=CONSUMER_MAP.get(i % 4)) consumer.start() consumer_threads.append(consumer) # this part continues to block even though all # queue items were processed :/ # LOG_QUEUE.join() # Commenting for now... # logging.debug('finished processing queue') while True: try: time.sleep(10) except KeyboardInterrupt: config.STOP_EVENT.set() print for consumer in consumer_threads: logging.info( '{0.name} sent {0.sent_records} records!'.format(consumer)) sys.exit('shutting down streamer...')
def main(): config = {} with open('./config.txt', 'r') as file: for line in file: line = line.rstrip() key, val = line.split('=') config[key] = val file.close() captchaList = [] condition = threading.Condition() producer = Producer(sitekey=config['sitekey'], twocaptchaKey=config['twocaptchaKey'], condition=condition, producerNo=int(config['producerThread']), captchaList=captchaList) producer.start() for i in range(int(config['consumerThread'])): consumer = Consumer(url=config['url'], productNo=config['productNo'], raffleNo=config['raffleNo'], areCode=config['phoneAreaCode'], domain=config['catchAllDomain'], prefix=config['catchAllPrefix'], condition=condition, captchaList=captchaList) consumer.start()
def main(): # 创建 socket 对象 server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # 获取本地主机名 host = socket.gethostname() port = 9999 # 绑定端口号 server_socket.bind((host, port)) # 设置最大连接数,超过后排队 server_socket.listen(5) # 线程池队列 work_queue = queue.Queue() # 消息队列 message_queue = queue.Queue() # 创建一个有4个线程的线程池 thread_pool = ThreadPoolManger(4, work_queue) # 启动生产者进程 p = Producer() p.start() # 启动消费者进程 c = Consumer() c.start() while True: # 建立客户端连接 client_socket, addr = server_socket.accept() t = threading.Thread(target=tcp_link, args=(client_socket, addr, thread_pool, message_queue)) t.start()
def __init__(self): super().__init__() self.ui = Ui_MainWindow() self.ui.setupUi(self) # буфер для отправленных, но еще не обработанных сообщений self.buffer = queue.Queue() # потоки для обработки информации self.sentThread = QThread() self.sentObj = Producer(self.sem, self.buffer) self.sentObj.moveToThread(self.sentThread) self.n = 1 self.getThreadsPool = [QThread()] self.getObjs = [Consumer(self.sem, self.sem_forBuffer, 1, self.buffer)] self.getObjs[0].moveToThread(self.getThreadsPool[0]) self.ui.sendBt.clicked.connect(self.sentObj.run) self.ui.sendBt.clicked.connect(self.check) self.sentObj.message_sented.connect(self.getObjs[0].run) self.sentObj.message_sented.connect(self.addSendedMessage) self.getObjs[0].message_getted.connect(self.addGettedMessage) self.ui.okBt.clicked.connect(self.change_threadNumber) self.sem_forBuffer.release() self.sentThread.start() self.getThreadsPool[0].start()
def __init__(self): self.url_counter = 1 self.document_client = DocumentClient() self.indexing_client = IndexingClient() self.pagerank_client = PagerankClient() self.producer = Producer('url_queue') self.consumer = Consumer('url_queue')
def __init__(self): self.spark = SparkSession.builder.appName('kmeans').getOrCreate() self.conf = SparkConf().setMaster('local').setAppName('kmeans') self.sc = SparkContext(conf=self.conf) self.consumer = Consumer('bus', 'localhost') self.stream = self.consumer.get_stream() self.kafka_stream = ConsumerKafka('bus', 'localhost')
def main(flag, host): message = Consumer(flag, host) message = message.pull_message() print message.callback() res = es.index(index="bus", doc_type='json', body=message) # filter(res) # print res '''
def test_consumer_no_commits_returns_unix_start(db): conn, cur = db consumer = Consumer(conn, "foo", "offsets") cur.execute(consumer.create_query) conn.commit() offset = consumer.get_offset() assert offset == datetime(1970, 1, 1)
def __init__(self, broker_url='amqp://localhost', queue_prefix=None, exchange='messagebus'): self.broker_url = broker_url self.consumer = Consumer(self.broker_url, queue_prefix, exchange) self._queue_prefix = queue_prefix self.exchange = exchange
def test_qps(): # 创建生产者 p = Producer(ip=g_ip, user=g_user, password=g_password) p.producer_declare() p.create_exchange(g_exchange, "topic") # 创建消费者 consumers = [] for queue_name in g_queue_name: for i in range(0, 3): consumers.append( Consumer(ip=g_ip, user=g_user, password=g_password)) consumers[len(consumers) - 1].start_consumer( g_exchange, queue_name + str(i), queue_name + str(i)) time.sleep(10) # 等待10S, 让消费者绑定完成 log.info("[test_qps] starting ...") try: target_time = g_test_secs start = time.time() stop = False while not stop: for queue_name in g_queue_name: for i in range(0, 3): time.sleep(g_sleep_secs) p.publish(g_exchange, queue_name + str(i), '{"msg":"this is a test!"}') curr = time.time() if (curr - start) >= target_time: stop = True break if stop: break except Exception as err: log.error("[test_qps] error: " + str(err)) finally: for queue_name in g_queue_name: for i in range(0, 3): p.publish(g_exchange, queue_name + str(i), "quit") p.close() recev = 0 last_time = 0.0 for c in consumers: c.join() recev += c.number_of_msg() if c.stop_consume_time() > last_time: last_time = c.stop_consume_time() log.info("[test_qps] %d msg have been sent, start at %f" % (p.number_of_msg(), p.start_publish_time())) log.info("[test_qps] %d msg have been received, end at %f" % (recev, last_time)) log.info("[test_qps] QPS: %f" % (recev / (last_time - p.start_publish_time())))
def __init__(self): self.consumer = Consumer('bus', 'localhost') self.stream = self.consumer.get_stream() self.kafka_stream = ConsumerKafka('bus', 'localhost') self.cleaned_stream = self.stream.map(self.clean_up) self.conf = SparkConf().setMaster('local').setAppName( 'linear_regression') self.sc = SparkContext(conf=self.conf) self.spark = SparkSession(self.sc)
def start_stop_consumer(flag): consumer = TrendingHashtagConsumer() consumer1 = Consumer() if flag: consumer.start() consumer1.start() else: consumer.stop() consumer1.stop()
def test_consumer_returns_last_commit(db): conn, _ = db consumer = Consumer(conn, "foo", "offsets") new_offset = consumer.new_offset() consumer.commit(new_offset) offset = consumer.get_offset() assert offset == new_offset
def test_consumer_creates_table_if_not_exists(db): conn, _ = db consumer = Consumer(conn, "foo", "offsets") offset = consumer.get_offset() assert offset == datetime(1970, 1, 1) offset = consumer.get_offset() assert offset == datetime(1970, 1, 1)
def test_consumer_duplicate_email(db_session, duplicate_email): """ Test to detemine unique contraint violation is handled properly in code. """ c = Consumer() for data in duplicate_email: task_id = c.s(**data).apply_async() all_users = db_session.query(User).all() assert len(all_users) == len(duplicate_email) - 1
def run(): conn = InfluxDBClient(**config.INFLUX_CONN_SETTING) conn.create_database(config.INFLUX_CONN_SETTING['database']) conn.create_retention_policy( name=f"{config.influx_config['database']}_policy", duration=config.influx_config['retention'], replication='1', database=config.influx_config['database']) conn.close() Thread(target=Producer().run).start() Thread(target=Consumer().run).start()
def test_consumer_valid_data(db_session, valid_data): """ Test consumer class with valid data. Verify the results by querying the database. """ c = Consumer() task_id = c.s(**valid_data).apply_async() all_users = db_session.query(User).all() assert len(all_users) == 1 assert all_users[0].email == valid_data['email']
def test_consumer_invalid_data(db_session, invalid_data): """ Test with invalid data schema should not load anything to database. """ c = Consumer() task_id = c.s(**invalid_data).apply_async() all_users = db_session.query(User).all() assert isinstance(task_id, celery.result.EagerResult) assert len(all_users) == 0
def run(ctx): """Run Bounced Email Service""" from handler import Handler from consumer import Consumer try: handler = Handler(ctx.obj) consumer = Consumer(ctx.obj, handler) consumer.run() except KeyboardInterrupt: consumer.stop()
def test_consumer_passes_offsets_to_work_fn(db): conn, _ = db consumer = Consumer(conn, "foo", "offsets") new_offset = consumer.new_offset() consumer.commit(new_offset) def work(old, new): assert old == new_offset consumer.do_work(work)
def connect_consumer(retries=0): print('Connecting to RabbitMQ server') try: Consumer().start() except pika.exceptions.AMQPConnectionError: if retries >= 2: raise print('Connection error. Retrying in 5 seconds.') time.sleep(5) connect_consumer(retries + 1)
def consume_data(topic, borker, group_id): consumer = Consumer(brokers=[borker], group_id=group_id) consumer.subscribe([topic, "thing-event"]) for msg in consumer: try: value = msg.value print(value) except Exception as err: print(err) continue
def __init__(self, output_file_name): self.tasks_queue = multiprocessing.JoinableQueue() self.write_output_lock = multiprocessing.Lock() self.num_consumers = multiprocessing.cpu_count() * 2 self.consumers = [ Consumer(self.tasks_queue, output_file_name, self.write_output_lock) for i in range(self.num_consumers) ] logger.info('Starting %d consumers' % self.num_consumers) for w in self.consumers: w.start()
def test_consumer_throws_consumer_work_error_on_exception(db): conn, _ = db consumer = Consumer(conn, "foo", "offsets") new_offset = consumer.new_offset() consumer.commit(new_offset) def work(old, new): assert old != new_offset with pytest.raises(ConsumerWorkError): consumer.do_work(work)
def cmd(): def callback(cmd): cmd_name = next(iter(cmd['cmd'])) payload = { 'cmdexe': { cmd_name: f'processed {cmd["cmd"][cmd_name]}', } } producer = Producer(payload) Container(producer).run() consumer = Consumer(callback) Container(consumer).run()
def main(): # max_buffer = input.insert_number('insert maximum buffer capacity: ') # buffer = Buffer(max_buffer) monitor = Monitor() producer = Producer(monitor) consumer = Consumer(monitor) try: threading.Thread(target=producer.auto_produce).start() threading.Thread(target=consumer.auto_consume).start() except: print(Logger.error('Unable to start thread'))