def _cbk_wl_creation(self, ch: BlockingChannel, method: Basic.Deliver, _: BasicProperties, body: bytes) -> None: ch.basic_ack(method.delivery_tag) arr = body.decode().strip().split(',') logger = logging.getLogger('monitoring.workload_creation') logger.debug(f'{arr} is received from workload_creation queue') if len(arr) != 5: return wl_identifier, wl_type, pid, perf_pid, perf_interval = arr pid = int(pid) perf_pid = int(perf_pid) perf_interval = int(perf_interval) item = wl_identifier.split('_') wl_name = item[0] if not psutil.pid_exists(pid): return workload = Workload(wl_name, wl_type, pid, perf_pid, perf_interval) if wl_type == 'bg': logger.info(f'{workload} is background process') else: logger.info(f'{workload} is foreground process') self._pending_wl.add(workload) wl_queue_name = '{}({})'.format(wl_name, pid) ch.queue_declare(wl_queue_name) ch.basic_consume(functools.partial(self._cbk_wl_monitor, workload), wl_queue_name)
def ngram_create_consumer(ch: BlockingChannel, method: Basic.Deliver, properties, body: bytes): logger.debug("Recebido: " + body.decode(), extra={"received_args": body}) try: task_info = json.loads(body.decode()) logger.debug("Recuperando tarefa: " + task_info["task"]) ngram_create_task: NGramsCreateTaskModel = NGramsCreateTaskModel.objects(id=task_info["task"]).first() if ngram_create_task is None: raise Exception("Não foi encontrada nenhuma tarefa com o id " + task_info["task"]) except Exception as e: logger.error( "Erro ao recuperar a tarefa", exc_info=True, extra={"received_args": body} ) ch.basic_nack( delivery_tag=method.delivery_tag, requeue=False ) return False success_task = process_task(ngram_create_task) logger.debug("Finalizando tarefa: " + task_info["task"]) if success_task: ch.basic_ack(delivery_tag=method.delivery_tag) return True else: ch.basic_nack( delivery_tag=method.delivery_tag, requeue=False ) return False
def _cbk_wl_monitor(self, workload: Workload, ch: BlockingChannel, method: Basic.Deliver, _: BasicProperties, body: bytes) -> None: metric = json.loads(body.decode()) ch.basic_ack(method.delivery_tag) if self._node_type == NodeType.IntegratedGPU: item = BasicMetric(metric['llc_references'], metric['llc_misses'], metric['instructions'], metric['cycles'], metric['gpu_core_util'], metric['gpu_core_freq'], metric['gpu_emc_util'], metric['gpu_emc_freq'], workload.perf_interval) if self._node_type == NodeType.CPU: item = BasicMetric(metric['llc_references'], metric['llc_misses'], metric['instructions'], metric['cycles'], 0, 0, 0, 0, workload.perf_interval) logger = logging.getLogger(f'monitoring.metric.{workload}') logger.debug(f'{metric} is given from ') metric_que = workload.metrics if len(metric_que) == self._metric_buf_size: metric_que.pop() metric_que.appendleft(item)
def _cbk_connecting_nodes(self, ch: BlockingChannel, method: Basic.Deliver, _: BasicProperties, body: bytes) \ -> None: ch.basic_ack(method.delivery_tag) arr = body.decode().strip().split(',') logger = logging.getLogger('monitoring.tracking_nodes') logger.debug(f'{arr} is received from tracking_node queue') if len(arr) != 6: return ip_addr, aggr_contention, num_workloads, num_of_fg_wls, num_of_bg_wls, node_type = arr aggr_contention = float(aggr_contention) num_workloads = int(num_workloads) num_of_fg_wls = int(num_of_fg_wls) num_of_bg_wls = int(num_of_bg_wls) # node_type is either 'gpu' or 'cpu' tracked_node = self._cluster_nodes[ip_addr] tracked_node.aggr_contention = aggr_contention tracked_node.num_workloads = num_workloads tracked_node.num_of_fg_wls = num_of_fg_wls tracked_node.num_of_bg_wls = num_of_bg_wls tracked_node.node_type = node_type node_queue_name = '{}_node_({})'.format(tracked_node.node_type, tracked_node.ip_addr) ch.queue_declare(node_queue_name) ch.basic_consume( functools.partial(self._cbk_node_monitor, tracked_node), node_queue_name)
def do_consume(channel: BlockingChannel) -> None: events: List[Dict[str, Any]] = [] last_process = time.time() max_processed: Optional[int] = None self.is_consuming = True # This iterator technique will iteratively collect up to # batch_size events from the RabbitMQ queue (if present) # before calling the callback with the batch. If not # enough events are present, it will sleep for at most # timeout seconds before calling the callback with the # batch of events it has. for method, properties, body in channel.consume( queue_name, inactivity_timeout=timeout): if body is not None: assert method is not None events.append(orjson.loads(body)) max_processed = method.delivery_tag now = time.time() if len(events) >= batch_size or ( timeout and now >= last_process + timeout): if events: assert max_processed is not None try: callback(events) channel.basic_ack(max_processed, multiple=True) except BaseException: channel.basic_nack(max_processed, multiple=True) raise events = [] last_process = now if not self.is_consuming: break
def task_process(self, channel: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes) -> None: """ Process the received task :param channel: channel :param method: method :param properties: task properties :param body: task body :return: None """ raw_body = loads(body.decode(encoding="utf-8")) cases = raw_body.get("cases", {}) task = TaskItem(**raw_body.get("task", {})) try: results = list(self.manager.multi_case_runner(cases=cases)) for result in results: TaskCrud.create_task_result(task, result or {}) task.set_success( msg=f"Task done: {len(results)} out of {len(cases)} cases") except Exception as cases_err: task.set_error(msg=f"Task error: {str(cases_err)}") TaskCrud.update_task(task) logger.info(msg=f"Done task {task.task_id}") channel.basic_publish( exchange="", routing_key=properties.reply_to, properties=BasicProperties( correlation_id=properties.correlation_id), body=dumps(task.as_json()).encode(encoding="utf-8"), ) channel.basic_ack(delivery_tag=method.delivery_tag)
async def execute( self, channel: BlockingChannel, method, properties, body: CallInfo, ): task = self.tasks[body.func] if not is_coroutine_callable(task): raise Exception(f"{task.__name__} is not coroutine function.") func_name = body.func file = task.__code__.co_filename line = task.__code__.co_firstlineno try: result = await task.do(**body.kwargs) self.logger.info( f"[SUCCESS]{file} {line} {func_name}(**{body.kwargs!r})") except Exception as e: self.logger.warning( f"[FAILED]{file} {line} {func_name}(**{body.kwargs!r}) {e}") # TODO: 処理が数回失敗した場合は、デッドレターキューに格納する(現在はとりあえず削除) # channel.basic_reject(delivery_tag=method.delivery_tag, requeue=False) # メッセージを破棄する # channel.basic_reject(delivery_tag=method.delivery_tag, requeue=True) # メッセージを再度キューイングする。実行順序は、多分最後になる # channel.close() # コネクションやチャンネル開放時、未応答のキューは再度キューイングされる。順序はそのままらしいので、エラーが発生し続け、他の処理を妨害する恐れがある if not self.auto_ack: channel.basic_ack(delivery_tag=method.delivery_tag)
def callback(ch: BlockingChannel, method, properties, body): msg = str(body, "utf8").split(":") print('# [%s] %s 메세지를 받았습니다. \n %r' % (datetime.datetime.now(), msg[0], body)) time.sleep(int(str(msg[1])) / 10) print(" # [%s] 완료했습니다." % datetime.datetime.now()) ch.basic_ack(delivery_tag=method.delivery_tag)
def wrapped_consumer( ch: BlockingChannel, method: Basic.Deliver, properties: pika.BasicProperties, body: bytes, ) -> None: callback([orjson.loads(body)]) ch.basic_ack(delivery_tag=method.delivery_tag)
def on_rpc_response(ch: BlockingChannel, method, props, body): if props.correlation_id == self.corr_id: decoded_body = body.decode() log_info('GET RPC REPLY', queue=reply_queue, correlation_id=self.corr_id, body=decoded_body) self.result = jsons.loads(decoded_body) ch.basic_ack(delivery_tag=method.delivery_tag)
def _on_message( channel: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes, ) -> None: quote: StockQuote = pickle.loads(body) handler([quote]) channel.basic_ack(method.delivery_tag)
def opened(channel: BlockingChannel) -> None: while True: (meta, _, message) = channel.basic_get(queue_name) if message is None: break channel.basic_ack(meta.delivery_tag) messages.append(message)
def _message_callback(self, ch: BlockingChannel, method: Basic.Deliver, _, body: bytes): try: body_as_dict = RabbitMQBrokerConsumer._decode_body(body) self._callback(body_as_dict) except Exception as e: print(f'Error during handling message {body}: {e}') finally: ch.basic_ack(delivery_tag=method.delivery_tag)
def on_message(self, ch: BlockingChannel, method, properties, body: str): payload = json.loads(body) response = self.callback(self.service, payload) payload = {**payload, **response} self.channel.basic_publish("", self.queue_out, body=json.dumps(payload), mandatory=True) ch.basic_ack(delivery_tag=method.delivery_tag)
def wrapped_consumer(ch: BlockingChannel, method: Basic.Deliver, properties: pika.BasicProperties, body: str) -> None: try: consumer(ch, method, properties, body) ch.basic_ack(delivery_tag=method.delivery_tag) except Exception as e: ch.basic_nack(delivery_tag=method.delivery_tag) raise e
def on_request(self, channel: BlockingChannel, method, props, body): request = dill.loads(body) response = self.execute_request(request) channel.basic_publish(exchange='', routing_key=props.reply_to, properties=pika.BasicProperties(correlation_id = \ props.correlation_id), body=dill.dumps(response)) channel.basic_ack(delivery_tag=method.delivery_tag)
def req_callback(ch: BlockingChannel, method: spec.Basic.Deliver, _: spec.BasicProperties, body: bytes): body = load_object(body) ack = callback(body) if ack is None: ack = True if ack: ch.basic_ack(delivery_tag=method.delivery_tag) else: ch.basic_nack(delivery_tag=method.delivery_tag)
def raw_answer_callback(ch: BlockingChannel, method: spec.Basic.Deliver, _: spec.BasicProperties, body: bytes): body: AnsBody = load_object(body) ack = answer_callback(body['id'], body['ans']) if ack is None: ack = True if ack: ch.basic_ack(delivery_tag=method.delivery_tag) else: ch.basic_nack(delivery_tag=method.delivery_tag)
def ack_message(self, channel: BlockingChannel, delivery_tag: int) -> None: if channel.is_open: logging.info( "%s", f"Message (delivery_tag={delivery_tag}) acknowledged") channel.basic_ack(delivery_tag) else: logging.info( "%s", (f"Message (delivery_tag={delivery_tag}) " "NOT acknowledged (channel closed)"), )
def cb(ch: BlockingChannel, method: pika.spec.Basic.Deliver, properties: pika.spec.BasicProperties, body: bytes): key = properties.correlation_id try: callback(ch, method, properties, body) ch.basic_ack(delivery_tag=method.delivery_tag) self.logger.info(f" [x] Message {key} processed!") except Exception: self.logger.info( f" [-] failed to process message {key}: {traceback.format_exc()}" )
def on_rpc_response(ch: BlockingChannel, method, props, body): if props.correlation_id == self.corr_id: decoded_body = body.decode() self.result = jsons.loads(decoded_body) print({ 'action': 'get_rmq_rpc_reply', 'queue': reply_queue, 'correlation_id': self.corr_id, 'result': self.result }) self.replied = True ch.basic_ack(delivery_tag=method.delivery_tag)
def req_callback(ch: BlockingChannel, method: spec.Basic.Deliver, _: spec.BasicProperties, body: bytes): body: ReqBody = load_object(body) ack = request_callback( body['req'], lambda answer: answer_callback(body['id'], answer)) if ack is None: ack = True if ack: ch.basic_ack(delivery_tag=method.delivery_tag) else: ch.basic_nack(delivery_tag=method.delivery_tag)
def callback(self, ch: BlockingChannel, method: spec.Basic.Deliver, properties: spec.BasicProperties, body: bytes): log.debug(' -> Received ch: %s meth: %s props: %s body: %s', ch, method, properties, body) b = body.decode() try: # General format: {req_id, action, host} act = json.loads(b) # type: dict log.debug(' -> Decoded to %s', act) if 'action' not in act: raise ValueError('"action" was not in decoded json object...') if act['action'] not in self.ACTIONS: raise ValueError('Action "{}" is not a valid action...'.format( act['action'])) except (JSONDecodeError, TypeError, ValueError, AttributeError): log.exception('Error decoding json for %s', b) return ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False) try: run_act = self.ACTIONS[act['action']](**act) if run_act: log.debug('Acknowledging success to MQ') return ch.basic_ack(delivery_tag=method.delivery_tag) else: log.debug('Acknowledging failure (try later) to MQ') return ch.basic_nack(delivery_tag=method.delivery_tag) except (InvalidHostException, AttributeError, ValueError) as e: log.warning('Invalid host... Type: %s Msg: %s', type(e), str(e)) return ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False) except Exception: log.exception('Unknown exception while handling action call...') return ch.basic_nack(delivery_tag=method.delivery_tag)
def ner_resume_create_consumer(ch: BlockingChannel, method: Basic.Deliver, properties, body: bytes): logger.debug("Recebido: " + body.decode(), extra={"received_args": body}) try: task_info = json.loads(body.decode()) logger.debug("Recuperando tarefa: " + task_info["task"]) ner_resume_create_task: NerResumeCreateTaskModel = NerResumeCreateTaskModel.objects( id=task_info["task"]).first() if ner_resume_create_task is None: raise Exception("Não foi encontrada nenhuma tarefa com o id " + task_info["task"]) except Exception as e: logger.error("Erro ao recuperar a tarefa", exc_info=True, extra={"received_args": body}) ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False) return False try: if ner_resume_create_task.status == "success" or \ ner_resume_create_task.progress == ner_resume_create_task.total: ner_resume_create_task.status = "success" ner_resume_create_task.save() success_task = True else: success_task = process_task(ner_resume_create_task) logger.debug("Finalizando tarefa: " + task_info["task"]) if success_task: ch.basic_ack(delivery_tag=method.delivery_tag) return True except ChannelWrongStateError: return False except Exception: ner_resume_create_task.status = "error" ner_resume_create_task.save() logger.error( ner_resume_create_task.error, exc_info=True, extra={"received_args": ner_resume_create_task.to_mongo()}) ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False) return False
def handle_consumption_error( self, ch: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes, is_store: bool, ): if self.has_been_redelivered_too_much(properties): derived_action = self.send_to_dead_letter(ch, method, properties, body) else: derived_action = self.send_to_retry(ch, method, properties, body, is_store) ch.basic_ack(delivery_tag=method.delivery_tag) return derived_action
def __on_request( self, channel: BlockingChannel, method: pika.spec.Basic.Deliver, properties: pika.BasicProperties, body: bytes ): try: print('Got message') message = json.loads(body.decode()) print(message) metadata: dict = { 'user': message['user'] if 'user' in message else None } payload: dict = message['payload'] status_code, response = self.methods[message['method']](payload, metadata) print('Response', response) except ResponseException as e: print_exc() status_code, response = e.status_code, str(e) except Exception as e: print_exc() status_code, response = 500, 'Internal Server Error: ' + str(e) if properties.reply_to: channel.basic_publish( exchange='', routing_key=properties.reply_to, properties=pika.BasicProperties(correlation_id=properties.correlation_id), body=json.dumps({ 'statusCode': status_code, 'body': response }) ) else: print('Async, one way, message handled') channel.basic_ack(delivery_tag=method.delivery_tag) print('Acked')
def _consuming_callback(self, ch: BlockingChannel, method, properties, body): try: rabbit_message = json.loads(body.decode()) except json.JSONDecodeError: self._logger.error( '[RabbitConsumer] Task skipped. JSONDecodeError on "%s"', body.decode()) ch.basic_ack(delivery_tag=method.delivery_tag) else: self._thread_output.put(rabbit_message) self._logger.debug('Got msg: %s', rabbit_message) # waiting for main thread signal = self._wait_signal() if signal is SendAcknowledgeSignal: ch.basic_ack(delivery_tag=method.delivery_tag) elif signal is RejectSignal: ch.basic_reject(delivery_tag=method.delivery_tag) else: raise Exception('Excuse me what the type?')
def _cbk_job_submission(self, ch: BlockingChannel, method: Basic.Deliver, _: BasicProperties, body: bytes) -> None: ch.basic_ack(method.delivery_tag) arr = body.decode().strip().split(',') logger = logging.getLogger('monitoring.job_submission') logger.debug(f'{arr} is received from job_submission queue') if len(arr) != 4: return job_name, job_type, job_preferences, job_objective = arr job = Job(job_name, job_type, job_preferences, job_objective) if job_type == 'bg': logger.info(f'{job_name} is background job') else: logger.info(f'{job_name} is foreground job') self._pending_jobs.add(job)
def _cbk_wl_creation(self, ch: BlockingChannel, method: Basic.Deliver, _: BasicProperties, body: bytes) -> None: ch.basic_ack(method.delivery_tag) arr = body.decode().strip().split(',') logger = logging.getLogger('monitoring.workload_creation') logger.debug(f'{arr} is received from workload_creation queue') if len(arr) != 8: return wl_identifier, wl_type, pid, perf_pid, perf_interval, tegra_pid, tegra_interval, max_workloads = arr pid = int(pid) perf_pid = int(perf_pid) perf_interval = int(perf_interval) item = wl_identifier.split('_') wl_name = item[0] max_wls = int(max_workloads) if not psutil.pid_exists(pid): return workload = Workload(wl_name, wl_type, pid, perf_pid, perf_interval) #workload.check_gpu_task() if wl_type == 'bg': logger.info(f'{workload} is background process') else: logger.info(f'{workload} is foreground process') self._pending_wl.add(workload, max_wls) wl_queue_name = 'rmq-{}-{}({})'.format(self._rmq_host, wl_name, pid) ch.exchange_declare(exchange=self._rmq_creation_exchange, exchange_type='fanout') ch.queue_declare(wl_queue_name) self._rmq_bench_exchange = f'ex-{self._rmq_host}-{wl_name}({pid})' ch.queue_bind(exchange=self._rmq_bench_exchange, queue=wl_queue_name) ch.basic_consume(functools.partial(self._cbk_wl_monitor, workload), wl_queue_name)
def _cbk_node_monitor(self, node: Node, ch: BlockingChannel, method: Basic.Deliver, _: BasicProperties, body: bytes) -> None: metric = json.loads(body.decode( )) # Through json format, node monitor can get aggr contention info ch.basic_ack(method.delivery_tag) # FIXME: Hard coded (200ms as interval) item = BasicMetric(metric['llc_references'], metric['llc_misses'], metric['instructions'], metric['cycles'], metric['gpu_core_util'], metric['gpu_core_freq'], metric['gpu_emc_util'], metric['gpu_emc_freq'], 200) logger = logging.getLogger(f'monitoring.metric.{node}') logger.debug(f'{metric} is given from ') metric_que = node.metrics if len(metric_que) == self._metric_buf_size: metric_que.pop() metric_que.appendleft(item)