def _process(self, dryrun, msg, show): if msg: msg = MessageFactory.serialize(self.arg2msg(msg)) if not self.instance._Bot__source_pipeline: # is None if source pipeline does not exist self.instance._Bot__source_pipeline = Pipeline(None) self.instance._Bot__source_pipeline.receive = lambda: msg self.instance.logger.info( " * Message from cli will be used when processing.") if dryrun: self.instance.send_message = lambda msg, path="_default": self.instance.logger.info( "DRYRUN: Message would be sent now{}!".format( " to the {} path".format(path) if (path != "_default") else "")) self.instance.acknowledge_message = lambda: self.instance.logger.info( "DRYRUN: Message would be acknowledged now!") self.instance.logger.info( " * Dryrun only, no message will be really sent through.") if show: fn = self.instance.send_message self.instance.send_message = lambda msg, path="_default": [ self.pprint(msg), fn(msg, path=path) ] self.instance.logger.info("Processing...") self.instance.process()
def _process(self, dryrun, msg, show): if msg: msg = MessageFactory.serialize(self.arg2msg(msg)) if not self.instance._Bot__source_pipeline: # is None if source pipeline does not exist self.instance._Bot__source_pipeline = Pipeline(None) self.instance._Bot__source_pipeline.receive = lambda *args, **kwargs: msg self.instance._Bot__source_pipeline.acknowledge = lambda *args, **kwargs: None self.instance.logger.info( " * Message from cli will be used when processing.") if dryrun: self.instance.send_message = lambda *args, **kwargs: self.instance.logger.info( "DRYRUN: Message would be sent now to %r!", kwargs.get('path', "_default")) self.instance.acknowledge_message = lambda *args, **kwargs: self.instance.logger.info( "DRYRUN: Message would be acknowledged now!") self.instance.logger.info( " * Dryrun only, no message will be really sent through.") if show: fn = self.instance.send_message self.instance.send_message = lambda *args, **kwargs: [ self.pprint(args or "No message generated"), fn(*args, **kwargs) ] self.instance.logger.info("Processing...") self.instance.process()
def process(self): message = self.receive_message() if message: message = MessageFactory.serialize(message) self.logger.info(message) self.conn.send(body=message, destination=self.exchange)
def send_message(self, message): if not message: self.logger.warning("Empty message found.") return False self.message_counter += 1 if self.message_counter % 500 == 0: self.logger.info("Processed %s messages" % self.message_counter) raw_message = MessageFactory.serialize(message) self.destination_pipeline.send(raw_message)
def send_message(self, message): if not message: self.logger.warning("Sending Message: Empty message found.") return False self.logger.debug("Sending message.") self.message_counter += 1 if self.message_counter % 500 == 0: self.logger.info("Processed %s messages." % self.message_counter) raw_message = MessageFactory.serialize(message) self.destination_pipeline.send(raw_message)
def _process(self, dryrun, msg): if msg: msg = MessageFactory.serialize(self.arg2msg(msg)) self.instance._Bot__source_pipeline.receive = lambda: msg self.instance.logger.info(" * Message from cli will be used when processing.") if dryrun: self.instance.send_message = lambda msg: self.instance.logger.info("DRYRUN: Message would be sent now!") self.instance.acknowledge_message = lambda: self.instance.logger.info("DRYRUN: Message would be acknowledged now!") self.instance.logger.info(" * Dryrun only, no message will be really sent through.") self.instance.logger.info("Processing...") self.instance.process()
def send_message(self, *messages): for message in messages: if not message: self.logger.warning("Ignoring empty message at sending.") continue self.logger.debug("Sending message.") self.__message_counter += 1 if self.__message_counter % 500 == 0: self.logger.info("Processed %s messages." % self.__message_counter) raw_message = MessageFactory.serialize(message) self.__destination_pipeline.send(raw_message)
def send_message(self, *messages): for message in messages: if not message: self.logger.warning("Ignoring empty message at sending.") continue self.logger.debug("Sending message.") self.__message_counter += 1 if self.__message_counter % 500 == 0: self.logger.info("Processed %s messages." % self.__message_counter) raw_message = MessageFactory.serialize(message) self.__destination_pipeline.send(raw_message)
def _process(self, dryrun, msg): if msg: msg = MessageFactory.serialize(self.arg2msg(msg)) self.instance._Bot__source_pipeline.receive = lambda: msg self.instance.logger.info(" * Message from cli will be used when processing.") if dryrun: self.instance.send_message = lambda msg: self.instance.logger.info("DRYRUN: Message would be sent now!") self.instance.acknowledge_message = lambda: self.instance.logger.info("DRYRUN: Message would be acknowledged now!") self.instance.logger.info(" * Dryrun only, no message will be really sent through.") self.instance.logger.info("Processing...") self.instance.process()
def _process(self, dryrun, msg, show): if msg: msg = MessageFactory.serialize(self.arg2msg(msg)) self.instance._Bot__source_pipeline.receive = lambda: msg self.instance.logger.info(" * Message from cli will be used when processing.") if dryrun: self.instance.send_message = lambda msg, path="_default": self.instance.logger.info( "DRYRUN: Message would be sent now{}!".format(" to the {} path".format(path) if (path != "_default") else "")) self.instance.acknowledge_message = lambda: self.instance.logger.info("DRYRUN: Message would be acknowledged now!") self.instance.logger.info(" * Dryrun only, no message will be really sent through.") if show: fn = self.instance.send_message self.instance.send_message = lambda msg, path="_default": [self.pprint(msg), fn(msg, path=path)] self.instance.logger.info("Processing...") self.instance.process()
def process(self): message = self.receive_message() message = MessageFactory.serialize(message) self.conn.send(body=message, destination=self.exchange) self.acknowledge_message()
def process(self): message = self.receive_message() message = MessageFactory.serialize(message) self.conn.send(body=message, destination=self.exchange) self.acknowledge_message()
def submit(): parameters = handle_parameters(request.form) temp_file = get_temp_file() if not temp_file: return create_response('No file') destination_pipeline = PipelineFactory.create(PipelineParameters(), logger=app.logger, direction='destination') if not CONFIG.get('destination_pipeline_queue_formatted', False): destination_pipeline.set_queues(CONFIG['destination_pipeline_queue'], "destination") destination_pipeline.connect() time_observation = DateTime().generate_datetime_now() successful_lines = 0 with open(temp_file[0], encoding='utf8') as handle: reader = csv.reader(handle, delimiter=parameters['delimiter'], quotechar=parameters['quotechar'], skipinitialspace=parameters['skipInitialSpace'], escapechar=parameters['escapechar'], ) if parameters['has_header']: next(reader) for _ in range(parameters['skipInitialLines']): next(reader) for lineindex, line in enumerate(reader): event = Event() try: for columnindex, (column, value) in \ enumerate(zip(parameters['columns'], line)): if not column or not value: continue if column.startswith('time.'): parsed = dateutil.parser.parse(value) if not parsed.tzinfo: value += parameters['timezone'] parsed = dateutil.parser.parse(value) value = parsed.isoformat() if column == 'extra': value = handle_extra(value) event.add(column, value) for key, value in parameters.get('constant_fields', {}).items(): if key not in event: event.add(key, value) for key, value in request.form.items(): if not key.startswith('custom_'): continue key = key[7:] if key not in event: event.add(key, value) if CONFIG.get('destination_pipeline_queue_formatted', False): queue_name = CONFIG['destination_pipeline_queue'].format(ev=event) destination_pipeline.set_queues(queue_name, "destination") destination_pipeline.connect() except Exception: continue if 'classification.type' not in event: event.add('classification.type', parameters['classification.type']) if 'classification.identifier' not in event: event.add('classification.identifier', parameters['classification.identifier']) if 'feed.code' not in event: event.add('feed.code', parameters['feed.code']) if 'time.observation' not in event: event.add('time.observation', time_observation, sanitize=False) raw_message = MessageFactory.serialize(event) destination_pipeline.send(raw_message) successful_lines += 1 return create_response('Successfully processed %s lines.' % successful_lines)