def _process(self, dryrun, msg, show):
if msg:
msg = MessageFactory.serialize(self.arg2msg(msg))
if not self.instance._Bot__source_pipeline:
# is None if source pipeline does not exist
self.instance._Bot__source_pipeline = Pipeline(None)
self.instance._Bot__source_pipeline.receive = lambda: msg
self.instance.logger.info(
" * Message from cli will be used when processing.")
if dryrun:
self.instance.send_message = lambda msg, path="_default": self.instance.logger.info(
"DRYRUN: Message would be sent now{}!".format(
" to the {} path".format(path)
if (path != "_default") else ""))
self.instance.acknowledge_message = lambda: self.instance.logger.info(
"DRYRUN: Message would be acknowledged now!")
self.instance.logger.info(
" * Dryrun only, no message will be really sent through.")
if show:
fn = self.instance.send_message
self.instance.send_message = lambda msg, path="_default": [
self.pprint(msg), fn(msg, path=path)
]
self.instance.logger.info("Processing...")
self.instance.process()
def _process(self, dryrun, msg, show):
if msg:
msg = MessageFactory.serialize(self.arg2msg(msg))
if not self.instance._Bot__source_pipeline:
# is None if source pipeline does not exist
self.instance._Bot__source_pipeline = Pipeline(None)
self.instance._Bot__source_pipeline.receive = lambda *args, **kwargs: msg
self.instance._Bot__source_pipeline.acknowledge = lambda *args, **kwargs: None
self.instance.logger.info(
" * Message from cli will be used when processing.")
if dryrun:
self.instance.send_message = lambda *args, **kwargs: self.instance.logger.info(
"DRYRUN: Message would be sent now to %r!",
kwargs.get('path', "_default"))
self.instance.acknowledge_message = lambda *args, **kwargs: self.instance.logger.info(
"DRYRUN: Message would be acknowledged now!")
self.instance.logger.info(
" * Dryrun only, no message will be really sent through.")
if show:
fn = self.instance.send_message
self.instance.send_message = lambda *args, **kwargs: [
self.pprint(args or "No message generated"),
fn(*args, **kwargs)
]
self.instance.logger.info("Processing...")
self.instance.process()
def process(self):
message = self.receive_message()
if message:
message = MessageFactory.serialize(message)
self.logger.info(message)
self.conn.send(body=message, destination=self.exchange)
def send_message(self, message):
if not message:
self.logger.warning("Empty message found.")
return False
self.message_counter += 1
if self.message_counter % 500 == 0:
self.logger.info("Processed %s messages" % self.message_counter)
raw_message = MessageFactory.serialize(message)
self.destination_pipeline.send(raw_message)
def send_message(self, message):
if not message:
self.logger.warning("Sending Message: Empty message found.")
return False
self.logger.debug("Sending message.")
self.message_counter += 1
if self.message_counter % 500 == 0:
self.logger.info("Processed %s messages." % self.message_counter)
raw_message = MessageFactory.serialize(message)
self.destination_pipeline.send(raw_message)
def _process(self, dryrun, msg):
if msg:
msg = MessageFactory.serialize(self.arg2msg(msg))
self.instance._Bot__source_pipeline.receive = lambda: msg
self.instance.logger.info(" * Message from cli will be used when processing.")
if dryrun:
self.instance.send_message = lambda msg: self.instance.logger.info("DRYRUN: Message would be sent now!")
self.instance.acknowledge_message = lambda: self.instance.logger.info("DRYRUN: Message would be acknowledged now!")
self.instance.logger.info(" * Dryrun only, no message will be really sent through.")
self.instance.logger.info("Processing...")
self.instance.process()
def send_message(self, *messages):
for message in messages:
if not message:
self.logger.warning("Ignoring empty message at sending.")
continue
self.logger.debug("Sending message.")
self.__message_counter += 1
if self.__message_counter % 500 == 0:
self.logger.info("Processed %s messages." % self.__message_counter)
raw_message = MessageFactory.serialize(message)
self.__destination_pipeline.send(raw_message)
def send_message(self, *messages):
for message in messages:
if not message:
self.logger.warning("Ignoring empty message at sending.")
continue
self.logger.debug("Sending message.")
self.__message_counter += 1
if self.__message_counter % 500 == 0:
self.logger.info("Processed %s messages." % self.__message_counter)
raw_message = MessageFactory.serialize(message)
self.__destination_pipeline.send(raw_message)
def _process(self, dryrun, msg):
if msg:
msg = MessageFactory.serialize(self.arg2msg(msg))
self.instance._Bot__source_pipeline.receive = lambda: msg
self.instance.logger.info(" * Message from cli will be used when processing.")
if dryrun:
self.instance.send_message = lambda msg: self.instance.logger.info("DRYRUN: Message would be sent now!")
self.instance.acknowledge_message = lambda: self.instance.logger.info("DRYRUN: Message would be acknowledged now!")
self.instance.logger.info(" * Dryrun only, no message will be really sent through.")
self.instance.logger.info("Processing...")
self.instance.process()
def _process(self, dryrun, msg, show):
if msg:
msg = MessageFactory.serialize(self.arg2msg(msg))
self.instance._Bot__source_pipeline.receive = lambda: msg
self.instance.logger.info(" * Message from cli will be used when processing.")
if dryrun:
self.instance.send_message = lambda msg, path="_default": self.instance.logger.info(
"DRYRUN: Message would be sent now{}!".format(" to the {} path".format(path) if (path != "_default") else ""))
self.instance.acknowledge_message = lambda: self.instance.logger.info("DRYRUN: Message would be acknowledged now!")
self.instance.logger.info(" * Dryrun only, no message will be really sent through.")
if show:
fn = self.instance.send_message
self.instance.send_message = lambda msg, path="_default": [self.pprint(msg), fn(msg, path=path)]
self.instance.logger.info("Processing...")
self.instance.process()
def process(self):
message = self.receive_message()
message = MessageFactory.serialize(message)
self.conn.send(body=message, destination=self.exchange)
self.acknowledge_message()
def process(self):
message = self.receive_message()
message = MessageFactory.serialize(message)
self.conn.send(body=message, destination=self.exchange)
self.acknowledge_message()
def submit():
parameters = handle_parameters(request.form)
temp_file = get_temp_file()
if not temp_file:
return create_response('No file')
destination_pipeline = PipelineFactory.create(PipelineParameters(),
logger=app.logger,
direction='destination')
if not CONFIG.get('destination_pipeline_queue_formatted', False):
destination_pipeline.set_queues(CONFIG['destination_pipeline_queue'], "destination")
destination_pipeline.connect()
time_observation = DateTime().generate_datetime_now()
successful_lines = 0
with open(temp_file[0], encoding='utf8') as handle:
reader = csv.reader(handle, delimiter=parameters['delimiter'],
quotechar=parameters['quotechar'],
skipinitialspace=parameters['skipInitialSpace'],
escapechar=parameters['escapechar'],
)
if parameters['has_header']:
next(reader)
for _ in range(parameters['skipInitialLines']):
next(reader)
for lineindex, line in enumerate(reader):
event = Event()
try:
for columnindex, (column, value) in \
enumerate(zip(parameters['columns'], line)):
if not column or not value:
continue
if column.startswith('time.'):
parsed = dateutil.parser.parse(value)
if not parsed.tzinfo:
value += parameters['timezone']
parsed = dateutil.parser.parse(value)
value = parsed.isoformat()
if column == 'extra':
value = handle_extra(value)
event.add(column, value)
for key, value in parameters.get('constant_fields', {}).items():
if key not in event:
event.add(key, value)
for key, value in request.form.items():
if not key.startswith('custom_'):
continue
key = key[7:]
if key not in event:
event.add(key, value)
if CONFIG.get('destination_pipeline_queue_formatted', False):
queue_name = CONFIG['destination_pipeline_queue'].format(ev=event)
destination_pipeline.set_queues(queue_name, "destination")
destination_pipeline.connect()
except Exception:
continue
if 'classification.type' not in event:
event.add('classification.type', parameters['classification.type'])
if 'classification.identifier' not in event:
event.add('classification.identifier', parameters['classification.identifier'])
if 'feed.code' not in event:
event.add('feed.code', parameters['feed.code'])
if 'time.observation' not in event:
event.add('time.observation', time_observation, sanitize=False)
raw_message = MessageFactory.serialize(event)
destination_pipeline.send(raw_message)
successful_lines += 1
return create_response('Successfully processed %s lines.' % successful_lines)
