def process(self):
RT = rt.Rt(self.parameters.uri, self.parameters.user,
self.parameters.password)
if not RT.login():
raise ValueError('Login failed.')
query = RT.search(Queue=self.parameters.search_queue,
Subject__like=self.parameters.search_subject_like,
Owner=self.parameters.search_owner,
Status=self.parameters.search_status)
self.logger.info('{} results on search query.'.format(len(query)))
for ticket in query:
ticket_id = int(ticket['id'].split('/')[1])
self.logger.debug('Process ticket {}.'.format(ticket_id))
for (att_id, att_name, _, _) in RT.get_attachments(ticket_id):
if re.search(self.parameters.attachment_regex, att_name):
self.logger.debug('Found attachment {}: {!r}.'
''.format(att_id, att_name))
break
else:
self.logger.debug('No matching attachement name found.')
continue
attachment = RT.get_attachment_content(ticket_id, att_id)
if self.parameters.unzip_attachment:
file_obj = io.BytesIO(attachment)
zipped = zipfile.ZipFile(file_obj)
raw = zipped.read(zipped.namelist()[0])
else:
raw = attachment
if self.parameters.gnupg_decrypt:
raw = str(
self.gpg.decrypt(
raw,
always_trust=self.parameters.gnupg_trust,
passphrase=self.parameters.gnupg_passphrase))
self.logger.info('Successfully decrypted attachment.')
self.logger.debug(raw)
report = Report()
report.add("raw", raw, sanitize=True)
report.add("rtir_id", ticket_id, sanitize=True)
report.add("feed.name", self.parameters.feed, sanitize=True)
report.add("feed.accuracy",
self.parameters.accuracy,
sanitize=True)
time_observation = DateTime().generate_datetime_now()
report.add('time.observation', time_observation, sanitize=True)
self.send_message(report)
if self.parameters.take_ticket:
RT.edit_ticket(ticket_id, Owner=self.parameters.user)
def process(self):
self.logger.info("Downloading report through API")
otx = OTXv2(self.parameters.api_key)
pulses = otx.getall()
self.logger.info("Report downloaded.")
report = Report()
report.add("raw", json.dumps(pulses), sanitize=True)
report.add("feed.name", self.parameters.feed, sanitize=True)
report.add("feed.accuracy", self.parameters.accuracy, sanitize=True)
time_observation = DateTime().generate_datetime_now()
report.add('time.observation', time_observation, sanitize=True)
self.send_message(report)
def process(self):
mailbox = imbox.Imbox(self.parameters.mail_host,
self.parameters.mail_user,
self.parameters.mail_password,
self.parameters.mail_ssl)
emails = mailbox.messages(folder=self.parameters.mail_folder,
unread=True)
if emails:
for uid, message in emails:
if (self.parameters.mail_subject_regex and not re.search(
self.parameters.mail_subject_regex, message.subject)):
continue
self.logger.info("Reading email report")
for attach in message.attachments:
if not attach:
continue
# remove quote marks from filename
attach_name = attach['filename'][1:len(attach['filename']
) - 1]
if re.search(self.parameters.mail_attach_regex,
attach_name):
if self.parameters.mail_attach_unzip:
zipped = zipfile.ZipFile(attach['content'])
raw_report = zipped.read(zipped.namelist()[0])
else:
raw_report = attach['content'].read()
report = Report()
report.add("raw", raw_report, sanitize=True)
report.add("feed.name",
self.parameters.feed,
sanitize=True)
report.add("feed.accuracy",
self.parameters.accuracy,
sanitize=True)
time_observation = DateTime().generate_datetime_now()
report.add('time.observation',
time_observation,
sanitize=True)
self.send_message(report)
mailbox.mark_seen(uid)
self.logger.info("Email report read")
def on_message(self, headers, message):
self.n6stomper.logger.debug('Receive message '
'{!r}...'.format(message[:500]))
report = Report()
report.add("raw", message.rstrip(), sanitize=True)
report.add("feed.name", self.n6stomper.parameters.feed, sanitize=True)
report.add("feed.url",
"stomp://" + self.n6stomper.parameters.server + ":" +
str(self.n6stomper.parameters.port) + "/" +
self.n6stomper.parameters.exchange,
sanitize=True)
time_observation = DateTime().generate_datetime_now()
report.add('time.observation', time_observation, sanitize=True)
self.n6stomper.send_message(report)
self.n6stomper.logger.debug('Receiving Message.')
def process(self):
mailbox = imbox.Imbox(self.parameters.mail_host,
self.parameters.mail_user,
self.parameters.mail_password,
self.parameters.mail_ssl)
emails = mailbox.messages(folder=self.parameters.mail_folder, unread=True)
if emails:
for uid, message in emails:
if (self.parameters.mail_subject_regex and
not re.search(self.parameters.mail_subject_regex,
message.subject)):
continue
self.logger.info("Reading email report")
for body in message.body['plain']:
match = re.search(self.parameters.mail_url_regex, body)
if match:
url = match.group()
self.logger.info("Downloading report from %s" % url)
raw_report = fetch_url(url, timeout=60.0,
chunk_size=16384)
self.logger.info("Report downloaded.")
report = Report()
report.add("raw", raw_report, sanitize=True)
report.add("feed.name",
self.parameters.feed, sanitize=True)
report.add("feed.accuracy", self.parameters.accuracy, sanitize=True)
time_observation = DateTime().generate_datetime_now()
report.add('time.observation', time_observation,
sanitize=True)
self.send_message(report)
mailbox.mark_seen(uid)
self.logger.info("Email report read")
def process(self):
self.logger.info("Downloading report from %s" %
self.parameters.http_url)
resp = requests.get(url=self.parameters.http_url,
auth=self.auth,
proxies=self.proxy,
headers=self.http_header,
verify=self.http_verify_cert)
if resp.status_code // 100 != 2:
raise ValueError('HTTP response status code was {}.'
''.format(resp.status_code))
self.logger.info("Report downloaded.")
report = Report()
report.add("raw", resp.text, sanitize=True)
report.add("feed.name", self.parameters.feed, sanitize=True)
report.add("feed.url", self.parameters.http_url, sanitize=True)
report.add("feed.accuracy", self.parameters.accuracy, sanitize=True)
time_observation = DateTime().generate_datetime_now()
report.add('time.observation', time_observation, sanitize=True)
self.send_message(report)
def submit():
parameters = handle_parameters(request.form)
temp_file = get_temp_file()
if not temp_file:
return create_response('No file')
destination_pipeline = PipelineFactory.create(PipelineParameters(),
logger=app.logger,
direction='destination')
if not CONFIG.get('destination_pipeline_queue_formatted', False):
destination_pipeline.set_queues(CONFIG['destination_pipeline_queue'], "destination")
destination_pipeline.connect()
time_observation = DateTime().generate_datetime_now()
successful_lines = 0
with open(temp_file[0], encoding='utf8') as handle:
reader = csv.reader(handle, delimiter=parameters['delimiter'],
quotechar=parameters['quotechar'],
skipinitialspace=parameters['skipInitialSpace'],
escapechar=parameters['escapechar'],
)
if parameters['has_header']:
next(reader)
for _ in range(parameters['skipInitialLines']):
next(reader)
for lineindex, line in enumerate(reader):
event = Event()
try:
for columnindex, (column, value) in \
enumerate(zip(parameters['columns'], line)):
if not column or not value:
continue
if column.startswith('time.'):
parsed = dateutil.parser.parse(value)
if not parsed.tzinfo:
value += parameters['timezone']
parsed = dateutil.parser.parse(value)
value = parsed.isoformat()
if column == 'extra':
value = handle_extra(value)
event.add(column, value)
for key, value in parameters.get('constant_fields', {}).items():
if key not in event:
event.add(key, value)
for key, value in request.form.items():
if not key.startswith('custom_'):
continue
key = key[7:]
if key not in event:
event.add(key, value)
if CONFIG.get('destination_pipeline_queue_formatted', False):
queue_name = CONFIG['destination_pipeline_queue'].format(ev=event)
destination_pipeline.set_queues(queue_name, "destination")
destination_pipeline.connect()
except Exception:
continue
if 'classification.type' not in event:
event.add('classification.type', parameters['classification.type'])
if 'classification.identifier' not in event:
event.add('classification.identifier', parameters['classification.identifier'])
if 'feed.code' not in event:
event.add('feed.code', parameters['feed.code'])
if 'time.observation' not in event:
event.add('time.observation', time_observation, sanitize=False)
raw_message = MessageFactory.serialize(event)
destination_pipeline.send(raw_message)
successful_lines += 1
return create_response('Successfully processed %s lines.' % successful_lines)
def process(self):
mailbox = imbox.Imbox(self.parameters.mail_host, self.parameters.mail_user, utils.base64_decode(self.parameters.mail_password), self.parameters.mail_ssl)
self.logger.info("Connected to mail server")
emails = mailbox.messages(folder=self.parameters.folder, unread=True)
try:
if emails:
self.logger.info("Parsing emails in mailbox")
for uid, message in emails:
if self.parameters.subject_regex and not re.search(self.parameters.subject_regex, message.subject):
continue
self.logger.info("Reading email report")
if hasattr(message,'attachments') and message.attachments:
for attach in message.attachments:
if not attach:
continue
attach_name = attach['filename'][1:len(attach['filename'])-1] # remove quote marks from filename
if re.search(self.parameters.attach_regex, attach_name):
self.logger.info("Parsing attachment")
if self.parameters.attach_unzip:
zipped = zipfile.ZipFile(attach['content'])
raw_report = zipped.read(zipped.namelist()[0])
else:
raw_report = attach['content'].read()
self.logger.info('content read')
report = Report()
report.add("raw", raw_report, sanitize=True)
report.add("feed.name", self.parameters.feed,sanitize=True)
report.add("feed.accuracy", self.parameters.accuracy, sanitize=True)
time_observation = DateTime().generate_datetime_now()
#report.add('time.observation', time_observation)
report.add('feed.reportname', message.subject, sanitize=True)
self.logger.info('rocking in a free world')
self.send_message(report)
self.logger.info('just some administration left')
mailbox.mark_seen(uid)
self.logger.info("Email report read")
else:
# If no attachment, read from url
# update way of fetching from url to new way in http/
self.logger.info("No attachment found, trying collecting from URL")
for body in message.body['plain']:
self.logger.info("Parsing message body")
match = re.search(self.parameters.url_regex, body)
if match:
url = match.group()
self.logger.info("Downloading report from %s" % url)
resp = requests.get(url=url)
if resp.status_code // 100 != 2:
raise ValueError('HTTP response status code was {}.' ''.format(resp.status_code))
raw_report = resp.content
self.logger.info("Report downloaded.")
report = Report()
report.add("raw", raw_report, sanitize=True)
report.add("feed.name", self.parameters.feed, sanitize=True)
report.add("feed.accuracy", self.parameters.accuracy, sanitize=True)
self.logger.info("all is well sir")
time_observation = DateTime().generate_datetime_now()
#report.add('time.observation', time_observation, sanitize=True)
report.add('feed.reportname', message.subject, sanitize=True)
self.send_message(report)
mailbox.mark_seen(uid)
self.logger.info("Email report read")
except:
self.logger.info("ERROR with the collector ---")
