def collect_events(helper, ew):
"""Implement your data collection logic here """
opt_imap_server = helper.get_arg("imap_server")
opt_imap_mailbox = helper.get_arg("imap_mailbox")
opt_use_ssl = True
opt_global_account = helper.get_arg('global_account')
opt_resolve_ip = helper.get_arg('resolve_ip')
opt_validate_xml = helper.get_arg('validate_xml')
opt_validate_dkim = helper.get_arg('validate_dkim')
opt_output_format = helper.get_arg('output_format')
opt_batch_size = int(helper.get_arg('batch_size'))
loglevel = helper.get_log_level()
helper.set_log_level(loglevel)
tmp_dir = create_tmp_dir(helper)
i2d = Imap2Dir(helper, opt_imap_server, tmp_dir, opt_use_ssl,
opt_global_account, opt_imap_mailbox, opt_validate_dkim,
opt_batch_size)
try:
filelist = i2d.process_incoming()
if len(filelist) > 0:
d2s = Dir2Splunk(ew, helper, tmp_dir, 0, opt_resolve_ip,
opt_validate_xml, opt_output_format, False)
if d2s.check_dir():
d2s.process_incoming()
finally:
remove_tmp_dir(helper, tmp_dir)
def process_incoming(self):
""" Processes the main incoming directory
"""
self.helper.log_info(
"Start processing incoming directory %s with %d quiet_secs" %
(self.dir, self.quiet_secs))
try:
self.check_dir()
self.tmp_dir = create_tmp_dir(self.helper)
fileslist = self.filter_quiet_files(self.list_incoming())
if self.do_checkpoint:
fileslist = self.filter_seen_files(fileslist)
for file in fileslist:
ext = os.path.splitext(file)[1]
if ext == ".zip":
self.helper.log_info("Start processing zip file %s" % file)
for xmlfile in self.process_zipfile(file):
xmlfile = self.fix_xml_encoding(xmlfile)
if self.is_valid_rua_xmlfile(xmlfile):
lines = self.process_xmlfile(xmlfile)
self.write_event(lines)
else:
self.helper.log_debug(
"process_incoming: ignoring invalid xml file %s from %s"
% (xmlfile, file))
elif ext == ".gz":
self.helper.log_info("Start processing gz file %s" % file)
for xmlfile in self.process_gzfile(file):
xmlfile = self.fix_xml_encoding(xmlfile)
if self.is_valid_rua_xmlfile(xmlfile):
lines = self.process_xmlfile(xmlfile)
self.write_event(lines)
else:
self.helper.log_debug(
"process_incoming: ignoring invalid xml file %s from %s"
% (xmlfile, file))
elif ext == ".xml":
self.helper.log_info("Start processing xml file %s" % file)
file = self.fix_xml_encoding(file)
if self.is_valid_rua_xmlfile(file):
lines = self.process_xmlfile(file)
self.write_event(lines)
else:
self.helper.log_debug(
"process_incoming: ignoring invalid xml file %s" %
file)
else:
self.helper.log_debug(
"process_incoming: Ignoring file %s" % file)
if self.do_checkpoint:
self.save_check_point(file)
finally:
self.helper.log_info("Ended processing incoming directory %s" %
self.dir)
remove_tmp_dir(self.helper, self.tmp_dir)
self.helper.log_debug("process_incoming: removed tmp_dir %s" %
self.tmp_dir)
def validate_input(helper, definition):
opt_pop3_server = definition.parameters.get("pop3_server", None)
opt_use_ssl = True
opt_global_account = definition.parameters.get('global_account', None)
opt_validate_dkim = definition.parameters.get('validate_dkim', None)
try:
tmp_dir = create_tmp_dir(helper)
p2d = Pop2Dir(helper, opt_pop3_server, tmp_dir, opt_use_ssl, opt_global_account, opt_validate_dkim)
p2d.get_pop3_connectivity()
finally:
remove_tmp_dir(helper, tmp_dir)
def validate_input(helper, definition):
"""Implement your own validation logic to validate the input stanza configurations"""
opt_pop3_server = definition.parameters.get("pop3_server", None)
opt_use_ssl = True
opt_global_account = definition.parameters.get('global_account', None)
opt_validate_dkim = definition.parameters.get('validate_dkim', None)
try:
tmp_dir = create_tmp_dir(helper)
p2d = Pop2Dir(helper, opt_pop3_server, tmp_dir, opt_use_ssl, opt_global_account, opt_validate_dkim)
p2d.get_pop3_connectivity()
finally:
remove_tmp_dir(helper, tmp_dir)
def validate_input(helper, definition):
opt_imap_server = definition.parameters.get("imap_server", None)
opt_imap_mailbox = definition.parameters.get("imap_mailbox", None)
opt_use_ssl = True
opt_global_account = definition.parameters.get('global_account', None)
opt_validate_dkim = definition.parameters.get('validate_dkim', None)
opt_batch_size = int(definition.parameters.get('batch_size', None))
try:
tmp_dir = create_tmp_dir(helper)
i2d = Imap2Dir(helper, opt_imap_server, tmp_dir, opt_use_ssl,
opt_global_account, opt_imap_mailbox, opt_validate_dkim,
opt_batch_size)
i2d.get_imap_connectivity()
finally:
remove_tmp_dir(helper, tmp_dir)
def collect_events(helper, ew):
opt_imap_server = helper.get_arg("imap_server")
opt_imap_mailbox = helper.get_arg("imap_mailbox")
opt_use_ssl = True
opt_global_account = helper.get_arg('global_account')
opt_resolve_ip = helper.get_arg('resolve_ip')
opt_validate_xml = helper.get_arg('validate_xml')
opt_validate_dkim = helper.get_arg('validate_dkim')
opt_output_format = helper.get_arg('output_format')
opt_batch_size = int(helper.get_arg('batch_size'))
loglevel = helper.get_log_level()
helper.set_log_level(loglevel)
tmp_dir = create_tmp_dir(helper)
i2d = Imap2Dir(helper, opt_imap_server, tmp_dir, opt_use_ssl,
opt_global_account, opt_imap_mailbox, opt_validate_dkim,
opt_batch_size)
try:
filelist = i2d.process_incoming()
if len(filelist) > 0:
d2s = Dir2Splunk(ew, helper, tmp_dir, 0, opt_resolve_ip,
opt_validate_xml, opt_output_format, False)
if d2s.check_dir():
d2s.process_incoming()
finally:
remove_tmp_dir(helper, tmp_dir)
# PSEUDOCODE for refactor:
#
# mailbox = DMARCMailbox(imap, ssl, account)
# for uid, message in mailbox.get_dmarc_messages()
# mail = DMARCMail(message)
# dkimvrfy = mail.dkim_verify()
# for file in mail.get_dmarc_attachments()
# rua = DMARCfile(file)
# res_xmlvalidation = rua.get_xml_validation()
# res_feedback = rua.get_rua_feedback()
# event = DMARCEvent(res_feedback, res_xmlvalidation, dkimvrfy)
# event.save_event()
# mailbox.save_checkpoint(uid)