def parse_line(self, line, report):
if line.startswith('#') or len(line) == 0:
self.tempdata.append(line)
else:
event = Event(report)
line_contents = line.split('|')
if len(line_contents) != len(self.FILE_FORMAT) + 1:
raise ValueError('Incorrect format for feed {}, found line: "{}"'.format(event.get('feed.url'), line))
if line_contents[-1].strip() in self.CATEGORY:
event.update(self.CATEGORY[line_contents[-1].strip()])
else:
raise ValueError('Unknown data feed {}.'.format(line_contents[-1].strip()))
for field, setter in zip(line_contents, self.FILE_FORMAT):
value = setter[1](field.strip())
if value is not None:
event.add(setter[0], value)
event.add('raw', line)
yield event
def process(self):
report = self.receive_message()
columns = self.parameters.columns
if not report or not report.contains("raw"):
self.acknowledge_message()
return
if report:
raw_report = utils.base64_decode(report.get("raw"))
rows = csv.DictReader(StringIO(raw_report))
for row in rows:
event = Event(report)
for key, value in row.items():
key = columns[key]
if not value:
continue
value = value.strip()
if key == u'__IGNORE__' or key == u'__TBD__':
continue
# set timezone explicitly to UTC as it is absent in the input
if key == "time.source":
value += " UTC"
if "hash" in key:
key = key.replace(':','')
#if "reverse_dns" in key:
#try:
# ipaddress.ip_address(value)
# continue
#except:
# pass
if key == "destination.geolocation.cc" and (value == "**" or value =="--" or value == "??"):
continue
#There are a lot of non harmonizable values in reverse_dns, i decided to ignore them when they are wrong.
if "reverse_dns" in key:
try:
event.add(key, value, sanitize= True)
except:
continue
else:
event.add(key, value, sanitize= True)
event.add('classification.type', u'vulnerable service')
self.addextraparams(event, event.get('feed.name'))
self.send_message(event)
self.acknowledge_message()
def process(self):
report = self.receive_message()
columns = self.parameters.columns
if not report or not report.contains("raw"):
self.acknowledge_message()
return
if report:
raw_report = utils.base64_decode(report.get("raw"))
rows = csv.DictReader(StringIO(raw_report))
for row in rows:
event = Event(report)
for key, value in row.items():
key = columns[key]
if not value:
continue
value = value.strip()
if key == u'__IGNORE__' or key == u'__TBD__':
continue
# set timezone explicitly to UTC as it is absent in the input
if key == "time.source":
value += " UTC"
if "hash" in key:
key = key.replace(':', '')
#if "reverse_dns" in key:
#try:
# ipaddress.ip_address(value)
# continue
#except:
# pass
if key == "destination.geolocation.cc" and (
value == "**" or value == "--" or value == "??"):
continue
#There are a lot of non harmonizable values in reverse_dns, i decided to ignore them when they are wrong.
if "reverse_dns" in key:
try:
event.add(key, value, sanitize=True)
except:
continue
else:
event.add(key, value, sanitize=True)
event.add('classification.type', u'vulnerable service')
self.addextraparams(event, event.get('feed.name'))
self.send_message(event)
self.acknowledge_message()
