def purge_database():
"""
Archive old documents
"""
with mongo.Mongo() as database:
database.purge_old_documents()
sys.exit(0)
def get_reputation_events_for_source(addr, source, start_date):
"""
Get reputation events with full data (raw data included) for
a given ip and a given source.
:param str addr: Ip the reputation must be computed with
:param str source: Source short name to get events of
:param int start_date: Timestamp the events must be retrieved from
:rtype: array
:return: Array of events
"""
with mongo.Mongo() as database:
events = database.find_all_event_data_for_ip(addr, start_date, True)
result = [
event for event in events
if event['source'] == _map_source_from_shortname(source)
]
# Find the first data to determine whether data are b64 encoded or not.
is_encoded = False
for event in result:
if event['data']:
is_encoded = utils.is_base64_encoded(event['data'])
break
# If data are encoded, then decode all
if is_encoded:
for event in result:
event['data'] = event['data'].decode(
'base64') if event['data'] else event['data']
return result
def update_db(documents):
"""
Update MongoDB by inserting new entries or updating existing ones (last seen
date, is it still active, etc.)
:param tuple documents: A tuple of dict containing documents to upsert.
"""
with mongo.Mongo() as database:
database.update_spamhaus_entries(documents)
def send_reports(self):
"""
The only public method used to run the process of email sending.
"""
with mongo.Mongo() as database:
for entry in database.find_highest_scores():
subject = self._prepare_subject(entry['_id'], entry['value'])
raw = self._prepare_raw(database, entry['_id'])
body = self._prepare_body(entry['_id'], entry['value'], raw)
self._send_mail(subject, body)
def get_reputation_events_for_source(addr, source, start_date):
"""
Get reputation events with full data (raw data included) for
a given ip and a given source.
:param str addr: Ip the reputation must be computed with
:param int start_date: Timestamp the events must be retrieved from
:rtype: array
:return: Array of events
"""
with mongo.Mongo() as database:
events = database.find_all_event_data_for_ip(addr, start_date, True)
return [event for event in events if event['source'] == source]
def run(self):
"""
Run the parser.
"""
with mongo.Mongo() as database:
current = self.next()
while current:
try:
addr = self.get_ip(current)
if not addr:
LOGGER.info('Entry skipped because no specified IP.')
current = self.next()
continue
if not utils.is_managed_ip(addr):
LOGGER.debug('Not a managed IP [%s].', addr)
current = self.next()
continue
doc_ts = int(
time.mktime(self.get_date(current).timetuple()))
if doc_ts < YESTERDAY:
LOGGER.debug('This entry is too old [%s].',
self.get_date(current))
current = self.next()
continue
document = {
'ip': addr,
'timestamp': doc_ts,
'weight': self.compute_weight(current),
'source': self.get_source(current),
'raw': self.get_raw(current)
}
database.push_ip_document(document)
except Exception as exc:
LOGGER.error('Unexpected error: %s [%s]', type(exc),
exc.message)
LOGGER.error(traceback.format_exc())
current = self.next()
self.close()
def get_spamhaus_entries(is_active):
"""
Retrive from mongo DB spamhaus entry, active or not.
"""
# Query mongo
with mongo.Mongo() as database:
documents = database.find_spamhaus_entries(is_active)
if not documents:
return None
# Format dto
result = []
for doc in documents:
result.append({
'sblNumber': doc['sbl_number'],
'cidr': doc['cidr'],
'firstSeen': int(mktime(doc['first_seen'].timetuple())),
'lastSeen': int(mktime(doc['last_seen'].timetuple()))
})
return result
def aggregate_reputation_per_source(addr, start_date):
"""
Aggregate ip reputation per source returning for each source
the sum of the weights.
:param str addr: Ip the reputation must be computed with
:param int start_date: Timestamp the events must be retrieved from
:rtype: dict
:return: Dictionnary that gives for each source, the aggregated
weight
"""
with mongo.Mongo() as database:
events = database.find_all_events_for_ip(addr, start_date, True)
# Reduce by source
scores_by_source = _compute_score_by_source(events)
# Append sources which are missing in scores_by_source (no attached events)
for parser in parsers.keys():
if parser not in scores_by_source.keys():
scores_by_source[parser] = 0
# Format final dto
result = []
for source in scores_by_source.keys():
if source not in shortened_names.keys():
short_name = source
else:
short_name = shortened_names[source]
result.append({
'short_name': short_name,
'full_name': source,
'result': scores_by_source[source],
})
return result