def main():
s = get_args()
format_ = '%d-%m-%Y'
for day_counter in range(s.days_back):
until_param, until_param_string, since_param, since_param_string = \
utils.get_time_params(s.end_date, day_counter, format_)
output_file = 'threat_descriptors_' + since_param_string + '_to_' + \
until_param_string + '.csv'
with open(output_file,'wb') as fout:
writer = csv.writer(fout)
results = ThreatDescriptor.objects(
fields=ThreatDescriptor._fields,
include_expired=s.include_expired,
limit=1000,
max_confidence=s.max_confidence,
min_confidence=s.min_confidence,
owner=s.owner,
text=s.text,
review_status=s.review_status,
share_level=s.share_level,
status=s.status,
strict_text=s.strict_text,
threat_type=s.threat_type,
type_=s.type,
since=since_param_string,
until=until_param_string,
)
fields_list = [
TD.ID,
TD.ADDED_ON,
TD.CONFIDENCE,
TD.DESCRIPTION,
TD.EXPIRED_ON,
[TD.INDICATOR, TI.INDICATOR],
[TD.INDICATOR, TI.TYPE],
[TD.INDICATOR, TI.ID],
TD.LAST_UPDATED,
[TD.OWNER, TE.ID],
[TD.OWNER, TE.NAME],
[TD.OWNER, TE.EMAIL],
TD.PRECISION,
TD.RAW_INDICATOR,
TD.REVIEW_STATUS,
TD.SEVERITY,
TD.SHARE_LEVEL,
TD.STATUS,
TD.THREAT_TYPE,
]
# Headers
writer.writerow(map(utils.convert_to_header,fields_list))
for result in results:
writer.writerow(
map(lambda x: utils.get_data_field(x, result), fields_list)
)
def main():
s = get_args()
format_ = '%d-%m-%Y'
for day_counter in range(s.days_back):
until_param, until_param_string, since_param, since_param_string = \
utils.get_time_params(s.end_date, day_counter, format_)
output_file = 'threat_descriptors_' + since_param_string + '_to_' + \
until_param_string + '.csv'
with open(output_file, 'wb') as fout:
writer = csv.writer(fout)
results = ThreatDescriptor.objects(
fields=ThreatDescriptor._fields,
include_expired=s.include_expired,
limit=1000,
max_confidence=s.max_confidence,
min_confidence=s.min_confidence,
owner=s.owner,
text=s.text,
review_status=s.review_status,
share_level=s.share_level,
status=s.status,
strict_text=s.strict_text,
threat_type=s.threat_type,
type_=s.type,
since=since_param_string,
until=until_param_string,
)
fields_list = [
TD.ID,
TD.ADDED_ON,
TD.CONFIDENCE,
TD.DESCRIPTION,
TD.EXPIRED_ON,
[TD.INDICATOR, TI.INDICATOR],
[TD.INDICATOR, TI.TYPE],
[TD.INDICATOR, TI.ID],
TD.LAST_UPDATED,
[TD.OWNER, TE.ID],
[TD.OWNER, TE.NAME],
[TD.OWNER, TE.EMAIL],
TD.PRECISION,
TD.RAW_INDICATOR,
TD.REVIEW_STATUS,
TD.SEVERITY,
TD.SHARE_LEVEL,
TD.STATUS,
TD.THREAT_TYPE,
]
# Headers
writer.writerow(map(utils.convert_to_header, fields_list))
for result in results:
writer.writerow(
map(lambda x: utils.get_data_field(x, result),
fields_list))
def perform_feed_retrieval(self):
new_feed = self.create_feed()
tx_limit = self.bridge_options.get('tx_request_limit', 500) or 500
tx_retries = self.bridge_options.get('tx_request_retries', 5) or 5
now = datetime.utcnow()
since_date = now - timedelta(days=self.bridge_options["historical_days"])
since_date_str = since_date.strftime("%Y-%m-%d")
until_date = since_date
while until_date < now + timedelta(1):
until_date += timedelta(days=1)
until_date_str = until_date.strftime("%Y-%m-%d")
for ioc_type in self.bridge_options["ioc_types"]:
self.logger.info("Pulling %s IOCs (%s to %s)" % (ioc_type, since_date_str, until_date_str))
try:
count = 0
for result in ThreatDescriptor.objects(since=since_date_str,
until=until_date_str,
type_=ioc_type,
dict_generator=True,
limit=tx_limit,
retries=tx_retries,
fields="raw_indicator,owner,indicator{id,indicator},type,last_updated,share_level,severity,description,report_urls,status,confidence"):
new_reports = processing_engines.process_ioc(ioc_type,
result,
minimum_severity=self.bridge_options["minimum_severity"],
status_filter=self.bridge_options["status_filter"],
minimum_confidence=self.bridge_options["minimum_confidence"])
for report in new_reports:
new_feed.add_report(report)
count += 1
if count % 1000 == 0:
self.logger.info("%s added %d reports for this iteration" % (ioc_type, count))
self.logger.info("%s added %d reports TOTAL" % (ioc_type, count))
except pytxFetchError:
self.logger.warning("Could not retrieve some IOCs of type %s. Continuing." % ioc_type)
except Exception:
self.logger.exception("Unknown exception retrieving IOCs of type %s." % ioc_type)
# update the start date
since_date_str = until_date_str
with self.feed_lock:
self.feed = new_feed
def update(self, ioc_types, tx_limit=250, tx_retries=5, max_historical_days=2):
now = datetime.datetime.utcnow()
since_date = now - datetime.timedelta(days=max_historical_days)
to_date = now + datetime.timedelta(days=1)
try:
cur = self.dbconn.execute("SELECT max(last_updated) FROM indicator_descriptors")
(s, ) = cur.fetchone()
if s:
since_date = dateutil.parser.parse(s)
except sqlite3.OperationalError:
self.create_tables()
# clamp maximum historical time
if now - since_date > datetime.timedelta(days=max_historical_days):
logger.info("Clamping maximum historical query to %d days." % max_historical_days)
since_date = now - datetime.timedelta(days=max_historical_days)
since_date = since_date.strftime("%Y-%m-%dT%H:%M:%S+0000")
to_date = to_date.strftime("%Y-%m-%d")
for ioc_type in ioc_types:
try:
logger.info("Querying ThreatExchange for %s indicators from %s to %s" % (ioc_type, since_date, to_date))
for result in ThreatDescriptor.objects(since=since_date, until=to_date, type_=ioc_type,
limit=tx_limit, retries=tx_retries,
fields="raw_indicator,owner,indicator{id,indicator},type," +
"last_updated,share_level,severity,description," +
"report_urls,status,confidence,threat_type",
headers=headers):
cur = self.dbconn.cursor()
try:
add_one_result(cur, result)
except Exception as e:
logger.exception("Exception processing threat descriptor: %s" % result.to_dict())
self.dbconn.rollback()
else:
self.dbconn.commit()
owner_cache.add(result.owner["id"])
finally:
cur.close()
except pytxFetchError:
logger.warning("Could not retrieve some IOCs of type %s. Continuing." % ioc_type)
except Exception as e:
logger.exception("Unknown exception retrieving IOCs of type %s." % ioc_type)