def main():
s = get_args()
format_ = '%d-%m-%Y'
for day_counter in range(s.days_back):
until_param, until_param_string, since_param, since_param_string = \
utils.get_time_params(s.end_date, day_counter, format_)
output_file = 'threat_descriptors_' + since_param_string + '_to_' + \
until_param_string + '.csv'
with open(output_file,'wb') as fout:
writer = csv.writer(fout)
results = ThreatDescriptor.objects(
fields=ThreatDescriptor._fields,
include_expired=s.include_expired,
limit=1000,
max_confidence=s.max_confidence,
min_confidence=s.min_confidence,
owner=s.owner,
text=s.text,
review_status=s.review_status,
share_level=s.share_level,
status=s.status,
strict_text=s.strict_text,
threat_type=s.threat_type,
type_=s.type,
since=since_param_string,
until=until_param_string,
)
fields_list = [
TD.ID,
TD.ADDED_ON,
TD.CONFIDENCE,
TD.DESCRIPTION,
TD.EXPIRED_ON,
[TD.INDICATOR, TI.INDICATOR],
[TD.INDICATOR, TI.TYPE],
[TD.INDICATOR, TI.ID],
TD.LAST_UPDATED,
[TD.OWNER, TE.ID],
[TD.OWNER, TE.NAME],
[TD.OWNER, TE.EMAIL],
TD.PRECISION,
TD.RAW_INDICATOR,
TD.REVIEW_STATUS,
TD.SEVERITY,
TD.SHARE_LEVEL,
TD.STATUS,
TD.THREAT_TYPE,
]
# Headers
writer.writerow(map(utils.convert_to_header,fields_list))
for result in results:
writer.writerow(
map(lambda x: utils.get_data_field(x, result), fields_list)
)
def main():
s = get_args()
format_ = '%d-%m-%Y'
for day_counter in range(s.days_back):
until_param, until_param_string, since_param, since_param_string = \
utils.get_time_params(s.end_date, day_counter, format_)
output_file = 'threat_descriptors_' + since_param_string + '_to_' + \
until_param_string + '.csv'
with open(output_file, 'wb') as fout:
writer = csv.writer(fout)
results = ThreatDescriptor.objects(
fields=ThreatDescriptor._fields,
include_expired=s.include_expired,
limit=1000,
max_confidence=s.max_confidence,
min_confidence=s.min_confidence,
owner=s.owner,
text=s.text,
review_status=s.review_status,
share_level=s.share_level,
status=s.status,
strict_text=s.strict_text,
threat_type=s.threat_type,
type_=s.type,
since=since_param_string,
until=until_param_string,
)
fields_list = [
TD.ID,
TD.ADDED_ON,
TD.CONFIDENCE,
TD.DESCRIPTION,
TD.EXPIRED_ON,
[TD.INDICATOR, TI.INDICATOR],
[TD.INDICATOR, TI.TYPE],
[TD.INDICATOR, TI.ID],
TD.LAST_UPDATED,
[TD.OWNER, TE.ID],
[TD.OWNER, TE.NAME],
[TD.OWNER, TE.EMAIL],
TD.PRECISION,
TD.RAW_INDICATOR,
TD.REVIEW_STATUS,
TD.SEVERITY,
TD.SHARE_LEVEL,
TD.STATUS,
TD.THREAT_TYPE,
]
# Headers
writer.writerow(map(utils.convert_to_header, fields_list))
for result in results:
writer.writerow(
map(lambda x: utils.get_data_field(x, result),
fields_list))
def perform_feed_retrieval(self):
new_feed = self.create_feed()
tx_limit = self.bridge_options.get('tx_request_limit', 500) or 500
tx_retries = self.bridge_options.get('tx_request_retries', 5) or 5
now = datetime.utcnow()
since_date = now - timedelta(days=self.bridge_options["historical_days"])
since_date_str = since_date.strftime("%Y-%m-%d")
until_date = since_date
while until_date < now + timedelta(1):
until_date += timedelta(days=1)
until_date_str = until_date.strftime("%Y-%m-%d")
for ioc_type in self.bridge_options["ioc_types"]:
self.logger.info("Pulling %s IOCs (%s to %s)" % (ioc_type, since_date_str, until_date_str))
try:
count = 0
for result in ThreatDescriptor.objects(since=since_date_str,
until=until_date_str,
type_=ioc_type,
dict_generator=True,
limit=tx_limit,
retries=tx_retries,
fields="raw_indicator,owner,indicator{id,indicator},type,last_updated,share_level,severity,description,report_urls,status,confidence"):
new_reports = processing_engines.process_ioc(ioc_type,
result,
minimum_severity=self.bridge_options["minimum_severity"],
status_filter=self.bridge_options["status_filter"],
minimum_confidence=self.bridge_options["minimum_confidence"])
for report in new_reports:
new_feed.add_report(report)
count += 1
if count % 1000 == 0:
self.logger.info("%s added %d reports for this iteration" % (ioc_type, count))
self.logger.info("%s added %d reports TOTAL" % (ioc_type, count))
except pytxFetchError:
self.logger.warning("Could not retrieve some IOCs of type %s. Continuing." % ioc_type)
except Exception:
self.logger.exception("Unknown exception retrieving IOCs of type %s." % ioc_type)
# update the start date
since_date_str = until_date_str
with self.feed_lock:
self.feed = new_feed
def update(self, ioc_types, tx_limit=250, tx_retries=5, max_historical_days=2):
now = datetime.datetime.utcnow()
since_date = now - datetime.timedelta(days=max_historical_days)
to_date = now + datetime.timedelta(days=1)
try:
cur = self.dbconn.execute("SELECT max(last_updated) FROM indicator_descriptors")
(s, ) = cur.fetchone()
if s:
since_date = dateutil.parser.parse(s)
except sqlite3.OperationalError:
self.create_tables()
# clamp maximum historical time
if now - since_date > datetime.timedelta(days=max_historical_days):
logger.info("Clamping maximum historical query to %d days." % max_historical_days)
since_date = now - datetime.timedelta(days=max_historical_days)
since_date = since_date.strftime("%Y-%m-%dT%H:%M:%S+0000")
to_date = to_date.strftime("%Y-%m-%d")
for ioc_type in ioc_types:
try:
logger.info("Querying ThreatExchange for %s indicators from %s to %s" % (ioc_type, since_date, to_date))
for result in ThreatDescriptor.objects(since=since_date, until=to_date, type_=ioc_type,
limit=tx_limit, retries=tx_retries,
fields="raw_indicator,owner,indicator{id,indicator},type," +
"last_updated,share_level,severity,description," +
"report_urls,status,confidence,threat_type",
headers=headers):
cur = self.dbconn.cursor()
try:
add_one_result(cur, result)
except Exception as e:
logger.exception("Exception processing threat descriptor: %s" % result.to_dict())
self.dbconn.rollback()
else:
self.dbconn.commit()
owner_cache.add(result.owner["id"])
finally:
cur.close()
except pytxFetchError:
logger.warning("Could not retrieve some IOCs of type %s. Continuing." % ioc_type)
except Exception as e:
logger.exception("Unknown exception retrieving IOCs of type %s." % ioc_type)
def import_object(request, type_, id_):
setup_access()
if type_ == "Threat Descriptors":
obj = ThreatDescriptor(id=id_)
obj.details(
fields=[f for f in ThreatDescriptor._default_fields if f not in
(td.PRIVACY_MEMBERS, td.METADATA)]
)
itype = get_mapped_itype(obj.get(td.TYPE))
tags = obj.get(td.TAGS)
if itype is None:
return {'success': False,
'message': "Descriptor type is not supported by CRITs"}
results = handle_indicator_ind(
obj.get(td.RAW_INDICATOR),
"ThreatExchange",
itype,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
request.user.username,
method="ThreatExchange Service",
reference="id: %s, owner: %s, share_level: %s" % (obj.get(td.ID),
obj.get(td.OWNER)['name'],
obj.get(td.SHARE_LEVEL)),
add_domain=True,
add_relationship=True,
confidence=build_ci(obj.get(td.CONFIDENCE)),
description=obj.get(td.DESCRIPTION),
bucket_list=tags
)
return results
elif type_ == "Malware Analyses":
obj = Malware(id=id_)
obj.details(
fields=[f for f in Malware._fields if f not in
(m.METADATA)]
)
filename = obj.get(m.MD5)
tags = obj.get(m.TAGS)
try:
data = obj.rf
except:
data = None
results = handle_file(
filename,
data,
"ThreatExchange",
method="ThreatExchange Service",
reference="id: %s, share_level: %s" % (obj.get(td.ID),
obj.get(td.SHARE_LEVEL)),
user=request.user.username,
md5_digest = obj.get(m.MD5),
sha1_digest = obj.get(m.SHA1),
sha256_digest = obj.get(m.SHA256),
size = obj.get(m.SAMPLE_SIZE),
mimetype = obj.get(m.SAMPLE_TYPE),
bucket_list=tags,
)
return {'success': True,
'md5': results}
else:
return {'success': False,
'message': "Invalid Type"}
return {'success': True}
def import_object(request, type_, id_):
setup_access()
if type_ == "Threat Descriptors":
obj = ThreatDescriptor(id=id_)
obj.details(fields=[
f for f in ThreatDescriptor._default_fields
if f not in (td.PRIVACY_MEMBERS, td.METADATA)
])
itype = get_mapped_itype(obj.get(td.TYPE))
if itype is None:
return {
'success': False,
'message': "Descriptor type is not supported by CRITs"
}
ithreat_type = getattr(IndicatorThreatTypes, obj.get(td.THREAT_TYPE))
results = handle_indicator_ind(
obj.get(td.RAW_INDICATOR),
"ThreatExchange",
itype,
ithreat_type,
None,
request.user.username,
method="ThreatExchange Service",
reference="id: %s, owner: %s, share_level: %s" % (obj.get(
td.ID), obj.get(td.OWNER)['name'], obj.get(td.SHARE_LEVEL)),
add_domain=True,
add_relationship=True,
confidence=build_ci(obj.get(td.CONFIDENCE)),
description=obj.get(td.DESCRIPTION))
return results
elif type_ == "Malware Analyses":
obj = Malware(id=id_)
obj.details(
fields=[f for f in Malware._fields if f not in (m.METADATA)])
filename = obj.get(m.MD5)
try:
data = obj.rf
except:
data = None
results = handle_file(
filename,
data,
"ThreatExchange",
method="ThreatExchange Service",
reference="id: %s, share_level: %s" %
(obj.get(td.ID), obj.get(td.SHARE_LEVEL)),
user=request.user.username,
md5_digest=obj.get(m.MD5),
sha1_digest=obj.get(m.SHA1),
sha256_digest=obj.get(m.SHA256),
size=obj.get(m.SAMPLE_SIZE),
mimetype=obj.get(m.SAMPLE_TYPE),
)
return {'success': True, 'md5': results}
else:
return {'success': False, 'message': "Invalid Type"}
return {'success': True}
def post(args):
response = ThreatDescriptor.new(vars(args))
print(response)
def post(args):
response = ThreatDescriptor.new(vars(args))
print(response)
