def main():
s = get_args()
format_ = '%d-%m-%Y'
for day_counter in range(s.days_back):
until_param, until_param_string, since_param, since_param_string = \
utils.get_time_params(s.end_date, day_counter, format_)
output_file = 'threat_indicators_' + since_param_string + '_to_' + \
until_param_string + '.csv'
with open(output_file, 'wb') as fout:
writer = csv.writer(fout)
results = ThreatIndicator.objects(
fields=ThreatIndicator._fields,
limit=1000,
text=s.text,
strict_text=s.strict_text,
threat_type=s.threat_type,
type_=s.type,
since=since_param_string,
until=until_param_string,
)
fields_list = [
TI.ID,
TI.INDICATOR,
TI.TYPE,
]
# Headers
writer.writerow(map(utils.convert_to_header, fields_list))
for result in results:
writer.writerow(
map(lambda x: utils.get_data_field(x, result),
fields_list))
def main():
s = get_args()
format_ = '%d-%m-%Y'
for day_counter in range(s.days_back):
until_param, until_param_string, since_param, since_param_string = \
utils.get_time_params(s.end_date, day_counter, format_)
output_file = 'threat_indicators_' + since_param_string + '_to_' + \
until_param_string + '.csv'
with open(output_file,'wb') as fout:
writer = csv.writer(fout)
results = ThreatIndicator.objects(
fields=ThreatIndicator._fields,
limit=1000,
text=s.text,
strict_text=s.strict_text,
threat_type=s.threat_type,
type_=s.type,
since=since_param_string,
until=until_param_string,
)
fields_list = [
TI.ID,
TI.INDICATOR,
TI.TYPE,
]
# Headers
writer.writerow(map(utils.convert_to_header,fields_list))
for result in results:
writer.writerow(
map(lambda x: utils.get_data_field(x, result), fields_list)
)
def main():
s = get_args()
format_ = '%d-%m-%Y'
for day_counter in range(s.days_back):
until_param, until_param_string, since_param, since_param_string = \
utils.get_time_params(s.end_date, day_counter, format_)
output_file = 'threat_descriptors_' + since_param_string + '_to_' + \
until_param_string + '.csv'
with open(output_file,'wb') as fout:
writer = csv.writer(fout)
results = ThreatDescriptor.objects(
fields=ThreatDescriptor._fields,
include_expired=s.include_expired,
limit=1000,
max_confidence=s.max_confidence,
min_confidence=s.min_confidence,
owner=s.owner,
text=s.text,
review_status=s.review_status,
share_level=s.share_level,
status=s.status,
strict_text=s.strict_text,
threat_type=s.threat_type,
type_=s.type,
since=since_param_string,
until=until_param_string,
)
fields_list = [
TD.ID,
TD.ADDED_ON,
TD.CONFIDENCE,
TD.DESCRIPTION,
TD.EXPIRED_ON,
[TD.INDICATOR, TI.INDICATOR],
[TD.INDICATOR, TI.TYPE],
[TD.INDICATOR, TI.ID],
TD.LAST_UPDATED,
[TD.OWNER, TE.ID],
[TD.OWNER, TE.NAME],
[TD.OWNER, TE.EMAIL],
TD.PRECISION,
TD.RAW_INDICATOR,
TD.REVIEW_STATUS,
TD.SEVERITY,
TD.SHARE_LEVEL,
TD.STATUS,
TD.THREAT_TYPE,
]
# Headers
writer.writerow(map(utils.convert_to_header,fields_list))
for result in results:
writer.writerow(
map(lambda x: utils.get_data_field(x, result), fields_list)
)
def main():
s = get_args()
format_ = '%d-%m-%Y'
for day_counter in range(s.days_back):
until_param, until_param_string, since_param, since_param_string = \
utils.get_time_params(s.end_date, day_counter, format_)
output_file = 'threat_descriptors_' + since_param_string + '_to_' + \
until_param_string + '.csv'
with open(output_file, 'wb') as fout:
writer = csv.writer(fout)
results = ThreatDescriptor.objects(
fields=ThreatDescriptor._fields,
include_expired=s.include_expired,
limit=1000,
max_confidence=s.max_confidence,
min_confidence=s.min_confidence,
owner=s.owner,
text=s.text,
review_status=s.review_status,
share_level=s.share_level,
status=s.status,
strict_text=s.strict_text,
threat_type=s.threat_type,
type_=s.type,
since=since_param_string,
until=until_param_string,
)
fields_list = [
TD.ID,
TD.ADDED_ON,
TD.CONFIDENCE,
TD.DESCRIPTION,
TD.EXPIRED_ON,
[TD.INDICATOR, TI.INDICATOR],
[TD.INDICATOR, TI.TYPE],
[TD.INDICATOR, TI.ID],
TD.LAST_UPDATED,
[TD.OWNER, TE.ID],
[TD.OWNER, TE.NAME],
[TD.OWNER, TE.EMAIL],
TD.PRECISION,
TD.RAW_INDICATOR,
TD.REVIEW_STATUS,
TD.SEVERITY,
TD.SHARE_LEVEL,
TD.STATUS,
TD.THREAT_TYPE,
]
# Headers
writer.writerow(map(utils.convert_to_header, fields_list))
for result in results:
writer.writerow(
map(lambda x: utils.get_data_field(x, result),
fields_list))
def main():
s = get_args()
format_ = '%d-%m-%Y'
for day_counter in range(s.days_back):
until_param, until_param_string, since_param, since_param_string = \
utils.get_time_params(s.end_date, day_counter, format_)
output_file = 'malware_families_' + since_param_string + '_to_' + \
until_param_string + '.csv'
with open(output_file, 'wb') as fout:
writer = csv.writer(fout)
results = MalwareFamily.objects(
fields=MalwareFamily._fields,
limit=1000,
text=s.text,
strict_text=s.strict_text,
since=since_param_string,
until=until_param_string,
)
fields_list = [
MF.ID,
MF.ADDED_ON,
MF.ALIASES,
MF.DESCRIPTION,
MF.FAMILY_TYPE,
MF.MALICIOUS,
MF.NAME,
MF.SAMPLE_COUNT,
MF.SUBMITTER_COUNT,
]
# Headers
writer.writerow(map(utils.convert_to_header, fields_list))
for result in results:
writer.writerow(
map(lambda x: utils.get_data_field(x, result),
fields_list))
def main():
s = get_args()
format_ = '%d-%m-%Y'
for day_counter in range(s.days_back):
until_param, until_param_string, since_param, since_param_string = \
utils.get_time_params(s.end_date, day_counter, format_)
output_file = 'malware_families_' + since_param_string + '_to_' + \
until_param_string + '.csv'
with open(output_file,'wb') as fout:
writer = csv.writer(fout)
results = MalwareFamily.objects(
fields=MalwareFamily._fields,
limit=1000,
text=s.text,
strict_text=s.strict_text,
since=since_param_string,
until=until_param_string,
)
fields_list = [
MF.ID,
MF.ADDED_ON,
MF.ALIASES,
MF.DESCRIPTION,
MF.FAMILY_TYPE,
MF.MALICIOUS,
MF.NAME,
MF.SAMPLE_COUNT,
MF.SUBMITTER_COUNT,
]
# Headers
writer.writerow(map(utils.convert_to_header,fields_list))
for result in results:
writer.writerow(
map(lambda x: utils.get_data_field(x, result), fields_list)
)
def main():
s = get_args()
format_ = '%d-%m-%Y'
for day_counter in range(s.days_back):
until_param, until_param_string, since_param, since_param_string = \
utils.get_time_params(s.end_date, day_counter, format_)
output_file = 'malware_analyses_' + since_param_string + '_to_' + \
until_param_string + '.csv'
with open(output_file, 'wb') as fout:
writer = csv.writer(fout)
# TODO: Remove this once querying the fields related to a sample
# Doesn't break TX, and fix all the things below
fields = Malware._default_fields
if (s.full_sample):
fields += ['sample_size', 'sample']
results = Malware.objects(
fields=fields,
limit=1000,
sample_type=s.sample_type,
share_level=s.share_level,
text=s.text,
status=s.status,
strict_text=s.strict_text,
since=since_param_string,
until=until_param_string,
)
fields_list = [
MA.ID,
MA.ADDED_ON,
MA.CRX,
MA.IMPHASH,
MA.MD5,
MA.PASSWORD,
MA.PE_RICH_HEADER,
MA.SAMPLE_TYPE,
MA.SAMPLE_SIZE_COMPRESSED,
MA.SHA1,
MA.SHA256,
MA.SHARE_LEVEL,
MA.SSDEEP,
MA.STATUS,
MA.SUBMITTER_COUNT,
MA.VICTIM_COUNT,
MA.XPI,
]
if (s.full_sample):
fields_list += [
MA.SAMPLE,
MA.SAMPLE_SIZE,
]
# Headers
writer.writerow(map(utils.convert_to_header, fields_list))
for result in results:
writer.writerow(
map(lambda x: utils.get_data_field(x, result),
fields_list))
def query(args):
"""
Query the ThreatExchange API at the specified endpoint.
"""
# maximum number of indicators to fetch
result_limit = 1000
# write results to this stream
output_stream = '/dev/stdout' if not args.output else args.output
for day in range(args.days_back):
# format date parameters for HTTP request
until, until_str, since, since_str = utils.get_time_params(args.end_date, day, '%d-%m-%Y')
with open(output_stream, 'wb') as ostream:
print('Writing to %s...' % output_stream)
writer = csv.writer(ostream)
if args.object == 'exchange_member':
engine = ThreatExchangeMember
fields = [XM.ID, XM.NAME]
parameters = dict()
elif args.object == 'malware_analysis':
engine = Malware
fields = [
MA.ID,
MA.ADDED_ON,
MA.CRX,
MA.IMPHASH,
MA.MD5,
MA.PASSWORD,
MA.PE_RICH_HEADER,
MA.SAMPLE_TYPE,
MA.SAMPLE_SIZE_COMPRESSED,
MA.SHA1,
MA.SHA256,
MA.SHARE_LEVEL,
MA.SSDEEP,
MA.STATUS,
MA.VICTIM_COUNT,
MA.XPI,
]
param_fields = Malware._default_fields
if args.full_sample:
param_fields += ['sample_size', 'sample']
parameters = dict(
fields=param_fields,
limit=result_limit,
text=args.text,
strict_text=args.strict_text,
sample_type=args.malware_type,
status=args.status,
share_level=args.share_level,
since=since_str,
until=until_str
)
elif args.object == 'malware_family':
engine = MalwareFamily
fields = [
MF.ID,
MF.ADDED_ON,
MF.ALIASES,
MF.DESCRIPTION,
MF.FAMILY_TYPE,
MF.MALICIOUS,
MF.NAME,
MF.SAMPLE_COUNT
]
parameters = dict(
fields=MalwareFamily._fields,
limit=result_limit,
text=args.text,
strict_text=args.strict_text,
since=since_str,
until=until_str
)
elif args.object == 'threat_descriptor':
engine = ThreatDescriptor
fields = [
TD.ID,
TD.ADDED_ON,
TD.CONFIDENCE,
TD.DESCRIPTION,
TD.EXPIRED_ON,
[TD.INDICATOR, TI.INDICATOR],
[TD.INDICATOR, TI.TYPE],
[TD.INDICATOR, TI.ID],
TD.LAST_UPDATED,
[TD.OWNER, XM.ID],
[TD.OWNER, XM.NAME],
[TD.OWNER, XM.EMAIL],
TD.PRECISION,
TD.RAW_INDICATOR,
TD.REVIEW_STATUS,
TD.SEVERITY,
TD.SHARE_LEVEL,
TD.STATUS
]
parameters = dict(
fields=ThreatDescriptor._fields,
include_expired=args.include_expired,
min_confidence=args.confidence_lb,
max_confidence=args.confidence_ub,
owner=args.owner,
review_status=args.review_status,
share_level=args.share_level,
status=args.status,
limit=result_limit,
text=args.text,
strict_text=args.strict_text,
type_=args.indicator_type,
since=since_str,
until=until_str
)
elif args.object == 'threat_indicator':
engine = ThreatIndicator
fields = [TI.ID, TI.INDICATOR, TI.TYPE]
parameters = dict(
fields=ThreatIndicator._fields,
limit=result_limit,
text=args.text,
strict_text=args.strict_text,
type_=args.indicator_type,
since=since_str,
until=until_str
)
objects = engine.objects(**parameters)
headers = [utils.convert_to_header(f) for f in fields]
writer.writerow(headers)
for i, o in enumerate(objects):
data = [i] + [utils.get_data_field(f, o) for f in fields]
writer.writerow(data)
def query(args):
"""
Query the ThreatExchange API at the specified endpoint.
"""
# maximum number of indicators to fetch
result_limit = 1000
# write results to this stream
output_stream = '/dev/stdout' if not args.output else args.output
for day in range(args.days_back):
# format date parameters for HTTP request
until, until_str, since, since_str = utils.get_time_params(args.end_date, day, '%d-%m-%Y')
with open(output_stream, 'wb') as ostream:
print('Writing to %s...' % output_stream)
writer = csv.writer(ostream)
if args.object == 'exchange_member':
engine = ThreatExchangeMember
fields = [XM.ID, XM.NAME]
parameters = dict()
elif args.object == 'malware_analysis':
engine = Malware
fields = [
MA.ID,
MA.ADDED_ON,
MA.CRX,
MA.IMPHASH,
MA.MD5,
MA.PASSWORD,
MA.PE_RICH_HEADER,
MA.SAMPLE_TYPE,
MA.SAMPLE_SIZE_COMPRESSED,
MA.SHA1,
MA.SHA256,
MA.SHARE_LEVEL,
MA.SSDEEP,
MA.STATUS,
MA.VICTIM_COUNT,
MA.XPI,
]
param_fields = Malware._default_fields
if args.full_sample:
param_fields += ['sample_size', 'sample']
parameters = dict(
fields=param_fields,
limit=result_limit,
text=args.text,
strict_text=args.strict_text,
sample_type=args.malware_type,
status=args.status,
share_level=args.share_level,
since=since_str,
until=until_str
)
elif args.object == 'malware_family':
engine = MalwareFamily
fields = [
MF.ID,
MF.ADDED_ON,
MF.ALIASES,
MF.DESCRIPTION,
MF.FAMILY_TYPE,
MF.MALICIOUS,
MF.NAME,
MF.SAMPLE_COUNT
]
parameters = dict(
fields=MalwareFamily._fields,
limit=result_limit,
text=args.text,
strict_text=args.strict_text,
since=since_str,
until=until_str
)
elif args.object == 'threat_descriptor':
engine = ThreatDescriptor
fields = [
TD.ID,
TD.ADDED_ON,
TD.CONFIDENCE,
TD.DESCRIPTION,
TD.EXPIRED_ON,
[TD.INDICATOR, TI.INDICATOR],
[TD.INDICATOR, TI.TYPE],
[TD.INDICATOR, TI.ID],
TD.LAST_UPDATED,
[TD.OWNER, XM.ID],
[TD.OWNER, XM.NAME],
[TD.OWNER, XM.EMAIL],
TD.PRECISION,
TD.RAW_INDICATOR,
TD.REVIEW_STATUS,
TD.SEVERITY,
TD.SHARE_LEVEL,
TD.STATUS
]
parameters = dict(
fields=ThreatDescriptor._fields,
include_expired=args.include_expired,
min_confidence=args.confidence_lb,
max_confidence=args.confidence_ub,
owner=args.owner,
review_status=args.review_status,
share_level=args.share_level,
status=args.status,
limit=result_limit,
text=args.text,
strict_text=args.strict_text,
type_=args.indicator_type,
since=since_str,
until=until_str
)
elif args.object == 'threat_indicator':
engine = ThreatIndicator
fields = [TI.ID, TI.INDICATOR, TI.TYPE]
parameters = dict(
fields=ThreatIndicator._fields,
limit=result_limit,
text=args.text,
strict_text=args.strict_text,
type_=args.indicator_type,
since=since_str,
until=until_str
)
objects = engine.objects(**parameters)
headers = [utils.convert_to_header(f) for f in fields]
writer.writerow(headers)
for i, o in enumerate(objects):
data = [i] + [utils.get_data_field(f, o) for f in fields]
writer.writerow(data)
def main():
s = get_args()
format_ = '%d-%m-%Y'
for day_counter in range(s.days_back):
until_param, until_param_string, since_param, since_param_string = \
utils.get_time_params(s.end_date, day_counter, format_)
output_file = 'malware_analyses_' + since_param_string + '_to_' + \
until_param_string + '.csv'
with open(output_file,'wb') as fout:
writer = csv.writer(fout)
# TODO: Remove this once querying the fields related to a sample
# Doesn't break TX, and fix all the things below
fields = Malware._default_fields
if (s.full_sample):
fields += ['sample_size', 'sample']
results = Malware.objects(
fields=fields,
limit=1000,
sample_type=s.sample_type,
share_level=s.share_level,
text=s.text,
status=s.status,
strict_text=s.strict_text,
since=since_param_string,
until=until_param_string,
)
fields_list = [
MA.ID,
MA.ADDED_ON,
MA.CRX,
MA.IMPHASH,
MA.MD5,
MA.PASSWORD,
MA.PE_RICH_HEADER,
MA.SAMPLE_TYPE,
MA.SAMPLE_SIZE_COMPRESSED,
MA.SHA1,
MA.SHA256,
MA.SHARE_LEVEL,
MA.SSDEEP,
MA.STATUS,
MA.SUBMITTER_COUNT,
MA.VICTIM_COUNT,
MA.XPI,
]
if (s.full_sample):
fields_list += [
MA.SAMPLE,
MA.SAMPLE_SIZE,
]
# Headers
writer.writerow(map(utils.convert_to_header,fields_list))
for result in results:
writer.writerow(
map(lambda x: utils.get_data_field(x, result), fields_list)
)
