def main(): s = get_args() format_ = '%d-%m-%Y' for day_counter in range(s.days_back): until_param, until_param_string, since_param, since_param_string = \ utils.get_time_params(s.end_date, day_counter, format_) output_file = 'threat_indicators_' + since_param_string + '_to_' + \ until_param_string + '.csv' with open(output_file, 'wb') as fout: writer = csv.writer(fout) results = ThreatIndicator.objects( fields=ThreatIndicator._fields, limit=1000, text=s.text, strict_text=s.strict_text, threat_type=s.threat_type, type_=s.type, since=since_param_string, until=until_param_string, ) fields_list = [ TI.ID, TI.INDICATOR, TI.TYPE, ] # Headers writer.writerow(map(utils.convert_to_header, fields_list)) for result in results: writer.writerow( map(lambda x: utils.get_data_field(x, result), fields_list))
def main(): s = get_args() format_ = '%d-%m-%Y' for day_counter in range(s.days_back): until_param, until_param_string, since_param, since_param_string = \ utils.get_time_params(s.end_date, day_counter, format_) output_file = 'threat_indicators_' + since_param_string + '_to_' + \ until_param_string + '.csv' with open(output_file,'wb') as fout: writer = csv.writer(fout) results = ThreatIndicator.objects( fields=ThreatIndicator._fields, limit=1000, text=s.text, strict_text=s.strict_text, threat_type=s.threat_type, type_=s.type, since=since_param_string, until=until_param_string, ) fields_list = [ TI.ID, TI.INDICATOR, TI.TYPE, ] # Headers writer.writerow(map(utils.convert_to_header,fields_list)) for result in results: writer.writerow( map(lambda x: utils.get_data_field(x, result), fields_list) )
def main(): s = get_args() format_ = '%d-%m-%Y' for day_counter in range(s.days_back): until_param, until_param_string, since_param, since_param_string = \ utils.get_time_params(s.end_date, day_counter, format_) output_file = 'threat_descriptors_' + since_param_string + '_to_' + \ until_param_string + '.csv' with open(output_file,'wb') as fout: writer = csv.writer(fout) results = ThreatDescriptor.objects( fields=ThreatDescriptor._fields, include_expired=s.include_expired, limit=1000, max_confidence=s.max_confidence, min_confidence=s.min_confidence, owner=s.owner, text=s.text, review_status=s.review_status, share_level=s.share_level, status=s.status, strict_text=s.strict_text, threat_type=s.threat_type, type_=s.type, since=since_param_string, until=until_param_string, ) fields_list = [ TD.ID, TD.ADDED_ON, TD.CONFIDENCE, TD.DESCRIPTION, TD.EXPIRED_ON, [TD.INDICATOR, TI.INDICATOR], [TD.INDICATOR, TI.TYPE], [TD.INDICATOR, TI.ID], TD.LAST_UPDATED, [TD.OWNER, TE.ID], [TD.OWNER, TE.NAME], [TD.OWNER, TE.EMAIL], TD.PRECISION, TD.RAW_INDICATOR, TD.REVIEW_STATUS, TD.SEVERITY, TD.SHARE_LEVEL, TD.STATUS, TD.THREAT_TYPE, ] # Headers writer.writerow(map(utils.convert_to_header,fields_list)) for result in results: writer.writerow( map(lambda x: utils.get_data_field(x, result), fields_list) )
def main(): s = get_args() format_ = '%d-%m-%Y' for day_counter in range(s.days_back): until_param, until_param_string, since_param, since_param_string = \ utils.get_time_params(s.end_date, day_counter, format_) output_file = 'threat_descriptors_' + since_param_string + '_to_' + \ until_param_string + '.csv' with open(output_file, 'wb') as fout: writer = csv.writer(fout) results = ThreatDescriptor.objects( fields=ThreatDescriptor._fields, include_expired=s.include_expired, limit=1000, max_confidence=s.max_confidence, min_confidence=s.min_confidence, owner=s.owner, text=s.text, review_status=s.review_status, share_level=s.share_level, status=s.status, strict_text=s.strict_text, threat_type=s.threat_type, type_=s.type, since=since_param_string, until=until_param_string, ) fields_list = [ TD.ID, TD.ADDED_ON, TD.CONFIDENCE, TD.DESCRIPTION, TD.EXPIRED_ON, [TD.INDICATOR, TI.INDICATOR], [TD.INDICATOR, TI.TYPE], [TD.INDICATOR, TI.ID], TD.LAST_UPDATED, [TD.OWNER, TE.ID], [TD.OWNER, TE.NAME], [TD.OWNER, TE.EMAIL], TD.PRECISION, TD.RAW_INDICATOR, TD.REVIEW_STATUS, TD.SEVERITY, TD.SHARE_LEVEL, TD.STATUS, TD.THREAT_TYPE, ] # Headers writer.writerow(map(utils.convert_to_header, fields_list)) for result in results: writer.writerow( map(lambda x: utils.get_data_field(x, result), fields_list))
def main(): s = get_args() format_ = '%d-%m-%Y' for day_counter in range(s.days_back): until_param, until_param_string, since_param, since_param_string = \ utils.get_time_params(s.end_date, day_counter, format_) output_file = 'malware_families_' + since_param_string + '_to_' + \ until_param_string + '.csv' with open(output_file, 'wb') as fout: writer = csv.writer(fout) results = MalwareFamily.objects( fields=MalwareFamily._fields, limit=1000, text=s.text, strict_text=s.strict_text, since=since_param_string, until=until_param_string, ) fields_list = [ MF.ID, MF.ADDED_ON, MF.ALIASES, MF.DESCRIPTION, MF.FAMILY_TYPE, MF.MALICIOUS, MF.NAME, MF.SAMPLE_COUNT, MF.SUBMITTER_COUNT, ] # Headers writer.writerow(map(utils.convert_to_header, fields_list)) for result in results: writer.writerow( map(lambda x: utils.get_data_field(x, result), fields_list))
def main(): s = get_args() format_ = '%d-%m-%Y' for day_counter in range(s.days_back): until_param, until_param_string, since_param, since_param_string = \ utils.get_time_params(s.end_date, day_counter, format_) output_file = 'malware_families_' + since_param_string + '_to_' + \ until_param_string + '.csv' with open(output_file,'wb') as fout: writer = csv.writer(fout) results = MalwareFamily.objects( fields=MalwareFamily._fields, limit=1000, text=s.text, strict_text=s.strict_text, since=since_param_string, until=until_param_string, ) fields_list = [ MF.ID, MF.ADDED_ON, MF.ALIASES, MF.DESCRIPTION, MF.FAMILY_TYPE, MF.MALICIOUS, MF.NAME, MF.SAMPLE_COUNT, MF.SUBMITTER_COUNT, ] # Headers writer.writerow(map(utils.convert_to_header,fields_list)) for result in results: writer.writerow( map(lambda x: utils.get_data_field(x, result), fields_list) )
def main(): s = get_args() format_ = '%d-%m-%Y' for day_counter in range(s.days_back): until_param, until_param_string, since_param, since_param_string = \ utils.get_time_params(s.end_date, day_counter, format_) output_file = 'malware_analyses_' + since_param_string + '_to_' + \ until_param_string + '.csv' with open(output_file, 'wb') as fout: writer = csv.writer(fout) # TODO: Remove this once querying the fields related to a sample # Doesn't break TX, and fix all the things below fields = Malware._default_fields if (s.full_sample): fields += ['sample_size', 'sample'] results = Malware.objects( fields=fields, limit=1000, sample_type=s.sample_type, share_level=s.share_level, text=s.text, status=s.status, strict_text=s.strict_text, since=since_param_string, until=until_param_string, ) fields_list = [ MA.ID, MA.ADDED_ON, MA.CRX, MA.IMPHASH, MA.MD5, MA.PASSWORD, MA.PE_RICH_HEADER, MA.SAMPLE_TYPE, MA.SAMPLE_SIZE_COMPRESSED, MA.SHA1, MA.SHA256, MA.SHARE_LEVEL, MA.SSDEEP, MA.STATUS, MA.SUBMITTER_COUNT, MA.VICTIM_COUNT, MA.XPI, ] if (s.full_sample): fields_list += [ MA.SAMPLE, MA.SAMPLE_SIZE, ] # Headers writer.writerow(map(utils.convert_to_header, fields_list)) for result in results: writer.writerow( map(lambda x: utils.get_data_field(x, result), fields_list))
def query(args): """ Query the ThreatExchange API at the specified endpoint. """ # maximum number of indicators to fetch result_limit = 1000 # write results to this stream output_stream = '/dev/stdout' if not args.output else args.output for day in range(args.days_back): # format date parameters for HTTP request until, until_str, since, since_str = utils.get_time_params(args.end_date, day, '%d-%m-%Y') with open(output_stream, 'wb') as ostream: print('Writing to %s...' % output_stream) writer = csv.writer(ostream) if args.object == 'exchange_member': engine = ThreatExchangeMember fields = [XM.ID, XM.NAME] parameters = dict() elif args.object == 'malware_analysis': engine = Malware fields = [ MA.ID, MA.ADDED_ON, MA.CRX, MA.IMPHASH, MA.MD5, MA.PASSWORD, MA.PE_RICH_HEADER, MA.SAMPLE_TYPE, MA.SAMPLE_SIZE_COMPRESSED, MA.SHA1, MA.SHA256, MA.SHARE_LEVEL, MA.SSDEEP, MA.STATUS, MA.VICTIM_COUNT, MA.XPI, ] param_fields = Malware._default_fields if args.full_sample: param_fields += ['sample_size', 'sample'] parameters = dict( fields=param_fields, limit=result_limit, text=args.text, strict_text=args.strict_text, sample_type=args.malware_type, status=args.status, share_level=args.share_level, since=since_str, until=until_str ) elif args.object == 'malware_family': engine = MalwareFamily fields = [ MF.ID, MF.ADDED_ON, MF.ALIASES, MF.DESCRIPTION, MF.FAMILY_TYPE, MF.MALICIOUS, MF.NAME, MF.SAMPLE_COUNT ] parameters = dict( fields=MalwareFamily._fields, limit=result_limit, text=args.text, strict_text=args.strict_text, since=since_str, until=until_str ) elif args.object == 'threat_descriptor': engine = ThreatDescriptor fields = [ TD.ID, TD.ADDED_ON, TD.CONFIDENCE, TD.DESCRIPTION, TD.EXPIRED_ON, [TD.INDICATOR, TI.INDICATOR], [TD.INDICATOR, TI.TYPE], [TD.INDICATOR, TI.ID], TD.LAST_UPDATED, [TD.OWNER, XM.ID], [TD.OWNER, XM.NAME], [TD.OWNER, XM.EMAIL], TD.PRECISION, TD.RAW_INDICATOR, TD.REVIEW_STATUS, TD.SEVERITY, TD.SHARE_LEVEL, TD.STATUS ] parameters = dict( fields=ThreatDescriptor._fields, include_expired=args.include_expired, min_confidence=args.confidence_lb, max_confidence=args.confidence_ub, owner=args.owner, review_status=args.review_status, share_level=args.share_level, status=args.status, limit=result_limit, text=args.text, strict_text=args.strict_text, type_=args.indicator_type, since=since_str, until=until_str ) elif args.object == 'threat_indicator': engine = ThreatIndicator fields = [TI.ID, TI.INDICATOR, TI.TYPE] parameters = dict( fields=ThreatIndicator._fields, limit=result_limit, text=args.text, strict_text=args.strict_text, type_=args.indicator_type, since=since_str, until=until_str ) objects = engine.objects(**parameters) headers = [utils.convert_to_header(f) for f in fields] writer.writerow(headers) for i, o in enumerate(objects): data = [i] + [utils.get_data_field(f, o) for f in fields] writer.writerow(data)
def main(): s = get_args() format_ = '%d-%m-%Y' for day_counter in range(s.days_back): until_param, until_param_string, since_param, since_param_string = \ utils.get_time_params(s.end_date, day_counter, format_) output_file = 'malware_analyses_' + since_param_string + '_to_' + \ until_param_string + '.csv' with open(output_file,'wb') as fout: writer = csv.writer(fout) # TODO: Remove this once querying the fields related to a sample # Doesn't break TX, and fix all the things below fields = Malware._default_fields if (s.full_sample): fields += ['sample_size', 'sample'] results = Malware.objects( fields=fields, limit=1000, sample_type=s.sample_type, share_level=s.share_level, text=s.text, status=s.status, strict_text=s.strict_text, since=since_param_string, until=until_param_string, ) fields_list = [ MA.ID, MA.ADDED_ON, MA.CRX, MA.IMPHASH, MA.MD5, MA.PASSWORD, MA.PE_RICH_HEADER, MA.SAMPLE_TYPE, MA.SAMPLE_SIZE_COMPRESSED, MA.SHA1, MA.SHA256, MA.SHARE_LEVEL, MA.SSDEEP, MA.STATUS, MA.SUBMITTER_COUNT, MA.VICTIM_COUNT, MA.XPI, ] if (s.full_sample): fields_list += [ MA.SAMPLE, MA.SAMPLE_SIZE, ] # Headers writer.writerow(map(utils.convert_to_header,fields_list)) for result in results: writer.writerow( map(lambda x: utils.get_data_field(x, result), fields_list) )