def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=('Query TruSTAR indicators and store them in a CSV file\n' 'Example:\n\n' 'python latest_indicators.py -s' + ' INCIDENT_REPORT -t "IP,URL" -l 50 -i 12')) parser.add_argument('-s', '--source', required=True, dest='source', help='Source can be INCIDENT_REPORT or OSINT') parser.add_argument('-t', '--types', required=False, dest='types', help='Types of indicators') parser.add_argument('-l', '--limit', required=False, dest='limit', help='Limit on the returned number of indicators') parser.add_argument('-i', '--intervalSize', required=False, dest='interval_size', help='Interval size in hours') parser.add_argument('-f', '--fileName', required=False, dest='file_name') ts = TruStar(config_role="trustar") args = parser.parse_args() source_type = args.source if args.types: indicator_types = args.types.split(",") else: indicator_types = DEFAULT_TYPE_STRING if args.limit: limit = args.limit else: limit = DEFAULT_LIMIT if args.interval_size: interval_size = args.interval_size else: interval_size = DEFAULT_INTERVAL_SIZE if args.file_name: file_name = args.file_name else: file_name = FILE_NAME response = ts.query_latest_indicators(ts.get_token(), source_type, indicator_types, limit, interval_size) print(json.dumps(response, indent=2)) save_to_file(response['indicators'], file_name, source_type, indicator_types)
def __init__(self, attribute, config): config['enclave_ids'] = config.get('enclave_ids', "").strip().split(',') config['client_metatag'] = self.CLIENT_METATAG self.ts_client = TruStar(config=config) self.misp_event = MISPEvent() self.misp_attribute = MISPAttribute() self.misp_attribute.from_dict(**attribute) self.misp_event.add_attribute(**self.misp_attribute)
def __init__( self, config_file_path, # type: str config_stanza): # type: (..., str) -> None configs = ConfigLoader.from_(config_file_path, config_stanza) # type: Dict configs = {k.lower(): v for k, v in configs.items()} configs[ 'client_metatag'] = TruStarGuardDutyLambdaHandler.CLIENT_METATAG self.ts = TruStar(config=configs) self.ts.logger.setLevel("DEBUG") """ ch = StreamHandler() ch.setLevel("DEBUG") self.ts.logger.addHandler(ch) """ self.enclave_id = configs['enclave_id']
def from_env_vars(cls, client_metatag): # type: (str) -> TruStar """ Builds TruStar client. """ if not client_metatag: raise Exception("must specify a client_metatag.") config = {param: os.environ.get(param.upper()) for param in cls.TRUSTAR_CLIENT_PARAMS} config['client_metatag'] = client_metatag return TruStar(config=config)
def build_ts_client(self): """ Builds TruStar client using configs from environ vars. """ logger.info("Building TruStar client.") ts_config = { 'user_api_key': os.environ['USER_API_KEY'], 'user_api_secret': os.environ['USER_API_SECRET'], 'client_metatag': TruStarGuardDutyLambdaHandler.CLIENT_METATAG } ts = TruStar(config=ts_config) logger.info("Done building TruStar client.") return ts
def main(): ts = TruStar(config_role="trustar") token = ts.get_token() # process all files in directory print("Processing and submitting each source file in %s as a TruSTAR Incident Report" % SOURCE_REPORT_DIR) for (dirpath, dirnames, filenames) in os.walk(SOURCE_REPORT_DIR): for file in filenames: print("Processing source file %s " % file) try: path = os.path.join(SOURCE_REPORT_DIR, file) report_body_txt = ts.process_file(path) response_json = ts.submit_report(token, report_body_txt, "COMMUNITY: " + file) response_json = ts.submit_report(token, report_body_txt, "ENCLAVE: " + file, enclave=True) report_id = response_json['reportId'] print("SUCCESSFULLY SUBMITTED REPORT, TRUSTAR REPORT as Incident Report ID {0}".format(report_id)) if 'reportIndicators' in response_json: print("Extracted the following indicators: {}".format(response_json['reportIndicators'])) else: print("No indicators returned from report id {0}".format(report_id)) if 'correlatedIndicators' in response_json: print( "Extracted the following correlated indicators: {}".format( response_json['correlatedIndicators'])) else: print("No correlatedIndicators found in report id {0}".format(report_id)) except: print("Problem with file %s, exception: " % file) continue time.sleep(2)
def process_file(source_file): """ Extract text from a file (pdf, txt, eml, csv, json) :param source_file path to file to read :return text from file """ if source_file.endswith(('.pdf', '.PDF')): txt = TruStar.extract_pdf(source_file) elif source_file.endswith(('.txt', '.eml', '.csv', '.json')): f = open(source_file, 'r') txt = f.read() else: raise ValueError('UNSUPPORTED FILE EXTENSION') return txt
from trustar import TruStar, get_logger, datetime_to_millis from datetime import datetime, timedelta import csv # initialize SDK ts = TruStar() # initialize logger logger = get_logger(__name__) # set 'from' to the start of yesterday and 'to' to the end of yesterday to_time = datetime.now() from_time = to_time - timedelta(days=7) # convert to millis since epoch to_time = datetime_to_millis(to_time) from_time = datetime_to_millis(from_time) # define CSV column names HEADERS = [ "report_id", "report_title", "report_tags", "indicator_value", "indicator_type" ] # open the output csv to create it with open('indicators.csv', 'w') as f: # create csv writer object writer = csv.DictWriter(f, HEADERS) # write header row writer.writeheader()
def trustar(): return TruStar(config_role='staging')
from trustar import TruStar, datetime_to_millis from datetime import datetime, timedelta from keys import misp_url, misp_key, misp_verifycert from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation # TODO: Work on reducing false positive attributes # (e.g. email addresses from RH-ISAC members being pulled from TruStar reports) # Signature whitelisting can be done on MISP with regex entries (in the standard php # regex /{regex}/{modifier} format) entered on http://MISP.local/admin/whitelists/index # to restrict matching attributes from being included in the IDS flag sensitive exports # initialize TruStar() tru = TruStar() # initialize ExpandedPyMISP() misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert) now = datetime.now() # date range for pulling reports can be hours= or days= to_time = datetime.now() from_time = to_time - timedelta(hours=6) # convert to millis since epoch to_time = datetime_to_millis(to_time) from_time = datetime_to_millis(from_time) rhisac = "7a33144f-aef3-442b-87d4-dbf70d8afdb0" # TruStar RH-ISAC enclave uuid reports = tru.get_reports(from_time=from_time, to_time=to_time,
def main(): role = "trustar" if len(sys.argv) > 1: role = sys.argv[1] ts = TruStar(config_file="trustar.conf", config_role=role) # generate random id to use as external_id external_id = str(randint(1, 100000)) # or use a specific external_id # external_id = "321" report_guid = None current_time = int(time.time()) * 1000 yesterday_time = current_time - to_milliseconds(days=1) if do_latest_reports: logger.info("Getting Latest Accessible Incident Reports Since 24 hours ago ...") try: # get each successive page of reports report_generator = ts.get_reports(from_time=yesterday_time, to_time=current_time, is_enclave=True, enclave_ids=ts.enclave_ids) for report in report_generator: logger.info(report) except Exception as e: logger.error('Could not get latest reports, error: %s' % e) print('') if do_reports_by_community: two_days_ago = current_time - to_milliseconds(days=2) logger.info("Getting community only reports for the previous day ...") try: reports = ts.get_reports(from_time=two_days_ago, to_time=yesterday_time, is_enclave=False) for report in reports: logger.info(report) except Exception as e: logger.error('Could not get community reports, error: %s' % e) print('') if do_reports_by_enclave: a_week_ago = current_time - to_milliseconds(days=7) logger.info("Getting enclave only reports for the previous week ...") try: reports = ts.get_reports(from_time=a_week_ago, to_time=current_time, is_enclave=True, enclave_ids=ts.enclave_ids) for report in reports: logger.info(report) except Exception as e: logger.error('Could not get community reports, error: %s' % e) print('') if do_correlated: logger.info("Querying Accessible Correlated Reports...") try: report_ids = ts.get_correlated_report_ids(search_string) logger.info(report_ids) logger.info("%d report(s) correlated with indicators '%s':\n" % (len(report_ids), search_string)) logger.info("\n".join(report_ids)) except Exception as e: logger.error('Could not get correlated reports, error: %s' % e) print('') if do_community_trends: logger.info("Get community trends...") try: indicators = ts.get_community_trends(indicator_type=None, days_back=1) for indicator in indicators: logger.info(indicator) except Exception as e: logger.error('Could not get community trends, error: %s' % e) print('') if do_query_indicators: try: logger.info("Getting related indicators...") indicators = ts.get_related_indicators(indicators=search_string) for indicator in indicators: logger.info(indicator) except Exception as e: logger.error('Could not get related indicators, error: %s' % e) # Submit simple test report to community if do_comm_submissions: logger.info("Submit New Community Incident Report") try: report = Report(title="COMMUNITY API SUBMISSION TEST", body=submit_indicators, time_began="2017-02-01T01:23:45", is_enclave=False) report = ts.submit_report(report) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) except Exception as e: logger.error('Could not submit community report, error: %s' % e) print('') # Submit simple test report to your enclave if do_enclave_submissions: logger.info("Submit New Enclave Incident Report") try: report = Report(title="ENCLAVE API SUBMISSION TEST ", body=submit_indicators, time_began="2017-02-01T01:23:45", enclave_ids=ts.enclave_ids) report = ts.submit_report(report) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) logger.info(report) except Exception as e: logger.error('Could not submit enclave report, error: %s' % e) print('') # Submit a test report and retrieve it if do_submit_report: logger.info("Submit New Enclave Incident Report with External ID") try: report = Report(title="Sample SDK Test Report", body=submit_indicators, time_began="2017-02-01T01:23:45", is_enclave=True, enclave_ids=ts.enclave_ids, external_id=external_id) report = ts.submit_report(report) logger.info("Report Submitted") logger.info("\texternalTrackingId: %s" % report.external_id) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) except Exception as e: logger.error('Could not submit report, error: %s' % e) print('') # Get test report previously submitted if do_report_details_by_ext_id: logger.info("Get Incident Report By External ID") try: report = ts.get_report_details(report_id=external_id, id_type=Report.ID_TYPE_EXTERNAL) logger.info("\ttitle: %s" % report.title) logger.info("\texternalTrackingId: %s" % report.external_id) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) report_guid = report.id except Exception as e: logger.error('Could not get report, error: %s' % e) print('') # Update a test report and test with get report if do_update_report_by_ext_id: logger.info("Update Incident Report By External ID") try: report = Report(title="Updated Sample Title", body="updated report body: 21.22.23.24", external_id=external_id, enclave_ids=ts.enclave_ids) report = ts.update_report(report) logger.info("\texternalTrackingId: %s" % report.external_id) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) except Exception as e: logger.error('Could not update report, error: %s' % e) print('') # Get test report previously submitted if do_report_details_by_guid: logger.info("Get Incident Report Details by GUID (TruSTAR internal ID)") try: report = ts.get_report_details(report_guid) logger.info("\ttitle: %s" % report.title) logger.info("\texternalTrackingId: %s" % report.external_id) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) except Exception as e: logger.error('Could not get report, error: %s' % e) print('') # Update a test report and test with get report if do_update_report_by_guid: logger.info("Update Incident Report by GUID (TruSTAR internal ID)") try: report = Report(id=report_guid, title="New Sample Title", body="new sample body - 7.8.9.10", enclave_ids=ts.enclave_ids) report = ts.update_report(report) logger.info("Updated Report using GUID") logger.info("\texternalTrackingId: %s" % report.external_id) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) except Exception as e: logger.error('Could not update report, error: %s' % e) print('') # Get test report previously submitted if do_report_details_by_guid: logger.info("Get Report by GUID (TruSTAR internal ID)") try: report = ts.get_report_details(report_guid) logger.info("\ttitle: %s" % report.title) logger.info("\texternalTrackingId: %s" % report.external_id) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) except Exception as e: logger.error('Could not get report, error: %s' % e) print('') # Release report to community if do_release_report_by_ext_id: logger.info("Release Incident Report by External ID") try: report = Report(external_id=external_id, is_enclave=False) report = ts.update_report(report) logger.info("Report Released using External ID:") logger.info("\texternalTrackingId: %s" % report.external_id) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) except Exception as e: logger.error('Could not release report, error: %s' % e) print('') # Get test report previously submitted if do_report_details_by_ext_id_2: logger.info("Get Incident Report Details by External ID") try: report = ts.get_report_details(report_id=external_id, id_type=Report.ID_TYPE_EXTERNAL) logger.info("\ttitle: %s" % report.title) logger.info("\texternalTrackingId: %s" % report.external_id) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) except Exception as e: logger.error('Could not get report, error: %s' % e) print('') # Delete test report previously submitted if do_delete_report_by_ext_id: logger.info("Delete Incident Report by External ID") try: ts.delete_report(report_id=external_id, id_type=Report.ID_TYPE_EXTERNAL) logger.info("Report Deleted using External ID\n") except Exception as e: logger.error('Could not delete report, error: %s' % e) print('') # Add an enclave tag to a newly created report if do_add_enclave_tag: logger.info("Add enclave tag to incident report") try: # submit report report = Report(title="Enclave report with tag", body=submit_indicators, is_enclave=True, enclave_ids=ts.enclave_ids) report = ts.submit_report(report) logger.info("\tId of new report %s\n" % report.id) # get back report details, including the enclave it's in report = ts.get_report_details(report_id=report.id) enclave_id = report.enclave_ids[0] # add an enclave tag tag_id = ts.add_enclave_tag(report_id=report.id, name="triage", enclave_id=enclave_id) # logger.info the added enclave tag logger.info("\tId of new enclave tag %s\n" % tag_id) # add another enclave tag tag_id = ts.add_enclave_tag(report_id=report.id, name="resolved", enclave_id=enclave_id) # logger.info the added enclave tag logger.info("\tId of new enclave tag %s\n" % tag_id) # Get enclave tag info if do_get_enclave_tags: logger.info("Get enclave tags for report") tags = ts.get_enclave_tags(report.id) logger.info("\tEnclave tags for report %s\n" % report.id) logger.info(tags) # delete enclave tag by name if do_delete_enclave_tag: logger.info("Delete enclave tag from report") response = ts.delete_enclave_tag(report.id, tag_id) logger.info("\tDeleted enclave tag for report %s\n" % report.id) logger.info(response) # add it back ts.add_enclave_tag(report_id=report.id, name="triage", enclave_id=enclave_id) # List all enclave tags tags = ts.get_all_enclave_tags(enclave_ids=ts.enclave_ids) logger.info("List of enclave tags for enclave %s\n" % enclave_id) logger.info(tags) # Search report by tag logger.info("Getting reports tagged 'triage'.") reports = ts.get_reports(from_time=yesterday_time, to_time=current_time, enclave_ids=ts.enclave_ids, tag="triage") for report in reports: logger.info(report) except Exception as e: logger.error('Could not handle enclave tag operation, error: %s' % e) print('') # search for reports containing term "abc" if do_search_reports: try: logger.info("Searching reports:") reports = ts.search_reports("abc") for report in reports: logger.info(report) except Exception as e: logger.error("Could not search reports, error: %s" % e) print('') # search for indicators matching pattern "abc" if do_search_indicators: try: logger.info("Searching indicators:") indicators = ts.search_indicators("abc") for indicator in indicators: logger.info(indicator) except Exception as e: logger.error("Could not search indicators, error: %s" % e) print('') if do_redact_report: try: logger.info("Redacting report:") redacted_report = ts.redact_report(title="amazon phishing scam", report_body="apple, microsoft, and amazon suffered " "from a phishing scam via [email protected]") logger.info(redacted_report) except Exception as e: logger.error("Could not redact report, error: %s" % e) print('')
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description= ('Submit TruSTAR reports from a CSV file\n' 'Example:\n\n' 'python ingest_csv.py -c "TargetIP,SourceIP,Info,Analysis,Indicators" -t "TrackingNumber" -f august_incident_report.csv' )) parser.add_argument('-f', '--file', required=True, dest='file_name', help='csv file to import') parser.add_argument('-t', '--title', required=True, dest='title_col', help='Name of column to use as title field') parser.add_argument('-d', '--datetime', required=False, dest='datetime_col', help='Name of column to use as report date/time') parser.add_argument('-c', '--columns', required=False, dest='cols', help='List of comma-separated column names to include') parser.add_argument( '-n', '--num-reports', required=False, dest='num_reports', type=int, default=1000, help='Max number of reports to submit (top-down order)') args = parser.parse_args() allowed_keys_content = [] if args.cols: allowed_keys_content = args.cols.split(",") # noinspection PyCallingNonCallable ts = TruStar(config_role="trustar") token = ts.get_token() df = pd.read_csv(args.file_name, nrows=args.num_reports) # Create title and report content from the provided column names (if any) all_reports = [] for report_num in range(0, len(df)): current_content = '' current_title = '' current_datetime = None current_report = {} for key in df: content = "{}:\n {}\n ".format(key, str(df[key][report_num])) if not allowed_keys_content or key in allowed_keys_content: current_content += content if key == args.title_col: current_title = str(df[key][report_num]) if key == args.datetime_col: current_datetime = str(df[key][report_num]) current_report['reportTitle'] = current_title current_report['reportDateTime'] = current_datetime current_report['reportContent'] = current_content all_reports.append(current_report) if do_enclave_submissions: num_submitted = 0 for staged_report in all_reports: successful = False attempts = 0 while not successful and attempts < 5: attempts += 1 try: response = ts.submit_report( token, staged_report['reportContent'], staged_report['reportTitle'], discovered_time_str=staged_report['reportDateTime'], enclave=True) if 'error' in response: print("Submission failed with error: {}, {}".format( response['error'], response['message'])) # if response['message'] == "Access token expired": if response['error'] in ("Internal Server Error", "Access token expired", "Authentication error"): print("Auth token expired, requesting new one") token = ts.get_token() else: raise Exception else: num_submitted += 1 successful = True print( "Submitted report #{}-{} title {} as TruSTAR IR {}" .format(num_submitted, attempts, staged_report['reportTitle'], response['reportId'])) if 'reportIndicators' in response and len( response['reportIndicators']) > 0: print("Extracted the following indicators: {}".format( json.dumps(response['reportIndicators']))) print() except: e = sys.exc_info()[0] print("Problem submitting report") time.sleep(5) # Sleep between submissions time.sleep(90)
def main(inputfile): ts = TruStar() token = ts.get_token() df = pd.read_json(process_alert(inputfile)) process_time = time.strftime('%Y-%m-%d %H:%M', time.localtime(time.time())) filtered_falsepositive = filter_false_positive(df, process_time) filtered_winmethodology = filter_win_methodology(filtered_falsepositive, process_time) filtered_bashshellshock = filter_bash_shellshock(filtered_winmethodology, process_time) filtered_data = filter_webapp_attack(filtered_bashshellshock, process_time) all_reports = [] for alert in filtered_data: title = str(alert['displayId']) + ' ' + str( alert['message'].encode('utf-8')) content = "" for key in alert: type_value = type(alert[key]) if type_value == list or type_value == int or type_value == long or type_value == bool \ or type_value == dict or alert[key] is None: content += key + ': ' + str(alert[key]).replace('u\'', '\'') + '\n' else: content += key + ': ' + str(alert[key].encode( 'ascii', 'ignore')) + '\n' created_time = str(alert['createDate']) current_report = { 'reportTitle': title, 'reportContent': content, 'reportDateTime': created_time } all_reports.append(current_report) if do_enclave_submissions: for staged_report in all_reports: start_time = time.time() response = ts.submit_report( token, staged_report['reportContent'], staged_report['reportTitle'], discovered_time_str=staged_report['reportDateTime'], enclave=True) print(response) if 'error' in response: print("Submission failed with error: {}, {}".format( response['error'], response['message'])) if response['error'] in ("Internal Server Error", "Access token expired", "Authentication error"): print("Auth token expired, requesting new one") token = ts.get_token() else: raise Exception else: end_time = time.time() delta_time = end_time - start_time print("Submitted report title {} as TruSTAR IR {}".format( staged_report['reportTitle'], response['reportId']) + " Time:" + str(delta_time)) if 'reportIndicators' in response and len( response['reportIndicators']) > 0: print("Extracted the following indicators: {}".format( json.dumps(response['reportIndicators']))) print() time.sleep(3)
""" This script will delete all reports for your enclaves that were submitted yesterday. This is just an example, DO NOT RUN THIS UNLESS YOU ARE SURE YOU REALLY WANT TO!!! """ import logging from trustar import TruStar, datetime_to_millis from datetime import datetime, timedelta # initialize SDK ts = TruStar() # initialize logger logger = logging.getLogger(__name__) # set 'from' to the start of yesterday and 'to' to the end of yesterday to_time = datetime.now() - timedelta(days=1) from_time = to_time - timedelta(days=1) # convert to millis since epoch to_time = datetime_to_millis(to_time) from_time = datetime_to_millis(from_time) # keep a count of how many reports have been deleted count = 0 # initialize reports list to None reports = None # Loop until no reports remain. We can use the "get_reports_page" method # without adjusting the time frame on subsequent calls, since we know that
def main(): parser = argparse.ArgumentParser( description= "Retrieve IOCs from TruSTAR and enrich them. Filter based on time range, enclave(s), and tag(s). Just use -r to retrieve IOCs submitted since midnight last night." ) parser.add_argument( "-r", "--retrieve", action="store_true", help= "Retrieve IOCs based on the start/end dates, enclaves, and tags. Mutually exclusive with -i/--iocs." ) parser.add_argument( "-s", "--startdate", help= "Starting date (YYYY-MM-DD) to search for IOCs. (default start of today)" ) parser.add_argument( "-e", "--enddate", help= "Ending date (YYYY-MM-DD) to search for IOCs (default end of today)") parser.add_argument( "-n", "--enclaves", nargs="+", help= "Space separated list of enclaves to search (default BCBS-Domain), '-n s' to select enclaves from a list of valid enclaves." ) parser.add_argument( "-t", "--tags", nargs="+", help= "Space separated list of tags that IOCs must be tagged with, default is to return IOCs regardless of tags. '-t s' to select tags from a list of valid tags." ) parser.add_argument( "-i", "--iocs", nargs="+", help= "Get enrichment data for IOCs. Specify a space separated list of IOCs. Mutually exclusive with -r/--retrieve." ) parser.add_argument("-o", "--output", help="Filename of file where results are to be saved.") parser.add_argument("-c", "--config", help="Trustar configuration file.", default=os.getcwd() + '/trustar.conf') args = parser.parse_args() if not args.iocs and not args.retrieve: parser.error("Must specify either -r/--retrieve or -i/--iocs.") if args.iocs and (args.retrieve or args.startdate or args.enddate or args.enclaves or args.tags): parser.error( "Ony IOC(s) can be specified when -i/--iocs is used. No other options are valid when -i/--iocs is specified." ) if args.startdate: try: starttime = calendar.timegm( parse_date(args.startdate).timetuple()) * 1000 except: parser.error("Invalid start date specified {}.".format( args.startdate)) else: # The default start date is the earliest time today. # e.g. the start date on 2018-06-19 is 00:00:00 2018-06-19 t = time.gmtime() starttime = calendar.timegm((t.tm_year, t.tm_mon, t.tm_mday, 0, 0, 0, t.tm_wday, t.tm_yday, t.tm_isdst)) * 1000 if args.enddate: # Must add 1 to the date so that the end date is actually 00:00:00 on the next day. # This ensures getting all of the data on enddate. try: endtime = parse_date(args.enddate) + datetime.timedelta(days=1) endtime = calendar.timegm(endtime.timetuple()) * 1000 except: parser.error("Invalid end date specified {}.".format(args.enddate)) else: endtime = int(time.time()) * 1000 try: ts = TruStar(config_file=args.config, config_role="trustar") except Exception as e: print('Config file not found or invalid: {}'.format(args.config)) sys.exit(-1) enclaves = {} enclaveNames = {} for enclave in ts.get_user_enclaves(): if enclave.read: enclaves[enclave.name] = enclave.id enclaveNames[enclave.id] = enclave.name enclaveSelections = [] if args.enclaves: if 's' in args.enclaves: enclaveSelections = selectFromList(enclaves, "Enclaves") if enclaveSelections == None: sys.exit() else: for enclave in args.enclaves: if enclave in enclaves: enclaveSelections.append(enclaves[enclave]) else: print("Invalid enclave: {}".format(enclave)) sys.exit() tagIds = getTagIds(ts, enclaveSelections) tagSelections = [] if args.tags: if 's' in args.tags: tagSelections = selectFromList(tagIds, "Tags") if tagSelections == None: sys.exit() else: for tag in args.tags: tagSelections.append(tagIds[tag]) outputFile = sys.stdout if args.output: outputFile = open(args.output, "w") if args.iocs: for ioc in args.iocs: outputIOC(ts, enclaveNames, ioc, outputFile) else: # print("enclaves: {}".format(enclaveSelections), file=sys.stderr) # print("tags: {}".format(tagSelections), file=sys.stderr) # print(starttime) # print(endtime) # print(enclaveSelections) # print(tagSelections) indicators = ts.get_indicators(from_time=starttime, to_time=endtime, enclave_ids=enclaveSelections, included_tag_ids=tagSelections, page_size=500) for indicator in indicators: print("{},{}".format(indicator.value, indicator.type), file=outputFile) if args.output: outputFile.close()
class TruSTARParser: ENTITY_TYPE_MAPPINGS = { 'BITCOIN_ADDRESS': "btc", 'CIDR_BLOCK': "ip-src", 'CVE': "vulnerability", 'URL': "url", 'EMAIL_ADDRESS': "email-src", 'SOFTWARE': "filename", 'IP': "ip-src", 'MALWARE': "malware-type", 'MD5': "md5", 'REGISTRY_KEY': "regkey", 'SHA1': "sha1", 'SHA256': "sha256" } REPORT_BASE_URL = "https://station.trustar.co/constellation/reports/{}" CLIENT_METATAG = "MISP-{}".format(pymisp.__version__) def __init__(self, attribute, config): config['enclave_ids'] = config.get('enclave_ids', "").strip().split(',') config['client_metatag'] = self.CLIENT_METATAG self.ts_client = TruStar(config=config) self.misp_event = MISPEvent() self.misp_attribute = MISPAttribute() self.misp_attribute.from_dict(**attribute) self.misp_event.add_attribute(**self.misp_attribute) def get_results(self): """ Returns the MISP Event enriched with TruSTAR indicator summary data. """ event = json.loads(self.misp_event.to_json()) results = { key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key]) } return {'results': results} def generate_trustar_links(self, entity_value): """ Generates links to TruSTAR reports if they exist. :param entity_value: <str> Value of entity. """ report_links = list() trustar_reports = self.ts_client.search_reports(entity_value) for report in trustar_reports: report_links.append(self.REPORT_BASE_URL.format(report.id)) return report_links def parse_indicator_summary(self, summaries): """ Converts a response from the TruSTAR /1.3/indicators/summaries endpoint a MISP trustar_report object and adds the summary data and links as attributes. :param summaries: <generator> A TruSTAR Python SDK Page.generator object for generating indicator summaries pages. """ for summary in summaries: trustar_obj = MISPObject('trustar_report') indicator_type = summary.indicator_type indicator_value = summary.value if indicator_type in self.ENTITY_TYPE_MAPPINGS: trustar_obj.add_attribute( indicator_type, attribute_type=self.ENTITY_TYPE_MAPPINGS[indicator_type], value=indicator_value) trustar_obj.add_attribute("INDICATOR_SUMMARY", attribute_type="text", value=json.dumps(summary.to_dict(), sort_keys=True, indent=4)) report_links = self.generate_trustar_links(indicator_value) for link in report_links: trustar_obj.add_attribute("REPORT_LINK", attribute_type="link", value=link) self.misp_event.add_object(**trustar_obj)
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=( 'Submit one or more reports from local files (txt, pdf, docx, etc) ' 'in a directory\n\n' 'Example:\n' 'python bulk_upload.py ./sample_reports')) parser.add_argument('dir', help='Path containing report files') parser.add_argument( '-i', '--ignore', dest='ignore', action='store_true', help='Ignore history and resubmit already procesed files') args = parser.parse_args() source_report_dir = args.dir ts = TruStar(config_role="trustar") token = ts.get_token() # process all files in directory print( "Processing and submitting each source file in %s as a TruSTAR Incident Report" % source_report_dir) processed_files = set() processed_files_file = os.path.join(source_report_dir, "processed_files.log") if os.path.isfile(processed_files_file) and not args.ignore: processed_files = set(line.strip() for line in open(processed_files_file)) with open(processed_files_file, 'a', 0) as pf: for (dirpath, dirnames, filenames) in os.walk(source_report_dir): for source_file in filenames: if source_file in processed_files: continue print("Processing source file %s " % source_file) try: path = os.path.join(source_report_dir, source_file) report_body_txt = process_file(path) # response_json = ts.submit_report(token, report_body_txt, "COMMUNITY: " + file) response_json = ts.submit_report(token, report_body_txt, "ENCLAVE: " + source_file, enclave=True) report_id = response_json['reportId'] print( "SUCCESSFULLY SUBMITTED REPORT, TRUSTAR REPORT as Incident Report ID %s" % report_id) pf.write("%s\n" % source_file) # if 'reportIndicators' in response_json: # print("Extracted the following indicators: {}".format(response_json['reportIndicators'])) # else: # print("No indicators returned from report id {0}".format(report_id)) # # # if 'correlatedIndicators' in response_json: # print( # "Extracted the following correlated indicators: {}".format( # response_json['correlatedIndicators'])) # else: # print("No correlatedIndicators found in report id {0}".format(report_id)) except Exception as e: print("Problem with file %s, exception: %s " % (source_file, e)) continue time.sleep(2)
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description= ('Submit one or more reports from local files (txt, pdf, docx, etc) ' 'in a directory\n\n' 'Example:\n' 'python bulk_upload.py --dir ./sample_reports --ts_conf ./trustar.conf' )) parser.add_argument('--dir', '-d', help='Path containing report files', required=True) parser.add_argument('--ts_config', '-c', help='Path containing trustar api config', nargs='?', default="./trustar.conf") parser.add_argument( '-i', '--ignore', dest='ignore', action='store_true', help='Ignore history and resubmit already procesed files') args = parser.parse_args() source_report_dir = args.dir ts_config = args.ts_config ts = TruStar(config_file=ts_config) # process all files in directory logger.info( "Processing and submitting each source file in %s as a TruSTAR Incident Report" % source_report_dir) processed_files = set() processed_files_file = os.path.join(source_report_dir, "processed_files.log") if os.path.isfile(processed_files_file) and not args.ignore: processed_files = set(line.strip() for line in open(processed_files_file)) skipped_files_file = os.path.join(source_report_dir, "skipped_files.log") with open(processed_files_file, 'a', 0) as pf: for (dirpath, dirnames, filenames) in os.walk(source_report_dir): for source_file in filenames: if (source_file == "processed_files.log" or source_file == "skipped_files.log"): continue if source_file in processed_files: logger.debug( "File {} was already processed. Ignoring.".format( source_file)) continue logger.info("Processing source file %s " % source_file) try: path = os.path.join(source_report_dir, source_file) report_body = process_file(path) if not report_body: logger.debug( "File {} ignored for no data".format(source_file)) raise # response_json = ts.submit_report(token, report_body, "COMMUNITY: " + file) logger.info("Report {}".format(report_body)) try: report = Report(title="ENCLAVE: %s" % source_file, body=report_body, is_enclave=True, enclave_ids=ts.enclave_ids) report = ts.submit_report(report) logger.info("SUCCESSFULLY SUBMITTED REPORT, " + "TRUSTAR REPORT as Incident Report ID %s" % report.id) pf.write("%s\n" % source_file) if report.indicators is not None: print("Extracted the following indicators: {}". format( json.dumps([ x.to_dict() for x in report.indicators ], indent=2))) else: print("No indicators returned from report id {0}". format(report.id)) except Exception as e: if '413' in e.message: logger.warn( "Could not submit file {}. Contains more indicators than currently supported." .format(source_file)) else: raise except Exception as e: logger.error("Problem with file %s, exception: %s " % (source_file, e)) with open(skipped_files_file, 'w', 0) as sf: sf.write("{}\n".format(source_file)) continue time.sleep(2)
""" Reads reports from a CSV and submits them to TruSTAR. """ from trustar import log, Report, TruStar import csv logger = log.get_logger(__name__) # mapping of CSV column names to report fields MAPPING = {"title": "name", "body": "content", "external_id": "id"} CSV_PATH = "reports.csv" # initialize SDK ts = TruStar() # read in CSV with open(CSV_PATH, 'r') as f: reader = csv.DictReader(f) # iterate over rows for row in reader: # define method to get report field from CSV row def get_field(field): return row.get(MAPPING.get(field)) # construct report from CSV row report = Report(title=get_field('title'), body=get_field('body'),
def main(): ts = TruStar(config_role="trustar") token = ts.get_token() if do_latest: print("Get Latest Reports") results = ts.get_latest_reports(token) for result in results: print("\t{}, {}, {}".format(result['id'], result['distributionType'], result['title'])) print() if do_correlated: print("Querying Correlated Reports") results = ts.get_correlated_reports(token, query_indicators) print("{} report(s) correlated with indicators '{}': ".format( len(results), query_indicators)) for result in results: print("\t%s" % result) print() if do_query_indicator: print("Querying correlated indicators with '{}' (first 100)".format( query_indicators)) results = ts.query_indicator(token, query_indicators, "100") print("Correlated Incident Report indicators:") for indicator_type, indicator_list in list( results["indicators"].items()): print("\n%s:\n\t%s" % (indicator_type, "\n\t".join( ['{}'.format(value) for value in indicator_list]))) print() print("Correlated Open Source documents:") for os_url in list(results["openSourceCorrelations"]): print("\t%s" % (os_url)) print() print("External Intelligence hits:") for exint_url in list(results["externalIntelligence"]): print("\t%s" % (exint_url)) print() # Submit simple test report to community if do_comm_submissions: community_response = ts.submit_report( token, submit_indicators, "COMMUNITY API SUBMISSION TEST ") print("Community submission response: {0}".format( json.dumps(community_response))) if 'reportIndicators' in community_response: print("Extracted the following community indicators: {}".format( community_response['reportIndicators'])) # Submit simple test report to your enclave if do_enclave_submissions: enclave_response = ts.submit_report(token, submit_indicators, "ENCLAVE API SUBMISSION TEST ", enclave=True) print("Enclave submission response: {0}".format( json.dumps(enclave_response))) if 'reportIndicators' in enclave_response: print("Extracted the following enclave indicators: {}".format( enclave_response['reportIndicators']))
class TestReportSubmit: body = 'test body' title = 'test title' external_url = 'https://testurl.com' external_id = 'test_ext_id' def __init__( self, config_file_path, # type: str config_stanza): # type: (..., str) -> None configs = ConfigLoader.from_(config_file_path, config_stanza) # type: Dict configs = {k.lower(): v for k, v in configs.items()} configs[ 'client_metatag'] = TruStarGuardDutyLambdaHandler.CLIENT_METATAG self.ts = TruStar(config=configs) self.ts.logger.setLevel("DEBUG") """ ch = StreamHandler() ch.setLevel("DEBUG") self.ts.logger.addHandler(ch) """ self.enclave_id = configs['enclave_id'] def go(self): time_begans = [ '1577865600000', 1577865600456, '1577865600', 1577865601, '2020-02-01T00:00:01+00:00', '2020-02-01T00:00:01.12345+00:00', '2020-02-01T00:00:01.74839+00:00', #datetime.now() - timedelta(days=5) ] test_reports = [] # type: List[Report] for i, t in enumerate(time_begans): test_reports.append(self.build_test_report(i, t)) for r in test_reports: try: self.ts.delete_report(r.external_id, id_type=IdType.EXTERNAL) except: pass _ = self.ts.submit_report(r) logger.info("Sleeping 20 seconds.") sleep(20) for r in test_reports: report = self.ts.get_report_details( r.external_id, id_type=IdType.EXTERNAL) # type: Report if not report.body == r.body: logger.error("body") if not report.title == r.title: logger.error("title") if not report.external_url == r.external_url: logger.error("external_url") if not report.external_id == r.external_id: logger.error("external_id") if not report.time_began == r.time_began: logger.error("time_began submitted: '{}', retrieved: " "'{}'".format(r.time_began, report.time_began)) def build_test_report(self, i, time_began): r = Report() r.body = self.body r.title = self.title r.external_id = self.external_id + str(i) r.external_url = self.external_url r.enclave_ids = [self.enclave_id] #r.set_time_began(time_began) r.time_began = time_began return r
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description= ('Submit TruSTAR reports from a CSV file\n' 'Example:\n\n' 'python ingest_csv.py -c "TargetIP,SourceIP,Info,Analysis,Indicators" -t "TrackingNumber" -d "ReportTime" -cn "CaseName" -f reportname.csv' )) parser.add_argument('-f', '--file', required=True, dest='file_name', help='csv file to import') parser.add_argument('-t', '--title', required=True, dest='title_col', help='Name of column to use as title field') parser.add_argument('-d', '--datetime', required=False, dest='datetime_col', help='Name of column to use as report date/time') parser.add_argument('-c', '--columns', required=False, dest='cols', help='List of comma-separated column names to include') parser.add_argument( '-n', '--num-reports', required=False, dest='num_reports', type=int, default=1000, help='Max number of reports to submit (top-down order)') parser.add_argument( '-o', '--output', required=False, dest='cef_output_file', default='trustar.cef', help= 'Common Event Format (CEF) output log file, one event is generated per successful submission' ) parser.add_argument( '-ci', '--case-id', required=False, dest='caseid_col', help='Name of column to use as report case ID for CEF export') args = parser.parse_args() allowed_keys_content = [] if args.cols: allowed_keys_content = args.cols.split(",") ts = TruStar(config_role="trustar") token = ts.get_token() df = pd.read_csv(args.file_name, nrows=args.num_reports, encoding="latin1") # Create title and report content from the provided column names (if any) all_reports = [] for report_num in range(0, len(df)): current_content = '' current_title = '' current_datetime = None current_case_id = 0 current_report = {} for key in df: # ignore empty cells, which are float64 NaNs cell_value = df[key][report_num] if pd.isnull(cell_value): continue cell_value = "%s" % cell_value # encode any unicode chars string_value = cell_value.encode('utf-8').strip() if string_value == "nan": print("%s -> %s" % (key, string_value)) continue content = "{}:\n {}\n \n".format(key, string_value) if not allowed_keys_content or key in allowed_keys_content: current_content += content if key == args.title_col: current_title = str(df[key][report_num]) if key == args.datetime_col: current_datetime = str(df[key][report_num]) if key == args.caseid_col: current_case_id = str(df[key][report_num]) current_report['reportTitle'] = current_title current_report['reportDateTime'] = current_datetime current_report['reportContent'] = current_content current_report['reportCaseId'] = current_case_id all_reports.append(current_report) if do_enclave_submissions: num_submitted = 0 for staged_report in all_reports: successful = False attempts = 0 while not successful and attempts < 5: attempts += 1 try: response = ts.submit_report( token, report_body_txt=staged_report['reportContent'], report_name=staged_report['reportTitle'], began_time=staged_report['reportDateTime'], enclave=True) if 'error' in response: print("Submission failed with error: %s, %s" % (response['error'], response['message'])) if response['error'] in ("Internal Server Error", "Access token expired", "Authentication error"): print("Auth token expired, requesting new one") token = ts.get_token() else: raise Exception else: num_submitted += 1 successful = True print( "Submitted report #%s-%s title %s as TruSTAR IR %s with case ID: %s" % (num_submitted, attempts, staged_report['reportTitle'], response['reportId'], staged_report['reportCaseId'])) print("URL: %s" % ts.get_report_url(response['reportId'])) # Build CEF output: # - HTTP_USER_AGENT is the cs1 field # - example CEF output: CEF:version|vendor|product|device_version|signature|name|severity|cs1=(num_submitted) cs2=(report_url) config = { 'cef.version': '0.5', 'cef.vendor': 'TruSTAR', 'cef.device_version': '2.0', 'cef.product': 'API', 'cef': True, 'cef.file': args.cef_output_file } environ = { 'REMOTE_ADDR': '127.0.0.1', 'HTTP_HOST': '127.0.0.1', 'HTTP_USER_AGENT': staged_report['reportTitle'] } log_cef('SUBMISSION', 1, environ, config, signature="INFO", cs2=staged_report['reportCaseId'], cs3=ts.get_report_url(response['reportId'])) #### # TODO: ADD YOUR CUSTOM POST-PROCESSING CODE FOR THIS SUBMISSION HERE #### if 'reportIndicators' in response and len( response['reportIndicators']) > 0: print("Indicators:\n %s" % (json.dumps(response['reportIndicators']))) print() except Exception as e: traceback.print_exc(file=sys.stdout) print("Problem submitting report: %s" % e) time.sleep(5) # Sleep between submissions time.sleep(5)
def main(): ts = TruStar(config_role="trustar") token = ts.get_token(verify=verify) # generate random id to use as external_id external_id = str(randint(1, 100000)) # or use a specific external_id # external_id = "321" report_guid = None # Submit a test report and retrieve it if do_submit_report: print("Submit Report") submission_response = ts.submit_report_v12(token, submit_indicators, "Sample SDK Test Report", external_id=external_id, began_time="2017-02-01T01:23:45", enclave=True, verify=verify) print("Report Submitted") print("\texternalTrackingId: %s" % submission_response['externalTrackingId']) print("\tindicators: %s" % submission_response['reportIndicators']) print("\tURL: %s\n" % ts.get_report_url(submission_response['reportId'])) # Get test report previously submitted if do_report_details_by_ext_id: print("Get Report") result = ts.get_report_details_v12(token, external_id, id_type="external", verify=verify) print("Report Details using External ID") print("\ttitle: %s" % result['title']) print("\texternalTrackingId: %s" % result['externalTrackingId']) print("\tindicators: %s" % result['indicators']) print("\tURL: %s\n" % ts.get_report_url(result['id'])) report_guid = result['id'] # Update a test report and test with get report if do_update_report_by_ext_id: print("Update Report") title = "NEW CC REPORT" body = "updated report body: 21.22.23.24" update_response = ts.update_report(token, external_id, id_type="external", title=title, report_body=body, verify=verify) print("Updated Report using External ID") print("\texternalTrackingId: %s" % update_response['externalTrackingId']) print("\tindicators: %s" % update_response['reportIndicators']) print("\tURL: %s\n" % ts.get_report_url(update_response['reportId'])) # Get test report previously submitted if do_report_details_by_guid: print("Get Report") result = ts.get_report_details_v12(token, report_guid, id_type="internal", verify=verify) print("Report Details using Guid") print("\ttitle: %s" % result['title']) print("\texternalTrackingId: %s" % result['externalTrackingId']) print("\tindicators: %s" % result['indicators']) print("\tURL: %s\n" % ts.get_report_url(result['id'])) # Update a test report and test with get report if do_update_report_by_guid: print("Update Report") title = "New Sample Title" body = "new sample body - 7.8.9.10" update_response = ts.update_report(token, report_guid, id_type="internal", title=title, report_body=body, verify=verify) print("Updated Report using GUID") print("\texternalTrackingId: %s" % update_response['externalTrackingId']) print("\tindicators: %s" % update_response['reportIndicators']) print("\tURL: %s\n" % ts.get_report_url(update_response['reportId'])) # Get test report previously submitted if do_report_details_by_guid: print("Get Report") result = ts.get_report_details_v12(token, report_guid, id_type="internal", verify=verify) print("Report Details using GUID") print("\ttitle: %s" % result['title']) print("\texternalTrackingId: %s" % result['externalTrackingId']) print("\tindicators: %s" % result['indicators']) print("\tURL: %s\n" % ts.get_report_url(result['id'])) # Release report to community if do_release_report_by_ext_id: print("Release Report") update_response = ts.update_report(token, external_id, id_type='external', distribution="COMMUNITY", verify=verify) print("Report Released using External ID") print("\texternalTrackingId: %s" % update_response['externalTrackingId']) print("\tindicators: %s" % update_response['reportIndicators']) print("\tURL: %s\n" % ts.get_report_url(update_response['reportId'])) # Get test report previously submitted if do_report_details_by_ext_id_2: print("Get Report") result = ts.get_report_details_v12(token, external_id, id_type="external", verify=verify) print("Report Details using External ID") print("\ttitle: %s" % result['title']) print("\texternalTrackingId: %s" % result['externalTrackingId']) print("\tindicators: %s" % result['indicators']) print("\tURL: %s\n" % ts.get_report_url(result['id'])) # Delete test report previously submitted if do_delete_report_by_ext_id: print("Delete Report") response = ts.delete_report(token, external_id, id_type="external", verify=verify) print("Report Deleted using External ID")
def main(): ts = TruStar(config_role="trustar") token = ts.get_token() if do_latest_reports: print("Getting Latest Accessible Reports...") results = ts.get_latest_reports(token) for result in results: print("\t%s, %s, %s" % (result['id'], result['distributionType'], result['title'])) print() if do_correlated: print("Querying Accessible Correlated Reports...") results = ts.get_correlated_reports(token, search_string) print("%d report(s) correlated with indicators '%s':\n" % (len(results), search_string)) print("\n".join(results)) print() if do_latest_indicators: print("Get Latest Indicators (first 100)") results = ts.query_latest_indicators(token, source='INCIDENT_REPORT', indicator_types='ALL', interval_size=24, limit=100) if 'indicators' in results: for ioc_type, value in results['indicators'].items(): if len(value) > 0: print("\t%s: %s" % (ioc_type, ','.join(value))) print() if do_report_details: print("Get Report Details") reports = ts.get_latest_reports(token) for report in reports: result = ts.get_report_details(token, report['id']) print("Getting Report Details using '%s': \n%s" % (report['id'], json.dumps(result, indent=4))) print() if do_query_indicators: print( "Querying correlated indicators with search string '%s' (first 100)" % search_string) results = ts.query_indicators(token, search_string, '100') indicator_hits = list(results["indicators"]) if len(indicator_hits) > 0: print("Correlated Incident Report Indicators:") for indicator_type, indicator_list in list( results["indicators"].items()): print("\n%s:\n\t%s" % (indicator_type, "\n\t".join( ['{}'.format(value) for value in indicator_list]))) print() os_hits = list(results["openSourceCorrelations"]) if len(os_hits) > 0: print("Correlated Open Source Documents:") for os_url in os_hits: print("\t%s" % os_url) print() exint_hits = list(results["externalIntelligence"]) if len(exint_hits) > 0: print("External Intelligence hits:") print('\t'.join(exint_hits)) print() # Submit simple test report to community if do_comm_submissions: community_response = ts.submit_report(token, submit_indicators, "COMMUNITY API SUBMISSION TEST", began_time="2017-02-01T01:23:45") print("\tURL: %s\n" % ts.get_report_url(community_response['reportId'])) if 'reportIndicators' in community_response: print("Extracted the following community indicators: \n%s\n" % json.dumps(community_response['reportIndicators'], indent=2)) # Submit simple test report to your enclave if do_enclave_submissions: enclave_response = ts.submit_report(token, submit_indicators, "ENCLAVE API SUBMISSION TEST ", enclave=True) print("\tURL: %s\n" % ts.get_report_url(enclave_response['reportId'])) print(enclave_response) if 'reportIndicators' in enclave_response: print("Extracted the following enclave indicators: \n%s\n" % json.dumps(enclave_response['reportIndicators'], indent=2))