def build_for(self, finding): # type: (Dict) -> Report """ Builds a Report for an event. Note: Does NOT validate report attribute values to ensure they are valid / reasonable. """ detail = finding.get('detail') if not detail: raise Exception("Guard Duty Finding dictionary's 'detail' kv pair" "is empty. This integration builds TruSTAR " "reports from that key's value.") title = detail.get('title', self.DEFAULT_TITLE) body = self._body_from_detail(detail) time_began = self._time_began_from_detail(detail) external_url = detail.get('arn') external_id = self.ext_id_encoder.reversible(self.enclave_id, detail.get('id')) r = Report(title=title, body=body, time_began=time_began, external_url=external_url, external_id=external_id, enclave_ids=[self.enclave_id]) logger.info("TimeBegan in ReportBuilder after assigning to the " "report object: '{}'".format(r.time_began)) logger.info("TimeBegan type: '{}'.".format(str(type(r.time_began)))) d = r.to_dict() logger.info("TimeBegan in ReportBuiilder after converting report to " "dict: '{}'.".format(d['timeBegan'])) return r
def report(trustar, current_time_millis, milliseconds_in_a_day): original_report = Report(title="Report 1", id=FAKE_REPORT_ID, body="Blah blah blah", time_began=current_time_millis - milliseconds_in_a_day, enclave_ids=trustar.enclave_ids) return original_report
def build_test_report(self, i, time_began): r = Report() r.body = self.body r.title = self.title r.external_id = self.external_id + str(i) r.external_url = self.external_url r.enclave_ids = [self.enclave_id] #r.set_time_began(time_began) r.time_began = time_began return r
def __init__( self, sample_event_file_path, # type: str expected_report_file_path, # type: str config_file_path, # type: str delete_reports_when_done=False # type: bool ): """ Some prep work. """ configs = self.load_configs(config_file_path) # type: dict self.__env_vars = self.env_vars_from_configs(configs) # type: Dict self.set_env_vars_to_os(self.__env_vars) self.ts = self.build_ts_client() self.test_event = self.dict_from_file( sample_event_file_path) # type: Dict self.expected_report = Report.from_dict( self.dict_from_file(expected_report_file_path)) # type: Dict self.delete_reports_when_done = delete_reports_when_done # type: bool
def main(): role = "trustar" if len(sys.argv) > 1: role = sys.argv[1] ts = TruStar(config_file="trustar.conf", config_role=role) # generate random id to use as external_id external_id = str(randint(1, 100000)) # or use a specific external_id # external_id = "321" report_guid = None current_time = int(time.time()) * 1000 yesterday_time = current_time - to_milliseconds(days=1) if do_latest_reports: logger.info("Getting Latest Accessible Incident Reports Since 24 hours ago ...") try: # get each successive page of reports report_generator = ts.get_reports(from_time=yesterday_time, to_time=current_time, is_enclave=True, enclave_ids=ts.enclave_ids) for report in report_generator: logger.info(report) except Exception as e: logger.error('Could not get latest reports, error: %s' % e) print('') if do_reports_by_community: two_days_ago = current_time - to_milliseconds(days=2) logger.info("Getting community only reports for the previous day ...") try: reports = ts.get_reports(from_time=two_days_ago, to_time=yesterday_time, is_enclave=False) for report in reports: logger.info(report) except Exception as e: logger.error('Could not get community reports, error: %s' % e) print('') if do_reports_by_enclave: a_week_ago = current_time - to_milliseconds(days=7) logger.info("Getting enclave only reports for the previous week ...") try: reports = ts.get_reports(from_time=a_week_ago, to_time=current_time, is_enclave=True, enclave_ids=ts.enclave_ids) for report in reports: logger.info(report) except Exception as e: logger.error('Could not get community reports, error: %s' % e) print('') if do_correlated: logger.info("Querying Accessible Correlated Reports...") try: report_ids = ts.get_correlated_report_ids(search_string) logger.info(report_ids) logger.info("%d report(s) correlated with indicators '%s':\n" % (len(report_ids), search_string)) logger.info("\n".join(report_ids)) except Exception as e: logger.error('Could not get correlated reports, error: %s' % e) print('') if do_community_trends: logger.info("Get community trends...") try: indicators = ts.get_community_trends(indicator_type=None, days_back=1) for indicator in indicators: logger.info(indicator) except Exception as e: logger.error('Could not get community trends, error: %s' % e) print('') if do_query_indicators: try: logger.info("Getting related indicators...") indicators = ts.get_related_indicators(indicators=search_string) for indicator in indicators: logger.info(indicator) except Exception as e: logger.error('Could not get related indicators, error: %s' % e) # Submit simple test report to community if do_comm_submissions: logger.info("Submit New Community Incident Report") try: report = Report(title="COMMUNITY API SUBMISSION TEST", body=submit_indicators, time_began="2017-02-01T01:23:45", is_enclave=False) report = ts.submit_report(report) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) except Exception as e: logger.error('Could not submit community report, error: %s' % e) print('') # Submit simple test report to your enclave if do_enclave_submissions: logger.info("Submit New Enclave Incident Report") try: report = Report(title="ENCLAVE API SUBMISSION TEST ", body=submit_indicators, time_began="2017-02-01T01:23:45", enclave_ids=ts.enclave_ids) report = ts.submit_report(report) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) logger.info(report) except Exception as e: logger.error('Could not submit enclave report, error: %s' % e) print('') # Submit a test report and retrieve it if do_submit_report: logger.info("Submit New Enclave Incident Report with External ID") try: report = Report(title="Sample SDK Test Report", body=submit_indicators, time_began="2017-02-01T01:23:45", is_enclave=True, enclave_ids=ts.enclave_ids, external_id=external_id) report = ts.submit_report(report) logger.info("Report Submitted") logger.info("\texternalTrackingId: %s" % report.external_id) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) except Exception as e: logger.error('Could not submit report, error: %s' % e) print('') # Get test report previously submitted if do_report_details_by_ext_id: logger.info("Get Incident Report By External ID") try: report = ts.get_report_details(report_id=external_id, id_type=Report.ID_TYPE_EXTERNAL) logger.info("\ttitle: %s" % report.title) logger.info("\texternalTrackingId: %s" % report.external_id) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) report_guid = report.id except Exception as e: logger.error('Could not get report, error: %s' % e) print('') # Update a test report and test with get report if do_update_report_by_ext_id: logger.info("Update Incident Report By External ID") try: report = Report(title="Updated Sample Title", body="updated report body: 21.22.23.24", external_id=external_id, enclave_ids=ts.enclave_ids) report = ts.update_report(report) logger.info("\texternalTrackingId: %s" % report.external_id) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) except Exception as e: logger.error('Could not update report, error: %s' % e) print('') # Get test report previously submitted if do_report_details_by_guid: logger.info("Get Incident Report Details by GUID (TruSTAR internal ID)") try: report = ts.get_report_details(report_guid) logger.info("\ttitle: %s" % report.title) logger.info("\texternalTrackingId: %s" % report.external_id) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) except Exception as e: logger.error('Could not get report, error: %s' % e) print('') # Update a test report and test with get report if do_update_report_by_guid: logger.info("Update Incident Report by GUID (TruSTAR internal ID)") try: report = Report(id=report_guid, title="New Sample Title", body="new sample body - 7.8.9.10", enclave_ids=ts.enclave_ids) report = ts.update_report(report) logger.info("Updated Report using GUID") logger.info("\texternalTrackingId: %s" % report.external_id) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) except Exception as e: logger.error('Could not update report, error: %s' % e) print('') # Get test report previously submitted if do_report_details_by_guid: logger.info("Get Report by GUID (TruSTAR internal ID)") try: report = ts.get_report_details(report_guid) logger.info("\ttitle: %s" % report.title) logger.info("\texternalTrackingId: %s" % report.external_id) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) except Exception as e: logger.error('Could not get report, error: %s' % e) print('') # Release report to community if do_release_report_by_ext_id: logger.info("Release Incident Report by External ID") try: report = Report(external_id=external_id, is_enclave=False) report = ts.update_report(report) logger.info("Report Released using External ID:") logger.info("\texternalTrackingId: %s" % report.external_id) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) except Exception as e: logger.error('Could not release report, error: %s' % e) print('') # Get test report previously submitted if do_report_details_by_ext_id_2: logger.info("Get Incident Report Details by External ID") try: report = ts.get_report_details(report_id=external_id, id_type=Report.ID_TYPE_EXTERNAL) logger.info("\ttitle: %s" % report.title) logger.info("\texternalTrackingId: %s" % report.external_id) logger.info("\tURL: %s\n" % ts.get_report_url(report.id)) except Exception as e: logger.error('Could not get report, error: %s' % e) print('') # Delete test report previously submitted if do_delete_report_by_ext_id: logger.info("Delete Incident Report by External ID") try: ts.delete_report(report_id=external_id, id_type=Report.ID_TYPE_EXTERNAL) logger.info("Report Deleted using External ID\n") except Exception as e: logger.error('Could not delete report, error: %s' % e) print('') # Add an enclave tag to a newly created report if do_add_enclave_tag: logger.info("Add enclave tag to incident report") try: # submit report report = Report(title="Enclave report with tag", body=submit_indicators, is_enclave=True, enclave_ids=ts.enclave_ids) report = ts.submit_report(report) logger.info("\tId of new report %s\n" % report.id) # get back report details, including the enclave it's in report = ts.get_report_details(report_id=report.id) enclave_id = report.enclave_ids[0] # add an enclave tag tag_id = ts.add_enclave_tag(report_id=report.id, name="triage", enclave_id=enclave_id) # logger.info the added enclave tag logger.info("\tId of new enclave tag %s\n" % tag_id) # add another enclave tag tag_id = ts.add_enclave_tag(report_id=report.id, name="resolved", enclave_id=enclave_id) # logger.info the added enclave tag logger.info("\tId of new enclave tag %s\n" % tag_id) # Get enclave tag info if do_get_enclave_tags: logger.info("Get enclave tags for report") tags = ts.get_enclave_tags(report.id) logger.info("\tEnclave tags for report %s\n" % report.id) logger.info(tags) # delete enclave tag by name if do_delete_enclave_tag: logger.info("Delete enclave tag from report") response = ts.delete_enclave_tag(report.id, tag_id) logger.info("\tDeleted enclave tag for report %s\n" % report.id) logger.info(response) # add it back ts.add_enclave_tag(report_id=report.id, name="triage", enclave_id=enclave_id) # List all enclave tags tags = ts.get_all_enclave_tags(enclave_ids=ts.enclave_ids) logger.info("List of enclave tags for enclave %s\n" % enclave_id) logger.info(tags) # Search report by tag logger.info("Getting reports tagged 'triage'.") reports = ts.get_reports(from_time=yesterday_time, to_time=current_time, enclave_ids=ts.enclave_ids, tag="triage") for report in reports: logger.info(report) except Exception as e: logger.error('Could not handle enclave tag operation, error: %s' % e) print('') # search for reports containing term "abc" if do_search_reports: try: logger.info("Searching reports:") reports = ts.search_reports("abc") for report in reports: logger.info(report) except Exception as e: logger.error("Could not search reports, error: %s" % e) print('') # search for indicators matching pattern "abc" if do_search_indicators: try: logger.info("Searching indicators:") indicators = ts.search_indicators("abc") for indicator in indicators: logger.info(indicator) except Exception as e: logger.error("Could not search indicators, error: %s" % e) print('') if do_redact_report: try: logger.info("Redacting report:") redacted_report = ts.redact_report(title="amazon phishing scam", report_body="apple, microsoft, and amazon suffered " "from a phishing scam via [email protected]") logger.info(redacted_report) except Exception as e: logger.error("Could not redact report, error: %s" % e) print('')
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description= ('Submit one or more reports from local files (txt, pdf, docx, etc) ' 'in a directory\n\n' 'Example:\n' 'python bulk_upload.py --dir ./sample_reports --ts_conf ./trustar.conf' )) parser.add_argument('--dir', '-d', help='Path containing report files', required=True) parser.add_argument('--ts_config', '-c', help='Path containing trustar api config', nargs='?', default="./trustar.conf") parser.add_argument( '-i', '--ignore', dest='ignore', action='store_true', help='Ignore history and resubmit already procesed files') args = parser.parse_args() source_report_dir = args.dir ts_config = args.ts_config ts = TruStar(config_file=ts_config) # process all files in directory logger.info( "Processing and submitting each source file in %s as a TruSTAR Incident Report" % source_report_dir) processed_files = set() processed_files_file = os.path.join(source_report_dir, "processed_files.log") if os.path.isfile(processed_files_file) and not args.ignore: processed_files = set(line.strip() for line in open(processed_files_file)) skipped_files_file = os.path.join(source_report_dir, "skipped_files.log") with open(processed_files_file, 'a', 0) as pf: for (dirpath, dirnames, filenames) in os.walk(source_report_dir): for source_file in filenames: if (source_file == "processed_files.log" or source_file == "skipped_files.log"): continue if source_file in processed_files: logger.debug( "File {} was already processed. Ignoring.".format( source_file)) continue logger.info("Processing source file %s " % source_file) try: path = os.path.join(source_report_dir, source_file) report_body = process_file(path) if not report_body: logger.debug( "File {} ignored for no data".format(source_file)) raise # response_json = ts.submit_report(token, report_body, "COMMUNITY: " + file) logger.info("Report {}".format(report_body)) try: report = Report(title="ENCLAVE: %s" % source_file, body=report_body, is_enclave=True, enclave_ids=ts.enclave_ids) report = ts.submit_report(report) logger.info("SUCCESSFULLY SUBMITTED REPORT, " + "TRUSTAR REPORT as Incident Report ID %s" % report.id) pf.write("%s\n" % source_file) if report.indicators is not None: print("Extracted the following indicators: {}". format( json.dumps([ x.to_dict() for x in report.indicators ], indent=2))) else: print("No indicators returned from report id {0}". format(report.id)) except Exception as e: if '413' in e.message: logger.warn( "Could not submit file {}. Contains more indicators than currently supported." .format(source_file)) else: raise except Exception as e: logger.error("Problem with file %s, exception: %s " % (source_file, e)) with open(skipped_files_file, 'w', 0) as sf: sf.write("{}\n".format(source_file)) continue time.sleep(2)
# mapping of CSV column names to report fields MAPPING = {"title": "name", "body": "content", "external_id": "id"} CSV_PATH = "reports.csv" # initialize SDK ts = TruStar() # read in CSV with open(CSV_PATH, 'r') as f: reader = csv.DictReader(f) # iterate over rows for row in reader: # define method to get report field from CSV row def get_field(field): return row.get(MAPPING.get(field)) # construct report from CSV row report = Report(title=get_field('title'), body=get_field('body'), external_id=get_field('external_id'), is_enclave=True, enclave_ids=ts.enclave_ids) # submit report ts.submit_report(report) logger.info("Submitted report: %s" % report)
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description= ('Submit TruSTAR reports from a CSV file\n' 'Example:\n\n' 'python ingest_csv.py -c "TargetIP,SourceIP,Info,Analysis,Indicators" -t "TrackingNumber" -d "ReportTime" -cn "CaseName" -f reportname.csv' )) parser.add_argument('-f', '--file', required=True, dest='file_name', help='csv file to import') parser.add_argument('-t', '--title', required=True, dest='title_col', help='Name of column to use as title field') parser.add_argument('-d', '--datetime', required=False, dest='datetime_col', help='Name of column to use as report date/time') parser.add_argument('-c', '--columns', required=False, dest='cols', help='List of comma-separated column names to include') parser.add_argument( '-n', '--num-reports', required=False, dest='num_reports', type=int, default=1000, help='Max number of reports to submit (top-down order)') parser.add_argument( '-o', '--output', required=False, dest='cef_output_file', default='trustar.cef', help= 'Common Event Format (CEF) output log file, one event is generated per successful submission' ) parser.add_argument( '-ci', '--case-id', required=False, dest='caseid_col', help='Name of column to use as report case ID for CEF export') args = parser.parse_args() allowed_keys_content = [] if args.cols: allowed_keys_content = args.cols.split(",") ts = TruStar(config_role="trustar") df = pd.read_csv(args.file_name, nrows=args.num_reports, encoding="latin1") # Create title and report content from the provided column names (if any) all_reports = [] for report_num in range(0, len(df)): current_content = '' current_title = '' current_datetime = None current_case_id = 0 for key in df: # ignore empty cells, which are float64 NaNs cell_value = df[key][report_num] if pd.isnull(cell_value): continue cell_value = "%s" % cell_value # encode any unicode chars string_value = cell_value.encode('utf-8').strip() if string_value == "nan": print("%s -> %s" % (key, string_value)) continue content = "{}:\n {}\n \n".format(key, string_value) if not allowed_keys_content or key in allowed_keys_content: current_content += content if key == args.title_col: current_title = str(df[key][report_num]) if key == args.datetime_col: current_datetime = str(df[key][report_num]) if key == args.caseid_col: current_case_id = str(df[key][report_num]) report = Report(title=current_title, time_began=current_datetime, body=current_content, external_id=current_case_id, is_enclave=True, enclave_ids=ts.enclave_ids) all_reports.append(report) if do_enclave_submissions: num_submitted = 0 for staged_report in all_reports: successful = False attempts = 0 while not successful and attempts < 5: attempts += 1 try: report = ts.submit_report(report=staged_report) num_submitted += 1 successful = True print( "Submitted report #%s-%s title %s as TruSTAR IR %s with case ID: %s" % (num_submitted, attempts, report.title, report.id, report.external_id)) print("URL: %s" % ts.get_report_url(report.id)) # Build CEF output: # - HTTP_USER_AGENT is the cs1 field # - example CEF output: CEF:version|vendor|product|device_version|signature|name|severity|cs1=(num_submitted) cs2=(report_url) config = { 'cef.version': '0.5', 'cef.vendor': 'TruSTAR', 'cef.device_version': '2.0', 'cef.product': 'API', 'cef': True, 'cef.file': args.cef_output_file } environ = { 'REMOTE_ADDR': '127.0.0.1', 'HTTP_HOST': '127.0.0.1', 'HTTP_USER_AGENT': report.title } log_cef('SUBMISSION', 1, environ, config, signature="INFO", cs2=report.external_id, cs3=ts.get_report_url(report.id)) #### # TODO: ADD YOUR CUSTOM POST-PROCESSING CODE FOR THIS SUBMISSION HERE #### print() except Exception as e: traceback.print_exc(file=sys.stdout) print("Problem submitting report: %s" % e) time.sleep(5) # Sleep between submissions time.sleep(5)
def attempt_lambda(self, test_event): # type: (Dict) -> Report """ Attempts the lambda. """ logger.info("attempting lambda.") d = lambda_handler(test_event, {}) logger.info("lambda attempt complete, success.") return Report.from_dict(d)