def build_for(self, finding): # type: (Dict) -> Report
""" Builds a Report for an event.
Note: Does NOT validate report attribute values to ensure they
are valid / reasonable. """
detail = finding.get('detail')
if not detail:
raise Exception("Guard Duty Finding dictionary's 'detail' kv pair"
"is empty. This integration builds TruSTAR "
"reports from that key's value.")
title = detail.get('title', self.DEFAULT_TITLE)
body = self._body_from_detail(detail)
time_began = self._time_began_from_detail(detail)
external_url = detail.get('arn')
external_id = self.ext_id_encoder.reversible(self.enclave_id,
detail.get('id'))
r = Report(title=title,
body=body,
time_began=time_began,
external_url=external_url,
external_id=external_id,
enclave_ids=[self.enclave_id])
logger.info("TimeBegan in ReportBuilder after assigning to the "
"report object: '{}'".format(r.time_began))
logger.info("TimeBegan type: '{}'.".format(str(type(r.time_began))))
d = r.to_dict()
logger.info("TimeBegan in ReportBuiilder after converting report to "
"dict: '{}'.".format(d['timeBegan']))
return r
def report(trustar, current_time_millis, milliseconds_in_a_day):
original_report = Report(title="Report 1",
id=FAKE_REPORT_ID,
body="Blah blah blah",
time_began=current_time_millis - milliseconds_in_a_day,
enclave_ids=trustar.enclave_ids)
return original_report
def build_test_report(self, i, time_began):
r = Report()
r.body = self.body
r.title = self.title
r.external_id = self.external_id + str(i)
r.external_url = self.external_url
r.enclave_ids = [self.enclave_id]
#r.set_time_began(time_began)
r.time_began = time_began
return r
def __init__(
self,
sample_event_file_path, # type: str
expected_report_file_path, # type: str
config_file_path, # type: str
delete_reports_when_done=False # type: bool
):
""" Some prep work. """
configs = self.load_configs(config_file_path) # type: dict
self.__env_vars = self.env_vars_from_configs(configs) # type: Dict
self.set_env_vars_to_os(self.__env_vars)
self.ts = self.build_ts_client()
self.test_event = self.dict_from_file(
sample_event_file_path) # type: Dict
self.expected_report = Report.from_dict(
self.dict_from_file(expected_report_file_path)) # type: Dict
self.delete_reports_when_done = delete_reports_when_done # type: bool
def main():
role = "trustar"
if len(sys.argv) > 1:
role = sys.argv[1]
ts = TruStar(config_file="trustar.conf", config_role=role)
# generate random id to use as external_id
external_id = str(randint(1, 100000))
# or use a specific external_id
# external_id = "321"
report_guid = None
current_time = int(time.time()) * 1000
yesterday_time = current_time - to_milliseconds(days=1)
if do_latest_reports:
logger.info("Getting Latest Accessible Incident Reports Since 24 hours ago ...")
try:
# get each successive page of reports
report_generator = ts.get_reports(from_time=yesterday_time,
to_time=current_time,
is_enclave=True,
enclave_ids=ts.enclave_ids)
for report in report_generator:
logger.info(report)
except Exception as e:
logger.error('Could not get latest reports, error: %s' % e)
print('')
if do_reports_by_community:
two_days_ago = current_time - to_milliseconds(days=2)
logger.info("Getting community only reports for the previous day ...")
try:
reports = ts.get_reports(from_time=two_days_ago,
to_time=yesterday_time,
is_enclave=False)
for report in reports:
logger.info(report)
except Exception as e:
logger.error('Could not get community reports, error: %s' % e)
print('')
if do_reports_by_enclave:
a_week_ago = current_time - to_milliseconds(days=7)
logger.info("Getting enclave only reports for the previous week ...")
try:
reports = ts.get_reports(from_time=a_week_ago,
to_time=current_time,
is_enclave=True,
enclave_ids=ts.enclave_ids)
for report in reports:
logger.info(report)
except Exception as e:
logger.error('Could not get community reports, error: %s' % e)
print('')
if do_correlated:
logger.info("Querying Accessible Correlated Reports...")
try:
report_ids = ts.get_correlated_report_ids(search_string)
logger.info(report_ids)
logger.info("%d report(s) correlated with indicators '%s':\n" % (len(report_ids), search_string))
logger.info("\n".join(report_ids))
except Exception as e:
logger.error('Could not get correlated reports, error: %s' % e)
print('')
if do_community_trends:
logger.info("Get community trends...")
try:
indicators = ts.get_community_trends(indicator_type=None,
days_back=1)
for indicator in indicators:
logger.info(indicator)
except Exception as e:
logger.error('Could not get community trends, error: %s' % e)
print('')
if do_query_indicators:
try:
logger.info("Getting related indicators...")
indicators = ts.get_related_indicators(indicators=search_string)
for indicator in indicators:
logger.info(indicator)
except Exception as e:
logger.error('Could not get related indicators, error: %s' % e)
# Submit simple test report to community
if do_comm_submissions:
logger.info("Submit New Community Incident Report")
try:
report = Report(title="COMMUNITY API SUBMISSION TEST",
body=submit_indicators,
time_began="2017-02-01T01:23:45",
is_enclave=False)
report = ts.submit_report(report)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
except Exception as e:
logger.error('Could not submit community report, error: %s' % e)
print('')
# Submit simple test report to your enclave
if do_enclave_submissions:
logger.info("Submit New Enclave Incident Report")
try:
report = Report(title="ENCLAVE API SUBMISSION TEST ",
body=submit_indicators,
time_began="2017-02-01T01:23:45",
enclave_ids=ts.enclave_ids)
report = ts.submit_report(report)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
logger.info(report)
except Exception as e:
logger.error('Could not submit enclave report, error: %s' % e)
print('')
# Submit a test report and retrieve it
if do_submit_report:
logger.info("Submit New Enclave Incident Report with External ID")
try:
report = Report(title="Sample SDK Test Report",
body=submit_indicators,
time_began="2017-02-01T01:23:45",
is_enclave=True,
enclave_ids=ts.enclave_ids,
external_id=external_id)
report = ts.submit_report(report)
logger.info("Report Submitted")
logger.info("\texternalTrackingId: %s" % report.external_id)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
except Exception as e:
logger.error('Could not submit report, error: %s' % e)
print('')
# Get test report previously submitted
if do_report_details_by_ext_id:
logger.info("Get Incident Report By External ID")
try:
report = ts.get_report_details(report_id=external_id, id_type=Report.ID_TYPE_EXTERNAL)
logger.info("\ttitle: %s" % report.title)
logger.info("\texternalTrackingId: %s" % report.external_id)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
report_guid = report.id
except Exception as e:
logger.error('Could not get report, error: %s' % e)
print('')
# Update a test report and test with get report
if do_update_report_by_ext_id:
logger.info("Update Incident Report By External ID")
try:
report = Report(title="Updated Sample Title",
body="updated report body: 21.22.23.24",
external_id=external_id,
enclave_ids=ts.enclave_ids)
report = ts.update_report(report)
logger.info("\texternalTrackingId: %s" % report.external_id)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
except Exception as e:
logger.error('Could not update report, error: %s' % e)
print('')
# Get test report previously submitted
if do_report_details_by_guid:
logger.info("Get Incident Report Details by GUID (TruSTAR internal ID)")
try:
report = ts.get_report_details(report_guid)
logger.info("\ttitle: %s" % report.title)
logger.info("\texternalTrackingId: %s" % report.external_id)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
except Exception as e:
logger.error('Could not get report, error: %s' % e)
print('')
# Update a test report and test with get report
if do_update_report_by_guid:
logger.info("Update Incident Report by GUID (TruSTAR internal ID)")
try:
report = Report(id=report_guid,
title="New Sample Title",
body="new sample body - 7.8.9.10",
enclave_ids=ts.enclave_ids)
report = ts.update_report(report)
logger.info("Updated Report using GUID")
logger.info("\texternalTrackingId: %s" % report.external_id)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
except Exception as e:
logger.error('Could not update report, error: %s' % e)
print('')
# Get test report previously submitted
if do_report_details_by_guid:
logger.info("Get Report by GUID (TruSTAR internal ID)")
try:
report = ts.get_report_details(report_guid)
logger.info("\ttitle: %s" % report.title)
logger.info("\texternalTrackingId: %s" % report.external_id)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
except Exception as e:
logger.error('Could not get report, error: %s' % e)
print('')
# Release report to community
if do_release_report_by_ext_id:
logger.info("Release Incident Report by External ID")
try:
report = Report(external_id=external_id,
is_enclave=False)
report = ts.update_report(report)
logger.info("Report Released using External ID:")
logger.info("\texternalTrackingId: %s" % report.external_id)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
except Exception as e:
logger.error('Could not release report, error: %s' % e)
print('')
# Get test report previously submitted
if do_report_details_by_ext_id_2:
logger.info("Get Incident Report Details by External ID")
try:
report = ts.get_report_details(report_id=external_id, id_type=Report.ID_TYPE_EXTERNAL)
logger.info("\ttitle: %s" % report.title)
logger.info("\texternalTrackingId: %s" % report.external_id)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
except Exception as e:
logger.error('Could not get report, error: %s' % e)
print('')
# Delete test report previously submitted
if do_delete_report_by_ext_id:
logger.info("Delete Incident Report by External ID")
try:
ts.delete_report(report_id=external_id, id_type=Report.ID_TYPE_EXTERNAL)
logger.info("Report Deleted using External ID\n")
except Exception as e:
logger.error('Could not delete report, error: %s' % e)
print('')
# Add an enclave tag to a newly created report
if do_add_enclave_tag:
logger.info("Add enclave tag to incident report")
try:
# submit report
report = Report(title="Enclave report with tag",
body=submit_indicators,
is_enclave=True,
enclave_ids=ts.enclave_ids)
report = ts.submit_report(report)
logger.info("\tId of new report %s\n" % report.id)
# get back report details, including the enclave it's in
report = ts.get_report_details(report_id=report.id)
enclave_id = report.enclave_ids[0]
# add an enclave tag
tag_id = ts.add_enclave_tag(report_id=report.id, name="triage", enclave_id=enclave_id)
# logger.info the added enclave tag
logger.info("\tId of new enclave tag %s\n" % tag_id)
# add another enclave tag
tag_id = ts.add_enclave_tag(report_id=report.id, name="resolved", enclave_id=enclave_id)
# logger.info the added enclave tag
logger.info("\tId of new enclave tag %s\n" % tag_id)
# Get enclave tag info
if do_get_enclave_tags:
logger.info("Get enclave tags for report")
tags = ts.get_enclave_tags(report.id)
logger.info("\tEnclave tags for report %s\n" % report.id)
logger.info(tags)
# delete enclave tag by name
if do_delete_enclave_tag:
logger.info("Delete enclave tag from report")
response = ts.delete_enclave_tag(report.id, tag_id)
logger.info("\tDeleted enclave tag for report %s\n" % report.id)
logger.info(response)
# add it back
ts.add_enclave_tag(report_id=report.id, name="triage", enclave_id=enclave_id)
# List all enclave tags
tags = ts.get_all_enclave_tags(enclave_ids=ts.enclave_ids)
logger.info("List of enclave tags for enclave %s\n" % enclave_id)
logger.info(tags)
# Search report by tag
logger.info("Getting reports tagged 'triage'.")
reports = ts.get_reports(from_time=yesterday_time,
to_time=current_time,
enclave_ids=ts.enclave_ids,
tag="triage")
for report in reports:
logger.info(report)
except Exception as e:
logger.error('Could not handle enclave tag operation, error: %s' % e)
print('')
# search for reports containing term "abc"
if do_search_reports:
try:
logger.info("Searching reports:")
reports = ts.search_reports("abc")
for report in reports:
logger.info(report)
except Exception as e:
logger.error("Could not search reports, error: %s" % e)
print('')
# search for indicators matching pattern "abc"
if do_search_indicators:
try:
logger.info("Searching indicators:")
indicators = ts.search_indicators("abc")
for indicator in indicators:
logger.info(indicator)
except Exception as e:
logger.error("Could not search indicators, error: %s" % e)
print('')
if do_redact_report:
try:
logger.info("Redacting report:")
redacted_report = ts.redact_report(title="amazon phishing scam",
report_body="apple, microsoft, and amazon suffered "
"from a phishing scam via [email protected]")
logger.info(redacted_report)
except Exception as e:
logger.error("Could not redact report, error: %s" % e)
print('')
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=
('Submit one or more reports from local files (txt, pdf, docx, etc) '
'in a directory\n\n'
'Example:\n'
'python bulk_upload.py --dir ./sample_reports --ts_conf ./trustar.conf'
))
parser.add_argument('--dir',
'-d',
help='Path containing report files',
required=True)
parser.add_argument('--ts_config',
'-c',
help='Path containing trustar api config',
nargs='?',
default="./trustar.conf")
parser.add_argument(
'-i',
'--ignore',
dest='ignore',
action='store_true',
help='Ignore history and resubmit already procesed files')
args = parser.parse_args()
source_report_dir = args.dir
ts_config = args.ts_config
ts = TruStar(config_file=ts_config)
# process all files in directory
logger.info(
"Processing and submitting each source file in %s as a TruSTAR Incident Report"
% source_report_dir)
processed_files = set()
processed_files_file = os.path.join(source_report_dir,
"processed_files.log")
if os.path.isfile(processed_files_file) and not args.ignore:
processed_files = set(line.strip()
for line in open(processed_files_file))
skipped_files_file = os.path.join(source_report_dir, "skipped_files.log")
with open(processed_files_file, 'a', 0) as pf:
for (dirpath, dirnames, filenames) in os.walk(source_report_dir):
for source_file in filenames:
if (source_file == "processed_files.log"
or source_file == "skipped_files.log"):
continue
if source_file in processed_files:
logger.debug(
"File {} was already processed. Ignoring.".format(
source_file))
continue
logger.info("Processing source file %s " % source_file)
try:
path = os.path.join(source_report_dir, source_file)
report_body = process_file(path)
if not report_body:
logger.debug(
"File {} ignored for no data".format(source_file))
raise
# response_json = ts.submit_report(token, report_body, "COMMUNITY: " + file)
logger.info("Report {}".format(report_body))
try:
report = Report(title="ENCLAVE: %s" % source_file,
body=report_body,
is_enclave=True,
enclave_ids=ts.enclave_ids)
report = ts.submit_report(report)
logger.info("SUCCESSFULLY SUBMITTED REPORT, " +
"TRUSTAR REPORT as Incident Report ID %s" %
report.id)
pf.write("%s\n" % source_file)
if report.indicators is not None:
print("Extracted the following indicators: {}".
format(
json.dumps([
x.to_dict()
for x in report.indicators
],
indent=2)))
else:
print("No indicators returned from report id {0}".
format(report.id))
except Exception as e:
if '413' in e.message:
logger.warn(
"Could not submit file {}. Contains more indicators than currently supported."
.format(source_file))
else:
raise
except Exception as e:
logger.error("Problem with file %s, exception: %s " %
(source_file, e))
with open(skipped_files_file, 'w', 0) as sf:
sf.write("{}\n".format(source_file))
continue
time.sleep(2)
# mapping of CSV column names to report fields
MAPPING = {"title": "name", "body": "content", "external_id": "id"}
CSV_PATH = "reports.csv"
# initialize SDK
ts = TruStar()
# read in CSV
with open(CSV_PATH, 'r') as f:
reader = csv.DictReader(f)
# iterate over rows
for row in reader:
# define method to get report field from CSV row
def get_field(field):
return row.get(MAPPING.get(field))
# construct report from CSV row
report = Report(title=get_field('title'),
body=get_field('body'),
external_id=get_field('external_id'),
is_enclave=True,
enclave_ids=ts.enclave_ids)
# submit report
ts.submit_report(report)
logger.info("Submitted report: %s" % report)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=
('Submit TruSTAR reports from a CSV file\n'
'Example:\n\n'
'python ingest_csv.py -c "TargetIP,SourceIP,Info,Analysis,Indicators" -t "TrackingNumber" -d "ReportTime" -cn "CaseName" -f reportname.csv'
))
parser.add_argument('-f',
'--file',
required=True,
dest='file_name',
help='csv file to import')
parser.add_argument('-t',
'--title',
required=True,
dest='title_col',
help='Name of column to use as title field')
parser.add_argument('-d',
'--datetime',
required=False,
dest='datetime_col',
help='Name of column to use as report date/time')
parser.add_argument('-c',
'--columns',
required=False,
dest='cols',
help='List of comma-separated column names to include')
parser.add_argument(
'-n',
'--num-reports',
required=False,
dest='num_reports',
type=int,
default=1000,
help='Max number of reports to submit (top-down order)')
parser.add_argument(
'-o',
'--output',
required=False,
dest='cef_output_file',
default='trustar.cef',
help=
'Common Event Format (CEF) output log file, one event is generated per successful submission'
)
parser.add_argument(
'-ci',
'--case-id',
required=False,
dest='caseid_col',
help='Name of column to use as report case ID for CEF export')
args = parser.parse_args()
allowed_keys_content = []
if args.cols:
allowed_keys_content = args.cols.split(",")
ts = TruStar(config_role="trustar")
df = pd.read_csv(args.file_name, nrows=args.num_reports, encoding="latin1")
# Create title and report content from the provided column names (if any)
all_reports = []
for report_num in range(0, len(df)):
current_content = ''
current_title = ''
current_datetime = None
current_case_id = 0
for key in df:
# ignore empty cells, which are float64 NaNs
cell_value = df[key][report_num]
if pd.isnull(cell_value):
continue
cell_value = "%s" % cell_value
# encode any unicode chars
string_value = cell_value.encode('utf-8').strip()
if string_value == "nan":
print("%s -> %s" % (key, string_value))
continue
content = "{}:\n {}\n \n".format(key, string_value)
if not allowed_keys_content or key in allowed_keys_content:
current_content += content
if key == args.title_col:
current_title = str(df[key][report_num])
if key == args.datetime_col:
current_datetime = str(df[key][report_num])
if key == args.caseid_col:
current_case_id = str(df[key][report_num])
report = Report(title=current_title,
time_began=current_datetime,
body=current_content,
external_id=current_case_id,
is_enclave=True,
enclave_ids=ts.enclave_ids)
all_reports.append(report)
if do_enclave_submissions:
num_submitted = 0
for staged_report in all_reports:
successful = False
attempts = 0
while not successful and attempts < 5:
attempts += 1
try:
report = ts.submit_report(report=staged_report)
num_submitted += 1
successful = True
print(
"Submitted report #%s-%s title %s as TruSTAR IR %s with case ID: %s"
% (num_submitted, attempts, report.title, report.id,
report.external_id))
print("URL: %s" % ts.get_report_url(report.id))
# Build CEF output:
# - HTTP_USER_AGENT is the cs1 field
# - example CEF output: CEF:version|vendor|product|device_version|signature|name|severity|cs1=(num_submitted) cs2=(report_url)
config = {
'cef.version': '0.5',
'cef.vendor': 'TruSTAR',
'cef.device_version': '2.0',
'cef.product': 'API',
'cef': True,
'cef.file': args.cef_output_file
}
environ = {
'REMOTE_ADDR': '127.0.0.1',
'HTTP_HOST': '127.0.0.1',
'HTTP_USER_AGENT': report.title
}
log_cef('SUBMISSION',
1,
environ,
config,
signature="INFO",
cs2=report.external_id,
cs3=ts.get_report_url(report.id))
####
# TODO: ADD YOUR CUSTOM POST-PROCESSING CODE FOR THIS SUBMISSION HERE
####
print()
except Exception as e:
traceback.print_exc(file=sys.stdout)
print("Problem submitting report: %s" % e)
time.sleep(5)
# Sleep between submissions
time.sleep(5)
def attempt_lambda(self, test_event): # type: (Dict) -> Report
""" Attempts the lambda. """
logger.info("attempting lambda.")
d = lambda_handler(test_event, {})
logger.info("lambda attempt complete, success.")
return Report.from_dict(d)