def main():
ts = TruStar(config_role="trustar")
token = ts.get_token()
# process all files in directory
print("Processing and submitting each source file in %s as a TruSTAR Incident Report" % SOURCE_REPORT_DIR)
for (dirpath, dirnames, filenames) in os.walk(SOURCE_REPORT_DIR):
for file in filenames:
print("Processing source file %s " % file)
try:
path = os.path.join(SOURCE_REPORT_DIR, file)
report_body_txt = ts.process_file(path)
response_json = ts.submit_report(token, report_body_txt, "COMMUNITY: " + file)
response_json = ts.submit_report(token, report_body_txt, "ENCLAVE: " + file, enclave=True)
report_id = response_json['reportId']
print("SUCCESSFULLY SUBMITTED REPORT, TRUSTAR REPORT as Incident Report ID {0}".format(report_id))
if 'reportIndicators' in response_json:
print("Extracted the following indicators: {}".format(response_json['reportIndicators']))
else:
print("No indicators returned from report id {0}".format(report_id))
if 'correlatedIndicators' in response_json:
print(
"Extracted the following correlated indicators: {}".format(
response_json['correlatedIndicators']))
else:
print("No correlatedIndicators found in report id {0}".format(report_id))
except:
print("Problem with file %s, exception: " % file)
continue
time.sleep(2)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=('Query TruSTAR indicators and store them in a CSV file\n'
'Example:\n\n'
'python latest_indicators.py -s' +
' INCIDENT_REPORT -t "IP,URL" -l 50 -i 12'))
parser.add_argument('-s',
'--source',
required=True,
dest='source',
help='Source can be INCIDENT_REPORT or OSINT')
parser.add_argument('-t',
'--types',
required=False,
dest='types',
help='Types of indicators')
parser.add_argument('-l',
'--limit',
required=False,
dest='limit',
help='Limit on the returned number of indicators')
parser.add_argument('-i',
'--intervalSize',
required=False,
dest='interval_size',
help='Interval size in hours')
parser.add_argument('-f', '--fileName', required=False, dest='file_name')
ts = TruStar(config_role="trustar")
args = parser.parse_args()
source_type = args.source
if args.types:
indicator_types = args.types.split(",")
else:
indicator_types = DEFAULT_TYPE_STRING
if args.limit:
limit = args.limit
else:
limit = DEFAULT_LIMIT
if args.interval_size:
interval_size = args.interval_size
else:
interval_size = DEFAULT_INTERVAL_SIZE
if args.file_name:
file_name = args.file_name
else:
file_name = FILE_NAME
response = ts.query_latest_indicators(ts.get_token(), source_type,
indicator_types, limit,
interval_size)
print(json.dumps(response, indent=2))
save_to_file(response['indicators'], file_name, source_type,
indicator_types)
def from_env_vars(cls, client_metatag): # type: (str) -> TruStar
""" Builds TruStar client. """
if not client_metatag:
raise Exception("must specify a client_metatag.")
config = {param: os.environ.get(param.upper()) for param in
cls.TRUSTAR_CLIENT_PARAMS}
config['client_metatag'] = client_metatag
return TruStar(config=config)
def __init__(self, attribute, config):
config['enclave_ids'] = config.get('enclave_ids',
"").strip().split(',')
config['client_metatag'] = self.CLIENT_METATAG
self.ts_client = TruStar(config=config)
self.misp_event = MISPEvent()
self.misp_attribute = MISPAttribute()
self.misp_attribute.from_dict(**attribute)
self.misp_event.add_attribute(**self.misp_attribute)
def build_ts_client(self):
""" Builds TruStar client using configs from environ vars. """
logger.info("Building TruStar client.")
ts_config = {
'user_api_key': os.environ['USER_API_KEY'],
'user_api_secret': os.environ['USER_API_SECRET'],
'client_metatag': TruStarGuardDutyLambdaHandler.CLIENT_METATAG
}
ts = TruStar(config=ts_config)
logger.info("Done building TruStar client.")
return ts
def __init__(
self,
config_file_path, # type: str
config_stanza): # type: (..., str) -> None
configs = ConfigLoader.from_(config_file_path,
config_stanza) # type: Dict
configs = {k.lower(): v for k, v in configs.items()}
configs[
'client_metatag'] = TruStarGuardDutyLambdaHandler.CLIENT_METATAG
self.ts = TruStar(config=configs)
self.ts.logger.setLevel("DEBUG")
"""
ch = StreamHandler()
ch.setLevel("DEBUG")
self.ts.logger.addHandler(ch)
"""
self.enclave_id = configs['enclave_id']
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=(
'Submit one or more reports from local files (txt, pdf, docx, etc) '
'in a directory\n\n'
'Example:\n'
'python bulk_upload.py ./sample_reports'))
parser.add_argument('dir', help='Path containing report files')
parser.add_argument(
'-i',
'--ignore',
dest='ignore',
action='store_true',
help='Ignore history and resubmit already procesed files')
args = parser.parse_args()
source_report_dir = args.dir
ts = TruStar(config_role="trustar")
token = ts.get_token()
# process all files in directory
print(
"Processing and submitting each source file in %s as a TruSTAR Incident Report"
% source_report_dir)
processed_files = set()
processed_files_file = os.path.join(source_report_dir,
"processed_files.log")
if os.path.isfile(processed_files_file) and not args.ignore:
processed_files = set(line.strip()
for line in open(processed_files_file))
with open(processed_files_file, 'a', 0) as pf:
for (dirpath, dirnames, filenames) in os.walk(source_report_dir):
for source_file in filenames:
if source_file in processed_files:
continue
print("Processing source file %s " % source_file)
try:
path = os.path.join(source_report_dir, source_file)
report_body_txt = process_file(path)
# response_json = ts.submit_report(token, report_body_txt, "COMMUNITY: " + file)
response_json = ts.submit_report(token,
report_body_txt,
"ENCLAVE: " + source_file,
enclave=True)
report_id = response_json['reportId']
print(
"SUCCESSFULLY SUBMITTED REPORT, TRUSTAR REPORT as Incident Report ID %s"
% report_id)
pf.write("%s\n" % source_file)
# if 'reportIndicators' in response_json:
# print("Extracted the following indicators: {}".format(response_json['reportIndicators']))
# else:
# print("No indicators returned from report id {0}".format(report_id))
#
# # if 'correlatedIndicators' in response_json:
# print(
# "Extracted the following correlated indicators: {}".format(
# response_json['correlatedIndicators']))
# else:
# print("No correlatedIndicators found in report id {0}".format(report_id))
except Exception as e:
print("Problem with file %s, exception: %s " %
(source_file, e))
continue
time.sleep(2)
from trustar import TruStar, get_logger, datetime_to_millis
from datetime import datetime, timedelta
import csv
# initialize SDK
ts = TruStar()
# initialize logger
logger = get_logger(__name__)
# set 'from' to the start of yesterday and 'to' to the end of yesterday
to_time = datetime.now()
from_time = to_time - timedelta(days=7)
# convert to millis since epoch
to_time = datetime_to_millis(to_time)
from_time = datetime_to_millis(from_time)
# define CSV column names
HEADERS = [
"report_id", "report_title", "report_tags", "indicator_value",
"indicator_type"
]
# open the output csv to create it
with open('indicators.csv', 'w') as f:
# create csv writer object
writer = csv.DictWriter(f, HEADERS)
# write header row
writer.writeheader()
def main():
role = "trustar"
if len(sys.argv) > 1:
role = sys.argv[1]
ts = TruStar(config_file="trustar.conf", config_role=role)
# generate random id to use as external_id
external_id = str(randint(1, 100000))
# or use a specific external_id
# external_id = "321"
report_guid = None
current_time = int(time.time()) * 1000
yesterday_time = current_time - to_milliseconds(days=1)
if do_latest_reports:
logger.info("Getting Latest Accessible Incident Reports Since 24 hours ago ...")
try:
# get each successive page of reports
report_generator = ts.get_reports(from_time=yesterday_time,
to_time=current_time,
is_enclave=True,
enclave_ids=ts.enclave_ids)
for report in report_generator:
logger.info(report)
except Exception as e:
logger.error('Could not get latest reports, error: %s' % e)
print('')
if do_reports_by_community:
two_days_ago = current_time - to_milliseconds(days=2)
logger.info("Getting community only reports for the previous day ...")
try:
reports = ts.get_reports(from_time=two_days_ago,
to_time=yesterday_time,
is_enclave=False)
for report in reports:
logger.info(report)
except Exception as e:
logger.error('Could not get community reports, error: %s' % e)
print('')
if do_reports_by_enclave:
a_week_ago = current_time - to_milliseconds(days=7)
logger.info("Getting enclave only reports for the previous week ...")
try:
reports = ts.get_reports(from_time=a_week_ago,
to_time=current_time,
is_enclave=True,
enclave_ids=ts.enclave_ids)
for report in reports:
logger.info(report)
except Exception as e:
logger.error('Could not get community reports, error: %s' % e)
print('')
if do_correlated:
logger.info("Querying Accessible Correlated Reports...")
try:
report_ids = ts.get_correlated_report_ids(search_string)
logger.info(report_ids)
logger.info("%d report(s) correlated with indicators '%s':\n" % (len(report_ids), search_string))
logger.info("\n".join(report_ids))
except Exception as e:
logger.error('Could not get correlated reports, error: %s' % e)
print('')
if do_community_trends:
logger.info("Get community trends...")
try:
indicators = ts.get_community_trends(indicator_type=None,
days_back=1)
for indicator in indicators:
logger.info(indicator)
except Exception as e:
logger.error('Could not get community trends, error: %s' % e)
print('')
if do_query_indicators:
try:
logger.info("Getting related indicators...")
indicators = ts.get_related_indicators(indicators=search_string)
for indicator in indicators:
logger.info(indicator)
except Exception as e:
logger.error('Could not get related indicators, error: %s' % e)
# Submit simple test report to community
if do_comm_submissions:
logger.info("Submit New Community Incident Report")
try:
report = Report(title="COMMUNITY API SUBMISSION TEST",
body=submit_indicators,
time_began="2017-02-01T01:23:45",
is_enclave=False)
report = ts.submit_report(report)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
except Exception as e:
logger.error('Could not submit community report, error: %s' % e)
print('')
# Submit simple test report to your enclave
if do_enclave_submissions:
logger.info("Submit New Enclave Incident Report")
try:
report = Report(title="ENCLAVE API SUBMISSION TEST ",
body=submit_indicators,
time_began="2017-02-01T01:23:45",
enclave_ids=ts.enclave_ids)
report = ts.submit_report(report)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
logger.info(report)
except Exception as e:
logger.error('Could not submit enclave report, error: %s' % e)
print('')
# Submit a test report and retrieve it
if do_submit_report:
logger.info("Submit New Enclave Incident Report with External ID")
try:
report = Report(title="Sample SDK Test Report",
body=submit_indicators,
time_began="2017-02-01T01:23:45",
is_enclave=True,
enclave_ids=ts.enclave_ids,
external_id=external_id)
report = ts.submit_report(report)
logger.info("Report Submitted")
logger.info("\texternalTrackingId: %s" % report.external_id)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
except Exception as e:
logger.error('Could not submit report, error: %s' % e)
print('')
# Get test report previously submitted
if do_report_details_by_ext_id:
logger.info("Get Incident Report By External ID")
try:
report = ts.get_report_details(report_id=external_id, id_type=Report.ID_TYPE_EXTERNAL)
logger.info("\ttitle: %s" % report.title)
logger.info("\texternalTrackingId: %s" % report.external_id)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
report_guid = report.id
except Exception as e:
logger.error('Could not get report, error: %s' % e)
print('')
# Update a test report and test with get report
if do_update_report_by_ext_id:
logger.info("Update Incident Report By External ID")
try:
report = Report(title="Updated Sample Title",
body="updated report body: 21.22.23.24",
external_id=external_id,
enclave_ids=ts.enclave_ids)
report = ts.update_report(report)
logger.info("\texternalTrackingId: %s" % report.external_id)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
except Exception as e:
logger.error('Could not update report, error: %s' % e)
print('')
# Get test report previously submitted
if do_report_details_by_guid:
logger.info("Get Incident Report Details by GUID (TruSTAR internal ID)")
try:
report = ts.get_report_details(report_guid)
logger.info("\ttitle: %s" % report.title)
logger.info("\texternalTrackingId: %s" % report.external_id)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
except Exception as e:
logger.error('Could not get report, error: %s' % e)
print('')
# Update a test report and test with get report
if do_update_report_by_guid:
logger.info("Update Incident Report by GUID (TruSTAR internal ID)")
try:
report = Report(id=report_guid,
title="New Sample Title",
body="new sample body - 7.8.9.10",
enclave_ids=ts.enclave_ids)
report = ts.update_report(report)
logger.info("Updated Report using GUID")
logger.info("\texternalTrackingId: %s" % report.external_id)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
except Exception as e:
logger.error('Could not update report, error: %s' % e)
print('')
# Get test report previously submitted
if do_report_details_by_guid:
logger.info("Get Report by GUID (TruSTAR internal ID)")
try:
report = ts.get_report_details(report_guid)
logger.info("\ttitle: %s" % report.title)
logger.info("\texternalTrackingId: %s" % report.external_id)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
except Exception as e:
logger.error('Could not get report, error: %s' % e)
print('')
# Release report to community
if do_release_report_by_ext_id:
logger.info("Release Incident Report by External ID")
try:
report = Report(external_id=external_id,
is_enclave=False)
report = ts.update_report(report)
logger.info("Report Released using External ID:")
logger.info("\texternalTrackingId: %s" % report.external_id)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
except Exception as e:
logger.error('Could not release report, error: %s' % e)
print('')
# Get test report previously submitted
if do_report_details_by_ext_id_2:
logger.info("Get Incident Report Details by External ID")
try:
report = ts.get_report_details(report_id=external_id, id_type=Report.ID_TYPE_EXTERNAL)
logger.info("\ttitle: %s" % report.title)
logger.info("\texternalTrackingId: %s" % report.external_id)
logger.info("\tURL: %s\n" % ts.get_report_url(report.id))
except Exception as e:
logger.error('Could not get report, error: %s' % e)
print('')
# Delete test report previously submitted
if do_delete_report_by_ext_id:
logger.info("Delete Incident Report by External ID")
try:
ts.delete_report(report_id=external_id, id_type=Report.ID_TYPE_EXTERNAL)
logger.info("Report Deleted using External ID\n")
except Exception as e:
logger.error('Could not delete report, error: %s' % e)
print('')
# Add an enclave tag to a newly created report
if do_add_enclave_tag:
logger.info("Add enclave tag to incident report")
try:
# submit report
report = Report(title="Enclave report with tag",
body=submit_indicators,
is_enclave=True,
enclave_ids=ts.enclave_ids)
report = ts.submit_report(report)
logger.info("\tId of new report %s\n" % report.id)
# get back report details, including the enclave it's in
report = ts.get_report_details(report_id=report.id)
enclave_id = report.enclave_ids[0]
# add an enclave tag
tag_id = ts.add_enclave_tag(report_id=report.id, name="triage", enclave_id=enclave_id)
# logger.info the added enclave tag
logger.info("\tId of new enclave tag %s\n" % tag_id)
# add another enclave tag
tag_id = ts.add_enclave_tag(report_id=report.id, name="resolved", enclave_id=enclave_id)
# logger.info the added enclave tag
logger.info("\tId of new enclave tag %s\n" % tag_id)
# Get enclave tag info
if do_get_enclave_tags:
logger.info("Get enclave tags for report")
tags = ts.get_enclave_tags(report.id)
logger.info("\tEnclave tags for report %s\n" % report.id)
logger.info(tags)
# delete enclave tag by name
if do_delete_enclave_tag:
logger.info("Delete enclave tag from report")
response = ts.delete_enclave_tag(report.id, tag_id)
logger.info("\tDeleted enclave tag for report %s\n" % report.id)
logger.info(response)
# add it back
ts.add_enclave_tag(report_id=report.id, name="triage", enclave_id=enclave_id)
# List all enclave tags
tags = ts.get_all_enclave_tags(enclave_ids=ts.enclave_ids)
logger.info("List of enclave tags for enclave %s\n" % enclave_id)
logger.info(tags)
# Search report by tag
logger.info("Getting reports tagged 'triage'.")
reports = ts.get_reports(from_time=yesterday_time,
to_time=current_time,
enclave_ids=ts.enclave_ids,
tag="triage")
for report in reports:
logger.info(report)
except Exception as e:
logger.error('Could not handle enclave tag operation, error: %s' % e)
print('')
# search for reports containing term "abc"
if do_search_reports:
try:
logger.info("Searching reports:")
reports = ts.search_reports("abc")
for report in reports:
logger.info(report)
except Exception as e:
logger.error("Could not search reports, error: %s" % e)
print('')
# search for indicators matching pattern "abc"
if do_search_indicators:
try:
logger.info("Searching indicators:")
indicators = ts.search_indicators("abc")
for indicator in indicators:
logger.info(indicator)
except Exception as e:
logger.error("Could not search indicators, error: %s" % e)
print('')
if do_redact_report:
try:
logger.info("Redacting report:")
redacted_report = ts.redact_report(title="amazon phishing scam",
report_body="apple, microsoft, and amazon suffered "
"from a phishing scam via [email protected]")
logger.info(redacted_report)
except Exception as e:
logger.error("Could not redact report, error: %s" % e)
print('')
from trustar import TruStar, datetime_to_millis
from datetime import datetime, timedelta
from keys import misp_url, misp_key, misp_verifycert
from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation
# TODO: Work on reducing false positive attributes
# (e.g. email addresses from RH-ISAC members being pulled from TruStar reports)
# Signature whitelisting can be done on MISP with regex entries (in the standard php
# regex /{regex}/{modifier} format) entered on http://MISP.local/admin/whitelists/index
# to restrict matching attributes from being included in the IDS flag sensitive exports
# initialize TruStar()
tru = TruStar()
# initialize ExpandedPyMISP()
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
now = datetime.now()
# date range for pulling reports can be hours= or days=
to_time = datetime.now()
from_time = to_time - timedelta(hours=6)
# convert to millis since epoch
to_time = datetime_to_millis(to_time)
from_time = datetime_to_millis(from_time)
rhisac = "7a33144f-aef3-442b-87d4-dbf70d8afdb0" # TruStar RH-ISAC enclave uuid
reports = tru.get_reports(from_time=from_time,
to_time=to_time,
def main(inputfile):
ts = TruStar()
token = ts.get_token()
df = pd.read_json(process_alert(inputfile))
process_time = time.strftime('%Y-%m-%d %H:%M', time.localtime(time.time()))
filtered_falsepositive = filter_false_positive(df, process_time)
filtered_winmethodology = filter_win_methodology(filtered_falsepositive,
process_time)
filtered_bashshellshock = filter_bash_shellshock(filtered_winmethodology,
process_time)
filtered_data = filter_webapp_attack(filtered_bashshellshock, process_time)
all_reports = []
for alert in filtered_data:
title = str(alert['displayId']) + ' ' + str(
alert['message'].encode('utf-8'))
content = ""
for key in alert:
type_value = type(alert[key])
if type_value == list or type_value == int or type_value == long or type_value == bool \
or type_value == dict or alert[key] is None:
content += key + ': ' + str(alert[key]).replace('u\'',
'\'') + '\n'
else:
content += key + ': ' + str(alert[key].encode(
'ascii', 'ignore')) + '\n'
created_time = str(alert['createDate'])
current_report = {
'reportTitle': title,
'reportContent': content,
'reportDateTime': created_time
}
all_reports.append(current_report)
if do_enclave_submissions:
for staged_report in all_reports:
start_time = time.time()
response = ts.submit_report(
token,
staged_report['reportContent'],
staged_report['reportTitle'],
discovered_time_str=staged_report['reportDateTime'],
enclave=True)
print(response)
if 'error' in response:
print("Submission failed with error: {}, {}".format(
response['error'], response['message']))
if response['error'] in ("Internal Server Error",
"Access token expired",
"Authentication error"):
print("Auth token expired, requesting new one")
token = ts.get_token()
else:
raise Exception
else:
end_time = time.time()
delta_time = end_time - start_time
print("Submitted report title {} as TruSTAR IR {}".format(
staged_report['reportTitle'], response['reportId']) +
" Time:" + str(delta_time))
if 'reportIndicators' in response and len(
response['reportIndicators']) > 0:
print("Extracted the following indicators: {}".format(
json.dumps(response['reportIndicators'])))
print()
time.sleep(3)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=
('Submit TruSTAR reports from a CSV file\n'
'Example:\n\n'
'python ingest_csv.py -c "TargetIP,SourceIP,Info,Analysis,Indicators" -t "TrackingNumber" -f august_incident_report.csv'
))
parser.add_argument('-f',
'--file',
required=True,
dest='file_name',
help='csv file to import')
parser.add_argument('-t',
'--title',
required=True,
dest='title_col',
help='Name of column to use as title field')
parser.add_argument('-d',
'--datetime',
required=False,
dest='datetime_col',
help='Name of column to use as report date/time')
parser.add_argument('-c',
'--columns',
required=False,
dest='cols',
help='List of comma-separated column names to include')
parser.add_argument(
'-n',
'--num-reports',
required=False,
dest='num_reports',
type=int,
default=1000,
help='Max number of reports to submit (top-down order)')
args = parser.parse_args()
allowed_keys_content = []
if args.cols:
allowed_keys_content = args.cols.split(",")
# noinspection PyCallingNonCallable
ts = TruStar(config_role="trustar")
token = ts.get_token()
df = pd.read_csv(args.file_name, nrows=args.num_reports)
# Create title and report content from the provided column names (if any)
all_reports = []
for report_num in range(0, len(df)):
current_content = ''
current_title = ''
current_datetime = None
current_report = {}
for key in df:
content = "{}:\n {}\n ".format(key, str(df[key][report_num]))
if not allowed_keys_content or key in allowed_keys_content:
current_content += content
if key == args.title_col:
current_title = str(df[key][report_num])
if key == args.datetime_col:
current_datetime = str(df[key][report_num])
current_report['reportTitle'] = current_title
current_report['reportDateTime'] = current_datetime
current_report['reportContent'] = current_content
all_reports.append(current_report)
if do_enclave_submissions:
num_submitted = 0
for staged_report in all_reports:
successful = False
attempts = 0
while not successful and attempts < 5:
attempts += 1
try:
response = ts.submit_report(
token,
staged_report['reportContent'],
staged_report['reportTitle'],
discovered_time_str=staged_report['reportDateTime'],
enclave=True)
if 'error' in response:
print("Submission failed with error: {}, {}".format(
response['error'], response['message']))
# if response['message'] == "Access token expired":
if response['error'] in ("Internal Server Error",
"Access token expired",
"Authentication error"):
print("Auth token expired, requesting new one")
token = ts.get_token()
else:
raise Exception
else:
num_submitted += 1
successful = True
print(
"Submitted report #{}-{} title {} as TruSTAR IR {}"
.format(num_submitted, attempts,
staged_report['reportTitle'],
response['reportId']))
if 'reportIndicators' in response and len(
response['reportIndicators']) > 0:
print("Extracted the following indicators: {}".format(
json.dumps(response['reportIndicators'])))
print()
except:
e = sys.exc_info()[0]
print("Problem submitting report")
time.sleep(5)
# Sleep between submissions
time.sleep(90)
def trustar():
return TruStar(config_role='staging')
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=
('Submit one or more reports from local files (txt, pdf, docx, etc) '
'in a directory\n\n'
'Example:\n'
'python bulk_upload.py --dir ./sample_reports --ts_conf ./trustar.conf'
))
parser.add_argument('--dir',
'-d',
help='Path containing report files',
required=True)
parser.add_argument('--ts_config',
'-c',
help='Path containing trustar api config',
nargs='?',
default="./trustar.conf")
parser.add_argument(
'-i',
'--ignore',
dest='ignore',
action='store_true',
help='Ignore history and resubmit already procesed files')
args = parser.parse_args()
source_report_dir = args.dir
ts_config = args.ts_config
ts = TruStar(config_file=ts_config)
# process all files in directory
logger.info(
"Processing and submitting each source file in %s as a TruSTAR Incident Report"
% source_report_dir)
processed_files = set()
processed_files_file = os.path.join(source_report_dir,
"processed_files.log")
if os.path.isfile(processed_files_file) and not args.ignore:
processed_files = set(line.strip()
for line in open(processed_files_file))
skipped_files_file = os.path.join(source_report_dir, "skipped_files.log")
with open(processed_files_file, 'a', 0) as pf:
for (dirpath, dirnames, filenames) in os.walk(source_report_dir):
for source_file in filenames:
if (source_file == "processed_files.log"
or source_file == "skipped_files.log"):
continue
if source_file in processed_files:
logger.debug(
"File {} was already processed. Ignoring.".format(
source_file))
continue
logger.info("Processing source file %s " % source_file)
try:
path = os.path.join(source_report_dir, source_file)
report_body = process_file(path)
if not report_body:
logger.debug(
"File {} ignored for no data".format(source_file))
raise
# response_json = ts.submit_report(token, report_body, "COMMUNITY: " + file)
logger.info("Report {}".format(report_body))
try:
report = Report(title="ENCLAVE: %s" % source_file,
body=report_body,
is_enclave=True,
enclave_ids=ts.enclave_ids)
report = ts.submit_report(report)
logger.info("SUCCESSFULLY SUBMITTED REPORT, " +
"TRUSTAR REPORT as Incident Report ID %s" %
report.id)
pf.write("%s\n" % source_file)
if report.indicators is not None:
print("Extracted the following indicators: {}".
format(
json.dumps([
x.to_dict()
for x in report.indicators
],
indent=2)))
else:
print("No indicators returned from report id {0}".
format(report.id))
except Exception as e:
if '413' in e.message:
logger.warn(
"Could not submit file {}. Contains more indicators than currently supported."
.format(source_file))
else:
raise
except Exception as e:
logger.error("Problem with file %s, exception: %s " %
(source_file, e))
with open(skipped_files_file, 'w', 0) as sf:
sf.write("{}\n".format(source_file))
continue
time.sleep(2)
def main():
ts = TruStar(config_role="trustar")
token = ts.get_token()
if do_latest:
print("Get Latest Reports")
results = ts.get_latest_reports(token)
for result in results:
print("\t{}, {}, {}".format(result['id'],
result['distributionType'],
result['title']))
print()
if do_correlated:
print("Querying Correlated Reports")
results = ts.get_correlated_reports(token, query_indicators)
print("{} report(s) correlated with indicators '{}': ".format(
len(results), query_indicators))
for result in results:
print("\t%s" % result)
print()
if do_query_indicator:
print("Querying correlated indicators with '{}' (first 100)".format(
query_indicators))
results = ts.query_indicator(token, query_indicators, "100")
print("Correlated Incident Report indicators:")
for indicator_type, indicator_list in list(
results["indicators"].items()):
print("\n%s:\n\t%s" % (indicator_type, "\n\t".join(
['{}'.format(value) for value in indicator_list])))
print()
print("Correlated Open Source documents:")
for os_url in list(results["openSourceCorrelations"]):
print("\t%s" % (os_url))
print()
print("External Intelligence hits:")
for exint_url in list(results["externalIntelligence"]):
print("\t%s" % (exint_url))
print()
# Submit simple test report to community
if do_comm_submissions:
community_response = ts.submit_report(
token, submit_indicators, "COMMUNITY API SUBMISSION TEST ")
print("Community submission response: {0}".format(
json.dumps(community_response)))
if 'reportIndicators' in community_response:
print("Extracted the following community indicators: {}".format(
community_response['reportIndicators']))
# Submit simple test report to your enclave
if do_enclave_submissions:
enclave_response = ts.submit_report(token,
submit_indicators,
"ENCLAVE API SUBMISSION TEST ",
enclave=True)
print("Enclave submission response: {0}".format(
json.dumps(enclave_response)))
if 'reportIndicators' in enclave_response:
print("Extracted the following enclave indicators: {}".format(
enclave_response['reportIndicators']))
def main():
parser = argparse.ArgumentParser(
description=
"Retrieve IOCs from TruSTAR and enrich them. Filter based on time range, enclave(s), and tag(s). Just use -r to retrieve IOCs submitted since midnight last night."
)
parser.add_argument(
"-r",
"--retrieve",
action="store_true",
help=
"Retrieve IOCs based on the start/end dates, enclaves, and tags. Mutually exclusive with -i/--iocs."
)
parser.add_argument(
"-s",
"--startdate",
help=
"Starting date (YYYY-MM-DD) to search for IOCs. (default start of today)"
)
parser.add_argument(
"-e",
"--enddate",
help=
"Ending date (YYYY-MM-DD) to search for IOCs (default end of today)")
parser.add_argument(
"-n",
"--enclaves",
nargs="+",
help=
"Space separated list of enclaves to search (default BCBS-Domain), '-n s' to select enclaves from a list of valid enclaves."
)
parser.add_argument(
"-t",
"--tags",
nargs="+",
help=
"Space separated list of tags that IOCs must be tagged with, default is to return IOCs regardless of tags. '-t s' to select tags from a list of valid tags."
)
parser.add_argument(
"-i",
"--iocs",
nargs="+",
help=
"Get enrichment data for IOCs. Specify a space separated list of IOCs. Mutually exclusive with -r/--retrieve."
)
parser.add_argument("-o",
"--output",
help="Filename of file where results are to be saved.")
parser.add_argument("-c",
"--config",
help="Trustar configuration file.",
default=os.getcwd() + '/trustar.conf')
args = parser.parse_args()
if not args.iocs and not args.retrieve:
parser.error("Must specify either -r/--retrieve or -i/--iocs.")
if args.iocs and (args.retrieve or args.startdate or args.enddate
or args.enclaves or args.tags):
parser.error(
"Ony IOC(s) can be specified when -i/--iocs is used. No other options are valid when -i/--iocs is specified."
)
if args.startdate:
try:
starttime = calendar.timegm(
parse_date(args.startdate).timetuple()) * 1000
except:
parser.error("Invalid start date specified {}.".format(
args.startdate))
else:
# The default start date is the earliest time today.
# e.g. the start date on 2018-06-19 is 00:00:00 2018-06-19
t = time.gmtime()
starttime = calendar.timegm((t.tm_year, t.tm_mon, t.tm_mday, 0, 0, 0,
t.tm_wday, t.tm_yday, t.tm_isdst)) * 1000
if args.enddate:
# Must add 1 to the date so that the end date is actually 00:00:00 on the next day.
# This ensures getting all of the data on enddate.
try:
endtime = parse_date(args.enddate) + datetime.timedelta(days=1)
endtime = calendar.timegm(endtime.timetuple()) * 1000
except:
parser.error("Invalid end date specified {}.".format(args.enddate))
else:
endtime = int(time.time()) * 1000
try:
ts = TruStar(config_file=args.config, config_role="trustar")
except Exception as e:
print('Config file not found or invalid: {}'.format(args.config))
sys.exit(-1)
enclaves = {}
enclaveNames = {}
for enclave in ts.get_user_enclaves():
if enclave.read:
enclaves[enclave.name] = enclave.id
enclaveNames[enclave.id] = enclave.name
enclaveSelections = []
if args.enclaves:
if 's' in args.enclaves:
enclaveSelections = selectFromList(enclaves, "Enclaves")
if enclaveSelections == None:
sys.exit()
else:
for enclave in args.enclaves:
if enclave in enclaves:
enclaveSelections.append(enclaves[enclave])
else:
print("Invalid enclave: {}".format(enclave))
sys.exit()
tagIds = getTagIds(ts, enclaveSelections)
tagSelections = []
if args.tags:
if 's' in args.tags:
tagSelections = selectFromList(tagIds, "Tags")
if tagSelections == None:
sys.exit()
else:
for tag in args.tags:
tagSelections.append(tagIds[tag])
outputFile = sys.stdout
if args.output:
outputFile = open(args.output, "w")
if args.iocs:
for ioc in args.iocs:
outputIOC(ts, enclaveNames, ioc, outputFile)
else:
# print("enclaves: {}".format(enclaveSelections), file=sys.stderr)
# print("tags: {}".format(tagSelections), file=sys.stderr)
# print(starttime)
# print(endtime)
# print(enclaveSelections)
# print(tagSelections)
indicators = ts.get_indicators(from_time=starttime,
to_time=endtime,
enclave_ids=enclaveSelections,
included_tag_ids=tagSelections,
page_size=500)
for indicator in indicators:
print("{},{}".format(indicator.value, indicator.type),
file=outputFile)
if args.output:
outputFile.close()
def main():
ts = TruStar(config_role="trustar")
token = ts.get_token()
if do_latest_reports:
print("Getting Latest Accessible Reports...")
results = ts.get_latest_reports(token)
for result in results:
print("\t%s, %s, %s" %
(result['id'], result['distributionType'], result['title']))
print()
if do_correlated:
print("Querying Accessible Correlated Reports...")
results = ts.get_correlated_reports(token, search_string)
print("%d report(s) correlated with indicators '%s':\n" %
(len(results), search_string))
print("\n".join(results))
print()
if do_latest_indicators:
print("Get Latest Indicators (first 100)")
results = ts.query_latest_indicators(token,
source='INCIDENT_REPORT',
indicator_types='ALL',
interval_size=24,
limit=100)
if 'indicators' in results:
for ioc_type, value in results['indicators'].items():
if len(value) > 0:
print("\t%s: %s" % (ioc_type, ','.join(value)))
print()
if do_report_details:
print("Get Report Details")
reports = ts.get_latest_reports(token)
for report in reports:
result = ts.get_report_details(token, report['id'])
print("Getting Report Details using '%s': \n%s" %
(report['id'], json.dumps(result, indent=4)))
print()
if do_query_indicators:
print(
"Querying correlated indicators with search string '%s' (first 100)"
% search_string)
results = ts.query_indicators(token, search_string, '100')
indicator_hits = list(results["indicators"])
if len(indicator_hits) > 0:
print("Correlated Incident Report Indicators:")
for indicator_type, indicator_list in list(
results["indicators"].items()):
print("\n%s:\n\t%s" % (indicator_type, "\n\t".join(
['{}'.format(value) for value in indicator_list])))
print()
os_hits = list(results["openSourceCorrelations"])
if len(os_hits) > 0:
print("Correlated Open Source Documents:")
for os_url in os_hits:
print("\t%s" % os_url)
print()
exint_hits = list(results["externalIntelligence"])
if len(exint_hits) > 0:
print("External Intelligence hits:")
print('\t'.join(exint_hits))
print()
# Submit simple test report to community
if do_comm_submissions:
community_response = ts.submit_report(token,
submit_indicators,
"COMMUNITY API SUBMISSION TEST",
began_time="2017-02-01T01:23:45")
print("\tURL: %s\n" %
ts.get_report_url(community_response['reportId']))
if 'reportIndicators' in community_response:
print("Extracted the following community indicators: \n%s\n" %
json.dumps(community_response['reportIndicators'], indent=2))
# Submit simple test report to your enclave
if do_enclave_submissions:
enclave_response = ts.submit_report(token,
submit_indicators,
"ENCLAVE API SUBMISSION TEST ",
enclave=True)
print("\tURL: %s\n" % ts.get_report_url(enclave_response['reportId']))
print(enclave_response)
if 'reportIndicators' in enclave_response:
print("Extracted the following enclave indicators: \n%s\n" %
json.dumps(enclave_response['reportIndicators'], indent=2))
def main():
ts = TruStar(config_role="trustar")
token = ts.get_token(verify=verify)
# generate random id to use as external_id
external_id = str(randint(1, 100000))
# or use a specific external_id
# external_id = "321"
report_guid = None
# Submit a test report and retrieve it
if do_submit_report:
print("Submit Report")
submission_response = ts.submit_report_v12(token, submit_indicators, "Sample SDK Test Report", external_id=external_id,
began_time="2017-02-01T01:23:45", enclave=True, verify=verify)
print("Report Submitted")
print("\texternalTrackingId: %s" % submission_response['externalTrackingId'])
print("\tindicators: %s" % submission_response['reportIndicators'])
print("\tURL: %s\n" % ts.get_report_url(submission_response['reportId']))
# Get test report previously submitted
if do_report_details_by_ext_id:
print("Get Report")
result = ts.get_report_details_v12(token, external_id, id_type="external", verify=verify)
print("Report Details using External ID")
print("\ttitle: %s" % result['title'])
print("\texternalTrackingId: %s" % result['externalTrackingId'])
print("\tindicators: %s" % result['indicators'])
print("\tURL: %s\n" % ts.get_report_url(result['id']))
report_guid = result['id']
# Update a test report and test with get report
if do_update_report_by_ext_id:
print("Update Report")
title = "NEW CC REPORT"
body = "updated report body: 21.22.23.24"
update_response = ts.update_report(token, external_id, id_type="external", title=title, report_body=body, verify=verify)
print("Updated Report using External ID")
print("\texternalTrackingId: %s" % update_response['externalTrackingId'])
print("\tindicators: %s" % update_response['reportIndicators'])
print("\tURL: %s\n" % ts.get_report_url(update_response['reportId']))
# Get test report previously submitted
if do_report_details_by_guid:
print("Get Report")
result = ts.get_report_details_v12(token, report_guid, id_type="internal", verify=verify)
print("Report Details using Guid")
print("\ttitle: %s" % result['title'])
print("\texternalTrackingId: %s" % result['externalTrackingId'])
print("\tindicators: %s" % result['indicators'])
print("\tURL: %s\n" % ts.get_report_url(result['id']))
# Update a test report and test with get report
if do_update_report_by_guid:
print("Update Report")
title = "New Sample Title"
body = "new sample body - 7.8.9.10"
update_response = ts.update_report(token, report_guid, id_type="internal", title=title, report_body=body, verify=verify)
print("Updated Report using GUID")
print("\texternalTrackingId: %s" % update_response['externalTrackingId'])
print("\tindicators: %s" % update_response['reportIndicators'])
print("\tURL: %s\n" % ts.get_report_url(update_response['reportId']))
# Get test report previously submitted
if do_report_details_by_guid:
print("Get Report")
result = ts.get_report_details_v12(token, report_guid, id_type="internal", verify=verify)
print("Report Details using GUID")
print("\ttitle: %s" % result['title'])
print("\texternalTrackingId: %s" % result['externalTrackingId'])
print("\tindicators: %s" % result['indicators'])
print("\tURL: %s\n" % ts.get_report_url(result['id']))
# Release report to community
if do_release_report_by_ext_id:
print("Release Report")
update_response = ts.update_report(token, external_id, id_type='external', distribution="COMMUNITY", verify=verify)
print("Report Released using External ID")
print("\texternalTrackingId: %s" % update_response['externalTrackingId'])
print("\tindicators: %s" % update_response['reportIndicators'])
print("\tURL: %s\n" % ts.get_report_url(update_response['reportId']))
# Get test report previously submitted
if do_report_details_by_ext_id_2:
print("Get Report")
result = ts.get_report_details_v12(token, external_id, id_type="external", verify=verify)
print("Report Details using External ID")
print("\ttitle: %s" % result['title'])
print("\texternalTrackingId: %s" % result['externalTrackingId'])
print("\tindicators: %s" % result['indicators'])
print("\tURL: %s\n" % ts.get_report_url(result['id']))
# Delete test report previously submitted
if do_delete_report_by_ext_id:
print("Delete Report")
response = ts.delete_report(token, external_id, id_type="external", verify=verify)
print("Report Deleted using External ID")
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=
('Submit TruSTAR reports from a CSV file\n'
'Example:\n\n'
'python ingest_csv.py -c "TargetIP,SourceIP,Info,Analysis,Indicators" -t "TrackingNumber" -d "ReportTime" -cn "CaseName" -f reportname.csv'
))
parser.add_argument('-f',
'--file',
required=True,
dest='file_name',
help='csv file to import')
parser.add_argument('-t',
'--title',
required=True,
dest='title_col',
help='Name of column to use as title field')
parser.add_argument('-d',
'--datetime',
required=False,
dest='datetime_col',
help='Name of column to use as report date/time')
parser.add_argument('-c',
'--columns',
required=False,
dest='cols',
help='List of comma-separated column names to include')
parser.add_argument(
'-n',
'--num-reports',
required=False,
dest='num_reports',
type=int,
default=1000,
help='Max number of reports to submit (top-down order)')
parser.add_argument(
'-o',
'--output',
required=False,
dest='cef_output_file',
default='trustar.cef',
help=
'Common Event Format (CEF) output log file, one event is generated per successful submission'
)
parser.add_argument(
'-ci',
'--case-id',
required=False,
dest='caseid_col',
help='Name of column to use as report case ID for CEF export')
args = parser.parse_args()
allowed_keys_content = []
if args.cols:
allowed_keys_content = args.cols.split(",")
ts = TruStar(config_role="trustar")
token = ts.get_token()
df = pd.read_csv(args.file_name, nrows=args.num_reports, encoding="latin1")
# Create title and report content from the provided column names (if any)
all_reports = []
for report_num in range(0, len(df)):
current_content = ''
current_title = ''
current_datetime = None
current_case_id = 0
current_report = {}
for key in df:
# ignore empty cells, which are float64 NaNs
cell_value = df[key][report_num]
if pd.isnull(cell_value):
continue
cell_value = "%s" % cell_value
# encode any unicode chars
string_value = cell_value.encode('utf-8').strip()
if string_value == "nan":
print("%s -> %s" % (key, string_value))
continue
content = "{}:\n {}\n \n".format(key, string_value)
if not allowed_keys_content or key in allowed_keys_content:
current_content += content
if key == args.title_col:
current_title = str(df[key][report_num])
if key == args.datetime_col:
current_datetime = str(df[key][report_num])
if key == args.caseid_col:
current_case_id = str(df[key][report_num])
current_report['reportTitle'] = current_title
current_report['reportDateTime'] = current_datetime
current_report['reportContent'] = current_content
current_report['reportCaseId'] = current_case_id
all_reports.append(current_report)
if do_enclave_submissions:
num_submitted = 0
for staged_report in all_reports:
successful = False
attempts = 0
while not successful and attempts < 5:
attempts += 1
try:
response = ts.submit_report(
token,
report_body_txt=staged_report['reportContent'],
report_name=staged_report['reportTitle'],
began_time=staged_report['reportDateTime'],
enclave=True)
if 'error' in response:
print("Submission failed with error: %s, %s" %
(response['error'], response['message']))
if response['error'] in ("Internal Server Error",
"Access token expired",
"Authentication error"):
print("Auth token expired, requesting new one")
token = ts.get_token()
else:
raise Exception
else:
num_submitted += 1
successful = True
print(
"Submitted report #%s-%s title %s as TruSTAR IR %s with case ID: %s"
% (num_submitted, attempts,
staged_report['reportTitle'],
response['reportId'],
staged_report['reportCaseId']))
print("URL: %s" %
ts.get_report_url(response['reportId']))
# Build CEF output:
# - HTTP_USER_AGENT is the cs1 field
# - example CEF output: CEF:version|vendor|product|device_version|signature|name|severity|cs1=(num_submitted) cs2=(report_url)
config = {
'cef.version': '0.5',
'cef.vendor': 'TruSTAR',
'cef.device_version': '2.0',
'cef.product': 'API',
'cef': True,
'cef.file': args.cef_output_file
}
environ = {
'REMOTE_ADDR': '127.0.0.1',
'HTTP_HOST': '127.0.0.1',
'HTTP_USER_AGENT': staged_report['reportTitle']
}
log_cef('SUBMISSION',
1,
environ,
config,
signature="INFO",
cs2=staged_report['reportCaseId'],
cs3=ts.get_report_url(response['reportId']))
####
# TODO: ADD YOUR CUSTOM POST-PROCESSING CODE FOR THIS SUBMISSION HERE
####
if 'reportIndicators' in response and len(
response['reportIndicators']) > 0:
print("Indicators:\n %s" %
(json.dumps(response['reportIndicators'])))
print()
except Exception as e:
traceback.print_exc(file=sys.stdout)
print("Problem submitting report: %s" % e)
time.sleep(5)
# Sleep between submissions
time.sleep(5)