def add_to_act(actapi: act.api.Act,
doc: Dict,
output_format: Text = "json") -> None:
"""Add a report to the ACT platform"""
report_id: Text = doc["hexdigest"]
title: Text = doc.get("title", "No title")
indicators: Dict = doc.get("indicators", {})
try:
# Report title
handle_fact(
actapi.fact("name", title).source("report", report_id),
output_format)
except act.api.base.ResponseError as e:
error("Unable to create fact: %s" % e)
# Loop over all items under indicators in report
for scio_indicator_type in EXTRACT_INDICATORS:
# Get object type from ACT (default to object type in SCIO)
act_indicator_type = SCIO_INDICATOR_ACT_MAP.get(
scio_indicator_type, scio_indicator_type)
report_mentions_fact(actapi, act_indicator_type,
indicators.get(scio_indicator_type, []),
report_id, output_format)
# For SHA256, create content object
for sha256 in list(set(indicators.get("sha256", []))):
handle_fact(
actapi.fact("represents").source("hash", sha256).destination(
"content", sha256), output_format)
# Add all URI components
for uri in list(set(indicators.get("uri", []))):
try:
handle_uri(actapi, uri, output_format=output_format)
except act.api.schema.MissingField:
error("Unable to create facts from uri: {}".format(uri))
# Locations (countries, regions, sub regions)
for location_type in EXTRACT_GEONAMES:
locations = doc.get("geonames", {}).get(location_type, [])
report_mentions_fact(actapi, SCIO_GEONAMES_ACT_MAP[location_type],
locations, report_id, output_format)
# Threat actor
report_mentions_fact(actapi, "threatActor",
doc.get("threat-actor", {}).get("names", []),
report_id, output_format)
# Tools
report_mentions_fact(
actapi, "tool",
[tool.lower() for tool in doc.get("tools", {}).get("names", [])],
report_id, output_format)
# Sector
report_mentions_fact(actapi, "sector", doc.get("sectors", []), report_id,
output_format)
def add_indicators_to_act(actapi: act.api.Act,
indicators: Dict,
report_id: Text,
output_format: Text) -> None:
"""Create facts of indicators map"""
# Loop over all items under indicators in report
for scio_indicator_type in EXTRACT_INDICATORS:
# Get object type from ACT (default to object type in SCIO)
act_indicator_type = SCIO_INDICATOR_ACT_MAP.get(scio_indicator_type,
scio_indicator_type)
value_fn = ACT_FN_MAP.get(act_indicator_type, lambda x: x)
values = {value_fn(value) for value
in indicators.get(scio_indicator_type, [])}
report_mentions_fact(
actapi,
act_indicator_type,
values,
report_id,
output_format)
# For IPv4+IPv6 addresses, create mention facts
# Use ip_obj to return exploded, normalized IPv4 and IPv6 address
for ip in set(indicators.get("ipv4", []) + indicators.get("ipv6", [])):
try:
handle_fact(
actapi.fact("mentions")
.source("report", report_id)
.destination(*act.api.helpers.ip_obj(ip)),
output_format
)
except ValueError as err:
warning(f"Creating fact to {ip} fails on IP validation {err}")
# For SHA256, create content object
for sha256 in set(indicators.get("sha256", [])):
handle_fact(
actapi.fact("represents")
.source("hash", sha256)
.destination("content", sha256),
output_format
)
# Add emails as URI components
for email in set(indicators.get("email", [])):
try:
email_uri = f"email://{email}"
handle_uri(actapi, email_uri, output_format=output_format)
handle_fact(
actapi.fact("mentions")
.source("report", report_id)
.destination("uri", email_uri),
output_format
)
except act.api.base.ValidationError as err:
warning(f"Fact from {email_uri} failes du to URI validation {err}")
except act.api.schema.MissingField:
warning(f"Unable to create facts from uri: {email_uri}")
# Add all URI components
for uri in set(indicators.get("uri", [])):
try:
handle_uri(actapi, uri, output_format=output_format)
except act.api.base.ValidationError as err:
warning(f"Fact from {uri} failes du to URI validation {err}")
except act.api.schema.MissingField:
warning(f"Unable to create facts from uri: {uri}")
def main() -> None:
"""main function"""
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = cli.handle_args(parseargs())
actapi = worker.init_act(args)
if not (args.privatekey and args.publickey):
cli.fatal(
"You must specify --privatekey and --publickey on command line or in config file"
)
proxies = (
{"http": args.proxy_string, "https": args.proxy_string}
if args.proxy_string
else None
)
iSightHandler = ISightAPIRequestHandler(args.root, args.privatekey, args.publickey)
data = iSightHandler.indicators(days=args.days, proxies=proxies)
if "success" not in data or not data["success"]:
logging.error(
"Unable to download from isight API [%s]",
data["message"] if "message" in data else "NA",
)
return
timestamp = int(time.time())
### DEBUG -- dump json to disc for each run
if args.debugdir:
with open(
os.path.join(args.debugdir, "error-{0}.json".format(timestamp)), "w"
) as f:
json.dump(data, f)
for i, dp in enumerate(data["message"]):
### --- Handle mentions facts
# Create report ID from the url (same approach as for feeds) and title to this ID.
reportID = hashlib.sha256(dp["webLink"].encode("utf8")).hexdigest()
handle_fact(actapi.fact("name", dp["title"]).source("report", reportID))
for obj in OBJECT_MAP: # run through all fields that we want to mention
if obj in dp and dp[obj]: # if the report contains data in the field
factType = OBJECT_MAP[obj](dp[obj]) # translate to ACT fact type
handle_fact(
actapi.fact("mentions") # and create fact from field
.source("report", reportID)
.destination(factType, dp[obj].lower())
)
if dp["url"]:
handle_fact(
actapi.fact("mentions")
.source("report", reportID)
.destination("uri", dp["url"])
)
handle_uri(actapi, dp["url"])
### --- IP -> malwareFamily
if dp["malwareFamily"] and dp["ip"]:
handle_facts(
act.api.fact.fact_chain(
actapi.fact("connectsTo")
.source("content", "*")
.destination("uri", "*"),
actapi.fact("componentOf")
.source("ipv4", dp["ip"])
.destination("uri", "*"),
actapi.fact("classifiedAs")
.source("content", "*")
.destination("tool", dp["malwareFamily"]),
)
)
### --- URL -> malwareFamily
elif dp["networkType"] == "url" and dp["malwareFamily"]:
handle_uri(actapi, dp["url"])
handle_facts(
act.api.fact.fact_chain(
actapi.fact("connectsTo")
.source("content", "*")
.destination("uri", dp["url"]),
actapi.fact("classifiedAs")
.source("content", "*")
.destination("tool", dp["malwareFamily"]),
)
)
### --- FQDN -> malwareFamily
elif dp["networkType"] == "network" and dp["domain"] and dp["malwareFamily"]:
handle_facts(
act.api.fact.fact_chain(
actapi.fact("connectsTo")
.source("content", "*")
.destination("uri", "*"),
actapi.fact("componentOf")
.source("fqdn", dp["domain"])
.destination("uri", "*"),
actapi.fact("classifiedAs")
.source("content", "*")
.destination("tool", dp["malwareFamily"]),
)
)
### --- hash -> malwareFamily
elif (
dp["fileType"]
and dp["malwareFamily"]
and (dp["sha1"] or dp["sha256"] or dp["md5"])
):
for digest_type in ["md5", "sha1", "sha256"]:
### In some cases the iSight api does not return a sha256 hashdigest
### so we need to make a chain through a placeholder content
if not dp["sha256"]:
if dp[digest_type]:
handle_facts(
act.api.fact.fact_chain(
actapi.fact("represents")
.source("hash", dp[digest_type])
.destination("content", "*"),
actapi.fact("classifiedAs")
.source("content", "*")
.destination("tool", dp["malwareFamily"]),
)
)
else: ## There is a sha256, so we do _not_ need a chain
if dp[digest_type]:
handle_fact(
actapi.fact("classifiedAs")
.source("content", dp["sha256"])
.destination("tool", dp["malwareFamily"])
)
handle_fact(
actapi.fact("represents")
.source("hash", dp[digest_type])
.destination("content", dp["sha256"])
)
### -- Hash --> actor
elif (
dp["fileType"] and dp["actor"] and (dp["sha1"] or dp["sha256"] or dp["md5"])
):
for digest_type in ["md5", "sha1", "sha256"]:
### In some cases the iSight api does not return a sha256 hashdigest
### so we need to make a chain through a placeholder content
if not dp["sha256"]:
if dp[digest_type]:
handle_facts(
act.api.fact.fact_chain(
actapi.fact("represents")
.source("hash", dp[digest_type])
.destination("content", "*"),
actapi.fact("observedIn")
.source("content", "*")
.destination("incident", "*"),
actapi.fact("attributedTo")
.source("incident", "*")
.destination("threatActor", dp["actor"]),
)
)
else: ## There is a sha256, so we do _not_ need a chain between all the way from hexdigest
if dp[digest_type]:
handle_fact(
actapi.fact("represents")
.source("hash", dp[digest_type])
.destination("content", dp["sha256"])
)
handle_facts(
act.api.fact.fact_chain(
actapi.fact("observedIn")
.source("content", dp["sha256"])
.destination("incident", "*"),
actapi.fact("attributedTo")
.source("incident", "*")
.destination("threatActor", dp["actor"]),
)
)
### We do have a sha256 of a file (but possibly nothing else). Add the content to hexdigest facts
elif dp["fileType"] and dp["sha256"]:
for digest in ["sha1", "md5", "sha256"]:
if dp[digest]:
handle_fact(
actapi.fact("represents")
.source("hash", dp[digest])
.destination("content", dp["sha256"])
)
if args.debugdir:
fields = [
k
for k, v in dp.items()
if v
and k
not in [
"reportId",
"title",
"ThreatScape",
"audience",
"intelligenceType",
"publishDate",
"reportLink",
"webLink",
]
]
logging.error(
"[%s] Extra fields while handeling index[%s] '%s'",
timestamp,
i,
", ".join(fields),
)
### -- DEBUG!
else:
if args.debugdir:
fields = [
k
for k, v in dp.items()
if v
and k
not in [
"reportId",
"title",
"ThreatScape",
"audience",
"intelligenceType",
"publishDate",
"reportLink",
"webLink",
]
]
logging.error(
"[%s] Unable to handle index[%s] with fields '%s'",
timestamp,
i,
", ".join(fields),
)
def add_to_act(actapi: act.api.Act,
doc: Dict,
output_format: Text = "json") -> None:
"""Add a report to the ACT platform"""
report_id: Text = doc["hexdigest"]
title: Text = doc.get("title", "No title")
indicators: Dict = doc.get("indicators", {})
try:
# Report title
handle_fact(
actapi.fact("name", title).source("report", report_id),
output_format)
except act.api.base.ResponseError as e:
error("Unable to create fact: %s" % e)
# Loop over all items under indicators in report
for scio_indicator_type in EXTRACT_INDICATORS:
# Get object type from ACT (default to object type in SCIO)
act_indicator_type = SCIO_INDICATOR_ACT_MAP.get(
scio_indicator_type, scio_indicator_type)
value_fn = ACT_FN_MAP.get(act_indicator_type, lambda x: x)
values = {
value_fn(value)
for value in indicators.get(scio_indicator_type, [])
}
report_mentions_fact(actapi, act_indicator_type, values, report_id,
output_format)
# For IPv4+IPv6 addresses, create mention facts
# Use ip_obj to return exploded, normalized IPv4 and IPv6 address
for ip in set(indicators.get("ipv4", []) + indicators.get("ipv6", [])):
try:
handle_fact(
actapi.fact("mentions").source(
"report",
report_id).destination(*act.api.helpers.ip_obj(ip)),
output_format)
except ValueError as err:
warning("Creating fact to {} fails on IP validation {}".format(
ip, err))
# For SHA256, create content object
for sha256 in set(indicators.get("sha256", [])):
handle_fact(
actapi.fact("represents").source("hash", sha256).destination(
"content", sha256), output_format)
# Add emails as URI components
for email in set(indicators.get("email", [])):
try:
email_uri = "email://{}".format(email)
handle_uri(actapi, email_uri, output_format=output_format)
handle_fact(
actapi.fact("mentions").source("report",
report_id).destination(
"uri", email_uri),
output_format)
except act.api.base.ValidationError as err:
warning(
"Creating fact from {} failes du to URI validation {}".format(
email_uri, err))
except act.api.schema.MissingField:
warning("Unable to create facts from uri: {}".format(email_uri))
# Add all URI components
for uri in set(indicators.get("uri", [])):
try:
handle_uri(actapi, uri, output_format=output_format)
except act.api.base.ValidationError as err:
warning(
"Creating fact from {} failes du to URI validation {}".format(
uri, err))
except act.api.schema.MissingField:
warning("Unable to create facts from uri: {}".format(uri))
# Locations (countries, regions, sub regions)
for location_type in EXTRACT_GEONAMES:
locations = doc.get("geonames", {}).get(location_type, [])
report_mentions_fact(actapi, SCIO_GEONAMES_ACT_MAP[location_type],
locations, report_id, output_format)
# Threat actor
report_mentions_fact(actapi, "threatActor",
doc.get("threat-actor", {}).get("names", []),
report_id, output_format)
# Tools
report_mentions_fact(
actapi, "tool",
[tool.lower() for tool in doc.get("tools", {}).get("names", [])],
report_id, output_format)
# Sector
report_mentions_fact(actapi, "sector", doc.get("sectors", []), report_id,
output_format)
def main() -> None:
"""main function"""
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = worker.handle_args(parseargs())
actapi = worker.init_act(args)
if not (args.privatekey and args.publickey):
worker.fatal("You must specify --privatekey and --publickey on command line or in config file")
proxies = {
'http': args.proxy_string,
'https': args.proxy_string
} if args.proxy_string else None
iSightHandler = ISightAPIRequestHandler(args.root, args.privatekey, args.publickey)
data = iSightHandler.indicators(days=args.days, proxies=proxies)
if 'success' not in data or not data['success']:
logging.error("Unable to download from isight API [%s]", data['message'] if 'message' in data else "NA")
return
timestamp = int(time.time())
### DEBUG -- dump json to disc for each run
if args.debugdir:
with open(os.path.join(args.debugdir, "error-{0}.json".format(timestamp)), "w") as f:
json.dump(data, f)
for i, dp in enumerate(data['message']):
### --- Handle mentions facts
# Create report ID from the url (same approach as for feeds) and title to this ID.
reportID = hashlib.sha256(dp['webLink'].encode('utf8')).hexdigest()
handle_fact(actapi.fact('name', dp['title']).source('report', reportID))
for obj in OBJECT_MAP: # run through all fields that we want to mention
if obj in dp and dp[obj]: # if the report contains data in the field
factType = OBJECT_MAP[obj](dp[obj]) # translate to ACT fact type
handle_fact(actapi.fact('mentions') # and create fact from field
.source('report', reportID)
.destination(factType, dp[obj].lower()))
if dp['url']:
handle_fact(actapi.fact('mentions')
.source('report', reportID)
.destination('uri', dp['url']))
try:
handle_uri(actapi, dp['url'])
except act.api.base.ValidationError as err:
logging.error("%s while storing url from mentions [%s]", err, dp['url'])
### --- IP -> malwareFamily
if dp['malwareFamily'] and dp['ip']:
chain = act.api.fact.fact_chain(
actapi.fact('connectsTo')
.source('content', '*')
.destination('uri', '*'),
actapi.fact('componentOf')
.source('ipv4', dp['ip'])
.destination('uri', '*'),
actapi.fact('classifiedAs')
.source('content', '*')
.destination('tool', dp['malwareFamily'].lower()))
for fact in chain:
handle_fact(fact)
### --- URL -> malwareFamily
elif dp['networkType'] == 'url' and dp['malwareFamily']:
try:
handle_uri(actapi, dp['url'])
except act.api.base.ValidationError as err:
logging.error("%s while storing url from mentions [%s]", err, dp['url'])
chain = act.api.fact.fact_chain(
actapi.fact('connectsTo')
.source('content', '*')
.destination('uri', dp['url']),
actapi.fact('classifiedAs')
.source('content', '*')
.destination('tool', dp['malwareFamily'].lower()))
for fact in chain:
handle_fact(fact)
### --- FQDN -> malwareFamily
elif dp['networkType'] == 'network' and dp['domain'] and dp['malwareFamily']:
chain = act.api.fact.fact_chain(
actapi.fact('connectsTo')
.source('content', '*')
.destination('uri', '*'),
actapi.fact('componentOf')
.source('fqdn', dp['domain'])
.destination('uri', '*'),
actapi.fact('classifiedAs')
.source('content', '*')
.destination('tool', dp['malwareFamily'].lower()))
for fact in chain:
handle_fact(fact)
### --- hash -> malwareFamily
elif dp['fileType'] and dp['malwareFamily'] and (dp['sha1'] or dp['sha256'] or dp['md5']):
for digest_type in ['md5', 'sha1', 'sha256']:
### In some cases the iSight api does not return a sha256 hashdigest
### so we need to make a chain through a placeholder content
if not dp['sha256']:
if dp[digest_type]:
chain = act.api.fact.fact_chain(
actapi.fact('represents')
.source('hash', dp[digest_type])
.destination('content', '*'),
actapi.fact('classifiedAs')
.source('content', '*')
.destination('tool', dp['malwareFamily']))
for fact in chain:
handle_fact(fact)
else: ## There is a sha256, so we do _not_ need a chain
if dp[digest_type]:
handle_fact(actapi.fact('classifiedAs')
.source('content', dp['sha256'])
.destination('tool', dp['malwareFamily']))
handle_fact(actapi.fact('represents')
.source('hash', dp[digest_type])
.destination('content', dp['sha256']))
### -- Hash --> actor
elif dp['fileType'] and dp['actor'] and (dp['sha1'] or dp['sha256'] or dp['md5']):
for digest_type in ['md5', 'sha1', 'sha256']:
### In some cases the iSight api does not return a sha256 hashdigest
### so we need to make a chain through a placeholder content
if not dp['sha256']:
if dp[digest_type]:
chain = act.api.fact.fact_chain(
actapi.fact('represents')
.source('hash', dp[digest_type])
.destination('content', '*'),
actapi.fact('observedIn')
.source('content', '*')
.destination('event', '*'),
actapi.fact('attributedTo')
.source('event', '*')
.destination('incident', '*'),
actapi.fact('attributedTo')
.source('incident', '*')
.destination('threatActor', dp['actor']))
for fact in chain:
handle_fact(fact)
else: ## There is a sha256, so we do _not_ need a chain between all the way from hexdigest
if dp[digest_type]:
handle_fact(actapi.fact('represents')
.source('hash', dp[digest_type])
.destination('content', dp['sha256']))
chain = act.api.fact.fact_chain(
actapi.fact('observedIn')
.source('content', dp['sha256'])
.destination('event', '*'),
actapi.fact('attributedTo')
.source('event', '*')
.destination('incident', '*'),
actapi.fact('attributedTo')
.source('incident', '*')
.destination('threatActor', dp['actor']))
for fact in chain:
handle_fact(fact)
### We do have a sha256 of a file (but possibly nothing else). Add the content to hexdigest facts
elif dp['fileType'] and dp['sha256']:
for digest in ['sha1', 'md5', 'sha256']:
if dp[digest]:
handle_fact(actapi.fact('represents')
.source('hash', dp[digest])
.destination('content', dp['sha256']))
if args.debugdir:
fields = [k for k, v in dp.items() if v and k not in ['reportId', 'title', 'ThreatScape',
'audience', 'intelligenceType',
'publishDate', 'reportLink', 'webLink']]
logging.error("[%s] Extra fields while handeling index[%s] '%s'", timestamp, i, ", ".join(fields))
### -- DEBUG!
else:
if args.debugdir:
fields = [k for k, v in dp.items() if v and k not in ['reportId', 'title', 'ThreatScape',
'audience', 'intelligenceType',
'publishDate', 'reportLink', 'webLink']]
logging.error("[%s] Unable to handle index[%s] with fields '%s'", timestamp, i, ", ".join(fields))