def main() -> None:
"""Main function"""
# Look for default ini file in "/etc/actworkers.ini" and
# ~/config/actworkers/actworkers.ini (or replace .config with
# $XDG_CONFIG_DIR if set)
args = worker.handle_args(parseargs())
# setup logging
act.api.utils.setup_logging(args.loglevel, prefix='act-alienvault-otx')
# check necessary configuration items for errors
for item in [
'config_path', 'otx_baseurl', 'otx_apikey', 'otx_path_lastretrived'
]:
if item not in args:
raise ConfigurationError(
'Missing configuration item {}'.format(item))
# initialise act api
actapi = worker.init_act(args)
# initialise otx api
otxapi = AlienvaultOTXAPI(args)
# create facts for indicators in each event
for event in otxapi.get_subscribed():
handle_facts(actapi, event)
def main_log_error() -> None:
"Main function. Log all exceptions to error"
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = cli.handle_args(parseargs())
actapi = worker.init_act(args)
if not (args.act_baseurl and args.user_id):
sys.stderr.write("Must specify --baseurl and --user-id\n")
sys.exit(1)
if not os.path.isfile(args.search_jobs):
sys.stderr.write("File not found: {}\n".format(args.search_jobs))
sys.exit(2)
if not args.search_jobs:
sys.stderr.write("Must specify config file with search jobs\n")
sys.exit(2)
try:
for name, options in parse_search_jobs(args.search_jobs).items():
process(actapi, args.output_path, name, options, args.workers)
except Exception:
error("Unhandled exception", exc_info=True)
raise
def main() -> None:
"""main function"""
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = cli.handle_args(parseargs())
actapi = worker.init_act(args)
if not args.apikey:
cli.fatal("You must specify --apikey on command line or in config file")
in_data = sys.stdin.read().strip()
proxies = (
{"http": args.proxy_string, "https": args.proxy_string}
if args.proxy_string
else None
)
vtapi = VirusTotalApi(args.apikey, proxies=proxies)
if args.hexdigest:
handle_hexdigest(actapi, vtapi, in_data, output_format=args.output_format)
elif args.ip:
handle_ip(actapi, vtapi, in_data, output_format=args.output_format)
elif args.domain:
handle_domain(actapi, vtapi, in_data, output_format=args.output_format)
else: # Type not specified, autodetect
handle_ioc(actapi, vtapi, in_data, output_format=args.output_format)
def main() -> None:
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = worker.handle_args(parseargs())
actapi = worker.init_act(args)
process(actapi, args.pdns_baseurl, args.apikey, args.timeout, args.proxy_string, args.output_format)
def main() -> None:
"""main function"""
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = worker.handle_args(parseargs())
actapi = worker.init_act(args)
# if not args.apikey:
# worker.fatal("You must specify --apikey on command line or in config file")
proxies = {
'http': args.proxy_string,
'https': args.proxy_string
} if args.proxy_string else None
params = {
'actapi': actapi,
'user_agent': args.user_agent,
'proxies': proxies,
'verify_ssl': args.no_check_certificate,
'output_format': args.output_format
}
if args.feed:
handle_feed(**params)
else:
params['apikey'] = args.apikey
for line in sys.stdin:
params['hashdigest'] = line.strip()
handle_hash(**params)
def main() -> None:
"""main function"""
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = cli.handle_args(parseargs())
actapi = worker.init_act(args)
# if not args.apikey:
# cli.fatal("You must specify --apikey on command line or in config file")
proxies = (
{"http": args.proxy_string, "https": args.proxy_string}
if args.proxy_string
else None
)
params = {
"actapi": actapi,
"user_agent": args.user_agent,
"proxies": proxies,
"verify_ssl": args.no_check_certificate,
"output_format": args.output_format,
}
if args.feed:
handle_feed(**params)
else:
params["apikey"] = args.apikey
for line in sys.stdin:
params["hashdigest"] = line.strip()
handle_hash(**params)
def main() -> None:
"""program entry point"""
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = worker.handle_args(parseargs())
manifest_dir = args.manifest_dir
actapi = worker.init_act(args)
verify_manifest_dir(manifest_dir)
misp_feeds_file = os.path.join(manifest_dir, "misp_feeds.txt")
with open(misp_feeds_file) as f:
for line in f:
feed_data = handle_feed(manifest_dir, line.strip(),
args.proxy_string, args.cert_file)
for event in feed_data:
n = 0
e = 0
act.api.helpers.handle_fact(actapi.fact("name",
event.info).source(
"report",
str(event.uuid)),
output_format=args.output_format)
n += 1
try:
act.api.helpers.handle_fact(
actapi.fact("externalLink").source(
"uri",
"{0}/{1}.json".format(line.strip(),
event.uuid)).destination(
"report",
str(event.uuid)),
output_format=args.output_format)
n += 1
except act.api.base.ResponseError as err:
e += 1
error("Error adding fact to platform", exc_info=True)
for attribute in event.attributes:
if not attribute.act_type:
continue
try:
act.api.helpers.handle_fact(
actapi.fact("mentions").source(
"report", str(event.uuid)).destination(
attribute.act_type, attribute.value),
output_format=args.output_format)
n += 1
except act.api.base.ResponseError as err:
e += 1
error("Error adding fact to platform", exc_info=True)
info("{0} facts. {1} errors.".format(n, e))
def main() -> None:
"""main function"""
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = cli.handle_args(parseargs())
actapi = worker.init_act(args)
if not args.country_codes:
cli.fatal("You must specify --country-codes on command line or in config file")
if not args.veris_prefix:
cli.fatal("You must specify --veris-prefix")
if not (args.veris_url or args.veris_file or args.stdin):
cli.fatal("You must specify --veris-url, --veris-file or --stdin")
args.veris_prefix = args.veris_prefix.upper()
if not os.path.isfile(args.country_codes):
cli.fatal(
"Country/region file not found at specified location: {}".format(
args.country_codes
),
2,
)
args.threat_actor_variety = [
variety.strip() for variety in args.threat_actor_variety.split(",")
]
# Configuration object that will be passed around to functions
config = {
# act API
"actapi": actapi,
# Map of CC -> Country Name
"cn_map": get_cn_map(args.country_codes),
# Map of CC -> Country Name
"campaign_map": get_campaigns(args.veris_prefix, args.veris_campaign)
if args.veris_campaign
else {},
# Cache of url > sha256
"db_cache": urlcache.URLCache(
requests_common_kwargs={
"proxies": {"http": args.proxy_string, "https": args.proxy_string},
"timeout": args.http_timeout,
}
),
"proxies": {"http": args.proxy_string, "https": args.proxy_string}
if args.proxy_string
else None,
}
# Add all arguments from args to config
config.update(vars(args))
process(config)
def main() -> None:
"""Main function"""
# Look for default ini file in "/etc/actworkers.ini" and
# ~/config/actworkers/actworkers.ini (or replace .config with
# $XDG_CONFIG_DIR if set)
args = worker.handle_args(worker.parseargs(WORKER_NAME))
actapi = worker.init_act(args)
process(actapi, args.output_format)
def main_log_error() -> None:
"Call main() and log all exceptions as errors"
try:
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = worker.handle_args(worker.parseargs("Generic uploader"))
actapi = worker.init_act(args)
main(actapi)
except Exception:
error("Unhandled exception: {}".format(traceback.format_exc()))
raise
def main() -> None:
"main function"
# Look for default ini file in "/etc/actworkers.ini" and
# ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = cli.handle_args(parseargs())
if not args.argus_apikey:
cli.fatal("You must specify --argus-apikey on command line or in config file")
actapi = worker.init_act(args)
process(actapi, args)
def main() -> None:
"""Main function"""
args = cli.handle_args(parseargs())
actapi = worker.init_act(args)
if not (args.act_baseurl and args.user_id):
error("Worker must be configured with --act-baseurl and --userid")
sys.exit(1)
tools = search_tools(actapi, args.exclude_tools)
aliases = get_aliases(actapi, list(tools.keys()))
for tool1, tool2 in process(tools, aliases, args.threshold):
handle_alias(actapi, tool1, tool2, args.submit, args.output_format)
def main() -> None:
"""main function"""
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = cli.handle_args(parseargs())
actapi = worker.init_act(args)
if not args.country_codes:
cli.fatal("You must specify --country-codes on command line or in config file")
if not os.path.isfile(args.country_codes):
cli.fatal(
"Country/region file not found at specified location: {}".format(
args.country_codes
),
2,
)
# Get map of CC -> Country Name
cn_map = get_cn_map(args.country_codes)
db_cache = get_db_cache(CACHE_DIR)
# Read IPs from stdin
if args.stdin:
in_data = sys.stdin.read().split("\n")
handle_ip(
actapi, cn_map, in_data, db_cache, args.proxy_string, args.output_format
)
# Bulk lookup
elif args.bulk:
all_ips = open(args.bulk, "r").readlines()
batch_size = 50
i = 0
while i < len(all_ips):
handle_ip(
actapi,
cn_map,
(all_ips[i : i + batch_size]),
db_cache,
args.proxy_string,
args.output_format,
)
i += batch_size
time.sleep(1)
db_cache.close()
def main() -> None:
"""main function"""
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = worker.handle_args(parseargs())
actapi = worker.init_act(args)
# Add IOCs from reports to the ACT platform
add_to_act(
actapi,
get_scio_report(),
args.output_format,
)
def main_log_error() -> None:
"Main function. Log all exceptions to error"
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = worker.handle_args(parseargs())
actapi = worker.init_act(args)
try:
process(
actapi,
worker.fetch_json(args.country_region_url, args.proxy_string,
args.http_timeout), args.output_format)
except Exception:
error("Unhandled exception: {}".format(traceback.format_exc()))
raise
def main() -> None:
"""Main function"""
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = cli.handle_args(parseargs())
actapi = worker.init_act(args)
proxies = ({
"http": args.proxy_string,
"https": args.proxy_string
} if args.proxy_string else None)
attack = Attck(proxies=proxies)
types = [args.type] if args.type else MITRE_TYPES
for mitre_type in types:
if mitre_type not in MITRE_TYPES:
error("Unknown mitre type: {}. Valid types: {}".format(
mitre_type, ",".join(MITRE_TYPES)))
sys.exit(2)
cache = notify_cache(args.notifycache)
model = getattr(attack, mitre_type)
techniques_notify = add_techniques(actapi, model, args.output_format)
groups_notify = add_groups(actapi, model, args.output_format)
software_notify = add_software(actapi, model, args.output_format)
# filter revoked objects from those allready notified
notify = [
notify
for notify in techniques_notify + groups_notify + software_notify
if notify.id not in cache
]
if notify:
notified = send_notification(notify, args.smtphost, args.sender,
args.recipient, mitre_type)
for object_id in notified:
# Add object to cache, so we will not be notified on the same object on the next run
add_to_cache(args.notifycache, object_id)
def test_args_origin_id(monkeypatch: _pytest.monkeypatch.MonkeyPatch) -> None:
"""test argument origin-id"""
origin_id = "00000000-0000-0000-0000-000000000001"
monkeypatch.setattr(sys, "argv",
["./test-worker.py", "--origin-id", origin_id])
args = cli.handle_args(worker.parseargs("Test worker"))
actapi = worker.init_act(args)
assert actapi.config.origin_id == origin_id
fact = (actapi.fact("mentions").source("report", "xyz").destination(
"fqdn", "test.com"))
assert fact.origin.id == origin_id
def main() -> None:
""" Main function """
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = worker.handle_args(parseargs())
actapi = worker.init_act(args)
if args.type:
types = [args.type]
else:
types = list(MITRE_URLS.keys())
for mitre_type in types:
url = MITRE_URLS.get(mitre_type.lower())
if not url:
error("Unknown mitre type: {}. Valid types: {}".format(
mitre_type, ",".join(MITRE_URLS.keys())))
sys.exit(2)
cache = notify_cache(args.notifycache)
# Get attack dataset as Stix Memory Store
attack = get_attack(url, args.proxy_string, args.http_timeout)
techniques_notify = add_techniques(actapi, attack, args.output_format)
groups_notify = add_groups(actapi, attack, args.output_format)
software_notify = add_software(actapi, attack, args.output_format)
# filter revoked objects from those allready notified
notify = [
notify
for notify in techniques_notify + groups_notify + software_notify
if notify.id not in cache
]
if notify:
notified = send_notification(notify, args.smtphost, args.sender,
args.recipient, url)
for object_id in notified:
# Add object to cache, so we will not be notified on the same object on the next run
add_to_cache(args.notifycache, object_id)
def main_log_error() -> None:
"Call main() and log all exceptions as errors"
try:
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = cli.handle_args(parseargs())
actapi = worker.init_act(args)
uploader(
actapi,
sys.stdin,
args.timing,
args.allow_default_origin,
args.no_exit_on_error,
)
except Exception:
error("Unhandled exception: {}".format(traceback.format_exc()))
raise
def main() -> None:
"""Main function"""
args = cli.handle_args(parseargs())
actapi = worker.init_act(args)
ta_cards = worker.fetch_json(
args.thaicert_url, args.proxy_string, args.http_timeout
)
process(actapi, ta_cards["values"])
vocab = worker.fetch_json(STIX_VOCAB, args.proxy_string, args.http_timeout)
add_sectors(actapi, ta_cards["values"], vocab)
countries = worker.fetch_json(COUNTRY_REGIONS, args.proxy_string, args.http_timeout)
countries = [country["name"].lower() for country in countries]
add_countries(actapi, ta_cards["values"], countries)
tools = worker.fetch_json(THAICERT_TOOLS_URL, args.proxy_string, args.http_timeout)
add_tools(actapi, ta_cards["values"], tools["values"])
def test_args_origin_name(
monkeypatch: _pytest.monkeypatch.MonkeyPatch) -> None:
""" test argument origin-name """
origin_name = "test-origin"
monkeypatch.setattr(sys, "argv",
["./test-worker.py", "--origin-name", origin_name])
args = worker.handle_args(worker.parseargs("Test worker"))
actapi = worker.init_act(args)
assert actapi.config.origin_name == origin_name
fact = actapi.fact("mentions") \
.source("report", "xyz")\
.destination("fqdn", "test.com")
assert fact.origin.name == origin_name
def main() -> None:
"""Main function"""
# Look for default ini file in "/etc/actworkers.ini" and
# ~/config/actworkers/actworkers.ini (or replace .config with
# $XDG_CONFIG_DIR if set)
args = parseargs()
try:
shorteners = [x.strip() for x in args.url_shorteners.split(",")]
except AttributeError:
worker.fatal("Empty list of shorteners?")
actapi = worker.init_act(args)
proxies = {
'http': args.proxy_string,
'https': args.proxy_string
} if args.proxy_string else None
process(actapi, shorteners, args.user_agent, proxies, args.output_format)
def main() -> None:
"main function"
try:
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = worker.handle_args(parseargs())
if not (args.act_baseurl and args.user_id):
fatal("--act-baseurl and --user-id must be specified")
if not (args.list or args.add or args.delete):
fatal("Specify either --list, --add or --delete")
if (args.delete) and not (args.origin_id):
fatal("Specify --origin-id to delete an origin")
actapi = worker.init_act(args)
origin_handler(actapi, args)
except Exception:
error("Unhandled exception: {}".format(traceback.format_exc()))
raise
def main() -> None:
"""main function"""
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = cli.handle_args(parseargs())
actapi = worker.init_act(args)
fact_type_definition_path = (pathlib.Path(
args.fact_type_definition).expanduser().resolve())
if not fact_type_definition_path.is_file():
print(f"{fact_type_definition_path} is not a file.")
sys.exit(1)
with fact_type_definition_path.open() as typedef:
graph = graph_from_type_def(typedef, args.avoid, args.include,
args.avoid_cost)
start_type, start_value = args.start
end_type, end_value = args.end
start, end = find_start_and_end_nodes(graph, start_type, end_type)
if not start:
print(f"{start_type} is not an object type")
sys.exit(1)
if not end:
print(f"{end_type} is not an object type")
sys.exit(1)
res = dijkstra(graph, start, end)
chain = fact_chain_from_path_result(actapi, res, start, end, start_value,
end_value)
for fact in chain:
handle_fact(fact, output_format=args.output_format)
def main() -> None:
"""program entry point"""
# Look for default ini file in "/etc/actworkers.ini" and
# ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = cli.handle_args(parseargs())
if not args.feed_uri:
cli.fatal("--feed-uri not specified")
if args.dump_dir and not args.dump_dir.is_dir():
os.makedirs(args.dump_dir)
proxies = ({
"http": args.proxy_string,
"https": args.proxy_string
} if args.proxy_string else None)
actapi = worker.init_act(args)
try:
# Get "updated" from last successful run
last_run_filename = get_last_run_filename(args.feed_cache,
args.feed_uri)
with pid.PidFile(force_tmpdir=True, pidname="act_feed.pid"):
handle_feed(
last_run_filename,
args.feed_uri,
actapi,
args.dump_dir,
proxies,
args.cert_file,
args.no_exit_on_error,
)
except pid.base.PidFileAlreadyLockedError:
error("pid file found - feed is already running")
def run() -> None:
args = cli.handle_args(parseargs())
actapi = worker.init_act(args)
if not args.apikey or not args.secret:
cli.fatal(
"You must specify --apikey and --secret on command line or in config file"
)
proxies = ({
"http": args.proxy_string,
"https": args.proxy_string
} if args.proxy_string else None)
for in_data in sys.stdin:
in_data = in_data.strip()
if args.hexdigest or HASH_RE.search(in_data):
handle_hexdigest(
actapi,
in_data,
args.apikey,
args.secret,
proxies,
output_format=args.output_format,
)
elif args.domain:
handle_domain(
actapi,
in_data,
args.apikey,
args.secret,
proxies,
output_format=args.output_format,
)
def main() -> None:
""" Main function """
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = worker.handle_args(parseargs())
actapi = worker.init_act(args)
if not args.ta:
sys.stderr.write(
"You must specify Threat Actor with --ta <THREAT ACTOR>\n")
sys.exit(1)
# Normalize and lookup country/technique/sectors
# Print error with suggested match if not found
ok = True
try:
args.target_countries = [
country_lookup(args, country) for country in args.target_countries
]
except (VocabularyException, FileNotFoundError,
worker.UnsupportedScheme) as e:
sys.stderr.write(str(e))
ok = False
try:
args.techniques = [technique_lookup(tech) for tech in args.techniques]
except VocabularyException as e:
sys.stderr.write(str(e))
ok = False
try:
if args.ta_located_in:
args.ta_located_in = country_lookup(args, args.ta_located_in)
except VocabularyException as e:
sys.stderr.write(str(e))
ok = False
try:
args.sectors = [sector_lookup(args, sector) for sector in args.sectors]
except VocabularyException as e:
sys.stderr.write(str(e))
ok = False
if not ok:
sys.exit(1)
if args.target_countries:
add_ta_target_country(actapi, args.output_format, args.ta,
args.target_countries)
if args.campaign:
add_ta_campaign(actapi, args.output_format, args.ta, args.campaign)
if args.tools:
add_ta_tools(actapi, args.output_format, args.ta, args.tools)
if args.techniques:
add_ta_techniques(actapi, args.output_format, args.ta, args.techniques)
if args.sectors:
add_ta_sectors(actapi, args.output_format, args.ta, args.sectors)
if args.ta_located_in:
add_ta_located_in(actapi, args.output_format, args.ta,
args.ta_located_in)
def main() -> None:
"""main function"""
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = worker.handle_args(parseargs())
actapi = worker.init_act(args)
if not (args.privatekey and args.publickey):
worker.fatal("You must specify --privatekey and --publickey on command line or in config file")
proxies = {
'http': args.proxy_string,
'https': args.proxy_string
} if args.proxy_string else None
iSightHandler = ISightAPIRequestHandler(args.root, args.privatekey, args.publickey)
data = iSightHandler.indicators(days=args.days, proxies=proxies)
if 'success' not in data or not data['success']:
logging.error("Unable to download from isight API [%s]", data['message'] if 'message' in data else "NA")
return
timestamp = int(time.time())
### DEBUG -- dump json to disc for each run
if args.debugdir:
with open(os.path.join(args.debugdir, "error-{0}.json".format(timestamp)), "w") as f:
json.dump(data, f)
for i, dp in enumerate(data['message']):
### --- Handle mentions facts
# Create report ID from the url (same approach as for feeds) and title to this ID.
reportID = hashlib.sha256(dp['webLink'].encode('utf8')).hexdigest()
handle_fact(actapi.fact('name', dp['title']).source('report', reportID))
for obj in OBJECT_MAP: # run through all fields that we want to mention
if obj in dp and dp[obj]: # if the report contains data in the field
factType = OBJECT_MAP[obj](dp[obj]) # translate to ACT fact type
handle_fact(actapi.fact('mentions') # and create fact from field
.source('report', reportID)
.destination(factType, dp[obj].lower()))
if dp['url']:
handle_fact(actapi.fact('mentions')
.source('report', reportID)
.destination('uri', dp['url']))
try:
handle_uri(actapi, dp['url'])
except act.api.base.ValidationError as err:
logging.error("%s while storing url from mentions [%s]", err, dp['url'])
### --- IP -> malwareFamily
if dp['malwareFamily'] and dp['ip']:
chain = act.api.fact.fact_chain(
actapi.fact('connectsTo')
.source('content', '*')
.destination('uri', '*'),
actapi.fact('componentOf')
.source('ipv4', dp['ip'])
.destination('uri', '*'),
actapi.fact('classifiedAs')
.source('content', '*')
.destination('tool', dp['malwareFamily'].lower()))
for fact in chain:
handle_fact(fact)
### --- URL -> malwareFamily
elif dp['networkType'] == 'url' and dp['malwareFamily']:
try:
handle_uri(actapi, dp['url'])
except act.api.base.ValidationError as err:
logging.error("%s while storing url from mentions [%s]", err, dp['url'])
chain = act.api.fact.fact_chain(
actapi.fact('connectsTo')
.source('content', '*')
.destination('uri', dp['url']),
actapi.fact('classifiedAs')
.source('content', '*')
.destination('tool', dp['malwareFamily'].lower()))
for fact in chain:
handle_fact(fact)
### --- FQDN -> malwareFamily
elif dp['networkType'] == 'network' and dp['domain'] and dp['malwareFamily']:
chain = act.api.fact.fact_chain(
actapi.fact('connectsTo')
.source('content', '*')
.destination('uri', '*'),
actapi.fact('componentOf')
.source('fqdn', dp['domain'])
.destination('uri', '*'),
actapi.fact('classifiedAs')
.source('content', '*')
.destination('tool', dp['malwareFamily'].lower()))
for fact in chain:
handle_fact(fact)
### --- hash -> malwareFamily
elif dp['fileType'] and dp['malwareFamily'] and (dp['sha1'] or dp['sha256'] or dp['md5']):
for digest_type in ['md5', 'sha1', 'sha256']:
### In some cases the iSight api does not return a sha256 hashdigest
### so we need to make a chain through a placeholder content
if not dp['sha256']:
if dp[digest_type]:
chain = act.api.fact.fact_chain(
actapi.fact('represents')
.source('hash', dp[digest_type])
.destination('content', '*'),
actapi.fact('classifiedAs')
.source('content', '*')
.destination('tool', dp['malwareFamily']))
for fact in chain:
handle_fact(fact)
else: ## There is a sha256, so we do _not_ need a chain
if dp[digest_type]:
handle_fact(actapi.fact('classifiedAs')
.source('content', dp['sha256'])
.destination('tool', dp['malwareFamily']))
handle_fact(actapi.fact('represents')
.source('hash', dp[digest_type])
.destination('content', dp['sha256']))
### -- Hash --> actor
elif dp['fileType'] and dp['actor'] and (dp['sha1'] or dp['sha256'] or dp['md5']):
for digest_type in ['md5', 'sha1', 'sha256']:
### In some cases the iSight api does not return a sha256 hashdigest
### so we need to make a chain through a placeholder content
if not dp['sha256']:
if dp[digest_type]:
chain = act.api.fact.fact_chain(
actapi.fact('represents')
.source('hash', dp[digest_type])
.destination('content', '*'),
actapi.fact('observedIn')
.source('content', '*')
.destination('event', '*'),
actapi.fact('attributedTo')
.source('event', '*')
.destination('incident', '*'),
actapi.fact('attributedTo')
.source('incident', '*')
.destination('threatActor', dp['actor']))
for fact in chain:
handle_fact(fact)
else: ## There is a sha256, so we do _not_ need a chain between all the way from hexdigest
if dp[digest_type]:
handle_fact(actapi.fact('represents')
.source('hash', dp[digest_type])
.destination('content', dp['sha256']))
chain = act.api.fact.fact_chain(
actapi.fact('observedIn')
.source('content', dp['sha256'])
.destination('event', '*'),
actapi.fact('attributedTo')
.source('event', '*')
.destination('incident', '*'),
actapi.fact('attributedTo')
.source('incident', '*')
.destination('threatActor', dp['actor']))
for fact in chain:
handle_fact(fact)
### We do have a sha256 of a file (but possibly nothing else). Add the content to hexdigest facts
elif dp['fileType'] and dp['sha256']:
for digest in ['sha1', 'md5', 'sha256']:
if dp[digest]:
handle_fact(actapi.fact('represents')
.source('hash', dp[digest])
.destination('content', dp['sha256']))
if args.debugdir:
fields = [k for k, v in dp.items() if v and k not in ['reportId', 'title', 'ThreatScape',
'audience', 'intelligenceType',
'publishDate', 'reportLink', 'webLink']]
logging.error("[%s] Extra fields while handeling index[%s] '%s'", timestamp, i, ", ".join(fields))
### -- DEBUG!
else:
if args.debugdir:
fields = [k for k, v in dp.items() if v and k not in ['reportId', 'title', 'ThreatScape',
'audience', 'intelligenceType',
'publishDate', 'reportLink', 'webLink']]
logging.error("[%s] Unable to handle index[%s] with fields '%s'", timestamp, i, ", ".join(fields))
def main() -> None:
"""Main function"""
args = cli.handle_args(worker.parseargs("IP Filter"))
worker.init_act(args)
process()
def main() -> None:
"""main function"""
# Look for default ini file in "/etc/actworkers.ini" and ~/config/actworkers/actworkers.ini
# (or replace .config with $XDG_CONFIG_DIR if set)
args = cli.handle_args(parseargs())
actapi = worker.init_act(args)
if not (args.privatekey and args.publickey):
cli.fatal(
"You must specify --privatekey and --publickey on command line or in config file"
)
proxies = (
{"http": args.proxy_string, "https": args.proxy_string}
if args.proxy_string
else None
)
iSightHandler = ISightAPIRequestHandler(args.root, args.privatekey, args.publickey)
data = iSightHandler.indicators(days=args.days, proxies=proxies)
if "success" not in data or not data["success"]:
logging.error(
"Unable to download from isight API [%s]",
data["message"] if "message" in data else "NA",
)
return
timestamp = int(time.time())
### DEBUG -- dump json to disc for each run
if args.debugdir:
with open(
os.path.join(args.debugdir, "error-{0}.json".format(timestamp)), "w"
) as f:
json.dump(data, f)
for i, dp in enumerate(data["message"]):
### --- Handle mentions facts
# Create report ID from the url (same approach as for feeds) and title to this ID.
reportID = hashlib.sha256(dp["webLink"].encode("utf8")).hexdigest()
handle_fact(actapi.fact("name", dp["title"]).source("report", reportID))
for obj in OBJECT_MAP: # run through all fields that we want to mention
if obj in dp and dp[obj]: # if the report contains data in the field
factType = OBJECT_MAP[obj](dp[obj]) # translate to ACT fact type
handle_fact(
actapi.fact("mentions") # and create fact from field
.source("report", reportID)
.destination(factType, dp[obj].lower())
)
if dp["url"]:
handle_fact(
actapi.fact("mentions")
.source("report", reportID)
.destination("uri", dp["url"])
)
handle_uri(actapi, dp["url"])
### --- IP -> malwareFamily
if dp["malwareFamily"] and dp["ip"]:
handle_facts(
act.api.fact.fact_chain(
actapi.fact("connectsTo")
.source("content", "*")
.destination("uri", "*"),
actapi.fact("componentOf")
.source("ipv4", dp["ip"])
.destination("uri", "*"),
actapi.fact("classifiedAs")
.source("content", "*")
.destination("tool", dp["malwareFamily"]),
)
)
### --- URL -> malwareFamily
elif dp["networkType"] == "url" and dp["malwareFamily"]:
handle_uri(actapi, dp["url"])
handle_facts(
act.api.fact.fact_chain(
actapi.fact("connectsTo")
.source("content", "*")
.destination("uri", dp["url"]),
actapi.fact("classifiedAs")
.source("content", "*")
.destination("tool", dp["malwareFamily"]),
)
)
### --- FQDN -> malwareFamily
elif dp["networkType"] == "network" and dp["domain"] and dp["malwareFamily"]:
handle_facts(
act.api.fact.fact_chain(
actapi.fact("connectsTo")
.source("content", "*")
.destination("uri", "*"),
actapi.fact("componentOf")
.source("fqdn", dp["domain"])
.destination("uri", "*"),
actapi.fact("classifiedAs")
.source("content", "*")
.destination("tool", dp["malwareFamily"]),
)
)
### --- hash -> malwareFamily
elif (
dp["fileType"]
and dp["malwareFamily"]
and (dp["sha1"] or dp["sha256"] or dp["md5"])
):
for digest_type in ["md5", "sha1", "sha256"]:
### In some cases the iSight api does not return a sha256 hashdigest
### so we need to make a chain through a placeholder content
if not dp["sha256"]:
if dp[digest_type]:
handle_facts(
act.api.fact.fact_chain(
actapi.fact("represents")
.source("hash", dp[digest_type])
.destination("content", "*"),
actapi.fact("classifiedAs")
.source("content", "*")
.destination("tool", dp["malwareFamily"]),
)
)
else: ## There is a sha256, so we do _not_ need a chain
if dp[digest_type]:
handle_fact(
actapi.fact("classifiedAs")
.source("content", dp["sha256"])
.destination("tool", dp["malwareFamily"])
)
handle_fact(
actapi.fact("represents")
.source("hash", dp[digest_type])
.destination("content", dp["sha256"])
)
### -- Hash --> actor
elif (
dp["fileType"] and dp["actor"] and (dp["sha1"] or dp["sha256"] or dp["md5"])
):
for digest_type in ["md5", "sha1", "sha256"]:
### In some cases the iSight api does not return a sha256 hashdigest
### so we need to make a chain through a placeholder content
if not dp["sha256"]:
if dp[digest_type]:
handle_facts(
act.api.fact.fact_chain(
actapi.fact("represents")
.source("hash", dp[digest_type])
.destination("content", "*"),
actapi.fact("observedIn")
.source("content", "*")
.destination("incident", "*"),
actapi.fact("attributedTo")
.source("incident", "*")
.destination("threatActor", dp["actor"]),
)
)
else: ## There is a sha256, so we do _not_ need a chain between all the way from hexdigest
if dp[digest_type]:
handle_fact(
actapi.fact("represents")
.source("hash", dp[digest_type])
.destination("content", dp["sha256"])
)
handle_facts(
act.api.fact.fact_chain(
actapi.fact("observedIn")
.source("content", dp["sha256"])
.destination("incident", "*"),
actapi.fact("attributedTo")
.source("incident", "*")
.destination("threatActor", dp["actor"]),
)
)
### We do have a sha256 of a file (but possibly nothing else). Add the content to hexdigest facts
elif dp["fileType"] and dp["sha256"]:
for digest in ["sha1", "md5", "sha256"]:
if dp[digest]:
handle_fact(
actapi.fact("represents")
.source("hash", dp[digest])
.destination("content", dp["sha256"])
)
if args.debugdir:
fields = [
k
for k, v in dp.items()
if v
and k
not in [
"reportId",
"title",
"ThreatScape",
"audience",
"intelligenceType",
"publishDate",
"reportLink",
"webLink",
]
]
logging.error(
"[%s] Extra fields while handeling index[%s] '%s'",
timestamp,
i,
", ".join(fields),
)
### -- DEBUG!
else:
if args.debugdir:
fields = [
k
for k, v in dp.items()
if v
and k
not in [
"reportId",
"title",
"ThreatScape",
"audience",
"intelligenceType",
"publishDate",
"reportLink",
"webLink",
]
]
logging.error(
"[%s] Unable to handle index[%s] with fields '%s'",
timestamp,
i,
", ".join(fields),
)
