def ValhallaReporter(values):
vallaha_val = []
for usrInput in values:
valFramework = copy.deepcopy(empty)
chk_256 = list(iocextract.extract_sha256_hashes(usrInput))
if chk_256:
usrInput = chk_256[0]
v = ValhallaAPI(api_key=valhallaKey)
response = v.get_hash_info(hash=usrInput)
if response.get('status') == 'empty':
pass
else:
dataReport = response.get('results')[0]
if len(dataReport) > 0:
valFramework.update({'Action On': usrInput})
valFramework.update(
{'Positives': dataReport.get('positives')})
valFramework.update(
{'Rule Name': dataReport.get('rulename')})
valFramework.update({'Tags': dataReport.get('tags')})
valFramework.update(
{'Timestamp': dataReport.get('timestamp')})
valFramework.update({'Total': dataReport.get('total')})
valFramework.update({
'Score':
str(dataReport.get('positives')) + '/' +
str(dataReport.get('total'))
})
vallaha_val.append({'Vallaha': valFramework})
return vallaha_val
def test_quote():
"""
Tests the quote page to check if the service can be access
:return:
"""
v = ValhallaAPI(api_key="")
assert "brave shall live forever" in v.get_quote()
def test_status():
"""
Retrieves the API status
:return:
"""
v = ValhallaAPI(api_key=DEMO_KEY)
status = v.get_status()
assert status["status"] == "green"
def test_demo_rules_json():
"""
Retrieves the demo rules from the rule feed
:return:
"""
v = ValhallaAPI(api_key=DEMO_KEY)
rules_response = v.get_rules_json()
assert len(rules_response['rules']) > 0
def test_invalid_key():
"""
Trying to retrieve rules with an invalid key
:return:
"""
v = ValhallaAPI(api_key=INVALID_KEY)
with pytest.raises(Exception):
v.get_rules_text()
def test_get_rule_info():
"""
Trying to retrieve info for a certain rule (the only one accessible with DEMO key)
:return:
"""
v = ValhallaAPI(api_key=DEMO_KEY)
response = v.get_rule_info('Casing_Anomaly_ByPass')
assert len(response) > 1
assert len(response['rule_matches']) > 0
def test_demo_rules_text():
"""
Retrieves the demo rules from the rule feed
:return:
"""
v = ValhallaAPI(api_key=DEMO_KEY)
response = v.get_rules_text()
assert RULES_TEXT in response
assert len(response) > 500
def test_get_hash_info():
"""
Trying to retrieve info for a certain hash (the only one accessible with DEMO key)
:return:
"""
v = ValhallaAPI(api_key=DEMO_KEY)
response = v.get_hash_info(
'8a883a74702f83a273e6c292c672f1144fd1cce8ee126cd90c95131e870744af')
assert len(response) > 1
assert len(response['results']) > 0
def test_invalid_key():
"""
Trying to retrieve rules with an invalid key
:return:
"""
v = ValhallaAPI(api_key=INVALID_KEY)
response = v.get_rules_text()
assert 'status' in response
assert 'error' in response
assert len(response) < 500
def test_demo_rules_search_limited():
"""
Retrieves the demo rules from the rule feed with custom expressions
:return:
"""
v = ValhallaAPI(api_key=DEMO_KEY)
rules_response1 = v.get_rules_json()
rules_response2 = v.get_rules_json(search="Mimikatz")
assert len(rules_response1['rules']) > 1
assert len(rules_response2['rules']) > 1
assert len(rules_response1['rules']) > len(rules_response2['rules'])
def test_demo_rules_tag_limited():
"""
Retrieves the demo rules from the rule feed with custom expressions
:return:
"""
v = ValhallaAPI(api_key=DEMO_KEY)
rules_response1 = v.get_rules_json()
rules_response2 = v.get_rules_json(tags=['APT'])
assert len(rules_response1['rules']) > 0
assert len(rules_response2['rules']) > 0
assert len(rules_response1['rules']) > len(rules_response2['rules'])
def test_demo_rule_info():
"""
Retrieves the demo rule info
:return:
"""
v = ValhallaAPI(api_key=DEMO_KEY)
rules_response1 = v.get_rule_info(RULE_INFO_DISALLOWED)
assert 'rule_matches' not in rules_response1
rules_response2 = v.get_rule_info(RULE_INFO_TEST)
assert 'rule_matches' in rules_response2
assert len(rules_response2['rule_matches']) > 0
def test_subscription():
"""
Retrieves the subscription status of the current user
:return:
"""
v = ValhallaAPI()
response = v.get_subscription()
print(response)
assert len(response) == 5
assert response["subscription"] == "limited"
assert response["tags"] == ['DEMO']
def test_demo_rules_product_limited():
"""
Retrieves the demo rules from the rule feed with a product set
:return:
"""
v = ValhallaAPI(api_key=DEMO_KEY)
rules_response = v.get_rules_json()
rules_response_limited = v.get_rules_json(product="DummyTest")
assert len(rules_response['rules']) > 0
assert len(rules_response['rules']) > len(rules_response_limited['rules'])
rules_response_limited2 = v.get_rules_json(product="CarbonBlack")
assert len(rules_response_limited2['rules']) > 0
def test_demo_rules_custom_limited():
"""
Retrieves the demo rules from the rule feed with custom expressions
:return:
"""
v = ValhallaAPI(api_key=DEMO_KEY)
rules_response1 = v.get_rules_json(product="DummyTest")
rules_response2 = v.get_rules_json(max_version="3.2.0", modules=['pe'])
rules_response3 = v.get_rules_json(max_version="3.2.0", modules=['pe'], with_crypto=False)
assert len(rules_response1['rules']) > 0
assert len(rules_response2['rules']) > 0
assert len(rules_response1['rules']) < len(rules_response2['rules'])
assert len(rules_response3['rules']) < len(rules_response2['rules'])
def test_demo_rules_combo_limited():
"""
Retrieves the demo rules from the rule feed with custom expressions
:return:
"""
v = ValhallaAPI(api_key=DEMO_KEY)
rules_response1 = v.get_rules_json()
rules_response2 = v.get_rules_json(score=60)
rules_response3 = v.get_rules_json(tags=['SUSP'], score=60)
assert len(rules_response1['rules']) > 1
assert len(rules_response2['rules']) > 1
assert len(rules_response3['rules']) > 1
assert len(rules_response1['rules']) > len(rules_response2['rules'])
assert len(rules_response2['rules']) > len(rules_response3['rules'])
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/../config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
# Extra config
self.confidence_level = get_config_variable(
"CONNECTOR_CONFIDENCE_LEVEL",
["connector", "confidence_level"],
config,
isNumber=True,
)
self.update_existing_data = get_config_variable(
"CONNECTOR_UPDATE_EXISTING_DATA",
["connector", "update_existing_data"],
config,
)
self.API_KEY = get_config_variable("VALHALLA_API_KEY",
["valhalla", "api_key"], config)
self.INTERVAL_SEC = get_config_variable(
"VALHALLA_INTERVAL_SEC",
["valhalla", "interval_sec"],
config,
isNumber=True,
)
self.helper = OpenCTIConnectorHelper(config)
self.helper.log_info(f"loaded valhalla config: {config}")
# If we run without API key we can assume all data is TLP:WHITE else we
# default to TLP:AMBER to be safe.
if self.API_KEY == "" or self.API_KEY is None:
self.default_marking = self.helper.api.marking_definition.read(
id=TLP_WHITE["id"])
self.valhalla_client = ValhallaAPI()
else:
self.default_marking = self.helper.api.marking_definition.read(
id=TLP_AMBER["id"])
self.valhalla_client = ValhallaAPI(api_key=self.API_KEY)
self.knowledge_importer = KnowledgeImporter(
self.helper,
self.confidence_level,
self.update_existing_data,
self.default_marking,
self.valhalla_client,
)
def main():
"""
Main Function (used as entry point)
:return:
"""
# Parse Arguments
parser = argparse.ArgumentParser(description='Valhalla-CLI')
parser.add_argument('-k',
help='API KEY',
metavar='apikey',
default=ValhallaAPI.DEMO_KEY)
parser.add_argument('-c',
help='Config file (see README for details)',
metavar='config-file',
default=os.path.join(str(Path.home()), ".valhalla"))
parser.add_argument('-o',
help='output file',
metavar='output-file',
default=ValhallaAPI.DEFAULT_OUTPUT_FILE)
parser.add_argument('--check',
action='store_true',
default=False,
help='Check subscription info and total rule count')
parser.add_argument('--debug',
action='store_true',
default=False,
help='Debug output')
group_proxy = parser.add_argument_group(
'=======================================================================\nProxy'
)
group_proxy.add_argument('-p',
help='proxy URL (e.g. https://my.proxy.net:8080)',
metavar='proxy-url',
default='')
group_proxy.add_argument('-pu',
help='proxy user',
metavar='proxy-user',
default='')
group_proxy.add_argument('-pp',
help='proxy password',
metavar='proxy-pass',
default='')
group_filter = parser.add_argument_group(
'=======================================================================\nFilter'
)
group_filter.add_argument('-fp',
help='filter product (valid products are: %s)' %
", ".join(ValhallaAPI.PRODUCT_IDENTIFIER),
metavar='product',
default='')
group_filter.add_argument(
'-fv',
help='get rules that support the given YARA version and lower',
metavar='yara-version',
default='')
group_filter.add_argument(
'-fm',
help=
'set a list of modules that your product supports (e.g. "-fm pe hash") '
'(setting no modules means that all modules are supported by your product)',
action='append',
nargs='+',
metavar='modules')
group_filter.add_argument(
'-ft',
help='set a list of tags to receive (e.g. "-ft APT MAL")',
action='append',
nargs='+',
metavar='tags')
group_filter.add_argument(
'-fs',
help='minimum score of rules to retrieve (e.g. "-fs 75")',
metavar='score',
default=0)
group_filter.add_argument(
'-fq',
help=
'get only rules that match a certain keyword in name or description '
'(e.g. "-fq Mimikatz")',
metavar='query',
default='')
group_filter.add_argument(
'--nocrypto',
help='filter all rules that require YARA to be compiled with crypto '
'support (OpenSSL)',
action='store_false',
default=True)
group_proxy = parser.add_argument_group(
'=======================================================================\nLookups'
)
group_proxy.add_argument(
'-lr',
help='Lookup a certain rule (returns matching samples)',
metavar='lookup-rule',
default='')
group_proxy.add_argument(
'-lh',
help='Lookup a certain sample hash (sha256) (returns matching rules)',
metavar='lookup-hash',
default='')
group_proxy.add_argument(
'-lk',
help='Lookup rules with a certain keyword (returns matching rules)',
metavar='lookup-keyword',
default='')
group_proxy.add_argument(
'-lkm',
help=
'Lookup hashes of samples on which rules have matches that contain a certain '
'keyword (returns matching sample hashes)',
metavar='lookup-keyword',
default='')
group_proxy.add_argument('-lo',
help='Output file for the lookup output',
metavar='lookup-output',
default='')
args = parser.parse_args()
print(" ")
print("===========================================================")
print(" _ __ ____ ____ _______ ____ ")
print(" | | / /__ _/ / / ___ _/ / /__ _____/ ___/ / / _/ ")
print(" | |/ / _ `/ / _ \\/ _ `/ / / _ `/___/ /__/ /___/ / ")
print(" |___/\\_,_/_/_//_/\\_,_/_/_/\\_,_/ \\___/____/___/ ")
print(" Ver. %s, Florian Roth, 2021 " %
__version__)
print(" ")
print("===========================================================")
print(" ")
# Logging
logFormatter = logging.Formatter("[%(levelname)-5.5s] %(message)s")
logFormatterRemote = logging.Formatter(
"{0} [%(levelname)-5.5s] %(message)s".format(platform.uname()[1]))
Log = logging.getLogger(__name__)
Log.setLevel(logging.INFO)
# Console Handler
consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(logFormatter)
Log.addHandler(consoleHandler)
# API Key
apikey = args.k
Log.info(
"Trying to read Valhalla config file at '%s' (set manually with -c)" %
args.c)
if os.path.exists(args.c):
Log.debug("Config file found at '%s'" % args.c)
config = configparser.ConfigParser()
config.read(args.c)
if 'DEFAULT' not in config:
Log.error(
"section [DEFAULT] missing in config file - skipping this config"
)
else:
apikey = config['DEFAULT']['APIKEY']
Log.info("Successfully read config file")
else:
Log.info(
"No config file found, will rely on API KEY passed via cmd line arguments (-k)"
)
# Check key
if apikey == ValhallaAPI.DEMO_KEY:
Log.warning(
"You are using the DEMO API key and will only retrieve the reduced open source signature set"
)
Log.warning(
"Set your private API key with '-k APIKEY' to get the rule sets that your have subscribed"
)
# Create the ValhallaAPI object
v = ValhallaAPI(api_key=apikey)
# Subscription check
if args.check:
status = v.get_subscription()
if 'active' in status:
if status['active']:
Log.info("Account is active: %s" % status)
sys.exit(0)
else:
Log.error("Account is inactive: %s" % status)
sys.exit(1)
else:
Log.error("Error: %s" % status['message'])
sys.exit(1)
# Proxy
if args.p:
Log.info("Setting proxy URL: %s USER: %s PASS: (hidden)" %
(args.p, args.pu))
if args.p.startswith("http:"):
Log.warning(
"URL starts with http instead of https - you should use a TLS encrypted connection"
)
v.set_proxy(args.p, args.pu, args.pp)
# Default: Get all rules that the set API key is subscribed to
# prepare some variables
modules = []
if args.fm:
modules = args.fm[0]
tags = []
if args.ft:
tags = args.ft[0]
# Lookups
if args.lr or args.lh or args.lk or args.lkm:
# Rule Lookup
if args.lr != "":
r = v.get_rule_info(args.lr)
# Hash Lookup
if args.lh != "":
r = v.get_hash_info(args.lh)
# Keyword to Rules Lookup
if args.lk != "":
r = v.get_keyword_rules(args.lk)
# Keyword to Rule Matches Lookup
if args.lkm != "":
r = v.get_keyword_rule_matches(args.lkm)
# Write them to an output file
if args.lo:
with open(args.lo, 'w') as fh:
fh.write(json.dumps(r, indent=4, sort_keys=True))
else:
# Show results
print(json.dumps(r, indent=4, sort_keys=True))
sys.exit(0)
# Score warning
if args.fs == 0:
Log.warning(
"Note that an unfiltered set (-fs 0) contains low scoring rules used for threat hunting purposes"
)
# Info output
Log.info(
"Retrieving rules with params PRODUCT: %s MAX_VERSION: %s MODULES: %s WITH_CRYPTO: %s TAGS: %s "
"SCORE: %s QUERY: %s" %
(args.fp, args.fv, ", ".join(modules), str(
args.nocrypto), ", ".join(tags), str(args.fs), args.fq))
# Retrieve rules
try:
response = v.get_rules_text(
product=args.fp,
max_version=args.fv,
modules=modules,
with_crypto=args.nocrypto,
tags=tags,
score=int(args.fs),
search=args.fq,
)
except UnknownProductError as e:
Log.error("Unknown product identifier - please use one of these: %s",
", ".join(ValhallaAPI.PRODUCT_IDENTIFIER))
sys.exit(1)
except ApiError as e:
Log.error(e.message)
sys.exit(1)
# Response information
Log.info("Number of retrieved rules: %d" % v.last_retrieved_rules_count)
# Output
output_file = args.o
# Tanium accepts only the ".yara" extension for imports
if args.fp == "Tanium" and output_file == "valhalla-rules.yar":
output_file = "valhalla-rules.yara"
# Write to the output file
Log.info("Writing retrieved rules into: %s" % output_file)
with open(output_file, 'w') as fh:
fh.write(response)
class Valhalla:
"""OpenCTI valhalla main class"""
_DEMO_API_KEY = "1111111111111111111111111111111111111111111111111111111111111111"
_STATE_LAST_RUN = "last_run"
_VALHALLA_LAST_VERSION = "valhalla_last_version"
def __init__(self):
# Instantiate the connector helper from config
config_file_path = os.path.dirname(
os.path.abspath(__file__)) + "/../config.yml"
config = (yaml.load(open(config_file_path), Loader=yaml.FullLoader)
if os.path.isfile(config_file_path) else {})
# Extra config
self.confidence_level = get_config_variable(
"CONNECTOR_CONFIDENCE_LEVEL",
["connector", "confidence_level"],
config,
isNumber=True,
)
self.update_existing_data = get_config_variable(
"CONNECTOR_UPDATE_EXISTING_DATA",
["connector", "update_existing_data"],
config,
)
self.API_KEY = get_config_variable("VALHALLA_API_KEY",
["valhalla", "api_key"], config)
self.INTERVAL_SEC = get_config_variable(
"VALHALLA_INTERVAL_SEC",
["valhalla", "interval_sec"],
config,
isNumber=True,
)
self.helper = OpenCTIConnectorHelper(config)
self.helper.log_info(f"loaded valhalla config: {config}")
# If we run without API key we can assume all data is TLP:WHITE else we
# default to TLP:AMBER to be safe.
if self.API_KEY == "" or self.API_KEY is None:
self.default_marking = self.helper.api.marking_definition.read(
id=TLP_WHITE["id"])
self.valhalla_client = ValhallaAPI()
else:
self.default_marking = self.helper.api.marking_definition.read(
id=TLP_AMBER["id"])
self.valhalla_client = ValhallaAPI(api_key=self.API_KEY)
self.knowledge_importer = KnowledgeImporter(
self.helper,
self.confidence_level,
self.update_existing_data,
self.default_marking,
self.valhalla_client,
)
def run(self):
self.helper.log_info("starting valhalla connector...")
while True:
try:
status_data = self.valhalla_client.get_status()
api_status = Status.parse_obj(status_data)
self.helper.log_info(f"current valhalla status: {api_status}")
current_time = int(datetime.utcnow().timestamp())
current_state = self._load_state()
self.helper.log_info(f"loaded state: {current_state}")
last_run = self._get_state_value(current_state,
self._STATE_LAST_RUN)
last_valhalla_version = self._get_state_value(
current_state, self._VALHALLA_LAST_VERSION)
if self._is_scheduled(
last_run, current_time) and self._check_version(
last_valhalla_version, api_status.version):
self.helper.log_info("running importers")
knowledge_importer_state = self._run_knowledge_importer(
current_state)
self.helper.log_info("done with running importers")
new_state = current_state.copy()
new_state.update(knowledge_importer_state)
new_state[self._STATE_LAST_RUN] = int(
datetime.utcnow().timestamp())
new_state[self._VALHALLA_LAST_VERSION] = api_status.version
self.helper.log_info(f"storing new state: {new_state}")
self.helper.set_state(new_state)
self.helper.log_info(
f"state stored, next run in: {self._get_interval()} seconds"
)
else:
new_interval = self._get_interval() - (current_time -
last_run)
self.helper.log_info(
f"connector will not run, next run in: {new_interval} seconds"
)
# After a successful run pause at least 60sec
time.sleep(60)
except (KeyboardInterrupt, SystemExit):
self.helper.log_info("connector stop")
exit(0)
except Exception as e:
self.helper.log_error(str(e))
exit(0)
def _run_knowledge_importer(
self, current_state: Mapping[str, Any]) -> Mapping[str, Any]:
return self.knowledge_importer.run(current_state)
def _get_interval(self) -> int:
return int(self.INTERVAL_SEC)
def _load_state(self) -> Dict[str, Any]:
current_state = self.helper.get_state()
if not current_state:
return {}
return current_state
@staticmethod
def _get_state_value(state: Optional[Mapping[str, Any]],
key: str,
default: Optional[Any] = None) -> Any:
if state is not None:
return state.get(key, default)
return default
def _is_scheduled(self, last_run: Optional[int],
current_time: int) -> bool:
if last_run is None:
return True
time_diff = current_time - last_run
return time_diff >= self._get_interval()
def _check_version(self, last_version: Optional[int],
current_version: int) -> bool:
if last_version is None:
return True
return current_version > last_version