def detect_imports(rule):
"""Detect imports: DEPRECATED."""
from plyara.utils import detect_imports
import warnings
warnings.warn(
'Static method detect_imports() is deprecated: use plyara.utils',
DeprecationWarning)
return detect_imports(rule)
def test_detect_imports(self):
for imp in ('androguard', 'cuckoo', 'dotnet', 'elf', 'hash', 'magic',
'math', 'pe'):
with data_dir.joinpath(
'import_ruleset_{}.yar'.format(imp)).open('r') as fh:
inputString = fh.read()
results = Plyara().parse_string(inputString)
for rule in results:
self.assertEqual(detect_imports(rule), [imp])
def generate_kwargs_from_parsed_rule(parsed_rule):
# Generate parsed rule kwargs for saving a rule
name = parsed_rule['rule_name']
tags = parsed_rule.get('tags', [])
scopes = parsed_rule.get('scopes', [])
# TODO : Update when Plyara moves to clean Python types
metadata = parsed_rule.get('metadata', {})
for key, value in metadata.items():
if value not in ('true', 'false'):
try:
value = int(value)
except ValueError:
metadata[key] = '"' + value + '"'
strings = parsed_rule.get('strings', [])
condition = parsed_rule['condition_terms']
# TODO : Update when Plyara moves to stripping quotes from detect_imports module
imports = [imp.strip('"') for imp in utils.detect_imports(parsed_rule)]
comments = parsed_rule.get('comments', [])
dependencies = utils.detect_dependencies(parsed_rule)
# Calculate hash value of rule strings and condition
logic_hash = utils.generate_logic_hash(parsed_rule)
# TEMP FIX - Use only a single instance of a metakey
# until YaraGuardian models and functions can be updated
for key, value in metadata.items():
if isinstance(value, list):
metadata[key] = value[0]
return {
'name': name,
'tags': list(set(tags)),
'scopes': list(set(scopes)),
'imports': list(set(imports)),
'comments': list(set(comments)),
'metadata': metadata,
'strings': strings,
'condition': condition,
'dependencies': dependencies,
'logic_hash': logic_hash
}
def _save_signatures(self,
signatures,
source,
default_status=DEFAULT_STATUS,
default_classification=None):
if len(signatures) == 0:
self.log.info(f"There are no signatures for {source}, skipping...")
return False
order = 1
upload_list = []
for signature in signatures:
classification = default_classification or self.classification.UNRESTRICTED
signature_id = None
version = 1
status = default_status
for meta in signature.get('metadata', {}):
for k, v in meta.items():
if k in ["classification", "sharing"]:
classification = v
elif k in ['id', 'rule_id', 'signature_id']:
signature_id = v
elif k in ['version', 'rule_version', 'revision']:
version = v
elif k in ['status', 'al_status']:
status = v
# Convert CCCS YARA status to AL signature status
if status == "RELEASED":
status = "DEPLOYED"
elif status == "DEPRECATED":
status = "DISABLED"
# Fallback status
if status not in [
"DEPLOYED", "NOISY", "DISABLED", "STAGING", "TESTING",
"INVALID"
]:
status = default_status
# Fix imports and remove cuckoo
signature['imports'] = utils.detect_imports(signature)
if "cuckoo" not in signature['imports']:
sig = Signature(
dict(
classification=classification,
data=utils.rebuild_yara_rule(signature),
name=signature.get('rule_name'),
order=order,
revision=int(float(version)),
signature_id=signature_id
or signature.get('rule_name'),
source=source,
status=status,
type=self.importer_type,
))
upload_list.append(sig.as_primitives())
else:
self.log.warning(
f"Signature '{signature.get('rule_name')}' skipped because it uses cuckoo module."
)
order += 1
r = self.update_client.signature.add_update_many(
source, self.importer_type, upload_list)
self.log.info(
f"Imported {r['success']}/{order - 1} signatures from {source} into Assemblyline"
)
return r['success']
def yara_update(updater_type, update_config_path, update_output_path,
download_directory, externals, cur_logger) -> None:
"""
Using an update configuration file as an input, which contains a list of sources, download all the file(s).
"""
# noinspection PyBroadException
try:
# Load updater configuration
update_config = {}
if update_config_path and os.path.exists(update_config_path):
with open(update_config_path, 'r') as yml_fh:
update_config = yaml.safe_load(yml_fh)
else:
cur_logger.error(f"Update configuration file doesn't exist: {update_config_path}")
exit()
# Exit if no update sources given
if 'sources' not in update_config.keys() or not update_config['sources']:
cur_logger.error(f"Update configuration does not contain any source to update from")
exit()
# Initialise al_client
server = update_config['ui_server']
user = update_config['api_user']
api_key = update_config['api_key']
cur_logger.info(f"Connecting to Assemblyline API: {server}...")
al_client = get_client(server, apikey=(user, api_key), verify=False)
cur_logger.info(f"Connected!")
# Parse updater configuration
previous_update = update_config.get('previous_update', None)
previous_hash = json.loads(update_config.get('previous_hash', None) or "{}")
sources = {source['name']: source for source in update_config['sources']}
files_sha256 = {}
files_default_classification = {}
# Create working directory
updater_working_dir = os.path.join(tempfile.gettempdir(), 'updater_working_dir')
if os.path.exists(updater_working_dir):
shutil.rmtree(updater_working_dir)
os.makedirs(updater_working_dir)
# Go through each source and download file
for source_name, source in sources.items():
os.makedirs(os.path.join(updater_working_dir, source_name))
# 1. Download signatures
cur_logger.info(f"Downloading files from: {source['uri']}")
uri: str = source['uri']
if uri.endswith('.git'):
files = git_clone_repo(download_directory, source, cur_logger, previous_update=previous_update)
else:
files = [url_download(download_directory, source, cur_logger, previous_update=previous_update)]
processed_files = set()
# 2. Aggregate files
file_name = os.path.join(updater_working_dir, f"{source_name}.yar")
mode = "w"
for file in files:
# File has already been processed before, skip it to avoid duplication of rules
if file in processed_files:
continue
cur_logger.info(f"Processing file: {file}")
file_dirname = os.path.dirname(file)
processed_files.add(os.path.normpath(file))
with open(file, 'r') as f:
f_lines = f.readlines()
temp_lines = []
for i, f_line in enumerate(f_lines):
if f_line.startswith("include"):
lines, processed_files = replace_include(f_line, file_dirname, processed_files, cur_logger)
temp_lines.extend(lines)
else:
temp_lines.append(f_line)
# guess the type of files that we have in the current file
guessed_category = guess_category(file)
parser = Plyara()
signatures = parser.parse_string("\n".join(temp_lines))
# Ignore "cuckoo" rules
if "cuckoo" in parser.imports:
parser.imports.remove("cuckoo")
# Guess category
if guessed_category:
for s in signatures:
if 'metadata' not in s:
s['metadata'] = []
# Do not override category with guessed category if it already exists
for meta in s['metadata']:
if 'category' in meta:
continue
s['metadata'].append({'category': guessed_category})
s['metadata'].append({guessed_category: s.get('rule_name')})
# Save all rules from source into single file
with open(file_name, mode) as f:
for s in signatures:
# Fix imports and remove cuckoo
s['imports'] = utils.detect_imports(s)
if "cuckoo" not in s['imports']:
f.write(utils.rebuild_yara_rule(s))
if mode == "w":
mode = "a"
# Check if the file is the same as the last run
if os.path.exists(file_name):
cache_name = os.path.basename(file_name)
sha256 = get_sha256_for_file(file_name)
if sha256 != previous_hash.get(cache_name, None):
files_sha256[cache_name] = sha256
files_default_classification[cache_name] = source.get('default_classification',
classification.UNRESTRICTED)
else:
cur_logger.info(f'File {cache_name} has not changed since last run. Skipping it...')
if files_sha256:
cur_logger.info(f"Found new {updater_type.upper()} rules files to process!")
yara_importer = YaraImporter(updater_type, al_client, logger=cur_logger)
# Validating and importing the different signatures
for base_file in files_sha256:
cur_logger.info(f"Validating output file: {base_file}")
cur_file = os.path.join(updater_working_dir, base_file)
source_name = os.path.splitext(os.path.basename(cur_file))[0]
default_classification = files_default_classification.get(base_file, classification.UNRESTRICTED)
try:
_compile_rules(cur_file, externals, cur_logger)
yara_importer.import_file(cur_file, source_name, default_classification=default_classification)
except Exception as e:
raise e
else:
cur_logger.info(f'No new {updater_type.upper()} rules files to process...')
# Check if new signatures have been added
if al_client.signature.update_available(since=previous_update or '', sig_type=updater_type)['update_available']:
cur_logger.info("An update is available for download from the datastore")
if not os.path.exists(update_output_path):
os.makedirs(update_output_path)
temp_zip_file = os.path.join(update_output_path, 'temp.zip')
al_client.signature.download(output=temp_zip_file,
query=f"type:{updater_type} AND (status:NOISY OR status:DEPLOYED)")
if os.path.exists(temp_zip_file):
with ZipFile(temp_zip_file, 'r') as zip_f:
zip_f.extractall(update_output_path)
os.remove(temp_zip_file)
# Create the response yaml
with open(os.path.join(update_output_path, 'response.yaml'), 'w') as yml_fh:
yaml.safe_dump(dict(hash=json.dumps(files_sha256)), yml_fh)
cur_logger.info(f"New ruleset successfully downloaded and ready to use")
cur_logger.info(f"{updater_type.upper()} updater completed successfully")
except Exception:
cur_logger.exception("Updater ended with an exception!")