def rebuild_yara_rule(rule):
"""Rebuild rule: DEPRECATED."""
from plyara.utils import rebuild_yara_rule
import warnings
warnings.warn(
'Static method rebuild_yara_rule() is deprecated: use plyara.utils',
DeprecationWarning)
return rebuild_yara_rule(rule)
def test_rebuild_yara_rule(self):
with data_dir.joinpath('rebuild_ruleset.yar').open(
'r', encoding='utf-8') as fh:
inputString = fh.read()
result = Plyara().parse_string(inputString)
rebuilt_rules = str()
for rule in result:
rebuilt_rules += rebuild_yara_rule(rule)
self.assertEqual(inputString, rebuilt_rules)
def format_rule(self):
raw_rule = {}
raw_rule['rule_name'] = self.name
raw_rule['tags'] = self.tags
raw_rule['imports'] = self.imports
raw_rule['metadata'] = self.metadata
raw_rule['strings'] = self.strings
raw_rule['condition_terms'] = self.condition
raw_rule['scopes'] = self.scopes
formatted_rule = utils.rebuild_yara_rule(raw_rule)
return formatted_rule
def exclude_rules(rules_file_name):
with open(rules_file_name, "r") as rules_json_file:
rules_json = json.loads(rules_json_file.read())
for repository in rules_json["repositories"]:
excluded_rules = repository["excludeRules"]
for file_name in excluded_rules.keys():
rule_names = excluded_rules[file_name]
with open(file_name, 'r') as file:
yara_rules = parser.parse_string(file.read())
filtered_rules = [
rule for rule in yara_rules
if rule['rule_name'] not in rule_names
]
parser.clear()
with open(file_name, 'w') as file:
for rule in filtered_rules:
file.write(rebuild_yara_rule(rule))
def test_rebuild_yara_rule_metadata(self):
test_rule = """
rule check_meta {
meta:
string_value = "TEST STRING"
string_value = "DIFFERENT TEST STRING"
bool_value = true
bool_value = false
digit_value = 5
digit_value = 10
condition:
true
}
"""
parsed = Plyara().parse_string(test_rule)
for rule in parsed:
unparsed = rebuild_yara_rule(rule)
self.assertIn('string_value = "TEST STRING"', unparsed)
self.assertIn('string_value = "DIFFERENT TEST STRING"', unparsed)
self.assertIn('bool_value = true', unparsed)
self.assertIn('bool_value = false', unparsed)
self.assertIn('digit_value = 5', unparsed)
self.assertIn('digit_value = 10', unparsed)
def measure(rule, cycles, show_score=True, c_duration=0):
"""
Measure rule performance
:param rule: the YARA rule to test
:param cycles: number of iterations over the sample set
:param show_score: show the performance score
:param c_duration: duration of the calibration run
:return duration: duration in seconds
:return count: count of samples in the given samples folders
"""
yara_rule_string = plutils.rebuild_yara_rule(rule)
y = yara.compile(source=yara_rule_string)
Log.info("Scanning sample set with rule: %s" % rule['rule_name'])
start = time.time()
count = 0
for s in SAMPLE_SET:
if not os.path.exists(s):
Log.error("[E] Error: sample directory '%s' doesn't exist" % s)
else:
for (dirpath, dirnames, filenames) in os.walk(s):
for filename in filenames:
count += 1
for _ in range(cycles):
sample_file = os.path.join(dirpath, filename)
with open(sample_file, 'rb') as fh:
fdata = fh.read()
matches = y.match(data=fdata)
end = time.time()
duration = end - start
if show_score:
# If a calibration duration has been evaluated
if c_duration > 0:
print("Duration: %.2f (%0.2f)" % (duration,
(duration - c_duration)))
else:
print("Duration: %.2f" % duration)
return duration, count
# first test all at once (this might easily fail on multiple files with duplicate rulenames)
measure_rule = CALIBRATION_RULES + rules_all
rule_name = "All " + str(
len(rules_list)) + " rules from all input files"
measure(measure_rule,
cycles,
progress,
show_score=True,
c_duration=crule_duration,
rule_name=rule_name,
alert_diff=alert_diff)
# Scan files
for r in rules_list:
yara_rule_string = plutils.rebuild_yara_rule(r)
rule_name = r['rule_name']
if len(yara_rule_string) > 20000:
log_warning_rich(
"Big rule: " + rule_name + " has " +
str(len(yara_rule_string)) + " bytes", yara_rule_string)
measure_rule = CALIBRATION_RULES + yara_rule_string
measure(measure_rule,
cycles,
progress,
show_score=True,
c_duration=crule_duration,
rule_name=rule_name,
alert_diff=alert_diff,
def _save_signatures(self,
signatures,
source,
default_status=DEFAULT_STATUS,
default_classification=None):
if len(signatures) == 0:
self.log.info(f"There are no signatures for {source}, skipping...")
return False
order = 1
upload_list = []
for signature in signatures:
classification = default_classification or self.classification.UNRESTRICTED
signature_id = None
version = 1
status = default_status
for meta in signature.get('metadata', {}):
for k, v in meta.items():
if k in ["classification", "sharing"]:
classification = v
elif k in ['id', 'rule_id', 'signature_id']:
signature_id = v
elif k in ['version', 'rule_version', 'revision']:
version = v
elif k in ['status', 'al_status']:
status = v
# Convert CCCS YARA status to AL signature status
if status == "RELEASED":
status = "DEPLOYED"
elif status == "DEPRECATED":
status = "DISABLED"
# Fallback status
if status not in [
"DEPLOYED", "NOISY", "DISABLED", "STAGING", "TESTING",
"INVALID"
]:
status = default_status
# Fix imports and remove cuckoo
signature['imports'] = utils.detect_imports(signature)
if "cuckoo" not in signature['imports']:
sig = Signature(
dict(
classification=classification,
data=utils.rebuild_yara_rule(signature),
name=signature.get('rule_name'),
order=order,
revision=int(float(version)),
signature_id=signature_id
or signature.get('rule_name'),
source=source,
status=status,
type=self.importer_type,
))
upload_list.append(sig.as_primitives())
else:
self.log.warning(
f"Signature '{signature.get('rule_name')}' skipped because it uses cuckoo module."
)
order += 1
r = self.update_client.signature.add_update_many(
source, self.importer_type, upload_list)
self.log.info(
f"Imported {r['success']}/{order - 1} signatures from {source} into Assemblyline"
)
return r['success']
with open(file, 'r') as f:
try:
yararules = parser.parse_string(f.read())
rules += yararules
except Exception as e:
print(f'File: {file} error to parse {str(e)}')
parser.clear()
#print(len(rules), rules)
rebuilt_rules = str()
name_rule = []
count = 0
countb = 0
for rule in rules:
#verify if compile
if rule['rule_name'] in name_rule:
continue
name_rule.append(rule['rule_name'])
rverif = rebuild_yara_rule(rule)
try:
comp = yara.compile(source=rverif, error_on_warning=True)
rebuilt_rules += rverif
count += 1
except Exception as e:
print(f'Rule error to parse {str(rverif)}')
countb += 1
with open('merged.yara', 'w') as f:
print(rebuilt_rules, file=f)
print('Write ' + str(count) + " rules and " + str(countb) +
" rules removed for warning")
def create_rules_in_tc(rules, raw_rules, ruleset_name, id_lookup_table,
mappings):
# TODO: Add meta fields as tags (Sandbox Restricted)
groups = tcex.ti.group(
owner=OWNER) # Owner might need to be brought up as an arg
parameters = {'includes': ['additional', 'attributes', 'labels', 'tags']}
for rule in rules:
rule_name = rule['rule_name']
rule_content = utils.rebuild_yara_rule(rule)
lookup_context = id_lookup_table.setdefault(ruleset_name,
dict()).setdefault(
rule_name, dict())
versions = lookup_context.setdefault('versions', dict())
groups = lookup_context.setdefault('groups', list())
group_created = False
lookup_context['priority'] = next(
filter(lambda x: 'priority' in x, rule.get('metadata', [])), None)
lookup_context['restricted'] = next(
filter(lambda x: 'sandbox_restricted' in x,
rule.get('metadata', [])), None)
version = next(
filter(lambda x: 'version' in x, rule.get('metadata', [])),
None) or '1.0'
signature_name = '{} : {} V{}'.format(ruleset_name, rule_name, version)
kwargs = {
'group_type': 'Signature',
'name': signature_name,
'file_name': '{}.yara'.format(rule_name),
'file_type': 'YARA',
'file_text': rule_content,
'owner': OWNER
}
try:
existing_id = lookup_context['versions'][version]
except KeyError:
existing_id = False
signature_ti = tcex.ti.group(**kwargs)
r = signature_ti.create()
response_id = r.json()['data']['signature']['id']
for v in versions:
version_ti = tcex.ti.group(group_type='Signature',
unique_id=versions[v],
owner=OWNER)
signature_ti.add_association(target=version_ti)
versions[version] = response_id
lookup_context['latest'] = response_id
else:
kwargs['unique_id'] = existing_id
signature_ti = tcex.ti.group(**kwargs)
list(
map(signature_ti.delete_attribute,
[item['id'] for item in signature_ti.attributes()]))
list(
map(signature_ti.delete_tag,
[item['name'] for item in signature_ti.tags()]))
signature_ti.update()
for meta in rule.get('metadata', []):
if not meta:
continue
attr_type, attr_value = meta.popitem()
if attr_type in mappings.get('associations', dict()):
group_type = mappings['associations'][attr_type]
for group_name in attr_value.split(','):
filters = tcex.ti.filters()
filters.add_filter('name', '=', group_name)
a_groups = tcex.ti.group(group_type=group_type,
owner=OWNER)
returned_groups = list(
a_groups.many(filters=filters, params=parameters))
if not returned_groups:
group_ti = tcex.ti.group(group_type=group_type,
name=group_name,
owner=OWNER)
r = group_ti.create()
groups.append({
'name':
group_name,
'id':
r.json()['data'][group_type.lower()]['id'],
'type':
group_type
})
signature_ti.add_association(target=group_ti)
continue
for g in returned_groups:
group_ti = tcex.ti.group(group_type=group_type,
unique_id=g['id'],
owner=OWNER)
groups.append({
'name': group_name,
'id': g['id'],
'type': group_type
})
signature_ti.add_association(target=group_ti)
group_created = True
elif attr_type == 'tags' and mappings.get('tags'):
list(
map(
lambda tag: signature_ti.add_tag(name='τ {}'.format(tag
)),
attr_value.split(',')))
elif attr_type in mappings.get('boolean_tags', []) and attr_value:
signature_ti.add_tag(
name='ζ {}'.format(attr_type.replace('_', ' ').title()))
elif mappings.get('attributes', dict()).get(attr_type):
signature_ti.add_attribute(
attribute_type=mappings['attributes'].get(attr_type),
attribute_value=attr_value)
if not group_created and mappings.get('default_association'):
group_ti = tcex.ti.group(
group_type=mappings['default_association'],
name=rule_name,
owner=OWNER)
r = group_ti.create()
groups.append({
'name':
rule_name,
'id':
r.json()['data'][mappings['default_association'].lower()]
['id'],
'type':
mappings['default_association']
})
signature_ti.add_association(target=group_ti)
signature_ti.add_tag(name='α {}'.format(ruleset_name))
signature_ti.add_tag(name='β {}'.format(rule_name))
def yara_update(updater_type, update_config_path, update_output_path,
download_directory, externals, cur_logger) -> None:
"""
Using an update configuration file as an input, which contains a list of sources, download all the file(s).
"""
# noinspection PyBroadException
try:
# Load updater configuration
update_config = {}
if update_config_path and os.path.exists(update_config_path):
with open(update_config_path, 'r') as yml_fh:
update_config = yaml.safe_load(yml_fh)
else:
cur_logger.error(f"Update configuration file doesn't exist: {update_config_path}")
exit()
# Exit if no update sources given
if 'sources' not in update_config.keys() or not update_config['sources']:
cur_logger.error(f"Update configuration does not contain any source to update from")
exit()
# Initialise al_client
server = update_config['ui_server']
user = update_config['api_user']
api_key = update_config['api_key']
cur_logger.info(f"Connecting to Assemblyline API: {server}...")
al_client = get_client(server, apikey=(user, api_key), verify=False)
cur_logger.info(f"Connected!")
# Parse updater configuration
previous_update = update_config.get('previous_update', None)
previous_hash = json.loads(update_config.get('previous_hash', None) or "{}")
sources = {source['name']: source for source in update_config['sources']}
files_sha256 = {}
files_default_classification = {}
# Create working directory
updater_working_dir = os.path.join(tempfile.gettempdir(), 'updater_working_dir')
if os.path.exists(updater_working_dir):
shutil.rmtree(updater_working_dir)
os.makedirs(updater_working_dir)
# Go through each source and download file
for source_name, source in sources.items():
os.makedirs(os.path.join(updater_working_dir, source_name))
# 1. Download signatures
cur_logger.info(f"Downloading files from: {source['uri']}")
uri: str = source['uri']
if uri.endswith('.git'):
files = git_clone_repo(download_directory, source, cur_logger, previous_update=previous_update)
else:
files = [url_download(download_directory, source, cur_logger, previous_update=previous_update)]
processed_files = set()
# 2. Aggregate files
file_name = os.path.join(updater_working_dir, f"{source_name}.yar")
mode = "w"
for file in files:
# File has already been processed before, skip it to avoid duplication of rules
if file in processed_files:
continue
cur_logger.info(f"Processing file: {file}")
file_dirname = os.path.dirname(file)
processed_files.add(os.path.normpath(file))
with open(file, 'r') as f:
f_lines = f.readlines()
temp_lines = []
for i, f_line in enumerate(f_lines):
if f_line.startswith("include"):
lines, processed_files = replace_include(f_line, file_dirname, processed_files, cur_logger)
temp_lines.extend(lines)
else:
temp_lines.append(f_line)
# guess the type of files that we have in the current file
guessed_category = guess_category(file)
parser = Plyara()
signatures = parser.parse_string("\n".join(temp_lines))
# Ignore "cuckoo" rules
if "cuckoo" in parser.imports:
parser.imports.remove("cuckoo")
# Guess category
if guessed_category:
for s in signatures:
if 'metadata' not in s:
s['metadata'] = []
# Do not override category with guessed category if it already exists
for meta in s['metadata']:
if 'category' in meta:
continue
s['metadata'].append({'category': guessed_category})
s['metadata'].append({guessed_category: s.get('rule_name')})
# Save all rules from source into single file
with open(file_name, mode) as f:
for s in signatures:
# Fix imports and remove cuckoo
s['imports'] = utils.detect_imports(s)
if "cuckoo" not in s['imports']:
f.write(utils.rebuild_yara_rule(s))
if mode == "w":
mode = "a"
# Check if the file is the same as the last run
if os.path.exists(file_name):
cache_name = os.path.basename(file_name)
sha256 = get_sha256_for_file(file_name)
if sha256 != previous_hash.get(cache_name, None):
files_sha256[cache_name] = sha256
files_default_classification[cache_name] = source.get('default_classification',
classification.UNRESTRICTED)
else:
cur_logger.info(f'File {cache_name} has not changed since last run. Skipping it...')
if files_sha256:
cur_logger.info(f"Found new {updater_type.upper()} rules files to process!")
yara_importer = YaraImporter(updater_type, al_client, logger=cur_logger)
# Validating and importing the different signatures
for base_file in files_sha256:
cur_logger.info(f"Validating output file: {base_file}")
cur_file = os.path.join(updater_working_dir, base_file)
source_name = os.path.splitext(os.path.basename(cur_file))[0]
default_classification = files_default_classification.get(base_file, classification.UNRESTRICTED)
try:
_compile_rules(cur_file, externals, cur_logger)
yara_importer.import_file(cur_file, source_name, default_classification=default_classification)
except Exception as e:
raise e
else:
cur_logger.info(f'No new {updater_type.upper()} rules files to process...')
# Check if new signatures have been added
if al_client.signature.update_available(since=previous_update or '', sig_type=updater_type)['update_available']:
cur_logger.info("An update is available for download from the datastore")
if not os.path.exists(update_output_path):
os.makedirs(update_output_path)
temp_zip_file = os.path.join(update_output_path, 'temp.zip')
al_client.signature.download(output=temp_zip_file,
query=f"type:{updater_type} AND (status:NOISY OR status:DEPLOYED)")
if os.path.exists(temp_zip_file):
with ZipFile(temp_zip_file, 'r') as zip_f:
zip_f.extractall(update_output_path)
os.remove(temp_zip_file)
# Create the response yaml
with open(os.path.join(update_output_path, 'response.yaml'), 'w') as yml_fh:
yaml.safe_dump(dict(hash=json.dumps(files_sha256)), yml_fh)
cur_logger.info(f"New ruleset successfully downloaded and ready to use")
cur_logger.info(f"{updater_type.upper()} updater completed successfully")
except Exception:
cur_logger.exception("Updater ended with an exception!")
