def rebuild_yara_rule(rule): """Rebuild rule: DEPRECATED.""" from plyara.utils import rebuild_yara_rule import warnings warnings.warn( 'Static method rebuild_yara_rule() is deprecated: use plyara.utils', DeprecationWarning) return rebuild_yara_rule(rule)
def test_rebuild_yara_rule(self): with data_dir.joinpath('rebuild_ruleset.yar').open( 'r', encoding='utf-8') as fh: inputString = fh.read() result = Plyara().parse_string(inputString) rebuilt_rules = str() for rule in result: rebuilt_rules += rebuild_yara_rule(rule) self.assertEqual(inputString, rebuilt_rules)
def format_rule(self): raw_rule = {} raw_rule['rule_name'] = self.name raw_rule['tags'] = self.tags raw_rule['imports'] = self.imports raw_rule['metadata'] = self.metadata raw_rule['strings'] = self.strings raw_rule['condition_terms'] = self.condition raw_rule['scopes'] = self.scopes formatted_rule = utils.rebuild_yara_rule(raw_rule) return formatted_rule
def exclude_rules(rules_file_name): with open(rules_file_name, "r") as rules_json_file: rules_json = json.loads(rules_json_file.read()) for repository in rules_json["repositories"]: excluded_rules = repository["excludeRules"] for file_name in excluded_rules.keys(): rule_names = excluded_rules[file_name] with open(file_name, 'r') as file: yara_rules = parser.parse_string(file.read()) filtered_rules = [ rule for rule in yara_rules if rule['rule_name'] not in rule_names ] parser.clear() with open(file_name, 'w') as file: for rule in filtered_rules: file.write(rebuild_yara_rule(rule))
def test_rebuild_yara_rule_metadata(self): test_rule = """ rule check_meta { meta: string_value = "TEST STRING" string_value = "DIFFERENT TEST STRING" bool_value = true bool_value = false digit_value = 5 digit_value = 10 condition: true } """ parsed = Plyara().parse_string(test_rule) for rule in parsed: unparsed = rebuild_yara_rule(rule) self.assertIn('string_value = "TEST STRING"', unparsed) self.assertIn('string_value = "DIFFERENT TEST STRING"', unparsed) self.assertIn('bool_value = true', unparsed) self.assertIn('bool_value = false', unparsed) self.assertIn('digit_value = 5', unparsed) self.assertIn('digit_value = 10', unparsed)
def measure(rule, cycles, show_score=True, c_duration=0): """ Measure rule performance :param rule: the YARA rule to test :param cycles: number of iterations over the sample set :param show_score: show the performance score :param c_duration: duration of the calibration run :return duration: duration in seconds :return count: count of samples in the given samples folders """ yara_rule_string = plutils.rebuild_yara_rule(rule) y = yara.compile(source=yara_rule_string) Log.info("Scanning sample set with rule: %s" % rule['rule_name']) start = time.time() count = 0 for s in SAMPLE_SET: if not os.path.exists(s): Log.error("[E] Error: sample directory '%s' doesn't exist" % s) else: for (dirpath, dirnames, filenames) in os.walk(s): for filename in filenames: count += 1 for _ in range(cycles): sample_file = os.path.join(dirpath, filename) with open(sample_file, 'rb') as fh: fdata = fh.read() matches = y.match(data=fdata) end = time.time() duration = end - start if show_score: # If a calibration duration has been evaluated if c_duration > 0: print("Duration: %.2f (%0.2f)" % (duration, (duration - c_duration))) else: print("Duration: %.2f" % duration) return duration, count
# first test all at once (this might easily fail on multiple files with duplicate rulenames) measure_rule = CALIBRATION_RULES + rules_all rule_name = "All " + str( len(rules_list)) + " rules from all input files" measure(measure_rule, cycles, progress, show_score=True, c_duration=crule_duration, rule_name=rule_name, alert_diff=alert_diff) # Scan files for r in rules_list: yara_rule_string = plutils.rebuild_yara_rule(r) rule_name = r['rule_name'] if len(yara_rule_string) > 20000: log_warning_rich( "Big rule: " + rule_name + " has " + str(len(yara_rule_string)) + " bytes", yara_rule_string) measure_rule = CALIBRATION_RULES + yara_rule_string measure(measure_rule, cycles, progress, show_score=True, c_duration=crule_duration, rule_name=rule_name, alert_diff=alert_diff,
def _save_signatures(self, signatures, source, default_status=DEFAULT_STATUS, default_classification=None): if len(signatures) == 0: self.log.info(f"There are no signatures for {source}, skipping...") return False order = 1 upload_list = [] for signature in signatures: classification = default_classification or self.classification.UNRESTRICTED signature_id = None version = 1 status = default_status for meta in signature.get('metadata', {}): for k, v in meta.items(): if k in ["classification", "sharing"]: classification = v elif k in ['id', 'rule_id', 'signature_id']: signature_id = v elif k in ['version', 'rule_version', 'revision']: version = v elif k in ['status', 'al_status']: status = v # Convert CCCS YARA status to AL signature status if status == "RELEASED": status = "DEPLOYED" elif status == "DEPRECATED": status = "DISABLED" # Fallback status if status not in [ "DEPLOYED", "NOISY", "DISABLED", "STAGING", "TESTING", "INVALID" ]: status = default_status # Fix imports and remove cuckoo signature['imports'] = utils.detect_imports(signature) if "cuckoo" not in signature['imports']: sig = Signature( dict( classification=classification, data=utils.rebuild_yara_rule(signature), name=signature.get('rule_name'), order=order, revision=int(float(version)), signature_id=signature_id or signature.get('rule_name'), source=source, status=status, type=self.importer_type, )) upload_list.append(sig.as_primitives()) else: self.log.warning( f"Signature '{signature.get('rule_name')}' skipped because it uses cuckoo module." ) order += 1 r = self.update_client.signature.add_update_many( source, self.importer_type, upload_list) self.log.info( f"Imported {r['success']}/{order - 1} signatures from {source} into Assemblyline" ) return r['success']
with open(file, 'r') as f: try: yararules = parser.parse_string(f.read()) rules += yararules except Exception as e: print(f'File: {file} error to parse {str(e)}') parser.clear() #print(len(rules), rules) rebuilt_rules = str() name_rule = [] count = 0 countb = 0 for rule in rules: #verify if compile if rule['rule_name'] in name_rule: continue name_rule.append(rule['rule_name']) rverif = rebuild_yara_rule(rule) try: comp = yara.compile(source=rverif, error_on_warning=True) rebuilt_rules += rverif count += 1 except Exception as e: print(f'Rule error to parse {str(rverif)}') countb += 1 with open('merged.yara', 'w') as f: print(rebuilt_rules, file=f) print('Write ' + str(count) + " rules and " + str(countb) + " rules removed for warning")
def create_rules_in_tc(rules, raw_rules, ruleset_name, id_lookup_table, mappings): # TODO: Add meta fields as tags (Sandbox Restricted) groups = tcex.ti.group( owner=OWNER) # Owner might need to be brought up as an arg parameters = {'includes': ['additional', 'attributes', 'labels', 'tags']} for rule in rules: rule_name = rule['rule_name'] rule_content = utils.rebuild_yara_rule(rule) lookup_context = id_lookup_table.setdefault(ruleset_name, dict()).setdefault( rule_name, dict()) versions = lookup_context.setdefault('versions', dict()) groups = lookup_context.setdefault('groups', list()) group_created = False lookup_context['priority'] = next( filter(lambda x: 'priority' in x, rule.get('metadata', [])), None) lookup_context['restricted'] = next( filter(lambda x: 'sandbox_restricted' in x, rule.get('metadata', [])), None) version = next( filter(lambda x: 'version' in x, rule.get('metadata', [])), None) or '1.0' signature_name = '{} : {} V{}'.format(ruleset_name, rule_name, version) kwargs = { 'group_type': 'Signature', 'name': signature_name, 'file_name': '{}.yara'.format(rule_name), 'file_type': 'YARA', 'file_text': rule_content, 'owner': OWNER } try: existing_id = lookup_context['versions'][version] except KeyError: existing_id = False signature_ti = tcex.ti.group(**kwargs) r = signature_ti.create() response_id = r.json()['data']['signature']['id'] for v in versions: version_ti = tcex.ti.group(group_type='Signature', unique_id=versions[v], owner=OWNER) signature_ti.add_association(target=version_ti) versions[version] = response_id lookup_context['latest'] = response_id else: kwargs['unique_id'] = existing_id signature_ti = tcex.ti.group(**kwargs) list( map(signature_ti.delete_attribute, [item['id'] for item in signature_ti.attributes()])) list( map(signature_ti.delete_tag, [item['name'] for item in signature_ti.tags()])) signature_ti.update() for meta in rule.get('metadata', []): if not meta: continue attr_type, attr_value = meta.popitem() if attr_type in mappings.get('associations', dict()): group_type = mappings['associations'][attr_type] for group_name in attr_value.split(','): filters = tcex.ti.filters() filters.add_filter('name', '=', group_name) a_groups = tcex.ti.group(group_type=group_type, owner=OWNER) returned_groups = list( a_groups.many(filters=filters, params=parameters)) if not returned_groups: group_ti = tcex.ti.group(group_type=group_type, name=group_name, owner=OWNER) r = group_ti.create() groups.append({ 'name': group_name, 'id': r.json()['data'][group_type.lower()]['id'], 'type': group_type }) signature_ti.add_association(target=group_ti) continue for g in returned_groups: group_ti = tcex.ti.group(group_type=group_type, unique_id=g['id'], owner=OWNER) groups.append({ 'name': group_name, 'id': g['id'], 'type': group_type }) signature_ti.add_association(target=group_ti) group_created = True elif attr_type == 'tags' and mappings.get('tags'): list( map( lambda tag: signature_ti.add_tag(name='τ {}'.format(tag )), attr_value.split(','))) elif attr_type in mappings.get('boolean_tags', []) and attr_value: signature_ti.add_tag( name='ζ {}'.format(attr_type.replace('_', ' ').title())) elif mappings.get('attributes', dict()).get(attr_type): signature_ti.add_attribute( attribute_type=mappings['attributes'].get(attr_type), attribute_value=attr_value) if not group_created and mappings.get('default_association'): group_ti = tcex.ti.group( group_type=mappings['default_association'], name=rule_name, owner=OWNER) r = group_ti.create() groups.append({ 'name': rule_name, 'id': r.json()['data'][mappings['default_association'].lower()] ['id'], 'type': mappings['default_association'] }) signature_ti.add_association(target=group_ti) signature_ti.add_tag(name='α {}'.format(ruleset_name)) signature_ti.add_tag(name='β {}'.format(rule_name))
def yara_update(updater_type, update_config_path, update_output_path, download_directory, externals, cur_logger) -> None: """ Using an update configuration file as an input, which contains a list of sources, download all the file(s). """ # noinspection PyBroadException try: # Load updater configuration update_config = {} if update_config_path and os.path.exists(update_config_path): with open(update_config_path, 'r') as yml_fh: update_config = yaml.safe_load(yml_fh) else: cur_logger.error(f"Update configuration file doesn't exist: {update_config_path}") exit() # Exit if no update sources given if 'sources' not in update_config.keys() or not update_config['sources']: cur_logger.error(f"Update configuration does not contain any source to update from") exit() # Initialise al_client server = update_config['ui_server'] user = update_config['api_user'] api_key = update_config['api_key'] cur_logger.info(f"Connecting to Assemblyline API: {server}...") al_client = get_client(server, apikey=(user, api_key), verify=False) cur_logger.info(f"Connected!") # Parse updater configuration previous_update = update_config.get('previous_update', None) previous_hash = json.loads(update_config.get('previous_hash', None) or "{}") sources = {source['name']: source for source in update_config['sources']} files_sha256 = {} files_default_classification = {} # Create working directory updater_working_dir = os.path.join(tempfile.gettempdir(), 'updater_working_dir') if os.path.exists(updater_working_dir): shutil.rmtree(updater_working_dir) os.makedirs(updater_working_dir) # Go through each source and download file for source_name, source in sources.items(): os.makedirs(os.path.join(updater_working_dir, source_name)) # 1. Download signatures cur_logger.info(f"Downloading files from: {source['uri']}") uri: str = source['uri'] if uri.endswith('.git'): files = git_clone_repo(download_directory, source, cur_logger, previous_update=previous_update) else: files = [url_download(download_directory, source, cur_logger, previous_update=previous_update)] processed_files = set() # 2. Aggregate files file_name = os.path.join(updater_working_dir, f"{source_name}.yar") mode = "w" for file in files: # File has already been processed before, skip it to avoid duplication of rules if file in processed_files: continue cur_logger.info(f"Processing file: {file}") file_dirname = os.path.dirname(file) processed_files.add(os.path.normpath(file)) with open(file, 'r') as f: f_lines = f.readlines() temp_lines = [] for i, f_line in enumerate(f_lines): if f_line.startswith("include"): lines, processed_files = replace_include(f_line, file_dirname, processed_files, cur_logger) temp_lines.extend(lines) else: temp_lines.append(f_line) # guess the type of files that we have in the current file guessed_category = guess_category(file) parser = Plyara() signatures = parser.parse_string("\n".join(temp_lines)) # Ignore "cuckoo" rules if "cuckoo" in parser.imports: parser.imports.remove("cuckoo") # Guess category if guessed_category: for s in signatures: if 'metadata' not in s: s['metadata'] = [] # Do not override category with guessed category if it already exists for meta in s['metadata']: if 'category' in meta: continue s['metadata'].append({'category': guessed_category}) s['metadata'].append({guessed_category: s.get('rule_name')}) # Save all rules from source into single file with open(file_name, mode) as f: for s in signatures: # Fix imports and remove cuckoo s['imports'] = utils.detect_imports(s) if "cuckoo" not in s['imports']: f.write(utils.rebuild_yara_rule(s)) if mode == "w": mode = "a" # Check if the file is the same as the last run if os.path.exists(file_name): cache_name = os.path.basename(file_name) sha256 = get_sha256_for_file(file_name) if sha256 != previous_hash.get(cache_name, None): files_sha256[cache_name] = sha256 files_default_classification[cache_name] = source.get('default_classification', classification.UNRESTRICTED) else: cur_logger.info(f'File {cache_name} has not changed since last run. Skipping it...') if files_sha256: cur_logger.info(f"Found new {updater_type.upper()} rules files to process!") yara_importer = YaraImporter(updater_type, al_client, logger=cur_logger) # Validating and importing the different signatures for base_file in files_sha256: cur_logger.info(f"Validating output file: {base_file}") cur_file = os.path.join(updater_working_dir, base_file) source_name = os.path.splitext(os.path.basename(cur_file))[0] default_classification = files_default_classification.get(base_file, classification.UNRESTRICTED) try: _compile_rules(cur_file, externals, cur_logger) yara_importer.import_file(cur_file, source_name, default_classification=default_classification) except Exception as e: raise e else: cur_logger.info(f'No new {updater_type.upper()} rules files to process...') # Check if new signatures have been added if al_client.signature.update_available(since=previous_update or '', sig_type=updater_type)['update_available']: cur_logger.info("An update is available for download from the datastore") if not os.path.exists(update_output_path): os.makedirs(update_output_path) temp_zip_file = os.path.join(update_output_path, 'temp.zip') al_client.signature.download(output=temp_zip_file, query=f"type:{updater_type} AND (status:NOISY OR status:DEPLOYED)") if os.path.exists(temp_zip_file): with ZipFile(temp_zip_file, 'r') as zip_f: zip_f.extractall(update_output_path) os.remove(temp_zip_file) # Create the response yaml with open(os.path.join(update_output_path, 'response.yaml'), 'w') as yml_fh: yaml.safe_dump(dict(hash=json.dumps(files_sha256)), yml_fh) cur_logger.info(f"New ruleset successfully downloaded and ready to use") cur_logger.info(f"{updater_type.upper()} updater completed successfully") except Exception: cur_logger.exception("Updater ended with an exception!")