def test_yara_rule_rest(rule_id):
"""Synchronously test yara rule associated with rule_id against all files attached to it
Return: results dictionary"""
yara_rule_entity = yara_rule.Yara_rule.query.get(rule_id)
if not yara_rule_entity:
abort(500)
try:
files_to_test = []
is_neg_test = str(request.args.get("negative", "0"))
if is_neg_test == "1":
neg_test_dir = Cfg_settings.get_setting(
"NEGATIVE_TESTING_FILE_DIRECTORY")
for root_path, dirs, dir_files in os.walk(neg_test_dir):
for f_name in dir_files:
files_to_test.append(os.path.join(root_path, f_name))
else:
for f in yara_rule_entity.files:
file_store_path = Cfg_settings.get_setting("FILE_STORE_PATH")
if not file_store_path:
raise Exception(
'FILE_STORE_PATH configuration setting not set.')
files_to_test.append(
os.path.join(
file_store_path,
str(f.entity_type)
if f.entity_type is not None else "",
str(f.entity_id) if f.entity_id is not None else "",
str(f.filename)))
return jsonify(
test_yara_rule(yara_rule_entity, files_to_test,
current_user.id)), 200
except Exception as e:
return e.message, 500
def does_rule_compile(yara_dict):
rule = Yara_rule.to_yara_rule_string(yara_dict, include_imports=True)
rule_temp_path = "/tmp/%s.yar" % (str(uuid.uuid4()).replace("-", "")[0:8])
with open(rule_temp_path, "w") as f:
f.write(rule)
yara_command = Cfg_settings.get_setting("SIGNATURE_TESTING_COMMAND")
command = yara_command.replace("RULE", rule_temp_path).replace(
"FILE_PATH", rule_temp_path)
proc = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
proc.wait()
stdout, stderr = proc.communicate()
return_code = proc.returncode
return (return_code == 0, return_code, stdout, stderr)
def test_yara_rule_task(rule_id, user, is_neg_test):
print("in test_yara_rule_task")
files_to_test = []
yara_rule_entity = yara_rule.Yara_rule.query.get(rule_id)
if not yara_rule_entity:
abort(500)
print("yara rule entity %s" % (yara_rule_entity))
neg_test_dir = Cfg_settings.get_setting("NEGATIVE_TESTING_FILE_DIRECTORY")
if not os.path.exists(neg_test_dir):
abort(500)
for root_path, dirs, dir_files in os.walk(neg_test_dir):
for f_name in dir_files:
files_to_test.append(os.path.join(root_path, f_name))
print("files to test %s" % (str(files_to_test)))
return test_yara_rule(yara_rule_entity, files_to_test, user, True,
is_neg_test)
def test_yara_rule_rest(rule_id):
"""Synchronously test yara rule associated with rule_id against all files attached to it
Return: results dictionary"""
yara_rule_entity = yara_rule.Yara_rule.query.get(rule_id)
if not yara_rule_entity:
abort(500)
try:
files_to_test = []
is_neg_test = str(request.args.get("negative", "0"))
is_neg_test = True if is_neg_test == "1" else False
if not is_neg_test:
for f in yara_rule_entity.files:
file_store_path = Cfg_settings.get_setting("FILE_STORE_PATH")
if not file_store_path:
raise Exception(
'FILE_STORE_PATH configuration setting not set.')
files_to_test.append(
os.path.join(
file_store_path,
str(f.entity_type)
if f.entity_type is not None else "",
str(f.entity_id) if f.entity_id is not None else "",
str(f.filename)))
if not is_neg_test:
return jsonify(
test_yara_rule(yara_rule_entity, files_to_test,
current_user.id, False, is_neg_test)), 200
else:
test_yara_rule_task.delay(yara_rule_entity.id, current_user.id,
is_neg_test)
return jsonify({
"status": "ok",
"message": "Background job created"
}), 200
except Exception as e:
return e.message, 500
def test_yara_rule(yara_rule_entity, files_to_test, user, is_async=False):
old_avg = db.session.query(func.avg(Yara_testing_history.avg_millis_per_file).label('average')) \
.filter(Yara_testing_history.yara_rule_id == yara_rule_entity.id) \
.scalar()
start_time = time.time()
start_time_str = datetime.datetime.fromtimestamp(start_time).strftime(
'%Y-%m-%d %H:%M:%S')
rule = get_yara_rule(yara_rule_entity)
total_file_count, count_of_files_matched, tests_terminated, total_file_time, errors_encountered = 0, 0, 0, 0, 0
error_msgs = []
threshold = float(
Cfg_settings.get_private_setting(
"MAX_MILLIS_PER_FILE_THRESHOLD")) or 3.0
processes = []
manager_dicts = []
for file_path in files_to_test:
total_file_count += 1
if not os.path.exists(file_path):
errors_encountered += 1
error_msgs.append(
ntpath.basename(file_path) + " not in File Store Path.")
continue
if not is_async:
manager = multiprocessing.Manager()
manager_dict = manager.dict()
if old_avg:
p = multiprocessing.Process(target=perform_rule_match,
args=(rule, file_path,
manager_dict))
processes.append(p)
p.start()
else:
perform_rule_match(rule, file_path, manager_dict)
manager_dicts.append(manager_dict)
else:
manager_dicts.append(perform_rule_match(rule, file_path, dict()))
if old_avg and not is_async:
time.sleep((old_avg * threshold) / 1000.0)
for p in processes:
p.join()
if p.is_alive():
tests_terminated += 1
p.terminate()
p.join()
for managers in manager_dicts:
if managers['duration']:
total_file_time += managers['duration']
if managers['match']:
count_of_files_matched += 1
end_time = time.time()
end_time_str = datetime.datetime.fromtimestamp(end_time).strftime(
'%Y-%m-%d %H:%M:%S')
if total_file_count > 0:
db.session.add(
Yara_testing_history(
yara_rule_id=yara_rule_entity.id,
revision=yara_rule_entity.revision,
start_time=start_time_str,
end_time=end_time_str,
files_tested=total_file_count,
files_matched=count_of_files_matched,
avg_millis_per_file=((total_file_time / total_file_count) *
1000),
user_id=user))
db.session.commit()
return dict(duration=(total_file_time * 1000),
files_tested=total_file_count,
files_matched=count_of_files_matched,
tests_terminated=tests_terminated,
errors_encountered=errors_encountered,
error_msgs=error_msgs)
def update_yara_rule(id):
"""Update yara_rule artifact
From Data: name (str), state(str), category (str), condition (str), strings (str)
Return: yara_rule artifact dictionary"""
do_not_bump_revision = request.json.get("do_not_bump_revision", False)
entity = yara_rule.Yara_rule.query.get(id)
if not entity:
abort(404)
if not current_user.admin and entity.owner_user_id != current_user.id:
abort(403)
release_state = cfg_states.Cfg_states.query.filter(cfg_states.Cfg_states.is_release_state > 0).first()
draft_state = cfg_states.Cfg_states.query.filter(cfg_states.Cfg_states.is_staging_state > 0).first()
old_state = entity.state
try:
rule_state = request.json.get("state", None).get("state", None)
except:
rule_state = request.json.get("state", None)
unique_rule_name_enforcement = Cfg_settings.get_setting("ENFORCE_UNIQUE_YARA_RULE_NAMES")
if unique_rule_name_enforcement and distutils.util.strtobool(unique_rule_name_enforcement):
if any([True for rule in
db.session.query(yara_rule.Yara_rule).filter(yara_rule.Yara_rule.name == request.json['name']).all() if
not rule.id == id]):
raise Exception("You cannot save two rules with the same name.")
compile_on_save = Cfg_settings.get_setting("COMPILE_YARA_RULE_ON_SAVE")
if compile_on_save and distutils.util.strtobool(compile_on_save) and (
rule_state == release_state.state or rule_state == draft_state.state):
test_result, return_code, stdout, stderr = test_yara_rule.does_rule_compile(request.json)
if not test_result:
raise Exception(
"State submitted is " + str(
rule_state) + " and the rule could not be saved because it does not compile.\n\nerror_code=" + str(
return_code) + "\n\n" + stderr)
if not release_state or not draft_state:
raise Exception("You must set a release, draft, and retirement state before modifying signatures")
if not do_not_bump_revision:
db.session.add(yara_rule.Yara_rule_history(date_created=datetime.datetime.now(), revision=entity.revision,
rule_json=json.dumps(entity.to_revision_dict()),
user_id=current_user.id,
yara_rule_id=entity.id,
state=entity.state))
if not entity.revision:
entity.revision = 1
temp_sig_id = entity.eventid
get_new_sig_id = False
if request.json['category'] and 'category' in request.json['category'] and not entity.category == \
request.json['category']['category']:
get_new_sig_id = True
if not request.json['category']['current']:
temp_sig_id = request.json['category']['range_min']
else:
temp_sig_id = request.json['category']['current'] + 1
if temp_sig_id > request.json['category']['range_max']:
abort(400)
entity = yara_rule.Yara_rule(
state=request.json['state']['state'] if request.json['state'] and 'state' in request.json['state'] else
request.json['state'],
name=request.json['name'],
description=request.json.get("description", None),
references=request.json.get("references", None),
category=request.json['category']['category'] if request.json['category'] and 'category' in request
.json['category'] else request.json['category'],
condition=yara_rule.Yara_rule.make_yara_sane(request.json["condition"], "condition:"),
strings=yara_rule.Yara_rule.make_yara_sane(request.json["strings"], "strings:"),
eventid=temp_sig_id,
id=id,
created_user_id=entity.created_user_id,
creation_date=entity.creation_date,
modified_user_id=current_user.id,
last_revision_date=datetime.datetime.now(),
owner_user_id=request.json['owner_user']['id'] if request.json.get("owner_user", None) and request
.json["owner_user"].get("id", None) else None,
revision=entity.revision if do_not_bump_revision else entity.revision + 1,
imports=yara_rule.Yara_rule.get_imports_from_string(request.json.get("imports", None)),
active=request.json.get("active", entity.active)
)
mitre_techniques = Cfg_settings.get_setting("MITRE_TECHNIQUES").split(",")
entity.mitre_techniques = request.json.get("mitre_techniques", [])
matches = [technique for technique in entity.mitre_techniques if technique not in mitre_techniques]
if matches:
raise (Exception(
"The following techniques were not found in the configuration: %s. Check 'MITRE_TECHNIQUES' on the settings page" % (
matches)))
mitre_tactics = Cfg_settings.get_setting("MITRE_TACTICS").split(",")
entity.mitre_tactics = request.json.get("mitre_tactics", [])
matches = [tactic for tactic in entity.mitre_tactics if tactic not in mitre_tactics]
if matches:
raise (Exception(
"The following tactics were not found in the configuration: %s. Check 'MITRE_TACTICS' on the settings page" % (
matches)))
if old_state == release_state.state and entity.state == release_state.state and not do_not_bump_revision:
entity.state = draft_state.state
db.session.merge(entity)
db.session.commit()
dirty = False
for name, value_dict in request.json.get("metadata_values", {}).iteritems():
if not name or not value_dict:
continue
m = db.session.query(MetadataMapping).join(Metadata, Metadata.id == MetadataMapping.metadata_id).filter(
Metadata.key == name).filter(Metadata.artifact_type == ENTITY_MAPPING["SIGNATURE"]).filter(
MetadataMapping.artifact_id == entity.id).first()
if m:
m.value = value_dict["value"]
db.session.add(m)
dirty = True
else:
m = db.session.query(Metadata).filter(Metadata.key == name).filter(
Metadata.artifact_type == ENTITY_MAPPING["SIGNATURE"]).first()
db.session.add(MetadataMapping(value=value_dict["value"], metadata_id=m.id, artifact_id=entity.id,
created_user_id=current_user.id))
dirty = True
if dirty:
db.session.commit()
# THIS IS UGLY. FIGURE OUT WHY MERGE ISN'T WORKING
entity = yara_rule.Yara_rule.query.get(entity.id)
if get_new_sig_id:
update_cfg_category_range_mapping_current(request.json['category']['id'], temp_sig_id)
delete_tags_mapping(entity.__tablename__, entity.id)
create_tags_mapping(entity.__tablename__, entity.id, request.json['tags'])
return jsonify(entity.to_dict()), 200
def create_yara_rule():
"""Create yara_rule artifact
From Data: name (str), state(str), category (str), condition (str), strings (str)
Return: yara_rule artifact dictionary"""
new_sig_id = 0
release_state = cfg_states.Cfg_states.query.filter(cfg_states.Cfg_states.is_release_state > 0).first()
draft_state = cfg_states.Cfg_states.query.filter(cfg_states.Cfg_states.is_staging_state > 0).first()
if not release_state or not draft_state:
raise Exception("You must set a release, draft, and retirement state before modifying signatures")
try:
rule_state = request.json.get("state", None).get("state", None)
except:
rule_state = request.json.get("state", None)
unique_rule_name_enforcement = Cfg_settings.get_setting("ENFORCE_UNIQUE_YARA_RULE_NAMES")
if unique_rule_name_enforcement and distutils.util.strtobool(unique_rule_name_enforcement):
if db.session.query(yara_rule.Yara_rule).filter(yara_rule.Yara_rule.name == request.json['name']).first():
raise Exception("You cannot save two rules with the same name.")
compile_on_save = Cfg_settings.get_setting("COMPILE_YARA_RULE_ON_SAVE")
if compile_on_save and distutils.util.strtobool(compile_on_save) and (
rule_state == release_state.state or rule_state == draft_state.state):
test_result, return_code, stdout, stderr = test_yara_rule.does_rule_compile(request.json)
if not test_result:
raise Exception(
"State submitted is " + str(
rule_state) + " and the rule could not be saved because it does not compile.\n\nerror_code=" + str(
return_code) + "\n\n" + stderr)
if request.json['category'] and 'category' in request.json['category']:
new_sig_id = request.json['category']['current'] + 1
entity = yara_rule.Yara_rule(
state=request.json['state']['state'] if 'state' in request.json['state'] else None,
name=request.json['name'],
description=request.json.get("description", None),
references=request.json.get("references", None),
category=request.json['category']['category'] if 'category' in request.json['category'] else None,
condition=yara_rule.Yara_rule.make_yara_sane(request.json['condition'], "condition:"),
strings=yara_rule.Yara_rule.make_yara_sane(request.json['strings'], "strings:"),
eventid=new_sig_id,
created_user_id=current_user.id,
modified_user_id=current_user.id,
owner_user_id=current_user.id,
imports=yara_rule.Yara_rule.get_imports_from_string(request.json.get("imports", None)),
active=request.json.get("active", True)
)
mitre_techniques = Cfg_settings.get_setting("MITRE_TECHNIQUES").split(",")
entity.mitre_techniques = request.json.get("mitre_techniques", [])
matches = [technique for technique in entity.mitre_techniques if technique not in mitre_techniques]
if matches:
raise (Exception(
"The following techniques were not found in the configuration: %s. Check 'MITRE_TECHNIQUES' on the settings page" % (
matches)))
mitre_tactics = Cfg_settings.get_setting("MITRE_TACTICS").split(",")
entity.mitre_tactics = request.json.get("mitre_tactics", [])
matches = [tactic for tactic in entity.mitre_tactics if tactic not in mitre_tactics]
if matches:
raise (Exception(
"The following tactics were not found in the configuration: %s. Check 'MITRE_TACTICS' on the settings page" % (
matches)))
if entity.state == release_state:
entity.state = draft_state.state
db.session.add(entity)
db.session.commit()
entity.tags = create_tags_mapping(entity.__tablename__, entity.id, request.json['tags'])
if request.json.get('new_comment', None):
create_comment(request.json['new_comment'],
ENTITY_MAPPING["SIGNATURE"],
entity.id,
current_user.id)
if new_sig_id > 0:
update_cfg_category_range_mapping_current(request.json['category']['id'], new_sig_id)
dirty = False
for name, value_dict in request.json.get("metadata_values", {}).iteritems():
if not name or not value_dict:
continue
m = db.session.query(MetadataMapping).join(Metadata, Metadata.id == MetadataMapping.metadata_id).filter(
Metadata.key == name).filter(Metadata.artifact_type == ENTITY_MAPPING["SIGNATURE"]).filter(
MetadataMapping.artifact_id == entity.id).first()
if m:
m.value = value_dict["value"]
db.session.add(m)
dirty = True
else:
m = db.session.query(Metadata).filter(Metadata.key == name).filter(
Metadata.artifact_type == ENTITY_MAPPING["SIGNATURE"]).first()
db.session.add(MetadataMapping(value=value_dict["value"], metadata_id=m.id, artifact_id=entity.id,
created_user_id=current_user.id))
dirty = True
if dirty:
db.session.commit()
return jsonify(entity.to_dict()), 201
def test_yara_rule(yara_rule_entity,
files_to_test,
user,
is_async=False,
is_neg_test=False):
if type(yara_rule_entity) is int:
yara_rule_entity = yara_rule.Yara_rule.query.get(yara_rule_entity)
old_avg = db.session.query(func.avg(Yara_testing_history.avg_millis_per_file).label('average')) \
.filter(Yara_testing_history.yara_rule_id == yara_rule_entity.id) \
.scalar()
start_time = time.time()
start_time_str = datetime.datetime.fromtimestamp(start_time).strftime(
'%Y-%m-%d %H:%M:%S')
rule = get_yara_rule(yara_rule_entity)
total_file_count, count_of_files_matched, tests_terminated, total_file_time, errors_encountered = 0, 0, 0, 0, 0
error_msgs = []
files_matches = []
threshold = float(
Cfg_settings.get_setting("MAX_MILLIS_PER_FILE_THRESHOLD") or 3.0)
yara_command = Cfg_settings.get_setting("SIGNATURE_TESTING_COMMAND")
yara_test_regex = Cfg_settings.get_setting(
"SIGNATURE_TESTING_COMMAND_SUCCESS_REGEX")
yrh = Yara_testing_history(
yara_rule_id=yara_rule_entity.id,
revision=yara_rule_entity.revision,
start_time=start_time_str,
end_time=None,
files_tested=0,
files_matched=0,
avg_millis_per_file=0,
total_files=len(files_to_test),
status=Yara_testing_history.STATUS_RUNNING,
test_type=Yara_testing_history.TEST_TYPE_NEGATIVE
if is_neg_test else Yara_testing_history.TEST_TYPE_POSITIVE,
user_id=user)
db.session.add(yrh)
db.session.commit()
processes = []
manager_dicts = []
for file_path in files_to_test:
total_file_count += 1
if (total_file_count % 10) == 0:
yrh.files_tested = total_file_count
db.session.add(yrh)
db.session.commit()
if not os.path.exists(file_path):
errors_encountered += 1
error_msgs.append(
ntpath.basename(file_path) + " not in File Store Path.")
continue
if not is_async:
manager = multiprocessing.Manager()
manager_dict = manager.dict()
if old_avg:
p = multiprocessing.Process(target=perform_rule_match,
args=(rule, file_path,
manager_dict, yara_command,
yara_test_regex))
processes.append(p)
p.start()
else:
perform_rule_match(rule, file_path, manager_dict, yara_command,
yara_test_regex)
manager_dicts.append(manager_dict)
else:
manager_dicts.append(
perform_rule_match(rule, file_path, dict(), yara_command,
yara_test_regex))
if old_avg and not is_async:
time.sleep((old_avg * threshold) / 1000.0)
for p in processes:
p.join()
if p.is_alive():
tests_terminated += 1
p.terminate()
p.join()
for managers in manager_dicts:
if managers['duration']:
total_file_time += managers['duration']
if managers['match']:
count_of_files_matched += 1
files_matches.append(
Yara_testing_history_files_matches(
run_time=managers['duration'],
path=managers['file_path'],
stdout=managers['stdout'],
stderr=managers['stderr'],
command=managers['command'],
command_match_test_regex=managers[
'command_match_test_regex']))
end_time = time.time()
end_time_str = datetime.datetime.fromtimestamp(end_time).strftime(
'%Y-%m-%d %H:%M:%S')
if total_file_count > 0:
yrh.status = Yara_testing_history.STATUS_COMPLETE
yrh.end_time = end_time_str
yrh.files_tested = total_file_count
yrh.files_matched = count_of_files_matched
yrh.avg_millis_per_file = ((total_file_time) / total_file_count)
db.session.add(yrh)
db.session.commit()
for match in files_matches:
match.history = yrh
db.session.add(match)
db.session.commit()
return dict(duration=(total_file_time * 1000),
files_tested=total_file_count,
files_matched=count_of_files_matched,
tests_terminated=tests_terminated,
errors_encountered=errors_encountered,
error_msgs=error_msgs)