def get_random_tags() -> dict:
desired_tag_types = [
'attribution.actor', 'network.static.ip', 'network.dynamic.ip',
'network.static.domain', 'network.dynamic.domain',
'network.static.uri', 'network.dynamic.uri', 'av.virus_name',
'attribution.implant', 'file.rule.yara', 'file.behavior',
'attribution.exploit', 'technique.packer', 'technique.obfuscation'
]
out = {}
flat_fields = Tagging.flat_fields()
tag_list = random.choices(list(flat_fields.keys()), k=random.randint(0, 2))
tag_list.extend(random.choices(desired_tag_types, k=random.randint(1, 2)))
for key in tag_list:
parts = key.split(".")
d = out
for part in parts[:-1]:
if part not in d:
d[part] = dict()
d = d[part]
if parts[-1] not in d:
d[parts[-1]] = []
for _ in range(random.randint(1, 2)):
d[parts[-1]].append(
random_data_for_field(flat_fields[key],
key.split(".")[-1]))
return out
def __init__(self, data, log=None):
valid_tags = set(Tagging.flat_fields().keys())
self.match = data.get('match', {})
self.regex = data.get('regex', {})
self.log = log
# Validate matches and regex
for section, item in {
'match': self.match,
'regex': self.regex
}.items():
if not isinstance(item, dict):
raise InvalidWhitelist(
f"Section {section} should be of type: DICT")
for k, v in item.items():
if not isinstance(v, list):
raise InvalidWhitelist(
f"Values in the {section} section should all be of type: LIST"
)
if k not in valid_tags:
raise InvalidWhitelist(
f"Key ({k}) in the {section} section is not a valid tag."
)
if section == 'regex':
self.regex[k] = [re.compile(x) for x in v]
def get_systems_constants(**_):
"""
Return the current system configuration constants which includes:
* Priorities
* File types
* Service tag types
* Service tag contexts
Variables:
None
Arguments:
None
Data Block:
None
Result example:
{
"priorities": {},
"file_types": [],
"tag_types": [],
"tag_contexts": []
}
"""
accepts_map = {}
rejects_map = {}
default_list = []
for srv in STORAGE.list_all_services(as_obj=False):
name = srv.get('name', None)
if name:
accept = srv.get('accepts', DEFAULT_SERVICE_ACCEPTS)
reject = srv.get('rejects', DEFAULT_SERVICE_REJECTS)
if accept == DEFAULT_SERVICE_ACCEPTS and reject == DEFAULT_SERVICE_REJECTS:
default_list.append(name)
else:
accepts_map[name] = re.compile(accept)
rejects_map[name] = re.compile(reject)
out = {
"max_priority":
constants.MAX_PRIORITY,
"priorities":
constants.PRIORITIES,
"file_types": [[
t,
sorted([
x for x in accepts_map.keys() if re.match(accepts_map[x], t)
and not re.match(rejects_map[x], t)
])
] for t in sorted(constants.RECOGNIZED_TYPES.keys())],
"tag_types":
sorted(list(Tagging.flat_fields().keys()))
}
out['file_types'].insert(0, ["*", default_list])
return make_api_response(out)
def start(self):
yara_externals = {
f'al_{x.replace(".", "_")}': ""
for x in Tagging.flat_fields().keys()
}
yara_externals.update({"al_file_rule_yara": ""})
file_parsers, tag_parsers = cli.compile(yara_externals)
self.log.info(f"loaded {file_parsers}")
cli.validate_parser_config()
self.file_parsers = file_parsers
self.tag_parsers = tag_parsers
def put_tag_safelist(**_):
"""
Save a new version of the tag_safelist file
Variables:
None
Arguments:
None
Data Block:
<new tag_safelist.yml file>
Result example:
{"success": true}
"""
tag_safelist_yml = request.json
try:
yml_data = yaml.safe_load(tag_safelist_yml)
for key in yml_data.keys():
if key not in ['match', 'regex']:
raise Exception('Invalid key found.')
fields = Tagging.flat_fields()
for tag_type in ['match', 'regex']:
for key, value in yml_data[tag_type].items():
if key not in fields:
raise Exception(f'{key} is not a valid tag type')
if not isinstance(value, list):
raise Exception(
f'Value for {key} should be a list of strings')
except Exception as e:
return make_api_response(
None, f"Invalid tag_safelist.yml file submitted: {str(e)}", 400)
with forge.get_cachestore('system', config=config,
datastore=STORAGE) as cache:
cache.save('tag_safelist_yml',
tag_safelist_yml.encode('utf-8'),
ttl=ADMIN_FILE_TTL,
force=True)
return make_api_response({'success': True})
def __init__(self, config=None):
externals = list(Tagging.flat_fields().keys())
super().__init__(config, name="TagCheck", externals=externals)
def get_systems_constants(**_):
"""
Return the current system configuration constants which includes:
* Priorities
* File types
* Service tag types
* Service tag contexts
Variables:
None
Arguments:
None
Data Block:
None
Result example:
{
"priorities": {},
"file_types": [],
"tag_types": [],
"tag_contexts": []
}
"""
accepts_map = {}
rejects_map = {}
default_list = []
recognized_types = set(IDENTIFY.trusted_mimes.values())
recognized_types = recognized_types.union(
set([x['al_type'] for x in IDENTIFY.magic_patterns]))
with open(IDENTIFY.magic_file.split(":")[0]) as fh:
for values in magic_custom.findall(fh.read()):
recognized_types.add(values)
with open(IDENTIFY.yara_file) as fh:
for values in yara_custom.findall(fh.read()):
recognized_types.add(values)
for srv in STORAGE.list_all_services(as_obj=False):
name = srv.get('name', None)
if name:
accept = srv.get('accepts', DEFAULT_SERVICE_ACCEPTS)
reject = srv.get('rejects', DEFAULT_SERVICE_REJECTS)
if accept == DEFAULT_SERVICE_ACCEPTS and reject == DEFAULT_SERVICE_REJECTS:
default_list.append(name)
else:
accepts_map[name] = re.compile(accept)
rejects_map[name] = re.compile(reject)
out = {
"max_priority":
constants.MAX_PRIORITY,
"priorities":
constants.PRIORITIES,
"file_types": [[
t,
sorted([
x for x in accepts_map.keys() if re.match(accepts_map[x], t)
and not re.match(rejects_map[x], t)
])
] for t in sorted(list(recognized_types))],
"tag_types":
sorted(list(Tagging.flat_fields().keys()))
}
out['file_types'].insert(0, ["*", default_list])
return make_api_response(out)
def test_update_single_alert(config, datastore):
persistent_redis = get_client(
host=config.core.redis.persistent.host,
port=config.core.redis.persistent.port,
private=False,
)
alerter = Alerter()
# Swap our alerter onto a private queue so our test doesn't get intercepted
alerter.alert_queue = alert_queue = NamedQueue(uuid.uuid4().hex,
persistent_redis)
# Get a random submission
submission = random.choice(all_submissions)
all_submissions.remove(submission)
# Generate a task for the submission
ingest_msg = random_model_obj(IngestTask)
ingest_msg.submission.sid = submission.sid
ingest_msg.submission.metadata = submission.metadata
ingest_msg.submission.params = submission.params
ingest_msg.submission.files = submission.files
alert_queue.push(ingest_msg.as_primitives())
alert_type = alerter.run_once()
assert alert_type == 'create'
datastore.alert.commit()
original_alert = datastore.alert.get(
datastore.alert.search(f"sid:{submission.sid}", fl="id",
as_obj=False)['items'][0]['id'])
assert original_alert is not None
# Generate a children task
child_submission = Submission(submission.as_primitives())
child_submission.sid = get_random_id()
child_submission.params.psid = submission.sid
# Alter the result of one of the services
r = None
while r is None:
r = datastore.result.get(random.choice(child_submission.results))
for s in r.result.sections:
old_tags = s.tags.as_primitives(strip_null=True)
s.tags = Tagging(recursive_extend(old_tags, get_random_tags()))
datastore.result.save(r.build_key(), r)
datastore.result.commit()
datastore.submission.save(child_submission.sid, child_submission)
datastore.submission.commit()
child_ingest_msg = random_model_obj(IngestTask)
child_ingest_msg.submission.sid = child_submission.sid
child_ingest_msg.submission.metadata = child_submission.metadata
child_ingest_msg.submission.params = child_submission.params
child_ingest_msg.submission.files = child_submission.files
child_ingest_msg.submission.time = ingest_msg.submission.time
alert_queue.push(child_ingest_msg.as_primitives())
alert_type = alerter.run_once()
assert alert_type == 'update'
datastore.alert.commit()
updated_alert = datastore.alert.get(
datastore.alert.search(f"sid:{child_submission.sid}",
fl="id",
as_obj=False)['items'][0]['id'])
assert updated_alert is not None
assert updated_alert != original_alert
from assemblyline.odm.models.tagging import Tagging
from yara_.yara_updater import *
UPDATE_CONFIGURATION_PATH = os.environ.get(
'UPDATE_CONFIGURATION_PATH', "/tmp/tagcheck_updater_config.yaml")
UPDATE_OUTPUT_PATH = os.environ.get('UPDATE_OUTPUT_PATH',
"/tmp/tagcheck_updater_output")
UPDATE_DIR = os.path.join(tempfile.gettempdir(), 'tagcheck_updates')
YARA_EXTERNALS = {
f'al_{x.replace(".", "_")}': ''
for x in list(Tagging.flat_fields().keys())
}
if __name__ == '__main__':
al_log.init_logging('updater.tagcheck')
logger = logging.getLogger('assemblyline.updater.tagcheck')
yara_update("tagcheck", UPDATE_CONFIGURATION_PATH, UPDATE_OUTPUT_PATH,
UPDATE_DIR, YARA_EXTERNALS, logger)