def load_user_settings(user):
default_settings = get_default_user_settings(user)
settings = STORAGE.user_settings.get(user['uname'], as_obj=False)
srv_list = [
x for x in STORAGE.list_all_services(as_obj=False, full=True)
if x['enabled']
]
if not settings:
def_srv_list = None
settings = default_settings
else:
# Make sure all defaults are there
for key, item in default_settings.items():
if key not in settings:
settings[key] = item
# Remove all obsolete keys
for key in list(settings.keys()):
if key not in default_settings:
del settings[key]
def_srv_list = settings.get('services', {}).get('selected', None)
settings['service_spec'] = get_default_service_spec(srv_list)
settings['services'] = get_default_service_list(srv_list, def_srv_list)
# Normalize the user's classification
settings['classification'] = Classification.normalize_classification(
settings['classification'])
return settings
def get_default_service_list(srv_list=None, default_selection=None):
if not default_selection:
default_selection = DEFAULT_SRV_SEL
if not srv_list:
srv_list = STORAGE.list_all_services(as_obj=False, full=True)
services = {}
for item in srv_list:
grp = item['category']
if grp not in services:
services[grp] = []
services[grp].append({
"name":
item["name"],
"category":
grp,
"selected": (grp in default_selection
or item['name'] in default_selection),
"is_external":
item["is_external"]
})
return [{
"name": k,
"selected": k in default_selection,
"services": v
} for k, v in services.items()]
def get_default_service_spec(srv_list=None):
if not srv_list:
srv_list = STORAGE.list_all_services(as_obj=False, full=True)
return [{
"name": x['name'],
"params": x["submission_params"]
} for x in srv_list if x["submission_params"]]
def get_systems_constants(**_):
"""
Return the current system configuration constants which includes:
* Priorities
* File types
* Service tag types
* Service tag contexts
Variables:
None
Arguments:
None
Data Block:
None
Result example:
{
"priorities": {},
"file_types": [],
"tag_types": [],
"tag_contexts": []
}
"""
accepts_map = {}
rejects_map = {}
default_list = []
for srv in STORAGE.list_all_services(as_obj=False):
name = srv.get('name', None)
if name:
accept = srv.get('accepts', DEFAULT_SERVICE_ACCEPTS)
reject = srv.get('rejects', DEFAULT_SERVICE_REJECTS)
if accept == DEFAULT_SERVICE_ACCEPTS and reject == DEFAULT_SERVICE_REJECTS:
default_list.append(name)
else:
accepts_map[name] = re.compile(accept)
rejects_map[name] = re.compile(reject)
out = {
"max_priority":
constants.MAX_PRIORITY,
"priorities":
constants.PRIORITIES,
"file_types": [[
t,
sorted([
x for x in accepts_map.keys() if re.match(accepts_map[x], t)
and not re.match(rejects_map[x], t)
])
] for t in sorted(constants.RECOGNIZED_TYPES.keys())],
"tag_types":
sorted(list(Tagging.flat_fields().keys()))
}
out['file_types'].insert(0, ["*", default_list])
return make_api_response(out)
def check_for_service_updates(**_):
"""
Check for potential updates for the given services.
Variables:
None
Arguments:
None
Data Block:
None
Result example:
{
'ResultSample': {
'latest_tag': 'v4.0.0dev163',
'update_available': true
}, ...
}
"""
output = {}
for service in STORAGE.list_all_services(full=True, as_obj=False):
update_info = latest_service_tags.get(service['name']) or {}
if update_info:
latest_tag = update_info.get(service['update_channel'], None)
output[service['name']] = {
"auth":
update_info['auth'],
"image":
f"{update_info['image']}:{latest_tag or 'latest'}",
"latest_tag":
latest_tag,
"update_available":
latest_tag is not None
and latest_tag.replace('stable', '') != service['version'],
"updating":
service_update.exists(service['name'])
}
return make_api_response(output)
def list_all_services(**_):
"""
List all service configurations of the system.
Variables:
None
Arguments:
None
Data Block:
None
Result example:
[
{'accepts': ".*"
'category': 'Extraction',
'classpath': 'al_services.alsvc_extract.Extract',
'description': "Extracts some stuff",
'enabled': True,
'name': 'Extract',
'rejects': 'empty'
'stage': 'CORE'
},
...
]
"""
resp = [{
'accepts': x.get('accepts', None),
'category': x.get('category', None),
'description': x.get('description', None),
'enabled': x.get('enabled', False),
'name': x.get('name', None),
'privileged': x.get('privileged', False),
'rejects': x.get('rejects', None),
'stage': x.get('stage', None),
'version': x.get('version', None)
} for x in STORAGE.list_all_services(full=True, as_obj=False)]
return make_api_response(resp)
def get_signature_sources(**_):
"""
Get all signature sources
Variables:
None
Arguments:
None
Data Block:
None
Result example:
{
'Yara': {
{
"uri": "http://somesite/file_to_get", # URI to fetch for parsing the rules
"name": "signature_file.yar", # Name of the file we will parse the rules as
"username": null, # Username used to get to the URI
"password": null, # Password used to get to the URI
"header": { # Header sent during the request to the URI
"X_TOKEN": "SOME RANDOM TOKEN" # Exemple of header
},
"private_key": null, # Private key used to get to the URI
"pattern": "^*.yar$" # Regex pattern use to get appropriate files from the URI
}, ...
}, ...
}
"""
services = STORAGE.list_all_services(full=True, as_obj=False)
out = {}
for service in services:
if service.get("update_config", {}).get("generates_signatures", False):
out[service['name']] = service['update_config']['sources']
# Save the signature
return make_api_response(out)
def get_system_configuration(**_):
"""
Return the current system configuration:
* Max file size
* Max number of embedded files
* Extraction's max depth
* and many others...
Variables:
None
Arguments:
None
Data Block:
None
Result example:
{
"<CONFIGURATION_ITEM>": <CONFIGURATION_VALUE>
}
"""
def get_config_item(parent, cur_item):
if "." in cur_item:
key, remainder = cur_item.split(".", 1)
return get_config_item(parent.get(key, {}), remainder)
else:
return parent.get(cur_item, None)
cat_map = {}
stg_map = {}
for srv in STORAGE.list_all_services(as_obj=False):
name = srv.get('name', None)
cat = srv.get('category', None)
if cat and name:
temp_cat = cat_map.get(cat, [])
temp_cat.append(name)
cat_map[cat] = temp_cat
stg = srv.get('stage', None)
if stg and name:
temp_stg = stg_map.get(stg, [])
temp_stg.append(name)
stg_map[stg] = temp_stg
shareable_config_items = [
"core.ingester.default_max_extracted",
"core.ingester.default_max_supplementary", "services.categories",
"services.min_service_workers", "services.preferred_update_channel",
"services.stages", "submission.default_max_extracted",
"submission.default_max_supplementary", "submission.dtl",
"submission.max_extraction_depth", "submission.max_file_size",
"submission.max_metadata_length", "submission.tag_types.attribution",
"submission.tag_types.behavior", "submission.tag_types.ioc",
"ui.allow_raw_downloads", "ui.audit", "ui.download_encoding",
"ui.enforce_quota", "ui.ingest_max_priority"
]
out = {}
config_dict = config.as_primitives()
for item in shareable_config_items:
out[item] = get_config_item(config_dict, item)
out["services.categories"] = [[x, cat_map.get(x, [])]
for x in out.get("services.categories", None)
]
out["services.stages"] = [[x, stg_map.get(x, [])]
for x in out.get("services.stages", None)]
return make_api_response(out)
def get_systems_constants(**_):
"""
Return the current system configuration constants which includes:
* Priorities
* File types
* Service tag types
* Service tag contexts
Variables:
None
Arguments:
None
Data Block:
None
Result example:
{
"priorities": {},
"file_types": [],
"tag_types": [],
"tag_contexts": []
}
"""
accepts_map = {}
rejects_map = {}
default_list = []
recognized_types = set(IDENTIFY.trusted_mimes.values())
recognized_types = recognized_types.union(
set([x['al_type'] for x in IDENTIFY.magic_patterns]))
with open(IDENTIFY.magic_file.split(":")[0]) as fh:
for values in magic_custom.findall(fh.read()):
recognized_types.add(values)
with open(IDENTIFY.yara_file) as fh:
for values in yara_custom.findall(fh.read()):
recognized_types.add(values)
for srv in STORAGE.list_all_services(as_obj=False):
name = srv.get('name', None)
if name:
accept = srv.get('accepts', DEFAULT_SERVICE_ACCEPTS)
reject = srv.get('rejects', DEFAULT_SERVICE_REJECTS)
if accept == DEFAULT_SERVICE_ACCEPTS and reject == DEFAULT_SERVICE_REJECTS:
default_list.append(name)
else:
accepts_map[name] = re.compile(accept)
rejects_map[name] = re.compile(reject)
out = {
"max_priority":
constants.MAX_PRIORITY,
"priorities":
constants.PRIORITIES,
"file_types": [[
t,
sorted([
x for x in accepts_map.keys() if re.match(accepts_map[x], t)
and not re.match(rejects_map[x], t)
])
] for t in sorted(list(recognized_types))],
"tag_types":
sorted(list(Tagging.flat_fields().keys()))
}
out['file_types'].insert(0, ["*", default_list])
return make_api_response(out)