def get_default_service_spec(srv_list=None):
if not srv_list:
srv_list = STORAGE.list_services()
return [{"name": x['name'],
"params": x["submission_params"]}
for x in srv_list if x["submission_params"]]
def load_user_settings(user):
default_settings = copy.deepcopy(SETTINGS_DEFAULT)
default_settings[
'classification'] = Classification.default_user_classification(user)
options = STORAGE.get_user_options(user['uname'])
srv_list = [x for x in STORAGE.list_services() if x['enabled']]
if not options:
def_srv_list = None
options = default_settings
else:
# Make sure all defaults are there
for key, item in default_settings.iteritems():
if key not in options:
options[key] = item
# Remove all obsolete keys
for key in options.keys():
if key not in default_settings:
del options[key]
def_srv_list = options.get('services', None)
options['service_spec'] = get_default_service_spec(srv_list)
options['services'] = get_default_service_list(srv_list, def_srv_list)
# Normalize the user's classification
options['classification'] = Classification.normalize_classification(
options['classification'])
return options
def get_systems_constants(**_):
"""
Return the current system configuration constants which includes:
* Priorities
* File types
* Service tag types
* Service tag contexts
Variables:
None
Arguments:
None
Data Block:
None
Result example:
{
"priorities": {},
"file_types": [],
"tag_types": [],
"tag_contexts": []
}
"""
accepts_map = {}
rejects_map = {}
default_list = []
for srv in STORAGE.list_services():
name = srv.get('name', None)
if name:
accept = srv.get('accepts', ".*")
reject = srv.get('rejects', "empty")
if accept == ServiceBase.SERVICE_ACCEPTS and reject == ServiceBase.SERVICE_REJECTS:
default_list.append(name)
else:
accepts_map[name] = re.compile(accept)
rejects_map[name] = re.compile(reject)
out = {
"priorities":
constants.PRIORITIES,
"file_types": [[
t,
sorted([
x for x in accepts_map.keys() if re.match(accepts_map[x], t)
and not re.match(rejects_map[x], t)
])
] for t in sorted(constants.RECOGNIZED_TAGS.keys())],
"tag_types":
sorted([x[0] for x in constants.STANDARD_TAG_TYPES]),
"tag_contexts":
sorted([x[0] for x in constants.STANDARD_TAG_CONTEXTS])
}
out['file_types'].insert(0, ["*", default_list])
return make_api_response(out)
def list_services_workers(**_):
"""
List number of workers for each services in the system.
Variables:
None
Arguments:
None
Data Block:
None
Result example:
{"MY SERVICE": 1, ... } # Dictionary of services and number of workers
"""
services = {s["name"]: 0 for s in STORAGE.list_services() if s['enabled']}
profiles = STORAGE.get_profiles_dict(
list(
set([
p["_yz_rk"] for p in STORAGE.stream_search(
"profile", "_yz_rk:*", fl="_yz_rk")
])))
used_profiles = {
n['mac_address']: n['profile']
for n in STORAGE.get_nodes(STORAGE.list_node_keys())
if n['profile'] != ""
}
for _mac, used_p in used_profiles.iteritems():
if used_p in profiles:
for srv, cfg in profiles[used_p]["services"].iteritems():
if srv in services:
services[srv] += cfg["workers"]
for srv, cfg in profiles[used_p]["virtual_machines"].iteritems():
if srv in services:
vm = STORAGE.get_virtualmachine(srv)
if not vm:
continue
services[srv] += vm['num_workers'] * cfg['num_instances']
return make_api_response(services, err=[profiles, used_profiles, services])
def get_default_service_list(srv_list=None, default_selection=None):
if not default_selection:
default_selection = ["Extraction", "Static Analysis", "Filtering", "Antivirus", "Post-Processing"]
if not srv_list:
srv_list = STORAGE.list_services()
services = {}
for item in srv_list:
grp = item['category']
if grp == SYSTEM_SERVICE_CATEGORY_NAME:
continue
if not services.has_key(grp):
services[grp] = []
services[grp].append({"name": item["name"],
"category": grp,
"selected": (grp in default_selection or item['name'] in default_selection),
"is_external": item["is_external"]})
return [{"name": k, "selected": k in default_selection, "services": v} for k, v in services.iteritems()]
def list_services(**__):
"""
List all service configurations of the system.
Variables:
None
Arguments:
None
Data Block:
None
Result example:
[
{'accepts': ".*"
'category': 'Extraction',
'classpath': 'al_services.alsvc_extract.Extract',
'description': "Extracts some stuff",
'enabled': True,
'name': 'Extract',
'rejects': 'empty'
'stage': 'CORE'
},
...
]
"""
resp = [{
'accepts': x.get('accepts', None),
'category': x.get('category', None),
'classpath': x.get('classpath', None),
'description': x.get('description', None),
'enabled': x.get('enabled', False),
'name': x.get('name', None),
'rejects': x.get('rejects', None),
'stage': x.get('stage', None)
} for x in STORAGE.list_services()]
return make_api_response(resp)
def list_queue_sizes(**_):
"""
List services queue size for each services in the system.
Variables:
None
Arguments:
None
Data Block:
None
Result example:
{"MY SERVICE": 1, ... } # Dictionnary of services and number item in queue
"""
services = list(
set([s.get("classpath", None) for s in STORAGE.list_services()]))
queue_lengths = {}
for svc in services:
queue_lengths[svc.split(".")[-1]] = get_service_queue_length(svc)
return make_api_response(queue_lengths)
def load_system_info(**kwargs):
"""
Load the full system information
Variables:
None
Arguments:
None
Data Block:
None
Result example:
{
"vms": {}, # Map of vms that are available in the system
"services": {}, # Map of service that are available in the system
"hosts": [] # List of physical hosts configured
}
"""
temp_service_map = {x['name']: x for x in STORAGE.list_services()}
vm_list = STORAGE.list_virtualmachines()
hosts = STORAGE.list_node_keys()
host_list = sorted(
[
host for host in STORAGE.get_nodes(hosts)
if host is not None and 'hostagent' in host.get('roles', [])
],
key=lambda k:
(k.get('machine_info', {}).get('cores', 1), k.get('machine_info', {}).
get('memory', '11.7'), k.get('machine_info', {}).get('name', 1)))
service_map = copy.copy(temp_service_map)
out_vm_map = {}
for vm in vm_list:
service_name = vm['name']
srv_list = {service_name: vm['num_workers']}
cpu_usage = temp_service_map.get(service_name, {}).get(
'cpu_cores', 1) * vm['num_workers']
out_vm_map[service_name] = {
"cpu_usage": cpu_usage,
"ram_usage": vm['ram'],
"services": srv_list,
"enabled": vm.get("enabled", False)
}
try:
del service_map[service_name]
except KeyError:
continue
out_service_map = {}
for service in service_map.itervalues():
out_service_map[service['name']] = {
"cpu_usage": service.get('cpu_cores', 1),
"ram_usage": service.get('ram_mb', 1024),
"enabled": service.get("enabled", False)
}
out_host_list = []
for host in host_list:
out_host_list.append({
"hostname": host['machine_info']['name'],
"profile": host['profile'],
"cores": host['machine_info']['cores'],
"memory": float(host['machine_info']['memory']) * 1024,
"mac": host['mac_address']
})
return make_api_response({
'vms': out_vm_map,
'services': out_service_map,
"hosts": out_host_list
})
def get_system_configuration_overview(**_):
"""
Display a system configuration overview.
Variables:
None
Arguments:
None
Data Block:
None
Result example:
{
"errors": { # Errors in the current config
"profiles": [], # Profiles in error
"services": [] }, # Services in error
"services": { # Services overview
"SRV_NAME": { # Single service overview
"enabled": True, # is enabled?
"profiles" : [], # profiles referencing it
"queue": 0, # items in queue
"workers": 1 }, # number of workers
...,}
}
"""
errors = {"services": [], "profiles": []}
services = {
s["name"]: {
"workers": 0,
"enabled": s["enabled"],
"profiles": [],
"queue": 0
}
for s in STORAGE.list_services()
}
profiles = STORAGE.get_profiles_dict(
list(
set([
p["_yz_rk"] for p in STORAGE.stream_search(
"profile", "_yz_rk:*", fl="_yz_rk")
])))
used_profiles = {
n['mac_address']: n['profile']
for n in STORAGE.get_nodes(STORAGE.list_node_keys())
if n['profile'] != ""
}
for mac, used_p in used_profiles.iteritems():
if profiles.has_key(used_p):
for srv, cfg in profiles[used_p]["services"].iteritems():
if not services.has_key(srv):
errors["services"].append({
"service": srv,
"profile": used_p
})
continue
services[srv]["workers"] += cfg["workers"]
if used_p not in services[srv]["profiles"]:
services[srv]["profiles"].append(used_p)
for srv, cfg in profiles[used_p]["virtual_machines"].iteritems():
if not services.has_key(srv):
errors["services"].append({
"service": srv,
"profile": used_p
})
continue
vm = STORAGE.get_virtualmachine(srv)
if not vm:
errors["services"].append({
"service": srv,
"profile": used_p
})
continue
services[srv][
"workers"] += vm['num_workers'] * cfg['num_instances']
if used_p not in services[srv]["profiles"]:
services[srv]["profiles"].append(used_p)
else:
errors["profiles"].append({"profile": used_p, "mac": mac})
for srv in services:
services[srv]["queue"] = get_service_queue_length(srv)
return make_api_response({"services": services, "errors": errors})
from flask import request
from assemblyline.al.common.heuristics import list_all_heuristics
from al_ui.apiv3 import core
from assemblyline.al.common import forge
from al_ui.config import STORAGE
from al_ui.api_base import api_login, make_api_response
SUB_API = 'heuristics'
Classification = forge.get_classification()
heuristics_api = core.make_subapi_blueprint(SUB_API)
heuristics_api._doc = "View the different heuristics of the system"
HEUR, HEUR_MAP = list_all_heuristics(STORAGE.list_services())
@heuristics_api.route("/<heuristic_id>/", methods=["GET"])
@api_login()
def get_heuristic(heuristic_id, **kwargs):
"""
Get a specific heuristic's detail from the system
Variables:
heuristic_id => ID of the heuristic
Arguments:
None
Data Block:
def get_system_configuration(**_):
"""
Return the current system configuration:
* Max file size
* Max number of embedded files
* Extraction's max depth
* and many others...
Variables:
None
Arguments:
None
Data Block:
None
Result example:
{
"<CONFIGURATION_ITEM>": <CONFIGURATION_VALUE>
}
"""
def get_config_item(parent, cur_item):
if "." in cur_item:
key, remainder = cur_item.split(".", 1)
return get_config_item(parent[key], remainder)
else:
return parent.get(cur_item, None)
cat_map = {}
stg_map = {}
for srv in STORAGE.list_services():
name = srv.get('name', None)
cat = srv.get('category', None)
if cat and name:
temp_cat = cat_map.get(cat, [])
temp_cat.append(name)
cat_map[cat] = temp_cat
stg = srv.get('stage', None)
if stg and name:
temp_stg = stg_map.get(stg, [])
temp_stg.append(name)
stg_map[stg] = temp_stg
shareable_config_items = [
"core.middleman.max_extracted", "core.middleman.max_supplementary",
"services.categories", "services.limits.max_extracted",
"services.limits.max_supplementary", "services.stages",
"services.system_category", "submissions.max.priority",
"submissions.max.size", "submissions.ttl", "ui.allow_raw_downloads",
"ui.audit", "ui.download_encoding", "ui.enforce_quota"
]
out = {}
for item in shareable_config_items:
out[item] = get_config_item(config, item)
out["services.categories"] = [[x, cat_map.get(x, [])]
for x in out.get("services.categories", None)
]
out["services.stages"] = [[x, stg_map.get(x, [])]
for x in out.get("services.stages", None)]
return make_api_response(out)
