def get_default_service_spec(srv_list=None): if not srv_list: srv_list = STORAGE.list_services() return [{"name": x['name'], "params": x["submission_params"]} for x in srv_list if x["submission_params"]]
def load_user_settings(user): default_settings = copy.deepcopy(SETTINGS_DEFAULT) default_settings[ 'classification'] = Classification.default_user_classification(user) options = STORAGE.get_user_options(user['uname']) srv_list = [x for x in STORAGE.list_services() if x['enabled']] if not options: def_srv_list = None options = default_settings else: # Make sure all defaults are there for key, item in default_settings.iteritems(): if key not in options: options[key] = item # Remove all obsolete keys for key in options.keys(): if key not in default_settings: del options[key] def_srv_list = options.get('services', None) options['service_spec'] = get_default_service_spec(srv_list) options['services'] = get_default_service_list(srv_list, def_srv_list) # Normalize the user's classification options['classification'] = Classification.normalize_classification( options['classification']) return options
def get_systems_constants(**_): """ Return the current system configuration constants which includes: * Priorities * File types * Service tag types * Service tag contexts Variables: None Arguments: None Data Block: None Result example: { "priorities": {}, "file_types": [], "tag_types": [], "tag_contexts": [] } """ accepts_map = {} rejects_map = {} default_list = [] for srv in STORAGE.list_services(): name = srv.get('name', None) if name: accept = srv.get('accepts', ".*") reject = srv.get('rejects', "empty") if accept == ServiceBase.SERVICE_ACCEPTS and reject == ServiceBase.SERVICE_REJECTS: default_list.append(name) else: accepts_map[name] = re.compile(accept) rejects_map[name] = re.compile(reject) out = { "priorities": constants.PRIORITIES, "file_types": [[ t, sorted([ x for x in accepts_map.keys() if re.match(accepts_map[x], t) and not re.match(rejects_map[x], t) ]) ] for t in sorted(constants.RECOGNIZED_TAGS.keys())], "tag_types": sorted([x[0] for x in constants.STANDARD_TAG_TYPES]), "tag_contexts": sorted([x[0] for x in constants.STANDARD_TAG_CONTEXTS]) } out['file_types'].insert(0, ["*", default_list]) return make_api_response(out)
def list_services_workers(**_): """ List number of workers for each services in the system. Variables: None Arguments: None Data Block: None Result example: {"MY SERVICE": 1, ... } # Dictionary of services and number of workers """ services = {s["name"]: 0 for s in STORAGE.list_services() if s['enabled']} profiles = STORAGE.get_profiles_dict( list( set([ p["_yz_rk"] for p in STORAGE.stream_search( "profile", "_yz_rk:*", fl="_yz_rk") ]))) used_profiles = { n['mac_address']: n['profile'] for n in STORAGE.get_nodes(STORAGE.list_node_keys()) if n['profile'] != "" } for _mac, used_p in used_profiles.iteritems(): if used_p in profiles: for srv, cfg in profiles[used_p]["services"].iteritems(): if srv in services: services[srv] += cfg["workers"] for srv, cfg in profiles[used_p]["virtual_machines"].iteritems(): if srv in services: vm = STORAGE.get_virtualmachine(srv) if not vm: continue services[srv] += vm['num_workers'] * cfg['num_instances'] return make_api_response(services, err=[profiles, used_profiles, services])
def get_default_service_list(srv_list=None, default_selection=None): if not default_selection: default_selection = ["Extraction", "Static Analysis", "Filtering", "Antivirus", "Post-Processing"] if not srv_list: srv_list = STORAGE.list_services() services = {} for item in srv_list: grp = item['category'] if grp == SYSTEM_SERVICE_CATEGORY_NAME: continue if not services.has_key(grp): services[grp] = [] services[grp].append({"name": item["name"], "category": grp, "selected": (grp in default_selection or item['name'] in default_selection), "is_external": item["is_external"]}) return [{"name": k, "selected": k in default_selection, "services": v} for k, v in services.iteritems()]
def list_services(**__): """ List all service configurations of the system. Variables: None Arguments: None Data Block: None Result example: [ {'accepts': ".*" 'category': 'Extraction', 'classpath': 'al_services.alsvc_extract.Extract', 'description': "Extracts some stuff", 'enabled': True, 'name': 'Extract', 'rejects': 'empty' 'stage': 'CORE' }, ... ] """ resp = [{ 'accepts': x.get('accepts', None), 'category': x.get('category', None), 'classpath': x.get('classpath', None), 'description': x.get('description', None), 'enabled': x.get('enabled', False), 'name': x.get('name', None), 'rejects': x.get('rejects', None), 'stage': x.get('stage', None) } for x in STORAGE.list_services()] return make_api_response(resp)
def list_queue_sizes(**_): """ List services queue size for each services in the system. Variables: None Arguments: None Data Block: None Result example: {"MY SERVICE": 1, ... } # Dictionnary of services and number item in queue """ services = list( set([s.get("classpath", None) for s in STORAGE.list_services()])) queue_lengths = {} for svc in services: queue_lengths[svc.split(".")[-1]] = get_service_queue_length(svc) return make_api_response(queue_lengths)
def load_system_info(**kwargs): """ Load the full system information Variables: None Arguments: None Data Block: None Result example: { "vms": {}, # Map of vms that are available in the system "services": {}, # Map of service that are available in the system "hosts": [] # List of physical hosts configured } """ temp_service_map = {x['name']: x for x in STORAGE.list_services()} vm_list = STORAGE.list_virtualmachines() hosts = STORAGE.list_node_keys() host_list = sorted( [ host for host in STORAGE.get_nodes(hosts) if host is not None and 'hostagent' in host.get('roles', []) ], key=lambda k: (k.get('machine_info', {}).get('cores', 1), k.get('machine_info', {}). get('memory', '11.7'), k.get('machine_info', {}).get('name', 1))) service_map = copy.copy(temp_service_map) out_vm_map = {} for vm in vm_list: service_name = vm['name'] srv_list = {service_name: vm['num_workers']} cpu_usage = temp_service_map.get(service_name, {}).get( 'cpu_cores', 1) * vm['num_workers'] out_vm_map[service_name] = { "cpu_usage": cpu_usage, "ram_usage": vm['ram'], "services": srv_list, "enabled": vm.get("enabled", False) } try: del service_map[service_name] except KeyError: continue out_service_map = {} for service in service_map.itervalues(): out_service_map[service['name']] = { "cpu_usage": service.get('cpu_cores', 1), "ram_usage": service.get('ram_mb', 1024), "enabled": service.get("enabled", False) } out_host_list = [] for host in host_list: out_host_list.append({ "hostname": host['machine_info']['name'], "profile": host['profile'], "cores": host['machine_info']['cores'], "memory": float(host['machine_info']['memory']) * 1024, "mac": host['mac_address'] }) return make_api_response({ 'vms': out_vm_map, 'services': out_service_map, "hosts": out_host_list })
def get_system_configuration_overview(**_): """ Display a system configuration overview. Variables: None Arguments: None Data Block: None Result example: { "errors": { # Errors in the current config "profiles": [], # Profiles in error "services": [] }, # Services in error "services": { # Services overview "SRV_NAME": { # Single service overview "enabled": True, # is enabled? "profiles" : [], # profiles referencing it "queue": 0, # items in queue "workers": 1 }, # number of workers ...,} } """ errors = {"services": [], "profiles": []} services = { s["name"]: { "workers": 0, "enabled": s["enabled"], "profiles": [], "queue": 0 } for s in STORAGE.list_services() } profiles = STORAGE.get_profiles_dict( list( set([ p["_yz_rk"] for p in STORAGE.stream_search( "profile", "_yz_rk:*", fl="_yz_rk") ]))) used_profiles = { n['mac_address']: n['profile'] for n in STORAGE.get_nodes(STORAGE.list_node_keys()) if n['profile'] != "" } for mac, used_p in used_profiles.iteritems(): if profiles.has_key(used_p): for srv, cfg in profiles[used_p]["services"].iteritems(): if not services.has_key(srv): errors["services"].append({ "service": srv, "profile": used_p }) continue services[srv]["workers"] += cfg["workers"] if used_p not in services[srv]["profiles"]: services[srv]["profiles"].append(used_p) for srv, cfg in profiles[used_p]["virtual_machines"].iteritems(): if not services.has_key(srv): errors["services"].append({ "service": srv, "profile": used_p }) continue vm = STORAGE.get_virtualmachine(srv) if not vm: errors["services"].append({ "service": srv, "profile": used_p }) continue services[srv][ "workers"] += vm['num_workers'] * cfg['num_instances'] if used_p not in services[srv]["profiles"]: services[srv]["profiles"].append(used_p) else: errors["profiles"].append({"profile": used_p, "mac": mac}) for srv in services: services[srv]["queue"] = get_service_queue_length(srv) return make_api_response({"services": services, "errors": errors})
from flask import request from assemblyline.al.common.heuristics import list_all_heuristics from al_ui.apiv3 import core from assemblyline.al.common import forge from al_ui.config import STORAGE from al_ui.api_base import api_login, make_api_response SUB_API = 'heuristics' Classification = forge.get_classification() heuristics_api = core.make_subapi_blueprint(SUB_API) heuristics_api._doc = "View the different heuristics of the system" HEUR, HEUR_MAP = list_all_heuristics(STORAGE.list_services()) @heuristics_api.route("/<heuristic_id>/", methods=["GET"]) @api_login() def get_heuristic(heuristic_id, **kwargs): """ Get a specific heuristic's detail from the system Variables: heuristic_id => ID of the heuristic Arguments: None Data Block:
def get_system_configuration(**_): """ Return the current system configuration: * Max file size * Max number of embedded files * Extraction's max depth * and many others... Variables: None Arguments: None Data Block: None Result example: { "<CONFIGURATION_ITEM>": <CONFIGURATION_VALUE> } """ def get_config_item(parent, cur_item): if "." in cur_item: key, remainder = cur_item.split(".", 1) return get_config_item(parent[key], remainder) else: return parent.get(cur_item, None) cat_map = {} stg_map = {} for srv in STORAGE.list_services(): name = srv.get('name', None) cat = srv.get('category', None) if cat and name: temp_cat = cat_map.get(cat, []) temp_cat.append(name) cat_map[cat] = temp_cat stg = srv.get('stage', None) if stg and name: temp_stg = stg_map.get(stg, []) temp_stg.append(name) stg_map[stg] = temp_stg shareable_config_items = [ "core.middleman.max_extracted", "core.middleman.max_supplementary", "services.categories", "services.limits.max_extracted", "services.limits.max_supplementary", "services.stages", "services.system_category", "submissions.max.priority", "submissions.max.size", "submissions.ttl", "ui.allow_raw_downloads", "ui.audit", "ui.download_encoding", "ui.enforce_quota" ] out = {} for item in shareable_config_items: out[item] = get_config_item(config, item) out["services.categories"] = [[x, cat_map.get(x, [])] for x in out.get("services.categories", None) ] out["services.stages"] = [[x, stg_map.get(x, [])] for x in out.get("services.stages", None)] return make_api_response(out)