def create_services(ds: AssemblylineDatastore, log=None, limit=None): if not limit: limit = len(SERVICES) for svc_name, svc in list(SERVICES.items())[:limit]: service_data = { "name": svc_name, "enabled": True, "category": svc[0], "stage": svc[1], "version": "3.3.0", "docker_config": { "image": f"cccs/alsvc_{svc_name.lower()}:latest", }, } if random.choice([True, False]): service_data['update_config'] = { "method": "run", "sources": [random_model_obj(UpdateSource)], "update_interval_seconds": 600, "generates_signatures": True } service_data = Service(service_data) # Save a v3 service ds.service.save(f"{service_data.name}_{service_data.version}", service_data) # Save the same service as v4 service_data.version = "4.0.0" ds.service.save(f"{service_data.name}_{service_data.version}", service_data) # Save the default delta entry ds.service_delta.save(service_data.name, {"version": service_data.version}) if log: log.info(f'\t{svc_name}') ds.service_delta.commit() ds.service.commit()
def register_service(self, service_data, log_prefix=""): keep_alive = True try: # Get heuristics list heuristics = service_data.pop('heuristics', None) # Patch update_channel, registry_type before Service registration object creation service_data['update_channel'] = service_data.get( 'update_channel', self.config.services.preferred_update_channel) service_data['docker_config']['registry_type'] = service_data['docker_config'] \ .get('registry_type', self.config.services.preferred_registry_type) service_data['privileged'] = service_data.get( 'privileged', self.config.services.prefer_service_privileged) for dep in service_data.get('dependencies', {}).values(): dep['container']['registry_type'] = dep.get( 'registry_type', self.config.services.preferred_registry_type) # Pop unused registration service_data for x in ['file_required', 'tool_version']: service_data.pop(x, None) # Create Service registration object service = Service(service_data) # Fix service version, we don't need to see the stable label service.version = service.version.replace('stable', '') # Save service if it doesn't already exist if not self.datastore.service.exists( f'{service.name}_{service.version}'): self.datastore.service.save( f'{service.name}_{service.version}', service) self.datastore.service.commit() self.log.info(f"{log_prefix}{service.name} registered") keep_alive = False # Save service delta if it doesn't already exist if not self.datastore.service_delta.exists(service.name): self.datastore.service_delta.save(service.name, {'version': service.version}) self.datastore.service_delta.commit() self.log.info(f"{log_prefix}{service.name} " f"version ({service.version}) registered") new_heuristics = [] if heuristics: plan = self.datastore.heuristic.get_bulk_plan() for index, heuristic in enumerate(heuristics): heuristic_id = f'#{index}' # Set heuristic id to it's position in the list for logging purposes try: # Append service name to heuristic ID heuristic[ 'heur_id'] = f"{service.name.upper()}.{str(heuristic['heur_id'])}" # Attack_id field is now a list, make it a list if we receive otherwise attack_id = heuristic.get('attack_id', None) if isinstance(attack_id, str): heuristic['attack_id'] = [attack_id] heuristic = Heuristic(heuristic) heuristic_id = heuristic.heur_id plan.add_upsert_operation(heuristic_id, heuristic) except Exception as e: msg = f"{service.name} has an invalid heuristic ({heuristic_id}): {str(e)}" self.log.exception(f"{log_prefix}{msg}") raise ValueError(msg) for item in self.datastore.heuristic.bulk(plan)['items']: if item['update']['result'] != "noop": new_heuristics.append(item['update']['_id']) self.log.info( f"{log_prefix}{service.name} " f"heuristic {item['update']['_id']}: {item['update']['result'].upper()}" ) self.datastore.heuristic.commit() service_config = self.datastore.get_service_with_delta( service.name, as_obj=False) # Notify components watching for service config changes self.event_sender.send(service.name, { 'operation': Operation.Added, 'name': service.name }) except ValueError as e: # Catch errors when building Service or Heuristic model(s) raise e return dict(keep_alive=keep_alive, new_heuristics=new_heuristics, service_config=service_config or dict())
def add_service(**_): """ Add a service using its yaml manifest Variables: None Arguments: None Data Block: <service_manifest.yml content> Result example: { "success": true } # Return true is the service was added """ data = request.data enable_allowed = True try: if b"$SERVICE_TAG" in data: tmp_service = yaml.safe_load(data) tmp_service.pop('tool_version', None) tmp_service.pop('file_required', None) tmp_service.pop('heuristics', []) # Apply global preferences, if missing, to get the appropriate container image tags tmp_service['update_channel'] = tmp_service.get( 'update_channel', config.services.preferred_update_channel) tmp_service['docker_config']['registry_type'] = tmp_service['docker_config'] \ .get('registry_type', config.services.preferred_registry_type) _, tag_name, _ = get_latest_tag_for_service( Service(tmp_service), config, LOGGER) enable_allowed = bool(tag_name) tag_name = tag_name.encode() if tag_name else b'latest' data = data.replace(b"$SERVICE_TAG", tag_name) service = yaml.safe_load(data) # Pop the data not part of service model service.pop('tool_version', None) service.pop('file_required', None) heuristics = service.pop('heuristics', []) # Validate submission params for sp in service.get('submission_params', []): if sp['type'] == 'list' and 'list' not in sp: return make_api_response( "", err= f"Missing list field for submission param: {sp['name']}", status_code=400) if sp['default'] != sp['value']: return make_api_response( "", err= f"Default and value mismatch for submission param: {sp['name']}", status_code=400) # Apply default global configurations (if absent in service configuration) if not service.get('update_channel'): service[ 'update_channel'] = config.services.preferred_update_channel if not service.get('docker_config', {}).get('registry_type'): service['docker_config'][ 'registry_type'] = config.services.preferred_registry_type # Privilege can be set explicitly but also granted to services that don't require the file for analysis service['privileged'] = service.get( 'privileged', config.services.prefer_service_privileged) for dep in service.get('dependencies', {}).values(): dep['container']['registry_type'] = dep.get( 'registry_type', config.services.preferred_registry_type) service['enabled'] = service['enabled'] and enable_allowed # Load service info service = Service(service) # Fix service version, we don't want to see stable if it's a stable container service.version = service.version.replace("stable", "") # Save service if it doesn't already exist if not STORAGE.service.get_if_exists( f'{service.name}_{service.version}'): STORAGE.service.save(f'{service.name}_{service.version}', service) STORAGE.service.commit() # Save service delta if it doesn't already exist if not STORAGE.service_delta.get_if_exists(service.name): STORAGE.service_delta.save(service.name, {'version': service.version}) STORAGE.service_delta.commit() # Notify components watching for service config changes event_sender.send(service.name, { 'operation': Operation.Added, 'name': service.name }) new_heuristics = [] if heuristics: plan = STORAGE.heuristic.get_bulk_plan() for _, heuristic in enumerate(heuristics): try: # Append service name to heuristic ID heuristic[ 'heur_id'] = f"{service.name.upper()}.{str(heuristic['heur_id'])}" # Attack_id field is now a list, make it a list if we receive otherwise attack_id = heuristic.get('attack_id', None) if isinstance(attack_id, str): heuristic['attack_id'] = [attack_id] heuristic = Heuristic(heuristic) heuristic_id = heuristic.heur_id plan.add_upsert_operation(heuristic_id, heuristic) except Exception: raise ValueError("Error parsing heuristics") for item in STORAGE.heuristic.bulk(plan)['items']: if item['update']['result'] != "noop": new_heuristics.append(item['update']['_id']) STORAGE.heuristic.commit() return make_api_response( dict(service_name=service.name, new_heuristics=new_heuristics)) except ValueError as e: # Catch errors when building Service or Heuristic model(s) return make_api_response("", err=str(e), status_code=400)
def add_service(**_): """ Add a service using its yaml manifest Variables: None Arguments: None Data Block: <service_manifest.yml content> Result example: { "success": true } # Return true is the service was added """ data = request.data try: if b"$SERVICE_TAG" in data: tmp_service = yaml.safe_load(data) tmp_service.pop('tool_version', None) tmp_service.pop('file_required', None) tmp_service.pop('heuristics', []) tmp_service[ 'update_channel'] = config.services.preferred_update_channel image_name, tag_name, _ = get_latest_tag_for_service( Service(tmp_service), config, LOGGER) if tag_name: tag_name = tag_name.encode() else: tag_name = b'latest' data = data.replace(b"$SERVICE_TAG", tag_name) service = yaml.safe_load(data) # Pop the data not part of service model service.pop('tool_version', None) service.pop('file_required', None) heuristics = service.pop('heuristics', []) # Validate submission params for sp in service.get('submission_params', []): if sp['type'] == 'list' and 'list' not in sp: return make_api_response( "", err= f"Missing list field for submission param: {sp['name']}", status_code=400) if sp['default'] != sp['value']: return make_api_response( "", err= f"Default and value mismatch for submission param: {sp['name']}", status_code=400) # Fix update_channel with the system default service['update_channel'] = config.services.preferred_update_channel # Load service info service = Service(service) # Fix service version, we don't want to see stable if it's a stable container service.version = service.version.replace("stable", "") # Save service if it doesn't already exist if not STORAGE.service.get_if_exists( f'{service.name}_{service.version}'): STORAGE.service.save(f'{service.name}_{service.version}', service) STORAGE.service.commit() # Save service delta if it doesn't already exist if not STORAGE.service_delta.get_if_exists(service.name): STORAGE.service_delta.save(service.name, {'version': service.version}) STORAGE.service_delta.commit() new_heuristics = [] if heuristics: plan = STORAGE.heuristic.get_bulk_plan() for index, heuristic in enumerate(heuristics): try: # Append service name to heuristic ID heuristic[ 'heur_id'] = f"{service.name.upper()}.{str(heuristic['heur_id'])}" # Attack_id field is now a list, make it a list if we receive otherwise attack_id = heuristic.get('attack_id', None) if isinstance(attack_id, str): heuristic['attack_id'] = [attack_id] heuristic = Heuristic(heuristic) heuristic_id = heuristic.heur_id plan.add_upsert_operation(heuristic_id, heuristic) except Exception: raise ValueError("Error parsing heuristics") for item in STORAGE.heuristic.bulk(plan)['items']: if item['update']['result'] != "noop": new_heuristics.append(item['update']['_id']) STORAGE.heuristic.commit() return make_api_response( dict(service_name=service.name, new_heuristics=new_heuristics)) except ValueError as e: # Catch errors when building Service or Heuristic model(s) return make_api_response("", err=str(e), status_code=400)