def create_services(ds: AssemblylineDatastore, log=None, limit=None):
if not limit:
limit = len(SERVICES)
for svc_name, svc in list(SERVICES.items())[:limit]:
service_data = {
"name": svc_name,
"enabled": True,
"category": svc[0],
"stage": svc[1],
"version": "3.3.0",
"docker_config": {
"image": f"cccs/alsvc_{svc_name.lower()}:latest",
},
}
if random.choice([True, False]):
service_data['update_config'] = {
"method": "run",
"sources": [random_model_obj(UpdateSource)],
"update_interval_seconds": 600,
"generates_signatures": True
}
service_data = Service(service_data)
# Save a v3 service
ds.service.save(f"{service_data.name}_{service_data.version}",
service_data)
# Save the same service as v4
service_data.version = "4.0.0"
ds.service.save(f"{service_data.name}_{service_data.version}",
service_data)
# Save the default delta entry
ds.service_delta.save(service_data.name,
{"version": service_data.version})
if log:
log.info(f'\t{svc_name}')
ds.service_delta.commit()
ds.service.commit()
def register_service(self, service_data, log_prefix=""):
keep_alive = True
try:
# Get heuristics list
heuristics = service_data.pop('heuristics', None)
# Patch update_channel, registry_type before Service registration object creation
service_data['update_channel'] = service_data.get(
'update_channel',
self.config.services.preferred_update_channel)
service_data['docker_config']['registry_type'] = service_data['docker_config'] \
.get('registry_type', self.config.services.preferred_registry_type)
service_data['privileged'] = service_data.get(
'privileged', self.config.services.prefer_service_privileged)
for dep in service_data.get('dependencies', {}).values():
dep['container']['registry_type'] = dep.get(
'registry_type',
self.config.services.preferred_registry_type)
# Pop unused registration service_data
for x in ['file_required', 'tool_version']:
service_data.pop(x, None)
# Create Service registration object
service = Service(service_data)
# Fix service version, we don't need to see the stable label
service.version = service.version.replace('stable', '')
# Save service if it doesn't already exist
if not self.datastore.service.exists(
f'{service.name}_{service.version}'):
self.datastore.service.save(
f'{service.name}_{service.version}', service)
self.datastore.service.commit()
self.log.info(f"{log_prefix}{service.name} registered")
keep_alive = False
# Save service delta if it doesn't already exist
if not self.datastore.service_delta.exists(service.name):
self.datastore.service_delta.save(service.name,
{'version': service.version})
self.datastore.service_delta.commit()
self.log.info(f"{log_prefix}{service.name} "
f"version ({service.version}) registered")
new_heuristics = []
if heuristics:
plan = self.datastore.heuristic.get_bulk_plan()
for index, heuristic in enumerate(heuristics):
heuristic_id = f'#{index}' # Set heuristic id to it's position in the list for logging purposes
try:
# Append service name to heuristic ID
heuristic[
'heur_id'] = f"{service.name.upper()}.{str(heuristic['heur_id'])}"
# Attack_id field is now a list, make it a list if we receive otherwise
attack_id = heuristic.get('attack_id', None)
if isinstance(attack_id, str):
heuristic['attack_id'] = [attack_id]
heuristic = Heuristic(heuristic)
heuristic_id = heuristic.heur_id
plan.add_upsert_operation(heuristic_id, heuristic)
except Exception as e:
msg = f"{service.name} has an invalid heuristic ({heuristic_id}): {str(e)}"
self.log.exception(f"{log_prefix}{msg}")
raise ValueError(msg)
for item in self.datastore.heuristic.bulk(plan)['items']:
if item['update']['result'] != "noop":
new_heuristics.append(item['update']['_id'])
self.log.info(
f"{log_prefix}{service.name} "
f"heuristic {item['update']['_id']}: {item['update']['result'].upper()}"
)
self.datastore.heuristic.commit()
service_config = self.datastore.get_service_with_delta(
service.name, as_obj=False)
# Notify components watching for service config changes
self.event_sender.send(service.name, {
'operation': Operation.Added,
'name': service.name
})
except ValueError as e: # Catch errors when building Service or Heuristic model(s)
raise e
return dict(keep_alive=keep_alive,
new_heuristics=new_heuristics,
service_config=service_config or dict())
def add_service(**_):
"""
Add a service using its yaml manifest
Variables:
None
Arguments:
None
Data Block:
<service_manifest.yml content>
Result example:
{ "success": true } # Return true is the service was added
"""
data = request.data
enable_allowed = True
try:
if b"$SERVICE_TAG" in data:
tmp_service = yaml.safe_load(data)
tmp_service.pop('tool_version', None)
tmp_service.pop('file_required', None)
tmp_service.pop('heuristics', [])
# Apply global preferences, if missing, to get the appropriate container image tags
tmp_service['update_channel'] = tmp_service.get(
'update_channel', config.services.preferred_update_channel)
tmp_service['docker_config']['registry_type'] = tmp_service['docker_config'] \
.get('registry_type', config.services.preferred_registry_type)
_, tag_name, _ = get_latest_tag_for_service(
Service(tmp_service), config, LOGGER)
enable_allowed = bool(tag_name)
tag_name = tag_name.encode() if tag_name else b'latest'
data = data.replace(b"$SERVICE_TAG", tag_name)
service = yaml.safe_load(data)
# Pop the data not part of service model
service.pop('tool_version', None)
service.pop('file_required', None)
heuristics = service.pop('heuristics', [])
# Validate submission params
for sp in service.get('submission_params', []):
if sp['type'] == 'list' and 'list' not in sp:
return make_api_response(
"",
err=
f"Missing list field for submission param: {sp['name']}",
status_code=400)
if sp['default'] != sp['value']:
return make_api_response(
"",
err=
f"Default and value mismatch for submission param: {sp['name']}",
status_code=400)
# Apply default global configurations (if absent in service configuration)
if not service.get('update_channel'):
service[
'update_channel'] = config.services.preferred_update_channel
if not service.get('docker_config', {}).get('registry_type'):
service['docker_config'][
'registry_type'] = config.services.preferred_registry_type
# Privilege can be set explicitly but also granted to services that don't require the file for analysis
service['privileged'] = service.get(
'privileged', config.services.prefer_service_privileged)
for dep in service.get('dependencies', {}).values():
dep['container']['registry_type'] = dep.get(
'registry_type', config.services.preferred_registry_type)
service['enabled'] = service['enabled'] and enable_allowed
# Load service info
service = Service(service)
# Fix service version, we don't want to see stable if it's a stable container
service.version = service.version.replace("stable", "")
# Save service if it doesn't already exist
if not STORAGE.service.get_if_exists(
f'{service.name}_{service.version}'):
STORAGE.service.save(f'{service.name}_{service.version}', service)
STORAGE.service.commit()
# Save service delta if it doesn't already exist
if not STORAGE.service_delta.get_if_exists(service.name):
STORAGE.service_delta.save(service.name,
{'version': service.version})
STORAGE.service_delta.commit()
# Notify components watching for service config changes
event_sender.send(service.name, {
'operation': Operation.Added,
'name': service.name
})
new_heuristics = []
if heuristics:
plan = STORAGE.heuristic.get_bulk_plan()
for _, heuristic in enumerate(heuristics):
try:
# Append service name to heuristic ID
heuristic[
'heur_id'] = f"{service.name.upper()}.{str(heuristic['heur_id'])}"
# Attack_id field is now a list, make it a list if we receive otherwise
attack_id = heuristic.get('attack_id', None)
if isinstance(attack_id, str):
heuristic['attack_id'] = [attack_id]
heuristic = Heuristic(heuristic)
heuristic_id = heuristic.heur_id
plan.add_upsert_operation(heuristic_id, heuristic)
except Exception:
raise ValueError("Error parsing heuristics")
for item in STORAGE.heuristic.bulk(plan)['items']:
if item['update']['result'] != "noop":
new_heuristics.append(item['update']['_id'])
STORAGE.heuristic.commit()
return make_api_response(
dict(service_name=service.name, new_heuristics=new_heuristics))
except ValueError as e: # Catch errors when building Service or Heuristic model(s)
return make_api_response("", err=str(e), status_code=400)
def add_service(**_):
"""
Add a service using its yaml manifest
Variables:
None
Arguments:
None
Data Block:
<service_manifest.yml content>
Result example:
{ "success": true } # Return true is the service was added
"""
data = request.data
try:
if b"$SERVICE_TAG" in data:
tmp_service = yaml.safe_load(data)
tmp_service.pop('tool_version', None)
tmp_service.pop('file_required', None)
tmp_service.pop('heuristics', [])
tmp_service[
'update_channel'] = config.services.preferred_update_channel
image_name, tag_name, _ = get_latest_tag_for_service(
Service(tmp_service), config, LOGGER)
if tag_name:
tag_name = tag_name.encode()
else:
tag_name = b'latest'
data = data.replace(b"$SERVICE_TAG", tag_name)
service = yaml.safe_load(data)
# Pop the data not part of service model
service.pop('tool_version', None)
service.pop('file_required', None)
heuristics = service.pop('heuristics', [])
# Validate submission params
for sp in service.get('submission_params', []):
if sp['type'] == 'list' and 'list' not in sp:
return make_api_response(
"",
err=
f"Missing list field for submission param: {sp['name']}",
status_code=400)
if sp['default'] != sp['value']:
return make_api_response(
"",
err=
f"Default and value mismatch for submission param: {sp['name']}",
status_code=400)
# Fix update_channel with the system default
service['update_channel'] = config.services.preferred_update_channel
# Load service info
service = Service(service)
# Fix service version, we don't want to see stable if it's a stable container
service.version = service.version.replace("stable", "")
# Save service if it doesn't already exist
if not STORAGE.service.get_if_exists(
f'{service.name}_{service.version}'):
STORAGE.service.save(f'{service.name}_{service.version}', service)
STORAGE.service.commit()
# Save service delta if it doesn't already exist
if not STORAGE.service_delta.get_if_exists(service.name):
STORAGE.service_delta.save(service.name,
{'version': service.version})
STORAGE.service_delta.commit()
new_heuristics = []
if heuristics:
plan = STORAGE.heuristic.get_bulk_plan()
for index, heuristic in enumerate(heuristics):
try:
# Append service name to heuristic ID
heuristic[
'heur_id'] = f"{service.name.upper()}.{str(heuristic['heur_id'])}"
# Attack_id field is now a list, make it a list if we receive otherwise
attack_id = heuristic.get('attack_id', None)
if isinstance(attack_id, str):
heuristic['attack_id'] = [attack_id]
heuristic = Heuristic(heuristic)
heuristic_id = heuristic.heur_id
plan.add_upsert_operation(heuristic_id, heuristic)
except Exception:
raise ValueError("Error parsing heuristics")
for item in STORAGE.heuristic.bulk(plan)['items']:
if item['update']['result'] != "noop":
new_heuristics.append(item['update']['_id'])
STORAGE.heuristic.commit()
return make_api_response(
dict(service_name=service.name, new_heuristics=new_heuristics))
except ValueError as e: # Catch errors when building Service or Heuristic model(s)
return make_api_response("", err=str(e), status_code=400)