def test_edit_service(datastore, login_session):
_, session, host = login_session
ds = datastore
delta_data = ds.service_delta.search("id:*", rows=100, as_obj=False)
svc_data = ds.service.search("id:*", rows=100, as_obj=False)
service = random.choice(list(TEMP_SERVICES.keys()))
service_data = Service({
"name": service,
"enabled": True,
"category": TEMP_SERVICES[service][0],
"stage": TEMP_SERVICES[service][1],
"version": "3.3.0",
"docker_config": {
"image": f"cccs/alsvc_{service.lower()}:latest",
},
}).as_primitives()
resp = get_api_data(session, f"{host}/api/v4/service/{service}/", method="POST", data=json.dumps(service_data))
assert resp['success']
ds.service_delta.commit()
ds.service.commit()
new_delta_data = ds.service_delta.search("id:*", rows=100, as_obj=False)
new_svc_data = ds.service.search("id:*", rows=100, as_obj=False)
assert new_delta_data != delta_data
assert new_svc_data == svc_data
for svc in new_delta_data['items']:
if svc['id'] == service:
assert svc['version'] == '3.3.0'
else:
assert svc['version'] == '4.0.0'
def load_service_manifest(self, return_heuristics=False) -> Union[None, Dict]:
service_manifest_yml = os.path.join(os.getcwd(), 'service_manifest.yml')
if os.path.exists(service_manifest_yml):
with open(service_manifest_yml) as yml_fh:
service_manifest_data = yaml.safe_load(yml_fh.read())
heuristics = service_manifest_data.get('heuristics', None)
# Pop the 'extra' data from the service manifest
for x in ['file_required', 'tool_version', 'heuristics']:
service_manifest_data.pop(x, None)
# Validate the service manifest
try:
self.service = Service(service_manifest_data)
except Exception as e:
LOG.error(f"Invalid service manifest: {str(e)}")
service_config = {}
if service_manifest_data:
service_config = service_manifest_data.get('config', {})
self.submission_params = {x['name']: x['default'] for x in service_manifest_data.get('submission_params', [])}
self.service = self.service_class(config=service_config)
if return_heuristics:
return heuristics
else:
raise Exception("Service manifest YAML file not found in root folder of service.")
def suricata_change_config(service_conf, login_session):
_, session, host = login_session
service_data = Service(service_conf).as_primitives()
resp = get_api_data(session,
f"{host}/api/v4/service/Suricata/",
method="POST",
data=json.dumps(service_data))
return resp['success']
def test_service_delta_to_service_model():
try:
data = random_model_obj(ServiceDelta).as_primitives()
Service(data).as_primitives()
except (ValueError, TypeError, KeyError):
pytest.fail(
"Could not use a 'ServiceDelta' object to create a 'Service' object."
)
def suricata_init_config(datastore, login_session):
_, session, host = login_session
service_conf = {
"name": "Suricata",
"enabled": True,
"category": "Networking",
"stage": "CORE",
"version": "4.0.0",
"docker_config": {
"image": f"cccs/assemblyline-service-suricata:4.0.0.dev69",
},
"update_config": {
"generates_signatures":
True,
"method":
"run",
"run_options": {
"allow_internet_access":
True,
"command": ["python", "-m", "suricata_.suricata_updater"],
"image":
"${REGISTRY}cccs/assemblyline-service-suricata:4.0.0.dev69"
},
"sources": [{
"name":
"old",
"pattern":
".*\\.rules",
"uri":
"https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz"
}, {
"name":
"old with space",
"pattern":
".*\\.rules",
"uri":
"https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz"
}],
"update_interval_seconds":
60 # Quarter-day (every 6 hours)
}
}
service_data = Service(service_conf).as_primitives()
resp = get_api_data(session,
f"{host}/api/v4/service/Suricata/",
method="POST",
data=json.dumps(service_data))
if resp['success']:
datastore.service_delta.commit()
datastore.service.commit()
delta_sources = datastore.get_service_with_delta(
"Suricata", as_obj=False)['update_config']['sources']
passed = delta_sources[0]['name'] == "old" and delta_sources[1][
'name'] == "old_with_space"
return passed, service_conf
return False
def create_services(ds: AssemblylineDatastore, log=None, limit=None):
if not limit:
limit = len(SERVICES)
for svc_name, svc in list(SERVICES.items())[:limit]:
service_data = {
"name": svc_name,
"enabled": True,
"category": svc[0],
"stage": svc[1],
"version": "3.3.0",
"docker_config": {
"image": f"cccs/alsvc_{svc_name.lower()}:latest",
},
}
if random.choice([True, False]):
service_data['update_config'] = {
"method": "run",
"sources": [random_model_obj(UpdateSource)],
"update_interval_seconds": 600,
"generates_signatures": True
}
service_data = Service(service_data)
# Save a v3 service
ds.service.save(f"{service_data.name}_{service_data.version}",
service_data)
# Save the same service as v4
service_data.version = "4.0.0"
ds.service.save(f"{service_data.name}_{service_data.version}",
service_data)
# Save the default delta entry
ds.service_delta.save(service_data.name,
{"version": service_data.version})
if log:
log.info(f'\t{svc_name}')
ds.service_delta.commit()
ds.service.commit()
def get_service_attributes() -> Service:
service_manifest_data = get_service_manifest()
# Pop the 'extra' data from the service manifest
for x in ['file_required', 'tool_version', 'heuristics']:
service_manifest_data.pop(x, None)
try:
service_attributes = Service(service_manifest_data)
except ValueError as e:
raise ValueError(f"Service manifest yaml contains invalid parameter(s): {str(e)}")
return service_attributes
def dummy_service(name, stage, category='static', accepts='', rejects=None):
return Service({
'name': name,
'stage': stage,
'category': category,
'accepts': accepts,
'rejects': rejects,
'version': '0',
'enabled': True,
'timeout': 2,
'docker_config': {
'image': 'somefakedockerimage:latest'
}
})
def get_service_with_delta(self, service_name, version=None, as_obj=True):
svc = self.ds.service_delta.get(service_name)
if svc is None:
return svc
if version is not None:
svc.version = version
svc_version_data = self.ds.service.get(f"{service_name}_{svc.version}")
if svc_version_data is None:
return svc_version_data
svc_version_data = recursive_update(
svc_version_data.as_primitives(strip_null=True),
svc.as_primitives(strip_null=True))
if as_obj:
return Service(svc_version_data)
else:
return svc_version_data
def list_all_services(self,
as_obj=True,
full=False) -> Union[List[dict], List[Service]]:
"""
:param as_obj: Return ODM objects rather than dicts
:param full: If true retrieve all the fields of the service object, otherwise only
fields returned by search are given.
"""
items = list(self.ds.service_delta.stream_search("id:*", as_obj=False))
if full:
service_data = self.ds.service.multiget(
[f"{item['id']}_{item['version']}" for item in items],
as_dictionary=False)
service_delta = self.ds.service_delta.multiget(
[item['id'] for item in items], as_dictionary=False)
services = [
recursive_update(data.as_primitives(strip_null=True),
delta.as_primitives(strip_null=True))
for data, delta in zip(service_data, service_delta)
]
else:
services_versions = {
item['id']: item
for item in self.ds.service.stream_search("id:*", as_obj=False)
}
services = [
recursive_update(
services_versions[f"{item['id']}_{item['version']}"], item)
for item in items
if f"{item['id']}_{item['version']}" in services_versions
]
if as_obj:
mask = None
if not full and services:
mask = services[0].keys()
return [Service(s, mask=mask) for s in services]
else:
return services
def dummy_service(name,
stage,
category='static',
accepts='',
rejects=None,
docid=None,
extra_data=False):
return Service(
{
'name': name,
'stage': stage,
'category': category,
'accepts': accepts,
'uses_temp_submission_data': extra_data,
'uses_tags': extra_data,
'rejects': rejects,
'version': '0',
'enabled': True,
'timeout': 2,
'docker_config': {
'image': 'somefakedockerimage:latest'
}
},
docid=docid)
def register_service(self, service_data, log_prefix=""):
keep_alive = True
try:
# Get heuristics list
heuristics = service_data.pop('heuristics', None)
# Patch update_channel, registry_type before Service registration object creation
service_data['update_channel'] = service_data.get(
'update_channel',
self.config.services.preferred_update_channel)
service_data['docker_config']['registry_type'] = service_data['docker_config'] \
.get('registry_type', self.config.services.preferred_registry_type)
service_data['privileged'] = service_data.get(
'privileged', self.config.services.prefer_service_privileged)
for dep in service_data.get('dependencies', {}).values():
dep['container']['registry_type'] = dep.get(
'registry_type',
self.config.services.preferred_registry_type)
# Pop unused registration service_data
for x in ['file_required', 'tool_version']:
service_data.pop(x, None)
# Create Service registration object
service = Service(service_data)
# Fix service version, we don't need to see the stable label
service.version = service.version.replace('stable', '')
# Save service if it doesn't already exist
if not self.datastore.service.exists(
f'{service.name}_{service.version}'):
self.datastore.service.save(
f'{service.name}_{service.version}', service)
self.datastore.service.commit()
self.log.info(f"{log_prefix}{service.name} registered")
keep_alive = False
# Save service delta if it doesn't already exist
if not self.datastore.service_delta.exists(service.name):
self.datastore.service_delta.save(service.name,
{'version': service.version})
self.datastore.service_delta.commit()
self.log.info(f"{log_prefix}{service.name} "
f"version ({service.version}) registered")
new_heuristics = []
if heuristics:
plan = self.datastore.heuristic.get_bulk_plan()
for index, heuristic in enumerate(heuristics):
heuristic_id = f'#{index}' # Set heuristic id to it's position in the list for logging purposes
try:
# Append service name to heuristic ID
heuristic[
'heur_id'] = f"{service.name.upper()}.{str(heuristic['heur_id'])}"
# Attack_id field is now a list, make it a list if we receive otherwise
attack_id = heuristic.get('attack_id', None)
if isinstance(attack_id, str):
heuristic['attack_id'] = [attack_id]
heuristic = Heuristic(heuristic)
heuristic_id = heuristic.heur_id
plan.add_upsert_operation(heuristic_id, heuristic)
except Exception as e:
msg = f"{service.name} has an invalid heuristic ({heuristic_id}): {str(e)}"
self.log.exception(f"{log_prefix}{msg}")
raise ValueError(msg)
for item in self.datastore.heuristic.bulk(plan)['items']:
if item['update']['result'] != "noop":
new_heuristics.append(item['update']['_id'])
self.log.info(
f"{log_prefix}{service.name} "
f"heuristic {item['update']['_id']}: {item['update']['result'].upper()}"
)
self.datastore.heuristic.commit()
service_config = self.datastore.get_service_with_delta(
service.name, as_obj=False)
# Notify components watching for service config changes
self.event_sender.send(service.name, {
'operation': Operation.Added,
'name': service.name
})
except ValueError as e: # Catch errors when building Service or Heuristic model(s)
raise e
return dict(keep_alive=keep_alive,
new_heuristics=new_heuristics,
service_config=service_config or dict())
def add_service(**_):
"""
Add a service using its yaml manifest
Variables:
None
Arguments:
None
Data Block:
<service_manifest.yml content>
Result example:
{ "success": true } # Return true is the service was added
"""
data = request.data
enable_allowed = True
try:
if b"$SERVICE_TAG" in data:
tmp_service = yaml.safe_load(data)
tmp_service.pop('tool_version', None)
tmp_service.pop('file_required', None)
tmp_service.pop('heuristics', [])
# Apply global preferences, if missing, to get the appropriate container image tags
tmp_service['update_channel'] = tmp_service.get(
'update_channel', config.services.preferred_update_channel)
tmp_service['docker_config']['registry_type'] = tmp_service['docker_config'] \
.get('registry_type', config.services.preferred_registry_type)
_, tag_name, _ = get_latest_tag_for_service(
Service(tmp_service), config, LOGGER)
enable_allowed = bool(tag_name)
tag_name = tag_name.encode() if tag_name else b'latest'
data = data.replace(b"$SERVICE_TAG", tag_name)
service = yaml.safe_load(data)
# Pop the data not part of service model
service.pop('tool_version', None)
service.pop('file_required', None)
heuristics = service.pop('heuristics', [])
# Validate submission params
for sp in service.get('submission_params', []):
if sp['type'] == 'list' and 'list' not in sp:
return make_api_response(
"",
err=
f"Missing list field for submission param: {sp['name']}",
status_code=400)
if sp['default'] != sp['value']:
return make_api_response(
"",
err=
f"Default and value mismatch for submission param: {sp['name']}",
status_code=400)
# Apply default global configurations (if absent in service configuration)
if not service.get('update_channel'):
service[
'update_channel'] = config.services.preferred_update_channel
if not service.get('docker_config', {}).get('registry_type'):
service['docker_config'][
'registry_type'] = config.services.preferred_registry_type
# Privilege can be set explicitly but also granted to services that don't require the file for analysis
service['privileged'] = service.get(
'privileged', config.services.prefer_service_privileged)
for dep in service.get('dependencies', {}).values():
dep['container']['registry_type'] = dep.get(
'registry_type', config.services.preferred_registry_type)
service['enabled'] = service['enabled'] and enable_allowed
# Load service info
service = Service(service)
# Fix service version, we don't want to see stable if it's a stable container
service.version = service.version.replace("stable", "")
# Save service if it doesn't already exist
if not STORAGE.service.get_if_exists(
f'{service.name}_{service.version}'):
STORAGE.service.save(f'{service.name}_{service.version}', service)
STORAGE.service.commit()
# Save service delta if it doesn't already exist
if not STORAGE.service_delta.get_if_exists(service.name):
STORAGE.service_delta.save(service.name,
{'version': service.version})
STORAGE.service_delta.commit()
# Notify components watching for service config changes
event_sender.send(service.name, {
'operation': Operation.Added,
'name': service.name
})
new_heuristics = []
if heuristics:
plan = STORAGE.heuristic.get_bulk_plan()
for _, heuristic in enumerate(heuristics):
try:
# Append service name to heuristic ID
heuristic[
'heur_id'] = f"{service.name.upper()}.{str(heuristic['heur_id'])}"
# Attack_id field is now a list, make it a list if we receive otherwise
attack_id = heuristic.get('attack_id', None)
if isinstance(attack_id, str):
heuristic['attack_id'] = [attack_id]
heuristic = Heuristic(heuristic)
heuristic_id = heuristic.heur_id
plan.add_upsert_operation(heuristic_id, heuristic)
except Exception:
raise ValueError("Error parsing heuristics")
for item in STORAGE.heuristic.bulk(plan)['items']:
if item['update']['result'] != "noop":
new_heuristics.append(item['update']['_id'])
STORAGE.heuristic.commit()
return make_api_response(
dict(service_name=service.name, new_heuristics=new_heuristics))
except ValueError as e: # Catch errors when building Service or Heuristic model(s)
return make_api_response("", err=str(e), status_code=400)
def add_service(**_):
"""
Add a service using its yaml manifest
Variables:
None
Arguments:
None
Data Block:
<service_manifest.yml content>
Result example:
{ "success": true } # Return true is the service was added
"""
data = request.data
try:
if b"$SERVICE_TAG" in data:
tmp_service = yaml.safe_load(data)
tmp_service.pop('tool_version', None)
tmp_service.pop('file_required', None)
tmp_service.pop('heuristics', [])
tmp_service[
'update_channel'] = config.services.preferred_update_channel
image_name, tag_name, _ = get_latest_tag_for_service(
Service(tmp_service), config, LOGGER)
if tag_name:
tag_name = tag_name.encode()
else:
tag_name = b'latest'
data = data.replace(b"$SERVICE_TAG", tag_name)
service = yaml.safe_load(data)
# Pop the data not part of service model
service.pop('tool_version', None)
service.pop('file_required', None)
heuristics = service.pop('heuristics', [])
# Validate submission params
for sp in service.get('submission_params', []):
if sp['type'] == 'list' and 'list' not in sp:
return make_api_response(
"",
err=
f"Missing list field for submission param: {sp['name']}",
status_code=400)
if sp['default'] != sp['value']:
return make_api_response(
"",
err=
f"Default and value mismatch for submission param: {sp['name']}",
status_code=400)
# Fix update_channel with the system default
service['update_channel'] = config.services.preferred_update_channel
# Load service info
service = Service(service)
# Fix service version, we don't want to see stable if it's a stable container
service.version = service.version.replace("stable", "")
# Save service if it doesn't already exist
if not STORAGE.service.get_if_exists(
f'{service.name}_{service.version}'):
STORAGE.service.save(f'{service.name}_{service.version}', service)
STORAGE.service.commit()
# Save service delta if it doesn't already exist
if not STORAGE.service_delta.get_if_exists(service.name):
STORAGE.service_delta.save(service.name,
{'version': service.version})
STORAGE.service_delta.commit()
new_heuristics = []
if heuristics:
plan = STORAGE.heuristic.get_bulk_plan()
for index, heuristic in enumerate(heuristics):
try:
# Append service name to heuristic ID
heuristic[
'heur_id'] = f"{service.name.upper()}.{str(heuristic['heur_id'])}"
# Attack_id field is now a list, make it a list if we receive otherwise
attack_id = heuristic.get('attack_id', None)
if isinstance(attack_id, str):
heuristic['attack_id'] = [attack_id]
heuristic = Heuristic(heuristic)
heuristic_id = heuristic.heur_id
plan.add_upsert_operation(heuristic_id, heuristic)
except Exception:
raise ValueError("Error parsing heuristics")
for item in STORAGE.heuristic.bulk(plan)['items']:
if item['update']['result'] != "noop":
new_heuristics.append(item['update']['_id'])
STORAGE.heuristic.commit()
return make_api_response(
dict(service_name=service.name, new_heuristics=new_heuristics))
except ValueError as e: # Catch errors when building Service or Heuristic model(s)
return make_api_response("", err=str(e), status_code=400)