class SubmissionParamsDelta(odm.Model):
default = odm.Optional(
odm.Any(),
description=
"Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)"
)
name = odm.Optional(
odm.Keyword(),
description=
"Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)"
)
type = odm.Optional(
odm.Enum(values=['str', 'int', 'list', 'bool']),
description=
"Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)"
)
value = odm.Optional(
odm.Any(),
description=
"Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)"
)
list = odm.Optional(
odm.Any(),
description=
"Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)"
)
hide = odm.Optional(
odm.Boolean(),
description=
"Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)"
)
class SubmissionParams(odm.Model):
default = odm.Any(
description="Default value (must match value in `value` field)")
name = odm.Keyword(description="Name of parameter")
type = odm.Enum(values=['str', 'int', 'list', 'bool'],
description="Type of parameter")
value = odm.Any(
description="Default value (must match value in `default` field)")
list = odm.Optional(odm.Any(),
description="List of values if `type: list`")
hide = odm.Boolean(default=False,
description="Should this parameter be hidden?")
class Service(odm.Model):
# Regexes applied to assemblyline style file type string
accepts = odm.Keyword(store=True, default=DEFAULT_SERVICE_ACCEPTS)
rejects = odm.Optional(
odm.Keyword(store=True, default=DEFAULT_SERVICE_REJECTS))
category = odm.Keyword(store=True,
default="Static Analysis",
copyto="__text__")
config = odm.Mapping(odm.Any(), default={}, index=False, store=False)
description = odm.Text(store=True, default="NA", copyto="__text__")
default_result_classification = odm.ClassificationString(
default=Classification.UNRESTRICTED)
enabled = odm.Boolean(store=True, default=False)
is_external = odm.Boolean(default=False)
licence_count = odm.Integer(default=0)
name = odm.Keyword(store=True, copyto="__text__")
version = odm.Keyword(store=True)
# Should the result cache be disabled for this service
disable_cache = odm.Boolean(default=False)
stage = odm.Keyword(store=True, default="CORE", copyto="__text__")
submission_params: SubmissionParams = odm.List(
odm.Compound(SubmissionParams), index=False, default=[])
timeout = odm.Integer(default=60)
docker_config: DockerConfig = odm.Compound(DockerConfig)
dependencies = odm.Mapping(odm.Compound(DependencyConfig), default={})
update_channel: str = odm.Enum(values=["stable", "rc", "beta", "dev"],
default='stable')
update_config: UpdateConfig = odm.Optional(odm.Compound(UpdateConfig))
class Task(odm.Model):
sid = odm.UUID()
fileinfo: FileInfo = odm.Compound(FileInfo) # File info block
filename = odm.Keyword()
service_name = odm.Keyword()
service_config = odm.Mapping(odm.Any(), default={}) # Service specific parameters
depth = odm.Integer(default=0)
max_files = odm.Integer()
ttl = odm.Integer(default=0)
tags = odm.List(odm.Compound(TagItem), default=[])
temporary_submission_data = odm.List(odm.Compound(DataItem), default=[])
deep_scan = odm.Boolean(default=False)
# Whether the service cache should be ignored during the processing of this task
ignore_cache = odm.Boolean(default=False)
# Priority for processing order
priority = odm.Integer(default=0)
@staticmethod
def make_key(sid, service_name, sha):
return f"{sid}_{service_name}_{sha}"
def key(self):
return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class Task(odm.Model):
sid = odm.UUID()
metadata = odm.FlattenedObject() # Metadata associated to the submission
min_classification = odm.Classification(
) # Minimum classification of the file being scanned
fileinfo: FileInfo = odm.Compound(FileInfo) # File info block
filename = odm.Keyword()
service_name = odm.Keyword()
service_config = odm.Mapping(odm.Any(),
default={}) # Service specific parameters
depth = odm.Integer(default=0)
max_files = odm.Integer()
ttl = odm.Integer(default=0)
tags = odm.List(odm.Compound(TagItem), default=[])
temporary_submission_data = odm.List(odm.Compound(DataItem), default=[])
deep_scan = odm.Boolean(default=False)
# Whether the service cache should be ignored during the processing of this task
ignore_cache = odm.Boolean(default=False)
# Whether the service should ignore the dynamic recursion prevention or not
ignore_dynamic_recursion_prevention = odm.Boolean(default=False)
# Priority for processing order
priority = odm.Integer(default=0)
@staticmethod
def make_key(sid, service_name, sha):
return f"{sid}_{service_name}_{sha}"
def key(self):
return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class ServiceDelta(odm.Model):
accepts = odm.Optional(odm.Keyword(), store=True)
rejects = odm.Optional(odm.Keyword(), store=True)
category = odm.Optional(odm.Keyword(), store=True, copyto="__text__")
config = odm.Optional(odm.Mapping(odm.Any()), index=False)
description = odm.Optional(odm.Text(), store=True, copyto="__text__")
default_result_classification = odm.Optional(odm.ClassificationString())
enabled = odm.Optional(odm.Boolean(), store=True)
is_external = odm.Optional(odm.Boolean())
licence_count = odm.Optional(odm.Integer())
name = odm.Optional(odm.Keyword(), store=True, copyto="__text__")
version = odm.Keyword(store=True)
disable_cache = odm.Optional(odm.Boolean())
stage = odm.Optional(odm.Keyword(), store=True, copyto="__text__")
submission_params = odm.Optional(odm.List(
odm.Compound(SubmissionParamsDelta)),
index=False)
timeout = odm.Optional(odm.Integer())
docker_config: DockerConfigDelta = odm.Optional(
odm.Compound(DockerConfigDelta))
dependencies: DependencyConfigDelta = odm.Mapping(
odm.Compound(DependencyConfigDelta), default={})
update_channel = odm.Optional(
odm.Enum(values=["stable", "rc", "beta", "dev"]))
update_config: UpdateConfigDelta = odm.Optional(
odm.Compound(UpdateConfigDelta))
class Task(odm.Model):
sid = odm.UUID(description="Submission ID")
metadata = odm.FlattenedObject(
description="Metadata associated to the submission")
min_classification = odm.Classification(
description="Minimum classification of the file being scanned")
fileinfo: FileInfo = odm.Compound(FileInfo, description="File info block")
filename = odm.Keyword(description="File name")
service_name = odm.Keyword(description="Service name")
service_config = odm.Mapping(odm.Any(),
default={},
description="Service specific parameters")
depth = odm.Integer(
default=0,
description="File depth relative to initital submitted file")
max_files = odm.Integer(
description="Maximum number of files that submission can have")
ttl = odm.Integer(default=0, description="Task TTL")
tags = odm.List(odm.Compound(TagItem),
default=[],
description="List of tags")
temporary_submission_data = odm.List(
odm.Compound(DataItem),
default=[],
description="Temporary submission data")
deep_scan = odm.Boolean(default=False, description="Perform deep scanning")
ignore_cache = odm.Boolean(
default=False,
description=
"Whether the service cache should be ignored during the processing of this task"
)
ignore_dynamic_recursion_prevention = odm.Boolean(
default=False,
description=
"Whether the service should ignore the dynamic recursion prevention or not"
)
ignore_filtering = odm.Boolean(
default=False, description="Should the service filter it's output?")
priority = odm.Integer(default=0,
description="Priority for processing order")
safelist_config = odm.Compound(
ServiceSafelist,
description=
"Safelisting configuration (as defined in global configuration)",
default={'enabled': False})
@staticmethod
def make_key(sid, service_name, sha):
return f"{sid}_{service_name}_{sha}"
def key(self):
return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class LDAP(odm.Model):
enabled: bool = odm.Boolean()
admin_dn: str = odm.Optional(odm.Keyword())
auto_create: bool = odm.Boolean()
auto_sync: bool = odm.Boolean()
base: str = odm.Keyword()
classification_mappings: Dict[str, str] = odm.Any()
email_field: str = odm.Keyword()
group_lookup_query: str = odm.Keyword()
image_field: str = odm.Keyword()
image_format: str = odm.Keyword()
name_field: str = odm.Keyword()
signature_importer_dn: str = odm.Optional(odm.Keyword())
signature_manager_dn: str = odm.Optional(odm.Keyword())
uid_field: str = odm.Keyword()
uri: str = odm.Keyword()
class UserSettings(odm.Model):
classification = odm.Classification(
default=Classification.UNRESTRICTED,
description="Default submission classification")
deep_scan = odm.Boolean(default=False,
description="Should a deep scan be performed?")
description = odm.Keyword(default="", description="Default description")
download_encoding = odm.Enum(
values=ENCODINGS,
default="cart",
description="Default download encoding when downloading files")
default_zip_password = odm.Text(
default="zippy",
description=
"Default user-defined password for creating password protected ZIPs when downloading files"
)
expand_min_score = odm.Integer(
default=500,
description="Auto-expand section when score bigger then this")
ignore_cache = odm.Boolean(default=False,
description="Ignore service caching?")
ignore_dynamic_recursion_prevention = odm.Boolean(
default=False, description="Ignore dynamic recursion prevention?")
ignore_filtering = odm.Boolean(default=False,
description="Ignore filtering services?")
malicious = odm.Boolean(
default=False,
description="Is the file submitted already known to be malicious?")
priority = odm.Integer(default=1000,
description="Default priority for the submissions")
profile = odm.Boolean(
default=False, description="Should the submission do extra profiling?")
service_spec = odm.Mapping(odm.Mapping(odm.Any()),
default={},
description="Default service specific settings")
services = odm.Compound(ServiceSelection,
default={},
description="Default service selection")
submission_view = odm.Enum(
values=VIEWS,
default="report",
description="Default view for completed submissions")
ttl = odm.Integer(default=30,
description="Default submission TTL, in days")
class ServiceClient(odm.Model):
"""Session data for a service worker.
This is not saved in the datastore, and its not shared with any other components
"""
client_id: str = odm.Keyword() # Session ID of the client
container_id: str = odm.Keyword() # Docker container ID of the client
ip: str = odm.IP() # IP address of the client
service_name: str = odm.Keyword(
) # Name of the service running on the client
service_version: str = odm.Keyword(
) # Version of the service running on the client
service_tool_version: Opt[str] = odm.Optional(
odm.Keyword()) # Tool version of the service running on the client
service_timeout: int = odm.Integer(
) # Timeout of the service running on the client
current: Opt[Current] = odm.Optional(odm.Compound(
Current)) # Info about the current status and task assigned
tasking_counters = odm.Optional(
odm.Any()) # MetricsFactory counters for the service
class LDAP(odm.Model):
enabled: bool = odm.Boolean(description="Should LDAP be enabled or not?")
admin_dn: str = odm.Optional(
odm.Keyword(),
description="DN of the group or the user who will get admin privileges"
)
bind_user: str = odm.Optional(
odm.Keyword(), description="User use to query the LDAP server")
bind_pass: str = odm.Optional(
odm.Keyword(), description="Password used to query the LDAP server")
auto_create: bool = odm.Boolean(
description="Auto-create users if they are missing")
auto_sync: bool = odm.Boolean(
description=
"Should we automatically sync with LDAP server on each login?")
base: str = odm.Keyword(description="Base DN for the users")
classification_mappings: Dict[str, str] = odm.Any(
description="Classification mapping")
email_field: str = odm.Keyword(
description="Name of the field containing the email address")
group_lookup_query: str = odm.Keyword(
description="How the group lookup is queried")
image_field: str = odm.Keyword(
description="Name of the field containing the user's avatar")
image_format: str = odm.Keyword(
description="Type of image used to store the avatar")
name_field: str = odm.Keyword(
description="Name of the field containing the user's name")
signature_importer_dn: str = odm.Optional(
odm.Keyword(),
description=
"DN of the group or the user who will get signature_importer role")
signature_manager_dn: str = odm.Optional(
odm.Keyword(),
description=
"DN of the group or the user who will get signature_manager role")
uid_field: str = odm.Keyword(description="Field name for the UID")
uri: str = odm.Keyword(description="URI to the LDAP server")
class SubmissionParamsDelta(odm.Model):
default = odm.Optional(odm.Any())
name = odm.Optional(odm.Keyword())
type = odm.Optional(odm.Enum(values=['str', 'int', 'list', 'bool']))
value = odm.Optional(odm.Any())
list = odm.Optional(odm.Any())
class ServiceDelta(odm.Model):
accepts = odm.Optional(
odm.Keyword(),
store=True,
description="Refer to:<br>[Service](../service/#service)")
rejects = odm.Optional(
odm.Keyword(),
store=True,
description="Refer to:<br>[Service](../service/#service)")
category = odm.Optional(
odm.Keyword(),
store=True,
copyto="__text__",
description="Refer to:<br>[Service](../service/#service)")
config = odm.Optional(
odm.Mapping(odm.Any()),
index=False,
description="Refer to:<br>[Service](../service/#service)")
description = odm.Optional(
odm.Text(),
store=True,
copyto="__text__",
description="Refer to:<br>[Service](../service/#service)")
default_result_classification = odm.Optional(
odm.ClassificationString(),
description="Refer to:<br>[Service](../service/#service)")
enabled = odm.Optional(
odm.Boolean(),
store=True,
description="Refer to:<br>[Service](../service/#service)")
is_external = odm.Optional(
odm.Boolean(),
description="Refer to:<br>[Service](../service/#service)")
licence_count = odm.Optional(
odm.Integer(),
description="Refer to:<br>[Service](../service/#service)")
max_queue_length = odm.Optional(
odm.Integer(),
description="Refer to:<br>[Service](../service/#service)")
uses_tags: bool = odm.Optional(
odm.Boolean(),
description="Refer to:<br>[Service](../service/#service)")
uses_temp_submission_data: bool = odm.Optional(
odm.Boolean(),
description="Refer to:<br>[Service](../service/#service)")
uses_metadata: bool = odm.Optional(
odm.Boolean(),
description="Refer to:<br>[Service](../service/#service)")
name = odm.Optional(
odm.Keyword(),
store=True,
copyto="__text__",
description="Refer to:<br>[Service](../service/#service)")
version = odm.Keyword(
store=True, description="Refer to:<br>[Service](../service/#service)")
privileged = odm.Optional(
odm.Boolean(),
description="Refer to:<br>[Service](../service/#service)")
disable_cache = odm.Optional(
odm.Boolean(),
description="Refer to:<br>[Service](../service/#service)")
stage = odm.Optional(
odm.Keyword(),
store=True,
copyto="__text__",
description="Refer to:<br>[Service](../service/#service)")
submission_params = odm.Optional(
odm.List(odm.Compound(SubmissionParamsDelta)),
index=False,
description="Refer to:<br>[Service](../service/#service)")
timeout = odm.Optional(
odm.Integer(),
description="Refer to:<br>[Service](../service/#service)")
docker_config: DockerConfigDelta = odm.Optional(
odm.Compound(DockerConfigDelta),
description="Refer to:<br>[Service](../service/#service)")
dependencies: DependencyConfigDelta = odm.Mapping(
odm.Compound(DependencyConfigDelta),
default={},
description="Refer to:<br>[Service](../service/#service)")
update_channel = odm.Optional(
odm.Enum(values=["stable", "rc", "beta", "dev"]),
description="Refer to:<br>[Service](../service/#service)")
update_config: UpdateConfigDelta = odm.Optional(
odm.Compound(UpdateConfigDelta),
description="Refer to:<br>[Service](../service/#service)")
class DataItem(odm.Model):
name = odm.Keyword()
value = odm.Any()
class Test(odm.Model):
number = odm.Integer()
other = odm.Any(index=False)
class DispatcherCommandMessage(odm.Model):
kind: str = odm.Enum(values=list(MESSAGE_CLASSES.keys()))
payload_data = odm.Any()
def payload(self):
return MESSAGE_CLASSES[self.kind](self.payload_data)
class SubmissionParams(odm.Model):
classification = odm.Classification(
default=Classification.UNRESTRICTED,
description="Original classification of the submission")
deep_scan = odm.Boolean(default=False,
description="Should a deep scan be performed?")
description = odm.Text(store=True,
copyto="__text__",
description="Description of the submission")
generate_alert = odm.Boolean(
default=False, description="Should this submission generate an alert?")
groups = odm.List(odm.Keyword(),
default=["USERS"],
description="List of groups related to this scan")
ignore_cache = odm.Boolean(
default=False, description="Ignore the cached service results?")
ignore_dynamic_recursion_prevention = odm.Boolean(
default=False,
description="Should we ignore dynamic recursion prevention?")
ignore_filtering = odm.Boolean(
default=False, description="Should we ignore filtering services?")
ignore_size = odm.Boolean(default=False,
description="Ignore the file size limits?")
never_drop = odm.Boolean(
default=False, description="Exempt from being dropped by ingester?")
malicious = odm.Boolean(
default=False,
description="Is the file submitted already known to be malicious?")
max_extracted = odm.Integer(default=500,
description="Max number of extracted files")
max_supplementary = odm.Integer(
default=500, description="Max number of supplementary files")
priority = odm.Integer(default=1000, description="Priority of the scan")
profile = odm.Boolean(
default=False, description="Should the submission do extra profiling?")
psid = odm.Optional(odm.UUID(), description="Parent submission ID")
quota_item = odm.Boolean(
default=False, description="Does this submission count against quota?")
services = odm.Compound(ServiceSelection,
default={},
description="Service selection")
service_spec = odm.Mapping(odm.Mapping(odm.Any()),
default={},
index=False,
store=False,
description="Service-specific parameters")
submitter = odm.Keyword(store=True,
copyto="__text__",
description="User who submitted the file")
ttl = odm.Integer(default=0,
description="Time, in days, to live for this submission")
type = odm.Keyword(default="USER", description="Type of submission")
initial_data = odm.Optional(
odm.Text(index=False),
description="Initialization for temporary submission data")
def get_hashing_keys(self):
"""Get the sections of the submission parameters that should be used in result hashes."""
data = self.as_primitives()
return {k: v for k, v in data.items() if k in _KEY_HASHED_FIELDS}
def create_filescore_key(self, sha256, services: list = None):
"""This is the key used to store the final score of a submission for fast lookup.
This lookup is one of the methods used to check for duplication in ingestion process,
so this key is fairly sensitive.
"""
# TODO do we need this version thing still be here?
# One up this if the cache is ever messed up and we
# need to quickly invalidate all old cache entries.
version = 0
if services is None:
services = self.services.selected
data = self.get_hashing_keys()
data['service_spec'] = sorted(
(key, sorted(values.items()))
for key, values in self.service_spec.items())
data['sha256'] = sha256
data['services'] = [str(x) for x in services]
s = ', '.join([f"{k}: {data[k]}" for k in sorted(data.keys())])
return 'v'.join(
[str(hashlib.md5(s.encode()).hexdigest()),
str(version)])
class Service(odm.Model):
# Regexes applied to assemblyline style file type string
accepts = odm.Keyword(
store=True,
default=DEFAULT_SERVICE_ACCEPTS,
description="Regex to accept files as identified by Assemblyline")
rejects = odm.Optional(
odm.Keyword(store=True, default=DEFAULT_SERVICE_REJECTS),
description="Regex to reject files as identified by Assemblyline")
category = odm.Keyword(
store=True,
default="Static Analysis",
copyto="__text__",
description="Which category does this service belong to?")
config = odm.Mapping(odm.Any(),
default={},
index=False,
store=False,
description="Service Configuration")
description = odm.Text(store=True,
default="NA",
copyto="__text__",
description="Description of service")
default_result_classification = odm.ClassificationString(
default=Classification.UNRESTRICTED,
description="Default classification assigned to service results")
enabled: bool = odm.Boolean(
store=True,
default=False,
description="Is the service enabled (by default)?")
is_external: bool = odm.Boolean(
default=False,
description=
"Does this service perform analysis outside of Assemblyline?")
licence_count: int = odm.Integer(
default=0,
description="How many licences is the service allowed to use?")
max_queue_length: int = odm.Integer(
default=0,
description=
"If more than this many jobs are queued for this service drop those over this limit. 0 is unlimited."
)
uses_tags: bool = odm.Boolean(
default=False,
description=
"Does this service use tags from other services for analysis?")
uses_temp_submission_data: bool = odm.Boolean(
default=False,
description=
"Does this service use temp data from other services for analysis?")
uses_metadata: bool = odm.Boolean(
default=False,
description="Does this service use submission metadata for analysis?")
name: str = odm.Keyword(store=True,
copyto="__text__",
description="Name of service")
version = odm.Keyword(store=True, description="Version of service")
privileged = odm.Boolean(
default=False,
description=
"Should the service be able to talk to core infrastructure or just service-server for tasking?"
)
disable_cache = odm.Boolean(
default=False,
description="Should the result cache be disabled for this service?")
stage = odm.Keyword(
store=True,
default="CORE",
copyto="__text__",
description="Which execution stage does this service run in?")
submission_params: SubmissionParams = odm.List(
odm.Compound(SubmissionParams),
index=False,
default=[],
description="Submission parameters of service")
timeout: int = odm.Integer(default=60,
description="Service task timeout, in seconds")
docker_config: DockerConfig = odm.Compound(
DockerConfig, description="Docker configuration for service")
dependencies: dict[str, DependencyConfig] = odm.Mapping(
odm.Compound(DependencyConfig),
default={},
description="Dependency configuration for service")
update_channel: str = odm.Enum(
values=["stable", "rc", "beta", "dev"],
default='stable',
description="What channel to watch for service updates?")
update_config: UpdateConfig = odm.Optional(
odm.Compound(UpdateConfig),
description="Update configuration for fetching external resources")
class DispatcherCommandMessage(odm.Model):
kind: str = odm.Enum(values=list(MESSAGE_CLASSES.keys()), description="Kind of message")
payload_data = odm.Any(description="Message payload")
def payload(self):
return MESSAGE_CLASSES[self.kind](self.payload_data)
class SubmissionParams(odm.Model):
default = odm.Any()
name = odm.Keyword()
type = odm.Enum(values=['str', 'int', 'list', 'bool'])
value = odm.Any()
