class Service(odm.Model):
# Regexes applied to assemblyline style file type string
accepts = odm.Keyword(store=True, default=DEFAULT_SERVICE_ACCEPTS)
rejects = odm.Optional(
odm.Keyword(store=True, default=DEFAULT_SERVICE_REJECTS))
category = odm.Keyword(store=True,
default="Static Analysis",
copyto="__text__")
config = odm.Mapping(odm.Any(), default={}, index=False, store=False)
description = odm.Text(store=True, default="NA", copyto="__text__")
default_result_classification = odm.ClassificationString(
default=Classification.UNRESTRICTED)
enabled = odm.Boolean(store=True, default=False)
is_external = odm.Boolean(default=False)
licence_count = odm.Integer(default=0)
name = odm.Keyword(store=True, copyto="__text__")
version = odm.Keyword(store=True)
# Should the result cache be disabled for this service
disable_cache = odm.Boolean(default=False)
stage = odm.Keyword(store=True, default="CORE", copyto="__text__")
submission_params: SubmissionParams = odm.List(
odm.Compound(SubmissionParams), index=False, default=[])
timeout = odm.Integer(default=60)
docker_config: DockerConfig = odm.Compound(DockerConfig)
dependencies = odm.Mapping(odm.Compound(DependencyConfig), default={})
update_channel: str = odm.Enum(values=["stable", "rc", "beta", "dev"],
default='stable')
update_config: UpdateConfig = odm.Optional(odm.Compound(UpdateConfig))
class ServiceDelta(odm.Model):
accepts = odm.Optional(odm.Keyword(), store=True)
rejects = odm.Optional(odm.Keyword(), store=True)
category = odm.Optional(odm.Keyword(), store=True, copyto="__text__")
config = odm.Optional(odm.Mapping(odm.Any()), index=False)
description = odm.Optional(odm.Text(), store=True, copyto="__text__")
default_result_classification = odm.Optional(odm.ClassificationString())
enabled = odm.Optional(odm.Boolean(), store=True)
is_external = odm.Optional(odm.Boolean())
licence_count = odm.Optional(odm.Integer())
name = odm.Optional(odm.Keyword(), store=True, copyto="__text__")
version = odm.Keyword(store=True)
disable_cache = odm.Optional(odm.Boolean())
stage = odm.Optional(odm.Keyword(), store=True, copyto="__text__")
submission_params = odm.Optional(odm.List(
odm.Compound(SubmissionParamsDelta)),
index=False)
timeout = odm.Optional(odm.Integer())
docker_config: DockerConfigDelta = odm.Optional(
odm.Compound(DockerConfigDelta))
dependencies: DependencyConfigDelta = odm.Mapping(
odm.Compound(DependencyConfigDelta), default={})
update_channel = odm.Optional(
odm.Enum(values=["stable", "rc", "beta", "dev"]))
update_config: UpdateConfigDelta = odm.Optional(
odm.Compound(UpdateConfigDelta))
class User(odm.Model):
agrees_with_tos = odm.Optional(
odm.Date(index=False, store=False),
description="Date the user agree with terms of service")
api_quota = odm.Integer(
default=10,
store=False,
description="Maximum number of concurrent API requests")
apikeys = odm.Mapping(odm.Compound(ApiKey),
default={},
index=False,
store=False,
description="Mapping of API keys")
apps = odm.Mapping(odm.Compound(Apps),
default={},
index=False,
store=False,
description="Applications with access to the account")
can_impersonate = odm.Boolean(
default=False,
index=False,
store=False,
description="Allowed to query on behalf of others?")
classification = odm.Classification(
is_user_classification=True,
copyto="__text__",
default=Classification.UNRESTRICTED,
description="Maximum classification for the user")
dn = odm.Optional(odm.Keyword(store=False, copyto="__text__"),
description="User's LDAP DN")
email = odm.Optional(odm.Email(copyto="__text__"),
description="User's email address")
groups = odm.List(odm.Keyword(),
copyto="__text__",
default=["USERS"],
description="List of groups the user submits to")
is_active = odm.Boolean(default=True, description="Is the user active?")
name = odm.Keyword(copyto="__text__", description="Full name of the user")
otp_sk = odm.Optional(
odm.Keyword(index=False, store=False),
description="Secret key to generate one time passwords")
password = odm.Keyword(index=False,
store=False,
description="BCrypt hash of the user's password")
submission_quota = odm.Integer(
default=5,
store=False,
description="Maximum number of concurrent submissions")
type = odm.List(odm.Enum(values=USER_TYPES),
default=['user'],
description="Type of user")
security_tokens = odm.Mapping(odm.Keyword(),
index=False,
store=False,
default={},
description="Map of security tokens")
uname = odm.Keyword(copyto="__text__", description="Username")
class UI(odm.Model):
# Allow user to tell in advance the system that a file is malicious
allow_malicious_hinting: bool = odm.Boolean()
# Allow to user to download raw files
allow_raw_downloads: bool = odm.Boolean()
# Allow file submissions via url
allow_url_submissions: bool = odm.Boolean()
# Should API calls be audited and saved to a separate log file?
audit: bool = odm.Boolean()
# Banner message display on the main page (format: {<language_code>: message})
banner: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()))
# Banner message display on the main page (format: {<language_code>: message})
banner_level: str = odm.Enum(
values=["info", "warning", "success", "error"])
# Turn on debugging
debug: bool = odm.Boolean()
# Which encoding will be used
download_encoding = odm.Enum(values=["raw", "cart"])
# Assemblyline admins email address
email: str = odm.Optional(odm.Email())
# Enforce the user's quotas
enforce_quota: bool = odm.Boolean()
# Fully qualified domain name to use for the 2-factor authentication validation
fqdn: str = odm.Text()
# Maximum priority for ingest API
ingest_max_priority: int = odm.Integer()
# Turn on read only mode in the UI
read_only: bool = odm.Boolean()
# Offset of the read only mode for all paging and searches
read_only_offset: str = odm.Keyword(default="")
# Flask secret key to store cookies and stuff
secret_key: str = odm.Keyword()
# Duration of the user session before the user has to login again
session_duration: int = odm.Integer()
# Statistics configuration
statistics: Statistics = odm.Compound(Statistics,
default=DEFAULT_STATISTICS)
# Terms of service
tos: str = odm.Optional(odm.Text())
# Lock out user after accepting the terms of service
tos_lockout: bool = odm.Boolean()
# List of admins to notify when a user gets locked out
tos_lockout_notify: bool = odm.Optional(odm.List(odm.Keyword()))
# Headers that will be used by the url_download method
url_submission_headers: Dict[str, str] = odm.Optional(
odm.Mapping(odm.Keyword()))
# Proxy that will be used by the url_download method
url_submission_proxies: Dict[str, str] = odm.Optional(
odm.Mapping(odm.Keyword()))
# Validate if the session ip matches the ip the session was created from
validate_session_ip: bool = odm.Boolean()
# Validate if the session useragent matches the useragent the session was created with
validate_session_useragent: bool = odm.Boolean()
class BaseTestModel(odm.Model):
classification = odm.Classification(default="UNRESTRICTED", yml_config=yml_config)
flavour = odm.Text(copyto='features', default="EMPTY")
height = odm.Integer()
no_store = odm.Optional(odm.Keyword(store=False))
no_index = odm.Optional(odm.Keyword(index=False, store=False))
dots = odm.Mapping(odm.Compound(Position), default={})
birthday = odm.Date()
tags = odm.List(odm.Enum({'silly', 'cats', '10'}), default=[], copyto='features')
size = odm.Compound(MeasurementModel, default={'depth': 100, 'width': 100})
features = odm.List(odm.Text(), default=[])
metadata = odm.Mapping(odm.Text(), default={})
things = odm.List(odm.Compound(ThingsModel), default=[])
class NetworkHTTP(odm.Model):
connection_details = odm.Compound(
NetworkConnection,
description="The low-level details of the HTTP request")
request_uri = odm.URI(description="The URI requested")
request_headers = odm.Mapping(
odm.Json(), description="Headers included in the request")
request_body = odm.Optional(odm.Text(),
description="The body of the request")
request_method = odm.Enum(
[
# Standard HTTP methods
"GET",
"POST",
"PUT",
"DELETE",
"HEAD",
"CONNECT",
"OPTIONS",
"TRACE",
"PATCH",
# WebDAV HTTP methods
"BCOPY",
"BDELETE",
"BMOVE",
"BPROPFIND",
"BPROPPATCH",
"COPY",
"DELETE",
"LOCK",
"MKCOL",
"MOVE",
"NOTIFY",
"POLL",
"PROPFIND",
"PROPPATCH",
"SEARCH",
"SUBSCRIBE",
"UNLOCK",
"UNSUBSCRIBE",
"X-MS-ENUMATTS"
],
description="The method of the request")
response_headers = odm.Mapping(
odm.Json(), description="Headers included in the response")
response_status_code = odm.Optional(
odm.Integer(), description="The status code of the response")
response_body = odm.Optional(odm.Text(),
description="The body of the response")
class UserSettings(odm.Model): # User's default settings
classification = odm.Classification(default=Classification.UNRESTRICTED
) # Default submission classification
deep_scan = odm.Boolean(default=False) # Should a deep scan be performed
description = odm.Keyword(default="") # Default description
download_encoding = odm.Enum(
values=ENCODINGS,
default="cart") # Default download encoding when downloading files
expand_min_score = odm.Integer(
default=500) # Auto-expand section when score bigger then this
ignore_cache = odm.Boolean(default=False) # Ignore service caching
ignore_dynamic_recursion_prevention = odm.Boolean(
default=False) # Ignore dynamic recursion prevention
ignore_filtering = odm.Boolean(default=False) # Ignore filtering services
priority = odm.Integer(
default=1000) # Default priority for the submissions
profile = odm.Boolean(
default=False) # Should the submission do extra profiling
service_spec = odm.Mapping(odm.Keyword(),
default={}) # Default service specific settings
services = odm.Compound(ServiceSelection,
default={}) # Default service selection
submission_view = odm.Enum(
values=VIEWS,
default="report") # Default view for completed submissions
ttl = odm.Integer(default=0) # Default submission Time to Live (days)
class OAuthProvider(odm.Model):
auto_create: str = odm.Boolean(default=True)
auto_sync: str = odm.Boolean(default=False)
auto_properties: List[OAuthAutoProperty] = odm.List(
odm.Compound(OAuthAutoProperty), default=[])
app_provider: AppProvider = odm.Optional(odm.Compound(AppProvider))
uid_randomize: str = odm.Boolean(default=False)
uid_randomize_digits: str = odm.Integer(default=0)
uid_randomize_delimiter: str = odm.Keyword(default="-")
uid_regex: str = odm.Optional(odm.Keyword())
uid_format: str = odm.Optional(odm.Keyword())
client_id: str = odm.Optional(odm.Keyword())
client_secret: str = odm.Optional(odm.Keyword())
request_token_url: str = odm.Optional(odm.Keyword())
request_token_params: str = odm.Optional(odm.Keyword())
access_token_url: str = odm.Optional(odm.Keyword())
access_token_params: str = odm.Optional(odm.Keyword())
authorize_url: str = odm.Optional(odm.Keyword())
authorize_params: str = odm.Optional(odm.Keyword())
api_base_url: str = odm.Optional(odm.Keyword())
client_kwargs: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()))
jwks_uri: str = odm.Optional(odm.Keyword())
uid_field: str = odm.Optional(odm.Keyword())
user_get: str = odm.Optional(odm.Keyword())
user_groups: str = odm.Optional(odm.Keyword())
user_groups_data_field: str = odm.Optional(odm.Keyword())
user_groups_name_field: str = odm.Optional(odm.Keyword())
use_new_callback_format: str = odm.Boolean(default=False)
class Task(odm.Model):
sid = odm.UUID()
metadata = odm.FlattenedObject() # Metadata associated to the submission
min_classification = odm.Classification(
) # Minimum classification of the file being scanned
fileinfo: FileInfo = odm.Compound(FileInfo) # File info block
filename = odm.Keyword()
service_name = odm.Keyword()
service_config = odm.Mapping(odm.Any(),
default={}) # Service specific parameters
depth = odm.Integer(default=0)
max_files = odm.Integer()
ttl = odm.Integer(default=0)
tags = odm.List(odm.Compound(TagItem), default=[])
temporary_submission_data = odm.List(odm.Compound(DataItem), default=[])
deep_scan = odm.Boolean(default=False)
# Whether the service cache should be ignored during the processing of this task
ignore_cache = odm.Boolean(default=False)
# Whether the service should ignore the dynamic recursion prevention or not
ignore_dynamic_recursion_prevention = odm.Boolean(default=False)
# Priority for processing order
priority = odm.Integer(default=0)
@staticmethod
def make_key(sid, service_name, sha):
return f"{sid}_{service_name}_{sha}"
def key(self):
return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class Config(odm.Model):
auth: Auth = odm.Compound(
Auth,
default=DEFAULT_AUTH,
description="Authentication module configuration")
core: Core = odm.Compound(Core,
default=DEFAULT_CORE,
description="Core component configuration")
datastore: Datastore = odm.Compound(Datastore,
default=DEFAULT_DATASTORE,
description="Datastore configuration")
datasources: Dict[str, Datasource] = odm.Mapping(
odm.Compound(Datasource),
default=DEFAULT_DATASOURCES,
description="Datasources configuration")
filestore: Filestore = odm.Compound(Filestore,
default=DEFAULT_FILESTORE,
description="Filestore configuration")
logging: Logging = odm.Compound(Logging,
default=DEFAULT_LOGGING,
description="Logging configuration")
services: Services = odm.Compound(Services,
default=DEFAULT_SERVICES,
description="Service configuration")
system: System = odm.Compound(System,
default=DEFAULT_SYSTEM,
description="System configuration")
ui: UI = odm.Compound(UI,
default=DEFAULT_UI,
description="UI configuration parameters")
submission: Submission = odm.Compound(
Submission,
default=DEFAULT_SUBMISSION,
description="Options for how submissions will be processed")
class OAuth(odm.Model):
enabled: bool = odm.Boolean(description="Enable use of OAuth?")
gravatar_enabled: bool = odm.Boolean(description="Enable gravatar?")
providers: Dict[str, OAuthProvider] = odm.Mapping(
odm.Compound(OAuthProvider),
default=DEFAULT_OAUTH_PROVIDERS,
description="OAuth provider configuration")
class Task(odm.Model):
sid = odm.UUID()
fileinfo: FileInfo = odm.Compound(FileInfo) # File info block
filename = odm.Keyword()
service_name = odm.Keyword()
service_config = odm.Mapping(odm.Any(), default={}) # Service specific parameters
depth = odm.Integer(default=0)
max_files = odm.Integer()
ttl = odm.Integer(default=0)
tags = odm.List(odm.Compound(TagItem), default=[])
temporary_submission_data = odm.List(odm.Compound(DataItem), default=[])
deep_scan = odm.Boolean(default=False)
# Whether the service cache should be ignored during the processing of this task
ignore_cache = odm.Boolean(default=False)
# Priority for processing order
priority = odm.Integer(default=0)
@staticmethod
def make_key(sid, service_name, sha):
return f"{sid}_{service_name}_{sha}"
def key(self):
return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class Ingester(odm.Model):
default_user: str = odm.Keyword()
default_services: List[str] = odm.List(odm.Keyword())
default_resubmit_services: List[str] = odm.List(odm.Keyword())
# When a description is automatically generated, it will be the
# hash prefixed by this string
description_prefix: str = odm.Keyword()
# Path to a callback function filtering ingestion tasks that should have their
# priority forcefully reset to low
is_low_priority: str = odm.Keyword()
get_whitelist_verdict: str = odm.Keyword()
whitelist: str = odm.Keyword()
# Default values for parameters that may be overridden on a per submission basis
# How many extracted files may be added to a Submission
default_max_extracted: int = odm.Integer()
# How many supplementary files may be added to a submission
default_max_supplementary: int = odm.Integer()
# Drop a task altogether after this many seconds
expire_after: int = odm.Integer()
stale_after_seconds: int = odm.Integer()
# How long should scores be cached in the ingester
incomplete_expire_after_seconds: int = odm.Integer()
incomplete_stale_after_seconds: int = odm.Integer()
# How long can a queue get before we start dropping files
sampling_at: Dict[str, int] = odm.Mapping(odm.Integer())
max_inflight = odm.Integer()
# How long are files results cached
cache_dtl: int = odm.Integer()
class Task(odm.Model):
sid = odm.UUID(description="Submission ID")
metadata = odm.FlattenedObject(
description="Metadata associated to the submission")
min_classification = odm.Classification(
description="Minimum classification of the file being scanned")
fileinfo: FileInfo = odm.Compound(FileInfo, description="File info block")
filename = odm.Keyword(description="File name")
service_name = odm.Keyword(description="Service name")
service_config = odm.Mapping(odm.Any(),
default={},
description="Service specific parameters")
depth = odm.Integer(
default=0,
description="File depth relative to initital submitted file")
max_files = odm.Integer(
description="Maximum number of files that submission can have")
ttl = odm.Integer(default=0, description="Task TTL")
tags = odm.List(odm.Compound(TagItem),
default=[],
description="List of tags")
temporary_submission_data = odm.List(
odm.Compound(DataItem),
default=[],
description="Temporary submission data")
deep_scan = odm.Boolean(default=False, description="Perform deep scanning")
ignore_cache = odm.Boolean(
default=False,
description=
"Whether the service cache should be ignored during the processing of this task"
)
ignore_dynamic_recursion_prevention = odm.Boolean(
default=False,
description=
"Whether the service should ignore the dynamic recursion prevention or not"
)
ignore_filtering = odm.Boolean(
default=False, description="Should the service filter it's output?")
priority = odm.Integer(default=0,
description="Priority for processing order")
safelist_config = odm.Compound(
ServiceSafelist,
description=
"Safelisting configuration (as defined in global configuration)",
default={'enabled': False})
@staticmethod
def make_key(sid, service_name, sha):
return f"{sid}_{service_name}_{sha}"
def key(self):
return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class UserSettings(odm.Model):
classification = odm.Classification(
default=Classification.UNRESTRICTED,
description="Default submission classification")
deep_scan = odm.Boolean(default=False,
description="Should a deep scan be performed?")
description = odm.Keyword(default="", description="Default description")
download_encoding = odm.Enum(
values=ENCODINGS,
default="cart",
description="Default download encoding when downloading files")
default_zip_password = odm.Text(
default="zippy",
description=
"Default user-defined password for creating password protected ZIPs when downloading files"
)
expand_min_score = odm.Integer(
default=500,
description="Auto-expand section when score bigger then this")
ignore_cache = odm.Boolean(default=False,
description="Ignore service caching?")
ignore_dynamic_recursion_prevention = odm.Boolean(
default=False, description="Ignore dynamic recursion prevention?")
ignore_filtering = odm.Boolean(default=False,
description="Ignore filtering services?")
malicious = odm.Boolean(
default=False,
description="Is the file submitted already known to be malicious?")
priority = odm.Integer(default=1000,
description="Default priority for the submissions")
profile = odm.Boolean(
default=False, description="Should the submission do extra profiling?")
service_spec = odm.Mapping(odm.Mapping(odm.Any()),
default={},
description="Default service specific settings")
services = odm.Compound(ServiceSelection,
default={},
description="Default service selection")
submission_view = odm.Enum(
values=VIEWS,
default="report",
description="Default view for completed submissions")
ttl = odm.Integer(default=30,
description="Default submission TTL, in days")
class Services(odm.Model):
categories: List[str] = odm.List(
odm.Keyword(),
description="List of categories a service can be assigned to")
default_timeout: int = odm.Integer(
description="Default service timeout time in seconds")
min_service_workers: int = odm.Integer(
description=
"The minimum number of service instances to always be running.")
stages: List[str] = odm.List(
odm.Keyword(),
description="List of execution stages a service can be assigned to")
image_variables: Dict[str, str] = odm.Mapping(
odm.Keyword(default=''),
description="Substitution variables for image paths "
"(for custom registry support)")
update_image_variables: Dict[str, str] = odm.Mapping(
odm.Keyword(default=''),
description=
"Similar to `image_variables` but only applied to the updater. "
"Intended for use with local registries.")
preferred_update_channel: str = odm.Keyword(
description="Default update channel to be used for new services")
allow_insecure_registry: bool = odm.Boolean(
description="Allow fetching container images from insecure registries")
preferred_registry_type: str = odm.Enum(
values=["docker", "harbor"],
default='docker',
description=
"Global registry type to be used for fetching updates for a service (overridable by a service)"
)
prefer_service_privileged: bool = odm.Boolean(
default=False,
description="Global preference that controls if services should be "
"privileged to communicate with core infrastucture")
cpu_reservation: float = odm.Float(
description=
"How much CPU do we want to reserve relative to the service's request?<br>"
"At `1`, a service's full CPU request will be reserved for them.<br>"
"At `0` (only for very small appliances/dev boxes), the service's CPU will be limited "
"but no CPU will be reserved allowing for more flexible scheduling of containers."
)
safelist = odm.Compound(ServiceSafelist)
class DependencyConfig(odm.Model):
container: DockerConfig = odm.Compound(
DockerConfig,
description="Docker container configuration for dependency")
volumes = odm.Mapping(odm.Compound(PersistentVolume),
default={},
description="Volume configuration for dependency")
run_as_core: bool = odm.Boolean(
default=False,
description="Should this dependency run as other core components?")
class Services(odm.Model):
# Different possible categories
categories: List[str] = odm.List(odm.Keyword())
# Default service timeout time in seconds
default_timeout: int = odm.Integer()
# How many instances of a service should be kept in reserve running even
# when there doesn't seem to be any work for them to do
min_service_workers: int = odm.Integer()
# Different stages of execution in order
stages: List[str] = odm.List(odm.Keyword())
# Substitution variables for image paths (for custom registry support)
image_variables: Dict[str, str] = odm.Mapping(odm.Keyword(default=''))
class Heuristic(odm.Model):
attack_id = odm.List(odm.Keyword(copyto="__text__"), default=[], description="List of all associated ATT&CK IDs")
classification = odm.Classification(default=Classification.UNRESTRICTED,
description="Classification of the heuristic")
description = odm.Text(copyto="__text__", description="Description of the heuristic")
filetype = odm.Keyword(copyto="__text__", description="What type of files does this heuristic target?")
heur_id = odm.Keyword(copyto="__text__", description="ID of the Heuristic")
name = odm.Keyword(copyto="__text__", description="Name of the heuristic")
score = odm.Integer(description="Default score of the heuristic")
signature_score_map = odm.Mapping(odm.Integer(), default={},
description="Score of signatures for this heuristic")
stats = odm.Compound(Statistics, default={}, description="Statistics related to the Heuristic")
max_score = odm.Optional(odm.Integer(), description="Maximum score for heuristic")
class Heuristic(odm.Model):
attack_id = odm.List(odm.Enum(values=ATTACK_ID_LIST, copyto="__text__"),
default=[]) # List of all associated Att&ck IDs
classification = odm.Classification(
default=Classification.UNRESTRICTED) # Classification of the heuristic
description = odm.Text(copyto="__text__") # Description of the heuristic
filetype = odm.Keyword(copyto="__text__") # Type of file targeted
heur_id = odm.Keyword(copyto="__text__") # Heuristic ID
name = odm.Keyword(copyto="__text__") # Name of the heuristic
score = odm.Integer() # Default score of the heuristic
signature_score_map = odm.Mapping(
odm.Integer(), default={}) # Score of signatures for this heuristic
max_score = odm.Optional(odm.Integer()) # Maximum score for heuristic
class User(odm.Model):
agrees_with_tos = odm.Optional(odm.Date(
index=False, store=False)) # Date the user agree with terms of service
api_quota = odm.Integer(
default=10, store=False) # Max number of concurrent API requests
apikeys = odm.Mapping(odm.Compound(ApiKey),
default={},
index=False,
store=False) # Mapping of api keys
can_impersonate = odm.Boolean(
default=False, index=False,
store=False) # Allowed to query on behalf of others
classification = odm.Classification(
is_user_classification=True,
copyto="__text__",
default=Classification.UNRESTRICTED) # Max classification for the user
dn = odm.Optional(odm.Keyword(store=False,
copyto="__text__")) # User ldap DN
email = odm.Optional(
odm.Keyword(copyto="__text__")) # User's email address
groups = odm.List(odm.Keyword(), copyto="__text__",
default=["USERS"]) # List of groups the user submits to
is_active = odm.Boolean(default=True) # is the user active
name = odm.Keyword(copyto="__text__") # Full name of the user
otp_sk = odm.Optional(odm.Keyword(
index=False, store=False)) # Secret key to generate one time passwords
password = odm.Keyword(index=False,
store=False) # BCrypt hash of the user's password
submission_quota = odm.Integer(
default=5, store=False) # Maximum number of concurrent submissions
type = odm.List(odm.Enum(values=USER_TYPES),
default=['user']) # Type of user
security_tokens = odm.Mapping(odm.Keyword(),
index=False,
store=False,
default={}) # Map of security tokens
uname = odm.Keyword(copyto="__text__") # Username
class OAuthProvider(odm.Model):
auto_create: str = odm.Boolean(default=True)
auto_sync: str = odm.Boolean(default=False)
auto_properties: List[OAuthAutoProperty] = odm.List(odm.Compound(OAuthAutoProperty), default=[])
uid_regex: str = odm.Optional(odm.Keyword())
uid_format: str = odm.Optional(odm.Keyword())
client_id: str = odm.Optional(odm.Keyword())
client_secret: str = odm.Optional(odm.Keyword())
request_token_url: str = odm.Optional(odm.Keyword())
request_token_params: str = odm.Optional(odm.Keyword())
access_token_url: str = odm.Optional(odm.Keyword())
access_token_params: str = odm.Optional(odm.Keyword())
authorize_url: str = odm.Optional(odm.Keyword())
authorize_params: str = odm.Optional(odm.Keyword())
api_base_url: str = odm.Optional(odm.Keyword())
client_kwargs: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()))
user_get: str = odm.Keyword()
class DependencyConfigDelta(odm.Model):
container = odm.Optional(
odm.Compound(DockerConfigDelta),
description=
"Refer to:<br>[Service - DependencyConfig](../service/#dependencyconfig)"
)
volumes = odm.Mapping(
odm.Compound(PersistentVolumeDelta),
default={},
description=
"Refer to:<br>[Service - DependencyConfig](../service/#dependencyconfig)"
)
run_as_core: bool = odm.Optional(
odm.Boolean(),
description=
"Refer to:<br>[Service - DependencyConfig](../service/#dependencyconfig)"
)
class Ingester(odm.Model):
default_user: str = odm.Keyword(
description="Default user for bulk ingestion and unattended submissions"
)
default_services: List[str] = odm.List(
odm.Keyword(), description="Default service selection")
default_resubmit_services: List[str] = odm.List(
odm.Keyword(), description="Default service selection for resubmits")
description_prefix: str = odm.Keyword(
description=
"A prefix for descriptions. When a description is automatically generated, it will be "
"the hash prefixed by this string")
is_low_priority: str = odm.Keyword(
description=
"Path to a callback function filtering ingestion tasks that should have their priority "
"forcefully reset to low")
get_whitelist_verdict: str = odm.Keyword()
whitelist: str = odm.Keyword()
default_max_extracted: int = odm.Integer(
description=
"How many extracted files may be added to a Submission. Overrideable via submission parameters."
)
default_max_supplementary: int = odm.Integer(
description=
"How many supplementary files may be added to a Submission. Overrideable via submission parameters"
)
expire_after: int = odm.Integer(
description="Period, in seconds, in which a task should be expired")
stale_after_seconds: int = odm.Integer(
description="Drop a task altogether after this many seconds")
incomplete_expire_after_seconds: int = odm.Integer(
description="How long should scores be kept before expiry")
incomplete_stale_after_seconds: int = odm.Integer(
description="How long should scores be cached in the ingester")
sampling_at: Dict[str, int] = odm.Mapping(
odm.Integer(),
description="Thresholds at certain buckets before sampling")
max_inflight = odm.Integer(
description="How long can a queue get before we start dropping files")
cache_dtl: int = odm.Integer(
description="How long are files results cached")
class Config(odm.Model):
# Authentication module configuration
auth: Auth = odm.Compound(Auth, default=DEFAULT_AUTH)
# Core component configuration
core: Core = odm.Compound(Core, default=DEFAULT_CORE)
# Datastore configuration
datastore: Datastore = odm.Compound(Datastore, default=DEFAULT_DATASTORE)
# Datasources configuration
datasources: Dict[str, Datasource] = odm.Mapping(odm.Compound(Datasource), default=DEFAULT_DATASOURCES)
# Filestore configuration
filestore: Filestore = odm.Compound(Filestore, default=DEFAULT_FILESTORE)
# Logging configuration
logging: Logging = odm.Compound(Logging, default=DEFAULT_LOGGING)
# Service configuration
services: Services = odm.Compound(Services, default=DEFAULT_SERVICES)
# System configuration
system: System = odm.Compound(System, default=DEFAULT_SYSTEM)
# UI configuration parameters
ui: UI = odm.Compound(UI, default=DEFAULT_UI)
# Options for how submissions will be processed
submission: Submission = odm.Compound(Submission, default=DEFAULT_SUBMISSION)
class Alert(odm.Model):
alert_id = odm.Keyword(copyto="__text__") # ID of the alert
al = odm.Compound(ALResults) # Assemblyline result block
archive_ts = odm.Date(store=False) # Archiving timestamp
classification = odm.Classification() # Classification of the alert
expiry_ts = odm.Optional(odm.Date(store=False)) # Expiry timestamp
extended_scan = odm.Enum(values=EXTENDED_SCAN_VALUES,
store=False) # Status of the extended scan
file = odm.Compound(File) # File block
label = odm.List(odm.Keyword(), copyto="__text__",
default=[]) # List of labels applied to the alert
metadata = odm.Mapping(odm.Keyword(),
store=False) # Metadata submitted with the file
owner = odm.Optional(odm.Keyword()) # Owner of the alert
priority = odm.Optional(
odm.Enum(values=PRIORITIES)) # Priority applied to the alert
reporting_ts = odm.Date() # Time at which the alert was created
sid = odm.UUID(store=False) # ID of the submission related to this alert
status = odm.Optional(
odm.Enum(values=STATUSES)) # Status applied to the alert
ts = odm.Date() # Timestamp at which the file was submitted
type = odm.Keyword() # Type of alert
verdict = odm.Compound(Verdict, default={}) # Verdict timing
class DependencyConfigDelta(odm.Model):
container = odm.Optional(odm.Compound(DockerConfigDelta))
volumes = odm.Mapping(odm.Compound(PersistentVolumeDelta), default={})
class Datasource(odm.Model):
classpath: str = odm.Keyword()
config: Dict[str, str] = odm.Mapping(odm.Keyword())
class OAuthProvider(odm.Model):
auto_create: str = odm.Boolean(
default=True, description="Auto-create users if they are missing")
auto_sync: str = odm.Boolean(
default=False,
description="Should we automatically sync with OAuth provider?")
auto_properties: List[OAuthAutoProperty] = odm.List(
odm.Compound(OAuthAutoProperty),
default=[],
description="Automatic role and classification assignments")
app_provider: AppProvider = odm.Optional(odm.Compound(AppProvider))
uid_randomize: str = odm.Boolean(
default=False,
description=
"Should we generate a random username for the authenticated user?")
uid_randomize_digits: str = odm.Integer(
default=0,
description="How many digits should we add at the end of the username?"
)
uid_randomize_delimiter: str = odm.Keyword(
default="-",
description="What is the delimiter used by the random name generator?")
uid_regex: str = odm.Optional(
odm.Keyword(),
description=
"Regex used to parse an email address and capture parts to create a user ID out of it"
)
uid_format: str = odm.Optional(
odm.Keyword(),
description=
"Format of the user ID based on the captured parts from the regex")
client_id: str = odm.Optional(
odm.Keyword(),
description=
"ID of your application to authenticate to the OAuth provider")
client_secret: str = odm.Optional(
odm.Keyword(),
description=
"Password to your application to authenticate to the OAuth provider")
request_token_url: str = odm.Optional(odm.Keyword(),
description="URL to request token")
request_token_params: str = odm.Optional(
odm.Keyword(), description="Parameters to request token")
access_token_url: str = odm.Optional(odm.Keyword(),
description="URL to get access token")
access_token_params: str = odm.Optional(
odm.Keyword(), description="Parameters to get access token")
authorize_url: str = odm.Optional(
odm.Keyword(),
description="URL used to authorize access to a resource")
authorize_params: str = odm.Optional(
odm.Keyword(),
description="Parameters used to authorize access to a resource")
api_base_url: str = odm.Optional(
odm.Keyword(),
description="Base URL for downloading the user's and groups info")
client_kwargs: Dict[str, str] = odm.Optional(
odm.Mapping(odm.Keyword()),
description="Keyword arguments passed to the different URLs")
jwks_uri: str = odm.Optional(
odm.Keyword(),
description="URL used to verify if a returned JWKS token is valid")
uid_field: str = odm.Optional(
odm.Keyword(),
description="Name of the field that will contain the user ID")
user_get: str = odm.Optional(
odm.Keyword(),
description="Path from the base_url to fetch the user info")
user_groups: str = odm.Optional(
odm.Keyword(),
description="Path from the base_url to fetch the group info")
user_groups_data_field: str = odm.Optional(
odm.Keyword(),
description=
"Field return by the group info API call that contains the list of groups"
)
user_groups_name_field: str = odm.Optional(
odm.Keyword(),
description=
"Name of the field in the list of groups that contains the name of the group"
)
use_new_callback_format: str = odm.Boolean(
default=False, description="Should we use the new callback method?")
class UI(odm.Model):
alerting_meta: AlertingMeta = odm.Compound(
AlertingMeta,
default=DEFAULT_ALERTING_META,
description="Alerting metadata fields")
allow_malicious_hinting: bool = odm.Boolean(
description=
"Allow user to tell in advance the system that a file is malicious?")
allow_raw_downloads: bool = odm.Boolean(
description="Allow user to download raw files?")
allow_zip_downloads: bool = odm.Boolean(
description="Allow user to download files as password protected ZIPs?")
allow_replay: bool = odm.Boolean(
description="Allow users to request replay on another server?")
allow_url_submissions: bool = odm.Boolean(
description="Allow file submissions via url?")
audit: bool = odm.Boolean(
description=
"Should API calls be audited and saved to a separate log file?")
banner: Dict[str, str] = odm.Optional(
odm.Mapping(odm.Keyword()),
description=
"Banner message display on the main page (format: {<language_code>: message})"
)
banner_level: str = odm.Enum(
values=["info", "warning", "success", "error"],
description="Banner message level")
debug: bool = odm.Boolean(description="Enable debugging?")
discover_url: str = odm.Optional(odm.Keyword(), description="Discover URL")
download_encoding = odm.Enum(
values=["raw", "cart"],
description="Which encoding will be used for downloads?")
email: str = odm.Optional(odm.Email(),
description="Assemblyline admins email address")
enforce_quota: bool = odm.Boolean(description="Enforce the user's quotas?")
fqdn: str = odm.Text(
description=
"Fully qualified domain name to use for the 2-factor authentication validation"
)
ingest_max_priority: int = odm.Integer(
description="Maximum priority for ingest API")
read_only: bool = odm.Boolean(
description="Turn on read only mode in the UI")
read_only_offset: str = odm.Keyword(
default="",
description="Offset of the read only mode for all paging and searches")
secret_key: str = odm.Keyword(
description="Flask secret key to store cookies, etc.")
session_duration: int = odm.Integer(
description=
"Duration of the user session before the user has to login again")
statistics: Statistics = odm.Compound(
Statistics,
default=DEFAULT_STATISTICS,
description="Statistics configuration")
tos: str = odm.Optional(odm.Text(), description="Terms of service")
tos_lockout: bool = odm.Boolean(
description="Lock out user after accepting the terms of service?")
tos_lockout_notify: List[str] = odm.Optional(
odm.List(odm.Keyword()),
description="List of admins to notify when a user gets locked out")
url_submission_headers: Dict[str, str] = odm.Optional(
odm.Mapping(odm.Keyword()),
description="Headers used by the url_download method")
url_submission_proxies: Dict[str, str] = odm.Optional(
odm.Mapping(odm.Keyword()),
description="Proxy used by the url_download method")
validate_session_ip: bool = \
odm.Boolean(description="Validate if the session IP matches the IP the session was created from")
validate_session_useragent: bool = \
odm.Boolean(description="Validate if the session useragent matches the useragent the session was created with")
