class UserSettings(odm.Model): # User's default settings
classification = odm.Classification(default=Classification.UNRESTRICTED
) # Default submission classification
deep_scan = odm.Boolean(default=False) # Should a deep scan be performed
description = odm.Keyword(default="") # Default description
download_encoding = odm.Enum(
values=ENCODINGS,
default="cart") # Default download encoding when downloading files
expand_min_score = odm.Integer(
default=500) # Auto-expand section when score bigger then this
ignore_cache = odm.Boolean(default=False) # Ignore service caching
ignore_dynamic_recursion_prevention = odm.Boolean(
default=False) # Ignore dynamic recursion prevention
ignore_filtering = odm.Boolean(default=False) # Ignore filtering services
priority = odm.Integer(
default=1000) # Default priority for the submissions
profile = odm.Boolean(
default=False) # Should the submission do extra profiling
service_spec = odm.Mapping(odm.Keyword(),
default={}) # Default service specific settings
services = odm.Compound(ServiceSelection,
default={}) # Default service selection
submission_view = odm.Enum(
values=VIEWS,
default="report") # Default view for completed submissions
ttl = odm.Integer(default=0) # Default submission Time to Live (days)
class SubmissionMessage(odm.Model):
msg = odm.Compound(Submission) # Body of the message
msg_loader = odm.Enum(
values={LOADER_CLASS},
default=LOADER_CLASS) # Class to use to load the message as an object
msg_type = odm.Enum(values=MSG_TYPES) # Type of message
sender = odm.Keyword() # Sender of the message
class Alert(odm.Model):
alert_id = odm.Keyword(copyto="__text__") # ID of the alert
al = odm.Compound(ALResults) # Assemblyline result block
archive_ts = odm.Date(store=False) # Archiving timestamp
attack = odm.Compound(Attack) # Attack result block
classification = odm.Classification() # Classification of the alert
expiry_ts = odm.Optional(odm.Date(store=False)) # Expiry timestamp
extended_scan = odm.Enum(values=EXTENDED_SCAN_VALUES,
store=False) # Status of the extended scan
file = odm.Compound(File) # File block
filtered = odm.Boolean(default=False) # Are the alert result filtered
heuristic = odm.Compound(Heuristic) # Heuristic result block
label = odm.List(odm.Keyword(), copyto="__text__",
default=[]) # List of labels applied to the alert
metadata = odm.FlattenedObject(
default={}, store=False) # Metadata submitted with the file
owner = odm.Optional(odm.Keyword()) # Owner of the alert
priority = odm.Optional(
odm.Enum(values=PRIORITIES)) # Priority applied to the alert
reporting_ts = odm.Date() # Time at which the alert was created
sid = odm.UUID(store=False) # ID of the submission related to this alert
status = odm.Optional(
odm.Enum(values=STATUSES)) # Status applied to the alert
ts = odm.Date() # Timestamp at which the file was submitted
type = odm.Keyword() # Type of alert
verdict = odm.Compound(Verdict, default={}) # Verdict timing
class SubmissionMessage(odm.Model):
msg = odm.Compound(Submission, description="Body of the message")
msg_loader = odm.Enum(
values={LOADER_CLASS},
default=LOADER_CLASS,
description="Class to use to load the message as an object") #
msg_type = odm.Enum(values=MSG_TYPES, description="Type of message")
sender = odm.Keyword(description="Sender of the message")
class ScalerMessage(odm.Model):
msg = odm.Compound(Heartbeat, description="Heartbeat message")
msg_loader = odm.Enum(values={LOADER_CLASS},
default=LOADER_CLASS,
description="Loader class of message")
msg_type = odm.Enum(values=MSG_TYPES,
default="ScalerHeartbeat",
description="Type of message")
sender = odm.Keyword(description="Sender of message")
class ArchiveMessage(odm.Model):
msg = odm.Compound(Heartbeat, description="Heartbeat message")
msg_loader = odm.Enum(values={LOADER_CLASS},
default=LOADER_CLASS,
description="Loader class for message")
msg_type = odm.Enum(values=MSG_TYPES,
default="ArchiveHeartbeat",
description="Message type")
sender = odm.Keyword(description="Sender of message")
class AlertMessage(odm.Model):
msg = odm.Compound(Alert, description="Message of alert")
msg_loader = odm.Enum(values={LOADER_CLASS},
default=LOADER_CLASS,
description="Loader class for messages")
msg_type = odm.Enum(values=MSG_TYPES,
default="AlertCreated",
description="Type of Message")
sender = odm.Keyword(description="Sender of message")
class MetricsMessage(odm.Model):
msg = odm.Compound(Metrics, description="Metrics message")
msg_loader = odm.Enum(values={LOADER_CLASS},
default=LOADER_CLASS,
description="Loader class for message")
msg_type = odm.Enum(values=MSG_TYPES,
default="MetricsCounter",
description="Type of message")
sender = odm.Keyword(description="Sender of message")
class VM(odm.Model):
enabled = odm.Boolean(default=True)
name = odm.Keyword(copyto="__text__")
num_workers = odm.Integer(default=1)
os_type = odm.Enum(values=OS_TYPES)
os_variant = odm.Enum(values=OS_VARIANTS)
ram = odm.Integer(default=1024)
revert_every = odm.Integer(default=600)
vcpus = odm.Integer(default=1)
virtual_disk_url = odm.Keyword()
class TCSignature(odm.Model):
al_score = odm.Enum(values=SCORES, default="HIGH")
al_status = odm.Enum(values=STATUSES, default="TESTING")
callback = odm.Optional(odm.Keyword())
classification = odm.Classification(default=Classification.UNRESTRICTED)
comment = odm.Optional(odm.Keyword())
implant_family = odm.Optional(odm.Keyword())
last_modified = odm.Date(default="NOW")
name = odm.Keyword(copyto="__text__")
threat_actor = odm.Optional(odm.Keyword())
values = odm.List(odm.Keyword())
class UI(odm.Model):
# Allow user to tell in advance the system that a file is malicious
allow_malicious_hinting: bool = odm.Boolean()
# Allow to user to download raw files
allow_raw_downloads: bool = odm.Boolean()
# Allow file submissions via url
allow_url_submissions: bool = odm.Boolean()
# Should API calls be audited and saved to a separate log file?
audit: bool = odm.Boolean()
# Banner message display on the main page (format: {<language_code>: message})
banner: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()))
# Banner message display on the main page (format: {<language_code>: message})
banner_level: str = odm.Enum(
values=["info", "warning", "success", "error"])
# Turn on debugging
debug: bool = odm.Boolean()
# Which encoding will be used
download_encoding = odm.Enum(values=["raw", "cart"])
# Assemblyline admins email address
email: str = odm.Optional(odm.Email())
# Enforce the user's quotas
enforce_quota: bool = odm.Boolean()
# Fully qualified domain name to use for the 2-factor authentication validation
fqdn: str = odm.Text()
# Maximum priority for ingest API
ingest_max_priority: int = odm.Integer()
# Turn on read only mode in the UI
read_only: bool = odm.Boolean()
# Offset of the read only mode for all paging and searches
read_only_offset: str = odm.Keyword(default="")
# Flask secret key to store cookies and stuff
secret_key: str = odm.Keyword()
# Duration of the user session before the user has to login again
session_duration: int = odm.Integer()
# Statistics configuration
statistics: Statistics = odm.Compound(Statistics,
default=DEFAULT_STATISTICS)
# Terms of service
tos: str = odm.Optional(odm.Text())
# Lock out user after accepting the terms of service
tos_lockout: bool = odm.Boolean()
# List of admins to notify when a user gets locked out
tos_lockout_notify: bool = odm.Optional(odm.List(odm.Keyword()))
# Headers that will be used by the url_download method
url_submission_headers: Dict[str, str] = odm.Optional(
odm.Mapping(odm.Keyword()))
# Proxy that will be used by the url_download method
url_submission_proxies: Dict[str, str] = odm.Optional(
odm.Mapping(odm.Keyword()))
# Validate if the session ip matches the ip the session was created from
validate_session_ip: bool = odm.Boolean()
# Validate if the session useragent matches the useragent the session was created with
validate_session_useragent: bool = odm.Boolean()
class Logging(odm.Model):
# What level of logging should we have
log_level: str = odm.Enum(
values=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", "DISABLED"])
# Should we log to console?
log_to_console: bool = odm.Boolean()
# Should we log to files on the server?
log_to_file: bool = odm.Boolean()
# if yes, what is the log directory
log_directory: str = odm.Keyword()
# Should logs be sent to a syslog server?
log_to_syslog: bool = odm.Boolean()
# if yes, what is the syslog server hostname/ip?
syslog_host: str = odm.Keyword()
syslog_port: int = odm.Integer()
# How often should counters log their values (seconds)
export_interval: int = odm.Integer()
# Log in JSON format
log_as_json: bool = odm.Boolean()
# If set, core components will touch this path regularly to tell the container
# environment it is healthy
heartbeat_file: str = odm.Optional(odm.Keyword())
class ESMetrics(odm.Model):
hosts: str = odm.Optional(odm.List(odm.Keyword()))
host_certificates: str = odm.Optional(odm.Keyword())
warm: int = odm.Integer()
cold: int = odm.Integer()
delete: int = odm.Integer()
unit = odm.Enum(['d', 'h', 'm'])
class Apps(odm.Model):
client_id = odm.Keyword(
description="Username allowed to impersonate the current user")
netloc = odm.Keyword(description="DNS hostname for the server")
scope = odm.Enum(values=SCOPES,
description="Scope of access for the API key")
server = odm.Keyword(description="Name of the server that has access")
class UpdateConfigDelta(odm.Model):
generates_signatures = odm.Optional(
odm.Boolean(),
index=True,
description=
"Refer to:<br>[Service - UpdateConfig](../service/#updateconfig)")
sources = odm.Optional(
odm.List(odm.Compound(UpdateSourceDelta)),
description=
"Refer to:<br>[Service - UpdateConfig](../service/#updateconfig)")
update_interval_seconds = odm.Optional(
odm.Integer(),
description=
"Refer to:<br>[Service - UpdateConfig](../service/#updateconfig)")
wait_for_update = odm.Optional(
odm.Boolean(),
description=
"Refer to:<br>[Service - UpdateConfig](../service/#updateconfig)")
signature_delimiter = odm.Optional(
odm.Enum(values=SIGNATURE_DELIMITERS.keys()),
description=
"Refer to:<br>[Service - UpdateConfig](../service/#updateconfig)")
custom_delimiter = odm.Optional(
odm.Keyword(),
description=
"Refer to:<br>[Service - UpdateConfig](../service/#updateconfig)")
class Submission(odm.Model):
archive_ts = odm.Date(store=False) # Archiving timestamp
classification = odm.Classification() # Classification of the submission
error_count = odm.Integer() # Total number of errors in the submission
errors = odm.List(odm.Keyword(), store=False) # List of error keys
expiry_ts = odm.Optional(odm.Date(store=False)) # Expiry timestamp
file_count = odm.Integer() # Total number of files in the submission
files: List[File] = odm.List(
odm.Compound(File)) # List of files that were originally submitted
max_score = odm.Integer() # Maximum score of all the files in the scan
metadata = odm.FlattenedObject(
store=False) # Metadata associated to the submission
params: SubmissionParams = odm.Compound(
SubmissionParams) # Submission detail blocs
results: List[str] = odm.List(odm.Keyword(),
store=False) # List of result keys
sid = odm.UUID(copyto="__text__") # Submission ID
state = odm.Enum(values=SUBMISSION_STATES) # Status of the submission
times = odm.Compound(Times, default={}) # Timing bloc
verdict = odm.Compound(Verdict, default={}) # Verdict timing
def is_submit(self):
return self.state == 'submitted'
def is_complete(self):
return self.state == 'completed'
def is_initial(self):
return self.is_submit() and not self.params.psid
class Service(odm.Model):
# Regexes applied to assemblyline style file type string
accepts = odm.Keyword(store=True, default=DEFAULT_SERVICE_ACCEPTS)
rejects = odm.Optional(
odm.Keyword(store=True, default=DEFAULT_SERVICE_REJECTS))
category = odm.Keyword(store=True,
default="Static Analysis",
copyto="__text__")
config = odm.Mapping(odm.Any(), default={}, index=False, store=False)
description = odm.Text(store=True, default="NA", copyto="__text__")
default_result_classification = odm.ClassificationString(
default=Classification.UNRESTRICTED)
enabled = odm.Boolean(store=True, default=False)
is_external = odm.Boolean(default=False)
licence_count = odm.Integer(default=0)
name = odm.Keyword(store=True, copyto="__text__")
version = odm.Keyword(store=True)
# Should the result cache be disabled for this service
disable_cache = odm.Boolean(default=False)
stage = odm.Keyword(store=True, default="CORE", copyto="__text__")
submission_params: SubmissionParams = odm.List(
odm.Compound(SubmissionParams), index=False, default=[])
timeout = odm.Integer(default=60)
docker_config: DockerConfig = odm.Compound(DockerConfig)
dependencies = odm.Mapping(odm.Compound(DependencyConfig), default={})
update_channel: str = odm.Enum(values=["stable", "rc", "beta", "dev"],
default='stable')
update_config: UpdateConfig = odm.Optional(odm.Compound(UpdateConfig))
class DockerConfig(odm.Model):
allow_internet_access: bool = odm.Boolean(
default=False, description="Does the container have internet-access?")
command: Opt[list[str]] = odm.Optional(
odm.List(odm.Keyword()),
description="Command to run when container starts up.")
cpu_cores: float = odm.Float(default=1.0, description="CPU allocation")
environment: list[EnvironmentVariable] = odm.List(
odm.Compound(EnvironmentVariable),
default=[],
description="Additional environemnt variables for the container")
image: str = odm.Keyword(
description=
"Complete name of the Docker image with tag, may include registry")
registry_username: Opt[str] = odm.Optional(
odm.Keyword(),
description="The username to use when pulling the image")
registry_password: Opt[str] = odm.Optional(
odm.Keyword(),
description="The password or token to use when pulling the image")
registry_type: str = odm.Enum(values=["docker", "harbor"],
default='docker',
description="The type of container registry")
ports: list[str] = odm.List(
odm.Keyword(),
default=[],
description="What ports of container to expose?")
ram_mb: int = odm.Integer(default=512, description="Container RAM limit")
ram_mb_min: int = odm.Integer(default=128,
description="Container RAM request")
class System(odm.Model):
# Module path to the assemblyline constants
constants: str = odm.Keyword()
# Organisation acronym used for signatures
organisation: str = odm.Text()
# Type of system (production, staging, development)
type: str = odm.Enum(values=['production', 'staging', 'development'])
class IngestTask(odm.Model):
# Submission Parameters
submission: Submission = odm.Compound(Submission)
# Shortcut for properties of the submission
@property
def file_size(self) -> int:
return sum(file.size for file in self.submission.files)
@property
def params(self) -> SubmissionParams:
return self.submission.params
@property
def sha256(self) -> str:
return self.submission.files[0].sha256
# Information about the ingestion itself, parameters irrelevant
scan_key = odm.Optional(odm.Keyword()) # the filescore key
retries = odm.Integer(default=0)
# Fields added after a submission is complete for notification/bookkeeping processes
failure = odm.Text(
default='') # If the ingestion has failed for some reason, what is it?
score = odm.Optional(
odm.Integer()) # Score from previous processing of this file
extended_scan = odm.Enum(EXTENDED_SCAN_VALUES, default="skipped")
ingest_id = odm.UUID()
ingest_time = odm.Date(default="NOW")
class ServiceDelta(odm.Model):
accepts = odm.Optional(odm.Keyword(), store=True)
rejects = odm.Optional(odm.Keyword(), store=True)
category = odm.Optional(odm.Keyword(), store=True, copyto="__text__")
config = odm.Optional(odm.Mapping(odm.Any()), index=False)
description = odm.Optional(odm.Text(), store=True, copyto="__text__")
default_result_classification = odm.Optional(odm.ClassificationString())
enabled = odm.Optional(odm.Boolean(), store=True)
is_external = odm.Optional(odm.Boolean())
licence_count = odm.Optional(odm.Integer())
name = odm.Optional(odm.Keyword(), store=True, copyto="__text__")
version = odm.Keyword(store=True)
disable_cache = odm.Optional(odm.Boolean())
stage = odm.Optional(odm.Keyword(), store=True, copyto="__text__")
submission_params = odm.Optional(odm.List(
odm.Compound(SubmissionParamsDelta)),
index=False)
timeout = odm.Optional(odm.Integer())
docker_config: DockerConfigDelta = odm.Optional(
odm.Compound(DockerConfigDelta))
dependencies: DependencyConfigDelta = odm.Mapping(
odm.Compound(DependencyConfigDelta), default={})
update_channel = odm.Optional(
odm.Enum(values=["stable", "rc", "beta", "dev"]))
update_config: UpdateConfigDelta = odm.Optional(
odm.Compound(UpdateConfigDelta))
class System(odm.Model):
constants: str = odm.Keyword(
description="Module path to the assemblyline constants")
organisation: str = odm.Text(
description="Organisation acronym used for signatures")
type: str = odm.Enum(values=['production', 'staging', 'development'],
description="Type of system")
class Current(odm.Model):
"""The current assignment for a service worker"""
status: str = odm.Enum(values=STATUSES,
default='INITIALIZING') # Status of the client
task: Opt[Task] = odm.Optional(odm.Compound(Task))
task_timeout: Opt[datetime] = odm.Optional(
odm.Date()) # Time the task was assigned to the client
class Logging(odm.Model):
log_level: str = odm.Enum(
values=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", "DISABLED"],
description="What level of logging should we have?")
log_to_console: bool = odm.Boolean(description="Should we log to console?")
log_to_file: bool = odm.Boolean(
description="Should we log to files on the server?")
log_directory: str = odm.Keyword(
description=
"If `log_to_file: true`, what is the directory to store logs?")
log_to_syslog: bool = odm.Boolean(
description="Should logs be sent to a syslog server?")
syslog_host: str = odm.Keyword(
description=
"If `log_to_syslog: true`, provide hostname/IP of the syslog server?")
syslog_port: int = odm.Integer(
description=
"If `log_to_syslog: true`, provide port of the syslog server?")
export_interval: int = odm.Integer(
description="How often, in seconds, should counters log their values?")
log_as_json: bool = odm.Boolean(description="Log in JSON format?")
heartbeat_file: str = odm.Optional(
odm.Keyword(),
description="Add a health check to core components.<br>"
"If `true`, core components will touch this path regularly to tell the container environment it is healthy"
)
class SubmissionParamsDelta(odm.Model):
default = odm.Optional(
odm.Any(),
description=
"Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)"
)
name = odm.Optional(
odm.Keyword(),
description=
"Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)"
)
type = odm.Optional(
odm.Enum(values=['str', 'int', 'list', 'bool']),
description=
"Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)"
)
value = odm.Optional(
odm.Any(),
description=
"Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)"
)
list = odm.Optional(
odm.Any(),
description=
"Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)"
)
hide = odm.Optional(
odm.Boolean(),
description=
"Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)"
)
class UpdateConfigDelta(odm.Model):
# build_options = odm.Optional(odm.Compound(DockerfileConfigDelta))
generates_signatures = odm.Optional(odm.Boolean(), index=True)
method = odm.Optional(odm.Enum(values=['run', 'build']))
run_options = odm.Optional(odm.Compound(DockerConfigDelta))
sources = odm.Optional(odm.List(odm.Compound(UpdateSourceDelta)))
update_interval_seconds = odm.Optional(odm.Integer())
wait_for_update = odm.Optional(odm.Boolean())
class Datastore(odm.Model):
hosts: List[str] = odm.List(
odm.Keyword(), description="List of hosts used for the datastore")
ilm = odm.Compound(ILM,
default=DEFAULT_ILM,
description="Index Lifecycle Management Policy")
type = odm.Enum({"elasticsearch"},
description="Type of application used for the datastore")
class OAuthAutoProperty(odm.Model):
field: str = odm.Keyword(description="Field to apply `pattern` to")
pattern: str = odm.Keyword(
description="Regex pattern for auto-prop assignment")
type: str = odm.Enum(
OAUTH_AUTO_PROPERTY_TYPE,
description="Type of property assignment on pattern match")
value: str = odm.Keyword(description="Assigned property value")
class Heuristic(odm.Model):
heur_id = odm.Keyword(copyto="__text__") # Triggered heuristic
name = odm.Keyword(copyto="__text__") # Name of the heuristics
attack_id = odm.Optional(odm.Enum(values=PATTERNS, copyto="__text__")) # Attack matrix ID
attack_pattern = odm.Optional(odm.Keyword(copyto="__text__")) # Attack matrix Pattern Name
attack_categories = odm.Optional(odm.List(odm.Keyword())) # Attack matrix Categories
signature = odm.Optional(odm.Keyword()) # Signature that triggered the heuristic
score = odm.Integer() # Heuristic's score
class Section(odm.Model):
body = odm.Optional(odm.Text(copyto="__text__")) # Text body of the result section
classification = odm.Classification() # Classification of the section
body_format = odm.Enum(values=BODY_FORMAT, index=False) # Type of body in this section
depth = odm.Integer(index=False) # Depth of the section
heuristic = odm.Optional(odm.Compound(Heuristic)) # Heuristic used to score result section
tags = odm.Compound(Tagging, default={}) # List of tags associated to this section
title_text = odm.Text(copyto="__text__") # Title of the section