class Logging(odm.Model):
log_level: str = odm.Enum(
values=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", "DISABLED"],
description="What level of logging should we have?")
log_to_console: bool = odm.Boolean(description="Should we log to console?")
log_to_file: bool = odm.Boolean(
description="Should we log to files on the server?")
log_directory: str = odm.Keyword(
description=
"If `log_to_file: true`, what is the directory to store logs?")
log_to_syslog: bool = odm.Boolean(
description="Should logs be sent to a syslog server?")
syslog_host: str = odm.Keyword(
description=
"If `log_to_syslog: true`, provide hostname/IP of the syslog server?")
syslog_port: int = odm.Integer(
description=
"If `log_to_syslog: true`, provide port of the syslog server?")
export_interval: int = odm.Integer(
description="How often, in seconds, should counters log their values?")
log_as_json: bool = odm.Boolean(description="Log in JSON format?")
heartbeat_file: str = odm.Optional(
odm.Keyword(),
description="Add a health check to core components.<br>"
"If `true`, core components will touch this path regularly to tell the container environment it is healthy"
)
class DockerConfig(odm.Model):
allow_internet_access: bool = odm.Boolean(
default=False, description="Does the container have internet-access?")
command: Opt[list[str]] = odm.Optional(
odm.List(odm.Keyword()),
description="Command to run when container starts up.")
cpu_cores: float = odm.Float(default=1.0, description="CPU allocation")
environment: list[EnvironmentVariable] = odm.List(
odm.Compound(EnvironmentVariable),
default=[],
description="Additional environemnt variables for the container")
image: str = odm.Keyword(
description=
"Complete name of the Docker image with tag, may include registry")
registry_username: Opt[str] = odm.Optional(
odm.Keyword(),
description="The username to use when pulling the image")
registry_password: Opt[str] = odm.Optional(
odm.Keyword(),
description="The password or token to use when pulling the image")
registry_type: str = odm.Enum(values=["docker", "harbor"],
default='docker',
description="The type of container registry")
ports: list[str] = odm.List(
odm.Keyword(),
default=[],
description="What ports of container to expose?")
ram_mb: int = odm.Integer(default=512, description="Container RAM limit")
ram_mb_min: int = odm.Integer(default=128,
description="Container RAM request")
class Icon(odm.Model):
icon_id = odm.Optional(odm.Integer())
planes = odm.Optional(odm.Integer())
height = odm.Optional(odm.Integer())
width = odm.Optional(odm.Integer())
lang = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
sublang = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
class ESMetrics(odm.Model):
hosts: str = odm.Optional(odm.List(odm.Keyword()))
host_certificates: str = odm.Optional(odm.Keyword())
warm: int = odm.Integer()
cold: int = odm.Integer()
delete: int = odm.Integer()
unit = odm.Enum(['d', 'h', 'm'])
class Export(odm.Model):
@odm.model(index=True, store=False)
class Entry(odm.Model):
@odm.model(index=True, store=False)
class Forward_Information(odm.Model):
function = odm.Optional(
odm.EmptyableKeyword(copyto="__text__"))
library = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
address = odm.Optional(odm.Integer())
forward_information = odm.Optional(
odm.Compound(Forward_Information))
function_rva = odm.Optional(odm.Integer())
is_extern = odm.Optional(odm.Boolean())
name = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
ordinal = odm.Optional(odm.Integer())
entries = odm.Optional(odm.List(odm.Compound(Entry)))
export_flags = odm.Optional(odm.Integer())
major_version = odm.Optional(odm.Integer())
minor_version = odm.Optional(odm.Integer())
name = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
ordinal_base = odm.Optional(odm.Integer())
timestamp = odm.Optional(odm.Integer())
hr_timestamp = odm.Optional(odm.Date())
class Service(odm.Model):
# Regexes applied to assemblyline style file type string
accepts = odm.Keyword(store=True, default=DEFAULT_SERVICE_ACCEPTS)
rejects = odm.Optional(
odm.Keyword(store=True, default=DEFAULT_SERVICE_REJECTS))
category = odm.Keyword(store=True,
default="Static Analysis",
copyto="__text__")
config = odm.Mapping(odm.Any(), default={}, index=False, store=False)
description = odm.Text(store=True, default="NA", copyto="__text__")
default_result_classification = odm.ClassificationString(
default=Classification.UNRESTRICTED)
enabled = odm.Boolean(store=True, default=False)
is_external = odm.Boolean(default=False)
licence_count = odm.Integer(default=0)
name = odm.Keyword(store=True, copyto="__text__")
version = odm.Keyword(store=True)
# Should the result cache be disabled for this service
disable_cache = odm.Boolean(default=False)
stage = odm.Keyword(store=True, default="CORE", copyto="__text__")
submission_params: SubmissionParams = odm.List(
odm.Compound(SubmissionParams), index=False, default=[])
timeout = odm.Integer(default=60)
docker_config: DockerConfig = odm.Compound(DockerConfig)
dependencies = odm.Mapping(odm.Compound(DependencyConfig), default={})
update_channel: str = odm.Enum(values=["stable", "rc", "beta", "dev"],
default='stable')
update_config: UpdateConfig = odm.Optional(odm.Compound(UpdateConfig))
class Dispatcher(odm.Model):
timeout: float = odm.Integer(
description=
"Time between re-dispatching attempts, as long as some action (submission or any task completion) "
"happens before this timeout ends, the timeout resets.")
max_inflight: int = odm.Integer(
description="Maximum submissions allowed to be in-flight")
class Metrics(odm.Model):
files_completed = odm.Integer()
submissions_completed = odm.Integer()
cpu_seconds = PerformanceTimer()
cpu_seconds_count = odm.Integer()
busy_seconds = PerformanceTimer()
busy_seconds_count = odm.Integer()
class Alerter(odm.Model):
alert_ttl: int = odm.Integer(
description="Time to live (days) for an alert in the system")
constant_alert_fields: List[str] = odm.List(
odm.Keyword(),
description=
"List of fields that should not change during an alert update")
default_group_field: str = odm.Keyword(
description="Default field used for alert grouping view")
delay: int = odm.Integer(
description=
"Time in seconds that we give extended scans and workflow to complete their work "
"before we start showing alerts in the alert viewer.")
filtering_group_fields: List[str] = odm.List(
odm.Keyword(),
description=
"List of group fields that when selected will ignore certain alerts where this field is missing."
)
non_filtering_group_fields: List[str] = odm.List(
odm.Keyword(),
description=
"List of group fields that are sure to be present in all alerts.")
process_alert_message: str = odm.Keyword(
description=
"Python path to the function that will process an alert message.")
threshold: int = odm.Integer(
description=
"Minimum score to reach for a submission to be considered an alert.")
class Certificate(odm.Model):
@odm.model(index=True, store=False)
class RSA_Info(odm.Model):
d_param = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
e_param = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
n_param = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
p_param = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
q_param = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
version = odm.Optional(odm.Integer())
subject = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
issuer = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
serial_number = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
key_size = odm.Optional(odm.Integer())
key_type = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
key_usage = odm.Optional(odm.List(odm.EmptyableKeyword(copyto="__text__")))
certificate_policies = odm.Optional(
odm.List(odm.EmptyableKeyword(copyto="__text__")))
ext_key_usage = odm.Optional(
odm.List(odm.EmptyableKeyword(copyto="__text__")))
valid_from = odm.Optional(odm.Date())
valid_to = odm.Optional(odm.Date())
signature = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
signature_algorithm = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
is_trusted = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
raw_hex = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
rsa_info = odm.Optional(odm.Compound(RSA_Info))
class ServiceDelta(odm.Model):
accepts = odm.Optional(odm.Keyword(), store=True)
rejects = odm.Optional(odm.Keyword(), store=True)
category = odm.Optional(odm.Keyword(), store=True, copyto="__text__")
config = odm.Optional(odm.Mapping(odm.Any()), index=False)
description = odm.Optional(odm.Text(), store=True, copyto="__text__")
default_result_classification = odm.Optional(odm.ClassificationString())
enabled = odm.Optional(odm.Boolean(), store=True)
is_external = odm.Optional(odm.Boolean())
licence_count = odm.Optional(odm.Integer())
name = odm.Optional(odm.Keyword(), store=True, copyto="__text__")
version = odm.Keyword(store=True)
disable_cache = odm.Optional(odm.Boolean())
stage = odm.Optional(odm.Keyword(), store=True, copyto="__text__")
submission_params = odm.Optional(odm.List(
odm.Compound(SubmissionParamsDelta)),
index=False)
timeout = odm.Optional(odm.Integer())
docker_config: DockerConfigDelta = odm.Optional(
odm.Compound(DockerConfigDelta))
dependencies: DependencyConfigDelta = odm.Mapping(
odm.Compound(DependencyConfigDelta), default={})
update_channel = odm.Optional(
odm.Enum(values=["stable", "rc", "beta", "dev"]))
update_config: UpdateConfigDelta = odm.Optional(
odm.Compound(UpdateConfigDelta))
class Internal(odm.Model):
enabled: bool = odm.Boolean()
failure_ttl: int = odm.Integer()
max_failures: int = odm.Integer()
password_requirements: PasswordRequirement = odm.Compound(
PasswordRequirement, default=DEFAULT_PASSWORD_REQUIREMENTS)
signup: Signup = odm.Compound(Signup, default=DEFAULT_SIGNUP)
class UserSettings(odm.Model): # User's default settings
classification = odm.Classification(default=Classification.UNRESTRICTED
) # Default submission classification
deep_scan = odm.Boolean(default=False) # Should a deep scan be performed
description = odm.Keyword(default="") # Default description
download_encoding = odm.Enum(
values=ENCODINGS,
default="cart") # Default download encoding when downloading files
expand_min_score = odm.Integer(
default=500) # Auto-expand section when score bigger then this
ignore_cache = odm.Boolean(default=False) # Ignore service caching
ignore_dynamic_recursion_prevention = odm.Boolean(
default=False) # Ignore dynamic recursion prevention
ignore_filtering = odm.Boolean(default=False) # Ignore filtering services
priority = odm.Integer(
default=1000) # Default priority for the submissions
profile = odm.Boolean(
default=False) # Should the submission do extra profiling
service_spec = odm.Mapping(odm.Keyword(),
default={}) # Default service specific settings
services = odm.Compound(ServiceSelection,
default={}) # Default service selection
submission_view = odm.Enum(
values=VIEWS,
default="report") # Default view for completed submissions
ttl = odm.Integer(default=0) # Default submission Time to Live (days)
class IngestTask(odm.Model):
# Submission Parameters
submission: Submission = odm.Compound(Submission)
# Shortcut for properties of the submission
@property
def file_size(self) -> int:
return sum(file.size for file in self.submission.files)
@property
def params(self) -> SubmissionParams:
return self.submission.params
@property
def sha256(self) -> str:
return self.submission.files[0].sha256
# Information about the ingestion itself, parameters irrelevant
scan_key = odm.Optional(odm.Keyword()) # the filescore key
retries = odm.Integer(default=0)
# Fields added after a submission is complete for notification/bookkeeping processes
failure = odm.Text(
default='') # If the ingestion has failed for some reason, what is it?
score = odm.Optional(
odm.Integer()) # Score from previous processing of this file
extended_scan = odm.Enum(EXTENDED_SCAN_VALUES, default="skipped")
ingest_id = odm.UUID()
ingest_time = odm.Date(default="NOW")
class Submission(odm.Model):
archive_ts = odm.Date(store=False) # Archiving timestamp
classification = odm.Classification() # Classification of the submission
error_count = odm.Integer() # Total number of errors in the submission
errors = odm.List(odm.Keyword(), store=False) # List of error keys
expiry_ts = odm.Optional(odm.Date(store=False)) # Expiry timestamp
file_count = odm.Integer() # Total number of files in the submission
files: List[File] = odm.List(
odm.Compound(File)) # List of files that were originally submitted
max_score = odm.Integer() # Maximum score of all the files in the scan
metadata = odm.FlattenedObject(
store=False) # Metadata associated to the submission
params: SubmissionParams = odm.Compound(
SubmissionParams) # Submission detail blocs
results: List[str] = odm.List(odm.Keyword(),
store=False) # List of result keys
sid = odm.UUID(copyto="__text__") # Submission ID
state = odm.Enum(values=SUBMISSION_STATES) # Status of the submission
times = odm.Compound(Times, default={}) # Timing bloc
verdict = odm.Compound(Verdict, default={}) # Verdict timing
def is_submit(self):
return self.state == 'submitted'
def is_complete(self):
return self.state == 'completed'
def is_initial(self):
return self.is_submit() and not self.params.psid
class Task(odm.Model):
sid = odm.UUID()
fileinfo: FileInfo = odm.Compound(FileInfo) # File info block
filename = odm.Keyword()
service_name = odm.Keyword()
service_config = odm.Mapping(odm.Any(), default={}) # Service specific parameters
depth = odm.Integer(default=0)
max_files = odm.Integer()
ttl = odm.Integer(default=0)
tags = odm.List(odm.Compound(TagItem), default=[])
temporary_submission_data = odm.List(odm.Compound(DataItem), default=[])
deep_scan = odm.Boolean(default=False)
# Whether the service cache should be ignored during the processing of this task
ignore_cache = odm.Boolean(default=False)
# Priority for processing order
priority = odm.Integer(default=0)
@staticmethod
def make_key(sid, service_name, sha):
return f"{sid}_{service_name}_{sha}"
def key(self):
return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class Logging(odm.Model):
# What level of logging should we have
log_level: str = odm.Enum(
values=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", "DISABLED"])
# Should we log to console?
log_to_console: bool = odm.Boolean()
# Should we log to files on the server?
log_to_file: bool = odm.Boolean()
# if yes, what is the log directory
log_directory: str = odm.Keyword()
# Should logs be sent to a syslog server?
log_to_syslog: bool = odm.Boolean()
# if yes, what is the syslog server hostname/ip?
syslog_host: str = odm.Keyword()
syslog_port: int = odm.Integer()
# How often should counters log their values (seconds)
export_interval: int = odm.Integer()
# Log in JSON format
log_as_json: bool = odm.Boolean()
# If set, core components will touch this path regularly to tell the container
# environment it is healthy
heartbeat_file: str = odm.Optional(odm.Keyword())
class Task(odm.Model):
sid = odm.UUID()
metadata = odm.FlattenedObject() # Metadata associated to the submission
min_classification = odm.Classification(
) # Minimum classification of the file being scanned
fileinfo: FileInfo = odm.Compound(FileInfo) # File info block
filename = odm.Keyword()
service_name = odm.Keyword()
service_config = odm.Mapping(odm.Any(),
default={}) # Service specific parameters
depth = odm.Integer(default=0)
max_files = odm.Integer()
ttl = odm.Integer(default=0)
tags = odm.List(odm.Compound(TagItem), default=[])
temporary_submission_data = odm.List(odm.Compound(DataItem), default=[])
deep_scan = odm.Boolean(default=False)
# Whether the service cache should be ignored during the processing of this task
ignore_cache = odm.Boolean(default=False)
# Whether the service should ignore the dynamic recursion prevention or not
ignore_dynamic_recursion_prevention = odm.Boolean(default=False)
# Priority for processing order
priority = odm.Integer(default=0)
@staticmethod
def make_key(sid, service_name, sha):
return f"{sid}_{service_name}_{sha}"
def key(self):
return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class Queues(odm.Model):
ingest = odm.Integer(description="Number of submissions in ingest queue")
start = odm.List(odm.Integer(),
description="Number of submissions that started")
result = odm.List(odm.Integer(), description="Number of results in queue")
command = odm.List(odm.Integer(),
description="Number of commands in queue")
class Statistics(odm.Model):
count = odm.Integer(default=0)
min = odm.Integer(default=0)
max = odm.Integer(default=0)
avg = odm.Integer(default=0)
sum = odm.Integer(default=0)
first_hit = odm.Optional(odm.Date())
last_hit = odm.Optional(odm.Date())
class Alerter(odm.Model):
alert_ttl: int = odm.Integer()
constant_alert_fields: List[str] = odm.List(odm.Keyword())
default_group_field: str = odm.Keyword()
delay: int = odm.Integer()
filtering_group_fields: List[str] = odm.List(odm.Keyword())
non_filtering_group_fields: List[str] = odm.List(odm.Keyword())
process_alert_message: str = odm.Keyword()
class Statistics(odm.Model):
count = odm.Integer(default=0, description="Count of statistical hits")
min = odm.Integer(default=0, description="Minimum value of all stastical hits")
max = odm.Integer(default=0, description="Maximum value of all stastical hits")
avg = odm.Integer(default=0, description="Anerage of all stastical hits")
sum = odm.Integer(default=0, description="Sum of all stastical hits")
first_hit = odm.Optional(odm.Date(), description="Date of first hit of statistic")
last_hit = odm.Optional(odm.Date(), description="Date of last hit of statistic")
class ScalerServiceDefaults(odm.Model):
"""A set of default values to be used running a service when no other value is set."""
growth: int = odm.Integer()
shrink: int = odm.Integer()
backlog: int = odm.Integer()
min_instances: int = odm.Integer()
environment: List[EnvironmentVariable] = odm.List(
odm.Compound(EnvironmentVariable), default=[])
class Task(odm.Model):
sid = odm.UUID(description="Submission ID")
metadata = odm.FlattenedObject(
description="Metadata associated to the submission")
min_classification = odm.Classification(
description="Minimum classification of the file being scanned")
fileinfo: FileInfo = odm.Compound(FileInfo, description="File info block")
filename = odm.Keyword(description="File name")
service_name = odm.Keyword(description="Service name")
service_config = odm.Mapping(odm.Any(),
default={},
description="Service specific parameters")
depth = odm.Integer(
default=0,
description="File depth relative to initital submitted file")
max_files = odm.Integer(
description="Maximum number of files that submission can have")
ttl = odm.Integer(default=0, description="Task TTL")
tags = odm.List(odm.Compound(TagItem),
default=[],
description="List of tags")
temporary_submission_data = odm.List(
odm.Compound(DataItem),
default=[],
description="Temporary submission data")
deep_scan = odm.Boolean(default=False, description="Perform deep scanning")
ignore_cache = odm.Boolean(
default=False,
description=
"Whether the service cache should be ignored during the processing of this task"
)
ignore_dynamic_recursion_prevention = odm.Boolean(
default=False,
description=
"Whether the service should ignore the dynamic recursion prevention or not"
)
ignore_filtering = odm.Boolean(
default=False, description="Should the service filter it's output?")
priority = odm.Integer(default=0,
description="Priority for processing order")
safelist_config = odm.Compound(
ServiceSafelist,
description=
"Safelisting configuration (as defined in global configuration)",
default={'enabled': False})
@staticmethod
def make_key(sid, service_name, sha):
return f"{sid}_{service_name}_{sha}"
def key(self):
return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class ScalerProfile(odm.Model):
"""Minimal description for an assemblyline core component controlled by the scaler."""
growth: int = odm.Optional(odm.Integer())
shrink: int = odm.Optional(odm.Integer())
backlog: int = odm.Optional(odm.Integer())
min_instances: int = odm.Optional(odm.Integer())
max_instances: int = odm.Optional(odm.Integer())
queue: str = odm.Keyword()
container_config: DockerConfig = odm.Compound(DockerConfig)
class Import(odm.Model):
library = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
data = odm.Optional(odm.Integer())
hint = odm.Optional(odm.Integer())
iat_address = odm.Optional(odm.Integer())
iat_value = odm.Optional(odm.Integer())
is_ordinal = odm.Optional(odm.Boolean())
name = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
ordinal = odm.Optional(odm.Integer())
class User(odm.Model):
agrees_with_tos = odm.Optional(
odm.Date(index=False, store=False),
description="Date the user agree with terms of service")
api_quota = odm.Integer(
default=10,
store=False,
description="Maximum number of concurrent API requests")
apikeys = odm.Mapping(odm.Compound(ApiKey),
default={},
index=False,
store=False,
description="Mapping of API keys")
apps = odm.Mapping(odm.Compound(Apps),
default={},
index=False,
store=False,
description="Applications with access to the account")
can_impersonate = odm.Boolean(
default=False,
index=False,
store=False,
description="Allowed to query on behalf of others?")
classification = odm.Classification(
is_user_classification=True,
copyto="__text__",
default=Classification.UNRESTRICTED,
description="Maximum classification for the user")
dn = odm.Optional(odm.Keyword(store=False, copyto="__text__"),
description="User's LDAP DN")
email = odm.Optional(odm.Email(copyto="__text__"),
description="User's email address")
groups = odm.List(odm.Keyword(),
copyto="__text__",
default=["USERS"],
description="List of groups the user submits to")
is_active = odm.Boolean(default=True, description="Is the user active?")
name = odm.Keyword(copyto="__text__", description="Full name of the user")
otp_sk = odm.Optional(
odm.Keyword(index=False, store=False),
description="Secret key to generate one time passwords")
password = odm.Keyword(index=False,
store=False,
description="BCrypt hash of the user's password")
submission_quota = odm.Integer(
default=5,
store=False,
description="Maximum number of concurrent submissions")
type = odm.List(odm.Enum(values=USER_TYPES),
default=['user'],
description="Type of user")
security_tokens = odm.Mapping(odm.Keyword(),
index=False,
store=False,
default={},
description="Map of security tokens")
uname = odm.Keyword(copyto="__text__", description="Username")
class FileScore(odm.Model):
psid = odm.Optional(
odm.UUID()) # ID of the parent submission to the associated submission
expiry_ts = odm.Date(
index=True) # Expiry timestamp, used for garbage collection.
score = odm.Integer() # Maximum score for the associated submission
errors = odm.Integer(
) # Number of errors that occurred during the previous analysis
sid = odm.UUID() # ID of the associated submission
time = odm.Float() # Epoch time at which the FileScore entry was created
class VM(odm.Model):
enabled = odm.Boolean(default=True)
name = odm.Keyword(copyto="__text__")
num_workers = odm.Integer(default=1)
os_type = odm.Enum(values=OS_TYPES)
os_variant = odm.Enum(values=OS_VARIANTS)
ram = odm.Integer(default=1024)
revert_every = odm.Integer(default=600)
vcpus = odm.Integer(default=1)
virtual_disk_url = odm.Keyword()
class Header(odm.Model):
characteristics_hash = odm.Optional(odm.Integer())
characteristics_list = odm.Optional(
odm.List(odm.EmptyableKeyword(copyto="__text__")))
machine = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
numberof_sections = odm.Optional(odm.Integer())
numberof_symbols = odm.Optional(odm.Integer())
signature = odm.Optional(odm.List(odm.Integer()))
timestamp = odm.Optional(odm.Integer())
hr_timestamp = odm.Optional(odm.Date())
