class Logging(odm.Model): log_level: str = odm.Enum( values=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", "DISABLED"], description="What level of logging should we have?") log_to_console: bool = odm.Boolean(description="Should we log to console?") log_to_file: bool = odm.Boolean( description="Should we log to files on the server?") log_directory: str = odm.Keyword( description= "If `log_to_file: true`, what is the directory to store logs?") log_to_syslog: bool = odm.Boolean( description="Should logs be sent to a syslog server?") syslog_host: str = odm.Keyword( description= "If `log_to_syslog: true`, provide hostname/IP of the syslog server?") syslog_port: int = odm.Integer( description= "If `log_to_syslog: true`, provide port of the syslog server?") export_interval: int = odm.Integer( description="How often, in seconds, should counters log their values?") log_as_json: bool = odm.Boolean(description="Log in JSON format?") heartbeat_file: str = odm.Optional( odm.Keyword(), description="Add a health check to core components.<br>" "If `true`, core components will touch this path regularly to tell the container environment it is healthy" )
class DockerConfig(odm.Model): allow_internet_access: bool = odm.Boolean( default=False, description="Does the container have internet-access?") command: Opt[list[str]] = odm.Optional( odm.List(odm.Keyword()), description="Command to run when container starts up.") cpu_cores: float = odm.Float(default=1.0, description="CPU allocation") environment: list[EnvironmentVariable] = odm.List( odm.Compound(EnvironmentVariable), default=[], description="Additional environemnt variables for the container") image: str = odm.Keyword( description= "Complete name of the Docker image with tag, may include registry") registry_username: Opt[str] = odm.Optional( odm.Keyword(), description="The username to use when pulling the image") registry_password: Opt[str] = odm.Optional( odm.Keyword(), description="The password or token to use when pulling the image") registry_type: str = odm.Enum(values=["docker", "harbor"], default='docker', description="The type of container registry") ports: list[str] = odm.List( odm.Keyword(), default=[], description="What ports of container to expose?") ram_mb: int = odm.Integer(default=512, description="Container RAM limit") ram_mb_min: int = odm.Integer(default=128, description="Container RAM request")
class Icon(odm.Model): icon_id = odm.Optional(odm.Integer()) planes = odm.Optional(odm.Integer()) height = odm.Optional(odm.Integer()) width = odm.Optional(odm.Integer()) lang = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) sublang = odm.Optional(odm.EmptyableKeyword(copyto="__text__"))
class ESMetrics(odm.Model): hosts: str = odm.Optional(odm.List(odm.Keyword())) host_certificates: str = odm.Optional(odm.Keyword()) warm: int = odm.Integer() cold: int = odm.Integer() delete: int = odm.Integer() unit = odm.Enum(['d', 'h', 'm'])
class Export(odm.Model): @odm.model(index=True, store=False) class Entry(odm.Model): @odm.model(index=True, store=False) class Forward_Information(odm.Model): function = odm.Optional( odm.EmptyableKeyword(copyto="__text__")) library = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) address = odm.Optional(odm.Integer()) forward_information = odm.Optional( odm.Compound(Forward_Information)) function_rva = odm.Optional(odm.Integer()) is_extern = odm.Optional(odm.Boolean()) name = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) ordinal = odm.Optional(odm.Integer()) entries = odm.Optional(odm.List(odm.Compound(Entry))) export_flags = odm.Optional(odm.Integer()) major_version = odm.Optional(odm.Integer()) minor_version = odm.Optional(odm.Integer()) name = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) ordinal_base = odm.Optional(odm.Integer()) timestamp = odm.Optional(odm.Integer()) hr_timestamp = odm.Optional(odm.Date())
class Service(odm.Model): # Regexes applied to assemblyline style file type string accepts = odm.Keyword(store=True, default=DEFAULT_SERVICE_ACCEPTS) rejects = odm.Optional( odm.Keyword(store=True, default=DEFAULT_SERVICE_REJECTS)) category = odm.Keyword(store=True, default="Static Analysis", copyto="__text__") config = odm.Mapping(odm.Any(), default={}, index=False, store=False) description = odm.Text(store=True, default="NA", copyto="__text__") default_result_classification = odm.ClassificationString( default=Classification.UNRESTRICTED) enabled = odm.Boolean(store=True, default=False) is_external = odm.Boolean(default=False) licence_count = odm.Integer(default=0) name = odm.Keyword(store=True, copyto="__text__") version = odm.Keyword(store=True) # Should the result cache be disabled for this service disable_cache = odm.Boolean(default=False) stage = odm.Keyword(store=True, default="CORE", copyto="__text__") submission_params: SubmissionParams = odm.List( odm.Compound(SubmissionParams), index=False, default=[]) timeout = odm.Integer(default=60) docker_config: DockerConfig = odm.Compound(DockerConfig) dependencies = odm.Mapping(odm.Compound(DependencyConfig), default={}) update_channel: str = odm.Enum(values=["stable", "rc", "beta", "dev"], default='stable') update_config: UpdateConfig = odm.Optional(odm.Compound(UpdateConfig))
class Dispatcher(odm.Model): timeout: float = odm.Integer( description= "Time between re-dispatching attempts, as long as some action (submission or any task completion) " "happens before this timeout ends, the timeout resets.") max_inflight: int = odm.Integer( description="Maximum submissions allowed to be in-flight")
class Metrics(odm.Model): files_completed = odm.Integer() submissions_completed = odm.Integer() cpu_seconds = PerformanceTimer() cpu_seconds_count = odm.Integer() busy_seconds = PerformanceTimer() busy_seconds_count = odm.Integer()
class Alerter(odm.Model): alert_ttl: int = odm.Integer( description="Time to live (days) for an alert in the system") constant_alert_fields: List[str] = odm.List( odm.Keyword(), description= "List of fields that should not change during an alert update") default_group_field: str = odm.Keyword( description="Default field used for alert grouping view") delay: int = odm.Integer( description= "Time in seconds that we give extended scans and workflow to complete their work " "before we start showing alerts in the alert viewer.") filtering_group_fields: List[str] = odm.List( odm.Keyword(), description= "List of group fields that when selected will ignore certain alerts where this field is missing." ) non_filtering_group_fields: List[str] = odm.List( odm.Keyword(), description= "List of group fields that are sure to be present in all alerts.") process_alert_message: str = odm.Keyword( description= "Python path to the function that will process an alert message.") threshold: int = odm.Integer( description= "Minimum score to reach for a submission to be considered an alert.")
class Certificate(odm.Model): @odm.model(index=True, store=False) class RSA_Info(odm.Model): d_param = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) e_param = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) n_param = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) p_param = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) q_param = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) version = odm.Optional(odm.Integer()) subject = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) issuer = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) serial_number = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) key_size = odm.Optional(odm.Integer()) key_type = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) key_usage = odm.Optional(odm.List(odm.EmptyableKeyword(copyto="__text__"))) certificate_policies = odm.Optional( odm.List(odm.EmptyableKeyword(copyto="__text__"))) ext_key_usage = odm.Optional( odm.List(odm.EmptyableKeyword(copyto="__text__"))) valid_from = odm.Optional(odm.Date()) valid_to = odm.Optional(odm.Date()) signature = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) signature_algorithm = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) is_trusted = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) raw_hex = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) rsa_info = odm.Optional(odm.Compound(RSA_Info))
class ServiceDelta(odm.Model): accepts = odm.Optional(odm.Keyword(), store=True) rejects = odm.Optional(odm.Keyword(), store=True) category = odm.Optional(odm.Keyword(), store=True, copyto="__text__") config = odm.Optional(odm.Mapping(odm.Any()), index=False) description = odm.Optional(odm.Text(), store=True, copyto="__text__") default_result_classification = odm.Optional(odm.ClassificationString()) enabled = odm.Optional(odm.Boolean(), store=True) is_external = odm.Optional(odm.Boolean()) licence_count = odm.Optional(odm.Integer()) name = odm.Optional(odm.Keyword(), store=True, copyto="__text__") version = odm.Keyword(store=True) disable_cache = odm.Optional(odm.Boolean()) stage = odm.Optional(odm.Keyword(), store=True, copyto="__text__") submission_params = odm.Optional(odm.List( odm.Compound(SubmissionParamsDelta)), index=False) timeout = odm.Optional(odm.Integer()) docker_config: DockerConfigDelta = odm.Optional( odm.Compound(DockerConfigDelta)) dependencies: DependencyConfigDelta = odm.Mapping( odm.Compound(DependencyConfigDelta), default={}) update_channel = odm.Optional( odm.Enum(values=["stable", "rc", "beta", "dev"])) update_config: UpdateConfigDelta = odm.Optional( odm.Compound(UpdateConfigDelta))
class Internal(odm.Model): enabled: bool = odm.Boolean() failure_ttl: int = odm.Integer() max_failures: int = odm.Integer() password_requirements: PasswordRequirement = odm.Compound( PasswordRequirement, default=DEFAULT_PASSWORD_REQUIREMENTS) signup: Signup = odm.Compound(Signup, default=DEFAULT_SIGNUP)
class UserSettings(odm.Model): # User's default settings classification = odm.Classification(default=Classification.UNRESTRICTED ) # Default submission classification deep_scan = odm.Boolean(default=False) # Should a deep scan be performed description = odm.Keyword(default="") # Default description download_encoding = odm.Enum( values=ENCODINGS, default="cart") # Default download encoding when downloading files expand_min_score = odm.Integer( default=500) # Auto-expand section when score bigger then this ignore_cache = odm.Boolean(default=False) # Ignore service caching ignore_dynamic_recursion_prevention = odm.Boolean( default=False) # Ignore dynamic recursion prevention ignore_filtering = odm.Boolean(default=False) # Ignore filtering services priority = odm.Integer( default=1000) # Default priority for the submissions profile = odm.Boolean( default=False) # Should the submission do extra profiling service_spec = odm.Mapping(odm.Keyword(), default={}) # Default service specific settings services = odm.Compound(ServiceSelection, default={}) # Default service selection submission_view = odm.Enum( values=VIEWS, default="report") # Default view for completed submissions ttl = odm.Integer(default=0) # Default submission Time to Live (days)
class IngestTask(odm.Model): # Submission Parameters submission: Submission = odm.Compound(Submission) # Shortcut for properties of the submission @property def file_size(self) -> int: return sum(file.size for file in self.submission.files) @property def params(self) -> SubmissionParams: return self.submission.params @property def sha256(self) -> str: return self.submission.files[0].sha256 # Information about the ingestion itself, parameters irrelevant scan_key = odm.Optional(odm.Keyword()) # the filescore key retries = odm.Integer(default=0) # Fields added after a submission is complete for notification/bookkeeping processes failure = odm.Text( default='') # If the ingestion has failed for some reason, what is it? score = odm.Optional( odm.Integer()) # Score from previous processing of this file extended_scan = odm.Enum(EXTENDED_SCAN_VALUES, default="skipped") ingest_id = odm.UUID() ingest_time = odm.Date(default="NOW")
class Submission(odm.Model): archive_ts = odm.Date(store=False) # Archiving timestamp classification = odm.Classification() # Classification of the submission error_count = odm.Integer() # Total number of errors in the submission errors = odm.List(odm.Keyword(), store=False) # List of error keys expiry_ts = odm.Optional(odm.Date(store=False)) # Expiry timestamp file_count = odm.Integer() # Total number of files in the submission files: List[File] = odm.List( odm.Compound(File)) # List of files that were originally submitted max_score = odm.Integer() # Maximum score of all the files in the scan metadata = odm.FlattenedObject( store=False) # Metadata associated to the submission params: SubmissionParams = odm.Compound( SubmissionParams) # Submission detail blocs results: List[str] = odm.List(odm.Keyword(), store=False) # List of result keys sid = odm.UUID(copyto="__text__") # Submission ID state = odm.Enum(values=SUBMISSION_STATES) # Status of the submission times = odm.Compound(Times, default={}) # Timing bloc verdict = odm.Compound(Verdict, default={}) # Verdict timing def is_submit(self): return self.state == 'submitted' def is_complete(self): return self.state == 'completed' def is_initial(self): return self.is_submit() and not self.params.psid
class Task(odm.Model): sid = odm.UUID() fileinfo: FileInfo = odm.Compound(FileInfo) # File info block filename = odm.Keyword() service_name = odm.Keyword() service_config = odm.Mapping(odm.Any(), default={}) # Service specific parameters depth = odm.Integer(default=0) max_files = odm.Integer() ttl = odm.Integer(default=0) tags = odm.List(odm.Compound(TagItem), default=[]) temporary_submission_data = odm.List(odm.Compound(DataItem), default=[]) deep_scan = odm.Boolean(default=False) # Whether the service cache should be ignored during the processing of this task ignore_cache = odm.Boolean(default=False) # Priority for processing order priority = odm.Integer(default=0) @staticmethod def make_key(sid, service_name, sha): return f"{sid}_{service_name}_{sha}" def key(self): return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class Logging(odm.Model): # What level of logging should we have log_level: str = odm.Enum( values=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", "DISABLED"]) # Should we log to console? log_to_console: bool = odm.Boolean() # Should we log to files on the server? log_to_file: bool = odm.Boolean() # if yes, what is the log directory log_directory: str = odm.Keyword() # Should logs be sent to a syslog server? log_to_syslog: bool = odm.Boolean() # if yes, what is the syslog server hostname/ip? syslog_host: str = odm.Keyword() syslog_port: int = odm.Integer() # How often should counters log their values (seconds) export_interval: int = odm.Integer() # Log in JSON format log_as_json: bool = odm.Boolean() # If set, core components will touch this path regularly to tell the container # environment it is healthy heartbeat_file: str = odm.Optional(odm.Keyword())
class Task(odm.Model): sid = odm.UUID() metadata = odm.FlattenedObject() # Metadata associated to the submission min_classification = odm.Classification( ) # Minimum classification of the file being scanned fileinfo: FileInfo = odm.Compound(FileInfo) # File info block filename = odm.Keyword() service_name = odm.Keyword() service_config = odm.Mapping(odm.Any(), default={}) # Service specific parameters depth = odm.Integer(default=0) max_files = odm.Integer() ttl = odm.Integer(default=0) tags = odm.List(odm.Compound(TagItem), default=[]) temporary_submission_data = odm.List(odm.Compound(DataItem), default=[]) deep_scan = odm.Boolean(default=False) # Whether the service cache should be ignored during the processing of this task ignore_cache = odm.Boolean(default=False) # Whether the service should ignore the dynamic recursion prevention or not ignore_dynamic_recursion_prevention = odm.Boolean(default=False) # Priority for processing order priority = odm.Integer(default=0) @staticmethod def make_key(sid, service_name, sha): return f"{sid}_{service_name}_{sha}" def key(self): return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class Queues(odm.Model): ingest = odm.Integer(description="Number of submissions in ingest queue") start = odm.List(odm.Integer(), description="Number of submissions that started") result = odm.List(odm.Integer(), description="Number of results in queue") command = odm.List(odm.Integer(), description="Number of commands in queue")
class Statistics(odm.Model): count = odm.Integer(default=0) min = odm.Integer(default=0) max = odm.Integer(default=0) avg = odm.Integer(default=0) sum = odm.Integer(default=0) first_hit = odm.Optional(odm.Date()) last_hit = odm.Optional(odm.Date())
class Alerter(odm.Model): alert_ttl: int = odm.Integer() constant_alert_fields: List[str] = odm.List(odm.Keyword()) default_group_field: str = odm.Keyword() delay: int = odm.Integer() filtering_group_fields: List[str] = odm.List(odm.Keyword()) non_filtering_group_fields: List[str] = odm.List(odm.Keyword()) process_alert_message: str = odm.Keyword()
class Statistics(odm.Model): count = odm.Integer(default=0, description="Count of statistical hits") min = odm.Integer(default=0, description="Minimum value of all stastical hits") max = odm.Integer(default=0, description="Maximum value of all stastical hits") avg = odm.Integer(default=0, description="Anerage of all stastical hits") sum = odm.Integer(default=0, description="Sum of all stastical hits") first_hit = odm.Optional(odm.Date(), description="Date of first hit of statistic") last_hit = odm.Optional(odm.Date(), description="Date of last hit of statistic")
class ScalerServiceDefaults(odm.Model): """A set of default values to be used running a service when no other value is set.""" growth: int = odm.Integer() shrink: int = odm.Integer() backlog: int = odm.Integer() min_instances: int = odm.Integer() environment: List[EnvironmentVariable] = odm.List( odm.Compound(EnvironmentVariable), default=[])
class Task(odm.Model): sid = odm.UUID(description="Submission ID") metadata = odm.FlattenedObject( description="Metadata associated to the submission") min_classification = odm.Classification( description="Minimum classification of the file being scanned") fileinfo: FileInfo = odm.Compound(FileInfo, description="File info block") filename = odm.Keyword(description="File name") service_name = odm.Keyword(description="Service name") service_config = odm.Mapping(odm.Any(), default={}, description="Service specific parameters") depth = odm.Integer( default=0, description="File depth relative to initital submitted file") max_files = odm.Integer( description="Maximum number of files that submission can have") ttl = odm.Integer(default=0, description="Task TTL") tags = odm.List(odm.Compound(TagItem), default=[], description="List of tags") temporary_submission_data = odm.List( odm.Compound(DataItem), default=[], description="Temporary submission data") deep_scan = odm.Boolean(default=False, description="Perform deep scanning") ignore_cache = odm.Boolean( default=False, description= "Whether the service cache should be ignored during the processing of this task" ) ignore_dynamic_recursion_prevention = odm.Boolean( default=False, description= "Whether the service should ignore the dynamic recursion prevention or not" ) ignore_filtering = odm.Boolean( default=False, description="Should the service filter it's output?") priority = odm.Integer(default=0, description="Priority for processing order") safelist_config = odm.Compound( ServiceSafelist, description= "Safelisting configuration (as defined in global configuration)", default={'enabled': False}) @staticmethod def make_key(sid, service_name, sha): return f"{sid}_{service_name}_{sha}" def key(self): return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class ScalerProfile(odm.Model): """Minimal description for an assemblyline core component controlled by the scaler.""" growth: int = odm.Optional(odm.Integer()) shrink: int = odm.Optional(odm.Integer()) backlog: int = odm.Optional(odm.Integer()) min_instances: int = odm.Optional(odm.Integer()) max_instances: int = odm.Optional(odm.Integer()) queue: str = odm.Keyword() container_config: DockerConfig = odm.Compound(DockerConfig)
class Import(odm.Model): library = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) data = odm.Optional(odm.Integer()) hint = odm.Optional(odm.Integer()) iat_address = odm.Optional(odm.Integer()) iat_value = odm.Optional(odm.Integer()) is_ordinal = odm.Optional(odm.Boolean()) name = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) ordinal = odm.Optional(odm.Integer())
class User(odm.Model): agrees_with_tos = odm.Optional( odm.Date(index=False, store=False), description="Date the user agree with terms of service") api_quota = odm.Integer( default=10, store=False, description="Maximum number of concurrent API requests") apikeys = odm.Mapping(odm.Compound(ApiKey), default={}, index=False, store=False, description="Mapping of API keys") apps = odm.Mapping(odm.Compound(Apps), default={}, index=False, store=False, description="Applications with access to the account") can_impersonate = odm.Boolean( default=False, index=False, store=False, description="Allowed to query on behalf of others?") classification = odm.Classification( is_user_classification=True, copyto="__text__", default=Classification.UNRESTRICTED, description="Maximum classification for the user") dn = odm.Optional(odm.Keyword(store=False, copyto="__text__"), description="User's LDAP DN") email = odm.Optional(odm.Email(copyto="__text__"), description="User's email address") groups = odm.List(odm.Keyword(), copyto="__text__", default=["USERS"], description="List of groups the user submits to") is_active = odm.Boolean(default=True, description="Is the user active?") name = odm.Keyword(copyto="__text__", description="Full name of the user") otp_sk = odm.Optional( odm.Keyword(index=False, store=False), description="Secret key to generate one time passwords") password = odm.Keyword(index=False, store=False, description="BCrypt hash of the user's password") submission_quota = odm.Integer( default=5, store=False, description="Maximum number of concurrent submissions") type = odm.List(odm.Enum(values=USER_TYPES), default=['user'], description="Type of user") security_tokens = odm.Mapping(odm.Keyword(), index=False, store=False, default={}, description="Map of security tokens") uname = odm.Keyword(copyto="__text__", description="Username")
class FileScore(odm.Model): psid = odm.Optional( odm.UUID()) # ID of the parent submission to the associated submission expiry_ts = odm.Date( index=True) # Expiry timestamp, used for garbage collection. score = odm.Integer() # Maximum score for the associated submission errors = odm.Integer( ) # Number of errors that occurred during the previous analysis sid = odm.UUID() # ID of the associated submission time = odm.Float() # Epoch time at which the FileScore entry was created
class VM(odm.Model): enabled = odm.Boolean(default=True) name = odm.Keyword(copyto="__text__") num_workers = odm.Integer(default=1) os_type = odm.Enum(values=OS_TYPES) os_variant = odm.Enum(values=OS_VARIANTS) ram = odm.Integer(default=1024) revert_every = odm.Integer(default=600) vcpus = odm.Integer(default=1) virtual_disk_url = odm.Keyword()
class Header(odm.Model): characteristics_hash = odm.Optional(odm.Integer()) characteristics_list = odm.Optional( odm.List(odm.EmptyableKeyword(copyto="__text__"))) machine = odm.Optional(odm.EmptyableKeyword(copyto="__text__")) numberof_sections = odm.Optional(odm.Integer()) numberof_symbols = odm.Optional(odm.Integer()) signature = odm.Optional(odm.List(odm.Integer())) timestamp = odm.Optional(odm.Integer()) hr_timestamp = odm.Optional(odm.Date())