class SubmissionParamsDelta(odm.Model): default = odm.Optional( odm.Any(), description= "Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)" ) name = odm.Optional( odm.Keyword(), description= "Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)" ) type = odm.Optional( odm.Enum(values=['str', 'int', 'list', 'bool']), description= "Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)" ) value = odm.Optional( odm.Any(), description= "Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)" ) list = odm.Optional( odm.Any(), description= "Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)" ) hide = odm.Optional( odm.Boolean(), description= "Refer to:<br>[Service - SubmissionParams](../service/#submissionparams)" )
class SubmissionParams(odm.Model): default = odm.Any( description="Default value (must match value in `value` field)") name = odm.Keyword(description="Name of parameter") type = odm.Enum(values=['str', 'int', 'list', 'bool'], description="Type of parameter") value = odm.Any( description="Default value (must match value in `default` field)") list = odm.Optional(odm.Any(), description="List of values if `type: list`") hide = odm.Boolean(default=False, description="Should this parameter be hidden?")
class Service(odm.Model): # Regexes applied to assemblyline style file type string accepts = odm.Keyword(store=True, default=DEFAULT_SERVICE_ACCEPTS) rejects = odm.Optional( odm.Keyword(store=True, default=DEFAULT_SERVICE_REJECTS)) category = odm.Keyword(store=True, default="Static Analysis", copyto="__text__") config = odm.Mapping(odm.Any(), default={}, index=False, store=False) description = odm.Text(store=True, default="NA", copyto="__text__") default_result_classification = odm.ClassificationString( default=Classification.UNRESTRICTED) enabled = odm.Boolean(store=True, default=False) is_external = odm.Boolean(default=False) licence_count = odm.Integer(default=0) name = odm.Keyword(store=True, copyto="__text__") version = odm.Keyword(store=True) # Should the result cache be disabled for this service disable_cache = odm.Boolean(default=False) stage = odm.Keyword(store=True, default="CORE", copyto="__text__") submission_params: SubmissionParams = odm.List( odm.Compound(SubmissionParams), index=False, default=[]) timeout = odm.Integer(default=60) docker_config: DockerConfig = odm.Compound(DockerConfig) dependencies = odm.Mapping(odm.Compound(DependencyConfig), default={}) update_channel: str = odm.Enum(values=["stable", "rc", "beta", "dev"], default='stable') update_config: UpdateConfig = odm.Optional(odm.Compound(UpdateConfig))
class Task(odm.Model): sid = odm.UUID() fileinfo: FileInfo = odm.Compound(FileInfo) # File info block filename = odm.Keyword() service_name = odm.Keyword() service_config = odm.Mapping(odm.Any(), default={}) # Service specific parameters depth = odm.Integer(default=0) max_files = odm.Integer() ttl = odm.Integer(default=0) tags = odm.List(odm.Compound(TagItem), default=[]) temporary_submission_data = odm.List(odm.Compound(DataItem), default=[]) deep_scan = odm.Boolean(default=False) # Whether the service cache should be ignored during the processing of this task ignore_cache = odm.Boolean(default=False) # Priority for processing order priority = odm.Integer(default=0) @staticmethod def make_key(sid, service_name, sha): return f"{sid}_{service_name}_{sha}" def key(self): return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class Task(odm.Model): sid = odm.UUID() metadata = odm.FlattenedObject() # Metadata associated to the submission min_classification = odm.Classification( ) # Minimum classification of the file being scanned fileinfo: FileInfo = odm.Compound(FileInfo) # File info block filename = odm.Keyword() service_name = odm.Keyword() service_config = odm.Mapping(odm.Any(), default={}) # Service specific parameters depth = odm.Integer(default=0) max_files = odm.Integer() ttl = odm.Integer(default=0) tags = odm.List(odm.Compound(TagItem), default=[]) temporary_submission_data = odm.List(odm.Compound(DataItem), default=[]) deep_scan = odm.Boolean(default=False) # Whether the service cache should be ignored during the processing of this task ignore_cache = odm.Boolean(default=False) # Whether the service should ignore the dynamic recursion prevention or not ignore_dynamic_recursion_prevention = odm.Boolean(default=False) # Priority for processing order priority = odm.Integer(default=0) @staticmethod def make_key(sid, service_name, sha): return f"{sid}_{service_name}_{sha}" def key(self): return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class ServiceDelta(odm.Model): accepts = odm.Optional(odm.Keyword(), store=True) rejects = odm.Optional(odm.Keyword(), store=True) category = odm.Optional(odm.Keyword(), store=True, copyto="__text__") config = odm.Optional(odm.Mapping(odm.Any()), index=False) description = odm.Optional(odm.Text(), store=True, copyto="__text__") default_result_classification = odm.Optional(odm.ClassificationString()) enabled = odm.Optional(odm.Boolean(), store=True) is_external = odm.Optional(odm.Boolean()) licence_count = odm.Optional(odm.Integer()) name = odm.Optional(odm.Keyword(), store=True, copyto="__text__") version = odm.Keyword(store=True) disable_cache = odm.Optional(odm.Boolean()) stage = odm.Optional(odm.Keyword(), store=True, copyto="__text__") submission_params = odm.Optional(odm.List( odm.Compound(SubmissionParamsDelta)), index=False) timeout = odm.Optional(odm.Integer()) docker_config: DockerConfigDelta = odm.Optional( odm.Compound(DockerConfigDelta)) dependencies: DependencyConfigDelta = odm.Mapping( odm.Compound(DependencyConfigDelta), default={}) update_channel = odm.Optional( odm.Enum(values=["stable", "rc", "beta", "dev"])) update_config: UpdateConfigDelta = odm.Optional( odm.Compound(UpdateConfigDelta))
class Task(odm.Model): sid = odm.UUID(description="Submission ID") metadata = odm.FlattenedObject( description="Metadata associated to the submission") min_classification = odm.Classification( description="Minimum classification of the file being scanned") fileinfo: FileInfo = odm.Compound(FileInfo, description="File info block") filename = odm.Keyword(description="File name") service_name = odm.Keyword(description="Service name") service_config = odm.Mapping(odm.Any(), default={}, description="Service specific parameters") depth = odm.Integer( default=0, description="File depth relative to initital submitted file") max_files = odm.Integer( description="Maximum number of files that submission can have") ttl = odm.Integer(default=0, description="Task TTL") tags = odm.List(odm.Compound(TagItem), default=[], description="List of tags") temporary_submission_data = odm.List( odm.Compound(DataItem), default=[], description="Temporary submission data") deep_scan = odm.Boolean(default=False, description="Perform deep scanning") ignore_cache = odm.Boolean( default=False, description= "Whether the service cache should be ignored during the processing of this task" ) ignore_dynamic_recursion_prevention = odm.Boolean( default=False, description= "Whether the service should ignore the dynamic recursion prevention or not" ) ignore_filtering = odm.Boolean( default=False, description="Should the service filter it's output?") priority = odm.Integer(default=0, description="Priority for processing order") safelist_config = odm.Compound( ServiceSafelist, description= "Safelisting configuration (as defined in global configuration)", default={'enabled': False}) @staticmethod def make_key(sid, service_name, sha): return f"{sid}_{service_name}_{sha}" def key(self): return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class LDAP(odm.Model): enabled: bool = odm.Boolean() admin_dn: str = odm.Optional(odm.Keyword()) auto_create: bool = odm.Boolean() auto_sync: bool = odm.Boolean() base: str = odm.Keyword() classification_mappings: Dict[str, str] = odm.Any() email_field: str = odm.Keyword() group_lookup_query: str = odm.Keyword() image_field: str = odm.Keyword() image_format: str = odm.Keyword() name_field: str = odm.Keyword() signature_importer_dn: str = odm.Optional(odm.Keyword()) signature_manager_dn: str = odm.Optional(odm.Keyword()) uid_field: str = odm.Keyword() uri: str = odm.Keyword()
class UserSettings(odm.Model): classification = odm.Classification( default=Classification.UNRESTRICTED, description="Default submission classification") deep_scan = odm.Boolean(default=False, description="Should a deep scan be performed?") description = odm.Keyword(default="", description="Default description") download_encoding = odm.Enum( values=ENCODINGS, default="cart", description="Default download encoding when downloading files") default_zip_password = odm.Text( default="zippy", description= "Default user-defined password for creating password protected ZIPs when downloading files" ) expand_min_score = odm.Integer( default=500, description="Auto-expand section when score bigger then this") ignore_cache = odm.Boolean(default=False, description="Ignore service caching?") ignore_dynamic_recursion_prevention = odm.Boolean( default=False, description="Ignore dynamic recursion prevention?") ignore_filtering = odm.Boolean(default=False, description="Ignore filtering services?") malicious = odm.Boolean( default=False, description="Is the file submitted already known to be malicious?") priority = odm.Integer(default=1000, description="Default priority for the submissions") profile = odm.Boolean( default=False, description="Should the submission do extra profiling?") service_spec = odm.Mapping(odm.Mapping(odm.Any()), default={}, description="Default service specific settings") services = odm.Compound(ServiceSelection, default={}, description="Default service selection") submission_view = odm.Enum( values=VIEWS, default="report", description="Default view for completed submissions") ttl = odm.Integer(default=30, description="Default submission TTL, in days")
class ServiceClient(odm.Model): """Session data for a service worker. This is not saved in the datastore, and its not shared with any other components """ client_id: str = odm.Keyword() # Session ID of the client container_id: str = odm.Keyword() # Docker container ID of the client ip: str = odm.IP() # IP address of the client service_name: str = odm.Keyword( ) # Name of the service running on the client service_version: str = odm.Keyword( ) # Version of the service running on the client service_tool_version: Opt[str] = odm.Optional( odm.Keyword()) # Tool version of the service running on the client service_timeout: int = odm.Integer( ) # Timeout of the service running on the client current: Opt[Current] = odm.Optional(odm.Compound( Current)) # Info about the current status and task assigned tasking_counters = odm.Optional( odm.Any()) # MetricsFactory counters for the service
class LDAP(odm.Model): enabled: bool = odm.Boolean(description="Should LDAP be enabled or not?") admin_dn: str = odm.Optional( odm.Keyword(), description="DN of the group or the user who will get admin privileges" ) bind_user: str = odm.Optional( odm.Keyword(), description="User use to query the LDAP server") bind_pass: str = odm.Optional( odm.Keyword(), description="Password used to query the LDAP server") auto_create: bool = odm.Boolean( description="Auto-create users if they are missing") auto_sync: bool = odm.Boolean( description= "Should we automatically sync with LDAP server on each login?") base: str = odm.Keyword(description="Base DN for the users") classification_mappings: Dict[str, str] = odm.Any( description="Classification mapping") email_field: str = odm.Keyword( description="Name of the field containing the email address") group_lookup_query: str = odm.Keyword( description="How the group lookup is queried") image_field: str = odm.Keyword( description="Name of the field containing the user's avatar") image_format: str = odm.Keyword( description="Type of image used to store the avatar") name_field: str = odm.Keyword( description="Name of the field containing the user's name") signature_importer_dn: str = odm.Optional( odm.Keyword(), description= "DN of the group or the user who will get signature_importer role") signature_manager_dn: str = odm.Optional( odm.Keyword(), description= "DN of the group or the user who will get signature_manager role") uid_field: str = odm.Keyword(description="Field name for the UID") uri: str = odm.Keyword(description="URI to the LDAP server")
class SubmissionParamsDelta(odm.Model): default = odm.Optional(odm.Any()) name = odm.Optional(odm.Keyword()) type = odm.Optional(odm.Enum(values=['str', 'int', 'list', 'bool'])) value = odm.Optional(odm.Any()) list = odm.Optional(odm.Any())
class ServiceDelta(odm.Model): accepts = odm.Optional( odm.Keyword(), store=True, description="Refer to:<br>[Service](../service/#service)") rejects = odm.Optional( odm.Keyword(), store=True, description="Refer to:<br>[Service](../service/#service)") category = odm.Optional( odm.Keyword(), store=True, copyto="__text__", description="Refer to:<br>[Service](../service/#service)") config = odm.Optional( odm.Mapping(odm.Any()), index=False, description="Refer to:<br>[Service](../service/#service)") description = odm.Optional( odm.Text(), store=True, copyto="__text__", description="Refer to:<br>[Service](../service/#service)") default_result_classification = odm.Optional( odm.ClassificationString(), description="Refer to:<br>[Service](../service/#service)") enabled = odm.Optional( odm.Boolean(), store=True, description="Refer to:<br>[Service](../service/#service)") is_external = odm.Optional( odm.Boolean(), description="Refer to:<br>[Service](../service/#service)") licence_count = odm.Optional( odm.Integer(), description="Refer to:<br>[Service](../service/#service)") max_queue_length = odm.Optional( odm.Integer(), description="Refer to:<br>[Service](../service/#service)") uses_tags: bool = odm.Optional( odm.Boolean(), description="Refer to:<br>[Service](../service/#service)") uses_temp_submission_data: bool = odm.Optional( odm.Boolean(), description="Refer to:<br>[Service](../service/#service)") uses_metadata: bool = odm.Optional( odm.Boolean(), description="Refer to:<br>[Service](../service/#service)") name = odm.Optional( odm.Keyword(), store=True, copyto="__text__", description="Refer to:<br>[Service](../service/#service)") version = odm.Keyword( store=True, description="Refer to:<br>[Service](../service/#service)") privileged = odm.Optional( odm.Boolean(), description="Refer to:<br>[Service](../service/#service)") disable_cache = odm.Optional( odm.Boolean(), description="Refer to:<br>[Service](../service/#service)") stage = odm.Optional( odm.Keyword(), store=True, copyto="__text__", description="Refer to:<br>[Service](../service/#service)") submission_params = odm.Optional( odm.List(odm.Compound(SubmissionParamsDelta)), index=False, description="Refer to:<br>[Service](../service/#service)") timeout = odm.Optional( odm.Integer(), description="Refer to:<br>[Service](../service/#service)") docker_config: DockerConfigDelta = odm.Optional( odm.Compound(DockerConfigDelta), description="Refer to:<br>[Service](../service/#service)") dependencies: DependencyConfigDelta = odm.Mapping( odm.Compound(DependencyConfigDelta), default={}, description="Refer to:<br>[Service](../service/#service)") update_channel = odm.Optional( odm.Enum(values=["stable", "rc", "beta", "dev"]), description="Refer to:<br>[Service](../service/#service)") update_config: UpdateConfigDelta = odm.Optional( odm.Compound(UpdateConfigDelta), description="Refer to:<br>[Service](../service/#service)")
class DataItem(odm.Model): name = odm.Keyword() value = odm.Any()
class Test(odm.Model): number = odm.Integer() other = odm.Any(index=False)
class DispatcherCommandMessage(odm.Model): kind: str = odm.Enum(values=list(MESSAGE_CLASSES.keys())) payload_data = odm.Any() def payload(self): return MESSAGE_CLASSES[self.kind](self.payload_data)
class SubmissionParams(odm.Model): classification = odm.Classification( default=Classification.UNRESTRICTED, description="Original classification of the submission") deep_scan = odm.Boolean(default=False, description="Should a deep scan be performed?") description = odm.Text(store=True, copyto="__text__", description="Description of the submission") generate_alert = odm.Boolean( default=False, description="Should this submission generate an alert?") groups = odm.List(odm.Keyword(), default=["USERS"], description="List of groups related to this scan") ignore_cache = odm.Boolean( default=False, description="Ignore the cached service results?") ignore_dynamic_recursion_prevention = odm.Boolean( default=False, description="Should we ignore dynamic recursion prevention?") ignore_filtering = odm.Boolean( default=False, description="Should we ignore filtering services?") ignore_size = odm.Boolean(default=False, description="Ignore the file size limits?") never_drop = odm.Boolean( default=False, description="Exempt from being dropped by ingester?") malicious = odm.Boolean( default=False, description="Is the file submitted already known to be malicious?") max_extracted = odm.Integer(default=500, description="Max number of extracted files") max_supplementary = odm.Integer( default=500, description="Max number of supplementary files") priority = odm.Integer(default=1000, description="Priority of the scan") profile = odm.Boolean( default=False, description="Should the submission do extra profiling?") psid = odm.Optional(odm.UUID(), description="Parent submission ID") quota_item = odm.Boolean( default=False, description="Does this submission count against quota?") services = odm.Compound(ServiceSelection, default={}, description="Service selection") service_spec = odm.Mapping(odm.Mapping(odm.Any()), default={}, index=False, store=False, description="Service-specific parameters") submitter = odm.Keyword(store=True, copyto="__text__", description="User who submitted the file") ttl = odm.Integer(default=0, description="Time, in days, to live for this submission") type = odm.Keyword(default="USER", description="Type of submission") initial_data = odm.Optional( odm.Text(index=False), description="Initialization for temporary submission data") def get_hashing_keys(self): """Get the sections of the submission parameters that should be used in result hashes.""" data = self.as_primitives() return {k: v for k, v in data.items() if k in _KEY_HASHED_FIELDS} def create_filescore_key(self, sha256, services: list = None): """This is the key used to store the final score of a submission for fast lookup. This lookup is one of the methods used to check for duplication in ingestion process, so this key is fairly sensitive. """ # TODO do we need this version thing still be here? # One up this if the cache is ever messed up and we # need to quickly invalidate all old cache entries. version = 0 if services is None: services = self.services.selected data = self.get_hashing_keys() data['service_spec'] = sorted( (key, sorted(values.items())) for key, values in self.service_spec.items()) data['sha256'] = sha256 data['services'] = [str(x) for x in services] s = ', '.join([f"{k}: {data[k]}" for k in sorted(data.keys())]) return 'v'.join( [str(hashlib.md5(s.encode()).hexdigest()), str(version)])
class Service(odm.Model): # Regexes applied to assemblyline style file type string accepts = odm.Keyword( store=True, default=DEFAULT_SERVICE_ACCEPTS, description="Regex to accept files as identified by Assemblyline") rejects = odm.Optional( odm.Keyword(store=True, default=DEFAULT_SERVICE_REJECTS), description="Regex to reject files as identified by Assemblyline") category = odm.Keyword( store=True, default="Static Analysis", copyto="__text__", description="Which category does this service belong to?") config = odm.Mapping(odm.Any(), default={}, index=False, store=False, description="Service Configuration") description = odm.Text(store=True, default="NA", copyto="__text__", description="Description of service") default_result_classification = odm.ClassificationString( default=Classification.UNRESTRICTED, description="Default classification assigned to service results") enabled: bool = odm.Boolean( store=True, default=False, description="Is the service enabled (by default)?") is_external: bool = odm.Boolean( default=False, description= "Does this service perform analysis outside of Assemblyline?") licence_count: int = odm.Integer( default=0, description="How many licences is the service allowed to use?") max_queue_length: int = odm.Integer( default=0, description= "If more than this many jobs are queued for this service drop those over this limit. 0 is unlimited." ) uses_tags: bool = odm.Boolean( default=False, description= "Does this service use tags from other services for analysis?") uses_temp_submission_data: bool = odm.Boolean( default=False, description= "Does this service use temp data from other services for analysis?") uses_metadata: bool = odm.Boolean( default=False, description="Does this service use submission metadata for analysis?") name: str = odm.Keyword(store=True, copyto="__text__", description="Name of service") version = odm.Keyword(store=True, description="Version of service") privileged = odm.Boolean( default=False, description= "Should the service be able to talk to core infrastructure or just service-server for tasking?" ) disable_cache = odm.Boolean( default=False, description="Should the result cache be disabled for this service?") stage = odm.Keyword( store=True, default="CORE", copyto="__text__", description="Which execution stage does this service run in?") submission_params: SubmissionParams = odm.List( odm.Compound(SubmissionParams), index=False, default=[], description="Submission parameters of service") timeout: int = odm.Integer(default=60, description="Service task timeout, in seconds") docker_config: DockerConfig = odm.Compound( DockerConfig, description="Docker configuration for service") dependencies: dict[str, DependencyConfig] = odm.Mapping( odm.Compound(DependencyConfig), default={}, description="Dependency configuration for service") update_channel: str = odm.Enum( values=["stable", "rc", "beta", "dev"], default='stable', description="What channel to watch for service updates?") update_config: UpdateConfig = odm.Optional( odm.Compound(UpdateConfig), description="Update configuration for fetching external resources")
class DispatcherCommandMessage(odm.Model): kind: str = odm.Enum(values=list(MESSAGE_CLASSES.keys()), description="Kind of message") payload_data = odm.Any(description="Message payload") def payload(self): return MESSAGE_CLASSES[self.kind](self.payload_data)
class SubmissionParams(odm.Model): default = odm.Any() name = odm.Keyword() type = odm.Enum(values=['str', 'int', 'list', 'bool']) value = odm.Any()