class FileScore(odm.Model): psid = odm.Optional( odm.UUID()) # ID of the parent submission to the associated submission expiry_ts = odm.Date( index=True) # Expiry timestamp, used for garbage collection. score = odm.Integer() # Maximum score for the associated submission errors = odm.Integer( ) # Number of errors that occurred during the previous analysis sid = odm.UUID() # ID of the associated submission time = odm.Float() # Epoch time at which the FileScore entry was created
class FileScore(odm.Model): psid = odm.Optional( odm.UUID(), description=" Parent submission ID of the associated submission") expiry_ts = odm.Date( index=True, description="Expiry timestamp, used for garbage collection") score = odm.Integer( description="Maximum score for the associated submission") errors = odm.Integer( description= "Number of errors that occurred during the previous analysis") sid = odm.UUID(description="ID of the associated submission") time = odm.Float( description="Epoch time at which the FileScore entry was created")
class Alert(odm.Model): alert_id = odm.Keyword(copyto="__text__") # ID of the alert al = odm.Compound(ALResults) # Assemblyline result block archive_ts = odm.Date(store=False) # Archiving timestamp attack = odm.Compound(Attack) # Attack result block classification = odm.Classification() # Classification of the alert expiry_ts = odm.Optional(odm.Date(store=False)) # Expiry timestamp extended_scan = odm.Enum(values=EXTENDED_SCAN_VALUES, store=False) # Status of the extended scan file = odm.Compound(File) # File block filtered = odm.Boolean(default=False) # Are the alert result filtered heuristic = odm.Compound(Heuristic) # Heuristic result block label = odm.List(odm.Keyword(), copyto="__text__", default=[]) # List of labels applied to the alert metadata = odm.FlattenedObject( default={}, store=False) # Metadata submitted with the file owner = odm.Optional(odm.Keyword()) # Owner of the alert priority = odm.Optional( odm.Enum(values=PRIORITIES)) # Priority applied to the alert reporting_ts = odm.Date() # Time at which the alert was created sid = odm.UUID(store=False) # ID of the submission related to this alert status = odm.Optional( odm.Enum(values=STATUSES)) # Status applied to the alert ts = odm.Date() # Timestamp at which the file was submitted type = odm.Keyword() # Type of alert verdict = odm.Compound(Verdict, default={}) # Verdict timing
class IngestTask(odm.Model): # Submission Parameters submission: Submission = odm.Compound(Submission) # Shortcut for properties of the submission @property def file_size(self) -> int: return sum(file.size for file in self.submission.files) @property def params(self) -> SubmissionParams: return self.submission.params @property def sha256(self) -> str: return self.submission.files[0].sha256 # Information about the ingestion itself, parameters irrelevant scan_key = odm.Optional(odm.Keyword()) # the filescore key retries = odm.Integer(default=0) # Fields added after a submission is complete for notification/bookkeeping processes failure = odm.Text( default='') # If the ingestion has failed for some reason, what is it? score = odm.Optional( odm.Integer()) # Score from previous processing of this file extended_scan = odm.Enum(EXTENDED_SCAN_VALUES, default="skipped") ingest_id = odm.UUID() ingest_time = odm.Date(default="NOW")
class Task(odm.Model): sid = odm.UUID() fileinfo: FileInfo = odm.Compound(FileInfo) # File info block filename = odm.Keyword() service_name = odm.Keyword() service_config = odm.Mapping(odm.Any(), default={}) # Service specific parameters depth = odm.Integer(default=0) max_files = odm.Integer() ttl = odm.Integer(default=0) tags = odm.List(odm.Compound(TagItem), default=[]) temporary_submission_data = odm.List(odm.Compound(DataItem), default=[]) deep_scan = odm.Boolean(default=False) # Whether the service cache should be ignored during the processing of this task ignore_cache = odm.Boolean(default=False) # Priority for processing order priority = odm.Integer(default=0) @staticmethod def make_key(sid, service_name, sha): return f"{sid}_{service_name}_{sha}" def key(self): return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class Submission(odm.Model): archive_ts = odm.Date(store=False) # Archiving timestamp classification = odm.Classification() # Classification of the submission error_count = odm.Integer() # Total number of errors in the submission errors = odm.List(odm.Keyword(), store=False) # List of error keys expiry_ts = odm.Optional(odm.Date(store=False)) # Expiry timestamp file_count = odm.Integer() # Total number of files in the submission files: List[File] = odm.List( odm.Compound(File)) # List of files that were originally submitted max_score = odm.Integer() # Maximum score of all the files in the scan metadata = odm.FlattenedObject( store=False) # Metadata associated to the submission params: SubmissionParams = odm.Compound( SubmissionParams) # Submission detail blocs results: List[str] = odm.List(odm.Keyword(), store=False) # List of result keys sid = odm.UUID(copyto="__text__") # Submission ID state = odm.Enum(values=SUBMISSION_STATES) # Status of the submission times = odm.Compound(Times, default={}) # Timing bloc verdict = odm.Compound(Verdict, default={}) # Verdict timing def is_submit(self): return self.state == 'submitted' def is_complete(self): return self.state == 'completed' def is_initial(self): return self.is_submit() and not self.params.psid
class Task(odm.Model): sid = odm.UUID() metadata = odm.FlattenedObject() # Metadata associated to the submission min_classification = odm.Classification( ) # Minimum classification of the file being scanned fileinfo: FileInfo = odm.Compound(FileInfo) # File info block filename = odm.Keyword() service_name = odm.Keyword() service_config = odm.Mapping(odm.Any(), default={}) # Service specific parameters depth = odm.Integer(default=0) max_files = odm.Integer() ttl = odm.Integer(default=0) tags = odm.List(odm.Compound(TagItem), default=[]) temporary_submission_data = odm.List(odm.Compound(DataItem), default=[]) deep_scan = odm.Boolean(default=False) # Whether the service cache should be ignored during the processing of this task ignore_cache = odm.Boolean(default=False) # Whether the service should ignore the dynamic recursion prevention or not ignore_dynamic_recursion_prevention = odm.Boolean(default=False) # Priority for processing order priority = odm.Integer(default=0) @staticmethod def make_key(sid, service_name, sha): return f"{sid}_{service_name}_{sha}" def key(self): return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class Submission(odm.Model): sid = odm.UUID() # Submission ID to use time = odm.Date(default="NOW") files: List[File] = odm.List(odm.Compound(File), default=[]) # File block metadata: Dict[str, str] = odm.FlattenedObject(default={}) # Metadata submitted with the file notification: Notification = odm.Compound(Notification, default={}) # Notification queue parameters params: SubmissionParams = odm.Compound(SubmissionParams) # Parameters of the submission
class Task(odm.Model): sid = odm.UUID(description="Submission ID") metadata = odm.FlattenedObject( description="Metadata associated to the submission") min_classification = odm.Classification( description="Minimum classification of the file being scanned") fileinfo: FileInfo = odm.Compound(FileInfo, description="File info block") filename = odm.Keyword(description="File name") service_name = odm.Keyword(description="Service name") service_config = odm.Mapping(odm.Any(), default={}, description="Service specific parameters") depth = odm.Integer( default=0, description="File depth relative to initital submitted file") max_files = odm.Integer( description="Maximum number of files that submission can have") ttl = odm.Integer(default=0, description="Task TTL") tags = odm.List(odm.Compound(TagItem), default=[], description="List of tags") temporary_submission_data = odm.List( odm.Compound(DataItem), default=[], description="Temporary submission data") deep_scan = odm.Boolean(default=False, description="Perform deep scanning") ignore_cache = odm.Boolean( default=False, description= "Whether the service cache should be ignored during the processing of this task" ) ignore_dynamic_recursion_prevention = odm.Boolean( default=False, description= "Whether the service should ignore the dynamic recursion prevention or not" ) ignore_filtering = odm.Boolean( default=False, description="Should the service filter it's output?") priority = odm.Integer(default=0, description="Priority for processing order") safelist_config = odm.Compound( ServiceSafelist, description= "Safelisting configuration (as defined in global configuration)", default={'enabled': False}) @staticmethod def make_key(sid, service_name, sha): return f"{sid}_{service_name}_{sha}" def key(self): return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class Submission(odm.Model): sid = odm.UUID(description="Submission ID to use") time = odm.Date(default="NOW", description="Message time") files: List[File] = odm.List(odm.Compound(File), default=[], description="File block") metadata: Dict[str, str] = odm.FlattenedObject( default={}, description="Metadata submitted with the file") notification: Notification = odm.Compound( Notification, default={}, description="Notification queue parameters") params: SubmissionParams = odm.Compound( SubmissionParams, description="Parameters of the submission") scan_key: Opt[str] = odm.Optional(odm.Keyword())
class Submission(odm.Model): archive_ts = odm.Date(store=False, description="Archiving timestamp") classification = odm.Classification( description="Classification of the submission") error_count = odm.Integer( description="Total number of errors in the submission") errors: list[str] = odm.List(odm.Keyword(), store=False, description="List of error keys") expiry_ts = odm.Optional(odm.Date(store=False), description="Expiry timestamp") file_count = odm.Integer( description="Total number of files in the submission") files: list[File] = odm.List( odm.Compound(File), description="List of files that were originally submitted") max_score = odm.Integer( description="Maximum score of all the files in the scan") metadata = odm.FlattenedObject( store=False, description="Metadata associated to the submission") params: SubmissionParams = odm.Compound( SubmissionParams, description="Submission parameter details") results: list[str] = odm.List(odm.Keyword(), store=False, description="List of result keys") sid = odm.UUID(copyto="__text__", description="Submission ID") state = odm.Enum(values=SUBMISSION_STATES, description="Status of the submission") times = odm.Compound(Times, default={}, description="Submission-specific times") verdict = odm.Compound(Verdict, default={}, description="Malicious verdict details") # the filescore key, used in deduplication. This is a non-unique key, that is # shared by submissions that may be processed as duplicates. scan_key = odm.Optional(odm.Keyword(store=False, index=False)) def is_submit(self): return self.state == 'submitted' def is_complete(self): return self.state == 'completed' def is_initial(self): return self.is_submit() and not self.params.psid
class Workflow(odm.Model): classification = odm.Classification( copyto="__text__", default=Classification.UNRESTRICTED) # Classification of the workflow creation_date = odm.Date(default="NOW") # Creation date of the workflow creator = odm.Keyword() # UID of the creator edited_by = odm.Keyword() # UID of the last edit user hit_count = odm.Integer(default=0) # Number of time workflow hit labels = odm.List(odm.Keyword(), copyto="__text__", default=[]) # Labels applied by the workflow last_edit = odm.Date(default="NOW") # Last edit date last_seen = odm.Optional(odm.Date()) # Last hit date name = odm.Keyword(copyto="__text__") # Name of the workflow priority = odm.Optional( odm.Enum(copyto="__text__", values=PRIORITIES)) # Priority applied by the workflow query = odm.Keyword() # Query that the workflow runs status = odm.Optional(odm.Enum( copyto="__text__", values=STATUSES)) # Status applied by the workflow workflow_id = odm.Optional(odm.UUID()) # ID of the workflow
class Alert(odm.Model): alert_id = odm.Keyword(copyto="__text__", description="ID of the alert") al = odm.Compound(ALResults, description="Assemblyline Result Block") archive_ts = odm.Date(store=False, description="Archiving timestamp") attack = odm.Compound(Attack, description="ATT&CK Block") classification = odm.Classification(description="Classification of the alert") expiry_ts = odm.Optional(odm.Date(store=False), description="Expiry timestamp") extended_scan = odm.Enum(values=EXTENDED_SCAN_VALUES, description="Status of the extended scan") file = odm.Compound(File, description="File Block") filtered = odm.Boolean(default=False, description="Are the alert results filtered?") heuristic = odm.Compound(Heuristic, description="Heuristic Block") label = odm.List(odm.Keyword(), copyto="__text__", default=[], description="List of labels applied to the alert") metadata = odm.FlattenedObject(default={}, store=False, description="Metadata submitted with the file") owner = odm.Optional(odm.Keyword(), description="Owner of the alert") priority = odm.Optional(odm.Enum(values=PRIORITIES), description="Priority applied to the alert") reporting_ts = odm.Date(description="Alert creation timestamp") sid = odm.UUID(description="Submission ID related to this alert") status = odm.Optional(odm.Enum(values=STATUSES), description="Status applied to the alert") ts = odm.Date(description="File submission timestamp") type = odm.Keyword(description="Type of alert") verdict = odm.Compound(Verdict, default={}, description="Verdict Block") workflows_completed = odm.Boolean(default=False, description="Have all workflows ran on this alert?")
class SubmissionParams(odm.Model): classification = odm.Classification( default=Classification.UNRESTRICTED, description="Original classification of the submission") deep_scan = odm.Boolean(default=False, description="Should a deep scan be performed?") description = odm.Text(store=True, copyto="__text__", description="Description of the submission") generate_alert = odm.Boolean( default=False, description="Should this submission generate an alert?") groups = odm.List(odm.Keyword(), default=["USERS"], description="List of groups related to this scan") ignore_cache = odm.Boolean( default=False, description="Ignore the cached service results?") ignore_dynamic_recursion_prevention = odm.Boolean( default=False, description="Should we ignore dynamic recursion prevention?") ignore_filtering = odm.Boolean( default=False, description="Should we ignore filtering services?") ignore_size = odm.Boolean(default=False, description="Ignore the file size limits?") never_drop = odm.Boolean( default=False, description="Exempt from being dropped by ingester?") malicious = odm.Boolean( default=False, description="Is the file submitted already known to be malicious?") max_extracted = odm.Integer(default=500, description="Max number of extracted files") max_supplementary = odm.Integer( default=500, description="Max number of supplementary files") priority = odm.Integer(default=1000, description="Priority of the scan") profile = odm.Boolean( default=False, description="Should the submission do extra profiling?") psid = odm.Optional(odm.UUID(), description="Parent submission ID") quota_item = odm.Boolean( default=False, description="Does this submission count against quota?") services = odm.Compound(ServiceSelection, default={}, description="Service selection") service_spec = odm.Mapping(odm.Mapping(odm.Any()), default={}, index=False, store=False, description="Service-specific parameters") submitter = odm.Keyword(store=True, copyto="__text__", description="User who submitted the file") ttl = odm.Integer(default=0, description="Time, in days, to live for this submission") type = odm.Keyword(default="USER", description="Type of submission") initial_data = odm.Optional( odm.Text(index=False), description="Initialization for temporary submission data") def get_hashing_keys(self): """Get the sections of the submission parameters that should be used in result hashes.""" data = self.as_primitives() return {k: v for k, v in data.items() if k in _KEY_HASHED_FIELDS} def create_filescore_key(self, sha256, services: list = None): """This is the key used to store the final score of a submission for fast lookup. This lookup is one of the methods used to check for duplication in ingestion process, so this key is fairly sensitive. """ # TODO do we need this version thing still be here? # One up this if the cache is ever messed up and we # need to quickly invalidate all old cache entries. version = 0 if services is None: services = self.services.selected data = self.get_hashing_keys() data['service_spec'] = sorted( (key, sorted(values.items())) for key, values in self.service_spec.items()) data['sha256'] = sha256 data['services'] = [str(x) for x in services] s = ', '.join([f"{k}: {data[k]}" for k in sorted(data.keys())]) return 'v'.join( [str(hashlib.md5(s.encode()).hexdigest()), str(version)])