class FileScore(odm.Model):
psid = odm.Optional(
odm.UUID()) # ID of the parent submission to the associated submission
expiry_ts = odm.Date(
index=True) # Expiry timestamp, used for garbage collection.
score = odm.Integer() # Maximum score for the associated submission
errors = odm.Integer(
) # Number of errors that occurred during the previous analysis
sid = odm.UUID() # ID of the associated submission
time = odm.Float() # Epoch time at which the FileScore entry was created
class FileScore(odm.Model):
psid = odm.Optional(
odm.UUID(),
description=" Parent submission ID of the associated submission")
expiry_ts = odm.Date(
index=True,
description="Expiry timestamp, used for garbage collection")
score = odm.Integer(
description="Maximum score for the associated submission")
errors = odm.Integer(
description=
"Number of errors that occurred during the previous analysis")
sid = odm.UUID(description="ID of the associated submission")
time = odm.Float(
description="Epoch time at which the FileScore entry was created")
class Alert(odm.Model):
alert_id = odm.Keyword(copyto="__text__") # ID of the alert
al = odm.Compound(ALResults) # Assemblyline result block
archive_ts = odm.Date(store=False) # Archiving timestamp
attack = odm.Compound(Attack) # Attack result block
classification = odm.Classification() # Classification of the alert
expiry_ts = odm.Optional(odm.Date(store=False)) # Expiry timestamp
extended_scan = odm.Enum(values=EXTENDED_SCAN_VALUES,
store=False) # Status of the extended scan
file = odm.Compound(File) # File block
filtered = odm.Boolean(default=False) # Are the alert result filtered
heuristic = odm.Compound(Heuristic) # Heuristic result block
label = odm.List(odm.Keyword(), copyto="__text__",
default=[]) # List of labels applied to the alert
metadata = odm.FlattenedObject(
default={}, store=False) # Metadata submitted with the file
owner = odm.Optional(odm.Keyword()) # Owner of the alert
priority = odm.Optional(
odm.Enum(values=PRIORITIES)) # Priority applied to the alert
reporting_ts = odm.Date() # Time at which the alert was created
sid = odm.UUID(store=False) # ID of the submission related to this alert
status = odm.Optional(
odm.Enum(values=STATUSES)) # Status applied to the alert
ts = odm.Date() # Timestamp at which the file was submitted
type = odm.Keyword() # Type of alert
verdict = odm.Compound(Verdict, default={}) # Verdict timing
class IngestTask(odm.Model):
# Submission Parameters
submission: Submission = odm.Compound(Submission)
# Shortcut for properties of the submission
@property
def file_size(self) -> int:
return sum(file.size for file in self.submission.files)
@property
def params(self) -> SubmissionParams:
return self.submission.params
@property
def sha256(self) -> str:
return self.submission.files[0].sha256
# Information about the ingestion itself, parameters irrelevant
scan_key = odm.Optional(odm.Keyword()) # the filescore key
retries = odm.Integer(default=0)
# Fields added after a submission is complete for notification/bookkeeping processes
failure = odm.Text(
default='') # If the ingestion has failed for some reason, what is it?
score = odm.Optional(
odm.Integer()) # Score from previous processing of this file
extended_scan = odm.Enum(EXTENDED_SCAN_VALUES, default="skipped")
ingest_id = odm.UUID()
ingest_time = odm.Date(default="NOW")
class Task(odm.Model):
sid = odm.UUID()
fileinfo: FileInfo = odm.Compound(FileInfo) # File info block
filename = odm.Keyword()
service_name = odm.Keyword()
service_config = odm.Mapping(odm.Any(), default={}) # Service specific parameters
depth = odm.Integer(default=0)
max_files = odm.Integer()
ttl = odm.Integer(default=0)
tags = odm.List(odm.Compound(TagItem), default=[])
temporary_submission_data = odm.List(odm.Compound(DataItem), default=[])
deep_scan = odm.Boolean(default=False)
# Whether the service cache should be ignored during the processing of this task
ignore_cache = odm.Boolean(default=False)
# Priority for processing order
priority = odm.Integer(default=0)
@staticmethod
def make_key(sid, service_name, sha):
return f"{sid}_{service_name}_{sha}"
def key(self):
return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class Submission(odm.Model):
archive_ts = odm.Date(store=False) # Archiving timestamp
classification = odm.Classification() # Classification of the submission
error_count = odm.Integer() # Total number of errors in the submission
errors = odm.List(odm.Keyword(), store=False) # List of error keys
expiry_ts = odm.Optional(odm.Date(store=False)) # Expiry timestamp
file_count = odm.Integer() # Total number of files in the submission
files: List[File] = odm.List(
odm.Compound(File)) # List of files that were originally submitted
max_score = odm.Integer() # Maximum score of all the files in the scan
metadata = odm.FlattenedObject(
store=False) # Metadata associated to the submission
params: SubmissionParams = odm.Compound(
SubmissionParams) # Submission detail blocs
results: List[str] = odm.List(odm.Keyword(),
store=False) # List of result keys
sid = odm.UUID(copyto="__text__") # Submission ID
state = odm.Enum(values=SUBMISSION_STATES) # Status of the submission
times = odm.Compound(Times, default={}) # Timing bloc
verdict = odm.Compound(Verdict, default={}) # Verdict timing
def is_submit(self):
return self.state == 'submitted'
def is_complete(self):
return self.state == 'completed'
def is_initial(self):
return self.is_submit() and not self.params.psid
class Task(odm.Model):
sid = odm.UUID()
metadata = odm.FlattenedObject() # Metadata associated to the submission
min_classification = odm.Classification(
) # Minimum classification of the file being scanned
fileinfo: FileInfo = odm.Compound(FileInfo) # File info block
filename = odm.Keyword()
service_name = odm.Keyword()
service_config = odm.Mapping(odm.Any(),
default={}) # Service specific parameters
depth = odm.Integer(default=0)
max_files = odm.Integer()
ttl = odm.Integer(default=0)
tags = odm.List(odm.Compound(TagItem), default=[])
temporary_submission_data = odm.List(odm.Compound(DataItem), default=[])
deep_scan = odm.Boolean(default=False)
# Whether the service cache should be ignored during the processing of this task
ignore_cache = odm.Boolean(default=False)
# Whether the service should ignore the dynamic recursion prevention or not
ignore_dynamic_recursion_prevention = odm.Boolean(default=False)
# Priority for processing order
priority = odm.Integer(default=0)
@staticmethod
def make_key(sid, service_name, sha):
return f"{sid}_{service_name}_{sha}"
def key(self):
return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class Submission(odm.Model):
sid = odm.UUID() # Submission ID to use
time = odm.Date(default="NOW")
files: List[File] = odm.List(odm.Compound(File), default=[]) # File block
metadata: Dict[str, str] = odm.FlattenedObject(default={}) # Metadata submitted with the file
notification: Notification = odm.Compound(Notification, default={}) # Notification queue parameters
params: SubmissionParams = odm.Compound(SubmissionParams) # Parameters of the submission
class Task(odm.Model):
sid = odm.UUID(description="Submission ID")
metadata = odm.FlattenedObject(
description="Metadata associated to the submission")
min_classification = odm.Classification(
description="Minimum classification of the file being scanned")
fileinfo: FileInfo = odm.Compound(FileInfo, description="File info block")
filename = odm.Keyword(description="File name")
service_name = odm.Keyword(description="Service name")
service_config = odm.Mapping(odm.Any(),
default={},
description="Service specific parameters")
depth = odm.Integer(
default=0,
description="File depth relative to initital submitted file")
max_files = odm.Integer(
description="Maximum number of files that submission can have")
ttl = odm.Integer(default=0, description="Task TTL")
tags = odm.List(odm.Compound(TagItem),
default=[],
description="List of tags")
temporary_submission_data = odm.List(
odm.Compound(DataItem),
default=[],
description="Temporary submission data")
deep_scan = odm.Boolean(default=False, description="Perform deep scanning")
ignore_cache = odm.Boolean(
default=False,
description=
"Whether the service cache should be ignored during the processing of this task"
)
ignore_dynamic_recursion_prevention = odm.Boolean(
default=False,
description=
"Whether the service should ignore the dynamic recursion prevention or not"
)
ignore_filtering = odm.Boolean(
default=False, description="Should the service filter it's output?")
priority = odm.Integer(default=0,
description="Priority for processing order")
safelist_config = odm.Compound(
ServiceSafelist,
description=
"Safelisting configuration (as defined in global configuration)",
default={'enabled': False})
@staticmethod
def make_key(sid, service_name, sha):
return f"{sid}_{service_name}_{sha}"
def key(self):
return Task.make_key(self.sid, self.service_name, self.fileinfo.sha256)
class Submission(odm.Model):
sid = odm.UUID(description="Submission ID to use")
time = odm.Date(default="NOW", description="Message time")
files: List[File] = odm.List(odm.Compound(File),
default=[],
description="File block")
metadata: Dict[str, str] = odm.FlattenedObject(
default={}, description="Metadata submitted with the file")
notification: Notification = odm.Compound(
Notification, default={}, description="Notification queue parameters")
params: SubmissionParams = odm.Compound(
SubmissionParams, description="Parameters of the submission")
scan_key: Opt[str] = odm.Optional(odm.Keyword())
class Submission(odm.Model):
archive_ts = odm.Date(store=False, description="Archiving timestamp")
classification = odm.Classification(
description="Classification of the submission")
error_count = odm.Integer(
description="Total number of errors in the submission")
errors: list[str] = odm.List(odm.Keyword(),
store=False,
description="List of error keys")
expiry_ts = odm.Optional(odm.Date(store=False),
description="Expiry timestamp")
file_count = odm.Integer(
description="Total number of files in the submission")
files: list[File] = odm.List(
odm.Compound(File),
description="List of files that were originally submitted")
max_score = odm.Integer(
description="Maximum score of all the files in the scan")
metadata = odm.FlattenedObject(
store=False, description="Metadata associated to the submission")
params: SubmissionParams = odm.Compound(
SubmissionParams, description="Submission parameter details")
results: list[str] = odm.List(odm.Keyword(),
store=False,
description="List of result keys")
sid = odm.UUID(copyto="__text__", description="Submission ID")
state = odm.Enum(values=SUBMISSION_STATES,
description="Status of the submission")
times = odm.Compound(Times,
default={},
description="Submission-specific times")
verdict = odm.Compound(Verdict,
default={},
description="Malicious verdict details")
# the filescore key, used in deduplication. This is a non-unique key, that is
# shared by submissions that may be processed as duplicates.
scan_key = odm.Optional(odm.Keyword(store=False, index=False))
def is_submit(self):
return self.state == 'submitted'
def is_complete(self):
return self.state == 'completed'
def is_initial(self):
return self.is_submit() and not self.params.psid
class Workflow(odm.Model):
classification = odm.Classification(
copyto="__text__",
default=Classification.UNRESTRICTED) # Classification of the workflow
creation_date = odm.Date(default="NOW") # Creation date of the workflow
creator = odm.Keyword() # UID of the creator
edited_by = odm.Keyword() # UID of the last edit user
hit_count = odm.Integer(default=0) # Number of time workflow hit
labels = odm.List(odm.Keyword(), copyto="__text__",
default=[]) # Labels applied by the workflow
last_edit = odm.Date(default="NOW") # Last edit date
last_seen = odm.Optional(odm.Date()) # Last hit date
name = odm.Keyword(copyto="__text__") # Name of the workflow
priority = odm.Optional(
odm.Enum(copyto="__text__",
values=PRIORITIES)) # Priority applied by the workflow
query = odm.Keyword() # Query that the workflow runs
status = odm.Optional(odm.Enum(
copyto="__text__", values=STATUSES)) # Status applied by the workflow
workflow_id = odm.Optional(odm.UUID()) # ID of the workflow
class Alert(odm.Model):
alert_id = odm.Keyword(copyto="__text__", description="ID of the alert")
al = odm.Compound(ALResults, description="Assemblyline Result Block")
archive_ts = odm.Date(store=False, description="Archiving timestamp")
attack = odm.Compound(Attack, description="ATT&CK Block")
classification = odm.Classification(description="Classification of the alert")
expiry_ts = odm.Optional(odm.Date(store=False), description="Expiry timestamp")
extended_scan = odm.Enum(values=EXTENDED_SCAN_VALUES, description="Status of the extended scan")
file = odm.Compound(File, description="File Block")
filtered = odm.Boolean(default=False, description="Are the alert results filtered?")
heuristic = odm.Compound(Heuristic, description="Heuristic Block")
label = odm.List(odm.Keyword(), copyto="__text__", default=[], description="List of labels applied to the alert")
metadata = odm.FlattenedObject(default={}, store=False, description="Metadata submitted with the file")
owner = odm.Optional(odm.Keyword(), description="Owner of the alert")
priority = odm.Optional(odm.Enum(values=PRIORITIES), description="Priority applied to the alert")
reporting_ts = odm.Date(description="Alert creation timestamp")
sid = odm.UUID(description="Submission ID related to this alert")
status = odm.Optional(odm.Enum(values=STATUSES), description="Status applied to the alert")
ts = odm.Date(description="File submission timestamp")
type = odm.Keyword(description="Type of alert")
verdict = odm.Compound(Verdict, default={}, description="Verdict Block")
workflows_completed = odm.Boolean(default=False, description="Have all workflows ran on this alert?")
class SubmissionParams(odm.Model):
classification = odm.Classification(
default=Classification.UNRESTRICTED,
description="Original classification of the submission")
deep_scan = odm.Boolean(default=False,
description="Should a deep scan be performed?")
description = odm.Text(store=True,
copyto="__text__",
description="Description of the submission")
generate_alert = odm.Boolean(
default=False, description="Should this submission generate an alert?")
groups = odm.List(odm.Keyword(),
default=["USERS"],
description="List of groups related to this scan")
ignore_cache = odm.Boolean(
default=False, description="Ignore the cached service results?")
ignore_dynamic_recursion_prevention = odm.Boolean(
default=False,
description="Should we ignore dynamic recursion prevention?")
ignore_filtering = odm.Boolean(
default=False, description="Should we ignore filtering services?")
ignore_size = odm.Boolean(default=False,
description="Ignore the file size limits?")
never_drop = odm.Boolean(
default=False, description="Exempt from being dropped by ingester?")
malicious = odm.Boolean(
default=False,
description="Is the file submitted already known to be malicious?")
max_extracted = odm.Integer(default=500,
description="Max number of extracted files")
max_supplementary = odm.Integer(
default=500, description="Max number of supplementary files")
priority = odm.Integer(default=1000, description="Priority of the scan")
profile = odm.Boolean(
default=False, description="Should the submission do extra profiling?")
psid = odm.Optional(odm.UUID(), description="Parent submission ID")
quota_item = odm.Boolean(
default=False, description="Does this submission count against quota?")
services = odm.Compound(ServiceSelection,
default={},
description="Service selection")
service_spec = odm.Mapping(odm.Mapping(odm.Any()),
default={},
index=False,
store=False,
description="Service-specific parameters")
submitter = odm.Keyword(store=True,
copyto="__text__",
description="User who submitted the file")
ttl = odm.Integer(default=0,
description="Time, in days, to live for this submission")
type = odm.Keyword(default="USER", description="Type of submission")
initial_data = odm.Optional(
odm.Text(index=False),
description="Initialization for temporary submission data")
def get_hashing_keys(self):
"""Get the sections of the submission parameters that should be used in result hashes."""
data = self.as_primitives()
return {k: v for k, v in data.items() if k in _KEY_HASHED_FIELDS}
def create_filescore_key(self, sha256, services: list = None):
"""This is the key used to store the final score of a submission for fast lookup.
This lookup is one of the methods used to check for duplication in ingestion process,
so this key is fairly sensitive.
"""
# TODO do we need this version thing still be here?
# One up this if the cache is ever messed up and we
# need to quickly invalidate all old cache entries.
version = 0
if services is None:
services = self.services.selected
data = self.get_hashing_keys()
data['service_spec'] = sorted(
(key, sorted(values.items()))
for key, values in self.service_spec.items())
data['sha256'] = sha256
data['services'] = [str(x) for x in services]
s = ', '.join([f"{k}: {data[k]}" for k in sorted(data.keys())])
return 'v'.join(
[str(hashlib.md5(s.encode()).hexdigest()),
str(version)])
