class SubmissionSummary(odm.Model):
classification = odm.Classification(default=Classification.UNRESTRICTED) # Classification of the cache
filtered = odm.Boolean(default=False) # Has this cache entry been filtered
expiry_ts = odm.Date(index=True) # Expiry date
tags = odm.Text() # Tags cache
attack_matrix = odm.Text() # Att&ck Matrix cache
heuristics = odm.Text() # Heuristics cache
class Section(odm.Model):
body = odm.Optional(odm.Text(copyto="__text__")) # Text body of the result section
classification = odm.Classification() # Classification of the section
body_format = odm.Enum(values=BODY_FORMAT, index=False) # Type of body in this section
depth = odm.Integer(index=False) # Depth of the section
heuristic = odm.Optional(odm.Compound(Heuristic)) # Heuristic used to score result section
tags = odm.Compound(Tagging, default={}) # List of tags associated to this section
title_text = odm.Text(copyto="__text__") # Title of the section
class Subject(odm.Model):
ip = odm.Optional(odm.IP(), description="Subject's IP")
domain = odm.Optional(odm.Domain(), description="Subject's domain")
uri = odm.Optional(odm.URI(), description="Subject's URI")
process = odm.Optional(odm.Compound(Process),
description="Subject's process")
file = odm.Optional(odm.Text(), description="Subject's file")
registry = odm.Optional(odm.Text(),
description="Subject's registry key")
class ObjectID(odm.Model):
guid = odm.Text(description="The GUID associated with the object")
tag = odm.Optional(odm.Text(),
description="The normalized tag of the object")
treeid = odm.Optional(odm.Text(), description="The hash of the tree ID")
processtree = odm.Optional(
odm.Keyword(),
description="Human-readable tree ID (concatenation of tags)")
time_observed = odm.Optional(
odm.Date(), description="The time at which the object was observed")
class SubmissionTree(odm.Model):
classification = odm.Classification(
default=Classification.UNRESTRICTED,
description="Classification of the cache")
filtered = odm.Boolean(default=False,
description="Has this cache entry been filtered?")
expiry_ts = odm.Date(description="Expiry timestamp")
supplementary = odm.Text(index=False,
description="Tree of supplementary files")
tree = odm.Text(index=False, description="File tree cache")
class UI(odm.Model):
# Allow user to tell in advance the system that a file is malicious
allow_malicious_hinting: bool = odm.Boolean()
# Allow to user to download raw files
allow_raw_downloads: bool = odm.Boolean()
# Allow file submissions via url
allow_url_submissions: bool = odm.Boolean()
# Should API calls be audited and saved to a separate log file?
audit: bool = odm.Boolean()
# Banner message display on the main page (format: {<language_code>: message})
banner: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()))
# Banner message display on the main page (format: {<language_code>: message})
banner_level: str = odm.Enum(
values=["info", "warning", "success", "error"])
# Turn on debugging
debug: bool = odm.Boolean()
# Which encoding will be used
download_encoding = odm.Enum(values=["raw", "cart"])
# Assemblyline admins email address
email: str = odm.Optional(odm.Email())
# Enforce the user's quotas
enforce_quota: bool = odm.Boolean()
# Fully qualified domain name to use for the 2-factor authentication validation
fqdn: str = odm.Text()
# Maximum priority for ingest API
ingest_max_priority: int = odm.Integer()
# Turn on read only mode in the UI
read_only: bool = odm.Boolean()
# Offset of the read only mode for all paging and searches
read_only_offset: str = odm.Keyword(default="")
# Flask secret key to store cookies and stuff
secret_key: str = odm.Keyword()
# Duration of the user session before the user has to login again
session_duration: int = odm.Integer()
# Statistics configuration
statistics: Statistics = odm.Compound(Statistics,
default=DEFAULT_STATISTICS)
# Terms of service
tos: str = odm.Optional(odm.Text())
# Lock out user after accepting the terms of service
tos_lockout: bool = odm.Boolean()
# List of admins to notify when a user gets locked out
tos_lockout_notify: bool = odm.Optional(odm.List(odm.Keyword()))
# Headers that will be used by the url_download method
url_submission_headers: Dict[str, str] = odm.Optional(
odm.Mapping(odm.Keyword()))
# Proxy that will be used by the url_download method
url_submission_proxies: Dict[str, str] = odm.Optional(
odm.Mapping(odm.Keyword()))
# Validate if the session ip matches the ip the session was created from
validate_session_ip: bool = odm.Boolean()
# Validate if the session useragent matches the useragent the session was created with
validate_session_useragent: bool = odm.Boolean()
class SubmissionSummary(odm.Model):
classification = odm.Classification(
default=Classification.UNRESTRICTED,
description="Classification of the cache")
filtered = odm.Boolean(default=False,
description="Has this cache entry been filtered?")
expiry_ts = odm.Date(index=True, description="Expiry timestamp")
tags = odm.Text(description="Tags cache")
attack_matrix = odm.Text(description="ATT&CK Matrix cache")
heuristics = odm.Text(description="Heuristics cache")
heuristic_sections = odm.Text(
description="All sections mapping to the heuristics")
heuristic_name_map = odm.Text(description="Map of heuristic names to IDs")
class BaseTestModel(odm.Model):
classification = odm.Classification(default="UNRESTRICTED", yml_config=yml_config)
flavour = odm.Text(copyto='features', default="EMPTY")
height = odm.Integer()
no_store = odm.Optional(odm.Keyword(store=False))
no_index = odm.Optional(odm.Keyword(index=False, store=False))
dots = odm.Mapping(odm.Compound(Position), default={})
birthday = odm.Date()
tags = odm.List(odm.Enum({'silly', 'cats', '10'}), default=[], copyto='features')
size = odm.Compound(MeasurementModel, default={'depth': 100, 'width': 100})
features = odm.List(odm.Text(), default=[])
metadata = odm.Mapping(odm.Text(), default={})
things = odm.List(odm.Compound(ThingsModel), default=[])
class NetworkHTTP(odm.Model):
connection_details = odm.Compound(
NetworkConnection,
description="The low-level details of the HTTP request")
request_uri = odm.URI(description="The URI requested")
request_headers = odm.Mapping(
odm.Json(), description="Headers included in the request")
request_body = odm.Optional(odm.Text(),
description="The body of the request")
request_method = odm.Enum(
[
# Standard HTTP methods
"GET",
"POST",
"PUT",
"DELETE",
"HEAD",
"CONNECT",
"OPTIONS",
"TRACE",
"PATCH",
# WebDAV HTTP methods
"BCOPY",
"BDELETE",
"BMOVE",
"BPROPFIND",
"BPROPPATCH",
"COPY",
"DELETE",
"LOCK",
"MKCOL",
"MOVE",
"NOTIFY",
"POLL",
"PROPFIND",
"PROPPATCH",
"SEARCH",
"SUBSCRIBE",
"UNLOCK",
"UNSUBSCRIBE",
"X-MS-ENUMATTS"
],
description="The method of the request")
response_headers = odm.Mapping(
odm.Json(), description="Headers included in the response")
response_status_code = odm.Optional(
odm.Integer(), description="The status code of the response")
response_body = odm.Optional(odm.Text(),
description="The body of the response")
class Service(odm.Model):
# Regexes applied to assemblyline style file type string
accepts = odm.Keyword(store=True, default=DEFAULT_SERVICE_ACCEPTS)
rejects = odm.Optional(
odm.Keyword(store=True, default=DEFAULT_SERVICE_REJECTS))
category = odm.Keyword(store=True,
default="Static Analysis",
copyto="__text__")
config = odm.Mapping(odm.Any(), default={}, index=False, store=False)
description = odm.Text(store=True, default="NA", copyto="__text__")
default_result_classification = odm.ClassificationString(
default=Classification.UNRESTRICTED)
enabled = odm.Boolean(store=True, default=False)
is_external = odm.Boolean(default=False)
licence_count = odm.Integer(default=0)
name = odm.Keyword(store=True, copyto="__text__")
version = odm.Keyword(store=True)
# Should the result cache be disabled for this service
disable_cache = odm.Boolean(default=False)
stage = odm.Keyword(store=True, default="CORE", copyto="__text__")
submission_params: SubmissionParams = odm.List(
odm.Compound(SubmissionParams), index=False, default=[])
timeout = odm.Integer(default=60)
docker_config: DockerConfig = odm.Compound(DockerConfig)
dependencies = odm.Mapping(odm.Compound(DependencyConfig), default={})
update_channel: str = odm.Enum(values=["stable", "rc", "beta", "dev"],
default='stable')
update_config: UpdateConfig = odm.Optional(odm.Compound(UpdateConfig))
class System(odm.Model):
# Module path to the assemblyline constants
constants: str = odm.Keyword()
# Organisation acronym used for signatures
organisation: str = odm.Text()
# Type of system (production, staging, development)
type: str = odm.Enum(values=['production', 'staging', 'development'])
class System(odm.Model):
constants: str = odm.Keyword(
description="Module path to the assemblyline constants")
organisation: str = odm.Text(
description="Organisation acronym used for signatures")
type: str = odm.Enum(values=['production', 'staging', 'development'],
description="Type of system")
class ServiceDelta(odm.Model):
accepts = odm.Optional(odm.Keyword(), store=True)
rejects = odm.Optional(odm.Keyword(), store=True)
category = odm.Optional(odm.Keyword(), store=True, copyto="__text__")
config = odm.Optional(odm.Mapping(odm.Any()), index=False)
description = odm.Optional(odm.Text(), store=True, copyto="__text__")
default_result_classification = odm.Optional(odm.ClassificationString())
enabled = odm.Optional(odm.Boolean(), store=True)
is_external = odm.Optional(odm.Boolean())
licence_count = odm.Optional(odm.Integer())
name = odm.Optional(odm.Keyword(), store=True, copyto="__text__")
version = odm.Keyword(store=True)
disable_cache = odm.Optional(odm.Boolean())
stage = odm.Optional(odm.Keyword(), store=True, copyto="__text__")
submission_params = odm.Optional(odm.List(
odm.Compound(SubmissionParamsDelta)),
index=False)
timeout = odm.Optional(odm.Integer())
docker_config: DockerConfigDelta = odm.Optional(
odm.Compound(DockerConfigDelta))
dependencies: DependencyConfigDelta = odm.Mapping(
odm.Compound(DependencyConfigDelta), default={})
update_channel = odm.Optional(
odm.Enum(values=["stable", "rc", "beta", "dev"]))
update_config: UpdateConfigDelta = odm.Optional(
odm.Compound(UpdateConfigDelta))
class IngestTask(odm.Model):
# Submission Parameters
submission: Submission = odm.Compound(Submission)
# Shortcut for properties of the submission
@property
def file_size(self) -> int:
return sum(file.size for file in self.submission.files)
@property
def params(self) -> SubmissionParams:
return self.submission.params
@property
def sha256(self) -> str:
return self.submission.files[0].sha256
# Information about the ingestion itself, parameters irrelevant
scan_key = odm.Optional(odm.Keyword()) # the filescore key
retries = odm.Integer(default=0)
# Fields added after a submission is complete for notification/bookkeeping processes
failure = odm.Text(
default='') # If the ingestion has failed for some reason, what is it?
score = odm.Optional(
odm.Integer()) # Score from previous processing of this file
extended_scan = odm.Enum(EXTENDED_SCAN_VALUES, default="skipped")
ingest_id = odm.UUID()
ingest_time = odm.Date(default="NOW")
class Antivirus(odm.Model):
@odm.model(description="Antivirus Detection Model")
class Detection(odm.Model):
engine_name = odm.Keyword(description="Name of antivirus engine")
engine_version = odm.Optional(
odm.Keyword(), description="Version of antivirus engine")
engine_definition_version = odm.Optional(
odm.Keyword(), description="Version of definition set")
virus_name = odm.Optional(odm.Keyword(),
description="The name of the virus")
# What category does the verdict fall under?
category = odm.Optional(
odm.Enum([
'type-unsupported', 'undetected', 'failure', 'suspicious',
'malicious'
]),
description="What category does the verdict fall under?<br><ul>"
"<li>`type-unsupported`: File sent to antivirus is unsupported</li>"
"<li>`undetected`: File not detected by antivirus</li>"
"<li>`failure`: Antivirus failed during detection</li>"
"<li>`suspicious`: Antivirus deems suspicious</li>"
"<li>`malicious`: Antivirus deems malicious</li></ul>")
odm_version = odm.Text(
default="2.0", description="Version of antivirus ontological result")
detections = odm.List(odm.Compound(Detection),
description="List of antivirus detections")
class Scaler(odm.Model):
service_defaults: ScalerServiceDefaults = odm.Compound(
ScalerServiceDefaults)
# only available for docker hosts, not kubernetes
cpu_overallocation: float = odm.Float(default=1)
memory_overallocation: float = odm.Float(default=1)
# Additional labels to be applied to deployments in kubernetes('=' delimited)
additional_labels: List[str] = odm.Optional(odm.List(odm.Text()))
class NetworkDNS(odm.Model):
connection_details = odm.Compound(
NetworkConnection,
description="The low-level details of the DNS request")
domain = odm.Domain(description="The domain requested")
resolved_ips = odm.List(odm.IP(),
description="A list of IPs that were resolved")
lookup_type = odm.Text(description="The type of DNS request")
class Heuristic(odm.Model):
attack_id = odm.Optional(odm.Enum(values=PATTERNS, copyto="__text__")) # Att&ck matrix pattern
classification = odm.Classification(default=Classification.UNRESTRICTED) # Classification of the heuristic
description = odm.Text(copyto="__text__") # Description of the heuristic
filetype = odm.Keyword(copyto="__text__") # Type of file targeted
heur_id = odm.Keyword(copyto="__text__") # Heuristic ID
name = odm.Keyword(copyto="__text__") # Name of the heuristic
score = odm.Integer() # Score of the heuristic
class Response(odm.Model):
message = odm.Text(copyto="__text__") # Error message
service_debug_info = odm.Optional(
odm.Keyword()) # Info about where the service was processed
service_name = odm.Keyword(
copyto="__text__") # Name of the service that had an error
service_tool_version = odm.Optional(
odm.Keyword(copyto="__text__")) # Tool version of the service
service_version = odm.Keyword() # Version of the service
status = odm.Enum(values=STATUSES) # Status of the error
class Response(odm.Model):
message = odm.Text(copyto="__text__", description="Error message")
service_debug_info = odm.Optional(
odm.Keyword(),
description="Information about where the service was processed")
service_name = odm.Keyword(copyto="__text__", description="Service Name")
service_tool_version = odm.Optional(odm.Keyword(copyto="__text__"),
description="Service Tool Version")
service_version = odm.Keyword(description="Service Version")
status = odm.Enum(values=STATUSES,
description="Status of error produced by service")
class Scaler(odm.Model):
service_defaults: ScalerServiceDefaults = odm.Compound(
ScalerServiceDefaults,
description="Defaults Scaler will assign to a service.")
cpu_overallocation: float = odm.Float(
description="Percentage of CPU overallocation")
memory_overallocation: float = odm.Float(
description="Percentage of RAM overallocation")
additional_labels: List[str] = odm.Optional(
odm.List(odm.Text()),
description="Additional labels to be applied to services('=' delimited)"
)
class Heuristic(odm.Model):
attack_id = odm.List(odm.Enum(values=ATTACK_ID_LIST, copyto="__text__"),
default=[]) # List of all associated Att&ck IDs
classification = odm.Classification(
default=Classification.UNRESTRICTED) # Classification of the heuristic
description = odm.Text(copyto="__text__") # Description of the heuristic
filetype = odm.Keyword(copyto="__text__") # Type of file targeted
heur_id = odm.Keyword(copyto="__text__") # Heuristic ID
name = odm.Keyword(copyto="__text__") # Name of the heuristic
score = odm.Integer() # Default score of the heuristic
signature_score_map = odm.Mapping(
odm.Integer(), default={}) # Score of signatures for this heuristic
max_score = odm.Optional(odm.Integer()) # Maximum score for heuristic
class Heuristic(odm.Model):
attack_id = odm.List(odm.Keyword(copyto="__text__"), default=[], description="List of all associated ATT&CK IDs")
classification = odm.Classification(default=Classification.UNRESTRICTED,
description="Classification of the heuristic")
description = odm.Text(copyto="__text__", description="Description of the heuristic")
filetype = odm.Keyword(copyto="__text__", description="What type of files does this heuristic target?")
heur_id = odm.Keyword(copyto="__text__", description="ID of the Heuristic")
name = odm.Keyword(copyto="__text__", description="Name of the heuristic")
score = odm.Integer(description="Default score of the heuristic")
signature_score_map = odm.Mapping(odm.Integer(), default={},
description="Score of signatures for this heuristic")
stats = odm.Compound(Statistics, default={}, description="Statistics related to the Heuristic")
max_score = odm.Optional(odm.Integer(), description="Maximum score for heuristic")
class Signature(odm.Model):
classification = odm.Classification(store=True,
default=Classification.UNRESTRICTED)
data = odm.Text(index=False, store=False)
last_modified = odm.Date(default="NOW")
name = odm.Keyword(copyto="__text__")
order = odm.Integer(default=1, store=False)
revision = odm.Keyword(default="1")
signature_id = odm.Optional(odm.Keyword())
source = odm.Keyword()
state_change_date = odm.Optional(odm.Date(store=False))
state_change_user = odm.Optional(odm.Keyword(store=False))
status = odm.Enum(values=RULE_STATUSES, copyto="__text__")
type = odm.Keyword(copyto="__text__")
class UserSettings(odm.Model):
classification = odm.Classification(
default=Classification.UNRESTRICTED,
description="Default submission classification")
deep_scan = odm.Boolean(default=False,
description="Should a deep scan be performed?")
description = odm.Keyword(default="", description="Default description")
download_encoding = odm.Enum(
values=ENCODINGS,
default="cart",
description="Default download encoding when downloading files")
default_zip_password = odm.Text(
default="zippy",
description=
"Default user-defined password for creating password protected ZIPs when downloading files"
)
expand_min_score = odm.Integer(
default=500,
description="Auto-expand section when score bigger then this")
ignore_cache = odm.Boolean(default=False,
description="Ignore service caching?")
ignore_dynamic_recursion_prevention = odm.Boolean(
default=False, description="Ignore dynamic recursion prevention?")
ignore_filtering = odm.Boolean(default=False,
description="Ignore filtering services?")
malicious = odm.Boolean(
default=False,
description="Is the file submitted already known to be malicious?")
priority = odm.Integer(default=1000,
description="Default priority for the submissions")
profile = odm.Boolean(
default=False, description="Should the submission do extra profiling?")
service_spec = odm.Mapping(odm.Mapping(odm.Any()),
default={},
description="Default service specific settings")
services = odm.Compound(ServiceSelection,
default={},
description="Default service selection")
submission_view = odm.Enum(
values=VIEWS,
default="report",
description="Default view for completed submissions")
ttl = odm.Integer(default=30,
description="Default submission TTL, in days")
class Process(odm.Model):
objectid = odm.Compound(ObjectID, description="The object ID of the process object")
# Parent process details
pobjectid = odm.Compound(ObjectID, description="The object ID of the parent process object")
pimage = odm.Optional(odm.Text(), description="The image of the parent process that spawned this process")
pcommand_line = odm.Optional(odm.Text(), description="The command line that the parent process ran")
ppid = odm.Optional(odm.Integer(), description="The process ID of the parent process")
pid = odm.Optional(odm.Integer(), description="The process ID")
image = odm.Text(default="<unknown_image>", description="The image of the process")
command_line = odm.Optional(odm.Text(), description="The command line that the process ran")
start_time = odm.Date(description="The time of creation for the process")
end_time = odm.Date(description="The time of termination for the process")
integrity_level = odm.Optional(odm.Text(), description="The integrity level of the process")
image_hash = odm.Optional(odm.Text(), description="The hash of the file run")
original_file_name = odm.Optional(odm.Text(), description="The original name of the file")
class UI(odm.Model):
alerting_meta: AlertingMeta = odm.Compound(
AlertingMeta,
default=DEFAULT_ALERTING_META,
description="Alerting metadata fields")
allow_malicious_hinting: bool = odm.Boolean(
description=
"Allow user to tell in advance the system that a file is malicious?")
allow_raw_downloads: bool = odm.Boolean(
description="Allow user to download raw files?")
allow_zip_downloads: bool = odm.Boolean(
description="Allow user to download files as password protected ZIPs?")
allow_replay: bool = odm.Boolean(
description="Allow users to request replay on another server?")
allow_url_submissions: bool = odm.Boolean(
description="Allow file submissions via url?")
audit: bool = odm.Boolean(
description=
"Should API calls be audited and saved to a separate log file?")
banner: Dict[str, str] = odm.Optional(
odm.Mapping(odm.Keyword()),
description=
"Banner message display on the main page (format: {<language_code>: message})"
)
banner_level: str = odm.Enum(
values=["info", "warning", "success", "error"],
description="Banner message level")
debug: bool = odm.Boolean(description="Enable debugging?")
discover_url: str = odm.Optional(odm.Keyword(), description="Discover URL")
download_encoding = odm.Enum(
values=["raw", "cart"],
description="Which encoding will be used for downloads?")
email: str = odm.Optional(odm.Email(),
description="Assemblyline admins email address")
enforce_quota: bool = odm.Boolean(description="Enforce the user's quotas?")
fqdn: str = odm.Text(
description=
"Fully qualified domain name to use for the 2-factor authentication validation"
)
ingest_max_priority: int = odm.Integer(
description="Maximum priority for ingest API")
read_only: bool = odm.Boolean(
description="Turn on read only mode in the UI")
read_only_offset: str = odm.Keyword(
default="",
description="Offset of the read only mode for all paging and searches")
secret_key: str = odm.Keyword(
description="Flask secret key to store cookies, etc.")
session_duration: int = odm.Integer(
description=
"Duration of the user session before the user has to login again")
statistics: Statistics = odm.Compound(
Statistics,
default=DEFAULT_STATISTICS,
description="Statistics configuration")
tos: str = odm.Optional(odm.Text(), description="Terms of service")
tos_lockout: bool = odm.Boolean(
description="Lock out user after accepting the terms of service?")
tos_lockout_notify: List[str] = odm.Optional(
odm.List(odm.Keyword()),
description="List of admins to notify when a user gets locked out")
url_submission_headers: Dict[str, str] = odm.Optional(
odm.Mapping(odm.Keyword()),
description="Headers used by the url_download method")
url_submission_proxies: Dict[str, str] = odm.Optional(
odm.Mapping(odm.Keyword()),
description="Proxy used by the url_download method")
validate_session_ip: bool = \
odm.Boolean(description="Validate if the session IP matches the IP the session was created from")
validate_session_useragent: bool = \
odm.Boolean(description="Validate if the session useragent matches the useragent the session was created with")
class SubmissionParams(odm.Model):
classification = odm.Classification(
default=Classification.UNRESTRICTED,
description="Original classification of the submission")
deep_scan = odm.Boolean(default=False,
description="Should a deep scan be performed?")
description = odm.Text(store=True,
copyto="__text__",
description="Description of the submission")
generate_alert = odm.Boolean(
default=False, description="Should this submission generate an alert?")
groups = odm.List(odm.Keyword(),
default=["USERS"],
description="List of groups related to this scan")
ignore_cache = odm.Boolean(
default=False, description="Ignore the cached service results?")
ignore_dynamic_recursion_prevention = odm.Boolean(
default=False,
description="Should we ignore dynamic recursion prevention?")
ignore_filtering = odm.Boolean(
default=False, description="Should we ignore filtering services?")
ignore_size = odm.Boolean(default=False,
description="Ignore the file size limits?")
never_drop = odm.Boolean(
default=False, description="Exempt from being dropped by ingester?")
malicious = odm.Boolean(
default=False,
description="Is the file submitted already known to be malicious?")
max_extracted = odm.Integer(default=500,
description="Max number of extracted files")
max_supplementary = odm.Integer(
default=500, description="Max number of supplementary files")
priority = odm.Integer(default=1000, description="Priority of the scan")
profile = odm.Boolean(
default=False, description="Should the submission do extra profiling?")
psid = odm.Optional(odm.UUID(), description="Parent submission ID")
quota_item = odm.Boolean(
default=False, description="Does this submission count against quota?")
services = odm.Compound(ServiceSelection,
default={},
description="Service selection")
service_spec = odm.Mapping(odm.Mapping(odm.Any()),
default={},
index=False,
store=False,
description="Service-specific parameters")
submitter = odm.Keyword(store=True,
copyto="__text__",
description="User who submitted the file")
ttl = odm.Integer(default=0,
description="Time, in days, to live for this submission")
type = odm.Keyword(default="USER", description="Type of submission")
initial_data = odm.Optional(
odm.Text(index=False),
description="Initialization for temporary submission data")
def get_hashing_keys(self):
"""Get the sections of the submission parameters that should be used in result hashes."""
data = self.as_primitives()
return {k: v for k, v in data.items() if k in _KEY_HASHED_FIELDS}
def create_filescore_key(self, sha256, services: list = None):
"""This is the key used to store the final score of a submission for fast lookup.
This lookup is one of the methods used to check for duplication in ingestion process,
so this key is fairly sensitive.
"""
# TODO do we need this version thing still be here?
# One up this if the cache is ever messed up and we
# need to quickly invalidate all old cache entries.
version = 0
if services is None:
services = self.services.selected
data = self.get_hashing_keys()
data['service_spec'] = sorted(
(key, sorted(values.items()))
for key, values in self.service_spec.items())
data['sha256'] = sha256
data['services'] = [str(x) for x in services]
s = ', '.join([f"{k}: {data[k]}" for k in sorted(data.keys())])
return 'v'.join(
[str(hashlib.md5(s.encode()).hexdigest()),
str(version)])
class SubmissionSummary(odm.Model):
expiry_ts = odm.Date(index=True) # Expiry date
tags = odm.Text() # Tags cache
attack_matrix = odm.Text() # Att&ck Matrix cache
heuristics = odm.Text() # Heuristics cache
class ThingsModel(odm.Model):
count = odm.Integer()
thing = odm.Text()