def format_result(user_classification,
r,
min_classification,
build_hierarchy=False):
if not CLASSIFICATION.is_accessible(user_classification,
min_classification):
return None
# Drop sections user does not have access and set others to at least min classification
max_classification, r['result']['sections'] = filter_sections(
r['result']['sections'], user_classification, min_classification)
# Drop supplementary and extracted files that the user does not have access to
for ftype in ['supplementary', 'extracted']:
r['response'][ftype] = [
x for x in r['response'][ftype] if CLASSIFICATION.is_accessible(
user_classification, x['classification'])
]
# Set result classification to at least min but no more then viewable result classification
r['classification'] = CLASSIFICATION.max_classification(
max_classification, min_classification)
if build_hierarchy:
try:
section_hierarchy, _ = build_heirarchy_rec(r['result']['sections'])
r['section_hierarchy'] = section_hierarchy['children']
except InvalidSectionList:
LOGGER.warning(
f"Could not generate section hierarchy for {r['response']['service_name']} "
f"service. Will use old display method.")
return r
def setup_watch_queue(sid, **kwargs):
"""
Starts a watch queue to get live results
Variables:
sid => Submission ID
Arguments:
None
Data Block:
None
Result example:
{"wq_id": "D-c7668cfa-...-c4132285142e-WQ"} #ID of the watch queue
"""
data = STORAGE.submission.get(sid, as_obj=False)
user = kwargs['user']
if user and data and Classification.is_accessible(user['classification'], data['classification']):
wq_id = DispatchClient(datastore=STORAGE).setup_watch_queue(sid)
if wq_id:
return make_api_response({"wq_id": wq_id})
return make_api_response("", "No dispatchers are processing this submission.", 404)
else:
return make_api_response("", "You are not allowed to access this submissions.", 403)
def get_alert(alert_id, **kwargs):
"""
Get the alert details for a given alert key
Variables:
alert_id => ID of the alert to get the details for
Arguments:
None
Data Block:
None
API call example:
/api/v4/alert/1234567890/
Result example:
{
KEY: VALUE, # All fields of an alert in key/value pair
}
"""
user = kwargs['user']
data = STORAGE.alert.get(alert_id, as_obj=False)
if not data:
return make_api_response("", "This alert does not exists...", 404)
if user and Classification.is_accessible(user['classification'], data['classification']):
return make_api_response(data)
else:
return make_api_response("", "You are not allowed to see this alert...", 403)
def get_workflow(workflow_id, **kwargs):
"""
Load the user account information.
Variables:
workflow_id => ID of the workflow
Arguments:
None
Data Block:
None
Result example:
{
"name": "Workflow name", # Name of the workflow
"classification": "", # Max classification for workflow
"label": ['label1'], # Labels for the workflow
"priority": "LOW", # Priority of the workflow
"status": "MALICIOUS", # Status of the workflow
"query": "*:*" # Query to match the data
}
"""
wf = STORAGE.workflow.get(workflow_id, as_obj=False)
if wf:
if CLASSIFICATION.is_accessible(kwargs['user']['classification'], wf['classification']):
return make_api_response(wf)
else:
return make_api_response({},
err="You're not allowed to view workflow ID: %s" % workflow_id,
status_code=403)
else:
return make_api_response({},
err="Workflow ID %s does not exist" % workflow_id,
status_code=404)
def request_replay(index, doc_id, **kwargs):
"""
Request an alert or a submission to be transfered to another system
Variables:
index => Type of document to be transfered (alert or submission)
doc_id => ID of the document to transfer
Arguments:
None
Data Block:
None
Result example:
{"success": true}
"""
user = kwargs['user']
if index not in ['alert', 'submission']:
return make_api_response(
"", f"{index.upper()} is not a valid index for this API.", 400)
index_ds = STORAGE.get_collection(index)
doc = index_ds.get_if_exists(doc_id, as_obj=False)
if not doc or not Classification.is_accessible(user['classification'],
doc['classification']):
return make_api_response(
"",
f"You are not allowed to modify the {index} with the following ID: {doc_id}",
403)
operations = [(index_ds.UPDATE_SET, 'metadata.replay', REPLAY_REQUESTED)]
return make_api_response({'success': index_ds.update(doc_id, operations)})
def get_heuristic(heuristic_id, **kwargs):
"""
Get a specific heuristic's detail from the system
Variables:
heuristic_id => ID of the heuristic
Arguments:
None
Data Block:
None
Result example:
{"id": "AL_HEUR_001", # Heuristics ID
"filetype": ".*", # Target file type
"name": "HEURISTICS_NAME", # Heuristics name
"description": ""} # Heuristics description
"""
user = kwargs['user']
h = STORAGE.heuristic.get(heuristic_id, as_obj=False)
if not h:
return make_api_response("", "Heuristic not found", 404)
if user and Classification.is_accessible(user['classification'], h['classification']):
# Always refresh stats when someone get a heuristic
h.update({'stats': STORAGE.get_stat_for_heuristic(heuristic_id)})
return make_api_response(h)
else:
return make_api_response("", "You are not allowed to see this heuristic...", 403)
def delete_submission(sid, **kwargs):
"""
Delete a submission as well as all related
files, results and errors
Variables:
sid => Submission ID to be deleted
Arguments:
None
Data Block:
None
Result example:
{success: true}
"""
user = kwargs['user']
submission = STORAGE.submission.get(sid, as_obj=False)
if not submission:
return make_api_response("", f"There are not submission with sid: {sid}", 404)
if Classification.is_accessible(user['classification'], submission['classification']) \
and (submission['params']['submitter'] == user['uname'] or 'admin' in user['type']):
STORAGE.delete_submission_tree_bulk(sid, Classification, transport=FILESTORE)
STORAGE.submission.commit()
return make_api_response({"success": True})
else:
return make_api_response("", "Your are not allowed to delete this submission.", 403)
def create_bundle(sid, **kwargs):
"""
Creates a bundle containing the submission results and the associated files
Variables:
sid => ID of the submission to create the bundle for
Arguments:
use_alert => The ID provided is from an alert and we will use it to create the bundle
Data Block:
None
API call example:
/api/v4/bundle/create/234f334-...-31232/
Result example:
-- THE BUNDLE FILE BINARY --
"""
user = kwargs['user']
use_alert = request.args.get('use_alert', 'false').lower() in ['true', '']
if use_alert:
data = STORAGE.alert.get(sid, as_obj=False)
else:
data = STORAGE.submission.get(sid, as_obj=False)
if user and data and Classification.is_accessible(user['classification'],
data['classification']):
temp_target_file = None
try:
temp_target_file = bundle_create(sid,
working_dir=BUNDLING_DIR,
use_alert=use_alert)
f_size = os.path.getsize(temp_target_file)
return stream_file_response(open(temp_target_file, 'rb'),
"%s.al_bundle" % sid, f_size)
except SubmissionNotFound as snf:
return make_api_response(
"", "Submission %s does not exist. [%s]" % (sid, str(snf)),
404)
except BundlingException as be:
return make_api_response(
"", "An error occured while bundling submission %s. [%s]" %
(sid, str(be)), 404)
finally:
try:
if temp_target_file:
os.remove(temp_target_file)
except Exception:
pass
else:
return make_api_response(
"",
f"You are not allowed create a bundle for this {'alert' if use_alert else 'submission'}...",
403)
def get_file_children(sha256, **kwargs):
"""
Get the list of children files for a given file
Variables:
sha256 => A resource locator for the file (sha256)
Arguments:
None
Data Block:
None
API call example:
/api/v4/file/children/123456...654321/
Result example:
[ # List of children
{"name": "NAME OF FILE", # Name of the children
"sha256": "123..DEF"}, # sha256 of the children
]
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
if file_obj:
if user and Classification.is_accessible(user['classification'],
file_obj['classification']):
output = []
response = STORAGE.result.grouped_search(
"response.service_name",
query=f"id:{sha256}* AND response.extracted:*",
fl="*",
rows=100,
sort="created desc",
access_control=user['access_control'],
as_obj=False)
processed_srl = []
for r in response['items']:
for extracted in r['items'][0]['response']['extracted']:
if extracted['sha256'] not in processed_srl:
processed_srl.append(extracted['sha256'])
output.append({
'sha256': extracted['sha256'],
'name': extracted['name']
})
return make_api_response(output)
else:
return make_api_response({},
"You are not allowed to view this file.",
403)
else:
return make_api_response({}, "This file does not exists.", 404)
def get_file_score(sha256, **kwargs):
"""
Get the score of the latest service run for a given file.
Variables:
sha256 => A resource locator for the file (SHA256)
Arguments:
None
Data Block:
None
API call example:
/api/v4/file/score/123456...654321/
Result example:
{"file_info": {}, # File info Block
"result_keys": [<keys>] # List of keys used to compute the score
"score": 0} # Latest score for the file
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
if not file_obj:
return make_api_response([], "This file does not exists", 404)
if user and Classification.is_accessible(user['classification'],
file_obj['classification']):
score = 0
keys = []
res = STORAGE.result.grouped_search(
"response.service_name",
f"id:{sha256}*",
fl="result.score,id",
sort="created desc",
access_control=user["access_control"],
rows=100,
as_obj=False)
for s in res['items']:
for d in s['items']:
score += d['result']['score']
keys.append(d["id"])
return make_api_response({
"file_info": file_obj,
"score": score,
"result_keys": keys
})
else:
return make_api_response([], "You are not allowed to view this file",
403)
def get_file_hex(sha256, **kwargs):
"""
Returns the file hex representation
Variables:
sha256 => A resource locator for the file (sha256)
Arguments:
bytes_only => Only return bytes with no formatting
length => Number of bytes per lines
Data Block:
None
API call example:
/api/v4/file/hex/123456...654321/
Result example:
<THE FILE HEX REPRESENTATION>
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
bytes_only = request.args.get('bytes_only',
'false').lower() in ['true', '']
length = int(request.args.get('length', '16'))
if not file_obj:
return make_api_response({}, "The file was not found in the system.",
404)
if file_obj['size'] > API_MAX_SIZE:
return make_api_response(
{}, "This file is too big to be seen through this API.", 403)
if user and Classification.is_accessible(user['classification'],
file_obj['classification']):
data = FILESTORE.get(sha256)
if not data:
return make_api_response({},
"This file was not found in the system.",
404)
if bytes_only:
return make_api_response(dump(data).decode())
else:
return make_api_response(hexdump(data, length=length))
else:
return make_api_response({}, "You are not allowed to view this file.",
403)
def get_file_strings(sha256, **kwargs):
"""
Return all strings in a given file
Variables:
sha256 => A resource locator for the file (sha256)
Arguments:
len => Minimum length for a string
Data Block:
None
Result example:
<THE LIST OF STRINGS>
"""
user = kwargs['user']
hlen = request.args.get('len', "6")
file_obj = STORAGE.file.get(sha256, as_obj=False)
if file_obj['size'] > API_MAX_SIZE:
return make_api_response(
{}, "This file is too big to be seen through this API.", 403)
if not file_obj:
return make_api_response({}, "The file was not found in the system.",
404)
if user and Classification.is_accessible(user['classification'],
file_obj['classification']):
data = FILESTORE.get(sha256)
if not data:
return make_api_response({},
"This file was not found in the system.",
404)
# Ascii strings (we use decode with replace on to create delimiters)
pattern = "[\x1f-\x7e]{%s,}" % hlen
string_list = re.findall(pattern, data.decode("ascii",
errors="replace"))
# UTF-16 strings
string_list += re.findall(pattern,
data.decode("utf-16", errors="replace"))
return make_api_response("\n".join(string_list))
else:
return make_api_response({}, "You are not allowed to view this file.",
403)
def get_submission(sid, **kwargs):
"""
Get the submission details for a given Submission ID
Variables:
sid => Submission ID to get the details for
Arguments:
None
Data Block:
None
Result example:
{"files": [ # List of source files
["FNAME", "sha256"], ...], # Each file = List of name/sha256
"errors": [], # List of error keys (sha256.ServiceName)
"submission": { # Submission Block
"profile": true, # Should keep stats about execution?
"description": "", # Submission description
"ttl": 30, # Submission days to live
"ignore_filtering": false, # Ignore filtering services?
"priority": 1000, # Submission priority, higher = faster
"ignore_cache": true, # Force reprocess even is result exist?
"groups": ["group", ...], # List of groups with access
"sid": "ab9...956", # Submission ID
"submitter": "user", # Uname of the submitter
"max_score": 1422, }, # Score of highest scoring file
"results": [], # List of Results keys (sha256.ServiceName.Version.Config)
"times": { # Timing block
"completed": "2014-...", # Completed time
"submitted": "2014-..." # Submitted time
},
"state": "completed", # State of the submission
"services": { # Service Block
"selected": ["mcafee"], # List of selected services
"params": {}, # Service specific parameters
"excluded": [] # List of excluded services
}
}
"""
user = kwargs['user']
data = STORAGE.submission.get(sid, as_obj=False)
if data is None:
return make_api_response("", "Submission ID %s does not exists." % sid, 404)
if data and user and Classification.is_accessible(user['classification'], data['classification']):
return make_api_response(data)
else:
return make_api_response("", "You are not allowed to view the data of this submission", 403)
def set_verdict(alert_id, verdict, **kwargs):
"""
Set the verdict of an alert based on its ID.
Variables:
submission_id -> ID of the alert to give a verdict to
verdict -> verdict that the user think the alert is: malicious or non_malicious
Arguments:
None
Data Block:
None
Result example:
{"success": True} # Has the verdict been set or not
"""
reverse_verdict = {
'malicious': 'non_malicious',
'non_malicious': 'malicious'
}
user = kwargs['user']
if verdict not in ['malicious', 'non_malicious']:
return make_api_response({"success": False}, f"'{verdict}' is not a valid verdict.", 400)
document = STORAGE.alert.get(alert_id, as_obj=False)
if not document:
return make_api_response({"success": False}, f"There are no alert with id: {alert_id}", 404)
if not Classification.is_accessible(user['classification'], document['classification']):
return make_api_response({"success": False}, "You are not allowed to give verdict on alert with "
f"ID: {alert_id}", 403)
resp = STORAGE.alert.update_by_query(f"sid:{document['sid']}", [
('REMOVE', f'verdict.{verdict}', user['uname']),
('APPEND', f'verdict.{verdict}', user['uname']),
('REMOVE', f'verdict.{reverse_verdict[verdict]}', user['uname'])
])
STORAGE.submission.update(document['sid'], [
('REMOVE', f'verdict.{verdict}', user['uname']),
('APPEND', f'verdict.{verdict}', user['uname']),
('REMOVE', f'verdict.{reverse_verdict[verdict]}', user['uname'])
])
return make_api_response({"success": resp != 0})
def get_file_information(sha256, **kwargs):
"""
Get information about the file like:
Hashes, size, frequency count, etc...
Variables:
sha256 => A resource locator for the file (sha256)
Arguments:
None
Data Block:
None
API call example:
/api/v4/file/info/123456...654321/
Result example:
{ # File information block
"ascii": "PK..", # First 64 bytes as ASCII
"classification": "UNRESTRICTED", # Access control for the file
"entropy": 7.99, # File's entropy
"hex": "504b...c0b2", # First 64 bytes as hex
"magic": "Zip archive data", # File's identification description (from magic)
"md5": "8f31...a048", # File's MD5 hash
"mime": "application/zip", # Mimetype of the file (from magic)
"seen_count": 7, # Number of time we've seen this file
"seen_first": "2015-03-04T21:59:13.204861Z", # Time at which we first seen this file
"seen_last": "2015-03-10T19:42:04.587233Z", # Last time we've seen the file
"sha256": "e021...4de2", # File's sha256 hash
"sha1": "354f...fdab", # File's sha1 hash
"size": 3417, # Size of the file
"ssdeep": "4:Smm...OHY+", # File's SSDEEP hash
"tag": "archive/zip" # Type of file that we identified
}
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
if file_obj:
if user and Classification.is_accessible(user['classification'],
file_obj['classification']):
return make_api_response(file_obj)
else:
return make_api_response({},
"You are not allowed to view this file.",
403)
else:
return make_api_response({}, "This file does not exists.", 404)
def get_file_results_for_service(sha256, service, **kwargs):
"""
Get the all the file results of a specific file and a specific query.
Variables:
sha256 => A resource locator for the file (SHA256)
Arguments:
all => if all argument is present, it will return all versions
NOTE: Max to 100 results...
Data Block:
None
API call example:
/api/v4/file/result/123456...654321/service_name/
Result example:
{"file_info": {}, # File info Block
"results": {}} # Full result list for the service
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
if not file_obj:
return make_api_response([], "This file does not exists", 404)
if user and Classification.is_accessible(user['classification'],
file_obj['classification']):
res = STORAGE.result.search(f"id:{sha256}.{service}*",
sort="created desc",
fl="*",
rows=100 if "all" in request.args else 1,
access_control=user["access_control"],
as_obj=False,
use_archive=True)
results = []
for r in res['items']:
result = format_result(user['classification'], r,
file_obj['classification'])
if result:
results.append(result)
return make_api_response({"file_info": file_obj, "results": results})
else:
return make_api_response([], "You are not allowed to view this file",
403)
def filter_sections(sections, user_classification, min_classification):
max_classification = min_classification
# Filtering section you do not have access to
temp_sections = [
s for s in sections if CLASSIFICATION.is_accessible(
user_classification, s['classification'])
]
final_sections = []
for section in temp_sections:
# TODO: Depth analysis should be done before returning sections
try:
# Recalculation max classification using the currently accessible sections
section['classification'] = CLASSIFICATION.max_classification(
section['classification'], min_classification)
max_classification = CLASSIFICATION.max_classification(
section['classification'], max_classification)
except InvalidClassification:
continue
if section['body_format'] in [
"GRAPH_DATA", "URL", "JSON", "KEY_VALUE"
] and isinstance(section['body'], str):
# Loading JSON formatted sections
try:
section['body'] = json.loads(section['body'])
except ValueError:
pass
# Changing tags to a list
section['tags'] = tag_dict_to_list(section['tags'])
final_sections.append(section)
# Telling the user a section was hidden
if len(sections) != len(final_sections):
hidden_section = dict(
body=
"One of the sections produced by the service has been removed because you do not have enough "
"priviledges to see its results. \n\nContact system administrators for more information.",
title_text="WARNING: Service sections have been sanitized",
depth=0,
classification=CLASSIFICATION.UNRESTRICTED,
tags={},
heuristic=None,
body_format="TEXT")
final_sections.insert(0, hidden_section)
return max_classification, final_sections
def get_file_image_datastream(sha256, **kwargs):
"""
Returns the image file as a datastream
Variables:
sha256 => A resource locator for the file (sha256)
Arguments:
None
Data Block:
None
API call example:
/api/v4/file/image/123456...654321/
Result example:
data:image/png;base64,...
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
if not file_obj:
return make_api_response({}, "The file was not found in the system.",
404)
if not file_obj.get('is_section_image',
False) or not file_obj['type'].startswith("image/"):
return make_api_response(
{}, "This file is not allowed to be downloaded as a datastream.",
403)
if user and Classification.is_accessible(user['classification'],
file_obj['classification']):
data = FILESTORE.get(sha256)
if not data:
return make_api_response({},
"This file was not found in the system.",
404)
return make_api_response(
f"data:{file_obj['type']};base64,{base64.b64encode(data).decode()}"
)
else:
return make_api_response({}, "You are not allowed to view this file.",
403)
def get_user_favorites(username, **kwargs):
"""
Loads the user's favorites.
Variables:
username => Name of the user you want to get the avatar for
Arguments:
None
Data Block:
None
Result example:
{ # Dictionary of
"<name_of_query>": # Named queries
"*:*", # The actual query to run
...
}
"""
user = kwargs['user']
favorites = {
"alert": [],
"search": [],
"signature": [],
"submission": [],
"error": []
}
res_favorites = STORAGE.user_favorites.get(username, as_obj=False)
if res_favorites:
if username == "__global__" or username != user['uname']:
for key in favorites.keys():
for fav in res_favorites[key]:
if 'classification' in fav:
if CLASSIFICATION.is_accessible(
user['classification'], fav['classification']):
favorites[key].append(fav)
else:
favorites[key].append(fav)
else:
favorites.update(res_favorites)
return make_api_response(favorites)
def change_priority(alert_id, **kwargs):
"""
Change the priority of a given alert
Variables:
alert_id => ID of the alert to change the priority
Arguments:
"HIGH" => New priority for the alert
Data Block:
None
API call example:
/api/v4/alert/priority/12345...67890/
Result example:
{"success": true}
"""
user = kwargs['user']
try:
priority = request.json
priority = priority.upper()
if priority not in PRIORITIES:
raise ValueError("Not in priorities")
except ValueError:
return make_api_response({"success": False}, err="Invalid priority received.", status_code=400)
alert = STORAGE.alert.get(alert_id, as_obj=False)
if not alert:
return make_api_response({"success": False},
err="Alert ID %s not found" % alert_id,
status_code=404)
if not Classification.is_accessible(user['classification'], alert['classification']):
return make_api_response("", "You are not allowed to see this alert...", 403)
if priority != alert.get('priority', None):
return make_api_response({
"success": STORAGE.alert.update(alert_id, [(STORAGE.alert.UPDATE_SET, 'priority', priority)])})
else:
return make_api_response({"success": True})
def change_status(alert_id, **kwargs):
"""
Change the status of a given alert
Variables:
alert_id => ID of the alert to change the status
Arguments:
None
Data Block:
"MALICIOUS" => New status for the alert
API call example:
/api/v4/alert/status/12345...67890/
Result example:
{"success": true}
"""
user = kwargs['user']
try:
status = request.json
status = status.upper()
if status not in STATUSES:
raise ValueError("Not in priorities")
except ValueError:
return make_api_response({"success": False}, err="Invalid status received.", status_code=400)
alert = STORAGE.alert.get(alert_id, as_obj=False)
if not alert:
return make_api_response({"success": False},
err="Alert ID %s not found" % alert_id,
status_code=404)
if not Classification.is_accessible(user['classification'], alert['classification']):
return make_api_response("", "You are not allowed to see this alert...", 403)
if status != alert.get('status', None):
return make_api_response({
"success": STORAGE.alert.update(alert_id, [(STORAGE.alert.UPDATE_SET, 'status', status)])})
else:
return make_api_response({"success": True})
def add_labels(alert_id, **kwargs):
"""
Add one or multiple labels to a given alert
Variables:
alert_id => ID of the alert to add the label to
Arguments:
None
Data Block:
["LBL1", "LBL2"] => List of labels to add as comma separated string
API call example:
/api/v4/alert/label/12345...67890/
Result example:
{"success": true}
"""
user = kwargs['user']
try:
labels = set(request.json)
except ValueError:
return make_api_response({"success": False}, err="Invalid list of labels received.", status_code=400)
alert = STORAGE.alert.get(alert_id, as_obj=False)
if not alert:
return make_api_response({"success": False}, err="Alert ID %s not found" % alert_id, status_code=404)
if not Classification.is_accessible(user['classification'], alert['classification']):
return make_api_response("", "You are not allowed to see this alert...", 403)
cur_label = set(alert.get('label', []))
label_diff = labels.difference(labels.intersection(cur_label))
if label_diff:
return make_api_response({
"success": STORAGE.alert.update(alert_id, [(STORAGE.alert.UPDATE_APPEND_IF_MISSING, 'label', lbl)
for lbl in label_diff])})
else:
return make_api_response({"success": True})
def get_file_ascii(sha256, **kwargs):
"""
Return the ascii values for a file where ascii chars are replaced by DOTs.
Variables:
sha256 => A resource locator for the file (sha256)
Arguments:
None
Data Block:
None
Result example:
<THE ASCII FILE>
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
if file_obj['size'] > API_MAX_SIZE:
return make_api_response(
{}, "This file is too big to be seen through this API.", 403)
if not file_obj:
return make_api_response({}, "The file was not found in the system.",
404)
if user and Classification.is_accessible(user['classification'],
file_obj['classification']):
data = FILESTORE.get(sha256)
if not data:
return make_api_response({},
"This file was not found in the system.",
404)
return make_api_response(data.translate(FILTER_ASCII).decode())
else:
return make_api_response({}, "You are not allowed to view this file.",
403)
def generate_ontology_file(results, user, updates={}, fnames={}):
# Load ontology files
sio = StringIO()
for r in results:
for supp in r.get('response', {}).get('supplementary', {}):
if supp['name'].endswith('.ontology'):
try:
ontology = json.loads(FILESTORE.get(supp['sha256']))
sha256 = ontology['header']['sha256']
c12n = ontology['header']['classification']
if sha256 == r['sha256'] and Classification.is_accessible(
user['classification'], c12n):
# Update the ontology with the live values
ontology['header'].update(updates)
# Set filenames if any
if sha256 in fnames:
ontology['header']['filenames'] = fnames[sha256]
elif 'filenames' in ontology['header']:
del ontology['header']['filenames']
# Make sure parent is not equal to current hash
if 'parent' in ontology['header'] and ontology[
'header']['parent'] == sha256:
del ontology['header']['parent']
sio.write(
json.dumps(
ontology, indent=None, separators=(',', ':')) +
'\n')
except Exception as e:
LOGGER.warning(
f"An error occured while fetching ontology files: {str(e)}"
)
# Flush and reset buffer
sio.flush()
return sio
def take_ownership(alert_id, **kwargs):
"""
Take ownership of a given alert
Variables:
alert_id => ID of the alert to send to take ownership
Arguments:
None
Data Block:
None
API call example:
/api/v4/alert/ownership/12345...67890/
Result example:
{"success": true}
"""
user = kwargs['user']
alert = STORAGE.alert.get(alert_id, as_obj=False)
if not alert:
return make_api_response({"success": False},
err="Alert ID %s not found" % alert_id,
status_code=404)
if not Classification.is_accessible(user['classification'], alert['classification']):
return make_api_response({"success": False}, "You are not allowed to see this alert...", 403)
current_owner = alert.get('owner', None)
if current_owner is None:
return make_api_response({
"success": STORAGE.alert.update(alert_id, [(STORAGE.alert.UPDATE_SET, 'owner', user['uname'])])})
else:
return make_api_response({"success": False},
err="Alert is already owned by %s" % current_owner,
status_code=403)
def get_file_tree(sid, **kwargs):
"""
Get the file hierarchy of a given Submission ID. This is
an N deep recursive process but is limited to the max depth
set in the system settings.
Variables:
sid => Submission ID to get the tree for
Arguments:
None
Data Block:
None
API call example:
/api/v4/submission/tree/12345678-1234-1234-1234-1234567890AB/
Result example:
{ # Dictionary of file blocks
"1f...11": { # File sha256 (sha256)
"score": 923, # Score for the file
"name": ["file.exe",...] # List of possible names for the file
"children": {...} # Dictionary of children file blocks
}, ...
"""
user = kwargs['user']
data = STORAGE.submission.get(sid, as_obj=False)
if data is None:
return make_api_response("", "Submission ID %s does not exists." % sid, 404)
if data and user and Classification.is_accessible(user['classification'], data['classification']):
return make_api_response(STORAGE.get_or_create_file_tree(data, config.submission.max_extraction_depth,
cl_engine=Classification,
user_classification=user['classification']))
else:
return make_api_response("", "You are not allowed to view the data of this submission", 403)
def delete_signature(signature_id, **kwargs):
"""
Delete a signature based of its ID
Variables:
signature_id => Signature ID
Arguments:
None
Data Block:
None
Result example:
{"success": True} # Signature delete successful
"""
user = kwargs['user']
data = STORAGE.signature.get(signature_id, as_obj=False)
if data:
if not Classification.is_accessible(
user['classification'],
data.get('classification', Classification.UNRESTRICTED)):
return make_api_response(
"", "Your are not allowed to delete this signature.", 403)
ret_val = STORAGE.signature.delete(signature_id)
_reset_service_updates(data['type'])
event_sender.send(
data['type'], {
'signature_id': signature_id,
'signature_type': data['type'],
'source': data['source'],
'operation': Operation.Removed
})
return make_api_response({"success": ret_val})
else:
return make_api_response("", f"Signature not found. ({signature_id})",
404)
def get_signature(signature_id, **kwargs):
"""
Get the detail of a signature based of its ID and revision
Variables:
signature_id => Signature ID
Arguments:
None
Data Block:
None
Result example:
{}
"""
user = kwargs['user']
data = STORAGE.signature.get(signature_id, as_obj=False)
if data:
if not Classification.is_accessible(
user['classification'],
data.get('classification', Classification.UNRESTRICTED)):
return make_api_response(
"", "Your are not allowed to view this signature.", 403)
# Always refresh stats when someone get a signature
data.update({
'stats':
STORAGE.get_stat_for_signature(signature_id, data['source'],
data['name'], data['type'])
})
return make_api_response(data)
else:
return make_api_response("", f"Signature not found. ({signature_id})",
404)
def filter_sections(sections, user_classification, min_classification):
max_classification = min_classification
# Filtering section you do not have access to
temp_sections = [
s for s in sections if CLASSIFICATION.is_accessible(
user_classification, s['classification'])
]
final_sections = []
for section in temp_sections:
try:
# Recalculation max classification using the currently accessible sections
section['classification'] = CLASSIFICATION.max_classification(
section['classification'], min_classification)
max_classification = CLASSIFICATION.max_classification(
section['classification'], max_classification)
except InvalidClassification:
continue
final_sections.append(fix_section_data(section))
# Telling the user a section was hidden
if len(sections) != len(final_sections):
hidden_section = dict(
body=
"One of the sections produced by the service has been removed because you do not have enough "
"priviledges to see its results. \n\nContact system administrators for more information.",
title_text="WARNING: Service sections have been sanitized",
depth=0,
classification=CLASSIFICATION.UNRESTRICTED,
tags=[],
heuristic=None,
body_format="TEXT")
final_sections.insert(0, hidden_section)
return max_classification, final_sections
def outstanding_services(sid, **kwargs):
"""
List outstanding services and the number of file each
of them still have to process.
Variables:
sid => Submission ID
Arguments:
None
Data Block:
None
Result example:
{"MY SERVICE": 1, ... } # Dictionnary of services and number of files
"""
data = STORAGE.submission.get(sid, as_obj=False)
user = kwargs['user']
if user and data and Classification.is_accessible(user['classification'], data['classification']):
return make_api_response(DispatchClient(datastore=STORAGE).outstanding_services(sid))
else:
return make_api_response({}, "You are not allowed to access this submissions.", 403)
