def add_access_control(user):
user.update(
Classification.get_access_control_parts(user.get(
"classification", Classification.UNRESTRICTED),
user_classification=True))
gl2_query = " OR ".join(
['__access_grp2__:__EMPTY__'] +
['__access_grp2__:"%s"' % x for x in user["__access_grp2__"]])
gl2_query = "(%s) AND " % gl2_query
gl1_query = " OR ".join(
['__access_grp1__:__EMPTY__'] +
['__access_grp1__:"%s"' % x for x in user["__access_grp1__"]])
gl1_query = "(%s) AND " % gl1_query
req = list(
set(Classification.get_access_control_req()).difference(
set(user["__access_req__"])))
req_query = " OR ".join(['__access_req__:"%s"' % r for r in req])
if req_query:
req_query = "-(%s) AND " % req_query
lvl_query = "__access_lvl__:[0 TO %s]" % user["__access_lvl__"]
query = "".join([gl2_query, gl1_query, req_query, lvl_query])
user['access_control'] = safe_str(query)
def format_result(user_classification,
r,
min_classification,
build_hierarchy=False):
if not CLASSIFICATION.is_accessible(user_classification,
min_classification):
return None
# Drop sections user does not have access and set others to at least min classification
max_classification, r['result']['sections'] = filter_sections(
r['result']['sections'], user_classification, min_classification)
# Drop supplementary and extracted files that the user does not have access to
for ftype in ['supplementary', 'extracted']:
r['response'][ftype] = [
x for x in r['response'][ftype] if CLASSIFICATION.is_accessible(
user_classification, x['classification'])
]
# Set result classification to at least min but no more then viewable result classification
r['classification'] = CLASSIFICATION.max_classification(
max_classification, min_classification)
if build_hierarchy:
try:
section_hierarchy, _ = build_heirarchy_rec(r['result']['sections'])
r['section_hierarchy'] = section_hierarchy['children']
except InvalidSectionList:
LOGGER.warning(
f"Could not generate section hierarchy for {r['response']['service_name']} "
f"service. Will use old display method.")
return r
def _merge_safe_hashes(new, old):
try:
# Check if hash types match
if new['type'] != old['type']:
raise InvalidSafehash(
f"Safe hash type mismatch: {new['type']} != {old['type']}")
# Use max classification
old['classification'] = CLASSIFICATION.max_classification(
old['classification'], new['classification'])
# Update updated time
old['updated'] = now_as_iso()
# Update hashes
old['hashes'].update(new['hashes'])
# Update type specific info
if old['type'] == 'file':
old.setdefault('file', {})
new_names = new.get('file', {}).pop('name', [])
if 'name' in old['file']:
for name in new_names:
if name not in old['file']['name']:
old['file']['name'].append(name)
elif new_names:
old['file']['name'] = new_names
old['file'].update(new.get('file', {}))
elif old['type'] == 'tag':
old['tag'] = new['tag']
# Merge sources
src_map = {x['name']: x for x in new['sources']}
if not src_map:
raise InvalidSafehash("No valid source found")
old_src_map = {x['name']: x for x in old['sources']}
for name, src in src_map.items():
src_cl = src.get('classification', None)
if src_cl:
old['classification'] = CLASSIFICATION.max_classification(
old['classification'], src_cl)
if name not in old_src_map:
old_src_map[name] = src
else:
old_src = old_src_map[name]
if old_src['type'] != src['type']:
raise InvalidSafehash(
f"Source {name} has a type conflict: {old_src['type']} != {src['type']}"
)
for reason in src['reason']:
if reason not in old_src['reason']:
old_src['reason'].append(reason)
old['sources'] = old_src_map.values()
return old
except Exception as e:
raise InvalidSafehash(f"Invalid data provided: {str(e)}")
def filter_sections(sections, user_classification, min_classification):
max_classification = min_classification
# Filtering section you do not have access to
temp_sections = [
s for s in sections if CLASSIFICATION.is_accessible(
user_classification, s['classification'])
]
final_sections = []
for section in temp_sections:
# TODO: Depth analysis should be done before returning sections
try:
# Recalculation max classification using the currently accessible sections
section['classification'] = CLASSIFICATION.max_classification(
section['classification'], min_classification)
max_classification = CLASSIFICATION.max_classification(
section['classification'], max_classification)
except InvalidClassification:
continue
if section['body_format'] in [
"GRAPH_DATA", "URL", "JSON", "KEY_VALUE"
] and isinstance(section['body'], str):
# Loading JSON formatted sections
try:
section['body'] = json.loads(section['body'])
except ValueError:
pass
# Changing tags to a list
section['tags'] = tag_dict_to_list(section['tags'])
final_sections.append(section)
# Telling the user a section was hidden
if len(sections) != len(final_sections):
hidden_section = dict(
body=
"One of the sections produced by the service has been removed because you do not have enough "
"priviledges to see its results. \n\nContact system administrators for more information.",
title_text="WARNING: Service sections have been sanitized",
depth=0,
classification=CLASSIFICATION.UNRESTRICTED,
tags={},
heuristic=None,
body_format="TEXT")
final_sections.insert(0, hidden_section)
return max_classification, final_sections
def get_default_user_settings(user):
return UserSettings({
"classification":
Classification.default_user_classification(user),
"ttl":
config.submission.dtl
}).as_primitives()
def setup_watch_queue(sid, **kwargs):
"""
Starts a watch queue to get live results
Variables:
sid => Submission ID
Arguments:
None
Data Block:
None
Result example:
{"wq_id": "D-c7668cfa-...-c4132285142e-WQ"} #ID of the watch queue
"""
data = STORAGE.submission.get(sid, as_obj=False)
user = kwargs['user']
if user and data and Classification.is_accessible(user['classification'], data['classification']):
wq_id = DispatchClient(datastore=STORAGE).setup_watch_queue(sid)
if wq_id:
return make_api_response({"wq_id": wq_id})
return make_api_response("", "No dispatchers are processing this submission.", 404)
else:
return make_api_response("", "You are not allowed to access this submissions.", 403)
def request_replay(index, doc_id, **kwargs):
"""
Request an alert or a submission to be transfered to another system
Variables:
index => Type of document to be transfered (alert or submission)
doc_id => ID of the document to transfer
Arguments:
None
Data Block:
None
Result example:
{"success": true}
"""
user = kwargs['user']
if index not in ['alert', 'submission']:
return make_api_response(
"", f"{index.upper()} is not a valid index for this API.", 400)
index_ds = STORAGE.get_collection(index)
doc = index_ds.get_if_exists(doc_id, as_obj=False)
if not doc or not Classification.is_accessible(user['classification'],
doc['classification']):
return make_api_response(
"",
f"You are not allowed to modify the {index} with the following ID: {doc_id}",
403)
operations = [(index_ds.UPDATE_SET, 'metadata.replay', REPLAY_REQUESTED)]
return make_api_response({'success': index_ds.update(doc_id, operations)})
def get_heuristic(heuristic_id, **kwargs):
"""
Get a specific heuristic's detail from the system
Variables:
heuristic_id => ID of the heuristic
Arguments:
None
Data Block:
None
Result example:
{"id": "AL_HEUR_001", # Heuristics ID
"filetype": ".*", # Target file type
"name": "HEURISTICS_NAME", # Heuristics name
"description": ""} # Heuristics description
"""
user = kwargs['user']
h = STORAGE.heuristic.get(heuristic_id, as_obj=False)
if not h:
return make_api_response("", "Heuristic not found", 404)
if user and Classification.is_accessible(user['classification'], h['classification']):
# Always refresh stats when someone get a heuristic
h.update({'stats': STORAGE.get_stat_for_heuristic(heuristic_id)})
return make_api_response(h)
else:
return make_api_response("", "You are not allowed to see this heuristic...", 403)
def get_workflow(workflow_id, **kwargs):
"""
Load the user account information.
Variables:
workflow_id => ID of the workflow
Arguments:
None
Data Block:
None
Result example:
{
"name": "Workflow name", # Name of the workflow
"classification": "", # Max classification for workflow
"label": ['label1'], # Labels for the workflow
"priority": "LOW", # Priority of the workflow
"status": "MALICIOUS", # Status of the workflow
"query": "*:*" # Query to match the data
}
"""
wf = STORAGE.workflow.get(workflow_id, as_obj=False)
if wf:
if CLASSIFICATION.is_accessible(kwargs['user']['classification'], wf['classification']):
return make_api_response(wf)
else:
return make_api_response({},
err="You're not allowed to view workflow ID: %s" % workflow_id,
status_code=403)
else:
return make_api_response({},
err="Workflow ID %s does not exist" % workflow_id,
status_code=404)
def load_user_settings(user):
default_settings = get_default_user_settings(user)
settings = STORAGE.user_settings.get(user['uname'], as_obj=False)
srv_list = [
x for x in STORAGE.list_all_services(as_obj=False, full=True)
if x['enabled']
]
if not settings:
def_srv_list = None
settings = default_settings
else:
# Make sure all defaults are there
for key, item in default_settings.items():
if key not in settings:
settings[key] = item
# Remove all obsolete keys
for key in list(settings.keys()):
if key not in default_settings:
del settings[key]
def_srv_list = settings.get('services', {}).get('selected', None)
settings['service_spec'] = get_default_service_spec(srv_list)
settings['services'] = get_default_service_list(srv_list, def_srv_list)
# Normalize the user's classification
settings['classification'] = Classification.normalize_classification(
settings['classification'])
return settings
def delete_submission(sid, **kwargs):
"""
Delete a submission as well as all related
files, results and errors
Variables:
sid => Submission ID to be deleted
Arguments:
None
Data Block:
None
Result example:
{success: true}
"""
user = kwargs['user']
submission = STORAGE.submission.get(sid, as_obj=False)
if not submission:
return make_api_response("", f"There are not submission with sid: {sid}", 404)
if Classification.is_accessible(user['classification'], submission['classification']) \
and (submission['params']['submitter'] == user['uname'] or 'admin' in user['type']):
STORAGE.delete_submission_tree_bulk(sid, Classification, transport=FILESTORE)
STORAGE.submission.commit()
return make_api_response({"success": True})
else:
return make_api_response("", "Your are not allowed to delete this submission.", 403)
def get_alert(alert_id, **kwargs):
"""
Get the alert details for a given alert key
Variables:
alert_id => ID of the alert to get the details for
Arguments:
None
Data Block:
None
API call example:
/api/v4/alert/1234567890/
Result example:
{
KEY: VALUE, # All fields of an alert in key/value pair
}
"""
user = kwargs['user']
data = STORAGE.alert.get(alert_id, as_obj=False)
if not data:
return make_api_response("", "This alert does not exists...", 404)
if user and Classification.is_accessible(user['classification'], data['classification']):
return make_api_response(data)
else:
return make_api_response("", "You are not allowed to see this alert...", 403)
def create_bundle(sid, **kwargs):
"""
Creates a bundle containing the submission results and the associated files
Variables:
sid => ID of the submission to create the bundle for
Arguments:
use_alert => The ID provided is from an alert and we will use it to create the bundle
Data Block:
None
API call example:
/api/v4/bundle/create/234f334-...-31232/
Result example:
-- THE BUNDLE FILE BINARY --
"""
user = kwargs['user']
use_alert = request.args.get('use_alert', 'false').lower() in ['true', '']
if use_alert:
data = STORAGE.alert.get(sid, as_obj=False)
else:
data = STORAGE.submission.get(sid, as_obj=False)
if user and data and Classification.is_accessible(user['classification'],
data['classification']):
temp_target_file = None
try:
temp_target_file = bundle_create(sid,
working_dir=BUNDLING_DIR,
use_alert=use_alert)
f_size = os.path.getsize(temp_target_file)
return stream_file_response(open(temp_target_file, 'rb'),
"%s.al_bundle" % sid, f_size)
except SubmissionNotFound as snf:
return make_api_response(
"", "Submission %s does not exist. [%s]" % (sid, str(snf)),
404)
except BundlingException as be:
return make_api_response(
"", "An error occured while bundling submission %s. [%s]" %
(sid, str(be)), 404)
finally:
try:
if temp_target_file:
os.remove(temp_target_file)
except Exception:
pass
else:
return make_api_response(
"",
f"You are not allowed create a bundle for this {'alert' if use_alert else 'submission'}...",
403)
def get_file_children(sha256, **kwargs):
"""
Get the list of children files for a given file
Variables:
sha256 => A resource locator for the file (sha256)
Arguments:
None
Data Block:
None
API call example:
/api/v4/file/children/123456...654321/
Result example:
[ # List of children
{"name": "NAME OF FILE", # Name of the children
"sha256": "123..DEF"}, # sha256 of the children
]
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
if file_obj:
if user and Classification.is_accessible(user['classification'],
file_obj['classification']):
output = []
response = STORAGE.result.grouped_search(
"response.service_name",
query=f"id:{sha256}* AND response.extracted:*",
fl="*",
rows=100,
sort="created desc",
access_control=user['access_control'],
as_obj=False)
processed_srl = []
for r in response['items']:
for extracted in r['items'][0]['response']['extracted']:
if extracted['sha256'] not in processed_srl:
processed_srl.append(extracted['sha256'])
output.append({
'sha256': extracted['sha256'],
'name': extracted['name']
})
return make_api_response(output)
else:
return make_api_response({},
"You are not allowed to view this file.",
403)
else:
return make_api_response({}, "This file does not exists.", 404)
def get_or_create_summary(sid, results, user_classification, completed):
user_classification = CLASSIFICATION.normalize_classification(user_classification, long_format=False)
cache_key = f"{sid}_{user_classification}_with_sections"
for illegal_char in [" ", ":", "/"]:
cache_key = cache_key.replace(illegal_char, "")
summary_cache = STORAGE.submission_summary.get_if_exists(cache_key, as_obj=False)
if not summary_cache:
summary = STORAGE.get_summary_from_keys(
results, cl_engine=CLASSIFICATION, user_classification=user_classification,
keep_heuristic_sections=True)
expiry = now_as_iso(config.datastore.ilm.days_until_archive * 24 * 60 * 60)
partial = not completed or "missing_results" in summary or "missing_files" in summary
# Do not cache partial summary
if not partial:
summary_cache = {
"attack_matrix": json.dumps(summary['attack_matrix']),
"tags": json.dumps(summary['tags']),
"expiry_ts": expiry,
"heuristics": json.dumps(summary['heuristics']),
"classification": summary['classification'],
"filtered": summary["filtered"],
"heuristic_sections": json.dumps(summary['heuristic_sections']),
"heuristic_name_map": json.dumps(summary['heuristic_name_map'])
}
STORAGE.submission_summary.save(cache_key, summary_cache)
return {
"attack_matrix": summary['attack_matrix'],
"tags": summary['tags'],
"expiry_ts": expiry,
"heuristics": summary['heuristics'],
"classification": summary['classification'],
"filtered": summary["filtered"],
"partial": partial,
"heuristic_sections": summary['heuristic_sections'],
"heuristic_name_map": summary['heuristic_name_map']
}
return {
"attack_matrix": json.loads(summary_cache['attack_matrix']),
"tags": json.loads(summary_cache['tags']),
"expiry_ts": summary_cache["expiry_ts"],
"heuristics": json.loads(summary_cache['heuristics']),
"classification": summary_cache['classification'],
"filtered": summary_cache["filtered"],
"partial": False,
"heuristic_sections": json.loads(summary_cache['heuristic_sections']),
"heuristic_name_map": json.loads(summary_cache['heuristic_name_map'])
}
def get_file_score(sha256, **kwargs):
"""
Get the score of the latest service run for a given file.
Variables:
sha256 => A resource locator for the file (SHA256)
Arguments:
None
Data Block:
None
API call example:
/api/v4/file/score/123456...654321/
Result example:
{"file_info": {}, # File info Block
"result_keys": [<keys>] # List of keys used to compute the score
"score": 0} # Latest score for the file
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
if not file_obj:
return make_api_response([], "This file does not exists", 404)
if user and Classification.is_accessible(user['classification'],
file_obj['classification']):
score = 0
keys = []
res = STORAGE.result.grouped_search(
"response.service_name",
f"id:{sha256}*",
fl="result.score,id",
sort="created desc",
access_control=user["access_control"],
rows=100,
as_obj=False)
for s in res['items']:
for d in s['items']:
score += d['result']['score']
keys.append(d["id"])
return make_api_response({
"file_info": file_obj,
"score": score,
"result_keys": keys
})
else:
return make_api_response([], "You are not allowed to view this file",
403)
def get_file_hex(sha256, **kwargs):
"""
Returns the file hex representation
Variables:
sha256 => A resource locator for the file (sha256)
Arguments:
bytes_only => Only return bytes with no formatting
length => Number of bytes per lines
Data Block:
None
API call example:
/api/v4/file/hex/123456...654321/
Result example:
<THE FILE HEX REPRESENTATION>
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
bytes_only = request.args.get('bytes_only',
'false').lower() in ['true', '']
length = int(request.args.get('length', '16'))
if not file_obj:
return make_api_response({}, "The file was not found in the system.",
404)
if file_obj['size'] > API_MAX_SIZE:
return make_api_response(
{}, "This file is too big to be seen through this API.", 403)
if user and Classification.is_accessible(user['classification'],
file_obj['classification']):
data = FILESTORE.get(sha256)
if not data:
return make_api_response({},
"This file was not found in the system.",
404)
if bytes_only:
return make_api_response(dump(data).decode())
else:
return make_api_response(hexdump(data, length=length))
else:
return make_api_response({}, "You are not allowed to view this file.",
403)
def get_submission(sid, **kwargs):
"""
Get the submission details for a given Submission ID
Variables:
sid => Submission ID to get the details for
Arguments:
None
Data Block:
None
Result example:
{"files": [ # List of source files
["FNAME", "sha256"], ...], # Each file = List of name/sha256
"errors": [], # List of error keys (sha256.ServiceName)
"submission": { # Submission Block
"profile": true, # Should keep stats about execution?
"description": "", # Submission description
"ttl": 30, # Submission days to live
"ignore_filtering": false, # Ignore filtering services?
"priority": 1000, # Submission priority, higher = faster
"ignore_cache": true, # Force reprocess even is result exist?
"groups": ["group", ...], # List of groups with access
"sid": "ab9...956", # Submission ID
"submitter": "user", # Uname of the submitter
"max_score": 1422, }, # Score of highest scoring file
"results": [], # List of Results keys (sha256.ServiceName.Version.Config)
"times": { # Timing block
"completed": "2014-...", # Completed time
"submitted": "2014-..." # Submitted time
},
"state": "completed", # State of the submission
"services": { # Service Block
"selected": ["mcafee"], # List of selected services
"params": {}, # Service specific parameters
"excluded": [] # List of excluded services
}
}
"""
user = kwargs['user']
data = STORAGE.submission.get(sid, as_obj=False)
if data is None:
return make_api_response("", "Submission ID %s does not exists." % sid, 404)
if data and user and Classification.is_accessible(user['classification'], data['classification']):
return make_api_response(data)
else:
return make_api_response("", "You are not allowed to view the data of this submission", 403)
def get_file_strings(sha256, **kwargs):
"""
Return all strings in a given file
Variables:
sha256 => A resource locator for the file (sha256)
Arguments:
len => Minimum length for a string
Data Block:
None
Result example:
<THE LIST OF STRINGS>
"""
user = kwargs['user']
hlen = request.args.get('len', "6")
file_obj = STORAGE.file.get(sha256, as_obj=False)
if file_obj['size'] > API_MAX_SIZE:
return make_api_response(
{}, "This file is too big to be seen through this API.", 403)
if not file_obj:
return make_api_response({}, "The file was not found in the system.",
404)
if user and Classification.is_accessible(user['classification'],
file_obj['classification']):
data = FILESTORE.get(sha256)
if not data:
return make_api_response({},
"This file was not found in the system.",
404)
# Ascii strings (we use decode with replace on to create delimiters)
pattern = "[\x1f-\x7e]{%s,}" % hlen
string_list = re.findall(pattern, data.decode("ascii",
errors="replace"))
# UTF-16 strings
string_list += re.findall(pattern,
data.decode("utf-16", errors="replace"))
return make_api_response("\n".join(string_list))
else:
return make_api_response({}, "You are not allowed to view this file.",
403)
def set_verdict(alert_id, verdict, **kwargs):
"""
Set the verdict of an alert based on its ID.
Variables:
submission_id -> ID of the alert to give a verdict to
verdict -> verdict that the user think the alert is: malicious or non_malicious
Arguments:
None
Data Block:
None
Result example:
{"success": True} # Has the verdict been set or not
"""
reverse_verdict = {
'malicious': 'non_malicious',
'non_malicious': 'malicious'
}
user = kwargs['user']
if verdict not in ['malicious', 'non_malicious']:
return make_api_response({"success": False}, f"'{verdict}' is not a valid verdict.", 400)
document = STORAGE.alert.get(alert_id, as_obj=False)
if not document:
return make_api_response({"success": False}, f"There are no alert with id: {alert_id}", 404)
if not Classification.is_accessible(user['classification'], document['classification']):
return make_api_response({"success": False}, "You are not allowed to give verdict on alert with "
f"ID: {alert_id}", 403)
resp = STORAGE.alert.update_by_query(f"sid:{document['sid']}", [
('REMOVE', f'verdict.{verdict}', user['uname']),
('APPEND', f'verdict.{verdict}', user['uname']),
('REMOVE', f'verdict.{reverse_verdict[verdict]}', user['uname'])
])
STORAGE.submission.update(document['sid'], [
('REMOVE', f'verdict.{verdict}', user['uname']),
('APPEND', f'verdict.{verdict}', user['uname']),
('REMOVE', f'verdict.{reverse_verdict[verdict]}', user['uname'])
])
return make_api_response({"success": resp != 0})
def get_file_information(sha256, **kwargs):
"""
Get information about the file like:
Hashes, size, frequency count, etc...
Variables:
sha256 => A resource locator for the file (sha256)
Arguments:
None
Data Block:
None
API call example:
/api/v4/file/info/123456...654321/
Result example:
{ # File information block
"ascii": "PK..", # First 64 bytes as ASCII
"classification": "UNRESTRICTED", # Access control for the file
"entropy": 7.99, # File's entropy
"hex": "504b...c0b2", # First 64 bytes as hex
"magic": "Zip archive data", # File's identification description (from magic)
"md5": "8f31...a048", # File's MD5 hash
"mime": "application/zip", # Mimetype of the file (from magic)
"seen_count": 7, # Number of time we've seen this file
"seen_first": "2015-03-04T21:59:13.204861Z", # Time at which we first seen this file
"seen_last": "2015-03-10T19:42:04.587233Z", # Last time we've seen the file
"sha256": "e021...4de2", # File's sha256 hash
"sha1": "354f...fdab", # File's sha1 hash
"size": 3417, # Size of the file
"ssdeep": "4:Smm...OHY+", # File's SSDEEP hash
"tag": "archive/zip" # Type of file that we identified
}
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
if file_obj:
if user and Classification.is_accessible(user['classification'],
file_obj['classification']):
return make_api_response(file_obj)
else:
return make_api_response({},
"You are not allowed to view this file.",
403)
else:
return make_api_response({}, "This file does not exists.", 404)
def filter_sections(sections, user_classification, min_classification):
max_classification = min_classification
# Filtering section you do not have access to
temp_sections = [
s for s in sections if CLASSIFICATION.is_accessible(
user_classification, s['classification'])
]
final_sections = []
for section in temp_sections:
try:
# Recalculation max classification using the currently accessible sections
section['classification'] = CLASSIFICATION.max_classification(
section['classification'], min_classification)
max_classification = CLASSIFICATION.max_classification(
section['classification'], max_classification)
except InvalidClassification:
continue
final_sections.append(fix_section_data(section))
# Telling the user a section was hidden
if len(sections) != len(final_sections):
hidden_section = dict(
body=
"One of the sections produced by the service has been removed because you do not have enough "
"priviledges to see its results. \n\nContact system administrators for more information.",
title_text="WARNING: Service sections have been sanitized",
depth=0,
classification=CLASSIFICATION.UNRESTRICTED,
tags=[],
heuristic=None,
body_format="TEXT")
final_sections.insert(0, hidden_section)
return max_classification, final_sections
def check_for_source_change(delta_list, source):
change_list = {}
for delta in delta_list:
if delta['name'] == source['name']:
# Classification update
if delta['default_classification'] != source[
'default_classification']:
class_norm = Classification.normalize_classification(
delta['default_classification'])
change_list[
'default_classification'] = STORAGE.signature.update_by_query(
query=f'source:"{source["name"]}"',
operations=[("SET", "classification", class_norm)])
return change_list
def get_file_results_for_service(sha256, service, **kwargs):
"""
Get the all the file results of a specific file and a specific query.
Variables:
sha256 => A resource locator for the file (SHA256)
Arguments:
all => if all argument is present, it will return all versions
NOTE: Max to 100 results...
Data Block:
None
API call example:
/api/v4/file/result/123456...654321/service_name/
Result example:
{"file_info": {}, # File info Block
"results": {}} # Full result list for the service
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
if not file_obj:
return make_api_response([], "This file does not exists", 404)
if user and Classification.is_accessible(user['classification'],
file_obj['classification']):
res = STORAGE.result.search(f"id:{sha256}.{service}*",
sort="created desc",
fl="*",
rows=100 if "all" in request.args else 1,
access_control=user["access_control"],
as_obj=False,
use_archive=True)
results = []
for r in res['items']:
result = format_result(user['classification'], r,
file_obj['classification'])
if result:
results.append(result)
return make_api_response({"file_info": file_obj, "results": results})
else:
return make_api_response([], "You are not allowed to view this file",
403)
def get_file_image_datastream(sha256, **kwargs):
"""
Returns the image file as a datastream
Variables:
sha256 => A resource locator for the file (sha256)
Arguments:
None
Data Block:
None
API call example:
/api/v4/file/image/123456...654321/
Result example:
data:image/png;base64,...
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
if not file_obj:
return make_api_response({}, "The file was not found in the system.",
404)
if not file_obj.get('is_section_image',
False) or not file_obj['type'].startswith("image/"):
return make_api_response(
{}, "This file is not allowed to be downloaded as a datastream.",
403)
if user and Classification.is_accessible(user['classification'],
file_obj['classification']):
data = FILESTORE.get(sha256)
if not data:
return make_api_response({},
"This file was not found in the system.",
404)
return make_api_response(
f"data:{file_obj['type']};base64,{base64.b64encode(data).decode()}"
)
else:
return make_api_response({}, "You are not allowed to view this file.",
403)
def get_user_classification(self, group_dn_list):
"""
Extend the users classification information with the configured group information
NB: This is not fully implemented at this point
:param group_dn_list: list of DNs the user is member of
:return:
"""
ret = CLASSIFICATION.UNRESTRICTED
for group_dn in group_dn_list:
if group_dn in self.classification_mappings:
ret = CLASSIFICATION.max_classification(
ret, self.classification_mappings[group_dn])
return ret
def get_user_favorites(username, **kwargs):
"""
Loads the user's favorites.
Variables:
username => Name of the user you want to get the avatar for
Arguments:
None
Data Block:
None
Result example:
{ # Dictionary of
"<name_of_query>": # Named queries
"*:*", # The actual query to run
...
}
"""
user = kwargs['user']
favorites = {
"alert": [],
"search": [],
"signature": [],
"submission": [],
"error": []
}
res_favorites = STORAGE.user_favorites.get(username, as_obj=False)
if res_favorites:
if username == "__global__" or username != user['uname']:
for key in favorites.keys():
for fav in res_favorites[key]:
if 'classification' in fav:
if CLASSIFICATION.is_accessible(
user['classification'], fav['classification']):
favorites[key].append(fav)
else:
favorites[key].append(fav)
else:
favorites.update(res_favorites)
return make_api_response(favorites)
def change_priority(alert_id, **kwargs):
"""
Change the priority of a given alert
Variables:
alert_id => ID of the alert to change the priority
Arguments:
"HIGH" => New priority for the alert
Data Block:
None
API call example:
/api/v4/alert/priority/12345...67890/
Result example:
{"success": true}
"""
user = kwargs['user']
try:
priority = request.json
priority = priority.upper()
if priority not in PRIORITIES:
raise ValueError("Not in priorities")
except ValueError:
return make_api_response({"success": False}, err="Invalid priority received.", status_code=400)
alert = STORAGE.alert.get(alert_id, as_obj=False)
if not alert:
return make_api_response({"success": False},
err="Alert ID %s not found" % alert_id,
status_code=404)
if not Classification.is_accessible(user['classification'], alert['classification']):
return make_api_response("", "You are not allowed to see this alert...", 403)
if priority != alert.get('priority', None):
return make_api_response({
"success": STORAGE.alert.update(alert_id, [(STORAGE.alert.UPDATE_SET, 'priority', priority)])})
else:
return make_api_response({"success": True})
def change_status(alert_id, **kwargs):
"""
Change the status of a given alert
Variables:
alert_id => ID of the alert to change the status
Arguments:
None
Data Block:
"MALICIOUS" => New status for the alert
API call example:
/api/v4/alert/status/12345...67890/
Result example:
{"success": true}
"""
user = kwargs['user']
try:
status = request.json
status = status.upper()
if status not in STATUSES:
raise ValueError("Not in priorities")
except ValueError:
return make_api_response({"success": False}, err="Invalid status received.", status_code=400)
alert = STORAGE.alert.get(alert_id, as_obj=False)
if not alert:
return make_api_response({"success": False},
err="Alert ID %s not found" % alert_id,
status_code=404)
if not Classification.is_accessible(user['classification'], alert['classification']):
return make_api_response("", "You are not allowed to see this alert...", 403)
if status != alert.get('status', None):
return make_api_response({
"success": STORAGE.alert.update(alert_id, [(STORAGE.alert.UPDATE_SET, 'status', status)])})
else:
return make_api_response({"success": True})
def get_file_ascii(sha256, **kwargs):
"""
Return the ascii values for a file where ascii chars are replaced by DOTs.
Variables:
sha256 => A resource locator for the file (sha256)
Arguments:
None
Data Block:
None
Result example:
<THE ASCII FILE>
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
if file_obj['size'] > API_MAX_SIZE:
return make_api_response(
{}, "This file is too big to be seen through this API.", 403)
if not file_obj:
return make_api_response({}, "The file was not found in the system.",
404)
if user and Classification.is_accessible(user['classification'],
file_obj['classification']):
data = FILESTORE.get(sha256)
if not data:
return make_api_response({},
"This file was not found in the system.",
404)
return make_api_response(data.translate(FILTER_ASCII).decode())
else:
return make_api_response({}, "You are not allowed to view this file.",
403)
