def get_service_result(cache_key, **kwargs):
"""
Get the result for a given service cache key.
Variables:
cache_key => Service result cache key
as SRL.ServiceName.ServiceVersion.ConfigHash
Arguments:
None
Data Block:
None
Result example:
{"response": { # Service Response
"milestones": {}, # Timing object
"supplementary": [], # Supplementary files
"service_name": "Mcafee", # Service Name
"message": "", # Service error message
"extracted": [], # Extracted files
"service_version": "v0"}, # Service Version
"result": { # Result objects
"score": 1302, # Total score for the file
"sections": [{ # Result sections
"body": "Text goes here", # Body of the section (TEXT)
"classification": "", # Classification
"links": [], # Links inside the section
"title_text": "Title", # Title of the section
"depth": 0, # Depth (for Display purposes)
"score": 500, # Section's score
"body_format": null, # Body format
"subsections": [] # List of sub-sections
}, ... ],
"classification": "", # Maximum classification for service
"tags": [{ # Generated Tags
"usage": "IDENTIFICATION", # Tag usage
"value": "Tag Value", # Tag value
"type": "Tag Type", # Tag type
"weight": 50, # Tag Weight
"classification": "" # Tag Classification
}, ...]
}
}
"""
user = kwargs['user']
data = STORAGE.get_result(cache_key)
if data is None:
return make_api_response("",
"Cache key %s does not exists." % cache_key,
404)
cur_file = STORAGE.get_file(cache_key[:64])
data = format_result(user['classification'], data,
cur_file['classification'])
if not data:
return make_api_response(
"", "You are not allowed to view the results for this key", 403)
return make_api_response(data)
def get_file_hex(srl, **kwargs):
"""
Returns the file hex representation
Variables:
srl => A resource locator for the file (sha256)
Arguments:
None
Data Block:
None
API call example:
/api/v3/file/hex/123456...654321/
Result example:
<THE FILE HEX REPRESENTATION>
"""
user = kwargs['user']
file_obj = STORAGE.get_file(srl)
if not file_obj:
return make_api_response({}, "The file was not found in the system.", 404)
if user and Classification.is_accessible(user['classification'], file_obj['classification']):
with forge.get_filestore() as f_transport:
data = f_transport.get(srl)
if not data:
return make_api_response({}, "This file was not found in the system.", 404)
return make_api_response(hexdump(data))
else:
return make_api_response({}, "You are not allowed to view this file.", 403)
def download_file(srl, **kwargs):
"""
Download the file using the default encoding method. This api
will force the browser in download mode.
Variables:
srl => A resource locator for the file (sha256)
Arguments:
name => Name of the file to download
format => Format to encode the file in
password => Password of the password protected zip
Data Block:
None
API call example:
/api/v3/file/download/123456...654321/
Result example:
<THE FILE BINARY ENCODED IN SPECIFIED FORMAT>
"""
user = kwargs['user']
file_obj = STORAGE.get_file(srl)
if not file_obj:
return make_api_response({}, "The file was not found in the system.", 404)
if user and Classification.is_accessible(user['classification'], file_obj['classification']):
params = load_user_settings(user)
name = request.args.get('name', srl)
if name == "":
name = srl
else:
name = basename(name)
name = safe_str(name)
file_format = request.args.get('format', params['download_encoding'])
if file_format == "raw" and not ALLOW_RAW_DOWNLOADS:
return make_api_response({}, "RAW file download has been disabled by administrators.", 403)
password = request.args.get('password', None)
with forge.get_filestore() as f_transport:
data = f_transport.get(srl)
if not data:
return make_api_response({}, "The file was not found in the system.", 404)
data, error, already_encoded = encode_file(data, file_format, name, password)
if error:
return make_api_response({}, error['text'], error['code'])
if file_format != "raw" and not already_encoded:
name = "%s.%s" % (name, file_format)
return make_file_response(data, name, len(data))
else:
return make_api_response({}, "You are not allowed to download this file.", 403)
def get_file_children(srl, **kwargs):
"""
Get the list of children files for a given file
Variables:
srl => A resource locator for the file (sha256)
Arguments:
None
Data Block:
None
API call example:
/api/v3/file/children/123456...654321/
Result example:
[ # List of children
{"name": "NAME OF FILE", # Name of the children
"srl": "123..DEF"}, # SRL of the children (SHA256)
]
"""
user = kwargs['user']
file_obj = STORAGE.get_file(srl)
if file_obj:
if user and Classification.is_accessible(user['classification'], file_obj['classification']):
return make_api_response(STORAGE.list_file_childrens(srl, access_control=user["access_control"]))
else:
return make_api_response({}, "You are not allowed to view this file.", 403)
else:
return make_api_response({}, "This file does not exists.", 404)
def get_file_raw(srl, **kwargs):
"""
Return the raw values for a file where non-utf8 chars are replaced by DOTs.
Variables:
srl => A resource locator for the file (sha256)
Arguments:
None
Data Block:
None
Result example:
<THE RAW FILE>
"""
user = kwargs['user']
file_obj = STORAGE.get_file(srl)
if not file_obj:
return make_api_response({}, "The file was not found in the system.", 404)
if user and Classification.is_accessible(user['classification'], file_obj['classification']):
with forge.get_filestore() as f_transport:
data = f_transport.get(srl)
if not data:
return make_api_response({}, "This file was not found in the system.", 404)
return make_api_response(data.translate(FILTER_RAW))
else:
return make_api_response({}, "You are not allowed to view this file.", 403)
def get_file_score(srl, **kwargs):
"""
Get the score of the latest service run for a given file.
Variables:
srl => A resource locator for the file (SHA256)
Arguments:
None
Data Block:
None
API call example:
/api/v3/file/score/123456...654321/
Result example:
{"file_info": {}, # File info Block
"result_keys": [<keys>] # List of keys used to compute the score
"score": 0} # Latest score for the file
"""
user = kwargs['user']
file_obj = STORAGE.get_file(srl)
if not file_obj:
return make_api_response([], "This file does not exists", 404)
args = [
("group", "on"),
("group.field", "response.service_name"),
("group.format", "simple"),
("fl", "result.score,_yz_rk"),
("sort", "created desc"),
("rows", "100")
]
if user and Classification.is_accessible(user['classification'], file_obj['classification']):
score = 0
keys = []
res = STORAGE.direct_search("result", "_yz_rk:%s*" % srl, args,
__access_control__=user["access_control"])
docs = res['grouped']['response.service_name']['doclist']['docs']
for d in docs:
score += d['result.score']
keys.append(d["_yz_rk"])
return make_api_response({"file_info": file_obj, "score": score, "result_keys": keys})
else:
return make_api_response([], "You are not allowed to view this file", 403)
def get_file_results_for_service(srl, service, **kwargs):
"""
Get the all the file results of a specific file and a specific query.
Variables:
srl => A resource locator for the file (SHA256)
Arguments:
all => if all argument is present, it will return all versions
NOTE: Max to 100 results...
Data Block:
None
API call example:
/api/v3/file/result/123456...654321/service_name/
Result example:
{"file_info": {}, # File info Block
"results": {}} # Full result list for the service
"""
user = kwargs['user']
file_obj = STORAGE.get_file(srl)
args = [("fl", "_yz_rk"),
("sort", "created desc")]
if "all" in request.args:
args.append(("rows", "100"))
else:
args.append(("rows", "1"))
if not file_obj:
return make_api_response([], "This file does not exists", 404)
if user and Classification.is_accessible(user['classification'], file_obj['classification']):
res = STORAGE.direct_search("result", "_yz_rk:%s.%s*" % (srl, service), args,
__access_control__=user["access_control"])['response']['docs']
keys = [k["_yz_rk"] for k in res]
results = []
for r in STORAGE.get_results(keys):
result = format_result(user['classification'], r, file_obj['classification'])
if result:
results.append(result)
return make_api_response({"file_info": file_obj, "results": results})
else:
return make_api_response([], "You are not allowed to view this file", 403)
def file_viewer(**kwargs):
user = kwargs['user']
srl = angular_safe(request.args.get("srl", None))
if not srl:
abort(404)
data = STORAGE.get_file(srl)
if not data:
abort(404)
if not Classification.is_accessible(user['classification'],
data['classification']):
abort(403)
return custom_render("file_viewer.html", srl=srl, **kwargs)
def get_file_information(srl, **kwargs):
"""
Get information about the file like:
Hashes, size, frequency count, etc...
Variables:
srl => A resource locator for the file (sha256)
Arguments:
None
Data Block:
None
API call example:
/api/v3/file/info/123456...654321/
Result example:
{ # File information block
"ascii": "PK..", # First 64 bytes as ASCII
"classification": "UNRESTRICTED", # Access control for the file
"entropy": 7.99, # File's entropy
"hex": "504b...c0b2", # First 64 bytes as hex
"magic": "Zip archive data", # File's identification description (from magic)
"md5": "8f31...a048", # File's MD5 hash
"mime": "application/zip", # Mimetype of the file (from magic)
"seen_count": 7, # Number of time we've seen this file
"seen_first": "2015-03-04T21:59:13.204861Z", # Time at which we first seen this file
"seen_last": "2015-03-10T19:42:04.587233Z", # Last time we've seen the file
"sha256": "e021...4de2", # File's sha256 hash
"sha1": "354f...fdab", # File's sha1 hash
"size": 3417, # Size of the file
"ssdeep": "4:Smm...OHY+", # File's SSDEEP hash
"tag": "archive/zip" # Type of file that we identified
}
"""
user = kwargs['user']
file_obj = STORAGE.get_file(srl)
if file_obj:
if user and Classification.is_accessible(user['classification'], file_obj['classification']):
return make_api_response(file_obj)
else:
return make_api_response({}, "You are not allowed to view this file.", 403)
else:
return make_api_response({}, "This file does not exists.", 404)
def get_file_strings(srl, **kwargs):
"""
Return all strings in a given file
Variables:
srl => A resource locator for the file (sha256)
Arguments:
len => Minimum length for a string
Data Block:
None
Result example:
<THE LIST OF STRINGS>
"""
user = kwargs['user']
hlen = request.args.get('len', "6")
file_obj = STORAGE.get_file(srl)
if not file_obj:
return make_api_response({}, "The file was not found in the system.", 404)
if user and Classification.is_accessible(user['classification'], file_obj['classification']):
with forge.get_filestore() as f_transport:
data = f_transport.get(srl)
if not data:
return make_api_response({}, "This file was not found in the system.", 404)
# Ascii strings
pattern = "[\x1f-\x7e]{%s,}" % hlen
string_list = re.findall(pattern, data)
# UTF-16 strings
try:
string_list += re.findall(pattern, data.decode("utf-16", errors="ignore"))
except UnicodeDecodeError:
pass
return make_api_response("\n".join(string_list))
else:
return make_api_response({}, "You are not allowed to view this file.", 403)
def get_file_results(srl, **kwargs):
"""
Get the all the file results of a specific file.
Variables:
srl => A resource locator for the file (SHA256)
Arguments:
None
Data Block:
None
API call example:
/api/v3/file/result/123456...654321/
Result example:
{"file_info": {}, # File info Block
"results": {}, # Full result list
"errors": {}, # Full error list
"parents": {}, # List of possible parents
"childrens": {}, # List of children files
"tags": {}, # List tags generated
"metadata": {}, # Metadata facets results
"file_viewer_only": True } # UI switch to disable features
"""
user = kwargs['user']
file_obj = STORAGE.get_file(srl)
if not file_obj:
return make_api_response({}, "This file does not exists", 404)
if user and Classification.is_accessible(user['classification'], file_obj['classification']):
output = {"file_info": {}, "results": [], "tags": []}
plan = [
(STORAGE.list_file_active_keys, (srl, user["access_control"]), "results"),
(STORAGE.list_file_parents, (srl, user["access_control"]), "parents"),
(STORAGE.list_file_childrens, (srl, user["access_control"]), "children"),
(STORAGE.get_file_submission_meta, (srl, user["access_control"]), "meta"),
]
temp = execute_concurrently(plan)
active_keys, alternates = temp['results']
output['parents'] = temp['parents']
output['childrens'] = temp['children']
output['metadata'] = temp['meta']
output['file_info'] = file_obj
output['results'] = []
output['alternates'] = {}
res = STORAGE.get_results(active_keys)
for r in res:
res = format_result(user['classification'], r, file_obj['classification'])
if res:
output['results'].append(res)
for i in alternates:
if i['response']['service_name'] not in output["alternates"]:
output["alternates"][i['response']['service_name']] = []
i['response']['service_version'] = i['_yz_rk'].split(".", 3)[2].replace("_", ".")
output["alternates"][i['response']['service_name']].append(i)
output['errors'] = []
output['file_viewer_only'] = True
for res in output['results']:
# noinspection PyBroadException
try:
if "result" in res:
if 'tags' in res['result']:
output['tags'].extend(res['result']['tags'])
except:
pass
return make_api_response(output)
else:
return make_api_response({}, "You are not allowed to view this file", 403)
def get_file_submission_results(sid, srl, **kwargs):
"""
Get the all the results and errors of a specific file
for a specific Submission ID
Variables:
sid => Submission ID to get the result for
srl => Resource locator to get the result for
Arguments (POST only):
extra_result_keys => List of extra result keys to get
extra_error_keys => List of extra error keys to get
Data Block:
None
Result example:
{"errors": [], # List of error blocks
"file_info": {}, # File information block (md5, ...)
"results": [], # List of result blocks
"tags": [] } # List of generated tags
"""
user = kwargs['user']
# Check if submission exist
data = STORAGE.get_submission(sid)
if data is None:
return make_api_response("", "Submission ID %s does not exists." % sid, 404)
if data and user and Classification.is_accessible(user['classification'], data['classification']):
# Prepare output
output = {"file_info": {}, "results": [], "tags": [], "errors": []}
# Extra keys - This is a live mode optimisation
res_keys = data.get("results", [])
err_keys = data.get("errors", [])
if request.method == "POST" and request.json is not None and data['state'] != "completed":
extra_rkeys = request.json.get("extra_result_keys", [])
extra_ekeys = request.json.get("extra_error_keys", [])
# Load keys
res_keys.extend(extra_rkeys)
err_keys.extend(extra_ekeys)
res_keys = list(set(res_keys))
err_keys = list(set(err_keys))
# Get File, results and errors
temp_file = STORAGE.get_file(srl)
if not Classification.is_accessible(user['classification'], temp_file['classification']):
return make_api_response("", "You are not allowed to view the data of this file", 403)
output['file_info'] = temp_file
temp_results = STORAGE.get_results(set([x for x in res_keys if x.startswith(srl)]))
results = []
for r in temp_results:
r = format_result(user['classification'], r, temp_file['classification'])
if r:
results.append(r)
output['results'] = results
output['errors'] = STORAGE.get_errors(set([x for x in err_keys if x.startswith(srl)]))
output['metadata'] = STORAGE.get_file_submission_meta(srl, user["access_control"])
# Generate tag list
temp = {}
for res in output['results']:
try:
if res.has_key('result'):
if res['result'].has_key('tags'):
temp.update({"__".join([v["type"], v['value']]): v for v in res['result']['tags']})
except:
pass
output["tags"] = temp.values()
return make_api_response(output)
else:
return make_api_response("", "You are not allowed to view the data of this submission", 403)
