def submit(**kwargs):
"""
Submit a single file, sha256 or url for analysis
Note 1:
If you are submitting a sh256 or a URL, you must use the application/json encoding and one of
sha256 or url parameters must be included in the data block.
Note 2:
If you are submitting a file directly, you have to use multipart/form-data encoding this
was done to reduce the memory footprint and speedup file transfers
** Read documentation of mime multipart standard if your library does not support it**
The multipart/form-data for sending binary has two parts:
- The first part contains a JSON dump of the optional params and uses the name 'json'
- The last part conatins the file binary, uses the name 'bin' and includes a filename
Variables:
None
Arguments:
None
Data Block (SHA256 or URL):
{
// REQUIRED: One of the two following
"sha256": "123...DEF", # SHA256 hash of the file already in the datastore
"url": "http://...", # Url to fetch the file from
// OPTIONAL VALUES
"name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url
"metadata": { # Submission metadata
"key": val, # Key/Value pair metadata values
},
"params": { # Submission parameters
"key": val, # Key/Value pair for params that different then defaults
}, # Default params can be fetch at /api/v3/user/submission_params/<user>/
}
Data Block (Binary):
--0b34a3c50d3c02dd804a172329a0b2aa <-- Randomly generated boundary for this http request
Content-Disposition: form-data; name="json" <-- JSON data blob part (only previous optional values valid)
{"metadata": {"hello": "world"}}
--0b34a3c50d3c02dd804a172329a0b2aa <-- Switch to next part, file part
Content-Disposition: form-data; name="bin"; filename="name_of_the_file_to_scan.bin"
<BINARY DATA OF THE FILE TO SCAN... DOES NOT NEED TO BE ENCODDED>
--0b34a3c50d3c02dd804a172329a0b2aa-- <-- End of HTTP transmission
Result example:
<Submission message object as a json dictionary>
"""
user = kwargs['user']
quota_error = check_submission_quota(user)
if quota_error:
return make_api_response("", quota_error, 503)
out_dir = os.path.join(TEMP_SUBMIT_DIR, get_random_id())
with forge.get_filestore() as f_transport:
try:
# Get data block and binary blob
if 'multipart/form-data' in request.content_type:
if 'json' in request.values:
data = json.loads(request.values['json'])
else:
data = {}
binary = request.files['bin']
name = data.get("name", binary.filename)
sha256 = None
url = None
elif 'application/json' in request.content_type:
data = request.json
binary = None
sha256 = data.get('sha256', None)
url = data.get('url', None)
name = data.get(
"name", None) or sha256 or os.path.basename(url) or None
else:
return make_api_response({}, "Invalid content type", 400)
if data is None:
return make_api_response({}, "Missing data block", 400)
if not name:
return make_api_response({}, "Filename missing", 400)
name = os.path.basename(name)
if not name:
return make_api_response({}, "Invalid filename", 400)
# Create task object
if "ui_params" in data:
s_params = ui_to_submission_params(data['ui_params'])
else:
s_params = ui_to_submission_params(
STORAGE.user_settings.get(user['uname'], as_obj=False))
if not s_params:
s_params = get_default_user_settings(user)
s_params.update(data.get("params", {}))
if 'groups' not in s_params:
s_params['groups'] = user['groups']
s_params['quota_item'] = True
s_params['submitter'] = user['uname']
if not s_params['description']:
s_params['description'] = "Inspection of file: %s" % name
if not Classification.is_accessible(user['classification'],
s_params['classification']):
return make_api_response(
{}, "You cannot start a scan with higher "
"classification then you're allowed to see", 400)
# Prepare the output directory
try:
os.makedirs(out_dir)
except Exception:
pass
out_file = os.path.join(out_dir, name)
# Get the output file
extra_meta = {}
if not binary:
if sha256:
if f_transport.exists(sha256):
f_transport.download(sha256, out_file)
else:
return make_api_response(
{}, "SHA256 does not exist in our datastore", 404)
else:
if url:
if not config.ui.allow_url_submissions:
return make_api_response(
{},
"URL submissions are disabled in this system",
400)
try:
safe_download(url, out_file)
extra_meta['submitted_url'] = url
except FileTooBigException:
return make_api_response(
{}, "File too big to be scanned.", 400)
except InvalidUrlException:
return make_api_response(
{}, "Url provided is invalid.", 400)
except ForbiddenLocation:
return make_api_response(
{}, "Hostname in this URL cannot be resolved.",
400)
else:
return make_api_response(
{},
"Missing file to scan. No binary, sha256 or url provided.",
400)
else:
with open(out_file, "wb") as my_file:
my_file.write(binary.read())
try:
metadata = flatten(data.get('metadata', {}))
metadata.update(extra_meta)
submission_obj = Submission({
"files": [],
"metadata": metadata,
"params": s_params
})
except (ValueError, KeyError) as e:
return make_api_response("", err=str(e), status_code=400)
# Submit the task to the system
try:
result = SubmissionClient(datastore=STORAGE,
filestore=f_transport,
config=config).submit(
submission_obj,
local_files=[out_file],
cleanup=False)
except SubmissionException as e:
return make_api_response("", err=str(e), status_code=400)
return make_api_response(result.as_primitives())
finally:
try:
# noinspection PyUnboundLocalVariable
os.unlink(out_file)
except Exception:
pass
try:
shutil.rmtree(out_dir, ignore_errors=True)
except Exception:
pass
def start_ui_submission(ui_sid, **kwargs):
"""
Start UI submission.
Starts processing after files where uploaded to the server.
Variables:
ui_sid => UUID for the current UI file upload
Arguments:
None
Data Block (REQUIRED):
Dictionary of UI specific user settings
Result example:
{
'started': True, # Has the submission started processing?
'sid' : "c7668cfa-...-c4132285142e" # Submission ID
}
"""
user = kwargs['user']
quota_error = check_submission_quota(user)
if quota_error:
return make_api_response("", quota_error, 503)
ui_params = request.json
ui_params['groups'] = kwargs['user']['groups']
ui_params['quota_item'] = True
ui_params['submitter'] = user['uname']
if not Classification.is_accessible(user['classification'],
ui_params['classification']):
return make_api_response({
"started": False,
"sid": None
}, "You cannot start a scan with higher "
"classification then you're allowed to see",
403)
request_files = []
request_dirs = []
fnames = []
try:
flist = glob.glob(TEMP_DIR + ui_sid + "*")
if len(flist) > 0:
# Generate file list
for fpath in flist:
request_dirs.append(fpath)
files = os.listdir(fpath)
for myfile in files:
request_files.append(os.path.join(fpath, myfile))
if myfile not in fnames:
fnames.append(myfile)
if not ui_params['description']:
ui_params['description'] = "Inspection of file%s: %s" % (
{
True: "s",
False: ""
}[len(fnames) > 1], ", ".join(fnames))
# Submit to dispatcher
try:
submission_obj = Submission({
"files": [],
"params":
ui_to_submission_params(ui_params)
})
except (ValueError, KeyError) as e:
return make_api_response("", err=str(e), status_code=400)
with forge.get_filestore() as f_transport:
try:
result = SubmissionClient(datastore=STORAGE,
filestore=f_transport,
config=config).submit(
submission_obj,
local_files=request_files,
cleanup=False)
except SubmissionException as e:
return make_api_response("", err=str(e), status_code=400)
return make_api_response({"started": True, "sid": result.sid})
else:
return make_api_response({
"started": False,
"sid": None
}, "No files where found for ID %s. "
"Try again..." % ui_sid, 404)
finally:
# Remove files
for myfile in request_files:
try:
os.unlink(myfile)
except Exception:
pass
# Remove dirs
for fpath in request_dirs:
try:
os.rmdir(fpath)
except Exception:
pass
def resubmit_submission_for_analysis(sid, *args, **kwargs):
"""
Resubmit a submission for analysis with the exact same parameters as before
Variables:
sid => Submission ID to re-submit
Arguments:
None
Data Block:
None
Result example:
# Submission message object as a json dictionary
"""
user = kwargs['user']
quota_error = check_submission_quota(user)
if quota_error:
return make_api_response("", quota_error, 503)
submit_result = None
try:
submission = STORAGE.submission.get(sid, as_obj=False)
if submission:
if not Classification.is_accessible(user['classification'],
submission['classification']):
return make_api_response(
"",
"You are not allowed to re-submit a submission that you don't have access to",
403)
submission_params = submission['params']
submission_params['classification'] = submission['classification']
else:
return make_api_response({},
"Submission %s does not exists." % sid,
status_code=404)
submission_params['submitter'] = user['uname']
submission_params['quota_item'] = True
submission_params[
'description'] = "Resubmit %s for analysis" % ", ".join(
[x['name'] for x in submission["files"]])
try:
submission_obj = Submission({
"files": submission["files"],
"metadata": submission['metadata'],
"params": submission_params
})
except (ValueError, KeyError) as e:
return make_api_response("", err=str(e), status_code=400)
with forge.get_filestore() as f_transport:
submit_result = SubmissionClient(
datastore=STORAGE, filestore=f_transport,
config=config).submit(submission_obj)
submission_received(submission_obj)
return make_api_response(submit_result.as_primitives())
except SubmissionException as e:
return make_api_response("", err=str(e), status_code=400)
finally:
if submit_result is None:
decrement_submission_quota(user)
def resubmit_for_dynamic(sha256, *args, **kwargs):
"""
Resubmit a file for dynamic analysis
Variables:
sha256 => Resource locator (SHA256)
Arguments (Optional):
copy_sid => Mimic the attributes of this SID.
name => Name of the file for the submission
Data Block:
None
Result example:
# Submission message object as a json dictionary
"""
user = kwargs['user']
quota_error = check_submission_quota(user)
if quota_error:
return make_api_response("", quota_error, 503)
submit_result = None
try:
copy_sid = request.args.get('copy_sid', None)
name = safe_str(request.args.get('name', sha256))
if copy_sid:
submission = STORAGE.submission.get(copy_sid, as_obj=False)
else:
submission = None
if submission:
if not Classification.is_accessible(user['classification'],
submission['classification']):
return make_api_response(
"",
"You are not allowed to re-submit a submission that you don't have access to",
403)
submission_params = submission['params']
submission_params['classification'] = submission['classification']
else:
submission_params = ui_to_submission_params(
load_user_settings(user))
with forge.get_filestore() as f_transport:
if not f_transport.exists(sha256):
return make_api_response(
{},
"File %s cannot be found on the server therefore it cannot be resubmitted."
% sha256,
status_code=404)
files = [{'name': name, 'sha256': sha256}]
submission_params['submitter'] = user['uname']
submission_params['quota_item'] = True
if 'priority' not in submission_params:
submission_params['priority'] = 500
submission_params[
'description'] = "Resubmit %s for Dynamic Analysis" % name
if "Dynamic Analysis" not in submission_params['services'][
'selected']:
submission_params['services']['selected'].append(
"Dynamic Analysis")
try:
submission_obj = Submission({
"files": files,
"params": submission_params
})
except (ValueError, KeyError) as e:
return make_api_response("", err=str(e), status_code=400)
submit_result = SubmissionClient(
datastore=STORAGE, filestore=f_transport,
config=config).submit(submission_obj)
submission_received(submission_obj)
return make_api_response(submit_result.as_primitives())
except SubmissionException as e:
return make_api_response("", err=str(e), status_code=400)
finally:
if submit_result is None:
decrement_submission_quota(user)
def start_ui_submission(ui_sid, **kwargs):
"""
Start UI submission.
Starts processing after files where uploaded to the server.
Variables:
ui_sid => UUID for the current UI file upload
Arguments:
None
Data Block (REQUIRED):
Dictionary of UI specific user settings
Result example:
{
'started': True, # Has the submission started processing?
'sid' : "c7668cfa-...-c4132285142e" # Submission ID
}
"""
user = kwargs['user']
ui_params = request.json
ui_params['groups'] = kwargs['user']['groups']
ui_params['quota_item'] = True
ui_params['submitter'] = user['uname']
if not Classification.is_accessible(user['classification'],
ui_params['classification']):
return make_api_response({
"started": False,
"sid": None
}, "You cannot start a scan with higher "
"classification then you're allowed to see",
403)
quota_error = check_submission_quota(user)
if quota_error:
return make_api_response("", quota_error, 503)
submit_result = None
submitted_file = None
try:
# Download the file from the cache
with forge.get_cachestore("flowjs", config) as cache:
ui_sid = get_cache_name(ui_sid)
if cache.exists(ui_sid):
target_dir = os.path.join(TEMP_DIR, ui_sid)
os.makedirs(target_dir, exist_ok=True)
target_file = os.path.join(target_dir,
ui_params.pop('filename', ui_sid))
if os.path.exists(target_file):
os.unlink(target_file)
# Save the reconstructed file
cache.download(ui_sid, target_file)
submitted_file = target_file
# Submit the file
if submitted_file is not None:
with open(submitted_file, 'rb') as fh:
if is_cart(fh.read(256)):
meta = get_metadata_only(submitted_file)
if meta.get('al',
{}).get('type',
'unknown') == 'archive/bundle/al':
try:
submission = import_bundle(submitted_file,
allow_incomplete=True,
identify=IDENTIFY)
except Exception as e:
return make_api_response("",
err=str(e),
status_code=400)
return make_api_response({
"started": True,
"sid": submission['sid']
})
if not ui_params['description']:
ui_params[
'description'] = f"Inspection of file: {os.path.basename(submitted_file)}"
# Submit to dispatcher
try:
params = ui_to_submission_params(ui_params)
# Enforce maximum DTL
if config.submission.max_dtl > 0:
params['ttl'] = min(int(
params['ttl']), config.submission.max_dtl) if int(
params['ttl']) else config.submission.max_dtl
submission_obj = Submission({"files": [], "params": params})
except (ValueError, KeyError) as e:
return make_api_response("", err=str(e), status_code=400)
try:
submit_result = SubmissionClient(
datastore=STORAGE,
filestore=FILESTORE,
config=config,
identify=IDENTIFY).submit(submission_obj,
local_files=[submitted_file])
submission_received(submission_obj)
except SubmissionException as e:
return make_api_response("", err=str(e), status_code=400)
return make_api_response({
"started": True,
"sid": submit_result.sid
})
else:
return make_api_response({
"started": False,
"sid": None
}, "No files where found for ID %s. "
"Try again..." % ui_sid, 404)
finally:
if submit_result is None:
decrement_submission_quota(user)
# Remove file
if os.path.exists(submitted_file):
os.unlink(submitted_file)
# Remove dir
if os.path.exists(target_dir) and os.path.isdir(target_dir):
os.rmdir(target_dir)