def load_user_settings(user):
default_settings = copy.deepcopy(SETTINGS_DEFAULT)
default_settings[
'classification'] = Classification.default_user_classification(user)
options = STORAGE.get_user_options(user['uname'])
srv_list = [x for x in STORAGE.list_services() if x['enabled']]
if not options:
def_srv_list = None
options = default_settings
else:
# Make sure all defaults are there
for key, item in default_settings.iteritems():
if key not in options:
options[key] = item
# Remove all obsolete keys
for key in options.keys():
if key not in default_settings:
del options[key]
def_srv_list = options.get('services', None)
options['service_spec'] = get_default_service_spec(srv_list)
options['services'] = get_default_service_list(srv_list, def_srv_list)
# Normalize the user's classification
options['classification'] = Classification.normalize_classification(
options['classification'])
return options
def submit_file(*args, **kwargs):
"""
Submit a single file inline
Variables:
None
Arguments:
None
Data Block (REQUIRED):
{
"name": "file.exe", # Name of the file
"binary": "A24AB..==", # Base64 encoded file binary
"params": { # Submission parameters
"key": val, # Key/Value pair for params that different then defaults
}, # Default params can be fetch at /api/v3/user/submission_params/<user>/
"srv_spec": { # Service specifics parameters
"Extract": {
"password": "******"
},
}
}
Result example:
{
"submission":{}, # Submission Block
"times": {}, # Timing Block
"state": "submitted", # Submission state
"services": {}, # Service selection Block
"fileinfo": {} # File information Block
"files": [] # List of submitted files
"request": {} # Request detail block
}
"""
user = kwargs['user']
check_submission_quota(user)
out_dir = os.path.join(TEMP_SUBMIT_DIR, uuid4().get_hex())
try:
data = request.json
if not data:
return make_api_response({}, "Missing data block", 400)
name = data.get("name", None)
if not name:
return make_api_response({}, "Filename missing", 400)
out_file = os.path.join(out_dir, os.path.basename(name))
binary = data.get("binary", None)
if not binary:
return make_api_response({}, "File binary missing", 400)
else:
try:
os.makedirs(out_dir)
except:
pass
with open(out_file, "wb") as my_file:
my_file.write(base64.b64decode(binary))
# Create task object
task = STORAGE.get_user_options(user['uname'])
if not task:
task = get_default_user_settings(user)
task.update(data.get("params", {}))
if 'groups' not in task:
task['groups'] = user['groups']
task["params"] = data.get("srv_spec", {})
if 'services' in task and "selected" not in task:
task["selected"] = task["services"]
task['quota_item'] = True
task['submitter'] = user['uname']
task['sid'] = str(uuid4())
if not task['description']:
task['description'] = "Inspection of file: %s" % name
with forge.get_filestore() as f_transport:
result = SubmissionWrapper.submit_inline(STORAGE, f_transport, [out_file],
**remove_ui_specific_options(task))
if result['submission']['sid'] != task['sid']:
raise Exception('ID does not match what was returned by the dispatcher. Cancelling request...')
return make_api_response(result)
finally:
try:
# noinspection PyUnboundLocalVariable
os.unlink(out_file)
except:
pass
try:
shutil.rmtree(out_dir, ignore_errors=True)
except:
pass
def ingest_single_file(**kwargs):
"""
Ingest a single file in the system
Note:
Binary and sha256 fields are optional but at least one of them has to be there
notification_queue, notification_threshold and generate_alert fields are optional
Note 2:
The ingest API uses the user's default settings to submit files to the system
unless these settings are overridden in the 'params' field. Although, there are
exceptions to that rule. Fields deep_scan, ignore_filtering, ignore_cache,
ignore_tag are resetted to False because the lead to dangerous behavior in the
system.
Variables:
None
Arguments:
None
Data Block:
{
"name": "file.exe", # Name of the file
"binary": "A24AB..==", # Base64 encoded file binary
"metadata": { # Submission Metadata
"key": val, # Key/Value pair for metadata parameters
},
"params": { # Submission parameters
"key": val, # Key/Value pair for params that differ from the user's defaults
}, # DEFAULT: /api/v3/user/submission_params/<user>/
"sha256": "1234...CDEF" # SHA256 hash of the file
"srv_spec": { # Service specifics parameters
"Extract": {
"password": "******"
},
},
"type": "SUBMISSION_TYPE" # Required type field,
"notification_queue": None, # Name of the notification queue
"notification_threshold": None, # Threshold for notification
"generate_alert": False # Generate an alert in our alerting system or not
}
Result example:
{ "success": true }
"""
user = kwargs['user']
out_dir = os.path.join(TEMP_SUBMIT_DIR, uuid4().get_hex())
with forge.get_filestore() as f_transport:
try:
data = request.json
if not data:
return make_api_response({}, "Missing data block", 400)
notification_queue = data.get('notification_queue', None)
if notification_queue:
notification_queue = "nq-%s" % notification_queue
notification_threshold = data.get('notification_threshold', None)
if not isinstance(notification_threshold,
int) and notification_threshold:
return make_api_response(
{}, "notification_threshold should be and int", 400)
generate_alert = data.get('generate_alert', False)
if not isinstance(generate_alert, bool):
return make_api_response({},
"generate_alert should be a boolean",
400)
name = data.get("name", None)
if not name:
return make_api_response({}, "Filename missing", 400)
ingest_msg_type = data.get("type", None)
if not ingest_msg_type:
return make_api_response({}, "Required type field missing",
400)
out_file = os.path.join(out_dir, os.path.basename(name))
try:
os.makedirs(out_dir)
except:
pass
binary = data.get("binary", None)
if not binary:
sha256 = data.get('sha256', None)
if sha256:
if f_transport.exists(sha256):
f_transport.download(sha256, out_file)
else:
return make_api_response(
{}, "SHA256 does not exist in our datastore", 404)
else:
return make_api_response(
{}, "Both file binary and sha256 missing", 400)
else:
with open(out_file, "wb") as my_file:
my_file.write(base64.b64decode(binary))
overrides = STORAGE.get_user_options(user['uname'])
overrides['selected'] = overrides['services']
overrides.update({
'deep_scan': False,
"priority": 150,
"ignore_cache": False,
"ignore_filtering": False,
"ignore_tag": False,
})
overrides.update(data.get("params", {}))
overrides.update({
'description':
"[%s] Inspection of file: %s" % (ingest_msg_type, name),
'generate_alert':
generate_alert,
'max_extracted':
100,
'max_supplementary':
100,
'params':
data.get("srv_spec", {}),
'submitter':
user['uname'],
})
if notification_queue:
overrides.update({
"notification_queue":
notification_queue,
"notification_threshold":
notification_threshold
})
overrides['priority'] = min(overrides.get("priority", 150), 250)
metadata = data.get("metadata", {})
metadata['type'] = ingest_msg_type
if 'ts' not in metadata:
metadata['ts'] = now_as_iso()
digests = identify.get_digests_for_file(out_file)
digests.pop('path', None)
sha256 = digests['sha256']
if not f_transport.exists(sha256):
f_transport.put(out_file, sha256, location='far')
msg = {
"priority": overrides['priority'],
"type": ingest_msg_type,
"overrides": remove_ui_specific_options(overrides),
"metadata": metadata
}
msg.update(digests)
ingest.push(forge.determine_ingest_queue(sha256), msg)
return make_api_response({"success": True})
finally:
try:
# noinspection PyUnboundLocalVariable
os.unlink(out_file)
except:
pass
try:
shutil.rmtree(out_dir, ignore_errors=True)
except:
pass
def resubmit_for_dynamic(srl, *args, **kwargs):
"""
Resubmit a file for dynamic analysis
Variables:
srl => Resource locator (SHA256)
Arguments (Optional):
copy_sid => Mimic the attributes of this SID.
name => Name of the file for the submission
Data Block:
None
Result example:
{
"submission":{}, # Submission Block
"request": {}, # Request Block
"times": {}, # Timing Block
"state": "submitted", # Submission state
"services": {}, # Service selection Block
"fileinfo": {} # File information Block
}
"""
user = kwargs['user']
copy_sid = request.args.get('copy_sid', None)
name = request.args.get('name', srl)
if copy_sid:
submission = STORAGE.get_submission(copy_sid)
else:
submission = None
if submission:
if not Classification.is_accessible(user['classification'], submission['classification']):
return make_api_response("", "You are not allowed to re-submit a submission that you don't have access to",
403)
task = {k: v for k, v in submission['submission'].iteritems() if k not in STRIP_KW}
task.update({k: v for k, v in submission['services'].iteritems() if k not in STRIP_KW})
task['classification'] = submission['classification']
else:
params = STORAGE.get_user_options(user['uname'])
task = {k: v for k, v in params.iteritems() if k not in STRIP_KW}
task['selected'] = params["services"]
task['classification'] = params['classification']
task['sha256'] = srl
with forge.get_filestore() as f_transport:
if not f_transport.exists(srl):
return make_api_response({}, "File %s cannot be found on the server therefore it cannot be resubmitted."
% srl, status_code=404)
task['path'] = name
task['submitter'] = user['uname']
if 'priority' not in task:
task['priority'] = 500
task['description'] = "Resubmit %s for Dynamic Analysis" % name
if "Dynamic Analysis" not in task['selected']:
task['selected'].append("Dynamic Analysis")
submit_result = SubmissionWrapper.submit(f_transport, STORAGE, **task)
return make_api_response(submit_result)
def base(*args, **kwargs):
# Validate User-Agent
user_agent = request.environ.get("HTTP_USER_AGENT",
"Unknown browser")
if "MSIE 8" in user_agent or "MSIE 9" in user_agent or "MSIE 7" in user_agent or "MSIE 6" in user_agent:
return redirect(redirect_helper("/unsupported.html"))
# Create Path
path = request.path + "?" + request.query_string
# Login
try:
session_id = flsk_session.get("session_id", None)
if not session_id:
abort(401)
session = KV_SESSION.get(session_id)
if not session:
abort(401)
else:
session = json.loads(session)
cur_time = now()
if session.get('expire_at', 0) < cur_time:
KV_SESSION.pop(session_id)
abort(401)
else:
session['expire_at'] = cur_time + session.get(
'duration', 3600)
if request.headers.get("X-Forward-For", None) != session.get('ip', None) or \
request.headers.get("User-Agent", None) != session.get('user_agent', None):
abort(401)
KV_SESSION.set(session_id, session)
logged_in_uname = session.get("username", None)
if not set(self.required_priv).intersection(
set(session.get("privileges", []))):
abort(401)
user = login(logged_in_uname, path)
if self.require_admin and not user['is_admin']:
raise AccessDeniedException(
"Url '%s' requires ADMIN privileges" % request.path)
except AccessDeniedException:
raise
if self.audit:
json_blob = request.json
if not isinstance(json_blob, dict):
json_blob = {}
params_list = list(args) + \
["%s=%s" % (k, v) for k, v in kwargs.iteritems() if k in AUDIT_KW_TARGET] + \
["%s=%s" % (k, v) for k, v in request.args.iteritems() if k in AUDIT_KW_TARGET] + \
["%s=%s" % (k, v) for k, v in json_blob.iteritems() if k in AUDIT_KW_TARGET]
AUDIT_LOG.info("%s [%s] :: %s(%s)" %
(logged_in_uname, user['classification'],
func.func_name, ", ".join(params_list)))
# Dump Generic KWARGS
kwargs['build_master'] = "%s.%s" % (BUILD_MASTER, BUILD_LOWER)
kwargs['user'] = user
kwargs['user_js'] = json.dumps(user)
kwargs['debug'] = str(DEBUG).lower()
kwargs['menu'] = create_menu(user, path)
kwargs['avatar'] = STORAGE.get_user_avatar(user['uname'])
kwargs['is_prod'] = SYSTEM_NAME == "production"
options = STORAGE.get_user_options(user['uname'])
if not request.path == "/terms.html":
if not user.get('agrees_with_tos', False) and config.ui.get(
"tos", None) is not None:
return redirect(redirect_helper("/terms.html"))
if not options and not request.path == "/settings.html":
return redirect(redirect_helper("/settings.html?forced"))
if self.load_options:
kwargs['options'] = json.dumps(options)
kwargs["build_no"] = BUILD_NO
return func(*args, **kwargs)
