def format_result(user_classification,
r,
min_classification,
build_hierarchy=False):
if not CLASSIFICATION.is_accessible(user_classification,
min_classification):
return None
# Drop sections user does not have access and set others to at least min classification
max_classification, r['result']['sections'] = filter_sections(
r['result']['sections'], user_classification, min_classification)
# Drop supplementary and extracted files that the user does not have access to
for ftype in ['supplementary', 'extracted']:
r['response'][ftype] = [
x for x in r['response'][ftype] if CLASSIFICATION.is_accessible(
user_classification, x['classification'])
]
# Set result classification to at least min but no more then viewable result classification
r['classification'] = CLASSIFICATION.max_classification(
max_classification, min_classification)
if build_hierarchy:
try:
section_hierarchy, _ = build_heirarchy_rec(r['result']['sections'])
r['section_hierarchy'] = section_hierarchy['children']
except InvalidSectionList:
LOGGER.warning(
f"Could not generate section hierarchy for {r['response']['service_name']} "
f"service. Will use old display method.")
return r
def signup_validate(**_):
"""
Validate a user's signup request
Variables:
None
Arguments:
None
Data Block:
{
"registration_key": "234234...ADFCB" # Key used to validate the user's signup process
}
Result example:
{
"success": true
}
"""
if not config.auth.internal.signup.enabled:
return make_api_response({"success": False},
"Signup process has been disabled", 403)
try:
data = request.json
except BadRequest:
data = request.values
registration_key = data.get('registration_key', None)
if registration_key:
try:
signup_queue = get_signup_queue(registration_key)
members = signup_queue.members()
signup_queue.delete()
if members:
user_info = members[0]
# Add dynamic classification group
user_info['classification'] = get_dynamic_classification(
user_info.get('classification',
Classification.UNRESTRICTED),
user_info['email'])
user = User(user_info)
username = user.uname
STORAGE.user.save(username, user)
return make_api_response({"success": True})
except (KeyError, ValueError) as e:
LOGGER.warning(f"Fail to signup user: {str(e)}")
pass
else:
return make_api_response(
{"success": False},
"Not enough information to proceed with user creation", 400)
return make_api_response({"success": False}, "Invalid registration key",
400)
def auto_auth_check(self):
apikey = request.environ.get('HTTP_X_APIKEY', None)
uname = request.environ.get('HTTP_X_USER', None)
if apikey is not None and uname is not None:
ip = request.headers.get("X-Forwarded-For", request.remote_addr)
with elasticapm.capture_span(name="@api_login:auto_auth_check()",
span_type="authentication"):
try:
# TODO: apikey_handler is slow to verify the password (bcrypt's fault)
# We could fix this by saving the hash of the combinaison of the
# APIkey and the username in an ExpiringSet and looking it up for
# sub-sequent calls...
validated_user, priv = validate_apikey(
uname, apikey, STORAGE)
except AuthenticationException:
msg = "Invalid user or APIKey"
LOGGER.warning(
f"Authentication failure. (U:{uname} - IP:{ip}) [{msg}]"
)
abort(401, msg)
return
if validated_user:
LOGGER.info(f"Login successful. (U:{uname} - IP:{ip})")
if not set(self.required_priv).intersection(set(priv)):
abort(
403,
"The method you've used to login does not give you access to this API"
)
return
return validated_user
return None
def check_submission_quota(user, num=1) -> Optional[str]:
quota_user = user['uname']
quota = user.get('submission_quota', 5)
count = num + Hash('submissions-' + quota_user, **persistent).length()
if count > quota:
LOGGER.info("User %s exceeded their submission quota. [%s/%s]",
quota_user, count, quota)
return "You've exceeded your maximum submission quota of %s " % quota
return None
def check_submission_quota(user) -> Optional[str]:
quota_user = user['uname']
max_quota = user.get('submission_quota', 5)
if config.ui.enforce_quota and not SUBMISSION_TRACKER.begin(
quota_user, max_quota):
LOGGER.info("User %s exceeded their submission quota of %s.",
quota_user, max_quota)
return "You've exceeded your maximum submission quota of %s " % max_quota
return None
def agree_with_tos(username, **kwargs):
"""
Specified user send agreement to Terms of Service
Variables:
username => Name of the user that agrees with tos
Arguments:
None
Data Block:
None
Result example:
{
"success": true # Saving the user info succeded
}
"""
logged_in_user = kwargs['user']
if logged_in_user['uname'] != username:
return make_api_response(
{"success": False},
"You can't agree to Terms Of Service on behalf of someone else!",
400)
user = STORAGE.user.get(username)
if not user:
return make_api_response({"success": False},
"User %s does not exist." % username, 403)
else:
user.agrees_with_tos = now_as_iso()
if config.ui.tos_lockout:
user.is_active = False
if config.ui.tos_lockout and config.ui.tos_lockout_notify:
# noinspection PyBroadException
try:
for adr in config.ui.tos_lockout_notify:
send_authorize_email(adr, username, user.email or "")
except Exception as e:
LOGGER.error(
f"An error occurred while sending confirmation emails: {str(e)}"
)
return make_api_response(
{"success": False},
"The system was unable to send confirmation emails "
"to the administrators. Retry again later...", 400)
STORAGE.user.save(username, user)
return make_api_response({"success": True})
def _get_cached_signatures(signature_cache, query_hash):
try:
s = signature_cache.get(query_hash)
if s is None or s == b'':
return s
return make_file_response(s,
f"al_signatures_{query_hash[:7]}.zip",
len(s),
content_type="application/zip")
except Exception: # pylint: disable=W0702
LOGGER.exception('Failed to read cached signatures:')
return None
def get_file_infos(keys):
infos = {}
missing = []
retry = 0
while keys and retry < max_retry:
if retry:
time.sleep(2 ** (retry - 7))
try:
infos.update(STORAGE.file.multiget(keys, as_obj=False))
except MultiKeyError as e:
LOGGER.warning(f"Trying to get multiple files but some are missing: {str(e.keys)}")
infos.update(e.partial_output)
missing.extend(e.keys)
keys = [x for x in keys if x not in infos and x not in missing]
retry += 1
return infos, missing
def get_errors(keys):
out = {}
err = {}
missing = []
retry = 0
while keys and retry < max_retry:
if retry:
time.sleep(2 ** (retry - 7))
try:
err.update(STORAGE.error.multiget(keys, as_obj=False))
except MultiKeyError as e:
LOGGER.warning(f"Trying to get multiple errors but some are missing: {str(e.keys)}")
err.update(e.partial_output)
missing.extend(e.keys)
keys = [x for x in keys if x not in err and x not in missing]
retry += 1
out["errors"] = err
out["missing_error_keys"] = keys + missing
return out
def generate_ontology_file(results, user, updates={}, fnames={}):
# Load ontology files
sio = StringIO()
for r in results:
for supp in r.get('response', {}).get('supplementary', {}):
if supp['name'].endswith('.ontology'):
try:
ontology = json.loads(FILESTORE.get(supp['sha256']))
sha256 = ontology['header']['sha256']
c12n = ontology['header']['classification']
if sha256 == r['sha256'] and Classification.is_accessible(
user['classification'], c12n):
# Update the ontology with the live values
ontology['header'].update(updates)
# Set filenames if any
if sha256 in fnames:
ontology['header']['filenames'] = fnames[sha256]
elif 'filenames' in ontology['header']:
del ontology['header']['filenames']
# Make sure parent is not equal to current hash
if 'parent' in ontology['header'] and ontology[
'header']['parent'] == sha256:
del ontology['header']['parent']
sio.write(
json.dumps(
ontology, indent=None, separators=(',', ':')) +
'\n')
except Exception as e:
LOGGER.warning(
f"An error occured while fetching ontology files: {str(e)}"
)
# Flush and reset buffer
sio.flush()
return sio
def oauth_validate(**_):
"""
Validate and oAuth session and return it's associated username, avatar and oAuth Token
Variables:
None
Arguments:
provider => Which oAuth provider to validate the token against
* => All parameters returned by your oAuth provider callback...
Data Block:
None
Result example:
{
"avatar": "data:image...",
"oauth_token": "123123...123213",
"username": "******"
}
"""
oauth_provider = request.values.get('provider', None)
avatar = None
username = None
email_adr = None
oauth_token = None
if config.auth.oauth.enabled:
oauth = current_app.extensions.get('authlib.integrations.flask_client')
provider = oauth.create_client(oauth_provider)
if provider:
# noinspection PyBroadException
try:
oauth_provider_config = config.auth.oauth.providers[
oauth_provider]
if oauth_provider_config.app_provider:
# Validate the token that we've received using the secret
token = provider.authorize_access_token(
client_secret=oauth_provider_config.client_secret)
# Initialize the app_provider
app_provider = OAuth2Session(
oauth_provider_config.app_provider.client_id
or oauth_provider_config.client_id,
oauth_provider_config.app_provider.client_secret
or oauth_provider_config.client_secret,
scope=oauth_provider_config.app_provider.scope)
app_provider.fetch_token(
oauth_provider_config.app_provider.access_token_url,
grant_type="client_credentials")
else:
# Validate the token
token = provider.authorize_access_token()
app_provider = None
user_data = None
if oauth_provider_config.jwks_uri:
user_data = provider.parse_id_token(token)
# Get user data from endpoint
if app_provider and oauth_provider_config.app_provider.user_get:
url = oauth_provider_config.app_provider.user_get
uid = user_data.get('id', None)
if not uid and user_data and oauth_provider_config.uid_field:
uid = user_data.get(oauth_provider_config.uid_field,
None)
if uid:
url = url.format(id=uid)
resp = app_provider.get(url)
if resp.ok:
user_data = resp.json()
elif not user_data:
resp = provider.get(oauth_provider_config.user_get)
if resp.ok:
user_data = resp.json()
# Add group data if API is configured for it
groups = []
if app_provider and oauth_provider_config.app_provider.group_get:
url = oauth_provider_config.app_provider.group_get
uid = user_data.get('id', None)
if not uid and user_data and oauth_provider_config.uid_field:
uid = user_data.get(oauth_provider_config.uid_field,
None)
if uid:
url = url.format(id=uid)
resp_grp = app_provider.get(url)
if resp_grp.ok:
groups = resp_grp.json()
elif oauth_provider_config.user_groups:
resp_grp = provider.get(oauth_provider_config.user_groups)
if resp_grp.ok:
groups = resp_grp.json()
if groups:
if oauth_provider_config.user_groups_data_field:
groups = groups[
oauth_provider_config.user_groups_data_field]
if oauth_provider_config.user_groups_name_field:
groups = [
x[oauth_provider_config.user_groups_name_field]
for x in groups
]
user_data['groups'] = groups
if user_data:
data = parse_profile(user_data, oauth_provider_config)
has_access = data.pop('access', False)
if has_access and data['email'] is not None:
oauth_avatar = data.pop('avatar', None)
# Find if user already exists
users = STORAGE.user.search(f"email:{data['email']}",
fl="*",
as_obj=False)['items']
if users:
cur_user = users[0]
# Do not update username and password from the current user
data['uname'] = cur_user.get(
'uname', data['uname'])
data['password'] = cur_user.get(
'password', data['password'])
else:
if data['uname'] != data['email']:
# Username was computed using a regular expression, lets make sure we don't
# assign the same username to two users
res = STORAGE.user.search(
f"uname:{data['uname']}",
rows=0,
as_obj=False)
if res['total'] > 0:
cnt = res['total']
new_uname = f"{data['uname']}{cnt}"
while STORAGE.user.get(
new_uname) is not None:
cnt += 1
new_uname = f"{data['uname']}{cnt}"
data['uname'] = new_uname
cur_user = {}
username = data['uname']
email_adr = data['email']
# Add add dynamic classification group
data['classification'] = get_dynamic_classification(
data['classification'], data['email'])
# Make sure the user exists in AL and is in sync
if (not cur_user and oauth_provider_config.auto_create) or \
(cur_user and oauth_provider_config.auto_sync):
# Update the current user
cur_user.update(data)
# Save avatar
if oauth_avatar:
avatar = fetch_avatar(oauth_avatar, provider,
oauth_provider_config)
if avatar:
STORAGE.user_avatar.save(username, avatar)
# Save updated user
STORAGE.user.save(username, cur_user)
if cur_user:
if avatar is None:
avatar = STORAGE.user_avatar.get(
username
) or "/static/images/user_default.png"
oauth_token = hashlib.sha256(
str(token).encode(
"utf-8", errors='replace')).hexdigest()
get_token_store(username).add(oauth_token)
else:
return make_api_response(
{"err_code": 3},
err="User auto-creation is disabled",
status_code=403)
else:
return make_api_response(
{"err_code": 2},
err="This user is not allowed access to the system",
status_code=403)
except OAuthError as err:
return make_api_response({"err_code": 1},
err=str(err),
status_code=401)
except Exception as err:
LOGGER.exception(str(err))
return make_api_response(
{
"err_code": 1,
"exception": str(err)
},
err=
"Unhandled exception occured while processing oAuth token",
status_code=401)
if username is None:
return make_api_response({"err_code": 0},
err="oAuth disabled on the server",
status_code=401)
return make_api_response({
"avatar": avatar,
"username": username,
"oauth_token": oauth_token,
"email_adr": email_adr
})
def base(*args, **kwargs):
if 'user' in kwargs:
if kwargs['user'].get('authenticated', False):
return func(*args, **kwargs)
else:
abort(403, "Invalid pre-authenticated user")
return
self.test_readonly("API")
logged_in_uname = self.get_logged_in_user()
impersonator = None
# Impersonate
authorization = request.environ.get("HTTP_AUTHORIZATION", None)
if authorization:
# noinspection PyBroadException
try:
bearer_token = authorization.split(" ")[-1]
headers = jwt.get_unverified_header(bearer_token)
decoded = jwt.decode(
bearer_token,
hashlib.sha256(f"{SECRET_KEY}_{headers['token_id']}".
encode()).hexdigest(),
algorithms=[headers.get('alg', "HS256")])
except Exception:
abort(400, "Malformed bearer token")
return
target_user = STORAGE.user.get(headers['user'], as_obj=False)
if target_user:
target_token = target_user.get('apps', {}).get(
headers['token_id'], {})
if target_token == decoded and target_token[
'client_id'] == logged_in_uname:
impersonator = logged_in_uname
logged_in_uname = headers['user']
LOGGER.info(
f"{impersonator} is impersonating {logged_in_uname} for query: {request.path}"
)
if not set(self.required_priv).intersection(
set(SCOPES[decoded["scope"]])):
abort(
403,
"The method you've used to login does not give you access to this API"
)
return
else:
abort(403, "Invalid bearer token")
return
else:
abort(404, "User not found")
return
user = login(logged_in_uname)
# Terms of Service
if request.path not in ["/api/v4/help/tos/", "/api/v4/user/whoami/",
f"/api/v4/user/tos/{logged_in_uname}/",
"/api/v4/auth/logout/"] \
and not user.get('agrees_with_tos', False) and config.ui.tos is not None:
abort(
403,
"Agree to Terms of Service before you can make any API calls"
)
return
self.test_require_type(user, "API")
#############################################
# Special username api query validation
#
# If an API call requests a username, the username as to match
# the logged in user or the user has to be ADMIN
#
# API that needs this special validation need to make sure their
# variable name for the username is as an optional parameter
# inside 'username_key'. Default: 'username'
if self.username_key in kwargs:
if kwargs[self.username_key] != user['uname'] \
and not kwargs[self.username_key] == "__global__" \
and not kwargs[self.username_key] == "__workflow__" \
and not kwargs[self.username_key].lower() == "__current__" \
and 'admin' not in user['type']:
return make_api_response(
{}, "Your username does not match requested username",
403)
self.audit_if_required(args,
kwargs,
logged_in_uname,
user,
func,
impersonator=impersonator)
# Save user credential in user kwarg for future reference
kwargs['user'] = user
if config.core.metrics.apm_server.server_url is not None:
elasticapm.set_user_context(username=user.get('name', None),
email=user.get('email', None),
user_id=user.get('uname', None))
# Check current user quota
quota_user = user['uname']
flsk_session['quota_user'] = quota_user
flsk_session['quota_set'] = True
quota = user.get('api_quota', 10)
if not QUOTA_TRACKER.begin(quota_user, quota):
if config.ui.enforce_quota:
LOGGER.info(
f"User {quota_user} was prevented from using the api due to exceeded quota."
)
return make_api_response(
"", f"You've exceeded your maximum quota of {quota}",
503)
else:
LOGGER.debug(
f"Quota of {quota} exceeded for user {quota_user}.")
else:
LOGGER.debug(
f"{quota_user}'s quota is under or equal its limit of {quota}"
)
return func(*args, **kwargs)
def set_user_account(username, **kwargs):
"""
Save the user account information.
Variables:
username => Name of the user to get the account info
Arguments:
None
Data Block:
{
"name": "Test user", # Name of the user
"is_active": true, # Is the user active?
"classification": "", # Max classification for user
"uname": "usertest", # Username
"type": ['user'], # List of all types the user is member of
"avatar": null, # Avatar of the user
"groups": ["TEST"] # Groups the user is member of
}
Result example:
{
"success": true # Saving the user info succeded
}
"""
try:
data = request.json
new_pass = data.pop('new_pass', None)
old_user = STORAGE.user.get(username, as_obj=False)
if not old_user:
return make_api_response({"success": False}, "User %s does not exists" % username, 404)
if not data['name']:
return make_api_response({"success": False}, "Full name of the user cannot be empty", 400)
data['apikeys'] = old_user.get('apikeys', [])
data['otp_sk'] = old_user.get('otp_sk', None)
data['security_tokens'] = old_user.get('security_tokens', {}) or {}
if new_pass:
password_requirements = config.auth.internal.password_requirements.as_primitives()
if not check_password_requirements(new_pass, **password_requirements):
error_msg = get_password_requirement_message(**password_requirements)
return make_api_response({"success": False}, error_msg, 469)
data['password'] = get_password_hash(new_pass)
data.pop('new_pass_confirm', None)
else:
data['password'] = old_user.get('password', "__NO_PASSWORD__") or "__NO_PASSWORD__"
# Apply dynamic classification
data['classification'] = get_dynamic_classification(data['classification'], data['email'])
ret_val = save_user_account(username, data, kwargs['user'])
if ret_val and \
not old_user['is_active'] \
and data['is_active'] \
and config.ui.tos_lockout \
and config.ui.tos_lockout_notify:
try:
email = data['email'] or ""
for adr in config.ui.tos_lockout_notify:
send_activated_email(adr, username, email, kwargs['user']['uname'])
if email:
send_activated_email(email, username, email, kwargs['user']['uname'])
except Exception as e:
# We can't send confirmation email, Rollback user change and mark this a failure
STORAGE.user.save(username, old_user)
LOGGER.error(f"An error occured while sending confirmation emails: {str(e)}")
return make_api_response({"success": False}, "The system was unable to send confirmation emails. "
"Retry again later...", 404)
return make_api_response({"success": ret_val})
except AccessDeniedException as e:
return make_api_response({"success": False}, str(e), 403)
except InvalidDataException as e:
return make_api_response({"success": False}, str(e), 400)
def get_file_submission_results(sid, sha256, **kwargs):
"""
Get the all the results and errors of a specific file
for a specific Submission ID
Variables:
sid => Submission ID to get the result for
sha256 => Resource locator to get the result for
Arguments (POST only):
extra_result_keys => List of extra result keys to get
extra_error_keys => List of extra error keys to get
Data Block:
None
Result example:
{"errors": [], # List of error blocks
"file_info": {}, # File information block (md5, ...)
"results": [], # List of result blocks
"tags": [] } # List of generated tags
"""
user = kwargs['user']
# Check if submission exist
data = STORAGE.submission.get(sid, as_obj=False)
if data is None:
return make_api_response("", "Submission ID %s does not exists." % sid,
404)
if data and user and Classification.is_accessible(user['classification'],
data['classification']):
# Prepare output
output = {
"file_info": {},
"results": [],
"tags": {},
"errors": [],
"attack_matrix": {},
'heuristics': {},
"signatures": set()
}
# Extra keys - This is a live mode optimisation
res_keys = data.get("results", [])
err_keys = data.get("errors", [])
if request.method == "POST" and request.json is not None and data[
'state'] != "completed":
extra_rkeys = request.json.get("extra_result_keys", [])
extra_ekeys = request.json.get("extra_error_keys", [])
# Load keys
res_keys.extend(extra_rkeys)
err_keys.extend(extra_ekeys)
res_keys = list(set(res_keys))
err_keys = list(set(err_keys))
# Get File, results and errors
temp_file = STORAGE.file.get(sha256, as_obj=False)
if not temp_file:
output['file_info']['sha256'] = sha256
output['signatures'] = list(output['signatures'])
output['missing'] = True
return make_api_response(
output,
"The file you are trying to view is missing from the system",
404)
if not Classification.is_accessible(user['classification'],
temp_file['classification']):
return make_api_response(
"", "You are not allowed to view the data of this file", 403)
output['file_info'] = temp_file
max_c12n = output['file_info']['classification']
temp_results = list(
STORAGE.get_multiple_results(
[x for x in res_keys if x.startswith(sha256)],
cl_engine=Classification,
as_obj=False).values())
results = []
for r in temp_results:
r = format_result(user['classification'],
r,
temp_file['classification'],
build_hierarchy=True)
if r:
max_c12n = Classification.max_classification(
max_c12n, r['classification'])
results.append(r)
output['results'] = results
try:
output['errors'] = STORAGE.error.multiget(
[x for x in err_keys if x.startswith(sha256)],
as_obj=False,
as_dictionary=False)
except MultiKeyError as e:
LOGGER.warning(
f"Trying to get multiple errors but some are missing: {str(e.keys)}"
)
output['errors'] = e.partial_output
output['metadata'] = STORAGE.get_file_submission_meta(
sha256, config.ui.statistics.submission, user["access_control"])
for res in output['results']:
for sec in res['result']['sections']:
h_type = "info"
if sec.get('heuristic', False):
# Get the heuristics data
if sec['heuristic']['score'] < 100:
h_type = "info"
elif sec['heuristic']['score'] < 1000:
h_type = "suspicious"
else:
h_type = "malicious"
item = (sec['heuristic']['heur_id'],
sec['heuristic']['name'])
output['heuristics'].setdefault(h_type, [])
if item not in output['heuristics'][h_type]:
output['heuristics'][h_type].append(item)
# Process Attack matrix
for attack in sec['heuristic'].get('attack', []):
attack_id = attack['attack_id']
for cat in attack['categories']:
output['attack_matrix'].setdefault(cat, [])
item = (attack_id, attack['pattern'], h_type)
if item not in output['attack_matrix'][cat]:
output['attack_matrix'][cat].append(item)
# Process Signatures
for signature in sec['heuristic'].get('signature', []):
sig = (signature['name'], h_type)
if sig not in output['signatures']:
output['signatures'].add(sig)
# Process tags
for t in sec['tags']:
output["tags"].setdefault(t['type'], {})
current_htype = output["tags"][t['type']].get(
t['value'], None)
if not current_htype:
output["tags"][t['type']][t['value']] = h_type
else:
if current_htype == 'malicious' or h_type == 'malicious':
output["tags"][t['type']][t['value']] = 'malicious'
elif current_htype == 'suspicious' or h_type == 'suspicious':
output["tags"][t['type']][
t['value']] = 'suspicious'
else:
output["tags"][t['type']][t['value']] = 'info'
for t_type in output["tags"]:
output["tags"][t_type] = [
(k, v) for k, v in output['tags'][t_type].items()
]
output['signatures'] = list(output['signatures'])
output['file_info']['classification'] = max_c12n
return make_api_response(output)
else:
return make_api_response(
"", "You are not allowed to view the data of this submission", 403)
def login(**_):
"""
Login the user onto the system
Variables:
None
Arguments:
None
Data Block:
{
"user": <UID>,
"password": <ENCRYPTED_PASSWORD>,
"otp": <OTP_TOKEN>,
"apikey": <ENCRYPTED_APIKEY>,
"webauthn_auth_resp": <RESPONSE_TO_CHALLENGE_FROM_WEBAUTHN>
}
Result example:
{
"username": <Logged in user>, # Username for the logged in user
"privileges": ["R", "W"], # Different privileges that the user will get for this session
"session_duration": 60 # Time after which this session becomes invalid
# Note: The timer reset after each call
}
"""
data = request.json
if not data:
data = request.values
user = data.get('user', None)
password = data.get('password', None)
apikey = data.get('apikey', None)
webauthn_auth_resp = data.get('webauthn_auth_resp', None)
oauth_provider = data.get('oauth_provider', None)
oauth_token = data.get('oauth_token', None)
if config.auth.oauth.enabled and oauth_provider:
oauth = current_app.extensions.get('authlib.integrations.flask_client')
provider = oauth.create_client(oauth_provider)
if provider:
redirect_uri = f'https://{request.host}/login.html?provider={oauth_provider}'
return provider.authorize_redirect(redirect_uri=redirect_uri)
try:
otp = int(data.get('otp', 0) or 0)
except Exception:
raise AuthenticationException('Invalid OTP token')
if (user and password) or (user and apikey) or (user and oauth_token):
auth = {
'username': user,
'password': password,
'otp': otp,
'webauthn_auth_resp': webauthn_auth_resp,
'apikey': apikey,
'oauth_token': oauth_token
}
logged_in_uname = None
ip = request.headers.get("X-Forwarded-For", request.remote_addr)
try:
logged_in_uname, priv = default_authenticator(
auth, request, flsk_session, STORAGE)
session_duration = config.ui.session_duration
cur_time = now()
xsrf_token = generate_random_secret()
current_session = {
'duration': session_duration,
'ip': ip,
'privileges': priv,
'time': int(cur_time) - (int(cur_time) % session_duration),
'user_agent': request.headers.get("User-Agent", None),
'username': logged_in_uname,
'xsrf_token': xsrf_token
}
session_id = hashlib.sha512(
str(current_session).encode("UTF-8")).hexdigest()
current_session['expire_at'] = cur_time + session_duration
flsk_session['session_id'] = session_id
KV_SESSION.add(session_id, current_session)
return make_api_response(
{
"username": logged_in_uname,
"privileges": priv,
"session_duration": session_duration
},
cookies={'XSRF-TOKEN': xsrf_token})
except AuthenticationException as wpe:
uname = auth.get('username', '(None)')
LOGGER.warning(
f"Authentication failure. (U:{uname} - IP:{ip}) [{wpe}]")
return make_api_response("", err=str(wpe), status_code=401)
finally:
if logged_in_uname:
LOGGER.info(
f"Login successful. (U:{logged_in_uname} - IP:{ip})")
return make_api_response(
"", "Not enough information to proceed with authentication", 401)
def signup(**_):
"""
Signup a new user into the system
Variables:
None
Arguments:
None
Data Block:
{
"user": <UID>,
"password": <DESIRED_PASSWORD>,
"password_confirm": <DESIRED_PASSWORD_CONFIRMATION>,
"email": <EMAIL_ADDRESS>
}
Result example:
{
"success": true
}
"""
if not config.auth.internal.signup.enabled:
return make_api_response({"success": False},
"Signup process has been disabled", 403)
try:
data = request.json
except BadRequest:
data = request.values
uname = data.get('user', None)
password = data.get('password', None)
password_confirm = data.get('password_confirm', None)
email = data.get('email', None)
if not uname or not password or not password_confirm or not email:
return make_api_response(
{"success": False},
"Not enough information to proceed with user creation", 400)
if STORAGE.user.get(uname) or len(uname) < 3:
return make_api_response(
{"success": False},
"There is already a user registered with this name", 460)
else:
for c in uname:
if not 97 <= ord(c) <= 122 and not ord(c) == 45:
return make_api_response(
{"success": False},
"Invalid username. [Lowercase letters and dashes "
"only with at least 3 letters]", 460)
if password_confirm != password:
return make_api_response("", "Passwords do not match", 469)
password_requirements = config.auth.internal.password_requirements.as_primitives(
)
if not check_password_requirements(password, **password_requirements):
error_msg = get_password_requirement_message(**password_requirements)
return make_api_response({"success": False}, error_msg, 469)
if STORAGE.user.search(f"email:{email.lower()}").get('total', 0) != 0:
return make_api_response(
{"success": False},
"There is already a user registered with this email address", 466)
# Normalize email address
email = email.lower()
email_valid = False
for r in config.auth.internal.signup.valid_email_patterns:
matcher = re.compile(r)
if matcher.findall(email):
email_valid = True
break
if not email_valid:
extra = ""
if config.ui.email:
extra = f". Contact {config.ui.email} for more information."
return make_api_response({"success": False},
f"Invalid email address{extra}", 466)
password = get_password_hash(password)
key = hashlib.sha256(
get_random_password(length=512).encode('utf-8')).hexdigest()
try:
send_signup_email(email, key)
get_signup_queue(key).add({
"uname": uname,
"password": password,
"email": email,
"groups": ['USERS'],
"name": uname
})
except Exception as e:
LOGGER.warning(f"Sending email for signup process failed: {str(e)}")
return make_api_response(
{"success": False},
"The system failed to send signup confirmation link.", 400)
return make_api_response({"success": True})
def get_multiple_service_results(**kwargs):
"""
Get multiple result and error keys at the same time
Variables:
None
Arguments:
None
Data Block:
{"error": [], #List of error keys to lookup
"result": [] #List of result keys to lookup
}
Result example:
{"error": {}, #Dictionary of error object matching the keys
"result": {} #Dictionary of result object matching the keys
}
"""
user = kwargs['user']
data = request.json
try:
errors = STORAGE.error.multiget(data.get('error', []),
as_dictionary=True,
as_obj=False)
except MultiKeyError as e:
LOGGER.warning(
f"Trying to get multiple errors but some are missing: {str(e.keys)}"
)
errors = e.partial_output
results = STORAGE.get_multiple_results(data.get('result', []),
CLASSIFICATION,
as_obj=False)
try:
file_infos = STORAGE.file.multiget(list(
set([x[:64] for x in results.keys()])),
as_dictionary=True,
as_obj=False)
except MultiKeyError as e:
LOGGER.warning(
f"Trying to get multiple files but some are missing: {str(e.keys)}"
)
file_infos = e.partial_output
for r_key in list(results.keys()):
r_value = format_result(user['classification'],
results[r_key],
file_infos.get(r_key[:64], {}).get(
'classification',
CLASSIFICATION.UNRESTRICTED),
build_hierarchy=True)
if not r_value:
del results[r_key]
else:
results[r_key] = r_value
out = {"error": errors, "result": results}
return make_api_response(out)
def base(*args, **kwargs):
if 'user' in kwargs:
if kwargs['user'].get('authenticated', False):
return func(*args, **kwargs)
else:
abort(403, "Invalid pre-authenticated user")
return
self.test_readonly("API")
logged_in_uname = self.get_logged_in_user()
user = login(logged_in_uname)
# Terms of Service
if request.path not in ["/api/v4/help/tos/", "/api/v4/user/whoami/",
f"/api/v4/user/tos/{logged_in_uname}/",
"/api/v4/auth/logout/"] \
and not user.get('agrees_with_tos', False) and config.ui.tos is not None:
abort(
403,
"Agree to Terms of Service before you can make any API calls"
)
return
self.test_require_type(user, "API")
#############################################
# Special username api query validation
#
# If an API call requests a username, the username as to match
# the logged in user or the user has to be ADMIN
#
# API that needs this special validation need to make sure their
# variable name for the username is as an optional parameter
# inside 'username_key'. Default: 'username'
if self.username_key in kwargs:
if kwargs[self.username_key] != user['uname'] \
and not kwargs[self.username_key] == "__global__" \
and not kwargs[self.username_key] == "__workflow__" \
and not kwargs[self.username_key].lower() == "__current__" \
and 'admin' not in user['type']:
return make_api_response(
{}, "Your username does not match requested username",
403)
self.audit_if_required(args, kwargs, logged_in_uname, user, func)
# Save user credential in user kwarg for future reference
kwargs['user'] = user
if config.core.metrics.apm_server.server_url is not None:
elasticapm.set_user_context(username=user.get('name', None),
email=user.get('email', None),
user_id=user.get('uname', None))
# Check current user quota
quota_user = user['uname']
flsk_session['quota_user'] = quota_user
flsk_session['quota_set'] = True
quota = user.get('api_quota', 10)
if not QUOTA_TRACKER.begin(quota_user, quota):
if config.ui.enforce_quota:
LOGGER.info(
f"User {quota_user} was prevented from using the api due to exceeded quota."
)
return make_api_response(
"", f"You've exceeded your maximum quota of {quota}",
503)
else:
LOGGER.debug(
f"Quota of {quota} exceeded for user {quota_user}.")
else:
LOGGER.debug(
f"{quota_user}'s quota is under or equal its limit of {quota}"
)
return func(*args, **kwargs)
def reset_pwd(**_):
"""
Reset the password for the specified reset ID
Variables:
None
Arguments:
None
Data Block:
{
"reset_id": <RESET_HASH>,
"password": <PASSWORD TO RESET TO>,
"password_confirm": <CONFIRMATION OF PASSWORD TO RESET TO>
}
Result example:
{
"success": true
}
"""
if not config.auth.internal.signup.enabled:
return make_api_response({"success": False},
"Signup process has been disabled", 403)
try:
data = request.json
except BadRequest:
data = request.values
reset_id = data.get('reset_id', None)
password = data.get('password', None)
password_confirm = data.get('password_confirm', None)
if reset_id and password and password_confirm:
if password != password_confirm:
return make_api_response({"success": False},
err="Password mismatch",
status_code=469)
password_requirements = config.auth.internal.password_requirements.as_primitives(
)
if not check_password_requirements(password, **password_requirements):
error_msg = get_password_requirement_message(
**password_requirements)
return make_api_response({"success": False}, error_msg, 469)
try:
reset_queue = get_reset_queue(reset_id)
members = reset_queue.members()
reset_queue.delete()
if members:
email = members[0]
res = STORAGE.user.search(f"email:{email}", fl="*")
if res.get('total', 0) == 1:
user = res['items'][0]
user.password = get_password_hash(password)
STORAGE.user.save(user.uname, user)
return make_api_response({"success": True})
except Exception as e:
LOGGER.warning(f"Failed to reset the user's password: {str(e)}")
pass
return make_api_response(
{"success": False},
err=
"This reset link has expired, please restart the password reset process",
status_code=403)
return make_api_response({"success": False},
err="Invalid parameters passed",
status_code=400)
# TODO: this needs testing that can only be done when a service datasource is available.
path = cfg
cfg = config
for point in path.split('.'):
if 'enabled' in cfg:
if not cfg['enabled']:
raise SkipDatasource()
cfg = cfg.get(point)
cls = load_module_by_path(classpath)
obj = cls(LOGGER, **cfg)
sources[name] = create_query_datasource(obj)
except SkipDatasource:
continue
except Exception:
LOGGER.exception(
"Problem creating %s datasource (%s)", name, classpath
)
except Exception:
LOGGER.exception("No datasources")
# noinspection PyUnusedLocal
@hash_search_api.route("/<file_hash>/", methods=["GET"])
@api_login(required_priv=['R'])
def search_hash(file_hash, *args, **kwargs):
"""
Search for a hash in multiple data sources as configured in the seed.
Variables:
file_hash => Hash to search in the multiple data sources
[MD5, SHA1 or SHA256]
