def set_ui4(**kwargs):
"""
Set the UI 4 redirect ON or OFF
Variables:
None
Arguments:
None
Data Block:
True/False
Result example:
{
"success": True
}
"""
ui4 = request.json
if not isinstance(ui4, bool):
return make_api_response("", err="Invalid data block", status_code=400)
user_settings = STORAGE.user_settings.get(kwargs['user']['uname'])
if not user_settings:
return make_api_response("", err="Cannot find user", status_code=404)
user_settings.ui4 = ui4
user_settings.ui4_ask = False
return make_api_response({"success": STORAGE.user_settings.save(kwargs['user']['uname'], user_settings)})
def remove_workflow(workflow_id, **_):
"""
Remove the specified workflow.
Variables:
workflow_id => ID of the workflow to remove
Arguments:
None
Data Block:
None
Result example:
{
"success": true # Was the remove successful?
}
"""
wf = STORAGE.workflow.get(workflow_id)
if wf:
return make_api_response({"success": STORAGE.workflow.delete(workflow_id)})
else:
return make_api_response({"success": False},
err="Workflow ID %s does not exist" % workflow_id,
status_code=404)
def delete_signature(sid, **kwargs):
"""
Delete a signature based of its ID
Variables:
sid => Signature ID
Arguments:
None
Data Block:
None
Result example:
{"success": True} # Signature delete successful
"""
user = kwargs['user']
data = STORAGE.signature.get(sid, as_obj=False)
if data:
if not Classification.is_accessible(
user['classification'],
data.get('classification', Classification.UNRESTRICTED)):
return make_api_response(
"", "Your are not allowed to delete this signature.", 403)
ret_val = STORAGE.signature.delete(sid)
_reset_service_updates(data['type'])
return make_api_response({"success": ret_val})
else:
return make_api_response("", f"Signature not found. ({sid})", 404)
def get_tag_safelist(**_):
"""
Get the current tag_safelist
Variables:
None
Arguments:
default => Load the default values that came with the system
Data Block:
None
Result example:
<current tag_safelist.yml file>
"""
default = request.args.get('default', 'false').lower() in ['true', '']
with forge.get_cachestore('system', config=config,
datastore=STORAGE) as cache:
tag_safelist_yml = cache.get('tag_safelist_yml')
if not tag_safelist_yml or default:
yml_data = forge.get_tag_safelist_data()
if yml_data:
return make_api_response(yaml.safe_dump(yml_data))
return make_api_response(
None, "Could not find the tag_safelist.yml file", 404)
return make_api_response(safe_str(tag_safelist_yml))
def setup_watch_queue(sid, **kwargs):
"""
Starts a watch queue to get live results
Variables:
sid => Submission ID
Arguments:
None
Data Block:
None
Result example:
{"wq_id": "D-c7668cfa-...-c4132285142e-WQ"} #ID of the watch queue
"""
data = STORAGE.submission.get(sid, as_obj=False)
user = kwargs['user']
if user and data and Classification.is_accessible(user['classification'], data['classification']):
wq_id = DispatchClient(datastore=STORAGE).setup_watch_queue(sid)
if wq_id:
return make_api_response({"wq_id": wq_id})
return make_api_response("", "No dispatchers are processing this submission.", 404)
else:
return make_api_response("", "You are not allowed to access this submissions.", 403)
def set_user_avatar(username, **kwargs):
"""
Sets the user's Avatar
Variables:
username => Name of the user you want to set the avatar for
Arguments:
None
Data Block:
"data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEASABIAAD..."
Result example:
{
"success": true # Was saving the avatar successful ?
}
"""
if username != kwargs['user']['uname']:
return make_api_response({"success": False}, "Cannot save the avatar of another user.", 403)
data = request.data
if data:
data = data.decode('utf-8')
if not isinstance(data, str) or not STORAGE.user_avatar.save(username, data):
make_api_response({"success": False}, "Data block should be a base64 encoded image "
"that starts with 'data:image/<format>;base64,'", 400)
else:
STORAGE.user_avatar.delete(username)
return make_api_response({"success": True})
def get_identify_custom_yara_file(**_):
"""
Get identify's current custom Yara file
Variables:
None
Arguments:
default => Load the default values that came with the system
Data Block:
None
Result example:
<current custom.yara file>
"""
default = request.args.get('default', 'false').lower() in ['true', '']
with forge.get_cachestore('system', config=config,
datastore=STORAGE) as cache:
custom_yara = cache.get('custom_yara')
if not custom_yara or default:
with open(constants.YARA_RULE_PATH) as mfh:
return make_api_response(mfh.read())
return make_api_response(custom_yara.decode('utf-8'))
def outstanding_services(sid, **kwargs):
"""
List outstanding services and the number of file each
of them still have to process.
Variables:
sid => Submission ID
Arguments:
None
Data Block:
None
Result example:
{"MY SERVICE": 1, ... } # Dictionnary of services and number of files
"""
data = STORAGE.submission.get(sid, as_obj=False)
user = kwargs['user']
if user and data and Classification.is_accessible(user['classification'],
data['classification']):
return make_api_response(
DispatchClient(datastore=STORAGE).outstanding_services(sid))
else:
return make_api_response(
{}, "You are not allowed to access this submissions.", 403)
def list_index_fields(index, **kwargs):
"""
List all available fields for a given index
Variables:
index => Which specific index you want to know the fields for
Arguments:
None
Data Block:
None
Result example:
{
"<<FIELD_NAME>>": { # For a given field
indexed: True, # Is the field indexed
stored: False, # Is the field stored
type: string # What type of data in the field
},
...
}
"""
user = kwargs['user']
collection = get_collection(index, user)
if collection is not None:
return make_api_response(collection.fields())
elif index == "ALL":
return make_api_response(list_all_fields(user))
else:
return make_api_response("",
f"Not a valid index to search in: {index}",
400)
def get_service_error(cache_key, **_):
"""
Get the content off a given service error cache key.
Variables:
cache_key => Service result cache key
as SHA256.ServiceName.ServiceVersion.Configuration.e
Arguments:
None
Data Block:
None
Result example:
{"created": "1900-01-01T00:00:00Z", # Time at which the error was created
"response": { # Service Response
"message": "Err message", # Error Message
"service_debug_info": "", # Infromation about where the job was processed
"service_name": "NSRL", # Service Name
"service_version": "", # Service Version
"status": "FAIL"} # Status
"sha256": "123456...123456"} # SHA256 of the files in error
"""
data = STORAGE.error.get(cache_key, as_obj=False)
if data is None:
return make_api_response("",
"Cache key %s does not exists." % cache_key,
404)
return make_api_response(data)
def list_errors(**_):
"""
List all error in the system (per page)
Variables:
None
Arguments:
offset => Offset at which we start giving errors
query => Query to apply to the error list
rows => Numbers of errors to return
Data Block:
None
Result example:
{"total": 201, # Total errors found
"offset": 0, # Offset in the error list
"count": 100, # Number of errors returned
"items": [] # List of error blocks
}
"""
offset = int(request.args.get('offset', 0))
rows = int(request.args.get('rows', 100))
query = request.args.get('query', "id:*") or "id:*"
try:
return make_api_response(STORAGE.error.search(query, offset=offset, rows=rows, as_obj=False,
sort="created desc"))
except SearchException as e:
return make_api_response("", f"The specified search query is not valid. ({e})", 400)
def get_file_ascii(sha256, **kwargs):
"""
Return the ascii values for a file where ascii chars are replaced by DOTs.
Variables:
sha256 => A resource locator for the file (sha256)
Arguments:
None
Data Block:
None
Result example:
<THE ASCII FILE>
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
if not file_obj:
return make_api_response({}, "The file was not found in the system.", 404)
if user and Classification.is_accessible(user['classification'], file_obj['classification']):
with forge.get_filestore() as f_transport:
data = f_transport.get(sha256)
if not data:
return make_api_response({}, "This file was not found in the system.", 404)
return make_api_response(data.translate(FILTER_ASCII).decode())
else:
return make_api_response({}, "You are not allowed to view this file.", 403)
def get_file_hex(sha256, **kwargs):
"""
Returns the file hex representation
Variables:
sha256 => A resource locator for the file (sha256)
Arguments:
None
Data Block:
None
API call example:
/api/v4/file/hex/123456...654321/
Result example:
<THE FILE HEX REPRESENTATION>
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
if not file_obj:
return make_api_response({}, "The file was not found in the system.", 404)
if user and Classification.is_accessible(user['classification'], file_obj['classification']):
with forge.get_filestore() as f_transport:
data = f_transport.get(sha256)
if not data:
return make_api_response({}, "This file was not found in the system.", 404)
return make_api_response(hexdump(data))
else:
return make_api_response({}, "You are not allowed to view this file.", 403)
def delete_submission(sid, **kwargs):
"""
Delete a submission as well as all related
files, results and errors
Variables:
sid => Submission ID to be deleted
Arguments:
None
Data Block:
None
Result example:
{success: true}
"""
user = kwargs['user']
submission = STORAGE.submission.get(sid, as_obj=False)
if not submission:
return make_api_response("", f"There are not submission with sid: {sid}", 404)
if Classification.is_accessible(user['classification'], submission['classification']) \
and (submission['params']['submitter'] == user['uname'] or 'admin' in user['type']):
STORAGE.delete_submission_tree_bulk(sid, Classification, transport=FILESTORE)
STORAGE.submission.commit()
return make_api_response({"success": True})
else:
return make_api_response("", "Your are not allowed to delete this submission.", 403)
def get_alert(alert_id, **kwargs):
"""
Get the alert details for a given alert key
Variables:
alert_id => ID of the alert to get the details for
Arguments:
None
Data Block:
None
API call example:
/api/v4/alert/1234567890/
Result example:
{
KEY: VALUE, # All fields of an alert in key/value pair
}
"""
user = kwargs['user']
data = STORAGE.alert.get(alert_id, as_obj=False)
if not data:
return make_api_response("", "This alert does not exists...", 404)
if user and Classification.is_accessible(user['classification'],
data['classification']):
return make_api_response(data)
else:
return make_api_response("",
"You are not allowed to see this alert...",
403)
def delete_obo_token(token_id, **kwargs):
"""
Delete an application access to your profile
Variables:
None
Arguments:
token_id => ID of the application token to delete
Data Block:
None
Result example:
{'success': true}
"""
uname = kwargs['user']['uname']
user_data = STORAGE.user.get(uname, as_obj=False)
if token_id not in user_data.get('apps', {}):
return make_api_response({"success": False}, "Token ID does not exist",
404)
user_data['apps'].pop(token_id)
STORAGE.user.save(uname, user_data)
return make_api_response({"success": True})
def remove_user_account(username, **_):
"""
Remove the account specified by the username.
Variables:
username => Name of the user to get the account info
Arguments:
None
Data Block:
None
Result example:
{
"success": true # Was the remove successful?
}
"""
user_data = STORAGE.user.get(username)
if user_data:
user_deleted = STORAGE.user.delete(username)
avatar_deleted = STORAGE.user_avatar.delete(username)
favorites_deleted = STORAGE.user_favorites.delete(username)
settings_deleted = STORAGE.user_settings.delete(username)
if not user_deleted or not avatar_deleted or not favorites_deleted or not settings_deleted:
return make_api_response({"success": False})
return make_api_response({"success": True})
else:
return make_api_response({"success": False},
err=f"User {username} does not exist",
status_code=404)
def logout(**_):
"""
Logout from the system clearing the current session
Variables:
None
Arguments:
None
Data Block:
None
Result example:
{
"success": true
}
"""
try:
session_id = flsk_session.get('session_id', None)
if session_id:
KV_SESSION.pop(session_id)
flsk_session.clear()
res = make_api_response({"success": True})
res.set_cookie('XSRF-TOKEN', '', max_age=0)
return res
except ValueError:
return make_api_response("", err="No user logged in?", status_code=400)
def get_identify_magic_patterns(**_):
"""
Get identify's magic patterns
Variables:
None
Arguments:
default => Load the default values that came with the system
Data Block:
None
Result example:
<current identify's magic patterns>
"""
default = request.args.get('default', 'false').lower() in ['true', '']
with forge.get_cachestore('system', config=config,
datastore=STORAGE) as cache:
custom_patterns = cache.get('custom_patterns')
if not custom_patterns or default:
return make_api_response(yaml.safe_dump(magic_patterns))
return make_api_response(custom_patterns.decode('utf-8'))
def signup_validate(**_):
"""
Validate a user's signup request
Variables:
None
Arguments:
None
Data Block:
{
"registration_key": "234234...ADFCB" # Key used to validate the user's signup process
}
Result example:
{
"success": true
}
"""
if not config.auth.internal.signup.enabled:
return make_api_response({"success": False},
"Signup process has been disabled", 403)
try:
data = request.json
except BadRequest:
data = request.values
registration_key = data.get('registration_key', None)
if registration_key:
try:
signup_queue = get_signup_queue(registration_key)
members = signup_queue.members()
signup_queue.delete()
if members:
user_info = members[0]
# Add dynamic classification group
user_info['classification'] = get_dynamic_classification(
user_info.get('classification',
Classification.UNRESTRICTED),
user_info['email'])
user = User(user_info)
username = user.uname
STORAGE.user.save(username, user)
return make_api_response({"success": True})
except (KeyError, ValueError) as e:
LOGGER.warning(f"Fail to signup user: {str(e)}")
pass
else:
return make_api_response(
{"success": False},
"Not enough information to proceed with user creation", 400)
return make_api_response({"success": False}, "Invalid registration key",
400)
def set_system_message(**kwargs):
"""
Set the current system message
Variables:
None
Arguments:
None
Data Block:
{
"title": "Message title",
"severity": "info",
"message": "This is a test message"
}
Result example:
{"success": true}
"""
msg = request.json
if isinstance(msg, dict) and 'severity' in msg and 'message' in msg:
msg['user'] = kwargs['user']['uname']
msg = {
k: v
for k, v in msg.items()
if k in ['severity', 'message', 'title', 'user']
}
UI_MESSAGING.set('system_message', msg)
return make_api_response({"success": True})
return make_api_response(None, "Invalid system message submitted.", 400)
def get_potential_versions(servicename, **_):
"""
List the different versions of a service stored in the system
Variables:
servicename => Name of the service to get the versions
Arguments:
None
Data Block:
None
Result example:
['3.1.0', '3.2.0', '3.3.0', '4.0.0', ...] # List of service versions
"""
service = STORAGE.service_delta.get(servicename)
if service:
return make_api_response(
sorted([
item.version
for item in STORAGE.service.stream_search(f"id:{servicename}*",
fl="version")
],
key=lambda x: parse(x),
reverse=True))
else:
return make_api_response("",
err=f"{servicename} service does not exist",
status_code=404)
def list_bucket_fields(bucket, **kwargs):
"""
List all available fields for a given bucket
Variables:
bucket => Which specific bucket you want to know the fields for
Arguments:
None
Data Block:
None
Result example:
{
"<<FIELD_NAME>>": { # For a given field
indexed: True, # Is the field indexed
stored: False, # Is the field stored
type: string # What type of data in the field
},
...
}
"""
if bucket in BUCKET_MAP or ():
return make_api_response(BUCKET_MAP[bucket].fields())
elif 'admin' in kwargs['user']['type'] and hasattr(STORAGE, bucket):
return make_api_response(getattr(STORAGE, bucket).fields())
elif bucket == "ALL":
return make_api_response(list_all_fields())
else:
return make_api_response("",
f"Not a valid bucket to search in: {bucket}",
400)
def get_message(message_type, **_):
"""
Read a message from the queue
Variables:
message_type => Type of message (file | submission | alert)
Arguments:
None
Data Block:
None
Result example:
{
... # Message from the queue
}
"""
if message_type not in ['file', 'submission', 'alert']:
return make_api_response(
"",
f"{message_type.upper()} is not a valid message type for this API.",
400)
return make_api_response(QUEUE_MAP[message_type].pop(blocking=True,
timeout=30))
def get_workflow(workflow_id, **kwargs):
"""
Load the user account information.
Variables:
workflow_id => ID of the workflow
Arguments:
None
Data Block:
None
Result example:
{
"name": "Workflow name", # Name of the workflow
"classification": "", # Max classification for workflow
"label": ['label1'], # Labels for the workflow
"priority": "LOW", # Priority of the workflow
"status": "MALICIOUS", # Status of the workflow
"query": "*:*" # Query to match the data
}
"""
wf = STORAGE.workflow.get(workflow_id, as_obj=False)
if wf:
if CLASSIFICATION.is_accessible(kwargs['user']['classification'], wf['classification']):
return make_api_response(wf)
else:
return make_api_response({},
err="You're not allowed to view workflow ID: %s" % workflow_id,
status_code=403)
else:
return make_api_response({},
err="Workflow ID %s does not exist" % workflow_id,
status_code=404)
def request_replay(index, doc_id, **kwargs):
"""
Request an alert or a submission to be transfered to another system
Variables:
index => Type of document to be transfered (alert or submission)
doc_id => ID of the document to transfer
Arguments:
None
Data Block:
None
Result example:
{"success": true}
"""
user = kwargs['user']
if index not in ['alert', 'submission']:
return make_api_response(
"", f"{index.upper()} is not a valid index for this API.", 400)
index_ds = STORAGE.get_collection(index)
doc = index_ds.get_if_exists(doc_id, as_obj=False)
if not doc or not Classification.is_accessible(user['classification'],
doc['classification']):
return make_api_response(
"",
f"You are not allowed to modify the {index} with the following ID: {doc_id}",
403)
operations = [(index_ds.UPDATE_SET, 'metadata.replay', REPLAY_REQUESTED)]
return make_api_response({'success': index_ds.update(doc_id, operations)})
def get_error(error_key, **kwargs):
"""
Get the error details for a given error key
Variables:
error_key => Error key to get the details for
Arguments:
None
Data Block:
None
Result example:
{
KEY: VALUE, # All fields of an error in key/value pair
}
"""
user = kwargs['user']
data = STORAGE.error.get(error_key, as_obj=False)
if user and data:
return make_api_response(data)
else:
return make_api_response("",
"You are not allowed to see this error...",
403)
def get_stats_for_fields(fields, query, tc_start, tc, access_control):
if not tc_start and "no_delay" not in request.args and config.core.alerter.delay != 0:
tc_start = now_as_iso(config.core.alerter.delay * -1)
if tc and config.ui.read_only:
tc += config.ui.read_only_offset
timming_filter = get_timming_filter(tc_start, tc)
filters = [x for x in request.args.getlist("fq") if x != ""]
if timming_filter:
filters.append(timming_filter)
try:
if isinstance(fields, list):
with concurrent.futures.ThreadPoolExecutor(
len(fields)) as executor:
res = {
field: executor.submit(STORAGE.alert.facet,
field,
query=query,
filters=filters,
limit=100,
access_control=access_control)
for field in fields
}
return make_api_response({k: v.result() for k, v in res.items()})
else:
return make_api_response(
STORAGE.alert.facet(fields,
query=query,
filters=filters,
limit=100,
access_control=access_control))
except SearchException as e:
return make_api_response("", f"SearchException: {e}", 400)
def get_heuristic(heuristic_id, **kwargs):
"""
Get a specific heuristic's detail from the system
Variables:
heuristic_id => ID of the heuristic
Arguments:
None
Data Block:
None
Result example:
{"id": "AL_HEUR_001", # Heuristics ID
"filetype": ".*", # Target file type
"name": "HEURISTICS_NAME", # Heuristics name
"description": ""} # Heuristics description
"""
user = kwargs['user']
h = STORAGE.heuristic.get(heuristic_id, as_obj=False)
if not h:
return make_api_response("", "Heuristic not found", 404)
if user and Classification.is_accessible(user['classification'], h['classification']):
# Always refresh stats when someone get a heuristic
h.update({'stats': STORAGE.get_stat_for_heuristic(heuristic_id)})
return make_api_response(h)
else:
return make_api_response("", "You are not allowed to see this heuristic...", 403)
def update_service(**_):
"""
Update a given service
Variables:
None
Arguments:
None
Data Block:
{
"name": "ResultSample"
"image": "cccs/assemblyline-service-resultsample:4.0.0dev0"
}
Result example:
{
success: true
}
"""
data = request.json
service_key = f"{data['name']}_{data['update_data']['latest_tag'].replace('stable', '')}"
# Check is the version we are trying to update to already exists
if STORAGE.service.get_if_exists(service_key):
operations = [(STORAGE.service_delta.UPDATE_SET, 'version',
data['update_data']['latest_tag'].replace('stable',
''))]
if STORAGE.service_delta.update(data['name'], operations):
return make_api_response({'success': True, 'status': "updated"})
service_update.set(data['name'], data['update_data'])
return make_api_response({'success': True, 'status': "updating"})
