def get_tag_safelist(**_):
"""
Get the current tag_safelist
Variables:
None
Arguments:
default => Load the default values that came with the system
Data Block:
None
Result example:
<current tag_safelist.yml file>
"""
default = request.args.get('default', 'false').lower() in ['true', '']
with forge.get_cachestore('system', config=config,
datastore=STORAGE) as cache:
tag_safelist_yml = cache.get('tag_safelist_yml')
if not tag_safelist_yml or default:
yml_data = forge.get_tag_safelist_data()
if yml_data:
return make_api_response(yaml.safe_dump(yml_data))
return make_api_response(
None, "Could not find the tag_safelist.yml file", 404)
return make_api_response(safe_str(tag_safelist_yml))
def get_identify_custom_yara_file(**_):
"""
Get identify's current custom Yara file
Variables:
None
Arguments:
default => Load the default values that came with the system
Data Block:
None
Result example:
<current custom.yara file>
"""
default = request.args.get('default', 'false').lower() in ['true', '']
with forge.get_cachestore('system', config=config,
datastore=STORAGE) as cache:
custom_yara = cache.get('custom_yara')
if not custom_yara or default:
with open(constants.YARA_RULE_PATH) as mfh:
return make_api_response(mfh.read())
return make_api_response(custom_yara.decode('utf-8'))
def _load_magic_file(self):
self.magic_file = ':'.join(
(constants.MAGIC_RULE_PATH, '/usr/share/file/magic.mgc'))
if self.use_cache:
self.log.info("Checking for custom magic file...")
with get_cachestore('system',
config=self.config,
datastore=self.datastore) as cache:
try:
custom_magic = "/tmp/custom.magic"
cache.download('custom_magic', custom_magic)
self.magic_file = ':'.join(
(custom_magic, '/usr/share/file/magic.mgc'))
self.log.info("Custom magic file loaded!")
except FileStoreException:
self.log.info("No custom magic file found.")
with self.lock:
self.file_type = magic.magic_open(magic.MAGIC_CONTINUE +
magic.MAGIC_RAW)
magic.magic_load(self.file_type, self.magic_file)
self.mime_type = magic.magic_open(magic.MAGIC_CONTINUE +
magic.MAGIC_RAW +
magic.MAGIC_MIME)
magic.magic_load(self.mime_type, self.magic_file)
def get_identify_magic_patterns(**_):
"""
Get identify's magic patterns
Variables:
None
Arguments:
default => Load the default values that came with the system
Data Block:
None
Result example:
<current identify's magic patterns>
"""
default = request.args.get('default', 'false').lower() in ['true', '']
with forge.get_cachestore('system', config=config,
datastore=STORAGE) as cache:
custom_patterns = cache.get('custom_patterns')
if not custom_patterns or default:
return make_api_response(yaml.safe_dump(magic_patterns))
return make_api_response(custom_patterns.decode('utf-8'))
def cachestore(datastore_connection):
cachestore = forge.get_cachestore(COMPONENT,
datastore=datastore_connection)
cachestore.datastore.cached_file.delete_by_query("id:*")
cachestore.save(KEY, DATA)
cachestore.datastore.cached_file.commit()
return cachestore
def put_identify_magic_patterns(**_):
"""
Save a new version of identify's magic patterns file
Variables:
None
Arguments:
None
Data Block:
<new magic patterns file>
Result example:
{"success": True}
"""
data = request.json.encode('utf-8')
try:
patterns = yaml.safe_load(data)
for pattern in patterns:
if 'al_type' not in pattern:
raise ValueError(
f"Missing 'al_type' in pattern: {str(pattern)}")
if not al_re.match(pattern['al_type']):
raise ValueError(
f"Invalid 'al_type' in pattern: {str(pattern)}")
if 'regex' not in pattern:
raise ValueError(f"Missing 'regex' in pattern: {str(pattern)}")
try:
re.compile(pattern['regex'])
except Exception:
raise ValueError(
f"Invalid regular expression in pattern: {str(pattern)}")
except Exception as e:
return make_api_response({'success': False},
err=str(e),
status_code=400)
with forge.get_cachestore('system', config=config,
datastore=STORAGE) as cache:
if yaml.safe_dump(patterns) == yaml.safe_dump(magic_patterns):
cache.delete('custom_patterns')
else:
cache.save('custom_patterns', data, ttl=ADMIN_FILE_TTL, force=True)
# Notify components watching to reload magic patterns
event_sender.send('identify', 'patterns')
return make_api_response({'success': True})
def flowjs_check_chunk(**kwargs):
"""
Flowjs check file chunk.
This API is reserved for the FLOWJS file uploader. It allows FLOWJS
to check if the file chunk already exists on the server.
Variables:
None
Arguments (REQUIRED):
flowChunkNumber => Current chunk number
flowFilename => Original filename
flowTotalChunks => Total number of chunks
flowIdentifier => File unique identifier
flowCurrentChunkSize => Size of the current chunk
Data Block:
None
Result example:
{'exists': True} #Does the chunk exists on the server?
"""
flow_chunk_number = request.args.get("flowChunkNumber", None)
flow_chunk_size = request.args.get("flowChunkSize", None)
flow_total_size = request.args.get("flowTotalSize", None)
flow_filename = request.args.get("flowFilename", None)
flow_total_chunks = request.args.get("flowTotalChunks", None)
flow_identifier = request.args.get("flowIdentifier", None)
flow_current_chunk_size = request.args.get("flowCurrentChunkSize", None)
if not flow_chunk_number or not flow_identifier or not flow_current_chunk_size or not flow_filename \
or not flow_total_chunks or not flow_chunk_size or not flow_total_size:
return make_api_response(
"", "Required arguments missing. flowChunkNumber, flowIdentifier, "
"flowCurrentChunkSize, flowChunkSize and flowTotalSize "
"should always be present.", 412)
filename = get_cache_name(flow_identifier, flow_chunk_number)
with forge.get_cachestore("flowjs", config) as cache:
if cache.exists(filename):
return make_api_response({"exist": True})
else:
return make_api_response(
{
"exist": False,
"msg": "Chunk does not exist, please send it!"
},
status_code=204)
def put_identify_custom_yara_file(**_):
"""
Save a new version of identify's custom Yara file
Variables:
None
Arguments:
None
Data Block:
<current custom.yara file>
Result example:
{"success": True}
"""
data = request.json.encode('utf-8')
yara_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as tmp:
yara_file = tmp.name
tmp.write(data)
try:
yara_default_externals = {'mime': '', 'magic': '', 'type': ''}
yara.compile(filepaths={"default": yara_file},
externals=yara_default_externals)
except Exception as e:
message = str(e).replace(yara_file, "custom.yara line ")
return make_api_response(
{'success': False},
f"The Yara file you have submitted is invalid: {message}", 400)
finally:
if yara_file and os.path.exists(yara_file):
os.unlink(yara_file)
with forge.get_cachestore('system', config=config,
datastore=STORAGE) as cache:
if hashlib.sha256(data).hexdigest() == get_sha256_for_file(
constants.YARA_RULE_PATH):
cache.delete('custom_yara')
else:
cache.save('custom_yara', data, ttl=ADMIN_FILE_TTL, force=True)
# Notify components watching to reload yara file
event_sender.send('identify', 'yara')
return make_api_response({'success': True})
def put_identify_custom_magic_file(**_):
"""
Save a new version of identify's custom LibMagic file
Variables:
None
Arguments:
None
Data Block:
<current custom.magic file>
Result example:
{"success": True}
"""
data = request.json.encode('utf-8')
magic_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as tmp:
magic_file = tmp.name
tmp.write(data)
try:
test = magic.magic_open(magic.MAGIC_CONTINUE + magic.MAGIC_RAW)
magic.magic_load(test, magic_file)
except magic.MagicException:
return make_api_response(
{'success': False},
"The magic file you have submitted is invalid.", 400)
finally:
if magic_file and os.path.exists(magic_file):
os.unlink(magic_file)
with forge.get_cachestore('system', config=config,
datastore=STORAGE) as cache:
if hashlib.sha256(data).hexdigest() == get_sha256_for_file(
constants.MAGIC_RULE_PATH):
cache.delete('custom_magic')
else:
cache.save('custom_magic', data, ttl=ADMIN_FILE_TTL, force=True)
# Notify components watching to reload magic file
event_sender.send('identify', 'magic')
return make_api_response({'success': True})
def put_tag_safelist(**_):
"""
Save a new version of the tag_safelist file
Variables:
None
Arguments:
None
Data Block:
<new tag_safelist.yml file>
Result example:
{"success": true}
"""
tag_safelist_yml = request.json
try:
yml_data = yaml.safe_load(tag_safelist_yml)
for key in yml_data.keys():
if key not in ['match', 'regex']:
raise Exception('Invalid key found.')
fields = Tagging.flat_fields()
for tag_type in ['match', 'regex']:
for key, value in yml_data[tag_type].items():
if key not in fields:
raise Exception(f'{key} is not a valid tag type')
if not isinstance(value, list):
raise Exception(
f'Value for {key} should be a list of strings')
except Exception as e:
return make_api_response(
None, f"Invalid tag_safelist.yml file submitted: {str(e)}", 400)
with forge.get_cachestore('system', config=config,
datastore=STORAGE) as cache:
cache.save('tag_safelist_yml',
tag_safelist_yml.encode('utf-8'),
ttl=ADMIN_FILE_TTL,
force=True)
return make_api_response({'success': True})
def put_identify_trusted_mimetypes(**_):
"""
Save a new version of identify's trusted mimetypes file
Variables:
None
Arguments:
None
Data Block:
<new trusted mimetypes file>
Result example:
{"success": True}
"""
data = request.json.encode('utf-8')
try:
mimes = yaml.safe_load(data)
for k, v in mimes.items():
if not isinstance(k, str) or not al_re.match(k):
raise ValueError(f"Invalid mimetype in item: [{k}: {v}]")
if not isinstance(v, str) or not al_re.match(v):
raise ValueError(f"Invalid AL type in item [{k}: {v}]")
except Exception as e:
return make_api_response({'success': False},
err=str(e),
status_code=400)
with forge.get_cachestore('system', config=config,
datastore=STORAGE) as cache:
if yaml.safe_dump(mimes) == yaml.safe_dump(trusted_mimes):
cache.delete('custom_mimes')
else:
cache.save('custom_mimes', data, ttl=ADMIN_FILE_TTL, force=True)
# Notify components watching to reload trusted mimes
event_sender.send('identify', 'mimes')
return make_api_response({'success': True})
def get_safelisted_tags(self, tag_types):
if isinstance(tag_types, str):
tag_types = tag_types.split(',')
with forge.get_cachestore('system',
config=self.config,
datastore=self.datastore) as cache:
tag_safelist_yml = cache.get('tag_safelist_yml')
if tag_safelist_yml:
tag_safelist_data = yaml.safe_load(tag_safelist_yml)
else:
tag_safelist_data = forge.get_tag_safelist_data()
if tag_types:
output = {
'match': {
k: v
for k, v in tag_safelist_data.get('match', {}).items()
if k in tag_types or tag_types == []
},
'regex': {
k: v
for k, v in tag_safelist_data.get('regex', {}).items()
if k in tag_types or tag_types == []
},
}
for tag in tag_types:
for sl in self.datastore.safelist.stream_search(
f"type:tag AND enabled:true AND tag.type:{tag}",
as_obj=False):
output['match'].setdefault(sl['tag']['type'], [])
output['match'][sl['tag']['type']].append(
sl['tag']['value'])
else:
output = tag_safelist_data
for sl in self.datastore.safelist.stream_search(
"type:tag AND enabled:true", as_obj=False):
output['match'].setdefault(sl['tag']['type'], [])
output['match'][sl['tag']['type']].append(sl['tag']['value'])
return output
def _load_yara_file(self):
self.yara_file = constants.YARA_RULE_PATH
if self.use_cache:
self.log.info("Checking for custom yara file...")
with get_cachestore('system',
config=self.config,
datastore=self.datastore) as cache:
try:
custom_yara = "/tmp/custom.yara"
cache.download('custom_yara', custom_yara)
self.yara_file = custom_yara
self.log.info("Custom yara file loaded!")
except FileStoreException:
self.log.info("No custom magic file found.")
yara_rules = yara.compile(filepaths={"default": self.yara_file},
externals=self.yara_default_externals)
with self.lock:
self.yara_rules = yara_rules
def _load_trusted_mimes(self):
trusted_mimes = default_trusted_mimes
if self.use_cache:
self.log.info("Checking for custom trusted mimes...")
with get_cachestore('system',
config=self.config,
datastore=self.datastore) as cache:
try:
mimes = cache.get('custom_mimes')
if mimes:
trusted_mimes = yaml.safe_load(mimes)
self.log.info("Custom trusted mimes loaded!")
else:
self.log.info("No custom magic patterns found.")
except FileStoreException:
self.log.info("No custom trusted mimes found.")
with self.lock:
self.trusted_mimes = trusted_mimes
def _load_magic_patterns(self):
self.magic_patterns = default_magic_patterns
if self.use_cache:
self.log.info("Checking for custom magic patterns...")
with get_cachestore('system',
config=self.config,
datastore=self.datastore) as cache:
try:
patterns = cache.get('custom_patterns')
if patterns:
self.magic_patterns = yaml.safe_load(patterns)
self.log.info("Custom magic patterns loaded!")
else:
self.log.info("No custom magic patterns found.")
except FileStoreException:
self.log.info("No custom magic patterns found.")
compiled_patterns = [[
x['al_type'], re.compile(x['regex'], re.IGNORECASE)
] for x in self.magic_patterns]
with self.lock:
self.compiled_magic_patterns = compiled_patterns
def download_signatures(**kwargs):
"""
Download signatures from the system.
Variables:
None
Arguments:
query => Query used to filter the signatures
Default: All deployed signatures
Data Block:
None
Result example:
<A zip file containing all signatures files from the different sources>
"""
user = kwargs['user']
query = request.args.get('query', 'status:DEPLOYED')
access = user['access_control']
last_modified = STORAGE.get_signature_last_modified()
query_hash = sha256(
f'{query}.{access}.{last_modified}'.encode('utf-8')).hexdigest()
with forge.get_cachestore('al_ui.signature') as signature_cache:
response = _get_cached_signatures(signature_cache, query_hash)
if response:
return response
with Lock(f"al_signatures_{query_hash[:7]}.zip", 30):
response = _get_cached_signatures(signature_cache, query_hash)
if response:
return response
output_files = {}
keys = [
k['id'] for k in STORAGE.signature.stream_search(
query, fl="id", access_control=access, as_obj=False)
]
signature_list = sorted(STORAGE.signature.multiget(
keys, as_dictionary=False, as_obj=False),
key=lambda x: x['order'])
for sig in signature_list:
out_fname = f"{sig['type']}/{sig['source']}"
output_files.setdefault(out_fname, [])
output_files[out_fname].append(sig['data'])
output_zip = InMemoryZip()
for fname, data in output_files.items():
output_zip.append(fname, "\n\n".join(data))
rule_file_bin = output_zip.read()
signature_cache.save(query_hash,
rule_file_bin,
ttl=DEFAULT_CACHE_TTL)
return make_file_response(rule_file_bin,
f"al_signatures_{query_hash[:7]}.zip",
len(rule_file_bin),
content_type="application/zip")
def start_ui_submission(ui_sid, **kwargs):
"""
Start UI submission.
Starts processing after files where uploaded to the server.
Variables:
ui_sid => UUID for the current UI file upload
Arguments:
None
Data Block (REQUIRED):
Dictionary of UI specific user settings
Result example:
{
'started': True, # Has the submission started processing?
'sid' : "c7668cfa-...-c4132285142e" # Submission ID
}
"""
user = kwargs['user']
ui_params = request.json
ui_params['groups'] = kwargs['user']['groups']
ui_params['quota_item'] = True
ui_params['submitter'] = user['uname']
if not Classification.is_accessible(user['classification'],
ui_params['classification']):
return make_api_response({
"started": False,
"sid": None
}, "You cannot start a scan with higher "
"classification then you're allowed to see",
403)
quota_error = check_submission_quota(user)
if quota_error:
return make_api_response("", quota_error, 503)
submit_result = None
submitted_file = None
try:
# Download the file from the cache
with forge.get_cachestore("flowjs", config) as cache:
ui_sid = get_cache_name(ui_sid)
if cache.exists(ui_sid):
target_dir = os.path.join(TEMP_DIR, ui_sid)
os.makedirs(target_dir, exist_ok=True)
target_file = os.path.join(target_dir,
ui_params.pop('filename', ui_sid))
if os.path.exists(target_file):
os.unlink(target_file)
# Save the reconstructed file
cache.download(ui_sid, target_file)
submitted_file = target_file
# Submit the file
if submitted_file is not None:
with open(submitted_file, 'rb') as fh:
if is_cart(fh.read(256)):
meta = get_metadata_only(submitted_file)
if meta.get('al',
{}).get('type',
'unknown') == 'archive/bundle/al':
try:
submission = import_bundle(submitted_file,
allow_incomplete=True,
identify=IDENTIFY)
except Exception as e:
return make_api_response("",
err=str(e),
status_code=400)
return make_api_response({
"started": True,
"sid": submission['sid']
})
if not ui_params['description']:
ui_params[
'description'] = f"Inspection of file: {os.path.basename(submitted_file)}"
# Submit to dispatcher
try:
params = ui_to_submission_params(ui_params)
# Enforce maximum DTL
if config.submission.max_dtl > 0:
params['ttl'] = min(int(
params['ttl']), config.submission.max_dtl) if int(
params['ttl']) else config.submission.max_dtl
submission_obj = Submission({"files": [], "params": params})
except (ValueError, KeyError) as e:
return make_api_response("", err=str(e), status_code=400)
try:
submit_result = SubmissionClient(
datastore=STORAGE,
filestore=FILESTORE,
config=config,
identify=IDENTIFY).submit(submission_obj,
local_files=[submitted_file])
submission_received(submission_obj)
except SubmissionException as e:
return make_api_response("", err=str(e), status_code=400)
return make_api_response({
"started": True,
"sid": submit_result.sid
})
else:
return make_api_response({
"started": False,
"sid": None
}, "No files where found for ID %s. "
"Try again..." % ui_sid, 404)
finally:
if submit_result is None:
decrement_submission_quota(user)
# Remove file
if os.path.exists(submitted_file):
os.unlink(submitted_file)
# Remove dir
if os.path.exists(target_dir) and os.path.isdir(target_dir):
os.rmdir(target_dir)
def flowjs_upload_chunk(**kwargs):
"""
Flowjs upload file chunk.
This API is reserved for the FLOWJS file uploader. It allows
FLOWJS to upload a file chunk to the server.
Variables:
None
Arguments (REQUIRED):
flowChunkNumber => Current chunk number
flowChunkSize => Usual size of the chunks
flowCurrentChunkSize => Size of the current chunk
flowTotalSize => Total size for the file
flowIdentifier => File unique identifier
flowFilename => Original filename
flowRelativePath => Relative path of the file on the client
flowTotalChunks => Total number of chunks
Data Block:
None
Result example:
{
'success': True, #Did the upload succeeded?
'completed': False #Are all chunks received by the server?
}
"""
flow_chunk_number = request.form.get("flowChunkNumber", None)
flow_chunk_size = request.form.get("flowChunkSize", None)
flow_current_chunk_size = request.form.get("flowCurrentChunkSize", None)
flow_total_size = request.form.get("flowTotalSize", None)
flow_identifier = request.form.get("flowIdentifier", None)
flow_filename = safe_str(request.form.get("flowFilename", None))
flow_relative_path = request.form.get("flowRelativePath", None)
flow_total_chunks = request.form.get("flowTotalChunks", None)
completed = True
if not flow_chunk_number or not flow_chunk_size or not flow_current_chunk_size or not flow_total_size \
or not flow_identifier or not flow_filename or not flow_relative_path or not flow_total_chunks:
return make_api_response(
"", "Required arguments missing. flowChunkNumber, flowChunkSize, "
"flowCurrentChunkSize, flowTotalSize, flowIdentifier, flowFilename, "
"flowRelativePath and flowTotalChunks should always be present.",
412)
filename = get_cache_name(flow_identifier, flow_chunk_number)
with forge.get_cachestore("flowjs", config) as cache:
file_obj = request.files['file']
cache.save(filename, file_obj.stream.read())
# Test in reverse order to fail fast
for chunk in range(int(flow_total_chunks), 0, -1):
chunk_name = get_cache_name(flow_identifier, chunk)
if not cache.exists(chunk_name):
completed = False
break
if completed:
# Reconstruct the file
ui_sid = get_cache_name(flow_identifier)
target_file = os.path.join(TEMP_DIR, ui_sid)
try:
os.makedirs(TEMP_DIR)
except Exception:
pass
try:
os.unlink(target_file)
except Exception:
pass
for chunk in range(int(flow_total_chunks)):
chunk_name = get_cache_name(flow_identifier, chunk + 1)
with open(target_file, "ab") as t:
t.write(cache.get(chunk_name))
cache.delete(chunk_name)
# Save the reconstructed file
with open(target_file, "rb") as t:
cache.save(ui_sid, t.read())
os.unlink(target_file)
return make_api_response({'success': True, 'completed': completed})
