def test_simple(self):
plaintext = b'0123456789' * 10000000
pt_stream = BytesIO(plaintext)
ct_stream = BytesIO()
cart.pack_stream(pt_stream, ct_stream, {'name': 'hello.txt'},
{'digest': 'done'})
crypt_text = ct_stream.getvalue()
ct_stream = BytesIO(crypt_text)
pt_stream = BytesIO()
temp_file = tempfile.mkstemp()[1]
with open(temp_file, 'wb') as f:
f.write(ct_stream.getvalue())
(header, footer) = cart.unpack_stream(ct_stream, pt_stream)
inline_metadata = {}
if header:
inline_metadata.update(header)
if footer:
inline_metadata.update(footer)
plaintext_prime = pt_stream.getvalue()
self.assertEqual(plaintext_prime, plaintext)
metadata = cart.get_metadata_only(temp_file)
self.assertEqual(metadata, inline_metadata)
self.assertTrue(cart.is_cart(crypt_text))
def test_download_to_file(datastore, client):
file_id = random_id_from_collection(datastore, 'file')
download_output = "/tmp/download_{}".format(file_id)
try:
client.file.download(file_id, output=download_output)
assert open(download_output, 'rb').read(4) == b"CART"
metadata = cart.get_metadata_only(download_output)
assert file_id == metadata['sha256']
finally:
os.unlink(download_output)
def test_identify():
with forge.get_identify(use_cache=False) as identify:
# Setup test data
aaaa = f"{'A' * 10000}".encode()
sha256 = hashlib.sha256(aaaa).hexdigest()
# Prep temp file
_, input_path = tempfile.mkstemp()
output_path = f"{input_path}.cart"
try:
# Write temp file
with open(input_path, 'wb') as oh:
oh.write(aaaa)
# Create a cart file
with open(output_path, 'wb') as oh:
with open(input_path, 'rb') as ih:
pack_stream(ih, oh, {'name': 'test_identify.a'})
# Validate the cart file created
meta = get_metadata_only(output_path)
assert meta.get("sha256", None) == sha256
# Validate identify file detection
info = identify.fileinfo(output_path)
assert info.get("type", None) == "archive/cart"
# Validate identify hashing
output_sha256 = subprocess.check_output(['sha256sum', output_path
])[:64].decode()
assert info.get("sha256", None) == output_sha256
finally:
# Cleanup output file
if os.path.exists(output_path):
os.unlink(output_path)
# Cleanup input file
if os.path.exists(input_path):
os.unlink(input_path)
def cart_ident(path):
from cart import get_metadata_only
metadata = get_metadata_only(path)
return metadata.get('al', {}).get('tag', 'archive/cart')
def cart_ident(path: str) -> str:
try:
metadata = get_metadata_only(path)
except Exception:
return 'corrupted/cart'
return metadata.get('al', {}).get('type', 'archive/cart')
def start_ui_submission(ui_sid, **kwargs):
"""
Start UI submission.
Starts processing after files where uploaded to the server.
Variables:
ui_sid => UUID for the current UI file upload
Arguments:
None
Data Block (REQUIRED):
Dictionary of UI specific user settings
Result example:
{
'started': True, # Has the submission started processing?
'sid' : "c7668cfa-...-c4132285142e" # Submission ID
}
"""
user = kwargs['user']
ui_params = request.json
ui_params['groups'] = kwargs['user']['groups']
ui_params['quota_item'] = True
ui_params['submitter'] = user['uname']
if not Classification.is_accessible(user['classification'],
ui_params['classification']):
return make_api_response({
"started": False,
"sid": None
}, "You cannot start a scan with higher "
"classification then you're allowed to see",
403)
quota_error = check_submission_quota(user)
if quota_error:
return make_api_response("", quota_error, 503)
submit_result = None
submitted_file = None
try:
# Download the file from the cache
with forge.get_cachestore("flowjs", config) as cache:
ui_sid = get_cache_name(ui_sid)
if cache.exists(ui_sid):
target_dir = os.path.join(TEMP_DIR, ui_sid)
os.makedirs(target_dir, exist_ok=True)
target_file = os.path.join(target_dir,
ui_params.pop('filename', ui_sid))
if os.path.exists(target_file):
os.unlink(target_file)
# Save the reconstructed file
cache.download(ui_sid, target_file)
submitted_file = target_file
# Submit the file
if submitted_file is not None:
with open(submitted_file, 'rb') as fh:
if is_cart(fh.read(256)):
meta = get_metadata_only(submitted_file)
if meta.get('al',
{}).get('type',
'unknown') == 'archive/bundle/al':
try:
submission = import_bundle(submitted_file,
allow_incomplete=True,
identify=IDENTIFY)
except Exception as e:
return make_api_response("",
err=str(e),
status_code=400)
return make_api_response({
"started": True,
"sid": submission['sid']
})
if not ui_params['description']:
ui_params[
'description'] = f"Inspection of file: {os.path.basename(submitted_file)}"
# Submit to dispatcher
try:
params = ui_to_submission_params(ui_params)
# Enforce maximum DTL
if config.submission.max_dtl > 0:
params['ttl'] = min(int(
params['ttl']), config.submission.max_dtl) if int(
params['ttl']) else config.submission.max_dtl
submission_obj = Submission({"files": [], "params": params})
except (ValueError, KeyError) as e:
return make_api_response("", err=str(e), status_code=400)
try:
submit_result = SubmissionClient(
datastore=STORAGE,
filestore=FILESTORE,
config=config,
identify=IDENTIFY).submit(submission_obj,
local_files=[submitted_file])
submission_received(submission_obj)
except SubmissionException as e:
return make_api_response("", err=str(e), status_code=400)
return make_api_response({
"started": True,
"sid": submit_result.sid
})
else:
return make_api_response({
"started": False,
"sid": None
}, "No files where found for ID %s. "
"Try again..." % ui_sid, 404)
finally:
if submit_result is None:
decrement_submission_quota(user)
# Remove file
if os.path.exists(submitted_file):
os.unlink(submitted_file)
# Remove dir
if os.path.exists(target_dir) and os.path.isdir(target_dir):
os.rmdir(target_dir)
def cart_ident(path: str) -> str:
try:
metadata = get_metadata_only(path)
except Exception:
return "corrupted/cart"
return metadata.get("al", {}).get("type", "archive/cart")
def start_ui_submission(ui_sid, **kwargs):
"""
Start UI submission.
Starts processing after files where uploaded to the server.
Variables:
ui_sid => UUID for the current UI file upload
Arguments:
None
Data Block (REQUIRED):
Dictionary of UI specific user settings
Result example:
{
'started': True, # Has the submission started processing?
'sid' : "c7668cfa-...-c4132285142e" # Submission ID
}
"""
user = kwargs['user']
ui_params = request.json
ui_params['groups'] = kwargs['user']['groups']
ui_params['quota_item'] = True
ui_params['submitter'] = user['uname']
if not Classification.is_accessible(user['classification'], ui_params['classification']):
return make_api_response({"started": False, "sid": None}, "You cannot start a scan with higher "
"classification then you're allowed to see", 403)
quota_error = check_submission_quota(user)
if quota_error:
return make_api_response("", quota_error, 503)
submit_result = None
request_files = []
request_dirs = []
fnames = []
try:
flist = glob.glob(TEMP_DIR + ui_sid + "*")
if len(flist) > 0:
# Generate file list
for fpath in flist:
request_dirs.append(fpath)
files = os.listdir(fpath)
for myfile in files:
request_files.append(os.path.join(fpath, myfile))
if myfile not in fnames:
fnames.append(myfile)
with open(request_files[0], 'rb') as fh:
if is_cart(fh.read(256)):
meta = get_metadata_only(request_files[0])
if meta.get('al', {}).get('type', 'unknown') == 'archive/bundle/al':
try:
submission = import_bundle(request_files[0])
except Exception as e:
return make_api_response("", err=str(e), status_code=400)
return make_api_response({"started": True, "sid": submission['sid']})
if not ui_params['description']:
ui_params['description'] = "Inspection of file%s: %s" % ({True: "s", False: ""}[len(fnames) > 1],
", ".join(fnames))
# Submit to dispatcher
try:
submission_obj = Submission({
"files": [],
"params": ui_to_submission_params(ui_params)
})
except (ValueError, KeyError) as e:
return make_api_response("", err=str(e), status_code=400)
with forge.get_filestore() as f_transport:
try:
submit_result = SubmissionClient(datastore=STORAGE, filestore=f_transport, config=config)\
.submit(submission_obj, local_files=request_files, cleanup=False)
submission_received(submission_obj)
except SubmissionException as e:
return make_api_response("", err=str(e), status_code=400)
return make_api_response({"started": True, "sid": submit_result.sid})
else:
return make_api_response({"started": False, "sid": None}, "No files where found for ID %s. "
"Try again..." % ui_sid, 404)
finally:
if submit_result is None:
decrement_submission_quota(user)
# Remove files
for myfile in request_files:
try:
os.unlink(myfile)
except Exception:
pass
# Remove dirs
for fpath in request_dirs:
try:
os.rmdir(fpath)
except Exception:
pass