def test_empty(self):
"""
Empty input stream, empty opt header, empty opt footer, no digests.
"""
empty_stream = BytesIO()
output_stream = BytesIO()
header = footer = {}
# Pack with empty everything
cart.pack_stream(empty_stream,
output_stream,
header,
footer,
auto_digests=())
packed_text = output_stream.getvalue()
self.assert_valid_mandatory_header(packed_text)
# Now test unpacking the result.
packed_stream = BytesIO(packed_text)
plain_stream = BytesIO()
(opt_header, opt_footer) = cart.unpack_stream(packed_stream,
plain_stream)
plain_text = plain_stream.getvalue()
self.assertEqual(opt_header, {})
self.assertEqual(opt_footer, {})
self.assertEqual(len(plain_text), 0)
def test_rc4_override(self):
rc4_key = b"Test Da Key !"
tmp_header = {'name': 'hello.txt'}
tmp_footer = {'rc4_key': rc4_key.decode()}
plaintext = b'0123456789' * 100
pt_stream = BytesIO(plaintext)
ct_stream = BytesIO()
cart.pack_stream(pt_stream,
ct_stream,
optional_header=tmp_header,
optional_footer=tmp_footer,
arc4_key_override=rc4_key)
crypt_text = ct_stream.getvalue()
ct_stream = BytesIO(crypt_text)
pt_stream = BytesIO()
with self.assertRaises(cart.InvalidARC4KeyException):
cart.unpack_stream(ct_stream, pt_stream)
ct_stream = BytesIO(crypt_text)
pt_stream = BytesIO()
(header, footer) = cart.unpack_stream(ct_stream,
pt_stream,
arc4_key_override=rc4_key)
self.assertEqual(header, tmp_header)
self.assertEqual(footer, tmp_footer)
def test_simple(self):
plaintext = b'0123456789' * 10000000
pt_stream = BytesIO(plaintext)
ct_stream = BytesIO()
cart.pack_stream(pt_stream, ct_stream, {'name': 'hello.txt'},
{'digest': 'done'})
crypt_text = ct_stream.getvalue()
ct_stream = BytesIO(crypt_text)
pt_stream = BytesIO()
temp_file = tempfile.mkstemp()[1]
with open(temp_file, 'wb') as f:
f.write(ct_stream.getvalue())
(header, footer) = cart.unpack_stream(ct_stream, pt_stream)
inline_metadata = {}
if header:
inline_metadata.update(header)
if footer:
inline_metadata.update(footer)
plaintext_prime = pt_stream.getvalue()
self.assertEqual(plaintext_prime, plaintext)
metadata = cart.get_metadata_only(temp_file)
self.assertEqual(metadata, inline_metadata)
self.assertTrue(cart.is_cart(crypt_text))
def encode_file(input_path, name, metadata=None):
if metadata is None:
metadata = {}
_, output_path = tempfile.mkstemp()
with open(output_path, 'wb') as oh:
with open(input_path, 'rb') as ih:
data = ih.read(64)
if not is_cart(data):
ih.seek(0)
metadata.update({'name': name})
pack_stream(ih, oh, metadata)
return output_path, f"{name}.cart"
else:
return input_path, name
def encode_file(data, file_format, name, password=None):
error = {}
already_encoded = False
file_info = identify.ident(data, len(data))
if file_info['tag'] in NEUTERED_FORMAT:
already_encoded = True
elif file_format == 'cart':
from cart import pack_stream
from cStringIO import StringIO
ostream = StringIO()
pack_stream(StringIO(data), ostream, {"name": name})
data = ostream.getvalue()
elif file_format != 'raw':
error['code'] = 500
error['text'] = "Invalid file format specified."
return data, error, already_encoded
def test_identify():
with forge.get_identify(use_cache=False) as identify:
# Setup test data
aaaa = f"{'A' * 10000}".encode()
sha256 = hashlib.sha256(aaaa).hexdigest()
# Prep temp file
_, input_path = tempfile.mkstemp()
output_path = f"{input_path}.cart"
try:
# Write temp file
with open(input_path, 'wb') as oh:
oh.write(aaaa)
# Create a cart file
with open(output_path, 'wb') as oh:
with open(input_path, 'rb') as ih:
pack_stream(ih, oh, {'name': 'test_identify.a'})
# Validate the cart file created
meta = get_metadata_only(output_path)
assert meta.get("sha256", None) == sha256
# Validate identify file detection
info = identify.fileinfo(output_path)
assert info.get("type", None) == "archive/cart"
# Validate identify hashing
output_sha256 = subprocess.check_output(['sha256sum', output_path
])[:64].decode()
assert info.get("sha256", None) == output_sha256
finally:
# Cleanup output file
if os.path.exists(output_path):
os.unlink(output_path)
# Cleanup input file
if os.path.exists(input_path):
os.unlink(input_path)
def test_large(self):
"""
128MB stream, large opt header, large opt footer, default digests + testdigester.
"""
test_text = b'0' * 1024 * 1024 * 128
in_stream = BytesIO(test_text)
output_stream = BytesIO()
test_header = {}
test_footer = {}
# Pack with empty everything
cart.pack_stream(in_stream, output_stream, test_header, test_footer)
packed_text = output_stream.getvalue()
self.assert_valid_mandatory_header(packed_text)
# Now test unpacking the result.
packed_stream = BytesIO(packed_text)
plain_stream = BytesIO()
(opt_header, opt_footer) = cart.unpack_stream(packed_stream,
plain_stream)
plain_text = plain_stream.getvalue()
self.assertEqual(test_header, opt_header)
self.assertEqual(test_footer, opt_footer)
self.assertEqual(test_text, plain_text)
def test_small(self):
"""
1 byte stream, 1 element opt header, 1 element opt footer, default digests.
"""
test_text = b'a'
in_stream = BytesIO(test_text)
output_stream = BytesIO()
test_header = {'testkey': 'testvalue'}
test_footer = {'complete': 'yes'}
# Pack with empty everything
cart.pack_stream(in_stream, output_stream, test_header, test_footer)
packed_text = output_stream.getvalue()
self.assert_valid_mandatory_header(packed_text)
# Now test unpacking the result.
packed_stream = BytesIO(packed_text)
plain_stream = BytesIO()
(opt_header, opt_footer) = cart.unpack_stream(packed_stream,
plain_stream)
plain_text = plain_stream.getvalue()
self.assertEqual(test_header, opt_header)
self.assertEqual(test_footer, opt_footer)
self.assertEqual(test_text, plain_text)
def test_rc4_override(self):
from cStringIO import StringIO
rc4_key = "Test Da Key !"
tmp_header = {'name': 'hello.txt'}
tmp_footer = {'rc4_key': rc4_key}
plaintext = '0123456789' * 100
pt_stream = StringIO(plaintext)
ct_stream = StringIO()
cart.pack_stream(pt_stream,
ct_stream,
optional_header=tmp_header,
optional_footer=tmp_footer,
arc4_key_override=rc4_key)
crypt_text = ct_stream.getvalue()
ct_stream = StringIO(crypt_text)
pt_stream = StringIO()
try:
cart.unpack_stream(ct_stream, pt_stream)
except ValueError, e:
self.assertEqual(str(e),
"Invalid ARC4 Key, could not unpack header")
def create_bundle(sid, working_dir=WORK_DIR):
with forge.get_datastore() as datastore:
temp_bundle_file = f"bundle_{get_random_id()}"
current_working_dir = os.path.join(working_dir, temp_bundle_file)
try:
submission = datastore.submission.get(sid, as_obj=False)
if submission is None:
raise SubmissionNotFound("Can't find submission %s, skipping." % sid)
else:
target_file = os.path.join(working_dir, f"{temp_bundle_file}.cart")
tgz_file = os.path.join(working_dir, f"{temp_bundle_file}.tgz")
try:
os.makedirs(current_working_dir)
except PermissionError:
raise
except Exception:
pass
# Create file information data
file_tree = datastore.get_or_create_file_tree(submission,
config.submission.max_extraction_depth)['tree']
flatten_tree = list(set(recursive_flatten_tree(file_tree) +
[r[:64] for r in submission.get("results", [])]))
file_infos, _ = get_file_infos(copy(flatten_tree), datastore)
# Add bundling metadata
if 'bundle.source' not in submission['metadata']:
submission['metadata']['bundle.source'] = config.ui.fqdn
if Classification.enforce and 'bundle.classification' not in submission['metadata']:
submission['metadata']['bundle.classification'] = submission['classification']
data = {
'submission': submission,
'files': {"list": flatten_tree, "tree": file_tree, "infos": file_infos},
'results': get_results(submission.get("results", []), file_infos, datastore),
'errors': get_errors(submission.get("errors", []), datastore)
}
# Save result files
with open(os.path.join(current_working_dir, "results.json"), "w") as fp:
json.dump(data, fp)
# Download all related files
with forge.get_filestore() as filestore:
for sha256 in flatten_tree:
try:
filestore.download(sha256, os.path.join(current_working_dir, sha256))
except FileStoreException:
pass
# Create the bundle
subprocess.check_call("tar czf %s *" % tgz_file, shell=True, cwd=current_working_dir)
with open(target_file, 'wb') as oh:
with open(tgz_file, 'rb') as ih:
pack_stream(ih, oh, {'al': {"type": BUNDLE_TYPE}, 'name': f"{sid}.tgz"})
return target_file
except Exception as e:
raise BundlingException("Could not bundle submission '%s'. [%s: %s]" % (sid, type(e).__name__, str(e)))
finally:
if current_working_dir:
subprocess.check_call(["rm", "-rf", current_working_dir])
