def test_empty(self): """ Empty input stream, empty opt header, empty opt footer, no digests. """ empty_stream = BytesIO() output_stream = BytesIO() header = footer = {} # Pack with empty everything cart.pack_stream(empty_stream, output_stream, header, footer, auto_digests=()) packed_text = output_stream.getvalue() self.assert_valid_mandatory_header(packed_text) # Now test unpacking the result. packed_stream = BytesIO(packed_text) plain_stream = BytesIO() (opt_header, opt_footer) = cart.unpack_stream(packed_stream, plain_stream) plain_text = plain_stream.getvalue() self.assertEqual(opt_header, {}) self.assertEqual(opt_footer, {}) self.assertEqual(len(plain_text), 0)
def test_rc4_override(self): rc4_key = b"Test Da Key !" tmp_header = {'name': 'hello.txt'} tmp_footer = {'rc4_key': rc4_key.decode()} plaintext = b'0123456789' * 100 pt_stream = BytesIO(plaintext) ct_stream = BytesIO() cart.pack_stream(pt_stream, ct_stream, optional_header=tmp_header, optional_footer=tmp_footer, arc4_key_override=rc4_key) crypt_text = ct_stream.getvalue() ct_stream = BytesIO(crypt_text) pt_stream = BytesIO() with self.assertRaises(cart.InvalidARC4KeyException): cart.unpack_stream(ct_stream, pt_stream) ct_stream = BytesIO(crypt_text) pt_stream = BytesIO() (header, footer) = cart.unpack_stream(ct_stream, pt_stream, arc4_key_override=rc4_key) self.assertEqual(header, tmp_header) self.assertEqual(footer, tmp_footer)
def test_simple(self): plaintext = b'0123456789' * 10000000 pt_stream = BytesIO(plaintext) ct_stream = BytesIO() cart.pack_stream(pt_stream, ct_stream, {'name': 'hello.txt'}, {'digest': 'done'}) crypt_text = ct_stream.getvalue() ct_stream = BytesIO(crypt_text) pt_stream = BytesIO() temp_file = tempfile.mkstemp()[1] with open(temp_file, 'wb') as f: f.write(ct_stream.getvalue()) (header, footer) = cart.unpack_stream(ct_stream, pt_stream) inline_metadata = {} if header: inline_metadata.update(header) if footer: inline_metadata.update(footer) plaintext_prime = pt_stream.getvalue() self.assertEqual(plaintext_prime, plaintext) metadata = cart.get_metadata_only(temp_file) self.assertEqual(metadata, inline_metadata) self.assertTrue(cart.is_cart(crypt_text))
def encode_file(input_path, name, metadata=None): if metadata is None: metadata = {} _, output_path = tempfile.mkstemp() with open(output_path, 'wb') as oh: with open(input_path, 'rb') as ih: data = ih.read(64) if not is_cart(data): ih.seek(0) metadata.update({'name': name}) pack_stream(ih, oh, metadata) return output_path, f"{name}.cart" else: return input_path, name
def encode_file(data, file_format, name, password=None): error = {} already_encoded = False file_info = identify.ident(data, len(data)) if file_info['tag'] in NEUTERED_FORMAT: already_encoded = True elif file_format == 'cart': from cart import pack_stream from cStringIO import StringIO ostream = StringIO() pack_stream(StringIO(data), ostream, {"name": name}) data = ostream.getvalue() elif file_format != 'raw': error['code'] = 500 error['text'] = "Invalid file format specified." return data, error, already_encoded
def test_identify(): with forge.get_identify(use_cache=False) as identify: # Setup test data aaaa = f"{'A' * 10000}".encode() sha256 = hashlib.sha256(aaaa).hexdigest() # Prep temp file _, input_path = tempfile.mkstemp() output_path = f"{input_path}.cart" try: # Write temp file with open(input_path, 'wb') as oh: oh.write(aaaa) # Create a cart file with open(output_path, 'wb') as oh: with open(input_path, 'rb') as ih: pack_stream(ih, oh, {'name': 'test_identify.a'}) # Validate the cart file created meta = get_metadata_only(output_path) assert meta.get("sha256", None) == sha256 # Validate identify file detection info = identify.fileinfo(output_path) assert info.get("type", None) == "archive/cart" # Validate identify hashing output_sha256 = subprocess.check_output(['sha256sum', output_path ])[:64].decode() assert info.get("sha256", None) == output_sha256 finally: # Cleanup output file if os.path.exists(output_path): os.unlink(output_path) # Cleanup input file if os.path.exists(input_path): os.unlink(input_path)
def test_large(self): """ 128MB stream, large opt header, large opt footer, default digests + testdigester. """ test_text = b'0' * 1024 * 1024 * 128 in_stream = BytesIO(test_text) output_stream = BytesIO() test_header = {} test_footer = {} # Pack with empty everything cart.pack_stream(in_stream, output_stream, test_header, test_footer) packed_text = output_stream.getvalue() self.assert_valid_mandatory_header(packed_text) # Now test unpacking the result. packed_stream = BytesIO(packed_text) plain_stream = BytesIO() (opt_header, opt_footer) = cart.unpack_stream(packed_stream, plain_stream) plain_text = plain_stream.getvalue() self.assertEqual(test_header, opt_header) self.assertEqual(test_footer, opt_footer) self.assertEqual(test_text, plain_text)
def test_small(self): """ 1 byte stream, 1 element opt header, 1 element opt footer, default digests. """ test_text = b'a' in_stream = BytesIO(test_text) output_stream = BytesIO() test_header = {'testkey': 'testvalue'} test_footer = {'complete': 'yes'} # Pack with empty everything cart.pack_stream(in_stream, output_stream, test_header, test_footer) packed_text = output_stream.getvalue() self.assert_valid_mandatory_header(packed_text) # Now test unpacking the result. packed_stream = BytesIO(packed_text) plain_stream = BytesIO() (opt_header, opt_footer) = cart.unpack_stream(packed_stream, plain_stream) plain_text = plain_stream.getvalue() self.assertEqual(test_header, opt_header) self.assertEqual(test_footer, opt_footer) self.assertEqual(test_text, plain_text)
def test_rc4_override(self): from cStringIO import StringIO rc4_key = "Test Da Key !" tmp_header = {'name': 'hello.txt'} tmp_footer = {'rc4_key': rc4_key} plaintext = '0123456789' * 100 pt_stream = StringIO(plaintext) ct_stream = StringIO() cart.pack_stream(pt_stream, ct_stream, optional_header=tmp_header, optional_footer=tmp_footer, arc4_key_override=rc4_key) crypt_text = ct_stream.getvalue() ct_stream = StringIO(crypt_text) pt_stream = StringIO() try: cart.unpack_stream(ct_stream, pt_stream) except ValueError, e: self.assertEqual(str(e), "Invalid ARC4 Key, could not unpack header")
def create_bundle(sid, working_dir=WORK_DIR): with forge.get_datastore() as datastore: temp_bundle_file = f"bundle_{get_random_id()}" current_working_dir = os.path.join(working_dir, temp_bundle_file) try: submission = datastore.submission.get(sid, as_obj=False) if submission is None: raise SubmissionNotFound("Can't find submission %s, skipping." % sid) else: target_file = os.path.join(working_dir, f"{temp_bundle_file}.cart") tgz_file = os.path.join(working_dir, f"{temp_bundle_file}.tgz") try: os.makedirs(current_working_dir) except PermissionError: raise except Exception: pass # Create file information data file_tree = datastore.get_or_create_file_tree(submission, config.submission.max_extraction_depth)['tree'] flatten_tree = list(set(recursive_flatten_tree(file_tree) + [r[:64] for r in submission.get("results", [])])) file_infos, _ = get_file_infos(copy(flatten_tree), datastore) # Add bundling metadata if 'bundle.source' not in submission['metadata']: submission['metadata']['bundle.source'] = config.ui.fqdn if Classification.enforce and 'bundle.classification' not in submission['metadata']: submission['metadata']['bundle.classification'] = submission['classification'] data = { 'submission': submission, 'files': {"list": flatten_tree, "tree": file_tree, "infos": file_infos}, 'results': get_results(submission.get("results", []), file_infos, datastore), 'errors': get_errors(submission.get("errors", []), datastore) } # Save result files with open(os.path.join(current_working_dir, "results.json"), "w") as fp: json.dump(data, fp) # Download all related files with forge.get_filestore() as filestore: for sha256 in flatten_tree: try: filestore.download(sha256, os.path.join(current_working_dir, sha256)) except FileStoreException: pass # Create the bundle subprocess.check_call("tar czf %s *" % tgz_file, shell=True, cwd=current_working_dir) with open(target_file, 'wb') as oh: with open(tgz_file, 'rb') as ih: pack_stream(ih, oh, {'al': {"type": BUNDLE_TYPE}, 'name': f"{sid}.tgz"}) return target_file except Exception as e: raise BundlingException("Could not bundle submission '%s'. [%s: %s]" % (sid, type(e).__name__, str(e))) finally: if current_working_dir: subprocess.check_call(["rm", "-rf", current_working_dir])