def _finder_from_td(self): collector = OxidizedResourceCollector(policy="in-memory-only") for r in find_resources_in_path(self.td): collector.add_in_memory(r) f = OxidizedFinder() f.add_resources(collector.oxidize()[0]) return f
def _finder_from_td(self): collector = OxidizedResourceCollector(allowed_locations=["in-memory"]) for r in find_resources_in_path(self.td): collector.add_in_memory(r) f = OxidizedFinder() f.add_resources(collector.oxidize()[0]) return f
def _finder_from_td(self): collector = OxidizedResourceCollector(allowed_locations=["in-memory"]) for r in find_resources_in_path(self.td): collector.add_in_memory(r) f = OxidizedFinder() f.add_resources( collector.oxidize( python_exe=os.environ.get("PYTHON_SYS_EXECUTABLE"))[0]) return f
def test_add_resource_module(self): f = OxidizedFinder() resource = OxidizedResource() resource.name = "my_module" resource.flavor = "module" source = b"print('hello from my_module')" code = compile(source, "my_module.py", "exec") bytecode = marshal.dumps(code) resource.in_memory_source = source resource.in_memory_bytecode = bytecode f.add_resource(resource) resources = [r for r in f.indexed_resources() if r.name == "my_module"] self.assertEqual(len(resources), 1) spec = f.find_spec("my_module", None) self.assertIsInstance(spec, importlib.machinery.ModuleSpec) self.assertEqual(spec.name, "my_module") self.assertIsNone(spec.loader_state) self.assertIsNone(spec.submodule_search_locations) self.assertEqual(f.get_source("my_module"), source.decode("utf-8")) self.assertEqual(f.get_code("my_module"), code)
def get_resources_data(self) -> bytes: c = OxidizedResourceCollector(allowed_locations=["in-memory"]) for path in sys.path: if os.path.isdir(path): for i, resource in enumerate(find_resources_in_path(path)): c.add_in_memory(resource) if i == 10: break finder = OxidizedFinder() finder.add_resources(c.oxidize()[0]) return finder.serialize_indexed_resources()
def test_add_resources(self): f = OxidizedFinder() a = OxidizedResource() a.name = "foo_a" a.flavor = "module" b = OxidizedResource() b.name = "foo_b" b.flavor = "module" f.add_resources([a, b]) resources = [ r for r in f.indexed_resources() if r.name in ("foo_a", "foo_b") ] self.assertEqual(len(resources), 2)
def make_finder(*modules: Tuple[str, str, bool]) -> OxidizedFinder: """Create an ``OxidizedFinder`` with modules defined by ``modules``. ``modules`` must be tuples of the form (name, source_code, is_package). """ mpf = OxidizedFinder() for module_name, source, is_pkg in modules: # See example in OxidizedFinder.add_resource resource = OxidizedResource() resource.is_module = True resource.name = module_name resource.is_package = is_pkg resource.in_memory_source = source.encode("utf-8") resource.in_memory_bytecode = marshal.dumps( compile(source, module_name, "exec")) mpf.add_resource(resource) return mpf
def test_bad_utf8_pkg_name_raises(self): exe = sys.executable.encode("utf-8", "surrogatepass") foo = "fo\ud800o".encode("utf-8", "surrogatepass") exe = os.path.join(exe, foo) with self.assertRaisesRegex(ImportError, "cannot decode") as cm: OxidizedFinder().path_hook(exe) self.assertIsInstance(cm.exception.__cause__, UnicodeDecodeError) self.assertIn(foo, cm.exception.__cause__.object) self.assertEqual(cm.exception.__cause__.encoding.lower(), "utf-8")
def test_bad_unicode_executable_path_ok(self): # A non-Unicode path doesn't cause a panic. exe = sys.executable.encode("utf-8", "surrogatepass") # "fo\ud800o" contains an unpaired surrogate. foo = "fo\ud800o".encode("utf-8", "surrogatepass") exe += foo with self.assertRaises(ImportError) as cm: OxidizedFinder().path_hook(exe) self.assertEqual(cm.exception.path, exe)
def test_source_module(self): c = OxidizedResourceCollector(policy="in-memory-only") source_path = self.td / "foo.py" with source_path.open("wb") as fh: fh.write(b"import io\n") for resource in find_resources_in_path(self.td): c.add_in_memory(resource) f = OxidizedFinder() f.add_resources(c.oxidize()[0]) resources = [r for r in f.indexed_resources() if r.name == "foo"] self.assertEqual(len(resources), 1) r = resources[0] self.assertEqual(r.in_memory_source, b"import io\n")
def test_resources_builtins(self): f = OxidizedFinder() f.index_interpreter_builtin_extension_modules() resources = f.indexed_resources() self.assertIsInstance(resources, list) self.assertGreater(len(resources), 0) resources = sorted(resources, key=lambda x: x.name) resource = [x for x in resources if x.name == "_io"][0] self.assertIsInstance(resource, OxidizedResource) self.assertEqual(repr(resource), '<OxidizedResource name="_io">') self.assertFalse(resource.is_module) self.assertTrue(resource.is_builtin_extension_module) self.assertFalse(resource.is_frozen_module) self.assertFalse(resource.is_extension_module) self.assertFalse(resource.is_shared_library) self.assertIsInstance(resource.name, str) self.assertEqual(resource.name, "_io") self.assertFalse(resource.is_package) self.assertFalse(resource.is_namespace_package) self.assertIsNone(resource.in_memory_source) self.assertIsNone(resource.in_memory_bytecode) self.assertIsNone(resource.in_memory_bytecode_opt1) self.assertIsNone(resource.in_memory_bytecode_opt2) self.assertIsNone(resource.in_memory_extension_module_shared_library) self.assertIsNone(resource.in_memory_package_resources) self.assertIsNone(resource.in_memory_distribution_resources) self.assertIsNone(resource.in_memory_shared_library) self.assertIsNone(resource.shared_library_dependency_names) self.assertIsNone(resource.relative_path_module_source) self.assertIsNone(resource.relative_path_module_bytecode) self.assertIsNone(resource.relative_path_module_bytecode_opt1) self.assertIsNone(resource.relative_path_module_bytecode_opt2) self.assertIsNone( resource.relative_path_extension_module_shared_library) self.assertIsNone(resource.relative_path_package_resources) self.assertIsNone(resource.relative_path_distribution_resources)
def test_source_module(self): c = OxidizedResourceCollector(allowed_locations=["in-memory"]) source_path = self.td / "foo.py" with source_path.open("wb") as fh: fh.write(b"import io\n") for resource in find_resources_in_path(self.td): c.add_in_memory(resource) f = OxidizedFinder() python_exe = os.environ.get("PYTHON_SYS_EXECUTABLE") with assert_tempfile_cleaned_up(): oxide = c.oxidize(python_exe=python_exe) f.add_resources(oxide[0]) resources = [r for r in f.indexed_resources() if r.name == "foo"] self.assertEqual(len(resources), 1) r = resources[0] self.assertEqual(r.in_memory_source, b"import io\n")
def test_load_pkg_info(self): # In absence of a METADATA file, a PKG-INFO file will be read. pkginfo_path = self.td / "my_package-1.0.egg-info" / "PKG-INFO" pkginfo_path.parent.mkdir() with pkginfo_path.open("w", encoding="utf-8") as fh: fh.write("Name: my_package\n") fh.write("Version: 1.0\n") collector = OxidizedResourceCollector(policy="in-memory-only") for r in find_resources_in_path(self.td): collector.add_in_memory(r) f = OxidizedFinder() f.add_resources(collector.oxidize()[0]) dists = f.find_distributions() self.assertEqual(len(dists), 1) metadata = dists[0].metadata self.assertIsInstance(metadata, email.message.Message) self.assertEqual(metadata["Name"], "my_package") self.assertEqual(metadata["Version"], "1.0")
def test_add_sys_path(self): c = OxidizedResourceCollector( policy="prefer-in-memory-fallback-filesystem-relative:prefix") for path in sys.path: if os.path.isdir(path): for resource in find_resources_in_path(path): c.add_in_memory(resource) c.add_filesystem_relative("", resource) resources, file_installs = c.oxidize() f = OxidizedFinder() f.add_resources(resources) with (self.td / "serialized").open("wb") as fh: fh.write(f.serialize_indexed_resources()) f = OxidizedFinder(resources_file=self.td / "serialized") self.assertGreaterEqual(len(f.indexed_resources()), len(resources)) for r in f.indexed_resources(): r.in_memory_source r.in_memory_bytecode
def test_find_distributions_empty(self): f = OxidizedFinder() dists = f.find_distributions() self.assertIsInstance(dists, list) self.assertEqual(len(dists), 0)
def test_resources_frozen(self): f = OxidizedFinder() resources = f.indexed_resources() resource = [x for x in resources if x.name == "_frozen_importlib"][0] self.assertEqual(resource.flavor, "frozen")
def test_serialize_simple(self): f = OxidizedFinder() m = OxidizedResource() m.is_module = True m.name = "my_module" m.in_memory_source = b"import io" f.add_resource(m) m = OxidizedResource() m.is_module = True m.name = "module_b" m.in_memory_bytecode = b"dummy bytecode" f.add_resource(m) serialized = f.serialize_indexed_resources() self.assertIsInstance(serialized, bytes) f2 = OxidizedFinder() f2.index_bytes(serialized) modules = {r.name: r for r in f2.indexed_resources() if r.is_module} self.assertEqual(len(modules), 2) self.assertIn("my_module", modules) self.assertIn("module_b", modules) self.assertEqual(modules["my_module"].in_memory_source, b"import io") self.assertEqual(modules["module_b"].in_memory_bytecode, b"dummy bytecode")
def test_add_resource_bad_type(self): f = OxidizedFinder() with self.assertRaises(TypeError): f.add_resource(None)
def test_resources_frozen(self): f = OxidizedFinder() resources = f.indexed_resources() resource = [x for x in resources if x.name == "_frozen_importlib"][0] self.assertTrue(resource.is_frozen_module)
def test_origin_bad_value(self): with self.assertRaises(TypeError): OxidizedFinder(relative_path_origin=True)
def test_no_indices(self): OxidizedFinder( resources_data= b"pyembed\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00")
def test_resources_bad_magic(self): with self.assertRaisesRegex(ValueError, "unrecognized file format"): OxidizedFinder(resources_data=b"\xde\xad\xbe\xef\xaa\xaa\xaa\xaa")
def test_find_distributions_empty(self): f = OxidizedFinder() dists = f.find_distributions() self.assertIsInstance(dists, collections.abc.Iterator) dists = list(dists) self.assertEqual(len(dists), 0)
def test_none(self): f = OxidizedFinder(None) self.assertIsInstance(f, OxidizedFinder) f = OxidizedFinder(resources_data=None) self.assertIsInstance(f, OxidizedFinder)
def test_no_args(self): f = OxidizedFinder() self.assertIsInstance(f, OxidizedFinder)
def test_origin(self): OxidizedFinder(relative_path_origin="/path/to/origin")
def test_bad_resources_type(self): with self.assertRaises(TypeError): OxidizedFinder(resources_data="foo")
def test_add_sys_path(self): c = OxidizedResourceCollector( allowed_locations=["in-memory", "filesystem-relative"]) for path in sys.path: if os.path.isdir(path): for resource in find_resources_in_path(path): c.add_in_memory(resource) c.add_filesystem_relative("", resource) python_exe = os.environ.get("PYTHON_SYS_EXECUTABLE") with assert_tempfile_cleaned_up(): resources, file_installs = c.oxidize(python_exe=python_exe) f = OxidizedFinder() f.add_resources(resources) with (self.td / "serialized").open("wb") as fh: fh.write(f.serialize_indexed_resources()) f = OxidizedFinder() f.index_file_memory_mapped(self.td / "serialized") self.assertGreaterEqual(len(f.indexed_resources()), len(resources)) for r in f.indexed_resources(): r.in_memory_source r.in_memory_bytecode
def test_find_spec_missing(self): f = OxidizedFinder() self.assertIsNone(f.find_spec("my_package", None))
def test_resources_no_magic(self): with self.assertRaisesRegex(ValueError, "reading 8 byte"): OxidizedFinder(resources_data=b"foo")