def test_add_sys_path(self):
c = OxidizedResourceCollector(
allowed_locations=["in-memory", "filesystem-relative"])
for path in sys.path:
if os.path.isdir(path):
for resource in find_resources_in_path(path):
c.add_in_memory(resource)
c.add_filesystem_relative("", resource)
python_exe = os.environ.get("PYTHON_SYS_EXECUTABLE")
with assert_tempfile_cleaned_up():
resources, file_installs = c.oxidize(python_exe=python_exe)
f = OxidizedFinder()
f.add_resources(resources)
with (self.td / "serialized").open("wb") as fh:
fh.write(f.serialize_indexed_resources())
f = OxidizedFinder()
f.index_file_memory_mapped(self.td / "serialized")
self.assertGreaterEqual(len(f.indexed_resources()), len(resources))
for r in f.indexed_resources():
r.in_memory_source
r.in_memory_bytecode
def test_serialize_simple(self):
f = OxidizedFinder()
m = OxidizedResource()
m.name = "my_module"
m.flavor = "module"
m.in_memory_source = b"import io"
f.add_resource(m)
m = OxidizedResource()
m.name = "module_b"
m.flavor = "module"
m.in_memory_bytecode = b"dummy bytecode"
f.add_resource(m)
serialized = f.serialize_indexed_resources()
self.assertIsInstance(serialized, bytes)
f2 = OxidizedFinder(resources_data=serialized)
modules = {
r.name: r
for r in f2.indexed_resources() if r.flavor == "module"
}
self.assertEqual(len(modules), 2)
self.assertIn("my_module", modules)
self.assertIn("module_b", modules)
self.assertEqual(modules["my_module"].in_memory_source, b"import io")
self.assertEqual(modules["module_b"].in_memory_bytecode,
b"dummy bytecode")
def test_resources_frozen(self):
f = OxidizedFinder()
f.index_interpreter_frozen_modules()
resources = f.indexed_resources()
resource = [x for x in resources if x.name == "_frozen_importlib"][0]
self.assertTrue(resource.is_frozen_module)
def test_find_distributions_context_unknown_name(self):
f = OxidizedFinder()
dists = list(
f.find_distributions(
importlib.metadata.DistributionFinder.Context(name="missing")))
self.assertEqual(len(dists), 0)
def test_add_resource_module(self):
f = OxidizedFinder()
resource = OxidizedResource()
resource.name = "my_module"
resource.flavor = "module"
source = b"print('hello from my_module')"
code = compile(source, "my_module.py", "exec")
bytecode = marshal.dumps(code)
resource.in_memory_source = source
resource.in_memory_bytecode = bytecode
f.add_resource(resource)
resources = [r for r in f.indexed_resources() if r.name == "my_module"]
self.assertEqual(len(resources), 1)
spec = f.find_spec("my_module", None)
self.assertIsInstance(spec, importlib.machinery.ModuleSpec)
self.assertEqual(spec.name, "my_module")
self.assertIsNone(spec.loader_state)
self.assertIsNone(spec.submodule_search_locations)
self.assertEqual(f.get_source("my_module"), source.decode("utf-8"))
self.assertEqual(f.get_code("my_module"), code)
def test_load_pkg_info(self):
# In absence of a METADATA file, a PKG-INFO file will be read.
pkginfo_path = self.td / "my_package-1.0.egg-info" / "PKG-INFO"
pkginfo_path.parent.mkdir()
with pkginfo_path.open("w", encoding="utf-8") as fh:
fh.write("Name: my_package\n")
fh.write("Version: 1.0\n")
collector = OxidizedResourceCollector(allowed_locations=["in-memory"])
for r in find_resources_in_path(self.td):
collector.add_in_memory(r)
f = OxidizedFinder()
f.add_resources(
collector.oxidize(
python_exe=os.environ.get("PYTHON_SYS_EXECUTABLE"))[0])
dists = list(f.find_distributions())
self.assertEqual(len(dists), 1)
metadata = dists[0].metadata
self.assertIsInstance(metadata, email.message.Message)
self.assertEqual(metadata["Name"], "my_package")
self.assertEqual(metadata["Version"], "1.0")
def test_not_sys_executable_abs_str(self):
self.assertFalse(sys.prefix.startswith(sys.executable),
"bad assumption in test")
finder = OxidizedFinder()
with self.assertRaisesRegex(ImportError, self.NOT_FOUND_ERR) as cm:
finder.path_hook(sys.prefix)
self.assertEqual(cm.exception.path, sys.prefix)
def test_index_file_memory_mapped_simple(self):
path = self.td / "simple"
with path.open("wb") as fh:
fh.write(self.get_resources_data())
f = OxidizedFinder()
f.index_file_memory_mapped(path)
def test_iter_modules_empty(self):
f = OxidizedFinder()
self.assertIsInstance(f.iter_modules(), list)
self.assertEqual(f.iter_modules(), [])
sys.meta_path = [f]
sys.path = []
self.assertEqual(len(list(pkgutil.iter_modules())), 0)
def _finder_from_td(self):
collector = OxidizedResourceCollector(allowed_locations=["in-memory"])
for r in find_resources_in_path(self.td):
collector.add_in_memory(r)
f = OxidizedFinder()
f.add_resources(collector.oxidize()[0])
return f
def test_bad_unicode_executable_path_ok(self):
# A non-Unicode path doesn't cause a panic.
exe = sys.executable.encode("utf-8", "surrogatepass")
# "fo\ud800o" contains an unpaired surrogate.
foo = "fo\ud800o".encode("utf-8", "surrogatepass")
exe += foo
with self.assertRaises(ImportError) as cm:
OxidizedFinder().path_hook(exe)
self.assertEqual(cm.exception.path, exe)
def _finder_from_td(self):
collector = OxidizedResourceCollector(policy="in-memory-only")
for r in find_resources_in_path(self.td):
collector.add_in_memory(r)
f = OxidizedFinder()
f.add_resources(collector.oxidize()[0])
return f
def test_bad_utf8_pkg_name_raises(self):
exe = sys.executable.encode("utf-8", "surrogatepass")
foo = "fo\ud800o".encode("utf-8", "surrogatepass")
exe = os.path.join(exe, foo)
with self.assertRaisesRegex(ImportError, "cannot decode") as cm:
OxidizedFinder().path_hook(exe)
self.assertIsInstance(cm.exception.__cause__, UnicodeDecodeError)
self.assertIn(foo, cm.exception.__cause__.object)
self.assertEqual(cm.exception.__cause__.encoding.lower(), "utf-8")
def _finder_from_td(self):
collector = OxidizedResourceCollector(allowed_locations=["in-memory"])
for r in find_resources_in_path(self.td):
collector.add_in_memory(r)
f = OxidizedFinder()
f.add_resources(
collector.oxidize(
python_exe=os.environ.get("PYTHON_SYS_EXECUTABLE"))[0])
return f
def test_add_sys_path(self):
c = OxidizedResourceCollector(
policy="prefer-in-memory-fallback-filesystem-relative:prefix")
for path in sys.path:
if os.path.isdir(path):
for resource in find_resources_in_path(path):
c.add_in_memory(resource)
c.add_filesystem_relative("", resource)
resources, file_installs = c.oxidize()
f = OxidizedFinder()
f.add_resources(resources)
with (self.td / "serialized").open("wb") as fh:
fh.write(f.serialize_indexed_resources())
f = OxidizedFinder(resources_file=self.td / "serialized")
self.assertGreaterEqual(len(f.indexed_resources()), len(resources))
for r in f.indexed_resources():
r.in_memory_source
r.in_memory_bytecode
def get_resources_data(self) -> bytes:
c = OxidizedResourceCollector(allowed_locations=["in-memory"])
for path in sys.path:
if os.path.isdir(path):
for i, resource in enumerate(find_resources_in_path(path)):
c.add_in_memory(resource)
if i == 10:
break
finder = OxidizedFinder()
finder.add_resources(c.oxidize()[0])
return finder.serialize_indexed_resources()
def test_add_resources(self):
f = OxidizedFinder()
a = OxidizedResource()
a.name = "foo_a"
a.flavor = "module"
b = OxidizedResource()
b.name = "foo_b"
b.flavor = "module"
f.add_resources([a, b])
resources = [
r for r in f.indexed_resources() if r.name in ("foo_a", "foo_b")
]
self.assertEqual(len(resources), 2)
def make_finder(*modules: Tuple[str, str, bool]) -> OxidizedFinder:
"""Create an ``OxidizedFinder`` with modules defined by ``modules``.
``modules`` must be tuples of the form (name, source_code, is_package).
"""
mpf = OxidizedFinder()
for module_name, source, is_pkg in modules:
# See example in OxidizedFinder.add_resource
resource = OxidizedResource()
resource.is_module = True
resource.name = module_name
resource.is_package = is_pkg
resource.in_memory_source = source.encode("utf-8")
resource.in_memory_bytecode = marshal.dumps(
compile(source, module_name, "exec"))
mpf.add_resource(resource)
return mpf
def test_source_module(self):
c = OxidizedResourceCollector(policy="in-memory-only")
source_path = self.td / "foo.py"
with source_path.open("wb") as fh:
fh.write(b"import io\n")
for resource in find_resources_in_path(self.td):
c.add_in_memory(resource)
f = OxidizedFinder()
f.add_resources(c.oxidize()[0])
resources = [r for r in f.indexed_resources() if r.name == "foo"]
self.assertEqual(len(resources), 1)
r = resources[0]
self.assertEqual(r.in_memory_source, b"import io\n")
def test_resources_builtins(self):
f = OxidizedFinder()
f.index_interpreter_builtin_extension_modules()
resources = f.indexed_resources()
self.assertIsInstance(resources, list)
self.assertGreater(len(resources), 0)
resources = sorted(resources, key=lambda x: x.name)
resource = [x for x in resources if x.name == "_io"][0]
self.assertIsInstance(resource, OxidizedResource)
self.assertEqual(repr(resource), '<OxidizedResource name="_io">')
self.assertFalse(resource.is_module)
self.assertTrue(resource.is_builtin_extension_module)
self.assertFalse(resource.is_frozen_module)
self.assertFalse(resource.is_extension_module)
self.assertFalse(resource.is_shared_library)
self.assertIsInstance(resource.name, str)
self.assertEqual(resource.name, "_io")
self.assertFalse(resource.is_package)
self.assertFalse(resource.is_namespace_package)
self.assertIsNone(resource.in_memory_source)
self.assertIsNone(resource.in_memory_bytecode)
self.assertIsNone(resource.in_memory_bytecode_opt1)
self.assertIsNone(resource.in_memory_bytecode_opt2)
self.assertIsNone(resource.in_memory_extension_module_shared_library)
self.assertIsNone(resource.in_memory_package_resources)
self.assertIsNone(resource.in_memory_distribution_resources)
self.assertIsNone(resource.in_memory_shared_library)
self.assertIsNone(resource.shared_library_dependency_names)
self.assertIsNone(resource.relative_path_module_source)
self.assertIsNone(resource.relative_path_module_bytecode)
self.assertIsNone(resource.relative_path_module_bytecode_opt1)
self.assertIsNone(resource.relative_path_module_bytecode_opt2)
self.assertIsNone(
resource.relative_path_extension_module_shared_library)
self.assertIsNone(resource.relative_path_package_resources)
self.assertIsNone(resource.relative_path_distribution_resources)
def test_source_module(self):
c = OxidizedResourceCollector(allowed_locations=["in-memory"])
source_path = self.td / "foo.py"
with source_path.open("wb") as fh:
fh.write(b"import io\n")
for resource in find_resources_in_path(self.td):
c.add_in_memory(resource)
f = OxidizedFinder()
python_exe = os.environ.get("PYTHON_SYS_EXECUTABLE")
with assert_tempfile_cleaned_up():
oxide = c.oxidize(python_exe=python_exe)
f.add_resources(oxide[0])
resources = [r for r in f.indexed_resources() if r.name == "foo"]
self.assertEqual(len(resources), 1)
r = resources[0]
self.assertEqual(r.in_memory_source, b"import io\n")
def test_load_pkg_info(self):
# In absence of a METADATA file, a PKG-INFO file will be read.
pkginfo_path = self.td / "my_package-1.0.egg-info" / "PKG-INFO"
pkginfo_path.parent.mkdir()
with pkginfo_path.open("w", encoding="utf-8") as fh:
fh.write("Name: my_package\n")
fh.write("Version: 1.0\n")
collector = OxidizedResourceCollector(policy="in-memory-only")
for r in find_resources_in_path(self.td):
collector.add_in_memory(r)
f = OxidizedFinder()
f.add_resources(collector.oxidize()[0])
dists = f.find_distributions()
self.assertEqual(len(dists), 1)
metadata = dists[0].metadata
self.assertIsInstance(metadata, email.message.Message)
self.assertEqual(metadata["Name"], "my_package")
self.assertEqual(metadata["Version"], "1.0")
def test_find_spec_missing(self):
f = OxidizedFinder()
self.assertIsNone(f.find_spec("my_package", None))
def test_resources_bad_magic(self):
with self.assertRaisesRegex(ValueError, "unrecognized file format"):
OxidizedFinder(resources_data=b"\xde\xad\xbe\xef\xaa\xaa\xaa\xaa")
def test_find_distributions_empty(self):
f = OxidizedFinder()
dists = f.find_distributions()
self.assertIsInstance(dists, list)
self.assertEqual(len(dists), 0)
def test_no_indices(self):
OxidizedFinder(
resources_data=
b"pyembed\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00")
def test_origin_bad_value(self):
with self.assertRaises(TypeError):
OxidizedFinder(relative_path_origin=True)
def test_add_resource_bad_type(self):
f = OxidizedFinder()
with self.assertRaises(TypeError):
f.add_resource(None)
def test_origin(self):
OxidizedFinder(relative_path_origin="/path/to/origin")
def test_resources_frozen(self):
f = OxidizedFinder()
resources = f.indexed_resources()
resource = [x for x in resources if x.name == "_frozen_importlib"][0]
self.assertEqual(resource.flavor, "frozen")
