def test_scan_not_directory(self):
path = self.td / "file"
with path.open("wb"):
pass
with self.assertRaisesRegex(ValueError, "path is not a directory"):
find_resources_in_path(path)
def test_urllib(self):
c = OxidizedResourceCollector(policy="filesystem-relative-only:lib")
for path in sys.path:
if os.path.isdir(path):
for resource in find_resources_in_path(path):
if isinstance(resource, PythonModuleBytecode):
if resource.module.startswith("urllib"):
if resource.optimize_level == 0:
c.add_filesystem_relative("lib", resource)
resources, file_installs = c.oxidize()
self.assertEqual(len(resources), len(file_installs))
idx = None
for i, resource in enumerate(resources):
if resource.name == "urllib.request":
idx = i
break
self.assertIsNotNone(idx)
(path, data, executable) = file_installs[idx]
self.assertEqual(
path,
pathlib.Path("lib") / "urllib" / "__pycache__" /
("request.%s.pyc" % sys.implementation.cache_tag),
)
self.assertTrue(data.startswith(importlib.util.MAGIC_NUMBER))
def test_package_distribution_resource(self):
init_py = self.td / "foo" / "__init__.py"
init_py.parent.mkdir()
with init_py.open("wb"):
pass
resource = self.td / "foo-1.0.dist-info" / "METADATA"
resource.parent.mkdir()
with resource.open("wb") as fh:
fh.write(b"Name: foo\n")
fh.write(b"Version: 1.0\n")
resources = find_resources_in_path(self.td)
self.assertEqual(len(resources), 2)
r = resources[0]
self.assertIsInstance(r, PythonModuleSource)
self.assertEqual(r.module, "foo")
self.assertTrue(r.is_package)
r = resources[1]
self.assertIsInstance(r, PythonPackageDistributionResource)
self.assertEqual(r.package, "foo")
self.assertEqual(r.version, "1.0")
self.assertEqual(r.name, "METADATA")
self.assertEqual(r.data, b"Name: foo\nVersion: 1.0\n")
def test_urllib(self):
c = OxidizedResourceCollector(
allowed_locations=["filesystem-relative"])
for path in sys.path:
if os.path.isdir(path):
for resource in find_resources_in_path(path):
if isinstance(resource, PythonModuleBytecode):
if resource.module.startswith("urllib"):
if resource.optimize_level == 0:
c.add_filesystem_relative("lib", resource)
python_exe = os.environ.get("PYTHON_SYS_EXECUTABLE")
with assert_tempfile_cleaned_up():
resources, file_installs = c.oxidize(python_exe=python_exe)
self.assertEqual(len(resources), len(file_installs))
idx = None
for i, resource in enumerate(resources):
if resource.name == "urllib.request":
idx = i
break
self.assertIsNotNone(idx)
(path, data, executable) = file_installs[idx]
self.assertEqual(
path,
pathlib.Path("lib") / "urllib" / "__pycache__" /
("request.%s.pyc" % sys.implementation.cache_tag),
)
self.assertTrue(data.startswith(importlib.util.MAGIC_NUMBER))
def test_add_sys_path(self):
c = OxidizedResourceCollector(
allowed_locations=["in-memory", "filesystem-relative"])
for path in sys.path:
if os.path.isdir(path):
for resource in find_resources_in_path(path):
c.add_in_memory(resource)
c.add_filesystem_relative("", resource)
python_exe = os.environ.get("PYTHON_SYS_EXECUTABLE")
with assert_tempfile_cleaned_up():
resources, file_installs = c.oxidize(python_exe=python_exe)
f = OxidizedFinder()
f.add_resources(resources)
with (self.td / "serialized").open("wb") as fh:
fh.write(f.serialize_indexed_resources())
f = OxidizedFinder()
f.index_file_memory_mapped(self.td / "serialized")
self.assertGreaterEqual(len(f.indexed_resources()), len(resources))
for r in f.indexed_resources():
r.in_memory_source
r.in_memory_bytecode
def test_load_pkg_info(self):
# In absence of a METADATA file, a PKG-INFO file will be read.
pkginfo_path = self.td / "my_package-1.0.egg-info" / "PKG-INFO"
pkginfo_path.parent.mkdir()
with pkginfo_path.open("w", encoding="utf-8") as fh:
fh.write("Name: my_package\n")
fh.write("Version: 1.0\n")
collector = OxidizedResourceCollector(allowed_locations=["in-memory"])
for r in find_resources_in_path(self.td):
collector.add_in_memory(r)
f = OxidizedFinder()
f.add_resources(
collector.oxidize(
python_exe=os.environ.get("PYTHON_SYS_EXECUTABLE"))[0])
dists = list(f.find_distributions())
self.assertEqual(len(dists), 1)
metadata = dists[0].metadata
self.assertIsInstance(metadata, email.message.Message)
self.assertEqual(metadata["Name"], "my_package")
self.assertEqual(metadata["Version"], "1.0")
def _finder_from_td(self):
collector = OxidizedResourceCollector(policy="in-memory-only")
for r in find_resources_in_path(self.td):
collector.add_in_memory(r)
f = OxidizedFinder()
f.add_resources(collector.oxidize()[0])
return f
def _finder_from_td(self):
collector = OxidizedResourceCollector(allowed_locations=["in-memory"])
for r in find_resources_in_path(self.td):
collector.add_in_memory(r)
f = OxidizedFinder()
f.add_resources(collector.oxidize()[0])
return f
def _finder_from_td(self):
collector = OxidizedResourceCollector(allowed_locations=["in-memory"])
for r in find_resources_in_path(self.td):
collector.add_in_memory(r)
f = OxidizedFinder()
f.add_resources(
collector.oxidize(
python_exe=os.environ.get("PYTHON_SYS_EXECUTABLE"))[0])
return f
def test_source_file(self):
source_path = self.td / "foo.py"
with source_path.open("wb") as fh:
fh.write(b"import io\n")
resources = find_resources_in_path(self.td)
self.assertEqual(len(resources), 1)
r = resources[0]
self.assertIsInstance(r, PythonModuleSource)
self.assertEqual(r.module, "foo")
self.assertEqual(r.source, b"import io\n")
self.assertFalse(r.is_package)
def test_extension_module(self):
suffix = importlib.machinery.EXTENSION_SUFFIXES[0]
path = self.td / ("foo%s" % suffix)
with path.open("wb") as fh:
fh.write(b"dummy")
resources = find_resources_in_path(self.td)
self.assertEqual(len(resources), 1)
r = resources[0]
self.assertIsInstance(r, PythonExtensionModule)
self.assertEqual(r.name, "foo")
def get_resources_data(self) -> bytes:
c = OxidizedResourceCollector(allowed_locations=["in-memory"])
for path in sys.path:
if os.path.isdir(path):
for i, resource in enumerate(find_resources_in_path(path)):
c.add_in_memory(resource)
if i == 10:
break
finder = OxidizedFinder()
finder.add_resources(c.oxidize()[0])
return finder.serialize_indexed_resources()
def test_bytecode_file(self):
cache_tag = sys.implementation.cache_tag
path = self.td / "__pycache__" / ("foo.%s.pyc" % cache_tag)
path.parent.mkdir()
with path.open("wb") as fh:
# First 16 bytes are a header, which gets stripped.
fh.write(b"0123456789abcdefbytecode")
resources = find_resources_in_path(self.td)
self.assertEqual(len(resources), 1)
r = resources[0]
self.assertIsInstance(r, PythonModuleBytecode)
self.assertEqual(r.module, "foo")
self.assertEqual(r.bytecode, b"bytecode")
self.assertEqual(r.optimize_level, 0)
self.assertFalse(r.is_package)
def test_source_module(self):
c = OxidizedResourceCollector(policy="in-memory-only")
source_path = self.td / "foo.py"
with source_path.open("wb") as fh:
fh.write(b"import io\n")
for resource in find_resources_in_path(self.td):
c.add_in_memory(resource)
f = OxidizedFinder()
f.add_resources(c.oxidize()[0])
resources = [r for r in f.indexed_resources() if r.name == "foo"]
self.assertEqual(len(resources), 1)
r = resources[0]
self.assertEqual(r.in_memory_source, b"import io\n")
def test_source_module(self):
c = OxidizedResourceCollector(allowed_locations=["in-memory"])
source_path = self.td / "foo.py"
with source_path.open("wb") as fh:
fh.write(b"import io\n")
for resource in find_resources_in_path(self.td):
c.add_in_memory(resource)
f = OxidizedFinder()
python_exe = os.environ.get("PYTHON_SYS_EXECUTABLE")
with assert_tempfile_cleaned_up():
oxide = c.oxidize(python_exe=python_exe)
f.add_resources(oxide[0])
resources = [r for r in f.indexed_resources() if r.name == "foo"]
self.assertEqual(len(resources), 1)
r = resources[0]
self.assertEqual(r.in_memory_source, b"import io\n")
def test_load_pkg_info(self):
# In absence of a METADATA file, a PKG-INFO file will be read.
pkginfo_path = self.td / "my_package-1.0.egg-info" / "PKG-INFO"
pkginfo_path.parent.mkdir()
with pkginfo_path.open("w", encoding="utf-8") as fh:
fh.write("Name: my_package\n")
fh.write("Version: 1.0\n")
collector = OxidizedResourceCollector(policy="in-memory-only")
for r in find_resources_in_path(self.td):
collector.add_in_memory(r)
f = OxidizedFinder()
f.add_resources(collector.oxidize()[0])
dists = f.find_distributions()
self.assertEqual(len(dists), 1)
metadata = dists[0].metadata
self.assertIsInstance(metadata, email.message.Message)
self.assertEqual(metadata["Name"], "my_package")
self.assertEqual(metadata["Version"], "1.0")
def test_add_sys_path(self):
c = OxidizedResourceCollector(
policy="prefer-in-memory-fallback-filesystem-relative:prefix")
for path in sys.path:
if os.path.isdir(path):
for resource in find_resources_in_path(path):
c.add_in_memory(resource)
c.add_filesystem_relative("", resource)
resources, file_installs = c.oxidize()
f = OxidizedFinder()
f.add_resources(resources)
with (self.td / "serialized").open("wb") as fh:
fh.write(f.serialize_indexed_resources())
f = OxidizedFinder(resources_file=self.td / "serialized")
self.assertGreaterEqual(len(f.indexed_resources()), len(resources))
for r in f.indexed_resources():
r.in_memory_source
r.in_memory_bytecode
def test_package_resource(self):
init_py = self.td / "package" / "__init__.py"
init_py.parent.mkdir()
with init_py.open("wb"):
pass
resource = self.td / "package" / "resource.txt"
with resource.open("wb") as fh:
fh.write(b"resource file")
resources = find_resources_in_path(self.td)
self.assertEqual(len(resources), 2)
r = resources[0]
self.assertIsInstance(r, PythonModuleSource)
self.assertEqual(r.module, "package")
self.assertTrue(r.is_package)
r = resources[1]
self.assertIsInstance(r, PythonPackageResource)
self.assertEqual(r.package, "package")
self.assertEqual(r.name, "resource.txt")
self.assertEqual(r.data, b"resource file")
def test_scan_missing(self):
with self.assertRaisesRegex(ValueError, "path is not a directory"):
find_resources_in_path(self.td / "missing")
def scan_oxidized(path):
oxidized_importer.find_resources_in_path(path)
def test_scan_sys_path(self):
for path in sys.path:
if os.path.isdir(path):
find_resources_in_path(path)