def test_namespaces_cant_have_resources(self):
contents = set(resources.contents(
'test.test_importlib.data03.namespace'))
self.assertEqual(len(contents), 0)
# Even though there is a file in the namespace directory, it is not
# considered a resource, since namespace packages can't have them.
self.assertFalse(resources.is_resource(
'test.test_importlib.data03.namespace',
'resource1.txt'))
# We should get an exception if we try to read it or open it.
self.assertRaises(
FileNotFoundError,
resources.open_text,
'test.test_importlib.data03.namespace', 'resource1.txt')
self.assertRaises(
FileNotFoundError,
resources.open_binary,
'test.test_importlib.data03.namespace', 'resource1.txt')
self.assertRaises(
FileNotFoundError,
resources.read_text,
'test.test_importlib.data03.namespace', 'resource1.txt')
self.assertRaises(
FileNotFoundError,
resources.read_binary,
'test.test_importlib.data03.namespace', 'resource1.txt')
def test_package_has_no_reader_fallback(self):
# Test odd ball packages which:
# 1. Do not have a ResourceReader as a loader
# 2. Are not on the file system
# 3. Are not in a zip file
module = util.create_package(
file=data01, path=data01.__file__, contents=['A', 'B', 'C'])
# Give the module a dummy loader.
module.__loader__ = object()
# Give the module a dummy origin.
module.__file__ = '/path/which/shall/not/be/named'
if sys.version_info >= (3,):
module.__spec__.loader = module.__loader__
module.__spec__.origin = module.__file__
self.assertFalse(resources.is_resource(module, 'A'))
def test_package_has_no_reader_fallback(self):
# Test odd ball packages which:
# 1. Do not have a ResourceReader as a loader
# 2. Are not on the file system
# 3. Are not in a zip file
module = util.create_package(
file=data01, path=data01.__file__, contents=['A', 'B', 'C'])
# Give the module a dummy loader.
module.__loader__ = object()
# Give the module a dummy origin.
module.__file__ = '/path/which/shall/not/be/named'
if sys.version_info >= (3,):
module.__spec__.loader = module.__loader__
module.__spec__.origin = module.__file__
self.assertFalse(resources.is_resource(module, 'A'))
def _valid_key_cacheable(tzpath, key):
assert isinstance(tzpath, tuple) # zoneinfo changed, better update this function!
for root in tzpath:
if os.path.exists(os.path.join(root, key)): # pragma: no branch
# No branch because most systems only have one TZPATH component.
return True
else: # pragma: no cover
# This branch is only taken for names which are known to zoneinfo
# but not present on the filesystem, i.e. on Windows with tzdata,
# and so is never executed by our coverage tests.
*package_loc, resource_name = key.split("/")
package = "tzdata.zoneinfo." + ".".join(package_loc)
try:
return is_resource(package, resource_name)
except ModuleNotFoundError:
return False
def test(self):
expect = resources.read_text(
package=__package__,
resource=_EXPECTED_RESULT_FILENAME,
)
variables = None
if resources.is_resource(package=__package__,
name=_TEMPLATE_VARIABLES_FILENAME):
variables = yaml_parse(data=resources.read_text(
package=__package__,
resource=_TEMPLATE_VARIABLES_FILENAME,
), )
actual = render_role_template(
role_path=_ROLE_PATH,
template_filename=_TEMPLATE_FILENAME,
variables=variables,
)
self.assertEqual(expect, actual)
def note_card_paths() -> List[str]:
"""
Returns a list of strings representing filesystem paths
to PNG images of musical notes.
:return: List[str]
"""
pkg = "music_flash_cards.cards.chromatic_note_cards"
note_cards = []
for note_card_name in contents(pkg):
if is_resource(pkg,
note_card_name) and note_card_name.endswith(".png"):
with path(pkg, note_card_name) as note_card_path:
note_cards.append(str(note_card_path))
return note_cards
def run(self):
"""See `Task.run`"""
if not pkg_resources.is_resource(xresources, self.resource_name):
self._error("Resource '{}' does not exist".format(
self.resource_name))
return False
with pkg_resources.path(xresources, self.resource_name) as path:
self._info("Copying resources '{}' to {}".format(
self.resource_name, self.destination_path))
try:
shutil.copyfile(path, self.destination_path)
except Exception:
self._exception("An error occured during the copy")
return False
return True
def copy_resource_folder(package: Package,
destination_path: str,
exclude: List[str] = None):
"""
Copies the full content of provided package in destination folder.
Names of files that should not be copied have to be listed in `exclude`.
.. Warning ::
As the resources in the folder are discovered by browsing
through the folder, they are not explicitly listed in the Python code.
Therefore, to have the install process run smoothly, these resources need
to be listed in the MANIFEST.in file.
:param package: name of resource package to copy
:param destination_path: file system path of destination
:param exclude: list of item names that should not be copied
"""
exclusion_list = ["__pycache__"]
if exclude:
exclusion_list += exclude
for resource_name in contents(package):
if resource_name in exclusion_list:
continue
if is_resource(package, resource_name):
destination_file_path = pth.join(destination_path, resource_name)
copy_resource(package, resource_name, destination_file_path)
else:
# In case of subfolders that are only declared in MANIFEST.in,
# getattr(package, "resource_name") will fail (is there another way?).
# So we fall back to using package name as as string.
if isinstance(package, ModuleType):
package_name = package.__name__
else: # str
package_name = package
new_package_name = ".".join([package_name, resource_name])
new_destination_path = pth.join(destination_path, resource_name)
copy_resource_folder(new_package_name,
new_destination_path,
exclude=exclude)
def test_loader(loader_func, data_shape, target_shape, n_target, has_descr,
filenames):
bunch = loader_func()
assert isinstance(bunch, Bunch)
assert bunch.data.shape == data_shape
assert bunch.target.shape == target_shape
if hasattr(bunch, "feature_names"):
assert len(bunch.feature_names) == data_shape[1]
if n_target is not None:
assert len(bunch.target_names) == n_target
if has_descr:
assert bunch.DESCR
if filenames:
assert "data_module" in bunch
assert all([
f in bunch
and resources.is_resource(bunch["data_module"], bunch[f])
for f in filenames
])
def import_submodules(package):
"""Import all the submodules of a given package."""
try:
import_module(package)
entries = contents(package)
except Exception: # pylint: disable = broad-except
return
for name in entries:
if name.startswith('_'):
continue
if is_resource(package, name):
if name.endswith('.py'):
import_submodules(f'{package}.{name[:-3]}')
else:
try:
contents(f'{package}.{name}')
except TypeError:
continue
import_submodules(f'{package}.{name}')
def test_namespaces_cannot_have_resources(self):
contents = resources.contents('test.test_importlib.data03.namespace')
self.assertFalse(list(contents))
# Even though there is a file in the namespace directory, it is not
# considered a resource, since namespace packages can't have them.
self.assertFalse(
resources.is_resource('test.test_importlib.data03.namespace',
'resource1.txt'))
# We should get an exception if we try to read it or open it.
self.assertRaises(FileNotFoundError, resources.open_text,
'test.test_importlib.data03.namespace',
'resource1.txt')
self.assertRaises(FileNotFoundError, resources.open_binary,
'test.test_importlib.data03.namespace',
'resource1.txt')
self.assertRaises(FileNotFoundError, resources.read_text,
'test.test_importlib.data03.namespace',
'resource1.txt')
self.assertRaises(FileNotFoundError, resources.read_binary,
'test.test_importlib.data03.namespace',
'resource1.txt')
def load_schema(self):
schema = {}
try:
pkg_name = f'villas.controller.schemas.{self.category}.{self.type}'
pkg = importlib.import_module(pkg_name)
except ModuleNotFoundError:
self.logger.warn('Missing schemas!')
return schema
for res in resources.contents(pkg):
name, ext = os.path.splitext(res)
if resources.is_resource(pkg, res) and ext in ['.yaml', '.json']:
fo = resources.open_text(pkg, res)
loadedschema = yaml.load(fo, yaml.SafeLoader)
schema[name] = Draft202012Validator(loadedschema)
return schema
def load_stopwords(lang: str) -> Set[str]:
"""Load stopwords set.
Args:
lang (str): language, one of {"ru", }
Raises:
ValueError: if there is no list of stop words.
Returns:
Set[str]: set of stopwords.
"""
lang_file = f"{lang}.txt"
if not pkg_resources.is_resource(boosearch.resources.stopwords, lang_file):
raise ValueError(f"Stopwords list `{lang}` not founded")
raw_stopwords = pkg_resources.read_text(
boosearch.resources.stopwords, lang_file
)
stopwords = set(word.lower().strip() for word in raw_stopwords.split("\n"))
return stopwords
def load_configuration(descriptor):
""" Load configuration from the given descriptor. Could be
either a `spleeter:` prefixed embedded configuration name
or a file system path to read configuration from.
:param descriptor: Configuration descriptor to use for lookup.
:returns: Loaded description as dict.
:raise ValueError: If required embedded configuration does not exists.
:raise IOError: If required configuration file does not exists.
"""
# Embedded configuration reading.
if descriptor.startswith(_EMBEDDED_CONFIGURATION_PREFIX):
name = descriptor[len(_EMBEDDED_CONFIGURATION_PREFIX):]
if not loader.is_resource(resources, f'{name}.json'):
raise ValueError(f'No embedded configuration {name} found')
with loader.open_text(resources, f'{name}.json') as stream:
return json.load(stream)
# Standard file reading.
if not exists(descriptor):
raise IOError(f'Configuration file {descriptor} not found')
with open(descriptor, 'r') as stream:
return json.load(stream)
def load_config(cfg_file: Path = None):
global cfg_file_location
if cfg_file is not None:
cfg_file_location = cfg_file
return MasterConfig(**json.load(cfg_file.open()))
# Try to find a configuration file. First check the directory where the script is, then check the working directory, then the user directory
# Along with creating the configuration, also set the save location for the config file when the program exits
if resources.is_resource('DailyData', 'config.json'):
cfg_file_location = None
return MasterConfig(
**json.loads(resources.read_text(__name__, 'config.json')))
elif cwd_config.exists():
cfg_file_location = cwd_config
return MasterConfig(**json.load(cwd_config.open()))
elif usr_config.exists():
cfg_file_location = usr_config
return MasterConfig(**json.load(usr_config.open()))
else:
# Create default configuration if one doesn't exist
cfg_file_location = usr_config
return initial_setup()
def test_is_submodule_resource(self):
submodule = import_module('ziptestdata.subdirectory')
self.assertTrue(resources.is_resource(submodule, 'binary.file'))
def test_read_submodule_resource_by_name(self):
self.assertTrue(
resources.is_resource('ziptestdata.subdirectory', 'binary.file'))
import os
import json
try:
# python 3.7+
import importlib.resources as config_loader
except ImportError:
# python <= 3.6
import importlib_resources as config_loader
# get the configuration
config_name = os.environ.get('PROGRAM_ARGS', 'wm-sasl-example.json')
if not config_loader.is_resource('pyproducer.resources.env', config_name):
if not config_name.endswith('.json'):
config_name += '.json'
config = json.loads(
config_loader.read_text('pyproducer.resources.env', config_name)
)
def replace_envs(conf, composite_key=''):
for key, val in conf.items():
new_composite_key = f'{composite_key}.{key}' if composite_key else key
if isinstance(val, dict):
replace_envs(val, new_composite_key)
else:
if val == '_env_':
conf[key] = os.environ[new_composite_key.replace('.', '_').upper()]
try:
def test_read_submodule_resource_by_name(self):
self.assertTrue(
resources.is_resource('ziptestdata.subdirectory', 'binary.file'))
def test_is_resource_missing(self):
self.assertFalse(resources.is_resource(self.data, 'not-a-file'))
def test_resource_missing_is_not_resource(self):
package = util.create_package(file=data01,
path=data01.__file__,
contents=['A', 'B', 'C', 'D/E', 'D/F'])
self.assertFalse(resources.is_resource(package, 'Z'))
def test_is_resource_missing(package, resource):
assert not ir.is_resource(package, resource)
def test_is_resource_failure_does_not_keep_open(self):
c = resources.is_resource('ziptestdata', 'not-present')
self.zip_path.unlink()
del c
def test_resource_missing_is_not_resource(self):
package = util.create_package(
file=data01, path=data01.__file__,
contents=['A', 'B', 'C', 'D/E', 'D/F'])
self.assertFalse(resources.is_resource(package, 'Z'))
def test_is_resource_subresource_directory():
# Directories are not resources.
assert not ir.is_resource("resources", "submodule")
suffix_re = re.compile(r"\.sage$")
prefix_re = re.compile(r"^\d{2}_")
def attack_name(s):
if isinstance(s, Path):
s = s.stem
else:
s = suffix_re.sub("", s)
return prefix_re.sub("", s).replace("\n", " ")
builtin = {
attack_name(res): res
for res in sorted(contents(builtin_attacks))
if is_resource(builtin_attacks, res) and res.endswith(".sage")
}
installed = {
attack_name(f): f
for d in reversed(path) for f in sorted(d.glob("*.sage")) if f.is_file()
}
def attack_path(name):
try:
return installed[name]
except KeyError:
pass
try:
return resources.path(builtin_attacks, builtin[name])
def test_is_resource_missing(self):
self.assertFalse(resources.is_resource(self.data, 'not-a-file'))
def test_is_resource_good_path(self):
self.assertTrue(resources.is_resource(self.data, 'binary.file'))
def test_is_submodule_resource(self):
submodule = import_module('ziptestdata.subdirectory')
self.assertTrue(
resources.is_resource(submodule, 'binary.file'))
def test_is_resource_good_path(package, resource):
assert ir.is_resource(package, resource)
def test_is_resource_does_not_keep_open(self):
c = resources.is_resource('ziptestdata', 'binary.file')
self.zip_path.unlink()
del c
def is_resource(package, name):
return resources.is_resource( # pytype: disable=module-attr
pycompat.sysstr(package), encoding.strfromlocal(name))
def test_is_resource_good_path(self):
self.assertTrue(resources.is_resource(self.data, 'binary.file'))
def create_archive(sources: List[Path],
target: Path,
interpreter: str,
main: str,
env: Environment,
compressed: bool = True) -> None:
"""Create an application archive from SOURCE.
This function is a heavily modified version of stdlib's
`zipapp.create_archive <https://docs.python.org/3/library/zipapp.html#zipapp.create_archive>`_
"""
# Check that main has the right format.
mod, sep, fn = main.partition(":")
mod_ok = all(part.isidentifier() for part in mod.split("."))
fn_ok = all(part.isidentifier() for part in fn.split("."))
if not (sep == ":" and mod_ok and fn_ok):
raise zipapp.ZipAppError("Invalid entry point: " + main)
# Collect our timestamp data
main_py = MAIN_TEMPLATE.format(module=mod, fn=fn)
timestamp = datetime.strptime(
env.built_at,
BUILD_AT_TIMESTAMP_FORMAT).replace(tzinfo=timezone.utc).timestamp()
zipinfo_datetime: Tuple[int, int, int, int, int,
int] = time.gmtime(int(timestamp))[0:6]
with target.open(mode="wb") as fd:
# Write shebang.
write_file_prefix(fd, interpreter)
# Determine compression.
compression = zipfile.ZIP_DEFLATED if compressed else zipfile.ZIP_STORED
# Pack zipapp with dependencies.
with zipfile.ZipFile(fd, "w", compression=compression) as archive:
site_packages = Path("site-packages")
contents_hash = hashlib.sha256()
for source in sources:
# Glob is known to return results in non-deterministic order.
# We need to sort them by in-archive paths to ensure
# that archive contents are reproducible.
for path in sorted(source.rglob("*"), key=str):
# Skip compiled files and directories (as they are not required to be present in the zip).
if path.suffix == ".pyc" or path.is_dir():
continue
data = path.read_bytes()
# update the contents hash
contents_hash.update(data)
arcname = str(site_packages / path.relative_to(source))
write_to_zipapp(archive,
arcname,
data,
zipinfo_datetime,
compression,
stat=path.stat())
if env.build_id is None:
# Now that we have a hash of all the source files, use it as our build id if the user did not
# specify a custom one.
env.build_id = contents_hash.hexdigest()
# now let's add the shiv bootstrap code.
bootstrap_target = Path("_bootstrap")
for bootstrap_file in importlib_resources.contents(
bootstrap): # type: ignore
if importlib_resources.is_resource(
bootstrap, bootstrap_file): # type: ignore
with importlib_resources.path(
bootstrap, bootstrap_file) as path: # type: ignore
data = path.read_bytes()
write_to_zipapp(
archive,
str(bootstrap_target / path.name),
data,
zipinfo_datetime,
compression,
stat=path.stat(),
)
# Write environment info in json file.
#
# The environment file contains build_id which is a SHA-256 checksum of all **site-packages** contents.
# the bootstarp code, environment.json and __main__.py are not used to calculate the checksum, is it's
# only used for local caching of site-packages and these files are always read from archive.
write_to_zipapp(archive, "environment.json",
env.to_json().encode("utf-8"), zipinfo_datetime,
compression)
# write __main__
write_to_zipapp(archive, "__main__.py", main_py.encode("utf-8"),
zipinfo_datetime, compression)
# Make pyz executable (on windows this is no-op).
target.chmod(target.stat().st_mode | S_IXUSR | S_IXGRP | S_IXOTH)
def test_is_resource_subresource_directory(self):
# Directories are not resources.
self.assertFalse(resources.is_resource(self.data, 'subdirectory'))
def test_importlib(self):
self.assertTrue(
resources.is_resource("datapackage2.subdir", "data.txt"))
self.assertTrue(resources.is_resource("zfdatapackage", "data.txt"))
def test_is_resource_subresource_directory(self):
# Directories are not resources.
self.assertFalse(resources.is_resource(self.data, 'subdirectory'))
def is_resource(package, name):
return resources.is_resource(pycompat.sysstr(package),
encoding.strfromlocal(name))
