def test_importlib_resources(self):
# App ZIP
pkg = "android1"
names = ["subdir", "__init__.py", "a.txt", "b.so", "mod1.py"]
self.assertCountEqual(names, resources.contents(pkg))
for name in names:
with self.subTest(name=name):
self.assertEqual(resources.is_resource(pkg, name),
name != "subdir")
self.check_ir_resource(APP_ZIP, pkg, "__init__.py",
b"# This package is")
self.check_ir_resource(APP_ZIP, pkg, "a.txt", b"alpha\n")
self.check_ir_resource(APP_ZIP, pkg, "b.so", b"bravo\n")
self.assertFalse(resources.is_resource(pkg, "invalid.py"))
with self.assertRaisesRegex(FileNotFoundError, "invalid.py"):
resources.read_binary(pkg, "invalid.py")
with self.assertRaisesRegex(FileNotFoundError, "invalid.py"):
with resources.path(pkg, "invalid.py"):
pass
# Requirements ZIP
self.reset_package("murmurhash")
self.assertCountEqual([
"include", "tests", "__init__.pxd", "__init__.pyc", "about.pyc",
"mrmr.pxd", "mrmr.pyx", "mrmr.so"
], resources.contents("murmurhash"))
self.check_ir_resource(REQS_COMMON_ZIP, "murmurhash", "__init__.pyc",
MAGIC_NUMBER)
self.check_ir_resource(REQS_COMMON_ZIP, "murmurhash", "mrmr.pxd",
b"from libc.stdint")
self.check_ir_resource(REQS_ABI_ZIP, "murmurhash", "mrmr.so",
b"\x7fELF")
def add_static_paths(app):
"""Ensure CSS/JS is loaded."""
app.env.book_theme_resources_changed = False
output_static_folder = Path(app.outdir) / "_static"
theme_static_files = resources.contents(theme_static)
if (app.config.html_theme_options.get("theme_dev_mode", False)
and output_static_folder.exists()):
# during development, the JS/CSS may change, if this is the case,
# we want to remove the old files and ensure that the new files are loaded
for path in output_static_folder.glob("sphinx-book-theme*"):
if path.name not in theme_static_files:
app.env.book_theme_resources_changed = True
path.unlink()
# note sphinx treats theme css different to regular css
# (it is specified in theme.conf), so we don't directly use app.add_css_file
for fname in resources.contents(theme_static):
if fname.endswith(".css"):
if not (output_static_folder / fname).exists():
(output_static_folder / fname).write_bytes(
resources.read_binary(theme_static, fname))
app.env.book_theme_resources_changed = True
# add javascript
for fname in resources.contents(theme_static):
if fname.endswith(".js"):
app.add_js_file(fname)
def test_unrelated_contents(self):
"""
Test thata zip with two unrelated subpackages return
distinct resources. Ref python/importlib_resources#44.
"""
self.assertEqual(set(resources.contents('ziptestdata.one')),
{'__init__.py', 'resource1.txt'})
self.assertEqual(set(resources.contents('ziptestdata.two')),
{'__init__.py', 'resource2.txt'})
def test_unrelated_contents(self):
# https://gitlab.com/python-devs/importlib_resources/issues/44
#
# Here we have a zip file with two unrelated subpackages. The bug
# reports that getting the contents of a resource returns unrelated
# files.
self.assertEqual(set(resources.contents('ziptestdata.one')),
{'__init__.py', 'resource1.txt'})
self.assertEqual(set(resources.contents('ziptestdata.two')),
{'__init__.py', 'resource2.txt'})
def test_unrelated_contents(self):
# https://gitlab.com/python-devs/importlib_resources/issues/44
#
# Here we have a zip file with two unrelated subpackages. The bug
# reports that getting the contents of a resource returns unrelated
# files.
self.assertEqual(
set(resources.contents('ziptestdata.one')),
{'__init__.py', 'resource1.txt'})
self.assertEqual(
set(resources.contents('ziptestdata.two')),
{'__init__.py', 'resource2.txt'})
def update_web_pages(root_dir):
for resource in resources.contents(templates):
if resource.endswith('.html') or resource.endswith('.js') or resource.endswith('.hbs'):
if resource == 'index.html' and (root_dir / resource).exists():
continue
with resources.path(templates, resource) as file_path:
shutil.copy(file_path, root_dir / resource)
def iter_contents(cls,
module, # type: str
filter, # type: typing.Callable[[str, str], bool]
):
# type: (...) -> typing.Generator[typing.Tuple[str, str], None, None]
"""
Iterate resources from given module, recursively.
Only resources matching any of given given pattern will be
returned.
:param module: module name where resources will be search on
:type module: str
:param filter: resource filtering function
:type filter: typing.Callable[[str, str], bool]
:returns: iterator of tuples with module and resource name
:rtype: typing.Iterator[typing.Tuple[str, str], None]
"""
if module is None:
return
try:
for item in res.contents(module):
for subitem in cls.iter_content_items(module, item, filter):
yield subitem
except ImportError:
pass
def copy_resource(resource, destination, force):
"""Copy the content of one resource directory
Args:
resource (str): resource to copy.
destination (path.Path): directory where to copy the resource.
force (bool): if the destination exists and this flag is set to True,
overwrite the destination.
"""
if not force and destination.exists():
try:
result = strtobool(
input(f"Directory {destination} already exists, "
"overwrite it with its content? [y/N] "))
except ValueError:
result = False
if not result:
return
destination.makedirs_p()
for file_name in contents(resource):
# ignore Python files
if file_name.startswith("__"):
continue
with path(resource, file_name) as file:
Path(file).copy(destination)
def get_testfiles() -> list:
"""Returns the available test files.
Parameters
----------
None
Returns
-------
:
List with available test files.
Example
-------
>>> from araucaria.testdata import get_testfiles
>>> testfiles = get_testfiles()
>>> for file in testfiles:
... if 'dnd' in file:
... print(file)
dnd_glitchfile.dat
dnd_testfile1.dat
dnd_testfile2.dat
dnd_testfile3.dat
"""
olist = list(pkg_resources.contents(testdata))
clist = copy(olist)
for item in olist:
if ('__' in item) or ('utils' in item):
clist.remove(item)
clist.sort()
return (clist)
def get_metadata(app: str) -> Optional[MetaData]:
meta = []
control = []
if callable(MetaData):
metadata = MetaData()
for name in resources.contents(app):
if name == "models.py":
_module = import_module(f"{app}.{name[:-3]}")
for model in dir(_module):
table = getattr(_module, model)
if hasattr(table, "metadata") and hasattr(table, "database"):
table_module = getattr(table, "__module__", None)
if (
table_module == _module.__name__
and table.database == os.environ["CRAX_DB_NAME"]
):
meta.append(table)
control.append(table.table.name)
for base in meta:
for (table_name, table) in base.metadata.tables.items():
if table_name in control:
metadata._add_table(table_name, table.schema, table)
return metadata
def apply_migrations(engine: EngineStrategy) -> None:
Base.metadata.create_all(bind=engine, tables=[Version.__table__], checkfirst=True)
db: Session = SessionLocal()
current_version = db.query(Version).first()
if current_version is None:
current_version = Version(version="v000")
migrations = resources.contents("escarpolette.migrations")
for migration in sorted(migrations):
if not migration.endswith(".sql"):
continue
migration_version, _ = migration.split(".")
if migration_version <= current_version.version:
continue
sql = resources.read_text("escarpolette.migrations", migration)
queries = sql.split(";")
for query in queries:
db.execute(query)
current_version.version, _ = migration.split(".")
db.add(current_version)
db.commit()
def _copy_css(self):
css_path = self._configdir / "css"
if not css_path.exists():
css_path.mkdir()
for r in pkg_resources.contents(data):
if r.endswith(".css"):
(css_path / r).write_text(pkg_resources.read_text(data, r))
def list_suites(cores=None, verbose=False):
if cores is None:
cores = [mpas_core.name for mpas_core in get_mpas_cores()]
print('Suites:')
for core in cores:
try:
suites = contents('compass.{}.suites'.format(core))
except FileNotFoundError:
continue
for suite in sorted(suites):
print_header_if_cached = True
if suite.endswith('.txt'):
print(' -c {} -t {}'.format(core, os.path.splitext(suite)[0]))
if verbose:
text = resources.read_text(
'compass.{}.suites'.format(core), suite)
tests = list()
for test in text.split('\n'):
test = test.strip()
if test == 'cached':
print('\t Cached steps: all')
elif test.startswith('cached'):
if print_header_if_cached:
print('\t Cached steps:')
# don't print the header again if there are
# multiple lines of cached steps
print_header_if_cached = False
print(test.replace('cached: ', '\t '))
elif (len(test) > 0 and test not in tests
and not test.startswith('#')):
print("\t* {}".format(test))
print_header_if_cached = True
def _install_python_package(
self, package_name: str) -> Tuple[Set[Bundle], Set[str]]:
"""
Recursively loads indicated package.
:param package_name:
:return: A 2-tuple, with the list of installed bundles and the list
of failed modules names
"""
bundles = set()
failed = set()
try:
package_contents = list(contents(package_name))
except (TypeError, ModuleNotFoundError):
if package_name.endswith(".py"):
try:
bundle = self.context.install_bundle(package_name[:-3])
bundles.add(bundle)
except BundleException:
failed.add(package_name)
return bundles, failed
for item in package_contents:
sub_bundles, sub_failed = self._install_python_package(".".join(
[package_name, item]))
bundles.update(sub_bundles)
failed.update(sub_failed)
return bundles, failed
def test_namespaces_cant_have_resources(self):
contents = set(resources.contents(
'test.test_importlib.data03.namespace'))
self.assertEqual(len(contents), 0)
# Even though there is a file in the namespace directory, it is not
# considered a resource, since namespace packages can't have them.
self.assertFalse(resources.is_resource(
'test.test_importlib.data03.namespace',
'resource1.txt'))
# We should get an exception if we try to read it or open it.
self.assertRaises(
FileNotFoundError,
resources.open_text,
'test.test_importlib.data03.namespace', 'resource1.txt')
self.assertRaises(
FileNotFoundError,
resources.open_binary,
'test.test_importlib.data03.namespace', 'resource1.txt')
self.assertRaises(
FileNotFoundError,
resources.read_text,
'test.test_importlib.data03.namespace', 'resource1.txt')
self.assertRaises(
FileNotFoundError,
resources.read_binary,
'test.test_importlib.data03.namespace', 'resource1.txt')
def list_locales():
"""Return a list of available locales in xclim."""
locale_list = contents("xclim.data")
return [
locale.split(".")[0] for locale in locale_list
if locale.endswith(".json")
]
def get_available_layouts(size='normal'):
"""Return the names of the built-in layouts.
Parameters
----------
size : str
only return layouts of size 'small', 'normal', 'big', 'all'.
Default is 'normal'.
'small' -> width=16, height=8, food=10
'normal' -> width=32, height=16, food=30
'big' -> width=64, height=32, food=60
'all' -> all of the above
Returns
-------
layout_names : list of str
the available layouts
"""
# loop in layouts directory and look for layout files
valid = ('small', 'normal', 'big', 'all')
if size not in valid:
raise ValueError(
f"Invalid layout size ('{size}' given). Valid: {valid}")
if size == 'all':
size = ''
return [
item[:-(len('.layout'))]
for item in importlib_resources.contents('pelita._layouts')
if item.endswith('.layout') and size in item
]
def test_namespaces_cannot_have_resources(self):
contents = resources.contents('test.test_importlib.data03.namespace')
self.assertFalse(list(contents))
# Even though there is a file in the namespace directory, it is not
# considered a resource, since namespace packages can't have them.
self.assertFalse(resources.is_resource(
'test.test_importlib.data03.namespace',
'resource1.txt'))
# We should get an exception if we try to read it or open it.
self.assertRaises(
FileNotFoundError,
resources.open_text,
'test.test_importlib.data03.namespace', 'resource1.txt')
self.assertRaises(
FileNotFoundError,
resources.open_binary,
'test.test_importlib.data03.namespace', 'resource1.txt')
self.assertRaises(
FileNotFoundError,
resources.read_text,
'test.test_importlib.data03.namespace', 'resource1.txt')
self.assertRaises(
FileNotFoundError,
resources.read_binary,
'test.test_importlib.data03.namespace', 'resource1.txt')
def load_builtin_definitions(self) -> None:
"""Load the built-in key definitions from the ``defs`` directory"""
for fname in resources.contents(_defs):
if fname.endswith(".toml") and resources.is_resource(_defs, fname):
logger.debug("Loading defs from %s", fname)
with resources.open_text(_defs, fname) as file:
self.attach_defs(toml.load(file))
def __init__(self):
""" Parse Config files """
self.name = "default builder"
for name in resources.contents(__name__):
if name == type(self).__name__ + ".conf":
self.config = configparser.ConfigParser()
self.config.read('builders/' + name)
def read_variable_descriptions(cls,
file_parent: str,
update_existing: bool = True):
"""
Reads variable descriptions in indicated folder or package, if it contains some.
The file variable_descriptions.txt is looked for. Nothing is done if it is not
found (no error raised also).
Each line of the file should be formatted like::
my:variable||The description of my:variable, as long as needed, but on one line.
:param file_parent: the folder path or the package name that should contain the file
:param update_existing: if True, previous descriptions will be updated.
if False, previous descriptions will be erased.
"""
if not update_existing:
cls._variable_descriptions = {}
cls._loaded_descriptions = set()
variable_descriptions = None
description_file = None
if pth.isdir(file_parent):
file_path = pth.join(file_parent, DESCRIPTION_FILENAME)
if pth.isfile(file_path):
description_file = open(file_path)
else:
# Then it is a module name
if DESCRIPTION_FILENAME in contents(file_parent):
description_file = open_text(file_parent, DESCRIPTION_FILENAME)
if description_file is not None:
try:
variable_descriptions = np.genfromtxt(description_file,
delimiter="||",
dtype=str,
autostrip=True)
except Exception as exc:
# Reading the file is not mandatory, so let's just log the error.
_LOGGER.error(
"Could not read file %s in %s. Error log is:\n%s",
DESCRIPTION_FILENAME,
file_parent,
exc,
)
description_file.close()
if variable_descriptions is not None:
if np.shape(variable_descriptions) == (2, ):
# If the file contains only one line, np.genfromtxt() will return a (2,)-shaped
# array. We need a reshape for dict.update() to work correctly.
variable_descriptions = np.reshape(variable_descriptions,
(1, 2))
cls._loaded_descriptions.add(file_parent)
cls.update_variable_descriptions(variable_descriptions)
_LOGGER.info("Loaded variable descriptions in %s", file_parent)
def test_submodule_contents_by_name(self):
contents = set(resources.contents('namespacedata01'))
try:
contents.remove('__pycache__')
except KeyError:
pass
self.assertEqual(contents,
{'binary.file', 'utf-8.file', 'utf-16.file'})
def _import_all(package: str) -> None:
"""Import all actions in a package"""
files = resources.contents(package) # type: ignore
actions = [
f[:-3] for f in files if f.endswith(".py") and f[0] != "_" and not f.startswith(".#")
]
for action in actions:
_import(package, action)
def iterdir(modulepath):
modname = modulepath.replace('/', '.')
if modname == '__main__':
dirpath = _main_path()
for item in dirpath.iterdir():
yield item.name
else:
yield from impres.contents(modname)
def get_methods():
'''Return list of installed methods'''
contents = pkg_resources.contents(recipes)
methods = []
for i in contents:
if '.cfg' in i:
methods.append(i[0:-4])
return methods
def copy_bootstrap(bootstrap_target: Path) -> None:
"""Copy bootstrap code from shiv into the pyz.
:param bootstrap_target: The temporary directory where we are staging pyz contents.
"""
for bootstrap_file in importlib_resources.contents(bootstrap):
if importlib_resources.is_resource(bootstrap, bootstrap_file):
with importlib_resources.path(bootstrap, bootstrap_file) as f:
shutil.copyfile(f.absolute(), bootstrap_target / f.name)
def copy_resources(package, destination):
os.makedirs(destination, exist_ok=True)
for entry in resources.contents(package):
if not resources.is_resource(package, entry):
continue
with resources.path(package, entry) as resource_path:
shutil.copy2(str(resource_path), destination)
def built_in_yaml_definitions():
package = 'approxeng.input.yaml_controllers'
for item in resources.contents(package):
if item.endswith('.yaml') and resources.is_resource(package, item):
yaml_string = resources.read_text('approxeng.input.yaml_controllers', item)
profile = Profile(d=yaml.load(yaml_string, Loader=yaml.SafeLoader))
controller_class = profile.build_controller_class()
for vendor_id, product_id in controller_class.registration_ids():
yield f'{vendor_id}-{product_id}', controller_class
def update_css(app: Sphinx):
"""Compile SCSS to the build directory."""
# merge user CSS variables with defaults
css_variables = get_default_css_variables()
for key, value in app.config.panels_css_variables.items():
if key not in css_variables:
LOGGER.warning(f"key in 'panels_css_variables' is not recognised: {key}")
else:
css_variables[key] = value
# reset css changed attribute
app.env.panels_css_changed = False
# setup up new static path in output dir
static_path = (Path(app.outdir) / "_panels_static").absolute()
static_path.mkdir(exist_ok=True)
app.config.html_static_path.append(str(static_path))
old_resources = {path.name for path in static_path.glob("*") if path.is_file()}
css_resources = [("spanels-", css_panels)]
if app.config.panels_add_boostrap_css:
css_resources.append(("spanels-bootstrap-", css_bootstrap))
# add new resources
for prefix, module in css_resources:
for filename in resources.contents(module):
if not filename.endswith(".css"):
continue
out_name = prefix + filename
if not (static_path / out_name).exists():
content = resources.read_text(module, filename)
(static_path / out_name).write_text(content)
app.env.panels_css_changed = True
app.add_css_file(prefix + filename)
if out_name in old_resources:
old_resources.remove(out_name)
# add variables CSS file
css_lines = [":root {"]
for name, value in css_variables.items():
css_lines.append(f"--{name}: {value};")
css_lines.append("}")
css_str = "\n".join(css_lines)
css_variables_name = (
f"spanels-variables--{hashlib.md5(css_str.encode('utf8')).hexdigest()}.css"
)
if css_variables_name in old_resources:
old_resources.remove(css_variables_name)
app.env.panels_css_changed = True
(static_path / css_variables_name).write_text(css_str)
app.add_css_file(css_variables_name)
# remove old resources
for name in old_resources:
for path in Path(app.outdir).glob(f"**/{name}"):
path.unlink()
def all_kitten_names() -> FrozenSet[str]:
try:
from importlib.resources import contents
except ImportError:
from importlib_resources import contents # type: ignore
ans = []
for name in contents('kittens'):
if '__' not in name and '.' not in name and name != 'tui':
ans.append(name)
return frozenset(ans)
def mkdct(spkg):
data = {}
for sp in spkg:
data.update({
f.replace(".pkl", ""): load(ir.path(f"{base}.{sp}", f))
for f in ir.contents(f"{base}.{sp}")
if (name is None and f[0:2] != "__") or (
name is not None and f == (name + ".pkl"))
})
return data
def copy_resource_tree(package, dest):
package_name = package.__name__.split(".")[-1]
dest_subdir = Path(dest)/package_name
dest_subdir.mkdir(mode=0o755, exist_ok=True)
for x in resources.contents(package):
if resources.is_resource(package, x):
copy_resource(package, x, dest_subdir)
elif x != "__pycache__":
subpackage = import_module(f".{x}", package.__name__)
copy_resource_tree(subpackage, dest_subdir)
def get_enabled(mod_type: str, package='cslbot') -> Tuple[List[str], List[str]]:
enabled, disabled = [], []
for f in resources.contents("%s.%s" % (package, mod_type)):
if not f.endswith('.py'):
continue
name = basename(f).split('.')[0]
mod_name = "%s.%s.%s" % (package.lower(), mod_type, name)
if group_enabled(mod_type, name):
enabled.append(mod_name)
elif group_disabled(mod_type, name):
disabled.append(mod_name)
elif name != '__init__':
logging.error("%s must be either enabled or disabled in groups.cfg" % mod_name)
# default to disabled
disabled.append(mod_name)
return enabled, disabled
def test_contents(self):
contents = set(resources.contents(self.data))
# There may be cruft in the directory listing of the data directory.
# Under Python 3 we could have a __pycache__ directory, and under
# Python 2 we could have .pyc files. These are both artifacts of the
# test suite importing these modules and writing these caches. They
# aren't germane to this test, so just filter them out.
contents.discard('__pycache__')
contents.discard('__init__.pyc')
contents.discard('__init__.pyo')
self.assertEqual(contents, {
'__init__.py',
'subdirectory',
'utf-8.file',
'binary.file',
'utf-16.file',
})
def test_submodule_contents_by_name(self):
self.assertEqual(
set(resources.contents('ziptestdata.subdirectory')),
{'__init__.py', 'binary.file'})
def test_submodule_contents(self):
submodule = import_module('ziptestdata.subdirectory')
self.assertEqual(
set(resources.contents(submodule)),
{'__init__.py', 'binary.file'})
def _import_solutions():
"""Import all resources to register plug-ins"""
for name in resources.contents(__name__):
if name.endswith(".py"):
import_module(f"{__name__}.{name[:-3]}")
def test_resource_contents(self):
package = util.create_package(
file=data01, path=data01.__file__, contents=['A', 'B', 'C'])
self.assertEqual(
set(resources.contents(package)),
{'A', 'B', 'C'})
