def load_data(data, info=False):
''' Loads a dataset as a Pandas dataframe
Special case data == 'help' the help_text is printed to console
All datasets visible at:
https://github.com/astroDimitrios/astroedu/tree/main/src/astroedu/datasets
Args:
data -- string, name of the csv file to load (without extension)
info -- boolean, default False - if True prints a short description
of the data before loading the dataframe
Returns:
A pandas dataframe
Example:
>>> planets = load_data('planets')
>>> load_data('help')
'''
if data == 'help':
print(help_text)
return
data_csv = data + '.csv'
if not is_resource('astroedu.datasets', data_csv):
raise FileExistsError(f'No data file named {data_csv} found.')
with files('astroedu.datasets').joinpath(data_csv) as p:
if info:
with files('astroedu.datasets').joinpath(data + '.txt') as i:
InfoHandler = open(str(i), 'r')
print(InfoHandler.read())
return pd.read_csv(str(p))
def available(self) -> bool:
try:
ret = importlib_resources.is_resource(self.path, "__init__.py")
assert isinstance(ret, bool)
return ret
except ModuleNotFoundError:
return False
def copy_bootstrap(bootstrap_target: Path) -> None:
"""Copy bootstrap code from shiv into the pyz.
:param bootstrap_target: The temporary directory where we are staging pyz contents.
"""
for bootstrap_file in importlib_resources.contents(bootstrap):
if importlib_resources.is_resource(bootstrap, bootstrap_file):
with importlib_resources.path(bootstrap, bootstrap_file) as f:
shutil.copyfile(f.absolute(), bootstrap_target / f.name)
def copy_bootstrap(bootstrap_target: Path) -> None:
"""Copy bootstrap code from shiv into the pyz.
:param bootstrap_target: The temporary directory where we are staging pyz contents.
"""
for bootstrap_file in importlib_resources.contents(bootstrap):
if importlib_resources.is_resource(bootstrap, bootstrap_file):
with importlib_resources.path(bootstrap, bootstrap_file) as f:
shutil.copyfile(f.absolute(), bootstrap_target / f.name)
def copy_resources(package, destination):
makedirs(destination, exist_ok=True)
for entry in resources.contents(package):
if not resources.is_resource(package, entry):
continue
with resources.path(package, entry) as resource_path:
shutil.copy2(str(resource_path), destination)
def _load_interactive(*args):
''' Loads an interactive notebook
Called from startup() only
'''
interactive_name = args[1]
if not is_resource('astroedu.interactives', interactive_name+'.ipynb'):
raise FileExistsError(f'No interactive named {interactive_name} found.')
with files('astroedu').joinpath('interactives', interactive_name+'.ipynb') as p:
call = ['jupyter', 'lab']
call_args = list(args[2:]) if len(args) > 2 else []
run_interactive = call + [str(p)] + call_args
print(f'Loading interactive {interactive_name}!')
subprocess.run(run_interactive)
def resource_context(*path, **kwargs):
"""Provide a context manager that yields a pathlib.Path object to a resource file or directory.
If the resource does not already exist on its own on the file system,
a temporary directory/file will be created. If the directory/file was created, it
will be deleted upon exiting the context manager (no exception is
raised if the directory was deleted prior to the context manager
exiting).
"""
if len(path) == 0:
raise TypeError("must provide a path")
final_name = path[-1]
package = ".".join([RESOURCE_MODULE] + list(path[:-1]))
ignore = kwargs.pop("ignore", (".DS_Store", "__init__.py"))
if importlib_resources.is_resource(package, final_name):
# the resource is a file
with importlib_resources.path(package, final_name) as path:
yield path.absolute()
else:
# the resource is a directory
package = package + "." + final_name
# TODO if the package folder exists on the file system it would be ideal to just return that
# but importlib_resources doesn't provide a public API for that
resources = [
c for c in importlib_resources.contents(package)
if importlib_resources.is_resource(package, c) and c not in ignore
]
folder_path = pathlib.Path(tempfile.mkdtemp())
try:
for resource in resources:
with (folder_path / resource).open("wb") as handle:
handle.write(
importlib_resources.read_binary(package, resource))
yield folder_path
finally:
if folder_path.exists():
shutil.rmtree(str(folder_path))
def load_astroedu_config():
''' Loads the config.ini file
Returns:
The config dictionary
Example:
>>> astroedu_config = load_astroedu_config()
'''
if not is_resource('astroedu', 'config.ini'):
raise FileExistsError('No config.ini file found. Did you forget to run: \n astroedu build')
config = ConfigParser()
config.read(config_file_path)
return config['astroedu-config']
def test_package_has_no_reader_fallback(self):
# Test odd ball packages which:
# 1. Do not have a ResourceReader as a loader
# 2. Are not on the file system
# 3. Are not in a zip file
module = util.create_package(file=data01,
path=data01.__file__,
contents=['A', 'B', 'C'])
# Give the module a dummy loader.
module.__loader__ = object()
# Give the module a dummy origin.
module.__file__ = '/path/which/shall/not/be/named'
module.__spec__.loader = module.__loader__
module.__spec__.origin = module.__file__
self.assertFalse(resources.is_resource(module, 'A'))
def decorator(func):
missing_binaries = set()
for fn in filenames:
if not is_resource(aspire.data, f'{fn}'):
missing_binaries.add(fn)
if missing_binaries:
for fn in missing_binaries:
logger.error(f"Binary file {yellow(fn)} is required!")
@functools.wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
return wrapper
def copy_resource_folder(package: Package,
destination_path: str,
exclude: List[str] = None):
"""
Copies the full content of provided package in destination folder.
Names of files that should not be copied have to be listed in `exclude`.
.. Warning ::
As the resources in the folder are discovered by browsing
through the folder, they are not explicitly listed in the Python code.
Therefore, to have the install process run smoothly, these resources need
to be listed in the MANIFEST.in file.
:param package: name of resource package to copy
:param destination_path: file system path of destination
:param exclude: list of item names that should not be copied
"""
exclusion_list = ["__pycache__"]
if exclude:
exclusion_list += exclude
for resource_name in contents(package):
if resource_name in exclusion_list:
continue
if is_resource(package, resource_name):
destination_file_path = pth.join(destination_path, resource_name)
copy_resource(package, resource_name, destination_file_path)
else:
# In case of subfolders that are only declared in MANIFEST.in,
# getattr(package, "resource_name") will fail (is there another way?).
# So we fall back to using package name as as string.
if isinstance(package, ModuleType):
package_name = package.__name__
else: # str
package_name = package
new_package_name = ".".join([package_name, resource_name])
new_destination_path = pth.join(destination_path, resource_name)
copy_resource_folder(new_package_name,
new_destination_path,
exclude=exclude)
def cookiecutter(template_package, output_dir, **kwargs):
with tempfile.TemporaryDirectory() as template_dir:
if importlib_resources.is_resource(template_package, "cookiecutter.json"):
with importlib_resources.path(
template_package, "cookiecutter.json"
) as path:
extract_package_to_temp(
template_package, template_dir, os.path.dirname(path), ""
)
else:
raise BackendException(
f"Provided template package does not contain cookiecutter.json config: {template_package}"
)
replace_parameters_values(kwargs, [[None, ""], [True, "yes"], [False, "no"]])
cookiecutter_main(
template_dir, no_input=True, output_dir=output_dir, extra_context=kwargs
)
def test_namespaces_cannot_have_resources(self):
contents = resources.contents(
'importlib_resources.tests.data03.namespace')
self.assertFalse(list(contents))
# Even though there is a file in the namespace directory, it is not
# considered a resource, since namespace packages can't have them.
self.assertFalse(
resources.is_resource('importlib_resources.tests.data03.namespace',
'resource1.txt'))
# We should get an exception if we try to read it or open it.
self.assertRaises(FileNotFoundError, resources.open_text,
'importlib_resources.tests.data03.namespace',
'resource1.txt')
self.assertRaises(FileNotFoundError, resources.open_binary,
'importlib_resources.tests.data03.namespace',
'resource1.txt')
self.assertRaises(FileNotFoundError, resources.read_text,
'importlib_resources.tests.data03.namespace',
'resource1.txt')
self.assertRaises(FileNotFoundError, resources.read_binary,
'importlib_resources.tests.data03.namespace',
'resource1.txt')
def find_pluggable_components(subpackage, interface):
"""Find components which conform to a given interface.
This finds components which can be implemented in a plugin. It will
search for the interface in the named subpackage, where the Python import
path of the subpackage will be prepended by `mailman` for system
components, and the various plugin names for any external components.
:param subpackage: The subpackage to search. This is prepended by
'mailman' to search for system components, and each enabled plugin for
external components.
:type subpackage: str
:param interface: The interface that returned objects must conform to.
:type interface: `Interface`
:return: The sequence of matching components.
:rtype: Objects implementing `interface`
"""
# This can't be imported at module level because of circular imports.
from mailman.config import config
# Return the system components first.
yield from find_components('mailman.' + subpackage, interface)
# Return all the matching components in all the subpackages of all enabled
# plugins. Only enabled and existing plugins will appear in this
# dictionary.
for name, plugin_config in config.plugin_configs:
# If the plugin's configuration defines a components package, use
# that, falling back to the plugin's name.
package = plugin_config['component_package'].strip()
if len(package) == 0:
package = name
# It's possible that the plugin doesn't include the directory for this
# subpackage. That's fine.
if (subpackage in contents(package)
and not is_resource(package, subpackage)):
plugin_package = '{}.{}'.format(package, subpackage)
yield from find_components(plugin_package, interface)
def test_read_submodule_resource_by_name(self):
self.assertTrue(
resources.is_resource('ziptestdata.subdirectory', 'binary.file'))
def test_resource_missing_is_not_resource(self):
package = util.create_package(file=data01,
path=data01.__file__,
contents=['A', 'B', 'C', 'D/E', 'D/F'])
self.assertFalse(resources.is_resource(package, 'Z'))
def test_read_submodule_resource_by_name(self):
self.assertTrue(resources.is_resource('namespacedata01',
'binary.file'))
def test_is_submodule_resource(self):
self.assertTrue(
resources.is_resource(import_module('namespacedata01'),
'binary.file'))
def test_is_resource_subresource_directory(self):
# Directories are not resources.
self.assertFalse(resources.is_resource(self.data, 'subdirectory'))
def test_is_resource_missing(self):
self.assertFalse(resources.is_resource(self.data, 'not-a-file'))
def test_is_resource_failure_does_not_keep_open(self):
c = resources.is_resource('ziptestdata', 'not-present')
self.zip_path.unlink()
del c
def test_is_resource_does_not_keep_open(self):
c = resources.is_resource('ziptestdata', 'binary.file')
self.zip_path.unlink()
del c
def test_is_resource_good_path(self):
self.assertTrue(resources.is_resource(self.data, 'binary.file'))
def test_is_submodule_resource(self):
submodule = import_module('ziptestdata.subdirectory')
self.assertTrue(resources.is_resource(submodule, 'binary.file'))