def test_unrelated_contents(self): """ Test thata zip with two unrelated subpackages return distinct resources. Ref python/importlib_resources#44. """ self.assertEqual(set(resources.contents('ziptestdata.one')), {'__init__.py', 'resource1.txt'}) self.assertEqual(set(resources.contents('ziptestdata.two')), {'__init__.py', 'resource2.txt'})
def test_unrelated_contents(self): # https://gitlab.com/python-devs/importlib_resources/issues/44 # # Here we have a zip file with two unrelated subpackages. The bug # reports that getting the contents of a resource returns unrelated # files. self.assertEqual(set(resources.contents('ziptestdata.one')), {'__init__.py', 'resource1.txt'}) self.assertEqual(set(resources.contents('ziptestdata.two')), {'__init__.py', 'resource2.txt'})
def listcommands(): """ Returns a list of all available commands """ paths = map(pathlib.Path, importlib_resources.contents(command)) all = (path.stem for path in paths if not path.name.startswith('_')) return filter(_is_available, all)
def ncp_fw_update(): ''' Compare the NCP firmware with the one available in the 'ncp_fiwmare' folder and update if needed ''' # Find the DFU file that matches the required fw version dfu_file = None ver_num = kibra.__kinosver__.split(' v')[-1] for file_name in importlib_resources.contents(NCP_FW_FOLDER): if ver_num in file_name: # TODO: This relies on the file name, we could also check the file # contents to make sure dfu_file = file_name break if not dfu_file: logging.error('Required NCP firmware not present.') sys.exit() # Flash the NCP and re-enable it with importlib_resources.path(NCP_FW_FOLDER, dfu_file) as dfu_path: logging.warn('NCP will be updated with firmware v%s' % ver_num) try: dfu_file = kidfu.DfuFile(str(dfu_path)) kifwu.dfu_find_and_flash(dfu_file, unattended=True) except Exception as exc: logging.error('Problem updating NCP firmware: %s' % exc) sys.exit() logging.info('NCP updated successfully.')
def test_submodule_contents_by_name(self): contents = set(resources.contents('namespacedata01')) try: contents.remove('__pycache__') except KeyError: pass self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'})
def copy_bootstrap(bootstrap_target: Path) -> None: """Copy bootstrap code from shiv into the pyz. :param bootstrap_target: The temporary directory where we are staging pyz contents. """ for bootstrap_file in importlib_resources.contents(bootstrap): if importlib_resources.is_resource(bootstrap, bootstrap_file): with importlib_resources.path(bootstrap, bootstrap_file) as f: shutil.copyfile(f.absolute(), bootstrap_target / f.name)
def copy_resources(package, destination): makedirs(destination, exist_ok=True) for entry in resources.contents(package): if not resources.is_resource(package, entry): continue with resources.path(package, entry) as resource_path: shutil.copy2(str(resource_path), destination)
def test_all_exported_classes_have_slots(): """All "public" classes that form part of the library's interface should have `__slots__`. When defining a subclass of AWSObject, it is possible to accidentally forget to add an empty __slots__ class variable. This means that unknown variables can accidentally be set on that class - violating the well-defined behaviour of our types. """ # Modules to inspect public_modules = [flyingcircus.core, flyingcircus.intrinsic_function] # Packages to inspect public_packages = [flyingcircus.service] # Get public module from public packages for package in public_packages: for filename in importlib_resources.contents(package): # Ignore special filenames/modules if filename.startswith("__"): continue modulename = package.__name__ + "." + filename.split(".")[0] module = importlib.import_module(modulename) public_modules.append(module) # Verify that every AWSObject-like class in a public module has # a well-defined attribute list for module in public_modules: for name, obj in module.__dict__.items(): if not inspect.isclass(obj): continue if not issubclass(obj, AWSObject): continue if obj is flyingcircus.core.Resource: # Resource is abstract, so we can't check it directly continue full_class_name = f"{module.__name__}.{name}" try: instance = obj() except Exception as ex: assert False, f"Unable to instantiate {full_class_name}: " + str(ex) nonexistent_attrib_name = "ThisAttributeDoesNotExist" with pytest.raises( AttributeError, message=f"{full_class_name} hasn't defined __slots__", match=f"{obj.__name__}.*{nonexistent_attrib_name}", ): setattr(instance, nonexistent_attrib_name, 42)
def _resolve_resource( package: str, *, name: t.Optional[str], support_extensions=(".yaml", ".yml")) -> dict: logger.info("resolve resource, find resource from %s", package) for fname in importlib_resources.contents(package): if not os.path.splitext(fname)[1].endswith(support_extensions): continue name = fname break logger.info("resolve resource, load data from %s", name) with importlib_resources.open_text(package, name) as rf: return loading.load(rf)
def _sorted_file_migrations(): files = [] ids = [] for f in importlib_resources.contents(migrations): match = re.search(MIGRATION_FILE_PATTERN, f) if match: idx = int(match.group("id")) if idx in ids: raise click.ClickException( f"Inconsistent package (multiples migrations with {idx} as id)" ) ids.append(idx) files.append((idx, match.group("name"), f)) return sorted(files, key=lambda f: f[0])
def test_contents(self): contents = set(resources.contents(self.data)) # There may be cruft in the directory listing of the data directory. # It could have a __pycache__ directory, # an artifact of the # test suite importing these modules, which # are not germane to this test, so just filter them out. contents.discard('__pycache__') self.assertEqual( contents, { '__init__.py', 'subdirectory', 'utf-8.file', 'binary.file', 'utf-16.file', }, )
def test_contents(self): contents = set(resources.contents(self.data)) # There may be cruft in the directory listing of the data directory. # Under Python 3 we could have a __pycache__ directory, and under # Python 2 we could have .pyc files. These are both artifacts of the # test suite importing these modules and writing these caches. They # aren't germane to this test, so just filter them out. contents.discard('__pycache__') contents.discard('__init__.pyc') contents.discard('__init__.pyo') self.assertEqual( contents, { '__init__.py', 'subdirectory', 'utf-8.file', 'binary.file', 'utf-16.file', })
def copy_resource_folder(package: Package, destination_path: str, exclude: List[str] = None): """ Copies the full content of provided package in destination folder. Names of files that should not be copied have to be listed in `exclude`. .. Warning :: As the resources in the folder are discovered by browsing through the folder, they are not explicitly listed in the Python code. Therefore, to have the install process run smoothly, these resources need to be listed in the MANIFEST.in file. :param package: name of resource package to copy :param destination_path: file system path of destination :param exclude: list of item names that should not be copied """ exclusion_list = ["__pycache__"] if exclude: exclusion_list += exclude for resource_name in contents(package): if resource_name in exclusion_list: continue if is_resource(package, resource_name): destination_file_path = pth.join(destination_path, resource_name) copy_resource(package, resource_name, destination_file_path) else: # In case of subfolders that are only declared in MANIFEST.in, # getattr(package, "resource_name") will fail (is there another way?). # So we fall back to using package name as as string. if isinstance(package, ModuleType): package_name = package.__name__ else: # str package_name = package new_package_name = ".".join([package_name, resource_name]) new_destination_path = pth.join(destination_path, resource_name) copy_resource_folder(new_package_name, new_destination_path, exclude=exclude)
def retrieve_migrations() -> List[MigrationItem]: migrations = [] ids = [] for file_name in importlib_resources.contents(migrations_module): match = re.search(MIGRATION_FILE_PATTERN, file_name) if match: idx = int(match.group("id")) # Sanity check if idx in ids: raise AssertionError( f"Inconsistent package (multiples migrations with {idx} as id)" ) ids.append(idx) sql = importlib_resources.read_text(migrations_module, file_name) if not sql: raise AssertionError(f"Empty migration file {file_name}") migrations.append( MigrationItem(idx=idx, name=match.group("name"), file_name=file_name, sql=sql) ) return sorted(migrations, key=lambda item: item.idx)
def test_namespaces_cannot_have_resources(self): contents = resources.contents( 'importlib_resources.tests.data03.namespace') self.assertFalse(list(contents)) # Even though there is a file in the namespace directory, it is not # considered a resource, since namespace packages can't have them. self.assertFalse( resources.is_resource('importlib_resources.tests.data03.namespace', 'resource1.txt')) # We should get an exception if we try to read it or open it. self.assertRaises(FileNotFoundError, resources.open_text, 'importlib_resources.tests.data03.namespace', 'resource1.txt') self.assertRaises(FileNotFoundError, resources.open_binary, 'importlib_resources.tests.data03.namespace', 'resource1.txt') self.assertRaises(FileNotFoundError, resources.read_text, 'importlib_resources.tests.data03.namespace', 'resource1.txt') self.assertRaises(FileNotFoundError, resources.read_binary, 'importlib_resources.tests.data03.namespace', 'resource1.txt')
def resource_context(*path, **kwargs): """Provide a context manager that yields a pathlib.Path object to a resource file or directory. If the resource does not already exist on its own on the file system, a temporary directory/file will be created. If the directory/file was created, it will be deleted upon exiting the context manager (no exception is raised if the directory was deleted prior to the context manager exiting). """ if len(path) == 0: raise TypeError("must provide a path") final_name = path[-1] package = ".".join([RESOURCE_MODULE] + list(path[:-1])) ignore = kwargs.pop("ignore", (".DS_Store", "__init__.py")) if importlib_resources.is_resource(package, final_name): # the resource is a file with importlib_resources.path(package, final_name) as path: yield path.absolute() else: # the resource is a directory package = package + "." + final_name # TODO if the package folder exists on the file system it would be ideal to just return that # but importlib_resources doesn't provide a public API for that resources = [ c for c in importlib_resources.contents(package) if importlib_resources.is_resource(package, c) and c not in ignore ] folder_path = pathlib.Path(tempfile.mkdtemp()) try: for resource in resources: with (folder_path / resource).open("wb") as handle: handle.write( importlib_resources.read_binary(package, resource)) yield folder_path finally: if folder_path.exists(): shutil.rmtree(str(folder_path))
def find_components(package, interface): """Find components which conform to a given interface. Search all the modules in a given package, returning an iterator over all items found that conform to the given interface, unless that object is decorated with `@abstract_component`. :param package: The package path to search. :type package: string :param interface: The interface that returned objects must conform to. :type interface: `Interface` :return: The sequence of matching components. :rtype: items implementing `interface` """ for filename in contents(package): basename, extension = os.path.splitext(filename) if extension != '.py' or basename.startswith('.'): continue module_name = '{}.{}'.format(package, basename) module = import_module(module_name) if not hasattr(module, '__all__'): continue yield from scan_module(module, interface)
def find_pluggable_components(subpackage, interface): """Find components which conform to a given interface. This finds components which can be implemented in a plugin. It will search for the interface in the named subpackage, where the Python import path of the subpackage will be prepended by `mailman` for system components, and the various plugin names for any external components. :param subpackage: The subpackage to search. This is prepended by 'mailman' to search for system components, and each enabled plugin for external components. :type subpackage: str :param interface: The interface that returned objects must conform to. :type interface: `Interface` :return: The sequence of matching components. :rtype: Objects implementing `interface` """ # This can't be imported at module level because of circular imports. from mailman.config import config # Return the system components first. yield from find_components('mailman.' + subpackage, interface) # Return all the matching components in all the subpackages of all enabled # plugins. Only enabled and existing plugins will appear in this # dictionary. for name, plugin_config in config.plugin_configs: # If the plugin's configuration defines a components package, use # that, falling back to the plugin's name. package = plugin_config['component_package'].strip() if len(package) == 0: package = name # It's possible that the plugin doesn't include the directory for this # subpackage. That's fine. if (subpackage in contents(package) and not is_resource(package, subpackage)): plugin_package = '{}.{}'.format(package, subpackage) yield from find_components(plugin_package, interface)
import importlib_resources as resources import subprocess import shutil import glob import re from pathlib import Path, PureWindowsPath from ginjinn import config from ginjinn.core import Configuration from ginjinn.data_files import tf_config_templates, tf_script_templates # TODO: maybe put those constants in another file ''' Model configuration files that are available out of the box''' AVAILABLE_MODEL_CONFIGS = [ f for f in resources.contents(tf_config_templates) if f.endswith('.config') ] ''' Models (names) that are available out of the box''' AVAILABLE_MODELS = [ # [:-7] removes '.config' f[:-7] for f in AVAILABLE_MODEL_CONFIGS ] ''' Mapping of model names to model configuration file names''' MODEL_CONFIG_FILES = {m: f'{m}.config' for m in AVAILABLE_MODELS} ''' Mapping of model names (and configs) to urls where a coco-pretrained version can be downloaded ''' PRETRAINED_MODEL_URLS = { 'faster_rcnn_resnet50_coco': 'http://download.tensorflow.org/models/object_detection/faster_rcnn_resnet50_coco_2018_01_28.tar.gz', 'rfcn_resnet101_coco': 'http://download.tensorflow.org/models/object_detection/rfcn_resnet101_coco_2018_01_28.tar.gz', 'faster_rcnn_inception_resnet_v2_atrous_coco':
def test_contents(self): assert self.expected <= set(resources.contents(self.data))
import runpy import sys from io import StringIO from time import sleep import importlib_resources import pkg_resources from benchmarker import Benchmarker from importlib_resources import path from benchmarks.benchmark_utils import swap_attr for entry in [ entry for entry in importlib_resources.contents('examples') if not pkg_resources.resource_isdir('examples', entry) and entry. endswith(".py") and '__init__' not in entry and 'DELETE' not in entry ]: print("*****************************************************************") print("Benchmarking {}".format(entry)) print("*****************************************************************") with path('examples', entry) as file_in: with Benchmarker(cycle=20, extra=1) as bench: @bench(entry) def _(_): # prevent Benchmarker from doing "ZeroDivisionError: float division by zero: # ratio = base_time / real_time" sleep(0.001) # In order to pipe input into examples that have input(), # we use the test package, which is meant for internal use by Python only internal and
def test_resource_contents(self): package = util.create_package(file=data01, path=data01.__file__, contents=['A', 'B', 'C']) self.assertEqual(set(resources.contents(package)), {'A', 'B', 'C'})
def test_contents_does_not_keep_open(self): c = resources.contents('ziptestdata') self.zip_path.unlink() del c
cmd = ' '.join(cmd) + ' ' + parameters else: cmd = ' '.join(cmd + parameters) else: if isinstance(parameters, string_types): parameters = shlex.split(parameters) cmd += parameters p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE) stdout, stderr = p.communicate(input=stdin) if not (p.returncode == 0 or ignore_return_code): raise OSError(bytes_to_str(stderr)) return stdout.decode('utf8', 'replace'), stderr.decode('utf8', 'replace') for entry in [ entry for entry in importlib_resources.contents('notebooks') if not pkg_resources.resource_isdir('notebooks', entry) and entry.endswith(".ipynb") ]: print("*****************************************************************") print("Converting and running {}".format(entry)) print("*****************************************************************") with path('notebooks', entry) as file_in: out, err = run_python( " -m jupyter nbconvert --execute --to notebook --inplace " + str(file_in)) sys.stderr.write(err) sys.stderr.flush() sys.stdout.write(out) sys.stdout.flush()
def test_submodule_contents_by_name(self): self.assertEqual( set(resources.contents('ziptestdata.subdirectory')), {'__init__.py', 'binary.file'}, )
def test_submodule_contents(self): submodule = import_module('ziptestdata.subdirectory') self.assertEqual(set(resources.contents(submodule)), {'__init__.py', 'binary.file'})
def main(myst_file, header_file): """ Build a web page for the myst markdown file MYST_FILE, using the headers in headers.json usage: build_page quiz3.md [headers.json] if headers.json is missing and doesn't exist, a default file "headers.json" will be written and used. If headers.json does exist, it must be specified on the command line. headers.json is a json file with a dictionary with these keys (margin sizes in inches) \b { "page_title": "Day 03 quiz", "left_header": "Jan. 19, 2021", "center_header": "Day 03 quiz", "page_margin_top": 1, "page_margin_bottom": 1, "page_margin_left": 1, "page_margin_right": 1 } html output is written to the file _myst_file/myst_file.html To force a page break insert: <div class="page-break"></div> """ myst_file = Path(myst_file).resolve() header_list = list(header_file) if len(header_list) == 0: # # header_file may be missing so handle the case where # nargs=0 and create default headers.json. # the_headers = "headers.json" if Path(the_headers).is_file(): raise ValueError( f"{the_headers} exists but wasn't specified, won't overwrite") with ir.open_text('paged_html_theme.templates', 'headers.json', encoding='utf-8') as header_in: header_contents = header_in.read() with open(the_headers, 'w', encoding='utf-8') as header_out: header_out.write(header_contents) print( f"wrote default header file \n{header_contents} to \n{str(the_headers)}\n" ) elif len(header_list) == 1: the_headers = Path(header_list[0]).resolve() else: # # if nargs is not 0 or 1, give up # print(f"need a single header file, found {header_list}") stem_name = myst_file.stem root_dir = myst_file.parent output_dir = root_dir / f'_{stem_name}_build' with open(the_headers, 'r', encoding='utf-8') as input: header_dict = json.load(input) contents = list(ir.contents('paged_html_theme.templates')) with ir.open_text('paged_html_theme.templates', 'conf_py.j2', encoding='utf-8') as the_conf: conf_j2 = jinja2.Template(the_conf.read()) if not Path(myst_file).is_file(): raise ValueError(f"could not find {myst_file}") arglist = [ 'sphinx-build', '-v', '-b html', str(root_dir), str(output_dir), str(myst_file) ] argstring = ' '.join(arglist) header_dict['stem_name'] = stem_name conf_file = conf_j2.render(**header_dict) with cd(root_dir): with open('conf.py', 'w', encoding='utf-8') as conf_out: conf_out.write(conf_file) print(f"running the command \n{argstring}\n") result = subprocess.run(argstring, capture_output=True, shell=True) if result.stdout: print(f"stdout message: {result.stdout.decode('utf-8')}") out_file = output_dir / f"{stem_name}.html" if out_file.is_file(): print(f"full path to output: {out_file}") else: raise ValueError(f"can't find output file {out_file}") if result.stderr: print(f"stderror message: {result.stderr.decode('utf-8')}")