def visit_asset_node(translator: HTMLTranslator, node: AssetNode): """ Visit an :class:`~.AssetNode`. :param translator: :param node: The node being visited. """ if not hasattr(translator, "_asset_node_seen_files"): # Files that have already been seen translator._asset_node_seen_files = [] # type: ignore assets_out_dir = PathPlus(translator.builder.outdir) / "_assets" assets_out_dir.maybe_make(parents=True) source_file = PathPlus(translator.builder.confdir) / node["source_file"] if source_file not in translator._asset_node_seen_files and source_file.is_file(): # type: ignore # Avoid unnecessary copies of potentially large files. translator._asset_node_seen_files.append(source_file) # type: ignore shutil.copy2(source_file, assets_out_dir) elif not source_file.is_file(): stderr_writer(Fore.RED(f"{translator.builder.current_docname}: Asset file '{source_file}' not found.")) translator.context.append('') return # Create the HTML current_uri = (pathlib.PurePosixPath('/') / translator.builder.current_docname).parent refuri = posixpath.relpath(f"/_assets/{node['refuri']}", str(current_uri)) translator.body.append(f'<a class="reference external" href="{refuri}")/">') translator.context.append("</a>")
def copy_assets(app: Sphinx, exception: Optional[Exception] = None) -> None: """ Copy asset files to the output. :param app: The Sphinx application. :param exception: Any exception which occurred and caused Sphinx to abort. """ if exception: # pragma: no cover return style = {} for colour, hex_ in _colour_map.items(): style[ f"div.sphinx-highlights div.highlight-{colour} div.card-header"] = { "background-color": hex_ } # if app.config.html_theme in {"domdf_sphinx_theme", "sphinx_rtd_theme"}: # header_colour = app.config.html_theme_options.get("style_nav_header_background", "#2980B9") # # style.blankline() # style.extend([ # "div.sphinx-highlights div.card-header {", # f" background-color: {header_colour}", # '}', # ]) css_dir = PathPlus(app.builder.outdir) / "_static" / "css" css_dir.maybe_make(parents=True) dict2css.dump(style, css_dir / "sphinx_highlights.css")
def copy_assets(app: Sphinx, exception: Optional[Exception] = None) -> None: """ Copy asset files to the output. :param app: The Sphinx application. :param exception: Any exception which occurred and caused Sphinx to abort. """ if exception: # pragma: no cover return style = StringList([ ".docutils.container {", " padding-left: 0 !important;", " padding-right: 0 !important;", '}', '', # "div.sphinx-tabs.docutils.container {", # " padding-left: 0 !important;", # " padding-right: 0 !important;", # "}", # '', "div.ui.top.attached.tabular.menu.sphinx-menu.docutils.container {", # " padding-left: 0 !important;", # " padding-right: 0 !important;", " margin-left: 0 !important;", " margin-right: 0 !important;", '}', ]) css_dir = PathPlus(app.builder.outdir) / "_static" / "css" css_dir.maybe_make(parents=True) css_file = css_dir / "tabs_customise.css" css_file.write_lines(style)
def enable_docs( repo_path: pathlib.Path, templates: Environment, init_repo_templates: Environment, ) -> List[str]: docs_dir = PathPlus(repo_path / templates.globals["docs_dir"]) docs_dir.maybe_make() (docs_dir / "api").maybe_make() for filename in {"index.rst"}: template = init_repo_templates.get_template(filename) (docs_dir / filename).write_clean(template.render()) api_buf = StringList() header_line: str = '=' * (len(templates.globals["import_name"]) + 1) api_buf.append(header_line) api_buf.append(templates.globals["import_name"]) api_buf.append(header_line) api_buf.blankline(ensure_single=True) api_buf.append(f".. automodule:: {templates.globals['import_name']}") api_buf.blankline(ensure_single=True) (docs_dir / "api" / templates.globals["modname"]).with_suffix(".rst").write_lines(api_buf) return [ posixpath.join(templates.globals["docs_dir"], "api", f"{templates.globals['modname']}.rst"), posixpath.join(templates.globals["docs_dir"], "index.rst"), ]
def copy_asset_files(app: Sphinx, exception: Exception = None): """ Copy additional stylesheets into the HTML build directory. :param app: The Sphinx application. :param exception: Any exception which occurred and caused Sphinx to abort. .. versionadded:: 1.2.0 """ if exception: # pragma: no cover return if app.builder.format.lower() != "html": return style = { "span.regex_literal": { "color": "dimgrey" }, "span.regex_at": { "color": "orangered" }, "span.regex_repeat_brace": { "color": "orangered" }, "span.regex_branch": { "color": "orangered" }, "span.regex_subpattern": { "color": "dodgerblue" }, "span.regex_in": { "color": "darkorange" }, "span.regex_category": { "color": "darkseagreen" }, "span.regex_repeat": { "color": "orangered" }, "span.regex_any": { "color": "orangered" }, "code.regex": { "font-size": "80%" }, "span.regex": { "font-weight": "bold" }, } static_dir = PathPlus(app.outdir) / "_static" static_dir.maybe_make(parents=True) (static_dir / "regex.css").write_clean(dict2css.dumps(style, minify=True))
class ResultParser: """ Given a directory of CSV results exported from MassHunter, parse them to CSV and JSON. :param raw_results_dir: The directory in which the raw exports from MassHunter are stored. :param json_results_dir: The directory to store the output json files in. :param csv_results_dir: The directory to store the output csv files in. """ def __init__(self, raw_results_dir: PathLike, json_results_dir: PathLike, csv_results_dir: PathLike): self.raw_results_dir = PathPlus(raw_results_dir) self.json_results_dir = PathPlus(json_results_dir) self.json_results_dir.maybe_make(parents=True) self.csv_results_dir = PathPlus(csv_results_dir) self.csv_results_dir.maybe_make(parents=True) def parse_for_directory(self, directory: PathLike): """ Convert the "CSV Results.csv" file in the given directory to CSV and JSON. :param directory: """ (self.json_results_dir / directory).maybe_make() (self.csv_results_dir / directory).maybe_make() infile = self.raw_results_dir / directory / "CSV Results.csv" csv_outfile = self.csv_results_dir / directory / "CSV Results Parsed.csv" json_outfile = self.json_results_dir / directory / "results.json" print(f"{infile} -> {csv_outfile}") print(f"{' ' * len(str(infile))} -> {json_outfile}") parse_masshunter_csv(infile, csv_outfile, json_outfile) def parse_directory_list(self, directory_list: Iterable[PathLike]): """ Runs :meth:`.~ResultsParser.parse_for_directory` for each directory in ``directory_list``. :param directory_list: A list of directories to process. """ for directory in directory_list: print(f"Processing directory {directory}") self.parse_for_directory(directory)
def copy_asset_files(app: Sphinx, exception: Optional[Exception] = None): """ Copy additional stylesheets into the HTML build directory. :param app: The Sphinx application. :param exception: Any exception which occurred and caused Sphinx to abort. """ if exception: # pragma: no cover return if app.builder is None or app.builder.format.lower( ) != "html": # pragma: no cover return static_dir = PathPlus(app.outdir) / "_static" static_dir.maybe_make(parents=True) dict2css.dump(_css.regex_styles, static_dir / "regex.css", minify=True)
def BASE(tmp_pathplus: PathPlus): top_dir = tmp_pathplus tmp_pathplus = top_dir / "a/b/c/d" tmp_pathplus.maybe_make(parents=True) join = lambda *x: os.path.join(tmp_pathplus, *x) if os.name == "nt": # Workaround for http://bugs.python.org/issue13772. def dirlink(src, dest): os.symlink(src, dest, target_is_directory=True) else: def dirlink(src, dest): os.symlink(src, dest) os.mkdir(join("dirA")) os.mkdir(join("dirB")) os.mkdir(join("dirC")) os.mkdir(join("dirC", "dirD")) os.mkdir(join("dirE")) with open(join("fileA"), "wb") as f: f.write(b"this is file A\n") with open(join("dirB", "fileB"), "wb") as f: f.write(b"this is file B\n") with open(join("dirC", "fileC"), "wb") as f: f.write(b"this is file C\n") with open(join("dirC", "dirD", "fileD"), "wb") as f: f.write(b"this is file D\n") os.chmod(join("dirE"), 0) if not PYPY and sys.platform != "win32": # Relative symlinks. os.symlink("fileA", join("linkA")) os.symlink("non-existing", join("brokenLink")) dirlink("dirB", join("linkB")) dirlink(os.path.join("..", "dirB"), join("dirA", "linkC")) # This one goes upwards, creating a loop. dirlink(os.path.join("..", "dirB"), join("dirB", "linkD")) yield tmp_pathplus os.chmod(join("dirE"), 0o777) shutil.rmtree(top_dir)
def make_conda_actions_ci(repo_path: pathlib.Path, templates: Environment) -> List[str]: """ Add configuration for testing conda packages on `GitHub Actions` to the desired repo. :param repo_path: Path to the repository root. :param templates: """ workflows_dir = PathPlus(repo_path / ".github" / "workflows") conda_ci_file = workflows_dir / "conda_ci.yml" if templates.globals["enable_conda"]: actions = templates.get_template("github_conda_ci.yml") workflows_dir.maybe_make(parents=True) def no_pypy_versions(versions): """ Returns the subset of ``versions`` which does not end with ``-dev``. :param versions: """ return [ v for v in no_dev_versions(versions) if "pypy" not in v.lower() ] pip_dependencies = ["whey-conda"] pyproject_file = PathPlus(repo_path / "pyproject.toml") if pyproject_file.is_file(): data: DefaultDict[str, Any] = DefaultDict(dom_toml.load(pyproject_file)) pip_dependencies.extend(data["build-system"]["requires"]) conda_ci_file.write_clean( actions.render(no_dev_versions=no_pypy_versions, pip_dependencies=pip_dependencies)) else: conda_ci_file.unlink(missing_ok=True) return [conda_ci_file.relative_to(repo_path).as_posix()]
def make_issue_templates(repo_path: pathlib.Path, templates: Environment) -> List[str]: """ Add issue templates for GitHub to the desired repo. :param repo_path: Path to the repository root. :param templates: """ managed_files = [] issue_template_dir = PathPlus(repo_path / ".github" / "ISSUE_TEMPLATE") issue_template_dir.maybe_make(parents=True) for filename in ["bug_report.md", "feature_request.md"]: filepath = issue_template_dir / filename filepath.write_clean(templates.get_template(filename).render()) managed_files.append(filepath.relative_to(repo_path).as_posix()) return managed_files
class HTTPCache: """ Cache HTTP requests for up to 28 days and limit the rate of requests to no more than 5/second. :param app_name: The name of the app. This dictates the name of the cache directory. :param expires_after: The maximum time to cache responses for. """ app_name: str #: The name of the app. This dictates the name of the cache directory. cache_dir: PathPlus #: The location of the cache directory on disk. caches: Dict[str, Dict[str, Any]] #: Mapping of function names to their caches. def __init__(self, app_name: str, expires_after: datetime.timedelta = datetime.timedelta(days=28)): self.app_name: str = str(app_name) self.cache_dir = PathPlus(platformdirs.user_cache_dir(self.app_name)) self.cache_dir.maybe_make(parents=True) self.session: requests.Session = CacheControl( sess=requests.Session(), cache=FileCache(self.cache_dir), heuristic=ExpiresAfter( days=expires_after.days, seconds=expires_after.seconds, microseconds=expires_after.microseconds, ), adapter_class=RateLimitAdapter ) def clear(self) -> bool: """ Clear the cache. :returns: True to indicate success. False otherwise. """ try: shutil.rmtree(self.cache_dir) return True except Exception as e: # pragma: no cover warnings.warn(f"Could not remove cache. The error was: {e}") return False
def copy_asset_files(app: Sphinx, exception: Optional[Exception] = None) -> None: """ Copy asset files to the output. :param app: The Sphinx application. :param exception: Any exception which occurred and caused Sphinx to abort. .. versionchanged:: 2.7.0 Renamed from ``copy_assets``. The old name is deprecated an will be removed in 3.0.0 """ if exception: # pragma: no cover return if app.builder is None or app.builder.format.lower() != "html": # pragma: no cover return # style = StringList([ # ".docutils.container {", # " padding-left: 0 !important;", # " padding-right: 0 !important;", # '}', # '', # # "div.sphinx-tabs.docutils.container {", # # " padding-left: 0 !important;", # # " padding-right: 0 !important;", # # "}", # # '', # "div.ui.top.attached.tabular.menu.sphinx-menu.docutils.container {", # # " padding-left: 0 !important;", # # " padding-right: 0 !important;", # " margin-left: 0 !important;", # " margin-right: 0 !important;", # '}', # ]) css_static_dir = PathPlus(app.builder.outdir) / "_static" / "css" css_static_dir.maybe_make(parents=True) dict2css.dump(_css.tweaks_sphinx_panels_tabs_styles, css_static_dir / "tabs_customise.css")
def copy_asset_files(app: Sphinx, exception: Optional[Exception] = None): """ Copy additional stylesheets into the HTML build directory. :param app: The Sphinx application. :param exception: Any exception which occurred and caused Sphinx to abort. """ if exception: # pragma: no cover return if app.builder is None or app.builder.format.lower() != "html": # pragma: no cover return extensions_selector = ", ".join([ "p.sphinx-toolbox-extensions", "div.sphinx-toolbox-extensions.highlight-python", "div.sphinx-toolbox-extensions.highlight-python div.highlight", ]) rest_example_style = { "padding-left": "5px", "border-style": "dotted", "border-width": "1px", "border-color": "darkgray", } style: MutableMapping[str, dict2css.Style] = { "p.source-link": {"margin-bottom": 0}, "p.source-link + hr.docutils": {"margin-top": "10px"}, extensions_selector: {"margin-bottom": "10px"}, "div.rest-example.docutils.container": rest_example_style, **installation_styles, **shields_styles, **regex_styles, } css_static_dir = PathPlus(app.outdir) / "_static" / "css" css_static_dir.maybe_make(parents=True) dict2css.dump(style, css_static_dir / "sphinx-toolbox.css")
def copy_asset_files(app: Sphinx, exception: Optional[Exception] = None): """ Copy additional stylesheets into the HTML build directory. .. versionadded:: 1.2.0 :param app: The Sphinx application. :param exception: Any exception which occurred and caused Sphinx to abort. """ if exception: # pragma: no cover return if app.builder is None or app.builder.format.lower() != "html": # pragma: no cover return static_dir = PathPlus(app.outdir) / "_static" static_dir.maybe_make(parents=True) dict2css.dump(_css.installation_styles, static_dir / "sphinx_toolbox_installation.css", minify=True) (static_dir / "sphinx_toolbox_installation.js").write_lines([ "// Based on https://github.com/executablebooks/sphinx-tabs/blob/master/sphinx_tabs/static/tabs.js", "// Copyright (c) 2017 djungelorm", "// MIT Licensed", '', "function deselectTabset(target) {", " const parent = target.parentNode;", " const grandparent = parent.parentNode;", '', ' if (parent.parentNode.parentNode.getAttribute("id").startsWith("installation")) {', '', " // Hide all tabs in current tablist, but not nested", " Array.from(parent.children).forEach(t => {", ' if (t.getAttribute("name") !== target.getAttribute("name")) {', ' t.setAttribute("aria-selected", "false");', " }", " });", '', " // Hide all associated panels", " Array.from(grandparent.children).slice(1).forEach(p => { // Skip tablist", ' if (p.getAttribute("name") !== target.getAttribute("name")) {', ' p.setAttribute("hidden", "false")', " }", " });", " }", '', " else {", " // Hide all tabs in current tablist, but not nested", " Array.from(parent.children).forEach(t => {", ' t.setAttribute("aria-selected", "false");', " });", '', " // Hide all associated panels", " Array.from(grandparent.children).slice(1).forEach(p => { // Skip tablist", ' p.setAttribute("hidden", "true")', " });", " }", '', '}', '', "// Compatibility with sphinx-tabs 2.1.0 and later", "function deselectTabList(tab) {deselectTabset(tab)}", '', ])
def copy_asset_files(app: Sphinx, exception: Optional[Exception] = None): """ Copy additional stylesheets into the HTML build directory. .. versionadded:: 2.6.0 :param app: The Sphinx application. :param exception: Any exception which occurred and caused Sphinx to abort. """ if exception: # pragma: no cover return if app.builder is None or app.builder.format.lower( ) != "html": # pragma: no cover return prompt_style: dict2css.Style = { "user-select": None, "font-size": "13px", "font-family": '"SFMono-Regular", Menlo, Consolas, Monaco, Liberation Mono, Lucida Console, monospace', "border": None, "padding": "11px 0 0", "margin": "0 5px 0 0", "box-shadow": None, "wrap-option": None, "white-space": "nowrap", } container_style: dict2css.Style = { "padding-top": "5px", "display": "flex", "align-items": "stretch", "margin": 0, } code_style_string = "div.code-cell.container div.code-cell-code, div.output-cell.container div.output-cell-code" code_style: dict2css.Style = { "width": "100%", "padding-top": 0, "margin-top": 0, } style: MutableMapping[str, dict2css.Style] = { "div.code-cell.container div.prompt": { "color": "#307FC1" }, "div.output-cell.container div.prompt": { "color": "#BF5B3D" }, "div.code-cell.container div.prompt, div.output-cell.container div.prompt": prompt_style, "div.code-cell.container, div.output-cell.container": container_style, code_style_string: code_style, } static_dir = PathPlus(app.outdir) / "_static" static_dir.maybe_make(parents=True) dict2css.dump(style, static_dir / "sphinx-toolbox-code.css")
def make_module( name: str, module: ModuleType, attr_list: Iterable[str] = (), first_party_imports: Iterable[str] = (), converter=Converter() ) -> bool: """ Create type stubs for a module. :param name: The name of the module. :param module: The module object. :param attr_list: A list of attributes to create stubs for. :param first_party_imports: A list of first-party imports to include at the top of the file. """ buf = StringList() path = name.split('.') stubs_dir = PathPlus(f"{path[0]}-stubs") stubs_dir.maybe_make() (stubs_dir / '/'.join(x for x in path[1:-1])).maybe_make(parents=True) stub_file = stubs_dir / '/'.join(x for x in path[1:-1]) / f"{path[-1]}.pyi" import_name = name.replace(".__init__", '') for imp in (*make_imports(name), *first_party_imports): imp = re.sub( fr"import {import_name}\.([A-Za-z_]+)\.([A-Za-z_]+)\.([A-Za-z_]+)", r"from .\1.\2 import \3", imp) imp = re.sub(fr"import {import_name}\.([A-Za-z_]+)\.([A-Za-z_]+)", r"from .\1 import \2", imp) imp = re.sub(fr"import {import_name}\.([A-Za-z_]+)", r"from . import \1", imp) imp = re.sub(fr"import {import_name}$", '', imp) buf.append(imp) if import_name != "System.ComponentModel": if import_name == "System": buf.append("from .ComponentModel import MarshalByValueComponent") else: buf.append( "from System.ComponentModel import MarshalByValueComponent") for attr_name in dedup(attr_list): stub_code = walk_attrs(module, attr_name, converter=converter) stub_code = stub_code.replace(f": {import_name}.", ": ") stub_code = stub_code.replace(f" -> {import_name}.", " -> ") stub_code = stub_code.replace(f"[{import_name}.", '[') stub_code.replace( "System.Collections.Generic.IDictionary[System.String,System.String]", "Any") buf.blankline(ensure_single=True) buf.blankline() buf.append(stub_code) sorted_code = isort.code(str(buf), config=isort_config) sans_unneeded_imports = fix_code( sorted_code, additional_imports=None, expand_star_imports=False, remove_all_unused_imports=False, remove_duplicate_keys=False, remove_unused_variables=False, ignore_init_module_imports=False, ) stub_file.write_text(sans_unneeded_imports) return True
class ActionsManager: """ Responsible for creating, updating and removing GitHub Actions workflows. :param repo_path: Path to the repository root. :param templates: .. versionadded:: 2020.12.18 """ def __init__(self, repo_path: pathlib.Path, templates: Environment): self.repo_path = repo_path self.templates = templates self.actions = templates.get_template("github_ci.yml") self.workflows_dir = PathPlus(repo_path / ".github" / "workflows") self.workflows_dir.maybe_make(parents=True) code_file_filter: DelimitedList[str] = DelimitedList() if self.templates.globals["enable_docs"]: code_file_filter.append(f"{templates.globals['docs_dir']}/**") else: code_file_filter.append(f"doc-source/**") code_file_filter.extend([ "CONTRIBUTING.rst", ".imgbotconfig", ".pre-commit-config.yaml", ".pylintrc", ".readthedocs.yml", ]) # ".bumpversion.cfg", # ".style.yapf", # "stubs.txt", self._code_file_filter = f"!({code_file_filter:|})" def get_gh_actions_python_versions(self) -> Dict[str, str]: """ Prepares the mapping of Python versions to tox testenvs for use with GitHub Actions. .. versionadded:: 2020.12.21 """ gh_action_matrix = self.get_gh_actions_matrix() version_mapping: Dict[str, str] = {} for py_version, (testenvs, experimental) in gh_action_matrix.items(): version_mapping[str(py_version)] = testenvs return version_mapping def get_gh_actions_matrix(self) -> Dict[str, Tuple[str, bool]]: """ Determines the matrix of Python versions used in GitHub Actions. .. versionadded:: $VERSION """ config = self.templates.globals python_versions = config["python_versions"] tox_py_versions = config["tox_py_versions"] third_party_version_matrix = config["third_party_version_matrix"] # TODO: support multi-library matrices if third_party_version_matrix: third_party_library = list(third_party_version_matrix.keys())[0] third_party_versions = DelimitedList( third_party_version_matrix[third_party_library]) matrix_testenv_string = f"-{third_party_library}{{{third_party_versions:,}}}" else: matrix_testenv_string = '' output: Dict[str, Tuple[str, bool]] = {} for (py_version, metadata), gh_py_version, tox_py_version in zip( python_versions.items(), set_gh_actions_versions(python_versions), tox_py_versions, ): output[str(gh_py_version)] = ( f"{tox_py_version}{matrix_testenv_string},build", metadata["experimental"], ) return output @staticmethod def _is_experimental(version: str): if version in {"pypy-3.7", "pypy-3.8"}: return True try: return Version(version).is_prerelease except InvalidVersion: return False def make_windows(self) -> PathPlus: """ Create, update or remove the Windows action, as appropriate. """ platform_name = "Windows" ci_file = self.workflows_dir / "python_ci.yml" if platform_name in self.templates.globals["platforms"]: ci_file.write_clean( self.actions.render( no_dev_versions=no_dev_versions, ci_platform=platform_ci_names[platform_name], ci_name=platform_name, python_versions=set_gh_actions_versions( self.get_windows_ci_versions()), dependency_lines=self.get_windows_ci_requirements(), gh_actions_versions=self.get_gh_actions_matrix(), code_file_filter=self._code_file_filter, is_experimental=self._is_experimental)) elif ci_file.is_file(): ci_file.unlink() return ci_file def make_macos(self) -> PathPlus: """ Create, update or remove the macOS action, as appropriate. """ platform_name = "macOS" ci_file = self.workflows_dir / f"python_ci_{platform_name.lower()}.yml" if platform_name in self.templates.globals["platforms"]: ci_file.write_clean( self.actions.render( no_dev_versions=no_dev_versions, ci_platform=platform_ci_names[platform_name], ci_name=platform_name, python_versions=set_gh_actions_versions( self.get_macos_ci_versions()), dependency_lines=self.get_macos_ci_requirements(), gh_actions_versions=self.get_gh_actions_matrix(), code_file_filter=self._code_file_filter, is_experimental=self._is_experimental)) elif ci_file.is_file(): ci_file.unlink() return ci_file def make_linux(self) -> PathPlus: """ Create, update or remove the Linux action, as appropriate. """ platform_name = "Linux" ci_file = self.workflows_dir / f"python_ci_{platform_name.lower()}.yml" if platform_name in self.templates.globals["platforms"]: conda_pip_dependencies = ["mkrecipe"] pyproject_file = PathPlus(self.repo_path / "pyproject.toml") if pyproject_file.is_file(): data: DefaultDict[str, Any] = DefaultDict( dom_toml.load(pyproject_file)) conda_pip_dependencies.extend(data["build-system"]["requires"]) ci_file.write_clean( self.actions.render( no_dev_versions=no_dev_versions, python_versions=set_gh_actions_versions( self.get_linux_ci_versions()), ci_platform=platform_ci_names[platform_name], ci_name=platform_name, dependency_lines=self.get_linux_ci_requirements(), gh_actions_versions=self.get_gh_actions_matrix(), code_file_filter=self._code_file_filter, run_on_tags=" tags:\n - '*'", is_experimental=self._is_experimental, conda_pip_dependencies=conda_pip_dependencies, )) elif ci_file.is_file(): ci_file.unlink() return ci_file def make_rustpython(self) -> PathPlus: """ Create, update or remove the RustPython action, as appropriate. """ platform_name = "Linux" template = self.templates.get_template("github_ci_rustpython.yml") ci_file = self.workflows_dir / f"rustpython_ci_{platform_name.lower()}.yml" if platform_name in self.templates.globals[ "platforms"] and "rustpython" in self.get_linux_ci_versions(): ci_file.write_clean( template.render( ci_platform=platform_ci_names[platform_name], dependency_lines=self.get_linux_ci_requirements(), code_file_filter=self._code_file_filter, )) elif ci_file.is_file(): ci_file.unlink() return ci_file def make_mypy(self) -> PathPlus: """ Create, update or remove the mypy action, as appropriate. .. versionadded:: 2020.1.27 """ ci_file = self.workflows_dir / "mypy.yml" template = self.templates.get_template(ci_file.name) # TODO: handle case where Linux is not a supported platform platforms = set(self.templates.globals["platforms"]) if "macOS" in platforms: platforms.remove("macOS") platforms = set( filter(None, (platform_ci_names.get(p, None) for p in platforms))) dependency_lines = self.get_linux_mypy_requirements() linux_platform = platform_ci_names["Linux"] if dependency_lines == self.standard_python_install_lines: dependencies_block = StringList([ "- name: Install dependencies 🔧", " run: |", ]) with dependencies_block.with_indent(" ", 2): dependencies_block.extend(self.standard_python_install_lines) else: dependencies_block = StringList([ "- name: Install dependencies (Linux) 🔧", f" if: ${{{{ matrix.os == '{linux_platform}' && steps.changes.outputs.code == 'true' }}}}", " run: |", ]) with dependencies_block.with_indent(" ", 2): dependencies_block.extend(dependency_lines) if self.templates.globals["platforms"] != ["Linux"]: dependencies_block.blankline(ensure_single=True) dependencies_block.extend([ "- name: Install dependencies (Win/mac) 🔧", f" if: ${{{{ matrix.os != '{linux_platform}' && steps.changes.outputs.code == 'true' }}}}", " run: |", ]) with dependencies_block.with_indent(" ", 2): dependencies_block.extend( self.standard_python_install_lines) ci_file.write_clean( template.render( platforms=sorted(platforms), linux_platform=platform_ci_names["Linux"], dependencies_block=indent(str(dependencies_block), " "), code_file_filter=self._code_file_filter, )) return ci_file def make_flake8(self) -> PathPlus: """ Create, update or remove the flake8 action, as appropriate. .. versionadded:: 2021.8.11 """ ci_file = self.workflows_dir / "flake8.yml" template = self.templates.get_template(ci_file.name) # TODO: handle case where Linux is not a supported platform ci_file.write_clean( template.render(code_file_filter=self._code_file_filter)) return ci_file def get_windows_ci_versions(self) -> List[str]: """ Returns the Python versions to run tests for on Windows. """ py_versions: List[str] = list( self.templates.globals["python_versions"]) if not self.templates.globals["pure_python"] and "3.8" in py_versions: py_versions.remove( "3.8" ) # FIXME: Python 3.8 tests fail on Windows for native wheels. return py_versions def get_linux_ci_versions(self) -> List[str]: """ Returns the Python versions to run tests for on Linux. """ return self.templates.globals["python_versions"] def get_macos_ci_versions(self) -> List[str]: """ Returns the Python versions to run tests for on macOS. """ py_versions: List[str] = list( self.templates.globals["python_versions"]) if "pypy36" in py_versions: py_versions.remove( "pypy36") # PyPy 3.6 requires patching on Big Sur return py_versions standard_python_install_lines = [ "python -VV", "python -m site", "python -m pip install --upgrade pip setuptools wheel", "python -m pip install --upgrade tox virtualenv", ] def _get_additional_requirements(self) -> Iterator[str]: if self.templates.globals["travis_additional_requirements"]: additional_requirements = DelimitedList( self.templates.globals["travis_additional_requirements"]) yield f"python -m pip install --upgrade {additional_requirements: }" def get_windows_ci_requirements(self) -> List[str]: """ Returns the Python requirements to run tests for on Windows. """ dependency_lines = StringList( self.templates.globals["github_ci_requirements"]["Windows"]["pre"]) dependency_lines.extend(self.standard_python_install_lines) dependency_lines.extend(self._get_additional_requirements()) dependency_lines.extend( self.templates.globals["github_ci_requirements"]["Windows"] ["post"]) return dependency_lines def get_linux_ci_requirements(self) -> List[str]: """ Returns the Python requirements to run tests for on Linux. """ dependency_lines = StringList( self.templates.globals["github_ci_requirements"]["Linux"]["pre"]) dependency_lines.extend(self.standard_python_install_lines) if self.templates.globals["enable_tests"]: dependency_lines.append( "python -m pip install --upgrade coverage_pyver_pragma") dependency_lines.extend(self._get_additional_requirements()) dependency_lines.extend( self.templates.globals["github_ci_requirements"]["Linux"]["post"]) return dependency_lines def get_linux_mypy_requirements(self) -> List[str]: """ Returns the Python requirements to run tests for on Linux. """ dependency_lines = StringList( self.templates.globals["github_ci_requirements"]["Linux"]["pre"]) dependency_lines.extend(self.standard_python_install_lines) dependency_lines.extend( self.templates.globals["github_ci_requirements"]["Linux"]["post"]) return dependency_lines def get_macos_ci_requirements(self) -> List[str]: """ Returns the Python requirements to run tests for on macOS. """ dependency_lines = StringList( self.templates.globals["github_ci_requirements"]["macOS"]["pre"]) dependency_lines.extend(self.standard_python_install_lines) dependency_lines.extend(self._get_additional_requirements()) dependency_lines.extend( self.templates.globals["github_ci_requirements"]["macOS"]["post"]) return dependency_lines
class Cache: """ Cache function arguments to and in-memory dictionary and a JSON file. :param app_name: The name of the app. This dictates the name of the cache directory. """ app_name: str #: The name of the app. This dictates the name of the cache directory. cache_dir: PathPlus #: The location of the cache directory on disk. caches: Dict[str, Dict[str, Any]] #: Mapping of function names to their caches. def __init__(self, app_name: str): self.app_name: str = str(app_name) self.cache_dir = PathPlus( platformdirs.user_cache_dir(f"{self.app_name}_cache")) self.cache_dir.maybe_make(parents=True) # Mapping of function names to their caches self.caches: Dict[str, Dict[str, Any]] = {} def clear(self, func: Optional[Callable] = None) -> bool: """ Clear the cache. :param func: Optional function to clear the cache for. By default, the whole cache is cleared. :no-default func: :returns: True to indicate success. False otherwise. """ try: if func is None: shutil.rmtree(self.cache_dir) self.cache_dir.maybe_make() for function in self.caches: self.caches[function] = {} else: function_name = func.__name__ cache_file = self.cache_dir / f"{function_name}.json" if cache_file.is_file(): cache_file.unlink() if function_name in self.caches: del self.caches[function_name] self.caches[function_name] = {} return True except Exception as e: # pragma: no cover warnings.warn(f"Could not remove cache. The error was: {e}") return False def load_cache(self, func: Callable) -> None: """ Loads the cache for the given function. :param func: """ cache_file: PathPlus = self.cache_dir / f"{func.__name__}.json" if cache_file.is_file(): cache = json.loads(cache_file.read_text()) else: cache = {} self.caches[func.__name__] = cache return cache def __call__(self, func: Callable): """ Decorator to cache the return values of a function based on its inputs. :param func: """ function_name = func.__name__ posargs: Iterable[str] = inspect.getfullargspec(func).args cache_file: PathPlus = self.cache_dir / f"{function_name}.json" self.load_cache(func) @wraps(func) def wrapper(*args, **kwargs: Any): kwargs = posargs2kwargs(args, posargs, kwargs) key: str = json.dumps(kwargs) response: Any cache = self.caches[function_name] if key in cache: # Return cached response response = json.loads(cache[key]) else: response = func(**kwargs) if response is not None: # Don't cache None values. cache[key] = json.dumps(response) cache_file.write_text(json.dumps(cache)) return response return wrapper
"cache_dir", "resolve_url", "cache_dir_for_url", "download_objects_inv", "find_url", ] __author__: str = "Dominic Davis-Foster" __copyright__: str = "2021 Dominic Davis-Foster" __license__: str = "MIT License" __version__: str = "0.2.1" __email__: str = "*****@*****.**" #: Directory in which cached files are stored. cache_dir = PathPlus(appdirs.user_cache_dir("searchdocs")) cache_dir.maybe_make(parents=True) def resolve_url(url: Union[str, RequestsURL]) -> RequestsURL: """ Resolve any redirects in the given URL. :param url: """ return RequestsURL(RequestsURL(url).head(allow_redirects=True).url) @functools.lru_cache() def cache_dir_for_url(url: Union[str, URL]) -> PathPlus: """