def check_fn(obtained_filename: PathPlus, expected_filename: PathLike): expected_filename = PathPlus(expected_filename) template = Template( expected_filename.read_text(), block_start_string="<%", block_end_string="%>", variable_start_string="<<", variable_end_string=">>", comment_start_string="<#", comment_end_string="#>", ) expected_filename.write_text( template.render( sphinx_version=sphinx.version_info, python_version=sys.version_info, docutils_version=docutils_version, **jinja2_namespace or {}, )) return check_text_files(obtained_filename, expected_filename, encoding="UTF-8")
def run(self, filename: PathLike): """ Parse configuration from the given file. :param filename: The filename of the YAML configuration file. """ filename = PathPlus(filename) if not filename.is_file(): raise FileNotFoundError(str(filename)) with tempfile.TemporaryDirectory() as tmpdir: tmpdir_p = PathPlus(tmpdir) schema_file = tmpdir_p / "schema.json" schema = make_schema(*self.config_vars) schema["additionalProperties"] = self.allow_unknown_keys schema_file.dump_json(schema) validate_files(schema_file, filename) parsed_config_vars: MutableMapping[str, Any] = {} with filename.open() as file: raw_config_vars: Mapping[str, Any] = YAML(typ="safe", pure=True).load(file) for var in self.config_vars: parsed_config_vars[var.__name__] = getattr( self, f"visit_{var.__name__}", var.get)(raw_config_vars) return self.custom_parsing(raw_config_vars, parsed_config_vars, filename)
def convert_notebook( nb_file: PathLike, outfile: PathLike, ): """ Convert a notebook to a Python file. :param nb_file: Filename of the Jupyter Notebook to convert. :param outfile: The filename to store the Python output as. """ nb_file = PathPlus(nb_file) outfile = PathPlus(outfile) outfile.parent.maybe_make() script, *_ = py_exporter.from_file(str(nb_file)) outfile.write_clean(script) with importlib_resources.path("notebook2script", "isort.cfg") as isort_config: with importlib_resources.path("notebook2script", "style.yapf") as yapf_style: reformat_file(outfile, yapf_style=str(yapf_style), isort_config_file=str(isort_config)) linter.process_file(outfile) with open(outfile, "r+b") as f: fix_encoding_pragma(f, remove=True, expected_pragma=b"# coding: utf-8")
def maybe_make(directory: PathLike, mode: int = 0o777, parents: bool = False): """ Create a directory at the given path, but only if the directory does not already exist. .. attention:: This will fail silently if a file with the same name already exists. This appears to be due to the behaviour of :func:`os.mkdir`. :param directory: Directory to create :param mode: Combined with the process's umask value to determine the file mode and access flags :param parents: If :py:obj:`False` (the default), a missing parent raises a :class:`FileNotFoundError`. If :py:obj:`True`, any missing parents of this path are created as needed; they are created with the default permissions without taking mode into account (mimicking the POSIX ``mkdir -p`` command). :no-default parents: .. versionchanged:: 1.6.0 Removed the ``'exist_ok'`` option, since it made no sense in this context. """ if not isinstance(directory, pathlib.Path): directory = pathlib.Path(directory) try: directory.mkdir(mode, parents, exist_ok=True) except FileExistsError: pass
def get_metadata_for_file(filename: PathLike) -> Dict[str, Any]: """ Returns the EXIF metadata for ``filename``, as a ``key: value`` mapping. :param filename: """ filename = PathPlus(filename) if not filename.is_file(): raise FileNotFoundError(filename) # get the tags with filename.open("rb") as fp: data = exifread.process_file(fp, details=False, debug=False) if data: return {k: str(v) for k, v in data.items()} else: # using exiftool as a backup for some files including videos with exiftool.ExifTool() as et: try: data = et.get_metadata(str(filename)) except json.decoder.JSONDecodeError: raise ValueError( f"Could not parse EXIF data for {filename} or no EXIF data found." ) return dict(data)
def dump( data: Mapping[str, Any], filename: PathLike, encoder: Union[Type[toml.TomlEncoder], toml.TomlEncoder] = toml.TomlEncoder, ) -> str: r""" Writes out ``data`` as TOML to the given file. :param data: :param filename: The filename to write to. :param encoder: The :class:`toml.TomlEncoder` to use for constructing the output string. :returns: A string containing the ``TOML`` corresponding to ``data``. .. versionchanged:: 0.5.0 The default value for ``encoder`` changed from :py:obj:`None` to :class:`toml.TomlEncoder` Explicitly passing ``encoder=None`` is deprecated and support will be removed in 1.0.0 .. latex:clearpage:: """ filename = PathPlus(filename) as_toml = dumps(data, encoder=encoder) filename.write_clean(as_toml) return as_toml
def bump_version_for_file(self, filename: PathLike, config: BumpversionFileConfig): """ Bumps the version for the given file. :param filename: :param config: """ filename = self.repo.target_repo / filename filename.write_text(filename.read_text().replace(config["search"], config["replace"]))
def make_recipe(project_dir: PathLike, recipe_file: PathLike) -> None: """ Make a Conda ``meta.yaml`` recipe. :param project_dir: The project directory. :param recipe_file: The file to save the recipe as. """ recipe_file = PathPlus(recipe_file) recipe_file.parent.maybe_make(parents=True) recipe_file.write_clean(MaryBerry(project_dir).make())
def sort_requirements(filename: PathLike, allow_git: bool = False) -> int: """ Sort the requirements in the given file alphabetically. :param filename: The file to sort the requirements in. :param allow_git: Whether to allow lines that start with ``git+``, which are allowed by pip but not :pep:`508`. """ ret = PASS filename = PathPlus(filename) comments: List[str] requirements: Set[ComparableRequirement] git_lines: List[str] = [] requirements, comments, invalid_lines = read_requirements( req_file=filename, include_invalid=True, normalize_func=normalize_keep_dot, ) for line in invalid_lines: if line.startswith("git+") and allow_git: git_lines.append(line) else: ret |= FAIL # find and remove pkg-resources==0.0.0 # which is automatically added by broken pip package under Debian if ComparableRequirement("pkg-resources==0.0.0") in requirements: requirements.remove(ComparableRequirement("pkg-resources==0.0.0")) ret |= FAIL sorted_requirements = sorted(requirements) buf = StringList( [*comments, *git_lines, *[str(req) for req in sorted_requirements]]) buf.blankline(ensure_single=True) if (requirements != sorted_requirements and buf != filename.read_lines()) or ret: print('\n'.join(buf)) # print(coloured_diff( # filename.read_lines(), # buf, # str(filename), # str(filename), # "(original)", # "(sorted)", # lineterm='', # )) ret |= FAIL filename.write_lines(buf) return ret
def export_wordcloud(word_cloud: WordCloud, outfile: PathLike) -> None: """ Export a wordcloud to a file. :param word_cloud: :param outfile: The file to export the wordcloud to. """ outfile = pathlib.Path(outfile) if outfile.suffix == ".svg": outfile.write_text(word_cloud.to_svg()) else: word_cloud.to_file(str(outfile))
def __init__( self, lib_path: PathLike, lib_type: int = NISTMS_MAIN_LIB, work_dir: Optional[PathLike] = None, debug: bool = False, ): """ :param lib_path: The path to the mass spectral library. :param lib_type: The type of library. One of NISTMS_MAIN_LIB, NISTMS_USER_LIB, NISTMS_REP_LIB. :param work_dir: The path to the working directory. """ if not isinstance(lib_path, pathlib.Path): lib_path = pathlib.Path(lib_path) if not lib_path.is_dir(): raise FileNotFoundError( f"Library not found at the given path: {lib_path}") if lib_type not in {NISTMS_MAIN_LIB, NISTMS_USER_LIB, NISTMS_REP_LIB}: raise ValueError( "`lib_type` must be one of NISTMS_MAIN_LIB, NISTMS_USER_LIB, NISTMS_REP_LIB." ) # # Check if the server is already running # for container in client.containers.list(all=True, filters={"status": "running"}): # if container.name == "pyms-nist-server": # self.docker = container # break # else: # self.debug: bool = bool(debug) print("Launching Docker...") try: self.__launch_container(lib_path, lib_type) except docker.errors.ImageNotFound: print("Docker Image not found. Downloading now.") client.images.pull("domdfcoding/pywine-pyms-nist") self.__launch_container(lib_path, lib_type) atexit.register(self.uninit) retry_count = 0 # Wait for server to come online while retry_count < 240: try: if requests.get("http://localhost:5001/").text == "ready": self.initialised = True return except requests.exceptions.ConnectionError: time.sleep(0.5) retry_count += 1 raise TimeoutError("Unable to communicate with the search server.")
def make_recipe(repo_dir: PathLike, recipe_file: PathLike) -> None: """ Make a Conda ``meta.yaml`` recipe. :param repo_dir: The repository directory. :param recipe_file: The file to save the recipe as. .. versionadded:: 2020.11.10 """ # TODO: tests for this repo_dir = PathPlus(repo_dir) recipe_file = PathPlus(recipe_file) recipe_file.write_clean(CondaRecipeMaker(repo_dir).make())
def read_expr_list(file_name: PathLike) -> List[Experiment]: """ Reads the set of experiment files and returns a list of :class:`pyms.Experiment.Experiment` objects. :param file_name: The name of the file which lists experiment dump file names, one file per line. :return: A list of Experiment instances. :author: Vladimir Likic """ if not is_path(file_name): raise TypeError("'file_name' must be a string or a PathLike object") file_name = prepare_filepath(file_name, mkdirs=False) fp = file_name.open(encoding="UTF-8") exprfiles = fp.readlines() fp.close() expr_list = [] for exprfile in exprfiles: exprfile = exprfile.strip() expr = load_expr(exprfile) expr_list.append(expr) return expr_list
def write(self, file_name: PathLike, minutes: bool = False, formatting: bool = True): """ Writes the ion chromatogram to the specified file. :param file_name: The name of the output file :param minutes: A boolean value indicating whether to write time in minutes :param formatting: Whether to format the numbers in the output. :authors: Lewis Lee, Vladimir Likic, Dominic Davis-Foster (pathlib support) """ if not is_path(file_name): raise TypeError( "'file_name' must be a string or a PathLike object") file_name = prepare_filepath(file_name) with file_name.open('w', encoding="UTF-8") as fp: time_list = copy.deepcopy(self._time_list) if minutes: for ii in range(len(time_list)): time_list[ii] = time_list[ii] / 60.0 for ii in range(len(time_list)): if formatting: fp.write( f"{time_list[ii]:8.4f} {self._intensity_array[ii]:#.6e}\n" ) else: fp.write(f"{time_list[ii]} {self._intensity_array[ii]}\n")
def relpath(path: PathLike, relative_to: Optional[PathLike] = None) -> pathlib.Path: """ Returns the path for the given file or directory relative to the given directory or, if that would require path traversal, returns the absolute path. :param path: Path to find the relative path for :param relative_to: The directory to find the path relative to. Defaults to the current directory. :no-default relative_to: """ # noqa D400 if not isinstance(path, pathlib.Path): path = pathlib.Path(path) abs_path = path.absolute() if relative_to is None: relative_to = pathlib.Path().absolute() if not isinstance(relative_to, pathlib.Path): relative_to = pathlib.Path(relative_to) relative_to = relative_to.absolute() try: return abs_path.relative_to(relative_to) except ValueError: return abs_path
def make_document( outfile: PathLike, *elements: Iterable[str], glossary: str = '', ): r""" Construct a LaTeX document from the given elements. :param outfile: :param \*elements: :param glossary: """ outfile = PathPlus(outfile) outfile.write_clean( main_template.render(elements=elements, glossary=glossary))
def check_file( self, filename: PathLike, extension: Optional[str] = None, newline: Optional[str] = '\n', **kwargs, ): r""" Check the content of the given text file against the reference file. :param filename: :param extension: The extension of the reference file. If :py:obj:`None` the extension is determined from ``filename``. :param newline: Controls how universal newlines mode works. See :func:`open`. :param \*\*kwargs: Additional keyword arguments passed to :meth:`pytest_regressions.file_regression.FileRegressionFixture.check`. .. seealso:: :func:`~.check_file_output` """ filename = PathPlus(filename) data = filename.read_text(encoding="UTF-8") extension = extension or filename.suffix if extension == ".py": extension = "._py_" __tracebackhide__ = True return self.check(data, extension=extension, newline=newline, **kwargs)
def validate_files( schemafile: PathLike, *datafiles: PathLike, encoding: str = "utf-8", ) -> None: r""" Validate the given datafiles against the given schema. :param schemafile: The ``json`` or ``yaml`` formatted schema to validate with. :param \*datafiles: The ``json`` or ``yaml`` files to validate. :param encoding: Encoding to open the files with. .. versionadded:: 0.4.0 """ schemafile = pathlib.Path(schemafile) yaml = YAML(typ="safe", pure=True) schema = yaml.load(schemafile.read_text(encoding=encoding)) for filename in datafiles: for document in yaml.load_all( pathlib.Path(filename).read_text(encoding=encoding)): try: jsonschema.validate(document, schema, format_checker=jsonschema.FormatChecker()) except jsonschema.exceptions.ValidationError as e: e.filename = str(filename) raise e
def write_intensities_stream(self, file_name: PathLike): """ Loop over all scans and, for each scan, write the intensities to the given file, one intensity per line. Intensities from different scans are joined without any delimiters. :param file_name: Output file name. :authors: Vladimir Likic, Dominic Davis-Foster (pathlib support) """ # noqa: D400 if not is_path(file_name): raise TypeError( "'file_name' must be a string or a PathLike object") file_name = prepare_filepath(file_name) # n = len(self._scan_list) print(" -> Writing scans to a file") fp = file_name.open('w', encoding="UTF-8") for scan in self._scan_list: intensities = scan.intensity_list for i in intensities: fp.write(f"{i:8.4f}\n") fp.close()
def rmdir(path: PathLike) -> str: r""" Remove the given directory and its contents. :param path: :returns: A message in the format :file:`'removing {<path>}'` .. attention:: On Windows-like systems using ``\`` as a path separator you may need to use a *raw string* for the path as the ``toxinidir`` substitution may contain backslashes: .. code-block:: ini recreate_hook = builtin.rmdir(r"{toxinidir}/doc-source/build") """ path = PathPlus(path) if path.is_dir(): shutil.rmtree(path) return f"removing {path!s}"
def load_file(self, filename: PathLike) -> Union[Dict, List]: """ Load the given YAML file and return its contents. :param filename: """ filename = PathPlus(filename) return self.load(filename.read_text())
def dump( self, filename: PathLike, encoder: Union[Type[toml.TomlEncoder], toml.TomlEncoder] = PyProjectTomlEncoder, ): """ Write as TOML to the given file. :param filename: The filename to write to. :param encoder: The :class:`toml.TomlEncoder` to use for constructing the output string. :returns: A string containing the TOML representation. """ filename = PathPlus(filename) as_toml = self.dumps(encoder=encoder) filename.write_clean(as_toml) return as_toml
def __init__(self, filename: PathLike): if not isinstance(filename, pathlib.Path): filename = PathPlus(filename) self.filename: str = filename.as_posix() """ The file listing the packages to install. .. latex:clearpage:: """ super().__init__(f"Could not install from {self.filename!r}")
def savefig(fig: Figure, filename: PathLike, *args, **kwargs): if "facecolor" not in kwargs: kwargs["facecolor"] = fig.get_facecolor() if "edgecolor" not in kwargs: kwargs["edgecolor"] = None filename = str(filename) if filename.endswith(".svg"): return save_svg(fig, filename, *args, **kwargs) else: return fig.savefig(filename, *args, **kwargs)
def is_datafile(file_name: PathLike) -> bool: """ Returns whether the given path is a valid data file. :param file_name: name of the ``.d`` datafile """ if not isinstance(file_name, pathlib.Path): try: file_name = pathlib.Path(file_name) except TypeError: raise TypeError( f"'file_name' must be a string or a PathLike object, not {type(file_name)}" ) if file_name.exists(): if file_name.is_dir(): if ((file_name / "AcqData") / "Contents.xml").exists(): return True return False
def from_jcamp(cls: Type[_M], file_name: PathLike) -> _M: """ Create a MassSpectrum from a JCAMP-DX file. :param file_name: Path of the file to read. :authors: Qiao Wang, Andrew Isaac, Vladimir Likic, David Kainer, Dominic Davis-Foster """ if not is_path(file_name): raise TypeError( "'file_name' must be a string or a PathLike object") file_name = prepare_filepath(file_name, mkdirs=False) print(f" -> Reading JCAMP file '{file_name}'") lines_list = file_name.open('r', encoding="UTF-8") xydata = [] last_tag = None for line in lines_list: if line.strip(): if line.startswith("##"): # key word or information fields = line.split('=', 1) current_tag = fields[0] = fields[0].lstrip("##").upper() last_tag = fields[0] if current_tag.upper().startswith("END"): break else: if last_tag in xydata_tags: line_sub = re.split(r",| ", line.strip()) for item in line_sub: if not len(item.strip()) == 0: xydata.append(float(item.strip())) # By this point we should have all of the xydata if len(xydata) % 2 == 1: # TODO: This means the data is not in x, y pairs # Make a better error message raise ValueError("data not in pair !") mass_list = [] intensity_list = [] for i in range(len(xydata) // 2): mass_list.append(xydata[i * 2]) intensity_list.append(xydata[i * 2 + 1]) return cls(mass_list, intensity_list)
def export_leco_csv(self, file_name: PathLike): """ Exports data in LECO CSV format. :param file_name: The name of the output file. :authors: Andrew Isaac, Vladimir Likic, Dominic Davis-Foster (pathlib support) """ if not is_path(file_name): raise TypeError("'file_name' must be a string or a PathLike object") file_name = prepare_filepath(file_name, mkdirs=False) if not file_name.parent.is_dir(): file_name.parent.mkdir(parents=True) mass_list = self._mass_list time_list = self._time_list vals = self._intensity_array fp = file_name.open('w', encoding="UTF-8") # Format is text header with: # "Scan","Time",... # and the rest is "TIC" or m/z as text, i.e. "50","51"... # The following lines are: # scan_number,time,value,value,... # scan_number is an int, rest seem to be fixed format floats. # The format is 0.000000e+000 # write header fp.write('"Scan","Time"') for ii in mass_list: if is_number(ii): fp.write(f',"{int(ii):d}"') else: raise TypeError("mass list datum not a number") fp.write("\r\n") # windows CR/LF # write lines for ii, time_ in enumerate(time_list): fp.write(f"{ii},{time_:#.6e}") for jj in range(len(vals[ii])): if is_number(vals[ii][jj]): fp.write(f",{vals[ii][jj]:#.6e}") else: raise TypeError("datum not a number") fp.write("\r\n") fp.close()
def dump_to_file(self, data: Union[MutableMapping, Sequence], filename: PathLike, mode: str = 'w'): """ Dump the given data to the specified file. :param data: :param filename: :param mode: """ filename = PathPlus(filename) if 'w' in mode: filename.write_lines([ "# Configuration for 'repo_helper' (https://github.com/repo-helper/repo_helper)", self.dumps(data, explicit_start=True), ]) elif 'a' in mode: with filename.open('a') as fp: fp.write('\n') fp.write(self.dumps(data, explicit_start=False))
def frequency_from_directory( directory: PathLike, exclude_words: Sequence[str] = (), exclude_dirs: Sequence[PathLike] = (), ) -> Counter: """ Returns a dictionary mapping the words in files in ``directory`` to their frequencies. :param directory: The directory to process :param exclude_words: An optional list of words to exclude :param exclude_dirs: An optional list of directories to exclude. .. versionadded:: 0.2.0 """ # TODO: only certain file extensions directory = pathlib.Path(directory).absolute() exclude_dirs_list = [".git"] for d in exclude_dirs: d = pathlib.Path(d) if d.is_absolute(): d = d.relative_to(directory) exclude_dirs_list.append(str(d)) def is_excluded(path): for dir_name in exclude_dirs_list: if re.match(dir_name, path.relative_to(directory).as_posix()): return True return False word_counts: typing.Counter[str] = Counter() for file in directory.rglob("**/*.*"): if file.is_file() and not is_excluded(file): word_counts += get_tokens(file) for word in exclude_words: if word in word_counts: del word_counts[word] return word_counts
def load_object(file_name: PathLike) -> object: """ Loads an object previously dumped with :func:`~.dump_object`. :param file_name: Name of the object dump file. :return: Object contained in the file. :authors: Vladimir Likic, Dominic Davis-Foster (pathlib support) """ if not is_path(file_name): raise TypeError("'file_name' must be a string or a PathLike object") file_name = prepare_filepath(file_name) with file_name.open("wb") as fp: return pickle.load(fp)