def check_fn(obtained_filename: PathPlus,
expected_filename: PathLike):
expected_filename = PathPlus(expected_filename)
template = Template(
expected_filename.read_text(),
block_start_string="<%",
block_end_string="%>",
variable_start_string="<<",
variable_end_string=">>",
comment_start_string="<#",
comment_end_string="#>",
)
expected_filename.write_text(
template.render(
sphinx_version=sphinx.version_info,
python_version=sys.version_info,
docutils_version=docutils_version,
**jinja2_namespace or {},
))
return check_text_files(obtained_filename,
expected_filename,
encoding="UTF-8")
def run(self, filename: PathLike):
"""
Parse configuration from the given file.
:param filename: The filename of the YAML configuration file.
"""
filename = PathPlus(filename)
if not filename.is_file():
raise FileNotFoundError(str(filename))
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir_p = PathPlus(tmpdir)
schema_file = tmpdir_p / "schema.json"
schema = make_schema(*self.config_vars)
schema["additionalProperties"] = self.allow_unknown_keys
schema_file.dump_json(schema)
validate_files(schema_file, filename)
parsed_config_vars: MutableMapping[str, Any] = {}
with filename.open() as file:
raw_config_vars: Mapping[str, Any] = YAML(typ="safe",
pure=True).load(file)
for var in self.config_vars:
parsed_config_vars[var.__name__] = getattr(
self, f"visit_{var.__name__}", var.get)(raw_config_vars)
return self.custom_parsing(raw_config_vars, parsed_config_vars,
filename)
def convert_notebook(
nb_file: PathLike,
outfile: PathLike,
):
"""
Convert a notebook to a Python file.
:param nb_file: Filename of the Jupyter Notebook to convert.
:param outfile: The filename to store the Python output as.
"""
nb_file = PathPlus(nb_file)
outfile = PathPlus(outfile)
outfile.parent.maybe_make()
script, *_ = py_exporter.from_file(str(nb_file))
outfile.write_clean(script)
with importlib_resources.path("notebook2script", "isort.cfg") as isort_config:
with importlib_resources.path("notebook2script", "style.yapf") as yapf_style:
reformat_file(outfile, yapf_style=str(yapf_style), isort_config_file=str(isort_config))
linter.process_file(outfile)
with open(outfile, "r+b") as f:
fix_encoding_pragma(f, remove=True, expected_pragma=b"# coding: utf-8")
def maybe_make(directory: PathLike, mode: int = 0o777, parents: bool = False):
"""
Create a directory at the given path, but only if the directory does not already exist.
.. attention::
This will fail silently if a file with the same name already exists.
This appears to be due to the behaviour of :func:`os.mkdir`.
:param directory: Directory to create
:param mode: Combined with the process's umask value to determine the file mode and access flags
:param parents: If :py:obj:`False` (the default), a missing parent raises a :class:`FileNotFoundError`.
If :py:obj:`True`, any missing parents of this path are created as needed; they are created with the
default permissions without taking mode into account (mimicking the POSIX ``mkdir -p`` command).
:no-default parents:
.. versionchanged:: 1.6.0 Removed the ``'exist_ok'`` option, since it made no sense in this context.
"""
if not isinstance(directory, pathlib.Path):
directory = pathlib.Path(directory)
try:
directory.mkdir(mode, parents, exist_ok=True)
except FileExistsError:
pass
def get_metadata_for_file(filename: PathLike) -> Dict[str, Any]:
"""
Returns the EXIF metadata for ``filename``, as a ``key: value`` mapping.
:param filename:
"""
filename = PathPlus(filename)
if not filename.is_file():
raise FileNotFoundError(filename)
# get the tags
with filename.open("rb") as fp:
data = exifread.process_file(fp, details=False, debug=False)
if data:
return {k: str(v) for k, v in data.items()}
else:
# using exiftool as a backup for some files including videos
with exiftool.ExifTool() as et:
try:
data = et.get_metadata(str(filename))
except json.decoder.JSONDecodeError:
raise ValueError(
f"Could not parse EXIF data for {filename} or no EXIF data found."
)
return dict(data)
def dump(
data: Mapping[str, Any],
filename: PathLike,
encoder: Union[Type[toml.TomlEncoder],
toml.TomlEncoder] = toml.TomlEncoder,
) -> str:
r"""
Writes out ``data`` as TOML to the given file.
:param data:
:param filename: The filename to write to.
:param encoder: The :class:`toml.TomlEncoder` to use for constructing the output string.
:returns: A string containing the ``TOML`` corresponding to ``data``.
.. versionchanged:: 0.5.0
The default value for ``encoder`` changed from :py:obj:`None` to :class:`toml.TomlEncoder`
Explicitly passing ``encoder=None`` is deprecated and support will be removed in 1.0.0
.. latex:clearpage::
"""
filename = PathPlus(filename)
as_toml = dumps(data, encoder=encoder)
filename.write_clean(as_toml)
return as_toml
def bump_version_for_file(self, filename: PathLike, config: BumpversionFileConfig): """ Bumps the version for the given file. :param filename: :param config: """ filename = self.repo.target_repo / filename filename.write_text(filename.read_text().replace(config["search"], config["replace"]))
def make_recipe(project_dir: PathLike, recipe_file: PathLike) -> None:
"""
Make a Conda ``meta.yaml`` recipe.
:param project_dir: The project directory.
:param recipe_file: The file to save the recipe as.
"""
recipe_file = PathPlus(recipe_file)
recipe_file.parent.maybe_make(parents=True)
recipe_file.write_clean(MaryBerry(project_dir).make())
def sort_requirements(filename: PathLike, allow_git: bool = False) -> int:
"""
Sort the requirements in the given file alphabetically.
:param filename: The file to sort the requirements in.
:param allow_git: Whether to allow lines that start with ``git+``, which are allowed by pip but not :pep:`508`.
"""
ret = PASS
filename = PathPlus(filename)
comments: List[str]
requirements: Set[ComparableRequirement]
git_lines: List[str] = []
requirements, comments, invalid_lines = read_requirements(
req_file=filename,
include_invalid=True,
normalize_func=normalize_keep_dot,
)
for line in invalid_lines:
if line.startswith("git+") and allow_git:
git_lines.append(line)
else:
ret |= FAIL
# find and remove pkg-resources==0.0.0
# which is automatically added by broken pip package under Debian
if ComparableRequirement("pkg-resources==0.0.0") in requirements:
requirements.remove(ComparableRequirement("pkg-resources==0.0.0"))
ret |= FAIL
sorted_requirements = sorted(requirements)
buf = StringList(
[*comments, *git_lines, *[str(req) for req in sorted_requirements]])
buf.blankline(ensure_single=True)
if (requirements != sorted_requirements
and buf != filename.read_lines()) or ret:
print('\n'.join(buf))
# print(coloured_diff(
# filename.read_lines(),
# buf,
# str(filename),
# str(filename),
# "(original)",
# "(sorted)",
# lineterm='',
# ))
ret |= FAIL
filename.write_lines(buf)
return ret
def export_wordcloud(word_cloud: WordCloud, outfile: PathLike) -> None:
"""
Export a wordcloud to a file.
:param word_cloud:
:param outfile: The file to export the wordcloud to.
"""
outfile = pathlib.Path(outfile)
if outfile.suffix == ".svg":
outfile.write_text(word_cloud.to_svg())
else:
word_cloud.to_file(str(outfile))
def __init__(
self,
lib_path: PathLike,
lib_type: int = NISTMS_MAIN_LIB,
work_dir: Optional[PathLike] = None,
debug: bool = False,
):
"""
:param lib_path: The path to the mass spectral library.
:param lib_type: The type of library. One of NISTMS_MAIN_LIB, NISTMS_USER_LIB, NISTMS_REP_LIB.
:param work_dir: The path to the working directory.
"""
if not isinstance(lib_path, pathlib.Path):
lib_path = pathlib.Path(lib_path)
if not lib_path.is_dir():
raise FileNotFoundError(
f"Library not found at the given path: {lib_path}")
if lib_type not in {NISTMS_MAIN_LIB, NISTMS_USER_LIB, NISTMS_REP_LIB}:
raise ValueError(
"`lib_type` must be one of NISTMS_MAIN_LIB, NISTMS_USER_LIB, NISTMS_REP_LIB."
)
# # Check if the server is already running
# for container in client.containers.list(all=True, filters={"status": "running"}):
# if container.name == "pyms-nist-server":
# self.docker = container
# break
# else:
#
self.debug: bool = bool(debug)
print("Launching Docker...")
try:
self.__launch_container(lib_path, lib_type)
except docker.errors.ImageNotFound:
print("Docker Image not found. Downloading now.")
client.images.pull("domdfcoding/pywine-pyms-nist")
self.__launch_container(lib_path, lib_type)
atexit.register(self.uninit)
retry_count = 0
# Wait for server to come online
while retry_count < 240:
try:
if requests.get("http://localhost:5001/").text == "ready":
self.initialised = True
return
except requests.exceptions.ConnectionError:
time.sleep(0.5)
retry_count += 1
raise TimeoutError("Unable to communicate with the search server.")
def make_recipe(repo_dir: PathLike, recipe_file: PathLike) -> None:
"""
Make a Conda ``meta.yaml`` recipe.
:param repo_dir: The repository directory.
:param recipe_file: The file to save the recipe as.
.. versionadded:: 2020.11.10
"""
# TODO: tests for this
repo_dir = PathPlus(repo_dir)
recipe_file = PathPlus(recipe_file)
recipe_file.write_clean(CondaRecipeMaker(repo_dir).make())
def read_expr_list(file_name: PathLike) -> List[Experiment]:
"""
Reads the set of experiment files and returns a list of :class:`pyms.Experiment.Experiment` objects.
:param file_name: The name of the file which lists experiment dump file names, one file per line.
:return: A list of Experiment instances.
:author: Vladimir Likic
"""
if not is_path(file_name):
raise TypeError("'file_name' must be a string or a PathLike object")
file_name = prepare_filepath(file_name, mkdirs=False)
fp = file_name.open(encoding="UTF-8")
exprfiles = fp.readlines()
fp.close()
expr_list = []
for exprfile in exprfiles:
exprfile = exprfile.strip()
expr = load_expr(exprfile)
expr_list.append(expr)
return expr_list
def write(self,
file_name: PathLike,
minutes: bool = False,
formatting: bool = True):
"""
Writes the ion chromatogram to the specified file.
:param file_name: The name of the output file
:param minutes: A boolean value indicating whether to write time in minutes
:param formatting: Whether to format the numbers in the output.
:authors: Lewis Lee, Vladimir Likic, Dominic Davis-Foster (pathlib support)
"""
if not is_path(file_name):
raise TypeError(
"'file_name' must be a string or a PathLike object")
file_name = prepare_filepath(file_name)
with file_name.open('w', encoding="UTF-8") as fp:
time_list = copy.deepcopy(self._time_list)
if minutes:
for ii in range(len(time_list)):
time_list[ii] = time_list[ii] / 60.0
for ii in range(len(time_list)):
if formatting:
fp.write(
f"{time_list[ii]:8.4f} {self._intensity_array[ii]:#.6e}\n"
)
else:
fp.write(f"{time_list[ii]} {self._intensity_array[ii]}\n")
def relpath(path: PathLike,
relative_to: Optional[PathLike] = None) -> pathlib.Path:
"""
Returns the path for the given file or directory relative to the given
directory or, if that would require path traversal, returns the absolute path.
:param path: Path to find the relative path for
:param relative_to: The directory to find the path relative to.
Defaults to the current directory.
:no-default relative_to:
""" # noqa D400
if not isinstance(path, pathlib.Path):
path = pathlib.Path(path)
abs_path = path.absolute()
if relative_to is None:
relative_to = pathlib.Path().absolute()
if not isinstance(relative_to, pathlib.Path):
relative_to = pathlib.Path(relative_to)
relative_to = relative_to.absolute()
try:
return abs_path.relative_to(relative_to)
except ValueError:
return abs_path
def make_document(
outfile: PathLike,
*elements: Iterable[str],
glossary: str = '',
):
r"""
Construct a LaTeX document from the given elements.
:param outfile:
:param \*elements:
:param glossary:
"""
outfile = PathPlus(outfile)
outfile.write_clean(
main_template.render(elements=elements, glossary=glossary))
def check_file(
self,
filename: PathLike,
extension: Optional[str] = None,
newline: Optional[str] = '\n',
**kwargs,
):
r"""
Check the content of the given text file against the reference file.
:param filename:
:param extension: The extension of the reference file.
If :py:obj:`None` the extension is determined from ``filename``.
:param newline: Controls how universal newlines mode works. See :func:`open`.
:param \*\*kwargs: Additional keyword arguments passed to
:meth:`pytest_regressions.file_regression.FileRegressionFixture.check`.
.. seealso:: :func:`~.check_file_output`
"""
filename = PathPlus(filename)
data = filename.read_text(encoding="UTF-8")
extension = extension or filename.suffix
if extension == ".py":
extension = "._py_"
__tracebackhide__ = True
return self.check(data, extension=extension, newline=newline, **kwargs)
def validate_files(
schemafile: PathLike,
*datafiles: PathLike,
encoding: str = "utf-8",
) -> None:
r"""
Validate the given datafiles against the given schema.
:param schemafile: The ``json`` or ``yaml`` formatted schema to validate with.
:param \*datafiles: The ``json`` or ``yaml`` files to validate.
:param encoding: Encoding to open the files with.
.. versionadded:: 0.4.0
"""
schemafile = pathlib.Path(schemafile)
yaml = YAML(typ="safe", pure=True)
schema = yaml.load(schemafile.read_text(encoding=encoding))
for filename in datafiles:
for document in yaml.load_all(
pathlib.Path(filename).read_text(encoding=encoding)):
try:
jsonschema.validate(document,
schema,
format_checker=jsonschema.FormatChecker())
except jsonschema.exceptions.ValidationError as e:
e.filename = str(filename)
raise e
def write_intensities_stream(self, file_name: PathLike):
"""
Loop over all scans and, for each scan, write the intensities to the
given file, one intensity per line.
Intensities from different scans are joined without any delimiters.
:param file_name: Output file name.
:authors: Vladimir Likic, Dominic Davis-Foster (pathlib support)
""" # noqa: D400
if not is_path(file_name):
raise TypeError(
"'file_name' must be a string or a PathLike object")
file_name = prepare_filepath(file_name)
# n = len(self._scan_list)
print(" -> Writing scans to a file")
fp = file_name.open('w', encoding="UTF-8")
for scan in self._scan_list:
intensities = scan.intensity_list
for i in intensities:
fp.write(f"{i:8.4f}\n")
fp.close()
def rmdir(path: PathLike) -> str:
r"""
Remove the given directory and its contents.
:param path:
:returns: A message in the format :file:`'removing {<path>}'`
.. attention::
On Windows-like systems using ``\`` as a path separator
you may need to use a *raw string* for the path as the ``toxinidir``
substitution may contain backslashes:
.. code-block:: ini
recreate_hook = builtin.rmdir(r"{toxinidir}/doc-source/build")
"""
path = PathPlus(path)
if path.is_dir():
shutil.rmtree(path)
return f"removing {path!s}"
def load_file(self, filename: PathLike) -> Union[Dict, List]:
"""
Load the given YAML file and return its contents.
:param filename:
"""
filename = PathPlus(filename)
return self.load(filename.read_text())
def dump(
self,
filename: PathLike,
encoder: Union[Type[toml.TomlEncoder],
toml.TomlEncoder] = PyProjectTomlEncoder,
):
"""
Write as TOML to the given file.
:param filename: The filename to write to.
:param encoder: The :class:`toml.TomlEncoder` to use for constructing the output string.
:returns: A string containing the TOML representation.
"""
filename = PathPlus(filename)
as_toml = self.dumps(encoder=encoder)
filename.write_clean(as_toml)
return as_toml
def __init__(self, filename: PathLike):
if not isinstance(filename, pathlib.Path):
filename = PathPlus(filename)
self.filename: str = filename.as_posix()
"""
The file listing the packages to install.
.. latex:clearpage::
"""
super().__init__(f"Could not install from {self.filename!r}")
def savefig(fig: Figure, filename: PathLike, *args, **kwargs):
if "facecolor" not in kwargs:
kwargs["facecolor"] = fig.get_facecolor()
if "edgecolor" not in kwargs:
kwargs["edgecolor"] = None
filename = str(filename)
if filename.endswith(".svg"):
return save_svg(fig, filename, *args, **kwargs)
else:
return fig.savefig(filename, *args, **kwargs)
def is_datafile(file_name: PathLike) -> bool:
"""
Returns whether the given path is a valid data file.
:param file_name: name of the ``.d`` datafile
"""
if not isinstance(file_name, pathlib.Path):
try:
file_name = pathlib.Path(file_name)
except TypeError:
raise TypeError(
f"'file_name' must be a string or a PathLike object, not {type(file_name)}"
)
if file_name.exists():
if file_name.is_dir():
if ((file_name / "AcqData") / "Contents.xml").exists():
return True
return False
def from_jcamp(cls: Type[_M], file_name: PathLike) -> _M:
"""
Create a MassSpectrum from a JCAMP-DX file.
:param file_name: Path of the file to read.
:authors: Qiao Wang, Andrew Isaac, Vladimir Likic, David Kainer, Dominic Davis-Foster
"""
if not is_path(file_name):
raise TypeError(
"'file_name' must be a string or a PathLike object")
file_name = prepare_filepath(file_name, mkdirs=False)
print(f" -> Reading JCAMP file '{file_name}'")
lines_list = file_name.open('r', encoding="UTF-8")
xydata = []
last_tag = None
for line in lines_list:
if line.strip():
if line.startswith("##"):
# key word or information
fields = line.split('=', 1)
current_tag = fields[0] = fields[0].lstrip("##").upper()
last_tag = fields[0]
if current_tag.upper().startswith("END"):
break
else:
if last_tag in xydata_tags:
line_sub = re.split(r",| ", line.strip())
for item in line_sub:
if not len(item.strip()) == 0:
xydata.append(float(item.strip()))
# By this point we should have all of the xydata
if len(xydata) % 2 == 1:
# TODO: This means the data is not in x, y pairs
# Make a better error message
raise ValueError("data not in pair !")
mass_list = []
intensity_list = []
for i in range(len(xydata) // 2):
mass_list.append(xydata[i * 2])
intensity_list.append(xydata[i * 2 + 1])
return cls(mass_list, intensity_list)
def export_leco_csv(self, file_name: PathLike):
"""
Exports data in LECO CSV format.
:param file_name: The name of the output file.
:authors: Andrew Isaac, Vladimir Likic, Dominic Davis-Foster (pathlib support)
"""
if not is_path(file_name):
raise TypeError("'file_name' must be a string or a PathLike object")
file_name = prepare_filepath(file_name, mkdirs=False)
if not file_name.parent.is_dir():
file_name.parent.mkdir(parents=True)
mass_list = self._mass_list
time_list = self._time_list
vals = self._intensity_array
fp = file_name.open('w', encoding="UTF-8")
# Format is text header with:
# "Scan","Time",...
# and the rest is "TIC" or m/z as text, i.e. "50","51"...
# The following lines are:
# scan_number,time,value,value,...
# scan_number is an int, rest seem to be fixed format floats.
# The format is 0.000000e+000
# write header
fp.write('"Scan","Time"')
for ii in mass_list:
if is_number(ii):
fp.write(f',"{int(ii):d}"')
else:
raise TypeError("mass list datum not a number")
fp.write("\r\n") # windows CR/LF
# write lines
for ii, time_ in enumerate(time_list):
fp.write(f"{ii},{time_:#.6e}")
for jj in range(len(vals[ii])):
if is_number(vals[ii][jj]):
fp.write(f",{vals[ii][jj]:#.6e}")
else:
raise TypeError("datum not a number")
fp.write("\r\n")
fp.close()
def dump_to_file(self,
data: Union[MutableMapping, Sequence],
filename: PathLike,
mode: str = 'w'):
"""
Dump the given data to the specified file.
:param data:
:param filename:
:param mode:
"""
filename = PathPlus(filename)
if 'w' in mode:
filename.write_lines([
"# Configuration for 'repo_helper' (https://github.com/repo-helper/repo_helper)",
self.dumps(data, explicit_start=True),
])
elif 'a' in mode:
with filename.open('a') as fp:
fp.write('\n')
fp.write(self.dumps(data, explicit_start=False))
def frequency_from_directory(
directory: PathLike,
exclude_words: Sequence[str] = (),
exclude_dirs: Sequence[PathLike] = (),
) -> Counter:
"""
Returns a dictionary mapping the words in files in ``directory`` to their frequencies.
:param directory: The directory to process
:param exclude_words: An optional list of words to exclude
:param exclude_dirs: An optional list of directories to exclude.
.. versionadded:: 0.2.0
"""
# TODO: only certain file extensions
directory = pathlib.Path(directory).absolute()
exclude_dirs_list = [".git"]
for d in exclude_dirs:
d = pathlib.Path(d)
if d.is_absolute():
d = d.relative_to(directory)
exclude_dirs_list.append(str(d))
def is_excluded(path):
for dir_name in exclude_dirs_list:
if re.match(dir_name, path.relative_to(directory).as_posix()):
return True
return False
word_counts: typing.Counter[str] = Counter()
for file in directory.rglob("**/*.*"):
if file.is_file() and not is_excluded(file):
word_counts += get_tokens(file)
for word in exclude_words:
if word in word_counts:
del word_counts[word]
return word_counts
def load_object(file_name: PathLike) -> object:
"""
Loads an object previously dumped with :func:`~.dump_object`.
:param file_name: Name of the object dump file.
:return: Object contained in the file.
:authors: Vladimir Likic, Dominic Davis-Foster (pathlib support)
"""
if not is_path(file_name):
raise TypeError("'file_name' must be a string or a PathLike object")
file_name = prepare_filepath(file_name)
with file_name.open("wb") as fp:
return pickle.load(fp)
