def _parse_pip_requirements(pip_requirements):
"""
Parses an iterable of pip requirement strings or a pip requirements file.
:param pip_requirements: Either an iterable of pip requirement strings
(e.g. ``["scikit-learn", "-r requirements.txt"]``) or the string path to a pip requirements
file on the local filesystem (e.g. ``"requirements.txt"``). If ``None``, an empty list will
be returned.
:return: A tuple of parsed requirements and constraints.
"""
if pip_requirements is None:
return [], []
def _is_string(x):
return isinstance(x, str)
def _is_iterable(x):
try:
iter(x)
return True
except Exception:
return False
if _is_string(pip_requirements):
requirements = []
constraints = []
for req_or_con in _parse_requirements(pip_requirements,
is_constraint=False):
if req_or_con.is_constraint:
constraints.append(req_or_con.req_str)
else:
requirements.append(req_or_con.req_str)
return requirements, constraints
elif _is_iterable(pip_requirements) and all(
map(_is_string, pip_requirements)):
try:
# Create a temporary requirements file in the current working directory
tmp_req_file = tempfile.NamedTemporaryFile(
mode="w",
prefix="mlflow.",
suffix=".tmp.requirements.txt",
dir=os.getcwd(),
# Setting `delete` to True causes a permission-denied error on Windows
# while trying to read the generated temporary file.
delete=False,
)
tmp_req_file.write("\n".join(pip_requirements))
tmp_req_file.close()
return _parse_pip_requirements(tmp_req_file.name)
finally:
# Clean up the temporary requirements file
os.remove(tmp_req_file.name)
else:
raise TypeError(
"`pip_requirements` must be either a string path to a pip requirements file on the "
"local filesystem or an iterable of pip requirement strings, but got `{}`"
.format(type(pip_requirements)))
def _get_virtualenv_name(python_env, work_dir_path, env_id=None):
requirements = _parse_requirements(
python_env.dependencies,
is_constraint=False,
base_dir=work_dir_path,
)
return _get_mlflow_env_name(
str(python_env) + "".join(map(lambda x: x.req_str, requirements)) +
(env_id or ""))
def _parse_pip_requirements(pip_requirements):
"""
Parses an iterable of pip requirement strings or a pip requirements file.
:param pip_requirements: Either an iterable of pip requirement strings
(e.g. ``["scikit-learn", "-r requirements.txt"]``) or the string path to a pip requirements
file on the local filesystem (e.g. ``"requirements.txt"``). If ``None``, an empty list will
be returned.
:return: A tuple of parsed requirements and constraints.
"""
if pip_requirements is None:
return [], []
def _is_string(x):
return isinstance(x, str)
def _is_iterable(x):
try:
iter(x)
return True
except Exception:
return False
if _is_string(pip_requirements):
with open(pip_requirements) as f:
return _parse_pip_requirements(f.read().splitlines())
elif _is_iterable(pip_requirements) and all(map(_is_string, pip_requirements)):
requirements = []
constraints = []
for req_or_con in _parse_requirements(pip_requirements, is_constraint=False):
if req_or_con.is_constraint:
constraints.append(req_or_con.req_str)
else:
requirements.append(req_or_con.req_str)
return requirements, constraints
else:
raise TypeError(
"`pip_requirements` must be either a string path to a pip requirements file on the "
"local filesystem or an iterable of pip requirement strings, but got `{}`".format(
type(pip_requirements)
)
)
def test_parse_requirements(request, tmpdir):
"""
Ensures `_parse_requirements` returns the same result as `pip._internal.req.parse_requirements`
"""
from pip._internal.req import parse_requirements as pip_parse_requirements
from pip._internal.network.session import PipSession
root_req_src = """
# No version specifier
noverspec
no-ver-spec
# Version specifiers
verspec<1.0
ver-spec == 2.0
# Environment marker
env-marker; python_version < "3.8"
inline-comm # Inline comment
inlinecomm # Inline comment
# Git URIs
git+https://github.com/git/uri
git+https://github.com/sub/dir#subdirectory=subdir
# Requirements files
-r {relative_req}
--requirement {absolute_req}
# Constraints files
-c {relative_con}
--constraint {absolute_con}
# Line continuation
line-cont\
==\
1.0
# Line continuation with spaces
line-cont-space \
== \
1.0
# Line continuation with a blank line
line-cont-blank\
# Line continuation at EOF
line-cont-eof\
""".strip()
try:
os.chdir(tmpdir)
root_req = tmpdir.join("requirements.txt")
# Requirements files
rel_req = tmpdir.join("relative_req.txt")
abs_req = tmpdir.join("absolute_req.txt")
# Constraints files
rel_con = tmpdir.join("relative_con.txt")
abs_con = tmpdir.join("absolute_con.txt")
# pip's requirements parser collapses an absolute requirements file path:
# https://github.com/pypa/pip/issues/10121
# As a workaround, use a relative path on Windows.
absolute_req = abs_req.basename if os.name == "nt" else abs_req.strpath
absolute_con = abs_con.basename if os.name == "nt" else abs_con.strpath
root_req.write(
root_req_src.format(
relative_req=rel_req.basename,
absolute_req=absolute_req,
relative_con=rel_con.basename,
absolute_con=absolute_con,
))
rel_req.write("rel-req-xxx\nrel-req-yyy")
abs_req.write("abs-req-zzz")
rel_con.write("rel-con-xxx\nrel-con-yyy")
abs_con.write("abs-con-zzz")
expected_cons = [
"rel-con-xxx",
"rel-con-yyy",
"abs-con-zzz",
]
expected_reqs = [
"noverspec",
"no-ver-spec",
"verspec<1.0",
"ver-spec == 2.0",
'env-marker; python_version < "3.8"',
"inline-comm",
"inlinecomm",
"git+https://github.com/git/uri",
"git+https://github.com/sub/dir#subdirectory=subdir",
"rel-req-xxx",
"rel-req-yyy",
"abs-req-zzz",
"line-cont==1.0",
"line-cont-space == 1.0",
"line-cont-blank",
"line-cont-eof",
]
parsed_reqs = list(
_parse_requirements(root_req.basename, is_constraint=False))
pip_reqs = list(
pip_parse_requirements(root_req.basename, session=PipSession()))
# Requirements
assert [r.req_str for r in parsed_reqs
if not r.is_constraint] == expected_reqs
assert [r.requirement for r in pip_reqs
if not r.constraint] == expected_reqs
# Constraints
assert [r.req_str for r in parsed_reqs
if r.is_constraint] == expected_cons
assert [r.requirement for r in pip_reqs
if r.constraint] == expected_cons
finally:
os.chdir(request.config.invocation_dir)
