def find_missing_requirements(self):
"""
Find the required packages that are not in the requirements.txt file.
:return: set of missing packages.
:rtype: set[Requirement]
"""
requirements = self._reduce_by_version(self._get_requirements_dict_from_py_files())
debug('requirements:')
debug(pformat(requirements))
needed = []
needed.extend(sorted(compress_list(unique_list(requirements))))
filename = 'requirements.txt'
if not os.path.exists(filename):
debug("Missing: " + filename)
diff = sorted(set(needed))
else:
with open(filename) as in_file:
existing_requirements = []
for line in [line.strip() for line in in_file.readlines()]:
if line and not line.startswith('#'):
existing_requirements.append(Requirement(line))
existing = sorted(compress_list(unique_list(existing_requirements)))
difference = [req for req in needed if req not in existing]
diff = sorted(set([req for req in difference
if not req.markers or Requirement(req.package) not in needed]))
debug("find_missing_requirements.needed: {pkgs}".format(pkgs=pformat(needed)))
debug("find_missing_requirements.diff: {pkgs}".format(pkgs=pformat(diff)))
return diff
def required_files(self):
"""
Add required packages (specified in module docstrings) to the appropriate requirements text file(s).
"""
debug("requiredFiles")
needed_dict = Requirements(self._project).find_missing_requirements()
for filename in needed_dict.keys():
needed = needed_dict[filename]
debug("needed: %s" % repr(needed))
try:
requirements_filename = os.path.join(self._project.herringfile_dir, filename)
if not os.path.isfile(requirements_filename):
with open(requirements_filename, 'w') as req_file:
req_file.write('-e .\n\n')
with open(requirements_filename, 'a') as req_file:
for need in sorted(unique_list(list(needed))):
out_line = need.qualified(qualifiers=True)
if out_line:
req_file.write(out_line + "\n")
for need in sorted(unique_list(list(needed))):
out_line = need.qualified(qualifiers=False)
if out_line:
req_file.write(out_line + "\n")
except IOError as ex:
warning("Can not add the following to the {filename} file: {needed}\n{err}".format(
filename=filename, needed=repr(needed), err=str(ex)))
def test_unique_list():
"""removes duplicate entries in the list"""
assert unique_list([]) == []
assert unique_list([1]) == [1]
assert unique_list([1, 2]) == [1, 2]
assert unique_list([1, 1]) == [1]
assert unique_list([1, 1, 1]) == [1]
assert unique_list([1, 1, 2]) == [1, 2]
assert unique_list([1, 2, 1]) == [1, 2]
assert unique_list([2, 1, 1]) == [2, 1]
def mkvenvs():
"""Make virturalenvs used for wheel building. Requires Project.wheel_python_versions and virtualenvwrapper."""
venvs = VirtualenvInfo('python_versions')
if not venvs.in_virtualenv and venvs.defined:
for venv_info in venvs.infos(exists=False):
requirement_files = ['requirements.txt']
versioned_requirement_file = Project.versioned_requirements_file_format.format(ver=venv_info.ver)
if os.path.isfile(versioned_requirement_file):
requirement_files.append(versioned_requirement_file)
for requirement_file in unique_list(requirement_files):
with open(requirement_file) as file_:
requirements = file_.readlines()
# info(requirement_file)
# info('=' * len(requirement_file))
# info(pformat(requirements))
install_lines = [
'pip install --upgrade {pip_options} pip ; '.format(pip_options=Project.pip_options),
'pip install --upgrade {pip_options} setuptools ; '.format(pip_options=Project.pip_options),
'pip install --upgrade {pip_options} requests[security] ;'.format(pip_options=Project.pip_options),
]
if 'numpy' in requirements:
install_lines.append('pip install {pip_options} numpy ; '.format(pip_options=Project.pip_options))
if 'matplotlib' in requirements:
install_lines.append('pip install {pip_options} matplotlib ; '.format(pip_options=Project.pip_options))
for requirement_file in unique_list(requirement_files):
install_lines.append('pip install {pip_options} -r {requirement_file} ; '.format(
pip_options=Project.pip_options, requirement_file=requirement_file))
venv_info.mkvirtualenv()
info(''.join(install_lines))
venv_info.run(''.join(install_lines))
else:
info("To build with wheels, in your herringfile you must set Project.wheel_python_versions to a list"
"of compact version, for example: ['27', '33', '34'] will build wheels for "
"python 2.7, 3.3, and 3.4")
return
def find_missing_requirements(self, lib_files=None):
"""
Find the required packages that are not in the requirements.txt file.
:param lib_files: list of filenames to scan for requirement comments. Set to None to scan all py files
in the herringlib directory and the herringfile.
:type lib_files: None|list[str]
:return: key is requirement filename and value is the set of missing packages.
:rtype: dict[str,set[Requirement]]
"""
requirements_dict = self._get_requirements_dict_from_py_files(lib_files=lib_files)
diff_dict = {}
for requirement_filename in requirements_dict.keys():
requirements = self._reduce_by_version(requirements_dict[requirement_filename])
debug('requirements:')
debug(pformat(requirements))
needed = []
needed.extend(sorted(compress_list(unique_list(requirements))))
if not os.path.exists(requirement_filename):
debug("Missing: " + requirement_filename)
diff_dict[requirement_filename] = sorted(set(needed))
else:
with open(requirement_filename) as in_file:
existing_requirements = []
for line in [line.strip() for line in in_file.readlines()]:
if line and not line.startswith('#'):
existing_requirements.append(Requirement(line))
existing = sorted(compress_list(unique_list(existing_requirements)))
difference = [req for req in needed if req not in existing]
diff_dict[requirement_filename] = sorted(set([req for req in difference
if not req.markers or
Requirement(req.package) not in needed]))
debug("find_missing_requirements.needed: {pkgs}".format(pkgs=pformat(needed)))
debug("find_missing_requirements.diff: {pkgs}".format(pkgs=pformat(diff_dict[requirement_filename])))
return diff_dict
def mkvenvs():
"""
Make populated virtual environments. Requires Project.python_versions, Project.docs_venv, and virtualenvwrapper.
Currently there are two possibly overlapping sets of virtual environments, the set defined by 'python_versions',
and the virtual environment specified by 'docs_venv'. So we iterate across both sets and use the if exists
to avoid the overlap. Each set can have a set of requirements.txt files used to populate the virtual environment.
"""
for attr_name in Project.virtualenv_requirements.keys():
requirement_files = Project.virtualenv_requirements[attr_name]
venvs = VirtualenvInfo(attr_name)
if venvs.in_virtualenv:
warning('Please deactivate the current virtual environment then try running this task again.')
return
if venvs.defined:
pip_options = '{pip_opts}'.format(pip_opts=Project.pip_options)
for venv_info in venvs.infos(exists=False):
if venv_info.exists():
info("{venv} already exists".format(venv=venv_info.venv))
continue
versioned_requirement_file = Project.versioned_requirements_file_format.format(ver=venv_info.ver)
if os.path.isfile(versioned_requirement_file):
requirement_files.append(versioned_requirement_file)
for requirement_file in unique_list(requirement_files):
with open(requirement_file) as file_:
requirements = file_.readlines()
# info(requirement_file)
# info('=' * len(requirement_file))
# info(pformat(requirements))
install_lines = [
'pip install --upgrade pip',
'pip install wheel',
'pip install --upgrade {pip_options} setuptools'.format(pip_options=pip_options),
'pip install --upgrade {pip_options} cryptography'.format(pip_options=pip_options),
'pip install --upgrade {pip_options} pyopenssl ndg-httpsclient pyasn1'.format(
pip_options=pip_options),
'pip install --upgrade {pip_options} requests[security]'.format(pip_options=pip_options),
]
if 'numpy' in requirements:
install_lines.append('pip install --upgrade {pip_options} numpy'.format(pip_options=pip_options))
if 'matplotlib' in requirements:
install_lines.append('pip install --upgrade {pip_options} matplotlib'.format(
pip_options=pip_options))
for requirement_file in unique_list(requirement_files):
install_lines.append('pip install --upgrade {pip_options} --pre -r {requirement_file}'.format(
pip_options=pip_options, requirement_file=requirement_file))
venv_info.mkvirtualenv()
for line in install_lines:
# noinspection PyArgumentEqualDefault
venv_info.run(line, verbose=True)
else:
info("To build with wheels, in your herringfile you must set Project.wheel_python_versions to a list"
"of compact version, for example: ['27', '33', '34'] will build wheels for "
"python 2.7, 3.3, and 3.4")