def _create_module_diagrams(path):
"""
create module UML diagrams
:param path: the module path
:type path: str
"""
info("_create_module_diagrams")
if not executables_available(['pyreverse']):
warning('pyreverse not available')
return
with open(os.path.join(Project.docs_dir, "pyreverse.log"), "w") as outputter:
for module_path in [root for root, dirs, files in os.walk(path) if os.path.basename(root) != '__pycache__']:
debug("module_path: {path}".format(path=module_path))
init_filename = os.path.join(module_path, '__init__.py')
if os.path.exists(init_filename):
info(init_filename)
name = os.path.basename(module_path).split(".")[0]
output = run_python('pyreverse -o svg -p {name} {module} '.format(name=name, module=module_path),
verbose=True, ignore_errors=True)
outputter.write(output)
errors = [line for line in output.splitlines() if not line.startswith('parsing')]
if errors:
info(errors)
def __init__(self, logfile=None, verbose=False, prefix=None, postfix=None, password=None, environment=None,
virtualenv=None):
super(LocalShell, self).__init__(is_remote=False, verbose=verbose)
self.logfile = logfile
self.prefix = prefix
self.postfix = postfix
self.password = password
if environment:
self.prefix = Project.prefix[environment]
self.postfix = Project.postfix[environment]
if virtualenv:
activate = None
full_path = os.path.expanduser(virtualenv)
if os.path.isdir(full_path):
activate = ["cd {path} ; source bin/activate ; ".format(path=full_path)]
else:
# noinspection PyBroadException
try:
if virtualenv in [venv.strip() for venv in self.run('bash -c "lsvirtualenv -b"').splitlines()]:
activate = "workon {venv} ; python --version ; " \
"echo \"VirtualEnv: $VIRTUAL_ENV\" ; ".format(venv=virtualenv)
except:
warning('Not using virtualenv: {venv}'.format(venv=virtualenv))
if activate is not None:
if self.prefix:
self.prefix = activate + self.prefix
else:
self.prefix = activate
def _create_class_diagrams(path):
"""
Create class UML diagram
:param path: path to the module file.
:type path: str
"""
info("_create_class_diagrams")
if not executables_available(['pynsource']):
warning('pynsource not available')
return
files = [os.path.join(dir_path, f)
for dir_path, dir_names, files in os.walk(path)
for f in fnmatch.filter(files, '*.py')]
debug("files: {files}".format(files=repr(files)))
with open(os.path.join(Project.docs_dir, "pynsource.log"), "w") as outputter:
for src_file in files:
debug(src_file)
name = src_file.replace(Project.herringfile_dir + '/', '').replace('.py', '.png').replace('/', '.')
output = "classes_{name}".format(name=name)
debug(output)
if not os.path.isfile(output) or (os.path.isfile(output) and is_newer(output, src_file)):
output = run_python("pynsource -y {output} {source}".format(output=output, source=src_file),
verbose=False, ignore_errors=True)
outputter.write(output)
def required_files(self):
"""
Add required packages (specified in module docstrings) to the appropriate requirements text file(s).
"""
debug("requiredFiles")
needed_dict = Requirements(self._project).find_missing_requirements()
for filename in needed_dict.keys():
needed = needed_dict[filename]
debug("needed: %s" % repr(needed))
try:
requirements_filename = os.path.join(self._project.herringfile_dir, filename)
if not os.path.isfile(requirements_filename):
with open(requirements_filename, 'w') as req_file:
req_file.write('-e .\n\n')
with open(requirements_filename, 'a') as req_file:
for need in sorted(unique_list(list(needed))):
out_line = need.qualified(qualifiers=True)
if out_line:
req_file.write(out_line + "\n")
for need in sorted(unique_list(list(needed))):
out_line = need.qualified(qualifiers=False)
if out_line:
req_file.write(out_line + "\n")
except IOError as ex:
warning("Can not add the following to the {filename} file: {needed}\n{err}".format(
filename=filename, needed=repr(needed), err=str(ex)))
def rmvenvs():
"""Remove all the virtual environments"""
venvs = VirtualenvInfo('python_versions')
if not venvs.in_virtualenv and venvs.defined:
for venv_info in venvs.infos():
venv_info.rmvirtualenv()
else:
warning('Please deactivate the current virtual environment then try running this task again.')
def configured(self):
"""
Check if herring has been configured.
:return: Asserted if it looks like herringfile has been configured.
:rtype: bool
"""
if getattr(self, "herringfile_dir", None) is None:
warning("Your herringfile must set up herringfile_dir.")
return False
return True
def _class_line(module_name, class_name):
"""create the class figure lines for the given module and class"""
info("_class_line(%s, %s)" % (module_name, class_name))
line = ""
classes_image = "uml/classes_{module}.{name}.png".format(module=module_name, name=class_name)
image_path = os.path.join(Project.docs_dir, "_src", classes_image)
if os.path.exists(image_path):
info("adding figure %s" % image_path)
line += "\n.. figure:: {image}\n\n {name} Class\n\n".format(image=classes_image, name=class_name)
else:
warning("%s does not exist!" % image_path)
return line
def listvenvs():
"""Run "pip list" in each virtual environment."""
venvs = VirtualenvInfo('python_versions')
if venvs.in_virtualenv:
warning('Please deactivate the current virtual environment then try running this task again.')
return
info("Project Virtual Environments:")
if venvs.defined:
for venv_info in venvs.infos():
venv_info.run('pip list ')
info('')
def _package_line(module_name):
"""create the package figure lines for the given module"""
info("_package_line(%s)" % module_name)
line = ""
package_image = "uml/packages_{name}.svg".format(name=module_name.split(".")[-1])
classes_image = "uml/classes_{name}.svg".format(name=module_name.split(".")[-1])
image_path = os.path.join(Project.docs_dir, "_src", package_image)
if os.path.exists(image_path):
info("adding figure %s" % image_path)
line += "\n.. figure:: {image}\n :width: 1100 px\n\n {name} Packages\n\n".format(
image=package_image, name=module_name
)
line += "\n.. figure:: {image}\n\n {name} Classes\n\n".format(image=classes_image, name=module_name)
else:
warning("%s does not exist!" % image_path)
return line
def env_value(name, default_value=None, warn_if_unset=False):
"""
Safely get value from environment variable, get default value if not defined in environment
:param name: The environment variable name
:type name: str
:param default_value: the value to return if the variable is not in the environment
:type default_value: str|None
:param warn_if_unset: issue a warning if the environment variable is not set
:type warn_if_unset: bool
"""
value = os.environ.get(name)
if value is not None:
return value
if warn_if_unset:
warning("The \"{name}\" environment variable is not set".format(name=name))
return default_value
def rmvenvs():
"""
Remove all the virtual environments.
Note that we do not remove the docs_venv virtual environment as it is intended to be shared across projects.
"""
venvs = VirtualenvInfo('python_versions')
if venvs.in_virtualenv:
warning('Please deactivate the current virtual environment then try running this task again.')
return
if venvs.defined:
for venv_info in venvs.infos():
venv_info.rmvirtualenv()
else:
info("There are no virtual environments to remove.")
def executables_available(executable_list):
"""
Check if the given applications are on the path using the 'which' command.
:param executable_list: list of application names
:type executable_list: list of str instances
:return: asserted if all are available
:rtype: bool
"""
with LocalShell() as local:
for executable in executable_list:
if not local.system("which {name}".format(name=executable), verbose=False).strip():
if executable in HELP:
warning(HELP[executable])
else:
warning("Please install: %s" % executable)
return False
return True
def lsvenvs():
"""List the virtual environments"""
venvs = VirtualenvInfo('python_versions')
if venvs.in_virtualenv:
warning('Please deactivate the current virtual environment then try running this task again.')
return
info("Project Virtual Environments:")
if venvs.defined:
for venv_info in venvs.infos():
info(venv_info.venv)
info("Documentation Virtual Environment:")
venvs = VirtualenvInfo('docs_venv')
if venvs.defined:
for venv_info in venvs.infos():
info(venv_info.venv)
def wheels():
"""build wheels"""
info('')
info("=" * 70)
info('building wheels')
venvs = VirtualenvInfo('wheel_python_versions')
if not venvs.in_virtualenv and venvs.defined:
value = setup_cfg_value(section='wheel', key='universal')
if value is None or value != '0':
warning('To use wheels, you must disable universal in setup.cfg:\n [wheel]\n universal=0\n')
return
for venv_info in venvs.infos():
venv_info.run('{herring} build::wheel --python-tag py{ver}'.format(herring=Project.herring,
ver=venv_info.ver))
else:
info("To build wheels, in your herringfile you must set Project.wheel_python_versions to a list"
"of compact version, for example: ['27', '33', '34'] will build wheels for "
"python 2.7, 3.3, and 3.4")
return
def upvenvs():
"""Run "pip install --update -r requirements" in each virtual environment."""
pip_options = '{pip_opts}'.format(pip_opts=Project.pip_options)
for attr_name in Project.virtualenv_requirements.keys():
requirement_files = Project.virtualenv_requirements[attr_name]
venvs = VirtualenvInfo(attr_name)
if venvs.in_virtualenv:
warning('Please deactivate the current virtual environment then try running this task again.')
return
# info("Project Virtual Environments:")
if venvs.defined:
for venv_info in venvs.infos():
venv_info.run('pip install --upgrade pip')
venv_info.run('pip install --upgrade wheel')
venv_info.run('pip install --upgrade {pip_options} setuptools'.format(pip_options=pip_options))
for requirement_filename in requirement_files:
venv_info.run('pip install --upgrade {pip_options} --pre -r {req}'.format(pip_options=pip_options,
req=requirement_filename))
def mkvenvs():
"""
Make populated virtual environments. Requires Project.python_versions, Project.docs_venv, and virtualenvwrapper.
Currently there are two possibly overlapping sets of virtual environments, the set defined by 'python_versions',
and the virtual environment specified by 'docs_venv'. So we iterate across both sets and use the if exists
to avoid the overlap. Each set can have a set of requirements.txt files used to populate the virtual environment.
"""
for attr_name in Project.virtualenv_requirements.keys():
requirement_files = Project.virtualenv_requirements[attr_name]
venvs = VirtualenvInfo(attr_name)
if venvs.in_virtualenv:
warning('Please deactivate the current virtual environment then try running this task again.')
return
if venvs.defined:
pip_options = '{pip_opts}'.format(pip_opts=Project.pip_options)
for venv_info in venvs.infos(exists=False):
if venv_info.exists():
info("{venv} already exists".format(venv=venv_info.venv))
continue
versioned_requirement_file = Project.versioned_requirements_file_format.format(ver=venv_info.ver)
if os.path.isfile(versioned_requirement_file):
requirement_files.append(versioned_requirement_file)
for requirement_file in unique_list(requirement_files):
with open(requirement_file) as file_:
requirements = file_.readlines()
# info(requirement_file)
# info('=' * len(requirement_file))
# info(pformat(requirements))
install_lines = [
'pip install --upgrade pip',
'pip install wheel',
'pip install --upgrade {pip_options} setuptools'.format(pip_options=pip_options),
'pip install --upgrade {pip_options} cryptography'.format(pip_options=pip_options),
'pip install --upgrade {pip_options} pyopenssl ndg-httpsclient pyasn1'.format(
pip_options=pip_options),
'pip install --upgrade {pip_options} requests[security]'.format(pip_options=pip_options),
]
if 'numpy' in requirements:
install_lines.append('pip install --upgrade {pip_options} numpy'.format(pip_options=pip_options))
if 'matplotlib' in requirements:
install_lines.append('pip install --upgrade {pip_options} matplotlib'.format(
pip_options=pip_options))
for requirement_file in unique_list(requirement_files):
install_lines.append('pip install --upgrade {pip_options} --pre -r {requirement_file}'.format(
pip_options=pip_options, requirement_file=requirement_file))
venv_info.mkvirtualenv()
for line in install_lines:
# noinspection PyArgumentEqualDefault
venv_info.run(line, verbose=True)
else:
info("To build with wheels, in your herringfile you must set Project.wheel_python_versions to a list"
"of compact version, for example: ['27', '33', '34'] will build wheels for "
"python 2.7, 3.3, and 3.4")
