def _create_module_diagrams(path): """ create module UML diagrams :param path: the module path :type path: str """ info("_create_module_diagrams") if not executables_available(['pyreverse']): warning('pyreverse not available') return with open(os.path.join(Project.docs_dir, "pyreverse.log"), "w") as outputter: for module_path in [root for root, dirs, files in os.walk(path) if os.path.basename(root) != '__pycache__']: debug("module_path: {path}".format(path=module_path)) init_filename = os.path.join(module_path, '__init__.py') if os.path.exists(init_filename): info(init_filename) name = os.path.basename(module_path).split(".")[0] output = run_python('pyreverse -o svg -p {name} {module} '.format(name=name, module=module_path), verbose=True, ignore_errors=True) outputter.write(output) errors = [line for line in output.splitlines() if not line.startswith('parsing')] if errors: info(errors)
def __init__(self, logfile=None, verbose=False, prefix=None, postfix=None, password=None, environment=None, virtualenv=None): super(LocalShell, self).__init__(is_remote=False, verbose=verbose) self.logfile = logfile self.prefix = prefix self.postfix = postfix self.password = password if environment: self.prefix = Project.prefix[environment] self.postfix = Project.postfix[environment] if virtualenv: activate = None full_path = os.path.expanduser(virtualenv) if os.path.isdir(full_path): activate = ["cd {path} ; source bin/activate ; ".format(path=full_path)] else: # noinspection PyBroadException try: if virtualenv in [venv.strip() for venv in self.run('bash -c "lsvirtualenv -b"').splitlines()]: activate = "workon {venv} ; python --version ; " \ "echo \"VirtualEnv: $VIRTUAL_ENV\" ; ".format(venv=virtualenv) except: warning('Not using virtualenv: {venv}'.format(venv=virtualenv)) if activate is not None: if self.prefix: self.prefix = activate + self.prefix else: self.prefix = activate
def _create_class_diagrams(path): """ Create class UML diagram :param path: path to the module file. :type path: str """ info("_create_class_diagrams") if not executables_available(['pynsource']): warning('pynsource not available') return files = [os.path.join(dir_path, f) for dir_path, dir_names, files in os.walk(path) for f in fnmatch.filter(files, '*.py')] debug("files: {files}".format(files=repr(files))) with open(os.path.join(Project.docs_dir, "pynsource.log"), "w") as outputter: for src_file in files: debug(src_file) name = src_file.replace(Project.herringfile_dir + '/', '').replace('.py', '.png').replace('/', '.') output = "classes_{name}".format(name=name) debug(output) if not os.path.isfile(output) or (os.path.isfile(output) and is_newer(output, src_file)): output = run_python("pynsource -y {output} {source}".format(output=output, source=src_file), verbose=False, ignore_errors=True) outputter.write(output)
def required_files(self): """ Add required packages (specified in module docstrings) to the appropriate requirements text file(s). """ debug("requiredFiles") needed_dict = Requirements(self._project).find_missing_requirements() for filename in needed_dict.keys(): needed = needed_dict[filename] debug("needed: %s" % repr(needed)) try: requirements_filename = os.path.join(self._project.herringfile_dir, filename) if not os.path.isfile(requirements_filename): with open(requirements_filename, 'w') as req_file: req_file.write('-e .\n\n') with open(requirements_filename, 'a') as req_file: for need in sorted(unique_list(list(needed))): out_line = need.qualified(qualifiers=True) if out_line: req_file.write(out_line + "\n") for need in sorted(unique_list(list(needed))): out_line = need.qualified(qualifiers=False) if out_line: req_file.write(out_line + "\n") except IOError as ex: warning("Can not add the following to the {filename} file: {needed}\n{err}".format( filename=filename, needed=repr(needed), err=str(ex)))
def rmvenvs(): """Remove all the virtual environments""" venvs = VirtualenvInfo('python_versions') if not venvs.in_virtualenv and venvs.defined: for venv_info in venvs.infos(): venv_info.rmvirtualenv() else: warning('Please deactivate the current virtual environment then try running this task again.')
def configured(self): """ Check if herring has been configured. :return: Asserted if it looks like herringfile has been configured. :rtype: bool """ if getattr(self, "herringfile_dir", None) is None: warning("Your herringfile must set up herringfile_dir.") return False return True
def _class_line(module_name, class_name): """create the class figure lines for the given module and class""" info("_class_line(%s, %s)" % (module_name, class_name)) line = "" classes_image = "uml/classes_{module}.{name}.png".format(module=module_name, name=class_name) image_path = os.path.join(Project.docs_dir, "_src", classes_image) if os.path.exists(image_path): info("adding figure %s" % image_path) line += "\n.. figure:: {image}\n\n {name} Class\n\n".format(image=classes_image, name=class_name) else: warning("%s does not exist!" % image_path) return line
def listvenvs(): """Run "pip list" in each virtual environment.""" venvs = VirtualenvInfo('python_versions') if venvs.in_virtualenv: warning('Please deactivate the current virtual environment then try running this task again.') return info("Project Virtual Environments:") if venvs.defined: for venv_info in venvs.infos(): venv_info.run('pip list ') info('')
def _package_line(module_name): """create the package figure lines for the given module""" info("_package_line(%s)" % module_name) line = "" package_image = "uml/packages_{name}.svg".format(name=module_name.split(".")[-1]) classes_image = "uml/classes_{name}.svg".format(name=module_name.split(".")[-1]) image_path = os.path.join(Project.docs_dir, "_src", package_image) if os.path.exists(image_path): info("adding figure %s" % image_path) line += "\n.. figure:: {image}\n :width: 1100 px\n\n {name} Packages\n\n".format( image=package_image, name=module_name ) line += "\n.. figure:: {image}\n\n {name} Classes\n\n".format(image=classes_image, name=module_name) else: warning("%s does not exist!" % image_path) return line
def env_value(name, default_value=None, warn_if_unset=False): """ Safely get value from environment variable, get default value if not defined in environment :param name: The environment variable name :type name: str :param default_value: the value to return if the variable is not in the environment :type default_value: str|None :param warn_if_unset: issue a warning if the environment variable is not set :type warn_if_unset: bool """ value = os.environ.get(name) if value is not None: return value if warn_if_unset: warning("The \"{name}\" environment variable is not set".format(name=name)) return default_value
def rmvenvs(): """ Remove all the virtual environments. Note that we do not remove the docs_venv virtual environment as it is intended to be shared across projects. """ venvs = VirtualenvInfo('python_versions') if venvs.in_virtualenv: warning('Please deactivate the current virtual environment then try running this task again.') return if venvs.defined: for venv_info in venvs.infos(): venv_info.rmvirtualenv() else: info("There are no virtual environments to remove.")
def executables_available(executable_list): """ Check if the given applications are on the path using the 'which' command. :param executable_list: list of application names :type executable_list: list of str instances :return: asserted if all are available :rtype: bool """ with LocalShell() as local: for executable in executable_list: if not local.system("which {name}".format(name=executable), verbose=False).strip(): if executable in HELP: warning(HELP[executable]) else: warning("Please install: %s" % executable) return False return True
def lsvenvs(): """List the virtual environments""" venvs = VirtualenvInfo('python_versions') if venvs.in_virtualenv: warning('Please deactivate the current virtual environment then try running this task again.') return info("Project Virtual Environments:") if venvs.defined: for venv_info in venvs.infos(): info(venv_info.venv) info("Documentation Virtual Environment:") venvs = VirtualenvInfo('docs_venv') if venvs.defined: for venv_info in venvs.infos(): info(venv_info.venv)
def wheels(): """build wheels""" info('') info("=" * 70) info('building wheels') venvs = VirtualenvInfo('wheel_python_versions') if not venvs.in_virtualenv and venvs.defined: value = setup_cfg_value(section='wheel', key='universal') if value is None or value != '0': warning('To use wheels, you must disable universal in setup.cfg:\n [wheel]\n universal=0\n') return for venv_info in venvs.infos(): venv_info.run('{herring} build::wheel --python-tag py{ver}'.format(herring=Project.herring, ver=venv_info.ver)) else: info("To build wheels, in your herringfile you must set Project.wheel_python_versions to a list" "of compact version, for example: ['27', '33', '34'] will build wheels for " "python 2.7, 3.3, and 3.4") return
def upvenvs(): """Run "pip install --update -r requirements" in each virtual environment.""" pip_options = '{pip_opts}'.format(pip_opts=Project.pip_options) for attr_name in Project.virtualenv_requirements.keys(): requirement_files = Project.virtualenv_requirements[attr_name] venvs = VirtualenvInfo(attr_name) if venvs.in_virtualenv: warning('Please deactivate the current virtual environment then try running this task again.') return # info("Project Virtual Environments:") if venvs.defined: for venv_info in venvs.infos(): venv_info.run('pip install --upgrade pip') venv_info.run('pip install --upgrade wheel') venv_info.run('pip install --upgrade {pip_options} setuptools'.format(pip_options=pip_options)) for requirement_filename in requirement_files: venv_info.run('pip install --upgrade {pip_options} --pre -r {req}'.format(pip_options=pip_options, req=requirement_filename))
def mkvenvs(): """ Make populated virtual environments. Requires Project.python_versions, Project.docs_venv, and virtualenvwrapper. Currently there are two possibly overlapping sets of virtual environments, the set defined by 'python_versions', and the virtual environment specified by 'docs_venv'. So we iterate across both sets and use the if exists to avoid the overlap. Each set can have a set of requirements.txt files used to populate the virtual environment. """ for attr_name in Project.virtualenv_requirements.keys(): requirement_files = Project.virtualenv_requirements[attr_name] venvs = VirtualenvInfo(attr_name) if venvs.in_virtualenv: warning('Please deactivate the current virtual environment then try running this task again.') return if venvs.defined: pip_options = '{pip_opts}'.format(pip_opts=Project.pip_options) for venv_info in venvs.infos(exists=False): if venv_info.exists(): info("{venv} already exists".format(venv=venv_info.venv)) continue versioned_requirement_file = Project.versioned_requirements_file_format.format(ver=venv_info.ver) if os.path.isfile(versioned_requirement_file): requirement_files.append(versioned_requirement_file) for requirement_file in unique_list(requirement_files): with open(requirement_file) as file_: requirements = file_.readlines() # info(requirement_file) # info('=' * len(requirement_file)) # info(pformat(requirements)) install_lines = [ 'pip install --upgrade pip', 'pip install wheel', 'pip install --upgrade {pip_options} setuptools'.format(pip_options=pip_options), 'pip install --upgrade {pip_options} cryptography'.format(pip_options=pip_options), 'pip install --upgrade {pip_options} pyopenssl ndg-httpsclient pyasn1'.format( pip_options=pip_options), 'pip install --upgrade {pip_options} requests[security]'.format(pip_options=pip_options), ] if 'numpy' in requirements: install_lines.append('pip install --upgrade {pip_options} numpy'.format(pip_options=pip_options)) if 'matplotlib' in requirements: install_lines.append('pip install --upgrade {pip_options} matplotlib'.format( pip_options=pip_options)) for requirement_file in unique_list(requirement_files): install_lines.append('pip install --upgrade {pip_options} --pre -r {requirement_file}'.format( pip_options=pip_options, requirement_file=requirement_file)) venv_info.mkvirtualenv() for line in install_lines: # noinspection PyArgumentEqualDefault venv_info.run(line, verbose=True) else: info("To build with wheels, in your herringfile you must set Project.wheel_python_versions to a list" "of compact version, for example: ['27', '33', '34'] will build wheels for " "python 2.7, 3.3, and 3.4")