def _create_class_diagrams(path): """ Create class UML diagram :param path: path to the module file. :type path: str """ info("_create_class_diagrams") if not executables_available(['pynsource']): warning('pynsource not available') return files = [os.path.join(dir_path, f) for dir_path, dir_names, files in os.walk(path) for f in fnmatch.filter(files, '*.py')] debug("files: {files}".format(files=repr(files))) with open(os.path.join(Project.docs_dir, "pynsource.log"), "w") as outputter: for src_file in files: debug(src_file) name = src_file.replace(Project.herringfile_dir + '/', '').replace('.py', '.png').replace('/', '.') output = "classes_{name}".format(name=name) debug(output) if not os.path.isfile(output) or (os.path.isfile(output) and is_newer(output, src_file)): output = run_python("pynsource -y {output} {source}".format(output=output, source=src_file), verbose=False, ignore_errors=True) outputter.write(output)
def put(self, files, remote_path=None, out_stream=sys.stdout, verbose=False): """ Copy a file from the local system to the remote system. :param files: :param remote_path: :param out_stream: :param verbose: :return: :rtype: """ if remote_path is None: remote_path = files self.display("scp '{src}' '{dest}'".format(src=files, dest=remote_path), out_stream=out_stream, verbose=verbose) ssh = SSHClient() ssh.load_system_host_keys() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(Project.address, Project.port, Project.user, Project.password) scp = SCPClient(ssh.get_transport()) # scp = SCPClient(self.ssh.get_transport()) # noinspection PyBroadException try: info("\nfiles: %s" % repr(files)) info("remote_path: %s" % remote_path) output = scp.put(files, '"{dest}"'.format(dest=remote_path), recursive=True) or '' except Exception: try: output = scp.put(files, remote_path, recursive=True) or '' except Exception as ex: output = str(ex) self.display("\n" + output, out_stream=out_stream, verbose=verbose) return output
def get_project_version(project_package=None): r""" Get the version from __init__.py with a line: /^__version__\s*=\s*(\S+)/ If it doesn't exist try to load it from the VERSION.txt file. If still no joy, then return '0.0.0' :param project_package: the root package :type project_package: str :returns: the version string :rtype: str """ # trying __init__.py first try: file_name = _file_spec('__init__.py', project_package) debug("version_file => %s" % file_name) # noinspection PyBroadException try: # python3 with open(file_name, 'r', encoding='utf-8') as inFile: for line in inFile.readlines(): match = re.match(VERSION_REGEX, line) if match: return match.group(1) except: # python2 with open(file_name, 'r') as inFile: for line in inFile.readlines(): match = re.match(VERSION_REGEX, line) if match: return match.group(1) except IOError: pass # no joy, so try getting the version from a VERSION.txt file. try: file_name = _file_spec('VERSION.txt', project_package) info("version_file => %s" % file_name) # noinspection PyBroadException try: # python3 with open(file_name, 'r', encoding='utf-8') as in_file: return in_file.read().strip() except: # python2 with open(file_name, 'r') as in_file: return in_file.read().strip() except IOError: try: file_name = _file_spec('VERSION.txt', Project.herringfile_dir) info("version_file => %s" % file_name) with open(file_name) as in_file: return in_file.read().strip() except IOError: pass # no joy again, so set to initial version and try again set_project_version('0.0.1', project_package) return get_project_version(project_package)
def lsvenvs(): """List the virtual environments""" venvs = VirtualenvInfo('python_versions') info("Project Virtual Environments:") if not venvs.in_virtualenv and venvs.defined: for venv_info in venvs.infos(): info(venv_info.venv)
def init(): """ Initialize a new python project with default files. Default values from herring.conf and directory name. """ defaults = _project_defaults() if Project.prompt: defaults['name'] = prompt("Enter the project's name:", defaults['name']) defaults['package'] = prompt("Enter the project's package:", defaults['package']) defaults['author'] = prompt("Enter the project's author:", defaults['author']) defaults['author_email'] = prompt("Enter the project's author's email:", defaults['author_email']) defaults['description'] = prompt("Enter the project's description:", defaults['description']) # print("defaults:\n{defaults}".format(defaults=pformat(defaults))) if Project.use_templates: template = Template() for template_dir in [os.path.abspath(os.path.join(herringlib, 'herringlib', 'templates')) for herringlib in HerringFile.herringlib_paths]: info("template directory: %s" % template_dir) # noinspection PyArgumentEqualDefault template.generate(template_dir, defaults, overwrite=False)
def test(): """Run the unit tests.""" # Run the tests in each of the virtual environments defined in Project.test_python_versions # or if not defined, then in Project.wheel_python_versions. If neither are defined, then # run the test in the current environment. venvs = VirtualenvInfo('test_python_versions', 'wheel_python_versions') coverage = '--cov-report term-missing --cov={package}'.format(package=Project.package) reports = '--junitxml={quality}/tests.xml'.format(quality=Project.quality_dir) mkdir_p(Project.tests_dir) if not venvs.in_virtualenv and venvs.defined: for venv_info in venvs.infos(): info('Running unit tests using the {venv} virtual environment.'.format(venv=venv_info.venv)) venv_info.run('py.test {coverage} {reports} {tests_dir}'.format(coverage=coverage, reports=reports, tests_dir=Project.tests_dir), verbose=True) else: with LocalShell() as local: info('Running unit tests using the current python environment') local.run("py.test {coverage} {reports} {tests_dir}".format(coverage=coverage, reports=reports, tests_dir=Project.tests_dir), verbose=True)
def packages_required(package_names): """ Check that the given packages are installed. :param package_names: the package names :type package_names: list :return: asserted if all the packages are installed :rtype: bool """ # info("packages_required(%s)" % repr(package_names)) # noinspection PyBroadException try: result = True # info(package_names) # info(__pip_list) for requirement in [Requirement(name) for name in package_names]: if requirement.supported_python(): pkg_name = requirement.package if pkg_name.lower() not in __pip_list: try: # info('__import__("{name}")'.format(name=pkg_name)) __import__(pkg_name) except ImportError: info(pkg_name + " not installed!") missing_modules.append(pkg_name) result = False return result except: return False
def _create_class_diagrams(path): """ Create class UML diagram :param path: path to the module file. :type path: str """ info("_create_class_diagrams") if not executables_available(["pynsource"]): return files = [ os.path.join(dir_path, f) for dir_path, dir_names, files in os.walk(path) for f in fnmatch.filter(files, "*.py") ] debug("files: {files}".format(files=repr(files))) for src_file in files: debug(src_file) name = src_file.replace(Project.herringfile_dir + "/", "").replace(".py", ".png").replace("/", ".") output = "classes_{name}".format(name=name) debug(output) if not os.path.isfile(output) or (os.path.isfile(output) and is_newer(output, src_file)): run_python( "pynsource -y {output} {source}".format(output=output, source=src_file), verbose=False, ignore_errors=True, )
def publish(): """ copy latest docs to a linux base web server """ project_version_name = "{name}-{version}".format(name=Project.base_name, version=Project.version) project_latest_name = "{name}-latest".format(name=Project.base_name) doc_version = '{dir}/{file}'.format(dir=Project.docs_path, file=project_version_name) doc_latest = '{dir}/{file}'.format(dir=Project.docs_path, file=project_latest_name) docs_html_dir = '{dir}'.format(dir=Project.docs_html_dir) password = Project.docs_password if password is None and Project.doc_host_prompt_for_sudo_password: password = getpass("password for {user}@{host}: ".format(user=Project.docs_user, host=Project.docs_host)) Project.docs_password = password info("Publishing to {user}@{host}".format(user=Project.docs_user, host=Project.docs_host)) with RemoteShell(user=Project.docs_user, password=Project.docs_password, host=Project.docs_host, verbose=True) as remote: remote.run('mkdir -p \"{dir}\"'.format(dir=Project.docs_path)) remote.run('rm -rf \"{path}\"'.format(path=doc_latest)) remote.run('rm -rf \"{path}\"'.format(path=doc_version)) remote.run('mkdir -p \"{dir}\"'.format(dir=doc_version)) for file_ in [os.path.join(docs_html_dir, file_) for file_ in os.listdir(docs_html_dir)]: remote.put(file_, doc_version) remote.run('ln -s \"{src}\" \"{dest}\"'.format(src=doc_version, dest=doc_latest)) remote.run('sudo chown -R {user}:{group} \"{dest}\"'.format(user=Project.docs_user, group=Project.docs_group, dest=doc_version), accept_defaults=True, timeout=10) remote.run('sudo chmod -R g+w \"{dest}\"'.format(dest=doc_version), accept_defaults=True, timeout=10)
def _neon(text, file_name, animate_logo, fontsize): info("creating logo") pre = ' '.join(dedent("""\ -size 500x200 xc:lightblue -font Comic-Sans-MS-Bold -pointsize {fontsize} -gravity center -undercolor black -stroke none -strokewidth 3 """).format(fontsize=fontsize).strip().split('\n')) post = ' '.join(dedent("""\ -trim +repage -shave 1x1 -bordercolor black -border 20x20 """).strip().split('\n')) on = ' '.join(dedent("""\ convert {pre} -fill DeepSkyBlue -annotate +0+0 '{text}' {post} \( +clone -blur 0x25 -level 0%,50% \) -compose screen -composite {file}_on.png """.format(text=text, file=file_name, pre=pre, post=post)).strip().split('\n')) off = ' '.join(dedent("""\ convert {pre} -fill grey12 -annotate +0+0 '{text}' {post} {file}_off.png """.format(text=text, file=file_name, pre=pre, post=post)).strip().split('\n')) animated = ' '.join(dedent("""convert \ -adjoin -delay 100 -resize 240 {file}_on.png {file}_off.png {file}_animated.gif """.format(file=file_name)).strip().split('\n')) # noinspection PyArgumentEqualDefault with LocalShell(verbose=False) as local: if animate_logo: local.run(on) local.run(off) local.run(animated) local.run('bash -c "rm -f {file}_on.png {file}_off.png"'.format(file=file_name)) logo_image = "{file}_animated.gif".format(file=file_name) else: info(on) local.run(on) on_image = "{file}_on.png".format(file=file_name) logo_image = "{file}.png".format(file=file_name) if os.path.isfile(on_image): os.rename(on_image, logo_image) return logo_image
def design(): """Update the design.rst from the source module's docstrings""" info("Python version: {version}".format(version=version)) design_header = Project.design_header.strip() if design_header: py_files = _find_py_files(Project.package) if py_files: with open(Project.design_file, "w") as design_file: design_file.write("Design\n") design_file.write("======\n\n") design_file.write(design_header) design_file.write("\n\n") for py_file in sorted(py_files): docstring, functions, classes = _parse_py_file(py_file) design_file.write(py_file) design_file.write("\n") design_file.write("-" * len(py_file)) design_file.write("\n\n") design_file.write(docstring) design_file.write("\n\n") if functions: design_file.write("Functions:\n\n") for function in functions: design_file.write("* {name}\n".format(name=function)) design_file.write("\n\n") if classes: design_file.write("Classes:\n\n") for class_ in classes: design_file.write("* {name}\n".format(name=class_)) design_file.write("\n\n") else: touch(Project.design_file)
def listvenvs(): """Run "pip list" in each virtual environment.""" venvs = VirtualenvInfo('python_versions') info("Project Virtual Environments:") if not venvs.in_virtualenv and venvs.defined: for venv_info in venvs.infos(): venv_info.run('pip list ') info('')
def upvenvs(): """Run "pip install --update -r requirements" in each virtual environment.""" venvs = VirtualenvInfo('python_versions') info("Project Virtual Environments:") if not venvs.in_virtualenv and venvs.defined: for venv_info in venvs.infos(): venv_info.run('pip install --upgrade pip') venv_info.run('pip install --upgrade setuptools') venv_info.run('pip install --upgrade -r requirements.txt')
def publish_gh_pages(): """copy documentation to github pages""" if Project.github_url is not None: tmp_repo_path = None try: tmp_repo_path = tempfile.mkdtemp() with LocalShell(verbose=True) as local: # clone repo selecting gh-pages branch # git clone {git_url} {directory} # git branch --list local.run("git clone {url} {dir}".format(url=Project.github_url, dir=tmp_repo_path)) with cd(tmp_repo_path, verbose=True): remote_branches = [ line.lstrip(r"[*\s]*").strip() for line in local.run("git branch --list -r").splitlines() ] if "origin/gh-pages" in remote_branches: local.run("git pull origin") local.run("git checkout -b gh-pages origin/gh-pages") # select branch # git checkout gh-pages local.run("git checkout gh-pages") # remove github pages clone directory # clean_directory(tmp_repo_path) # touch .nojekyl touch(".nojekyll") # copy documentation if os.path.isdir(Project.docs_html_path): dir_util.copy_tree(Project.docs_html_path, tmp_repo_path) # commit and push to github local.run("git add --all") local.run("git status") now = datetime.now().strftime("%c") message = "{project} Documentation {version} {date}".format( project=Project.title, version=Project.version, date=now ) local.run("git commit -m '{message}'".format(message=message)) local.run("git push origin gh-pages") else: info("Please create a 'gh-pages' branch on the github repository.") finally: if tmp_repo_path is not None: try: info("removing {repo}".format(repo=tmp_repo_path)) shutil.rmtree(tmp_repo_path) except OSError as ex: if ex.errno != errno.ENOENT: raise
def __init__(self, *attr_names): self._ver_attr = None self._raise_when_in_venv = False debug(repr(attr_names)) for name in attr_names: debug(name) self._ver_attr = getattr(Project, name, None) if self._ver_attr is not None: info("_ver_attr: %s" % repr(self._ver_attr)) break
def pdf(): """Generate PDF API documents""" venvs = VirtualenvInfo("doc_python_version") if not venvs.in_virtualenv and venvs.defined: for venv_info in venvs.infos(): venv_info.run("{herring} doc::pdf_generate".format(herring=Project.herring)) else: info("Generating documentation using the current python environment") task_execute("doc::pdf_generate")
def slides(): """generate project slides""" venvs = VirtualenvInfo('docs_venv') info("venvs: {venvs}".format(venvs=repr(venvs.__dict__))) if not venvs.in_virtualenv and venvs.defined: for venv_info in venvs.infos(): venv_info.run('{herring} doc::hieroglyph_slides --python-tag py{ver}'.format(herring=Project.herring, ver=venv_info.ver)) else: info('Generating slides using the current python environment') task_execute('doc::hieroglyph_slides')
def watch(): """generate project documentation""" venvs = VirtualenvInfo('doc_python_version') info("venvs: {venvs}".format(venvs=repr(venvs.__dict__))) if not venvs.in_virtualenv and venvs.defined: for venv_info in venvs.infos(): venv_info.run('{herring} doc::watcher --python-tag py{ver}'.format(herring=Project.herring, ver=venv_info.ver)) else: info('Watching documentation using the current python environment') task_execute('doc::watcher')
def doc_no_api(): """generate project documentation without the api""" venvs = VirtualenvInfo('docs_venv') info("venvs: {venvs}".format(venvs=repr(venvs.__dict__))) if not venvs.in_virtualenv and venvs.defined: for venv_info in venvs.infos(): venv_info.run('{herring} doc::generate_no_api --python-tag py{ver}'.format(herring=Project.herring, ver=venv_info.ver)) else: info('Generating documentation using the current python environment') task_execute('doc::generate_no_api')
def _class_line(module_name, class_name): """create the class figure lines for the given module and class""" info("_class_line(%s, %s)" % (module_name, class_name)) line = "" classes_image = "uml/classes_{module}.{name}.png".format(module=module_name, name=class_name) image_path = os.path.join(Project.docs_dir, "_src", classes_image) if os.path.exists(image_path): info("adding figure %s" % image_path) line += "\n.. figure:: {image}\n\n {name} Class\n\n".format(image=classes_image, name=class_name) else: warning("%s does not exist!" % image_path) return line
def doc(): """generate project documentation""" venvs = VirtualenvInfo("doc_python_version") info("venvs: {venvs}".format(venvs=repr(venvs.__dict__))) if not venvs.in_virtualenv and venvs.defined: for venv_info in venvs.infos(): venv_info.run( "{herring} doc::generate --python-tag py{ver}".format(herring=Project.herring, ver=venv_info.ver) ) else: info("Generating documentation using the current python environment") task_execute("doc::generate")
def do_task(): """ This function create a task that will be a daemon """ app = QtGui.QApplication(sys.argv) path = os.path.join(Project.herringfile_dir, Project.docs_html_dir, 'index.html') info(path) url = 'file:///' + pathname2url(os.path.abspath(path)) url = QtCore.QUrl(url) wb = MainWindow(url) wb.show() app.exec_()
def listvenvs(): """Run "pip list" in each virtual environment.""" venvs = VirtualenvInfo('python_versions') if venvs.in_virtualenv: warning('Please deactivate the current virtual environment then try running this task again.') return info("Project Virtual Environments:") if venvs.defined: for venv_info in venvs.infos(): venv_info.run('pip list ') info('')
def __init__(self, user=None, password=None, host=None, logfile=None, environment=None, verbose=False, virtualenv=None, working_dir=None): """ :param user: the remote user account :type user: str :param password: the password for the remote user account :type password: str :param host: the remote machine :type host: str :param logfile: optional logfile :type logfile: str :param environment: Optional key to Project.prefix and Project.postfix dictionaries :type environment: str :param verbose: extra logging :type verbose: bool :param virtualenv: directory that contains a virtual environment to activate upon connection :type virtualenv: str """ super(RemoteShell, self).__init__(is_remote=True, verbose=verbose) if not user: user = Project.user if not host: host = Project.address Project.user = user Project.password = password Project.address = host self.ssh = pxssh(timeout=1200) try: if password: self.ssh.login(host, user, password) else: self.ssh.login(host, user) except ExceptionPxssh: if not password: password = Project.password if not password: password = getpass('password for {user}@{host}: '.format(user=user, host=host)) self.ssh.close() self.ssh = pxssh(timeout=1200) self.ssh.login(host, user, password) self.accept_defaults = False self.logfile = logfile self.prefix = [] self.postfix = [] if environment: self.prefix.extend(Project.prefix[environment] or []) self.postfix.extend(Project.postfix[environment] or []) if working_dir: self.prefix.insert(0, "cd {dir} ; ".format(dir=working_dir)) if virtualenv: self.prefix.insert(0, "source {path}/bin/activate ; ".format(path=virtualenv)) if verbose: info("remote: {user}@{host}".format(user=user or "", host=host or ""))
def sdist(): """ build source distribution""" info('') info("=" * 70) info('building source distribution') venvs = VirtualenvInfo('sdist_python_version') if not venvs.in_virtualenv and venvs.defined: for venv_info in venvs.infos(): info('Building sdist using {venv} virtual environment'.format(venv=venv_info.venv)) venv_info.run('python setup.py sdist') else: with LocalShell() as local: info("Building sdist using default environment") local.system("python setup.py sdist")
def update(): """ Regenerate files (except herringfile) from current templates. Delete the file(s) you want to update, then run this task. """ defaults = _project_defaults() template = Template() for template_dir in [os.path.abspath(os.path.join(herringlib, 'herringlib', 'templates')) for herringlib in HerringFile.herringlib_paths]: info("template directory: %s" % template_dir) # noinspection PyArgumentEqualDefault template.generate(template_dir, defaults, overwrite=False)
def rmvenvs(): """ Remove all the virtual environments. Note that we do not remove the docs_venv virtual environment as it is intended to be shared across projects. """ venvs = VirtualenvInfo('python_versions') if venvs.in_virtualenv: warning('Please deactivate the current virtual environment then try running this task again.') return if venvs.defined: for venv_info in venvs.infos(): venv_info.rmvirtualenv() else: info("There are no virtual environments to remove.")
def metrics(): """ Quality metrics """ # Run the metrics in each of the virtual environments defined in Project.metrics_python_versions # or if not defined, then in Project.wheel_python_versions. If neither are defined, then # run the test in the current environment. venvs = VirtualenvInfo('metrics_python_versions', 'wheel_python_versions') if not venvs.in_virtualenv and venvs.defined: for venv_info in venvs.infos(): info('Running metrics using the {venv} virtual environment.'.format(venv=venv_info.venv)) venv_info.run('herring metrics::all_metrics') else: info('Running metrics using the current python environment') task_execute('metrics::all_metrics')
def environment(): """ Display project environment """ venvs = VirtualenvInfo('python_versions') site_packages_cmdline = "python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())'" project_env = {} if not venvs.in_virtualenv and venvs.defined: for venv_info in venvs.infos(): site_packages = venv_info.run(site_packages_cmdline).strip().splitlines()[2] project_env[venv_info.venv + ': site-packages'] = site_packages else: with LocalShell() as local: site_packages = local.system(site_packages_cmdline).strip() project_env['site-packages'] = site_packages info(pformat(project_env)) return project_env
def _package_line(module_name): """create the package figure lines for the given module""" info("_package_line(%s)" % module_name) line = "" package_image = "uml/packages_{name}.svg".format(name=module_name.split(".")[-1]) classes_image = "uml/classes_{name}.svg".format(name=module_name.split(".")[-1]) image_path = os.path.join(Project.docs_dir, "_src", package_image) if os.path.exists(image_path): info("adding figure %s" % image_path) line += "\n.. figure:: {image}\n :width: 1100 px\n\n {name} Packages\n\n".format( image=package_image, name=module_name ) line += "\n.. figure:: {image}\n\n {name} Classes\n\n".format(image=classes_image, name=module_name) else: warning("%s does not exist!" % image_path) return line