def _create_class_diagrams(path):
"""
Create class UML diagram
:param path: path to the module file.
:type path: str
"""
info("_create_class_diagrams")
if not executables_available(['pynsource']):
warning('pynsource not available')
return
files = [os.path.join(dir_path, f)
for dir_path, dir_names, files in os.walk(path)
for f in fnmatch.filter(files, '*.py')]
debug("files: {files}".format(files=repr(files)))
with open(os.path.join(Project.docs_dir, "pynsource.log"), "w") as outputter:
for src_file in files:
debug(src_file)
name = src_file.replace(Project.herringfile_dir + '/', '').replace('.py', '.png').replace('/', '.')
output = "classes_{name}".format(name=name)
debug(output)
if not os.path.isfile(output) or (os.path.isfile(output) and is_newer(output, src_file)):
output = run_python("pynsource -y {output} {source}".format(output=output, source=src_file),
verbose=False, ignore_errors=True)
outputter.write(output)
def put(self, files, remote_path=None, out_stream=sys.stdout, verbose=False):
"""
Copy a file from the local system to the remote system.
:param files:
:param remote_path:
:param out_stream:
:param verbose:
:return: :rtype:
"""
if remote_path is None:
remote_path = files
self.display("scp '{src}' '{dest}'".format(src=files, dest=remote_path),
out_stream=out_stream, verbose=verbose)
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(Project.address, Project.port, Project.user, Project.password)
scp = SCPClient(ssh.get_transport())
# scp = SCPClient(self.ssh.get_transport())
# noinspection PyBroadException
try:
info("\nfiles: %s" % repr(files))
info("remote_path: %s" % remote_path)
output = scp.put(files, '"{dest}"'.format(dest=remote_path), recursive=True) or ''
except Exception:
try:
output = scp.put(files, remote_path, recursive=True) or ''
except Exception as ex:
output = str(ex)
self.display("\n" + output, out_stream=out_stream, verbose=verbose)
return output
def get_project_version(project_package=None):
r"""
Get the version from __init__.py with a line: /^__version__\s*=\s*(\S+)/
If it doesn't exist try to load it from the VERSION.txt file.
If still no joy, then return '0.0.0'
:param project_package: the root package
:type project_package: str
:returns: the version string
:rtype: str
"""
# trying __init__.py first
try:
file_name = _file_spec('__init__.py', project_package)
debug("version_file => %s" % file_name)
# noinspection PyBroadException
try:
# python3
with open(file_name, 'r', encoding='utf-8') as inFile:
for line in inFile.readlines():
match = re.match(VERSION_REGEX, line)
if match:
return match.group(1)
except:
# python2
with open(file_name, 'r') as inFile:
for line in inFile.readlines():
match = re.match(VERSION_REGEX, line)
if match:
return match.group(1)
except IOError:
pass
# no joy, so try getting the version from a VERSION.txt file.
try:
file_name = _file_spec('VERSION.txt', project_package)
info("version_file => %s" % file_name)
# noinspection PyBroadException
try:
# python3
with open(file_name, 'r', encoding='utf-8') as in_file:
return in_file.read().strip()
except:
# python2
with open(file_name, 'r') as in_file:
return in_file.read().strip()
except IOError:
try:
file_name = _file_spec('VERSION.txt', Project.herringfile_dir)
info("version_file => %s" % file_name)
with open(file_name) as in_file:
return in_file.read().strip()
except IOError:
pass
# no joy again, so set to initial version and try again
set_project_version('0.0.1', project_package)
return get_project_version(project_package)
def lsvenvs():
"""List the virtual environments"""
venvs = VirtualenvInfo('python_versions')
info("Project Virtual Environments:")
if not venvs.in_virtualenv and venvs.defined:
for venv_info in venvs.infos():
info(venv_info.venv)
def init():
"""
Initialize a new python project with default files. Default values from herring.conf and directory name.
"""
defaults = _project_defaults()
if Project.prompt:
defaults['name'] = prompt("Enter the project's name:", defaults['name'])
defaults['package'] = prompt("Enter the project's package:", defaults['package'])
defaults['author'] = prompt("Enter the project's author:", defaults['author'])
defaults['author_email'] = prompt("Enter the project's author's email:", defaults['author_email'])
defaults['description'] = prompt("Enter the project's description:", defaults['description'])
# print("defaults:\n{defaults}".format(defaults=pformat(defaults)))
if Project.use_templates:
template = Template()
for template_dir in [os.path.abspath(os.path.join(herringlib, 'herringlib', 'templates'))
for herringlib in HerringFile.herringlib_paths]:
info("template directory: %s" % template_dir)
# noinspection PyArgumentEqualDefault
template.generate(template_dir, defaults, overwrite=False)
def test():
"""Run the unit tests."""
# Run the tests in each of the virtual environments defined in Project.test_python_versions
# or if not defined, then in Project.wheel_python_versions. If neither are defined, then
# run the test in the current environment.
venvs = VirtualenvInfo('test_python_versions', 'wheel_python_versions')
coverage = '--cov-report term-missing --cov={package}'.format(package=Project.package)
reports = '--junitxml={quality}/tests.xml'.format(quality=Project.quality_dir)
mkdir_p(Project.tests_dir)
if not venvs.in_virtualenv and venvs.defined:
for venv_info in venvs.infos():
info('Running unit tests using the {venv} virtual environment.'.format(venv=venv_info.venv))
venv_info.run('py.test {coverage} {reports} {tests_dir}'.format(coverage=coverage,
reports=reports,
tests_dir=Project.tests_dir),
verbose=True)
else:
with LocalShell() as local:
info('Running unit tests using the current python environment')
local.run("py.test {coverage} {reports} {tests_dir}".format(coverage=coverage,
reports=reports,
tests_dir=Project.tests_dir),
verbose=True)
def packages_required(package_names):
"""
Check that the given packages are installed.
:param package_names: the package names
:type package_names: list
:return: asserted if all the packages are installed
:rtype: bool
"""
# info("packages_required(%s)" % repr(package_names))
# noinspection PyBroadException
try:
result = True
# info(package_names)
# info(__pip_list)
for requirement in [Requirement(name) for name in package_names]:
if requirement.supported_python():
pkg_name = requirement.package
if pkg_name.lower() not in __pip_list:
try:
# info('__import__("{name}")'.format(name=pkg_name))
__import__(pkg_name)
except ImportError:
info(pkg_name + " not installed!")
missing_modules.append(pkg_name)
result = False
return result
except:
return False
def _create_class_diagrams(path):
"""
Create class UML diagram
:param path: path to the module file.
:type path: str
"""
info("_create_class_diagrams")
if not executables_available(["pynsource"]):
return
files = [
os.path.join(dir_path, f)
for dir_path, dir_names, files in os.walk(path)
for f in fnmatch.filter(files, "*.py")
]
debug("files: {files}".format(files=repr(files)))
for src_file in files:
debug(src_file)
name = src_file.replace(Project.herringfile_dir + "/", "").replace(".py", ".png").replace("/", ".")
output = "classes_{name}".format(name=name)
debug(output)
if not os.path.isfile(output) or (os.path.isfile(output) and is_newer(output, src_file)):
run_python(
"pynsource -y {output} {source}".format(output=output, source=src_file),
verbose=False,
ignore_errors=True,
)
def publish():
""" copy latest docs to a linux base web server """
project_version_name = "{name}-{version}".format(name=Project.base_name, version=Project.version)
project_latest_name = "{name}-latest".format(name=Project.base_name)
doc_version = '{dir}/{file}'.format(dir=Project.docs_path, file=project_version_name)
doc_latest = '{dir}/{file}'.format(dir=Project.docs_path, file=project_latest_name)
docs_html_dir = '{dir}'.format(dir=Project.docs_html_dir)
password = Project.docs_password
if password is None and Project.doc_host_prompt_for_sudo_password:
password = getpass("password for {user}@{host}: ".format(user=Project.docs_user,
host=Project.docs_host))
Project.docs_password = password
info("Publishing to {user}@{host}".format(user=Project.docs_user, host=Project.docs_host))
with RemoteShell(user=Project.docs_user,
password=Project.docs_password,
host=Project.docs_host,
verbose=True) as remote:
remote.run('mkdir -p \"{dir}\"'.format(dir=Project.docs_path))
remote.run('rm -rf \"{path}\"'.format(path=doc_latest))
remote.run('rm -rf \"{path}\"'.format(path=doc_version))
remote.run('mkdir -p \"{dir}\"'.format(dir=doc_version))
for file_ in [os.path.join(docs_html_dir, file_) for file_ in os.listdir(docs_html_dir)]:
remote.put(file_, doc_version)
remote.run('ln -s \"{src}\" \"{dest}\"'.format(src=doc_version, dest=doc_latest))
remote.run('sudo chown -R {user}:{group} \"{dest}\"'.format(user=Project.docs_user,
group=Project.docs_group,
dest=doc_version),
accept_defaults=True, timeout=10)
remote.run('sudo chmod -R g+w \"{dest}\"'.format(dest=doc_version),
accept_defaults=True, timeout=10)
def _neon(text, file_name, animate_logo, fontsize):
info("creating logo")
pre = ' '.join(dedent("""\
-size 500x200
xc:lightblue
-font Comic-Sans-MS-Bold
-pointsize {fontsize}
-gravity center
-undercolor black
-stroke none
-strokewidth 3
""").format(fontsize=fontsize).strip().split('\n'))
post = ' '.join(dedent("""\
-trim
+repage
-shave 1x1
-bordercolor black
-border 20x20
""").strip().split('\n'))
on = ' '.join(dedent("""\
convert
{pre}
-fill DeepSkyBlue
-annotate +0+0 '{text}'
{post}
\( +clone -blur 0x25 -level 0%,50% \)
-compose screen -composite
{file}_on.png
""".format(text=text, file=file_name, pre=pre, post=post)).strip().split('\n'))
off = ' '.join(dedent("""\
convert
{pre}
-fill grey12
-annotate +0+0 '{text}'
{post}
{file}_off.png
""".format(text=text, file=file_name, pre=pre, post=post)).strip().split('\n'))
animated = ' '.join(dedent("""convert \
-adjoin -delay 100 -resize 240 {file}_on.png {file}_off.png {file}_animated.gif
""".format(file=file_name)).strip().split('\n'))
# noinspection PyArgumentEqualDefault
with LocalShell(verbose=False) as local:
if animate_logo:
local.run(on)
local.run(off)
local.run(animated)
local.run('bash -c "rm -f {file}_on.png {file}_off.png"'.format(file=file_name))
logo_image = "{file}_animated.gif".format(file=file_name)
else:
info(on)
local.run(on)
on_image = "{file}_on.png".format(file=file_name)
logo_image = "{file}.png".format(file=file_name)
if os.path.isfile(on_image):
os.rename(on_image, logo_image)
return logo_image
def design():
"""Update the design.rst from the source module's docstrings"""
info("Python version: {version}".format(version=version))
design_header = Project.design_header.strip()
if design_header:
py_files = _find_py_files(Project.package)
if py_files:
with open(Project.design_file, "w") as design_file:
design_file.write("Design\n")
design_file.write("======\n\n")
design_file.write(design_header)
design_file.write("\n\n")
for py_file in sorted(py_files):
docstring, functions, classes = _parse_py_file(py_file)
design_file.write(py_file)
design_file.write("\n")
design_file.write("-" * len(py_file))
design_file.write("\n\n")
design_file.write(docstring)
design_file.write("\n\n")
if functions:
design_file.write("Functions:\n\n")
for function in functions:
design_file.write("* {name}\n".format(name=function))
design_file.write("\n\n")
if classes:
design_file.write("Classes:\n\n")
for class_ in classes:
design_file.write("* {name}\n".format(name=class_))
design_file.write("\n\n")
else:
touch(Project.design_file)
def listvenvs():
"""Run "pip list" in each virtual environment."""
venvs = VirtualenvInfo('python_versions')
info("Project Virtual Environments:")
if not venvs.in_virtualenv and venvs.defined:
for venv_info in venvs.infos():
venv_info.run('pip list ')
info('')
def upvenvs():
"""Run "pip install --update -r requirements" in each virtual environment."""
venvs = VirtualenvInfo('python_versions')
info("Project Virtual Environments:")
if not venvs.in_virtualenv and venvs.defined:
for venv_info in venvs.infos():
venv_info.run('pip install --upgrade pip')
venv_info.run('pip install --upgrade setuptools')
venv_info.run('pip install --upgrade -r requirements.txt')
def publish_gh_pages():
"""copy documentation to github pages"""
if Project.github_url is not None:
tmp_repo_path = None
try:
tmp_repo_path = tempfile.mkdtemp()
with LocalShell(verbose=True) as local:
# clone repo selecting gh-pages branch
# git clone {git_url} {directory}
# git branch --list
local.run("git clone {url} {dir}".format(url=Project.github_url, dir=tmp_repo_path))
with cd(tmp_repo_path, verbose=True):
remote_branches = [
line.lstrip(r"[*\s]*").strip()
for line in local.run("git branch --list -r").splitlines()
]
if "origin/gh-pages" in remote_branches:
local.run("git pull origin")
local.run("git checkout -b gh-pages origin/gh-pages")
# select branch
# git checkout gh-pages
local.run("git checkout gh-pages")
# remove github pages clone directory
# clean_directory(tmp_repo_path)
# touch .nojekyl
touch(".nojekyll")
# copy documentation
if os.path.isdir(Project.docs_html_path):
dir_util.copy_tree(Project.docs_html_path, tmp_repo_path)
# commit and push to github
local.run("git add --all")
local.run("git status")
now = datetime.now().strftime("%c")
message = "{project} Documentation {version} {date}".format(
project=Project.title, version=Project.version, date=now
)
local.run("git commit -m '{message}'".format(message=message))
local.run("git push origin gh-pages")
else:
info("Please create a 'gh-pages' branch on the github repository.")
finally:
if tmp_repo_path is not None:
try:
info("removing {repo}".format(repo=tmp_repo_path))
shutil.rmtree(tmp_repo_path)
except OSError as ex:
if ex.errno != errno.ENOENT:
raise
def __init__(self, *attr_names):
self._ver_attr = None
self._raise_when_in_venv = False
debug(repr(attr_names))
for name in attr_names:
debug(name)
self._ver_attr = getattr(Project, name, None)
if self._ver_attr is not None:
info("_ver_attr: %s" % repr(self._ver_attr))
break
def pdf():
"""Generate PDF API documents"""
venvs = VirtualenvInfo("doc_python_version")
if not venvs.in_virtualenv and venvs.defined:
for venv_info in venvs.infos():
venv_info.run("{herring} doc::pdf_generate".format(herring=Project.herring))
else:
info("Generating documentation using the current python environment")
task_execute("doc::pdf_generate")
def slides():
"""generate project slides"""
venvs = VirtualenvInfo('docs_venv')
info("venvs: {venvs}".format(venvs=repr(venvs.__dict__)))
if not venvs.in_virtualenv and venvs.defined:
for venv_info in venvs.infos():
venv_info.run('{herring} doc::hieroglyph_slides --python-tag py{ver}'.format(herring=Project.herring,
ver=venv_info.ver))
else:
info('Generating slides using the current python environment')
task_execute('doc::hieroglyph_slides')
def watch():
"""generate project documentation"""
venvs = VirtualenvInfo('doc_python_version')
info("venvs: {venvs}".format(venvs=repr(venvs.__dict__)))
if not venvs.in_virtualenv and venvs.defined:
for venv_info in venvs.infos():
venv_info.run('{herring} doc::watcher --python-tag py{ver}'.format(herring=Project.herring,
ver=venv_info.ver))
else:
info('Watching documentation using the current python environment')
task_execute('doc::watcher')
def doc_no_api():
"""generate project documentation without the api"""
venvs = VirtualenvInfo('docs_venv')
info("venvs: {venvs}".format(venvs=repr(venvs.__dict__)))
if not venvs.in_virtualenv and venvs.defined:
for venv_info in venvs.infos():
venv_info.run('{herring} doc::generate_no_api --python-tag py{ver}'.format(herring=Project.herring,
ver=venv_info.ver))
else:
info('Generating documentation using the current python environment')
task_execute('doc::generate_no_api')
def _class_line(module_name, class_name):
"""create the class figure lines for the given module and class"""
info("_class_line(%s, %s)" % (module_name, class_name))
line = ""
classes_image = "uml/classes_{module}.{name}.png".format(module=module_name, name=class_name)
image_path = os.path.join(Project.docs_dir, "_src", classes_image)
if os.path.exists(image_path):
info("adding figure %s" % image_path)
line += "\n.. figure:: {image}\n\n {name} Class\n\n".format(image=classes_image, name=class_name)
else:
warning("%s does not exist!" % image_path)
return line
def doc():
"""generate project documentation"""
venvs = VirtualenvInfo("doc_python_version")
info("venvs: {venvs}".format(venvs=repr(venvs.__dict__)))
if not venvs.in_virtualenv and venvs.defined:
for venv_info in venvs.infos():
venv_info.run(
"{herring} doc::generate --python-tag py{ver}".format(herring=Project.herring, ver=venv_info.ver)
)
else:
info("Generating documentation using the current python environment")
task_execute("doc::generate")
def do_task():
"""
This function create a task that will be a daemon
"""
app = QtGui.QApplication(sys.argv)
path = os.path.join(Project.herringfile_dir, Project.docs_html_dir, 'index.html')
info(path)
url = 'file:///' + pathname2url(os.path.abspath(path))
url = QtCore.QUrl(url)
wb = MainWindow(url)
wb.show()
app.exec_()
def listvenvs():
"""Run "pip list" in each virtual environment."""
venvs = VirtualenvInfo('python_versions')
if venvs.in_virtualenv:
warning('Please deactivate the current virtual environment then try running this task again.')
return
info("Project Virtual Environments:")
if venvs.defined:
for venv_info in venvs.infos():
venv_info.run('pip list ')
info('')
def __init__(self, user=None, password=None, host=None, logfile=None, environment=None, verbose=False,
virtualenv=None, working_dir=None):
"""
:param user: the remote user account
:type user: str
:param password: the password for the remote user account
:type password: str
:param host: the remote machine
:type host: str
:param logfile: optional logfile
:type logfile: str
:param environment: Optional key to Project.prefix and Project.postfix dictionaries
:type environment: str
:param verbose: extra logging
:type verbose: bool
:param virtualenv: directory that contains a virtual environment to activate upon connection
:type virtualenv: str
"""
super(RemoteShell, self).__init__(is_remote=True, verbose=verbose)
if not user:
user = Project.user
if not host:
host = Project.address
Project.user = user
Project.password = password
Project.address = host
self.ssh = pxssh(timeout=1200)
try:
if password:
self.ssh.login(host, user, password)
else:
self.ssh.login(host, user)
except ExceptionPxssh:
if not password:
password = Project.password
if not password:
password = getpass('password for {user}@{host}: '.format(user=user, host=host))
self.ssh.close()
self.ssh = pxssh(timeout=1200)
self.ssh.login(host, user, password)
self.accept_defaults = False
self.logfile = logfile
self.prefix = []
self.postfix = []
if environment:
self.prefix.extend(Project.prefix[environment] or [])
self.postfix.extend(Project.postfix[environment] or [])
if working_dir:
self.prefix.insert(0, "cd {dir} ; ".format(dir=working_dir))
if virtualenv:
self.prefix.insert(0, "source {path}/bin/activate ; ".format(path=virtualenv))
if verbose:
info("remote: {user}@{host}".format(user=user or "", host=host or ""))
def sdist():
""" build source distribution"""
info('')
info("=" * 70)
info('building source distribution')
venvs = VirtualenvInfo('sdist_python_version')
if not venvs.in_virtualenv and venvs.defined:
for venv_info in venvs.infos():
info('Building sdist using {venv} virtual environment'.format(venv=venv_info.venv))
venv_info.run('python setup.py sdist')
else:
with LocalShell() as local:
info("Building sdist using default environment")
local.system("python setup.py sdist")
def update():
"""
Regenerate files (except herringfile) from current templates.
Delete the file(s) you want to update, then run this task.
"""
defaults = _project_defaults()
template = Template()
for template_dir in [os.path.abspath(os.path.join(herringlib, 'herringlib', 'templates'))
for herringlib in HerringFile.herringlib_paths]:
info("template directory: %s" % template_dir)
# noinspection PyArgumentEqualDefault
template.generate(template_dir, defaults, overwrite=False)
def rmvenvs():
"""
Remove all the virtual environments.
Note that we do not remove the docs_venv virtual environment as it is intended to be shared across projects.
"""
venvs = VirtualenvInfo('python_versions')
if venvs.in_virtualenv:
warning('Please deactivate the current virtual environment then try running this task again.')
return
if venvs.defined:
for venv_info in venvs.infos():
venv_info.rmvirtualenv()
else:
info("There are no virtual environments to remove.")
def metrics():
""" Quality metrics """
# Run the metrics in each of the virtual environments defined in Project.metrics_python_versions
# or if not defined, then in Project.wheel_python_versions. If neither are defined, then
# run the test in the current environment.
venvs = VirtualenvInfo('metrics_python_versions', 'wheel_python_versions')
if not venvs.in_virtualenv and venvs.defined:
for venv_info in venvs.infos():
info('Running metrics using the {venv} virtual environment.'.format(venv=venv_info.venv))
venv_info.run('herring metrics::all_metrics')
else:
info('Running metrics using the current python environment')
task_execute('metrics::all_metrics')
def environment():
""" Display project environment """
venvs = VirtualenvInfo('python_versions')
site_packages_cmdline = "python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())'"
project_env = {}
if not venvs.in_virtualenv and venvs.defined:
for venv_info in venvs.infos():
site_packages = venv_info.run(site_packages_cmdline).strip().splitlines()[2]
project_env[venv_info.venv + ': site-packages'] = site_packages
else:
with LocalShell() as local:
site_packages = local.system(site_packages_cmdline).strip()
project_env['site-packages'] = site_packages
info(pformat(project_env))
return project_env
def _package_line(module_name):
"""create the package figure lines for the given module"""
info("_package_line(%s)" % module_name)
line = ""
package_image = "uml/packages_{name}.svg".format(name=module_name.split(".")[-1])
classes_image = "uml/classes_{name}.svg".format(name=module_name.split(".")[-1])
image_path = os.path.join(Project.docs_dir, "_src", package_image)
if os.path.exists(image_path):
info("adding figure %s" % image_path)
line += "\n.. figure:: {image}\n :width: 1100 px\n\n {name} Packages\n\n".format(
image=package_image, name=module_name
)
line += "\n.. figure:: {image}\n\n {name} Classes\n\n".format(image=classes_image, name=module_name)
else:
warning("%s does not exist!" % image_path)
return line
