def radon():
""" Cyclomatic complexity metrics """
if not executables_available(['radon']):
return
mkdir_p(Project.quality_dir)
def qd(basename):
"""
get the relative path to report file in quality directory
:param basename: the report base name.
:returns: the relative path to the given report name in the quality directory.
"""
return os.path.join(Project.quality_dir, basename)
with LocalShell() as local:
local.system("radon cc -s --average --total-average {dir} > {out}".format(
dir=Project.package, out=qd('radon_cc.txt')))
local.system("radon cc -s --average --total-average --json {dir} > {out}".format(
dir=Project.package, out=qd('radon_cc.json')))
local.system("radon cc -s --average --total-average --xml {dir} > {out}".format(
dir=Project.package, out=qd('radon_cc.xml')))
local.system("radon mi -s {dir} > {out}".format(
dir=Project.package, out=qd('radon_mi.txt')))
local.system("radon raw -s {dir} > {out}".format(
dir=Project.package, out=qd('radon_raw.txt')))
local.system("radon raw -s --json {dir} > {out}".format(
dir=Project.package, out=qd('radon_raw.json')))
def pycodestyle():
"""Run pycodestyle checks"""
if not executables_available(['pycodestyle']):
return
mkdir_p(Project.quality_dir)
pycodestyle_text = os.path.join(Project.quality_dir, 'pycodestyle.txt')
pycodestyle_out = os.path.join(Project.quality_dir, 'pycodestyle.out')
pycodestyle_html = os.path.join(Project.quality_dir, 'pycodestyle.html')
os.system("rm -f %s" % pycodestyle_text)
os.system("PYTHONPATH=%s pycodestyle %s 2>/dev/null >%s" % (Project.pythonPath, Project.package, pycodestyle_text))
os.system("pepper8 -o %s %s" % (pycodestyle_html, pycodestyle_text))
# need to reorder the columns to make compatible with pylint file format
# pycodestyle output: "{file}:{line}:{column}: {err} {desc}"
# pylint output: "{file}:{line}: [{err}] {desc}"
# noinspection PyArgumentEqualDefault
with open(pycodestyle_text, 'r') as src_file:
lines = src_file.readlines()
with open(pycodestyle_out, 'w') as out_file:
for line in lines:
match = re.match(r"(.+):(\d+):(\d+):\s*(\S+)\s+(.+)", line)
if match:
out_file.write("{file}:{line}: [{err}] {desc}\n".format(file=match.group(1),
line=match.group(2),
err=match.group(4),
desc=match.group(5)))
def sloccount():
"""Generate SLOCCount output file, sloccount.sc, used by jenkins"""
if not executables_available(['sloccount']):
return
sloc_data = qd('slocdata')
mkdir_p(sloc_data)
sloc_filename = qd('sloccount.sc')
with LocalShell() as local:
output = local.run("sloccount --datadir {data} --wide --details {src}".format(data=sloc_data,
src=Project.package))
if os.path.isfile(sloc_filename):
os.remove(sloc_filename)
with open(sloc_filename, 'w') as sloc_file:
sloc_file.write(output)
counts = {'all': 0}
for line in output.splitlines():
match = re.match(r"^(\d+)\s+(\S+)\s+(\S+)\s+(\S+)", line)
if match:
language = match.group(2)
if language not in counts:
counts[language] = 0
counts[language] += int(match.group(1))
counts['all'] += int(match.group(1))
with open(qd("sloccount.js"), 'w') as out_file:
out_file.write("\n{name}_sloccount_data = {{\n".format(name=Project.name))
for key in sorted(counts.keys()):
out_file.write(" \"{key}\": \"{value}\",\n".format(key=key, value=counts[key]))
out_file.write("};\n")
def violations_report():
""" Quality, violations, metrics reports """
lines = ["""
<html>
<head>
<title>Violation Reports</title>
</head>
<body>
<h1>Violation Reports</h1>
"""]
mkdir_p(Project.quality_dir)
pylint_log = os.path.join(Project.quality_dir, 'pylint.log')
pep8_text = os.path.join(Project.quality_dir, 'pep8.txt')
index_html = os.path.join(Project.quality_dir, 'index.html')
for fileSpec in (pylint_log, pep8_text):
file_base = os.path.basename(fileSpec)
violations_name = "violations.%s" % file_base
summary = "violations.summary.%s" % file_base
lines.append(" <h2>%s</h2>" % os.path.splitext(file_base)[0])
lines.append(" <ul>")
lines.append(" <li><a href='%s'>Report</a></li>" % file_base)
lines.append(" <li><a href='%s'>Violations</a></li>" % violations_name)
lines.append(" <li><a href='%s'>Violation Summary</a></li>" % summary)
lines.append(" </ul>")
lines.append("""
</body>
</html>
""")
with open(index_html, 'w') as f:
f.write("\n".join(lines))
def diagrams():
"""Create UML diagrams"""
if Project.package is not None:
path = os.path.join(Project.herringfile_dir, Project.package)
mkdir_p(Project.uml_dir)
with cd(Project.uml_dir, verbose=True):
_create_module_diagrams(path)
_create_class_diagrams(path)
def bandit():
""" scan source code for possible security vulnerabilities """
mkdir_p(Project.quality_dir)
bandit_filename = qd("security.txt")
with LocalShell() as local:
output = local.run("bandit -r {src}".format(src=Project.package), verbose=True)
if os.path.isfile(bandit_filename):
os.remove(bandit_filename)
with open(bandit_filename, 'w') as sloc_file:
sloc_file.write(output)
def cheesecake():
""" Run the cheesecake kwalitee metric """
if not executables_available(['cheesecake_index']):
return
mkdir_p(Project.quality_dir)
cheesecake_log = os.path.join(Project.quality_dir, 'cheesecake.log')
with LocalShell() as local:
local.system("cheesecake_index --path=dist/%s-%s.tar.gz --keep-log -l %s" %
(Project.name,
Project.version,
cheesecake_log))
def lint():
""" Run pylint with project overrides from pylint.rc """
if not executables_available(['pylint']):
return
mkdir_p(Project.quality_dir)
options = ''
if os.path.exists(Project.pylintrc):
options += "--rcfile=pylint.rc"
pylint_log = os.path.join(Project.quality_dir, 'pylint.log')
with LocalShell() as local:
local.system("pylint {options} {dir} > {log}".format(options=options,
dir=Project.package,
log=pylint_log))
def complexity():
""" Run McCabe code complexity """
if not executables_available(['pymetrics']):
return
mkdir_p(Project.quality_dir)
quality_dir = Project.quality_dir
complexity_txt = os.path.join(quality_dir, 'complexity.txt')
graph = os.path.join(quality_dir, 'output.png')
acc = os.path.join(quality_dir, 'complexity_acc.txt')
metrics_html = os.path.join(quality_dir, 'complexity_metrics.html')
with LocalShell() as local:
local.system("touch %s" % complexity_txt)
local.system("touch %s" % acc)
local.system("pymetrics --nosql --nocsv `find %s/ -iname \"*.py\"` > %s" %
(Project.package, complexity_txt))
def flake8():
"""Run flake8 checks"""
if not executables_available(['flake8']):
return
mkdir_p(Project.quality_dir)
flake8_text = os.path.join(Project.quality_dir, 'flake8.txt')
flake8_out = os.path.join(Project.quality_dir, 'flake8.out')
flake8_html = os.path.join(Project.quality_dir, 'flake8.html')
flake8_js = os.path.join(Project.quality_dir, 'flake8.js')
os.system("rm -f %s" % flake8_text)
os.system("PYTHONPATH=%s flake8 --show-source --statistics %s 2>/dev/null >%s" % (Project.pythonPath, Project.package, flake8_text))
os.system("pepper8 -o %s %s" % (flake8_html, flake8_text))
# need to reorder the columns to make compatible with pylint file format
# flake8 output: "{file}:{line}:{column}: {err} {desc}"
# pylint output: "{file}:{line}: [{err}] {desc}"
# noinspection PyArgumentEqualDefault
with open(flake8_text, 'r') as src_file:
lines = src_file.readlines()
errors = 0
warnings = 0
others = 0
with open(flake8_out, 'w') as out_file:
for line in lines:
match = re.match(r"(.+):(\d+):(\d+):\s*(\S+)\s+(.+)", line)
if match:
if match.group(4).startswith('E'):
errors += 1
elif match.group(4).startswith('W'):
warnings += 1
else:
others += 1
out_file.write("{file}:{line}: [{err}] {desc}\n".format(file=match.group(1),
line=match.group(2),
err=match.group(4),
desc=match.group(5)))
with open(flake8_js, 'w') as out_file:
out_file.write(dedent("""
{name}_flake8_data = {{
"errors": "{errors}",
"warnings": "{warnings}",
"other": "{others}"
}};
""".format(name=Project.name, errors=errors, warnings=warnings, others=others)))
def sloc():
"""Run sloccount to get the source lines of code."""
if not executables_available(['sloccount']):
return
mkdir_p(Project.quality_dir)
sloc_json = os.path.join(Project.quality_dir, 'sloc.json')
totals_by_language = _sloc_totals_by_language()
total_sloc = 0
for value in totals_by_language.values():
total_sloc += value[0]
with open(sloc_json, 'w') as json_file:
json.dump(totals_by_language, json_file)
for lang in totals_by_language.keys():
info("{lang}: {total} ({percentage}%)".format(lang=lang,
total=totals_by_language[lang][0],
percentage=totals_by_language[lang][1]))
info("Total SLOC: {total}".format(total=total_sloc))
def new_image():
"""Create a new image directory and populate with a default Dockerfile."""
names = list(task.argv)
if not names:
if Project.prompt and task.arg_prompt is not None:
name = prompt(task.arg_prompt)
if name is not None and name.strip():
names.append(name)
for name in names:
container_dir = os.path.join(Project.docker_dir, Project.docker_containers_dir, name)
mkdir_p(container_dir)
# populate container dir with Dockerfile and .dockerignore
dockerfile = os.path.join(container_dir, 'Dockerfile')
dockerignore = os.path.join(container_dir, '.dockerignore')
shutil.copyfile(DOCKERFILE_TEMPLATE, dockerfile)
touch(dockerignore)
info("Created folder for new image at: {dir}".format(dir=container_dir))
def _render(self, template_filename, template_dir, dest_filename, defaults, overwrite=False):
# info('dest_filename: %s' % dest_filename)
if os.path.isdir(template_filename):
mkdir_p(template_filename)
else:
mkdir_p(os.path.dirname(dest_filename))
template_root, template_ext = os.path.splitext(template_filename)
if template_ext == '.template':
if not os.path.isdir(dest_filename):
if overwrite or not os.path.isfile(dest_filename) or os.path.getsize(dest_filename) == 0:
self._create_from_template(template_filename, dest_filename, **defaults)
else:
if overwrite or not os.path.isfile(dest_filename):
if os.path.join(template_dir, '__init__.py') != template_filename and os.path.join(
template_dir, 'bin', '__init__.py') != template_filename:
shutil.copyfile(template_filename, dest_filename)
if os.path.exists(template_filename) and os.path.exists(dest_filename):
shutil.copymode(template_filename, dest_filename)
def __directory(self, relative_name):
"""return the full path from the given path relative to the herringfile directory"""
if relative_name.startswith('~'):
directory_name = os.path.expanduser(relative_name)
elif relative_name.startswith('/'):
directory_name = os.path.abspath(relative_name)
else:
directory_name = os.path.join(self.herringfile_dir, relative_name)
return mkdir_p(directory_name)
def radon():
""" Cyclomatic complexity metrics """
if not executables_available(['radon']):
return
mkdir_p(Project.quality_dir)
with LocalShell() as local:
local.system("radon cc -s --average --total-average {dir} > {out}".format(
dir=Project.package, out=qd('radon_cc.txt')))
local.system("radon cc -s --average --total-average --json {dir} > {out}".format(
dir=Project.package, out=qd('radon_cc.json')))
local.system("radon cc -s --average --total-average --xml {dir} > {out}".format(
dir=Project.package, out=qd('radon_cc.xml')))
local.system("radon mi -s {dir} > {out}".format(
dir=Project.package, out=qd('radon_mi.txt')))
local.system("radon raw -s {dir} > {out}".format(
dir=Project.package, out=qd('radon_raw.txt')))
local.system("radon raw -s --json {dir} > {out}".format(
dir=Project.package, out=qd('radon_raw.json')))
grade_a = local.system("grep -c \" - A \" {txt}".format(txt=qd('radon_cc.txt'))).strip()
grade_b = local.system("grep -c \" - B \" {txt}".format(txt=qd('radon_cc.txt'))).strip()
grade_c = local.system("grep -c \" - C \" {txt}".format(txt=qd('radon_cc.txt'))).strip()
grade_d = local.system("grep -c \" - D \" {txt}".format(txt=qd('radon_cc.txt'))).strip()
grade_e = local.system("grep -c \" - E \" {txt}".format(txt=qd('radon_cc.txt'))).strip()
grade_f = local.system("grep -c \" - F \" {txt}".format(txt=qd('radon_cc.txt'))).strip()
with open(qd("radon_cc_summary.js"), 'w') as out_file:
out_file.write(dedent(r"""
{name}_code_complexity_data = {{
"A": "{a}",
"B": "{b}",
"C": "{c}",
"D": "{d}",
"E": "{e}",
"F": "{f}",
}};
""".format(name=Project.name, a=grade_a, b=grade_b, c=grade_c, d=grade_d, e=grade_e, f=grade_f)))
def violations():
"""Find the violations by inverting the results from the code analysis"""
mkdir_p(Project.quality_dir)
pylint_log = os.path.join(Project.quality_dir, 'pylint.log')
pep8_text = os.path.join(Project.quality_dir, 'pep8.txt')
for fileSpec in (pylint_log, pep8_text):
pyviolations = PyViolations()
pyviolations.process_file(fileSpec)
# noinspection PyArgumentEqualDefault
outputter = TextOutputter(summary=False)
pyviolations.report(outputter)
output_filespec = os.path.join(Project.quality_dir,
"violations.%s" % os.path.basename(fileSpec))
with open(output_filespec, 'w') as f:
f.write(outputter.to_string())
outputter = TextOutputter(summary=True)
pyviolations.report(outputter)
output_filespec = os.path.join(Project.quality_dir,
"violations.summary.%s" % os.path.basename(fileSpec))
with open(output_filespec, 'w') as f:
f.write(outputter.to_string())
def test_template_rendering():
"""
The goal when rendering is to never lose previous content and to only create a backup when necessary.
So we test creating backup files as needed:
* render file with param dict 0 verify no backup
* render file with param dict 0 verify no backup
* render file with param dict 1 verify one backup, ~ backup is file rendered with param dict 0
* render file with param dict 1 verify one backup, ~ backup is file rendered with param dict 0
* render file with param dict 2 verify two backups, ~ backup is file rendered with param dict 0,
~1 backup is file rendered with param dict 1
* render file with param dict 2 verify two backups, ~ backup is file rendered with param dict 0,
~1 backup is file rendered with param dict 1
The dummy.py.template renders to a script that prints the params hash so we can easily verify content.
This gives us a round trip:
* params[n] dict is rendered to dummy.py
* running dummy.py then eval'ing the output should give a dict equal to the original params[n] dict
"""
params = [
{
'name': 'Foo',
'package': 'foo',
'author': 'joe bob',
'author_email': '*****@*****.**',
'description': 'Just another foo',
},
{
'name': 'FooBar',
'package': 'foo',
'author': 'joe bob',
'author_email': '*****@*****.**',
'description': 'Just another foobar',
},
{
'name': 'FooBar',
'package': 'foobar',
'author': 'joe bob',
'author_email': '*****@*****.**',
'description': 'Just another foobar',
},
]
dest_dir = os.path.join(os.path.dirname(__file__), 'output')
if os.path.isdir(dest_dir):
shutil.rmtree(dest_dir)
mkdir_p(dest_dir)
source_name = os.path.join(os.path.dirname(__file__), 'dummy.py.template')
dest_name = os.path.join(dest_dir, 'dummy.py')
assert len(backup_files(dest_dir)) == 0
template = Template()
with LocalShell() as local:
template._create_from_template(src_filename=source_name, dest_filename=dest_name, **params[0])
# dist_dir: ['test.py']
assert len(backup_files(dest_dir)) == 0
assert eval(local.run("python {file}".format(file=dest_name))) == params[0]
template._create_from_template(src_filename=source_name, dest_filename=dest_name, **params[0])
# dist_dir: ['test.py']
assert len(backup_files(dest_dir)) == 0
assert eval(local.run("python {file}".format(file=dest_name))) == params[0]
template._create_from_template(src_filename=source_name, dest_filename=dest_name, **params[1])
# dist_dir: ['test.py', 'test.py~']
assert 'dummy.py~' in backup_files(dest_dir)
assert len(backup_files(dest_dir)) == 1
assert eval(local.run("python {file}".format(file=dest_name))) == params[1]
assert eval(local.run("python {file}~".format(file=dest_name))) == params[0]
template._create_from_template(src_filename=source_name, dest_filename=dest_name, **params[1])
# dist_dir: ['test.py', 'test.py~']
assert 'dummy.py~' in backup_files(dest_dir)
assert len(backup_files(dest_dir)) == 1
assert eval(local.run("python {file}".format(file=dest_name))) == params[1]
assert eval(local.run("python {file}~".format(file=dest_name))) == params[0]
template._create_from_template(src_filename=source_name, dest_filename=dest_name, **params[2])
# dist_dir: ['test.py', 'test.py~', 'test.py1~']
assert 'dummy.py~' in backup_files(dest_dir)
assert 'dummy.py1~' in backup_files(dest_dir)
assert len(backup_files(dest_dir)) == 2
assert eval(local.run("python {file}".format(file=dest_name))) == params[2]
assert eval(local.run("python {file}1~".format(file=dest_name))) == params[1]
assert eval(local.run("python {file}~".format(file=dest_name))) == params[0]
template._create_from_template(src_filename=source_name, dest_filename=dest_name, **params[2])
# dist_dir: ['test.py', 'test.py~', 'test.py1~']
assert 'dummy.py~' in backup_files(dest_dir)
assert 'dummy.py1~' in backup_files(dest_dir)
assert len(backup_files(dest_dir)) == 2
assert eval(local.run("python {file}".format(file=dest_name))) == params[2]
assert eval(local.run("python {file}1~".format(file=dest_name))) == params[1]
assert eval(local.run("python {file}~".format(file=dest_name))) == params[0]
def graph_complexity():
""" Create Cyclomatic Complexity graphs. """
import matplotlib
matplotlib.use('Agg') # Must be before importing matplotlib.pyplot or pylab!
from matplotlib import pyplot
if not executables_available(['radon']):
return
mkdir_p(Project.quality_dir)
graphic_type_ext = 'svg'
with LocalShell() as local:
data_json = local.run("radon cc -s --json {dir}".format(dir=Project.package))
data = json.loads(data_json)
# info(pformat(data))
components = {'function': {}, 'method': {}, 'class': {}}
for path in data.keys():
for component in data[path]:
if isinstance(component, dict):
complexity_score = component['complexity']
if complexity_score not in components[component['type']]:
components[component['type']][complexity_score] = []
# noinspection PyUnresolvedReferences
components[component['type']][complexity_score].append(component)
# else:
# warning("{path}: {component}".format(path=path, component=pformat(component)))
component_names = {
'all': 'Components',
'function': 'Functions',
'class': 'Classes',
'method': 'Methods'
}
fig_number = 1
x = {}
y = {}
for component_type in components.keys():
info(component_type)
x[component_type] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21, 22, 23, 24, 25]
y[component_type] = [0] * 25
for score in sorted(components[component_type].keys()):
cnt = len(components[component_type][score])
# info("{complexity}: {cnt}".format(complexity=score, cnt=cnt))
if score < 25:
y[component_type][score - 1] += cnt
else:
y[component_type][-1] += cnt
info("fig_number: %d" % fig_number)
# plot_number = 110 + fig_number
plot_number = 111
info("plot_number: %d" % plot_number)
fig = pyplot.figure(fig_number)
pyplot.subplot(plot_number)
fig.suptitle("Cyclomatic Complexity of {type}".format(type=component_names[component_type]))
pyplot.bar(x[component_type][0:4], y[component_type][0:4], align='center', color='green')
pyplot.bar(x[component_type][5:9], y[component_type][5:9], align='center', color='blue')
pyplot.bar(x[component_type][10:14], y[component_type][10:14], align='center', color='yellow')
pyplot.bar(x[component_type][15:19], y[component_type][15:19], align='center', color='orange')
pyplot.bar(x[component_type][20:], y[component_type][20:], align='center', color='red')
pyplot.xlabel('Cyclomatic Complexity')
pyplot.ylabel('Number of {type}'.format(type=component_names[component_type]))
pyplot.savefig(os.path.join(Project.quality_dir, "cc_{type}.{ext}".format(type=component_type,
ext=graphic_type_ext)))
fig_number += 1
info("fig_number: %d" % fig_number)
# plot_number = 110 + fig_number
plot_number = 111
info("plot_number: %d" % plot_number)
fig = pyplot.figure(fig_number)
pyplot.subplot(plot_number)
fig.suptitle("Cyclomatic Complexity of All Components")
hatch = {'class': '/', 'method': '+', 'function': '*'}
bottom = [0] * 25
legend_bar = {}
for component_type in components.keys():
legend_bar[component_type] = pyplot.bar(x[component_type][0:4], y[component_type][0:4], align='center',
color='green', hatch=hatch[component_type], bottom=bottom[0:4])
pyplot.bar(x[component_type][5:9], y[component_type][5:9], align='center', color='blue',
hatch=hatch[component_type], bottom=bottom[5:9])
pyplot.bar(x[component_type][10:14], y[component_type][10:14], align='center', color='yellow',
hatch=hatch[component_type], bottom=bottom[10:14])
pyplot.bar(x[component_type][15:19], y[component_type][15:19], align='center', color='orange',
hatch=hatch[component_type], bottom=bottom[15:19])
pyplot.bar(x[component_type][20:24], y[component_type][20:24], align='center', color='red',
hatch=hatch[component_type], bottom=bottom[20:24])
bottom = list([bottom[j] + y[component_type][j] for j in range(len(bottom))])
pyplot.xlabel('Cyclomatic Complexity')
pyplot.ylabel('Number of Components')
pyplot.legend((legend_bar[component_type] for component_type in components.keys()),
(component_type for component_type in components.keys()))
pyplot.savefig(os.path.join(Project.quality_dir, "cc_all.{ext}".format(ext=graphic_type_ext)))
if '--display' in task.argv:
pyplot.show(fig_number)