def _create_class_diagrams(path):
"""
Create class UML diagram
:param path: path to the module file.
:type path: str
"""
info("_create_class_diagrams")
if not executables_available(["pynsource"]):
return
files = [
os.path.join(dir_path, f)
for dir_path, dir_names, files in os.walk(path)
for f in fnmatch.filter(files, "*.py")
]
debug("files: {files}".format(files=repr(files)))
for src_file in files:
debug(src_file)
name = src_file.replace(Project.herringfile_dir + "/", "").replace(".py", ".png").replace("/", ".")
output = "classes_{name}".format(name=name)
debug(output)
if not os.path.isfile(output) or (os.path.isfile(output) and is_newer(output, src_file)):
run_python(
"pynsource -y {output} {source}".format(output=output, source=src_file),
verbose=False,
ignore_errors=True,
)
def pycodestyle():
"""Run pycodestyle checks"""
if not executables_available(['pycodestyle']):
return
mkdir_p(Project.quality_dir)
pycodestyle_text = os.path.join(Project.quality_dir, 'pycodestyle.txt')
pycodestyle_out = os.path.join(Project.quality_dir, 'pycodestyle.out')
pycodestyle_html = os.path.join(Project.quality_dir, 'pycodestyle.html')
os.system("rm -f %s" % pycodestyle_text)
os.system("PYTHONPATH=%s pycodestyle %s 2>/dev/null >%s" % (Project.pythonPath, Project.package, pycodestyle_text))
os.system("pepper8 -o %s %s" % (pycodestyle_html, pycodestyle_text))
# need to reorder the columns to make compatible with pylint file format
# pycodestyle output: "{file}:{line}:{column}: {err} {desc}"
# pylint output: "{file}:{line}: [{err}] {desc}"
# noinspection PyArgumentEqualDefault
with open(pycodestyle_text, 'r') as src_file:
lines = src_file.readlines()
with open(pycodestyle_out, 'w') as out_file:
for line in lines:
match = re.match(r"(.+):(\d+):(\d+):\s*(\S+)\s+(.+)", line)
if match:
out_file.write("{file}:{line}: [{err}] {desc}\n".format(file=match.group(1),
line=match.group(2),
err=match.group(4),
desc=match.group(5)))
def sloccount():
"""Generate SLOCCount output file, sloccount.sc, used by jenkins"""
if not executables_available(['sloccount']):
return
sloc_data = qd('slocdata')
mkdir_p(sloc_data)
sloc_filename = qd('sloccount.sc')
with LocalShell() as local:
output = local.run("sloccount --datadir {data} --wide --details {src}".format(data=sloc_data,
src=Project.package))
if os.path.isfile(sloc_filename):
os.remove(sloc_filename)
with open(sloc_filename, 'w') as sloc_file:
sloc_file.write(output)
counts = {'all': 0}
for line in output.splitlines():
match = re.match(r"^(\d+)\s+(\S+)\s+(\S+)\s+(\S+)", line)
if match:
language = match.group(2)
if language not in counts:
counts[language] = 0
counts[language] += int(match.group(1))
counts['all'] += int(match.group(1))
with open(qd("sloccount.js"), 'w') as out_file:
out_file.write("\n{name}_sloccount_data = {{\n".format(name=Project.name))
for key in sorted(counts.keys()):
out_file.write(" \"{key}\": \"{value}\",\n".format(key=key, value=counts[key]))
out_file.write("};\n")
def radon():
""" Cyclomatic complexity metrics """
if not executables_available(['radon']):
return
mkdir_p(Project.quality_dir)
def qd(basename):
"""
get the relative path to report file in quality directory
:param basename: the report base name.
:returns: the relative path to the given report name in the quality directory.
"""
return os.path.join(Project.quality_dir, basename)
with LocalShell() as local:
local.system("radon cc -s --average --total-average {dir} > {out}".format(
dir=Project.package, out=qd('radon_cc.txt')))
local.system("radon cc -s --average --total-average --json {dir} > {out}".format(
dir=Project.package, out=qd('radon_cc.json')))
local.system("radon cc -s --average --total-average --xml {dir} > {out}".format(
dir=Project.package, out=qd('radon_cc.xml')))
local.system("radon mi -s {dir} > {out}".format(
dir=Project.package, out=qd('radon_mi.txt')))
local.system("radon raw -s {dir} > {out}".format(
dir=Project.package, out=qd('radon_raw.txt')))
local.system("radon raw -s --json {dir} > {out}".format(
dir=Project.package, out=qd('radon_raw.json')))
def _create_class_diagrams(path):
"""
Create class UML diagram
:param path: path to the module file.
:type path: str
"""
info("_create_class_diagrams")
if not executables_available(['pynsource']):
warning('pynsource not available')
return
files = [os.path.join(dir_path, f)
for dir_path, dir_names, files in os.walk(path)
for f in fnmatch.filter(files, '*.py')]
debug("files: {files}".format(files=repr(files)))
with open(os.path.join(Project.docs_dir, "pynsource.log"), "w") as outputter:
for src_file in files:
debug(src_file)
name = src_file.replace(Project.herringfile_dir + '/', '').replace('.py', '.png').replace('/', '.')
output = "classes_{name}".format(name=name)
debug(output)
if not os.path.isfile(output) or (os.path.isfile(output) and is_newer(output, src_file)):
output = run_python("pynsource -y {output} {source}".format(output=output, source=src_file),
verbose=False, ignore_errors=True)
outputter.write(output)
def _create_module_diagrams(path):
"""
create module UML diagrams
:param path: the module path
:type path: str
"""
info("_create_module_diagrams")
if not executables_available(['pyreverse']):
warning('pyreverse not available')
return
with open(os.path.join(Project.docs_dir, "pyreverse.log"), "w") as outputter:
for module_path in [root for root, dirs, files in os.walk(path) if os.path.basename(root) != '__pycache__']:
debug("module_path: {path}".format(path=module_path))
init_filename = os.path.join(module_path, '__init__.py')
if os.path.exists(init_filename):
info(init_filename)
name = os.path.basename(module_path).split(".")[0]
output = run_python('pyreverse -o svg -p {name} {module} '.format(name=name, module=module_path),
verbose=True, ignore_errors=True)
outputter.write(output)
errors = [line for line in output.splitlines() if not line.startswith('parsing')]
if errors:
info(errors)
def cheesecake():
""" Run the cheesecake kwalitee metric """
if not executables_available(['cheesecake_index']):
return
mkdir_p(Project.quality_dir)
cheesecake_log = os.path.join(Project.quality_dir, 'cheesecake.log')
with LocalShell() as local:
local.system("cheesecake_index --path=dist/%s-%s.tar.gz --keep-log -l %s" %
(Project.name,
Project.version,
cheesecake_log))
def lint():
""" Run pylint with project overrides from pylint.rc """
if not executables_available(['pylint']):
return
mkdir_p(Project.quality_dir)
options = ''
if os.path.exists(Project.pylintrc):
options += "--rcfile=pylint.rc"
pylint_log = os.path.join(Project.quality_dir, 'pylint.log')
with LocalShell() as local:
local.system("pylint {options} {dir} > {log}".format(options=options,
dir=Project.package,
log=pylint_log))
def complexity():
""" Run McCabe code complexity """
if not executables_available(['pymetrics']):
return
mkdir_p(Project.quality_dir)
quality_dir = Project.quality_dir
complexity_txt = os.path.join(quality_dir, 'complexity.txt')
graph = os.path.join(quality_dir, 'output.png')
acc = os.path.join(quality_dir, 'complexity_acc.txt')
metrics_html = os.path.join(quality_dir, 'complexity_metrics.html')
with LocalShell() as local:
local.system("touch %s" % complexity_txt)
local.system("touch %s" % acc)
local.system("pymetrics --nosql --nocsv `find %s/ -iname \"*.py\"` > %s" %
(Project.package, complexity_txt))
def flake8():
"""Run flake8 checks"""
if not executables_available(['flake8']):
return
mkdir_p(Project.quality_dir)
flake8_text = os.path.join(Project.quality_dir, 'flake8.txt')
flake8_out = os.path.join(Project.quality_dir, 'flake8.out')
flake8_html = os.path.join(Project.quality_dir, 'flake8.html')
flake8_js = os.path.join(Project.quality_dir, 'flake8.js')
os.system("rm -f %s" % flake8_text)
os.system("PYTHONPATH=%s flake8 --show-source --statistics %s 2>/dev/null >%s" % (Project.pythonPath, Project.package, flake8_text))
os.system("pepper8 -o %s %s" % (flake8_html, flake8_text))
# need to reorder the columns to make compatible with pylint file format
# flake8 output: "{file}:{line}:{column}: {err} {desc}"
# pylint output: "{file}:{line}: [{err}] {desc}"
# noinspection PyArgumentEqualDefault
with open(flake8_text, 'r') as src_file:
lines = src_file.readlines()
errors = 0
warnings = 0
others = 0
with open(flake8_out, 'w') as out_file:
for line in lines:
match = re.match(r"(.+):(\d+):(\d+):\s*(\S+)\s+(.+)", line)
if match:
if match.group(4).startswith('E'):
errors += 1
elif match.group(4).startswith('W'):
warnings += 1
else:
others += 1
out_file.write("{file}:{line}: [{err}] {desc}\n".format(file=match.group(1),
line=match.group(2),
err=match.group(4),
desc=match.group(5)))
with open(flake8_js, 'w') as out_file:
out_file.write(dedent("""
{name}_flake8_data = {{
"errors": "{errors}",
"warnings": "{warnings}",
"other": "{others}"
}};
""".format(name=Project.name, errors=errors, warnings=warnings, others=others)))
def sloc():
"""Run sloccount to get the source lines of code."""
if not executables_available(['sloccount']):
return
mkdir_p(Project.quality_dir)
sloc_json = os.path.join(Project.quality_dir, 'sloc.json')
totals_by_language = _sloc_totals_by_language()
total_sloc = 0
for value in totals_by_language.values():
total_sloc += value[0]
with open(sloc_json, 'w') as json_file:
json.dump(totals_by_language, json_file)
for lang in totals_by_language.keys():
info("{lang}: {total} ({percentage}%)".format(lang=lang,
total=totals_by_language[lang][0],
percentage=totals_by_language[lang][1]))
info("Total SLOC: {total}".format(total=total_sloc))
def rstlint():
"""Check the RST in the source files"""
if not executables_available(['rst-lint']):
return
rst_files = [os.path.join(dir_path, f)
for dir_path, dir_names, files in os.walk(Project.herringfile_dir)
for f in fnmatch.filter(files, '*.rst')]
src_files = [os.path.join(dir_path, f)
for dir_path, dir_names, files in os.walk(Project.herringfile_dir)
for f in fnmatch.filter(files, '*.py')]
with LocalShell() as local:
for src_file in rst_files + src_files:
cmd_line = 'rst-lint {file}'.format(file=src_file)
result = local.system(cmd_line, verbose=False)
if not re.search(r'No problems found', result):
info(cmd_line)
info(result)
def radon():
""" Cyclomatic complexity metrics """
if not executables_available(['radon']):
return
mkdir_p(Project.quality_dir)
with LocalShell() as local:
local.system("radon cc -s --average --total-average {dir} > {out}".format(
dir=Project.package, out=qd('radon_cc.txt')))
local.system("radon cc -s --average --total-average --json {dir} > {out}".format(
dir=Project.package, out=qd('radon_cc.json')))
local.system("radon cc -s --average --total-average --xml {dir} > {out}".format(
dir=Project.package, out=qd('radon_cc.xml')))
local.system("radon mi -s {dir} > {out}".format(
dir=Project.package, out=qd('radon_mi.txt')))
local.system("radon raw -s {dir} > {out}".format(
dir=Project.package, out=qd('radon_raw.txt')))
local.system("radon raw -s --json {dir} > {out}".format(
dir=Project.package, out=qd('radon_raw.json')))
grade_a = local.system("grep -c \" - A \" {txt}".format(txt=qd('radon_cc.txt'))).strip()
grade_b = local.system("grep -c \" - B \" {txt}".format(txt=qd('radon_cc.txt'))).strip()
grade_c = local.system("grep -c \" - C \" {txt}".format(txt=qd('radon_cc.txt'))).strip()
grade_d = local.system("grep -c \" - D \" {txt}".format(txt=qd('radon_cc.txt'))).strip()
grade_e = local.system("grep -c \" - E \" {txt}".format(txt=qd('radon_cc.txt'))).strip()
grade_f = local.system("grep -c \" - F \" {txt}".format(txt=qd('radon_cc.txt'))).strip()
with open(qd("radon_cc_summary.js"), 'w') as out_file:
out_file.write(dedent(r"""
{name}_code_complexity_data = {{
"A": "{a}",
"B": "{b}",
"C": "{c}",
"D": "{d}",
"E": "{e}",
"F": "{f}",
}};
""".format(name=Project.name, a=grade_a, b=grade_b, c=grade_c, d=grade_d, e=grade_e, f=grade_f)))
def _create_module_diagrams(path):
"""
create module UML diagrams
:param path: the module path
:type path: str
"""
info("_create_module_diagrams")
if not executables_available(["pyreverse"]):
return
# TODO fixme hangs on tp-otto
for module_path in [root for root, dirs, files in os.walk(path)]:
info("module_path: {path}".format(path=module_path))
init_filename = os.path.join(module_path, "__init__.py")
if os.path.exists(init_filename):
info(init_filename)
name = os.path.basename(module_path).split(".")[0]
output = run_python(
"pyreverse -o svg -p {name} {module}".format(name=name, module=module_path),
verbose=True,
ignore_errors=True,
)
info([line for line in output.splitlines() if not line.startswith("parsing")])
def graph_complexity():
""" Create Cyclomatic Complexity graphs. """
import matplotlib
matplotlib.use('Agg') # Must be before importing matplotlib.pyplot or pylab!
from matplotlib import pyplot
if not executables_available(['radon']):
return
mkdir_p(Project.quality_dir)
graphic_type_ext = 'svg'
with LocalShell() as local:
data_json = local.run("radon cc -s --json {dir}".format(dir=Project.package))
data = json.loads(data_json)
# info(pformat(data))
components = {'function': {}, 'method': {}, 'class': {}}
for path in data.keys():
for component in data[path]:
if isinstance(component, dict):
complexity_score = component['complexity']
if complexity_score not in components[component['type']]:
components[component['type']][complexity_score] = []
# noinspection PyUnresolvedReferences
components[component['type']][complexity_score].append(component)
# else:
# warning("{path}: {component}".format(path=path, component=pformat(component)))
component_names = {
'all': 'Components',
'function': 'Functions',
'class': 'Classes',
'method': 'Methods'
}
fig_number = 1
x = {}
y = {}
for component_type in components.keys():
info(component_type)
x[component_type] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21, 22, 23, 24, 25]
y[component_type] = [0] * 25
for score in sorted(components[component_type].keys()):
cnt = len(components[component_type][score])
# info("{complexity}: {cnt}".format(complexity=score, cnt=cnt))
if score < 25:
y[component_type][score - 1] += cnt
else:
y[component_type][-1] += cnt
info("fig_number: %d" % fig_number)
# plot_number = 110 + fig_number
plot_number = 111
info("plot_number: %d" % plot_number)
fig = pyplot.figure(fig_number)
pyplot.subplot(plot_number)
fig.suptitle("Cyclomatic Complexity of {type}".format(type=component_names[component_type]))
pyplot.bar(x[component_type][0:4], y[component_type][0:4], align='center', color='green')
pyplot.bar(x[component_type][5:9], y[component_type][5:9], align='center', color='blue')
pyplot.bar(x[component_type][10:14], y[component_type][10:14], align='center', color='yellow')
pyplot.bar(x[component_type][15:19], y[component_type][15:19], align='center', color='orange')
pyplot.bar(x[component_type][20:], y[component_type][20:], align='center', color='red')
pyplot.xlabel('Cyclomatic Complexity')
pyplot.ylabel('Number of {type}'.format(type=component_names[component_type]))
pyplot.savefig(os.path.join(Project.quality_dir, "cc_{type}.{ext}".format(type=component_type,
ext=graphic_type_ext)))
fig_number += 1
info("fig_number: %d" % fig_number)
# plot_number = 110 + fig_number
plot_number = 111
info("plot_number: %d" % plot_number)
fig = pyplot.figure(fig_number)
pyplot.subplot(plot_number)
fig.suptitle("Cyclomatic Complexity of All Components")
hatch = {'class': '/', 'method': '+', 'function': '*'}
bottom = [0] * 25
legend_bar = {}
for component_type in components.keys():
legend_bar[component_type] = pyplot.bar(x[component_type][0:4], y[component_type][0:4], align='center',
color='green', hatch=hatch[component_type], bottom=bottom[0:4])
pyplot.bar(x[component_type][5:9], y[component_type][5:9], align='center', color='blue',
hatch=hatch[component_type], bottom=bottom[5:9])
pyplot.bar(x[component_type][10:14], y[component_type][10:14], align='center', color='yellow',
hatch=hatch[component_type], bottom=bottom[10:14])
pyplot.bar(x[component_type][15:19], y[component_type][15:19], align='center', color='orange',
hatch=hatch[component_type], bottom=bottom[15:19])
pyplot.bar(x[component_type][20:24], y[component_type][20:24], align='center', color='red',
hatch=hatch[component_type], bottom=bottom[20:24])
bottom = list([bottom[j] + y[component_type][j] for j in range(len(bottom))])
pyplot.xlabel('Cyclomatic Complexity')
pyplot.ylabel('Number of Components')
pyplot.legend((legend_bar[component_type] for component_type in components.keys()),
(component_type for component_type in components.keys()))
pyplot.savefig(os.path.join(Project.quality_dir, "cc_all.{ext}".format(ext=graphic_type_ext)))
if '--display' in task.argv:
pyplot.show(fig_number)
