示例#1
0
        def _create_class_diagrams(path):
            """
            Create class UML diagram

            :param path: path to the module file.
            :type path: str
            """
            info("_create_class_diagrams")
            if not executables_available(["pynsource"]):
                return
            files = [
                os.path.join(dir_path, f)
                for dir_path, dir_names, files in os.walk(path)
                for f in fnmatch.filter(files, "*.py")
            ]
            debug("files: {files}".format(files=repr(files)))
            for src_file in files:
                debug(src_file)
                name = src_file.replace(Project.herringfile_dir + "/", "").replace(".py", ".png").replace("/", ".")
                output = "classes_{name}".format(name=name)
                debug(output)
                if not os.path.isfile(output) or (os.path.isfile(output) and is_newer(output, src_file)):
                    run_python(
                        "pynsource -y {output} {source}".format(output=output, source=src_file),
                        verbose=False,
                        ignore_errors=True,
                    )
示例#2
0
    def pycodestyle():
        """Run pycodestyle checks"""
        if not executables_available(['pycodestyle']):
            return
        mkdir_p(Project.quality_dir)
        pycodestyle_text = os.path.join(Project.quality_dir, 'pycodestyle.txt')
        pycodestyle_out = os.path.join(Project.quality_dir, 'pycodestyle.out')
        pycodestyle_html = os.path.join(Project.quality_dir, 'pycodestyle.html')
        os.system("rm -f %s" % pycodestyle_text)
        os.system("PYTHONPATH=%s pycodestyle %s 2>/dev/null >%s" % (Project.pythonPath, Project.package, pycodestyle_text))
        os.system("pepper8 -o %s %s" % (pycodestyle_html, pycodestyle_text))

        # need to reorder the columns to make compatible with pylint file format
        # pycodestyle output:    "{file}:{line}:{column}: {err} {desc}"
        # pylint output:  "{file}:{line}: [{err}] {desc}"

        # noinspection PyArgumentEqualDefault
        with open(pycodestyle_text, 'r') as src_file:
            lines = src_file.readlines()

        with open(pycodestyle_out, 'w') as out_file:
            for line in lines:
                match = re.match(r"(.+):(\d+):(\d+):\s*(\S+)\s+(.+)", line)
                if match:
                    out_file.write("{file}:{line}: [{err}] {desc}\n".format(file=match.group(1),
                                                                            line=match.group(2),
                                                                            err=match.group(4),
                                                                            desc=match.group(5)))
示例#3
0
    def sloccount():
        """Generate SLOCCount output file, sloccount.sc, used by jenkins"""
        if not executables_available(['sloccount']):
            return
        sloc_data = qd('slocdata')
        mkdir_p(sloc_data)
        sloc_filename = qd('sloccount.sc')
        with LocalShell() as local:
            output = local.run("sloccount --datadir {data} --wide --details {src}".format(data=sloc_data,
                                                                                          src=Project.package))
            if os.path.isfile(sloc_filename):
                os.remove(sloc_filename)
            with open(sloc_filename, 'w') as sloc_file:
                sloc_file.write(output)

            counts = {'all': 0}
            for line in output.splitlines():
                match = re.match(r"^(\d+)\s+(\S+)\s+(\S+)\s+(\S+)", line)
                if match:
                    language = match.group(2)
                    if language not in counts:
                        counts[language] = 0
                    counts[language] += int(match.group(1))
                    counts['all'] += int(match.group(1))

            with open(qd("sloccount.js"), 'w') as out_file:
                out_file.write("\n{name}_sloccount_data = {{\n".format(name=Project.name))
                for key in sorted(counts.keys()):
                    out_file.write("    \"{key}\": \"{value}\",\n".format(key=key, value=counts[key]))
                out_file.write("};\n")
示例#4
0
        def radon():
            """ Cyclomatic complexity metrics """
            if not executables_available(['radon']):
                return
            mkdir_p(Project.quality_dir)

            def qd(basename):
                """
                get the relative path to report file in quality directory

                :param basename: the report base name.
                :returns: the relative path to the given report name in the quality directory.
                """
                return os.path.join(Project.quality_dir, basename)

            with LocalShell() as local:
                local.system("radon cc -s --average --total-average {dir} > {out}".format(
                    dir=Project.package, out=qd('radon_cc.txt')))
                local.system("radon cc -s --average --total-average --json {dir} > {out}".format(
                    dir=Project.package, out=qd('radon_cc.json')))
                local.system("radon cc -s --average --total-average --xml {dir} > {out}".format(
                    dir=Project.package, out=qd('radon_cc.xml')))
                local.system("radon mi -s {dir} > {out}".format(
                    dir=Project.package, out=qd('radon_mi.txt')))
                local.system("radon raw -s {dir} > {out}".format(
                    dir=Project.package, out=qd('radon_raw.txt')))
                local.system("radon raw -s --json {dir} > {out}".format(
                    dir=Project.package, out=qd('radon_raw.json')))
示例#5
0
    def _create_class_diagrams(path):
        """
        Create class UML diagram

        :param path: path to the module file.
        :type path: str
        """
        info("_create_class_diagrams")
        if not executables_available(['pynsource']):
            warning('pynsource not available')
            return

        files = [os.path.join(dir_path, f)
                 for dir_path, dir_names, files in os.walk(path)
                 for f in fnmatch.filter(files, '*.py')]
        debug("files: {files}".format(files=repr(files)))
        with open(os.path.join(Project.docs_dir, "pynsource.log"), "w") as outputter:
            for src_file in files:
                debug(src_file)
                name = src_file.replace(Project.herringfile_dir + '/', '').replace('.py', '.png').replace('/', '.')
                output = "classes_{name}".format(name=name)
                debug(output)
                if not os.path.isfile(output) or (os.path.isfile(output) and is_newer(output, src_file)):
                    output = run_python("pynsource -y {output} {source}".format(output=output, source=src_file),
                                        verbose=False, ignore_errors=True)
                    outputter.write(output)
示例#6
0
    def _create_module_diagrams(path):
        """
        create module UML diagrams

        :param path: the module path
         :type path: str
        """
        info("_create_module_diagrams")
        if not executables_available(['pyreverse']):
            warning('pyreverse not available')
            return

        with open(os.path.join(Project.docs_dir, "pyreverse.log"), "w") as outputter:
            for module_path in [root for root, dirs, files in os.walk(path) if os.path.basename(root) != '__pycache__']:
                debug("module_path: {path}".format(path=module_path))
                init_filename = os.path.join(module_path, '__init__.py')
                if os.path.exists(init_filename):
                    info(init_filename)
                    name = os.path.basename(module_path).split(".")[0]
                    output = run_python('pyreverse -o svg -p {name} {module} '.format(name=name, module=module_path),
                                        verbose=True, ignore_errors=True)
                    outputter.write(output)
                    errors = [line for line in output.splitlines() if not line.startswith('parsing')]
                    if errors:
                        info(errors)
示例#7
0
 def cheesecake():
     """ Run the cheesecake kwalitee metric """
     if not executables_available(['cheesecake_index']):
         return
     mkdir_p(Project.quality_dir)
     cheesecake_log = os.path.join(Project.quality_dir, 'cheesecake.log')
     with LocalShell() as local:
         local.system("cheesecake_index --path=dist/%s-%s.tar.gz --keep-log -l %s" %
                      (Project.name,
                       Project.version,
                       cheesecake_log))
示例#8
0
 def lint():
     """ Run pylint with project overrides from pylint.rc """
     if not executables_available(['pylint']):
         return
     mkdir_p(Project.quality_dir)
     options = ''
     if os.path.exists(Project.pylintrc):
         options += "--rcfile=pylint.rc"
     pylint_log = os.path.join(Project.quality_dir, 'pylint.log')
     with LocalShell() as local:
         local.system("pylint {options} {dir} > {log}".format(options=options,
                                                              dir=Project.package,
                                                              log=pylint_log))
示例#9
0
 def complexity():
     """ Run McCabe code complexity """
     if not executables_available(['pymetrics']):
         return
     mkdir_p(Project.quality_dir)
     quality_dir = Project.quality_dir
     complexity_txt = os.path.join(quality_dir, 'complexity.txt')
     graph = os.path.join(quality_dir, 'output.png')
     acc = os.path.join(quality_dir, 'complexity_acc.txt')
     metrics_html = os.path.join(quality_dir, 'complexity_metrics.html')
     with LocalShell() as local:
         local.system("touch %s" % complexity_txt)
         local.system("touch %s" % acc)
         local.system("pymetrics --nosql --nocsv `find %s/ -iname \"*.py\"` > %s" %
                      (Project.package, complexity_txt))
示例#10
0
    def flake8():
        """Run flake8 checks"""
        if not executables_available(['flake8']):
            return
        mkdir_p(Project.quality_dir)
        flake8_text = os.path.join(Project.quality_dir, 'flake8.txt')
        flake8_out = os.path.join(Project.quality_dir, 'flake8.out')
        flake8_html = os.path.join(Project.quality_dir, 'flake8.html')
        flake8_js = os.path.join(Project.quality_dir, 'flake8.js')
        os.system("rm -f %s" % flake8_text)
        os.system("PYTHONPATH=%s flake8 --show-source --statistics %s 2>/dev/null >%s" % (Project.pythonPath, Project.package, flake8_text))
        os.system("pepper8 -o %s %s" % (flake8_html, flake8_text))

        # need to reorder the columns to make compatible with pylint file format
        # flake8 output:    "{file}:{line}:{column}: {err} {desc}"
        # pylint output:  "{file}:{line}: [{err}] {desc}"

        # noinspection PyArgumentEqualDefault
        with open(flake8_text, 'r') as src_file:
            lines = src_file.readlines()

        errors = 0
        warnings = 0
        others = 0
        with open(flake8_out, 'w') as out_file:
            for line in lines:
                match = re.match(r"(.+):(\d+):(\d+):\s*(\S+)\s+(.+)", line)
                if match:
                    if match.group(4).startswith('E'):
                        errors += 1
                    elif match.group(4).startswith('W'):
                        warnings += 1
                    else:
                        others += 1
                    out_file.write("{file}:{line}: [{err}] {desc}\n".format(file=match.group(1),
                                                                            line=match.group(2),
                                                                            err=match.group(4),
                                                                            desc=match.group(5)))
        with open(flake8_js, 'w') as out_file:
            out_file.write(dedent("""
                    {name}_flake8_data = {{
                        "errors": "{errors}",
                        "warnings": "{warnings}",
                        "other": "{others}"
                    }};
                """.format(name=Project.name, errors=errors, warnings=warnings, others=others)))
示例#11
0
 def sloc():
     """Run sloccount to get the source lines of code."""
     if not executables_available(['sloccount']):
         return
     mkdir_p(Project.quality_dir)
     sloc_json = os.path.join(Project.quality_dir, 'sloc.json')
     totals_by_language = _sloc_totals_by_language()
     total_sloc = 0
     for value in totals_by_language.values():
         total_sloc += value[0]
     with open(sloc_json, 'w') as json_file:
         json.dump(totals_by_language, json_file)
     for lang in totals_by_language.keys():
         info("{lang}: {total} ({percentage}%)".format(lang=lang,
                                                       total=totals_by_language[lang][0],
                                                       percentage=totals_by_language[lang][1]))
     info("Total SLOC: {total}".format(total=total_sloc))
示例#12
0
文件: doc.py 项目: royw/herringlib
    def rstlint():
        """Check the RST in the source files"""
        if not executables_available(['rst-lint']):
            return
        rst_files = [os.path.join(dir_path, f)
                     for dir_path, dir_names, files in os.walk(Project.herringfile_dir)
                     for f in fnmatch.filter(files, '*.rst')]

        src_files = [os.path.join(dir_path, f)
                     for dir_path, dir_names, files in os.walk(Project.herringfile_dir)
                     for f in fnmatch.filter(files, '*.py')]

        with LocalShell() as local:
            for src_file in rst_files + src_files:
                cmd_line = 'rst-lint {file}'.format(file=src_file)
                result = local.system(cmd_line, verbose=False)
                if not re.search(r'No problems found', result):
                    info(cmd_line)
                    info(result)
示例#13
0
    def radon():
        """ Cyclomatic complexity metrics """
        if not executables_available(['radon']):
            return
        mkdir_p(Project.quality_dir)

        with LocalShell() as local:
            local.system("radon cc -s --average --total-average {dir} > {out}".format(
                dir=Project.package, out=qd('radon_cc.txt')))
            local.system("radon cc -s --average --total-average --json {dir} > {out}".format(
                dir=Project.package, out=qd('radon_cc.json')))
            local.system("radon cc -s --average --total-average --xml {dir} > {out}".format(
                dir=Project.package, out=qd('radon_cc.xml')))
            local.system("radon mi -s {dir} > {out}".format(
                dir=Project.package, out=qd('radon_mi.txt')))
            local.system("radon raw -s {dir} > {out}".format(
                dir=Project.package, out=qd('radon_raw.txt')))
            local.system("radon raw -s --json {dir} > {out}".format(
                dir=Project.package, out=qd('radon_raw.json')))

            grade_a = local.system("grep -c \" - A \" {txt}".format(txt=qd('radon_cc.txt'))).strip()
            grade_b = local.system("grep -c \" - B \" {txt}".format(txt=qd('radon_cc.txt'))).strip()
            grade_c = local.system("grep -c \" - C \" {txt}".format(txt=qd('radon_cc.txt'))).strip()
            grade_d = local.system("grep -c \" - D \" {txt}".format(txt=qd('radon_cc.txt'))).strip()
            grade_e = local.system("grep -c \" - E \" {txt}".format(txt=qd('radon_cc.txt'))).strip()
            grade_f = local.system("grep -c \" - F \" {txt}".format(txt=qd('radon_cc.txt'))).strip()

            with open(qd("radon_cc_summary.js"), 'w') as out_file:
                out_file.write(dedent(r"""
                    {name}_code_complexity_data = {{
                        "A": "{a}",
                        "B": "{b}",
                        "C": "{c}",
                        "D": "{d}",
                        "E": "{e}",
                        "F": "{f}",
                    }};
                """.format(name=Project.name, a=grade_a, b=grade_b, c=grade_c, d=grade_d, e=grade_e, f=grade_f)))
示例#14
0
        def _create_module_diagrams(path):
            """
            create module UML diagrams

            :param path: the module path
             :type path: str
            """
            info("_create_module_diagrams")
            if not executables_available(["pyreverse"]):
                return
            # TODO fixme hangs on tp-otto
            for module_path in [root for root, dirs, files in os.walk(path)]:
                info("module_path: {path}".format(path=module_path))
                init_filename = os.path.join(module_path, "__init__.py")
                if os.path.exists(init_filename):
                    info(init_filename)
                    name = os.path.basename(module_path).split(".")[0]
                    output = run_python(
                        "pyreverse -o svg -p {name} {module}".format(name=name, module=module_path),
                        verbose=True,
                        ignore_errors=True,
                    )
                    info([line for line in output.splitlines() if not line.startswith("parsing")])
示例#15
0
    def graph_complexity():
        """ Create Cyclomatic Complexity graphs. """
        import matplotlib
        matplotlib.use('Agg')  # Must be before importing matplotlib.pyplot or pylab!
        from matplotlib import pyplot

        if not executables_available(['radon']):
            return
        mkdir_p(Project.quality_dir)
        graphic_type_ext = 'svg'

        with LocalShell() as local:
            data_json = local.run("radon cc -s --json {dir}".format(dir=Project.package))
            data = json.loads(data_json)

        # info(pformat(data))
        components = {'function': {}, 'method': {}, 'class': {}}
        for path in data.keys():
            for component in data[path]:
                if isinstance(component, dict):
                    complexity_score = component['complexity']
                    if complexity_score not in components[component['type']]:
                        components[component['type']][complexity_score] = []
                    # noinspection PyUnresolvedReferences
                    components[component['type']][complexity_score].append(component)
                # else:
                #     warning("{path}: {component}".format(path=path, component=pformat(component)))

        component_names = {
            'all': 'Components',
            'function': 'Functions',
            'class': 'Classes',
            'method': 'Methods'
        }

        fig_number = 1
        x = {}
        y = {}
        for component_type in components.keys():
            info(component_type)
            x[component_type] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
                                 21, 22, 23, 24, 25]
            y[component_type] = [0] * 25
            for score in sorted(components[component_type].keys()):
                cnt = len(components[component_type][score])
                # info("{complexity}: {cnt}".format(complexity=score, cnt=cnt))
                if score < 25:
                    y[component_type][score - 1] += cnt
                else:
                    y[component_type][-1] += cnt

            info("fig_number: %d" % fig_number)
            # plot_number = 110 + fig_number
            plot_number = 111
            info("plot_number: %d" % plot_number)
            fig = pyplot.figure(fig_number)
            pyplot.subplot(plot_number)
            fig.suptitle("Cyclomatic Complexity of {type}".format(type=component_names[component_type]))
            pyplot.bar(x[component_type][0:4], y[component_type][0:4], align='center', color='green')
            pyplot.bar(x[component_type][5:9], y[component_type][5:9], align='center', color='blue')
            pyplot.bar(x[component_type][10:14], y[component_type][10:14], align='center', color='yellow')
            pyplot.bar(x[component_type][15:19], y[component_type][15:19], align='center', color='orange')
            pyplot.bar(x[component_type][20:], y[component_type][20:], align='center', color='red')

            pyplot.xlabel('Cyclomatic Complexity')
            pyplot.ylabel('Number of {type}'.format(type=component_names[component_type]))

            pyplot.savefig(os.path.join(Project.quality_dir, "cc_{type}.{ext}".format(type=component_type,
                                                                                      ext=graphic_type_ext)))
            fig_number += 1

        info("fig_number: %d" % fig_number)
        # plot_number = 110 + fig_number
        plot_number = 111
        info("plot_number: %d" % plot_number)
        fig = pyplot.figure(fig_number)
        pyplot.subplot(plot_number)
        fig.suptitle("Cyclomatic Complexity of All Components")
        hatch = {'class': '/', 'method': '+', 'function': '*'}
        bottom = [0] * 25
        legend_bar = {}
        for component_type in components.keys():
            legend_bar[component_type] = pyplot.bar(x[component_type][0:4], y[component_type][0:4], align='center',
                                                    color='green', hatch=hatch[component_type], bottom=bottom[0:4])
            pyplot.bar(x[component_type][5:9], y[component_type][5:9], align='center', color='blue',
                       hatch=hatch[component_type], bottom=bottom[5:9])
            pyplot.bar(x[component_type][10:14], y[component_type][10:14], align='center', color='yellow',
                       hatch=hatch[component_type], bottom=bottom[10:14])
            pyplot.bar(x[component_type][15:19], y[component_type][15:19], align='center', color='orange',
                       hatch=hatch[component_type], bottom=bottom[15:19])
            pyplot.bar(x[component_type][20:24], y[component_type][20:24], align='center', color='red',
                       hatch=hatch[component_type], bottom=bottom[20:24])
            bottom = list([bottom[j] + y[component_type][j] for j in range(len(bottom))])

        pyplot.xlabel('Cyclomatic Complexity')
        pyplot.ylabel('Number of Components')
        pyplot.legend((legend_bar[component_type] for component_type in components.keys()),
                      (component_type for component_type in components.keys()))

        pyplot.savefig(os.path.join(Project.quality_dir, "cc_all.{ext}".format(ext=graphic_type_ext)))

        if '--display' in task.argv:
            pyplot.show(fig_number)