Ejemplo n.º 1
0
def findbugs(deb, analysis):
    _, err, ret = run_command(["dpkg", "-x", deb, "binary"])

    if ret != 0:
        raise Exception("Cannot extract binary from deb:" + err)

    with cd('binary'):
        # Force english as findbugs is localized
        os.putenv("LANG", "C")
        out, err, _ = run_command(
            ['fb', 'analyze', '-effort:max', '-xml:withMessages', '.'])

        xmlbytes = out.encode("utf-8")

        failed = False
        #        if err.strip() == '':
        #            return (analysis, err, failed)

        for issue in parse_findbugs(xmlbytes):
            analysis.results.append(issue)
            if not failed and issue.severity in [
                    'performance', 'portability', 'error', 'warning'
            ]:
                failed = True

        return (analysis, err, failed, None, None)
Ejemplo n.º 2
0
def coccinelle(dsc, analysis):
    raise NotImplemented("Not ported")

    run_command(["dpkg-source", "-x", dsc, "source"])
    with cd('source'):
        log = ""
        failed = False
        for semantic_patch in list_semantic_patches():
            out, err, ret = run_command([
                "spatch",
                "-D", "firehose",
                "--cocci-file", semantic_patch,
                "--dir", ".",
                "--no-show-diff",
                "--timeout", "120",
            ])
            failed = (ret != 0) or failed

            parsed_results = parse_coccinelle(out)

            result_count = 0
            for result in parsed_results:
                analysis.results.append(result)
                result_count += 1

            log += "DEAL patch %s\n" % semantic_patch
            log += "  %d results\n" % result_count

    return (analysis, log, failed, None)
Ejemplo n.º 3
0
def coccinelle(dsc, analysis):

    run_command(["dpkg-source", "-x", dsc, "source"])
    os.environ['COCCI_SUT_TYPE'] = 'debian-source' # used by coccinelle firehose scripts

    with cd('source'):
        log = ""
        failed = False
        for semantic_patch in list_semantic_patches():
            logger.debug('Running patch: {0}'.format(semantic_patch))
            out, err, ret = run_command([
                "spatch",
                "-D", "firehose",
                "--cocci-file", semantic_patch,
                "--dir", ".",
                "--no-show-diff",
                "--timeout", "120",
            ])


            failed = (ret != 0) or failed
            logger.debug('Spatch output: {0}'.format(out))
            logger.debug('Spatch err: {0}'.format(err))
            parsed_results = parse_coccinelle(out)

            result_count = 0
            for result in parsed_results:
                analysis.results.append(result)
                result_count += 1

            log += "DEAL patch %s\n" % semantic_patch
            log += "  %d results\n" % result_count

    return (analysis, log, failed, None, None)
Ejemplo n.º 4
0
def coccinelle(dsc, analysis):
    run_command(["dpkg-source", "-x", dsc, "source-coccinelle"])
    os.environ['COCCI_SUT_TYPE'] = 'debian-source'  # used by coccinelle firehose scripts

    with cd('source-coccinelle'):
        log = ""
        failed = False
        for semantic_patch in list_semantic_patches():
            logger.debug('Running patch: {0}'.format(semantic_patch))
            out, err, ret = run_command([
                "spatch",
                "-D", "firehose",
                "--cocci-file", semantic_patch,
                "--dir", ".",
                "--no-show-diff",
                "--timeout", "120",
            ])

            failed = (ret != 0) or failed
            logger.debug('Spatch output: {0}'.format(out))
            logger.debug('Spatch err: {0}'.format(err))
            parsed_results = parse_coccinelle(out)

            result_count = 0
            for result in parsed_results:
                analysis.results.append(result)
                result_count += 1

            log += "DEAL patch %s\n" % semantic_patch
            log += "  %d results\n" % result_count

    return (analysis, log, failed, None, None)
Ejemplo n.º 5
0
def coccinelle(dsc, analysis):
    raise NotImplemented("Not ported")

    run_command(["dpkg-source", "-x", dsc, "source"])
    with cd('source'):
        log = ""
        failed = False
        for semantic_patch in list_semantic_patches():
            out, err, ret = run_command([
                "spatch",
                "-D",
                "firehose",
                "--cocci-file",
                semantic_patch,
                "--dir",
                ".",
                "--no-show-diff",
                "--timeout",
                "120",
            ])
            failed = (ret != 0) or failed

            parsed_results = parse_coccinelle(out)

            result_count = 0
            for result in parsed_results:
                analysis.results.append(result)
                result_count += 1

            log += "DEAL patch %s\n" % semantic_patch
            log += "  %d results\n" % result_count

    return (analysis, log, failed, None)
Ejemplo n.º 6
0
def roodi(dsc, analysis):
    run_command(["dpkg-source", "-x", dsc, "source-roodi"])
    with cd('source-roodi'):
        out, _, ret = run_command(['roodi', '.'])
        failed = ret != 0

        for issue in parse_roodi(out.splitlines()):
            analysis.results.append(issue)

        return (analysis, out, failed, None, None)
Ejemplo n.º 7
0
def pep8(dsc, analysis):
    run_command(["dpkg-source", "-x", dsc, "source"])
    with cd('source'):
        out, err, ret = run_command(['pep8', '.'])
        failed = ret != 0

        for issue in parse_pep8(out.splitlines()):
            analysis.results.append(issue)

        return (analysis, out, failed, None)
Ejemplo n.º 8
0
def pep8(dsc, analysis):
    run_command(["dpkg-source", "-x", dsc, "source-pep8"])
    with cd('source-pep8'):
        out, err, ret = run_command(['pep8', '.'])
        failed = ret != 0

        for issue in parse_pep8(out.splitlines()):
            analysis.results.append(issue)

        return (analysis, out, failed, None, None)
Ejemplo n.º 9
0
def test_findbugs():
    with cd("tmp"):
        report = firehose.model.Analysis.from_xml(
            open("/home/sylvestre/incoming/1.firehose.xml", 'r'))

        urllib.urlretrieve(
            "http://snapshot.debian.org/archive/debian/20130527T160533Z/pool/main/libj/libjlatexmath-java/libjlatexmath-fop-java_1.0.2-1_all.deb",
            "libjlatexmath-fop-java_1.0.2-1_all.deb")
        assert os.path.isfile("libjlatexmath-fop-java_1.0.2-1_all.deb")

        analysis = findbugs("libjlatexmath-fop-java_1.0.2-1_all.deb", report)
Ejemplo n.º 10
0
def test_findbugs():
    with cd("tmp"):
        report = firehose.model.Analysis.from_xml(open("/home/sylvestre/incoming/1.firehose.xml", "r"))

        urllib.urlretrieve(
            "http://snapshot.debian.org/archive/debian/20130527T160533Z/pool/main/libj/libjlatexmath-java/libjlatexmath-fop-java_1.0.2-1_all.deb",
            "libjlatexmath-fop-java_1.0.2-1_all.deb",
        )
        assert os.path.isfile("libjlatexmath-fop-java_1.0.2-1_all.deb")

        analysis = findbugs("libjlatexmath-fop-java_1.0.2-1_all.deb", report)
Ejemplo n.º 11
0
def jshint(dsc, analysis):
    run_command(["dpkg-source", "-x", dsc, "source-jshint"])
    with cd('source-jshint'):
        out, _, ret = run_command(['jshint', '--reporter=checkstyle', '.'])
        failed = ret != 0

        if out is not None:
            for issue in parse_jshint(out.splitlines()):
                analysis.results.append(issue)
        else:
            print "Output of JSHint checker command is None!"

        return (analysis, out, failed, None, None)
Ejemplo n.º 12
0
def test_dud():
    with cd("resources"):
        d = Dud()
        d.add_file("file1")
        d.add_file("file2")
        d['Created-By'] = "Test <*****@*****.**>"
        d['Source'] = "fnord"
        d['Version'] = "1.0"
        d['Architecture'] = "all"

        assert ["file1", "file2"] == list(d.files())

        d.write_dud("test.dud")
Ejemplo n.º 13
0
def checkout(package):
    with tdir() as path:
        with cd(path):
            if package['type'] == "source":
                safe_run(["dget", "-u", "-d", package['source']['dsc_url']])
                yield package['source']['dsc_filename']
            elif package['type'] == "binary":
                files = []
                for deb in package['binary']['debs']:
                    files += [deb['filename']]
                    safe_run(["dget", "-u", "-d", deb['url']])
                yield files
            else:
                raise Exception
Ejemplo n.º 14
0
def checkout(package):
    with tdir() as path:
        with cd(path):
            if package['type'] == "source":
                safe_run(["dget", "-u", "-d", package['source']['dsc_url']])
                yield package['source']['dsc_filename']
            elif package['type'] == "binary":
                files = []
                for deb in package['binary']['debs']:
                    files += [deb['filename']]
                    safe_run(["dget", "-u", "-d", deb['url']])
                yield files
            else:
                raise Exception
Ejemplo n.º 15
0
def checkout(job, package):
    with tdir() as path:
        with cd(path):
            server_info = proxy.get_info()
            src = package["source"]
            archive = "{url}/{group}".format(url=server_info["repo"]["base"], group=src["group"])
            if package["type"] == "source":
                yield aget(archive, src["suite"], "main", src["name"], src["version"])
            elif package["type"] == "binary":
                arch = job["arch"]
                if arch == "all":
                    arch = "amd64"  # XXX: THIS IS A HACK. FIX THIS.
                yield bget(archive, src["suite"], "main", arch, src["name"], src["version"])
            else:
                raise Exception
Ejemplo n.º 16
0
def perlcritic(dsc, analysis):
    run_command(["dpkg-source", "-x", dsc, "source-perlcritic"])
    with cd('source-perlcritic'):
        out, _, ret = run_command([
            'perlcritic', '--brutal', '.', '--verbose',
            '%f:%l:%c %s    %p    %m\n'
        ])
        if ret == 1:
            raise Exception("Perlcritic had an internal error")

        failed = ret == 2
        for issue in parse_perlcritic(out.splitlines()):
            analysis.results.append(issue)

        return (analysis, out, failed, None, None)
Ejemplo n.º 17
0
def fetch_and_upload(dist, source, version, **kwargs):
    from . import DEFAULT_MIRROR
    confFile = "/etc/debile-rebuild.ini"
    config = configparser.ConfigParser({'mirror': DEFAULT_MIRROR})
    if not os.path.isfile(confFile):
        raise Exception("Could not find " + confFile)
    config.read([confFile])
    gpg = config.get('config', 'signing-key')
    target = config.get('config', 'dput-target')
    mirror = config.get('config', 'mirror')

    eversion = version
    if ":" in eversion:
        _, eversion = version.rsplit(":", 1)

    if "incoming.debian.org" == mirror:
        DSC_URL = (
            "http://{mirror}/{source}_{version}.dsc".format(
                source=source,
                version=eversion,
                mirror=mirror,
            ))
    else:
        path = pool_path(source)
        DSC_URL = (
            "http://{mirror}/debian/pool/main/"
            "{path}/{source}_{version}.dsc".format(
                path=path,
                source=source,
                version=eversion,
                mirror=mirror,
            ))

    with tdir() as pth:
        with cd(pth):
            _, err, ret = run_command(['dget', '-u', DSC_URL])
            if ret == 0:
                dsc = os.path.basename(DSC_URL)
                try:
                    changes = write_changes(dsc, dist, **kwargs)
                except MissingChangesFieldException as e:
                    print('Could not issue rebuild. Missing Changes'
                          'Field: {0}'.format(e.value))
                else:
                    out, err = run(['debsign', '-k%s' % gpg, changes])
                    out, err = run(['dput', target, changes])
            else:
                print('Could not download %s' % DSC_URL)
Ejemplo n.º 18
0
def checkout(job, package):
    with tdir() as path:
        with cd(path):
            src = package['source']
            archive = proxy.get_archive_location(src['group'])
            if package['type'] == "source":
                yield aget(archive, src['suite'], 'main',
                           src['name'], src['version'])
            elif package['type'] == "binary":
                arch = job['arch']
                if arch == 'all':
                    arch = 'amd64'  # XXX: THIS IS A HACK. FIX THIS.
                yield bget(archive, src['suite'], 'main',
                           arch, src['name'], src['version'])
            else:
                raise Exception
Ejemplo n.º 19
0
def checkout(job, package):
    with tdir() as path:
        with cd(path):
            src = package['source']
            archive = package['group']['repo_url']
            if package['type'] == "source":
                yield aget(archive, src['suite'], src['component'],
                           src['name'], src['version'])
            elif package['type'] == "binary":
                arch = job['arch']
                if arch == 'all':
                    arch = 'amd64'  # XXX: THIS IS A HACK. FIX THIS.
                yield bget(archive, src['suite'], src['component'],
                           arch, src['name'], src['version'])
            else:
                raise Exception
Ejemplo n.º 20
0
def fetch_and_upload(dist, source, version, **kwargs):
    from . import DEFAULT_MIRROR
    confFile = "/etc/debile-rebuild.ini"
    config = configparser.ConfigParser({'mirror': DEFAULT_MIRROR})
    if not os.path.isfile(confFile):
        raise Exception("Could not find " + confFile)
    config.read([confFile])
    gpg = config.get('config', 'signing-key')
    target = config.get('config', 'dput-target')
    mirror = config.get('config', 'mirror')

    eversion = version
    if ":" in eversion:
        _, eversion = version.rsplit(":", 1)

    if "incoming.debian.org" == mirror:
        DSC_URL = ("http://{mirror}/{source}_{version}.dsc".format(
            source=source,
            version=eversion,
            mirror=mirror,
        ))
    else:
        path = pool_path(source)
        DSC_URL = ("http://{mirror}/debian/pool/main/"
                   "{path}/{source}_{version}.dsc".format(
                       path=path,
                       source=source,
                       version=eversion,
                       mirror=mirror,
                   ))

    with tdir() as pth:
        with cd(pth):
            _, err, ret = run_command(['dget', '-u', DSC_URL])
            if ret == 0:
                dsc = os.path.basename(DSC_URL)
                try:
                    changes = write_changes(dsc, dist, **kwargs)
                except MissingChangesFieldException as e:
                    print('Could not issue rebuild. Missing Changes'
                          'Field: {0}'.format(e.value))
                else:
                    out, err = run(['debsign', '-k%s' % gpg, changes])
                    out, err = run(['dput', target, changes])
            else:
                print('Could not download %s' % DSC_URL)
Ejemplo n.º 21
0
def cppcheck(dsc, analysis):
    run_command(["dpkg-source", "-x", dsc, "source-cppcheck"])
    with cd('source-cppcheck'):
        _, err, _ = run_command(
            ['cppcheck', '-j8', '--enable=all', '.', '--xml'])

        xmlbytes = err.encode()

        failed = False
        if err.strip() == '':
            return (analysis, err, failed, None, None)

        for issue in parse_cppcheck(xmlbytes):
            analysis.results.append(issue)
            if not failed and issue.severity in [
                    'performance', 'portability', 'error', 'warning'
            ]:
                failed = True

        return (analysis, err, failed, None, None)
Ejemplo n.º 22
0
def findbugs(deb, analysis):
    run_command(["dpkg", "-x", deb, "binary"])
    with cd('binary'):
        out, err, ret = run_command([
            'fb', 'analyze', '-effort:max', '-xml:withMessages', '.'
        ])

        xmlbytes = out.encode("utf-8")

        failed = False
#        if err.strip() == '':
#            return (analysis, err, failed)

        for issue in parse_findbugs(xmlbytes):
            analysis.results.append(issue)
            if not failed and issue.severity in [
                'performance', 'portability', 'error', 'warning'
            ]:
                failed = True

        return (analysis, err, failed, None, None)
Ejemplo n.º 23
0
def cppcheck(dsc, analysis):
    run_command(["dpkg-source", "-x", dsc, "source"])
    with cd('source'):
        out, err, ret = run_command([
            'cppcheck', '-j8', '--enable=all', '.', '--xml'
        ])

        xmlbytes = err.encode()

        failed = False
        if err.strip() == '':
            return (analysis, err, failed)

        for issue in parse_cppcheck(xmlbytes):
            analysis.results.append(issue)
            if not failed and issue.severity in [
                'performance', 'portability', 'error', 'warning'
            ]:
                failed = True

        return (analysis, err, failed, None)
Ejemplo n.º 24
0
def findbugs(deb, analysis):
    run_command(["dpkg", "-x", deb, "binary"])
    with cd('binary'):
        out, err, ret = run_command([
                'fb', 'analyze', '-effort:max', '-xml:withMessages', '.'
        ])

        xmlbytes = out.encode("utf-8")

        failed = False
#        if err.strip() == '':
#            return (analysis, err, failed)

        for issue in parse_findbugs(xmlbytes):
            analysis.results.append(issue)
            if not failed and issue.severity in [
                'performance', 'portability', 'error', 'warning'
            ]:
                failed = True

        return (analysis, err, failed, None)
Ejemplo n.º 25
0
def pylint(dsc, analysis):
    run_command(["dpkg-source", "-x", dsc, "source-pylint"])
    with cd('source-pylint'):
        sources = find_python_files('.')
        failed = False
        output = ""

        for source in sources:
            out, _, ret = run_command([
                'pylint', '-rn',
                '--msg-template="[{C}]{path}:{line},{column}:({msg_id})[{symbol}]{msg}"',
                source
            ])
            failed = ret != 0

            if out is not None:
                output += out

                for issue in parse_pylint(out.splitlines()):
                    analysis.results.append(issue)

        return (analysis, output, failed, None, None)
Ejemplo n.º 26
0
def findbugs(deb, analysis):
    _, err, ret = run_command(["dpkg", "-x", deb, "binary"])

    if ret != 0:
        raise Exception("Cannot extract binary from deb:" + err)

    with cd("binary"):
        # Force english as findbugs is localized
        os.putenv("LANG", "C")
        out, err, _ = run_command(["fb", "analyze", "-effort:max", "-xml:withMessages", "."])

        xmlbytes = out.encode("utf-8")

        failed = False
        #        if err.strip() == '':
        #            return (analysis, err, failed)

        for issue in parse_findbugs(xmlbytes):
            analysis.results.append(issue)
            if not failed and issue.severity in ["performance", "portability", "error", "warning"]:
                failed = True

        return (analysis, err, failed, None, None)
Ejemplo n.º 27
0
def checkout(package):
    proxy = get_proxy()
    _type = package['type']
    if _type not in ['binary', 'source']:
        raise ValueError("type sucks")

    def source():
        url = proxy.get_dsc_url(package['source_id'])
        dsc = os.path.basename(url)
        dget(url)
        yield dsc

    def binary():
        url = proxy.get_deb_url(package['binary_id'])
        deb = os.path.basename(url)
        out, err, ret = run_command(['wget', url])
        if ret != 0:
            raise Exception("zomgwtf")
        yield deb

    with tdir() as where:
        with cd(where):
            for x in {"source": source, "binary": binary}[_type]():
                yield x
Ejemplo n.º 28
0
def run(dsc, source, job, firehose):
    run_command(["dpkg-source", "-x", dsc, "source"])
    with cd('source'):
        return desktop_file_validate('source', firehose)