def findbugs(deb, analysis):
_, err, ret = run_command(["dpkg", "-x", deb, "binary"])
if ret != 0:
raise Exception("Cannot extract binary from deb:" + err)
with cd('binary'):
# Force english as findbugs is localized
os.putenv("LANG", "C")
out, err, _ = run_command(
['fb', 'analyze', '-effort:max', '-xml:withMessages', '.'])
xmlbytes = out.encode("utf-8")
failed = False
# if err.strip() == '':
# return (analysis, err, failed)
for issue in parse_findbugs(xmlbytes):
analysis.results.append(issue)
if not failed and issue.severity in [
'performance', 'portability', 'error', 'warning'
]:
failed = True
return (analysis, err, failed, None, None)
def coccinelle(dsc, analysis):
raise NotImplemented("Not ported")
run_command(["dpkg-source", "-x", dsc, "source"])
with cd('source'):
log = ""
failed = False
for semantic_patch in list_semantic_patches():
out, err, ret = run_command([
"spatch",
"-D", "firehose",
"--cocci-file", semantic_patch,
"--dir", ".",
"--no-show-diff",
"--timeout", "120",
])
failed = (ret != 0) or failed
parsed_results = parse_coccinelle(out)
result_count = 0
for result in parsed_results:
analysis.results.append(result)
result_count += 1
log += "DEAL patch %s\n" % semantic_patch
log += " %d results\n" % result_count
return (analysis, log, failed, None)
def coccinelle(dsc, analysis):
run_command(["dpkg-source", "-x", dsc, "source"])
os.environ['COCCI_SUT_TYPE'] = 'debian-source' # used by coccinelle firehose scripts
with cd('source'):
log = ""
failed = False
for semantic_patch in list_semantic_patches():
logger.debug('Running patch: {0}'.format(semantic_patch))
out, err, ret = run_command([
"spatch",
"-D", "firehose",
"--cocci-file", semantic_patch,
"--dir", ".",
"--no-show-diff",
"--timeout", "120",
])
failed = (ret != 0) or failed
logger.debug('Spatch output: {0}'.format(out))
logger.debug('Spatch err: {0}'.format(err))
parsed_results = parse_coccinelle(out)
result_count = 0
for result in parsed_results:
analysis.results.append(result)
result_count += 1
log += "DEAL patch %s\n" % semantic_patch
log += " %d results\n" % result_count
return (analysis, log, failed, None, None)
def coccinelle(dsc, analysis):
run_command(["dpkg-source", "-x", dsc, "source-coccinelle"])
os.environ['COCCI_SUT_TYPE'] = 'debian-source' # used by coccinelle firehose scripts
with cd('source-coccinelle'):
log = ""
failed = False
for semantic_patch in list_semantic_patches():
logger.debug('Running patch: {0}'.format(semantic_patch))
out, err, ret = run_command([
"spatch",
"-D", "firehose",
"--cocci-file", semantic_patch,
"--dir", ".",
"--no-show-diff",
"--timeout", "120",
])
failed = (ret != 0) or failed
logger.debug('Spatch output: {0}'.format(out))
logger.debug('Spatch err: {0}'.format(err))
parsed_results = parse_coccinelle(out)
result_count = 0
for result in parsed_results:
analysis.results.append(result)
result_count += 1
log += "DEAL patch %s\n" % semantic_patch
log += " %d results\n" % result_count
return (analysis, log, failed, None, None)
def coccinelle(dsc, analysis):
raise NotImplemented("Not ported")
run_command(["dpkg-source", "-x", dsc, "source"])
with cd('source'):
log = ""
failed = False
for semantic_patch in list_semantic_patches():
out, err, ret = run_command([
"spatch",
"-D",
"firehose",
"--cocci-file",
semantic_patch,
"--dir",
".",
"--no-show-diff",
"--timeout",
"120",
])
failed = (ret != 0) or failed
parsed_results = parse_coccinelle(out)
result_count = 0
for result in parsed_results:
analysis.results.append(result)
result_count += 1
log += "DEAL patch %s\n" % semantic_patch
log += " %d results\n" % result_count
return (analysis, log, failed, None)
def roodi(dsc, analysis):
run_command(["dpkg-source", "-x", dsc, "source-roodi"])
with cd('source-roodi'):
out, _, ret = run_command(['roodi', '.'])
failed = ret != 0
for issue in parse_roodi(out.splitlines()):
analysis.results.append(issue)
return (analysis, out, failed, None, None)
def pep8(dsc, analysis):
run_command(["dpkg-source", "-x", dsc, "source"])
with cd('source'):
out, err, ret = run_command(['pep8', '.'])
failed = ret != 0
for issue in parse_pep8(out.splitlines()):
analysis.results.append(issue)
return (analysis, out, failed, None)
def pep8(dsc, analysis):
run_command(["dpkg-source", "-x", dsc, "source-pep8"])
with cd('source-pep8'):
out, err, ret = run_command(['pep8', '.'])
failed = ret != 0
for issue in parse_pep8(out.splitlines()):
analysis.results.append(issue)
return (analysis, out, failed, None, None)
def test_findbugs():
with cd("tmp"):
report = firehose.model.Analysis.from_xml(
open("/home/sylvestre/incoming/1.firehose.xml", 'r'))
urllib.urlretrieve(
"http://snapshot.debian.org/archive/debian/20130527T160533Z/pool/main/libj/libjlatexmath-java/libjlatexmath-fop-java_1.0.2-1_all.deb",
"libjlatexmath-fop-java_1.0.2-1_all.deb")
assert os.path.isfile("libjlatexmath-fop-java_1.0.2-1_all.deb")
analysis = findbugs("libjlatexmath-fop-java_1.0.2-1_all.deb", report)
def test_findbugs():
with cd("tmp"):
report = firehose.model.Analysis.from_xml(open("/home/sylvestre/incoming/1.firehose.xml", "r"))
urllib.urlretrieve(
"http://snapshot.debian.org/archive/debian/20130527T160533Z/pool/main/libj/libjlatexmath-java/libjlatexmath-fop-java_1.0.2-1_all.deb",
"libjlatexmath-fop-java_1.0.2-1_all.deb",
)
assert os.path.isfile("libjlatexmath-fop-java_1.0.2-1_all.deb")
analysis = findbugs("libjlatexmath-fop-java_1.0.2-1_all.deb", report)
def jshint(dsc, analysis):
run_command(["dpkg-source", "-x", dsc, "source-jshint"])
with cd('source-jshint'):
out, _, ret = run_command(['jshint', '--reporter=checkstyle', '.'])
failed = ret != 0
if out is not None:
for issue in parse_jshint(out.splitlines()):
analysis.results.append(issue)
else:
print "Output of JSHint checker command is None!"
return (analysis, out, failed, None, None)
def test_dud():
with cd("resources"):
d = Dud()
d.add_file("file1")
d.add_file("file2")
d['Created-By'] = "Test <*****@*****.**>"
d['Source'] = "fnord"
d['Version'] = "1.0"
d['Architecture'] = "all"
assert ["file1", "file2"] == list(d.files())
d.write_dud("test.dud")
def checkout(package):
with tdir() as path:
with cd(path):
if package['type'] == "source":
safe_run(["dget", "-u", "-d", package['source']['dsc_url']])
yield package['source']['dsc_filename']
elif package['type'] == "binary":
files = []
for deb in package['binary']['debs']:
files += [deb['filename']]
safe_run(["dget", "-u", "-d", deb['url']])
yield files
else:
raise Exception
def checkout(package):
with tdir() as path:
with cd(path):
if package['type'] == "source":
safe_run(["dget", "-u", "-d", package['source']['dsc_url']])
yield package['source']['dsc_filename']
elif package['type'] == "binary":
files = []
for deb in package['binary']['debs']:
files += [deb['filename']]
safe_run(["dget", "-u", "-d", deb['url']])
yield files
else:
raise Exception
def checkout(job, package):
with tdir() as path:
with cd(path):
server_info = proxy.get_info()
src = package["source"]
archive = "{url}/{group}".format(url=server_info["repo"]["base"], group=src["group"])
if package["type"] == "source":
yield aget(archive, src["suite"], "main", src["name"], src["version"])
elif package["type"] == "binary":
arch = job["arch"]
if arch == "all":
arch = "amd64" # XXX: THIS IS A HACK. FIX THIS.
yield bget(archive, src["suite"], "main", arch, src["name"], src["version"])
else:
raise Exception
def perlcritic(dsc, analysis):
run_command(["dpkg-source", "-x", dsc, "source-perlcritic"])
with cd('source-perlcritic'):
out, _, ret = run_command([
'perlcritic', '--brutal', '.', '--verbose',
'%f:%l:%c %s %p %m\n'
])
if ret == 1:
raise Exception("Perlcritic had an internal error")
failed = ret == 2
for issue in parse_perlcritic(out.splitlines()):
analysis.results.append(issue)
return (analysis, out, failed, None, None)
def fetch_and_upload(dist, source, version, **kwargs):
from . import DEFAULT_MIRROR
confFile = "/etc/debile-rebuild.ini"
config = configparser.ConfigParser({'mirror': DEFAULT_MIRROR})
if not os.path.isfile(confFile):
raise Exception("Could not find " + confFile)
config.read([confFile])
gpg = config.get('config', 'signing-key')
target = config.get('config', 'dput-target')
mirror = config.get('config', 'mirror')
eversion = version
if ":" in eversion:
_, eversion = version.rsplit(":", 1)
if "incoming.debian.org" == mirror:
DSC_URL = (
"http://{mirror}/{source}_{version}.dsc".format(
source=source,
version=eversion,
mirror=mirror,
))
else:
path = pool_path(source)
DSC_URL = (
"http://{mirror}/debian/pool/main/"
"{path}/{source}_{version}.dsc".format(
path=path,
source=source,
version=eversion,
mirror=mirror,
))
with tdir() as pth:
with cd(pth):
_, err, ret = run_command(['dget', '-u', DSC_URL])
if ret == 0:
dsc = os.path.basename(DSC_URL)
try:
changes = write_changes(dsc, dist, **kwargs)
except MissingChangesFieldException as e:
print('Could not issue rebuild. Missing Changes'
'Field: {0}'.format(e.value))
else:
out, err = run(['debsign', '-k%s' % gpg, changes])
out, err = run(['dput', target, changes])
else:
print('Could not download %s' % DSC_URL)
def checkout(job, package):
with tdir() as path:
with cd(path):
src = package['source']
archive = proxy.get_archive_location(src['group'])
if package['type'] == "source":
yield aget(archive, src['suite'], 'main',
src['name'], src['version'])
elif package['type'] == "binary":
arch = job['arch']
if arch == 'all':
arch = 'amd64' # XXX: THIS IS A HACK. FIX THIS.
yield bget(archive, src['suite'], 'main',
arch, src['name'], src['version'])
else:
raise Exception
def checkout(job, package):
with tdir() as path:
with cd(path):
src = package['source']
archive = package['group']['repo_url']
if package['type'] == "source":
yield aget(archive, src['suite'], src['component'],
src['name'], src['version'])
elif package['type'] == "binary":
arch = job['arch']
if arch == 'all':
arch = 'amd64' # XXX: THIS IS A HACK. FIX THIS.
yield bget(archive, src['suite'], src['component'],
arch, src['name'], src['version'])
else:
raise Exception
def fetch_and_upload(dist, source, version, **kwargs):
from . import DEFAULT_MIRROR
confFile = "/etc/debile-rebuild.ini"
config = configparser.ConfigParser({'mirror': DEFAULT_MIRROR})
if not os.path.isfile(confFile):
raise Exception("Could not find " + confFile)
config.read([confFile])
gpg = config.get('config', 'signing-key')
target = config.get('config', 'dput-target')
mirror = config.get('config', 'mirror')
eversion = version
if ":" in eversion:
_, eversion = version.rsplit(":", 1)
if "incoming.debian.org" == mirror:
DSC_URL = ("http://{mirror}/{source}_{version}.dsc".format(
source=source,
version=eversion,
mirror=mirror,
))
else:
path = pool_path(source)
DSC_URL = ("http://{mirror}/debian/pool/main/"
"{path}/{source}_{version}.dsc".format(
path=path,
source=source,
version=eversion,
mirror=mirror,
))
with tdir() as pth:
with cd(pth):
_, err, ret = run_command(['dget', '-u', DSC_URL])
if ret == 0:
dsc = os.path.basename(DSC_URL)
try:
changes = write_changes(dsc, dist, **kwargs)
except MissingChangesFieldException as e:
print('Could not issue rebuild. Missing Changes'
'Field: {0}'.format(e.value))
else:
out, err = run(['debsign', '-k%s' % gpg, changes])
out, err = run(['dput', target, changes])
else:
print('Could not download %s' % DSC_URL)
def cppcheck(dsc, analysis):
run_command(["dpkg-source", "-x", dsc, "source-cppcheck"])
with cd('source-cppcheck'):
_, err, _ = run_command(
['cppcheck', '-j8', '--enable=all', '.', '--xml'])
xmlbytes = err.encode()
failed = False
if err.strip() == '':
return (analysis, err, failed, None, None)
for issue in parse_cppcheck(xmlbytes):
analysis.results.append(issue)
if not failed and issue.severity in [
'performance', 'portability', 'error', 'warning'
]:
failed = True
return (analysis, err, failed, None, None)
def findbugs(deb, analysis):
run_command(["dpkg", "-x", deb, "binary"])
with cd('binary'):
out, err, ret = run_command([
'fb', 'analyze', '-effort:max', '-xml:withMessages', '.'
])
xmlbytes = out.encode("utf-8")
failed = False
# if err.strip() == '':
# return (analysis, err, failed)
for issue in parse_findbugs(xmlbytes):
analysis.results.append(issue)
if not failed and issue.severity in [
'performance', 'portability', 'error', 'warning'
]:
failed = True
return (analysis, err, failed, None, None)
def cppcheck(dsc, analysis):
run_command(["dpkg-source", "-x", dsc, "source"])
with cd('source'):
out, err, ret = run_command([
'cppcheck', '-j8', '--enable=all', '.', '--xml'
])
xmlbytes = err.encode()
failed = False
if err.strip() == '':
return (analysis, err, failed)
for issue in parse_cppcheck(xmlbytes):
analysis.results.append(issue)
if not failed and issue.severity in [
'performance', 'portability', 'error', 'warning'
]:
failed = True
return (analysis, err, failed, None)
def findbugs(deb, analysis):
run_command(["dpkg", "-x", deb, "binary"])
with cd('binary'):
out, err, ret = run_command([
'fb', 'analyze', '-effort:max', '-xml:withMessages', '.'
])
xmlbytes = out.encode("utf-8")
failed = False
# if err.strip() == '':
# return (analysis, err, failed)
for issue in parse_findbugs(xmlbytes):
analysis.results.append(issue)
if not failed and issue.severity in [
'performance', 'portability', 'error', 'warning'
]:
failed = True
return (analysis, err, failed, None)
def pylint(dsc, analysis):
run_command(["dpkg-source", "-x", dsc, "source-pylint"])
with cd('source-pylint'):
sources = find_python_files('.')
failed = False
output = ""
for source in sources:
out, _, ret = run_command([
'pylint', '-rn',
'--msg-template="[{C}]{path}:{line},{column}:({msg_id})[{symbol}]{msg}"',
source
])
failed = ret != 0
if out is not None:
output += out
for issue in parse_pylint(out.splitlines()):
analysis.results.append(issue)
return (analysis, output, failed, None, None)
def findbugs(deb, analysis):
_, err, ret = run_command(["dpkg", "-x", deb, "binary"])
if ret != 0:
raise Exception("Cannot extract binary from deb:" + err)
with cd("binary"):
# Force english as findbugs is localized
os.putenv("LANG", "C")
out, err, _ = run_command(["fb", "analyze", "-effort:max", "-xml:withMessages", "."])
xmlbytes = out.encode("utf-8")
failed = False
# if err.strip() == '':
# return (analysis, err, failed)
for issue in parse_findbugs(xmlbytes):
analysis.results.append(issue)
if not failed and issue.severity in ["performance", "portability", "error", "warning"]:
failed = True
return (analysis, err, failed, None, None)
def checkout(package):
proxy = get_proxy()
_type = package['type']
if _type not in ['binary', 'source']:
raise ValueError("type sucks")
def source():
url = proxy.get_dsc_url(package['source_id'])
dsc = os.path.basename(url)
dget(url)
yield dsc
def binary():
url = proxy.get_deb_url(package['binary_id'])
deb = os.path.basename(url)
out, err, ret = run_command(['wget', url])
if ret != 0:
raise Exception("zomgwtf")
yield deb
with tdir() as where:
with cd(where):
for x in {"source": source, "binary": binary}[_type]():
yield x
def run(dsc, source, job, firehose):
run_command(["dpkg-source", "-x", dsc, "source"])
with cd('source'):
return desktop_file_validate('source', firehose)