def pep8(dsc, analysis):
run_command(["dpkg-source", "-x", dsc, "source"])
with cd('source'):
out, err, ret = run_command(['pep8', '.'])
failed = ret != 0
for issue in parse_pep8(out.splitlines()):
analysis.results.append(issue)
return (analysis, out, failed)
def pep8(dsc, analysis):
run_command(["dpkg-source", "-x", dsc, "source"])
with cd('source'):
out, err, ret = run_command(['pep8', '.'])
failed = ret != 0
for issue in parse_pep8(out.splitlines()):
analysis.results.append(issue)
return (analysis, out, failed)
def iterate():
suites = listize(config['suites'])
arches = listize(config['arches'])
with workon(suites, arches, list(PLUGINS.keys())) as job:
if job is None:
raise IDidNothingError("No more jobs")
package_id = job['package']
type_ = job['package_type']
logging.debug("Fetching the %s package, id=%s", type_, package_id)
package = None
if type_ == 'binary':
package = proxy.get_binary_package(package_id)
elif type_ == 'source':
package = proxy.get_source_package(package_id)
else:
raise IDidNothingError("SHIT")
handler, version_getter = load_module(job['type'])
firehose = create_firehose(package, version_getter)
with tdir() as fd:
with cd(fd):
with checkout(package) as target:
firehose, log, err = handler(target, package,
job, firehose)
type_ = {"sources": "source",
"binaries": "binary"}[package['_type']]
logging.info("Job worker returned, filing reports")
report = proxy.submit_report(firehose.to_json(),
job['_id'], err)
logging.info("Sending the XML firehose report to the pool")
open('firehose.xml', 'w').write(firehose.to_xml_bytes())
remote_firehose_path = proxy.get_firehose_write_location(report)
cmd = config['copy'].format(src='firehose.xml',
dest=remote_firehose_path)
out, err, ret = run_command(cmd)
logging.info("Sending the logs to the pool")
remote_path = proxy.get_log_write_location(report)
open('ethel-log', 'wb').write(log.encode('utf-8'))
cmd = config['copy'].format(src='ethel-log',
dest=remote_path)
out, err, ret = run_command(cmd)
if ret != 0:
print(out)
raise Exception("SHIT.")
def perlcritic(dsc, analysis):
run_command(["dpkg-source", "-x", dsc, "source"])
with cd('source'):
out, err, ret = run_command([
'perlcritic', '--brutal', '.', '--verbose',
'%f:%l:%c %s %p %m\n'
])
if ret == 1:
raise Exception("Perlcritic had an internal error")
failed = ret == 2
for issue in parse_perlcritic(out.splitlines()):
analysis.results.append(issue)
return (analysis, out, failed)
def sbuild(package, suite, arch):
chroot = "%s-%s" % (suite, arch)
dsc = os.path.basename(package)
if not dsc.endswith('.dsc'):
raise ValueError("WTF")
source, dsc = dsc.split("_", 1)
version, _ = dsc.rsplit(".", 1)
local = None
if "-" in version:
version, local = version.rsplit("-", 1)
suite, arch = chroot.split("-", 1)
sut = DebianBinary(source, version, local, arch)
out, err, ret = run_command([
"sbuild",
"-A",
"-c",
chroot,
"-v",
"-d",
suite,
"-j",
"8",
package,
])
ftbfs = ret != 0
info = parse_sbuild_log(out, sut=sut)
return info, out, ftbfs
def version():
out, err, ret = run_command(["sbuild", '--version'])
# TODO check ret
vline = out.splitlines()[0]
v = VERSION.match(vline)
vdict = v.groupdict()
return ('sbuild', vdict['version'])
def version():
out, err, ret = run_command(["sbuild", "--version"])
# TODO check ret
vline = out.splitlines()[0]
v = VERSION.match(vline)
vdict = v.groupdict()
return ("sbuild", vdict["version"])
def version(lintian_binary='lintian'):
out, err, ret = run_command([
lintian_binary, '--version'
])
#TODO: if ret != 0, not installed !
name, version = out.split(" ")
return (name, version.strip())
def version():
out, err, ret = run_command([
'cppcheck', '--version'
])
#TODO: if ret != 0, not installed !
name, version = out.split(" ")
return (name, version.strip())
def iterate():
suites = listize(config["suites"])
arches = listize(config["arches"])
with workon(suites, arches, list(PLUGINS.keys())) as job:
if job is None:
raise IDidNothingError("No more jobs")
package_id = job["package"]
type_ = job["package_type"]
logging.debug("Fetching the %s package, id=%s", type_, package_id)
package = None
if type_ == "binary":
package = proxy.get_binary_package(package_id)
elif type_ == "source":
package = proxy.get_source_package(package_id)
else:
raise IDidNothingError("SHIT")
handler, version_getter = load_module(job["type"])
firehose = create_firehose(package, version_getter)
with tdir() as fd:
with cd(fd):
with checkout(package) as target:
firehose, log, err = handler(target, package, job, firehose)
type_ = {"sources": "source", "binaries": "binary"}[package["_type"]]
logging.info("Job worker returned, filing reports")
report = proxy.submit_report(firehose.to_json(), job["_id"], err)
logging.info("Sending the XML firehose report to the pool")
open("firehose.xml", "w").write(firehose.to_xml_bytes())
remote_firehose_path = proxy.get_firehose_write_location(report)
cmd = config["copy"].format(src="firehose.xml", dest=remote_firehose_path)
out, err, ret = run_command(cmd)
logging.info("Sending the logs to the pool")
remote_path = proxy.get_log_write_location(report)
open("ethel-log", "wb").write(log.encode("utf-8"))
cmd = config["copy"].format(src="ethel-log", dest=remote_path)
out, err, ret = run_command(cmd)
if ret != 0:
print(out)
raise Exception("SHIT.")
def binary():
info = proxy.get_deb_info(package['_id'])
url_base = info['root']
out, err, ret = run_command(['wget'] + [
os.path.join(url_base, x) for x in info['packages']])
if ret != 0:
raise Exception("zomgwtf")
yield package['binaries']
def binary():
info = proxy.get_deb_info(package['_id'])
url_base = info['root']
out, err, ret = run_command(
['wget'] + [os.path.join(url_base, x) for x in info['packages']])
if ret != 0:
raise Exception("zomgwtf")
yield package['binaries']
def cppcheck(dsc, analysis):
run_command(["dpkg-source", "-x", dsc, "source"])
with cd('source'):
out, err, ret = run_command(['cppcheck', '--enable=all', '.', '--xml'])
xmlbytes = err.encode()
failed = False
if err.strip() == '':
return (analysis, err, failed)
for issue in parse_cppcheck(xmlbytes):
analysis.results.append(issue)
if not failed and issue.severity in [
'performance', 'portability', 'error', 'warning'
]:
failed = True
return (analysis, err, failed)
def cppcheck(dsc, analysis):
run_command(["dpkg-source", "-x", dsc, "source"])
with cd('source'):
out, err, ret = run_command([
'cppcheck', '--enable=all', '.', '--xml'
])
xmlbytes = err.encode()
failed = False
if err.strip() == '':
return (analysis, err, failed)
for issue in parse_cppcheck(xmlbytes):
analysis.results.append(issue)
if not failed and issue.severity in [
'performance', 'portability', 'error', 'warning'
]:
failed = True
return (analysis, err, failed)
def desktop_file_validate(package_root, analysis):
log = ""
failed = False
for dirpath, dirnames, filenames in os.walk(package_root):
for fp in filenames:
out, err, ret = run_command(['desktop-file-validate', fp])
for issue in parse_desktop_file_validate(out.splitlines()):
analysis.results.append(issue)
failed = True
log += out
log = log.strip()
return (analysis, log, failed)
def desktop_file_validate(package_root, analysis):
log = ""
failed = False
for dirpath, dirnames, filenames in os.walk(package_root):
for fp in filenames:
out, err, ret = run_command(['desktop-file-validate', fp])
for issue in parse_desktop_file_validate(out.splitlines()):
analysis.results.append(issue)
failed = True
log += out
log = log.strip()
return (analysis, log, failed)
def lintian(packages, analysis, lintian_binary='lintian'):
log = ""
failed = False
for package in packages:
out, err, ret = run_command([lintian_binary, "-IE", "--pedantic",
"--show-overrides", package])
for issue in parse_lintian(out.splitlines(), package):
analysis.results.append(issue)
if issue.severity in ['warning', 'error']:
failed = True
log += out
return (analysis, log, failed)
def sbuild(package, suite, arch):
chroot = "%s-%s" % (suite, arch)
dsc = os.path.basename(package)
if not dsc.endswith(".dsc"):
raise ValueError("WTF")
source, dsc = dsc.split("_", 1)
version, _ = dsc.rsplit(".", 1)
local = None
if "-" in version:
version, local = version.rsplit("-", 1)
suite, arch = chroot.split("-", 1)
sut = DebianBinary(source, version, local, arch)
out, err, ret = run_command(["sbuild", "-A", "-c", chroot, "-v", "-d", suite, "-j", "8", package])
ftbfs = ret != 0
info = parse_sbuild_log(out, sut=sut)
return info, out, ftbfs
def version():
out, err, ret = run_command(['pep8', '--version'])
#TODO: if ret != 0, not installed !
return ('pep8', out.strip())
def run(dsc, source, job, firehose):
run_command(["dpkg-source", "-x", dsc, "source"])
with cd('source'):
return desktop_file_validate('source', firehose)
def version():
out, err, ret = run_command(['cppcheck', '--version'])
#TODO: if ret != 0, not installed !
name, version = out.split(" ")
return (name, version.strip())
def version():
out, err, ret = run_command([
'pep8', '--version'
])
#TODO: if ret != 0, not installed !
return ('pep8', out.strip())
def run(dsc, source, job, firehose):
run_command(["dpkg-source", "-x", dsc, "source"])
with cd('source'):
return desktop_file_validate('source', firehose)
def version():
out, err, ret = run_command([ 'perlcritic', '--version' ])
#TODO : check ret for perlcritic installation
return ('perlcritic', out.strip())