def adequate(chroot_name, target, package_name, analysis):
with schroot(chroot_name) as chroot:
chroot.copy(target, "/tmp")
out, err, ret = chroot.run(['apt-get', 'install', '-y', 'adequate'],
user='******')
os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
out, err, ret = chroot.run(
['dpkg', '-i', "/tmp/%s" % target],
user='******',
return_codes=(0, 1),
preserve_environment=True)
out, err, ret = chroot.run(['apt-get', 'install', '-y', '-f'],
user='******',
preserve_environment=True)
out, err, ret = chroot.run(['adequate', package_name])
failed = False
for issue in parse_adequate(out.splitlines()):
failed = True
analysis.results.append(issue)
return analysis, out, failed, None
def adequate(chroot_name, target, package_name, analysis):
with schroot(chroot_name) as chroot:
chroot.copy(target, "/tmp")
out, err, ret = chroot.run([
'apt-get', 'install', '-y', 'adequate'
], user='******')
os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
out, err, ret = chroot.run([
'dpkg', '-i',
"/tmp/%s" % target
], user='******', return_codes=(0, 1), preserve_environment=True)
out, err, ret = chroot.run([
'apt-get', 'install', '-y', '-f'
], user='******', preserve_environment=True)
out, err, ret = chroot.run(['adequate', package_name])
failed = False
for issue in parse_adequate(out.splitlines()):
failed = True
analysis.results.append(issue)
return (analysis, out, failed, None, None)
def adequate(chroot_name, packages, analysis):
with schroot(chroot_name) as chroot:
for deb in packages:
chroot.copy(deb, "/tmp")
out, err, ret = chroot.run([
'apt-get', 'install', '-y', 'adequate'
], user='******')
out, err, ret = chroot.run([
'dpkg', '-i'
] + [
"/tmp/%s" % (x) for x in packages
], user='******', return_codes=(0, 1))
out, err, ret = chroot.run([
'apt-get', 'install', '-y', '-f'
], user='******')
out, err, ret = chroot.run(['adequate', deb.split("_", 1)[0]])
failed = False
for issue in parse_adequate(out.splitlines()):
failed = True
analysis.results.append(issue)
return analysis, out, failed
def clanganalyzer(package, suite, arch, analysis):
raise NotImplemented("Not ported")
chroot_name = "%s-%s" % (suite, arch)
with schroot(chroot_name) as chroot:
# We should have the dsc file to bulid
dsc = os.path.basename(package)
if not dsc.endswith(".dsc"):
raise ValueError("clanganalyzer runner must receive a dsc file")
# Setup the chroot for scan-build run
# 1/ install clang
# TODO: check the return codes
out, err, ret = chroot.run(["apt-get", "install", "-y", "clang", "wget"], user="******")
# 2/ fake dpkg-buildpackage in the schroot
# Replace the real dpkg-buildpackage by our script
out_, err, ret = chroot.run(
["mv", "/usr/bin/dpkg-buildpackage", "/usr/bin/dpkg-buildpackage.faked"], user="******"
)
out += out_
internal_report_dir = "/tmp/scan-build/"
# We will output the scan-build plist reports there
out_, err, ret = chroot.run(["mkdir", "-p", internal_report_dir], user="******")
out += out_
out_, err, ret = chroot.run(["chmod", "777", internal_report_dir], user="******")
out += out_
# Create the script
fake_dpkg_url = "http://leo.cavaille.net/public/dpkg-buildpackage-html"
out_, err, ret = chroot.run(["wget", "-O", "/usr/bin/dpkg-buildpackage", fake_dpkg_url], user="******")
out += out_
# Make it executable
out_, err, ret = chroot.run(["chmod", "755", "/usr/bin/dpkg-buildpackage"], user="******")
out += out_
# Now run sbuild in this session chroot for the package
out_, err, ret = run_command(
["sbuild", "-A", "--use-schroot-session", chroot.session, "-v", "-d", suite, "-j", "8", package]
)
out += out_
# Parse the plist reports into Firehose and return
# WARN : if the previous run did not delete the folder, this will fail
# worst, if we run several instances of virtual builders, this will
# fail because by default /tmp is a bind mount from the physical server
reports_dir = glob.glob(internal_report_dir + "*")
### SCANDALOUS HACK !!
return analysis, out, reports_dir, None
def upgrade(chroot):
print("Schrooting")
with schroot(chroot) as chroot:
print("updating")
out, err, ret = chroot.run([
"apt-get", "update"
], user='******')
print(out, err)
out, err, ret = chroot.run([
"apt-get", "upgrade", "-y"
], user='******')
print(out, err)
out, err, ret = chroot.run([
"apt-get", "dist-upgrade", "-y"
], user='******')
print(out, err)
def upgrade(chroot):
print("Schrooting")
with schroot(chroot) as chroot:
print("updating")
out, err, ret = chroot.run([
"apt-get", "update"
], user='******')
print(out, err)
out, err, ret = chroot.run([
"apt-get", "upgrade", "-y"
], user='******')
print(out, err)
out, err, ret = chroot.run([
"apt-get", "dist-upgrade", "-y"
], user='******')
print(out, err)
def piuparts(chroot, packages, analysis):
cfg = configparser.ConfigParser()
if cfg.read("/etc/schroot/chroot.d/%s" % (chroot)) == []:
raise Exception("Shit. No such tarball")
block = cfg[chroot]
if "file" not in block:
raise Exception("Chroot type isn't of tarball")
location = block['file']
copy_location = os.path.join("/tmp", os.path.basename(location))
with schroot(chroot) as chroot:
chroot.copy(location, copy_location)
for package in packages:
chroot.copy(package, "/tmp")
print("[ ] Installing...")
chroot.run(['apt-get', 'install', '-y', 'piuparts'], user='******')
print("[ ] Piuparts installed.")
failed = False
try:
print("[ ] Running Piuparts..")
out, err, ret = chroot.run([
'piuparts',
'-b',
copy_location,
] + ["/tmp/%s" % (x) for x in packages] + [
'--warn-on-debsums-errors',
'--pedantic-purge-test',
],
user='******')
except SchrootCommandError as e:
out, err = e.out, e.err
failed = True
for x in parse_piuparts(out.splitlines(), package):
analysis.results.append(x)
return analysis, out, failed
def piuparts(chroot, packages, analysis):
cfg = configparser.ConfigParser()
if cfg.read("/etc/schroot/chroot.d/%s" % (chroot)) == []:
raise Exception("Shit. No such tarball")
block = cfg[chroot]
if "file" not in block:
raise Exception("Chroot type isn't of tarball")
location = block['file']
copy_location = os.path.join("/tmp", os.path.basename(location))
with schroot(chroot) as chroot:
chroot.copy(location, copy_location)
for package in packages:
chroot.copy(package, "/tmp")
print("[ ] Installing...")
chroot.run(['apt-get', 'install', '-y', 'piuparts'], user='******')
print("[ ] Piuparts installed.")
failed = False
try:
print("[ ] Running Piuparts..")
out, err, ret = chroot.run([
'piuparts',
'-b', copy_location,
] + [ "/tmp/%s" % (x) for x in packages ] + [
'--warn-on-debsums-errors',
'--pedantic-purge-test',
], user='******')
except SchrootCommandError as e:
out, err = e.out, e.err
failed = True
for x in parse_piuparts(out.splitlines(), package):
analysis.results.append(x)
return analysis, out, failed
def adequate(chroot_name, packages, analysis):
with schroot(chroot_name) as chroot:
for deb in packages:
chroot.copy(deb, "/tmp")
out, err, ret = chroot.run(['apt-get', 'install', '-y', 'adequate'],
user='******')
out, err, ret = chroot.run(['dpkg', '-i'] +
["/tmp/%s" % (x) for x in packages],
user='******',
return_codes=(0, 1))
out, err, ret = chroot.run(['apt-get', 'install', '-y', '-f'],
user='******')
out, err, ret = chroot.run(['adequate', deb.split("_", 1)[0]])
failed = False
for issue in parse_adequate(out.splitlines()):
failed = True
analysis.results.append(issue)
return analysis, out, failed
def clanganalyzer(package, suite, arch, analysis):
chroot_name = "%s-%s" % (suite, arch)
with schroot(chroot_name) as chroot:
# We should have the dsc file to bulid
dsc = os.path.basename(package)
if not dsc.endswith('.dsc'):
raise ValueError("clanganalyzer runner must receive a dsc file")
# Setup the chroot for scan-build run
# 1/ install clang
# TODO: check the return codes
out, err, ret = chroot.run([
'apt-get', 'update'
], user='******')
out, err, ret = chroot.run([
'apt-get', 'install', '-y', 'clang', 'wget', 'dpkg-dev',
], user='******')
# 2/ fake dpkg-buildpackage in the schroot
# Replace the real dpkg-buildpackage by our script
out_, err, ret = chroot.run([
'mv',
'/usr/bin/dpkg-buildpackage',
'/usr/bin/dpkg-buildpackage.faked'
], user='******')
out += out_
# TODO: use tempfile to create a unique temporary directory
# this would allow running several instances of debile-slave
# at once
# difficulty: the directory must stil exist when debile-slave
# uploads the files to the master
internal_report_dir = "/tmp/scan-build/"
# clean up internal report dir
out_, err, ret = chroot.run([
'rm', '-rf', internal_report_dir
], user='******')
# We will output the scan-build plist reports there
out_, err, ret = chroot.run([
'mkdir', '-p', internal_report_dir
], user='******')
out += out_
out_, err, ret = chroot.run([
'chmod', '777', internal_report_dir
], user='******')
out += out_
# Create the script
fake_dpkg_url = "http://www.mux.me/debile/dpkg-buildpackage-html"
out_, err, ret = chroot.run([
'wget', '-O', '/usr/bin/dpkg-buildpackage', fake_dpkg_url
], user='******')
out += out_
# Make it executable
out_, err, ret = chroot.run([
'chmod', '755', '/usr/bin/dpkg-buildpackage'
], user='******')
out += out_
# Now run sbuild in this session chroot for the package
out_, _, _ = run_command([
"sbuild",
"-A",
"--use-schroot-session", chroot.session,
"-v",
"-d", suite,
"-j", "8",
package,
])
out += out_
# Parse the plist reports into Firehose and return
# WARN : if the previous run did not delete the folder, this will fail
# worst, if we run several instances of virtual builders, this will
# fail because by default /tmp is a bind mount from the physical server
reports_dir = glob.glob(internal_report_dir + '/*')
for dir in reports_dir:
for plist in parse_scandir(dir):
for issue in plist:
analysis.results.append(issue)
# SCANDALOUS HACK !!
files = list()
for dir in reports_dir:
for f in glob.glob(dir + '/*.html') + glob.glob(dir + '/*.js') \
+ glob.glob(dir + '/*.css'):
files.append(f)
return (analysis, out, reports_dir, None, files)
def honorcxx(package, suite, arch, analysis):
chroot_name = '{0}-{1}-honorcxx'.format(suite, arch)
with schroot(chroot_name) as chroot:
# Let's install the real compilers
out, err, code = chroot.run(['apt-get', 'update'], user='******')
out_, err, code = chroot.run([
'apt-get', '-y', '--no-install-recommends', 'install', 'gcc',
'cpp', 'g++'
],
user='******')
out += out_
out += err
# Let's create fake-compiler
with chroot.create_file(fake_compiler_path,
user='******') as fake_compiler_file:
fake_compiler_file.write(fake_gcc)
out_, err, code = chroot.run(['chmod', '755', fake_compiler_path],
user='******')
out += err
# Let's create the fake gcc
out_, err, code = chroot.run(['rm', '-f', '/usr/bin/gcc'], user='******')
out += err
out_, err, code = chroot.run(['rm', '-f', '/usr/bin/g++'], user='******')
out += err
out_, err, code = chroot.run(['rm', '-f', '/usr/bin/cpp'], user='******')
out += err
with chroot.create_file('/usr/bin/gcc', user='******') as fake_gcc_file:
fake_gcc_file.write(fake_gcc)
out_, err, code = chroot.run(['chmod', '755', '/usr/bin/gcc'],
user='******')
out += err
for bin in ['gcc', 'g++', 'cpp']:
out_, err, code = chroot.run(
['ln', '-s', '/usr/bin/gcc', '/usr/bin/{0}'.format(bin)],
user='******')
out += err
for version in gcc_versions:
out_, err, code = chroot.run(
['rm', '-f', '/usr/bin/{0}-{1}'.format(bin, version)],
user='******')
out += err
out_, err, code = chroot.run([
'ln', '-s', '/usr/bin/gcc', '/usr/bin/{0}-{1}'.format(
bin, version)
],
user='******')
out += err
out_, err, code = chroot.run([
'sh', '-c',
'echo {0}-{1} hold | dpkg --set-selections'.format(
bin, version)
],
user='******')
out += err
out_, err, code = chroot.run([
'sh', '-c',
'echo {0}-{1}-base hold | dpkg --set-selections'.format(
bin, version)
],
user='******')
out += err
# cleanup previous result file
if os.path.exists('/tmp/no-honor-cxx'):
os.remove('/tmp/no-honor-cxx')
# let's go
out_, _, _ = run_command([
'sbuild',
'-A',
'--use-schroot-session',
chroot.session,
"-v",
"-d",
suite,
"-j",
"1",
package,
])
out += out_
failed = False
if os.path.exists('/tmp/no-honor-cxx'):
failed = True
# FIXME: firewoes complains, some data might be missing
# File "/home/clemux/dev/debian/firewoes/firewoes/lib/debianutils.py",
# line 70, in get_source_url
# >>> url_quote(message)))
# TypeError: %d format: a number is required, not Undefined
analysis.results.append(
Issue(
cwe=None,
testid='0',
location=Location(file=File('n/a', None), function=None),
message=Message(text='Package does not honor CC/CXX'),
severity='error',
notes=None,
trace=None,
))
os.remove('/tmp/no-honor-cxx')
return (analysis, out, failed, None, None)
def clanganalyzer(package, suite, arch, analysis):
raise NotImplemented("Not ported")
chroot_name = "%s-%s" % (suite, arch)
with schroot(chroot_name) as chroot:
# We should have the dsc file to bulid
dsc = os.path.basename(package)
if not dsc.endswith('.dsc'):
raise ValueError("clanganalyzer runner must receive a dsc file")
# Setup the chroot for scan-build run
# 1/ install clang
# TODO: check the return codes
out, err, ret = chroot.run([
'apt-get', 'install', '-y', 'clang', 'wget'
], user='******')
# 2/ fake dpkg-buildpackage in the schroot
# Replace the real dpkg-buildpackage by our script
out_, err, ret = chroot.run([
'mv',
'/usr/bin/dpkg-buildpackage',
'/usr/bin/dpkg-buildpackage.faked'
], user='******')
out += out_
internal_report_dir = "/tmp/scan-build/"
# We will output the scan-build plist reports there
out_, err, ret = chroot.run([
'mkdir', '-p', internal_report_dir
], user='******')
out += out_
out_, err, ret = chroot.run([
'chmod', '777', internal_report_dir
], user='******')
out += out_
# Create the script
fake_dpkg_url = "http://leo.cavaille.net/public/dpkg-buildpackage-html"
out_, err, ret = chroot.run([
'wget', '-O', '/usr/bin/dpkg-buildpackage', fake_dpkg_url
], user='******')
out += out_
# Make it executable
out_, err, ret = chroot.run([
'chmod', '755', '/usr/bin/dpkg-buildpackage'
], user='******')
out += out_
# Now run sbuild in this session chroot for the package
out_, err, ret = run_command([
"sbuild",
"-A",
"--use-schroot-session", chroot.session,
"-v",
"-d", suite,
"-j", "8",
package,
])
out += out_
# Parse the plist reports into Firehose and return
# WARN : if the previous run did not delete the folder, this will fail
# worst, if we run several instances of virtual builders, this will
# fail because by default /tmp is a bind mount from the physical server
reports_dir = glob.glob(internal_report_dir+'*')
### SCANDALOUS HACK !!
return analysis, out, reports_dir, None
def clanganalyzer(package, suite, arch, analysis):
chroot_name = "%s-%s" % (suite, arch)
with schroot(chroot_name) as chroot:
# We should have the dsc file to bulid
dsc = os.path.basename(package)
if not dsc.endswith('.dsc'):
raise ValueError("clanganalyzer runner must receive a dsc file")
# Setup the chroot for scan-build run
# 1/ install clang
# TODO: check the return codes
out, err, ret = chroot.run(['apt-get', 'update'], user='******')
out, err, ret = chroot.run([
'apt-get',
'install',
'-y',
'clang',
'wget',
'dpkg-dev',
],
user='******')
# 2/ fake dpkg-buildpackage in the schroot
# Replace the real dpkg-buildpackage by our script
out_, err, ret = chroot.run([
'mv', '/usr/bin/dpkg-buildpackage',
'/usr/bin/dpkg-buildpackage.faked'
],
user='******')
out += out_
# TODO: use tempfile to create a unique temporary directory
# this would allow running several instances of debile-slave
# at once
# difficulty: the directory must stil exist when debile-slave
# uploads the files to the master
internal_report_dir = "/tmp/scan-build/"
# clean up internal report dir
out_, err, ret = chroot.run(['rm', '-rf', internal_report_dir],
user='******')
# We will output the scan-build plist reports there
out_, err, ret = chroot.run(['mkdir', '-p', internal_report_dir],
user='******')
out += out_
out_, err, ret = chroot.run(['chmod', '777', internal_report_dir],
user='******')
out += out_
# Create the script
fake_dpkg_url = "http://www.mux.me/debile/dpkg-buildpackage-html"
out_, err, ret = chroot.run(
['wget', '-O', '/usr/bin/dpkg-buildpackage', fake_dpkg_url],
user='******')
out += out_
# Make it executable
out_, err, ret = chroot.run(
['chmod', '755', '/usr/bin/dpkg-buildpackage'], user='******')
out += out_
# Now run sbuild in this session chroot for the package
out_, err, ret = run_command([
"sbuild",
"-A",
"--use-schroot-session",
chroot.session,
"-v",
"-d",
suite,
"-j",
"8",
package,
])
out += out_
# Parse the plist reports into Firehose and return
# WARN : if the previous run did not delete the folder, this will fail
# worst, if we run several instances of virtual builders, this will
# fail because by default /tmp is a bind mount from the physical server
reports_dir = glob.glob(internal_report_dir + '/*')
for dir in reports_dir:
for plist in parse_scandir(dir):
for issue in plist:
analysis.results.append(issue)
# SCANDALOUS HACK !!
files = list()
for dir in reports_dir:
for f in glob.glob(dir + '/*.html') + glob.glob(dir + '/*.js') \
+ glob.glob(dir + '/*.css'):
files.append(f)
return (analysis, out, reports_dir, None, files)
