def __init__(self, config):
self._conf = RapidumoConfig()
self._affinity_preference = config["affinity_preference"]
self._archive_path = "%s/%s" % (self._conf.archive_config["path"], self._conf.distro_name)
self._buildq_path = self._conf.archive_config["build_queues_path"]
self._pkginfo = PackageBuildInfoRetriever(self._conf)
self._bcheck = BuildCheck(self._conf)
def render_template(name, out_name=None, *args, **kwargs):
config = RapidumoConfig()
gcfg = config.general_config
out_dir = gcfg['html_output']
if not out_name:
out_path = os.path.join(out_dir, name)
else:
out_path = os.path.join(out_dir, out_name)
# create subdirectories if necessary
out_dir = os.path.dirname(os.path.realpath(out_path))
if not os.path.exists(out_dir):
os.makedirs(out_dir)
template = j2_env.get_template(name)
content = template.render(*args, **kwargs)
with codecs.open(out_path, 'w', encoding='utf-8') as f:
f.write(content)
class ArchiveDebileBridge:
def __init__(self, config):
self._conf = RapidumoConfig()
self._affinity_preference = config["affinity_preference"]
self._archive_path = "%s/%s" % (self._conf.archive_config["path"], self._conf.distro_name)
self._buildq_path = self._conf.archive_config["build_queues_path"]
self._pkginfo = PackageBuildInfoRetriever(self._conf)
self._bcheck = BuildCheck(self._conf)
def _create_debile_source(self, session, pkg):
user = session.query(Person).filter_by(email="*****@*****.**").one()
group_suite = (
session.query(GroupSuite)
.join(GroupSuite.group)
.join(GroupSuite.suite)
.filter(Group.name == "default", Suite.name == pkg.suite)
.one()
)
component = session.query(Component).filter(Component.name == pkg.component).one()
aroot = None
if pkg.queue_name:
# package is in a build-queue
aroot = self._buildq_path
else:
aroot = self._archive_path
dsc_fname = "{root}/{directory}/{filename}".format(root=aroot, directory=pkg.directory, filename=pkg.dsc)
dsc = Dsc(open(dsc_fname))
if "Build-Architecture-Indep" in dsc:
valid_affinities = dsc["Build-Architecture-Indep"]
elif "X-Build-Architecture-Indep" in dsc:
valid_affinities = dsc["X-Build-Architecture-Indep"]
elif "X-Arch-Indep-Build-Arch" in dsc:
valid_affinities = dsc["X-Arch-Indep-Build-Arch"]
else:
valid_affinities = "any"
source = create_source(dsc, group_suite, component, user, self._affinity_preference, valid_affinities)
source.directory = pkg.directory
source.dsc_filename = pkg.dsc
session.add(source)
for aname in pkg.installed_archs:
arch = session.query(Arch).filter_by(name=aname).one()
binary = Binary(source=source, arch=arch, uploaded_at=source.uploaded_at)
session.add(binary)
for name, arch, filename in pkg.binaries:
if arch == binary.arch.name:
directory, _, filename = filename.rpartition("/")
deb = Deb(binary=binary, directory=directory, filename=filename)
session.add(deb)
create_jobs(source, dose_report="No dose-builddebcheck report available yet.")
oldsources = session.query(Source).filter(Source.group_suite == source.group_suite, Source.name == source.name)
for oldsource in oldsources:
if version_compare(oldsource.version, source.version) >= 0:
continue
# Drop any old jobs that are still pending.
for job in oldsource.jobs:
if (job.check.build and not any(job.built_binaries)) or not any(job.results):
session.delete(job)
elif job.failed is None:
job.failed = any(result.failed for result in job.results)
job.builder = None
job.assigned_at = None
job.finished_at = None
# Actually remove jobs marked for deletion above.
session.commit()
# If after cleanup there is no build jobs left, remove the source completely
if not any(job.check.build for job in oldsource.jobs):
session.delete(oldsource)
print("Created source for %s %s" % (source.name, source.version))
emit("accept", "source", source.debilize())
def _create_debile_binaries(self, session, source, pkg):
arch_all = session.query(Arch).filter(Arch.name == "all").one()
arches = session.query(Arch).filter(Arch.name.in_(pkg.installed_archs)).all()
if arch_all in source.arches and arch_all not in arches and source.affinity in arches:
if not session.query(
exists().where((Job.source == source) & (Job.arch == arch_all) & Job.check.has(Check.build == True))
).scalar():
# We have the arch:affinity binary but is still lacking the arch:all binary
# Make sure debile builds the arch:all binary separately
check = session.query(Check).filter(Check.build == True).one()
job = Job(check=check, arch=arch_all, source=source, binary=None)
session.add(job)
for arch in arches:
if session.query(exists().where((Binary.source == source) & (Binary.arch == arch))).scalar():
continue
# Find the job for this binary
job = (
session.query(Job)
.join(Job.check)
.filter(Job.source == source, Job.arch == arch, Check.build == True)
.first()
)
if not job and arch == arch_all and source.affinity in arches:
# The arch:all binary might have been created by the arch:affinity build job.
job = (
session.query(Job)
.join(Job.check)
.filter(Job.source == source, Job.arch == source.affinity, Check.build == True)
.first()
)
if job and (not job.finished_at or job.failed is True):
# Dak accepted a binary upload that debile-master didn't ask for
if arch != arch_all and not any(job.built_binaries):
session.delete(job)
job = None
if job:
binary = job.new_binary(arch)
else:
binary = Binary(source=source, arch=arch, uploaded_at=datetime.utcnow())
session.add(binary)
for name, arch, filename in pkg.binaries:
if arch == binary.arch.name:
directory, _, filename = filename.rpartition("/")
deb = Deb(binary=binary, directory=directory, filename=filename)
session.add(deb)
print("Created binary for %s %s on %s" % (binary.name, binary.version, binary.arch))
emit("accept", "binary", binary.debilize())
def _create_depwait_report(self, suite):
base_suite = self._conf.get_base_suite(suite)
components = self._conf.get_supported_components(base_suite).split(" ")
supported_archs = self._conf.get_supported_archs(base_suite).split(" ")
bcheck_data = {}
for component in components:
bcheck_data[component] = {}
for arch in supported_archs:
yaml_data = self._bcheck.get_package_states_yaml(suite, component, arch)
yaml_data = yaml_data.replace("%3a", ":") # Support for wheezy version of dose-builddebcheck
report_data = yaml.safe_load(yaml_data)["report"]
if not report_data:
report_data = list()
bcheck_data[component][arch] = report_data
yaml_file = open("%s/depwait-%s-%s_%s.yml" % (NEEDSBUILD_EXPORT_DIR, suite, component, arch), "w")
yaml_file.write(yaml_data)
yaml_file.close()
return bcheck_data
def _get_package_depwait_report(self, bcheck_data, job):
arch = job.source.affinity if job.arch.name == "all" else job.arch
for nbpkg in bcheck_data[job.component.name][arch.name]:
if nbpkg["package"] == ("src:" + job.source.name) and (nbpkg["version"] == job.source.version):
return nbpkg
return None
def import_pkgs(self, suite):
pkg_dict = self._pkginfo.get_packages_dict(suite)
for pkg in pkg_dict.values():
if pkg.suite.startswith("buildq"):
_, _, main_suite = pkg.suite.partition("-")
pkg.queue_name = pkg.suite
pkg.suite = main_suite
try:
with session() as s:
source = (
s.query(Source)
.join(Source.group_suite)
.join(GroupSuite.group)
.join(GroupSuite.suite)
.filter(
Source.name == pkg.pkgname,
Source.version == pkg.version,
Group.name == "default",
Suite.name == pkg.suite,
)
.first()
)
if not source:
self._create_debile_source(s, pkg)
elif pkg.installed_archs:
self._create_debile_binaries(s, source, pkg)
except Exception as ex:
print("Skipping %s (%s) in %s due to error: %s" % (pkg.pkgname, pkg.version, pkg.suite, str(ex)))
def unblock_jobs(self, suite):
bcheck_data = self._create_depwait_report(suite)
with session() as s:
jobs = (
s.query(Job)
.join(Job.check)
.join(Job.source)
.join(Source.group_suite)
.join(GroupSuite.group)
.join(GroupSuite.suite)
.filter(
Group.name == "default",
Suite.name == suite,
Check.build == True,
(Job.dose_report != None) | ~Job.built_binaries.any(),
)
)
for job in jobs:
try:
report = self._get_package_depwait_report(bcheck_data, job)
if report and report["status"] != "ok":
dose_report = "Unknown problem"
for reason in report["reasons"]:
if "missing" in reason:
dose_report = "Unsat dependency %s" % (reason["missing"]["pkg"]["unsat-dependency"])
break
elif "conflict" in reason:
dose_report = "Conflict between %s and %s" % (
reason["conflict"]["pkg1"]["package"],
reason["conflict"]["pkg2"]["package"],
)
break
if job.dose_report != dose_report:
job.dose_report = dose_report
elif job.dose_report != None:
job.dose_report = None
print("Unblocked job %s (%s) %s" % (job.source.name, job.source.version, job.name))
except Exception as ex:
print(
"Skipping %s (%s) %s due to error: %s"
% (job.source.name, job.source.version, job.name, str(ex))
)
def prune_pkgs(self, suite):
base_suite = self._conf.get_base_suite(suite)
suites = [suite, base_suite] if suite != base_suite else [suite]
components = self._conf.get_supported_components(base_suite).split(" ")
pkg_list = []
for s in suites:
for c in components:
pkg_list += self._pkginfo._get_package_list(s, c)
pkgs = set()
pkgs.update(pkg.pkgname + " " + pkg.version for pkg in pkg_list)
with session() as s:
sources = (
s.query(Source)
.join(Source.group_suite)
.join(GroupSuite.group)
.join(GroupSuite.suite)
.filter(Group.name == "default", Suite.name == suite)
)
for source in sources:
if not (source.name + " " + source.version) in pkgs and not os.path.exists(source.dsc_path):
print("Removed obsolete source %s %s" % (source.name, source.version))
# Package no longer in the archive (neither in the index nor the pool)
s.delete(source)
def reschedule_jobs(self):
with session() as s:
cutoff = datetime.utcnow() - timedelta(days=1)
jobs = s.query(Job).filter(Job.failed.is_(None), Job.finished_at != None, Job.finished_at < cutoff)
for job in jobs:
# Still missing the .dud a day after the builder told debile-master it had finished the job
print("Rescheduling %s in %s due to missing *.dud upload" % (str(job), str(job.group_suite)))
job.failed = None
job.builder = None
job.assigned_at = None
job.finished_at = None
cutoff = datetime.utcnow() - timedelta(days=7)
jobs = (
s.query(Job)
.join(Job.check)
.filter(
Check.build == True,
Job.failed.is_(False),
~Job.built_binaries.any(),
Job.finished_at != None,
Job.finished_at < cutoff,
)
)
for job in jobs:
# Still missing the .changes a week after the builder told debile-master it had finished the build job
print("Rescheduling %s in %s due to missing *.changes upload" % (str(job), str(job.group_suite)))
job.failed = None
job.builder = None
job.assigned_at = None
job.finished_at = None
def clean_results(self):
path = None
dirs = set()
with session() as s:
group = s.query(Group).filter_by(name="default").one()
path = group.files_path
dirs.update(
x.directory
for x in s.query(Result)
.join(Result.job)
.join(Job.source)
.join(Source.group_suite)
.filter(GroupSuite.group == group)
)
old_cwd = os.getcwd()
try:
os.chdir(path)
for dir in glob.iglob("*/*/*"):
if os.path.isdir(dir) and dir not in dirs:
# An orphaned results path, remove it
shutil.rmtree(dir)
print("Removed orphaned result dir %s" % dir)
finally:
os.chdir(old_cwd)