def buildPackages():
qmgr = qmanager.QueueManager()
qmgr.transferAllPackagesToWorkQueue() #move all packages to workQueue for compilation
queue = copy.copy(qmgr.workQueue)
# We'll set home variable just after buidl process to make revdep work right
homeDir = os.environ['HOME']
packageList = []
deltaPackageList = []
isopackages = {}
if len(queue) == 0:
logger.info("Work Queue is empty...")
sys.exit(1)
# FIXME: Use fcntl.flock
f = open("/var/run/buildfarm", 'w')
f.close()
# Unpickle and load ISO package list here if config.generateDelta is true.
if config.generateDelta:
try:
isopackages = cPickle.Unpickler(open("data/packages.db", "rb")).load()
except:
logger.error("You have to create packages.db in data/ for delta generation.")
os.unlink("/var/run/buildfarm")
sys.exit(1)
# Compiling current workQueue
logger.raw("QUEUE")
logger.info("*** All packages to be compiled : %s" % qmgr.workQueue)
mailer.info("*** I'm starting to compile following packages (in the order below):\n\n%s" % "\n".join(queue))
logger.raw()
for pspec in queue:
packagename = getPackageNameFromPath(pspec)
build_output = open(os.path.join(config.outputDir, "%s.txt" % packagename), "w")
logger.raw()
logger.info(
"*** Compiling source %s (%d of %d)" %
(
packagename,
int(queue.index(pspec) + 1),
len(queue)
)
)
# This is here because farm captures the build output
pisi = pisiinterface.PisiApi(stdout = build_output, stderr = build_output, outputDir = config.workDir)
try:
try:
# Save current *.pisi file list in /var/pisi for further cleanup
pisiList = glob.glob1(config.workDir, "*.pisi")
# Build source package
# Returned values can also contain -dbginfo- packages.
(newBinaryPackages, oldBinaryPackages) = pisi.build(pspec)
# Reduce to filenames
newBinaryPackages = map(lambda x: os.path.basename(x), newBinaryPackages)
oldBinaryPackages = map(lambda x: os.path.basename(x), oldBinaryPackages)
# Filter debug packages because we don't need to build delta packages
# for debug packages
newDebugPackages = [p for p in newBinaryPackages if isdebug(p)]
oldDebugPackages = [p for p in oldBinaryPackages if isdebug(p)]
newBinaryPackages = list(set(newBinaryPackages).difference(newDebugPackages))
oldBinaryPackages = list(set(oldBinaryPackages).difference(oldDebugPackages))
newBinaryPackages.sort()
oldBinaryPackages.sort()
# Delta package generation using delta interface
# If the return value is None, delta generation is disabled
ret = pisi.delta(isopackages, oldBinaryPackages, newBinaryPackages)
if ret:
(deltasToInstall, deltaPackages, blacklistedPackages) = ret
else:
(deltasToInstall, deltaPackages, blacklistedPackages) = ([], [], [])
# Reduce to filenames
deltasToInstall = map(lambda x: os.path.basename(x), deltasToInstall)
deltaPackages = map(lambda x: os.path.basename(x), deltaPackages)
# If there exists incremental delta packages, install them.
if deltasToInstall:
packagesToInstall = deltasToInstall[:]
if len(newBinaryPackages) > len(oldBinaryPackages):
logger.info("*** There are new binaries, the package is probably splitted.")
# There exists some first builds, install them because they don't have delta.
packagesToInstall.extend(newBinaryPackages[len(oldBinaryPackages):])
logger.debug("(splitted package), packagesToInstall: %s" % packagesToInstall)
else:
# No delta, install full packages
packagesToInstall = newBinaryPackages[:]
if blacklistedPackages:
# Merge the blacklisted packages and unify the list
logger.debug("blacklistedPackages: %s" % blacklistedPackages)
packagesToInstall.extend(blacklistedPackages)
packagesToInstall = list(set(packagesToInstall))
logger.debug("packagesToInstall after merge: %s" % packagesToInstall)
# Merge the package lists
deltaPackages = deltaPackages + deltasToInstall
logger.debug("All delta packages: %s" % deltaPackages)
except Exception, e:
# Transfer source package to wait queue in case of a build error
qmgr.transferToWaitQueue(pspec)
# If somehow some binary packages could have been build, they'll stay in /var/pisi
# We should remove them here.
for p in set(glob.glob1(config.workDir, "*.pisi")).difference(pisiList):
logger.info("*** Removing stale package '%s' from '%s'" % (p, config.workDir))
removeBinaryPackageFromWorkDir(p)
errmsg = "Error occured for '%s' in BUILD process:\n %s" % (pspec, e)
logger.error(errmsg)
mailer.error(errmsg, pspec)
else:
try:
# If there exists multiple packages, reorder them in order to
# correctly install interdependent packages.
if len(packagesToInstall) > 1:
# packagesToInstall doesn't contain full paths
logger.info("*** Reordering packages to satisfy inner runtime dependencies...")
packagesToInstall = pisi.getInstallOrder(packagesToInstall)
logger.info("*** Installation order is: %s" % packagesToInstall)
for p in packagesToInstall:
# Install package
logger.info("*** Installing: %s" % os.path.join(config.workDir, p))
pisi.install(os.path.join(config.workDir, p))
except Exception, e:
# Transfer source package to wait queue in case of an install error
qmgr.transferToWaitQueue(pspec)
# FIXME: The packages before packagesToInstall[p] are already installed and therefore need to be
# uninstalled because p can't be installed.
if isdelta(p) and "no attribute 'old_files'" in str(e):
logger.info("*** %s was probably not installed on the system and the delta installation failed." % getName(p))
errmsg = "Error occured for '%s' in INSTALL process: %s" % (os.path.join(config.workDir, p), e)
logger.error(errmsg)
mailer.error(errmsg, pspec)
# The package should be removed from the related lists and WorkDir in case of an
# installation problem
for pa in deltaPackages+newBinaryPackages+newDebugPackages:
if pa in deltasToInstall:
deltasToInstall.remove(pa)
elif pa in newBinaryPackages:
newBinaryPackages.remove(pa)
logger.info("*** (Cleanup) Removing %s from %s" % (pa, config.workDir))
removeBinaryPackageFromWorkDir(pa)
else:
logger.info("*** (Cleanup) Removing %s from %s" % (pa, config.workDir))
removeBinaryPackageFromWorkDir(pa)
else:
qmgr.removeFromWorkQueue(pspec)
movePackages(newBinaryPackages, oldBinaryPackages, deltaPackages, newDebugPackages)
packageList += (map(lambda x: os.path.basename(x), newBinaryPackages))
deltaPackageList += (map(lambda x: os.path.basename(x), deltaPackages))
finally:
pisi.close()
os.environ['HOME'] = homeDir
logger.raw("QUEUE")
logger.info("*** Wait Queue: %s" % (qmgr.waitQueue))
if qmgr.waitQueue:
mailer.info("Queue finished with problems and those packages couldn't be compiled:\n\n%s\n\n\nNew binary packages are;\n\n%s\n\nnow in repository" % ("\n".join(qmgr.waitQueue), "\n".join(packageList)))
else:
mailer.info("Queue finished without a problem!...\n\n\nNew binary packages are:\n\n%s\n\n"
"New delta packages are:\n\n%s\n\nnow in repository..." % ("\n".join(packageList), "\n".join(deltaPackageList)))
logger.raw()
logger.raw()
# Save current path
current = os.getcwd()
# Set index paths
paths = [config.binaryPath, config.testPath]
if config.debugSupport:
# Enable debugSupport in config to generate an index
# for the debug repository.
paths.append(config.debugPath)
