def buildPackages(): qmgr = qmanager.QueueManager() qmgr.transferAllPackagesToWorkQueue() #move all packages to workQueue for compilation queue = copy.copy(qmgr.workQueue) # We'll set home variable just after buidl process to make revdep work right homeDir = os.environ['HOME'] packageList = [] deltaPackageList = [] isopackages = {} if len(queue) == 0: logger.info("Work Queue is empty...") sys.exit(1) # FIXME: Use fcntl.flock f = open("/var/run/buildfarm", 'w') f.close() # Unpickle and load ISO package list here if config.generateDelta is true. if config.generateDelta: try: isopackages = cPickle.Unpickler(open("data/packages.db", "rb")).load() except: logger.error("You have to create packages.db in data/ for delta generation.") os.unlink("/var/run/buildfarm") sys.exit(1) # Compiling current workQueue logger.raw("QUEUE") logger.info("*** All packages to be compiled : %s" % qmgr.workQueue) mailer.info("*** I'm starting to compile following packages (in the order below):\n\n%s" % "\n".join(queue)) logger.raw() for pspec in queue: packagename = getPackageNameFromPath(pspec) build_output = open(os.path.join(config.outputDir, "%s.txt" % packagename), "w") logger.raw() logger.info( "*** Compiling source %s (%d of %d)" % ( packagename, int(queue.index(pspec) + 1), len(queue) ) ) # This is here because farm captures the build output pisi = pisiinterface.PisiApi(stdout = build_output, stderr = build_output, outputDir = config.workDir) try: try: # Save current *.pisi file list in /var/pisi for further cleanup pisiList = glob.glob1(config.workDir, "*.pisi") # Build source package # Returned values can also contain -dbginfo- packages. (newBinaryPackages, oldBinaryPackages) = pisi.build(pspec) # Reduce to filenames newBinaryPackages = map(lambda x: os.path.basename(x), newBinaryPackages) oldBinaryPackages = map(lambda x: os.path.basename(x), oldBinaryPackages) # Filter debug packages because we don't need to build delta packages # for debug packages newDebugPackages = [p for p in newBinaryPackages if isdebug(p)] oldDebugPackages = [p for p in oldBinaryPackages if isdebug(p)] newBinaryPackages = list(set(newBinaryPackages).difference(newDebugPackages)) oldBinaryPackages = list(set(oldBinaryPackages).difference(oldDebugPackages)) newBinaryPackages.sort() oldBinaryPackages.sort() # Delta package generation using delta interface # If the return value is None, delta generation is disabled ret = pisi.delta(isopackages, oldBinaryPackages, newBinaryPackages) if ret: (deltasToInstall, deltaPackages, blacklistedPackages) = ret else: (deltasToInstall, deltaPackages, blacklistedPackages) = ([], [], []) # Reduce to filenames deltasToInstall = map(lambda x: os.path.basename(x), deltasToInstall) deltaPackages = map(lambda x: os.path.basename(x), deltaPackages) # If there exists incremental delta packages, install them. if deltasToInstall: packagesToInstall = deltasToInstall[:] if len(newBinaryPackages) > len(oldBinaryPackages): logger.info("*** There are new binaries, the package is probably splitted.") # There exists some first builds, install them because they don't have delta. packagesToInstall.extend(newBinaryPackages[len(oldBinaryPackages):]) logger.debug("(splitted package), packagesToInstall: %s" % packagesToInstall) else: # No delta, install full packages packagesToInstall = newBinaryPackages[:] if blacklistedPackages: # Merge the blacklisted packages and unify the list logger.debug("blacklistedPackages: %s" % blacklistedPackages) packagesToInstall.extend(blacklistedPackages) packagesToInstall = list(set(packagesToInstall)) logger.debug("packagesToInstall after merge: %s" % packagesToInstall) # Merge the package lists deltaPackages = deltaPackages + deltasToInstall logger.debug("All delta packages: %s" % deltaPackages) except Exception, e: # Transfer source package to wait queue in case of a build error qmgr.transferToWaitQueue(pspec) # If somehow some binary packages could have been build, they'll stay in /var/pisi # We should remove them here. for p in set(glob.glob1(config.workDir, "*.pisi")).difference(pisiList): logger.info("*** Removing stale package '%s' from '%s'" % (p, config.workDir)) removeBinaryPackageFromWorkDir(p) errmsg = "Error occured for '%s' in BUILD process:\n %s" % (pspec, e) logger.error(errmsg) mailer.error(errmsg, pspec) else: try: # If there exists multiple packages, reorder them in order to # correctly install interdependent packages. if len(packagesToInstall) > 1: # packagesToInstall doesn't contain full paths logger.info("*** Reordering packages to satisfy inner runtime dependencies...") packagesToInstall = pisi.getInstallOrder(packagesToInstall) logger.info("*** Installation order is: %s" % packagesToInstall) for p in packagesToInstall: # Install package logger.info("*** Installing: %s" % os.path.join(config.workDir, p)) pisi.install(os.path.join(config.workDir, p)) except Exception, e: # Transfer source package to wait queue in case of an install error qmgr.transferToWaitQueue(pspec) # FIXME: The packages before packagesToInstall[p] are already installed and therefore need to be # uninstalled because p can't be installed. if isdelta(p) and "no attribute 'old_files'" in str(e): logger.info("*** %s was probably not installed on the system and the delta installation failed." % getName(p)) errmsg = "Error occured for '%s' in INSTALL process: %s" % (os.path.join(config.workDir, p), e) logger.error(errmsg) mailer.error(errmsg, pspec) # The package should be removed from the related lists and WorkDir in case of an # installation problem for pa in deltaPackages+newBinaryPackages+newDebugPackages: if pa in deltasToInstall: deltasToInstall.remove(pa) elif pa in newBinaryPackages: newBinaryPackages.remove(pa) logger.info("*** (Cleanup) Removing %s from %s" % (pa, config.workDir)) removeBinaryPackageFromWorkDir(pa) else:
logger.info("*** (Cleanup) Removing %s from %s" % (pa, config.workDir)) removeBinaryPackageFromWorkDir(pa) else: qmgr.removeFromWorkQueue(pspec) movePackages(newBinaryPackages, oldBinaryPackages, deltaPackages, newDebugPackages) packageList += (map(lambda x: os.path.basename(x), newBinaryPackages)) deltaPackageList += (map(lambda x: os.path.basename(x), deltaPackages)) finally: pisi.close() os.environ['HOME'] = homeDir logger.raw("QUEUE") logger.info("*** Wait Queue: %s" % (qmgr.waitQueue)) if qmgr.waitQueue: mailer.info("Queue finished with problems and those packages couldn't be compiled:\n\n%s\n\n\nNew binary packages are;\n\n%s\n\nnow in repository" % ("\n".join(qmgr.waitQueue), "\n".join(packageList))) else: mailer.info("Queue finished without a problem!...\n\n\nNew binary packages are:\n\n%s\n\n" "New delta packages are:\n\n%s\n\nnow in repository..." % ("\n".join(packageList), "\n".join(deltaPackageList))) logger.raw() logger.raw() # Save current path current = os.getcwd() # Set index paths paths = [config.binaryPath, config.testPath] if config.debugSupport: # Enable debugSupport in config to generate an index # for the debug repository. paths.append(config.debugPath)