def cleanPackage(self, product, version, productRoot, location): """remove any distribution-specific remnants of a package installation. This gets run automatically after a successful installation; however, it should be run explicitly after a failed installation. This implementation does a "pacman remove" to purge the record of the package from the local pacman database. @param product the name of the product to clean up after @param version the version of the product @param productRoot the product directory tree under which the product is assumed to be installed @param location the distribution location used to install the package. The implementation may ignore this. @returns bool True, if any state was cleaned up or False if nothing needed to be done. Note that False is not an error. """ pacmanDir = self.getOption('pacmanDBRoot', productRoot) if os.path.exists(os.path.join(pacmanDir,'o..pacman..o')): cmd = 'cd %s && pacman -allow urllib2 -remove "%s" ' eupsServer.system(cmd % (pacmanDir, location), self.Eups.noaction, self.verbose, self.log) else: if self.verbose >= 0: print >> self.log, \ "Warning: pacman database not found under", pacmanDir return False return True
def expandTableFile(tablefile): cmd = "\n".join(setups + ["eups expandtable -i --force %s" % tablefile]) try: server.system(cmd) except OSError, e: print >> self.log, e
def cleanBuildDirFor(self, productRoot, product, version, options=None, force=False, flavor=None): """Clean out the build directory used to build a product. This implementation calls getBuildDirFor() to get the full path of the directory used; then, if it exists, the directory is removed. As precaution, this implementation will only remove the directory if it appears to be below the product root, unless force=True. @param productRoot the root directory where products are installed @param product the name of the built product @param version the product's version @param force override the removal restrictions @param flavor the product flavor. If None, assume the current default flavor """ buildDir = self.getBuildDirFor(productRoot, product, version, options, flavor) if os.path.exists(buildDir): if force or (productRoot and os.path.commonprefix([productRoot, buildDir]) == productRoot): if self.verbose > 1: print >> self.log, "removing", buildDir rmCmd = "rm -rf %s" % buildDir try: server.system(rmCmd, verbosity=-1, log=self.log) except OSError, e: rmCmd = r"find %s -exec chmod 775 {} \; && %s" % (buildDir, rmCmd) try: server.system(rmCmd, verbosity=self.verbose-1, log=self.log) except OSError, e: print >> self.log, "Error removing %s; Continuing" % (buildDir)
def cleanPackage(self, product, version, productRoot, location): """remove any distribution-specific remnants of a package installation. This gets run automatically after a successful installation; however, it should be run explicitly after a failed installation. This implementation does a "pacman remove" to purge the record of the package from the local pacman database. @param product the name of the product to clean up after @param version the version of the product @param productRoot the product directory tree under which the product is assumed to be installed @param location the distribution location used to install the package. The implementation may ignore this. @returns bool True, if any state was cleaned up or False if nothing needed to be done. Note that False is not an error. """ pacmanDir = self.getOption('pacmanDBRoot', productRoot) if os.path.exists(os.path.join(pacmanDir, 'o..pacman..o')): cmd = 'cd %s && pacman -allow urllib2 -remove "%s" ' eupsServer.system(cmd % (pacmanDir, location), self.Eups.noaction, self.verbose, self.log) else: if self.verbose >= 0: print >> self.log, \ "Warning: pacman database not found under", pacmanDir return False return True
def installPacmanPackage(self, location, productRoot, installDir, pacmanDir, setups=None): if not os.path.exists(os.path.join(pacmanDir, "o..pacman..o")) and \ self.verbose >= 0: print >> self.log, "Warning: Pacman database directory,", \ "o..pacman..o, not found in", pacmanDir try: eupsServer.system("""cd %s && pacman -allow urllib2 -install "%s" """ % \ (pacmanDir, location), self.Eups.noaction, self.verbose, self.log) except OSError, e: raise RuntimeError("Pacman failed to install " + location)
def setGroupPerms(self, file, descend=False): if self.getOption("obeygroups", False): if self.verbose > 1: print >> self.log, "Setting group access for", file group = self.Option("groupowner") recurse = '' if descend: recurse = "-R " change = "" if os.path.isdir(file): change = "x" if group is not None: try: server.system("chgrp %s%s %s" % (recurse, group, dir), self.Eups.noaction, self.verbose-2, self.log) except OSError: pass try: system("chmod %sg+rws%s %s" % (recurse, change, dir), self.Eups.noaction, self.verbose-2, self.log) except OSError: pass
def createPackage(self, serverDir, product, version, flavor=None, overwrite=False): """Write a package distribution into server directory tree and return the distribution ID @param serverDir a local directory representing the root of the package distribution tree @param product the name of the product to create the package distribution for @param version the name of the product version @param flavor the flavor of the target platform; this may be ignored by the implentation @param overwrite if True, this package will overwrite any previously existing distribution files even if Eups.force is false """ if flavor is None: flavor = self.Eups.flavor tarball = self.getDistIdForPackage(product, version, flavor) (baseDir, productDir) = self.getProductInstDir(product, version, flavor) if os.access("%s/%s" % (serverDir, tarball), os.R_OK) and not (self.Eups.force or overwrite): if self.verbose > 0: print >> self.log, "Not recreating", tarball return tarball if self.verbose > 0: print >> self.log, "Writing", tarball try: eupsServer.system("(cd %s && tar -cf - %s) | gzip > %s/%s" % (baseDir, productDir, serverDir, tarball), self.Eups.noaction, self.verbose-1, self.log) except Exception, e: try: os.unlink("%s/%s" % (self.base, tarball)) except: pass raise OSError, "Failed to write %s: %s" % (tarball, str(e))
try: (baseDir, pdir, vdir) = re.search(r"^(\S+)/([^/]+)/([^/]+)$", installDir).groups() unpackDir = os.path.join(unpackDir, baseDir) except AttributeError, e: pass if not os.path.exists(unpackDir): os.makedirs(unpackDir) if self.verbose > 0: print >> self.log, "installing %s into %s" % (tarball, unpackDir) try: eupsServer.system("cd %s && tar -zxmf %s" % (unpackDir, tfile), self.Eups.noaction, verbosity=self.verbose - 1) except Exception, e: raise RuntimeError, ("Failed to read %s: %s" % (tfile, e)) if installDir and installDir == "none": installDir = None if installDir: installDir = os.path.join(productRoot, self.Eups.flavor, installDir) else: installDir = os.path.join(unpackDir, product, version) if installDir and os.path.exists(installDir): self.setGroupPerms(installDir)
class Distrib(eupsDistrib.DefaultDistrib): """A class to encapsulate tarball-based product distribution OPTIONS: The behavior of a Distrib class is fine-tuned via options (a dictionary of named values) that are passed in at construction time. The options supported are: noeups do not use the local EUPS database for information while creating packages. obeyGroups when creating files (other on the user side or the server side), set group ownership and make group writable groupowner when obeyGroups is true, change the group owner of to this value buildDir a directory to use to build a package during install. If this is a relative path, the full path will be relative to the product root for the installation. """ NAME = "tarball" def __init__(self, Eups, distServ, flavor, tag="current", options=None, verbosity=0, log=sys.stderr): eupsDistrib.Distrib.__init__(self, Eups, distServ, flavor, tag, options, verbosity, log) # @staticmethod # requires python 2.4 def parseDistID(distID): """Return a valid package location if and only if we recognize the given distribution identifier This implementation return a location if it ends with ".tar.gz" """ if distID: suffix = ".tar.gz" distID = distID.strip() if distID.endswith(suffix): return distID return None parseDistID = staticmethod(parseDistID) # should work as of python 2.2 def initServerTree(self, serverDir): """initialize the given directory to serve as a package distribution tree. @param serverDir the directory to initialize """ eupsDistrib.DefaultDistrib.initServerTree(self, serverDir) config = os.path.join(serverDir, eupsServer.serverConfigFilename) if not os.path.exists(config): configcontents = """# Configuration for a tarball-based server MANIFEST_FILE_RE = ^(?P<product>[^-]+)-(?P<version>[^@]+)@(?P<flavor>.*)\.manifest$ MANIFEST_URL = %(base)s/manifests/%(product)s-%(version)s@%(flavor)s.manifest TARBALL_URL = %(base)s/%(path)s DIST_URL = %(base)s/%(path)s """ cf = open(config, 'a') try: cf.write(configcontents) finally: cf.close() def createPackage(self, serverDir, product, version, flavor=None, overwrite=False): """Write a package distribution into server directory tree and return the distribution ID @param serverDir a local directory representing the root of the package distribution tree @param product the name of the product to create the package distribution for @param version the name of the product version @param flavor the flavor of the target platform; this may be ignored by the implentation @param overwrite if True, this package will overwrite any previously existing distribution files even if Eups.force is false """ if flavor is None: flavor = self.Eups.flavor tarball = self.getDistIdForPackage(product, version, flavor) (baseDir, productDir) = self.getProductInstDir(product, version, flavor) if not baseDir: if productDir != "none": raise RuntimeError( "Please complain to RHL about %s %s (baseDir = '', productDir = %s)" % (product, version, productDir)) msg = "I don't know how to write a tarball for %s %s as it has no directory" % ( product, version) if self.verbose > 1: print >> self.log, msg return None if os.access("%s/%s" % (serverDir, tarball), os.R_OK) and not (self.Eups.force or overwrite): if self.verbose > 0: print >> self.log, "Not recreating", tarball return tarball if self.verbose > 0: print >> self.log, "Writing", tarball # # Record where the binary distro was installed originally (and presumably tested...) # pwdFile = os.path.join(baseDir, productDir, ".pwd") fd = None try: # "try ... except ... finally" and "with" are too new-fangled to use fd = open(pwdFile, "w") print >> fd, os.path.join(baseDir, productDir) del fd except Exception, e: if self.verbose > 0: print >> self.log, \ "Unable to write %s; installation will be unable to check paths: %s" % (pwdFile, e) fullTarball = os.path.join(serverDir, tarball) try: eupsServer.system( "(cd %s && tar -cf - %s) | gzip > %s" % (baseDir, productDir, fullTarball), self.Eups.noaction, self.verbose - 1, self.log) except Exception, e: try: os.unlink(pwdFile) except: pass try: os.unlink(fullTarball) except: pass raise OSError, "Failed to write %s: %s" % (tarball, str(e))
def createPackage(self, serverDir, product, version, flavor=None, overwrite=False): """Write a package distribution into server directory tree and return the distribution ID @param serverDir a local directory representing the root of the package distribution tree @param product the name of the product to create the package distribution for @param version the name of the product version @param flavor the flavor of the target platform; this may be ignored by the implentation @param overwrite if True, this package will overwrite any previously existing distribution files even if Eups.force is false """ distid = self.getDistIdForPackage(product, version) distid = "eupspkg:%s-%s.eupspkg" % (product, version) # Make sure it's an absolute path serverDir = os.path.abspath(serverDir) (baseDir, productDir) = self.getProductInstDir(product, version, flavor) eupspkg = os.path.join(baseDir, productDir, "ups", "eupspkg") if not os.path.exists(eupspkg): # Use the defalt build file eupspkg = os.path.join(os.environ["EUPS_DIR"], 'lib', 'eupspkg.sh') # Construct the package in a temporary directory pkgdir0 = tempfile.mkdtemp(suffix='.eupspkg') prodSubdir = "%s-%s" % (product, version) pkgdir = os.path.join(pkgdir0, prodSubdir) os.mkdir(pkgdir) q = pipes.quote try: # Execute 'eupspkg <create>' cmd = ("cd %(pkgdir)s && " + \ "%(eupspkg)s PREFIX=%(prefix)s PRODUCT=%(product)s VERSION=%(version)s FLAVOR=%(flavor)s %(qopts)s" + \ " create") % \ { 'pkgdir': q(pkgdir), 'prefix': q(os.path.join(baseDir, productDir)), 'product': q(product), 'version': q(version), 'flavor': q(flavor), 'eupspkg': q(eupspkg), 'qopts': self.qopts, } eupsServer.system(cmd) # Tarball the result and copy it to $serverDir/products productsDir = os.path.join(serverDir, "products") if not os.path.isdir(productsDir): try: os.makedirs(productsDir) except: raise RuntimeError, ("Failed to create %s" % (productsDir)) tfn = os.path.join(productsDir, "%s-%s.eupspkg" % (product, version)) if os.path.exists(tfn) and not (overwrite or self.Eups.force): if self.Eups.verbose > 1: print >> self.log, "Not recreating", tfn return distid if not self.Eups.noaction: if self.verbose > 1: print >> self.log, "Writing", tfn try: cmd = 'cd %s && tar czf %s %s' % (q(pkgdir0), q(tfn), q(prodSubdir)) eupsServer.system(cmd) except OSError, e: try: os.unlink(tfn) except OSError: pass # probably didn't exist raise RuntimeError ("Failed to write %s: %s" % (tfn, e)) finally: shutil.rmtree(pkgdir0) return distid
def installPackage(self, location, product, version, productRoot, installDir, setups=None, buildDir=None): """Install a package with a given server location into a given product directory tree. @param location the location of the package on the server. This value is a distribution ID (distID) that has been stripped of its build type prefix. @param productRoot the product directory tree under which the product should be installed @param installDir the preferred sub-directory under the productRoot to install the directory. This value, which should be a relative path name, may be ignored or over-ridden by the pacman scripts @param setups a list of EUPS setup commands that should be run to properly build this package. This is usually ignored by the pacman scripts. """ pkg = location if self.Eups.verbose >= 1: print >> self.log, "[dl]",; self.log.flush() tfname = self.distServer.getFileForProduct(pkg, product, version, self.Eups.flavor, ftype="eupspkg", noaction=self.Eups.noaction) logfile = os.path.join(buildDir, "build.log") # we'll log the build to this file uimsgfile = os.path.join(buildDir, "build.msg") # messages to be shown on the console go to this file # Determine temporary build directory if not buildDir: buildDir = self.getOption('buildDir', 'EupsBuildDir') if self.verbose > 0: print >> self.log, "Building package: %s" % pkg print >> self.log, "Building in directory:", buildDir print >> self.log, "Writing log to: %s" % (logfile) if self.Eups.noaction: print >> self.log, "skipping [noaction]" return # Make sure the buildDir is empty (to avoid interference from failed builds) shutil.rmtree(buildDir) os.mkdir(buildDir) # Construct the build script q = pipes.quote try: buildscript = os.path.join(buildDir, "build.sh") fp = open(buildscript, 'w') try: fp.write("""\ #!/bin/bash # ---- # ---- This script has been autogenerated by 'eups distrib install'. # ---- set -xe cd %(buildDir)s # make sure the EUPS environment is set up . "$EUPS_DIR/bin/setups.sh" # sanitize the environment: unsetup any packages that were setup-ed # # NOTE: this has been disabled as there are legitimate reasons to have EUPS # packages other than the explicit dependencies set up (i.e., compilers, # different version of git, etc.) # # for pkg in $(eups list -s | cut -d' ' -f 1); do # unsetup -j "$pkg" # done # Unpack the eupspkg tarball tar xzvf %(eupspkg)s # Enter the directory unpacked from the tarball PKGDIR="$(find . -maxdepth 1 -type d ! -name ".*" | head -n 1)" test ! -z $PKGDIR cd "$PKGDIR" # If ./ups/eupspkg is not present, symlink in the default if [[ ! -e ./ups/eupspkg ]]; then mkdir -p ./ups ln -s "$EUPS_DIR/lib/eupspkg.sh" ups/eupspkg fi # eups setup the dependencies %(setups)s # show what we're running with (for the log file) eups list -s # fetch package source ( ./ups/eupspkg %(qopts)s fetch ) || exit -1 # prepare for build (e.g., apply platform-specific patches) ( ./ups/eupspkg %(qopts)s prep ) || exit -2 # setup the package being built. note we're using -k # to ensure setup-ed dependencies aren't overridden by # the table file. we could've used -j instead, but then # 'eups distrib install -j ...' installs would fail as # these don't traverse and setup the dependencies. setup --type=build -k -r . # configure, build, and install ( ./ups/eupspkg %(qopts)s config ) || exit -3 ( ./ups/eupspkg %(qopts)s build ) || exit -4 ( ./ups/eupspkg %(qopts)s install ) || exit -5 """ % { 'buildDir' : q(buildDir), 'eupspkg' : q(tfname), 'setups' : "\n".join(setups), 'product' : q(product), 'version' : q(version), 'qopts' : self.qopts, } ) finally: fp.close() # Make executable (equivalent of 'chmod +x $buildscript') st = os.stat(buildscript) os.chmod(buildscript, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) # # Did they ask to have group permissions honoured? # self.setGroupPerms(buildDir + "*") # Run the build cmd = "(%s) >> %s 2>&1 4>%s" % (q(buildscript), q(logfile), q(uimsgfile)) if not self.nobuild: if self.Eups.verbose >= 1: print >> self.log, "[build]",; self.log.flush() eupsServer.system(cmd, self.Eups.noaction) # Copy the build log into the product install directory. It's useful to keep around. installDirUps = os.path.join(self.Eups.path[0], self.Eups.flavor, product, version, 'ups') if os.path.isdir(installDirUps): shutil.copy2(logfile, installDirUps) if self.verbose > 0: print >> self.log, "Build log file copied to %s/%s" % (installDirUps, os.path.basename(logfile)) else: print >> self.log, "Build log file not copied as %s does not exist (this shouldn't happen)." % installDirUps # Echo any lines from "messages" file # TODO: This should be piped in real time, not written out to a file and echoed. if os.path.getsize(uimsgfile) > 0: print >> self.log, "" fp = open(uimsgfile) for line in fp: self.log.write(" %s" % line) fp.close() except OSError, e: if self.verbose >= 0 and os.path.exists(logfile): try: print >> self.log, "\n\n***** error: from %s:" % logfile eupsServer.system("tail -20 %s 1>&2" % q(logfile)) except: pass raise RuntimeError("Failed to build %s: %s" % (pkg, str(e)))
def clean(self, product, version, flavor=None, options=None, installDir=None, uninstall=False): """clean up the remaining remants of the failed installation of a distribution. @param product the name of the product to clean up after @param version the version of the product @param flavor the flavor for the product to assume. This affects where we look for partially installed packages. None (the default) means the default flavor. @param options extra options for fine-tuning the distrib-specific cleaning as a dictionary @param installDir the directory where the product should be installed If None, a default location based on the above parameters will be assumed. @parma uninstall if True, run the equivalent of "eups remove" for this package. default: False. """ handlePartialInstalls = True productRoot = self.getInstallRoot() if not flavor: flavor = self.eups.flavor # check the build directory buildDir = self.getBuildDirFor(productRoot, product, version, options, flavor) if self.verbose > 1 or (self.verbose > 0 and not os.path.exists(buildDir)): msg = "Looking for build directory to cleanup: %s" % buildDir if not os.path.exists(buildDir): msg += "; not found" print >> self.log, msg if os.path.exists(buildDir): distidfile = os.path.join(buildDir, "distID.txt") if os.path.isfile(distidfile): (distId, pkgroot) = self._readDistIDFile(distidfile) if distId and pkgroot: if self.verbose > 1: print >> self.log, "Attempting distClean for", \ "build directory via ", distId self.distribClean(product, version, pkgroot, distId, flavor) self.cleanBuildDirFor(productRoot, product, version, options, flavor=flavor) # now look for a partially installed (but not yet eups-declared) package if handlePartialInstalls: if not installDir: installDir = os.path.join(productRoot, flavor, product, version) if self.verbose > 1: print >> self.log, "Looking for a partially installed package:",\ product, version if os.path.isdir(installDir): distidfile = os.path.join(installDir, "ups", "distID.txt") if os.path.isfile(distidfile): (pkgroot, distId) = self._readDistIDFile(distidfile) if distId: if self.verbose > 1: print >> self.log, "Attempting distClean for", \ "installation directory via ", distId self.distribClean(product,version,pkgroot,distId,flavor) # make sure this directory is not declared for any product installDirs = map(lambda x: x.dir, self.eups.findProducts()) if installDir not in installDirs: if not installDir.startswith(productRoot) and \ not self.eups.force: if self.verbose >= 0: print >> self.log, "Too scared to delete product dir",\ "that's not under the product root:", installDir else: if self.verbose > 0: print >> self.log, "Removing installation dir:", \ installDir if utils.isDbWritable(installDir): try: server.system("/bin/rm -rf %s" % installDir) except OSError, e: print >> self.log, "Error removing %s; Continuing" % (installDir) elif self.verbose >= 0: print >> self.log, "No permission on install dir %s" % (installDir)
if installDir and installDir != "none": try: (baseDir, pdir, vdir) = re.search(r"^(\S+)/([^/]+)/([^/]+)$", installDir).groups() unpackDir = os.path.join(unpackDir,baseDir) except AttributeError, e: pass if not os.path.exists(unpackDir): os.makedirs(unpackDir) if self.verbose > 0: print >> self.log, "installing %s into %s" % (tarball, unpackDir) try: eupsServer.system("cd %s && tar -zxmf %s" % (unpackDir, tfile), self.Eups.noaction, verbosity=self.verbose-1) except Exception, e: raise RuntimeError, ("Failed to read %s: %s" % (tfile, e)) if installDir and installDir == "none": installDir = None if installDir: installDir = os.path.join(productRoot, self.Eups.flavor, installDir) else: installDir = os.path.join(unpackDir, product, version) if installDir and os.path.exists(installDir): self.setGroupPerms(installDir) # # Try to check for potential problems with non-relocatable binaries
raise RuntimeError, ("Failed to write %s" % bfile) if self.verbose and not self.nobuild: print "Issuing commands:" print "\t", str.join("\n\t", cmd) print >> file(logfile, "w"), str.join("\n\t", cmd) if False: cmd = "(%s) 2>&1 | tee >> %s" % (str.join("\n", cmd), logfile) else: cmd = "(%s) >> %s 2>&1 " % (str.join("\n", cmd), logfile) if not self.nobuild: try: eupsServer.system(cmd, self.Eups.noaction) except OSError, e: if self.verbose >= 0 and os.path.exists(logfile): try: print >> self.log, "BUILD ERROR! From build log:" eupsServer.system("tail -20 %s 1>&2" % logfile) except: pass raise RuntimeError("Failed to build %s: %s" % (builder, str(e))) if self.verbose > 0: print >> self.log, "Builder %s successfully completed" % builder def findTableFile(self, productName, version, flavor): """Give the distrib a chance to produce a table file""" return self.find_file_on_path("%s.table" % productName)
raise RuntimeError, ("Failed to write %s" % bfile) if self.verbose and not self.nobuild: print "Issuing commands:" print "\t", str.join("\n\t", cmd) print >> file(logfile, "w"), str.join("\n\t", cmd) if False: cmd = "(%s) 2>&1 | tee >> %s" % (str.join("\n", cmd), logfile) else: cmd = "(%s) >> %s 2>&1 " % (str.join("\n", cmd), logfile) if not self.nobuild: try: eupsServer.system(cmd, self.Eups.noaction) except OSError, e: if self.verbose >= 0 and os.path.exists(logfile): try: print >> self.log, "BUILD ERROR! From build log:" eupsServer.system("tail -20 %s 1>&2" % logfile) except: pass raise RuntimeError("Failed to build %s: %s" % (builder, str(e))) if self.verbose > 0: print >> self.log, "Builder %s successfully completed" % builder def findTableFile(self, productName, version, flavor): """Give the distrib a chance to produce a table file"""