def createTaggedRelease(self, tag, product, version=None, flavor=None,
distrib=None):
"""
create and release a named collection of products based on the
known dependencies of a given product.
@param serverRoot the root directory of a local server distribution
tree
@param tag the name to give to this tagged release.
@param product the name of the product to create
@param version the version of the product to create. (Default:
the version marked current in the EUPS db)
@param flavor the flavor to associate with this release (Default:
"generic")
@param distrib a Distrib instance to use to determine which
products to include. If not provided, a default
will be used.
"""
if distrib is not None and not isinstance(distrib, Distrib):
raise TypeError("distrib parameter not a Distrib instance")
validTags = self.getSupportedTags()
if tag in validTags:
if not self.eups.force:
raise EupsException("Can't over-write existing tagged release "+
"without --force")
elif self.verbose > 0:
print("Over-writing existing tagged release for", tag, file=self.log)
elif self.verbose > 0:
print("Creating new tagged release for", tag, file=self.log)
if not flavor: flavor = "generic"
if not version:
version = self.eups.findPreferredProduct(product)
if version:
version = version.version
if not version:
msg = "No local version of %s found" % product
raise ProductNotFound(product, msg)
if distribTypeName:
distrib = \
self.distFactory.createDistribByName(distribTypeName,
options=self.options,
verbosity=self.verbose,
log=self.log)
else:
distrib = DefaultDistrib(self.eups, self.distServer, self.flavor,
options=self.options,
verbosity=self.verbose,
log=self.log)
release = distrib.createTaggedRelease(serverRoot, tag, product, version,
flavor)
distrib.writeTaggedRelease(self.pkgroot, tag, products, flavor,
self.eups.flavor)
def __init__(self, files=None, flavors=None, maxsave=None, msg=None):
"""
@param files the files that were found to be out of sync. If None
the files are unknown.
@param flavors the flavors handled by the files. If None, the flavors
are unknown
@param maxsave if this was raised during an attempt to persist
stack data, this will be set to True or False
indicating whether persisting to files that were
okay was completed. If None, this is either not
known or not relevent.
"""
message = msg
if message is None:
message = "In-memory stack data appears out of sync with " + "cache files"
if files is not None:
message += ": " + str(files)
elif flavors is not None:
message += " (flavors=%s)" % str(files)
EupsException.__init__(self, message)
self.files = files
self.flavors = flavors
self.maxsave = maxsave
def __init__(self, files=None, flavors=None, maxsave=None, msg=None):
"""
@param files the files that were found to be out of sync. If None
the files are unknown.
@param flavors the flavors handled by the files. If None, the flavors
are unknown
@param maxsave if this was raised during an attempt to persist
stack data, this will be set to True or False
indicating whether persisting to files that were
okay was completed. If None, this is either not
known or not relevent.
"""
message = msg
if message is None:
message = "In-memory stack data appears out of sync with " + \
"cache files"
if files is not None:
message += ": " + str(files)
elif flavors is not None:
message += " (flavors=%s)" % str(files)
EupsException.__init__(self, message)
self.files = files
self.flavors = flavors
self.maxsave = maxsave
class Distrib(object):
"""A class to encapsulate product distribution
This class is an abstract base class with some default implementation.
Subclasses will provide an implementation for specific ways to install
packages or create distributable packages that can be installed by others.
A DistribServer instance, passed in via the constructor, controls how a
user pulls packages and related information from a distribution server.
A Distrib class understands how to unpack and/or build a package and
install into the user's software stack; this may include assumptions of
conventions of where to put things (though usually that information is
brought down from the server). A Distrib class also understands how to
create a server that provides these packages to users: it understands not
only how to create the necessary files but also how to organize them on
the server.
An instance of implemented Distrib class is usually accomplished via a
DistribFactory, using either createDistrib() (when installing a package)
or createDistribByName() (when creating a package). During package
installation, the Distrib class pulls down distribution files from a
(usually remote) server using a DistribServer instance (provided when
creating the DistribFactory instance). With package creation, the
Distrib class creates the distribution files based on a locally installed
version and deploys them into a *local* server directory tree. Thus a
Distrib class can (in principle) be used to replicate an entire server
locally. By using different Distrib class on installation and creation,
one can convert a server from one convention to another. Similarly, one
can create a distribution of server with a mix of distribution types to,
for example, provide both generic, build-from-source packages and
flavor-spcific, binary alternatives.
Sub-class implementations must override the value of the static string
variable, NAME, which is used as its default name (for look-ups via
DistribFactory.createDistribByName()). They should also provide
implementations (at a minimum) for the following unimplemented functions:
createPackage()
packageCreated()
getDistIdForPackage()
installPackage()
getTaggedRelease()
writeTaggedRelease()
createDependencies()
updateDependencies()
writeManifest()
The DefaultDistrib subclass provides default implementations for the
last four functions.
OPTIONS:
The behavior of a Distrib class is fine-tuned via options (a dictionary
of named values) that are passed in at construction time. This base
implementation supports a core set; sub-classes usually support them as
well but are not required to. A sub-class may support additional options
as well. The core set supported here are:
noeups do not use the local EUPS database for information
while creating packages.
obeyGroups If true, then any newly written directory or files
will be set to groupWritable.
groupowner when obeyGroups is true, change the group owner of
to this value
buildDir a directory that may be used to build a package
during install. If this is a relative path, it
the full path (by default) will be relative to
the product root for the installation. (See
getBuildDirFor()).
useFlavor Create a flavor-specific installation (i.e. not "generic")
"""
NAME = None # sub-classes should provide a string value
PRUNE = False # True if manifests are complete, and there's no need for recursion
# to find all the needed products
def __init__(self, Eups, distServ, flavor=None, tag="current", options=None,
verbosity=0, log=sys.stderr):
"""create a Distrib instance.
A DistribServer instance (provided as distServ) is usually created
by calling 'eups.server.ServerConf.makeServer(packageBase)' where
packageBase the base URL for the server.
@param Eups the Eups controller instance to use
@param distServ the DistribServer object associated with the server
@param flavor the default platform type to assume when necessary.
If None, it will be set to Eups.flavor
@param tag the logical name of the release of packages to assume
(default: "current")
@param options a dictionary of named options that are used to fine-
tune the behavior of this Distrib class. See
discussion above for a description of the options
supported by this implementation; sub-classes may
support different ones.
"""
self.distServer = distServ
self.Eups = Eups
self.flavor = flavor
self.tag = tag
self.verbose = verbosity
self.log = log
if not self.flavor:
self.flavor = self.Eups.flavor
if options is None: options = {}
self.options = options
if not isinstance(self.options, dict):
raise RuntimeError("Non-dictionary passed to options parameter: " +
repr(self.options))
self.noeups = self.getOption("noeups", False)
if self.noeups in (False, "False", "false"):
self.noeups = False
elif self.noeups in (True, "True", "true"):
self.noeups = True
else:
raise RuntimeError("Unrecognised value of noeups: %s" % self.noeups)
self.buildDir = self.getOption('buildDir', 'EupsBuildDir')
# @staticmethod # requires python 2.4
def parseDistID(distID):
"""Return a valid package location if and only we recognize the
given distribution identifier
This implementation always returns None
"""
return None
parseDistID = staticmethod(parseDistID) # should work as of python 2.2
def checkInit(self, forserver=True):
"""Check that self is properly initialised; this matters for subclasses
with special needs"""
return True
def initServerTree(self, serverDir):
"""initialize the given directory to serve as a package distribution
tree.
@param serverDir the directory to initialize
"""
if not os.path.exists(serverDir):
os.makedirs(serverDir)
def createPackage(self, serverDir, product, version, flavor=None, overwrite=False):
"""Write a package distribution into server directory tree and
return the distribution ID. If a package is made up of several files,
all of them (except for the manifest) should be deployed by this
function. This includes the table file if it is not incorporated
another file.
@param serverDir a local directory representing the root of the
package distribution tree
@param product the name of the product to create the package
distribution for
@param version the name of the product version
@param flavor the flavor of the target platform; this may
be ignored by the implentation. None means
that a non-flavor-specific package is preferred,
if supported.
@param overwrite if True, this package will overwrite any
previously existing distribution files even if Eups.force is false
"""
self.unimplemented("createPackage");
def getDistIdForPackage(self, product, version, flavor=None):
"""return the distribution ID that for a package distribution created
by this Distrib class (via createPackage())
@param product the name of the product to create the package
distribution for
@param version the name of the product version
@param flavor the flavor of the target platform; this may
be ignored by the implentation. None means
that a non-flavor-specific ID is preferred,
if supported.
"""
self.unimplemented("getDistIdForPackage");
def packageCreated(self, serverDir, product, version, flavor=None):
"""return True if a distribution package for a given product has
apparently been deployed into the given server directory.
@param serverDir a local directory representing the root of the
package distribution tree
@param product the name of the product to create the package
distribution for
@param version the name of the product version
@param flavor the flavor of the target platform; this may
be ignored by the implentation. None means
that the status of a non-flavor-specific package
is of interest, if supported.
"""
self.unimplemented("packageCreated")
def installPackage(self, location, product, version, productRoot,
installDir=None, setups=None, buildDir=None):
"""Install a package with a given server location into a given
product directory tree.
@param location the location of the package on the server. This
value is a distribution ID (distID) that has
been stripped of its build type prefix.
@param product the name of the product installed by the package.
An implementation may ignore this parameter,
allowing the package to completely control
what is being installed.
@param version the name of the product version. Like the product
parameter, an implementatoin may ignore this
parameter.
@param productRoot the product directory tree under which the
product should be installed
@param installDir the preferred sub-directory under the productRoot
to install the directory. This value, which
should be a relative path name, may be
ignored or over-ridden by the implementation.
Implementations should be prepared that this
might be set to None or "none".
@param setups a list of EUPS setup commands that should be run
to properly build this package. This may be
ignored by the package.
@param buildDir a directory to use as a temporary building space.
An implementation should attempt to create this
directory if it needs to. Upon successful
completion, it may clean up the contents of
this directory, but it should not remove it.
If None (default), use the value of the
'buildDir' option.
"""
self.unimplemented("installPackage");
def cleanPackage(self, product, version, productRoot, location):
"""remove any distribution-specific remnants of a package installation.
Some distrib mechanisms (namely, Pacman) maintain some of their own
state about installed package that may need to get cleaned up if the
installation fails. Note, however, it is assumed okay to clean a
package once it has been declared to the EUPS system.
This implementation does nothing but return False. Subclasses should
override method if additional state is maintained. If an error
occurs, the appropriate exception should be raised.
@param product the name of the product to clean up after
@param version the version of the product
@param productRoot the product directory tree under which the
product is assumed to be installed
@param location the distribution location used to install the
package. The implementation may ignore this.
@returns bool True, if any state was cleaned up or False if nothing
needed to be done. Note that False is not an
error.
"""
return False
def createTaggedRelease(self, top_product, top_version, flavor=None,
tag=None):
"""create a list of products to include in a tagged release based on
what is considered current (or more precisely, what is associated with
the given tag name) in the local EUPS database and return it as a
TaggedProductList instance.
In this implementation, a list of dependencies for a top product is
generated; then each product dependency is recursively analyzed for
its dependencies.
@param top_product the product that determines via its dependencies
which products are included.
@param top_version the version of the top product.
@param flavor the target platform for the release (default:
"generic")
"""
release = TaggedProductList(tag, flavor, self.verbose-1, sys.log)
self._recurseProdDeps(release, top_product, top_version, flavor)
def _recurseProdDeps(self, release, product, version, flavor):
dependencies = self.createDependencies(product, version, flavor)
for dep in dependencies:
prevVer = release.getProductVersion()
if prevVer is None or \
VersionParser("%s < %s" % (prevVer, dep.version)).eval():
release.addProduct(dep.product, dep.version, flavor)
self._recurseProdDeps(release, dep.product, dep.version, flavor)
def getTaggedRelease(self, serverDir, tag, flavor=None):
"""get the collection of products that make up a tagged release and
return it as a TaggedProductList instance. If such a release has not
yet been created/written, return None. This is used for creating a
server (i.e. it does not contact a remote host for this info).
@param tag the name of the tagged release of interest
@param flavor the target flavor for this release. An
implementation may ignore this variable.
"""
self.unimplemented("getTaggedReleasePath")
def writeTaggedRelease(self, serverDir, tag, products, flavor=None,
force=False):
"""write a given tagged release into a server distribution tree
@param serverDir a local directory representing the root of the
package distribution tree
@param tag the name to give to the release
@param products a TaggedProductList instance containing the list
of products in the release
@param flavor the target flavor for this release. An
implementation may ignore this variable.
@param force if False, don't over-write file if it already
exists
"""
self.unimplemented("writeTaggedReleasePath")
def createDependencies(self, product, version, flavor=None, tag=None,
recursive=False):
"""create a list of product dependencies based on what is known from
the system.
This implementation will look up the product in the EUPS database and
analyze its table file.
A typical full implementation of this function in a sub-class would
call _createDeps() to get the initial list of dependencies, but then
iterate through the products, adding the name of the table file and
the distId.
@param product the name of the product to create the package
distribution for
@param version the name of the product version
@param flavor the flavor of the target platform; this may
be ignored by the implentation
@param tag where a dependency is loose, prefer the version
that is part of this tagged release. This
may be ignored by the implementation
@param recursive if False, this list will only contain the direct
dependencies of this product; otherwise, it
will include the dependencies of the dependencies
recursively. Default: False
"""
self.unimplemented("createDependencies")
def updateDependencies(self, productList, flavor=None, mapping=server.Mapping()):
"""fill in information in the list of product dependencies based
on what is known from the system
A typical full implementation of this function in a sub-class would
iterate through the products, adding the name of the table file and
the distId.
@param productList list of products (output from createDependencies)
@param flavor the flavor of the target platform; this may
be ignored by the implentation
@param mapping Mapping from desired product,version to existent product,version
"""
self.unimplemented("updateDependencies")
def _createDeps(self, productName, versionName, flavor=None, tag=None,
recursive=False, exact=False, mapping=server.Mapping()):
"""return a list of product dependencies for a given project. This
function returns a proto-dependency list providing only as much
generic information as is possible without knowing the details of
how the data is organized on the server. Thus, the dependency list
will not include
"""
if flavor is None: flavor = self.flavor
if tag is None: tag = self.tag
productList = Manifest(productName, versionName, self.Eups, self.verbose-1,
log=self.log)
# add a record for the top product
productList.addDependency(productName, versionName, flavor, None, None, None, False)
dependencies = None
tablefile = None
if self.noeups:
if recursive and self.verbose > 0:
print >> self.log, "Warning dependencies are not guaranteed", \
"to be recursive when using noeups option"
def getTableFile(product, version, flavor):
tablefile = self.findTableFile(product, version, flavor)
# use the server as source of package information
if not tablefile and self.distServer:
try:
tablefile = self.distServer.getTableFile(product, version, self.flavor)
except RemoteFileNotFound, e:
pass
return tablefile
tablefile = getTableFile(productName, versionName, self.flavor)
if not tablefile:
buildProduct, buildVersion = mapping.apply(productName, versionName, self.flavor)
if buildProduct == productName and buildVersion != versionName:
tablefile = getTableFile(productName, buildVersion, self.flavor)
if not tablefile and self.verbose > 0:
print >> self.log, "Failed to find %s's table file; trying eups" % productName
if tablefile:
dependencies = self.Eups.dependencies_from_table(tablefile)
else:
# consult the EUPS database
def getDependencies(productName, version):
try:
product = self.Eups.getProduct(productName, version)
dependencies = self.Eups.getDependentProducts(product, productDictionary={},
topological=True)
except:
return None
return dependencies
dependencies = getDependencies(productName, versionName)
if dependencies is None:
buildProduct, buildVersion = mapping.apply(productName, versionName, self.flavor)
if buildProduct == productName and buildVersion != versionName:
dependencies = getDependencies(productName, buildVersion)
if dependencies is None:
if self.noeups:
if self.verbose > 0:
print >> self.log, ("Unable to find dependencies for %s %s, assuming empty" %
(productName, versionName))
dependencies = []
else:
raise EupsException("Unable to determine dependencies for %s %s" %
(productName, versionName))
#
# Still no luck? If noeups we'll proceed without a tablefile
#
if self.noeups and dependencies is None:
if self.verbose > 0:
print >> self.log, "Unable to find a table file for %s; assuming no dependencies" % productName
dependencies = self.Eups.dependencies_from_table("none")
#
# We have our dependencies; proceed
#
# The first thing to do is to ensure that more deeply nested products are listed first as we need to
# build them first when installing
#
def byDepth(a, b):
"""Sort by recursion depth"""
return cmp(b[2], a[2])
dependencies.sort(byDepth)
for (dprod, dopt, recursionDepth) in dependencies:
dproductName = dprod.name
dversionName = dprod.version
product, vroReason = self.Eups.findProductFromVRO(dproductName, dversionName)
if product:
versionName = product.version
else:
if dopt:
continue
raise eups.ProductNotFound(dproductName, dversionName)
productList.addDependency(dproductName, versionName, flavor, None, None, None, dopt)
#
# We need to install those products in the correct order
#
productList.roll() # we put the top-level product at the start of the list
# now let's go back and fill in the product directory
for dprod in productList.getProducts():
if self.noeups and productName == dprod.product:
basedir, dprod.instDir = None, "/dev/null"
else:
(basedir, instDir) = self.getProductInstDir(dprod.product, dprod.version, dprod.flavor)
return productList
def _createDeps(self, productName, versionName, flavor=None, tag=None,
recursive=False, exact=False, mapping=server.Mapping()):
"""return a list of product dependencies for a given project. This
function returns a proto-dependency list providing only as much
generic information as is possible without knowing the details of
how the data is organized on the server. Thus, the dependency list
will not include
"""
if flavor is None: flavor = self.flavor
if tag is None: tag = self.tag
productList = server.Manifest(productName, versionName, self.Eups, self.verbose-1,
log=self.log)
# add a record for the top product
productList.addDependency(productName, versionName, flavor, None, None, None, False)
dependencies = None
tablefile = None
if self.noeups:
if recursive and self.verbose > 0:
print("Warning dependencies are not guaranteed", \
"to be recursive when using noeups option", file=self.log)
def getTableFile(product, version, flavor):
tablefile = self.findTableFile(product, version, flavor)
# use the server as source of package information
if not tablefile and self.distServer:
try:
tablefile = self.distServer.getTableFile(product, version, self.flavor)
except server.RemoteFileNotFound:
pass
return tablefile
tablefile = getTableFile(productName, versionName, self.flavor)
if not tablefile:
buildProduct, buildVersion = mapping.apply(productName, versionName, self.flavor)
if buildProduct == productName and buildVersion != versionName:
tablefile = getTableFile(productName, buildVersion, self.flavor)
if not tablefile and self.verbose > 0:
print("Failed to find %s's table file; trying eups" % productName, file=self.log)
if tablefile:
dependencies = self.Eups.dependencies_from_table(tablefile)
else:
# consult the EUPS database
def getDependencies(productName, version):
try:
product = self.Eups.getProduct(productName, version)
dependencies = self.Eups.getDependentProducts(product, productDictionary={},
topological=True)
except:
return None
return dependencies
dependencies = getDependencies(productName, versionName)
if dependencies is None:
buildProduct, buildVersion = mapping.apply(productName, versionName, self.flavor)
if buildProduct == productName and buildVersion != versionName:
dependencies = getDependencies(productName, buildVersion)
if dependencies is None:
if self.noeups:
if self.verbose > 0:
print(("Unable to find dependencies for %s %s, assuming empty" %
(productName, versionName)), file=self.log)
dependencies = []
else:
raise EupsException("Unable to determine dependencies for %s %s" %
(productName, versionName))
#
# Still no luck? If noeups we'll proceed without a tablefile
#
if self.noeups and dependencies is None:
if self.verbose > 0:
print("Unable to find a table file for %s; assuming no dependencies" % productName, file=self.log)
dependencies = self.Eups.dependencies_from_table("none")
#
# We have our dependencies; proceed
#
# The first thing to do is to ensure that more deeply nested products are listed first as we need to
# build them first when installing
#
def byDepth(a):
"""Sort by recursion depth"""
return -a[2]
dependencies.sort(key=byDepth)
for (dprod, dopt, recursionDepth) in dependencies:
dproductName = dprod.name
dversionName = dprod.version
product, vroReason = self.Eups.findProductFromVRO(dproductName, dversionName)
if product:
versionName = product.version
else:
if dopt:
continue
raise eups.ProductNotFound(dproductName, dversionName)
productList.addDependency(dproductName, versionName, flavor, None, None, None, dopt)
#
# We need to install those products in the correct order
#
productList.roll() # we put the top-level product at the start of the list
# now let's go back and fill in the product directory
for dprod in productList.getProducts():
if self.noeups and productName == dprod.product:
basedir, dprod.instDir = None, "/dev/null"
else:
basedir, dprod.instDir = self.getProductInstDir(dprod.product, dprod.version, dprod.flavor)
return productList