def __init__(self, pkgroots, options=None, eupsenv=None,
installFlavor=None, distribClasses=None, override=None, allowEmptyPkgroot=False,
verbosity=None, log=sys.stderr):
"""
@param pkgroots the base URLs for the distribution repositories. This
can either be a list or a pipe-delimited ("|")
string.
@param options a dictionary of named options that are used to fine-
tune the behavior of the repositories. These are
passed onto the constructors for the underlying
Reposistory classes.
@param eupsenv an instance of a Eups class containing the Eups
environment to assume
@param installFlavor the desired flavor any install requests
@param distribClasses a dictionary by name of the Distrib classes
to support. This will augmented by those specified
by a server.
@param override a dictionary of server configuration parameters that
should override the configuration received from
each server.
@param allowEmptyPkgroot we are creating a distribution, so it's OK for pkgroot to be empty
@param verbosity if > 0, print status messages; the higher the
number, the more messages that are printed
(default is the value of eupsenv.verbose).
@param log the destination for status messages (default:
sys.stderr)
"""
if isinstance(pkgroots, str):
pkgroots = map(lambda p: p.strip(), pkgroots.split("|"))
if not allowEmptyPkgroot and len(pkgroots) == 0:
raise EupsException("No package servers to query; set -r or $EUPS_PKGROOT")
# the Eups environment
self.eups = eupsenv
if not self.eups:
self.eups = Eups()
self.verbose = verbosity
if self.verbose is None:
self.verbose = self.eups.verbose
self.log = log
if self.log is None:
self.log = sys.stdout
if not distribClasses:
distribClasses = {}
# the list of repository base URLs
self.pkgroots = []
# a lookup of Repository instances by its base URL
self.repos = {}
# the preferred installation flavor
self.flavor = installFlavor
if not self.flavor:
self.flavor = self.eups.flavor
df = DistribFactory(self.eups)
for name in distribClasses.keys():
# note: this will override the server's recommendation
# if we want change this, use:
# if not df.supportsName(name):
# df.register(distribClasses[name], name)
#
df.register(distribClasses[name], name)
for pkgroot in pkgroots:
# if pkgroot == None:
# ds = None
# else:
# ds = ServerConf.makeServer(pkgroot, eupsenv=eupsenv,
# override=override,
# verbosity=self.eups.verbose)
#
try:
dist = Repository(self.eups, pkgroot, options=options,
flavor=installFlavor, distFactory=df,
verbosity=self.eups.verbose)
self.pkgroots += [pkgroot]
self.repos[pkgroot] = dist
except ImportError, e:
msg = "Unable to use server %s: \"%s\"" % (pkgroot, e)
if self.eups.force:
print >> self.log, msg + "; continuing"
else:
raise RuntimeError(msg + ". Remove server from PKGROOT or use force")
def __init__(self, eupsenv, pkgroot, flavor=None, options=None, distFactory=None, verbosity=0, log=sys.stderr):
"""
create a Repository for a given server base URL (pkgroot)
@param eupsenv the Eups controller instance to use
@param pkgroot the base URL for the package server to pull
packages from or deploy packages to.
@param flavor the platform flavor of interest.
#--CUT
When installing
packages, this value is ignored and the version
set in the Eups controller is assumed to be
the target platform. For all other actions
(creating server packages, listing available
packages), this value will be assumed.
#--
If
None or "generic", then a generic platform
is assumed.
@param options a dictionary of options to pass to Distrib
instances used to install and create packages
@param distFactory a DistFactory instance to use. If not provided
a default one is created.
@param verbosity if > 0, print status messages; the higher the
number, the more messages that are printed
(default=0).
@param log the destination for status messages (default:
sys.stderr)
"""
self.eups = eupsenv
if not flavor:
flavor = self.eups.flavor
self.flavor = flavor
self.distFactory = None
if distFactory:
self.distFactory = distFactory.clone()
self.options = options
self.verbose = verbosity
if not isinstance(self.verbose, int):
self.verbose = 0
self.log = log
self.distServer = None
if self.options is None:
self.options = {}
if not isinstance(self.options, dict):
raise RuntimeError("Non-dictionary passed to options parameter: " + repr(self.options))
self.pkgroot = pkgroot
if pkgroot:
override = None
if self.options.has_key("serverconf"):
override = options["serverconf"]
self.distServer = ServerConf.makeServer(
pkgroot, eupsenv=eupsenv, override=override, verbosity=self.verbose, log=self.log
)
if self.distFactory is None:
self.distFactory = DistribFactory(self.eups, self.distServer)
elif not self.distServer:
self.distFactory.resetDistribServer(self.distServer)
# a cache of the supported tag names
self._supportedTags = None
# a cache of supported packages
self._pkgList = None
# True if servers should always be queried when looking for a
# repository to get a package from. If False, an internal cache
# of available products will be used.
self._alwaysQueryServer = False
if self.options.has_key("alwaysQueryServer"):
if isinstance(self.options["alwaysQueryServer"], str):
self.options["alwaysQueryServer"] = self.options["alwaysQueryServer"].upper()
if "TRUE".startswith(self.options["alwaysQueryServer"]):
self.options["alwaysQueryServer"] = True
else:
self.options["alwaysQueryServer"] = False
if self.options["alwaysQueryServer"]:
self._alwaysQueryServer = True
if self.distServer is not None and self.distServer.NOCACHE:
self._alwaysQueryServer = True
def __init__(self, eupsenv, pkgroot, flavor=None, options=None,
distFactory=None, verbosity=0, log=sys.stderr):
"""
create a Repository for a given server base URL (pkgroot)
@param eupsenv the Eups controller instance to use
@param pkgroot the base URL for the package server to pull
packages from or deploy packages to.
@param flavor the platform flavor of interest.
#--CUT
When installing
packages, this value is ignored and the version
set in the Eups controller is assumed to be
the target platform. For all other actions
(creating server packages, listing available
packages), this value will be assumed.
#--
If
None or "generic", then a generic platform
is assumed.
@param options a dictionary of options to pass to Distrib
instances used to install and create packages
@param distFactory a DistFactory instance to use. If not provided
a default one is created.
@param verbosity if > 0, print status messages; the higher the
number, the more messages that are printed
(default=0).
@param log the destination for status messages (default:
sys.stderr)
"""
self.eups = eupsenv
if not flavor:
flavor = self.eups.flavor
self.flavor = flavor
self.distFactory = None
if distFactory:
self.distFactory = distFactory.clone()
self.options = options
self.verbose = verbosity
if not isinstance(self.verbose, int):
self.verbose = 0
self.log = log
self.distServer = None
if self.options is None:
self.options = {}
if not isinstance(self.options, dict):
raise RuntimeError("Non-dictionary passed to options parameter: " +
repr(self.options))
self.pkgroot = pkgroot
if pkgroot:
override = None
if self.options.has_key('serverconf'):
override = options['serverconf']
self.distServer = ServerConf.makeServer(pkgroot, eupsenv=eupsenv, override=override,
verbosity=self.verbose, log=self.log)
if self.distFactory is None:
self.distFactory = DistribFactory(self.eups, self.distServer)
elif not self.distServer:
self.distFactory.resetDistribServer(self.distServer)
# a cache of the supported tag names
self._supportedTags = None
# a cache of supported packages
self._pkgList = None
# True if servers should always be queried when looking for a
# repository to get a package from. If False, an internal cache
# of available products will be used.
self._alwaysQueryServer = False
if self.options.has_key("alwaysQueryServer"):
if isinstance(self.options["alwaysQueryServer"], str):
self.options["alwaysQueryServer"] = \
self.options["alwaysQueryServer"].upper()
if "TRUE".startswith(self.options["alwaysQueryServer"]):
self.options["alwaysQueryServer"] = True
else:
self.options["alwaysQueryServer"] = False
if self.options["alwaysQueryServer"]:
self._alwaysQueryServer = True
if self.distServer is not None and self.distServer.NOCACHE:
self._alwaysQueryServer = True
class Repository(object):
"""
an interface into a distribution server for handling package
installation and creation requests.
Applications should use install packages via the Repositories class
(via its install() function) because not only does it take responsibility
for installing a product's dependencies as well, it has the ability to
search across multiple servers to find a package. In contrast, a
Repository handles installation of a single product; this is accomplished
via a Distrib instance returned by getDistribFor().
If isWritable() returns True, this package can create
"""
def __init__(self, eupsenv, pkgroot, flavor=None, options=None, distFactory=None, verbosity=0, log=sys.stderr):
"""
create a Repository for a given server base URL (pkgroot)
@param eupsenv the Eups controller instance to use
@param pkgroot the base URL for the package server to pull
packages from or deploy packages to.
@param flavor the platform flavor of interest.
#--CUT
When installing
packages, this value is ignored and the version
set in the Eups controller is assumed to be
the target platform. For all other actions
(creating server packages, listing available
packages), this value will be assumed.
#--
If
None or "generic", then a generic platform
is assumed.
@param options a dictionary of options to pass to Distrib
instances used to install and create packages
@param distFactory a DistFactory instance to use. If not provided
a default one is created.
@param verbosity if > 0, print status messages; the higher the
number, the more messages that are printed
(default=0).
@param log the destination for status messages (default:
sys.stderr)
"""
self.eups = eupsenv
if not flavor:
flavor = self.eups.flavor
self.flavor = flavor
self.distFactory = None
if distFactory:
self.distFactory = distFactory.clone()
self.options = options
self.verbose = verbosity
if not isinstance(self.verbose, int):
self.verbose = 0
self.log = log
self.distServer = None
if self.options is None:
self.options = {}
if not isinstance(self.options, dict):
raise RuntimeError("Non-dictionary passed to options parameter: " + repr(self.options))
self.pkgroot = pkgroot
if pkgroot:
override = None
if self.options.has_key("serverconf"):
override = options["serverconf"]
self.distServer = ServerConf.makeServer(
pkgroot, eupsenv=eupsenv, override=override, verbosity=self.verbose, log=self.log
)
if self.distFactory is None:
self.distFactory = DistribFactory(self.eups, self.distServer)
elif not self.distServer:
self.distFactory.resetDistribServer(self.distServer)
# a cache of the supported tag names
self._supportedTags = None
# a cache of supported packages
self._pkgList = None
# True if servers should always be queried when looking for a
# repository to get a package from. If False, an internal cache
# of available products will be used.
self._alwaysQueryServer = False
if self.options.has_key("alwaysQueryServer"):
if isinstance(self.options["alwaysQueryServer"], str):
self.options["alwaysQueryServer"] = self.options["alwaysQueryServer"].upper()
if "TRUE".startswith(self.options["alwaysQueryServer"]):
self.options["alwaysQueryServer"] = True
else:
self.options["alwaysQueryServer"] = False
if self.options["alwaysQueryServer"]:
self._alwaysQueryServer = True
if self.distServer is not None and self.distServer.NOCACHE:
self._alwaysQueryServer = True
def _mergeOptions(self, override):
if self.options:
out = self.options.copy()
else:
out = {}
if isinstance(override, dict):
for key in override.keys():
out[key] = override[key]
if len(out.keys()) == 0:
return None
return out
def _getPackageLookup(self):
if not self.distServer:
return dict(_sortOrder=[])
# Look for both generic and flavor-specific packages
pkgs = self.distServer.listAvailableProducts(flavor=self.flavor)
if self.flavor != None:
for p in self.distServer.listAvailableProducts(flavor=None):
if not pkgs.count(p):
pkgs.append(p)
#
# arrange into a hierarchical lookup
#
lookup = {} # key is product == pkg[0]
for pkg in pkgs:
if not lookup.has_key(pkg[0]):
lookup[pkg[0]] = {} # key is flavor == pkg[2]
if not lookup[pkg[0]].has_key(pkg[2]):
lookup[pkg[0]][pkg[2]] = [] # list of versions == pkg[1]
lookup[pkg[0]][pkg[2]].append(pkg[1])
# now sort the contents
keys = lookup.keys()
keys.sort()
lookup["_sortOrder"] = keys
for prod in lookup["_sortOrder"]:
keys = filter(lambda f: f != "generic", lookup[prod].keys())
keys.sort()
if "generic" in lookup[prod].keys():
keys.insert(0, "generic")
lookup[prod]["_sortOrder"] = keys
for flav in lookup[prod]["_sortOrder"]:
lookup[prod][flav].sort(self.eups.version_cmp)
return lookup
def getTagNamesFor(self, product, version, flavor="generic", tags=None, noaction=False):
"""
return as a list of strings all of the tag names assigned to
the given product by the repository.
@param product the name of the product
@param version the product's version
@param flavor the platform flavor (default: generic)
@param tags if set, the returned list will be the intersection
of the tags assigned by the server with this list.
By providing this, one can remove the need to
query the server for its list of supported tags.
"""
return self.distServer.getTagNamesFor(product, version, flavor, tags)
def getManifest(self, product, version, flavor, noaction=False):
"""
request the manifest for a particular product and return it as
a Manifest instance.
@param product the desired product name
@param version the desired version of the product
@param flavor the flavor of the target platform
"""
return self.distServer.getManifest(product, version, flavor, noaction)
def findPackage(self, product, version=None, prefFlavors=None):
"""
if this repository can provide a requesed package, return a tuple
of (product, version, flavor) reflecting the exact version of the
product available. If it is not available, return None.
@param product the name of the product
@param version the desired version. This can either be a version
string or an instance of Tag. If None,
the tags preferred by the Eups environment will
be searched.
@param prefFlavors the preferred platform flavors in an ordered list.
A single flavor may be given as a string. If None,
flavors preferred by the Eups environment will
be searched.
"""
if prefFlavors is None:
prefFlavors = Flavor().getFallbackFlavors(self.flavor, True)
elif not isinstance(prefFlavors, list):
prefFlavors = [prefFlavors]
versions = [version]
supportedTags = self.getSupportedTags() + ["latest"]
if version and isinstance(version, Tag) and not version.isPseudo():
if not version.isGlobal():
raise TagNotRecognized(version.name, "global", msg='Non-global tag "%s" requested.' % version.name)
if not version:
versions = self.eups.getPreferredTags()
for vers in versions:
if isinstance(vers, Tag) and not vers.isPseudo() and vers.name not in supportedTags:
if self.verbose > 0:
print >>self.log, "Tag %s not supported at %s" % (vers.name, self.pkgroot)
continue
for flavor in prefFlavors:
try:
out = self.listPackages(product, version, flavor)
if out:
return out[0]
except TagNotRecognized:
pass
return None
def getSupportedTags(self):
"""
return a list of the tag names supported by this repository
"""
if self._supportedTags is None:
if self.distServer:
self._supportedTags = self.distServer.getTagNames()
else:
self._supportedTags = []
return self._supportedTags
def listPackages(self, product=None, version=None, flavor=None, tag=None, queryServer=None, noaction=False):
"""
return a list of available products on the server. Each item
in the list is a list of the form, (product, version, flavor).
The optional inputs will restrict the list to those matching the
values.
@param product the name of the product. If None, all available
products are returned.
@param version the desired version. This can either be a version
version string or an instance of Tag. If None,
all available versions are returned.
@param flavor the desired platform flavor. If None, all
available versions are returned.
@param tag list only products matching this tag. If version
is also specified, an empty list will be returned
if the version (or matching versions) is (are) not
assigned this tag.
@param queryServer if True, this will force a query to the repository
server. If False, an internal cache will be used
if possible. If None (default), the behavior is
controlled by the "alwaysQueryServer" passed to
the constructor of this Repository (which defaults
to False).
"""
if queryServer is None:
queryServer = self._alwaysQueryServer
if isinstance(version, Tag) and version.name != "latest":
tag = version
version = None
if queryServer or tag:
if self.distServer is None:
raise RuntimeError("No distribution server set")
if tag:
if isinstance(tag, Tag):
tagName = tag.name
else:
tagName = tag
tag = self.eups.tags.getTag(tag)
if not tag.isGlobal():
raise TagNotRecognized(tagName, "global", msg='Non-global tag "%s" requested.' % tagName)
if tagName == "latest":
return self._listLatestProducts(product, flavor)
if tagName not in self.getSupportedTags():
raise TagNotRecognized(tag, "global", msg="tag %s not supported by server" % tag)
return self.distServer.listAvailableProducts(product, version, flavor, tag)
else:
if self._pkgList is None:
self._pkgList = self._getPackageLookup()
out = []
prods = self._pkgList["_sortOrder"]
if product:
if not self._pkgList.has_key(product):
return []
prods = [product]
for prod in prods:
flavs = self._pkgList[prod]["_sortOrder"]
if flavor:
if not self._pkgList[prod].has_key(flavor):
continue
flavs = [flavor]
for flav in flavs:
if version is None:
out.extend(map(lambda v: (prod, v, flav), self._pkgList[prod][flav]))
elif version and isinstance(version, str):
if version not in self._pkgList[prod][flav]:
continue
out.append((prod, version, flav))
else:
# looking for latest
out.append((prod, self._pkgList[prod][flav][-1], flav))
return out
def _listLatestProducts(self, product, flavor):
prods = self.distServer.listAvailableProducts(product, None, flavor)
names = {}
flavors = {}
out = []
for p in prods:
names[p[0]] = 1
flavors[p[2]] = 1
names = names.keys()
names.sort()
flavors = filter(lambda f: f != "generic", flavors.keys())
flavors.sort()
flavors.insert(0, "generic")
for name in names:
for flav in flavors:
latest = filter(lambda p: p[0] == name and p[2] == flav, prods)
latest.sort(lambda a, b: self.eups.version_cmp(a[1], b[1]))
out.extend(latest)
return out
def getDistribFor(self, distId, options=None, flavor=None, tag=None):
"""
return a Distrib instance for a given package distribution identifier.
@param distId the distribution identifier.
@param options a set of options to pass to the Distrib instance
that fine-tunes its behavior. (See DistribFactory)
@param tag a default tag name to associate with the package.
This is normally only relevent for creating
packages for deployment on a server.
"""
if self.distServer is None:
raise RuntimeError("No distribution server set")
if options is not None and not isinstance(options, dict):
raise ValueError("Repository.getDistribFor(): options not a " + "dictionary")
if not flavor:
flavor = self.flavor
opts = self._mergeOptions(options)
return self.distFactory.createDistrib(distId, flavor, tag, opts, self.verbose, self.log)
def isWritable(self):
"""
return true if new packages, tag assignments, etc. can be added to
this repository.
This implementation returns True only if the repository is accessible
via local disk.
"""
return LocalTransporter.canHandle(self.pkgroot) and isDbWritable(self.pkgroot)
def create(
self,
distribTypeName,
product,
version,
tag=None,
nodepend=False,
options=None,
manifest=None,
packageId=None,
repositories=None,
):
"""create and all necessary files for making a particular package
available and deploy them into a local server directory. This creates
not only the requested product but also all of its dependencies unless
nodepends is True. Unless Eups.force is True, it will not recreate the
a package that is already deployed under the serverRoot directory.
If repositories is provided, then a dependency package is not deployed
if it is available from any of the repositories given.
@param distribTypeName the name of the distribution type to create.
The recognized names are those registered to the
DistribFactory passed to this Distribution's
constructor. Names recognized by default include
"builder", "pacman", and "tarball".
@param product the name of the product to create
@param version the version of the product to create
@param tag if not None, update (or create) the tagged release
list for this tag name. Only the information
for the given product will be added/updated.
(Default: True)
@param nodepend if True, only the requested product package will be
created. The product dependencies will be skipped.
(Default: False)
@param manifest an existing manifest filename; if provided, this
will be used as the list of dependencies to assume
for this product, rather than generating the
list from the EUPS database. The deployed manifest
will be a modified version in that the distIDs for
undeployed packages will be updated and, if
necessary, an entry will be added for the top
product (specified by the inputs to this function).
Note that this implementation will (unless
nodepend is True) consult the remote server to
determine if the package is available with the
given distID.
@param packageId name:version for distribution; default
product:version (either field may be omitted)
@param repositories if provided and nodepend=False, then dependency
products will not be deployed if they are
already deployed in any of the repositories
given.
"""
if not self.isWritable():
raise RuntimeError("Unable to create packages for this repository (Choose a local repository)")
opts = self._mergeOptions(options)
rebuildMapping = options.get("rebuildMapping", Mapping())
rebuildInverse = rebuildMapping.inverse()
rebuildProduct, rebuildVersion = rebuildMapping.apply(product, version)
try:
distrib = self.distFactory.createDistribByName(
distribTypeName, options=opts, flavor=self.flavor, verbosity=self.verbose
)
except KeyError:
distrib = None
if distrib is None:
raise RuntimeError(
'%s: Distrib name not recognized (known types are "%s")'
% (distribTypeName, '", "'.join(self.distFactory.lookup.keys()))
)
# load manifest data
if manifest is None:
# create it from what we (eups) know about it
man = distrib.createDependencies(
rebuildProduct, rebuildVersion, self.flavor, exact=opts["exact"], mapping=rebuildInverse
)
else:
# load it from the given file
man = Manifest.fromFile(manifest, self.eups, self.eups.verbose - 1)
man.remapEntries(mode="create", mapping=rebuildMapping)
distrib.updateDependencies(man.getProducts(), flavor=self.flavor, mapping=rebuildInverse)
# we will always overwrite the top package
created = {}
id = distrib.createPackage(self.pkgroot, rebuildProduct, rebuildVersion, self.flavor, overwrite=True)
created["%s-%s" % (rebuildProduct, rebuildVersion)] = man.getDependency(
rebuildProduct, version=rebuildVersion, flavor=self.flavor
)
if not nodepend:
self._recursiveCreate(distrib, man, created, True, repositories, mapping=rebuildMapping)
# update the manifest record for the requested product
dp = man.getDependency(rebuildProduct, rebuildVersion)
if dp is None:
# this product is not found in the manifest; this might happen if
# this function was passed a manifest file to use. Just in case,
# check the auto-generated manifest for a record and add it to
# the given one
tdp = distrib.createDependencies(rebuildProduct, rebuildVersion, self.flavor, mapping=rebuildInverse)
tdp = tdp.getDependency(rebuildProduct, rebuildVersion)
if tdp is not None:
man.getProducts().append(tdp)
else:
dp.distId = id
# deploy the manifest file
if packageId:
vals = packageId.split(":")
if len(vals) != 2:
raise RuntimeError, ('Expected package Id of form name:version, saw "%s"' % packageId)
if vals[0] == "":
vals[0] = product
if vals[1] == "":
vals[1] = version
packageName, packageVersion = vals
else:
packageName = product
packageVersion = version
# A final mapping to ensure everything's what's intended
man.remapEntries(mode="create", mapping=rebuildMapping)
distrib.updateDependencies(man.getProducts(), flavor=self.flavor, mapping=rebuildInverse)
packageName, packageVersion = rebuildMapping.apply(packageName, packageVersion, self.flavor)
distrib.writeManifest(
self.pkgroot, man.getProducts(), packageName, packageVersion, flavor=self.flavor, force=self.eups.force
)
def _recursiveCreate(self, distrib, manifest, created=None, recurse=True, repos=None, mapping=Mapping()):
if created is None:
created = {}
for pos, dp in enumerate(manifest.getProducts()):
pver = "%s-%s" % (dp.product, dp.version)
if created.has_key(pver):
if created[pver]:
manifest.getProducts()[pos] = created[pver]
continue
# check to see if it looks like it is available in the format
# given in the file
if distrib.parseDistID(dp.distId) is None:
# this may happen if create() was handed a manifest file
if self._availableAtLocation(dp):
continue
else:
flavor = dp.flavor
if dp.flavor == "generic":
flavor = None
if distrib.packageCreated(self.pkgroot, dp.product, dp.version, flavor):
if not self.eups.force:
if self.verbose > 0:
print >>self.log, "Dependency %s %s is already deployed; skipping" % (
dp.product,
dp.version,
)
created[pver] = dp
continue
elif self.verbose > 0:
print >>self.log, "Overwriting existing dependency,", dp.product, dp.version
#
# Check if this product is available elsewhere
#
if repos and not self.eups.force:
# look for the requested flavor
already_available = bool(repos.findPackage(dp.product, dp.version, dp.flavor))
if not already_available and dp.flavor != "generic":
already_available = bool(repos.findPackage(dp.product, dp.version, "generic"))
if already_available:
dp.distId = "search"
dp.tablefile = None
dp.instDir = None
created[pver] = dp
continue
# we now should attempt to create this package because it appears
# not to be available
try:
man = distrib.createDependencies(dp.product, dp.version, self.flavor, mapping=mapping.inverse())
man.remapEntries(mode="create", mapping=mapping)
distrib.updateDependencies(man.getProducts(), flavor=self.flavor, mapping=mapping.inverse())
except eups.EupsException, e:
raise RuntimeError(
"Creating manifest for %s:%s, dependency of %s %s: %s"
% (dp.product, dp.version, manifest.product, manifest.version, e)
)
id = distrib.createPackage(self.pkgroot, dp.product, dp.version, self.flavor)
created[pver] = dp
dp.distId = id
if recurse:
self._recursiveCreate(distrib, man, created, recurse, repos, mapping=mapping)
distrib.writeManifest(
self.pkgroot, man.getProducts(), dp.product, dp.version, flavor=self.flavor, force=self.eups.force
)
class Repository(object):
"""
an interface into a distribution server for handling package
installation and creation requests.
Applications should use install packages via the Repositories class
(via its install() function) because not only does it take responsibility
for installing a product's dependencies as well, it has the ability to
search across multiple servers to find a package. In contrast, a
Repository handles installation of a single product; this is accomplished
via a Distrib instance returned by getDistribFor().
If isWritable() returns True, this package can create
"""
def __init__(self, eupsenv, pkgroot, flavor=None, options=None,
distFactory=None, verbosity=0, log=sys.stderr):
"""
create a Repository for a given server base URL (pkgroot)
@param eupsenv the Eups controller instance to use
@param pkgroot the base URL for the package server to pull
packages from or deploy packages to.
@param flavor the platform flavor of interest.
#--CUT
When installing
packages, this value is ignored and the version
set in the Eups controller is assumed to be
the target platform. For all other actions
(creating server packages, listing available
packages), this value will be assumed.
#--
If
None or "generic", then a generic platform
is assumed.
@param options a dictionary of options to pass to Distrib
instances used to install and create packages
@param distFactory a DistFactory instance to use. If not provided
a default one is created.
@param verbosity if > 0, print status messages; the higher the
number, the more messages that are printed
(default=0).
@param log the destination for status messages (default:
sys.stderr)
"""
self.eups = eupsenv
if not flavor:
flavor = self.eups.flavor
self.flavor = flavor
self.distFactory = None
if distFactory:
self.distFactory = distFactory.clone()
self.options = options
self.verbose = verbosity
if not isinstance(self.verbose, int):
self.verbose = 0
self.log = log
self.distServer = None
if self.options is None:
self.options = {}
if not isinstance(self.options, dict):
raise RuntimeError("Non-dictionary passed to options parameter: " +
repr(self.options))
self.pkgroot = pkgroot
if pkgroot:
override = None
if self.options.has_key('serverconf'):
override = options['serverconf']
self.distServer = ServerConf.makeServer(pkgroot, eupsenv=eupsenv, override=override,
verbosity=self.verbose, log=self.log)
if self.distFactory is None:
self.distFactory = DistribFactory(self.eups, self.distServer)
elif not self.distServer:
self.distFactory.resetDistribServer(self.distServer)
# a cache of the supported tag names
self._supportedTags = None
# a cache of supported packages
self._pkgList = None
# True if servers should always be queried when looking for a
# repository to get a package from. If False, an internal cache
# of available products will be used.
self._alwaysQueryServer = False
if self.options.has_key("alwaysQueryServer"):
if isinstance(self.options["alwaysQueryServer"], str):
self.options["alwaysQueryServer"] = \
self.options["alwaysQueryServer"].upper()
if "TRUE".startswith(self.options["alwaysQueryServer"]):
self.options["alwaysQueryServer"] = True
else:
self.options["alwaysQueryServer"] = False
if self.options["alwaysQueryServer"]:
self._alwaysQueryServer = True
if self.distServer is not None and self.distServer.NOCACHE:
self._alwaysQueryServer = True
def _mergeOptions(self, override):
if self.options:
out = self.options.copy()
else:
out = {}
if isinstance(override, dict):
for key in override.keys():
out[key] = override[key]
if len(out.keys()) == 0:
return None
return out
def _getPackageLookup(self):
if not self.distServer:
return dict(_sortOrder=[])
# Look for both generic and flavor-specific packages
pkgs = self.distServer.listAvailableProducts(flavor=self.flavor)
if self.flavor != None:
for p in self.distServer.listAvailableProducts(flavor=None):
if not pkgs.count(p):
pkgs.append(p)
#
# arrange into a hierarchical lookup
#
lookup = {} # key is product == pkg[0]
for pkg in pkgs:
if not lookup.has_key(pkg[0]):
lookup[pkg[0]] = {} # key is flavor == pkg[2]
if not lookup[pkg[0]].has_key(pkg[2]):
lookup[pkg[0]][pkg[2]] = [] # list of versions == pkg[1]
lookup[pkg[0]][pkg[2]].append(pkg[1])
# now sort the contents
keys = lookup.keys()
keys.sort()
lookup["_sortOrder"] = keys
for prod in lookup["_sortOrder"]:
keys = filter(lambda f: f != "generic", lookup[prod].keys())
keys.sort()
if "generic" in lookup[prod].keys():
keys.insert(0, "generic")
lookup[prod]["_sortOrder"] = keys
for flav in lookup[prod]["_sortOrder"]:
lookup[prod][flav].sort(self.eups.version_cmp)
return lookup
def getTagNamesFor(self, product, version, flavor="generic",
tags=None, noaction=False):
"""
return as a list of strings all of the tag names assigned to
the given product by the repository.
@param product the name of the product
@param version the product's version
@param flavor the platform flavor (default: generic)
@param tags if set, the returned list will be the intersection
of the tags assigned by the server with this list.
By providing this, one can remove the need to
query the server for its list of supported tags.
"""
return self.distServer.getTagNamesFor(product, version, flavor, tags)
def getManifest(self, product, version, flavor, noaction=False):
"""
request the manifest for a particular product and return it as
a Manifest instance.
@param product the desired product name
@param version the desired version of the product
@param flavor the flavor of the target platform
"""
return self.distServer.getManifest(product, version, flavor, noaction)
def findPackage(self, product, version=None, prefFlavors=None):
"""
if this repository can provide a requesed package, return a tuple
of (product, version, flavor) reflecting the exact version of the
product available. If it is not available, return None.
@param product the name of the product
@param version the desired version. This can either be a version
string or an instance of Tag. If None,
the tags preferred by the Eups environment will
be searched.
@param prefFlavors the preferred platform flavors in an ordered list.
A single flavor may be given as a string. If None,
flavors preferred by the Eups environment will
be searched.
"""
if prefFlavors is None:
prefFlavors = Flavor().getFallbackFlavors(self.flavor, True)
elif not isinstance(prefFlavors, list):
prefFlavors = [prefFlavors]
versions = [version]
supportedTags = self.getSupportedTags() + ["latest"]
if version and isinstance(version, Tag) and not version.isPseudo():
if not version.isGlobal():
raise TagNotRecognized(version.name, "global",
msg="Non-global tag \"%s\" requested." % version.name)
if not version:
versions = self.eups.getPreferredTags()
for vers in versions:
if isinstance(vers, Tag) and not vers.isPseudo() and \
vers.name not in supportedTags:
if self.verbose > 0:
print >> self.log, \
"Tag %s not supported at %s" % (vers.name, self.pkgroot)
continue
for flavor in prefFlavors:
try:
out = self.listPackages(product, version, flavor)
if out: return out[0]
except TagNotRecognized:
pass
return None
def getSupportedTags(self):
"""
return a list of the tag names supported by this repository
"""
if self._supportedTags is None:
if self.distServer:
self._supportedTags = self.distServer.getTagNames()
else:
self._supportedTags = []
return self._supportedTags
def listPackages(self, product=None, version=None, flavor=None, tag=None,
queryServer=None, noaction=False):
"""
return a list of available products on the server. Each item
in the list is a list of the form, (product, version, flavor).
The optional inputs will restrict the list to those matching the
values.
@param product the name of the product. If None, all available
products are returned.
@param version the desired version. This can either be a version
version string or an instance of Tag. If None,
all available versions are returned.
@param flavor the desired platform flavor. If None, all
available versions are returned.
@param tag list only products matching this tag. If version
is also specified, an empty list will be returned
if the version (or matching versions) is (are) not
assigned this tag.
@param queryServer if True, this will force a query to the repository
server. If False, an internal cache will be used
if possible. If None (default), the behavior is
controlled by the "alwaysQueryServer" passed to
the constructor of this Repository (which defaults
to False).
"""
if queryServer is None:
queryServer = self._alwaysQueryServer
if isinstance(version, Tag) and version.name != "latest":
tag = version
version = None
if queryServer or tag:
if self.distServer is None:
raise RuntimeError("No distribution server set")
if tag:
if isinstance(tag, Tag):
tagName = tag.name
else:
tagName = tag
tag = self.eups.tags.getTag(tag)
if not tag.isGlobal():
raise TagNotRecognized(tagName, "global",
msg="Non-global tag \"%s\" requested." % tagName)
if tagName == "latest":
return self._listLatestProducts(product, flavor)
if tagName not in self.getSupportedTags():
raise TagNotRecognized(tag, "global",
msg="tag %s not supported by server" % tag)
return self.distServer.listAvailableProducts(product, version, flavor, tag)
else:
if self._pkgList is None:
self._pkgList = self._getPackageLookup()
out = []
prods = self._pkgList["_sortOrder"]
if product:
if not self._pkgList.has_key(product):
return []
prods = [product]
for prod in prods:
flavs = self._pkgList[prod]["_sortOrder"]
if flavor:
if not self._pkgList[prod].has_key(flavor):
continue
flavs = [flavor]
for flav in flavs:
if version is None:
out.extend( map(lambda v: (prod, v, flav),
self._pkgList[prod][flav]) )
elif version and isinstance(version, str):
if version not in self._pkgList[prod][flav]:
continue
out.append( (prod, version, flav) )
else:
# looking for latest
out.append((prod,self._pkgList[prod][flav][-1],flav))
return out
def _listLatestProducts(self, product, flavor):
prods = self.distServer.listAvailableProducts(product, None, flavor)
names = {}
flavors = {}
out = []
for p in prods:
names[p[0]] = 1
flavors[p[2]] = 1
names = names.keys()
names.sort()
flavors = filter(lambda f: f != "generic", flavors.keys())
flavors.sort()
flavors.insert(0, "generic")
for name in names:
for flav in flavors:
latest = filter(lambda p: p[0] == name and p[2] == flav, prods)
latest.sort(lambda a,b: self.eups.version_cmp(a[1],b[1]))
out.extend(latest)
return out
def getDistribFor(self, distId, options=None, flavor=None, tag=None):
"""
return a Distrib instance for a given package distribution identifier.
@param distId the distribution identifier.
@param options a set of options to pass to the Distrib instance
that fine-tunes its behavior. (See DistribFactory)
@param tag a default tag name to associate with the package.
This is normally only relevent for creating
packages for deployment on a server.
"""
if self.distServer is None:
raise RuntimeError("No distribution server set")
if options is not None and not isinstance(options, dict):
raise ValueError("Repository.getDistribFor(): options not a " +
"dictionary")
if not flavor:
flavor = self.flavor
opts = self._mergeOptions(options)
return self.distFactory.createDistrib(distId, flavor, tag, opts,
self.verbose, self.log)
def isWritable(self):
"""
return true if new packages, tag assignments, etc. can be added to
this repository.
This implementation returns True only if the repository is accessible
via local disk.
"""
return (LocalTransporter.canHandle(self.pkgroot) and
isDbWritable(self.pkgroot))
def create(self, distribTypeName, product, version, tag=None,
nodepend=False, options=None, manifest=None, packageId=None,
repositories=None):
"""create and all necessary files for making a particular package
available and deploy them into a local server directory. This creates
not only the requested product but also all of its dependencies unless
nodepends is True. Unless Eups.force is True, it will not recreate the
a package that is already deployed under the serverRoot directory.
If repositories is provided, then a dependency package is not deployed
if it is available from any of the repositories given.
@param distribTypeName the name of the distribution type to create.
The recognized names are those registered to the
DistribFactory passed to this Distribution's
constructor. Names recognized by default include
"builder", "pacman", and "tarball".
@param product the name of the product to create
@param version the version of the product to create
@param tag if not None, update (or create) the tagged release
list for this tag name. Only the information
for the given product will be added/updated.
(Default: True)
@param nodepend if True, only the requested product package will be
created. The product dependencies will be skipped.
(Default: False)
@param manifest an existing manifest filename; if provided, this
will be used as the list of dependencies to assume
for this product, rather than generating the
list from the EUPS database. The deployed manifest
will be a modified version in that the distIDs for
undeployed packages will be updated and, if
necessary, an entry will be added for the top
product (specified by the inputs to this function).
Note that this implementation will (unless
nodepend is True) consult the remote server to
determine if the package is available with the
given distID.
@param packageId name:version for distribution; default
product:version (either field may be omitted)
@param repositories if provided and nodepend=False, then dependency
products will not be deployed if they are
already deployed in any of the repositories
given.
"""
if not self.isWritable():
raise RuntimeError("Unable to create packages for this repository (Choose a local repository)")
opts = self._mergeOptions(options)
rebuildMapping = options.get("rebuildMapping", Mapping())
rebuildInverse = rebuildMapping.inverse()
rebuildProduct, rebuildVersion = rebuildMapping.apply(product, version)
try:
distrib = self.distFactory.createDistribByName(distribTypeName, options=opts,
flavor=self.flavor, verbosity=self.verbose)
except KeyError:
distrib = None
if distrib is None:
raise RuntimeError("%s: Distrib name not recognized (known types are \"%s\")" %
(distribTypeName, '", "'.join(self.distFactory.lookup.keys())))
# load manifest data
if manifest is None:
# create it from what we (eups) know about it
man = distrib.createDependencies(rebuildProduct, rebuildVersion, self.flavor, exact=opts["exact"],
mapping=rebuildInverse)
else:
# load it from the given file
man = Manifest.fromFile(manifest, self.eups, self.eups.verbose-1)
man.remapEntries(mode="create", mapping=rebuildMapping)
distrib.updateDependencies(man.getProducts(), flavor=self.flavor, mapping=rebuildInverse)
# we will always overwrite the top package
created = {}
id = distrib.createPackage(self.pkgroot, rebuildProduct, rebuildVersion, self.flavor, overwrite=True)
created["%s-%s" % (rebuildProduct, rebuildVersion)] = man.getDependency(rebuildProduct,
version=rebuildVersion,
flavor=self.flavor)
if not nodepend:
self._recursiveCreate(distrib, man, created, True, repositories, mapping=rebuildMapping)
# update the manifest record for the requested product
dp = man.getDependency(rebuildProduct, rebuildVersion)
if dp is None:
# this product is not found in the manifest; this might happen if
# this function was passed a manifest file to use. Just in case,
# check the auto-generated manifest for a record and add it to
# the given one
tdp = distrib.createDependencies(rebuildProduct, rebuildVersion, self.flavor,
mapping=rebuildInverse)
tdp = tdp.getDependency(rebuildProduct, rebuildVersion)
if tdp is not None:
man.getProducts().append(tdp)
else:
dp.distId = id
# deploy the manifest file
if packageId:
vals = packageId.split(":")
if len(vals) != 2:
raise RuntimeError, ("Expected package Id of form name:version, saw \"%s\"" % packageId)
if vals[0] == "":
vals[0] = product
if vals[1] == "":
vals[1] = version
packageName, packageVersion = vals
else:
packageName = product
packageVersion = version
# A final mapping to ensure everything's what's intended
man.remapEntries(mode="create", mapping=rebuildMapping)
distrib.updateDependencies(man.getProducts(), flavor=self.flavor, mapping=rebuildInverse)
packageName, packageVersion = rebuildMapping.apply(packageName, packageVersion, self.flavor)
distrib.writeManifest(self.pkgroot, man.getProducts(), packageName, packageVersion,
flavor=self.flavor, force=self.eups.force)
def _recursiveCreate(self, distrib, manifest, created=None, recurse=True, repos=None, mapping=Mapping()):
if created is None:
created = {}
for pos, dp in enumerate(manifest.getProducts()):
pver = "%s-%s" % (dp.product, dp.version)
if created.has_key(pver):
if created[pver]:
manifest.getProducts()[pos] = created[pver]
continue
# check to see if it looks like it is available in the format
# given in the file
if distrib.parseDistID(dp.distId) is None:
# this may happen if create() was handed a manifest file
if self._availableAtLocation(dp):
continue
else:
flavor = dp.flavor
if dp.flavor == "generic": flavor = None
if distrib.packageCreated(self.pkgroot, dp.product, dp.version, flavor):
if not self.eups.force:
if self.verbose > 0:
print >> self.log, "Dependency %s %s is already deployed; skipping" % \
(dp.product, dp.version)
created[pver] = dp
continue
elif self.verbose > 0:
print >> self.log, "Overwriting existing dependency,", dp.product, dp.version
#
# Check if this product is available elsewhere
#
if repos and not self.eups.force:
# look for the requested flavor
already_available = bool(repos.findPackage(dp.product, dp.version, dp.flavor))
if not already_available and dp.flavor != "generic":
already_available = bool(repos.findPackage(dp.product, dp.version, "generic"))
if already_available:
dp.distId = "search"
dp.tablefile = None
dp.instDir = None
created[pver] = dp
continue
# we now should attempt to create this package because it appears
# not to be available
try:
man = distrib.createDependencies(dp.product, dp.version, self.flavor,
mapping=mapping.inverse())
man.remapEntries(mode="create", mapping=mapping)
distrib.updateDependencies(man.getProducts(), flavor=self.flavor, mapping=mapping.inverse())
except eups.EupsException, e:
raise RuntimeError("Creating manifest for %s:%s, dependency of %s %s: %s" %
(dp.product, dp.version, manifest.product, manifest.version, e))
id = distrib.createPackage(self.pkgroot, dp.product, dp.version, self.flavor)
created[pver] = dp
dp.distId = id
if recurse:
self._recursiveCreate(distrib, man, created, recurse, repos, mapping=mapping)
distrib.writeManifest(self.pkgroot, man.getProducts(), dp.product, dp.version,
flavor=self.flavor, force=self.eups.force)
