def createBranch(self,
project,
name,
platformLabel,
namespace=None,
description=''):
project = self.getProject(project)
if project.project_branches and \
name in [b.name for b in project.project_branches]:
raise errors.RbuildError("Branch named '%s' already exists" % name)
doc = xobj.Document()
doc.project_branch = br = xobj.XObj()
br.project = xobj.XObj()
# Why?
for key in ('id', 'name', 'short_name', 'domain_name'):
setattr(br.project, key, getattr(project, key))
br.name = name
br.platform_label = unicode(platformLabel)
br.description = description
if namespace:
br.namespace = namespace
try:
br = project.project_branches.append(doc)
except robj.errors.HTTPConflictError:
raise errors.RbuildError("Branch named '%s' already exists" % name)
return br.label
def getImages(self,
name,
project=None,
branch=None,
stage=None,
trailingVersion=None):
'''
Get an image by name. project limits search to that project.
branch limits search to that branch. stage limits serach to that stage.
trailingVersion limits the image to ones built with that group
trailingVersion. If the query returns multiple images, return the most
recently created.
@param name: name of image
@type name: str
@param shortName: short name of project
@type shortName: str
@param stageName: name of stage, requires shortName
@type stageName: str
@param trailingVersion: trailing version of product group
@type trailingVersion: str or None
@return: image or None
@rtype: rObj(image)
'''
# we only support stage filter if we also got a project and branch
if stage and not (project and branch):
raise errors.RbuildError(
'Must provide project and branch if stage is provided')
client = self.api._client
uri = self.api._uri + '/images'
# filter by image name
filter_by = ';filter_by=AND(EQUAL(name,%s)' % name
if project:
filter_by += ',EQUAL(project.short_name,%s)' % project
if stage:
filter_by += ',EQUAL(project_branch.label,%s)' % branch
filter_by += ',EQUAL(stage_name,%s)' % stage
if trailingVersion:
# filter by group trailing version
filter_by += ',EQUAL(trailing_version,%s)' % trailingVersion
uri += filter_by + ')'
# always sort by most recently created first
uri += ';order_by=-time_created'
try:
return client.do_GET(uri)
except robj.errors.HTTPNotFoundError:
raise errors.RbuildError("Unable to fetch images at '%s'" + uri)
def getImageTypeDef(self, product, version, imageType, arch):
client = self.api._client
uri = ('/products/%s/versions/%s/imageTypeDefinitions' %
(product, version))
try:
for imageTypeDef in client.do_GET(uri):
if (imageTypeDef.container.name == imageType
and imageTypeDef.architecture.name == arch):
return imageTypeDef
except robj.errors.HTTPNotFoundError:
raise errors.RbuildError(
"Project '%s' and version '%s' not found" % (product, version))
raise errors.RbuildError("No image type definition with name '%s'"
" and architecture '%s'" % (imageType, arch))
def __init__(self, handle=None, baseDirectory=None):
ProductStore.__init__(self, handle)
productDirectory = getDefaultProductDirectory(baseDirectory)
if productDirectory is None:
if baseDirectory is None:
raise errors.RbuildError('Could not find product directory')
else:
raise errors.RbuildError("No product directory at '%s'" %
baseDirectory)
self._baseDirectory = os.path.realpath(productDirectory)
self._testProductDirectory(self._baseDirectory)
stageName = getStageNameFromDirectory(os.getcwd())
if stageName is not None:
# Cannot load product yet, so cannot validate
self._currentStage = stageName
self.statusStore = None
def getProjectBranches(self, *args, **kwargs):
'''
Get a list of branches for a project
'''
try:
project = kwargs.pop('project', args[0])
except IndexError:
raise TypeError("Required argument 'project' (pos 1) noti found")
order_by = kwargs.pop('order_by', None)
# name might be a label
shortName, _, domainName = project.partition('.')
project = self.getProject(shortName)
# if the user gave us full repo hostname make
# sure they gave us the right one
if (domainName and project.domain_name != domainName):
raise errors.RbuildError("could not find a project matching '%s'" %
'.'.join((shortName, domainName)))
if order_by:
reverse = (order_by[0] == '-')
if order_by[0] in ('-', '+'):
order_by = order_by[1:]
return sorted(
project.project_branches,
key=lambda b: getattr(b, order_by),
reverse=reverse,
)
else:
return project.project_branches
def createProject(self,
title,
shortName,
hostName=None,
domainName=None,
description='',
external=False,
external_params=None):
assert ((external and external_params is not None)
or (not external and external_params is None))
doc = xobj.Document()
doc.project = proj = xobj.XObj()
proj.name = title
proj.short_name = shortName
proj.hostname = hostName or shortName
proj.description = description or ''
if domainName:
proj.domain_name = domainName
if external:
proj.external = 'true'
(proj.labels, proj.upstream_url, proj.auth_type, proj.user_name,
proj.password, proj.entitlement) = external_params
else:
proj.external = 'false'
try:
return self.api.projects.append(doc).project_id
except robj.errors.HTTPConflictError:
raise errors.RbuildError("A project with conflicting "
"parameters already exists")
def createTarget(self, ttype, ddata):
'''
Create a target using the descriptor data provided
@param ttype: target type
@type ttype: string
@param ddata: descriptor data for target
@type: DescriptorData
@return: the created Target
@rtype: robj.HTTPData
'''
# Construct the target xml
target_doc = xobj.Document()
target_doc.target = target = xobj.XObj()
target.description = ddata.getField('description')
target.name = ddata.getField('name')
target.zone_name = ddata.getField('zone')
target.target_type_name = ttype
try:
target = self.api.targets.append(target_doc, tag='target')
except robj.errors.HTTPConflictError:
raise errors.RbuildError(
"A target with conflicting parameters already exists")
return target
def configureTargetCredentials(self, target, ddata):
'''
Configure credentials for a target
@param ddata: descriptor for target
@type ddata: DescriptorData
@param target: target to configure
@type target: rObj(target)
@return: the configured target
@rtype: rObj(target)
'''
# make sure our target object is up to date
target.refresh()
doc = xobj.Document()
doc.job = job = xobj.XObj()
job.job_type = target.actions[1]._root.job_type
job.descriptor = target.actions[1]._root.descriptor
job.descriptor_data = xobj.parse(ddata.toxml()).descriptor_data
jobObj = target.jobs.append(doc)
while jobObj.job_state.name in ['Queued', 'Running']:
jobObj.refresh()
if jobObj.job_state.name == 'Failed':
raise errors.RbuildError('Unable to set credentials')
return target
def commit(self, message):
"""
Commit a product definition change
"""
if not self._handle.facade.conary.commit(
self.getProductDefinitionDirectory(), message=message):
raise errors.RbuildError('Failed to commit product definition')
def getBranchIdFromName(self, productName, versionName):
#pylint: disable-msg=R0914
# not a great candidate for refactoring
productId = self.getProductId(productName)
error, versionList = self.server.getProductVersionListForProduct(
productId)
if error:
raise errors.RbuilderError(*versionList)
versionNames = []
# W0612: leave unused variables as documentation
# W0631: versionId is guaranteed to be defined
#pylint: disable-msg=W0612,W0631
for (versionId2, productId2, namespace, versionName2,
desc) in versionList:
versionNames.append(versionName2)
if versionName == versionName2:
return versionId2
errstr = '%s is not a valid version for product %s.' % \
(versionName, productName)
if versionNames:
errstr += '\nValid versions are: %s' % \
', '.join(versionNames)
else:
errstr += '\nNo versions found for product %s.' % productName
raise errors.RbuildError(errstr)
def startProductBuilds(self,
productName,
versionName,
stageName,
force=False):
productId = self.getProductId(productName)
error, versionList = self.server.getProductVersionListForProduct(
productId)
if error:
raise errors.RbuilderError(*versionList)
versionId = None
# W0612: leave unused variables as documentation
# W0631: versionId is guaranteed to be defined
#pylint: disable-msg=W0612,W0631
if versionList:
if len(versionList[0]) == 4:
#This is an older rBuilder
for (versionId2, productId2, versionName2,
desc) in versionList:
if versionName == versionName2:
versionId = versionId2
break
else:
for (versionId2, productId2, namespace, versionName2,
desc) in versionList:
if versionName == versionName2:
versionId = versionId2
break
if versionId is None:
raise errors.RbuildError(
"could not find version %r for product %r" %
(versionName, productName))
error, buildIds = self.server.newBuildsFromProductDefinition(
versionId, stageName, force)
if error:
if buildIds[0] == 'TroveNotFoundForBuildDefinition':
errFlavors = '\n'.join(buildIds[1][0])
raise errors.RbuildError(
'%s\n\nTo submit the partial set of '
'builds, re-run this command with --force' % errFlavors)
else:
raise errors.RbuilderError(*buildIds)
return buildIds
def getTarget(self, name):
client = self.api._client
uri = self.api._uri + '/targets'
uri += ';filter_by=[name,EQUAL,%s]' % (name, )
targets = client.do_GET(uri)
if targets:
return targets[0]
raise errors.RbuildError("Target '%s' not found" % (name, ))
def update(self):
"""
This is the only acceptable way to update a product definition
checkout, because it invalidates the cached copy of the data.
"""
if not self._handle.facade.conary.updateCheckout(
self.getProductDefinitionDirectory()):
raise errors.RbuildError('Failed to update product definition')
ProductStore.update(self)
def getProductId(self, productName):
error, productId = self.server.getProjectIdByHostname(productName)
if error:
if productId[0] == 'ItemNotFound':
raise errors.RbuildError('Product %s not found' % \
productName)
else:
raise errors.RbuilderError(*productId)
return productId
def watchImages(self, buildIds, timeout=0, interval=5, quiet=False):
interval = 10
st = time.time()
timedOut = False
dropped = 0
finalStatus = {}
activeBuilds = dict.fromkeys(buildIds)
while activeBuilds:
for buildId in list(activeBuilds):
try:
error, buildStatus = self.server.getBuildStatus(buildId)
except socket.timeout:
dropped += 1
if dropped >= 3:
raise errors.RbuildError(
'rBuilder connection timed out after 3 attempts')
self._handle.ui.info(
'Status request timed out, trying again')
time.sleep(interval)
continue
if error:
raise errors.RbuilderError(*buildStatus)
dropped = 0
if activeBuilds[buildId] != buildStatus:
st = time.time() # reset timeout counter if status changes
activeBuilds[buildId] = buildStatus
if not quiet:
self._handle.ui.write(
'%s: %s "%s"', buildId,
self.statusNames.get(buildStatus['status'],
self.statusNames[-1]),
buildStatus['message'])
if activeBuilds[buildId]['status'] > 200:
finalStatus[buildId] = activeBuilds.pop(buildId)
if activeBuilds:
time.sleep(interval)
if timeout and time.time() - st > timeout:
timedOut = True
break
if timedOut:
self._handle.ui.error(
'Timed out while waiting for build status'
' to change (%d seconds)', timeout)
else:
self._handle.ui.write('All jobs completed')
if activeBuilds:
self._handle.ui.warning('Unfinished builds:')
self._printStatus(activeBuilds, ' Last status: ')
self._handle.ui.write('Finished builds:')
self._printStatus(finalStatus, ' ')
if any(x['status'] != 300 for x in finalStatus.values()):
return False
else:
return True
def getProject(self, shortName):
# FIXME: robj allows neither URL construction nor searching/filtering,
# so the only "kosher" way to find a project is to iterate over all of
# them. So cheating looks attractive by comparison...
client = self.api._client
uri = self.api._uri + '/projects/' + shortName
try:
return client.do_GET(uri)
except robj.errors.HTTPNotFoundError:
raise errors.RbuildError("Project '%s' not found" % (shortName, ))
def api(self):
if self._api is None:
top = robj.connect(self._url)
for ver in top.api_versions:
if ver.name == 'v1':
break
else:
raise errors.RbuildError("No compatible REST API found on "
"rBuilder '%s'" %
self._url.__safe_str__())
self._api = ver
return self._api
def getGroups(self, shortName, label, **kwargs):
client = self.api._client
uri = ('/products/%s/repos/search?type=group&label=%s' %
(shortName, label))
try:
groups = client.do_GET(uri)
if groups:
return groups
return []
except robj.errors.HTTPNotFoundError:
raise errors.RbuildError("Project '%s' and label '%s' not found" %
(shortName, label))
def _testProductDirectory(baseDirectory):
"""
Test to see whether a product directory has been checked out
@param baseDirectory: name of product directory
@type baseDirectory: string
@raise errors.RbuildError: If no product definition is checked out
in an .rbuild directory under baseDirectory
"""
productDefPath = (baseDirectory +
'/.rbuild/product-definition/product-definition.xml')
if not os.path.exists(productDefPath):
raise errors.RbuildError('No product directory at %r' %
baseDirectory)
def getTargetTypes(self):
'''
Return all target types
@return: mapping of TargetType name to TargetType
@rtype: dict
'''
client = self.api._client
uri = self.api._uri + '/target_types'
try:
return dict((x.name, x) for x in client.do_GET(uri))
except robj.errors.HTTPNotFoundError:
raise errors.RbuildError(msg='Target types url not found: %s' %
uri)
def getImageDefDescriptor(self, imageType):
# image_type_definition_descriptors are not in a collection, and they
# have an xml header, which causes rObj to process them incorrectly
# thus this mess
client = self.api._client._client # Ew
uri = self.api._uri + \
'/platforms/image_type_definition_descriptors/' + imageType
request = client.do_GET(uri)
request.wait()
response = request.response
if response.status >= 400:
raise errors.RbuildError(response.reason)
return response.content
def getWindowsBuildService(self):
systems = self.api.inventory.infrastructure_systems
wbsSystems = [
x for x in systems
if x.system_type.name == 'infrastructure-windows-build-node'
]
if len(wbsSystems) == 0:
raise errors.RbuildError(
'Could not find any available Windows '
'Build Service systems on your rBuilder. A Windows Build '
'Service is required for building Windows related packages.')
wbs = random.choice(wbsSystems)
if len(wbs.networks) == 0:
raise errors.RbuildError('Could not find any usable networks on '
'the Windows Build Service.')
network = wbs.networks[0]
address = str(network.ip_address) or str(network.dns_name)
return address
def getProductDefinitionSchemaVersion(self):
# rBuilder 5.2.3 <= version < rBuilder 6.1.0
ver = getattr(self.api, 'proddefSchemaVersion', None)
if ver is not None:
return str(ver)
# version >= rBuilder 6.1.0
version_info = getattr(self.api, 'version_info', None)
if version_info is not None:
ver = getattr(version_info, 'product_definition_schema_version',
None)
if ver is not None:
return str(ver)
# proddefSchemaVersion was added in rBuilder 5.2.3, prior to that the
# schema version was 2.0.
raise errors.RbuildError("Unable to determine the product definition "
"version offered by rBuilder")
def _findTrove(self,
name,
version,
flavor=None,
labelPath=None,
defaultFlavor=None,
allowMissing=False):
#pylint: disable-msg=R0913
# findTrove really needs all these arguments to pass through
"""
Gets a reference to a trove in the repository.
@param name: package to find
@type name: string
@param version: version of package to find
@type version: string or C{conary.versions.Version} B{opaque}
@param flavor: flavor of package to find (optional)
@type flavor: string or C{deps.Flavor} B{opaque}
@param labelPath: label(s) to find package on
@type labelPath: None, conary.versions.Label, or list of
conary.versions.Label
@param defaultFlavor: Flavor to use for those troves specifying None
for their flavor.
@type defaultFlavor: str or None
@param allowMissing: if True, allow None as a return value if
the package was not found.
@return: C{(name, version, flavor)} tuple.
Note that C{version} and C{flavor} objects are B{opaque}.
@rtype: (string, conary.versions.Version conary.deps.deps.Flavor)
"""
repos = self._getRepositoryClient()
flavor = self._getFlavor(flavor)
defaultFlavor = self._getFlavor(defaultFlavor)
try:
results = repos.findTroves(labelPath, [(name, version, flavor)],
defaultFlavor=defaultFlavor,
allowMissing=allowMissing)
except conaryerrors.LabelPathNeeded:
errstr = "%s is not a label. Please specify a label where a " \
"product definition can be found, or specify a product " \
"short name and version." % str(version)
raise errors.RbuildError(errstr)
if not results:
return None
troveTup, = results[name, version, flavor]
return troveTup
def _pollBuild(self, buildId, interval=10, max_dropped=3):
dropped = 0
buildStatus = None
while buildStatus is None:
try:
error, buildStatus = self.server.getBuildStatus(buildId)
except socket.timeout:
dropped += 1
if dropped >= max_dropped:
raise errors.RbuildError(
'rBuilder connection timed out after 3 attempts')
self._handle.ui.info('Status request timed out, trying again')
time.sleep(interval)
continue
if error:
raise errors.RbuilderError(*buildStatus)
return buildStatus
def getStageDirectory(self, stageName=None):
"""
Get the absolute directory associated with a given stage name.
@param stageName: stage name, or None to use the active stage.
@type stageName: string
@return: string of absolute directory corrosponding to C{stageName},
if none found, then return None.
@rtype: string or None
"""
if stageName is None:
stageName = self.getActiveStageName()
if stageName is not None:
stageDirectory = '%s/%s' % (self._baseDirectory, stageName)
if not os.path.exists(stageDirectory):
raise errors.RbuildError('Stage directory for %r'
' does not exist' % stageName)
return stageDirectory
else:
return None
def updateCheckout(self, targetDir):
"""
Update a subdirectory containing a checkout of a conary
source package. Similar to the C{cvc update} command.
@param targetDir: subdirectory containing package to update
@type targetDir: string
@return: Status
@rtype: bool
"""
# Conary likes absolute paths RBLD-137
targetDir = os.path.abspath(targetDir)
try:
return checkin.nologUpdateSrc(self._getRepositoryClient(),
[targetDir])
except (builderrors.UpToDate, builderrors.NotCheckedInError):
# The end result is an up to date checkout, so ignore the exception
return True
except builderrors.CheckinError, e:
# All other exceptions are deemed failures
raise errors.RbuildError(str(e))
def createProject(self,
title,
shortName,
hostName=None,
domainName=None,
description=''):
doc = xobj.Document()
doc.project = proj = xobj.XObj()
proj.name = title
proj.short_name = shortName
proj.hostname = hostName or shortName
proj.description = description or ''
if domainName:
proj.domain_name = domainName
proj.external = 'false'
try:
return self.api.projects.append(doc).project_id
except robj.errors.HTTPConflictError:
raise errors.RbuildError("A project with conflicting "
"parameters already exists")
def promoteGroups(self, groupList, fromTo, infoOnly=False):
"""
Promote the troves in C{groupList} using the promote map in
C{fromTo}. The former should be a list of trove tuples, and the
latter a dictionary mapping of labels (C{from: to}).
@param groupList: List of group trove tuples to promote
@type groupList: [(name, version, flavor)]
@param fromTo: Mapping of labels to execute promote on
@type fromTo: {from: to}
@param infoOnly: If C{True}, return without committing anything
@type infoOnly: C{bool}
"""
def getLabelOrBranch(label):
if isinstance(label, types.StringTypes):
if label.startswith('/'):
return self._getVersion(label)
else:
return self._getLabel(label)
return label
promoteMap = dict(
(self._getLabel(fromLabel), getLabelOrBranch(toLabel))
for (fromLabel, toLabel) in fromTo.iteritems())
client = self._getConaryClient()
success, cs = client.createSiblingCloneChangeSet(promoteMap,
groupList,
cloneSources=True)
if not success:
raise errors.RbuildError('Promote failed.')
else:
packageList = [
x.getNewNameVersionFlavor() for x in cs.iterNewTroveList()
]
packageList = [(str(x[0]), str(x[1]), str(x[2]))
for x in packageList]
if not infoOnly:
self._getRepositoryClient().commitChangeSet(cs)
return packageList
def _getResources(self, resource, **kwargs):
'''
Get a fitlered and ordered list of resoruces
@param resource: resource collection name
@param order_by: field and direction to order results
@param uri: alternative uri
@return: list of resources
@rtype: list
'''
uri = kwargs.pop('uri', None)
order_by = kwargs.pop('order_by', None)
fullUri = self.api._uri + '/'
if uri is not None:
fullUri += uri.strip('/') + '/'
fullUri += resource
filter_by = []
for field, param in kwargs.items():
param = self._singleBackslashRe.sub('r\\\\', param)
param = param.replace('"', r'\"')
filter_by.append('EQUAL(%s,"%s")' % (field, param))
if filter_by:
fullUri += ';filter_by=AND(%s)' % ','.join(filter_by)
# always sort by most recently created first
if order_by:
fullUri += ';order_by=%s' % order_by
client = self.api._client
try:
results = client.do_GET(fullUri)
if results:
return results
return []
except robj.errors.HTTPNotFoundError:
raise errors.RbuildError("Unable to fetch resource '%s' at '%s'" %
(resource, fullUri))