def get(self, request, hostname, troveString, pathHash): repos, path, ver, fileObj = self._getFileInfo(hostname, troveString, pathHash) flags = fileObj.flags return models.TroveFile(hostname=hostname, trove=troveString, pathId=sha1helper.md5ToString(fileObj.pathId()), pathHash=pathHash, path=path, fileVersion=ver, fileId=sha1helper.sha1ToString(fileObj.fileId()), tags=','.join(fileObj.tags()), isConfig=flags.isConfig(), isInitialContents=flags.isInitialContents(), isSource=flags.isSource(), isAutoSource=flags.isAutoSource(), isTransient=flags.isTransient(), size=fileObj.contents.size(), sha1=sha1helper.sha1ToString(fileObj.contents.sha1()), permissions=fileObj.inode.permsString(), mtime=fileObj.inode.mtime(), owner=fileObj.inode.owner(), group=fileObj.inode.group(), provides=fileObj.provides(), requires=fileObj.requires())
def testFileIdWrong(self): # create an absolute changeset cs = changeset.ChangeSet() # add a pkg diff flavor = deps.deps.parseFlavor('') v = versions.VersionFromString('/%s/1.0-1-1' %self.cfg.buildLabel.asString()).copy() v.resetTimeStamps() t = trove.Trove('test:test', v, flavor, None) path = self.workDir + '/blah' f = open(path, 'w') f.write('hello, world!\n') f.close() pathId = sha1helper.md5String('/blah') f = files.FileFromFilesystem(path, pathId) # add the file, but munge the fileid brokenFileId = ''.join(reversed(f.fileId())) cs.addFile(None, brokenFileId, f.freeze()) t.addFile(pathId, '/blah', v, brokenFileId) t.computeDigests() diff = t.diff(None, absolute = 1)[0] cs.newTrove(diff) repos = self.openRepository() try: repos.commitChangeSet(cs) assert 0, "Integrity Error not raised" except errors.TroveIntegrityError, e: assert(str(e) == 'fileObj.fileId() != fileId in changeset for ' 'pathId %s' % sha1helper.md5ToString(pathId))
def _getCacheFilename(self, name, version, flavor, compNames): # hash the version and flavor to give a unique filename versionFlavor = '%s %s' % (version.asString(), flavor.freeze()) if compNames: # we could be generating a different set of troves, # say, :runtime only instead of :runtime + :devel, so add # in the components we're creating versionFlavor += ' %s' % ' '.join(compNames) h = sha1helper.md5ToString(sha1helper.md5String(versionFlavor)) return '%s-%s.ccs' %(name, h)
def get(self, request, hostname, troveString): repos = self.getRepos() name, version, flavor = self._getTuple(troveString) trv = repos.getTrove(name, version, flavor, withFiles=True) fileList = [] for pathId, path, fileId, fileVersion in trv.iterFileList(): pathHash = sha1helper.md5String(path) fileList.append(models.TroveFile( hostname=hostname, pathId=sha1helper.md5ToString(pathId), pathHash=sha1helper.md5ToString(pathHash), path=path, fileId=sha1helper.sha1ToString(fileId), trove=troveString, fileVersion=fileVersion)) troveModel = models.Trove(hostname=hostname, name=name, version=version, flavor=flavor, files=fileList) return troveModel
def _getCheckFilesList(self, csTrove, troveInfo, fileHostFilter, configRestoreList, normalRestoreList, restoreContents=True): checkFilesList = [] for (pathId, path, fileId, newVersion) in csTrove.getNewFileList(): if (fileHostFilter and newVersion.getHost() not in fileHostFilter): fileObj = None fileStream = None else: # New files don't always have streams in the changeset. # Filesets and clones don't include them for files which # are already known to be in the repository. fileStream = self.cs.getFileChange(None, fileId) if fileStream is None: if not fileHostFilter: # We are trying to commit to a database, but the # diff returned nothing raise KeyError checkFilesList.append((pathId, fileId, newVersion)) fileObj = None else: fileObj = files.ThawFile(fileStream, pathId) if fileObj and fileObj.fileId() != fileId: raise trove.TroveIntegrityError( csTrove.getName(), csTrove.getNewVersion(), csTrove.getNewFlavor(), "fileObj.fileId() != fileId in changeset " "for pathId %s" % sha1helper.md5ToString(pathId)) self.addFileVersion(troveInfo, pathId, path, fileId, newVersion, fileStream=fileStream, withContents=restoreContents) self._handleContents(pathId, fileId, fileStream, configRestoreList, normalRestoreList, restoreContents=restoreContents) return checkFilesList
def _restoreNormal(self, cs, normalRestoreList, preRestored): ptrRestores = [] ptrRefsAdded = {} lastRestore = None # restore each pathId,fileId combo once while normalRestoreList: (pathId, fileId, sha1, restoreContents) = normalRestoreList.pop(0) if preRestored is not None and sha1 in preRestored: continue if (pathId, fileId) == lastRestore: continue lastRestore = (pathId, fileId) try: (contType, fileContents) = cs.getFileContents(pathId, fileId, compressed=True) except KeyError: raise errors.IntegrityError( "Missing file contents for pathId %s, fileId %s" % (sha1helper.md5ToString(pathId), sha1helper.sha1ToString(fileId))) if contType == changeset.ChangedFileTypes.ptr: ptrRestores.append(sha1) target = util.decompressString(fileContents.get().read()) if util.tupleListBsearchInsert( normalRestoreList, (target[:16], target[16:], sha1, True), self.ptrCmp): # Item was inserted. This creates a reference in the # datastore; keep track of it to prevent a duplicate # reference count. ptrRefsAdded[sha1] = True continue assert (contType == changeset.ChangedFileTypes.file) self.addFileContents(sha1, fileContents, restoreContents, 0, precompressed=True) for sha1 in ptrRestores: # Increment the reference count for items which were ptr's # to a different file. if sha1 in ptrRefsAdded: del ptrRefsAdded[sha1] else: self.addFileContents(sha1, None, False, 0)
def _write(self, f): """ Returns a string representing file information for this trove trove, which can later be read by the read() method. This is only used to create the Conary control file when dealing with :source component checkins, so things like trove dependency information is not needed. The format of the string is: name <name> version <version> branch <branch> (lastmerged <version>)? (factory <name>)? <file count> PATHID1 PATH1 FILEID1 ISCONFIG1 REFRESH1 VERSION1 PATHID2 PATH2 FILEID2 ISCONFIG2 REFRESH2 VERSION2 . . . PATHIDn PATHn FILEIDn ISCONFIGn REFRESHn VERSIONn """ assert(len(self.strongTroves) == 0) assert(len(self.weakTroves) == 0) f.write("name %s\n" % self.getName()) f.write("version %s\n" % self.getVersion().freeze()) f.write("branch %s\n" % self.getBranch().freeze()) if self.getLastMerged() is not None: f.write("lastmerged %s\n" % self.getLastMerged().freeze()) if self.getFactory(): f.write("factory %s\n" % self.getFactory()) rc = [] rc.append("%d\n" % (len(list(self.iterFileList())))) rc += [ "%s %s %s %s %s\n" % (sha1helper.md5ToString(x[0]), x[1], sha1helper.sha1ToString(x[2]), self.fileInfo[x[0]], x[3].asString()) for x in sorted(self.iterFileList()) ] f.write("".join(rc))
def _write(self, f): """ Returns a string representing file information for this trove trove, which can later be read by the read() method. This is only used to create the Conary control file when dealing with :source component checkins, so things like trove dependency information is not needed. The format of the string is: name <name> version <version> branch <branch> (lastmerged <version>)? (factory <name>)? <file count> PATHID1 PATH1 FILEID1 ISCONFIG1 REFRESH1 VERSION1 PATHID2 PATH2 FILEID2 ISCONFIG2 REFRESH2 VERSION2 . . . PATHIDn PATHn FILEIDn ISCONFIGn REFRESHn VERSIONn """ assert (len(self.strongTroves) == 0) assert (len(self.weakTroves) == 0) f.write("name %s\n" % self.getName()) f.write("version %s\n" % self.getVersion().freeze()) f.write("branch %s\n" % self.getBranch().freeze()) if self.getLastMerged() is not None: f.write("lastmerged %s\n" % self.getLastMerged().freeze()) if self.getFactory(): f.write("factory %s\n" % self.getFactory()) rc = [] rc.append("%d\n" % (len(list(self.iterFileList())))) rc += [ "%s %s %s %s %s\n" % (sha1helper.md5ToString(x[0]), x[1], sha1helper.sha1ToString( x[2]), self.fileInfo[x[0]], x[3].asString()) for x in sorted(self.iterFileList()) ] f.write("".join(rc))
def _getCheckFilesList(self, csTrove, troveInfo, fileHostFilter, configRestoreList, normalRestoreList, restoreContents = True): checkFilesList = [] for (pathId, path, fileId, newVersion) in csTrove.getNewFileList(): if (fileHostFilter and newVersion.getHost() not in fileHostFilter): fileObj = None fileStream = None else: # New files don't always have streams in the changeset. # Filesets and clones don't include them for files which # are already known to be in the repository. fileStream = self.cs.getFileChange(None, fileId) if fileStream is None: if not fileHostFilter: # We are trying to commit to a database, but the # diff returned nothing raise KeyError checkFilesList.append((pathId, fileId, newVersion)) fileObj = None else: fileObj = files.ThawFile(fileStream, pathId) if fileObj and fileObj.fileId() != fileId: raise trove.TroveIntegrityError(csTrove.getName(), csTrove.getNewVersion(), csTrove.getNewFlavor(), "fileObj.fileId() != fileId in changeset " "for pathId %s" % sha1helper.md5ToString(pathId)) self.addFileVersion(troveInfo, pathId, path, fileId, newVersion, fileStream = fileStream, withContents = restoreContents) self._handleContents(pathId, fileId, fileStream, configRestoreList, normalRestoreList, restoreContents = restoreContents) return checkFilesList
def _restoreConfig(self, cs, configRestoreList): # config files are cached, so we don't have to worry about not # restoring the same fileId/pathId twice for (pathId, newFileId, sha1, oldfile, newFileId, oldVersion, oldFileId, restoreContents) in configRestoreList: if cs.configFileIsDiff(pathId, newFileId): (contType, fileContents) = cs.getFileContents(pathId, newFileId) # the content for this file is in the form of a # diff, which we need to apply against the file in # the repository assert (oldVersion) try: f = self.repos.getFileContents([(oldFileId, oldVersion, oldfile)])[0].get() except KeyError: raise errors.IntegrityError( "Missing file contents for pathId %s, fileId %s" % (sha1helper.md5ToString(pathId), sha1helper.sha1ToString(oldFileId))) oldLines = f.readlines() f.close() del f diff = fileContents.get().readlines() (newLines, failedHunks) = patch.patch(oldLines, diff) fileContents = filecontents.FromString("".join(newLines)) assert (not failedHunks) else: # config files are not always available compressed (due # to the config file cache) fileContents = filecontents.FromChangeSet( cs, pathId, newFileId) self.addFileContents(sha1, fileContents, restoreContents, 1)
def formatFile(self, pathId, path, fileId, version, fileObj=None, prefix='', indent=0): taglist = '' sha1 = '' id = '' flavor = '' dcfg = self.dcfg verbose = dcfg.isVerbose() if verbose and isinstance(fileObj, files.SymbolicLink): name = "%s -> %s" % (path, fileObj.target()) else: name = path if dcfg.fileFlavors: if not fileObj.flavor().isEmpty(): flavor = '[%s]' % fileObj.flavor() if dcfg.tags: tags = [] if fileObj.tags: tags.extend(fileObj.tags) if fileObj.flags.isInitialContents(): tags.append('initialContents') if fileObj.flags.isAutoSource(): tags.append('autosource') if fileObj.flags.isConfig(): tags.append('config') if fileObj.flags.isTransient(): tags.append('transient') if tags: taglist = ' {' + ' '.join(tags) + '}' if dcfg.sha1s: if hasattr(fileObj, 'contents') and fileObj.contents: sha1 = sha1ToString(fileObj.contents.sha1()) + ' ' else: sha1 = ' '*41 if dcfg.ids and pathId: id = md5ToString(pathId) + ' ' + sha1ToString(fileId) + ', ' if dcfg.fileVersions: if dcfg.useFullVersions(): verStr = ' %s' % version elif dcfg.showLabels: verStr = ' %s/%s' % (version.branch().label(), version.trailingRevision()) else: verStr = ' %s' % version.trailingRevision() else: verStr = '' spacer = ' ' * indent if fileObj: owner = fileObj.inode.owner() if owner[0] == '+': owner = owner[1:] group = fileObj.inode.group() if group[0] == '+': group = group[1:] if verbose: ln = "%s%s%s%s%s 1 %-8s %-8s %s %s %s%s%s%s" % \ (spacer, prefix, id, sha1, fileObj.modeString(), owner, group, fileObj.sizeString(), fileObj.timeString(), name, flavor, taglist, verStr) else: ln = "%s%s%s%s%s%s%s" % (spacer, id, sha1, path, flavor, taglist, verStr) yield ln if dcfg.fileDeps: for ln in self.formatDeps(fileObj.provides(), fileObj.requires(), indent + 1, showEmpty = False): yield ln
def _createInstallTroveObjects(self, fileHostFilter = [], callback = None, hidden = False, mirror = False, allowIncomplete = False, excludeCapsuleContents = False): # create the trove objects which need to be installed; the # file objects which map up with them are created later, but # we do need a map from pathId to the path and version of the # file we need, so build up a dictionary with that information configRestoreList = [] normalRestoreList = [] checkFilesList = [] newList = [ x for x in self.cs.iterNewTroveList() ] repos = self.repos cs = self.cs oldTrovesNeeded = [ x.getOldNameVersionFlavor() for x in newList if x.getOldVersion() ] oldTroveIter = repos.iterTroves(oldTrovesNeeded, hidden = True) troveNo = 0 for csTrove in newList: if csTrove.troveType() == trove.TROVE_TYPE_REMOVED: # deal with these later on to ensure any changesets which # are relative to removed troves can be processed continue troveNo += 1 if callback: callback.creatingDatabaseTransaction(troveNo, len(newList)) newVersion = csTrove.getNewVersion() oldTroveVersion = csTrove.getOldVersion() oldTroveFlavor = csTrove.getOldFlavor() troveName = csTrove.getName() troveFlavor = csTrove.getNewFlavor() if repos.hasTrove(troveName, newVersion, troveFlavor): raise errors.CommitError, \ "version %s of %s already exists" % \ (newVersion.asString(), csTrove.getName()) if oldTroveVersion: newTrove = oldTroveIter.next() assert(newTrove.getNameVersionFlavor() == csTrove.getOldNameVersionFlavor()) self.oldTrove(newTrove, csTrove, troveName, oldTroveVersion, oldTroveFlavor) oldCompatClass = newTrove.getCompatibilityClass() if csTrove.isRollbackFence( oldCompatibilityClass = oldCompatClass, update = True): self.invalidateRollbacks(set = True) else: newTrove = trove.Trove(csTrove.getName(), newVersion, troveFlavor, csTrove.getChangeLog(), setVersion = False) # FIXME: we reset the trove version # since in this case we need to use the fileMap returned # from applyChangeSet allowIncomplete = True newFileMap = newTrove.applyChangeSet(csTrove, needNewFileMap=True, allowIncomplete=allowIncomplete) skipContents = (excludeCapsuleContents and newTrove.troveInfo.capsule.type and newTrove.troveInfo.capsule.type()) if newTrove.troveInfo.incomplete(): log.warning('trove %s has schema version %s, which contains' ' information not handled by this client. This' ' version of Conary understands schema version %s.' ' Dropping extra information. Please upgrade conary.', newTrove.getName(), newTrove.troveInfo.troveVersion(), trove.TROVE_VERSION) self.checkTroveCompleteness(newTrove) self.checkTroveSignatures(newTrove, callback=callback) if oldTroveVersion is not None: troveInfo = self.addTrove( (troveName, oldTroveVersion, oldTroveFlavor), newTrove, csTrove, hidden = hidden) else: troveInfo = self.addTrove(None, newTrove, csTrove, hidden = hidden) checkFilesList += self._getCheckFilesList(csTrove, troveInfo, fileHostFilter, configRestoreList, normalRestoreList, restoreContents = not skipContents) for (pathId, path, fileId, newVersion) in \ newTrove.iterFileList(members = True, capsules = True): # handle files which haven't changed; we know which those # are because they're in the merged trove but they aren't # in the newFileMap if pathId in newFileMap: continue self.addFileVersion(troveInfo, pathId, path, fileId, newVersion, withContents = not skipContents) filesNeeded = [] for i, (pathId, path, fileId, newVersion) in enumerate(csTrove.getChangedFileList()): tup = newFileMap[pathId] (oldPath, oldFileId, oldVersion) = tup[-3:] if path is None: path = oldPath if fileId is None: oldFileId = fileId if newVersion is None: newVersion = oldVersion if (fileHostFilter and newVersion.getHost() not in fileHostFilter): fileStream = None elif (oldVersion == newVersion and oldFileId == fileId): # the file didn't change between versions; we can just # ignore it fileStream = None else: fileStream = cs.getFileChange(oldFileId, fileId) if fileStream and fileStream[0] == "\x01": if len(fileStream) != 2: # This is awful, but this is how we say a file # stream didn't change. Omitting it or at least '' # would be nicer, but would break clients. filesNeeded.append((i, (pathId, oldFileId, oldVersion))) continue fileStream = None # None is the file object self.addFileVersion(troveInfo, pathId, path, fileId, newVersion, fileStream = fileStream, withContents = not skipContents) if fileStream is not None: self._handleContents(pathId, fileId, fileStream, configRestoreList, normalRestoreList, oldFileId = oldFileId, oldVersion = oldVersion, oldfile = None, restoreContents = not skipContents) oldFileObjects = list(repos.getFileVersions( [ x[1] for x in filesNeeded ])) for i, (pathId, path, fileId, newVersion) in enumerate(csTrove.getChangedFileList()): if not filesNeeded or filesNeeded[0][0] != i: continue filesNeeded.pop(0) tup = newFileMap[pathId] (oldPath, oldFileId, oldVersion) = tup[-3:] if path is None: path = oldPath if fileId is None: oldFileId = fileId if newVersion is None: newVersion = oldVersion restoreContents = not skipContents diff = cs.getFileChange(oldFileId, fileId) # stored as a diff (the file type is the same # and (for *repository* commits) the file # is in the same repository between versions oldfile = oldFileObjects.pop(0) fileObj = oldfile.copy() fileObj.twm(diff, oldfile) assert(fileObj.pathId() == pathId) fileStream = fileObj.freeze() if (not mirror) and ( fileObj.hasContents and fileObj.contents.sha1() == oldfile.contents.sha1() and not (fileObj.flags.isConfig() and not oldfile.flags.isConfig())): # don't restore the contents here. we don't # need them, and they may be relative to # something from a different repository restoreContents = False if fileObj and fileObj.fileId() != fileId: raise trove.TroveIntegrityError(csTrove.getName(), csTrove.getNewVersion(), csTrove.getNewFlavor(), "fileObj.fileId() != fileId in changeset") self.addFileVersion(troveInfo, pathId, path, fileId, newVersion, fileStream = fileStream, withContents = restoreContents) self._handleContents(pathId, fileId, fileStream, configRestoreList, normalRestoreList, oldFileId = oldFileId, oldVersion = oldVersion, oldfile = oldfile, restoreContents = restoreContents) del newFileMap self.addTroveDone(troveInfo, mirror=mirror) try: # we need to actualize this, not just get a generator list(repos.getFileVersions(checkFilesList)) except errors.FileStreamMissing, e: info = [ x for x in checkFilesList if x[1] == e.fileId ] (pathId, fileId) = info[0][0:2] # Missing from the repo; raise exception raise errors.IntegrityError( "Incomplete changeset specified: missing pathId %s " "fileId %s" % (sha1helper.md5ToString(pathId), sha1helper.sha1ToString(fileId)))
def verifySignatures(f, validKeys=None): """ Given an extended file, compute signatures """ f.seek(0) h = readHeader(f) # Cheap test first: verify MD5 sig sigmd5 = h.get(SIG_MD5, None) if sigmd5 is not None: f.seek(0) readSignatureHeader(f) # verify md5 digest md5 = digestlib.md5() util.copyfileobj(f, NullWriter(), digest=md5) if md5.digest() != sigmd5: raise MD5SignatureError( "The MD5 digest fails to verify: expected %s, got %s" % (sha1helper.md5ToString(sigmd5), md5.hexdigest())) # Don't bother if no gpg signature was present, or no valid keys were # presented if validKeys is None: return sigString = h.get(SIG_GPG, None) if sigString is None: return # Skip to immutable header region f.seek(0) readSignatureHeader(f) sig = openpgpfile.readSignature(sigString) keyId = sig.getSignerKeyId() matchingKeys = [x for x in validKeys if x.hasKeyId(keyId)] if not matchingKeys: raise PGPSignatureError("Signature generated with key %s does " "not match valid keys %s" % (keyId, ', '.join(x.getKeyId() for x in validKeys))) key = matchingKeys[0] # signature verification assumes a seekable stream and will seek to the # beginning; use a SeekableNestedFile size = h.getHeaderPlusPayloadSize() if size is None: pos = f.tell() f.seek(0, 2) size = f.tell() f.seek(pos, 0) snf = None if hasattr(f, 'pread'): extFile = f elif hasattr(f, 'name'): extFile = util.ExtendedFile(f.name, buffering=False) else: # worst case scenario, we slurp everything in memory extFile = util.ExtendedStringIO(f.read()) snf = extFile if snf is None: snf = util.SeekableNestedFile(extFile, start=f.tell(), size=size) try: sig.verifyDocument(key.getCryptoKey(), snf) except openpgpfile.SignatureError: raise PGPSignatureError
def getTrove(cu, roleIds, name, version, flavor, mkUrl = None, thisHost = None, displayFlavor = None, excludeCapsules = False): def buildTupleList(tuples, name, mkUrl = mkUrl): l = getattr(datamodel.SingleTrove, name)() for troveInfo in sorted(tuples.iter()): l.append(name = troveInfo.name(), version = troveInfo.version(), flavor = troveInfo.flavor(), mkUrl = mkUrl) return l def fileQuery(gfcu, filesInstanceId, dirName = None): # XXX restricing by dirName seems and obvious thing to do here, # but it actually slows things down?? # # the distinct here is unfortunate, but conary repositories had # a bug for about a year which caused it to store duplicate paths # if a path was committed for the first time duplicate times in # a single commit job gfcu.execute(""" SELECT DISTINCT dirName, basename, version, pathId, fileId FROM TroveFiles JOIN Versions USING (versionId) JOIN FileStreams ON (TroveFiles.streamId = FileStreams.streamId) JOIN FilePaths ON (TroveFiles.filePathId = FilePaths.filePathId) JOIN DirNames ON FilePaths.dirNameId = DirNames.dirNameId JOIN Basenames ON (FilePaths.baseNameId = Basenames.baseNameId) WHERE TroveFiles.instanceId = ? ORDER BY dirName, basename """, filesInstanceId) cu.execute(""" SELECT Instances.instanceId, Nodes.timeStamps FROM Instances JOIN Nodes USING (itemId, versionId) JOIN Items USING (itemId) JOIN Versions ON (Instances.versionId = Versions.versionId) JOIN Flavors ON (Instances.flavorId = Flavors.flavorId) JOIN UserGroupInstancesCache AS ugi ON (instances.instanceId = ugi.instanceId AND ugi.userGroupId in (%s)) WHERE item = ? AND version = ? AND flavor = ? """ % ",".join( str(x) for x in roleIds), name, version, deps.parseFlavor(flavor).freeze()) l = [ (x[0], x[1]) for x in cu ] if not l: return None instanceId, timeStamps = l[0] frzVer = versions.strToFrozen(version, timeStamps.split(":")) verobj = versions.ThawVersion(frzVer) tupleLists = [ ( trove._TROVEINFO_TAG_BUILDDEPS, 'builddeps' ), ( trove._TROVEINFO_TAG_POLICY_PROV, 'policyprovider' ), ( trove._TROVEINFO_TAG_LOADEDTROVES, 'loadedtroves' ), ( trove._TROVEINFO_TAG_COPIED_FROM, 'copiedfrom' ), ( trove._TROVEINFO_TAG_DERIVEDFROM, 'derivedfrom' ) ] cu.execute(""" SELECT infoType, data FROM TroveInfo WHERE instanceId = ? AND infoType IN (%s) """ % ",".join(str(x) for x in [ trove._TROVEINFO_TAG_SOURCENAME, trove._TROVEINFO_TAG_CLONEDFROM, trove._TROVEINFO_TAG_CLONEDFROMLIST, trove._TROVEINFO_TAG_BUILDTIME, trove._TROVEINFO_TAG_SIZE, trove._TROVEINFO_TAG_METADATA, trove._TROVEINFO_TAG_CAPSULE, ] + [ x[0] for x in tupleLists ] ), instanceId) troveInfo = {} for infoType, data in cu: data = cu.frombinary(data) infoClass = trove.TroveInfo.streamDict[infoType][1] troveInfo[infoType] = infoClass(data) kwargs = { 'name' : name, 'version' : verobj, 'flavor' : flavor } if displayFlavor is not None: kwargs['displayflavor'] = displayFlavor if trove._TROVEINFO_TAG_BUILDTIME in troveInfo: kwargs['buildtime'] = int(troveInfo[trove._TROVEINFO_TAG_BUILDTIME]()) if trove._TROVEINFO_TAG_SOURCENAME in troveInfo: kwargs['source'] = (troveInfo[trove._TROVEINFO_TAG_SOURCENAME](), verobj.getSourceVersion(), '') if trove._TROVEINFO_TAG_SIZE in troveInfo: kwargs['size'] = troveInfo[trove._TROVEINFO_TAG_SIZE]() if trove._TROVEINFO_TAG_METADATA in troveInfo: md = troveInfo[trove._TROVEINFO_TAG_METADATA].get() kwargs['shortdesc'] = md['shortDesc'] kwargs['longdesc'] = md['longDesc'] if md['licenses']: kwargs['license'] = [ x for x in md['licenses' ]] if md['crypto']: kwargs['crypto'] = [ x for x in md['crypto'] ] for (tag, tagName) in tupleLists: if tag in troveInfo: kwargs[tagName] = buildTupleList(troveInfo[tag], tagName, mkUrl = mkUrl) t = datamodel.SingleTrove(mkUrl = mkUrl, thisHost = thisHost, **kwargs) if trove._TROVEINFO_TAG_CLONEDFROMLIST in troveInfo: clonedFromList = troveInfo[trove._TROVEINFO_TAG_CLONEDFROMLIST] elif (trove._TROVEINFO_TAG_CLONEDFROM in troveInfo): clonedFromList = [ troveInfo[trove._TROVEINFO_TAG_CLONEDFROM]() ] else: clonedFromList = [] for ver in clonedFromList: t.addClonedFrom(name, ver, flavor, mkUrl = mkUrl) hasCapsule = False if trove._TROVEINFO_TAG_CAPSULE in troveInfo: if troveInfo[trove._TROVEINFO_TAG_CAPSULE].type(): hasCapsule = True fileQuery(cu, instanceId) for (dirName, baseName, fileVersion, pathId, fileId) in cu: dirName = cu.frombinary(dirName) baseName = cu.frombinary(baseName) if pathId == trove.CAPSULE_PATHID: isCapsule = 1 contentAvailable = not excludeCapsules else: isCapsule = None contentAvailable = not hasCapsule fileObj = datamodel.FileReference( path = os.path.join(dirName, baseName), version = fileVersion, pathId = md5ToString(cu.frombinary(pathId)), fileId = sha1ToString(cu.frombinary(fileId)), isCapsule = isCapsule, contentAvailable = contentAvailable, mkUrl = mkUrl, thisHost = thisHost) t.addFile(fileObj) cu.execute(""" SELECT item, version, flavor, TroveTroves.includedId, Nodes.timeStamps FROM TroveTroves JOIN Instances ON (Instances.instanceId = TroveTroves.includedId) JOIN Nodes USING (itemId, versionId) JOIN Items USING (itemId) JOIN Versions ON (Versions.versionId = Instances.versionId) JOIN Flavors ON (Flavors.flavorId = Instances.flavorId) WHERE TroveTroves.instanceId = ? AND (TroveTroves.flags & %d) = 0 ORDER BY item, version, flavor """ % schema.TROVE_TROVES_WEAKREF, instanceId) for (subName, subVersion, subFlavor, refInstanceId, subTS) in cu: subFlavor = str(deps.ThawFlavor(subFlavor)) frzVer = versions.strToFrozen(subVersion, [ x for x in subTS.split(":") ]) subV = versions.ThawVersion(frzVer) t.addReferencedTrove(subName, subV, subFlavor, mkUrl = mkUrl) # It would be far better to use file tags to identify these build # logs, but it's significantly slower as well because they're in # the file objects rather than the trove (and those file objects # could be stored on a different repository) if not subName.endswith(':debuginfo'): continue fileQuery(cu, refInstanceId, dirName = '/usr/src/debug/buildlogs') logHost = subV.getHost() for (dirName, baseName, fileVersion, pathId, fileId) in cu: if (dirName) != '/usr/src/debug/buildlogs': continue if baseName.endswith('-log.bz2'): t.setBuildLog(logHost, sha1ToString(fileId)) elif baseName.endswith('-xml.bz2'): t.setXMLBuildLog(logHost, sha1ToString(fileId)) return t
def formatFile(self, pathId, path, fileId, version, fileObj=None, prefix='', indent=0): taglist = '' sha1 = '' id = '' flavor = '' dcfg = self.dcfg verbose = dcfg.isVerbose() if verbose and isinstance(fileObj, files.SymbolicLink): name = "%s -> %s" % (path, fileObj.target()) else: name = path if dcfg.fileFlavors: if not fileObj.flavor().isEmpty(): flavor = '[%s]' % fileObj.flavor() if dcfg.tags: tags = [] if fileObj.tags: tags.extend(fileObj.tags) if fileObj.flags.isInitialContents(): tags.append('initialContents') if fileObj.flags.isAutoSource(): tags.append('autosource') if fileObj.flags.isConfig(): tags.append('config') if fileObj.flags.isTransient(): tags.append('transient') if tags: taglist = ' {' + ' '.join(tags) + '}' if dcfg.sha1s: if hasattr(fileObj, 'contents') and fileObj.contents: sha1 = sha1ToString(fileObj.contents.sha1()) + ' ' else: sha1 = ' ' * 41 if dcfg.ids and pathId: id = md5ToString(pathId) + ' ' + sha1ToString(fileId) + ', ' if dcfg.fileVersions: if dcfg.useFullVersions(): verStr = ' %s' % version elif dcfg.showLabels: verStr = ' %s/%s' % (version.branch().label(), version.trailingRevision()) else: verStr = ' %s' % version.trailingRevision() else: verStr = '' spacer = ' ' * indent if fileObj: owner = fileObj.inode.owner() if owner[0] == '+': owner = owner[1:] group = fileObj.inode.group() if group[0] == '+': group = group[1:] if verbose: ln = "%s%s%s%s%s 1 %-8s %-8s %s %s %s%s%s%s" % \ (spacer, prefix, id, sha1, fileObj.modeString(), owner, group, fileObj.sizeString(), fileObj.timeString(), name, flavor, taglist, verStr) else: ln = "%s%s%s%s%s%s%s" % (spacer, id, sha1, path, flavor, taglist, verStr) yield ln if dcfg.fileDeps: for ln in self.formatDeps(fileObj.provides(), fileObj.requires(), indent + 1, showEmpty=False): yield ln
def __init__(self, repos, cs, fileHostFilter=[], callback=None, resetTimestamps=False, allowIncomplete=False, hidden=False, mirror=False, excludeCapsuleContents=False): self.repos = repos self.cs = cs self.invalidateRollbacksFlag = False newList = [x for x in cs.iterNewTroveList()] if resetTimestamps: # This depends intimiately on the versions cache. We don't # change the timestamps on each version, because the cache # ensures they are all a single underlying object. Slick, # but brittle? updated = {} for csTrove in newList: ver = csTrove.getNewVersion() if ver in updated: pass else: oldVer = ver.copy() ver.trailingRevision().resetTimeStamp() updated[oldVer] = ver del updated troveNo, configRestoreList, normalRestoreList = \ self._createInstallTroveObjects(fileHostFilter = fileHostFilter, callback = callback, mirror = mirror, hidden = hidden, allowIncomplete = allowIncomplete, excludeCapsuleContents = excludeCapsuleContents) configRestoreList, normalRestoreList = \ self._filterRestoreList(configRestoreList, normalRestoreList) # use a key to select data up to, but not including, the first # version. We can't sort on version because we don't have timestamps configRestoreList.sort(key=lambda x: x[0:5]) normalRestoreList.sort(key=lambda x: x[0:3]) # config files are cached, so we don't have to worry about not # restoring the same fileId/pathId twice for (pathId, newFileId, sha1, oldfile, newFileId, oldVersion, oldFileId, restoreContents) in configRestoreList: if cs.configFileIsDiff(pathId, newFileId): (contType, fileContents) = cs.getFileContents(pathId, newFileId) # the content for this file is in the form of a # diff, which we need to apply against the file in # the repository assert (oldVersion) try: f = self.repos.getFileContents([(oldFileId, oldVersion, oldfile)])[0].get() except KeyError: raise errors.IntegrityError( "Missing file contents for pathId %s, fileId %s" % (sha1helper.md5ToString(pathId), sha1helper.sha1ToString(fileId))) oldLines = f.readlines() f.close() del f diff = fileContents.get().readlines() (newLines, failedHunks) = patch.patch(oldLines, diff) fileContents = filecontents.FromString("".join(newLines)) assert (not failedHunks) else: # config files are not always available compressed (due # to the config file cache) fileContents = filecontents.FromChangeSet( cs, pathId, newFileId) self.addFileContents(sha1, fileContents, restoreContents, 1) ptrRestores = [] ptrRefsAdded = {} lastRestore = None # restore each pathId,fileId combo once while normalRestoreList: (pathId, fileId, sha1, restoreContents) = normalRestoreList.pop(0) if (pathId, fileId) == lastRestore: continue lastRestore = (pathId, fileId) try: (contType, fileContents) = cs.getFileContents(pathId, fileId, compressed=True) except KeyError: raise errors.IntegrityError( "Missing file contents for pathId %s, fileId %s" % (sha1helper.md5ToString(pathId), sha1helper.sha1ToString(fileId))) if contType == changeset.ChangedFileTypes.ptr: ptrRestores.append(sha1) target = util.decompressString(fileContents.get().read()) if util.tupleListBsearchInsert( normalRestoreList, (target[:16], target[16:], sha1, True), self.ptrCmp): # Item was inserted. This creates a reference in the # datastore; keep track of it to prevent a duplicate # reference count. ptrRefsAdded[sha1] = True continue assert (contType == changeset.ChangedFileTypes.file) self.addFileContents(sha1, fileContents, restoreContents, 0, precompressed=True) for sha1 in ptrRestores: # Increment the reference count for items which were ptr's # to a different file. if sha1 in ptrRefsAdded: del ptrRefsAdded[sha1] else: self.addFileContents(sha1, None, False, 0) #del configRestoreList #del normalRestoreList for csTrove in newList: if csTrove.troveType() != trove.TROVE_TYPE_REMOVED: continue troveNo += 1 if callback: callback.creatingDatabaseTransaction(troveNo, len(newList)) self.markTroveRemoved(csTrove.getName(), csTrove.getNewVersion(), csTrove.getNewFlavor()) for (troveName, version, flavor) in cs.getOldTroveList(): trv = self.repos.getTrove(troveName, version, flavor) self.oldTrove(trv, None, troveName, version, flavor)
def __init__(self, repos, cs, fileHostFilter = [], callback = None, resetTimestamps = False, allowIncomplete = False, hidden = False, mirror = False, excludeCapsuleContents = False): self.repos = repos self.cs = cs self.invalidateRollbacksFlag = False newList = [ x for x in cs.iterNewTroveList() ] if resetTimestamps: # This depends intimiately on the versions cache. We don't # change the timestamps on each version, because the cache # ensures they are all a single underlying object. Slick, # but brittle? updated = {} for csTrove in newList: ver = csTrove.getNewVersion() if ver in updated: pass else: oldVer = ver.copy() ver.trailingRevision().resetTimeStamp() updated[oldVer] = ver del updated troveNo, configRestoreList, normalRestoreList = \ self._createInstallTroveObjects(fileHostFilter = fileHostFilter, callback = callback, mirror = mirror, hidden = hidden, allowIncomplete = allowIncomplete, excludeCapsuleContents = excludeCapsuleContents) configRestoreList, normalRestoreList = \ self._filterRestoreList(configRestoreList, normalRestoreList) # use a key to select data up to, but not including, the first # version. We can't sort on version because we don't have timestamps configRestoreList.sort(key=lambda x: x[0:5]) normalRestoreList.sort(key=lambda x: x[0:3]) # config files are cached, so we don't have to worry about not # restoring the same fileId/pathId twice for (pathId, newFileId, sha1, oldfile, newFileId, oldVersion, oldFileId, restoreContents) in configRestoreList: if cs.configFileIsDiff(pathId, newFileId): (contType, fileContents) = cs.getFileContents(pathId, newFileId) # the content for this file is in the form of a # diff, which we need to apply against the file in # the repository assert(oldVersion) try: f = self.repos.getFileContents( [(oldFileId, oldVersion, oldfile)])[0].get() except KeyError: raise errors.IntegrityError( "Missing file contents for pathId %s, fileId %s" % ( sha1helper.md5ToString(pathId), sha1helper.sha1ToString(fileId))) oldLines = f.readlines() f.close() del f diff = fileContents.get().readlines() (newLines, failedHunks) = patch.patch(oldLines, diff) fileContents = filecontents.FromString( "".join(newLines)) assert(not failedHunks) else: # config files are not always available compressed (due # to the config file cache) fileContents = filecontents.FromChangeSet(cs, pathId, newFileId) self.addFileContents(sha1, fileContents, restoreContents, 1) ptrRestores = [] ptrRefsAdded = {} lastRestore = None # restore each pathId,fileId combo once while normalRestoreList: (pathId, fileId, sha1, restoreContents) = normalRestoreList.pop(0) if (pathId, fileId) == lastRestore: continue lastRestore = (pathId, fileId) try: (contType, fileContents) = cs.getFileContents(pathId, fileId, compressed = True) except KeyError: raise errors.IntegrityError( "Missing file contents for pathId %s, fileId %s" % ( sha1helper.md5ToString(pathId), sha1helper.sha1ToString(fileId))) if contType == changeset.ChangedFileTypes.ptr: ptrRestores.append(sha1) target = util.decompressString(fileContents.get().read()) if util.tupleListBsearchInsert(normalRestoreList, (target[:16], target[16:], sha1, True), self.ptrCmp): # Item was inserted. This creates a reference in the # datastore; keep track of it to prevent a duplicate # reference count. ptrRefsAdded[sha1] = True continue assert(contType == changeset.ChangedFileTypes.file) self.addFileContents(sha1, fileContents, restoreContents, 0, precompressed = True) for sha1 in ptrRestores: # Increment the reference count for items which were ptr's # to a different file. if sha1 in ptrRefsAdded: del ptrRefsAdded[sha1] else: self.addFileContents(sha1, None, False, 0) #del configRestoreList #del normalRestoreList for csTrove in newList: if csTrove.troveType() != trove.TROVE_TYPE_REMOVED: continue troveNo += 1 if callback: callback.creatingDatabaseTransaction(troveNo, len(newList)) self.markTroveRemoved(csTrove.getName(), csTrove.getNewVersion(), csTrove.getNewFlavor()) for (troveName, version, flavor) in cs.getOldTroveList(): trv = self.repos.getTrove(troveName, version, flavor) self.oldTrove(trv, None, troveName, version, flavor)
def _createInstallTroveObjects( self, fileHostFilter=[], callback=None, hidden=False, mirror=False, allowIncomplete=False, ): # create the trove objects which need to be installed; the # file objects which map up with them are created later, but # we do need a map from pathId to the path and version of the # file we need, so build up a dictionary with that information configRestoreList = [] normalRestoreList = [] checkFilesList = [] newList = [x for x in self.cs.iterNewTroveList()] repos = self.repos cs = self.cs oldTrovesNeeded = [ x.getOldNameVersionFlavor() for x in newList if x.getOldVersion() ] oldTroveIter = repos.iterTroves(oldTrovesNeeded, hidden=True) troveNo = 0 for csTrove in newList: if csTrove.troveType() == trove.TROVE_TYPE_REMOVED: # deal with these later on to ensure any changesets which # are relative to removed troves can be processed continue troveNo += 1 if callback: callback.creatingDatabaseTransaction(troveNo, len(newList)) newVersion = csTrove.getNewVersion() oldTroveVersion = csTrove.getOldVersion() oldTroveFlavor = csTrove.getOldFlavor() troveName = csTrove.getName() troveFlavor = csTrove.getNewFlavor() if repos.hasTrove(troveName, newVersion, troveFlavor): raise errors.CommitError, \ "version %s of %s already exists" % \ (newVersion.asString(), csTrove.getName()) if oldTroveVersion: newTrove = oldTroveIter.next() assert (newTrove.getNameVersionFlavor() == csTrove.getOldNameVersionFlavor()) self.oldTrove(newTrove, csTrove, troveName, oldTroveVersion, oldTroveFlavor) oldCompatClass = newTrove.getCompatibilityClass() if csTrove.isRollbackFence( oldCompatibilityClass=oldCompatClass, update=True): self.invalidateRollbacks(set=True) else: newTrove = trove.Trove(csTrove.getName(), newVersion, troveFlavor, csTrove.getChangeLog(), setVersion=False) # FIXME: we reset the trove version # since in this case we need to use the fileMap returned # from applyChangeSet allowIncomplete = True newFileMap = newTrove.applyChangeSet( csTrove, needNewFileMap=True, allowIncomplete=allowIncomplete) if newTrove.troveInfo.incomplete(): log.warning( 'trove %s has schema version %s, which contains' ' information not handled by this client. This' ' version of Conary understands schema version %s.' ' Dropping extra information. Please upgrade conary.', newTrove.getName(), newTrove.troveInfo.troveVersion(), trove.TROVE_VERSION) self.checkTroveCompleteness(newTrove) self.checkTroveSignatures(newTrove, callback=callback) if oldTroveVersion is not None: troveInfo = self.addTrove( (troveName, oldTroveVersion, oldTroveFlavor), newTrove, csTrove, hidden=hidden) else: troveInfo = self.addTrove(None, newTrove, csTrove, hidden=hidden) checkFilesList += self._getCheckFilesList(csTrove, troveInfo, fileHostFilter, configRestoreList, normalRestoreList, restoreContents=True) for (pathId, path, fileId, newVersion) in \ newTrove.iterFileList(members = True, capsules = True): # handle files which haven't changed; we know which those # are because they're in the merged trove but they aren't # in the newFileMap if pathId in newFileMap: continue self.addFileVersion(troveInfo, pathId, path, fileId, newVersion, withContents=True) filesNeeded = [] for i, (pathId, path, fileId, newVersion) in enumerate(csTrove.getChangedFileList()): tup = newFileMap[pathId] (oldPath, oldFileId, oldVersion) = tup[-3:] if path is None: path = oldPath if fileId is None: oldFileId = fileId if newVersion is None: newVersion = oldVersion if (fileHostFilter and newVersion.getHost() not in fileHostFilter): fileStream = None elif (oldVersion == newVersion and oldFileId == fileId): # the file didn't change between versions; we can just # ignore it fileStream = None else: fileStream = cs.getFileChange(oldFileId, fileId) if fileStream and fileStream[0] == "\x01": if len(fileStream) != 2: # This is awful, but this is how we say a file # stream didn't change. Omitting it or at least '' # would be nicer, but would break clients. filesNeeded.append( (i, (pathId, oldFileId, oldVersion))) continue fileStream = None # None is the file object self.addFileVersion(troveInfo, pathId, path, fileId, newVersion, fileStream=fileStream, withContents=True) if fileStream is not None: self._handleContents(pathId, fileId, fileStream, configRestoreList, normalRestoreList, oldFileId=oldFileId, oldVersion=oldVersion, oldfile=None, restoreContents=True) oldFileObjects = list( repos.getFileVersions([x[1] for x in filesNeeded])) for i, (pathId, path, fileId, newVersion) in enumerate(csTrove.getChangedFileList()): if not filesNeeded or filesNeeded[0][0] != i: continue filesNeeded.pop(0) tup = newFileMap[pathId] (oldPath, oldFileId, oldVersion) = tup[-3:] if path is None: path = oldPath if fileId is None: oldFileId = fileId if newVersion is None: newVersion = oldVersion restoreContents = True diff = cs.getFileChange(oldFileId, fileId) # stored as a diff (the file type is the same # and (for *repository* commits) the file # is in the same repository between versions oldfile = oldFileObjects.pop(0) fileObj = oldfile.copy() fileObj.twm(diff, oldfile) assert (fileObj.pathId() == pathId) fileStream = fileObj.freeze() if (not mirror) and ( fileObj.hasContents and fileObj.contents.sha1() == oldfile.contents.sha1() and not (fileObj.flags.isConfig() and not oldfile.flags.isConfig())): # don't restore the contents here. we don't # need them, and they may be relative to # something from a different repository restoreContents = False if fileObj and fileObj.fileId() != fileId: raise trove.TroveIntegrityError( csTrove.getName(), csTrove.getNewVersion(), csTrove.getNewFlavor(), "fileObj.fileId() != fileId in changeset") self.addFileVersion(troveInfo, pathId, path, fileId, newVersion, fileStream=fileStream, withContents=restoreContents) self._handleContents(pathId, fileId, fileStream, configRestoreList, normalRestoreList, oldFileId=oldFileId, oldVersion=oldVersion, oldfile=oldfile, restoreContents=restoreContents) del newFileMap self.addTroveDone(troveInfo, mirror=mirror) try: # we need to actualize this, not just get a generator list(repos.getFileVersions(checkFilesList)) except errors.FileStreamMissing, e: info = [x for x in checkFilesList if x[1] == e.fileId] (pathId, fileId) = info[0][0:2] # Missing from the repo; raise exception raise errors.IntegrityError( "Incomplete changeset specified: missing pathId %s " "fileId %s" % (sha1helper.md5ToString(pathId), sha1helper.sha1ToString(fileId)))
def verifySignatures(f, validKeys = None): """ Given an extended file, compute signatures """ f.seek(0) h = readHeader(f) # Cheap test first: verify MD5 sig sigmd5 = h.get(SIG_MD5, None) if sigmd5 is not None: f.seek(0) readSignatureHeader(f) # verify md5 digest md5 = digestlib.md5() util.copyfileobj(f, NullWriter(), digest = md5) if md5.digest() != sigmd5: raise MD5SignatureError( "The MD5 digest fails to verify: expected %s, got %s" % (sha1helper.md5ToString(sigmd5), md5.hexdigest())) # Don't bother if no gpg signature was present, or no valid keys were # presented if validKeys is None: return sigString = h.get(SIG_GPG, None) if sigString is None: return # Skip to immutable header region f.seek(0) readSignatureHeader(f) sig = openpgpfile.readSignature(sigString) keyId = sig.getSignerKeyId() matchingKeys = [ x for x in validKeys if x.hasKeyId(keyId) ] if not matchingKeys: raise PGPSignatureError("Signature generated with key %s does " "not match valid keys %s" % (keyId, ', '.join(x.getKeyId() for x in validKeys))) key = matchingKeys[0] # signature verification assumes a seekable stream and will seek to the # beginning; use a SeekableNestedFile size = h.getHeaderPlusPayloadSize() if size is None: pos = f.tell() f.seek(0, 2) size = f.tell() f.seek(pos, 0) snf = None if hasattr(f, 'pread'): extFile = f elif hasattr(f, 'name'): extFile = util.ExtendedFile(f.name, buffering = False) else: # worst case scenario, we slurp everything in memory extFile = util.ExtendedStringIO(f.read()) snf = extFile if snf is None: snf = util.SeekableNestedFile(extFile, start = f.tell(), size = size) try: sig.verifyDocument(key.getCryptoKey(), snf) except openpgpfile.SignatureError: raise PGPSignatureError
def checkMd5(self, path, md5): d = md5String(open(self.rootDir + path).read()) self.assertEquals(md5ToString(d), md5)
def getTrove(cu, roleIds, name, version, flavor, mkUrl = None, thisHost = None, displayFlavor = None): def buildTupleList(tuples, name, mkUrl = mkUrl): l = getattr(datamodel.SingleTrove, name)() for troveInfo in sorted(tuples.iter()): l.append(name = troveInfo.name(), version = troveInfo.version(), flavor = troveInfo.flavor(), mkUrl = mkUrl) return l def fileQuery(gfcu, filesInstanceId, dirName = None): # XXX restricing by dirName seems and obvious thing to do here, # but it actually slows things down?? # # the distinct here is unfortunate, but conary repositories had # a bug for about a year which caused it to store duplicate paths # if a path was committed for the first time duplicate times in # a single commit job gfcu.execute(""" SELECT DISTINCT dirName, basename, version, pathId, fileId FROM TroveFiles JOIN Versions USING (versionId) JOIN FileStreams ON (TroveFiles.streamId = FileStreams.streamId) JOIN FilePaths ON (TroveFiles.filePathId = FilePaths.filePathId) JOIN DirNames ON FilePaths.dirNameId = DirNames.dirNameId JOIN Basenames ON (FilePaths.baseNameId = Basenames.baseNameId) WHERE TroveFiles.instanceId = ? ORDER BY dirName, basename """, filesInstanceId) cu.execute(""" SELECT Instances.instanceId, Nodes.timeStamps FROM Instances JOIN Nodes USING (itemId, versionId) JOIN Items USING (itemId) JOIN Versions ON (Instances.versionId = Versions.versionId) JOIN Flavors ON (Instances.flavorId = Flavors.flavorId) JOIN UserGroupInstancesCache AS ugi ON (instances.instanceId = ugi.instanceId AND ugi.userGroupId in (%s)) WHERE item = ? AND version = ? AND flavor = ? """ % ",".join( str(x) for x in roleIds), name, version, deps.parseFlavor(flavor).freeze()) l = [ (x[0], x[1]) for x in cu ] if not l: return None instanceId, timeStamps = l[0] frzVer = versions.strToFrozen(version, timeStamps.split(":")) verobj = versions.ThawVersion(frzVer) tupleLists = [ ( trove._TROVEINFO_TAG_BUILDDEPS, 'builddeps' ), ( trove._TROVEINFO_TAG_POLICY_PROV, 'policyprovider' ), ( trove._TROVEINFO_TAG_LOADEDTROVES, 'loadedtroves' ), ( trove._TROVEINFO_TAG_COPIED_FROM, 'copiedfrom' ), ( trove._TROVEINFO_TAG_DERIVEDFROM, 'derivedfrom' ) ] cu.execute(""" SELECT infoType, data FROM TroveInfo WHERE instanceId = ? AND infoType IN (%s) """ % ",".join(str(x) for x in [ trove._TROVEINFO_TAG_SOURCENAME, trove._TROVEINFO_TAG_CLONEDFROM, trove._TROVEINFO_TAG_CLONEDFROMLIST, trove._TROVEINFO_TAG_BUILDTIME, trove._TROVEINFO_TAG_SIZE, trove._TROVEINFO_TAG_METADATA, trove._TROVEINFO_TAG_CAPSULE, ] + [ x[0] for x in tupleLists ] ), instanceId) troveInfo = {} for infoType, data in cu: data = cu.frombinary(data) infoClass = trove.TroveInfo.streamDict[infoType][1] troveInfo[infoType] = infoClass(data) kwargs = { 'name' : name, 'version' : verobj, 'flavor' : flavor } if displayFlavor is not None: kwargs['displayflavor'] = displayFlavor if trove._TROVEINFO_TAG_BUILDTIME in troveInfo: kwargs['buildtime'] = int(troveInfo[trove._TROVEINFO_TAG_BUILDTIME]()) if trove._TROVEINFO_TAG_SOURCENAME in troveInfo: kwargs['source'] = (troveInfo[trove._TROVEINFO_TAG_SOURCENAME](), verobj.getSourceVersion(), '') if trove._TROVEINFO_TAG_SIZE in troveInfo: kwargs['size'] = troveInfo[trove._TROVEINFO_TAG_SIZE]() if trove._TROVEINFO_TAG_METADATA in troveInfo: md = troveInfo[trove._TROVEINFO_TAG_METADATA].get() kwargs['shortdesc'] = md['shortDesc'] kwargs['longdesc'] = md['longDesc'] if md['licenses']: kwargs['license'] = [ x for x in md['licenses' ]] if md['crypto']: kwargs['crypto'] = [ x for x in md['crypto'] ] for (tag, tagName) in tupleLists: if tag in troveInfo: kwargs[tagName] = buildTupleList(troveInfo[tag], tagName, mkUrl = mkUrl) t = datamodel.SingleTrove(mkUrl = mkUrl, thisHost = thisHost, **kwargs) if trove._TROVEINFO_TAG_CLONEDFROMLIST in troveInfo: clonedFromList = troveInfo[trove._TROVEINFO_TAG_CLONEDFROMLIST] elif (trove._TROVEINFO_TAG_CLONEDFROM in troveInfo): clonedFromList = [ troveInfo[trove._TROVEINFO_TAG_CLONEDFROM]() ] else: clonedFromList = [] for ver in clonedFromList: t.addClonedFrom(name, ver, flavor, mkUrl = mkUrl) hasCapsule = False if trove._TROVEINFO_TAG_CAPSULE in troveInfo: if troveInfo[trove._TROVEINFO_TAG_CAPSULE].type(): hasCapsule = True fileQuery(cu, instanceId) for (dirName, baseName, fileVersion, pathId, fileId) in cu: dirName = cu.frombinary(dirName) baseName = cu.frombinary(baseName) if pathId == trove.CAPSULE_PATHID: isCapsule = 1 contentAvailable = True else: isCapsule = None contentAvailable = not hasCapsule fileObj = datamodel.FileReference( path = os.path.join(dirName, baseName), version = fileVersion, pathId = md5ToString(cu.frombinary(pathId)), fileId = sha1ToString(cu.frombinary(fileId)), isCapsule = isCapsule, contentAvailable = contentAvailable, mkUrl = mkUrl, thisHost = thisHost) t.addFile(fileObj) cu.execute(""" SELECT item, version, flavor, TroveTroves.includedId, Nodes.timeStamps FROM TroveTroves JOIN Instances ON (Instances.instanceId = TroveTroves.includedId) JOIN Nodes USING (itemId, versionId) JOIN Items USING (itemId) JOIN Versions ON (Versions.versionId = Instances.versionId) JOIN Flavors ON (Flavors.flavorId = Instances.flavorId) WHERE TroveTroves.instanceId = ? AND (TroveTroves.flags & %d) = 0 ORDER BY item, version, flavor """ % schema.TROVE_TROVES_WEAKREF, instanceId) for (subName, subVersion, subFlavor, refInstanceId, subTS) in cu: subFlavor = str(deps.ThawFlavor(subFlavor)) frzVer = versions.strToFrozen(subVersion, [ x for x in subTS.split(":") ]) subV = versions.ThawVersion(frzVer) t.addReferencedTrove(subName, subV, subFlavor, mkUrl = mkUrl) # It would be far better to use file tags to identify these build # logs, but it's significantly slower as well because they're in # the file objects rather than the trove (and those file objects # could be stored on a different repository) if not subName.endswith(':debuginfo'): continue fileQuery(cu, refInstanceId, dirName = '/usr/src/debug/buildlogs') logHost = subV.getHost() for (dirName, baseName, fileVersion, pathId, fileId) in cu: if (dirName) != '/usr/src/debug/buildlogs': continue if baseName.endswith('-log.bz2'): t.setBuildLog(logHost, sha1ToString(fileId)) elif baseName.endswith('-xml.bz2'): t.setXMLBuildLog(logHost, sha1ToString(fileId)) return t