def writeDescFile(self): """ Rewrites the desc file with the new patch information. """ if not self.anyChanges: # No need to rewrite. return xpackage = self.doc.FirstChildElement('package') if not xpackage: return packageSeq = SeqValue() packageSeq.loadXml(xpackage, 'seq') packageSeq += 1 packageSeq.storeXml(xpackage, 'seq') # Remove all of the old patch entries from the desc file # we read earlier. xremove = [] for value in ['base_version', 'top_version', 'patch']: xpatch = xpackage.FirstChildElement(value) while xpatch: xremove.append(xpatch) xpatch = xpatch.NextSiblingElement(value) for xelement in xremove: xpackage.RemoveChild(xelement) xpackage.RemoveAttribute('last_patch_version') # Now replace them with the current patch information. xpackage.SetAttribute('patch_version', str(self.patchVersion)) xarchive = TiXmlElement('base_version') self.baseFile.storeXml(xarchive) xpackage.InsertEndChild(xarchive) # The current version is now the top version. xarchive = TiXmlElement('top_version') self.currentFile.storeXml(xarchive) xpackage.InsertEndChild(xarchive) for patchfile in self.patches: xpatch = patchfile.makeXml(self) xpackage.InsertEndChild(xpatch) self.doc.SaveFile() # Also copy the seq to the import desc file, for # documentation purposes. importDescFilename = str(self.packageDesc)[:-3] + 'import.xml' importDescFullpath = Filename(self.patchMaker.installDir, importDescFilename) doc = TiXmlDocument(importDescFullpath.toOsSpecific()) if doc.LoadFile(): xpackage = doc.FirstChildElement('package') if xpackage: packageSeq.storeXml(xpackage, 'seq') doc.SaveFile() else: print("Couldn't read %s" % (importDescFullpath)) if self.contentsDocPackage: # Now that we've rewritten the xml file, we have to # change the contents.xml file that references it to # indicate the new file hash. fileSpec = FileSpec() fileSpec.fromFile(self.patchMaker.installDir, self.packageDesc) fileSpec.storeXml(self.contentsDocPackage) # Also important to update the import.xml hash. ximport = self.contentsDocPackage.FirstChildElement('import') if ximport: fileSpec = FileSpec() fileSpec.fromFile(self.patchMaker.installDir, importDescFilename) fileSpec.storeXml(ximport) # Also copy the package seq value into the # contents.xml file, mainly for documentation purposes # (the authoritative seq value is within the desc # file). packageSeq.storeXml(self.contentsDocPackage, 'seq')
class Patchfile: """ A single patchfile for a package. """ def __init__(self, package): self.package = package self.packageName = package.packageName self.platform = package.platform self.version = package.version self.hostUrl = None # FileSpec for the patchfile itself self.file = None # FileSpec for the package file that the patch is applied to self.sourceFile = None # FileSpec for the package file that the patch generates self.targetFile = None # The PackageVersion corresponding to our sourceFile self.fromPv = None # The PackageVersion corresponding to our targetFile self.toPv = None def getSourceKey(self): """ Returns the key for locating the package that this patchfile can be applied to. """ return (self.packageName, self.platform, self.version, self.hostUrl, self.sourceFile) def getTargetKey(self): """ Returns the key for locating the package that this patchfile will generate. """ return (self.packageName, self.platform, self.version, self.hostUrl, self.targetFile) def fromFile(self, packageDir, patchFilename, sourceFile, targetFile): """ Creates the data structures from an existing patchfile on disk. """ self.file = FileSpec() self.file.fromFile(packageDir, patchFilename) self.sourceFile = sourceFile self.targetFile = targetFile def loadXml(self, xpatch): """ Reads the data structures from an xml file. """ self.packageName = xpatch.Attribute('name') or self.packageName self.platform = xpatch.Attribute('platform') or self.platform self.version = xpatch.Attribute('version') or self.version self.hostUrl = xpatch.Attribute('host') or self.hostUrl self.file = FileSpec() self.file.loadXml(xpatch) xsource = xpatch.FirstChildElement('source') if xsource: self.sourceFile = FileSpec() self.sourceFile.loadXml(xsource) xtarget = xpatch.FirstChildElement('target') if xtarget: self.targetFile = FileSpec() self.targetFile.loadXml(xtarget) def makeXml(self, package): xpatch = TiXmlElement('patch') if self.packageName != package.packageName: xpatch.SetAttribute('name', self.packageName) if self.platform != package.platform: xpatch.SetAttribute('platform', self.platform) if self.version != package.version: xpatch.SetAttribute('version', self.version) if self.hostUrl != package.hostUrl: xpatch.SetAttribute('host', self.hostUrl) self.file.storeXml(xpatch) xsource = TiXmlElement('source') self.sourceFile.storeMiniXml(xsource) xpatch.InsertEndChild(xsource) xtarget = TiXmlElement('target') self.targetFile.storeMiniXml(xtarget) xpatch.InsertEndChild(xtarget) return xpatch
def readDescFile(self, doProcessing=False): """ Reads the existing package.xml file and stores it in this class for later rewriting. if doProcessing is true, it may massage the file and the directory contents in preparation for building patches. Returns true on success, false on failure. """ self.anyChanges = False packageDescFullpath = Filename(self.patchMaker.installDir, self.packageDesc) self.doc = TiXmlDocument(packageDescFullpath.toOsSpecific()) if not self.doc.LoadFile(): print("Couldn't read %s" % (packageDescFullpath)) return False xpackage = self.doc.FirstChildElement('package') if not xpackage: return False self.packageName = xpackage.Attribute('name') self.platform = xpackage.Attribute('platform') self.version = xpackage.Attribute('version') # All packages we defined in-line are assigned to the # "none" host. TODO: support patching from packages on # other hosts, which means we'll need to fill in a value # here for those hosts. self.hostUrl = None self.currentFile = None self.baseFile = None self.topFile = None self.compressedFilename = None compressedFile = None # Assume there are changes for this version, until we # discover that there aren't. isNewVersion = True # Get the actual current version. xarchive = xpackage.FirstChildElement('uncompressed_archive') if xarchive: self.currentFile = FileSpec() self.currentFile.loadXml(xarchive) # Get the top_version--the top (newest) of the patch # chain. xarchive = xpackage.FirstChildElement('top_version') if xarchive: self.topFile = FileSpec() self.topFile.loadXml(xarchive) if self.topFile.hash == self.currentFile.hash: # No new version this pass. isNewVersion = False else: # There's a new version this pass. Update it. self.anyChanges = True else: # If there isn't a top_version yet, we have to make # one, by duplicating the currentFile. self.topFile = copy.copy(self.currentFile) self.anyChanges = True # Get the current patch version. If we have a # patch_version attribute, it refers to this particular # instance of the file, and that is the current patch # version number. If we only have a last_patch_version # attribute, it means a patch has not yet been built for # this particular instance, and that number is the # previous version's patch version number. patchVersion = xpackage.Attribute('patch_version') if patchVersion: self.patchVersion = int(patchVersion) else: patchVersion = xpackage.Attribute('last_patch_version') if patchVersion: self.patchVersion = int(patchVersion) if isNewVersion: self.patchVersion += 1 self.anyChanges = True # Put the patchVersion in the compressed filename, for # cache-busting. This means when the version changes, its # URL will also change, guaranteeing that users will # download the latest version, and not some stale cache # file. xcompressed = xpackage.FirstChildElement('compressed_archive') if xcompressed: compressedFile = FileSpec() compressedFile.loadXml(xcompressed) oldCompressedFilename = compressedFile.filename self.compressedFilename = oldCompressedFilename if doProcessing: newCompressedFilename = '%s.%s.pz' % ( self.currentFile.filename, self.patchVersion) if newCompressedFilename != oldCompressedFilename: oldCompressedPathname = Filename( self.packageDir, oldCompressedFilename) newCompressedPathname = Filename( self.packageDir, newCompressedFilename) if oldCompressedPathname.renameTo( newCompressedPathname): compressedFile.fromFile(self.packageDir, newCompressedFilename) compressedFile.storeXml(xcompressed) self.compressedFilename = newCompressedFilename self.anyChanges = True # Get the base_version--the bottom (oldest) of the patch # chain. xarchive = xpackage.FirstChildElement('base_version') if xarchive: self.baseFile = FileSpec() self.baseFile.loadXml(xarchive) else: # If there isn't a base_version yet, we have to make # one, by duplicating the currentFile. self.baseFile = copy.copy(self.currentFile) # Note that the we only store the compressed version # of base_filename on disk, but we store the md5 of # the uncompressed version in the xml file. To # emphasize this, we name it without the .pz extension # in the xml file, even though the compressed file on # disk actually has a .pz extension. self.baseFile.filename += '.base' # Also duplicate the (compressed) file itself. if doProcessing and self.compressedFilename: fromPathname = Filename(self.packageDir, self.compressedFilename) toPathname = Filename(self.packageDir, self.baseFile.filename + '.pz') fromPathname.copyTo(toPathname) self.anyChanges = True self.patches = [] xpatch = xpackage.FirstChildElement('patch') while xpatch: patchfile = PatchMaker.Patchfile(self) patchfile.loadXml(xpatch) self.patches.append(patchfile) xpatch = xpatch.NextSiblingElement('patch') return True
def writeDescFile(self): """ Rewrites the desc file with the new patch information. """ if not self.anyChanges: # No need to rewrite. return xpackage = self.doc.FirstChildElement('package') if not xpackage: return packageSeq = SeqValue() packageSeq.loadXml(xpackage, 'seq') packageSeq += 1 packageSeq.storeXml(xpackage, 'seq') # Remove all of the old patch entries from the desc file # we read earlier. xremove = [] for value in ['base_version', 'top_version', 'patch']: xpatch = xpackage.FirstChildElement(value) while xpatch: xremove.append(xpatch) xpatch = xpatch.NextSiblingElement(value) for xelement in xremove: xpackage.RemoveChild(xelement) xpackage.RemoveAttribute('last_patch_version') # Now replace them with the current patch information. xpackage.SetAttribute('patch_version', str(self.patchVersion)) xarchive = TiXmlElement('base_version') self.baseFile.storeXml(xarchive) xpackage.InsertEndChild(xarchive) # The current version is now the top version. xarchive = TiXmlElement('top_version') self.currentFile.storeXml(xarchive) xpackage.InsertEndChild(xarchive) for patchfile in self.patches: xpatch = patchfile.makeXml(self) xpackage.InsertEndChild(xpatch) self.doc.SaveFile() # Also copy the seq to the import desc file, for # documentation purposes. importDescFilename = self.packageDesc.cStr()[:-3] + 'import.xml' importDescFullpath = Filename(self.patchMaker.installDir, importDescFilename) doc = TiXmlDocument(importDescFullpath.toOsSpecific()) if doc.LoadFile(): xpackage = doc.FirstChildElement('package') if xpackage: packageSeq.storeXml(xpackage, 'seq') doc.SaveFile() else: print "Couldn't read %s" % (importDescFullpath) if self.contentsDocPackage: # Now that we've rewritten the xml file, we have to # change the contents.xml file that references it to # indicate the new file hash. fileSpec = FileSpec() fileSpec.fromFile(self.patchMaker.installDir, self.packageDesc) fileSpec.storeXml(self.contentsDocPackage) # Also important to update the import.xml hash. ximport = self.contentsDocPackage.FirstChildElement('import') if ximport: fileSpec = FileSpec() fileSpec.fromFile(self.patchMaker.installDir, importDescFilename) fileSpec.storeXml(ximport) # Also copy the package seq value into the # contents.xml file, mainly for documentation purposes # (the authoritative seq value is within the desc # file). packageSeq.storeXml(self.contentsDocPackage, 'seq')
def readDescFile(self, doProcessing = False): """ Reads the existing package.xml file and stores it in this class for later rewriting. if doProcessing is true, it may massage the file and the directory contents in preparation for building patches. Returns true on success, false on failure. """ self.anyChanges = False packageDescFullpath = Filename(self.patchMaker.installDir, self.packageDesc) self.doc = TiXmlDocument(packageDescFullpath.toOsSpecific()) if not self.doc.LoadFile(): print "Couldn't read %s" % (packageDescFullpath) return False xpackage = self.doc.FirstChildElement('package') if not xpackage: return False self.packageName = xpackage.Attribute('name') self.platform = xpackage.Attribute('platform') self.version = xpackage.Attribute('version') # All packages we defined in-line are assigned to the # "none" host. TODO: support patching from packages on # other hosts, which means we'll need to fill in a value # here for those hosts. self.hostUrl = None self.currentFile = None self.baseFile = None self.topFile = None self.compressedFilename = None compressedFile = None # Assume there are changes for this version, until we # discover that there aren't. isNewVersion = True # Get the actual current version. xarchive = xpackage.FirstChildElement('uncompressed_archive') if xarchive: self.currentFile = FileSpec() self.currentFile.loadXml(xarchive) # Get the top_version--the top (newest) of the patch # chain. xarchive = xpackage.FirstChildElement('top_version') if xarchive: self.topFile = FileSpec() self.topFile.loadXml(xarchive) if self.topFile.hash == self.currentFile.hash: # No new version this pass. isNewVersion = False else: # There's a new version this pass. Update it. self.anyChanges = True else: # If there isn't a top_version yet, we have to make # one, by duplicating the currentFile. self.topFile = copy.copy(self.currentFile) self.anyChanges = True # Get the current patch version. If we have a # patch_version attribute, it refers to this particular # instance of the file, and that is the current patch # version number. If we only have a last_patch_version # attribute, it means a patch has not yet been built for # this particular instance, and that number is the # previous version's patch version number. patchVersion = xpackage.Attribute('patch_version') if patchVersion: self.patchVersion = int(patchVersion) else: patchVersion = xpackage.Attribute('last_patch_version') if patchVersion: self.patchVersion = int(patchVersion) if isNewVersion: self.patchVersion += 1 self.anyChanges = True # Put the patchVersion in the compressed filename, for # cache-busting. This means when the version changes, its # URL will also change, guaranteeing that users will # download the latest version, and not some stale cache # file. xcompressed = xpackage.FirstChildElement('compressed_archive') if xcompressed: compressedFile = FileSpec() compressedFile.loadXml(xcompressed) oldCompressedFilename = compressedFile.filename self.compressedFilename = oldCompressedFilename if doProcessing: newCompressedFilename = '%s.%s.pz' % (self.currentFile.filename, self.patchVersion) if newCompressedFilename != oldCompressedFilename: oldCompressedPathname = Filename(self.packageDir, oldCompressedFilename) newCompressedPathname = Filename(self.packageDir, newCompressedFilename) if oldCompressedPathname.renameTo(newCompressedPathname): compressedFile.fromFile(self.packageDir, newCompressedFilename) compressedFile.storeXml(xcompressed) self.compressedFilename = newCompressedFilename self.anyChanges = True # Get the base_version--the bottom (oldest) of the patch # chain. xarchive = xpackage.FirstChildElement('base_version') if xarchive: self.baseFile = FileSpec() self.baseFile.loadXml(xarchive) else: # If there isn't a base_version yet, we have to make # one, by duplicating the currentFile. self.baseFile = copy.copy(self.currentFile) # Note that the we only store the compressed version # of base_filename on disk, but we store the md5 of # the uncompressed version in the xml file. To # emphasize this, we name it without the .pz extension # in the xml file, even though the compressed file on # disk actually has a .pz extension. self.baseFile.filename += '.base' # Also duplicate the (compressed) file itself. if doProcessing and self.compressedFilename: fromPathname = Filename(self.packageDir, self.compressedFilename) toPathname = Filename(self.packageDir, self.baseFile.filename + '.pz') fromPathname.copyTo(toPathname) self.anyChanges = True self.patches = [] xpatch = xpackage.FirstChildElement('patch') while xpatch: patchfile = PatchMaker.Patchfile(self) patchfile.loadXml(xpatch) self.patches.append(patchfile) xpatch = xpatch.NextSiblingElement('patch') return True
def __buildInstallPlans(self): """ Sets up self.installPlans, a list of one or more "plans" to download and install the package. """ pc = PStatCollector(':App:PackageInstaller:buildInstallPlans') pc.start() self.hasPackage = False if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever: # We're not allowed to download anything. self.installPlans = [] pc.stop() return if self.asMirror: # If we're just downloading a mirror archive, we only need # to get the compressed archive file. # Build a one-item install plan to download the compressed # archive. downloadSize = self.compressedArchive.size func = lambda step, fileSpec=self.compressedArchive: self.__downloadFile( step, fileSpec, allowPartial=True) step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download') installPlan = [step] self.installPlans = [installPlan] pc.stop() return # The normal download process. Determine what we will need to # download, and build a plan (or two) to download it all. self.installPlans = None # We know we will at least need to unpack the archive contents # at the end. unpackSize = 0 for file in self.extracts: unpackSize += file.size step = self.InstallStep(self.__unpackArchive, unpackSize, self.unpackFactor, 'unpack') planA = [step] # If the uncompressed archive file is good, that's all we'll # need to do. self.uncompressedArchive.actualFile = None if self.uncompressedArchive.quickVerify(self.getPackageDir(), notify=self.notify): self.installPlans = [planA] pc.stop() return # Maybe the compressed archive file is good. if self.compressedArchive.quickVerify(self.getPackageDir(), notify=self.notify): uncompressSize = self.uncompressedArchive.size step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor, 'uncompress') planA = [step] + planA self.installPlans = [planA] pc.stop() return # Maybe we can download one or more patches. We'll come back # to that in a minute as plan A. For now, construct plan B, # which will be to download the whole archive. planB = planA[:] uncompressSize = self.uncompressedArchive.size step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor, 'uncompress') planB = [step] + planB downloadSize = self.compressedArchive.size func = lambda step, fileSpec=self.compressedArchive: self.__downloadFile( step, fileSpec, allowPartial=True) step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download') planB = [step] + planB # Now look for patches. Start with the md5 hash from the # uncompressedArchive file we have on disk, and see if we can # find a patch chain from this file to our target. pathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) fileSpec = self.uncompressedArchive.actualFile if fileSpec is None and pathname.exists(): fileSpec = FileSpec() fileSpec.fromFile(self.getPackageDir(), self.uncompressedArchive.filename) plan = None if fileSpec: plan = self.__findPatchChain(fileSpec) if plan: # We can download patches. Great! That means this is # plan A, and the full download is plan B (in case # something goes wrong with the patching). planA = plan + planA self.installPlans = [planA, planB] else: # There are no patches to download, oh well. Stick with # plan B as the only plan. self.installPlans = [planB] # In case of unexpected failures on the internet, we will retry # the full download instead of just giving up. for retry in range(ConfigVariableInt('package-full-dl-retries', 1)): self.installPlans.append(planB[:]) pc.stop()
def __buildInstallPlans(self): """ Sets up self.installPlans, a list of one or more "plans" to download and install the package. """ pc = PStatCollector(':App:PackageInstaller:buildInstallPlans') pc.start() self.hasPackage = False if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever: # We're not allowed to download anything. self.installPlans = [] pc.stop() return if self.asMirror: # If we're just downloading a mirror archive, we only need # to get the compressed archive file. # Build a one-item install plan to download the compressed # archive. downloadSize = self.compressedArchive.size func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec, allowPartial = True) step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download') installPlan = [step] self.installPlans = [installPlan] pc.stop() return # The normal download process. Determine what we will need to # download, and build a plan (or two) to download it all. self.installPlans = None # We know we will at least need to unpack the archive contents # at the end. unpackSize = 0 for file in self.extracts: unpackSize += file.size step = self.InstallStep(self.__unpackArchive, unpackSize, self.unpackFactor, 'unpack') planA = [step] # If the uncompressed archive file is good, that's all we'll # need to do. self.uncompressedArchive.actualFile = None if self.uncompressedArchive.quickVerify(self.getPackageDir(), notify = self.notify): self.installPlans = [planA] pc.stop() return # Maybe the compressed archive file is good. if self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify): uncompressSize = self.uncompressedArchive.size step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor, 'uncompress') planA = [step] + planA self.installPlans = [planA] pc.stop() return # Maybe we can download one or more patches. We'll come back # to that in a minute as plan A. For now, construct plan B, # which will be to download the whole archive. planB = planA[:] uncompressSize = self.uncompressedArchive.size step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor, 'uncompress') planB = [step] + planB downloadSize = self.compressedArchive.size func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec, allowPartial = True) step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download') planB = [step] + planB # Now look for patches. Start with the md5 hash from the # uncompressedArchive file we have on disk, and see if we can # find a patch chain from this file to our target. pathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) fileSpec = self.uncompressedArchive.actualFile if fileSpec is None and pathname.exists(): fileSpec = FileSpec() fileSpec.fromFile(self.getPackageDir(), self.uncompressedArchive.filename) plan = None if fileSpec: plan = self.__findPatchChain(fileSpec) if plan: # We can download patches. Great! That means this is # plan A, and the full download is plan B (in case # something goes wrong with the patching). planA = plan + planA self.installPlans = [planA, planB] else: # There are no patches to download, oh well. Stick with # plan B as the only plan. self.installPlans = [planB] # In case of unexpected failures on the internet, we will retry # the full download instead of just giving up. for retry in range(ConfigVariableInt('package-full-dl-retries', 1)): self.installPlans.append(planB[:]) pc.stop()