def loadXml(self, xpackage):
self.packageName = xpackage.Attribute('name')
self.platform = xpackage.Attribute('platform')
self.version = xpackage.Attribute('version')
solo = xpackage.Attribute('solo')
self.solo = int(solo or '0')
perPlatform = xpackage.Attribute('per_platform')
self.perPlatform = int(perPlatform or '0')
self.descFile = FileSpec()
self.descFile.loadXml(xpackage)
self.validatePackageContents()
self.descFile.quickVerify(packageDir=self.sourceDir,
notify=PackageMerger.notify,
correctSelf=True)
self.packageSeq = SeqValue()
self.packageSeq.loadXml(xpackage, 'seq')
self.packageSetVer = SeqValue()
self.packageSetVer.loadXml(xpackage, 'set_ver')
self.importDescFile = None
ximport = xpackage.FirstChildElement('import')
if ximport:
self.importDescFile = FileSpec()
self.importDescFile.loadXml(ximport)
self.importDescFile.quickVerify(packageDir=self.sourceDir,
notify=PackageMerger.notify,
correctSelf=True)
def validatePackageContents(self):
""" Validates the contents of the package directory itself
against the expected hashes and timestamps. Updates
hashes and timestamps where needed. """
if self.solo:
return
needsChange = False
packageDescFullpath = Filename(self.sourceDir,
self.descFile.filename)
packageDir = Filename(packageDescFullpath.getDirname())
doc = TiXmlDocument(packageDescFullpath.toOsSpecific())
if not doc.LoadFile():
message = "Could not read XML file: %s" % (
self.descFile.filename)
raise OSError, message
xpackage = doc.FirstChildElement('package')
if not xpackage:
message = "No package definition: %s" % (
self.descFile.filename)
raise OSError, message
xcompressed = xpackage.FirstChildElement('compressed_archive')
if xcompressed:
spec = FileSpec()
spec.loadXml(xcompressed)
if not spec.quickVerify(packageDir=packageDir,
notify=PackageMerger.notify,
correctSelf=True):
spec.storeXml(xcompressed)
needsChange = True
xpatch = xpackage.FirstChildElement('patch')
while xpatch:
spec = FileSpec()
spec.loadXml(xpatch)
if not spec.quickVerify(packageDir=packageDir,
notify=PackageMerger.notify,
correctSelf=True):
spec.storeXml(xpatch)
needsChange = True
xpatch = xpatch.NextSiblingElement('patch')
if needsChange:
PackageMerger.notify.info("Rewriting %s" %
(self.descFile.filename))
doc.SaveFile()
self.descFile.quickVerify(packageDir=self.sourceDir,
notify=PackageMerger.notify,
correctSelf=True)
def fromFile(self, packageDir, patchFilename, sourceFile, targetFile):
""" Creates the data structures from an existing patchfile
on disk. """
self.file = FileSpec()
self.file.fromFile(packageDir, patchFilename)
self.sourceFile = sourceFile
self.targetFile = targetFile
def loadXml(self, xpatch):
""" Reads the data structures from an xml file. """
self.packageName = xpatch.Attribute('name') or self.packageName
self.platform = xpatch.Attribute('platform') or self.platform
self.version = xpatch.Attribute('version') or self.version
self.hostUrl = xpatch.Attribute('host') or self.hostUrl
self.file = FileSpec()
self.file.loadXml(xpatch)
xsource = xpatch.FirstChildElement('source')
if xsource:
self.sourceFile = FileSpec()
self.sourceFile.loadXml(xsource)
xtarget = xpatch.FirstChildElement('target')
if xtarget:
self.targetFile = FileSpec()
self.targetFile.loadXml(xtarget)
def loadXml(self, xpackage):
self.packageName = xpackage.Attribute('name')
self.platform = xpackage.Attribute('platform')
self.version = xpackage.Attribute('version')
solo = xpackage.Attribute('solo')
self.solo = int(solo or '0')
self.descFile = FileSpec()
self.descFile.loadXml(xpackage)
self.packageSeq = SeqValue()
self.packageSeq.loadXml(xpackage, 'seq')
self.packageSetVer = SeqValue()
self.packageSetVer.loadXml(xpackage, 'set_ver')
self.importDescFile = None
ximport = xpackage.FirstChildElement('import')
if ximport:
self.importDescFile = FileSpec()
self.importDescFile.loadXml(ximport)
def writeDescFile(self):
""" Rewrites the desc file with the new patch
information. """
if not self.anyChanges:
# No need to rewrite.
return
xpackage = self.doc.FirstChildElement('package')
if not xpackage:
return
packageSeq = SeqValue()
packageSeq.loadXml(xpackage, 'seq')
packageSeq += 1
packageSeq.storeXml(xpackage, 'seq')
# Remove all of the old patch entries from the desc file
# we read earlier.
xremove = []
for value in ['base_version', 'top_version', 'patch']:
xpatch = xpackage.FirstChildElement(value)
while xpatch:
xremove.append(xpatch)
xpatch = xpatch.NextSiblingElement(value)
for xelement in xremove:
xpackage.RemoveChild(xelement)
xpackage.RemoveAttribute('last_patch_version')
# Now replace them with the current patch information.
xpackage.SetAttribute('patch_version', str(self.patchVersion))
xarchive = TiXmlElement('base_version')
self.baseFile.storeXml(xarchive)
xpackage.InsertEndChild(xarchive)
# The current version is now the top version.
xarchive = TiXmlElement('top_version')
self.currentFile.storeXml(xarchive)
xpackage.InsertEndChild(xarchive)
for patchfile in self.patches:
xpatch = patchfile.makeXml(self)
xpackage.InsertEndChild(xpatch)
self.doc.SaveFile()
# Also copy the seq to the import desc file, for
# documentation purposes.
importDescFilename = str(self.packageDesc)[:-3] + 'import.xml'
importDescFullpath = Filename(self.patchMaker.installDir,
importDescFilename)
doc = TiXmlDocument(importDescFullpath.toOsSpecific())
if doc.LoadFile():
xpackage = doc.FirstChildElement('package')
if xpackage:
packageSeq.storeXml(xpackage, 'seq')
doc.SaveFile()
else:
print("Couldn't read %s" % (importDescFullpath))
if self.contentsDocPackage:
# Now that we've rewritten the xml file, we have to
# change the contents.xml file that references it to
# indicate the new file hash.
fileSpec = FileSpec()
fileSpec.fromFile(self.patchMaker.installDir, self.packageDesc)
fileSpec.storeXml(self.contentsDocPackage)
# Also important to update the import.xml hash.
ximport = self.contentsDocPackage.FirstChildElement('import')
if ximport:
fileSpec = FileSpec()
fileSpec.fromFile(self.patchMaker.installDir,
importDescFilename)
fileSpec.storeXml(ximport)
# Also copy the package seq value into the
# contents.xml file, mainly for documentation purposes
# (the authoritative seq value is within the desc
# file).
packageSeq.storeXml(self.contentsDocPackage, 'seq')
def readDescFile(self, doProcessing=False):
""" Reads the existing package.xml file and stores it in
this class for later rewriting. if doProcessing is true,
it may massage the file and the directory contents in
preparation for building patches. Returns true on
success, false on failure. """
self.anyChanges = False
packageDescFullpath = Filename(self.patchMaker.installDir,
self.packageDesc)
self.doc = TiXmlDocument(packageDescFullpath.toOsSpecific())
if not self.doc.LoadFile():
print("Couldn't read %s" % (packageDescFullpath))
return False
xpackage = self.doc.FirstChildElement('package')
if not xpackage:
return False
self.packageName = xpackage.Attribute('name')
self.platform = xpackage.Attribute('platform')
self.version = xpackage.Attribute('version')
# All packages we defined in-line are assigned to the
# "none" host. TODO: support patching from packages on
# other hosts, which means we'll need to fill in a value
# here for those hosts.
self.hostUrl = None
self.currentFile = None
self.baseFile = None
self.topFile = None
self.compressedFilename = None
compressedFile = None
# Assume there are changes for this version, until we
# discover that there aren't.
isNewVersion = True
# Get the actual current version.
xarchive = xpackage.FirstChildElement('uncompressed_archive')
if xarchive:
self.currentFile = FileSpec()
self.currentFile.loadXml(xarchive)
# Get the top_version--the top (newest) of the patch
# chain.
xarchive = xpackage.FirstChildElement('top_version')
if xarchive:
self.topFile = FileSpec()
self.topFile.loadXml(xarchive)
if self.topFile.hash == self.currentFile.hash:
# No new version this pass.
isNewVersion = False
else:
# There's a new version this pass. Update it.
self.anyChanges = True
else:
# If there isn't a top_version yet, we have to make
# one, by duplicating the currentFile.
self.topFile = copy.copy(self.currentFile)
self.anyChanges = True
# Get the current patch version. If we have a
# patch_version attribute, it refers to this particular
# instance of the file, and that is the current patch
# version number. If we only have a last_patch_version
# attribute, it means a patch has not yet been built for
# this particular instance, and that number is the
# previous version's patch version number.
patchVersion = xpackage.Attribute('patch_version')
if patchVersion:
self.patchVersion = int(patchVersion)
else:
patchVersion = xpackage.Attribute('last_patch_version')
if patchVersion:
self.patchVersion = int(patchVersion)
if isNewVersion:
self.patchVersion += 1
self.anyChanges = True
# Put the patchVersion in the compressed filename, for
# cache-busting. This means when the version changes, its
# URL will also change, guaranteeing that users will
# download the latest version, and not some stale cache
# file.
xcompressed = xpackage.FirstChildElement('compressed_archive')
if xcompressed:
compressedFile = FileSpec()
compressedFile.loadXml(xcompressed)
oldCompressedFilename = compressedFile.filename
self.compressedFilename = oldCompressedFilename
if doProcessing:
newCompressedFilename = '%s.%s.pz' % (
self.currentFile.filename, self.patchVersion)
if newCompressedFilename != oldCompressedFilename:
oldCompressedPathname = Filename(
self.packageDir, oldCompressedFilename)
newCompressedPathname = Filename(
self.packageDir, newCompressedFilename)
if oldCompressedPathname.renameTo(
newCompressedPathname):
compressedFile.fromFile(self.packageDir,
newCompressedFilename)
compressedFile.storeXml(xcompressed)
self.compressedFilename = newCompressedFilename
self.anyChanges = True
# Get the base_version--the bottom (oldest) of the patch
# chain.
xarchive = xpackage.FirstChildElement('base_version')
if xarchive:
self.baseFile = FileSpec()
self.baseFile.loadXml(xarchive)
else:
# If there isn't a base_version yet, we have to make
# one, by duplicating the currentFile.
self.baseFile = copy.copy(self.currentFile)
# Note that the we only store the compressed version
# of base_filename on disk, but we store the md5 of
# the uncompressed version in the xml file. To
# emphasize this, we name it without the .pz extension
# in the xml file, even though the compressed file on
# disk actually has a .pz extension.
self.baseFile.filename += '.base'
# Also duplicate the (compressed) file itself.
if doProcessing and self.compressedFilename:
fromPathname = Filename(self.packageDir,
self.compressedFilename)
toPathname = Filename(self.packageDir,
self.baseFile.filename + '.pz')
fromPathname.copyTo(toPathname)
self.anyChanges = True
self.patches = []
xpatch = xpackage.FirstChildElement('patch')
while xpatch:
patchfile = PatchMaker.Patchfile(self)
patchfile.loadXml(xpatch)
self.patches.append(patchfile)
xpatch = xpatch.NextSiblingElement('patch')
return True
def readContentsFile(self, tempFilename = None, freshDownload = False):
""" Reads the contents.xml file for this particular host, once
it has been downloaded into the indicated temporary file.
Returns true on success, false if the contents file is not
already on disk or is unreadable.
If tempFilename is specified, it is the filename read, and it
is copied the file into the standard location if it's not
there already. If tempFilename is not specified, the standard
filename is read if it is known. """
if not hasattr(PandaModules, 'TiXmlDocument'):
return False
if not tempFilename:
if self.hostDir:
# If the filename is not specified, we can infer it
# if we already know our hostDir
hostDir = self.hostDir
else:
# Otherwise, we have to guess the hostDir.
hostDir = self.__determineHostDir(None, self.hostUrl)
tempFilename = Filename(hostDir, 'contents.xml')
doc = PandaModules.TiXmlDocument(tempFilename.toOsSpecific())
if not doc.LoadFile():
return False
xcontents = doc.FirstChildElement('contents')
if not xcontents:
return False
maxAge = xcontents.Attribute('max_age')
if maxAge:
try:
maxAge = int(maxAge)
except:
maxAge = None
if maxAge is None:
# Default max_age if unspecified (see p3d_plugin.h).
from direct.p3d.AppRunner import AppRunner
maxAge = AppRunner.P3D_CONTENTS_DEFAULT_MAX_AGE
# Get the latest possible expiration time, based on the max_age
# indication. Any expiration time later than this is in error.
now = int(time.time())
self.contentsExpiration = now + maxAge
if freshDownload:
self.contentsSpec.readHash(tempFilename)
# Update the XML with the new download information.
xorig = xcontents.FirstChildElement('orig')
while xorig:
xcontents.RemoveChild(xorig)
xorig = xcontents.FirstChildElement('orig')
xorig = PandaModules.TiXmlElement('orig')
self.contentsSpec.storeXml(xorig)
xorig.SetAttribute('expiration', str(self.contentsExpiration))
xcontents.InsertEndChild(xorig)
else:
# Read the download hash and expiration time from the XML.
expiration = None
xorig = xcontents.FirstChildElement('orig')
if xorig:
self.contentsSpec.loadXml(xorig)
expiration = xorig.Attribute('expiration')
if expiration:
try:
expiration = int(expiration)
except:
expiration = None
if not self.contentsSpec.hash:
self.contentsSpec.readHash(tempFilename)
if expiration is not None:
self.contentsExpiration = min(self.contentsExpiration, expiration)
# Look for our own entry in the hosts table.
if self.hostUrl:
self.__findHostXml(xcontents)
else:
assert self.hostDir
self.__findHostXmlForHostDir(xcontents)
if not self.hostDir:
self.hostDir = self.__determineHostDir(None, self.hostUrl)
# Get the list of packages available for download and/or import.
xpackage = xcontents.FirstChildElement('package')
while xpackage:
name = xpackage.Attribute('name')
platform = xpackage.Attribute('platform')
version = xpackage.Attribute('version')
try:
solo = int(xpackage.Attribute('solo') or '')
except ValueError:
solo = False
package = self.__makePackage(name, platform, version, solo)
package.descFile = FileSpec()
package.descFile.loadXml(xpackage)
package.setupFilenames()
package.importDescFile = None
ximport = xpackage.FirstChildElement('import')
if ximport:
package.importDescFile = FileSpec()
package.importDescFile.loadXml(ximport)
xpackage = xpackage.NextSiblingElement('package')
self.hasContentsFile = True
# Now save the contents.xml file into the standard location.
if not self.appRunner or self.appRunner.verifyContents != self.appRunner.P3DVCNever:
assert self.hostDir
filename = Filename(self.hostDir, 'contents.xml')
filename.makeDir()
if freshDownload:
doc.SaveFile(filename.toOsSpecific())
else:
if filename != tempFilename:
tempFilename.copyTo(filename)
return True
def __init__(self, hostUrl, appRunner = None, hostDir = None,
rootDir = None, asMirror = False, perPlatform = None):
""" You must specify either an appRunner or a hostDir to the
HostInfo constructor.
If you pass asMirror = True, it means that this HostInfo
object is to be used to populate a "mirror" folder, a
duplicate (or subset) of the contents hosted by a server.
This means when you use this HostInfo to download packages, it
will only download the compressed archive file and leave it
there. At the moment, mirror folders do not download old
patch files from the server.
If you pass perPlatform = True, then files are unpacked into a
platform-specific directory, which is appropriate when you
might be downloading multiple platforms. The default is
perPlatform = False, which means all files are unpacked into
the host directory directly, without an intervening
platform-specific directory name. If asMirror is True, then
the default is perPlatform = True. """
assert appRunner or rootDir or hostDir
self.__setHostUrl(hostUrl)
self.appRunner = appRunner
self.rootDir = rootDir
if rootDir is None and appRunner:
self.rootDir = appRunner.rootDir
if hostDir and not isinstance(hostDir, Filename):
hostDir = Filename.fromOsSpecific(hostDir)
self.hostDir = hostDir
self.asMirror = asMirror
self.perPlatform = perPlatform
if perPlatform is None:
self.perPlatform = asMirror
# Initially false, this is set true when the contents file is
# successfully read.
self.hasContentsFile = False
# This is the time value at which the current contents file is
# no longer valid.
self.contentsExpiration = 0
# Contains the md5 hash of the original contents.xml file.
self.contentsSpec = FileSpec()
# descriptiveName will be filled in later, when the
# contents file is read.
self.descriptiveName = None
# A list of known mirrors for this host, all URL's guaranteed
# to end with a slash.
self.mirrors = []
# A map of keyword -> altHost URL's. An altHost is different
# than a mirror; an altHost is an alternate URL to download a
# different (e.g. testing) version of this host's contents.
# It is rarely used.
self.altHosts = {}
# This is a dictionary of packages by (name, version). It
# will be filled in when the contents file is read.
self.packages = {}
if self.appRunner and self.appRunner.verifyContents != self.appRunner.P3DVCForce:
# Attempt to pre-read the existing contents.xml; maybe it
# will be current enough for our purposes.
self.readContentsFile()
def __buildInstallPlans(self):
""" Sets up self.installPlans, a list of one or more "plans"
to download and install the package. """
pc = PStatCollector(':App:PackageInstaller:buildInstallPlans')
pc.start()
self.hasPackage = False
if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
# We're not allowed to download anything.
self.installPlans = []
pc.stop()
return
if self.asMirror:
# If we're just downloading a mirror archive, we only need
# to get the compressed archive file.
# Build a one-item install plan to download the compressed
# archive.
downloadSize = self.compressedArchive.size
func = lambda step, fileSpec=self.compressedArchive: self.__downloadFile(
step, fileSpec, allowPartial=True)
step = self.InstallStep(func, downloadSize, self.downloadFactor,
'download')
installPlan = [step]
self.installPlans = [installPlan]
pc.stop()
return
# The normal download process. Determine what we will need to
# download, and build a plan (or two) to download it all.
self.installPlans = None
# We know we will at least need to unpack the archive contents
# at the end.
unpackSize = 0
for file in self.extracts:
unpackSize += file.size
step = self.InstallStep(self.__unpackArchive, unpackSize,
self.unpackFactor, 'unpack')
planA = [step]
# If the uncompressed archive file is good, that's all we'll
# need to do.
self.uncompressedArchive.actualFile = None
if self.uncompressedArchive.quickVerify(self.getPackageDir(),
notify=self.notify):
self.installPlans = [planA]
pc.stop()
return
# Maybe the compressed archive file is good.
if self.compressedArchive.quickVerify(self.getPackageDir(),
notify=self.notify):
uncompressSize = self.uncompressedArchive.size
step = self.InstallStep(self.__uncompressArchive, uncompressSize,
self.uncompressFactor, 'uncompress')
planA = [step] + planA
self.installPlans = [planA]
pc.stop()
return
# Maybe we can download one or more patches. We'll come back
# to that in a minute as plan A. For now, construct plan B,
# which will be to download the whole archive.
planB = planA[:]
uncompressSize = self.uncompressedArchive.size
step = self.InstallStep(self.__uncompressArchive, uncompressSize,
self.uncompressFactor, 'uncompress')
planB = [step] + planB
downloadSize = self.compressedArchive.size
func = lambda step, fileSpec=self.compressedArchive: self.__downloadFile(
step, fileSpec, allowPartial=True)
step = self.InstallStep(func, downloadSize, self.downloadFactor,
'download')
planB = [step] + planB
# Now look for patches. Start with the md5 hash from the
# uncompressedArchive file we have on disk, and see if we can
# find a patch chain from this file to our target.
pathname = Filename(self.getPackageDir(),
self.uncompressedArchive.filename)
fileSpec = self.uncompressedArchive.actualFile
if fileSpec is None and pathname.exists():
fileSpec = FileSpec()
fileSpec.fromFile(self.getPackageDir(),
self.uncompressedArchive.filename)
plan = None
if fileSpec:
plan = self.__findPatchChain(fileSpec)
if plan:
# We can download patches. Great! That means this is
# plan A, and the full download is plan B (in case
# something goes wrong with the patching).
planA = plan + planA
self.installPlans = [planA, planB]
else:
# There are no patches to download, oh well. Stick with
# plan B as the only plan.
self.installPlans = [planB]
# In case of unexpected failures on the internet, we will retry
# the full download instead of just giving up.
for retry in range(ConfigVariableInt('package-full-dl-retries', 1)):
self.installPlans.append(planB[:])
pc.stop()
def __readDescFile(self):
""" Reads the desc xml file for this particular package,
assuming it's been already downloaded and verified. Returns
true on success, false on failure. """
if self.hasDescFile:
# No need to read it again.
return True
if self.solo:
# If this is a "solo" package, we don't actually "read"
# the desc file; that's the entire contents of the
# package.
self.hasDescFile = True
self.hasPackage = True
return True
filename = Filename(self.getPackageDir(), self.descFileBasename)
if not hasattr(PandaModules, 'TiXmlDocument'):
return False
doc = PandaModules.TiXmlDocument(filename.toOsSpecific())
if not doc.LoadFile():
return False
xpackage = doc.FirstChildElement('package')
if not xpackage:
return False
try:
self.patchVersion = int(xpackage.Attribute('patch_version') or '')
except ValueError:
self.patchVersion = None
self.displayName = None
xconfig = xpackage.FirstChildElement('config')
if xconfig:
# The name for display to an English-speaking user.
self.displayName = xconfig.Attribute('display_name')
# True if any apps that use this package must be GUI apps.
guiApp = xconfig.Attribute('gui_app')
if guiApp:
self.guiApp = int(guiApp)
# The uncompressed archive, which will be mounted directly,
# and also used for patching.
xuncompressedArchive = xpackage.FirstChildElement(
'uncompressed_archive')
if xuncompressedArchive:
self.uncompressedArchive = FileSpec()
self.uncompressedArchive.loadXml(xuncompressedArchive)
# The compressed archive, which is what is downloaded.
xcompressedArchive = xpackage.FirstChildElement('compressed_archive')
if xcompressedArchive:
self.compressedArchive = FileSpec()
self.compressedArchive.loadXml(xcompressedArchive)
# The list of files that should be extracted to disk.
self.extracts = []
xextract = xpackage.FirstChildElement('extract')
while xextract:
file = FileSpec()
file.loadXml(xextract)
self.extracts.append(file)
xextract = xextract.NextSiblingElement('extract')
# The list of additional packages that must be installed for
# this package to function properly.
self.requires = []
xrequires = xpackage.FirstChildElement('requires')
while xrequires:
packageName = xrequires.Attribute('name')
version = xrequires.Attribute('version')
hostUrl = xrequires.Attribute('host')
if packageName and hostUrl:
host = self.host.appRunner.getHostWithAlt(hostUrl)
self.requires.append((packageName, version, host))
xrequires = xrequires.NextSiblingElement('requires')
self.hasDescFile = True
# Now that we've read the desc file, go ahead and use it to
# verify the download status.
if self.__checkArchiveStatus():
# It's all fully downloaded, unpacked, and ready.
self.hasPackage = True
return True
# Still have to download it.
self.__buildInstallPlans()
return True