示例#1
0
        def loadXml(self, xpackage):
            self.packageName = xpackage.Attribute('name')
            self.platform = xpackage.Attribute('platform')
            self.version = xpackage.Attribute('version')
            solo = xpackage.Attribute('solo')
            self.solo = int(solo or '0')
            perPlatform = xpackage.Attribute('per_platform')
            self.perPlatform = int(perPlatform or '0')

            self.descFile = FileSpec()
            self.descFile.loadXml(xpackage)

            self.validatePackageContents()

            self.descFile.quickVerify(packageDir = self.sourceDir, notify = PackageMerger.notify, correctSelf = True)

            self.packageSeq = SeqValue()
            self.packageSeq.loadXml(xpackage, 'seq')
            self.packageSetVer = SeqValue()
            self.packageSetVer.loadXml(xpackage, 'set_ver')

            self.importDescFile = None
            ximport = xpackage.FirstChildElement('import')
            if ximport:
                self.importDescFile = FileSpec()
                self.importDescFile.loadXml(ximport)
                self.importDescFile.quickVerify(packageDir = self.sourceDir, notify = PackageMerger.notify, correctSelf = True)
示例#2
0
        def loadXml(self, xpackage):
            self.packageName = xpackage.Attribute('name')
            self.platform = xpackage.Attribute('platform')
            self.version = xpackage.Attribute('version')
            solo = xpackage.Attribute('solo')
            self.solo = int(solo or '0')
            perPlatform = xpackage.Attribute('per_platform')
            self.perPlatform = int(perPlatform or '0')

            self.descFile = FileSpec()
            self.descFile.loadXml(xpackage)

            self.validatePackageContents()

            self.descFile.quickVerify(packageDir=self.sourceDir,
                                      notify=PackageMerger.notify,
                                      correctSelf=True)

            self.packageSeq = SeqValue()
            self.packageSeq.loadXml(xpackage, 'seq')
            self.packageSetVer = SeqValue()
            self.packageSetVer.loadXml(xpackage, 'set_ver')

            self.importDescFile = None
            ximport = xpackage.FirstChildElement('import')
            if ximport:
                self.importDescFile = FileSpec()
                self.importDescFile.loadXml(ximport)
                self.importDescFile.quickVerify(packageDir=self.sourceDir,
                                                notify=PackageMerger.notify,
                                                correctSelf=True)
示例#3
0
        def loadXml(self, xpackage):
            self.packageName = xpackage.Attribute("name")
            self.platform = xpackage.Attribute("platform")
            self.version = xpackage.Attribute("version")
            solo = xpackage.Attribute("solo")
            self.solo = int(solo or "0")
            perPlatform = xpackage.Attribute("per_platform")
            self.perPlatform = int(perPlatform or "0")

            self.descFile = FileSpec()
            self.descFile.loadXml(xpackage)

            self.validatePackageContents()

            self.descFile.quickVerify(packageDir=self.sourceDir, notify=PackageMerger.notify, correctSelf=True)

            self.packageSeq = SeqValue()
            self.packageSeq.loadXml(xpackage, "seq")
            self.packageSetVer = SeqValue()
            self.packageSetVer.loadXml(xpackage, "set_ver")

            self.importDescFile = None
            ximport = xpackage.FirstChildElement("import")
            if ximport:
                self.importDescFile = FileSpec()
                self.importDescFile.loadXml(ximport)
                self.importDescFile.quickVerify(
                    packageDir=self.sourceDir, notify=PackageMerger.notify, correctSelf=True
                )
示例#4
0
        def fromFile(self, packageDir, patchFilename, sourceFile, targetFile):
            """ Creates the data structures from an existing patchfile
            on disk. """

            self.file = FileSpec()
            self.file.fromFile(packageDir, patchFilename)
            self.sourceFile = sourceFile
            self.targetFile = targetFile
    class PackageEntry:
        """ This corresponds to a <package> entry in the contents.xml
        file. """
        
        def __init__(self, xpackage, sourceDir):
            self.sourceDir = sourceDir
            self.loadXml(xpackage)

        def getKey(self):
            """ Returns a tuple used for sorting the PackageEntry
            objects uniquely per package. """
            return (self.packageName, self.platform, self.version)

        def isNewer(self, other):
            return self.descFile.timestamp > other.descFile.timestamp

        def loadXml(self, xpackage):
            self.packageName = xpackage.Attribute('name')
            self.platform = xpackage.Attribute('platform')
            self.version = xpackage.Attribute('version')
            solo = xpackage.Attribute('solo')
            self.solo = int(solo or '0')

            self.descFile = FileSpec()
            self.descFile.loadXml(xpackage)

            self.packageSeq = SeqValue()
            self.packageSeq.loadXml(xpackage, 'seq')
            self.packageSetVer = SeqValue()
            self.packageSetVer.loadXml(xpackage, 'set_ver')

            self.importDescFile = None
            ximport = xpackage.FirstChildElement('import')
            if ximport:
                self.importDescFile = FileSpec()
                self.importDescFile.loadXml(ximport)

        def makeXml(self):
            """ Returns a new TiXmlElement. """
            xpackage = TiXmlElement('package')
            xpackage.SetAttribute('name', self.packageName)
            if self.platform:
                xpackage.SetAttribute('platform', self.platform)
            if self.version:
                xpackage.SetAttribute('version', self.version)
            if self.solo:
                xpackage.SetAttribute('solo', '1')

            self.descFile.storeXml(xpackage)
            self.packageSeq.storeXml(xpackage, 'seq')
            self.packageSetVer.storeXml(xpackage, 'set_ver')

            if self.importDescFile:
                ximport = TiXmlElement('import')
                self.importDescFile.storeXml(ximport)
                xpackage.InsertEndChild(ximport)
            
            return xpackage
    class PackageEntry:
        """ This corresponds to a <package> entry in the contents.xml
        file. """

        def __init__(self, xpackage, sourceDir):
            self.sourceDir = sourceDir
            self.loadXml(xpackage)

        def getKey(self):
            """ Returns a tuple used for sorting the PackageEntry
            objects uniquely per package. """
            return (self.packageName, self.platform, self.version)

        def isNewer(self, other):
            return self.descFile.timestamp > other.descFile.timestamp

        def loadXml(self, xpackage):
            self.packageName = xpackage.Attribute('name')
            self.platform = xpackage.Attribute('platform')
            self.version = xpackage.Attribute('version')
            solo = xpackage.Attribute('solo')
            self.solo = int(solo or '0')

            self.descFile = FileSpec()
            self.descFile.loadXml(xpackage)

            self.packageSeq = SeqValue()
            self.packageSeq.loadXml(xpackage, 'seq')
            self.packageSetVer = SeqValue()
            self.packageSetVer.loadXml(xpackage, 'set_ver')

            self.importDescFile = None
            ximport = xpackage.FirstChildElement('import')
            if ximport:
                self.importDescFile = FileSpec()
                self.importDescFile.loadXml(ximport)

        def makeXml(self):
            """ Returns a new TiXmlElement. """
            xpackage = TiXmlElement('package')
            xpackage.SetAttribute('name', self.packageName)
            if self.platform:
                xpackage.SetAttribute('platform', self.platform)
            if self.version:
                xpackage.SetAttribute('version', self.version)
            if self.solo:
                xpackage.SetAttribute('solo', '1')

            self.descFile.storeXml(xpackage)
            self.packageSeq.storeXml(xpackage, 'seq')
            self.packageSetVer.storeXml(xpackage, 'set_ver')

            if self.importDescFile:
                ximport = TiXmlElement('import')
                self.importDescFile.storeXml(ximport)
                xpackage.InsertEndChild(ximport)

            return xpackage
示例#7
0
 def fromFile(self, packageDir, patchFilename, sourceFile, targetFile):
     """ Creates the data structures from an existing patchfile
     on disk. """
     
     self.file = FileSpec()
     self.file.fromFile(packageDir, patchFilename)
     self.sourceFile = sourceFile
     self.targetFile = targetFile
        def loadXml(self, xpackage):
            self.packageName = xpackage.Attribute('name')
            self.platform = xpackage.Attribute('platform')
            self.version = xpackage.Attribute('version')
            solo = xpackage.Attribute('solo')
            self.solo = int(solo or '0')

            self.descFile = FileSpec()
            self.descFile.loadXml(xpackage)

            self.packageSeq = SeqValue()
            self.packageSeq.loadXml(xpackage, 'seq')
            self.packageSetVer = SeqValue()
            self.packageSetVer.loadXml(xpackage, 'set_ver')

            self.importDescFile = None
            ximport = xpackage.FirstChildElement('import')
            if ximport:
                self.importDescFile = FileSpec()
                self.importDescFile.loadXml(ximport)
        def loadXml(self, xpackage):
            self.packageName = xpackage.Attribute('name')
            self.platform = xpackage.Attribute('platform')
            self.version = xpackage.Attribute('version')
            solo = xpackage.Attribute('solo')
            self.solo = int(solo or '0')

            self.descFile = FileSpec()
            self.descFile.loadXml(xpackage)

            self.packageSeq = SeqValue()
            self.packageSeq.loadXml(xpackage, 'seq')
            self.packageSetVer = SeqValue()
            self.packageSetVer.loadXml(xpackage, 'set_ver')

            self.importDescFile = None
            ximport = xpackage.FirstChildElement('import')
            if ximport:
                self.importDescFile = FileSpec()
                self.importDescFile.loadXml(ximport)
示例#10
0
        def loadXml(self, xpatch):
            """ Reads the data structures from an xml file. """
            
            self.packageName = xpatch.Attribute('name') or self.packageName
            self.platform = xpatch.Attribute('platform') or self.platform
            self.version = xpatch.Attribute('version') or self.version
            self.hostUrl = xpatch.Attribute('host') or self.hostUrl

            self.file = FileSpec()
            self.file.loadXml(xpatch)

            xsource = xpatch.FirstChildElement('source')
            if xsource:
                self.sourceFile = FileSpec()
                self.sourceFile.loadXml(xsource)

            xtarget = xpatch.FirstChildElement('target')
            if xtarget:
                self.targetFile = FileSpec()
                self.targetFile.loadXml(xtarget)
示例#11
0
        def loadXml(self, xpatch):
            """ Reads the data structures from an xml file. """

            self.packageName = xpatch.Attribute('name') or self.packageName
            self.platform = xpatch.Attribute('platform') or self.platform
            self.version = xpatch.Attribute('version') or self.version
            self.hostUrl = xpatch.Attribute('host') or self.hostUrl

            self.file = FileSpec()
            self.file.loadXml(xpatch)

            xsource = xpatch.FirstChildElement('source')
            if xsource:
                self.sourceFile = FileSpec()
                self.sourceFile.loadXml(xsource)

            xtarget = xpatch.FirstChildElement('target')
            if xtarget:
                self.targetFile = FileSpec()
                self.targetFile.loadXml(xtarget)
示例#12
0
    class PackageEntry:
        """ This corresponds to a <package> entry in the contents.xml
        file. """

        def __init__(self, xpackage, sourceDir):
            self.sourceDir = sourceDir
            self.loadXml(xpackage)

        def getKey(self):
            """ Returns a tuple used for sorting the PackageEntry
            objects uniquely per package. """
            return (self.packageName, self.platform, self.version)

        def isNewer(self, other):
            return self.descFile.timestamp > other.descFile.timestamp

        def loadXml(self, xpackage):
            self.packageName = xpackage.Attribute('name')
            self.platform = xpackage.Attribute('platform')
            self.version = xpackage.Attribute('version')
            solo = xpackage.Attribute('solo')
            self.solo = int(solo or '0')
            perPlatform = xpackage.Attribute('per_platform')
            self.perPlatform = int(perPlatform or '0')

            self.descFile = FileSpec()
            self.descFile.loadXml(xpackage)

            self.validatePackageContents()

            self.descFile.quickVerify(packageDir = self.sourceDir, notify = PackageMerger.notify, correctSelf = True)

            self.packageSeq = SeqValue()
            self.packageSeq.loadXml(xpackage, 'seq')
            self.packageSetVer = SeqValue()
            self.packageSetVer.loadXml(xpackage, 'set_ver')

            self.importDescFile = None
            ximport = xpackage.FirstChildElement('import')
            if ximport:
                self.importDescFile = FileSpec()
                self.importDescFile.loadXml(ximport)
                self.importDescFile.quickVerify(packageDir = self.sourceDir, notify = PackageMerger.notify, correctSelf = True)

        def makeXml(self):
            """ Returns a new TiXmlElement. """
            xpackage = TiXmlElement('package')
            xpackage.SetAttribute('name', self.packageName)
            if self.platform:
                xpackage.SetAttribute('platform', self.platform)
            if self.version:
                xpackage.SetAttribute('version', self.version)
            if self.solo:
                xpackage.SetAttribute('solo', '1')
            if self.perPlatform:
                xpackage.SetAttribute('per_platform', '1')

            self.descFile.storeXml(xpackage)
            self.packageSeq.storeXml(xpackage, 'seq')
            self.packageSetVer.storeXml(xpackage, 'set_ver')

            if self.importDescFile:
                ximport = TiXmlElement('import')
                self.importDescFile.storeXml(ximport)
                xpackage.InsertEndChild(ximport)

            return xpackage

        def validatePackageContents(self):
            """ Validates the contents of the package directory itself
            against the expected hashes and timestamps.  Updates
            hashes and timestamps where needed. """

            if self.solo:
                return

            needsChange = False
            packageDescFullpath = Filename(self.sourceDir, self.descFile.filename)
            packageDir = Filename(packageDescFullpath.getDirname())
            doc = TiXmlDocument(packageDescFullpath.toOsSpecific())
            if not doc.LoadFile():
                message = "Could not read XML file: %s" % (self.descFile.filename)
                raise OSError(message)

            xpackage = doc.FirstChildElement('package')
            if not xpackage:
                message = "No package definition: %s" % (self.descFile.filename)
                raise OSError(message)

            xcompressed = xpackage.FirstChildElement('compressed_archive')
            if xcompressed:
                spec = FileSpec()
                spec.loadXml(xcompressed)
                if not spec.quickVerify(packageDir = packageDir, notify = PackageMerger.notify, correctSelf = True):
                    spec.storeXml(xcompressed)
                    needsChange = True

            xpatch = xpackage.FirstChildElement('patch')
            while xpatch:
                spec = FileSpec()
                spec.loadXml(xpatch)
                if not spec.quickVerify(packageDir = packageDir, notify = PackageMerger.notify, correctSelf = True):
                    spec.storeXml(xpatch)
                    needsChange = True

                xpatch = xpatch.NextSiblingElement('patch')

            if needsChange:
                PackageMerger.notify.info("Rewriting %s" % (self.descFile.filename))
                doc.SaveFile()
                self.descFile.quickVerify(packageDir = self.sourceDir, notify = PackageMerger.notify, correctSelf = True)
示例#13
0
class HostInfo:
    """ This class represents a particular download host serving up
    Panda3D packages.  It is the Python equivalent of the P3DHost
    class in the core API. """

    notify = directNotify.newCategory("HostInfo")

    def __init__(self, hostUrl, appRunner = None, hostDir = None,
                 rootDir = None, asMirror = False, perPlatform = None):

        """ You must specify either an appRunner or a hostDir to the
        HostInfo constructor.

        If you pass asMirror = True, it means that this HostInfo
        object is to be used to populate a "mirror" folder, a
        duplicate (or subset) of the contents hosted by a server.
        This means when you use this HostInfo to download packages, it
        will only download the compressed archive file and leave it
        there.  At the moment, mirror folders do not download old
        patch files from the server.

        If you pass perPlatform = True, then files are unpacked into a
        platform-specific directory, which is appropriate when you
        might be downloading multiple platforms.  The default is
        perPlatform = False, which means all files are unpacked into
        the host directory directly, without an intervening
        platform-specific directory name.  If asMirror is True, then
        the default is perPlatform = True.

        Note that perPlatform is also restricted by the individual
        package's specification.  """

        self.__setHostUrl(hostUrl)
        self.appRunner = appRunner
        self.rootDir = rootDir
        if rootDir is None and appRunner:
            self.rootDir = appRunner.rootDir

        if hostDir and not isinstance(hostDir, Filename):
            hostDir = Filename.fromOsSpecific(hostDir)

        self.hostDir = hostDir
        self.asMirror = asMirror
        self.perPlatform = perPlatform
        if perPlatform is None:
            self.perPlatform = asMirror

        # Initially false, this is set true when the contents file is
        # successfully read.
        self.hasContentsFile = False

        # This is the time value at which the current contents file is
        # no longer valid.
        self.contentsExpiration = 0

        # Contains the md5 hash of the original contents.xml file.
        self.contentsSpec = FileSpec()

        # descriptiveName will be filled in later, when the
        # contents file is read.
        self.descriptiveName = None

        # A list of known mirrors for this host, all URL's guaranteed
        # to end with a slash.
        self.mirrors = []

        # A map of keyword -> altHost URL's.  An altHost is different
        # than a mirror; an altHost is an alternate URL to download a
        # different (e.g. testing) version of this host's contents.
        # It is rarely used.
        self.altHosts = {}

        # This is a dictionary of packages by (name, version).  It
        # will be filled in when the contents file is read.
        self.packages = {}

        if self.appRunner and self.appRunner.verifyContents != self.appRunner.P3DVCForce:
            # Attempt to pre-read the existing contents.xml; maybe it
            # will be current enough for our purposes.
            self.readContentsFile()

    def __setHostUrl(self, hostUrl):
        """ Assigns self.hostUrl, and related values. """
        self.hostUrl = hostUrl

        if not self.hostUrl:
            # A special case: the URL will be set later.
            self.hostUrlPrefix = None
            self.downloadUrlPrefix = None
        else:
            # hostUrlPrefix is the host URL, but it is guaranteed to end
            # with a slash.
            self.hostUrlPrefix = hostUrl
            if self.hostUrlPrefix[-1] != '/':
                self.hostUrlPrefix += '/'

            # downloadUrlPrefix is the URL prefix that should be used for
            # everything other than the contents.xml file.  It might be
            # the same as hostUrlPrefix, but in the case of an
            # https-protected hostUrl, it will be the cleartext channel.
            self.downloadUrlPrefix = self.hostUrlPrefix

    def freshenFile(self, http, fileSpec, localPathname):
        """ Ensures that the localPathname is the most current version
        of the file defined by fileSpec, as offered by host.  If not,
        it downloads a new version on-the-spot.  Returns true on
        success, false on failure. """

        if fileSpec.quickVerify(pathname = localPathname):
            # It's good, keep it.
            return True

        # It's stale, get a new one.
        doc = None
        if self.appRunner and self.appRunner.superMirrorUrl:
            # Use the "super mirror" first.
            url = core.URLSpec(self.appRunner.superMirrorUrl + fileSpec.filename)
            self.notify.info("Freshening %s" % (url))
            doc = http.getDocument(url)

        if not doc or not doc.isValid():
            # Failing the super mirror, contact the actual host.
            url = core.URLSpec(self.hostUrlPrefix + fileSpec.filename)
            self.notify.info("Freshening %s" % (url))
            doc = http.getDocument(url)
            if not doc.isValid():
                return False

        file = Filename.temporary('', 'p3d_')
        if not doc.downloadToFile(file):
            # Failed to download.
            file.unlink()
            return False

        # Successfully downloaded!
        localPathname.makeDir()
        if not file.renameTo(localPathname):
            # Couldn't move it into place.
            file.unlink()
            return False

        if not fileSpec.fullVerify(pathname = localPathname, notify = self.notify):
            # No good after download.
            self.notify.info("%s is still no good after downloading." % (url))
            return False

        return True

    def downloadContentsFile(self, http, redownload = False,
                             hashVal = None):
        """ Downloads the contents.xml file for this particular host,
        synchronously, and then reads it.  Returns true on success,
        false on failure.  If hashVal is not None, it should be a
        HashVal object, which will be filled with the hash from the
        new contents.xml file."""

        if self.hasCurrentContentsFile():
            # We've already got one.
            return True

        if self.appRunner and self.appRunner.verifyContents == self.appRunner.P3DVCNever:
            # Not allowed to.
            return False

        rf = None
        if http:
            if not redownload and self.appRunner and self.appRunner.superMirrorUrl:
                # We start with the "super mirror", if it's defined.
                url = self.appRunner.superMirrorUrl + 'contents.xml'
                request = DocumentSpec(url)
                self.notify.info("Downloading contents file %s" % (request))

                rf = Ramfile()
                channel = http.makeChannel(False)
                channel.getDocument(request)
                if not channel.downloadToRam(rf):
                    self.notify.warning("Unable to download %s" % (url))
                    rf = None

            if not rf:
                # Then go to the main host, if our super mirror let us
                # down.

                url = self.hostUrlPrefix + 'contents.xml'
                # Append a uniquifying query string to the URL to force the
                # download to go all the way through any caches.  We use the
                # time in seconds; that's unique enough.
                url += '?' + str(int(time.time()))

                # We might as well explicitly request the cache to be disabled
                # too, since we have an interface for that via HTTPChannel.
                request = DocumentSpec(url)
                request.setCacheControl(DocumentSpec.CCNoCache)

                self.notify.info("Downloading contents file %s" % (request))
                statusCode = None
                statusString = ''
                for attempt in range(int(ConfigVariableInt('contents-xml-dl-attempts', 3))):
                    if attempt > 0:
                        self.notify.info("Retrying (%s)..."%(attempt,))
                    rf = Ramfile()
                    channel = http.makeChannel(False)
                    channel.getDocument(request)
                    if channel.downloadToRam(rf):
                        self.notify.info("Successfully downloaded %s" % (url,))
                        break
                    else:
                        rf = None
                        statusCode = channel.getStatusCode()
                        statusString = channel.getStatusString()
                        self.notify.warning("Could not contact download server at %s" % (url,))
                        self.notify.warning("Status code = %s %s" % (statusCode, statusString))

                if not rf:
                    self.notify.warning("Unable to download %s" % (url,))
                    try:
                        # Something screwed up.
                        if statusCode == HTTPChannel.SCDownloadOpenError or \
                           statusCode == HTTPChannel.SCDownloadWriteError:
                            launcher.setPandaErrorCode(2)
                        elif statusCode == 404:
                            # 404 not found
                            launcher.setPandaErrorCode(5)
                        elif statusCode < 100:
                            # statusCode < 100 implies the connection attempt itself
                            # failed.  This is usually due to firewall software
                            # interfering.  Apparently some firewall software might
                            # allow the first connection and disallow subsequent
                            # connections; how strange.
                            launcher.setPandaErrorCode(4)
                        else:
                            # There are other kinds of failures, but these will
                            # generally have been caught already by the first test; so
                            # if we get here there may be some bigger problem.  Just
                            # give the generic "big problem" message.
                            launcher.setPandaErrorCode(6)
                    except NameError as e:
                        # no launcher
                        pass
                    except AttributeError as e:
                        self.notify.warning("%s" % (str(e),))
                        pass
                    return False

        tempFilename = Filename.temporary('', 'p3d_', '.xml')
        if rf:
            f = open(tempFilename.toOsSpecific(), 'wb')
            f.write(rf.getData())
            f.close()
            if hashVal:
                hashVal.hashString(rf.getData())

            if not self.readContentsFile(tempFilename, freshDownload = True):
                self.notify.warning("Failure reading %s" % (url))
                tempFilename.unlink()
                return False

            tempFilename.unlink()
            return True

        # Couldn't download the file.  Maybe we should look for a
        # previously-downloaded copy already on disk?
        return False

    def redownloadContentsFile(self, http):
        """ Downloads a new contents.xml file in case it has changed.
        Returns true if the file has indeed changed, false if it has
        not. """
        assert self.hasContentsFile

        if self.appRunner and self.appRunner.verifyContents == self.appRunner.P3DVCNever:
            # Not allowed to.
            return False

        url = self.hostUrlPrefix + 'contents.xml'
        self.notify.info("Redownloading %s" % (url))

        # Get the hash of the original file.
        assert self.hostDir
        hv1 = HashVal()
        if self.contentsSpec.hash:
            hv1.setFromHex(self.contentsSpec.hash)
        else:
            filename = Filename(self.hostDir, 'contents.xml')
            hv1.hashFile(filename)

        # Now download it again.
        self.hasContentsFile = False
        hv2 = HashVal()
        if not self.downloadContentsFile(http, redownload = True,
                                         hashVal = hv2):
            return False

        if hv2 == HashVal():
            self.notify.info("%s didn't actually redownload." % (url))
            return False
        elif hv1 != hv2:
            self.notify.info("%s has changed." % (url))
            return True
        else:
            self.notify.info("%s has not changed." % (url))
            return False

    def hasCurrentContentsFile(self):
        """ Returns true if a contents.xml file has been successfully
        read for this host and is still current, false otherwise. """
        if not self.appRunner \
            or self.appRunner.verifyContents == self.appRunner.P3DVCNone \
            or self.appRunner.verifyContents == self.appRunner.P3DVCNever:
            # If we're not asking to verify contents, then
            # contents.xml files never expires.
            return self.hasContentsFile

        now = int(time.time())
        return now < self.contentsExpiration and self.hasContentsFile

    def readContentsFile(self, tempFilename = None, freshDownload = False):
        """ Reads the contents.xml file for this particular host, once
        it has been downloaded into the indicated temporary file.
        Returns true on success, false if the contents file is not
        already on disk or is unreadable.

        If tempFilename is specified, it is the filename read, and it
        is copied the file into the standard location if it's not
        there already.  If tempFilename is not specified, the standard
        filename is read if it is known. """

        if not hasattr(core, 'TiXmlDocument'):
            return False

        if not tempFilename:
            if self.hostDir:
                # If the filename is not specified, we can infer it
                # if we already know our hostDir
                hostDir = self.hostDir
            else:
                # Otherwise, we have to guess the hostDir.
                hostDir = self.__determineHostDir(None, self.hostUrl)

            tempFilename = Filename(hostDir, 'contents.xml')

        doc = core.TiXmlDocument(tempFilename.toOsSpecific())
        if not doc.LoadFile():
            return False

        xcontents = doc.FirstChildElement('contents')
        if not xcontents:
            return False

        maxAge = xcontents.Attribute('max_age')
        if maxAge:
            try:
                maxAge = int(maxAge)
            except:
                maxAge = None
        if maxAge is None:
            # Default max_age if unspecified (see p3d_plugin.h).
            from direct.p3d.AppRunner import AppRunner
            maxAge = AppRunner.P3D_CONTENTS_DEFAULT_MAX_AGE

        # Get the latest possible expiration time, based on the max_age
        # indication.  Any expiration time later than this is in error.
        now = int(time.time())
        self.contentsExpiration = now + maxAge

        if freshDownload:
            self.contentsSpec.readHash(tempFilename)

            # Update the XML with the new download information.
            xorig = xcontents.FirstChildElement('orig')
            while xorig:
                xcontents.RemoveChild(xorig)
                xorig = xcontents.FirstChildElement('orig')

            xorig = core.TiXmlElement('orig')
            self.contentsSpec.storeXml(xorig)
            xorig.SetAttribute('expiration', str(self.contentsExpiration))

            xcontents.InsertEndChild(xorig)

        else:
            # Read the download hash and expiration time from the XML.
            expiration = None
            xorig = xcontents.FirstChildElement('orig')
            if xorig:
                self.contentsSpec.loadXml(xorig)
                expiration = xorig.Attribute('expiration')
                if expiration:
                    try:
                        expiration = int(expiration)
                    except:
                        expiration = None
            if not self.contentsSpec.hash:
                self.contentsSpec.readHash(tempFilename)

            if expiration is not None:
                self.contentsExpiration = min(self.contentsExpiration, expiration)

        # Look for our own entry in the hosts table.
        if self.hostUrl:
            self.__findHostXml(xcontents)
        else:
            assert self.hostDir
            self.__findHostXmlForHostDir(xcontents)

        if self.rootDir and not self.hostDir:
            self.hostDir = self.__determineHostDir(None, self.hostUrl)

        # Get the list of packages available for download and/or import.
        xpackage = xcontents.FirstChildElement('package')
        while xpackage:
            name = xpackage.Attribute('name')
            platform = xpackage.Attribute('platform')
            version = xpackage.Attribute('version')
            try:
                solo = int(xpackage.Attribute('solo') or '')
            except ValueError:
                solo = False
            try:
                perPlatform = int(xpackage.Attribute('per_platform') or '')
            except ValueError:
                perPlatform = False

            package = self.__makePackage(name, platform, version, solo, perPlatform)
            package.descFile = FileSpec()
            package.descFile.loadXml(xpackage)
            package.setupFilenames()

            package.importDescFile = None
            ximport = xpackage.FirstChildElement('import')
            if ximport:
                package.importDescFile = FileSpec()
                package.importDescFile.loadXml(ximport)

            xpackage = xpackage.NextSiblingElement('package')

        self.hasContentsFile = True

        # Now save the contents.xml file into the standard location.
        if self.appRunner and self.appRunner.verifyContents != self.appRunner.P3DVCNever:
            assert self.hostDir
            filename = Filename(self.hostDir, 'contents.xml')
            filename.makeDir()
            if freshDownload:
                doc.SaveFile(filename.toOsSpecific())
            else:
                if filename != tempFilename:
                    tempFilename.copyTo(filename)

        return True

    def __findHostXml(self, xcontents):
        """ Looks for the <host> or <alt_host> entry in the
        contents.xml that corresponds to the URL that we actually
        downloaded from. """

        xhost = xcontents.FirstChildElement('host')
        while xhost:
            url = xhost.Attribute('url')
            if url == self.hostUrl:
                self.readHostXml(xhost)
                return

            xalthost = xhost.FirstChildElement('alt_host')
            while xalthost:
                url = xalthost.Attribute('url')
                if url == self.hostUrl:
                    self.readHostXml(xalthost)
                    return
                xalthost = xalthost.NextSiblingElement('alt_host')

            xhost = xhost.NextSiblingElement('host')

    def __findHostXmlForHostDir(self, xcontents):
        """ Looks for the <host> or <alt_host> entry in the
        contents.xml that corresponds to the host dir that we read the
        contents.xml from.  This is used when reading a contents.xml
        file found on disk, as opposed to downloading it from a
        site. """

        xhost = xcontents.FirstChildElement('host')
        while xhost:
            url = xhost.Attribute('url')
            hostDirBasename = xhost.Attribute('host_dir')
            hostDir = self.__determineHostDir(hostDirBasename, url)
            if hostDir == self.hostDir:
                self.__setHostUrl(url)
                self.readHostXml(xhost)
                return

            xalthost = xhost.FirstChildElement('alt_host')
            while xalthost:
                url = xalthost.Attribute('url')
                hostDirBasename = xalthost.Attribute('host_dir')
                hostDir = self.__determineHostDir(hostDirBasename, url)
                if hostDir == self.hostDir:
                    self.__setHostUrl(url)
                    self.readHostXml(xalthost)
                    return
                xalthost = xalthost.NextSiblingElement('alt_host')

            xhost = xhost.NextSiblingElement('host')

    def readHostXml(self, xhost):
        """ Reads a <host> or <alt_host> entry and applies the data to
        this object. """

        descriptiveName = xhost.Attribute('descriptive_name')
        if descriptiveName and not self.descriptiveName:
            self.descriptiveName = descriptiveName

        hostDirBasename = xhost.Attribute('host_dir')
        if self.rootDir and not self.hostDir:
            self.hostDir = self.__determineHostDir(hostDirBasename, self.hostUrl)

        # Get the "download" URL, which is the source from which we
        # download everything other than the contents.xml file.
        downloadUrl = xhost.Attribute('download_url')
        if downloadUrl:
            self.downloadUrlPrefix = downloadUrl
            if self.downloadUrlPrefix[-1] != '/':
                self.downloadUrlPrefix += '/'
        else:
            self.downloadUrlPrefix = self.hostUrlPrefix

        xmirror = xhost.FirstChildElement('mirror')
        while xmirror:
            url = xmirror.Attribute('url')
            if url:
                if url[-1] != '/':
                    url += '/'
                if url not in self.mirrors:
                    self.mirrors.append(url)
            xmirror = xmirror.NextSiblingElement('mirror')

        xalthost = xhost.FirstChildElement('alt_host')
        while xalthost:
            keyword = xalthost.Attribute('keyword')
            url = xalthost.Attribute('url')
            if url and keyword:
                self.altHosts[keyword] = url
            xalthost = xalthost.NextSiblingElement('alt_host')

    def __makePackage(self, name, platform, version, solo, perPlatform):
        """ Creates a new PackageInfo entry for the given name,
        version, and platform.  If there is already a matching
        PackageInfo, returns it. """

        if not platform:
            platform = None

        platforms = self.packages.setdefault((name, version or ""), {})
        package = platforms.get("", None)
        if not package:
            package = PackageInfo(self, name, version, platform = platform,
                                  solo = solo, asMirror = self.asMirror,
                                  perPlatform = perPlatform)
            platforms[platform or ""] = package

        return package

    def getPackage(self, name, version, platform = None):
        """ Returns a PackageInfo that matches the indicated name and
        version and the indicated platform or the current runtime
        platform, if one is provided by this host, or None if not. """

        assert self.hasContentsFile
        platforms = self.packages.get((name, version or ""), {})

        if platform:
            # In this case, we are looking for a specific platform
            # only.
            return platforms.get(platform, None)

        # We are looking for one matching the current runtime
        # platform.  First, look for a package matching the current
        # platform exactly.
        package = platforms.get(PandaSystem.getPlatform(), None)

        # If not found, look for one matching no particular platform.
        if not package:
            package = platforms.get("", None)

        return package

    def getPackages(self, name = None, platform = None):
        """ Returns a list of PackageInfo objects that match the
        indicated name and/or platform, with no particular regards to
        version.  If name is None, all packages are returned. """

        assert self.hasContentsFile

        packages = []
        for (pn, version), platforms in self.packages.items():
            if name and pn != name:
                continue

            if not platform:
                for p2 in platforms:
                    package = self.getPackage(pn, version, platform = p2)
                    if package:
                        packages.append(package)
            else:
                package = self.getPackage(pn, version, platform = platform)
                if package:
                    packages.append(package)

        return packages

    def getAllPackages(self, includeAllPlatforms = False):
        """ Returns a list of all available packages provided by this
        host. """

        result = []

        items = sorted(self.packages.items())
        for key, platforms in items:
            if self.perPlatform or includeAllPlatforms:
                # If we maintain a different answer per platform,
                # return all of them.
                pitems = sorted(platforms.items())
                for pkey, package in pitems:
                    result.append(package)
            else:
                # If we maintain a host for the current platform
                # only (e.g. a client copy), then return only the
                # current platform, or no particular platform.
                package = platforms.get(PandaSystem.getPlatform(), None)
                if not package:
                    package = platforms.get("", None)

                if package:
                    result.append(package)

        return result

    def deletePackages(self, packages):
        """ Removes all of the indicated packages from the disk,
        uninstalling them and deleting all of their files.  The
        packages parameter must be a list of one or more PackageInfo
        objects, for instance as returned by getPackage().  Returns
        the list of packages that were NOT found. """

        packages = packages[:]

        for key, platforms in list(self.packages.items()):
            for platform, package in list(platforms.items()):
                if package in packages:
                    self.__deletePackageFiles(package)
                    del platforms[platform]
                    packages.remove(package)

            if not platforms:
                # If we've removed all the platforms for a given
                # package, remove the key from the toplevel map.
                del self.packages[key]

        return packages

    def __deletePackageFiles(self, package):
        """ Called by deletePackage(), this actually removes the files
        for the indicated package. """

        if self.appRunner:
            self.notify.info("Deleting package %s: %s" % (package.packageName, package.getPackageDir()))
            self.appRunner.rmtree(package.getPackageDir())

            self.appRunner.sendRequest('forget_package', self.hostUrl, package.packageName, package.packageVersion or '')

    def __determineHostDir(self, hostDirBasename, hostUrl):
        """ Hashes the host URL into a (mostly) unique directory
        string, which will be the root of the host's install tree.
        Returns the resulting path, as a Filename.

        This code is duplicated in C++, in
        P3DHost::determine_host_dir(). """

        if hostDirBasename:
            # If the contents.xml specified a host_dir parameter, use
            # it.
            hostDir = str(self.rootDir) + '/hosts'
            for component in hostDirBasename.split('/'):
                if component:
                    if component[0] == '.':
                        # Forbid ".foo" or "..".
                        component = 'x' + component
                    hostDir += '/'
                    hostDir += component
            return Filename(hostDir)

        hostDir = 'hosts/'

        # Look for a server name in the URL.  Including this string in the
        # directory name makes it friendlier for people browsing the
        # directory.

        # We could use URLSpec, but we do it by hand instead, to make
        # it more likely that our hash code will exactly match the
        # similar logic in P3DHost.
        p = hostUrl.find('://')
        hostname = ''
        if p != -1:
            start = p + 3
            end = hostUrl.find('/', start)
            # Now start .. end is something like "username@host:port".

            at = hostUrl.find('@', start)
            if at != -1 and at < end:
                start = at + 1

            colon = hostUrl.find(':', start)
            if colon != -1 and colon < end:
                end = colon

            # Now start .. end is just the hostname.
            hostname = hostUrl[start : end]

        # Now build a hash string of the whole URL.  We'll use MD5 to
        # get a pretty good hash, with a minimum chance of collision.
        # Even if there is a hash collision, though, it's not the end
        # of the world; it just means that both hosts will dump their
        # packages into the same directory, and they'll fight over the
        # toplevel contents.xml file.  Assuming they use different
        # version numbers (which should be safe since they have the
        # same hostname), there will be minimal redownloading.

        hashSize = 16
        keepHash = hashSize
        if hostname:
            hostDir += hostname + '_'

            # If we successfully got a hostname, we don't really need the
            # full hash.  We'll keep half of it.
            keepHash = keepHash // 2

        md = HashVal()
        md.hashString(hostUrl)
        hostDir += md.asHex()[:keepHash * 2]

        hostDir = Filename(self.rootDir, hostDir)
        return hostDir
示例#14
0
        def readDescFile(self, doProcessing = False):
            """ Reads the existing package.xml file and stores it in
            this class for later rewriting.  if doProcessing is true,
            it may massage the file and the directory contents in
            preparation for building patches.  Returns true on
            success, false on failure. """

            self.anyChanges = False

            packageDescFullpath = Filename(self.patchMaker.installDir, self.packageDesc)
            self.doc = TiXmlDocument(packageDescFullpath.toOsSpecific())
            if not self.doc.LoadFile():
                print "Couldn't read %s" % (packageDescFullpath)
                return False
            
            xpackage = self.doc.FirstChildElement('package')
            if not xpackage:
                return False
            self.packageName = xpackage.Attribute('name')
            self.platform = xpackage.Attribute('platform')
            self.version = xpackage.Attribute('version')

            # All packages we defined in-line are assigned to the
            # "none" host.  TODO: support patching from packages on
            # other hosts, which means we'll need to fill in a value
            # here for those hosts.
            self.hostUrl = None
        
            self.currentFile = None
            self.baseFile = None
            self.topFile = None
            self.compressedFilename = None
            compressedFile = None

            # Assume there are changes for this version, until we
            # discover that there aren't.
            isNewVersion = True

            # Get the actual current version.
            xarchive = xpackage.FirstChildElement('uncompressed_archive')
            if xarchive:
                self.currentFile = FileSpec()
                self.currentFile.loadXml(xarchive)

            # Get the top_version--the top (newest) of the patch
            # chain.
            xarchive = xpackage.FirstChildElement('top_version')
            if xarchive:
                self.topFile = FileSpec()
                self.topFile.loadXml(xarchive)

                if self.topFile.hash == self.currentFile.hash:
                    # No new version this pass.
                    isNewVersion = False
                else:
                    # There's a new version this pass.  Update it.
                    self.anyChanges = True
                
            else:
                # If there isn't a top_version yet, we have to make
                # one, by duplicating the currentFile.
                self.topFile = copy.copy(self.currentFile)
                self.anyChanges = True

            # Get the current patch version.  If we have a
            # patch_version attribute, it refers to this particular
            # instance of the file, and that is the current patch
            # version number.  If we only have a last_patch_version
            # attribute, it means a patch has not yet been built for
            # this particular instance, and that number is the
            # previous version's patch version number.
            patchVersion = xpackage.Attribute('patch_version')
            if patchVersion:
                self.patchVersion = int(patchVersion)
            else:
                patchVersion = xpackage.Attribute('last_patch_version')
                if patchVersion:
                    self.patchVersion = int(patchVersion)
                    if isNewVersion:
                        self.patchVersion += 1
                self.anyChanges = True

            # Put the patchVersion in the compressed filename, for
            # cache-busting.  This means when the version changes, its
            # URL will also change, guaranteeing that users will
            # download the latest version, and not some stale cache
            # file.
            xcompressed = xpackage.FirstChildElement('compressed_archive')
            if xcompressed:
                compressedFile = FileSpec()
                compressedFile.loadXml(xcompressed)

                oldCompressedFilename = compressedFile.filename
                self.compressedFilename = oldCompressedFilename

                if doProcessing:
                    newCompressedFilename = '%s.%s.pz' % (self.currentFile.filename, self.patchVersion)
                    if newCompressedFilename != oldCompressedFilename:
                        oldCompressedPathname = Filename(self.packageDir, oldCompressedFilename)
                        newCompressedPathname = Filename(self.packageDir, newCompressedFilename)
                        if oldCompressedPathname.renameTo(newCompressedPathname):
                            compressedFile.fromFile(self.packageDir, newCompressedFilename)
                            compressedFile.storeXml(xcompressed)

                        self.compressedFilename = newCompressedFilename
                        self.anyChanges = True

            # Get the base_version--the bottom (oldest) of the patch
            # chain.
            xarchive = xpackage.FirstChildElement('base_version')
            if xarchive:
                self.baseFile = FileSpec()
                self.baseFile.loadXml(xarchive)
            else:
                # If there isn't a base_version yet, we have to make
                # one, by duplicating the currentFile.
                self.baseFile = copy.copy(self.currentFile)

                # Note that the we only store the compressed version
                # of base_filename on disk, but we store the md5 of
                # the uncompressed version in the xml file.  To
                # emphasize this, we name it without the .pz extension
                # in the xml file, even though the compressed file on
                # disk actually has a .pz extension.
                self.baseFile.filename += '.base'

                # Also duplicate the (compressed) file itself.
                if doProcessing and self.compressedFilename:
                    fromPathname = Filename(self.packageDir, self.compressedFilename)
                    toPathname = Filename(self.packageDir, self.baseFile.filename + '.pz')
                    fromPathname.copyTo(toPathname)
                self.anyChanges = True

            self.patches = []
            xpatch = xpackage.FirstChildElement('patch')
            while xpatch:
                patchfile = PatchMaker.Patchfile(self)
                patchfile.loadXml(xpatch)
                self.patches.append(patchfile)
                xpatch = xpatch.NextSiblingElement('patch')

            return True
示例#15
0
        def writeDescFile(self):
            """ Rewrites the desc file with the new patch
            information. """

            if not self.anyChanges:
                # No need to rewrite.
                return

            xpackage = self.doc.FirstChildElement('package')
            if not xpackage:
                return

            packageSeq = SeqValue()
            packageSeq.loadXml(xpackage, 'seq')
            packageSeq += 1
            packageSeq.storeXml(xpackage, 'seq')

            # Remove all of the old patch entries from the desc file
            # we read earlier.
            xremove = []
            for value in ['base_version', 'top_version', 'patch']:
                xpatch = xpackage.FirstChildElement(value)
                while xpatch:
                    xremove.append(xpatch)
                    xpatch = xpatch.NextSiblingElement(value)

            for xelement in xremove:
                xpackage.RemoveChild(xelement)

            xpackage.RemoveAttribute('last_patch_version')

            # Now replace them with the current patch information.
            xpackage.SetAttribute('patch_version', str(self.patchVersion))

            xarchive = TiXmlElement('base_version')
            self.baseFile.storeXml(xarchive)
            xpackage.InsertEndChild(xarchive)

            # The current version is now the top version.
            xarchive = TiXmlElement('top_version')
            self.currentFile.storeXml(xarchive)
            xpackage.InsertEndChild(xarchive)
            
            for patchfile in self.patches:
                xpatch = patchfile.makeXml(self)
                xpackage.InsertEndChild(xpatch)

            self.doc.SaveFile()

            # Also copy the seq to the import desc file, for
            # documentation purposes.

            importDescFilename = self.packageDesc.cStr()[:-3] + 'import.xml'
            importDescFullpath = Filename(self.patchMaker.installDir, importDescFilename)
            doc = TiXmlDocument(importDescFullpath.toOsSpecific())
            if doc.LoadFile():
                xpackage = doc.FirstChildElement('package')
                if xpackage:
                    packageSeq.storeXml(xpackage, 'seq')
                    doc.SaveFile()
            else:
                print "Couldn't read %s" % (importDescFullpath)

            if self.contentsDocPackage:
                # Now that we've rewritten the xml file, we have to
                # change the contents.xml file that references it to
                # indicate the new file hash.
                fileSpec = FileSpec()
                fileSpec.fromFile(self.patchMaker.installDir, self.packageDesc)
                fileSpec.storeXml(self.contentsDocPackage)

                # Also important to update the import.xml hash.
                ximport = self.contentsDocPackage.FirstChildElement('import')
                if ximport:
                    fileSpec = FileSpec()
                    fileSpec.fromFile(self.patchMaker.installDir, importDescFilename)
                    fileSpec.storeXml(ximport)

                # Also copy the package seq value into the
                # contents.xml file, mainly for documentation purposes
                # (the authoritative seq value is within the desc
                # file).
                packageSeq.storeXml(self.contentsDocPackage, 'seq')
示例#16
0
    def __readDescFile(self):
        """ Reads the desc xml file for this particular package,
        assuming it's been already downloaded and verified.  Returns
        true on success, false on failure. """

        if self.hasDescFile:
            # No need to read it again.
            return True

        if self.solo:
            # If this is a "solo" package, we don't actually "read"
            # the desc file; that's the entire contents of the
            # package.
            self.hasDescFile = True
            self.hasPackage = True
            return True

        filename = Filename(self.getPackageDir(), self.descFileBasename)

        if not hasattr(PandaModules, 'TiXmlDocument'):
            return False
        doc = PandaModules.TiXmlDocument(filename.toOsSpecific())
        if not doc.LoadFile():
            return False

        xpackage = doc.FirstChildElement('package')
        if not xpackage:
            return False

        try:
            self.patchVersion = int(xpackage.Attribute('patch_version') or '')
        except ValueError:
            self.patchVersion = None

        self.displayName = None
        xconfig = xpackage.FirstChildElement('config')
        if xconfig:
            # The name for display to an English-speaking user.
            self.displayName = xconfig.Attribute('display_name')

            # True if any apps that use this package must be GUI apps.
            guiApp = xconfig.Attribute('gui_app')
            if guiApp:
                self.guiApp = int(guiApp)

        # The uncompressed archive, which will be mounted directly,
        # and also used for patching.
        xuncompressedArchive = xpackage.FirstChildElement('uncompressed_archive')
        if xuncompressedArchive:
            self.uncompressedArchive = FileSpec()
            self.uncompressedArchive.loadXml(xuncompressedArchive)

        # The compressed archive, which is what is downloaded.
        xcompressedArchive = xpackage.FirstChildElement('compressed_archive')
        if xcompressedArchive:
            self.compressedArchive = FileSpec()
            self.compressedArchive.loadXml(xcompressedArchive)

        # The list of files that should be extracted to disk.
        self.extracts = []
        xextract = xpackage.FirstChildElement('extract')
        while xextract:
            file = FileSpec()
            file.loadXml(xextract)
            self.extracts.append(file)
            xextract = xextract.NextSiblingElement('extract')

        # The list of additional packages that must be installed for
        # this package to function properly.
        self.requires = []
        xrequires = xpackage.FirstChildElement('requires')
        while xrequires:
            packageName = xrequires.Attribute('name')
            version = xrequires.Attribute('version')
            hostUrl = xrequires.Attribute('host')
            if packageName and hostUrl:
                host = self.host.appRunner.getHostWithAlt(hostUrl)
                self.requires.append((packageName, version, host))
            xrequires = xrequires.NextSiblingElement('requires')

        self.hasDescFile = True

        # Now that we've read the desc file, go ahead and use it to
        # verify the download status.
        if self.__checkArchiveStatus():
            # It's all fully downloaded, unpacked, and ready.
            self.hasPackage = True
            return True

        # Still have to download it.
        self.__buildInstallPlans()
        return True
示例#17
0
    class Package:
        """ This is a particular package.  This contains all of the
        information needed to reconstruct the package's desc file. """
        
        def __init__(self, packageDesc, patchMaker, xpackage = None):
            self.packageDir = Filename(patchMaker.installDir, packageDesc.getDirname())
            self.packageDesc = packageDesc
            self.patchMaker = patchMaker
            self.contentsDocPackage = xpackage
            self.patchVersion = 1
            self.currentPv = None
            self.basePv = None
            self.topPv = None

            self.packageName = None
            self.platform = None
            self.version = None
            self.hostUrl = None
            self.currentFile = None
            self.baseFile = None

            self.doc = None
            self.anyChanges = False
            self.patches = []

        def getCurrentKey(self):
            """ Returns the key to locate the current version of this
            package. """
            
            return (self.packageName, self.platform, self.version, self.hostUrl, self.currentFile)

        def getBaseKey(self):
            """ Returns the key to locate the "base" or oldest version
            of this package. """
            
            return (self.packageName, self.platform, self.version, self.hostUrl, self.baseFile)

        def getTopKey(self):
            """ Returns the key to locate the "top" or newest version
            of this package. """
            
            return (self.packageName, self.platform, self.version, self.hostUrl, self.topFile)

        def getGenericKey(self, fileSpec):
            """ Returns the key that has the indicated hash. """
            return (self.packageName, self.platform, self.version, self.hostUrl, fileSpec)

        def readDescFile(self, doProcessing = False):
            """ Reads the existing package.xml file and stores it in
            this class for later rewriting.  if doProcessing is true,
            it may massage the file and the directory contents in
            preparation for building patches.  Returns true on
            success, false on failure. """

            self.anyChanges = False

            packageDescFullpath = Filename(self.patchMaker.installDir, self.packageDesc)
            self.doc = TiXmlDocument(packageDescFullpath.toOsSpecific())
            if not self.doc.LoadFile():
                print "Couldn't read %s" % (packageDescFullpath)
                return False
            
            xpackage = self.doc.FirstChildElement('package')
            if not xpackage:
                return False
            self.packageName = xpackage.Attribute('name')
            self.platform = xpackage.Attribute('platform')
            self.version = xpackage.Attribute('version')

            # All packages we defined in-line are assigned to the
            # "none" host.  TODO: support patching from packages on
            # other hosts, which means we'll need to fill in a value
            # here for those hosts.
            self.hostUrl = None
        
            self.currentFile = None
            self.baseFile = None
            self.topFile = None
            self.compressedFilename = None
            compressedFile = None

            # Assume there are changes for this version, until we
            # discover that there aren't.
            isNewVersion = True

            # Get the actual current version.
            xarchive = xpackage.FirstChildElement('uncompressed_archive')
            if xarchive:
                self.currentFile = FileSpec()
                self.currentFile.loadXml(xarchive)

            # Get the top_version--the top (newest) of the patch
            # chain.
            xarchive = xpackage.FirstChildElement('top_version')
            if xarchive:
                self.topFile = FileSpec()
                self.topFile.loadXml(xarchive)

                if self.topFile.hash == self.currentFile.hash:
                    # No new version this pass.
                    isNewVersion = False
                else:
                    # There's a new version this pass.  Update it.
                    self.anyChanges = True
                
            else:
                # If there isn't a top_version yet, we have to make
                # one, by duplicating the currentFile.
                self.topFile = copy.copy(self.currentFile)
                self.anyChanges = True

            # Get the current patch version.  If we have a
            # patch_version attribute, it refers to this particular
            # instance of the file, and that is the current patch
            # version number.  If we only have a last_patch_version
            # attribute, it means a patch has not yet been built for
            # this particular instance, and that number is the
            # previous version's patch version number.
            patchVersion = xpackage.Attribute('patch_version')
            if patchVersion:
                self.patchVersion = int(patchVersion)
            else:
                patchVersion = xpackage.Attribute('last_patch_version')
                if patchVersion:
                    self.patchVersion = int(patchVersion)
                    if isNewVersion:
                        self.patchVersion += 1
                self.anyChanges = True

            # Put the patchVersion in the compressed filename, for
            # cache-busting.  This means when the version changes, its
            # URL will also change, guaranteeing that users will
            # download the latest version, and not some stale cache
            # file.
            xcompressed = xpackage.FirstChildElement('compressed_archive')
            if xcompressed:
                compressedFile = FileSpec()
                compressedFile.loadXml(xcompressed)

                oldCompressedFilename = compressedFile.filename
                self.compressedFilename = oldCompressedFilename

                if doProcessing:
                    newCompressedFilename = '%s.%s.pz' % (self.currentFile.filename, self.patchVersion)
                    if newCompressedFilename != oldCompressedFilename:
                        oldCompressedPathname = Filename(self.packageDir, oldCompressedFilename)
                        newCompressedPathname = Filename(self.packageDir, newCompressedFilename)
                        if oldCompressedPathname.renameTo(newCompressedPathname):
                            compressedFile.fromFile(self.packageDir, newCompressedFilename)
                            compressedFile.storeXml(xcompressed)

                        self.compressedFilename = newCompressedFilename
                        self.anyChanges = True

            # Get the base_version--the bottom (oldest) of the patch
            # chain.
            xarchive = xpackage.FirstChildElement('base_version')
            if xarchive:
                self.baseFile = FileSpec()
                self.baseFile.loadXml(xarchive)
            else:
                # If there isn't a base_version yet, we have to make
                # one, by duplicating the currentFile.
                self.baseFile = copy.copy(self.currentFile)

                # Note that the we only store the compressed version
                # of base_filename on disk, but we store the md5 of
                # the uncompressed version in the xml file.  To
                # emphasize this, we name it without the .pz extension
                # in the xml file, even though the compressed file on
                # disk actually has a .pz extension.
                self.baseFile.filename += '.base'

                # Also duplicate the (compressed) file itself.
                if doProcessing and self.compressedFilename:
                    fromPathname = Filename(self.packageDir, self.compressedFilename)
                    toPathname = Filename(self.packageDir, self.baseFile.filename + '.pz')
                    fromPathname.copyTo(toPathname)
                self.anyChanges = True

            self.patches = []
            xpatch = xpackage.FirstChildElement('patch')
            while xpatch:
                patchfile = PatchMaker.Patchfile(self)
                patchfile.loadXml(xpatch)
                self.patches.append(patchfile)
                xpatch = xpatch.NextSiblingElement('patch')

            return True

        def writeDescFile(self):
            """ Rewrites the desc file with the new patch
            information. """

            if not self.anyChanges:
                # No need to rewrite.
                return

            xpackage = self.doc.FirstChildElement('package')
            if not xpackage:
                return

            packageSeq = SeqValue()
            packageSeq.loadXml(xpackage, 'seq')
            packageSeq += 1
            packageSeq.storeXml(xpackage, 'seq')

            # Remove all of the old patch entries from the desc file
            # we read earlier.
            xremove = []
            for value in ['base_version', 'top_version', 'patch']:
                xpatch = xpackage.FirstChildElement(value)
                while xpatch:
                    xremove.append(xpatch)
                    xpatch = xpatch.NextSiblingElement(value)

            for xelement in xremove:
                xpackage.RemoveChild(xelement)

            xpackage.RemoveAttribute('last_patch_version')

            # Now replace them with the current patch information.
            xpackage.SetAttribute('patch_version', str(self.patchVersion))

            xarchive = TiXmlElement('base_version')
            self.baseFile.storeXml(xarchive)
            xpackage.InsertEndChild(xarchive)

            # The current version is now the top version.
            xarchive = TiXmlElement('top_version')
            self.currentFile.storeXml(xarchive)
            xpackage.InsertEndChild(xarchive)
            
            for patchfile in self.patches:
                xpatch = patchfile.makeXml(self)
                xpackage.InsertEndChild(xpatch)

            self.doc.SaveFile()

            # Also copy the seq to the import desc file, for
            # documentation purposes.

            importDescFilename = self.packageDesc.cStr()[:-3] + 'import.xml'
            importDescFullpath = Filename(self.patchMaker.installDir, importDescFilename)
            doc = TiXmlDocument(importDescFullpath.toOsSpecific())
            if doc.LoadFile():
                xpackage = doc.FirstChildElement('package')
                if xpackage:
                    packageSeq.storeXml(xpackage, 'seq')
                    doc.SaveFile()
            else:
                print "Couldn't read %s" % (importDescFullpath)

            if self.contentsDocPackage:
                # Now that we've rewritten the xml file, we have to
                # change the contents.xml file that references it to
                # indicate the new file hash.
                fileSpec = FileSpec()
                fileSpec.fromFile(self.patchMaker.installDir, self.packageDesc)
                fileSpec.storeXml(self.contentsDocPackage)

                # Also important to update the import.xml hash.
                ximport = self.contentsDocPackage.FirstChildElement('import')
                if ximport:
                    fileSpec = FileSpec()
                    fileSpec.fromFile(self.patchMaker.installDir, importDescFilename)
                    fileSpec.storeXml(ximport)

                # Also copy the package seq value into the
                # contents.xml file, mainly for documentation purposes
                # (the authoritative seq value is within the desc
                # file).
                packageSeq.storeXml(self.contentsDocPackage, 'seq')
示例#18
0
    def __buildInstallPlans(self):
        """ Sets up self.installPlans, a list of one or more "plans"
        to download and install the package. """

        pc = PStatCollector(':App:PackageInstaller:buildInstallPlans')
        pc.start()

        self.hasPackage = False
        
        if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
            # We're not allowed to download anything.
            self.installPlans = []
            pc.stop()
            return

        if self.asMirror:
            # If we're just downloading a mirror archive, we only need
            # to get the compressed archive file.

            # Build a one-item install plan to download the compressed
            # archive.
            downloadSize = self.compressedArchive.size
            func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec, allowPartial = True)
            
            step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download')
            installPlan = [step]
            self.installPlans = [installPlan]
            pc.stop()
            return 

        # The normal download process.  Determine what we will need to
        # download, and build a plan (or two) to download it all.
        self.installPlans = None

        # We know we will at least need to unpack the archive contents
        # at the end.
        unpackSize = 0
        for file in self.extracts:
            unpackSize += file.size
        step = self.InstallStep(self.__unpackArchive, unpackSize, self.unpackFactor, 'unpack')
        planA = [step]

        # If the uncompressed archive file is good, that's all we'll
        # need to do.
        self.uncompressedArchive.actualFile = None
        if self.uncompressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
            self.installPlans = [planA]
            pc.stop()
            return

        # Maybe the compressed archive file is good.
        if self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
            uncompressSize = self.uncompressedArchive.size
            step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor, 'uncompress')
            planA = [step] + planA
            self.installPlans = [planA]
            pc.stop()
            return

        # Maybe we can download one or more patches.  We'll come back
        # to that in a minute as plan A.  For now, construct plan B,
        # which will be to download the whole archive.
        planB = planA[:]

        uncompressSize = self.uncompressedArchive.size
        step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor, 'uncompress')
        planB = [step] + planB

        downloadSize = self.compressedArchive.size
        func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec, allowPartial = True)

        step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download')
        planB = [step] + planB

        # Now look for patches.  Start with the md5 hash from the
        # uncompressedArchive file we have on disk, and see if we can
        # find a patch chain from this file to our target.
        pathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
        fileSpec = self.uncompressedArchive.actualFile
        if fileSpec is None and pathname.exists():
            fileSpec = FileSpec()
            fileSpec.fromFile(self.getPackageDir(), self.uncompressedArchive.filename)
        plan = None
        if fileSpec:
            plan = self.__findPatchChain(fileSpec)
        if plan:
            # We can download patches.  Great!  That means this is
            # plan A, and the full download is plan B (in case
            # something goes wrong with the patching).
            planA = plan + planA
            self.installPlans = [planA, planB]
        else:
            # There are no patches to download, oh well.  Stick with
            # plan B as the only plan.
            self.installPlans = [planB]

        # In case of unexpected failures on the internet, we will retry 
        # the full download instead of just giving up.
        for retry in range(ConfigVariableInt('package-full-dl-retries', 1)):
            self.installPlans.append(planB[:])

        pc.stop()
示例#19
0
    class Patchfile:
        """ A single patchfile for a package. """
        def __init__(self, package):
            self.package = package
            self.packageName = package.packageName
            self.platform = package.platform
            self.version = package.version
            self.hostUrl = None

            # FileSpec for the patchfile itself
            self.file = None

            # FileSpec for the package file that the patch is applied to
            self.sourceFile = None

            # FileSpec for the package file that the patch generates
            self.targetFile = None

            # The PackageVersion corresponding to our sourceFile
            self.fromPv = None

            # The PackageVersion corresponding to our targetFile
            self.toPv = None

        def getSourceKey(self):
            """ Returns the key for locating the package that this
            patchfile can be applied to. """
            return (self.packageName, self.platform, self.version,
                    self.hostUrl, self.sourceFile)

        def getTargetKey(self):
            """ Returns the key for locating the package that this
            patchfile will generate. """
            return (self.packageName, self.platform, self.version,
                    self.hostUrl, self.targetFile)

        def fromFile(self, packageDir, patchFilename, sourceFile, targetFile):
            """ Creates the data structures from an existing patchfile
            on disk. """

            self.file = FileSpec()
            self.file.fromFile(packageDir, patchFilename)
            self.sourceFile = sourceFile
            self.targetFile = targetFile

        def loadXml(self, xpatch):
            """ Reads the data structures from an xml file. """

            self.packageName = xpatch.Attribute('name') or self.packageName
            self.platform = xpatch.Attribute('platform') or self.platform
            self.version = xpatch.Attribute('version') or self.version
            self.hostUrl = xpatch.Attribute('host') or self.hostUrl

            self.file = FileSpec()
            self.file.loadXml(xpatch)

            xsource = xpatch.FirstChildElement('source')
            if xsource:
                self.sourceFile = FileSpec()
                self.sourceFile.loadXml(xsource)

            xtarget = xpatch.FirstChildElement('target')
            if xtarget:
                self.targetFile = FileSpec()
                self.targetFile.loadXml(xtarget)

        def makeXml(self, package):
            xpatch = TiXmlElement('patch')

            if self.packageName != package.packageName:
                xpatch.SetAttribute('name', self.packageName)
            if self.platform != package.platform:
                xpatch.SetAttribute('platform', self.platform)
            if self.version != package.version:
                xpatch.SetAttribute('version', self.version)
            if self.hostUrl != package.hostUrl:
                xpatch.SetAttribute('host', self.hostUrl)

            self.file.storeXml(xpatch)

            xsource = TiXmlElement('source')
            self.sourceFile.storeMiniXml(xsource)
            xpatch.InsertEndChild(xsource)

            xtarget = TiXmlElement('target')
            self.targetFile.storeMiniXml(xtarget)
            xpatch.InsertEndChild(xtarget)

            return xpatch
示例#20
0
        def validatePackageContents(self):
            """ Validates the contents of the package directory itself
            against the expected hashes and timestamps.  Updates
            hashes and timestamps where needed. """

            if self.solo:
                return

            needsChange = False
            packageDescFullpath = Filename(self.sourceDir, self.descFile.filename)
            packageDir = Filename(packageDescFullpath.getDirname())
            doc = TiXmlDocument(packageDescFullpath.toOsSpecific())
            if not doc.LoadFile():
                message = "Could not read XML file: %s" % (self.descFile.filename)
                raise OSError(message)

            xpackage = doc.FirstChildElement('package')
            if not xpackage:
                message = "No package definition: %s" % (self.descFile.filename)
                raise OSError(message)

            xcompressed = xpackage.FirstChildElement('compressed_archive')
            if xcompressed:
                spec = FileSpec()
                spec.loadXml(xcompressed)
                if not spec.quickVerify(packageDir = packageDir, notify = PackageMerger.notify, correctSelf = True):
                    spec.storeXml(xcompressed)
                    needsChange = True

            xpatch = xpackage.FirstChildElement('patch')
            while xpatch:
                spec = FileSpec()
                spec.loadXml(xpatch)
                if not spec.quickVerify(packageDir = packageDir, notify = PackageMerger.notify, correctSelf = True):
                    spec.storeXml(xpatch)
                    needsChange = True

                xpatch = xpatch.NextSiblingElement('patch')

            if needsChange:
                PackageMerger.notify.info("Rewriting %s" % (self.descFile.filename))
                doc.SaveFile()
                self.descFile.quickVerify(packageDir = self.sourceDir, notify = PackageMerger.notify, correctSelf = True)
示例#21
0
    def __init__(self, hostUrl, appRunner = None, hostDir = None,
                 rootDir = None, asMirror = False, perPlatform = None):

        """ You must specify either an appRunner or a hostDir to the
        HostInfo constructor.

        If you pass asMirror = True, it means that this HostInfo
        object is to be used to populate a "mirror" folder, a
        duplicate (or subset) of the contents hosted by a server.
        This means when you use this HostInfo to download packages, it
        will only download the compressed archive file and leave it
        there.  At the moment, mirror folders do not download old
        patch files from the server.

        If you pass perPlatform = True, then files are unpacked into a
        platform-specific directory, which is appropriate when you
        might be downloading multiple platforms.  The default is
        perPlatform = False, which means all files are unpacked into
        the host directory directly, without an intervening
        platform-specific directory name.  If asMirror is True, then
        the default is perPlatform = True. """

        assert appRunner or rootDir or hostDir

        self.__setHostUrl(hostUrl)
        self.appRunner = appRunner
        self.rootDir = rootDir
        if rootDir is None and appRunner:
            self.rootDir = appRunner.rootDir

        if hostDir and not isinstance(hostDir, Filename):
            hostDir = Filename.fromOsSpecific(hostDir)
            
        self.hostDir = hostDir
        self.asMirror = asMirror
        self.perPlatform = perPlatform
        if perPlatform is None:
            self.perPlatform = asMirror

        # Initially false, this is set true when the contents file is
        # successfully read.
        self.hasContentsFile = False

        # This is the time value at which the current contents file is
        # no longer valid.
        self.contentsExpiration = 0

        # Contains the md5 hash of the original contents.xml file.
        self.contentsSpec = FileSpec()

        # descriptiveName will be filled in later, when the
        # contents file is read.
        self.descriptiveName = None

        # A list of known mirrors for this host, all URL's guaranteed
        # to end with a slash.
        self.mirrors = []

        # A map of keyword -> altHost URL's.  An altHost is different
        # than a mirror; an altHost is an alternate URL to download a
        # different (e.g. testing) version of this host's contents.
        # It is rarely used.
        self.altHosts = {}

        # This is a dictionary of packages by (name, version).  It
        # will be filled in when the contents file is read.
        self.packages = {}

        if self.appRunner and self.appRunner.verifyContents != self.appRunner.P3DVCForce:
            # Attempt to pre-read the existing contents.xml; maybe it
            # will be current enough for our purposes.
            self.readContentsFile()
示例#22
0
    def __buildInstallPlans(self):
        """ Sets up self.installPlans, a list of one or more "plans"
        to download and install the package. """

        pc = PStatCollector(':App:PackageInstaller:buildInstallPlans')
        pc.start()

        self.hasPackage = False

        if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
            # We're not allowed to download anything.
            self.installPlans = []
            pc.stop()
            return

        if self.asMirror:
            # If we're just downloading a mirror archive, we only need
            # to get the compressed archive file.

            # Build a one-item install plan to download the compressed
            # archive.
            downloadSize = self.compressedArchive.size
            func = lambda step, fileSpec=self.compressedArchive: self.__downloadFile(
                step, fileSpec, allowPartial=True)

            step = self.InstallStep(func, downloadSize, self.downloadFactor,
                                    'download')
            installPlan = [step]
            self.installPlans = [installPlan]
            pc.stop()
            return

        # The normal download process.  Determine what we will need to
        # download, and build a plan (or two) to download it all.
        self.installPlans = None

        # We know we will at least need to unpack the archive contents
        # at the end.
        unpackSize = 0
        for file in self.extracts:
            unpackSize += file.size
        step = self.InstallStep(self.__unpackArchive, unpackSize,
                                self.unpackFactor, 'unpack')
        planA = [step]

        # If the uncompressed archive file is good, that's all we'll
        # need to do.
        self.uncompressedArchive.actualFile = None
        if self.uncompressedArchive.quickVerify(self.getPackageDir(),
                                                notify=self.notify):
            self.installPlans = [planA]
            pc.stop()
            return

        # Maybe the compressed archive file is good.
        if self.compressedArchive.quickVerify(self.getPackageDir(),
                                              notify=self.notify):
            uncompressSize = self.uncompressedArchive.size
            step = self.InstallStep(self.__uncompressArchive, uncompressSize,
                                    self.uncompressFactor, 'uncompress')
            planA = [step] + planA
            self.installPlans = [planA]
            pc.stop()
            return

        # Maybe we can download one or more patches.  We'll come back
        # to that in a minute as plan A.  For now, construct plan B,
        # which will be to download the whole archive.
        planB = planA[:]

        uncompressSize = self.uncompressedArchive.size
        step = self.InstallStep(self.__uncompressArchive, uncompressSize,
                                self.uncompressFactor, 'uncompress')
        planB = [step] + planB

        downloadSize = self.compressedArchive.size
        func = lambda step, fileSpec=self.compressedArchive: self.__downloadFile(
            step, fileSpec, allowPartial=True)

        step = self.InstallStep(func, downloadSize, self.downloadFactor,
                                'download')
        planB = [step] + planB

        # Now look for patches.  Start with the md5 hash from the
        # uncompressedArchive file we have on disk, and see if we can
        # find a patch chain from this file to our target.
        pathname = Filename(self.getPackageDir(),
                            self.uncompressedArchive.filename)
        fileSpec = self.uncompressedArchive.actualFile
        if fileSpec is None and pathname.exists():
            fileSpec = FileSpec()
            fileSpec.fromFile(self.getPackageDir(),
                              self.uncompressedArchive.filename)
        plan = None
        if fileSpec:
            plan = self.__findPatchChain(fileSpec)
        if plan:
            # We can download patches.  Great!  That means this is
            # plan A, and the full download is plan B (in case
            # something goes wrong with the patching).
            planA = plan + planA
            self.installPlans = [planA, planB]
        else:
            # There are no patches to download, oh well.  Stick with
            # plan B as the only plan.
            self.installPlans = [planB]

        # In case of unexpected failures on the internet, we will retry
        # the full download instead of just giving up.
        for retry in range(ConfigVariableInt('package-full-dl-retries', 1)):
            self.installPlans.append(planB[:])

        pc.stop()
示例#23
0
    def __readDescFile(self):
        """ Reads the desc xml file for this particular package,
        assuming it's been already downloaded and verified.  Returns
        true on success, false on failure. """

        if self.hasDescFile:
            # No need to read it again.
            return True

        if self.solo:
            # If this is a "solo" package, we don't actually "read"
            # the desc file; that's the entire contents of the
            # package.
            self.hasDescFile = True
            self.hasPackage = True
            return True

        filename = Filename(self.getPackageDir(), self.descFileBasename)

        if not hasattr(PandaModules, 'TiXmlDocument'):
            return False
        doc = PandaModules.TiXmlDocument(filename.toOsSpecific())
        if not doc.LoadFile():
            return False

        xpackage = doc.FirstChildElement('package')
        if not xpackage:
            return False

        try:
            self.patchVersion = int(xpackage.Attribute('patch_version') or '')
        except ValueError:
            self.patchVersion = None

        self.displayName = None
        xconfig = xpackage.FirstChildElement('config')
        if xconfig:
            # The name for display to an English-speaking user.
            self.displayName = xconfig.Attribute('display_name')

            # True if any apps that use this package must be GUI apps.
            guiApp = xconfig.Attribute('gui_app')
            if guiApp:
                self.guiApp = int(guiApp)

        # The uncompressed archive, which will be mounted directly,
        # and also used for patching.
        xuncompressedArchive = xpackage.FirstChildElement(
            'uncompressed_archive')
        if xuncompressedArchive:
            self.uncompressedArchive = FileSpec()
            self.uncompressedArchive.loadXml(xuncompressedArchive)

        # The compressed archive, which is what is downloaded.
        xcompressedArchive = xpackage.FirstChildElement('compressed_archive')
        if xcompressedArchive:
            self.compressedArchive = FileSpec()
            self.compressedArchive.loadXml(xcompressedArchive)

        # The list of files that should be extracted to disk.
        self.extracts = []
        xextract = xpackage.FirstChildElement('extract')
        while xextract:
            file = FileSpec()
            file.loadXml(xextract)
            self.extracts.append(file)
            xextract = xextract.NextSiblingElement('extract')

        # The list of additional packages that must be installed for
        # this package to function properly.
        self.requires = []
        xrequires = xpackage.FirstChildElement('requires')
        while xrequires:
            packageName = xrequires.Attribute('name')
            version = xrequires.Attribute('version')
            hostUrl = xrequires.Attribute('host')
            if packageName and hostUrl:
                host = self.host.appRunner.getHostWithAlt(hostUrl)
                self.requires.append((packageName, version, host))
            xrequires = xrequires.NextSiblingElement('requires')

        self.hasDescFile = True

        # Now that we've read the desc file, go ahead and use it to
        # verify the download status.
        if self.__checkArchiveStatus():
            # It's all fully downloaded, unpacked, and ready.
            self.hasPackage = True
            return True

        # Still have to download it.
        self.__buildInstallPlans()
        return True
示例#24
0
class PackageInfo:
    """ This class represents a downloadable Panda3D package file that
    can be (or has been) installed into the current runtime.  It is
    the Python equivalent of the P3DPackage class in the core API. """

    notify = directNotify.newCategory("PackageInfo")

    # Weight factors for computing download progress.  This
    # attempts to reflect the relative time-per-byte of each of
    # these operations.
    downloadFactor = 1
    uncompressFactor = 0.01
    unpackFactor = 0.01
    patchFactor = 0.01

    # These tokens are yielded (not returned) by __downloadFile() and
    # other InstallStep functions.
    stepComplete = 1
    stepFailed = 2
    restartDownload = 3
    stepContinue = 4

    UsageBasename = 'usage.xml'

    class InstallStep:
        """ This class is one step of the installPlan list; it
        represents a single atomic piece of the installation step, and
        the relative effort of that piece.  When the plan is executed,
        it will call the saved function pointer here. """
        def __init__(self, func, bytes, factor, stepType):
            self.__funcPtr = func
            self.bytesNeeded = bytes
            self.bytesDone = 0
            self.bytesFactor = factor
            self.stepType = stepType
            self.pStatCol = PStatCollector(':App:PackageInstaller:%s' %
                                           (stepType))

        def func(self):
            """ self.__funcPtr(self) will return a generator of
            tokens.  This function defines a new generator that yields
            each of those tokens, but wraps each call into the nested
            generator within a pair of start/stop collector calls. """

            self.pStatCol.start()
            for token in self.__funcPtr(self):
                self.pStatCol.stop()
                yield token
                self.pStatCol.start()

            # Shouldn't ever get here.
            self.pStatCol.stop()
            raise StopIteration

        def getEffort(self):
            """ Returns the relative amount of effort of this step. """
            return self.bytesNeeded * self.bytesFactor

        def getProgress(self):
            """ Returns the progress of this step, in the range
            0..1. """
            if self.bytesNeeded == 0:
                return 1
            return min(float(self.bytesDone) / float(self.bytesNeeded), 1)

    def __init__(self,
                 host,
                 packageName,
                 packageVersion,
                 platform=None,
                 solo=False,
                 asMirror=False):
        self.host = host
        self.packageName = packageName
        self.packageVersion = packageVersion
        self.platform = platform
        self.solo = solo
        self.asMirror = asMirror

        # This will be active while we are in the middle of a download
        # cycle.
        self.http = None

        # This will be filled in when the host's contents.xml file is
        # read.
        self.packageDir = None

        # These will be filled in by HostInfo when the package is read
        # from contents.xml.
        self.descFile = None
        self.importDescFile = None

        # These are filled in when the desc file is successfully read.
        self.hasDescFile = False
        self.patchVersion = None
        self.displayName = None
        self.guiApp = False
        self.uncompressedArchive = None
        self.compressedArchive = None
        self.extracts = []
        self.requires = []
        self.installPlans = None

        # This is updated during downloadPackage().  It is in the
        # range 0..1.
        self.downloadProgress = 0

        # This is set true when the package file has been fully
        # downloaded and unpacked.
        self.hasPackage = False

        # This is set true when the package has been "installed",
        # meaning it's been added to the paths and all.
        self.installed = False

        # This is set true when the package has been updated in this
        # session, but not yet written to usage.xml.
        self.updated = False
        self.diskSpace = None

    def getPackageDir(self):
        """ Returns the directory in which this package is installed.
        This may not be known until the host's contents.xml file has
        been downloaded, which informs us of the host's own install
        directory. """

        if not self.packageDir:
            if not self.host.hasContentsFile:
                if not self.host.readContentsFile():
                    self.host.downloadContentsFile(self.http)

            # Derive the packageDir from the hostDir.
            self.packageDir = Filename(self.host.hostDir, self.packageName)
            if self.packageVersion:
                self.packageDir = Filename(self.packageDir,
                                           self.packageVersion)

            if self.host.perPlatform:
                # The server directory contains the platform name,
                # though the client directory normally doesn't (unless
                # perPlatform is set true).

                if self.platform:
                    self.packageDir = Filename(self.packageDir, self.platform)

        return self.packageDir

    def getDownloadEffort(self):
        """ Returns the relative amount of effort it will take to
        download this package.  The units are meaningless, except
        relative to other packges."""

        if not self.installPlans:
            return 0

        # Return the size of plan A, assuming it will work.
        plan = self.installPlans[0]
        size = sum(map(lambda step: step.getEffort(), plan))

        return size

    def getPrevDownloadedEffort(self):
        """ Returns a rough estimate of this package's total download
        effort, even if it is already downloaded. """

        effort = 0
        if self.compressedArchive:
            effort += self.compressedArchive.size * self.downloadFactor
        if self.uncompressedArchive:
            effort += self.uncompressedArchive.size * self.uncompressFactor
        # Don't bother counting unpacking.

        return effort

    def getFormattedName(self):
        """ Returns the name of this package, for output to the user.
        This will be the "public" name of the package, as formatted
        for user consumption; it will include capital letters and
        spaces where appropriate. """

        if self.displayName:
            name = self.displayName
        else:
            name = self.packageName
            if self.packageVersion:
                name += ' %s' % (self.packageVersion)

        if self.patchVersion:
            name += ' rev %s' % (self.patchVersion)

        return name

    def setupFilenames(self):
        """ This is called by the HostInfo when the package is read
        from contents.xml, to set up the internal filenames and such
        that rely on some of the information from contents.xml. """

        dirname, basename = self.descFile.filename.rsplit('/', 1)
        self.descFileDirname = dirname
        self.descFileBasename = basename

    def checkStatus(self):
        """ Checks the current status of the desc file and the package
        contents on disk. """

        if self.hasPackage:
            return True

        if not self.hasDescFile:
            filename = Filename(self.getPackageDir(), self.descFileBasename)
            if self.descFile.quickVerify(self.getPackageDir(),
                                         pathname=filename,
                                         notify=self.notify):
                if self.__readDescFile():
                    # Successfully read.  We don't need to call
                    # checkArchiveStatus again, since readDescFile()
                    # has just done it.
                    return self.hasPackage

        if self.hasDescFile:
            if self.__checkArchiveStatus():
                # It's all good.
                self.hasPackage = True

        return self.hasPackage

    def hasCurrentDescFile(self):
        """ Returns true if a desc file file has been successfully
        read for this package and is still current, false
        otherwise. """

        if not self.host.hasCurrentContentsFile():
            return False

        return self.hasDescFile

    def downloadDescFile(self, http):
        """ Downloads the desc file for this particular package,
        synchronously, and then reads it.  Returns true on success,
        false on failure. """

        for token in self.downloadDescFileGenerator(http):
            if token != self.stepContinue:
                break
            Thread.considerYield()

        return (token == self.stepComplete)

    def downloadDescFileGenerator(self, http):
        """ A generator function that implements downloadDescFile()
        one piece at a time.  It yields one of stepComplete,
        stepFailed, or stepContinue. """

        assert self.descFile

        if self.hasDescFile:
            # We've already got one.
            yield self.stepComplete
            return

        if self.host.appRunner and self.host.appRunner.verifyContents != self.host.appRunner.P3DVCNever:
            # We're allowed to download it.
            self.http = http

            func = lambda step, self=self: self.__downloadFile(
                None,
                self.descFile,
                urlbase=self.descFile.filename,
                filename=self.descFileBasename)
            step = self.InstallStep(func, self.descFile.size,
                                    self.downloadFactor, 'downloadDesc')

            for token in step.func():
                if token == self.stepContinue:
                    yield token
                else:
                    break

            while token == self.restartDownload:
                # Try again.
                func = lambda step, self=self: self.__downloadFile(
                    None,
                    self.descFile,
                    urlbase=self.descFile.filename,
                    filename=self.descFileBasename)
                step = self.InstallStep(func, self.descFile.size,
                                        self.downloadFactor, 'downloadDesc')
                for token in step.func():
                    if token == self.stepContinue:
                        yield token
                    else:
                        break

            if token == self.stepFailed:
                # Couldn't download the desc file.
                yield self.stepFailed
                return

            assert token == self.stepComplete

            filename = Filename(self.getPackageDir(), self.descFileBasename)
            # Now that we've written the desc file, make it read-only.
            os.chmod(filename.toOsSpecific(), 0444)

        if not self.__readDescFile():
            # Weird, it passed the hash check, but we still can't read
            # it.
            filename = Filename(self.getPackageDir(), self.descFileBasename)
            self.notify.warning("Failure reading %s" % (filename))
            yield self.stepFailed
            return

        yield self.stepComplete
        return

    def __readDescFile(self):
        """ Reads the desc xml file for this particular package,
        assuming it's been already downloaded and verified.  Returns
        true on success, false on failure. """

        if self.hasDescFile:
            # No need to read it again.
            return True

        if self.solo:
            # If this is a "solo" package, we don't actually "read"
            # the desc file; that's the entire contents of the
            # package.
            self.hasDescFile = True
            self.hasPackage = True
            return True

        filename = Filename(self.getPackageDir(), self.descFileBasename)

        if not hasattr(PandaModules, 'TiXmlDocument'):
            return False
        doc = PandaModules.TiXmlDocument(filename.toOsSpecific())
        if not doc.LoadFile():
            return False

        xpackage = doc.FirstChildElement('package')
        if not xpackage:
            return False

        try:
            self.patchVersion = int(xpackage.Attribute('patch_version') or '')
        except ValueError:
            self.patchVersion = None

        self.displayName = None
        xconfig = xpackage.FirstChildElement('config')
        if xconfig:
            # The name for display to an English-speaking user.
            self.displayName = xconfig.Attribute('display_name')

            # True if any apps that use this package must be GUI apps.
            guiApp = xconfig.Attribute('gui_app')
            if guiApp:
                self.guiApp = int(guiApp)

        # The uncompressed archive, which will be mounted directly,
        # and also used for patching.
        xuncompressedArchive = xpackage.FirstChildElement(
            'uncompressed_archive')
        if xuncompressedArchive:
            self.uncompressedArchive = FileSpec()
            self.uncompressedArchive.loadXml(xuncompressedArchive)

        # The compressed archive, which is what is downloaded.
        xcompressedArchive = xpackage.FirstChildElement('compressed_archive')
        if xcompressedArchive:
            self.compressedArchive = FileSpec()
            self.compressedArchive.loadXml(xcompressedArchive)

        # The list of files that should be extracted to disk.
        self.extracts = []
        xextract = xpackage.FirstChildElement('extract')
        while xextract:
            file = FileSpec()
            file.loadXml(xextract)
            self.extracts.append(file)
            xextract = xextract.NextSiblingElement('extract')

        # The list of additional packages that must be installed for
        # this package to function properly.
        self.requires = []
        xrequires = xpackage.FirstChildElement('requires')
        while xrequires:
            packageName = xrequires.Attribute('name')
            version = xrequires.Attribute('version')
            hostUrl = xrequires.Attribute('host')
            if packageName and hostUrl:
                host = self.host.appRunner.getHostWithAlt(hostUrl)
                self.requires.append((packageName, version, host))
            xrequires = xrequires.NextSiblingElement('requires')

        self.hasDescFile = True

        # Now that we've read the desc file, go ahead and use it to
        # verify the download status.
        if self.__checkArchiveStatus():
            # It's all fully downloaded, unpacked, and ready.
            self.hasPackage = True
            return True

        # Still have to download it.
        self.__buildInstallPlans()
        return True

    def __buildInstallPlans(self):
        """ Sets up self.installPlans, a list of one or more "plans"
        to download and install the package. """

        pc = PStatCollector(':App:PackageInstaller:buildInstallPlans')
        pc.start()

        self.hasPackage = False

        if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
            # We're not allowed to download anything.
            self.installPlans = []
            pc.stop()
            return

        if self.asMirror:
            # If we're just downloading a mirror archive, we only need
            # to get the compressed archive file.

            # Build a one-item install plan to download the compressed
            # archive.
            downloadSize = self.compressedArchive.size
            func = lambda step, fileSpec=self.compressedArchive: self.__downloadFile(
                step, fileSpec, allowPartial=True)

            step = self.InstallStep(func, downloadSize, self.downloadFactor,
                                    'download')
            installPlan = [step]
            self.installPlans = [installPlan]
            pc.stop()
            return

        # The normal download process.  Determine what we will need to
        # download, and build a plan (or two) to download it all.
        self.installPlans = None

        # We know we will at least need to unpack the archive contents
        # at the end.
        unpackSize = 0
        for file in self.extracts:
            unpackSize += file.size
        step = self.InstallStep(self.__unpackArchive, unpackSize,
                                self.unpackFactor, 'unpack')
        planA = [step]

        # If the uncompressed archive file is good, that's all we'll
        # need to do.
        self.uncompressedArchive.actualFile = None
        if self.uncompressedArchive.quickVerify(self.getPackageDir(),
                                                notify=self.notify):
            self.installPlans = [planA]
            pc.stop()
            return

        # Maybe the compressed archive file is good.
        if self.compressedArchive.quickVerify(self.getPackageDir(),
                                              notify=self.notify):
            uncompressSize = self.uncompressedArchive.size
            step = self.InstallStep(self.__uncompressArchive, uncompressSize,
                                    self.uncompressFactor, 'uncompress')
            planA = [step] + planA
            self.installPlans = [planA]
            pc.stop()
            return

        # Maybe we can download one or more patches.  We'll come back
        # to that in a minute as plan A.  For now, construct plan B,
        # which will be to download the whole archive.
        planB = planA[:]

        uncompressSize = self.uncompressedArchive.size
        step = self.InstallStep(self.__uncompressArchive, uncompressSize,
                                self.uncompressFactor, 'uncompress')
        planB = [step] + planB

        downloadSize = self.compressedArchive.size
        func = lambda step, fileSpec=self.compressedArchive: self.__downloadFile(
            step, fileSpec, allowPartial=True)

        step = self.InstallStep(func, downloadSize, self.downloadFactor,
                                'download')
        planB = [step] + planB

        # Now look for patches.  Start with the md5 hash from the
        # uncompressedArchive file we have on disk, and see if we can
        # find a patch chain from this file to our target.
        pathname = Filename(self.getPackageDir(),
                            self.uncompressedArchive.filename)
        fileSpec = self.uncompressedArchive.actualFile
        if fileSpec is None and pathname.exists():
            fileSpec = FileSpec()
            fileSpec.fromFile(self.getPackageDir(),
                              self.uncompressedArchive.filename)
        plan = None
        if fileSpec:
            plan = self.__findPatchChain(fileSpec)
        if plan:
            # We can download patches.  Great!  That means this is
            # plan A, and the full download is plan B (in case
            # something goes wrong with the patching).
            planA = plan + planA
            self.installPlans = [planA, planB]
        else:
            # There are no patches to download, oh well.  Stick with
            # plan B as the only plan.
            self.installPlans = [planB]

        # In case of unexpected failures on the internet, we will retry
        # the full download instead of just giving up.
        for retry in range(ConfigVariableInt('package-full-dl-retries', 1)):
            self.installPlans.append(planB[:])

        pc.stop()

    def __scanDirectoryRecursively(self, dirname):
        """ Generates a list of Filename objects: all of the files
        (not directories) within and below the indicated dirname. """

        contents = []
        for dirpath, dirnames, filenames in os.walk(dirname.toOsSpecific()):
            dirpath = Filename.fromOsSpecific(dirpath)
            if dirpath == dirname:
                dirpath = Filename('')
            else:
                dirpath.makeRelativeTo(dirname)
            for filename in filenames:
                contents.append(Filename(dirpath, filename))
        return contents

    def __removeFileFromList(self, contents, filename):
        """ Removes the indicated filename from the given list, if it is
        present.  """
        try:
            contents.remove(Filename(filename))
        except ValueError:
            pass

    def __checkArchiveStatus(self):
        """ Returns true if the archive and all extractable files are
        already correct on disk, false otherwise. """

        if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
            # Assume that everything is just fine.
            return True

        # Get a list of all of the files in the directory, so we can
        # remove files that don't belong.
        contents = self.__scanDirectoryRecursively(self.getPackageDir())
        self.__removeFileFromList(contents, self.descFileBasename)
        self.__removeFileFromList(contents, self.compressedArchive.filename)
        self.__removeFileFromList(contents, self.UsageBasename)
        if not self.asMirror:
            self.__removeFileFromList(contents,
                                      self.uncompressedArchive.filename)
            for file in self.extracts:
                self.__removeFileFromList(contents, file.filename)

        # Now, any files that are still in the contents list don't
        # belong.  It's important to remove these files before we
        # start verifying the files that we expect to find here, in
        # case there is a problem with ambiguous filenames or
        # something (e.g. case insensitivity).
        for filename in contents:
            self.notify.info("Removing %s" % (filename))
            pathname = Filename(self.getPackageDir(), filename)
            pathname.unlink()
            self.updated = True

        if self.asMirror:
            return self.compressedArchive.quickVerify(self.getPackageDir(),
                                                      notify=self.notify)

        allExtractsOk = True
        if not self.uncompressedArchive.quickVerify(self.getPackageDir(),
                                                    notify=self.notify):
            self.notify.debug("File is incorrect: %s" %
                              (self.uncompressedArchive.filename))
            allExtractsOk = False

        if allExtractsOk:
            # OK, the uncompressed archive is good; that means there
            # shouldn't be a compressed archive file here.
            pathname = Filename(self.getPackageDir(),
                                self.compressedArchive.filename)
            pathname.unlink()

            for file in self.extracts:
                if not file.quickVerify(self.getPackageDir(),
                                        notify=self.notify):
                    self.notify.debug("File is incorrect: %s" %
                                      (file.filename))
                    allExtractsOk = False
                    break

        if allExtractsOk:
            self.notify.debug("All %s extracts of %s seem good." %
                              (len(self.extracts), self.packageName))

        return allExtractsOk

    def __updateStepProgress(self, step):
        """ This callback is made from within the several step
        functions as the download step proceeds.  It updates
        self.downloadProgress with the current progress, so the caller
        can asynchronously query this value. """

        size = self.totalPlanCompleted + self.currentStepEffort * step.getProgress(
        )
        self.downloadProgress = min(float(size) / float(self.totalPlanSize), 1)

    def downloadPackage(self, http):
        """ Downloads the package file, synchronously, then
        uncompresses and unpacks it.  Returns true on success, false
        on failure.

        This assumes that self.installPlans has already been filled
        in, which will have been done by self.__readDescFile().
        """

        for token in self.downloadPackageGenerator(http):
            if token != self.stepContinue:
                break
            Thread.considerYield()

        return (token == self.stepComplete)

    def downloadPackageGenerator(self, http):
        """ A generator function that implements downloadPackage() one
        piece at a time.  It yields one of stepComplete, stepFailed,
        or stepContinue. """

        assert self.hasDescFile

        if self.hasPackage:
            # We've already got one.
            yield self.stepComplete
            return

        if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
            # We're not allowed to download anything. Assume it's already downloaded.
            yield self.stepComplete
            return

        # We should have an install plan by the time we get here.
        assert self.installPlans

        self.http = http
        for token in self.__followInstallPlans():
            if token == self.stepContinue:
                yield token
            else:
                break

        while token == self.restartDownload:
            # Try again.
            for token in self.downloadDescFileGenerator(http):
                if token == self.stepContinue:
                    yield token
                else:
                    break
            if token == self.stepComplete:
                for token in self.__followInstallPlans():
                    if token == self.stepContinue:
                        yield token
                    else:
                        break

        if token == self.stepFailed:
            yield self.stepFailed
            return

        assert token == self.stepComplete
        yield self.stepComplete
        return

    def __followInstallPlans(self):
        """ Performs all of the steps in self.installPlans.  Yields
        one of stepComplete, stepFailed, restartDownload, or
        stepContinue. """

        if not self.installPlans:
            self.__buildInstallPlans()

        installPlans = self.installPlans
        self.installPlans = None
        for plan in installPlans:
            self.totalPlanSize = sum(map(lambda step: step.getEffort(), plan))
            self.totalPlanCompleted = 0
            self.downloadProgress = 0

            planFailed = False
            for step in plan:
                self.currentStepEffort = step.getEffort()

                for token in step.func():
                    if token == self.stepContinue:
                        yield token
                    else:
                        break

                if token == self.restartDownload:
                    yield token
                if token == self.stepFailed:
                    planFailed = True
                    break
                assert token == self.stepComplete

                self.totalPlanCompleted += self.currentStepEffort

            if not planFailed:
                # Successfully downloaded!
                yield self.stepComplete
                return

            if taskMgr.destroyed:
                yield self.stepFailed
                return

        # All plans failed.
        yield self.stepFailed
        return

    def __findPatchChain(self, fileSpec):
        """ Finds the chain of patches that leads from the indicated
        patch version to the current patch version.  If found,
        constructs an installPlan that represents the steps of the
        patch installation; otherwise, returns None. """

        from direct.p3d.PatchMaker import PatchMaker

        patchMaker = PatchMaker(self.getPackageDir())
        patchChain = patchMaker.getPatchChainToCurrent(self.descFileBasename,
                                                       fileSpec)
        if patchChain is None:
            # No path.
            patchMaker.cleanup()
            return None

        plan = []
        for patchfile in patchChain:
            downloadSize = patchfile.file.size
            func = lambda step, fileSpec=patchfile.file: self.__downloadFile(
                step, fileSpec, allowPartial=True)
            step = self.InstallStep(func, downloadSize, self.downloadFactor,
                                    'download')
            plan.append(step)

            patchSize = patchfile.targetFile.size
            func = lambda step, patchfile=patchfile: self.__applyPatch(
                step, patchfile)
            step = self.InstallStep(func, patchSize, self.patchFactor, 'patch')
            plan.append(step)

        patchMaker.cleanup()
        return plan

    def __downloadFile(self,
                       step,
                       fileSpec,
                       urlbase=None,
                       filename=None,
                       allowPartial=False):
        """ Downloads the indicated file from the host into
        packageDir.  Yields one of stepComplete, stepFailed, 
        restartDownload, or stepContinue. """

        if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
            # We're not allowed to download anything.
            yield self.stepFailed
            return

        self.updated = True

        if not urlbase:
            urlbase = self.descFileDirname + '/' + fileSpec.filename

        # Build up a list of URL's to try downloading from.  Unlike
        # the C++ implementation in P3DPackage.cxx, here we build the
        # URL's in forward order.
        tryUrls = []

        if self.host.appRunner and self.host.appRunner.superMirrorUrl:
            # We start with the "super mirror", if it's defined.
            url = self.host.appRunner.superMirrorUrl + urlbase
            tryUrls.append((url, False))

        if self.host.mirrors:
            # Choose two mirrors at random.
            mirrors = self.host.mirrors[:]
            for i in range(2):
                mirror = random.choice(mirrors)
                mirrors.remove(mirror)
                url = mirror + urlbase
                tryUrls.append((url, False))
                if not mirrors:
                    break

        # After trying two mirrors and failing (or if there are no
        # mirrors), go get it from the original host.
        url = self.host.downloadUrlPrefix + urlbase
        tryUrls.append((url, False))

        # And finally, if the original host also fails, try again with
        # a cache-buster.
        tryUrls.append((url, True))

        for url, cacheBust in tryUrls:
            request = DocumentSpec(url)

            if cacheBust:
                # On the last attempt to download a particular file,
                # we bust through the cache: append a query string to
                # do this.
                url += '?' + str(int(time.time()))
                request = DocumentSpec(url)
                request.setCacheControl(DocumentSpec.CCNoCache)

            self.notify.info("%s downloading %s" % (self.packageName, url))

            if not filename:
                filename = fileSpec.filename
            targetPathname = Filename(self.getPackageDir(), filename)
            targetPathname.setBinary()

            channel = self.http.makeChannel(False)

            # If there's a previous partial download, attempt to resume it.
            bytesStarted = 0
            if allowPartial and not cacheBust and targetPathname.exists():
                bytesStarted = targetPathname.getFileSize()

            if bytesStarted < 1024 * 1024:
                # Not enough bytes downloaded to be worth the risk of
                # a partial download.
                bytesStarted = 0
            elif bytesStarted >= fileSpec.size:
                # Couldn't possibly be our file.
                bytesStarted = 0

            if bytesStarted:
                self.notify.info(
                    "Resuming %s after %s bytes already downloaded" %
                    (url, bytesStarted))
                # Make sure the file is writable.
                os.chmod(targetPathname.toOsSpecific(), 0644)
                channel.beginGetSubdocument(request, bytesStarted, 0)
            else:
                # No partial download possible; get the whole file.
                targetPathname.makeDir()
                targetPathname.unlink()
                channel.beginGetDocument(request)

            channel.downloadToFile(targetPathname)
            while channel.run():
                if step:
                    step.bytesDone = channel.getBytesDownloaded(
                    ) + channel.getFirstByteDelivered()
                    if step.bytesDone > step.bytesNeeded:
                        # Oops, too much data.  Might as well abort;
                        # it's the wrong file.
                        self.notify.warning(
                            "Got more data than expected for download %s" %
                            (url))
                        break

                    self.__updateStepProgress(step)

                if taskMgr.destroyed:
                    # If the task manager has been destroyed, we must
                    # be shutting down.  Get out of here.
                    self.notify.warning("Task Manager destroyed, aborting %s" %
                                        (url))
                    yield self.stepFailed
                    return

                yield self.stepContinue

            if step:
                step.bytesDone = channel.getBytesDownloaded(
                ) + channel.getFirstByteDelivered()
                self.__updateStepProgress(step)

            if not channel.isValid():
                self.notify.warning("Failed to download %s" % (url))

            elif not fileSpec.fullVerify(self.getPackageDir(),
                                         pathname=targetPathname,
                                         notify=self.notify):
                self.notify.warning(
                    "After downloading, %s incorrect" %
                    (Filename(fileSpec.filename).getBasename()))

                # This attempt failed.  Maybe the original contents.xml
                # file is stale.  Try re-downloading it now, just to be
                # sure.
                if self.host.redownloadContentsFile(self.http):
                    # Yes!  Go back and start over from the beginning.
                    yield self.restartDownload
                    return

            else:
                # Success!
                yield self.stepComplete
                return

            # Maybe the mirror is bad.  Go back and try the next
            # mirror.

        # All attempts failed.  Maybe the original contents.xml file
        # is stale.  Try re-downloading it now, just to be sure.
        if self.host.redownloadContentsFile(self.http):
            # Yes!  Go back and start over from the beginning.
            yield self.restartDownload
            return

        # All mirrors failed; the server (or the internet connection)
        # must be just fubar.
        yield self.stepFailed
        return

    def __applyPatch(self, step, patchfile):
        """ Applies the indicated patching in-place to the current
        uncompressed archive.  The patchfile is removed after the
        operation.  Yields one of stepComplete, stepFailed, 
        restartDownload, or stepContinue. """

        self.updated = True

        origPathname = Filename(self.getPackageDir(),
                                self.uncompressedArchive.filename)
        patchPathname = Filename(self.getPackageDir(), patchfile.file.filename)
        result = Filename.temporary('', 'patch_')
        self.notify.info("Patching %s with %s" % (origPathname, patchPathname))

        p = PandaModules.Patchfile()  # The C++ class

        ret = p.initiate(patchPathname, origPathname, result)
        if ret == EUSuccess:
            ret = p.run()
        while ret == EUOk:
            step.bytesDone = step.bytesNeeded * p.getProgress()
            self.__updateStepProgress(step)
            if taskMgr.destroyed:
                # If the task manager has been destroyed, we must
                # be shutting down.  Get out of here.
                self.notify.warning(
                    "Task Manager destroyed, aborting patch %s" %
                    (origPathname))
                yield self.stepFailed
                return

            yield self.stepContinue
            ret = p.run()
        del p
        patchPathname.unlink()

        if ret < 0:
            self.notify.warning("Patching of %s failed." % (origPathname))
            result.unlink()
            yield self.stepFailed
            return

        if not result.renameTo(origPathname):
            self.notify.warning("Couldn't rename %s to %s" %
                                (result, origPathname))
            yield self.stepFailed
            return

        yield self.stepComplete
        return

    def __uncompressArchive(self, step):
        """ Turns the compressed archive into the uncompressed
        archive.  Yields one of stepComplete, stepFailed, 
        restartDownload, or stepContinue. """

        if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
            # We're not allowed to!
            yield self.stepFailed
            return

        self.updated = True

        sourcePathname = Filename(self.getPackageDir(),
                                  self.compressedArchive.filename)
        targetPathname = Filename(self.getPackageDir(),
                                  self.uncompressedArchive.filename)
        targetPathname.unlink()
        self.notify.info("Uncompressing %s to %s" %
                         (sourcePathname, targetPathname))
        decompressor = Decompressor()
        decompressor.initiate(sourcePathname, targetPathname)
        totalBytes = self.uncompressedArchive.size
        result = decompressor.run()
        while result == EUOk:
            step.bytesDone = int(totalBytes * decompressor.getProgress())
            self.__updateStepProgress(step)
            result = decompressor.run()
            if taskMgr.destroyed:
                # If the task manager has been destroyed, we must
                # be shutting down.  Get out of here.
                self.notify.warning(
                    "Task Manager destroyed, aborting decompresss %s" %
                    (sourcePathname))
                yield self.stepFailed
                return

            yield self.stepContinue

        if result != EUSuccess:
            yield self.stepFailed
            return

        step.bytesDone = totalBytes
        self.__updateStepProgress(step)

        if not self.uncompressedArchive.quickVerify(self.getPackageDir(),
                                                    notify=self.notify):
            self.notify.warning("after uncompressing, %s still incorrect" %
                                (self.uncompressedArchive.filename))
            yield self.stepFailed
            return

        # Now that we've verified the archive, make it read-only.
        os.chmod(targetPathname.toOsSpecific(), 0444)

        # Now we can safely remove the compressed archive.
        sourcePathname.unlink()
        yield self.stepComplete
        return

    def __unpackArchive(self, step):
        """ Unpacks any files in the archive that want to be unpacked
        to disk.  Yields one of stepComplete, stepFailed, 
        restartDownload, or stepContinue. """

        if not self.extracts:
            # Nothing to extract.
            self.hasPackage = True
            yield self.stepComplete
            return

        if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
            # We're not allowed to!
            yield self.stepFailed
            return

        self.updated = True

        mfPathname = Filename(self.getPackageDir(),
                              self.uncompressedArchive.filename)
        self.notify.info("Unpacking %s" % (mfPathname))
        mf = Multifile()
        if not mf.openRead(mfPathname):
            self.notify.warning("Couldn't open %s" % (mfPathname))
            yield self.stepFailed
            return

        allExtractsOk = True
        step.bytesDone = 0
        for file in self.extracts:
            i = mf.findSubfile(file.filename)
            if i == -1:
                self.notify.warning("Not in Multifile: %s" % (file.filename))
                allExtractsOk = False
                continue

            targetPathname = Filename(self.getPackageDir(), file.filename)
            targetPathname.setBinary()
            targetPathname.unlink()
            if not mf.extractSubfile(i, targetPathname):
                self.notify.warning("Couldn't extract: %s" % (file.filename))
                allExtractsOk = False
                continue

            if not file.quickVerify(self.getPackageDir(), notify=self.notify):
                self.notify.warning("After extracting, still incorrect: %s" %
                                    (file.filename))
                allExtractsOk = False
                continue

            # Make sure it's executable, and not writable.
            os.chmod(targetPathname.toOsSpecific(), 0555)

            step.bytesDone += file.size
            self.__updateStepProgress(step)
            if taskMgr.destroyed:
                # If the task manager has been destroyed, we must
                # be shutting down.  Get out of here.
                self.notify.warning(
                    "Task Manager destroyed, aborting unpacking %s" %
                    (mfPathname))
                yield self.stepFailed
                return

            yield self.stepContinue

        if not allExtractsOk:
            yield self.stepFailed
            return

        self.hasPackage = True
        yield self.stepComplete
        return

    def installPackage(self, appRunner):
        """ Mounts the package and sets up system paths so it becomes
        available for use.  Returns true on success, false on failure. """

        assert self.hasPackage
        if self.installed:
            # Already installed.
            return True
        assert self not in appRunner.installedPackages

        mfPathname = Filename(self.getPackageDir(),
                              self.uncompressedArchive.filename)
        mf = Multifile()
        if not mf.openRead(mfPathname):
            self.notify.warning("Couldn't open %s" % (mfPathname))
            return False

        # We mount it under its actual location on disk.
        root = self.getPackageDir().cStr()

        vfs = VirtualFileSystem.getGlobalPtr()
        vfs.mount(mf, root, vfs.MFReadOnly)

        # Add this to the Python search path, if it's not already
        # there.  We have to take a bit of care to check if it's
        # already there, since there can be some ambiguity in
        # os-specific path strings.
        osRoot = self.getPackageDir().toOsSpecific()
        foundOnPath = False
        for p in sys.path:
            if osRoot == p:
                # Already here, exactly.
                foundOnPath = True
                break
            elif osRoot == Filename.fromOsSpecific(p).toOsSpecific():
                # Already here, with some futzing.
                foundOnPath = True
                break

        if not foundOnPath:
            # Not already here; add it.
            sys.path.append(osRoot)

        # Put it on the model-path, too.  We do this indiscriminantly,
        # because the Panda3D runtime won't be adding things to the
        # model-path, so it shouldn't be already there.
        getModelPath().appendDirectory(self.getPackageDir())

        # Set the environment variable to reference the package root.
        envvar = '%s_ROOT' % (self.packageName.upper())
        ExecutionEnvironment.setEnvironmentVariable(envvar, osRoot)

        # Add the package root to the system paths.
        if sys.platform.startswith('win'):
            path = os.environ.get('PATH', '')
            os.environ['PATH'] = "%s;%s" % (osRoot, path)
        else:
            path = os.environ.get('PATH', '')
            os.environ['PATH'] = "%s:%s" % (osRoot, path)
            path = os.environ.get('LD_LIBRARY_PATH', '')
            os.environ['LD_LIBRARY_PATH'] = "%s:%s" % (osRoot, path)

        if sys.platform == "darwin":
            path = os.environ.get('DYLD_LIBRARY_PATH', '')
            os.environ['DYLD_LIBRARY_PATH'] = "%s:%s" % (osRoot, path)

        # Now that the environment variable is set, read all of the
        # prc files in the package.
        appRunner.loadMultifilePrcFiles(mf, self.getPackageDir())

        # Also, find any toplevel Python packages, and add these as
        # shared packages.  This will allow different packages
        # installed in different directories to share Python files as
        # if they were all in the same directory.
        for filename in mf.getSubfileNames():
            if filename.endswith('/__init__.pyc') or \
               filename.endswith('/__init__.pyo') or \
               filename.endswith('/__init__.py'):
                components = filename.split('/')[:-1]
                moduleName = '.'.join(components)
                VFSImporter.sharedPackages[moduleName] = True

        # Fix up any shared directories so we can load packages from
        # disparate locations.
        VFSImporter.reloadSharedPackages()

        self.installed = True
        appRunner.installedPackages.append(self)

        self.markUsed()

        return True

    def __measureDiskSpace(self):
        """ Returns the amount of space used by this package, in
        bytes, as determined by examining the actual contents of the
        package directory and its subdirectories. """

        thisDir = ScanDirectoryNode(self.getPackageDir(), ignoreUsageXml=True)
        diskSpace = thisDir.getTotalSize()
        self.notify.info("Package %s uses %s MB" %
                         (self.packageName, (diskSpace + 524288) / 1048576))
        return diskSpace

    def markUsed(self):
        """ Marks the package as having been used.  This is normally
        called automatically by installPackage(). """

        if not hasattr(PandaModules, 'TiXmlDocument'):
            return

        if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
            # Not allowed to write any files to the package directory.
            return

        if self.updated:
            # If we've just installed a new version of the package,
            # re-measure the actual disk space used.
            self.diskSpace = self.__measureDiskSpace()

        filename = Filename(self.getPackageDir(), self.UsageBasename)
        doc = TiXmlDocument(filename.toOsSpecific())
        if not doc.LoadFile():
            decl = TiXmlDeclaration("1.0", "utf-8", "")
            doc.InsertEndChild(decl)

        xusage = doc.FirstChildElement('usage')
        if not xusage:
            doc.InsertEndChild(TiXmlElement('usage'))
            xusage = doc.FirstChildElement('usage')

        now = int(time.time())

        count = xusage.Attribute('count_app')
        try:
            count = int(count or '')
        except ValueError:
            count = 0
            xusage.SetAttribute('first_use', str(now))
        count += 1
        xusage.SetAttribute('count_app', str(count))

        xusage.SetAttribute('last_use', str(now))

        if self.updated:
            xusage.SetAttribute('last_update', str(now))
            self.updated = False
        else:
            # Since we haven't changed the disk space, we can just
            # read it from the previous xml file.
            diskSpace = xusage.Attribute('disk_space')
            try:
                diskSpace = int(diskSpace or '')
            except ValueError:
                # Unless it wasn't set already.
                self.diskSpace = self.__measureDiskSpace()

        xusage.SetAttribute('disk_space', str(self.diskSpace))

        # Write the file to a temporary filename, then atomically move
        # it to its actual filename, to avoid race conditions when
        # updating this file.
        tfile = Filename.temporary(self.getPackageDir().cStr(), '.xml')
        if doc.SaveFile(tfile.toOsSpecific()):
            tfile.renameTo(filename)

    def getUsage(self):
        """ Returns the xusage element that is read from the usage.xml
        file, or None if there is no usage.xml file. """

        if not hasattr(PandaModules, 'TiXmlDocument'):
            return None

        filename = Filename(self.getPackageDir(), self.UsageBasename)
        doc = TiXmlDocument(filename.toOsSpecific())
        if not doc.LoadFile():
            return None

        xusage = doc.FirstChildElement('usage')
        if not xusage:
            return None

        return copy.copy(xusage)
示例#25
0
    class PackageEntry:
        """ This corresponds to a <package> entry in the contents.xml
        file. """
        def __init__(self, xpackage, sourceDir):
            self.sourceDir = sourceDir
            self.loadXml(xpackage)

        def getKey(self):
            """ Returns a tuple used for sorting the PackageEntry
            objects uniquely per package. """
            return (self.packageName, self.platform, self.version)

        def isNewer(self, other):
            return self.descFile.timestamp > other.descFile.timestamp

        def loadXml(self, xpackage):
            self.packageName = xpackage.Attribute('name')
            self.platform = xpackage.Attribute('platform')
            self.version = xpackage.Attribute('version')
            solo = xpackage.Attribute('solo')
            self.solo = int(solo or '0')
            perPlatform = xpackage.Attribute('per_platform')
            self.perPlatform = int(perPlatform or '0')

            self.descFile = FileSpec()
            self.descFile.loadXml(xpackage)

            self.validatePackageContents()

            self.descFile.quickVerify(packageDir=self.sourceDir,
                                      notify=PackageMerger.notify,
                                      correctSelf=True)

            self.packageSeq = SeqValue()
            self.packageSeq.loadXml(xpackage, 'seq')
            self.packageSetVer = SeqValue()
            self.packageSetVer.loadXml(xpackage, 'set_ver')

            self.importDescFile = None
            ximport = xpackage.FirstChildElement('import')
            if ximport:
                self.importDescFile = FileSpec()
                self.importDescFile.loadXml(ximport)
                self.importDescFile.quickVerify(packageDir=self.sourceDir,
                                                notify=PackageMerger.notify,
                                                correctSelf=True)

        def makeXml(self):
            """ Returns a new TiXmlElement. """
            xpackage = TiXmlElement('package')
            xpackage.SetAttribute('name', self.packageName)
            if self.platform:
                xpackage.SetAttribute('platform', self.platform)
            if self.version:
                xpackage.SetAttribute('version', self.version)
            if self.solo:
                xpackage.SetAttribute('solo', '1')
            if self.perPlatform:
                xpackage.SetAttribute('per_platform', '1')

            self.descFile.storeXml(xpackage)
            self.packageSeq.storeXml(xpackage, 'seq')
            self.packageSetVer.storeXml(xpackage, 'set_ver')

            if self.importDescFile:
                ximport = TiXmlElement('import')
                self.importDescFile.storeXml(ximport)
                xpackage.InsertEndChild(ximport)

            return xpackage

        def validatePackageContents(self):
            """ Validates the contents of the package directory itself
            against the expected hashes and timestamps.  Updates
            hashes and timestamps where needed. """

            if self.solo:
                return

            needsChange = False
            packageDescFullpath = Filename(self.sourceDir,
                                           self.descFile.filename)
            packageDir = Filename(packageDescFullpath.getDirname())
            doc = TiXmlDocument(packageDescFullpath.toOsSpecific())
            if not doc.LoadFile():
                message = "Could not read XML file: %s" % (
                    self.descFile.filename)
                raise OSError, message

            xpackage = doc.FirstChildElement('package')
            if not xpackage:
                message = "No package definition: %s" % (
                    self.descFile.filename)
                raise OSError, message

            xcompressed = xpackage.FirstChildElement('compressed_archive')
            if xcompressed:
                spec = FileSpec()
                spec.loadXml(xcompressed)
                if not spec.quickVerify(packageDir=packageDir,
                                        notify=PackageMerger.notify,
                                        correctSelf=True):
                    spec.storeXml(xcompressed)
                    needsChange = True

            xpatch = xpackage.FirstChildElement('patch')
            while xpatch:
                spec = FileSpec()
                spec.loadXml(xpatch)
                if not spec.quickVerify(packageDir=packageDir,
                                        notify=PackageMerger.notify,
                                        correctSelf=True):
                    spec.storeXml(xpatch)
                    needsChange = True

                xpatch = xpatch.NextSiblingElement('patch')

            if needsChange:
                PackageMerger.notify.info("Rewriting %s" %
                                          (self.descFile.filename))
                doc.SaveFile()
                self.descFile.quickVerify(packageDir=self.sourceDir,
                                          notify=PackageMerger.notify,
                                          correctSelf=True)
示例#26
0
        def readDescFile(self, doProcessing=False):
            """ Reads the existing package.xml file and stores it in
            this class for later rewriting.  if doProcessing is true,
            it may massage the file and the directory contents in
            preparation for building patches.  Returns true on
            success, false on failure. """

            self.anyChanges = False

            packageDescFullpath = Filename(self.patchMaker.installDir,
                                           self.packageDesc)
            self.doc = TiXmlDocument(packageDescFullpath.toOsSpecific())
            if not self.doc.LoadFile():
                print("Couldn't read %s" % (packageDescFullpath))
                return False

            xpackage = self.doc.FirstChildElement('package')
            if not xpackage:
                return False
            self.packageName = xpackage.Attribute('name')
            self.platform = xpackage.Attribute('platform')
            self.version = xpackage.Attribute('version')

            # All packages we defined in-line are assigned to the
            # "none" host.  TODO: support patching from packages on
            # other hosts, which means we'll need to fill in a value
            # here for those hosts.
            self.hostUrl = None

            self.currentFile = None
            self.baseFile = None
            self.topFile = None
            self.compressedFilename = None
            compressedFile = None

            # Assume there are changes for this version, until we
            # discover that there aren't.
            isNewVersion = True

            # Get the actual current version.
            xarchive = xpackage.FirstChildElement('uncompressed_archive')
            if xarchive:
                self.currentFile = FileSpec()
                self.currentFile.loadXml(xarchive)

            # Get the top_version--the top (newest) of the patch
            # chain.
            xarchive = xpackage.FirstChildElement('top_version')
            if xarchive:
                self.topFile = FileSpec()
                self.topFile.loadXml(xarchive)

                if self.topFile.hash == self.currentFile.hash:
                    # No new version this pass.
                    isNewVersion = False
                else:
                    # There's a new version this pass.  Update it.
                    self.anyChanges = True

            else:
                # If there isn't a top_version yet, we have to make
                # one, by duplicating the currentFile.
                self.topFile = copy.copy(self.currentFile)
                self.anyChanges = True

            # Get the current patch version.  If we have a
            # patch_version attribute, it refers to this particular
            # instance of the file, and that is the current patch
            # version number.  If we only have a last_patch_version
            # attribute, it means a patch has not yet been built for
            # this particular instance, and that number is the
            # previous version's patch version number.
            patchVersion = xpackage.Attribute('patch_version')
            if patchVersion:
                self.patchVersion = int(patchVersion)
            else:
                patchVersion = xpackage.Attribute('last_patch_version')
                if patchVersion:
                    self.patchVersion = int(patchVersion)
                    if isNewVersion:
                        self.patchVersion += 1
                self.anyChanges = True

            # Put the patchVersion in the compressed filename, for
            # cache-busting.  This means when the version changes, its
            # URL will also change, guaranteeing that users will
            # download the latest version, and not some stale cache
            # file.
            xcompressed = xpackage.FirstChildElement('compressed_archive')
            if xcompressed:
                compressedFile = FileSpec()
                compressedFile.loadXml(xcompressed)

                oldCompressedFilename = compressedFile.filename
                self.compressedFilename = oldCompressedFilename

                if doProcessing:
                    newCompressedFilename = '%s.%s.pz' % (
                        self.currentFile.filename, self.patchVersion)
                    if newCompressedFilename != oldCompressedFilename:
                        oldCompressedPathname = Filename(
                            self.packageDir, oldCompressedFilename)
                        newCompressedPathname = Filename(
                            self.packageDir, newCompressedFilename)
                        if oldCompressedPathname.renameTo(
                                newCompressedPathname):
                            compressedFile.fromFile(self.packageDir,
                                                    newCompressedFilename)
                            compressedFile.storeXml(xcompressed)

                        self.compressedFilename = newCompressedFilename
                        self.anyChanges = True

            # Get the base_version--the bottom (oldest) of the patch
            # chain.
            xarchive = xpackage.FirstChildElement('base_version')
            if xarchive:
                self.baseFile = FileSpec()
                self.baseFile.loadXml(xarchive)
            else:
                # If there isn't a base_version yet, we have to make
                # one, by duplicating the currentFile.
                self.baseFile = copy.copy(self.currentFile)

                # Note that the we only store the compressed version
                # of base_filename on disk, but we store the md5 of
                # the uncompressed version in the xml file.  To
                # emphasize this, we name it without the .pz extension
                # in the xml file, even though the compressed file on
                # disk actually has a .pz extension.
                self.baseFile.filename += '.base'

                # Also duplicate the (compressed) file itself.
                if doProcessing and self.compressedFilename:
                    fromPathname = Filename(self.packageDir,
                                            self.compressedFilename)
                    toPathname = Filename(self.packageDir,
                                          self.baseFile.filename + '.pz')
                    fromPathname.copyTo(toPathname)
                self.anyChanges = True

            self.patches = []
            xpatch = xpackage.FirstChildElement('patch')
            while xpatch:
                patchfile = PatchMaker.Patchfile(self)
                patchfile.loadXml(xpatch)
                self.patches.append(patchfile)
                xpatch = xpatch.NextSiblingElement('patch')

            return True
示例#27
0
        def validatePackageContents(self):
            """ Validates the contents of the package directory itself
            against the expected hashes and timestamps.  Updates
            hashes and timestamps where needed. """

            if self.solo:
                return

            needsChange = False
            packageDescFullpath = Filename(self.sourceDir,
                                           self.descFile.filename)
            packageDir = Filename(packageDescFullpath.getDirname())
            doc = TiXmlDocument(packageDescFullpath.toOsSpecific())
            if not doc.LoadFile():
                message = "Could not read XML file: %s" % (
                    self.descFile.filename)
                raise OSError, message

            xpackage = doc.FirstChildElement('package')
            if not xpackage:
                message = "No package definition: %s" % (
                    self.descFile.filename)
                raise OSError, message

            xcompressed = xpackage.FirstChildElement('compressed_archive')
            if xcompressed:
                spec = FileSpec()
                spec.loadXml(xcompressed)
                if not spec.quickVerify(packageDir=packageDir,
                                        notify=PackageMerger.notify,
                                        correctSelf=True):
                    spec.storeXml(xcompressed)
                    needsChange = True

            xpatch = xpackage.FirstChildElement('patch')
            while xpatch:
                spec = FileSpec()
                spec.loadXml(xpatch)
                if not spec.quickVerify(packageDir=packageDir,
                                        notify=PackageMerger.notify,
                                        correctSelf=True):
                    spec.storeXml(xpatch)
                    needsChange = True

                xpatch = xpatch.NextSiblingElement('patch')

            if needsChange:
                PackageMerger.notify.info("Rewriting %s" %
                                          (self.descFile.filename))
                doc.SaveFile()
                self.descFile.quickVerify(packageDir=self.sourceDir,
                                          notify=PackageMerger.notify,
                                          correctSelf=True)
示例#28
0
    def __init__(self, hostUrl, appRunner = None, hostDir = None,
                 rootDir = None, asMirror = False, perPlatform = None):

        """ You must specify either an appRunner or a hostDir to the
        HostInfo constructor.

        If you pass asMirror = True, it means that this HostInfo
        object is to be used to populate a "mirror" folder, a
        duplicate (or subset) of the contents hosted by a server.
        This means when you use this HostInfo to download packages, it
        will only download the compressed archive file and leave it
        there.  At the moment, mirror folders do not download old
        patch files from the server.

        If you pass perPlatform = True, then files are unpacked into a
        platform-specific directory, which is appropriate when you
        might be downloading multiple platforms.  The default is
        perPlatform = False, which means all files are unpacked into
        the host directory directly, without an intervening
        platform-specific directory name.  If asMirror is True, then
        the default is perPlatform = True.

        Note that perPlatform is also restricted by the individual
        package's specification.  """

        self.__setHostUrl(hostUrl)
        self.appRunner = appRunner
        self.rootDir = rootDir
        if rootDir is None and appRunner:
            self.rootDir = appRunner.rootDir

        if hostDir and not isinstance(hostDir, Filename):
            hostDir = Filename.fromOsSpecific(hostDir)

        self.hostDir = hostDir
        self.asMirror = asMirror
        self.perPlatform = perPlatform
        if perPlatform is None:
            self.perPlatform = asMirror

        # Initially false, this is set true when the contents file is
        # successfully read.
        self.hasContentsFile = False

        # This is the time value at which the current contents file is
        # no longer valid.
        self.contentsExpiration = 0

        # Contains the md5 hash of the original contents.xml file.
        self.contentsSpec = FileSpec()

        # descriptiveName will be filled in later, when the
        # contents file is read.
        self.descriptiveName = None

        # A list of known mirrors for this host, all URL's guaranteed
        # to end with a slash.
        self.mirrors = []

        # A map of keyword -> altHost URL's.  An altHost is different
        # than a mirror; an altHost is an alternate URL to download a
        # different (e.g. testing) version of this host's contents.
        # It is rarely used.
        self.altHosts = {}

        # This is a dictionary of packages by (name, version).  It
        # will be filled in when the contents file is read.
        self.packages = {}

        if self.appRunner and self.appRunner.verifyContents != self.appRunner.P3DVCForce:
            # Attempt to pre-read the existing contents.xml; maybe it
            # will be current enough for our purposes.
            self.readContentsFile()
示例#29
0
    def readContentsFile(self, tempFilename = None, freshDownload = False):
        """ Reads the contents.xml file for this particular host, once
        it has been downloaded into the indicated temporary file.
        Returns true on success, false if the contents file is not
        already on disk or is unreadable.

        If tempFilename is specified, it is the filename read, and it
        is copied the file into the standard location if it's not
        there already.  If tempFilename is not specified, the standard
        filename is read if it is known. """

        if not hasattr(PandaModules, 'TiXmlDocument'):
            return False

        if not tempFilename:
            if self.hostDir:
                # If the filename is not specified, we can infer it
                # if we already know our hostDir
                hostDir = self.hostDir
            else:
                # Otherwise, we have to guess the hostDir.
                hostDir = self.__determineHostDir(None, self.hostUrl)

            tempFilename = Filename(hostDir, 'contents.xml')

        doc = PandaModules.TiXmlDocument(tempFilename.toOsSpecific())
        if not doc.LoadFile():
            return False

        xcontents = doc.FirstChildElement('contents')
        if not xcontents:
            return False

        maxAge = xcontents.Attribute('max_age')
        if maxAge:
            try:
                maxAge = int(maxAge)
            except:
                maxAge = None
        if maxAge is None:
            # Default max_age if unspecified (see p3d_plugin.h).
            from direct.p3d.AppRunner import AppRunner
            maxAge = AppRunner.P3D_CONTENTS_DEFAULT_MAX_AGE

        # Get the latest possible expiration time, based on the max_age
        # indication.  Any expiration time later than this is in error.
        now = int(time.time())
        self.contentsExpiration = now + maxAge

        if freshDownload:
            self.contentsSpec.readHash(tempFilename)

            # Update the XML with the new download information.
            xorig = xcontents.FirstChildElement('orig')
            while xorig:
                xcontents.RemoveChild(xorig)
                xorig = xcontents.FirstChildElement('orig')

            xorig = PandaModules.TiXmlElement('orig')
            self.contentsSpec.storeXml(xorig)
            xorig.SetAttribute('expiration', str(self.contentsExpiration))

            xcontents.InsertEndChild(xorig)
            
        else:
            # Read the download hash and expiration time from the XML.
            expiration = None
            xorig = xcontents.FirstChildElement('orig')
            if xorig:
                self.contentsSpec.loadXml(xorig)
                expiration = xorig.Attribute('expiration')
                if expiration:
                    try:
                        expiration = int(expiration)
                    except:
                        expiration = None
            if not self.contentsSpec.hash:
                self.contentsSpec.readHash(tempFilename)

            if expiration is not None:
                self.contentsExpiration = min(self.contentsExpiration, expiration)

        # Look for our own entry in the hosts table.
        if self.hostUrl:
            self.__findHostXml(xcontents)
        else:
            assert self.hostDir
            self.__findHostXmlForHostDir(xcontents)

        if not self.hostDir:
            self.hostDir = self.__determineHostDir(None, self.hostUrl)

        # Get the list of packages available for download and/or import.
        xpackage = xcontents.FirstChildElement('package')
        while xpackage:
            name = xpackage.Attribute('name')
            platform = xpackage.Attribute('platform')
            version = xpackage.Attribute('version')
            try:
                solo = int(xpackage.Attribute('solo') or '')
            except ValueError:
                solo = False
                
            package = self.__makePackage(name, platform, version, solo)
            package.descFile = FileSpec()
            package.descFile.loadXml(xpackage)
            package.setupFilenames()

            package.importDescFile = None
            ximport = xpackage.FirstChildElement('import')
            if ximport:
                package.importDescFile = FileSpec()
                package.importDescFile.loadXml(ximport)

            xpackage = xpackage.NextSiblingElement('package')

        self.hasContentsFile = True

        # Now save the contents.xml file into the standard location.
        if not self.appRunner or self.appRunner.verifyContents != self.appRunner.P3DVCNever:
            assert self.hostDir
            filename = Filename(self.hostDir, 'contents.xml')
            filename.makeDir()
            if freshDownload:
                doc.SaveFile(filename.toOsSpecific())
            else:
                if filename != tempFilename:
                    tempFilename.copyTo(filename)

        return True
示例#30
0
class PackageInfo:

    """ This class represents a downloadable Panda3D package file that
    can be (or has been) installed into the current runtime.  It is
    the Python equivalent of the P3DPackage class in the core API. """

    notify = directNotify.newCategory("PackageInfo")

    # Weight factors for computing download progress.  This
    # attempts to reflect the relative time-per-byte of each of
    # these operations.
    downloadFactor = 1
    uncompressFactor = 0.01
    unpackFactor = 0.01
    patchFactor = 0.01

    # These tokens are yielded (not returned) by __downloadFile() and
    # other InstallStep functions.
    stepComplete = 1
    stepFailed = 2
    restartDownload = 3
    stepContinue = 4

    UsageBasename = 'usage.xml'

    class InstallStep:
        """ This class is one step of the installPlan list; it
        represents a single atomic piece of the installation step, and
        the relative effort of that piece.  When the plan is executed,
        it will call the saved function pointer here. """
        def __init__(self, func, bytes, factor, stepType):
            self.__funcPtr = func
            self.bytesNeeded = bytes
            self.bytesDone = 0
            self.bytesFactor = factor
            self.stepType = stepType
            self.pStatCol = PStatCollector(':App:PackageInstaller:%s' % (stepType))

        def func(self):
            """ self.__funcPtr(self) will return a generator of
            tokens.  This function defines a new generator that yields
            each of those tokens, but wraps each call into the nested
            generator within a pair of start/stop collector calls. """
            
            self.pStatCol.start()
            for token in self.__funcPtr(self):
                self.pStatCol.stop()
                yield token
                self.pStatCol.start()

            # Shouldn't ever get here.
            self.pStatCol.stop()
            raise StopIteration

        def getEffort(self):
            """ Returns the relative amount of effort of this step. """
            return self.bytesNeeded * self.bytesFactor

        def getProgress(self):
            """ Returns the progress of this step, in the range
            0..1. """
            if self.bytesNeeded == 0:
                return 1
            return min(float(self.bytesDone) / float(self.bytesNeeded), 1)
    
    def __init__(self, host, packageName, packageVersion, platform = None,
                 solo = False, asMirror = False):
        self.host = host
        self.packageName = packageName
        self.packageVersion = packageVersion
        self.platform = platform
        self.solo = solo
        self.asMirror = asMirror

        # This will be active while we are in the middle of a download
        # cycle.
        self.http = None

        # This will be filled in when the host's contents.xml file is
        # read.
        self.packageDir = None
            
        # These will be filled in by HostInfo when the package is read
        # from contents.xml.
        self.descFile = None
        self.importDescFile = None

        # These are filled in when the desc file is successfully read.
        self.hasDescFile = False
        self.patchVersion = None
        self.displayName = None
        self.guiApp = False
        self.uncompressedArchive = None
        self.compressedArchive = None
        self.extracts = []
        self.requires = []
        self.installPlans = None
 
        # This is updated during downloadPackage().  It is in the
        # range 0..1.
        self.downloadProgress = 0
        
        # This is set true when the package file has been fully
        # downloaded and unpacked.
        self.hasPackage = False

        # This is set true when the package has been "installed",
        # meaning it's been added to the paths and all.
        self.installed = False

        # This is set true when the package has been updated in this
        # session, but not yet written to usage.xml.
        self.updated = False
        self.diskSpace = None

    def getPackageDir(self):
        """ Returns the directory in which this package is installed.
        This may not be known until the host's contents.xml file has
        been downloaded, which informs us of the host's own install
        directory. """
        
        if not self.packageDir:
            if not self.host.hasContentsFile:
                if not self.host.readContentsFile():
                    self.host.downloadContentsFile(self.http)
            
            # Derive the packageDir from the hostDir.
            self.packageDir = Filename(self.host.hostDir, self.packageName)
            if self.packageVersion:
                self.packageDir = Filename(self.packageDir, self.packageVersion)

            if self.host.perPlatform:
                # The server directory contains the platform name,
                # though the client directory normally doesn't (unless
                # perPlatform is set true).
                
                if self.platform:
                    self.packageDir = Filename(self.packageDir, self.platform)

        return self.packageDir

    def getDownloadEffort(self):
        """ Returns the relative amount of effort it will take to
        download this package.  The units are meaningless, except
        relative to other packges."""

        if not self.installPlans:
            return 0

        # Return the size of plan A, assuming it will work.
        plan = self.installPlans[0]
        size = sum(map(lambda step: step.getEffort(), plan))
        
        return size

    def getPrevDownloadedEffort(self):
        """ Returns a rough estimate of this package's total download
        effort, even if it is already downloaded. """

        effort = 0
        if self.compressedArchive:
            effort += self.compressedArchive.size * self.downloadFactor
        if self.uncompressedArchive:
            effort += self.uncompressedArchive.size * self.uncompressFactor
        # Don't bother counting unpacking.

        return effort

    def getFormattedName(self):
        """ Returns the name of this package, for output to the user.
        This will be the "public" name of the package, as formatted
        for user consumption; it will include capital letters and
        spaces where appropriate. """

        if self.displayName:
            name = self.displayName
        else:
            name = self.packageName
            if self.packageVersion:
                name += ' %s' % (self.packageVersion)

        if self.patchVersion:
            name += ' rev %s' % (self.patchVersion)

        return name
        

    def setupFilenames(self):
        """ This is called by the HostInfo when the package is read
        from contents.xml, to set up the internal filenames and such
        that rely on some of the information from contents.xml. """
        
        dirname, basename = self.descFile.filename.rsplit('/', 1)
        self.descFileDirname = dirname
        self.descFileBasename = basename

    def checkStatus(self):
        """ Checks the current status of the desc file and the package
        contents on disk. """

        if self.hasPackage:
            return True

        if not self.hasDescFile:
            filename = Filename(self.getPackageDir(), self.descFileBasename)
            if self.descFile.quickVerify(self.getPackageDir(), pathname = filename, notify = self.notify):
                if self.__readDescFile():
                    # Successfully read.  We don't need to call
                    # checkArchiveStatus again, since readDescFile()
                    # has just done it.
                    return self.hasPackage

        if self.hasDescFile:
            if self.__checkArchiveStatus():
                # It's all good.
                self.hasPackage = True

        return self.hasPackage

    def hasCurrentDescFile(self):
        """ Returns true if a desc file file has been successfully
        read for this package and is still current, false
        otherwise. """

        if not self.host.hasCurrentContentsFile():
            return False

        return self.hasDescFile

    def downloadDescFile(self, http):
        """ Downloads the desc file for this particular package,
        synchronously, and then reads it.  Returns true on success,
        false on failure. """

        for token in self.downloadDescFileGenerator(http):
            if token != self.stepContinue:
                break
            Thread.considerYield()

        return (token == self.stepComplete)
    
    def downloadDescFileGenerator(self, http):
        """ A generator function that implements downloadDescFile()
        one piece at a time.  It yields one of stepComplete,
        stepFailed, or stepContinue. """

        assert self.descFile

        if self.hasDescFile:
            # We've already got one.
            yield self.stepComplete; return

        if self.host.appRunner and self.host.appRunner.verifyContents != self.host.appRunner.P3DVCNever:
            # We're allowed to download it.
            self.http = http

            func = lambda step, self = self: self.__downloadFile(
                None, self.descFile,
                urlbase = self.descFile.filename,
                filename = self.descFileBasename)
            step = self.InstallStep(func, self.descFile.size, self.downloadFactor, 'downloadDesc')

            for token in step.func():
                if token == self.stepContinue:
                    yield token
                else:
                    break

            while token == self.restartDownload:
                # Try again.
                func = lambda step, self = self: self.__downloadFile(
                    None, self.descFile,
                    urlbase = self.descFile.filename,
                    filename = self.descFileBasename)
                step = self.InstallStep(func, self.descFile.size, self.downloadFactor, 'downloadDesc')
                for token in step.func():
                    if token == self.stepContinue:
                        yield token
                    else:
                        break

            if token == self.stepFailed:
                # Couldn't download the desc file.
                yield self.stepFailed; return

            assert token == self.stepComplete

            filename = Filename(self.getPackageDir(), self.descFileBasename)
            # Now that we've written the desc file, make it read-only.
            os.chmod(filename.toOsSpecific(), 0444)

        if not self.__readDescFile():
            # Weird, it passed the hash check, but we still can't read
            # it.
            filename = Filename(self.getPackageDir(), self.descFileBasename)
            self.notify.warning("Failure reading %s" % (filename))
            yield self.stepFailed; return

        yield self.stepComplete; return

    def __readDescFile(self):
        """ Reads the desc xml file for this particular package,
        assuming it's been already downloaded and verified.  Returns
        true on success, false on failure. """

        if self.hasDescFile:
            # No need to read it again.
            return True

        if self.solo:
            # If this is a "solo" package, we don't actually "read"
            # the desc file; that's the entire contents of the
            # package.
            self.hasDescFile = True
            self.hasPackage = True
            return True

        filename = Filename(self.getPackageDir(), self.descFileBasename)

        if not hasattr(PandaModules, 'TiXmlDocument'):
            return False
        doc = PandaModules.TiXmlDocument(filename.toOsSpecific())
        if not doc.LoadFile():
            return False

        xpackage = doc.FirstChildElement('package')
        if not xpackage:
            return False

        try:
            self.patchVersion = int(xpackage.Attribute('patch_version') or '')
        except ValueError:
            self.patchVersion = None

        self.displayName = None
        xconfig = xpackage.FirstChildElement('config')
        if xconfig:
            # The name for display to an English-speaking user.
            self.displayName = xconfig.Attribute('display_name')

            # True if any apps that use this package must be GUI apps.
            guiApp = xconfig.Attribute('gui_app')
            if guiApp:
                self.guiApp = int(guiApp)

        # The uncompressed archive, which will be mounted directly,
        # and also used for patching.
        xuncompressedArchive = xpackage.FirstChildElement('uncompressed_archive')
        if xuncompressedArchive:
            self.uncompressedArchive = FileSpec()
            self.uncompressedArchive.loadXml(xuncompressedArchive)

        # The compressed archive, which is what is downloaded.
        xcompressedArchive = xpackage.FirstChildElement('compressed_archive')
        if xcompressedArchive:
            self.compressedArchive = FileSpec()
            self.compressedArchive.loadXml(xcompressedArchive)

        # The list of files that should be extracted to disk.
        self.extracts = []
        xextract = xpackage.FirstChildElement('extract')
        while xextract:
            file = FileSpec()
            file.loadXml(xextract)
            self.extracts.append(file)
            xextract = xextract.NextSiblingElement('extract')

        # The list of additional packages that must be installed for
        # this package to function properly.
        self.requires = []
        xrequires = xpackage.FirstChildElement('requires')
        while xrequires:
            packageName = xrequires.Attribute('name')
            version = xrequires.Attribute('version')
            hostUrl = xrequires.Attribute('host')
            if packageName and hostUrl:
                host = self.host.appRunner.getHostWithAlt(hostUrl)
                self.requires.append((packageName, version, host))
            xrequires = xrequires.NextSiblingElement('requires')

        self.hasDescFile = True

        # Now that we've read the desc file, go ahead and use it to
        # verify the download status.
        if self.__checkArchiveStatus():
            # It's all fully downloaded, unpacked, and ready.
            self.hasPackage = True
            return True

        # Still have to download it.
        self.__buildInstallPlans()
        return True

    def __buildInstallPlans(self):
        """ Sets up self.installPlans, a list of one or more "plans"
        to download and install the package. """

        pc = PStatCollector(':App:PackageInstaller:buildInstallPlans')
        pc.start()

        self.hasPackage = False
        
        if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
            # We're not allowed to download anything.
            self.installPlans = []
            pc.stop()
            return

        if self.asMirror:
            # If we're just downloading a mirror archive, we only need
            # to get the compressed archive file.

            # Build a one-item install plan to download the compressed
            # archive.
            downloadSize = self.compressedArchive.size
            func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec, allowPartial = True)
            
            step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download')
            installPlan = [step]
            self.installPlans = [installPlan]
            pc.stop()
            return 

        # The normal download process.  Determine what we will need to
        # download, and build a plan (or two) to download it all.
        self.installPlans = None

        # We know we will at least need to unpack the archive contents
        # at the end.
        unpackSize = 0
        for file in self.extracts:
            unpackSize += file.size
        step = self.InstallStep(self.__unpackArchive, unpackSize, self.unpackFactor, 'unpack')
        planA = [step]

        # If the uncompressed archive file is good, that's all we'll
        # need to do.
        self.uncompressedArchive.actualFile = None
        if self.uncompressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
            self.installPlans = [planA]
            pc.stop()
            return

        # Maybe the compressed archive file is good.
        if self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
            uncompressSize = self.uncompressedArchive.size
            step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor, 'uncompress')
            planA = [step] + planA
            self.installPlans = [planA]
            pc.stop()
            return

        # Maybe we can download one or more patches.  We'll come back
        # to that in a minute as plan A.  For now, construct plan B,
        # which will be to download the whole archive.
        planB = planA[:]

        uncompressSize = self.uncompressedArchive.size
        step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor, 'uncompress')
        planB = [step] + planB

        downloadSize = self.compressedArchive.size
        func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec, allowPartial = True)

        step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download')
        planB = [step] + planB

        # Now look for patches.  Start with the md5 hash from the
        # uncompressedArchive file we have on disk, and see if we can
        # find a patch chain from this file to our target.
        pathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
        fileSpec = self.uncompressedArchive.actualFile
        if fileSpec is None and pathname.exists():
            fileSpec = FileSpec()
            fileSpec.fromFile(self.getPackageDir(), self.uncompressedArchive.filename)
        plan = None
        if fileSpec:
            plan = self.__findPatchChain(fileSpec)
        if plan:
            # We can download patches.  Great!  That means this is
            # plan A, and the full download is plan B (in case
            # something goes wrong with the patching).
            planA = plan + planA
            self.installPlans = [planA, planB]
        else:
            # There are no patches to download, oh well.  Stick with
            # plan B as the only plan.
            self.installPlans = [planB]

        # In case of unexpected failures on the internet, we will retry 
        # the full download instead of just giving up.
        for retry in range(ConfigVariableInt('package-full-dl-retries', 1)):
            self.installPlans.append(planB[:])

        pc.stop()

    def __scanDirectoryRecursively(self, dirname):
        """ Generates a list of Filename objects: all of the files
        (not directories) within and below the indicated dirname. """
        
        contents = []
        for dirpath, dirnames, filenames in os.walk(dirname.toOsSpecific()):
            dirpath = Filename.fromOsSpecific(dirpath)
            if dirpath == dirname:
                dirpath = Filename('')
            else:
                dirpath.makeRelativeTo(dirname)
            for filename in filenames:
                contents.append(Filename(dirpath, filename))
        return contents

    def __removeFileFromList(self, contents, filename):
        """ Removes the indicated filename from the given list, if it is
        present.  """
        try:
            contents.remove(Filename(filename))
        except ValueError:
            pass

    def __checkArchiveStatus(self):
        """ Returns true if the archive and all extractable files are
        already correct on disk, false otherwise. """

        if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
            # Assume that everything is just fine.
            return True

        # Get a list of all of the files in the directory, so we can
        # remove files that don't belong.
        contents = self.__scanDirectoryRecursively(self.getPackageDir()) 
        self.__removeFileFromList(contents, self.descFileBasename)
        self.__removeFileFromList(contents, self.compressedArchive.filename)
        self.__removeFileFromList(contents, self.UsageBasename)
        if not self.asMirror:
            self.__removeFileFromList(contents, self.uncompressedArchive.filename)
            for file in self.extracts:
                self.__removeFileFromList(contents, file.filename)

        # Now, any files that are still in the contents list don't
        # belong.  It's important to remove these files before we
        # start verifying the files that we expect to find here, in
        # case there is a problem with ambiguous filenames or
        # something (e.g. case insensitivity).
        for filename in contents:
            self.notify.info("Removing %s" % (filename))
            pathname = Filename(self.getPackageDir(), filename)
            pathname.unlink()
            self.updated = True

        if self.asMirror:
            return self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify)
            
        allExtractsOk = True
        if not self.uncompressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
            self.notify.debug("File is incorrect: %s" % (self.uncompressedArchive.filename))
            allExtractsOk = False

        if allExtractsOk:
            # OK, the uncompressed archive is good; that means there
            # shouldn't be a compressed archive file here.
            pathname = Filename(self.getPackageDir(), self.compressedArchive.filename)
            pathname.unlink()
            
            for file in self.extracts:
                if not file.quickVerify(self.getPackageDir(), notify = self.notify):
                    self.notify.debug("File is incorrect: %s" % (file.filename))
                    allExtractsOk = False
                    break

        if allExtractsOk:
            self.notify.debug("All %s extracts of %s seem good." % (
                len(self.extracts), self.packageName))

        return allExtractsOk

    def __updateStepProgress(self, step):
        """ This callback is made from within the several step
        functions as the download step proceeds.  It updates
        self.downloadProgress with the current progress, so the caller
        can asynchronously query this value. """

        size = self.totalPlanCompleted + self.currentStepEffort * step.getProgress()
        self.downloadProgress = min(float(size) / float(self.totalPlanSize), 1)
    
    def downloadPackage(self, http):
        """ Downloads the package file, synchronously, then
        uncompresses and unpacks it.  Returns true on success, false
        on failure.

        This assumes that self.installPlans has already been filled
        in, which will have been done by self.__readDescFile().
        """

        for token in self.downloadPackageGenerator(http):
            if token != self.stepContinue:
                break
            Thread.considerYield()

        return (token == self.stepComplete)
    
    def downloadPackageGenerator(self, http):
        """ A generator function that implements downloadPackage() one
        piece at a time.  It yields one of stepComplete, stepFailed,
        or stepContinue. """

        assert self.hasDescFile

        if self.hasPackage:
            # We've already got one.
            yield self.stepComplete; return

        if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
            # We're not allowed to download anything. Assume it's already downloaded.
            yield self.stepComplete; return

        # We should have an install plan by the time we get here.
        assert self.installPlans

        self.http = http
        for token in self.__followInstallPlans():
            if token == self.stepContinue:
                yield token
            else:
                break
            
        while token == self.restartDownload:
            # Try again.
            for token in self.downloadDescFileGenerator(http):
                if token == self.stepContinue:
                    yield token
                else:
                    break
            if token == self.stepComplete:
                for token in self.__followInstallPlans():
                    if token == self.stepContinue:
                        yield token
                    else:
                        break

        if token == self.stepFailed:
            yield self.stepFailed; return

        assert token == self.stepComplete
        yield self.stepComplete; return
            

    def __followInstallPlans(self):
        """ Performs all of the steps in self.installPlans.  Yields
        one of stepComplete, stepFailed, restartDownload, or
        stepContinue. """

        if not self.installPlans:
            self.__buildInstallPlans()

        installPlans = self.installPlans
        self.installPlans = None
        for plan in installPlans:
            self.totalPlanSize = sum(map(lambda step: step.getEffort(), plan))
            self.totalPlanCompleted = 0
            self.downloadProgress = 0

            planFailed = False
            for step in plan:
                self.currentStepEffort = step.getEffort()

                for token in step.func():
                    if token == self.stepContinue:
                        yield token
                    else:
                        break
                    
                if token == self.restartDownload:
                    yield token
                if token == self.stepFailed:
                    planFailed = True
                    break
                assert token == self.stepComplete
                
                self.totalPlanCompleted += self.currentStepEffort
                
            if not planFailed:
                # Successfully downloaded!
                yield self.stepComplete; return

            if taskMgr.destroyed:
                yield self.stepFailed; return

        # All plans failed.
        yield self.stepFailed; return

    def __findPatchChain(self, fileSpec):
        """ Finds the chain of patches that leads from the indicated
        patch version to the current patch version.  If found,
        constructs an installPlan that represents the steps of the
        patch installation; otherwise, returns None. """

        from direct.p3d.PatchMaker import PatchMaker

        patchMaker = PatchMaker(self.getPackageDir())
        patchChain = patchMaker.getPatchChainToCurrent(self.descFileBasename, fileSpec)
        if patchChain is None:
            # No path.
            patchMaker.cleanup()
            return None

        plan = []
        for patchfile in patchChain:
            downloadSize = patchfile.file.size
            func = lambda step, fileSpec = patchfile.file: self.__downloadFile(step, fileSpec, allowPartial = True)
            step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download')
            plan.append(step)

            patchSize = patchfile.targetFile.size
            func = lambda step, patchfile = patchfile: self.__applyPatch(step, patchfile)
            step = self.InstallStep(func, patchSize, self.patchFactor, 'patch')
            plan.append(step)

        patchMaker.cleanup()
        return plan

    def __downloadFile(self, step, fileSpec, urlbase = None, filename = None,
                       allowPartial = False):
        """ Downloads the indicated file from the host into
        packageDir.  Yields one of stepComplete, stepFailed, 
        restartDownload, or stepContinue. """

        if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
            # We're not allowed to download anything.
            yield self.stepFailed; return

        self.updated = True

        if not urlbase:
            urlbase = self.descFileDirname + '/' + fileSpec.filename

        # Build up a list of URL's to try downloading from.  Unlike
        # the C++ implementation in P3DPackage.cxx, here we build the
        # URL's in forward order.
        tryUrls = []

        if self.host.appRunner and self.host.appRunner.superMirrorUrl:
            # We start with the "super mirror", if it's defined.
            url = self.host.appRunner.superMirrorUrl + urlbase
            tryUrls.append((url, False))

        if self.host.mirrors:
            # Choose two mirrors at random.
            mirrors = self.host.mirrors[:]
            for i in range(2):
                mirror = random.choice(mirrors)
                mirrors.remove(mirror)
                url = mirror + urlbase
                tryUrls.append((url, False))
                if not mirrors:
                    break

        # After trying two mirrors and failing (or if there are no
        # mirrors), go get it from the original host.
        url = self.host.downloadUrlPrefix + urlbase
        tryUrls.append((url, False))

        # And finally, if the original host also fails, try again with
        # a cache-buster.
        tryUrls.append((url, True))

        for url, cacheBust in tryUrls:
            request = DocumentSpec(url)

            if cacheBust:
                # On the last attempt to download a particular file,
                # we bust through the cache: append a query string to
                # do this.
                url += '?' + str(int(time.time()))
                request = DocumentSpec(url)
                request.setCacheControl(DocumentSpec.CCNoCache)
             
            self.notify.info("%s downloading %s" % (self.packageName, url))

            if not filename:
                filename = fileSpec.filename
            targetPathname = Filename(self.getPackageDir(), filename)
            targetPathname.setBinary()

            channel = self.http.makeChannel(False)

            # If there's a previous partial download, attempt to resume it.
            bytesStarted = 0
            if allowPartial and not cacheBust and targetPathname.exists():
                bytesStarted = targetPathname.getFileSize()

            if bytesStarted < 1024*1024:
                # Not enough bytes downloaded to be worth the risk of
                # a partial download.
                bytesStarted = 0
            elif bytesStarted >= fileSpec.size:
                # Couldn't possibly be our file.
                bytesStarted = 0

            if bytesStarted:
                self.notify.info("Resuming %s after %s bytes already downloaded" % (url, bytesStarted))
                # Make sure the file is writable.
                os.chmod(targetPathname.toOsSpecific(), 0644)
                channel.beginGetSubdocument(request, bytesStarted, 0)
            else:
                # No partial download possible; get the whole file.
                targetPathname.makeDir()
                targetPathname.unlink()
                channel.beginGetDocument(request)
                
            channel.downloadToFile(targetPathname)
            while channel.run():
                if step:
                    step.bytesDone = channel.getBytesDownloaded() + channel.getFirstByteDelivered()
                    if step.bytesDone > step.bytesNeeded:
                        # Oops, too much data.  Might as well abort;
                        # it's the wrong file.
                        self.notify.warning("Got more data than expected for download %s" % (url))
                        break
                    
                    self.__updateStepProgress(step)

                if taskMgr.destroyed:
                    # If the task manager has been destroyed, we must
                    # be shutting down.  Get out of here.
                    self.notify.warning("Task Manager destroyed, aborting %s" % (url))
                    yield self.stepFailed; return
                    
                yield self.stepContinue
                
            if step:
                step.bytesDone = channel.getBytesDownloaded() + channel.getFirstByteDelivered()
                self.__updateStepProgress(step)

            if not channel.isValid():
                self.notify.warning("Failed to download %s" % (url))

            elif not fileSpec.fullVerify(self.getPackageDir(), pathname = targetPathname, notify = self.notify):
                self.notify.warning("After downloading, %s incorrect" % (Filename(fileSpec.filename).getBasename()))

                # This attempt failed.  Maybe the original contents.xml
                # file is stale.  Try re-downloading it now, just to be
                # sure.
                if self.host.redownloadContentsFile(self.http):
                    # Yes!  Go back and start over from the beginning.
                    yield self.restartDownload; return

            else:
                # Success!
                yield self.stepComplete; return

            # Maybe the mirror is bad.  Go back and try the next
            # mirror.

        # All attempts failed.  Maybe the original contents.xml file
        # is stale.  Try re-downloading it now, just to be sure.
        if self.host.redownloadContentsFile(self.http):
            # Yes!  Go back and start over from the beginning.
            yield self.restartDownload; return

        # All mirrors failed; the server (or the internet connection)
        # must be just fubar.
        yield self.stepFailed; return

    def __applyPatch(self, step, patchfile):
        """ Applies the indicated patching in-place to the current
        uncompressed archive.  The patchfile is removed after the
        operation.  Yields one of stepComplete, stepFailed, 
        restartDownload, or stepContinue. """

        self.updated = True

        origPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
        patchPathname = Filename(self.getPackageDir(), patchfile.file.filename)
        result = Filename.temporary('', 'patch_')
        self.notify.info("Patching %s with %s" % (origPathname, patchPathname))

        p = PandaModules.Patchfile()  # The C++ class

        ret = p.initiate(patchPathname, origPathname, result)
        if ret == EUSuccess:
            ret = p.run()
        while ret == EUOk:
            step.bytesDone = step.bytesNeeded * p.getProgress()
            self.__updateStepProgress(step)
            if taskMgr.destroyed:
                # If the task manager has been destroyed, we must
                # be shutting down.  Get out of here.
                self.notify.warning("Task Manager destroyed, aborting patch %s" % (origPathname))
                yield self.stepFailed; return

            yield self.stepContinue
            ret = p.run()
        del p
        patchPathname.unlink()
        
        if ret < 0:
            self.notify.warning("Patching of %s failed." % (origPathname))
            result.unlink()
            yield self.stepFailed; return

        if not result.renameTo(origPathname):
            self.notify.warning("Couldn't rename %s to %s" % (result, origPathname))
            yield self.stepFailed; return
            
        yield self.stepComplete; return

    def __uncompressArchive(self, step):
        """ Turns the compressed archive into the uncompressed
        archive.  Yields one of stepComplete, stepFailed, 
        restartDownload, or stepContinue. """

        if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
            # We're not allowed to!
            yield self.stepFailed; return

        self.updated = True

        sourcePathname = Filename(self.getPackageDir(), self.compressedArchive.filename)
        targetPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
        targetPathname.unlink()
        self.notify.info("Uncompressing %s to %s" % (sourcePathname, targetPathname))
        decompressor = Decompressor()
        decompressor.initiate(sourcePathname, targetPathname)
        totalBytes = self.uncompressedArchive.size
        result = decompressor.run()
        while result == EUOk:
            step.bytesDone = int(totalBytes * decompressor.getProgress())
            self.__updateStepProgress(step)
            result = decompressor.run()
            if taskMgr.destroyed:
                # If the task manager has been destroyed, we must
                # be shutting down.  Get out of here.
                self.notify.warning("Task Manager destroyed, aborting decompresss %s" % (sourcePathname))
                yield self.stepFailed; return

            yield self.stepContinue

        if result != EUSuccess:
            yield self.stepFailed; return
            
        step.bytesDone = totalBytes
        self.__updateStepProgress(step)

        if not self.uncompressedArchive.quickVerify(self.getPackageDir(), notify= self.notify):
            self.notify.warning("after uncompressing, %s still incorrect" % (
                self.uncompressedArchive.filename))
            yield self.stepFailed; return

        # Now that we've verified the archive, make it read-only.
        os.chmod(targetPathname.toOsSpecific(), 0444)

        # Now we can safely remove the compressed archive.
        sourcePathname.unlink()
        yield self.stepComplete; return
    
    def __unpackArchive(self, step):
        """ Unpacks any files in the archive that want to be unpacked
        to disk.  Yields one of stepComplete, stepFailed, 
        restartDownload, or stepContinue. """

        if not self.extracts:
            # Nothing to extract.
            self.hasPackage = True
            yield self.stepComplete; return

        if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
            # We're not allowed to!
            yield self.stepFailed; return

        self.updated = True

        mfPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
        self.notify.info("Unpacking %s" % (mfPathname))
        mf = Multifile()
        if not mf.openRead(mfPathname):
            self.notify.warning("Couldn't open %s" % (mfPathname))
            yield self.stepFailed; return
        
        allExtractsOk = True
        step.bytesDone = 0
        for file in self.extracts:
            i = mf.findSubfile(file.filename)
            if i == -1:
                self.notify.warning("Not in Multifile: %s" % (file.filename))
                allExtractsOk = False
                continue

            targetPathname = Filename(self.getPackageDir(), file.filename)
            targetPathname.unlink()
            if not mf.extractSubfile(i, targetPathname):
                self.notify.warning("Couldn't extract: %s" % (file.filename))
                allExtractsOk = False
                continue
            
            if not file.quickVerify(self.getPackageDir(), notify = self.notify):
                self.notify.warning("After extracting, still incorrect: %s" % (file.filename))
                allExtractsOk = False
                continue

            # Make sure it's executable, and not writable.
            os.chmod(targetPathname.toOsSpecific(), 0555)

            step.bytesDone += file.size
            self.__updateStepProgress(step)
            if taskMgr.destroyed:
                # If the task manager has been destroyed, we must
                # be shutting down.  Get out of here.
                self.notify.warning("Task Manager destroyed, aborting unpacking %s" % (mfPathname))
                yield self.stepFailed; return

            yield self.stepContinue

        if not allExtractsOk:
            yield self.stepFailed; return

        self.hasPackage = True
        yield self.stepComplete; return

    def installPackage(self, appRunner):
        """ Mounts the package and sets up system paths so it becomes
        available for use.  Returns true on success, false on failure. """

        assert self.hasPackage
        if self.installed:
            # Already installed.
            return True
        assert self not in appRunner.installedPackages

        mfPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
        mf = Multifile()
        if not mf.openRead(mfPathname):
            self.notify.warning("Couldn't open %s" % (mfPathname))
            return False

        # We mount it under its actual location on disk.
        root = self.getPackageDir().cStr()

        vfs = VirtualFileSystem.getGlobalPtr()
        vfs.mount(mf, root, vfs.MFReadOnly)

        # Add this to the Python search path, if it's not already
        # there.  We have to take a bit of care to check if it's
        # already there, since there can be some ambiguity in
        # os-specific path strings.
        osRoot = self.getPackageDir().toOsSpecific()
        foundOnPath = False
        for p in sys.path:
            if osRoot == p:
                # Already here, exactly.
                foundOnPath = True
                break
            elif osRoot == Filename.fromOsSpecific(p).toOsSpecific():
                # Already here, with some futzing.
                foundOnPath = True
                break

        if not foundOnPath:
            # Not already here; add it.
            sys.path.append(osRoot)

        # Put it on the model-path, too.  We do this indiscriminantly,
        # because the Panda3D runtime won't be adding things to the
        # model-path, so it shouldn't be already there.
        getModelPath().appendDirectory(self.getPackageDir())

        # Set the environment variable to reference the package root.
        envvar = '%s_ROOT' % (self.packageName.upper())
        ExecutionEnvironment.setEnvironmentVariable(envvar, osRoot)

        # Now that the environment variable is set, read all of the
        # prc files in the package.
        appRunner.loadMultifilePrcFiles(mf, self.getPackageDir())

        # Also, find any toplevel Python packages, and add these as
        # shared packages.  This will allow different packages
        # installed in different directories to share Python files as
        # if they were all in the same directory.
        for filename in mf.getSubfileNames():
            if filename.endswith('/__init__.pyc') or \
               filename.endswith('/__init__.pyo') or \
               filename.endswith('/__init__.py'):
                components = filename.split('/')[:-1]
                moduleName = '.'.join(components)
                VFSImporter.sharedPackages[moduleName] = True

        # Fix up any shared directories so we can load packages from
        # disparate locations.
        VFSImporter.reloadSharedPackages()

        self.installed = True
        appRunner.installedPackages.append(self)

        self.markUsed()

        return True

    def __measureDiskSpace(self):
        """ Returns the amount of space used by this package, in
        bytes, as determined by examining the actual contents of the
        package directory and its subdirectories. """

        thisDir = ScanDirectoryNode(self.getPackageDir(), ignoreUsageXml = True)
        diskSpace = thisDir.getTotalSize()
        self.notify.info("Package %s uses %s MB" % (
            self.packageName, (diskSpace + 524288) / 1048576))
        return diskSpace

    def markUsed(self):
        """ Marks the package as having been used.  This is normally
        called automatically by installPackage(). """

        if not hasattr(PandaModules, 'TiXmlDocument'):
            return

        if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
            # Not allowed to write any files to the package directory.
            return

        if self.updated:
            # If we've just installed a new version of the package,
            # re-measure the actual disk space used.
            self.diskSpace = self.__measureDiskSpace()

        filename = Filename(self.getPackageDir(), self.UsageBasename)
        doc = TiXmlDocument(filename.toOsSpecific())
        if not doc.LoadFile():
            decl = TiXmlDeclaration("1.0", "utf-8", "")
            doc.InsertEndChild(decl)
            
        xusage = doc.FirstChildElement('usage')
        if not xusage:
            doc.InsertEndChild(TiXmlElement('usage'))
            xusage = doc.FirstChildElement('usage')

        now = int(time.time())
        
        count = xusage.Attribute('count_app')
        try:
            count = int(count or '')
        except ValueError:
            count = 0
            xusage.SetAttribute('first_use', str(now))
        count += 1
        xusage.SetAttribute('count_app', str(count))

        xusage.SetAttribute('last_use', str(now))

        if self.updated:
            xusage.SetAttribute('last_update', str(now))
            self.updated = False
        else:
            # Since we haven't changed the disk space, we can just
            # read it from the previous xml file.
            diskSpace = xusage.Attribute('disk_space')
            try:
                diskSpace = int(diskSpace or '')
            except ValueError:
                # Unless it wasn't set already.
                self.diskSpace = self.__measureDiskSpace()

        xusage.SetAttribute('disk_space', str(self.diskSpace))

        # Write the file to a temporary filename, then atomically move
        # it to its actual filename, to avoid race conditions when
        # updating this file.
        tfile = Filename.temporary(self.getPackageDir().cStr(), '.xml')
        if doc.SaveFile(tfile.toOsSpecific()):
            tfile.renameTo(filename)
        
    def getUsage(self):
        """ Returns the xusage element that is read from the usage.xml
        file, or None if there is no usage.xml file. """

        if not hasattr(PandaModules, 'TiXmlDocument'):
            return None

        filename = Filename(self.getPackageDir(), self.UsageBasename)
        doc = TiXmlDocument(filename.toOsSpecific())
        if not doc.LoadFile():
            return None
            
        xusage = doc.FirstChildElement('usage')
        if not xusage:
            return None

        return copy.copy(xusage)
示例#31
0
    class Package:
        """ This is a particular package.  This contains all of the
        information needed to reconstruct the package's desc file. """
        def __init__(self, packageDesc, patchMaker, xpackage=None):
            self.packageDir = Filename(patchMaker.installDir,
                                       packageDesc.getDirname())
            self.packageDesc = packageDesc
            self.patchMaker = patchMaker
            self.contentsDocPackage = xpackage
            self.patchVersion = 1
            self.currentPv = None
            self.basePv = None
            self.topPv = None

            self.packageName = None
            self.platform = None
            self.version = None
            self.hostUrl = None
            self.currentFile = None
            self.baseFile = None

            self.doc = None
            self.anyChanges = False
            self.patches = []

        def getCurrentKey(self):
            """ Returns the key to locate the current version of this
            package. """

            return (self.packageName, self.platform, self.version,
                    self.hostUrl, self.currentFile)

        def getBaseKey(self):
            """ Returns the key to locate the "base" or oldest version
            of this package. """

            return (self.packageName, self.platform, self.version,
                    self.hostUrl, self.baseFile)

        def getTopKey(self):
            """ Returns the key to locate the "top" or newest version
            of this package. """

            return (self.packageName, self.platform, self.version,
                    self.hostUrl, self.topFile)

        def getGenericKey(self, fileSpec):
            """ Returns the key that has the indicated hash. """
            return (self.packageName, self.platform, self.version,
                    self.hostUrl, fileSpec)

        def readDescFile(self, doProcessing=False):
            """ Reads the existing package.xml file and stores it in
            this class for later rewriting.  if doProcessing is true,
            it may massage the file and the directory contents in
            preparation for building patches.  Returns true on
            success, false on failure. """

            self.anyChanges = False

            packageDescFullpath = Filename(self.patchMaker.installDir,
                                           self.packageDesc)
            self.doc = TiXmlDocument(packageDescFullpath.toOsSpecific())
            if not self.doc.LoadFile():
                print("Couldn't read %s" % (packageDescFullpath))
                return False

            xpackage = self.doc.FirstChildElement('package')
            if not xpackage:
                return False
            self.packageName = xpackage.Attribute('name')
            self.platform = xpackage.Attribute('platform')
            self.version = xpackage.Attribute('version')

            # All packages we defined in-line are assigned to the
            # "none" host.  TODO: support patching from packages on
            # other hosts, which means we'll need to fill in a value
            # here for those hosts.
            self.hostUrl = None

            self.currentFile = None
            self.baseFile = None
            self.topFile = None
            self.compressedFilename = None
            compressedFile = None

            # Assume there are changes for this version, until we
            # discover that there aren't.
            isNewVersion = True

            # Get the actual current version.
            xarchive = xpackage.FirstChildElement('uncompressed_archive')
            if xarchive:
                self.currentFile = FileSpec()
                self.currentFile.loadXml(xarchive)

            # Get the top_version--the top (newest) of the patch
            # chain.
            xarchive = xpackage.FirstChildElement('top_version')
            if xarchive:
                self.topFile = FileSpec()
                self.topFile.loadXml(xarchive)

                if self.topFile.hash == self.currentFile.hash:
                    # No new version this pass.
                    isNewVersion = False
                else:
                    # There's a new version this pass.  Update it.
                    self.anyChanges = True

            else:
                # If there isn't a top_version yet, we have to make
                # one, by duplicating the currentFile.
                self.topFile = copy.copy(self.currentFile)
                self.anyChanges = True

            # Get the current patch version.  If we have a
            # patch_version attribute, it refers to this particular
            # instance of the file, and that is the current patch
            # version number.  If we only have a last_patch_version
            # attribute, it means a patch has not yet been built for
            # this particular instance, and that number is the
            # previous version's patch version number.
            patchVersion = xpackage.Attribute('patch_version')
            if patchVersion:
                self.patchVersion = int(patchVersion)
            else:
                patchVersion = xpackage.Attribute('last_patch_version')
                if patchVersion:
                    self.patchVersion = int(patchVersion)
                    if isNewVersion:
                        self.patchVersion += 1
                self.anyChanges = True

            # Put the patchVersion in the compressed filename, for
            # cache-busting.  This means when the version changes, its
            # URL will also change, guaranteeing that users will
            # download the latest version, and not some stale cache
            # file.
            xcompressed = xpackage.FirstChildElement('compressed_archive')
            if xcompressed:
                compressedFile = FileSpec()
                compressedFile.loadXml(xcompressed)

                oldCompressedFilename = compressedFile.filename
                self.compressedFilename = oldCompressedFilename

                if doProcessing:
                    newCompressedFilename = '%s.%s.pz' % (
                        self.currentFile.filename, self.patchVersion)
                    if newCompressedFilename != oldCompressedFilename:
                        oldCompressedPathname = Filename(
                            self.packageDir, oldCompressedFilename)
                        newCompressedPathname = Filename(
                            self.packageDir, newCompressedFilename)
                        if oldCompressedPathname.renameTo(
                                newCompressedPathname):
                            compressedFile.fromFile(self.packageDir,
                                                    newCompressedFilename)
                            compressedFile.storeXml(xcompressed)

                        self.compressedFilename = newCompressedFilename
                        self.anyChanges = True

            # Get the base_version--the bottom (oldest) of the patch
            # chain.
            xarchive = xpackage.FirstChildElement('base_version')
            if xarchive:
                self.baseFile = FileSpec()
                self.baseFile.loadXml(xarchive)
            else:
                # If there isn't a base_version yet, we have to make
                # one, by duplicating the currentFile.
                self.baseFile = copy.copy(self.currentFile)

                # Note that the we only store the compressed version
                # of base_filename on disk, but we store the md5 of
                # the uncompressed version in the xml file.  To
                # emphasize this, we name it without the .pz extension
                # in the xml file, even though the compressed file on
                # disk actually has a .pz extension.
                self.baseFile.filename += '.base'

                # Also duplicate the (compressed) file itself.
                if doProcessing and self.compressedFilename:
                    fromPathname = Filename(self.packageDir,
                                            self.compressedFilename)
                    toPathname = Filename(self.packageDir,
                                          self.baseFile.filename + '.pz')
                    fromPathname.copyTo(toPathname)
                self.anyChanges = True

            self.patches = []
            xpatch = xpackage.FirstChildElement('patch')
            while xpatch:
                patchfile = PatchMaker.Patchfile(self)
                patchfile.loadXml(xpatch)
                self.patches.append(patchfile)
                xpatch = xpatch.NextSiblingElement('patch')

            return True

        def writeDescFile(self):
            """ Rewrites the desc file with the new patch
            information. """

            if not self.anyChanges:
                # No need to rewrite.
                return

            xpackage = self.doc.FirstChildElement('package')
            if not xpackage:
                return

            packageSeq = SeqValue()
            packageSeq.loadXml(xpackage, 'seq')
            packageSeq += 1
            packageSeq.storeXml(xpackage, 'seq')

            # Remove all of the old patch entries from the desc file
            # we read earlier.
            xremove = []
            for value in ['base_version', 'top_version', 'patch']:
                xpatch = xpackage.FirstChildElement(value)
                while xpatch:
                    xremove.append(xpatch)
                    xpatch = xpatch.NextSiblingElement(value)

            for xelement in xremove:
                xpackage.RemoveChild(xelement)

            xpackage.RemoveAttribute('last_patch_version')

            # Now replace them with the current patch information.
            xpackage.SetAttribute('patch_version', str(self.patchVersion))

            xarchive = TiXmlElement('base_version')
            self.baseFile.storeXml(xarchive)
            xpackage.InsertEndChild(xarchive)

            # The current version is now the top version.
            xarchive = TiXmlElement('top_version')
            self.currentFile.storeXml(xarchive)
            xpackage.InsertEndChild(xarchive)

            for patchfile in self.patches:
                xpatch = patchfile.makeXml(self)
                xpackage.InsertEndChild(xpatch)

            self.doc.SaveFile()

            # Also copy the seq to the import desc file, for
            # documentation purposes.

            importDescFilename = str(self.packageDesc)[:-3] + 'import.xml'
            importDescFullpath = Filename(self.patchMaker.installDir,
                                          importDescFilename)
            doc = TiXmlDocument(importDescFullpath.toOsSpecific())
            if doc.LoadFile():
                xpackage = doc.FirstChildElement('package')
                if xpackage:
                    packageSeq.storeXml(xpackage, 'seq')
                    doc.SaveFile()
            else:
                print("Couldn't read %s" % (importDescFullpath))

            if self.contentsDocPackage:
                # Now that we've rewritten the xml file, we have to
                # change the contents.xml file that references it to
                # indicate the new file hash.
                fileSpec = FileSpec()
                fileSpec.fromFile(self.patchMaker.installDir, self.packageDesc)
                fileSpec.storeXml(self.contentsDocPackage)

                # Also important to update the import.xml hash.
                ximport = self.contentsDocPackage.FirstChildElement('import')
                if ximport:
                    fileSpec = FileSpec()
                    fileSpec.fromFile(self.patchMaker.installDir,
                                      importDescFilename)
                    fileSpec.storeXml(ximport)

                # Also copy the package seq value into the
                # contents.xml file, mainly for documentation purposes
                # (the authoritative seq value is within the desc
                # file).
                packageSeq.storeXml(self.contentsDocPackage, 'seq')
示例#32
0
    class Patchfile:
        """ A single patchfile for a package. """
        
        def __init__(self, package):
            self.package = package
            self.packageName = package.packageName
            self.platform = package.platform
            self.version = package.version
            self.hostUrl = None

            # FileSpec for the patchfile itself
            self.file = None

            # FileSpec for the package file that the patch is applied to
            self.sourceFile = None

            # FileSpec for the package file that the patch generates
            self.targetFile = None

            # The PackageVersion corresponding to our sourceFile
            self.fromPv = None

            # The PackageVersion corresponding to our targetFile
            self.toPv = None

        def getSourceKey(self):
            """ Returns the key for locating the package that this
            patchfile can be applied to. """
            return (self.packageName, self.platform, self.version, self.hostUrl, self.sourceFile)

        def getTargetKey(self):
            """ Returns the key for locating the package that this
            patchfile will generate. """
            return (self.packageName, self.platform, self.version, self.hostUrl, self.targetFile)

        def fromFile(self, packageDir, patchFilename, sourceFile, targetFile):
            """ Creates the data structures from an existing patchfile
            on disk. """
            
            self.file = FileSpec()
            self.file.fromFile(packageDir, patchFilename)
            self.sourceFile = sourceFile
            self.targetFile = targetFile

        def loadXml(self, xpatch):
            """ Reads the data structures from an xml file. """
            
            self.packageName = xpatch.Attribute('name') or self.packageName
            self.platform = xpatch.Attribute('platform') or self.platform
            self.version = xpatch.Attribute('version') or self.version
            self.hostUrl = xpatch.Attribute('host') or self.hostUrl

            self.file = FileSpec()
            self.file.loadXml(xpatch)

            xsource = xpatch.FirstChildElement('source')
            if xsource:
                self.sourceFile = FileSpec()
                self.sourceFile.loadXml(xsource)

            xtarget = xpatch.FirstChildElement('target')
            if xtarget:
                self.targetFile = FileSpec()
                self.targetFile.loadXml(xtarget)

        def makeXml(self, package):
            xpatch = TiXmlElement('patch')

            if self.packageName != package.packageName:
                xpatch.SetAttribute('name', self.packageName)
            if self.platform != package.platform:
                xpatch.SetAttribute('platform', self.platform)
            if self.version != package.version:
                xpatch.SetAttribute('version', self.version)
            if self.hostUrl != package.hostUrl:
                xpatch.SetAttribute('host', self.hostUrl)

            self.file.storeXml(xpatch)

            xsource = TiXmlElement('source')
            self.sourceFile.storeMiniXml(xsource)
            xpatch.InsertEndChild(xsource)

            xtarget = TiXmlElement('target')
            self.targetFile.storeMiniXml(xtarget)
            xpatch.InsertEndChild(xtarget)

            return xpatch
示例#33
0
        def writeDescFile(self):
            """ Rewrites the desc file with the new patch
            information. """

            if not self.anyChanges:
                # No need to rewrite.
                return

            xpackage = self.doc.FirstChildElement('package')
            if not xpackage:
                return

            packageSeq = SeqValue()
            packageSeq.loadXml(xpackage, 'seq')
            packageSeq += 1
            packageSeq.storeXml(xpackage, 'seq')

            # Remove all of the old patch entries from the desc file
            # we read earlier.
            xremove = []
            for value in ['base_version', 'top_version', 'patch']:
                xpatch = xpackage.FirstChildElement(value)
                while xpatch:
                    xremove.append(xpatch)
                    xpatch = xpatch.NextSiblingElement(value)

            for xelement in xremove:
                xpackage.RemoveChild(xelement)

            xpackage.RemoveAttribute('last_patch_version')

            # Now replace them with the current patch information.
            xpackage.SetAttribute('patch_version', str(self.patchVersion))

            xarchive = TiXmlElement('base_version')
            self.baseFile.storeXml(xarchive)
            xpackage.InsertEndChild(xarchive)

            # The current version is now the top version.
            xarchive = TiXmlElement('top_version')
            self.currentFile.storeXml(xarchive)
            xpackage.InsertEndChild(xarchive)

            for patchfile in self.patches:
                xpatch = patchfile.makeXml(self)
                xpackage.InsertEndChild(xpatch)

            self.doc.SaveFile()

            # Also copy the seq to the import desc file, for
            # documentation purposes.

            importDescFilename = str(self.packageDesc)[:-3] + 'import.xml'
            importDescFullpath = Filename(self.patchMaker.installDir,
                                          importDescFilename)
            doc = TiXmlDocument(importDescFullpath.toOsSpecific())
            if doc.LoadFile():
                xpackage = doc.FirstChildElement('package')
                if xpackage:
                    packageSeq.storeXml(xpackage, 'seq')
                    doc.SaveFile()
            else:
                print("Couldn't read %s" % (importDescFullpath))

            if self.contentsDocPackage:
                # Now that we've rewritten the xml file, we have to
                # change the contents.xml file that references it to
                # indicate the new file hash.
                fileSpec = FileSpec()
                fileSpec.fromFile(self.patchMaker.installDir, self.packageDesc)
                fileSpec.storeXml(self.contentsDocPackage)

                # Also important to update the import.xml hash.
                ximport = self.contentsDocPackage.FirstChildElement('import')
                if ximport:
                    fileSpec = FileSpec()
                    fileSpec.fromFile(self.patchMaker.installDir,
                                      importDescFilename)
                    fileSpec.storeXml(ximport)

                # Also copy the package seq value into the
                # contents.xml file, mainly for documentation purposes
                # (the authoritative seq value is within the desc
                # file).
                packageSeq.storeXml(self.contentsDocPackage, 'seq')
示例#34
0
class HostInfo:
    """ This class represents a particular download host serving up
    Panda3D packages.  It is the Python equivalent of the P3DHost
    class in the core API. """

    notify = directNotify.newCategory("HostInfo")

    def __init__(self, hostUrl, appRunner = None, hostDir = None,
                 rootDir = None, asMirror = False, perPlatform = None):

        """ You must specify either an appRunner or a hostDir to the
        HostInfo constructor.

        If you pass asMirror = True, it means that this HostInfo
        object is to be used to populate a "mirror" folder, a
        duplicate (or subset) of the contents hosted by a server.
        This means when you use this HostInfo to download packages, it
        will only download the compressed archive file and leave it
        there.  At the moment, mirror folders do not download old
        patch files from the server.

        If you pass perPlatform = True, then files are unpacked into a
        platform-specific directory, which is appropriate when you
        might be downloading multiple platforms.  The default is
        perPlatform = False, which means all files are unpacked into
        the host directory directly, without an intervening
        platform-specific directory name.  If asMirror is True, then
        the default is perPlatform = True.

        Note that perPlatform is also restricted by the individual
        package's specification.  """

        self.__setHostUrl(hostUrl)
        self.appRunner = appRunner
        self.rootDir = rootDir
        if rootDir is None and appRunner:
            self.rootDir = appRunner.rootDir

        if hostDir and not isinstance(hostDir, Filename):
            hostDir = Filename.fromOsSpecific(hostDir)

        self.hostDir = hostDir
        self.asMirror = asMirror
        self.perPlatform = perPlatform
        if perPlatform is None:
            self.perPlatform = asMirror

        # Initially false, this is set true when the contents file is
        # successfully read.
        self.hasContentsFile = False

        # This is the time value at which the current contents file is
        # no longer valid.
        self.contentsExpiration = 0

        # Contains the md5 hash of the original contents.xml file.
        self.contentsSpec = FileSpec()

        # descriptiveName will be filled in later, when the
        # contents file is read.
        self.descriptiveName = None

        # A list of known mirrors for this host, all URL's guaranteed
        # to end with a slash.
        self.mirrors = []

        # A map of keyword -> altHost URL's.  An altHost is different
        # than a mirror; an altHost is an alternate URL to download a
        # different (e.g. testing) version of this host's contents.
        # It is rarely used.
        self.altHosts = {}

        # This is a dictionary of packages by (name, version).  It
        # will be filled in when the contents file is read.
        self.packages = {}

        if self.appRunner and self.appRunner.verifyContents != self.appRunner.P3DVCForce:
            # Attempt to pre-read the existing contents.xml; maybe it
            # will be current enough for our purposes.
            self.readContentsFile()

    def __setHostUrl(self, hostUrl):
        """ Assigns self.hostUrl, and related values. """
        self.hostUrl = hostUrl

        if not self.hostUrl:
            # A special case: the URL will be set later.
            self.hostUrlPrefix = None
            self.downloadUrlPrefix = None
        else:
            # hostUrlPrefix is the host URL, but it is guaranteed to end
            # with a slash.
            self.hostUrlPrefix = hostUrl
            if self.hostUrlPrefix[-1] != '/':
                self.hostUrlPrefix += '/'

            # downloadUrlPrefix is the URL prefix that should be used for
            # everything other than the contents.xml file.  It might be
            # the same as hostUrlPrefix, but in the case of an
            # https-protected hostUrl, it will be the cleartext channel.
            self.downloadUrlPrefix = self.hostUrlPrefix

    def freshenFile(self, http, fileSpec, localPathname):
        """ Ensures that the localPathname is the most current version
        of the file defined by fileSpec, as offered by host.  If not,
        it downloads a new version on-the-spot.  Returns true on
        success, false on failure. """

        if fileSpec.quickVerify(pathname = localPathname):
            # It's good, keep it.
            return True

        # It's stale, get a new one.
        doc = None
        if self.appRunner and self.appRunner.superMirrorUrl:
            # Use the "super mirror" first.
            url = core.URLSpec(self.appRunner.superMirrorUrl + fileSpec.filename)
            self.notify.info("Freshening %s" % (url))
            doc = http.getDocument(url)

        if not doc or not doc.isValid():
            # Failing the super mirror, contact the actual host.
            url = core.URLSpec(self.hostUrlPrefix + fileSpec.filename)
            self.notify.info("Freshening %s" % (url))
            doc = http.getDocument(url)
            if not doc.isValid():
                return False

        file = Filename.temporary('', 'p3d_')
        if not doc.downloadToFile(file):
            # Failed to download.
            file.unlink()
            return False

        # Successfully downloaded!
        localPathname.makeDir()
        if not file.renameTo(localPathname):
            # Couldn't move it into place.
            file.unlink()
            return False

        if not fileSpec.fullVerify(pathname = localPathname, notify = self.notify):
            # No good after download.
            self.notify.info("%s is still no good after downloading." % (url))
            return False

        return True

    def downloadContentsFile(self, http, redownload = False,
                             hashVal = None):
        """ Downloads the contents.xml file for this particular host,
        synchronously, and then reads it.  Returns true on success,
        false on failure.  If hashVal is not None, it should be a
        HashVal object, which will be filled with the hash from the
        new contents.xml file."""

        if self.hasCurrentContentsFile():
            # We've already got one.
            return True

        if self.appRunner and self.appRunner.verifyContents == self.appRunner.P3DVCNever:
            # Not allowed to.
            return False

        rf = None
        if http:
            if not redownload and self.appRunner and self.appRunner.superMirrorUrl:
                # We start with the "super mirror", if it's defined.
                url = self.appRunner.superMirrorUrl + 'contents.xml'
                request = DocumentSpec(url)
                self.notify.info("Downloading contents file %s" % (request))

                rf = Ramfile()
                channel = http.makeChannel(False)
                channel.getDocument(request)
                if not channel.downloadToRam(rf):
                    self.notify.warning("Unable to download %s" % (url))
                    rf = None

            if not rf:
                # Then go to the main host, if our super mirror let us
                # down.

                url = self.hostUrlPrefix + 'contents.xml'
                # Append a uniquifying query string to the URL to force the
                # download to go all the way through any caches.  We use the
                # time in seconds; that's unique enough.
                url += '?' + str(int(time.time()))

                # We might as well explicitly request the cache to be disabled
                # too, since we have an interface for that via HTTPChannel.
                request = DocumentSpec(url)
                request.setCacheControl(DocumentSpec.CCNoCache)

                self.notify.info("Downloading contents file %s" % (request))
                statusCode = None
                statusString = ''
                for attempt in range(int(ConfigVariableInt('contents-xml-dl-attempts', 3))):
                    if attempt > 0:
                        self.notify.info("Retrying (%s)..."%(attempt,))
                    rf = Ramfile()
                    channel = http.makeChannel(False)
                    channel.getDocument(request)
                    if channel.downloadToRam(rf):
                        self.notify.info("Successfully downloaded %s" % (url,))
                        break
                    else:
                        rf = None
                        statusCode = channel.getStatusCode()
                        statusString = channel.getStatusString()
                        self.notify.warning("Could not contact download server at %s" % (url,))
                        self.notify.warning("Status code = %s %s" % (statusCode, statusString))

                if not rf:
                    self.notify.warning("Unable to download %s" % (url,))
                    try:
                        # Something screwed up.
                        if statusCode == HTTPChannel.SCDownloadOpenError or \
                           statusCode == HTTPChannel.SCDownloadWriteError:
                            launcher.setPandaErrorCode(2)
                        elif statusCode == 404:
                            # 404 not found
                            launcher.setPandaErrorCode(5)
                        elif statusCode < 100:
                            # statusCode < 100 implies the connection attempt itself
                            # failed.  This is usually due to firewall software
                            # interfering.  Apparently some firewall software might
                            # allow the first connection and disallow subsequent
                            # connections; how strange.
                            launcher.setPandaErrorCode(4)
                        else:
                            # There are other kinds of failures, but these will
                            # generally have been caught already by the first test; so
                            # if we get here there may be some bigger problem.  Just
                            # give the generic "big problem" message.
                            launcher.setPandaErrorCode(6)
                    except NameError as e:
                        # no launcher
                        pass
                    except AttributeError as e:
                        self.notify.warning("%s" % (str(e),))
                        pass
                    return False

        tempFilename = Filename.temporary('', 'p3d_', '.xml')
        if rf:
            f = open(tempFilename.toOsSpecific(), 'wb')
            f.write(rf.getData())
            f.close()
            if hashVal:
                hashVal.hashString(rf.getData())

            if not self.readContentsFile(tempFilename, freshDownload = True):
                self.notify.warning("Failure reading %s" % (url))
                tempFilename.unlink()
                return False

            tempFilename.unlink()
            return True

        # Couldn't download the file.  Maybe we should look for a
        # previously-downloaded copy already on disk?
        return False

    def redownloadContentsFile(self, http):
        """ Downloads a new contents.xml file in case it has changed.
        Returns true if the file has indeed changed, false if it has
        not. """
        assert self.hasContentsFile

        if self.appRunner and self.appRunner.verifyContents == self.appRunner.P3DVCNever:
            # Not allowed to.
            return False

        url = self.hostUrlPrefix + 'contents.xml'
        self.notify.info("Redownloading %s" % (url))

        # Get the hash of the original file.
        assert self.hostDir
        hv1 = HashVal()
        if self.contentsSpec.hash:
            hv1.setFromHex(self.contentsSpec.hash)
        else:
            filename = Filename(self.hostDir, 'contents.xml')
            hv1.hashFile(filename)

        # Now download it again.
        self.hasContentsFile = False
        hv2 = HashVal()
        if not self.downloadContentsFile(http, redownload = True,
                                         hashVal = hv2):
            return False

        if hv2 == HashVal():
            self.notify.info("%s didn't actually redownload." % (url))
            return False
        elif hv1 != hv2:
            self.notify.info("%s has changed." % (url))
            return True
        else:
            self.notify.info("%s has not changed." % (url))
            return False

    def hasCurrentContentsFile(self):
        """ Returns true if a contents.xml file has been successfully
        read for this host and is still current, false otherwise. """
        if not self.appRunner \
            or self.appRunner.verifyContents == self.appRunner.P3DVCNone \
            or self.appRunner.verifyContents == self.appRunner.P3DVCNever:
            # If we're not asking to verify contents, then
            # contents.xml files never expires.
            return self.hasContentsFile

        now = int(time.time())
        return now < self.contentsExpiration and self.hasContentsFile

    def readContentsFile(self, tempFilename = None, freshDownload = False):
        """ Reads the contents.xml file for this particular host, once
        it has been downloaded into the indicated temporary file.
        Returns true on success, false if the contents file is not
        already on disk or is unreadable.

        If tempFilename is specified, it is the filename read, and it
        is copied the file into the standard location if it's not
        there already.  If tempFilename is not specified, the standard
        filename is read if it is known. """

        if not hasattr(core, 'TiXmlDocument'):
            return False

        if not tempFilename:
            if self.hostDir:
                # If the filename is not specified, we can infer it
                # if we already know our hostDir
                hostDir = self.hostDir
            else:
                # Otherwise, we have to guess the hostDir.
                hostDir = self.__determineHostDir(None, self.hostUrl)

            tempFilename = Filename(hostDir, 'contents.xml')

        doc = core.TiXmlDocument(tempFilename.toOsSpecific())
        if not doc.LoadFile():
            return False

        xcontents = doc.FirstChildElement('contents')
        if not xcontents:
            return False

        maxAge = xcontents.Attribute('max_age')
        if maxAge:
            try:
                maxAge = int(maxAge)
            except:
                maxAge = None
        if maxAge is None:
            # Default max_age if unspecified (see p3d_plugin.h).
            from direct.p3d.AppRunner import AppRunner
            maxAge = AppRunner.P3D_CONTENTS_DEFAULT_MAX_AGE

        # Get the latest possible expiration time, based on the max_age
        # indication.  Any expiration time later than this is in error.
        now = int(time.time())
        self.contentsExpiration = now + maxAge

        if freshDownload:
            self.contentsSpec.readHash(tempFilename)

            # Update the XML with the new download information.
            xorig = xcontents.FirstChildElement('orig')
            while xorig:
                xcontents.RemoveChild(xorig)
                xorig = xcontents.FirstChildElement('orig')

            xorig = core.TiXmlElement('orig')
            self.contentsSpec.storeXml(xorig)
            xorig.SetAttribute('expiration', str(self.contentsExpiration))

            xcontents.InsertEndChild(xorig)

        else:
            # Read the download hash and expiration time from the XML.
            expiration = None
            xorig = xcontents.FirstChildElement('orig')
            if xorig:
                self.contentsSpec.loadXml(xorig)
                expiration = xorig.Attribute('expiration')
                if expiration:
                    try:
                        expiration = int(expiration)
                    except:
                        expiration = None
            if not self.contentsSpec.hash:
                self.contentsSpec.readHash(tempFilename)

            if expiration is not None:
                self.contentsExpiration = min(self.contentsExpiration, expiration)

        # Look for our own entry in the hosts table.
        if self.hostUrl:
            self.__findHostXml(xcontents)
        else:
            assert self.hostDir
            self.__findHostXmlForHostDir(xcontents)

        if self.rootDir and not self.hostDir:
            self.hostDir = self.__determineHostDir(None, self.hostUrl)

        # Get the list of packages available for download and/or import.
        xpackage = xcontents.FirstChildElement('package')
        while xpackage:
            name = xpackage.Attribute('name')
            platform = xpackage.Attribute('platform')
            version = xpackage.Attribute('version')
            try:
                solo = int(xpackage.Attribute('solo') or '')
            except ValueError:
                solo = False
            try:
                perPlatform = int(xpackage.Attribute('per_platform') or '')
            except ValueError:
                perPlatform = False

            package = self.__makePackage(name, platform, version, solo, perPlatform)
            package.descFile = FileSpec()
            package.descFile.loadXml(xpackage)
            package.setupFilenames()

            package.importDescFile = None
            ximport = xpackage.FirstChildElement('import')
            if ximport:
                package.importDescFile = FileSpec()
                package.importDescFile.loadXml(ximport)

            xpackage = xpackage.NextSiblingElement('package')

        self.hasContentsFile = True

        # Now save the contents.xml file into the standard location.
        if self.appRunner and self.appRunner.verifyContents != self.appRunner.P3DVCNever:
            assert self.hostDir
            filename = Filename(self.hostDir, 'contents.xml')
            filename.makeDir()
            if freshDownload:
                doc.SaveFile(filename.toOsSpecific())
            else:
                if filename != tempFilename:
                    tempFilename.copyTo(filename)

        return True

    def __findHostXml(self, xcontents):
        """ Looks for the <host> or <alt_host> entry in the
        contents.xml that corresponds to the URL that we actually
        downloaded from. """

        xhost = xcontents.FirstChildElement('host')
        while xhost:
            url = xhost.Attribute('url')
            if url == self.hostUrl:
                self.readHostXml(xhost)
                return

            xalthost = xhost.FirstChildElement('alt_host')
            while xalthost:
                url = xalthost.Attribute('url')
                if url == self.hostUrl:
                    self.readHostXml(xalthost)
                    return
                xalthost = xalthost.NextSiblingElement('alt_host')

            xhost = xhost.NextSiblingElement('host')

    def __findHostXmlForHostDir(self, xcontents):
        """ Looks for the <host> or <alt_host> entry in the
        contents.xml that corresponds to the host dir that we read the
        contents.xml from.  This is used when reading a contents.xml
        file found on disk, as opposed to downloading it from a
        site. """

        xhost = xcontents.FirstChildElement('host')
        while xhost:
            url = xhost.Attribute('url')
            hostDirBasename = xhost.Attribute('host_dir')
            hostDir = self.__determineHostDir(hostDirBasename, url)
            if hostDir == self.hostDir:
                self.__setHostUrl(url)
                self.readHostXml(xhost)
                return

            xalthost = xhost.FirstChildElement('alt_host')
            while xalthost:
                url = xalthost.Attribute('url')
                hostDirBasename = xalthost.Attribute('host_dir')
                hostDir = self.__determineHostDir(hostDirBasename, url)
                if hostDir == self.hostDir:
                    self.__setHostUrl(url)
                    self.readHostXml(xalthost)
                    return
                xalthost = xalthost.NextSiblingElement('alt_host')

            xhost = xhost.NextSiblingElement('host')

    def readHostXml(self, xhost):
        """ Reads a <host> or <alt_host> entry and applies the data to
        this object. """

        descriptiveName = xhost.Attribute('descriptive_name')
        if descriptiveName and not self.descriptiveName:
            self.descriptiveName = descriptiveName

        hostDirBasename = xhost.Attribute('host_dir')
        if self.rootDir and not self.hostDir:
            self.hostDir = self.__determineHostDir(hostDirBasename, self.hostUrl)

        # Get the "download" URL, which is the source from which we
        # download everything other than the contents.xml file.
        downloadUrl = xhost.Attribute('download_url')
        if downloadUrl:
            self.downloadUrlPrefix = downloadUrl
            if self.downloadUrlPrefix[-1] != '/':
                self.downloadUrlPrefix += '/'
        else:
            self.downloadUrlPrefix = self.hostUrlPrefix

        xmirror = xhost.FirstChildElement('mirror')
        while xmirror:
            url = xmirror.Attribute('url')
            if url:
                if url[-1] != '/':
                    url += '/'
                if url not in self.mirrors:
                    self.mirrors.append(url)
            xmirror = xmirror.NextSiblingElement('mirror')

        xalthost = xhost.FirstChildElement('alt_host')
        while xalthost:
            keyword = xalthost.Attribute('keyword')
            url = xalthost.Attribute('url')
            if url and keyword:
                self.altHosts[keyword] = url
            xalthost = xalthost.NextSiblingElement('alt_host')

    def __makePackage(self, name, platform, version, solo, perPlatform):
        """ Creates a new PackageInfo entry for the given name,
        version, and platform.  If there is already a matching
        PackageInfo, returns it. """

        if not platform:
            platform = None

        platforms = self.packages.setdefault((name, version or ""), {})
        package = platforms.get("", None)
        if not package:
            package = PackageInfo(self, name, version, platform = platform,
                                  solo = solo, asMirror = self.asMirror,
                                  perPlatform = perPlatform)
            platforms[platform or ""] = package

        return package

    def getPackage(self, name, version, platform = None):
        """ Returns a PackageInfo that matches the indicated name and
        version and the indicated platform or the current runtime
        platform, if one is provided by this host, or None if not. """

        assert self.hasContentsFile
        platforms = self.packages.get((name, version or ""), {})

        if platform:
            # In this case, we are looking for a specific platform
            # only.
            return platforms.get(platform, None)

        # We are looking for one matching the current runtime
        # platform.  First, look for a package matching the current
        # platform exactly.
        package = platforms.get(PandaSystem.getPlatform(), None)

        # If not found, look for one matching no particular platform.
        if not package:
            package = platforms.get("", None)

        return package

    def getPackages(self, name = None, platform = None):
        """ Returns a list of PackageInfo objects that match the
        indicated name and/or platform, with no particular regards to
        version.  If name is None, all packages are returned. """

        assert self.hasContentsFile

        packages = []
        for (pn, version), platforms in self.packages.items():
            if name and pn != name:
                continue

            if not platform:
                for p2 in platforms:
                    package = self.getPackage(pn, version, platform = p2)
                    if package:
                        packages.append(package)
            else:
                package = self.getPackage(pn, version, platform = platform)
                if package:
                    packages.append(package)

        return packages

    def getAllPackages(self, includeAllPlatforms = False):
        """ Returns a list of all available packages provided by this
        host. """

        result = []

        items = sorted(self.packages.items())
        for key, platforms in items:
            if self.perPlatform or includeAllPlatforms:
                # If we maintain a different answer per platform,
                # return all of them.
                pitems = sorted(platforms.items())
                for pkey, package in pitems:
                    result.append(package)
            else:
                # If we maintain a host for the current platform
                # only (e.g. a client copy), then return only the
                # current platform, or no particular platform.
                package = platforms.get(PandaSystem.getPlatform(), None)
                if not package:
                    package = platforms.get("", None)

                if package:
                    result.append(package)

        return result

    def deletePackages(self, packages):
        """ Removes all of the indicated packages from the disk,
        uninstalling them and deleting all of their files.  The
        packages parameter must be a list of one or more PackageInfo
        objects, for instance as returned by getPackage().  Returns
        the list of packages that were NOT found. """

        packages = packages[:]

        for key, platforms in list(self.packages.items()):
            for platform, package in list(platforms.items()):
                if package in packages:
                    self.__deletePackageFiles(package)
                    del platforms[platform]
                    packages.remove(package)

            if not platforms:
                # If we've removed all the platforms for a given
                # package, remove the key from the toplevel map.
                del self.packages[key]

        return packages

    def __deletePackageFiles(self, package):
        """ Called by deletePackage(), this actually removes the files
        for the indicated package. """

        if self.appRunner:
            self.notify.info("Deleting package %s: %s" % (package.packageName, package.getPackageDir()))
            self.appRunner.rmtree(package.getPackageDir())

            self.appRunner.sendRequest('forget_package', self.hostUrl, package.packageName, package.packageVersion or '')

    def __determineHostDir(self, hostDirBasename, hostUrl):
        """ Hashes the host URL into a (mostly) unique directory
        string, which will be the root of the host's install tree.
        Returns the resulting path, as a Filename.

        This code is duplicated in C++, in
        P3DHost::determine_host_dir(). """

        if hostDirBasename:
            # If the contents.xml specified a host_dir parameter, use
            # it.
            hostDir = str(self.rootDir) + '/hosts'
            for component in hostDirBasename.split('/'):
                if component:
                    if component[0] == '.':
                        # Forbid ".foo" or "..".
                        component = 'x' + component
                    hostDir += '/'
                    hostDir += component
            return Filename(hostDir)

        hostDir = 'hosts/'

        # Look for a server name in the URL.  Including this string in the
        # directory name makes it friendlier for people browsing the
        # directory.

        # We could use URLSpec, but we do it by hand instead, to make
        # it more likely that our hash code will exactly match the
        # similar logic in P3DHost.
        p = hostUrl.find('://')
        hostname = ''
        if p != -1:
            start = p + 3
            end = hostUrl.find('/', start)
            # Now start .. end is something like "username@host:port".

            at = hostUrl.find('@', start)
            if at != -1 and at < end:
                start = at + 1

            colon = hostUrl.find(':', start)
            if colon != -1 and colon < end:
                end = colon

            # Now start .. end is just the hostname.
            hostname = hostUrl[start : end]

        # Now build a hash string of the whole URL.  We'll use MD5 to
        # get a pretty good hash, with a minimum chance of collision.
        # Even if there is a hash collision, though, it's not the end
        # of the world; it just means that both hosts will dump their
        # packages into the same directory, and they'll fight over the
        # toplevel contents.xml file.  Assuming they use different
        # version numbers (which should be safe since they have the
        # same hostname), there will be minimal redownloading.

        hashSize = 16
        keepHash = hashSize
        if hostname:
            hostDir += hostname + '_'

            # If we successfully got a hostname, we don't really need the
            # full hash.  We'll keep half of it.
            keepHash = keepHash // 2

        md = HashVal()
        md.hashString(hostUrl)
        hostDir += md.asHex()[:keepHash * 2]

        hostDir = Filename(self.rootDir, hostDir)
        return hostDir