def __donePackage(self, pp, success):
""" Marks the indicated package as done, either successfully
or otherwise. """
assert not pp.done
if success:
pc = PStatCollector(':App:PackageInstaller:install:%s' %
(pp.package.packageName))
pc.start()
pp.package.installPackage(self.appRunner)
pc.stop()
self.packageLock.acquire()
try:
pp.done = True
pp.success = success
if success:
self.done.append(pp)
else:
self.failed.append(pp)
finally:
self.packageLock.release()
eventName = 'PackageInstaller-%s-packageDone' % self.uniqueId
messenger.send(eventName, [pp], taskChain='default')
def processEventPstats(self, event):
"""
Process a C++ event with pstats tracking
Duplicate any changes in processEvent
"""
# ********************************************************
# ******** Duplicate any changes in processEvent *********
# ********************************************************
# Get the event name
eventName = event.getName()
if eventName:
paramList = []
for i in range(event.getNumParameters()):
eventParameter = event.getParameter(i)
eventParameterData = self.parseEventParameter(eventParameter)
paramList.append(eventParameterData)
# Do not print the new frame debug, it is too noisy!
if (EventManager.notify.getDebug() and eventName != 'NewFrame'):
EventManager.notify.debug('received C++ event named: ' +
eventName + ' parameters: ' +
repr(paramList))
# Send the event, we used to send it with the event
# name as a parameter, but now you can use extraArgs for that
# ********************************************************
# ******** Duplicate any changes in processEvent *********
# ********************************************************
if self._wantPstats:
name = eventName
hyphen = name.find('-')
if hyphen >= 0:
name = name[0:hyphen]
pstatCollector = PStatCollector('App:Show code:eventManager:' +
name)
pstatCollector.start()
if self.eventHandler:
cppPstatCollector = PStatCollector(
'App:Show code:eventManager:' + name + ':C++')
if paramList:
messenger.send(eventName, paramList)
else:
messenger.send(eventName)
# Also send the event down into C++ land
if self.eventHandler:
if self._wantPstats:
cppPstatCollector.start()
self.eventHandler.dispatchEvent(event)
# ********************************************************
# ******** Duplicate any changes in processEvent *********
# ********************************************************
if self._wantPstats:
if self.eventHandler:
cppPstatCollector.stop()
pstatCollector.stop()
else:
# An unnamed event from C++ is probably a bad thing
EventManager.notify.warning('unnamed event in processEvent')
def processEventPstats(self, event):
"""
Process a C++ event with pstats tracking
Duplicate any changes in processEvent
"""
# ********************************************************
# ******** Duplicate any changes in processEvent *********
# ********************************************************
# Get the event name
eventName = event.getName()
if eventName:
paramList = []
for i in range(event.getNumParameters()):
eventParameter = event.getParameter(i)
eventParameterData = self.parseEventParameter(eventParameter)
paramList.append(eventParameterData)
# Do not print the new frame debug, it is too noisy!
if (EventManager.notify.getDebug() and eventName != 'NewFrame'):
EventManager.notify.debug('received C++ event named: ' + eventName +
' parameters: ' + repr(paramList))
# Send the event, we used to send it with the event
# name as a parameter, but now you can use extraArgs for that
# ********************************************************
# ******** Duplicate any changes in processEvent *********
# ********************************************************
if self._wantPstats:
name = eventName
hyphen = name.find('-')
if hyphen >= 0:
name = name[0:hyphen]
pstatCollector = PStatCollector('App:Show code:eventManager:' + name)
pstatCollector.start()
if self.eventHandler:
cppPstatCollector = PStatCollector(
'App:Show code:eventManager:' + name + ':C++')
if paramList:
messenger.send(eventName, paramList)
else:
messenger.send(eventName)
# Also send the event down into C++ land
if self.eventHandler:
if self._wantPstats:
cppPstatCollector.start()
self.eventHandler.dispatchEvent(event)
# ********************************************************
# ******** Duplicate any changes in processEvent *********
# ********************************************************
if self._wantPstats:
if self.eventHandler:
cppPstatCollector.stop()
pstatCollector.stop()
else:
# An unnamed event from C++ is probably a bad thing
EventManager.notify.warning('unnamed event in processEvent')
def __downloadPackageTask(self, task):
""" This task runs on the aysynchronous task chain; each pass,
it extracts one package from self.needsDownload and downloads
it. """
while True:
self.packageLock.acquire()
try:
# If we're done downloading, stop the task.
if self.state == self.S_done or not self.needsDownload:
self.downloadTask = None
self.packageLock.release()
yield task.done
return
assert self.state == self.S_started
pp = self.needsDownload[0]
del self.needsDownload[0]
except:
self.packageLock.release()
raise
self.packageLock.release()
# Now serve this one package.
eventName = 'PackageInstaller-%s-packageStarted' % self.uniqueId
messenger.send(eventName, [pp], taskChain='default')
if not pp.package.hasPackage:
for token in pp.package.downloadPackageGenerator(
self.appRunner.http):
if token == pp.package.stepContinue:
yield task.cont
else:
break
if token != pp.package.stepComplete:
pc = PStatCollector(
':App:PackageInstaller:donePackage:%s' %
(pp.package.packageName))
pc.start()
self.__donePackage(pp, False)
pc.stop()
yield task.cont
continue
# Successfully downloaded and installed.
pc = PStatCollector(':App:PackageInstaller:donePackage:%s' %
(pp.package.packageName))
pc.start()
self.__donePackage(pp, True)
pc.stop()
def __downloadPackageTask(self, task):
""" This task runs on the aysynchronous task chain; each pass,
it extracts one package from self.needsDownload and downloads
it. """
while True:
self.packageLock.acquire()
try:
# If we're done downloading, stop the task.
if self.state == self.S_done or not self.needsDownload:
self.downloadTask = None
self.packageLock.release()
yield task.done; return
assert self.state == self.S_started
pp = self.needsDownload[0]
del self.needsDownload[0]
except:
self.packageLock.release()
raise
self.packageLock.release()
# Now serve this one package.
eventName = 'PackageInstaller-%s-packageStarted' % self.uniqueId
messenger.send(eventName, [pp], taskChain = 'default')
if not pp.package.hasPackage:
for token in pp.package.downloadPackageGenerator(self.appRunner.http):
if token == pp.package.stepContinue:
yield task.cont
else:
break
if token != pp.package.stepComplete:
pc = PStatCollector(':App:PackageInstaller:donePackage:%s' % (pp.package.packageName))
pc.start()
self.__donePackage(pp, False)
pc.stop()
yield task.cont
continue
# Successfully downloaded and installed.
pc = PStatCollector(':App:PackageInstaller:donePackage:%s' % (pp.package.packageName))
pc.start()
self.__donePackage(pp, True)
pc.stop()
class BaseManager(DebugObject):
""" Base class for all managers, provides utility functions like timing
the update duration """
def __init__(self):
""" Inits the manager """
self._mgr_name = self.__class__.__name__
DebugObject.__init__(self, self._mgr_name)
self._update_collector = PStatCollector("App:Show code:RP_UpdateManagers:" + self._mgr_name)
assert hasattr(self, "do_update")
def update(self):
""" Updates the manager, this just calls the do_update() method and
times it """
self._update_collector.start()
self.do_update()
self._update_collector.stop()
class InstallStep:
""" This class is one step of the installPlan list; it
represents a single atomic piece of the installation step, and
the relative effort of that piece. When the plan is executed,
it will call the saved function pointer here. """
def __init__(self, func, bytes, factor, stepType):
self.__funcPtr = func
self.bytesNeeded = bytes
self.bytesDone = 0
self.bytesFactor = factor
self.stepType = stepType
self.pStatCol = PStatCollector(':App:PackageInstaller:%s' %
(stepType))
def func(self):
""" self.__funcPtr(self) will return a generator of
tokens. This function defines a new generator that yields
each of those tokens, but wraps each call into the nested
generator within a pair of start/stop collector calls. """
self.pStatCol.start()
for token in self.__funcPtr(self):
self.pStatCol.stop()
yield token
self.pStatCol.start()
# Shouldn't ever get here.
self.pStatCol.stop()
raise StopIteration
def getEffort(self):
""" Returns the relative amount of effort of this step. """
return self.bytesNeeded * self.bytesFactor
def getProgress(self):
""" Returns the progress of this step, in the range
0..1. """
if self.bytesNeeded == 0:
return 1
return min(float(self.bytesDone) / float(self.bytesNeeded), 1)
class BaseManager(DebugObject):
""" Base class for all managers, provides utility functions like timing
the update duration """
def __init__(self):
""" Inits the manager """
self._mgr_name = self.__class__.__name__
DebugObject.__init__(self, self._mgr_name)
self._update_collector = PStatCollector(
"App:Show code:RP_UpdateManagers:" + self._mgr_name)
def update(self):
""" Updates the manager, this just calls the do_update() method and
times it """
self._update_collector.start()
self.do_update()
self._update_collector.stop()
def do_update(self):
""" Abstract update method, all managers should implement this """
raise NotImplementedError()
class InstallStep:
""" This class is one step of the installPlan list; it
represents a single atomic piece of the installation step, and
the relative effort of that piece. When the plan is executed,
it will call the saved function pointer here. """
def __init__(self, func, bytes, factor, stepType):
self.__funcPtr = func
self.bytesNeeded = bytes
self.bytesDone = 0
self.bytesFactor = factor
self.stepType = stepType
self.pStatCol = PStatCollector(':App:PackageInstaller:%s' % (stepType))
def func(self):
""" self.__funcPtr(self) will return a generator of
tokens. This function defines a new generator that yields
each of those tokens, but wraps each call into the nested
generator within a pair of start/stop collector calls. """
self.pStatCol.start()
for token in self.__funcPtr(self):
self.pStatCol.stop()
yield token
self.pStatCol.start()
# Shouldn't ever get here.
self.pStatCol.stop()
raise StopIteration
def getEffort(self):
""" Returns the relative amount of effort of this step. """
return self.bytesNeeded * self.bytesFactor
def getProgress(self):
""" Returns the progress of this step, in the range
0..1. """
if self.bytesNeeded == 0:
return 1
return min(float(self.bytesDone) / float(self.bytesNeeded), 1)
def __donePackage(self, pp, success):
""" Marks the indicated package as done, either successfully
or otherwise. """
assert not pp.done
if success:
pc = PStatCollector(':App:PackageInstaller:install:%s' % (pp.package.packageName))
pc.start()
pp.package.installPackage(self.appRunner)
pc.stop()
self.packageLock.acquire()
try:
pp.done = True
pp.success = success
if success:
self.done.append(pp)
else:
self.failed.append(pp)
finally:
self.packageLock.release()
eventName = 'PackageInstaller-%s-packageDone' % self.uniqueId
messenger.send(eventName, [pp], taskChain = 'default')
def __buildInstallPlans(self):
""" Sets up self.installPlans, a list of one or more "plans"
to download and install the package. """
pc = PStatCollector(':App:PackageInstaller:buildInstallPlans')
pc.start()
self.hasPackage = False
if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
# We're not allowed to download anything.
self.installPlans = []
pc.stop()
return
if self.asMirror:
# If we're just downloading a mirror archive, we only need
# to get the compressed archive file.
# Build a one-item install plan to download the compressed
# archive.
downloadSize = self.compressedArchive.size
func = lambda step, fileSpec=self.compressedArchive: self.__downloadFile(
step, fileSpec, allowPartial=True)
step = self.InstallStep(func, downloadSize, self.downloadFactor,
'download')
installPlan = [step]
self.installPlans = [installPlan]
pc.stop()
return
# The normal download process. Determine what we will need to
# download, and build a plan (or two) to download it all.
self.installPlans = None
# We know we will at least need to unpack the archive contents
# at the end.
unpackSize = 0
for file in self.extracts:
unpackSize += file.size
step = self.InstallStep(self.__unpackArchive, unpackSize,
self.unpackFactor, 'unpack')
planA = [step]
# If the uncompressed archive file is good, that's all we'll
# need to do.
self.uncompressedArchive.actualFile = None
if self.uncompressedArchive.quickVerify(self.getPackageDir(),
notify=self.notify):
self.installPlans = [planA]
pc.stop()
return
# Maybe the compressed archive file is good.
if self.compressedArchive.quickVerify(self.getPackageDir(),
notify=self.notify):
uncompressSize = self.uncompressedArchive.size
step = self.InstallStep(self.__uncompressArchive, uncompressSize,
self.uncompressFactor, 'uncompress')
planA = [step] + planA
self.installPlans = [planA]
pc.stop()
return
# Maybe we can download one or more patches. We'll come back
# to that in a minute as plan A. For now, construct plan B,
# which will be to download the whole archive.
planB = planA[:]
uncompressSize = self.uncompressedArchive.size
step = self.InstallStep(self.__uncompressArchive, uncompressSize,
self.uncompressFactor, 'uncompress')
planB = [step] + planB
downloadSize = self.compressedArchive.size
func = lambda step, fileSpec=self.compressedArchive: self.__downloadFile(
step, fileSpec, allowPartial=True)
step = self.InstallStep(func, downloadSize, self.downloadFactor,
'download')
planB = [step] + planB
# Now look for patches. Start with the md5 hash from the
# uncompressedArchive file we have on disk, and see if we can
# find a patch chain from this file to our target.
pathname = Filename(self.getPackageDir(),
self.uncompressedArchive.filename)
fileSpec = self.uncompressedArchive.actualFile
if fileSpec is None and pathname.exists():
fileSpec = FileSpec()
fileSpec.fromFile(self.getPackageDir(),
self.uncompressedArchive.filename)
plan = None
if fileSpec:
plan = self.__findPatchChain(fileSpec)
if plan:
# We can download patches. Great! That means this is
# plan A, and the full download is plan B (in case
# something goes wrong with the patching).
planA = plan + planA
self.installPlans = [planA, planB]
else:
# There are no patches to download, oh well. Stick with
# plan B as the only plan.
self.installPlans = [planB]
# In case of unexpected failures on the internet, we will retry
# the full download instead of just giving up.
retries = core.ConfigVariableInt('package-full-dl-retries',
1).getValue()
for retry in range(retries):
self.installPlans.append(planB[:])
pc.stop()
def __buildInstallPlans(self):
""" Sets up self.installPlans, a list of one or more "plans"
to download and install the package. """
pc = PStatCollector(':App:PackageInstaller:buildInstallPlans')
pc.start()
self.hasPackage = False
if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
# We're not allowed to download anything.
self.installPlans = []
pc.stop()
return
if self.asMirror:
# If we're just downloading a mirror archive, we only need
# to get the compressed archive file.
# Build a one-item install plan to download the compressed
# archive.
downloadSize = self.compressedArchive.size
func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec, allowPartial = True)
step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download')
installPlan = [step]
self.installPlans = [installPlan]
pc.stop()
return
# The normal download process. Determine what we will need to
# download, and build a plan (or two) to download it all.
self.installPlans = None
# We know we will at least need to unpack the archive contents
# at the end.
unpackSize = 0
for file in self.extracts:
unpackSize += file.size
step = self.InstallStep(self.__unpackArchive, unpackSize, self.unpackFactor, 'unpack')
planA = [step]
# If the uncompressed archive file is good, that's all we'll
# need to do.
self.uncompressedArchive.actualFile = None
if self.uncompressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
self.installPlans = [planA]
pc.stop()
return
# Maybe the compressed archive file is good.
if self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
uncompressSize = self.uncompressedArchive.size
step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor, 'uncompress')
planA = [step] + planA
self.installPlans = [planA]
pc.stop()
return
# Maybe we can download one or more patches. We'll come back
# to that in a minute as plan A. For now, construct plan B,
# which will be to download the whole archive.
planB = planA[:]
uncompressSize = self.uncompressedArchive.size
step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor, 'uncompress')
planB = [step] + planB
downloadSize = self.compressedArchive.size
func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec, allowPartial = True)
step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download')
planB = [step] + planB
# Now look for patches. Start with the md5 hash from the
# uncompressedArchive file we have on disk, and see if we can
# find a patch chain from this file to our target.
pathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
fileSpec = self.uncompressedArchive.actualFile
if fileSpec is None and pathname.exists():
fileSpec = FileSpec()
fileSpec.fromFile(self.getPackageDir(), self.uncompressedArchive.filename)
plan = None
if fileSpec:
plan = self.__findPatchChain(fileSpec)
if plan:
# We can download patches. Great! That means this is
# plan A, and the full download is plan B (in case
# something goes wrong with the patching).
planA = plan + planA
self.installPlans = [planA, planB]
else:
# There are no patches to download, oh well. Stick with
# plan B as the only plan.
self.installPlans = [planB]
# In case of unexpected failures on the internet, we will retry
# the full download instead of just giving up.
retries = core.ConfigVariableInt('package-full-dl-retries', 1).getValue()
for retry in range(retries):
self.installPlans.append(planB[:])
pc.stop()