def __donePackage(self, pp, success):
""" Marks the indicated package as done, either successfully
or otherwise. """
assert not pp.done
if success:
pc = PStatCollector(':App:PackageInstaller:install:%s' %
(pp.package.packageName))
pc.start()
pp.package.installPackage(self.appRunner)
pc.stop()
self.packageLock.acquire()
try:
pp.done = True
pp.success = success
if success:
self.done.append(pp)
else:
self.failed.append(pp)
finally:
self.packageLock.release()
eventName = 'PackageInstaller-%s-packageDone' % self.uniqueId
messenger.send(eventName, [pp], taskChain='default')
def __downloadPackageTask(self, task):
""" This task runs on the aysynchronous task chain; each pass,
it extracts one package from self.needsDownload and downloads
it. """
while True:
self.packageLock.acquire()
try:
# If we're done downloading, stop the task.
if self.state == self.S_done or not self.needsDownload:
self.downloadTask = None
self.packageLock.release()
yield task.done
return
assert self.state == self.S_started
pp = self.needsDownload[0]
del self.needsDownload[0]
except:
self.packageLock.release()
raise
self.packageLock.release()
# Now serve this one package.
eventName = 'PackageInstaller-%s-packageStarted' % self.uniqueId
messenger.send(eventName, [pp], taskChain='default')
if not pp.package.hasPackage:
for token in pp.package.downloadPackageGenerator(
self.appRunner.http):
if token == pp.package.stepContinue:
yield task.cont
else:
break
if token != pp.package.stepComplete:
pc = PStatCollector(
':App:PackageInstaller:donePackage:%s' %
(pp.package.packageName))
pc.start()
self.__donePackage(pp, False)
pc.stop()
yield task.cont
continue
# Successfully downloaded and installed.
pc = PStatCollector(':App:PackageInstaller:donePackage:%s' %
(pp.package.packageName))
pc.start()
self.__donePackage(pp, True)
pc.stop()
def __downloadPackageTask(self, task):
""" This task runs on the aysynchronous task chain; each pass,
it extracts one package from self.needsDownload and downloads
it. """
while True:
self.packageLock.acquire()
try:
# If we're done downloading, stop the task.
if self.state == self.S_done or not self.needsDownload:
self.downloadTask = None
self.packageLock.release()
yield task.done; return
assert self.state == self.S_started
pp = self.needsDownload[0]
del self.needsDownload[0]
except:
self.packageLock.release()
raise
self.packageLock.release()
# Now serve this one package.
eventName = 'PackageInstaller-%s-packageStarted' % self.uniqueId
messenger.send(eventName, [pp], taskChain = 'default')
if not pp.package.hasPackage:
for token in pp.package.downloadPackageGenerator(self.appRunner.http):
if token == pp.package.stepContinue:
yield task.cont
else:
break
if token != pp.package.stepComplete:
pc = PStatCollector(':App:PackageInstaller:donePackage:%s' % (pp.package.packageName))
pc.start()
self.__donePackage(pp, False)
pc.stop()
yield task.cont
continue
# Successfully downloaded and installed.
pc = PStatCollector(':App:PackageInstaller:donePackage:%s' % (pp.package.packageName))
pc.start()
self.__donePackage(pp, True)
pc.stop()
class InstallStep:
""" This class is one step of the installPlan list; it
represents a single atomic piece of the installation step, and
the relative effort of that piece. When the plan is executed,
it will call the saved function pointer here. """
def __init__(self, func, bytes, factor, stepType):
self.__funcPtr = func
self.bytesNeeded = bytes
self.bytesDone = 0
self.bytesFactor = factor
self.stepType = stepType
self.pStatCol = PStatCollector(':App:PackageInstaller:%s' %
(stepType))
def func(self):
""" self.__funcPtr(self) will return a generator of
tokens. This function defines a new generator that yields
each of those tokens, but wraps each call into the nested
generator within a pair of start/stop collector calls. """
self.pStatCol.start()
for token in self.__funcPtr(self):
self.pStatCol.stop()
yield token
self.pStatCol.start()
# Shouldn't ever get here.
self.pStatCol.stop()
raise StopIteration
def getEffort(self):
""" Returns the relative amount of effort of this step. """
return self.bytesNeeded * self.bytesFactor
def getProgress(self):
""" Returns the progress of this step, in the range
0..1. """
if self.bytesNeeded == 0:
return 1
return min(float(self.bytesDone) / float(self.bytesNeeded), 1)
class InstallStep:
""" This class is one step of the installPlan list; it
represents a single atomic piece of the installation step, and
the relative effort of that piece. When the plan is executed,
it will call the saved function pointer here. """
def __init__(self, func, bytes, factor, stepType):
self.__funcPtr = func
self.bytesNeeded = bytes
self.bytesDone = 0
self.bytesFactor = factor
self.stepType = stepType
self.pStatCol = PStatCollector(":App:PackageInstaller:%s" % (stepType))
def func(self):
""" self.__funcPtr(self) will return a generator of
tokens. This function defines a new generator that yields
each of those tokens, but wraps each call into the nested
generator within a pair of start/stop collector calls. """
self.pStatCol.start()
for token in self.__funcPtr(self):
self.pStatCol.stop()
yield token
self.pStatCol.start()
# Shouldn't ever get here.
self.pStatCol.stop()
raise StopIteration
def getEffort(self):
""" Returns the relative amount of effort of this step. """
return self.bytesNeeded * self.bytesFactor
def getProgress(self):
""" Returns the progress of this step, in the range
0..1. """
if self.bytesNeeded == 0:
return 1
return min(float(self.bytesDone) / float(self.bytesNeeded), 1)
def __donePackage(self, pp, success):
""" Marks the indicated package as done, either successfully
or otherwise. """
assert not pp.done
if success:
pc = PStatCollector(':App:PackageInstaller:install:%s' % (pp.package.packageName))
pc.start()
pp.package.installPackage(self.appRunner)
pc.stop()
self.packageLock.acquire()
try:
pp.done = True
pp.success = success
if success:
self.done.append(pp)
else:
self.failed.append(pp)
finally:
self.packageLock.release()
eventName = 'PackageInstaller-%s-packageDone' % self.uniqueId
messenger.send(eventName, [pp], taskChain = 'default')
def __buildInstallPlans(self):
""" Sets up self.installPlans, a list of one or more "plans"
to download and install the package. """
pc = PStatCollector(':App:PackageInstaller:buildInstallPlans')
pc.start()
self.hasPackage = False
if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
# We're not allowed to download anything.
self.installPlans = []
pc.stop()
return
if self.asMirror:
# If we're just downloading a mirror archive, we only need
# to get the compressed archive file.
# Build a one-item install plan to download the compressed
# archive.
downloadSize = self.compressedArchive.size
func = lambda step, fileSpec=self.compressedArchive: self.__downloadFile(
step, fileSpec, allowPartial=True)
step = self.InstallStep(func, downloadSize, self.downloadFactor,
'download')
installPlan = [step]
self.installPlans = [installPlan]
pc.stop()
return
# The normal download process. Determine what we will need to
# download, and build a plan (or two) to download it all.
self.installPlans = None
# We know we will at least need to unpack the archive contents
# at the end.
unpackSize = 0
for file in self.extracts:
unpackSize += file.size
step = self.InstallStep(self.__unpackArchive, unpackSize,
self.unpackFactor, 'unpack')
planA = [step]
# If the uncompressed archive file is good, that's all we'll
# need to do.
self.uncompressedArchive.actualFile = None
if self.uncompressedArchive.quickVerify(self.getPackageDir(),
notify=self.notify):
self.installPlans = [planA]
pc.stop()
return
# Maybe the compressed archive file is good.
if self.compressedArchive.quickVerify(self.getPackageDir(),
notify=self.notify):
uncompressSize = self.uncompressedArchive.size
step = self.InstallStep(self.__uncompressArchive, uncompressSize,
self.uncompressFactor, 'uncompress')
planA = [step] + planA
self.installPlans = [planA]
pc.stop()
return
# Maybe we can download one or more patches. We'll come back
# to that in a minute as plan A. For now, construct plan B,
# which will be to download the whole archive.
planB = planA[:]
uncompressSize = self.uncompressedArchive.size
step = self.InstallStep(self.__uncompressArchive, uncompressSize,
self.uncompressFactor, 'uncompress')
planB = [step] + planB
downloadSize = self.compressedArchive.size
func = lambda step, fileSpec=self.compressedArchive: self.__downloadFile(
step, fileSpec, allowPartial=True)
step = self.InstallStep(func, downloadSize, self.downloadFactor,
'download')
planB = [step] + planB
# Now look for patches. Start with the md5 hash from the
# uncompressedArchive file we have on disk, and see if we can
# find a patch chain from this file to our target.
pathname = Filename(self.getPackageDir(),
self.uncompressedArchive.filename)
fileSpec = self.uncompressedArchive.actualFile
if fileSpec is None and pathname.exists():
fileSpec = FileSpec()
fileSpec.fromFile(self.getPackageDir(),
self.uncompressedArchive.filename)
plan = None
if fileSpec:
plan = self.__findPatchChain(fileSpec)
if plan:
# We can download patches. Great! That means this is
# plan A, and the full download is plan B (in case
# something goes wrong with the patching).
planA = plan + planA
self.installPlans = [planA, planB]
else:
# There are no patches to download, oh well. Stick with
# plan B as the only plan.
self.installPlans = [planB]
# In case of unexpected failures on the internet, we will retry
# the full download instead of just giving up.
for retry in range(ConfigVariableInt('package-full-dl-retries', 1)):
self.installPlans.append(planB[:])
pc.stop()
def __buildInstallPlans(self):
""" Sets up self.installPlans, a list of one or more "plans"
to download and install the package. """
pc = PStatCollector(':App:PackageInstaller:buildInstallPlans')
pc.start()
self.hasPackage = False
if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
# We're not allowed to download anything.
self.installPlans = []
pc.stop()
return
if self.asMirror:
# If we're just downloading a mirror archive, we only need
# to get the compressed archive file.
# Build a one-item install plan to download the compressed
# archive.
downloadSize = self.compressedArchive.size
func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec, allowPartial = True)
step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download')
installPlan = [step]
self.installPlans = [installPlan]
pc.stop()
return
# The normal download process. Determine what we will need to
# download, and build a plan (or two) to download it all.
self.installPlans = None
# We know we will at least need to unpack the archive contents
# at the end.
unpackSize = 0
for file in self.extracts:
unpackSize += file.size
step = self.InstallStep(self.__unpackArchive, unpackSize, self.unpackFactor, 'unpack')
planA = [step]
# If the uncompressed archive file is good, that's all we'll
# need to do.
self.uncompressedArchive.actualFile = None
if self.uncompressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
self.installPlans = [planA]
pc.stop()
return
# Maybe the compressed archive file is good.
if self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify):
uncompressSize = self.uncompressedArchive.size
step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor, 'uncompress')
planA = [step] + planA
self.installPlans = [planA]
pc.stop()
return
# Maybe we can download one or more patches. We'll come back
# to that in a minute as plan A. For now, construct plan B,
# which will be to download the whole archive.
planB = planA[:]
uncompressSize = self.uncompressedArchive.size
step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor, 'uncompress')
planB = [step] + planB
downloadSize = self.compressedArchive.size
func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec, allowPartial = True)
step = self.InstallStep(func, downloadSize, self.downloadFactor, 'download')
planB = [step] + planB
# Now look for patches. Start with the md5 hash from the
# uncompressedArchive file we have on disk, and see if we can
# find a patch chain from this file to our target.
pathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename)
fileSpec = self.uncompressedArchive.actualFile
if fileSpec is None and pathname.exists():
fileSpec = FileSpec()
fileSpec.fromFile(self.getPackageDir(), self.uncompressedArchive.filename)
plan = None
if fileSpec:
plan = self.__findPatchChain(fileSpec)
if plan:
# We can download patches. Great! That means this is
# plan A, and the full download is plan B (in case
# something goes wrong with the patching).
planA = plan + planA
self.installPlans = [planA, planB]
else:
# There are no patches to download, oh well. Stick with
# plan B as the only plan.
self.installPlans = [planB]
# In case of unexpected failures on the internet, we will retry
# the full download instead of just giving up.
for retry in range(ConfigVariableInt('package-full-dl-retries', 1)):
self.installPlans.append(planB[:])
pc.stop()