def writePackageVersions(self,
butler,
clobber=False,
doBackup=True,
dataset="packages"):
"""Compare and write package versions.
Parameters
----------
butler : `lsst.daf.persistence.Butler`
Data butler used to read/write the package versions.
clobber : `bool`, optional
A boolean flag that controls what happens if versions already have been saved:
- `True`: overwrite or rename the existing version info, depending on ``doBackup``.
- `False`: raise `TaskError` if this version info does not match the existing.
doBackup : `bool`, optional
If `True` and clobbering, old package version files are backed up.
dataset : `str`, optional
Name of dataset to read/write.
Raises
------
TaskError
Raised if there is a version mismatch with current and persisted lists of package versions.
Notes
-----
Note that this operation is subject to a race condition.
"""
packages = Packages.fromSystem()
if clobber:
return butler.put(packages, dataset, doBackup=doBackup)
if not butler.datasetExists(dataset, write=True):
return butler.put(packages, dataset)
try:
old = butler.get(dataset, immediate=True)
except Exception as exc:
raise type(exc)(
"Unable to read stored version dataset %s (%s); "
"consider using --clobber-versions or --no-versions" %
(dataset, exc))
# Note that because we can only detect python modules that have been imported, the stored
# list of products may be more or less complete than what we have now. What's important is
# that the products that are in common have the same version.
diff = packages.difference(old)
if diff:
raise TaskError(
"Version mismatch (" +
"; ".join("%s: %s vs %s" % (pkg, diff[pkg][1], diff[pkg][0])
for pkg in diff) +
"); consider using --clobber-versions or --no-versions")
# Update the old set of packages in case we have more packages that haven't been persisted.
extra = packages.extra(old)
if extra:
old.update(packages)
butler.put(old, dataset, doBackup=doBackup)
def savePackageVersions(self, graph):
"""Write versions of software packages to butler.
Parameters
----------
graph : `~lsst.pipe.base.QuantumGraph`
Execution graph.
Raises
------
Exception
Raised if ``checkExisting`` is ``True`` but versions are not
compatible.
"""
packages = Packages.fromSystem()
_LOG.debug("want to save packages: %s", packages)
datasetType = "packages"
dataId = {}
oldPackages = None
# start transaction to rollback any changes on exceptions
with self.butler.transaction():
if self.skipExisting:
try:
oldPackages = self.butler.get(
datasetType, dataId, collections=[self.butler.run])
_LOG.debug("old packages: %s", oldPackages)
except LookupError:
pass
if oldPackages is not None:
# Note that because we can only detect python modules that have been imported, the stored
# list of products may be more or less complete than what we have now. What's important is
# that the products that are in common have the same version.
diff = packages.difference(oldPackages)
if diff:
versions_str = "; ".join(
f"{pkg}: {diff[pkg][1]} vs {diff[pkg][0]}"
for pkg in diff)
raise TypeError(
f"Package versions mismatch: ({versions_str})")
else:
_LOG.debug("new packages are consistent with old")
# Update the old set of packages in case we have more packages that haven't been persisted.
extra = packages.extra(oldPackages)
if extra:
_LOG.debug("extra packages: %s", extra)
oldPackages.update(packages)
# have to remove existing dataset first, butler nas no replace option
ref = self.butler.registry.findDataset(
datasetType, dataId, collections=[self.butler.run])
self.butler.pruneDatasets([ref], unstore=True, purge=True)
self.butler.put(oldPackages, datasetType, dataId)
else:
self.butler.put(packages, datasetType, dataId)
def writePackageVersions(self, butler, clobber=False, doBackup=True, dataset="packages"):
"""Compare and write package versions.
Parameters
----------
butler : `lsst.daf.persistence.Butler`
Data butler used to read/write the package versions.
clobber : `bool`, optional
A boolean flag that controls what happens if versions already have been saved:
- `True`: overwrite or rename the existing version info, depending on ``doBackup``.
- `False`: raise `TaskError` if this version info does not match the existing.
doBackup : `bool`, optional
If `True` and clobbering, old package version files are backed up.
dataset : `str`, optional
Name of dataset to read/write.
Raises
------
TaskError
Raised if there is a version mismatch with current and persisted lists of package versions.
Notes
-----
Note that this operation is subject to a race condition.
"""
packages = Packages.fromSystem()
if clobber:
return butler.put(packages, dataset, doBackup=doBackup)
if not butler.datasetExists(dataset, write=True):
return butler.put(packages, dataset)
try:
old = butler.get(dataset, immediate=True)
except Exception as exc:
raise type(exc)("Unable to read stored version dataset %s (%s); "
"consider using --clobber-versions or --no-versions" %
(dataset, exc))
# Note that because we can only detect python modules that have been imported, the stored
# list of products may be more or less complete than what we have now. What's important is
# that the products that are in common have the same version.
diff = packages.difference(old)
if diff:
raise TaskError(
"Version mismatch (" +
"; ".join("%s: %s vs %s" % (pkg, diff[pkg][1], diff[pkg][0]) for pkg in diff) +
"); consider using --clobber-versions or --no-versions")
# Update the old set of packages in case we have more packages that haven't been persisted.
extra = packages.extra(old)
if extra:
old.update(packages)
butler.put(old, dataset, doBackup=doBackup)
def writePackageVersions(self,
butler,
clobber=False,
doBackup=True,
dataset="packages"):
"""!Compare and write package versions
We retrieve the persisted list of packages and compare with what we're currently using.
We raise TaskError if there's a version mismatch.
Note that this operation is subject to a race condition.
@param[in] butler data butler used to read/write the package versions
@param[in] clobber a boolean flag that controls what happens if versions already have been saved:
- True: overwrite or rename the existing version info, depending on `doBackup`
- False: raise TaskError if this version info does not match the existing
@param[in] doBackup if clobbering, should we backup the old files?
@param[in] dataset name of dataset to read/write
"""
packages = Packages.fromSystem()
if clobber:
return butler.put(packages, dataset, doBackup=doBackup)
if not butler.datasetExists(dataset):
return butler.put(packages, dataset)
try:
old = butler.get(dataset, immediate=True)
except Exception as exc:
raise type(exc)(
"Unable to read stored version dataset %s (%s); "
"consider using --clobber-versions or --no-versions" %
(dataset, exc))
# Note that because we can only detect python modules that have been imported, the stored
# list of products may be more or less complete than what we have now. What's important is
# that the products that are in common have the same version.
diff = packages.difference(old)
if diff:
raise TaskError(
"Version mismatch (" +
"; ".join("%s: %s vs %s" % (pkg, diff[pkg][1], diff[pkg][0])
for pkg in diff) +
"); consider using --clobber-versions or --no-versions")
# Update the old set of packages in case we have more packages that haven't been persisted.
extra = packages.extra(old)
if extra:
old.update(packages)
butler.put(old, dataset, doBackup=doBackup)
def testFundamentalTypes(self) -> None:
"""Ensure that some fundamental stack types round trip."""
ps = PropertySet()
ps["a.b"] = 5
ps["c.d.e"] = "string"
self.runFundamentalTypeTest("ps", ps)
pl = PropertyList()
pl["A"] = 1
pl.setComment("A", "An int comment")
pl["B"] = "string"
pl.setComment("B", "A string comment")
self.runFundamentalTypeTest("pl", pl)
pkg = Packages.fromSystem()
self.runFundamentalTypeTest("pkg", pkg)