def setUpClass(cls):
super().setUpClass()
cls.dataset = Dataset(cls.testDataset)
cls.INSTRUMENT = cls.dataset.instrument.getName()
cls.VISIT_ID = 204595
cls.DETECTOR_ID = 37
cls.rawData = [
{
'type': 'raw',
'file': 'lsst_a_204595_R11_S01_i.fits',
'exposure': cls.VISIT_ID,
'detector': cls.DETECTOR_ID,
'instrument': cls.INSTRUMENT
},
]
cls.calibData = [
{
'type': 'bias',
'file': 'bias-R11-S01-det037_2022-01-01.fits.gz',
'detector': cls.DETECTOR_ID,
'instrument': cls.INSTRUMENT
},
{
'type': 'flat',
'file': 'flat_i-R11-S01-det037_2022-08-06.fits.gz',
'detector': cls.DETECTOR_ID,
'instrument': cls.INSTRUMENT,
'physical_filter': 'i_sim_1.4'
},
]
def setUp(self):
# Repositories still get used by IngestTask despite Butler being a mock object
self._repo = self._calibRepo = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self._repo, ignore_errors=True)
# Fake Butler and RegisterTask to avoid initialization or DB overhead
def mockGet(datasetType, dataId=None):
"""Minimally fake a butler.get().
"""
if "raw_filename" in datasetType:
matchingFiles = [datum['file'] for datum in IngestionTestSuite.rawData
if datum['expId'] == dataId['expId']]
return [os.path.join(self._repo, file) for file in matchingFiles]
elif "bias_filename" in datasetType:
matchingFiles = [datum['file'] for datum in IngestionTestSuite.calibData
if datum['type'] == 'bias']
return [os.path.join(self._repo, file) for file in matchingFiles]
elif "flat_filename" in datasetType:
matchingFiles = [datum['file'] for datum in IngestionTestSuite.calibData
if datum['type'] == 'flat' and datum['filter'] == dataId['filter']]
return [os.path.join(self._repo, file) for file in matchingFiles]
elif "defects_filename" in datasetType:
return [os.path.join(self._repo, 'defects', 'defects.fits'), ]
elif "camera" in datasetType:
return IngestionTestSuite.mockCamera
else:
return None
butlerPatcher = unittest.mock.patch("lsst.daf.persistence.Butler")
self._butler = butlerPatcher.start()
self._butler.getMapperClass.return_value = lsst.obs.lsst.imsim.ImsimMapper
self._butler.return_value.get = mockGet
self.addCleanup(butlerPatcher.stop)
self._dataset = Dataset(self.datasetKey)
# Fake Workspace because it's too hard to make a real one with a fake Butler
self._workspace = unittest.mock.NonCallableMock(
spec=WorkspaceGen2,
dataRepo=self._repo,
calibRepo=self._calibRepo,
)
self._task = ingestion.DatasetIngestTask(config=IngestionTestSuite.config)
def testBadDataset(self):
"""Verify that Dataset construction fails gracefully on unsupported datasets.
"""
with self.assertRaises(ValueError):
Dataset("TotallyBogusDataset")
def testDatasets(self):
"""Verify that a Dataset knows its supported datasets.
"""
datasets = Dataset.getSupportedDatasets()
self.assertIn(DatasetTestSuite.datasetKey,
datasets) # assumed by other tests
def setUp(self):
self._testbed = Dataset(DatasetTestSuite.datasetKey)
class DatasetTestSuite(DataTestCase):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.obsPackage = 'obs_test'
cls.camera = 'test'
def setUp(self):
self._testbed = Dataset(DatasetTestSuite.datasetKey)
def testDatasets(self):
"""Verify that a Dataset knows its supported datasets.
"""
datasets = Dataset.getSupportedDatasets()
self.assertIn(DatasetTestSuite.datasetKey,
datasets) # assumed by other tests
def testBadDataset(self):
"""Verify that Dataset construction fails gracefully on unsupported datasets.
"""
with self.assertRaises(ValueError):
Dataset("TotallyBogusDataset")
def testDirectories(self):
"""Verify that a Dataset reports the desired directory structure.
"""
root = self._testbed.datasetRoot
self.assertEqual(self._testbed.rawLocation, os.path.join(root, 'raw'))
self.assertEqual(self._testbed.calibLocation,
os.path.join(root, 'calib'))
self.assertEqual(self._testbed.templateLocation,
os.path.join(root, 'templates'))
self.assertEqual(self._testbed.refcatsLocation,
os.path.join(root, 'refcats'))
def testObsPackage(self):
"""Verify that a Dataset knows its associated obs package and camera.
"""
self.assertEqual(self._testbed.obsPackage, DatasetTestSuite.obsPackage)
self.assertEqual(self._testbed.camera, DatasetTestSuite.camera)
def testOutput(self):
"""Verify that a Dataset can create an output repository as desired.
"""
testDir = tempfile.mkdtemp()
outputDir = os.path.join(testDir, 'goodOut')
calibRepoDir = outputDir
try:
self._testbed.makeCompatibleRepo(outputDir, calibRepoDir)
self.assertTrue(os.path.exists(outputDir),
'Output directory must exist.')
self.assertTrue(os.listdir(outputDir),
'Output directory must not be empty.')
self.assertTrue(
os.path.exists(os.path.join(outputDir, '_mapper')) or
os.path.exists(os.path.join(outputDir, 'repositoryCfg.yaml')),
'Output directory must have a _mapper or repositoryCfg.yaml file.'
)
finally:
if os.path.exists(testDir):
shutil.rmtree(testDir, ignore_errors=True)
def testExistingOutput(self):
"""Verify that a Dataset can handle pre-existing output directories,
including directories made by external code.
"""
testDir = tempfile.mkdtemp()
outputDir = os.path.join(testDir, 'badOut')
calibRepoDir = outputDir
try:
os.makedirs(outputDir)
output = os.path.join(outputDir, 'foo.txt')
with open(output, 'w') as dummy:
dummy.write('This is a test!')
self._testbed.makeCompatibleRepo(outputDir, calibRepoDir)
self.assertTrue(os.path.exists(outputDir),
'Output directory must exist.')
self.assertTrue(os.listdir(outputDir),
'Output directory must not be empty.')
self.assertTrue(
os.path.exists(os.path.join(outputDir, '_mapper')) or
os.path.exists(os.path.join(outputDir, 'repositoryCfg.yaml')),
'Output directory must have a _mapper or repositoryCfg.yaml file.'
)
finally:
if os.path.exists(testDir):
shutil.rmtree(testDir, ignore_errors=True)
def testBadDataset(self):
"""Verify that Dataset construction fails gracefully on nonexistent datasets.
"""
with self.assertRaises(ValueError):
Dataset("ap_verify_totally_bogus")
def setUp(self):
self._testbed = Dataset(DatasetTestSuite.testDataset)
class DatasetTestSuite(DataTestCase):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.obsPackage = 'obs_lsst'
cls.camera = 'imsim'
cls.gen3Camera = 'LSSTCam-imSim'
def setUp(self):
self._testbed = Dataset(DatasetTestSuite.testDataset)
def testRepr(self):
# Required to match constructor call
self.assertEqual(repr(self._testbed), "Dataset(" + repr(self.testDataset) + ")")
def testBadDataset(self):
"""Verify that Dataset construction fails gracefully on nonexistent datasets.
"""
with self.assertRaises(ValueError):
Dataset("ap_verify_totally_bogus")
def testDirectories(self):
"""Verify that a Dataset reports the desired directory structure.
"""
root = self._testbed.datasetRoot
self.assertEqual(self._testbed.rawLocation, os.path.join(root, 'raw'))
def testObsPackage(self):
"""Verify that a Dataset knows its associated obs package and camera.
"""
self.assertEqual(self._testbed.instrument.getName(), DatasetTestSuite.gen3Camera)
def _checkOutputGen3(self, repo):
"""Perform various integrity checks on a repository.
Parameters
----------
repo : `str`
The repository to test. Currently only filesystem repositories
are supported.
"""
self.assertTrue(os.path.exists(repo), 'Output directory must exist.')
# Call to Butler will fail if repo is corrupted
butler = dafButler.Butler(repo)
self.assertIn("LSSTCam-imSim/calib", butler.registry.queryCollections())
def testOutputGen3(self):
"""Verify that a Dataset can create an output repository as desired.
"""
testDir = tempfile.mkdtemp()
outputDir = os.path.join(testDir, 'goodOut')
try:
self._testbed.makeCompatibleRepoGen3(outputDir)
self._checkOutputGen3(outputDir)
finally:
if os.path.exists(testDir):
shutil.rmtree(testDir, ignore_errors=True)
def testExistingOutputGen3(self):
"""Verify that a Dataset can handle pre-existing output directories,
including directories made by external code.
"""
testDir = tempfile.mkdtemp()
outputDir = os.path.join(testDir, 'badOut')
try:
self._testbed.makeCompatibleRepoGen3(outputDir)
self._checkOutputGen3(outputDir)
self._testbed.makeCompatibleRepoGen3(outputDir)
self._checkOutputGen3(outputDir)
finally:
if os.path.exists(testDir):
shutil.rmtree(testDir, ignore_errors=True)
class DatasetTestSuite(DataTestCase):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.obsPackage = 'obs_lsst'
cls.camera = 'imsim'
cls.gen3Camera = 'LSSTCam-imSim'
def setUp(self):
self._testbed = Dataset(DatasetTestSuite.datasetKey)
def testRepr(self):
# Required to match constructor call
self.assertEqual(repr(self._testbed),
"Dataset(" + repr(self.datasetKey) + ")")
def testDatasets(self):
"""Verify that a Dataset knows its supported datasets.
"""
datasets = Dataset.getSupportedDatasets()
self.assertIn(DatasetTestSuite.datasetKey,
datasets) # assumed by other tests
def testBadDataset(self):
"""Verify that Dataset construction fails gracefully on unsupported datasets.
"""
with self.assertRaises(ValueError):
Dataset("TotallyBogusDataset")
def testDirectories(self):
"""Verify that a Dataset reports the desired directory structure.
"""
root = self._testbed.datasetRoot
self.assertEqual(self._testbed.rawLocation, os.path.join(root, 'raw'))
self.assertEqual(self._testbed.calibLocation,
os.path.join(root, 'calib'))
self.assertEqual(self._testbed.templateLocation,
os.path.join(root, 'templates'))
self.assertEqual(self._testbed.refcatsLocation,
os.path.join(root, 'refcats'))
def testObsPackage(self):
"""Verify that a Dataset knows its associated obs package and camera.
"""
self.assertEqual(self._testbed.obsPackage, DatasetTestSuite.obsPackage)
self.assertEqual(self._testbed.camera, DatasetTestSuite.camera)
self.assertEqual(self._testbed.instrument.getName(),
DatasetTestSuite.gen3Camera)
def testOutput(self):
"""Verify that a Dataset can create an output repository as desired.
"""
testDir = tempfile.mkdtemp()
outputDir = os.path.join(testDir, 'goodOut')
calibRepoDir = outputDir
try:
self._testbed.makeCompatibleRepo(outputDir, calibRepoDir)
self.assertTrue(os.path.exists(outputDir),
'Output directory must exist.')
self.assertTrue(os.listdir(outputDir),
'Output directory must not be empty.')
self.assertTrue(
os.path.exists(os.path.join(outputDir, '_mapper')) or
os.path.exists(os.path.join(outputDir, 'repositoryCfg.yaml')),
'Output directory must have a _mapper or repositoryCfg.yaml file.'
)
finally:
if os.path.exists(testDir):
shutil.rmtree(testDir, ignore_errors=True)
def testExistingOutput(self):
"""Verify that a Dataset can handle pre-existing output directories,
including directories made by external code.
"""
testDir = tempfile.mkdtemp()
outputDir = os.path.join(testDir, 'badOut')
calibRepoDir = outputDir
try:
os.makedirs(outputDir)
output = os.path.join(outputDir, 'foo.txt')
with open(output, 'w') as dummy:
dummy.write('This is a test!')
self._testbed.makeCompatibleRepo(outputDir, calibRepoDir)
self.assertTrue(os.path.exists(outputDir),
'Output directory must exist.')
self.assertTrue(os.listdir(outputDir),
'Output directory must not be empty.')
self.assertTrue(
os.path.exists(os.path.join(outputDir, '_mapper')) or
os.path.exists(os.path.join(outputDir, 'repositoryCfg.yaml')),
'Output directory must have a _mapper or repositoryCfg.yaml file.'
)
finally:
if os.path.exists(testDir):
shutil.rmtree(testDir, ignore_errors=True)
def _checkOutputGen3(self, repo):
"""Perform various integrity checks on a repository.
Parameters
----------
repo : `str`
The repository to test. Currently only filesystem repositories
are supported.
"""
self.assertTrue(os.path.exists(repo), 'Output directory must exist.')
# Call to Butler will fail if repo is corrupted
butler = dafButler.Butler(repo)
self.assertIn("LSSTCam-imSim/calib",
butler.registry.queryCollections())
def testOutputGen3(self):
"""Verify that a Dataset can create an output repository as desired.
"""
testDir = tempfile.mkdtemp()
outputDir = os.path.join(testDir, 'goodOut')
try:
self._testbed.makeCompatibleRepoGen3(outputDir)
self._checkOutputGen3(outputDir)
finally:
if os.path.exists(testDir):
shutil.rmtree(testDir, ignore_errors=True)
def testExistingOutputGen3(self):
"""Verify that a Dataset can handle pre-existing output directories,
including directories made by external code.
"""
testDir = tempfile.mkdtemp()
outputDir = os.path.join(testDir, 'badOut')
try:
self._testbed.makeCompatibleRepoGen3(outputDir)
self._checkOutputGen3(outputDir)
self._testbed.makeCompatibleRepoGen3(outputDir)
self._checkOutputGen3(outputDir)
finally:
if os.path.exists(testDir):
shutil.rmtree(testDir, ignore_errors=True)