def setUp(self):
numpy.random.seed(32)
self.bandpassPossibilities = ['u', 'g', 'r', 'i', 'z', 'y']
self.bandpassDir = os.path.join(getPackageDir('throughputs'), 'baseline')
self.sedDir = os.path.join(getPackageDir('sims_sed_library'))
self.sedDir = os.path.join(self.sedDir, 'galaxySED')
self.sedPossibilities = os.listdir(self.sedDir)
def setUp(self):
self.rng = np.random.RandomState(32)
self.bandpassPossibilities = ['u', 'g', 'r', 'i', 'z', 'y']
self.bandpassDir = os.path.join(getPackageDir('throughputs'), 'baseline')
self.sedDir = os.path.join(getPackageDir('sims_photUtils'),
'tests/cartoonSedTestData/galaxySed')
self.sedPossibilities = os.listdir(self.sedDir)
def setUpClass(cls):
cls.obs = ObservationMetaData(bandpassName=['u', 'g', 'r', 'i', 'z', 'y'],
m5 = [22.0, 23.0, 24.0, 25.0, 26.0, 27.0])
baselineDtype = np.dtype([(name, np.float) for name in baselineStarCatalog.column_outputs])
dbdtype = np.dtype([
('id', np.int),
('raJ2000', np.float),
('decJ2000', np.float),
('sedFilename', str, 100),
('magNorm', np.float),
('galacticAv', np.float)
])
inputDir = os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData')
inputFile = os.path.join(inputDir, 'IndicesTestCatalogStars.txt')
cls.db = fileDBObject(inputFile, runtable='test',
idColKey='id', dtype=dbdtype)
cat = baselineStarCatalog(cls.db, obs_metadata=cls.obs)
cls.catName = os.path.join(getPackageDir('sims_catUtils'), 'tests',
'scratchSpace', 'indicesStarsControlCat.txt')
cat.write_catalog(cls.catName)
cls.controlData = np.genfromtxt(cls.catName, dtype=baselineDtype, delimiter=',')
def requirePackage(packageName):
"""
Decorator to skip unit tests unless the package ``packageName`` is set up.
"""
try:
getPackageDir(packageName)
except pexExcept.NotFoundError as e:
return unittest.skip("Required package %s not available [%s]." % (packageName, str(e)))
return lambda func: func
def setUp(self):
self.normband = Bandpass()
self.normband.imsimBandpass()
self.uband = Bandpass()
self.uband.readThroughput(os.path.join(getPackageDir('sims_photUtils'), 'tests',
'cartoonSedTestData', 'test_bandpass_u.dat'))
self.gband = Bandpass()
self.gband.readThroughput(os.path.join(getPackageDir('sims_photUtils'), 'tests',
'cartoonSedTestData', 'test_bandpass_g.dat'))
def testBasics(self):
utilsPath = getPackageDir("utils")
self.assertTrue(os.path.isfile(os.path.join(utilsPath, "tests", "test_getPackageDir.py")))
# NOTE: one goal of this test is to confirm that the correct exception
# is raised even if we haven't explicitly imported pex.exceptions.
# Hence the odd structure of this assert block.
with self.assertRaises(Exception) as cm:
getPackageDir("nameOfNonexistendPackage2234q?#!")
self.assertIsInstance(cm.exception, lsst.pex.exceptions.NotFoundError)
def setUp(self):
rawPath = os.path.join(getPackageDir(TEST_DATA_PACKAGE), "hsc", "raw")
calibPath = os.path.join(getPackageDir(TEST_DATA_PACKAGE), "hsc", "calib")
self.repoPath = createDataRepository("lsst.obs.hsc.HscMapper", rawPath)
self.butler = dafPersist.Butler(root=self.repoPath, calibRoot=calibPath)
# The following properties of the provided data are known a priori.
self.visit = 904024
self.ccdNum = 50
self.filter = 'i'
self.rawSize = (2144, 4241)
self.ccdSize = (2048, 4176)
def testHalfLightRadiusOfImage(self):
"""
Test that GalSim is generating images of objects with the expected half light radius
by generating images with one object on them and comparing the total flux in the image
with the flux contained within the expected half light radius. Raise an exception
if the deviation is greater than 3-sigma.
"""
scratchDir = os.path.join(getPackageDir('sims_GalSimInterface'), 'tests', 'scratchSpace')
catName = os.path.join(scratchDir, 'hlr_test_Catalog.dat')
imageRoot = os.path.join(scratchDir, 'hlr_test_Image')
dbFileName = os.path.join(scratchDir, 'hlr_test_InputCatalog.dat')
baseDir = os.path.join(getPackageDir('sims_GalSimInterface'), 'tests', 'cameraData')
camera = ReturnCamera(baseDir)
detector = camera[0]
detName = detector.getName()
imageName = '%s_%s_u.fits' % (imageRoot, detName)
obs = ObservationMetaData(pointingRA = 75.0,
pointingDec = -12.0,
boundType = 'circle',
boundLength = 4.0,
rotSkyPos = 33.0,
mjd = 49250.0)
hlrTestList = [1.0, 2.0, 4.0]
for hlr in hlrTestList:
create_text_catalog(obs, dbFileName, np.array([3.0]), np.array([1.0]),
hlr=[hlr])
db = hlrFileDBObj(dbFileName, runtable='test')
cat = hlrCat(db, obs_metadata=obs)
cat.camera = camera
cat.write_catalog(catName)
cat.write_images(nameRoot=imageRoot)
totalFlux, hlrFlux = self.get_flux_in_half_light_radius(imageName, hlr, detector, camera, obs)
self.assertGreater(totalFlux, 1000.0) # make sure the image is not blank
# divide by gain because Poisson stats apply to photons
sigmaFlux = np.sqrt(0.5*totalFlux/cat.photParams.gain)
self.assertLess(np.abs(hlrFlux-0.5*totalFlux), 4.0*sigmaFlux)
if os.path.exists(catName):
os.unlink(catName)
if os.path.exists(dbFileName):
os.unlink(dbFileName)
if os.path.exists(imageName):
os.unlink(imageName)
def setUpClass(cls):
cls.scratchDir = os.path.join(getPackageDir('sims_GalSimInterface'), 'tests', 'scratchSpace')
cls.obs = ObservationMetaData(pointingRA=122.0, pointingDec=-29.1,
mjd=57381.2, rotSkyPos=43.2)
cls.camera = camTestUtils.CameraWrapper().camera
cls.dbFileName = os.path.join(cls.scratchDir, 'allowed_chips_test_db.txt')
if os.path.exists(cls.dbFileName):
os.unlink(cls.dbFileName)
cls.controlSed = Sed()
cls.controlSed.readSED_flambda(os.path.join(getPackageDir('sims_sed_library'),
'flatSED','sed_flat.txt.gz'))
cls.magNorm = 18.1
imsim = Bandpass()
imsim.imsimBandpass()
ff = cls.controlSed.calcFluxNorm(cls.magNorm, imsim)
cls.controlSed.multiplyFluxNorm(ff)
a_x, b_x = cls.controlSed.setupCCMab()
cls.controlSed.addCCMDust(a_x, b_x, A_v=0.1, R_v=3.1)
bpd = BandpassDict.loadTotalBandpassesFromFiles()
pp = PhotometricParameters()
cls.controlADU = cls.controlSed.calcADU(bpd['u'], pp)
cls.countSigma = np.sqrt(cls.controlADU/pp.gain)
cls.x_pix = 50
cls.y_pix = 50
x_list = []
y_list = []
name_list = []
for dd in cls.camera:
x_list.append(cls.x_pix)
y_list.append(cls.y_pix)
name_list.append(dd.getName())
x_list = np.array(x_list)
y_list = np.array(y_list)
ra_list, dec_list = raDecFromPixelCoords(x_list, y_list, name_list,
camera=cls.camera, obs_metadata=cls.obs,
epoch=2000.0)
dra_list = 3600.0*(ra_list-cls.obs.pointingRA)
ddec_list = 3600.0*(dec_list-cls.obs.pointingDec)
create_text_catalog(cls.obs, cls.dbFileName, dra_list, ddec_list,
mag_norm=[cls.magNorm]*len(dra_list))
cls.db = allowedChipsFileDBObj(cls.dbFileName, runtable='test')
def testFwhmOfImage(self):
"""
Test that GalSim generates images with the expected Full Width at Half Maximum.
"""
scratchDir = os.path.join(getPackageDir('sims_GalSimInterface'), 'tests', 'scratchSpace')
catName = os.path.join(scratchDir, 'fwhm_test_Catalog.dat')
imageRoot = os.path.join(scratchDir, 'fwhm_test_Image')
dbFileName = os.path.join(scratchDir, 'fwhm_test_InputCatalog.dat')
baseDir = os.path.join(getPackageDir('sims_GalSimInterface'), 'tests', 'cameraData')
camera = ReturnCamera(baseDir)
detector = camera[0]
detName = detector.getName()
imageName = '%s_%s_u.fits' % (imageRoot, detName)
obs = ObservationMetaData(pointingRA = 75.0,
pointingDec = -12.0,
boundType = 'circle',
boundLength = 4.0,
rotSkyPos = 33.0,
mjd = 49250.0)
create_text_catalog(obs, dbFileName, numpy.array([3.0]), \
numpy.array([1.0]), mag_norm=[13.0])
db = fwhmFileDBObj(dbFileName, runtable='test')
for fwhm in (0.5, 1.3):
cat = fwhmCat(db, obs_metadata=obs)
cat.camera = camera
psf = SNRdocumentPSF(fwhm=fwhm)
cat.setPSF(psf)
cat.write_catalog(catName)
cat.write_images(nameRoot=imageRoot)
self.verify_fwhm(imageName, fwhm, detector, camera, obs)
if os.path.exists(catName):
os.unlink(catName)
if os.path.exists(imageName):
os.unlink(imageName)
if os.path.exists(dbFileName):
os.unlink(dbFileName)
def __init__(self, catsim_cat, om10_cat='twinkles_lenses_v2.fits',
density_param=1.):
"""
Parameters
----------
catsim_cat: catsim catalog
The results array from an instance catalog.
om10_cat: optional, defaults to 'twinkles_tdc_rung4.fits
fits file with OM10 catalog
density_param: `np.float`, optioanl, defaults to 1.0
the fraction of eligible agn objects that become lensed and should
be between 0.0 and 1.0.
Returns
-------
updated_catalog:
A new results array with lens systems added.
"""
twinklesDir = getPackageDir('Twinkles')
om10_cat = os.path.join(twinklesDir, 'data', om10_cat)
self.catalog = catsim_cat
# ****** THIS ASSUMES THAT THE ENVIRONMENT VARIABLE OM10_DIR IS SET *******
lensdb = om10.DB(catalog=om10_cat)
self.lenscat = lensdb.lenses.copy()
self.density_param = density_param
self.bandpassDict = BandpassDict.loadTotalBandpassesFromFiles(bandpassNames=['i'])
specFileStart = 'Burst'
for key, val in sorted(iteritems(SpecMap.subdir_map)):
if re.match(key, specFileStart):
galSpecDir = str(val)
galDir = str(getPackageDir('sims_sed_library') + '/' + galSpecDir + '/')
self.LRG_name = 'Burst.25E09.1Z.spec'
self.LRG = Sed()
self.LRG.readSED_flambda(str(galDir + self.LRG_name))
#return
#Calculate imsimband magnitudes of source galaxies for matching
agn_sed = Sed()
agn_fname = str(getPackageDir('sims_sed_library') + '/agnSED/agn.spec.gz')
agn_sed.readSED_flambda(agn_fname)
src_iband = self.lenscat['MAGI_IN']
self.src_mag_norm = []
for src in src_iband:
self.src_mag_norm.append(matchBase().calcMagNorm([src],
agn_sed,
self.bandpassDict))
def testMags(self):
"""
Test that the interpolated mags are similar to mags computed from interpolated spectra
"""
throughPath = os.path.join(getPackageDir("throughputs"), "baseline")
filters = ["u", "g", "r", "i", "z", "y"]
bps = []
for filterName in filters:
bp = np.loadtxt(
os.path.join(throughPath, "filter_%s.dat" % filterName), dtype=zip(["wave", "trans"], [float] * 2)
)
lsst_bp = Bandpass()
lsst_bp.setBandpass(bp["wave"], bp["trans"])
bps.append(lsst_bp)
sm1 = sb.SkyModel()
sm1.setRaDecMjd([36.0], [-68.0], 49353.18, degrees=True)
sm1.computeSpec()
mags1 = []
for bp in bps:
mags1.append(sm1.computeMags(bandpass=bp))
mags1 = np.array(mags1)
sm2 = sb.SkyModel(mags=True)
sm2.setRaDecMjd([36.0], [-68.0], 49353.18, degrees=True)
sm2.computeSpec()
mag2 = sm2.computeMags()
np.testing.assert_allclose(mags1, mag2.T, rtol=1e-4)
def _loadSedList(self, wavelen_match):
"""
Method to load the member variable self._sedList, which is a SedList.
If self._sedList does not already exist, this method sets it up.
If it does already exist, this method flushes its contents and loads a new
chunk of Seds.
"""
sedNameList = self.column_by_name('sedFilename')
magNormList = self.column_by_name('magNorm')
galacticAvList = self.column_by_name('galacticAv')
if len(sedNameList)==0:
return np.ones((0))
if not hasattr(self, '_sedList'):
self._sedList = SedList(sedNameList, magNormList,
galacticAvList=galacticAvList,
wavelenMatch=wavelen_match,
fileDir=getPackageDir('sims_sed_library'),
specMap=defaultSpecMap)
else:
self._sedList.flush()
self._sedList.loadSedsFromList(sedNameList, magNormList,
galacticAvList=galacticAvList)
def setUpClass(cls):
cls.obs = ObservationMetaData(bandpassName=['u', 'g', 'r', 'i', 'z', 'y'],
m5=[24.0, 25.0, 26.0, 27.0, 28.0, 29.0])
dtype = np.dtype([
('id', np.int),
('sedFilenameBulge', str, 100),
('magNormBulge', np.float),
('sedFilenameDisk', str, 100),
('magNormDisk', np.float),
('sedFilenameAgn', str, 100),
('magNormAgn', np.float),
('internalAvBulge', np.float),
('internalAvDisk', np.float),
('galacticAv', np.float),
('redshift', np.float)
])
inputDir = os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData')
inputFile = os.path.join(inputDir, 'IndicesTestCatalogGalaxies.txt')
cls.db = fileDBObject(inputFile, dtype=dtype, runtable='test',
idColKey='id')
cls.db.objectTypeId = 44
cat = baselineGalaxyCatalog(cls.db, obs_metadata=cls.obs)
dtype = np.dtype([(name, np.float) for name in cat.column_outputs])
catName = tempfile.mktemp(dir=ROOT, prefix='', suffix='.txt')
cat.write_catalog(catName)
cls.controlData = np.genfromtxt(catName, dtype=dtype, delimiter=',')
os.remove(catName)
def initializeIGM(self, zMin = 1.5, zMax = 2.9, zDelta = 0.1, minWavelen = 300):
"""
Initialize an applyIGM object with the desired redshift grid.
If lookup tables are not evenly spaced in redshift then input manually
desired zRange array.
@param [in] zMin is the minimum redshift.
@param [in] zMax is the maximum redshift.
@param [in] zDelta is the redshift spacing.
@param [in] minWavelen is the minimum wavelength in the lookup tables
"""
self.zMin = zMin
self.zMax = zMax
self.zDelta = zDelta
self.minWavelen = minWavelen
#Don't have max wavelength since transmission goes to 1.0 at longest wavelengths
self.zRange = np.arange(zMin, zMax + (zDelta/2.), zDelta)
if self.tablesPresent == False:
table_dir = getPackageDir('sims_catUtils')
table_dir = os.path.join(table_dir,
'python/lsst/sims/catUtils/IGM/igm_tables')
self.loadTables(table_dir)
self.IGMisInitialized = True
def setUp(self):
# Read orbits.
self.orbits = Orbits()
self.jplDir = os.path.join(getPackageDir('sims_movingObjects'), 'tests/jpl_testdata')
self.orbits.readOrbits(os.path.join(self.jplDir, 'S0_n747.des'), skiprows=1)
# Read JPL ephems.
self.jpl = pd.read_table(os.path.join(self.jplDir, '807_n747.txt'), delim_whitespace=True)
# Add times in TAI and UTC, because.
t = Time(self.jpl['epoch_mjd'], format='mjd', scale='utc')
self.jpl['mjdTAI'] = t.tai.mjd
self.jpl['mjdUTC'] = t.utc.mjd
self.jpl = self.jpl.to_records(index=False)
# Generate interpolation coefficients for the time period in the JPL catalog.
self.coeffFile = 'test_coeffs'
self.residFile = 'test_resids'
self.failedFile = 'test_failed'
tStart = self.jpl['mjdTAI'].min() - 0.2
tSpan = 1.0
self.chebyFits = ChebyFits(self.orbits, tStart, tSpan,
ngran=64, skyTolerance=2.5,
nCoeff_position=14, obscode=807)
self.chebyFits.calcSegmentLength()
self.chebyFits.calcSegments()
self.chebyFits.write(self.coeffFile, self.residFile, self.failedFile, append=False)
self.coeffKeys = ['objId', 'tStart', 'tEnd', 'ra', 'dec', 'delta', 'vmag', 'elongation']
self.chebyValues = ChebyValues()
self.chebyValues.readCoefficients(self.coeffFile)
def testLSSTmags(self):
"""
Test that PhotometrySSM properly calculates LSST magnitudes
"""
cat = LSST_SSM_photCat(self.photDB)
dtype = np.dtype([('id', np.int), ('u', np.float), ('g', np.float),
('r', np.float), ('i', np.float), ('z', np.float),
('y', np.float)])
with lsst.utils.tests.getTempFilePath('.txt') as catName:
cat.write_catalog(catName)
testData = np.genfromtxt(catName, dtype=dtype, delimiter=',')
self.assertGreater(len(testData), 0)
controlData = np.genfromtxt(self.dbFile, dtype=self.dtype)
self.assertGreater(len(controlData), 0)
LSSTbandpasses = BandpassDict.loadTotalBandpassesFromFiles()
controlSedList = SedList(controlData['sedFilename'], controlData['magNorm'],
wavelenMatch=LSSTbandpasses.wavelenMatch,
fileDir=getPackageDir('sims_sed_library'),
specMap=defaultSpecMap)
controlMags = LSSTbandpasses.magListForSedList(controlSedList)
for ii in range(len(controlMags)):
for jj, bpName in enumerate(['u', 'g', 'r', 'i', 'z', 'y']):
self.assertAlmostEqual(controlMags[ii][jj], testData[bpName][ii], 10)
def __init__(self, compName=None, sortedOrder=['airmass', 'nightTimes'], mags=False):
"""
mags: Rather than the full spectrum, return the LSST ugrizy magnitudes.
"""
self.mags = mags
dataDir = os.path.join(getPackageDir('sims_skybrightness_data'), 'ESO_Spectra/'+compName)
filenames = sorted(glob.glob(dataDir+'/*.npz'))
self.spec, self.wave, self.filterWave = loadSpecFiles(filenames, mags=self.mags)
# Take the log of the spectra in case we want to interp in log space.
if not mags:
self.logSpec = np.zeros(self.spec['spectra'].shape, dtype=float)
good = np.where(self.spec['spectra'] != 0)
self.logSpec[good] = np.log10(self.spec['spectra'][good])
self.specSize = self.spec['spectra'][0].size
else:
self.specSize = 0
# What order are the dimesions sorted by (from how the .npz was packaged)
self.sortedOrder = sortedOrder
self.dimDict = {}
self.dimSizes = {}
for dt in self.sortedOrder:
self.dimDict[dt] = np.unique(self.spec[dt])
self.dimSizes[dt] = np.size(np.unique(self.spec[dt]))
# Set up and save the dict to order the filters once.
self.filterNameDict = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5}
def testLoadTotalBandpassesFromFiles(self):
"""
Test that the class method loadTotalBandpassesFromFiles produces the
expected result
"""
bandpassDir = os.path.join(getPackageDir('sims_photUtils'), 'tests', 'cartoonSedTestData')
bandpassNames = ['g', 'r', 'u']
bandpassRoot = 'test_bandpass_'
bandpassDict = BandpassDict.loadTotalBandpassesFromFiles(bandpassNames=bandpassNames,
bandpassDir=bandpassDir,
bandpassRoot = bandpassRoot)
controlBandpassList = []
for bpn in bandpassNames:
dummyBp = Bandpass()
dummyBp.readThroughput(os.path.join(bandpassDir,bandpassRoot+bpn+'.dat'))
controlBandpassList.append(dummyBp)
wMin = controlBandpassList[0].wavelen[0]
wMax = controlBandpassList[0].wavelen[-1]
wStep = controlBandpassList[0].wavelen[1]-controlBandpassList[0].wavelen[0]
for bp in controlBandpassList:
bp.resampleBandpass(wavelen_min=wMin, wavelen_max=wMax, wavelen_step=wStep)
for test, control in zip(bandpassDict.values(), controlBandpassList):
numpy.testing.assert_array_almost_equal(test.wavelen, control.wavelen, 19)
numpy.testing.assert_array_almost_equal(test.sb, control.sb, 19)
def loadTotalBandpassesFromFiles(cls,
bandpassNames=['u', 'g', 'r', 'i', 'z', 'y'],
bandpassDir = os.path.join(getPackageDir('throughputs'),'baseline'),
bandpassRoot = 'total_'):
"""
This will take the list of band passes named by bandpassNames and load them into
a BandpassDict
The bandpasses loaded this way are total bandpasses: they account for instrumental
and atmospheric transmission.
@param [in] bandpassNames is a list of names identifying each filter.
Defaults to ['u', 'g', 'r', 'i', 'z', 'y']
@param [in] bandpassDir is the name of the directory where the bandpass files are stored
@param [in] bandpassRoot contains the first part of the bandpass file name, i.e., it is assumed
that the bandpasses are stored in files of the type
bandpassDir/bandpassRoot_bandpassNames[i].dat
if we want to load bandpasses for a telescope other than LSST, we would do so
by altering bandpassDir and bandpassRoot
@param [out] bandpassDict is a BandpassDict containing the loaded throughputs
"""
bandpassList = []
for w in bandpassNames:
bandpassDummy = Bandpass()
bandpassDummy.readThroughput(os.path.join(bandpassDir,"%s.dat" % (bandpassRoot + w)))
bandpassList.append(bandpassDummy)
return cls(bandpassList, bandpassNames)
def setUp(self):
baseDir = os.path.join(getPackageDir('sims_GalSimInterface'),
'tests', 'cameraData')
self.camera = ReturnCamera(baseDir)
ra = 145.0
dec = -73.0
self.epoch = 2000.0
mjd = 49250.0
rotSkyPos = 45.0
self.obs = ObservationMetaData(unrefractedRA=ra,
unrefractedDec=dec,
boundType='circle',
boundLength=1.0,
mjd=mjd,
rotSkyPos=rotSkyPos)
raPointing, \
decPointing = _observedFromICRS(numpy.array([numpy.radians(ra)]),
numpy.array([numpy.radians(dec)]),
obs_metadata=self.obs,
epoch=self.epoch)
self.ra = raPointing[0]
self.dec = decPointing[0]
def _loadAgnSedList(self, wavelen_match):
"""
Load a SedList of galaxy AGN Seds.
The list will be stored in the variable self._bulgeSedList.
@param [in] wavelen_match is the wavelength grid (in nm)
on which the Seds are to be sampled.
"""
sedNameList = self.column_by_name('sedFilenameAgn')
magNormList = self.column_by_name('magNormAgn')
redshiftList = self.column_by_name('redshift')
cosmologicalDimming = not self._hasCosmoDistMod()
if len(sedNameList)==0:
return np.ones((0))
if not hasattr(self, '_agnSedList'):
self._agnSedList = SedList(sedNameList, magNormList,
redshiftList=redshiftList,
cosmologicalDimming=cosmologicalDimming,
wavelenMatch=wavelen_match,
fileDir=getPackageDir('sims_sed_library'),
specMap=defaultSpecMap)
else:
self._agnSedList.flush()
self._agnSedList.loadSedsFromList(sedNameList, magNormList,
redshiftList=redshiftList)
def __init__(self, catsim_cat, om10_cat='twinkles_tdc_rung4.fits', density_param = 1.):
"""
Input:
catsim_cat:
The results array from an instance catalog.
density_param:
A float between 0. and 1.0 that determines the fraction of eligible agn objects that become lensed.
Output:
updated_catalog:
A new results array with lens systems added.
"""
self.catalog = catsim_cat
# ****** THIS ASSUMES THAT THE ENVIRONMENT VARIABLE OM10_DIR IS SET *******
lensdb = om10.DB(catalog=om10_cat)
self.lenscat = lensdb.lenses.copy()
self.density_param = density_param
self.bandpassDict = BandpassDict.loadTotalBandpassesFromFiles(bandpassNames=['i'])
specFileStart = 'Burst'
for key, val in sorted(iteritems(SpecMap.subdir_map)):
if re.match(key, specFileStart):
galSpecDir = str(val)
galDir = str(getPackageDir('sims_sed_library') + '/' + galSpecDir + '/')
self.LRG_name = 'Burst.25E09.1Z.spec'
self.LRG = Sed()
self.LRG.readSED_flambda(str(galDir + self.LRG_name))
def setUpClass(cls):
cls.obs = ObservationMetaData(bandpassName=['u', 'g', 'r', 'i', 'z', 'y'],
m5 = [22.0, 23.0, 24.0, 25.0, 26.0, 27.0])
baselineDtype = np.dtype([(name, np.float) for name in baselineSSMCatalog.column_outputs])
dbdtype = np.dtype([
('id', np.int),
('sedFilename', str, 100),
('magNorm', np.float),
('velRA', np.float),
('velDec', np.float)
])
inputDir = os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData')
inputFile = os.path.join(inputDir, 'SSMphotometryCatalog.txt')
cls.db = fileDBObject(inputFile, runtable='test',
idColKey='id', dtype=dbdtype)
cat = baselineSSMCatalog(cls.db, obs_metadata=cls.obs)
catName = tempfile.mktemp(prefix='IndexTestCase_setUpClass')
cat.write_catalog(catName)
cls.controlData = np.genfromtxt(catName, dtype=baselineDtype, delimiter=',')
os.unlink(catName)
def readPhotSeds(sedDir=None):
"""
Read all the seds provided by this package, storing them in a
nested dictionary (sedDict['sedtype']['sedname']).
"""
if sedDir == None:
sedDir = os.path.join(getPackageDir('SYSENG_THROUGHPUTS_DIR'), 'seds')
# Set up the lists for the photometry reference SEDs.
# (I'm doing this by hand to keep these lists well-known over time.)
sedlists = {}
sedlists['quasar'] = ['quasar.dat',]
sedlists['stars'] = ['km10_7250.fits_g45', 'km10_6500.fits_g45',
'km10_6000.fits_g45', 'km10_5250.fits_g45',
'km10_4500.fits_g45', 'm3.0Full.dat']
sedlists['white_dwarf'] = ['wd_H_100000_80.dat', 'wd_H_15000_80.dat', 'wd_H_50000_80.dat',
'wd_H_5500_80.dat', 'wd_He_10000_80.dat', 'wd_He_15000_80.dat', 'wd_He_5500_80.dat']
sedlists['sn'] = ['sn1a_15.0.dat', 'sn1a_20.0.dat', 'sn1a_10.0.dat']
sedlists['galaxies'] = ['Sa_template_norm.sed.dat', 'Sdm_template_norm.sed0.dat',
'Ell2_template_norm.sed.dat']
sedlists['photoZ_outliers'] = ['xspec_172.sed.dat', 'xspec_173.sed.dat',
'xspec_175.sed.dat', 'xspec_176.sed.dat',
'xspec_90.sed.dat', 'xspec_91.sed.dat']
# Let's go read the files.
sedDict = {}
# Loop through quasar, stars, galaxies, sn in the sedlist dictionary.
for objtype in sedlists.keys():
sedDict[objtype] = {}
# And for each type of object, loop through and read each SED.
for s in sedlists[objtype]:
sedDict[objtype][s] = Sed()
sedDict[objtype][s].readSED_flambda(os.path.join(sedDir, objtype, s))
return sedDict
def test_eq(self):
"""
Test that __eq__ in Sed works correctly
"""
sed_dir = os.path.join(getPackageDir('sims_photUtils'), 'tests',
'cartoonSedTestData', 'starSed', 'kurucz')
list_of_seds = os.listdir(sed_dir)
sedname1 = os.path.join(sed_dir, list_of_seds[0])
sedname2 = os.path.join(sed_dir, list_of_seds[1])
ss1 = Sed()
ss1.readSED_flambda(sedname1)
ss2 = Sed()
ss2.readSED_flambda(sedname2)
ss3 = Sed()
ss3.readSED_flambda(sedname1)
self.assertFalse(ss1 == ss2)
self.assertTrue(ss1 != ss2)
self.assertTrue(ss1 == ss3)
self.assertFalse(ss1 != ss3)
ss3.flambdaTofnu()
self.assertFalse(ss1 == ss3)
self.assertTrue(ss1 != ss3)
def testMags(self):
"""
Test that the interpolated mags are similar to mags computed from interpolated spectra
"""
throughPath = os.path.join(getPackageDir('throughputs'), 'baseline')
filters = ['u', 'g', 'r', 'i', 'z', 'y']
bps = []
for filterName in filters:
bp = np.loadtxt(os.path.join(throughPath, 'filter_%s.dat' % filterName),
dtype=zip(['wave', 'trans'], [float]*2))
lsst_bp = Bandpass()
lsst_bp.setBandpass(bp['wave'], bp['trans'])
bps.append(lsst_bp)
sm1 = self.sm_spec
sm1.setRaDecMjd([36.], [-68.], 49353.18, degrees=True)
mags1 = []
for bp in bps:
mags1.append(sm1.returnMags(bandpass=bp))
mags1 = np.array(mags1)
sm2 = self.sm_mags
sm2.setRaDecMjd([36.], [-68.], 49353.18, degrees=True)
mag2 = sm2.returnMags()
for i, filtername in enumerate(filters):
np.testing.assert_allclose(mags1[i, :], mag2[filtername], rtol=1e-4)
def test_transformation(self):
"""
Test that transformations are applied to columns in an InstanceCatalog
"""
catName = os.path.join(getPackageDir('sims_catalogs_measures'), 'tests', 'scratchSpace',
'transformation_catalog.txt')
if os.path.exists(catName):
os.unlink(catName)
t = self.starDB.getCatalog('transformation_catalog', obs_metadata=self.obsMd)
t.write_catalog(catName)
dtype = np.dtype([
('id', np.int),
('raJ2000', np.float),
('decJ2000', np.float)
])
testData = np.genfromtxt(catName, delimiter=', ', dtype=dtype)
self.assertGreater(len(testData), 0)
for line in testData:
ic = np.where(self.starControlData['id']==line['id'])[0][0]
self.assertAlmostEqual(line['decJ2000'], self.starControlData['decJ2000'][ic], 5)
self.assertAlmostEqual(line['raJ2000'], 2.0*self.starControlData['raJ2000'][ic], 5)
if os.path.exists(catName):
os.unlink(catName)
def testDustMaps(self):
"""
Go through contents of DustMaps directory and check that files
are the size we expect them to be.
"""
mb = 1024*1024 # because os.path.getsize returns the size in bytes
kb = 1024
control_size_dict = {'SFD_dust_4096_ngp.fits': 64*mb,
'SFD_dust_4096_sgp.fits': 64*mb,
'dust_nside_1024.npz': 96*mb,
'dust_nside_128.npz': 1.5*mb,
'dust_nside_16.npz': 24*kb,
'dust_nside_2.npz': 582,
'dust_nside_256.npz': 6*mb,
'dust_nside_32.npz': 96*kb,
'dust_nside_4.npz': 1.7*kb,
'dust_nside_512.npz': 24*mb,
'dust_nside_64.npz': 384*kb,
'dust_nside_8.npz': 6.2*kb}
root_dir = getPackageDir('sims_maps')
for file_name in control_size_dict:
full_name = os.path.join(root_dir, 'DustMaps', file_name)
size = os.path.getsize(full_name)
self.assertLess(np.abs(size-control_size_dict[file_name]),
0.1*control_size_dict[file_name],
msg=self.lfs_msg)
def packageMergedSpec():
dataDir = getPackageDir('SIMS_SKYBRIGHTNESS_DATA')
outDir = os.path.join(dataDir, 'ESO_Spectra/MergedSpec')
# A large number of the background components only depend on Airmass, so we can merge those together
npzs = ['LowerAtm/Spectra.npz',
'ScatteredStarLight/scatteredStarLight.npz',
'UpperAtm/Spectra.npz']
files = [os.path.join(dataDir, 'ESO_Spectra', npz) for npz in npzs]
temp = np.load(files[0])
wave = temp['wave'].copy()
spec = temp['spec'].copy()
spec['spectra'] = spec['spectra']*0.
spec['mags'] = spec['mags']*0.
for filename in files:
restored = np.load(filename)
spec['spectra'] += restored['spec']['spectra']
try:
flux = 10.**(-0.4*(restored['spec']['mags']-np.log10(3631.)))
except:
import pdb ; pdb.set_trace()
flux[np.where(restored['spec']['mags'] == 0.)] = 0.
spec['mags'] += flux
spec['mags'] = -2.5*np.log10(spec['mags'])+np.log10(3631.)
np.savez(os.path.join(outDir,'mergedSpec.npz'), spec=spec, wave=wave, filterWave=temp['filterWave'])
def setDefaults(self):
"""Settings appropriate for most or all ap_pipe runs.
"""
# Always prefer decorrelation; may eventually become ImageDifferenceTask default
self.differencer.doDecorrelation = True
self.differencer.detection.thresholdValue = 5.0 # needed with doDecorrelation
# Don't have source catalogs for templates
self.differencer.doSelectSources = False
# make sure the db schema and config is set up for ap_association.
self.ppdb.dia_object_index = "baseline"
self.ppdb.dia_object_columns = []
self.ppdb.extra_schema_file = os.path.join(
getPackageDir("ap_association"), "data",
"ppdb-ap-pipe-schema-extra.yaml")
def read_fields():
"""
Read in the old Field coordinates
Returns
-------
numpy.array
With RA and dec in radians.
"""
names = ['id', 'RA', 'dec']
types = [int, float, float]
data_dir = os.path.join(getPackageDir('sims_featureScheduler'), 'python/lsst/sims/featureScheduler/')
filepath = os.path.join(data_dir, 'fieldID.lis')
fields = np.loadtxt(filepath, dtype=list(zip(names, types)))
fields['RA'] = np.radians(fields['RA'])
fields['dec'] = np.radians(fields['dec'])
return fields
def __init__(self, start_time, cloud_db=None, offset_year=0, scale=1e6):
self.cloud_db = cloud_db
if self.cloud_db is None:
self.cloud_db = os.path.join(getPackageDir('sims_cloudModel'),
'data', 'cloud.db')
# Cloud database starts in Jan 01 of the year of the start of the simulation.
year_start = start_time.datetime.year + offset_year
self.start_time = Time('%d-01-01' % year_start,
format='isot',
scale='tai')
self.cloud_dates = None
self.cloud_values = None
self.scale = scale
self.read_data()
def get_cartoon_disk_mags(self):
if not hasattr(self, 'cartoonBandpassDict'):
bandpassNames = ['u', 'g', 'r', 'i', 'z']
bandpassDir = getPackageDir('sims_photUtils')
bandpassDir = os.path.join(bandpassDir, 'tests',
'cartoonSedTestData')
self.cartoonBandpassDict = BandpassDict.loadTotalBandpassesFromFiles(
bandpassNames,
bandpassDir=bandpassDir,
bandpassRoot='test_bandpass_')
return self._quiescentMagnitudeGetter(
'disk', self.cartoonBandpassDict,
self.get_cartoon_disk_mags._colnames)
def __init__(self):
# XXX--need to move this out of featureScheduler!!!
path = getPackageDir('sims_speedObservatory')
datafile = os.path.join(
path, 'python/lsst/sims/speedObservatory/pre_slewtimes.npz')
data = np.load(datafile)
alt_array = data['alt_array'].copy()
az_array = data['az_array'].copy()
altitudes = np.radians(data['altitudes'].copy())
azimuths = np.radians(data['azimuths'].copy())
data.close()
# Generate interpolation objects
self.alt_interpolator = RectBivariateSpline(altitudes, altitudes,
alt_array)
self.az_interpolator = RectBivariateSpline(azimuths, azimuths,
az_array)
def testSSMSchema(self):
"""
Create a PhoSim InstanceCatalog of point sources (stars) formatted by the
PhoSimCatalogSSM class. Verify that the schema of the actual objects conforms
to what PhoSim expects, as defined here
https://bitbucket.org/phosim/phosim_release/wiki/Instance%20Catalog
"""
cat = PhoSimCatalogSSM(self.starDB, obs_metadata=self.obs_metadata)
cat.phoSimHeaderMap = test_header_map
cat_name = os.path.join(getPackageDir('sims_catUtils'), 'tests',
'scratchSpace', 'phosim_ssm_schema_cat.txt')
if os.path.exists(cat_name):
os.unlink(cat_name)
cat.write_catalog(cat_name)
with open(cat_name, 'r') as input_file:
cat_lines = input_file.readlines()
n_obj = 0
for line in cat_lines:
params = line.split()
if len(params) > 2:
n_obj += 1
self.assertEqual(len(params), 15)
self.assertEqual(params[0], 'object')
self.assertEqual(round(float(params[1])), float(params[1]),
10) # id
float(params[2]) # ra
float(params[3]) # dec
float(params[4]) # mag norm
self.assertIn('starSED', params[5]) # sed name
self.assertAlmostEqual(float(params[6]), 0.0, 10) # redshift
self.assertAlmostEqual(float(params[7]), 0.0, 10) # gamma1
self.assertAlmostEqual(float(params[8]), 0.0, 10) # gamma2
self.assertAlmostEqual(float(params[9]), 0.0, 10) # kappa
self.assertAlmostEqual(float(params[10]), 0.0, 10) # delta_ra
self.assertAlmostEqual(float(params[11]), 0.0, 10) # delta_dec
self.assertEqual(params[12], 'point') # source type
self.assertEqual(params[13], 'none') # internal dust
self.assertEqual(params[14], 'none') # Milky Way dust
self.assertGreater(n_obj, 0)
if os.path.exists(cat_name):
os.unlink(cat_name)
def testKolmogorovGaussianPSF(self):
scratchDir = tempfile.mkdtemp(prefix='testKolmogorovGaussianPSF',
dir=ROOT)
catName = os.path.join(scratchDir,
'kolmogorov_gaussian_test_Catalog.dat')
imageRoot = os.path.join(scratchDir, 'kolmogorov_gaussian_test_Image')
dbFileName = os.path.join(scratchDir,
'kolmogorov_gaussian_test_InputCatalog.dat')
baseDir = os.path.join(getPackageDir('sims_GalSimInterface'), 'tests',
'cameraData')
# instantiate a test camera with pixel_scale = 0.02 arcsec/pixel
camera = ReturnCamera(baseDir)
detector = camera[0]
detName = detector.getName()
imageName = '%s_%s_u.fits' % (imageRoot, detName)
obs = ObservationMetaData(pointingRA=75.0,
pointingDec=-12.0,
boundType='circle',
boundLength=4.0,
rotSkyPos=33.0,
mjd=49250.0)
create_text_catalog(obs,
dbFileName,
np.array([3.0]),
np.array([1.0]),
mag_norm=[13.0])
db = fwhmFileDBObj(dbFileName, runtable='test')
cat = fwhmCat(db, obs_metadata=obs)
cat.camera_wrapper = GalSimCameraWrapper(camera)
psf = Kolmogorov_and_Gaussian_PSF(rawSeeing=0.7,
airmass=1.05,
band='g')
cat.setPSF(psf)
cat.write_catalog(catName)
cat.write_images(nameRoot=imageRoot)
if os.path.exists(scratchDir):
shutil.rmtree(scratchDir)
def matchesToCatalog(matches, matchMeta):
"""Denormalise matches into a Catalog of "unpacked matches"
@param[in] matches unpacked matches, i.e. a list of Match objects whose schema
has "first" and "second" attributes which, resepectively, contain the
reference and source catalog entries, and a "distance" field (the
measured distance between the reference and source objects)
@param[in] matchMeta metadata for matches (must have .add attribute)
@return lsst.afw.table.BaseCatalog of matches (with ref_ and src_ prefix identifiers
for referece and source entries, respectively)
"""
if len(matches) == 0:
raise RuntimeError("No matches provided.")
refSchema = matches[0].first.getSchema()
srcSchema = matches[0].second.getSchema()
mergedSchema = makeMergedSchema(refSchema, Schema(), targetPrefix="ref_")
mergedSchema = makeMergedSchema(srcSchema,
mergedSchema,
targetPrefix="src_")
distKey = mergedSchema.addField("distance",
type=np.float64,
doc="Distance between ref and src")
mergedCatalog = BaseCatalog(mergedSchema)
copyIntoCatalog([m.first for m in matches],
mergedCatalog,
sourceSchema=refSchema,
targetPrefix="ref_")
copyIntoCatalog([m.second for m in matches],
mergedCatalog,
sourceSchema=srcSchema,
targetPrefix="src_")
for m, r in zip(matches, mergedCatalog):
r.set(distKey, m.distance)
# obtain reference catalog name if one is setup
try:
catalogName = os.path.basename(getPackageDir("astrometry_net_data"))
except pexExcept.NotFoundError:
catalogName = "NOT_SET"
matchMeta.add("REFCAT", catalogName)
mergedCatalog.getTable().setMetadata(matchMeta)
return mergedCatalog
def testIncompletDB(self):
"""
Test that if the mock OpSim database does not have all required columns, an exception
is raised.
"""
scratch_dir = os.path.join(getPackageDir('sims_catUtils'), 'tests',
'scratchSpace')
opsim_db_name = os.path.join(scratch_dir,
'incomplete_mock_opsim_sqlite.db')
if os.path.exists(opsim_db_name):
os.unlink(opsim_db_name)
conn = sqlite3.connect(opsim_db_name)
c = conn.cursor()
c.execute('''CREATE TABLE Summary (obsHistID int, expMJD real, '''
'''fieldRA real, filter text)''')
conn.commit()
rng = np.random.RandomState(77)
n_pointings = 100
ra_data = rng.random_sample(n_pointings) * 2.0 * np.pi
mjd_data = rng.random_sample(n_pointings) * 1000.0 + 59580.0
filter_dexes = rng.randint(0, 6, n_pointings)
bands = ('u', 'g', 'r', 'i', 'z', 'y')
filter_data = []
for ii in filter_dexes:
filter_data.append(bands[ii])
for ii in range(n_pointings):
cmd = '''INSERT INTO Summary VALUES(%i, %f, %f, '%s')''' % \
(ii, mjd_data[ii], ra_data[ii], filter_data[ii])
c.execute(cmd)
conn.commit()
conn.close()
incomplete_obs_gen = ObservationMetaDataGenerator(
database=opsim_db_name)
with self.assertRaises(RuntimeError) as context:
incomplete_obs_gen.getObservationMetaData(telescopeFilter='r')
self.assertIn(
"ObservationMetaDataGenerator requires that the database",
context.exception.args[0])
if os.path.exists(opsim_db_name):
os.unlink(opsim_db_name)
def setUpClass(cls):
cls._tile_radius = angularSeparation(0.0, 0.0, 2.0, 2.0)
dir_name = os.path.join(getPackageDir('sims_catUtils'), 'tests')
cls._tmpdir = tempfile.mkdtemp(dir=dir_name)
cls._temp_gal_db = os.path.join(cls._tmpdir, 'test_galaxies.db')
ra_min = -2.25
cls._ra_min = ra_min
ra_max = 2.251
cls._d_ra = 0.05
ra_grid = np.arange(ra_min, ra_max, cls._d_ra)
dec_grid = np.arange(ra_min, ra_max, cls._d_ra)
print('raw grid %d' % len(ra_grid))
ra_dec_mesh = np.meshgrid(ra_grid, dec_grid)
ra_grid = ra_dec_mesh[0].flatten()
dec_grid = ra_dec_mesh[1].flatten()
# add a very small offset so that numerical precision
# does not foul things up on the tile boundaries
rng = np.random.RandomState(7163)
ra_grid += 1.0e-5 * (rng.random_sample(len(ra_grid)) - 0.5)
dec_grid += 1.0e-5 * (rng.random_sample(len(dec_grid)) - 0.5)
galtag = (100 * np.round(45 + ra_grid / cls._d_ra) +
np.round(45 + dec_grid / cls._d_ra)).astype(int)
assert len(galtag) == len(np.unique(galtag))
htmid_grid = htm.findHtmid(ra_grid, dec_grid, 21)
print('got htmid %d' % len(htmid_grid))
print(htm.levelFromHtmid(htmid_grid.min()))
print(htm.levelFromHtmid(htmid_grid.max()))
assert htm.levelFromHtmid(htmid_grid.min()) == 21
assert htm.levelFromHtmid(htmid_grid.max()) == 21
gid = np.arange(len(ra_grid), dtype=int)
assert len(galtag) == len(np.unique(galtag))
print(ra_grid.max(), ra_grid.min())
with sqlite3.connect(cls._temp_gal_db) as conn:
c = conn.cursor()
query = '''CREATE TABLE galaxy(htmid int, id int,
galid text, ra real, dec real, galtag int, redshift)'''
c.execute(query).fetchall()
values = ((int(hh), int(ii), str(ii), r, d, int(g), 0.5)
for hh, ii, r, d, g in zip(htmid_grid, gid, ra_grid,
dec_grid, galtag))
c.executemany('INSERT INTO galaxy VALUES (?,?,?,?,?,?,?)', values)
class ImageSourceTestCase(unittest.TestCase):
"TestCase class for ImageSource."
imsim_dir = lsstUtils.getPackageDir('imsim')
eimage_file = os.path.join(imsim_dir, 'tests', 'data',
'lsst_e_161899_R22_S11_r.fits.gz')
image_source \
= desc.imsim.ImageSource.create_from_eimage(eimage_file, 'R22_S11')
def setUp(self):
imsim_dir = lsstUtils.getPackageDir('imsim')
self.eimage_file = os.path.join(imsim_dir, 'tests', 'data',
'lsst_e_161899_R22_S11_r.fits.gz')
self.image_source \
= desc.imsim.ImageSource.create_from_eimage(self.eimage_file,
'R22_S11')
def tearDown(self):
pass
def test_create_from_eimage(self):
"Test the .create_from_eimage static method."
self.assertAlmostEqual(self.image_source.exptime, 30.)
self.assertTupleEqual(self.image_source.eimage_data.shape,
(4072, 4000))
self.assertTupleEqual(
self.image_source.amp_images['R22_S11_C00'].getArray().shape,
(2048, 544))
def test_get_amplifier_hdu(self):
"Test the .get_amplifier_hdu method."
hdu = self.image_source.get_amplifier_hdu('R22_S11_C10')
self.assertEqual(hdu.header['DATASEC'], "[4:512,1:2000]")
self.assertEqual(hdu.header['DETSEC'], "[509:1,1:2000]")
hdu = self.image_source.get_amplifier_hdu('R22_S11_C17')
self.assertEqual(hdu.header['DATASEC'], "[4:512,1:2000]")
self.assertEqual(hdu.header['DETSEC'], "[4072:3564,1:2000]")
def test_raw_file_headers(self):
"Test contents of raw file headers."
outfile = 'raw_file_test.fits'
self.image_source.write_fits_file(outfile)
with fits.open(outfile) as hdus:
self.assertEqual(hdus[0].header['IMSIMVER'],
desc.imsim.__version__)
os.remove(outfile)
def testPrerequisiteLookupFunction(self):
"""This tests that a lookup function defined on a prerequisite input
is called when building a quantum graph.
"""
butler = dafButler.Butler(
os.path.join(getPackageDir("ci_hsc_gen3"), "DATA", "butler.yaml"))
pipeline = pipeBase.Pipeline("Test LookupFunction Pipeline")
pipeline.addTask(LookupTestPipelineTask, "test")
graphBuilder = pipeBase.GraphBuilder(butler.registry)
graph = graphBuilder.makeGraph(pipeline, ["HSC/calib"], None, None)
outputs = list(graph)
# verify the graph contains no datasetRefs for brighter fatter kernels
# instead of the datasetRefs that exist in the registry.
numberOfInputs = len(outputs[0].quantum.inputs['bfKernel'])
self.assertEqual(numberOfInputs, 0)
def setUp(self):
self.butler = Butler(os.path.join(getPackageDir("ci_hsc_gen3"),
"DATA"),
instrument="HSC",
skymap="discrete/ci_hsc",
writeable=False,
collections=["HSC/runs/ci_hsc"])
self._num_exposures = len(DATA_IDS)
self._num_exposures_good_templates = 29
self._num_visits = len({data_id["visit"] for data_id in DATA_IDS})
self._num_tracts = 1
self._num_patches = 1
self._num_bands = len(
{data_id["physical_filter"]
for data_id in DATA_IDS})
self._min_sources = 100
def get_magnitudes(self):
"""
Example photometry getter for alternative (i.e. non-LSST) bandpasses
"""
idNames = self.column_by_name('id')
columnNames = [name for name in self.get_magnitudes._colnames]
bandpassNames = ['u', 'g', 'r', 'i', 'z']
bandpassDir = os.path.join(getPackageDir('sims_photUtils'), 'tests',
'cartoonSedTestData')
if not hasattr(self, 'cartoonBandpassDict'):
self.cartoonBandpassDict = BandpassDict.loadTotalBandpassesFromFiles(
bandpassNames,
bandpassDir=bandpassDir,
bandpassRoot='test_bandpass_')
output = self._quiescentMagnitudeGetter(self.cartoonBandpassDict,
self.get_magnitudes._colnames)
#############################################################################
#Everything below this comment exists solely for the purposes of the unit test
#if you need to write a customized getter for photometry that uses non-LSST
#bandpasses, you only need to emulate the code above this comment.
magNormList = self.column_by_name('magNorm')
sedNames = self.column_by_name('sedFilename')
av = self.column_by_name('galacticAv')
#the two variables below will allow us to get at the SED and magnitude
#data from within the unit test class, so that we can be sure
#that the mixin loaded the correct bandpasses
sublist = SedList(sedNames, magNormList, galacticAvList=av)
for ss in sublist:
self.sedMasterList.append(ss)
if len(output) > 0:
for i in range(len(output[0])):
subList = []
for j in range(len(output)):
subList.append(output[j][i])
self.magnitudeMasterList.append(subList)
return output
def testNoise(self):
"""
Test that ExampleCCDNoise puts the expected counts on an image
by generating a flat image, adding noise and background to it,
and calculating the variance of counts in the image.
"""
lsstDefaults = LSSTdefaults()
gain = 2.5
readnoise = 6.0
photParams = PhotometricParameters(gain=gain, readnoise=readnoise)
img = galsim.Image(100, 100)
noise = ExampleCCDNoise(seed=42)
m5 = 24.5
bandpass = Bandpass()
bandpass.readThroughput(os.path.join(getPackageDir('throughputs'),
'baseline', 'total_r.dat'))
background = calcSkyCountsPerPixelForM5(m5, bandpass, FWHMeff=lsstDefaults.FWHMeff('r'),
photParams=photParams)
noisyImage = noise.addNoiseAndBackground(img, bandpass, m5=m5,
FWHMeff=lsstDefaults.FWHMeff('r'),
photParams=photParams)
mean = 0.0
var = 0.0
for ix in range(1, 101):
for iy in range(1, 101):
mean += noisyImage(ix, iy)
mean = mean/10000.0
for ix in range(1, 101):
for iy in range(1, 101):
var += (noisyImage(ix, iy) - mean)*(noisyImage(ix, iy) - mean)
var = var/9999.0
varElectrons = background*gain + readnoise
varADU = varElectrons/(gain*gain)
msg = 'background %e mean %e ' % (background, mean)
self.assertLess(np.abs(background/mean - 1.0), 0.05, msg=msg)
msg = 'var %e varADU %e ; ratio %e ; background %e' % (var, varADU, var/varADU, background)
self.assertLess(np.abs(var/varADU - 1.0), 0.05, msg=msg)
def makeCasuWfcam():
path = os.path.join(getPackageDir("obs_ukirt"), "casuWfcam", "camera")
for detNum, (gain, sat) in enumerate(zip(GAIN, SATURATION), 1):
det = AmpInfoCatalog(AmpInfoTable.makeMinimalSchema())
amp = det.addNew()
amp.setName(str(detNum))
amp.setBBox(Box2I())
amp.setGain(gain)
amp.setReadNoise(READNOISE)
amp.setSaturation(sat)
# amp.setSuspectLevel(float("nan"))
amp.setReadoutCorner(LL)
amp.setLinearityType("none")
amp.setHasRawInfo(False)
det.writeFits(os.path.join(path, "%d.fits" % (detNum, )))
def testLoadTables(self):
"Test Read-in of IGM Lookup Tables"
tableDirectory = str(
getPackageDir('sims_catUtils') +
'/python/lsst/sims/catUtils/IGM/igm_tables')
# First make sure that if variance option is turned on but
# there are no variance files that the correct error is raised
testIGM = ApplyIGM()
testIGM.initializeIGM(zMax=1.5)
with gzip.open('MeanLookupTable_zSource1.5.tbl.gz',
'wt') as testMeanLookupTable:
testMeanLookupTable.write('300.0 0.9999\n')
self.assertRaisesRegex(IOError, "Cannot find variance tables.",
testIGM.loadTables, os.getcwd())
os.remove('MeanLookupTable_zSource1.5.tbl.gz')
# Then make sure that the mean lookup tables and var lookup
# tables all get loaded into proper dicts
testIGMDicts = ApplyIGM()
testIGMDicts.initializeIGM()
testIGMDicts.loadTables(tableDirectory)
redshiftValues = [
'1.5', '1.6', '1.7', '1.8', '1.9', '2.0', '2.1', '2.2', '2.3',
'2.4', '2.5', '2.6', '2.7', '2.8', '2.9'
]
# Python 3 replaces assertItemsEqual() with assertCountEqual()
if hasattr(self, 'assertItemsEqual'):
self.assertItemsEqual(list(testIGMDicts.meanLookups.keys()),
redshiftValues)
self.assertItemsEqual(list(testIGMDicts.varLookups.keys()),
redshiftValues)
else:
self.assertCountEqual(list(testIGMDicts.meanLookups.keys()),
redshiftValues)
self.assertCountEqual(list(testIGMDicts.varLookups.keys()),
redshiftValues)
# Finally make sure that if Variance Boolean is false that
# nothing is passed in to varLookups
testIGMVar = ApplyIGM()
testIGMVar.initializeIGM()
testIGMVar.loadTables(tableDirectory, varianceTbl=False)
self.assertEqual(testIGMVar.varLookups, {})
def setUp(self):
baseDir = os.path.join(getPackageDir('sims_GalSimInterface'), 'tests',
'cameraData')
self.camera = ReturnCamera(baseDir)
ra = 145.0
dec = -73.0
self.epoch = 2000.0
mjd = 49250.0
rotSkyPos = 45.0
self.obs = ObservationMetaData(pointingRA=ra,
pointingDec=dec,
boundType='circle',
boundLength=1.0,
mjd=mjd,
rotSkyPos=rotSkyPos)
def _data_file_name(basename, module_name):
"""Return path name of a data file.
Parameters
----------
basename : `str`
Name of the file to add to the path string.
module_name : `str`
Name of lsst stack package environment variable.
Returns
-------
data_file_path : `str`
Full path of the file to load from the "data" directory in a given
repository.
"""
return os.path.join(getPackageDir(module_name), "data", basename)
def testParallax(self):
"""
This test will output a catalog of ICRS and observed positions.
It will also output the quantities (proper motion, radial velocity,
and parallax) needed to apply the transformaiton between the two.
It will then run the catalog through PALPY and verify that the catalog
generating code correctly applied the transformations.
"""
# create and write a catalog that performs astrometric transformations
# on a cartoon star database
cat = parallaxTestCatalog(self.starDBObject, obs_metadata=self.obs_metadata)
parallaxName = os.path.join(getPackageDir('sims_catUtils'), 'tests',
'scratchSpace', 'parallaxCatalog.sav')
if os.path.exists(parallaxName):
os.unlink(parallaxName)
cat.write_catalog(parallaxName)
data = np.genfromtxt(parallaxName, delimiter=',')
self.assertGreater(len(data), 0)
epoch = cat.db_obj.epoch
mjd = cat.obs_metadata.mjd
prms = pal.mappa(epoch, mjd.TDB)
for vv in data:
# run the PALPY routines that actuall do astrometry `by hand' and compare
# the results to the contents of the catalog
ra0 = np.radians(vv[0])
dec0 = np.radians(vv[1])
pmra = np.radians(vv[4])
pmdec = np.radians(vv[5])
rv = vv[6]
px = vv[7]
ra_apparent, dec_apparent = pal.mapqk(ra0, dec0, pmra, pmdec, px, rv, prms)
ra_apparent = np.array([ra_apparent])
dec_apparent = np.array([dec_apparent])
raObserved, decObserved = _observedFromAppGeo(ra_apparent, dec_apparent,
obs_metadata=cat.obs_metadata)
self.assertAlmostEqual(raObserved[0], np.radians(vv[2]), 7)
self.assertAlmostEqual(decObserved[0], np.radians(vv[3]), 7)
if os.path.exists(parallaxName):
os.unlink(parallaxName)
def setUpClass(cls):
if _USE_LSST_CAMERA:
cls.camera_wrapper = LSSTCameraWrapper()
cls.detector = cls.camera_wrapper.camera['R:1,1 S:2,2']
else:
baseDir = os.path.join(getPackageDir('sims_GalSimInterface'),
'tests', 'cameraData')
cls.camera_wrapper = GalSimCameraWrapper(ReturnCamera(baseDir))
cls.detector = cls.camera_wrapper.camera[0]
cls.obs = ObservationMetaData(pointingRA=25.0,
pointingDec=-10.0,
boundType='circle',
boundLength=1.0,
mjd=49250.0,
rotSkyPos=0.0)
cls.epoch = 2000.0
def setUp(self):
try:
self.datadir = getPackageDir("testdata_decam")
except pexExcept.NotFoundError:
message = "testdata_decam not setup. Skipping."
warnings.warn(message)
raise unittest.SkipTest(message)
self.outPath = tempfile.mkdtemp() if OutputName is None else OutputName
self.dataId = {'visit': 229388, 'ccdnum': 1}
argsList = [os.path.join(self.datadir, "rawData"), "--output", self.outPath, "--id"]
argsList += ["%s=%s" % (key, val) for key, val in self.dataId.iteritems()]
argsList += ["--config", "calibrate.doPhotoCal=False", "calibrate.doAstrometry=False",
# Temporary until DM-4232 is fixed.
"isr.assembleCcd.setGain=False"]
fullResult = ProcessCcdTask.parseAndRun(args=argsList, doReturnResults=True)
self.butler = fullResult.parsedCmd.butler
self.config = fullResult.parsedCmd.config
def applyDummy(self, valid_dexes, params, expmjd):
if len(params) == 0:
return np.array([[], [], [], [], [], []])
dtype = np.dtype([('t', float), ('du', float), ('dg', float),
('dr', float), ('di', float), ('dz', float),
('dy', float)])
dmag = np.zeros((6, self.num_variable_obj(params)))
for ix in valid_dexes[0]:
lc_name = os.path.join(getPackageDir('sims_catUtils'), 'tests',
'scratchSpace', params['lc'][ix])
lc_data = np.genfromtxt(lc_name, dtype=dtype)
dmag[0][ix] = np.interp(expmjd, lc_data['t'], lc_data['du'])
dmag[1][ix] = np.interp(expmjd, lc_data['t'], lc_data['dg'])
dmag[2][ix] = np.interp(expmjd, lc_data['t'], lc_data['dr'])
dmag[3][ix] = np.interp(expmjd, lc_data['t'], lc_data['di'])
dmag[4][ix] = np.interp(expmjd, lc_data['t'], lc_data['dz'])
dmag[5][ix] = np.interp(expmjd, lc_data['t'], lc_data['dy'])
return dmag
def test_flare_lc_failure(self):
"""
Test that the correct exception is thrown when you try
to interpolate from an MLT light curve cache that does not
exist
"""
cat_name = os.path.join(getPackageDir('sims_catUtils'), 'tests',
'scratchSpace', 'flaring_cat_failure.txt')
db = MLT_test_DB(database=self.db_name, driver='sqlite')
obs = ObservationMetaData(mjd=67432.1)
flare_cat = FlaringCatalog(db, obs_metadata=obs)
flare_cat._mlt_lc_file = 'a_nonexistent_cache'
with self.assertRaises(RuntimeError) as context:
flare_cat.write_catalog(cat_name)
self.assertIn('get_mdwarf_flares.sh', context.exception.args[0])
if os.path.exists(cat_name):
os.unlink(cat_name)
class testFakeSedCatalog(testFakeBandpassCatalog):
"""
tests the GalSim interface on fake seds and bandpasses
"""
sedDir = os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testSeds')
def get_sedFilepath(self):
"""
map the sedFilenames created by makePhoSimTestDB to the SEDs in
in testSeds/
"""
nameMap = {'km20_5750.fits_g40_5790': 'fakeSed1.dat',
'm2.0Full.dat': 'fakeSed2.dat',
'bergeron_6500_85.dat_6700': 'fakeSed3.dat'}
rawNames = self.column_by_name('sedFilename')
return np.array([nameMap[nn] for nn in rawNames])
def setUp(self):
self.butler = Butler(os.path.join(getPackageDir("ci_hsc_gen3"), "DATA"), writeable=False,
collections=["HSC/calib/2013-06-17", "HSC/runs/ci_hsc"])
# We need to provide a physical_filter value to fully identify a flat,
# but this still leaves the band as an implied value that this data ID
# doesn't know.
self.flatMinimalDataId = DataCoordinate.standardize(
instrument="HSC", detector=0, physical_filter="HSC-R",
universe=self.butler.registry.dimensions,
)
# For a calexp, the minimal data ID just has exposure and detector,
# so both band and physical_filter are implied and not known here.
self.calexpMinimalDataId = DataCoordinate.standardize(
instrument="HSC", detector=100, visit=903334,
universe=self.butler.registry.dimensions,
)
# Parameters with bbox to test that logic still works on subimage gets.
self.parameters = {"bbox": Box2I(Point2I(0, 0), Point2I(8, 7))}
def __init__(self,
start_time,
scheduled_downtime_db=None,
start_of_night_offset=-0.34):
self.scheduled_downtime_db = scheduled_downtime_db
if self.scheduled_downtime_db is None:
self.scheduled_downtime_db = os.path.join(
getPackageDir('sims_downtimeModel'), 'data',
'scheduled_downtime.db')
# downtime database starts in Jan 01 of the year of the start of the simulation.
year_start = start_time.datetime.year
self.night0 = Time('%d-01-01' % year_start, format='isot',
scale='tai') + start_of_night_offset
# Scheduled downtime data is a np.ndarray of start / end / activity for each scheduled downtime.
self.downtime = None
self.read_data()
def testObservedRaDec(self):
"""
Test that the mixins provided in Astrometry SSM really do convert ICRS RA, Dec
into observed RA, Dec
"""
catName = os.path.join(getPackageDir('sims_catUtils'), 'tests',
'scratchSpace', 'ssmAstrometryCat.txt')
dtype = np.dtype([('id', np.int), ('raObserved', np.float),
('decObserved', np.float)])
controlData = np.genfromtxt(self.dbFile, dtype=self.dtype)
rng = np.random.RandomState(42)
nTests = 5
raList = rng.random_sample(nTests) * 2.0 * np.pi
decList = (rng.random_sample(nTests) - 0.5) * np.pi
mjdList = rng.random_sample(nTests) * 5000.0 + 53850.0
for raPointing, decPointing, mjd in zip(raList, decList, mjdList):
obs = ObservationMetaData(pointingRA=raPointing,
pointingDec=decPointing,
mjd=mjd)
cat = SSM_astrometryCat(self.astDB, obs_metadata=obs)
cat.write_catalog(catName)
testData = np.genfromtxt(catName, dtype=dtype, delimiter=',')
self.assertGreater(len(testData), 0)
raObservedControl, decObservedControl = _observedFromICRS(
controlData['raJ2000'],
controlData['decJ2000'],
obs_metadata=obs,
epoch=2000.0,
includeRefraction=True)
np.testing.assert_array_almost_equal(raObservedControl,
testData['raObserved'], 10)
np.testing.assert_array_almost_equal(decObservedControl,
testData['decObserved'], 10)
if os.path.exists(catName):
os.unlink(catName)
def big_sky_dust(
nside=32,
weights={
'u': [0.31, 0.15, False],
'g': [0.44, 0.15],
'r': [1., 0.3],
'i': [1., 0.3],
'z': [0.9, 0.3],
'y': [0.9, 0.3, False]
},
dust_limit=0.19):
"""
Based on the Olsen et al Cadence White Paper
"""
ebvDataDir = getPackageDir('sims_maps')
filename = 'DustMaps/dust_nside_%i.npz' % nside
dustmap = np.load(os.path.join(ebvDataDir, filename))['ebvMap']
# wfd covers -72.25 < dec < 12.4. Avoid galactic plane |b| > 15 deg
wfd_north = np.radians(12.4)
wfd_south = np.radians(-72.25)
full_north = np.radians(30.)
ra, dec = ra_dec_hp_map(nside=nside)
total_map = np.zeros(ra.size)
# let's make a first pass here
total_map[np.where(dec < full_north)] = 1e-6
total_map[np.where((dec > wfd_south) & (dec < wfd_north)
& (dustmap < dust_limit))] = 1.
# Now let's break it down by filter
result = {}
for key in weights:
result[key] = total_map + 0.
result[key][np.where(result[key] == 1)] = weights[key][0]
result[key][np.where(result[key] == 1e-6)] = weights[key][1]
if len(weights[key]) == 3:
result[key][np.where(dec > wfd_north)] = 0.
return result
def get_sdss_agn_mags(self):
"""
An example getter for SDSS AGN magnitudes
"""
# load a BandpassDict of SDSS bandpasses, if not done already
if not hasattr(self, 'sdssBandpassDict'):
bandpassNames = ['u', 'g', 'r', 'i', 'z']
bandpassDir = os.path.join(getPackageDir('throughputs'), 'sdss')
bandpassRoot = 'sdss_'
self.sdssBandpassDict = BandpassDict.loadTotalBandpassesFromFiles(
bandpassNames,
bandpassRoot=bandpassRoot,
bandpassDir=bandpassDir)
# actually calculate the magnitudes
return self._magnitudeGetter('agn', self.sdssBandpassDict,
self.get_sdss_agn_mags._colnames)
