def test_pedantic(self):
with self.assertRaises(ValueError):
_parse_sec_keyword('blat')
#- should log a warning about large readnoise
rawimage = self.rawimage + np.random.normal(scale=2,
size=self.rawimage.shape)
image = preproc(rawimage, self.header)
#- should log an error about huge readnoise
rawimage = self.rawimage + np.random.normal(scale=10,
size=self.rawimage.shape)
image = preproc(rawimage, self.header)
#- should log a warning about small readnoise
rdnoise = 0.7 * np.mean(self.rdnoise.values())
rawimage = np.random.normal(scale=rdnoise, size=self.rawimage.shape)
image = preproc(rawimage, self.header)
#- should log a warning about tiny readnoise
rdnoise = 0.01 * np.mean(self.rdnoise.values())
rawimage = np.random.normal(scale=rdnoise, size=self.rawimage.shape)
image = preproc(rawimage, self.header)
#- Missing expected RDNOISE keywords shouldn't be fatal
hdr = self.header.copy()
del hdr['RDNOISE1']
del hdr['RDNOISE2']
del hdr['RDNOISE3']
del hdr['RDNOISE4']
image = preproc(self.rawimage, hdr)
#- Missing expected GAIN keywords should log error but not crash
hdr = self.header.copy()
del hdr['GAIN1']
del hdr['GAIN2']
del hdr['GAIN3']
del hdr['GAIN4']
image = preproc(self.rawimage, hdr)
def test_pedantic(self):
with self.assertRaises(ValueError):
_parse_sec_keyword('blat')
#- should log a warning about large readnoise
rawimage = self.rawimage + np.random.normal(scale=2, size=self.rawimage.shape)
image = preproc(rawimage, self.header)
#- should log an error about huge readnoise
rawimage = self.rawimage + np.random.normal(scale=10, size=self.rawimage.shape)
image = preproc(rawimage, self.header)
#- should log a warning about small readnoise
rdnoise = 0.7 * np.mean(self.rdnoise.values())
rawimage = np.random.normal(scale=rdnoise, size=self.rawimage.shape)
image = preproc(rawimage, self.header)
#- should log a warning about tiny readnoise
rdnoise = 0.01 * np.mean(self.rdnoise.values())
rawimage = np.random.normal(scale=rdnoise, size=self.rawimage.shape)
image = preproc(rawimage, self.header)
#- Missing expected RDNOISE keywords shouldn't be fatal
hdr = self.header.copy()
del hdr['RDNOISE1']
del hdr['RDNOISE2']
del hdr['RDNOISE3']
del hdr['RDNOISE4']
image = preproc(self.rawimage, hdr)
#- Missing expected GAIN keywords should log error but not crash
hdr = self.header.copy()
del hdr['GAIN1']
del hdr['GAIN2']
del hdr['GAIN3']
del hdr['GAIN4']
image = preproc(self.rawimage, hdr)
def run_qa(self,
image,
camera,
expid,
paname=None,
amps=False,
url=None,
qafig=None):
retval = {}
retval["EXPID"] = expid
retval["ARM"] = camera[0]
retval["SPECTROGRAPH"] = int(camera[1])
retval["PANAME"] = paname
t = datetime.datetime.now()
retval["MJD"] = Time(t).mjd
bias_overscan = []
for kk in ['1', '2', '3', '4']:
from desispec.preproc import _parse_sec_keyword
sel = _parse_sec_keyword(image.meta['BIASSEC' + kk])
pix = image.pix[sel]
#- Compute statistics of the bias region that only reject
# the 0.5% of smallest and largest values. (from sdssproc)
isort = np.sort(pix.ravel())
nn = isort.shape[0]
bias = np.mean(isort[long(0.005 * nn):long(0.995 * nn)])
bias_overscan.append(bias)
bias = np.mean(bias_overscan)
if amps:
bias_amps = np.array(bias_overscan)
retval["VALUE"] = {'BIAS': bias, 'BIAS_AMP': bias_amps}
else:
retval["VALUE"] = {'BIAS': bias}
#- http post if needed
if url is not None:
try:
import requests
response = requests.get(url)
#- Check if the api has json
api = response.json()
#- proceed with post
job = {"name": "QL", "status": 0, "dictionary": retval}
response = requests.post(api['job'],
json=job,
auth=("username", "password"))
except:
log.info("Skipping HTTP post...")
if qafig is not None:
from desispec.qa.qa_plots_ql import plot_bias_overscan
plot_bias_overscan(retval, qafig)
log.info("Output QA fig %s" % qafig)
return retval
def run_qa(self,
image,
camera,
expid,
paname=None,
amps=False,
url=None,
qafig=None):
retval = {}
retval["EXPID"] = expid
retval["ARM"] = camera[0]
retval["SPECTROGRAPH"] = int(camera[1])
retval["PANAME"] = paname
t = datetime.datetime.now()
retval["MJD"] = Time(t).mjd
rmsccd = getrms(
image.pix
) #- should we add dark current and/or readnoise to this as well?
if amps:
rms_amps = []
#- get amp boundary in pixels
from desispec.preproc import _parse_sec_keyword
for kk in ['1', '2', '3', '4']:
ampboundary = _parse_sec_keyword(image.meta["CCDSEC" + kk])
rmsamp = getrms(image.pix[ampboundary])
rms_amps.append(rmsamp)
retval["VALUE"] = {"RMS": rmsccd, "RMS_AMP": np.array(rms_amps)}
else:
retval["VALUE"] = {"RMS": rmsccd}
if url is not None:
try:
import requests
response = requests.get(url)
#- Check if the api has json
api = response.json()
#- proceed with post
job = {
"name": "QL",
"status": 0,
"dictionary": retval
} #- QLF should disintegrate dictionary
response = requests.post(
api['job'], json=job, auth=("username", "password")
) #- username, password not real but placeholder here.
except:
log.info("Skipping HTTP post...")
if qafig is not None:
from desispec.qa.qa_plots_ql import plot_RMS
plot_RMS(retval, qafig)
log.info("Output QA fig %s" % qafig)
return retval
def run_qa(self,image,camera,expid,paname=None,amps=False,url=None,qafig=None):
retval={}
retval["EXPID"]=expid
retval["ARM"]=camera[0]
retval["SPECTROGRAPH"]=int(camera[1])
retval["PANAME"]=paname
#retval["TIMESTAMP"]='{:%Y-%b-%d %H:%M:%S}'.format(datetime.datetime.now())
t=datetime.datetime.now()
retval["MJD"]=Time(t).mjd
#- get the counts over entire CCD
npix3sig=countpix(image.pix,nsig=3) #- above 3 sigma
npix100=countpix(image.pix,ncounts=100) #- above 100 pixel count
npix500=countpix(image.pix,ncounts=500) #- above 500 pixel count
#- get the counts for each amp
if amps:
npix3sig_amps=[]
npix100_amps=[]
npix500_amps=[]
#- get amp boundary in pixels
from desispec.preproc import _parse_sec_keyword
for kk in ['1','2','3','4']:
ampboundary=_parse_sec_keyword(image.meta["CCDSEC"+kk])
npix3sig=countpix(image.pix[ampboundary],nsig=3)
npix3sig_amps.append(npix3sig)
npix100=countpix(image.pix[ampboundary],ncounts=100)
npix100_amps.append(npix100)
npix500=countpix(image.pix[ampboundary],ncounts=500)
npix500_amps.append(npix500)
retval["VALUE"]={"NPIX3SIG":npix3sig,"NPIX100":npix100,"NPIX500":npix500, "NPIX3SIG_AMP": npix3sig_amps, "NPIX100_AMP": npix100_amps,"NPIX500_AMP": npix500_amps}
else:
retval["VALUE"]={"NPIX3SIG":npix3sig,"NPIX100":npix100,"NPIX500":npix500}
if url is not None:
try:
import requests
response=requests.get(url)
#- Check if the api has json
api=response.json()
#- proceed with post
job={"name":"QL","status":0,"dictionary":retval} #- QLF should disintegrate dictionary
response=requests.post(api['job'],json=job,auth=("username","password")) #- username, password not real but placeholder here.
except:
log.info("Skipping HTTP post...")
if qafig is not None:
from desispec.qa.qa_plots_ql import plot_countpix
plot_countpix(retval,qafig)
log.info("Output QA fig %s"%qafig)
return retval
def ampregion(image):
"""
Get the pixel boundary regions for amps
Args:
image: desispec.image.Image object
"""
pixboundary = []
for kk in ['1', '2', '3', '4']: #- 4 amps
#- get the amp region in pix
ampboundary = _parse_sec_keyword(image.meta["CCDSEC" + kk])
pixboundary.append(ampboundary)
return pixboundary
def ampregion(image):
"""
get the pixel boundary regions for amps
args: image: desispec.image.Image object
"""
from desispec.preproc import _parse_sec_keyword
pixboundary=[]
for kk in ['1','2','3','4']: #- 4 amps
#- get the amp region in pix
ampboundary=_parse_sec_keyword(image.meta["CCDSEC"+kk])
pixboundary.append(ampboundary)
return pixboundary
def run_qa(self,image,camera,expid,paname=None,amps=False,url=None,qafig=None):
retval={}
retval["EXPID"]=expid
retval["ARM"]=camera[0]
retval["SPECTROGRAPH"]=int(camera[1])
retval["PANAME"]=paname
t=datetime.datetime.now()
retval["MJD"]=Time(t).mjd
bias_overscan=[]
for kk in ['1','2','3','4']:
from desispec.preproc import _parse_sec_keyword
sel=_parse_sec_keyword(image.meta['BIASSEC'+kk])
pix=image.pix[sel]
#- Compute statistics of the bias region that only reject
# the 0.5% of smallest and largest values. (from sdssproc)
isort=np.sort(pix.ravel())
nn=isort.shape[0]
bias=np.mean(isort[long(0.005*nn) : long(0.995*nn)])
bias_overscan.append(bias)
bias=np.mean(bias_overscan)
if amps:
bias_amps=np.array(bias_overscan)
retval["VALUE"]={'BIAS':bias,'BIAS_AMP':bias_amps}
else:
retval["VALUE"]={'BIAS':bias}
#- http post if needed
if url is not None:
try:
import requests
response=requests.get(url)
#- Check if the api has json
api=response.json()
#- proceed with post
job={"name":"QL","status":0,"dictionary":retval}
response=requests.post(api['job'],json=job,auth=("username","password"))
except:
log.info("Skipping HTTP post...")
if qafig is not None:
from desispec.qa.qa_plots_ql import plot_bias_overscan
plot_bias_overscan(retval,qafig)
log.info("Output QA fig %s"%qafig)
return retval
def run_qa(self,image,camera,expid,paname=None,amps=False,url=None,qafig=None):
retval={}
retval["EXPID"]=expid
retval["ARM"]=camera[0]
retval["SPECTROGRAPH"]=int(camera[1])
retval["PANAME"]=paname
t=datetime.datetime.now()
retval["MJD"]=Time(t).mjd
rmsccd=getrms(image.pix) #- should we add dark current and/or readnoise to this as well?
if amps:
rms_amps=[]
#- get amp boundary in pixels
from desispec.preproc import _parse_sec_keyword
for kk in ['1','2','3','4']:
ampboundary=_parse_sec_keyword(image.meta["CCDSEC"+kk])
rmsamp=getrms(image.pix[ampboundary])
rms_amps.append(rmsamp)
retval["VALUE"]={"RMS":rmsccd,"RMS_AMP":np.array(rms_amps)}
else:
retval["VALUE"]={"RMS":rmsccd}
if url is not None:
try:
import requests
response=requests.get(url)
#- Check if the api has json
api=response.json()
#- proceed with post
job={"name":"QL","status":0,"dictionary":retval} #- QLF should disintegrate dictionary
response=requests.post(api['job'],json=job,auth=("username","password")) #- username, password not real but placeholder here.
except:
log.info("Skipping HTTP post...")
if qafig is not None:
from desispec.qa.qa_plots_ql import plot_RMS
plot_RMS(retval,qafig)
log.info("Output QA fig %s"%qafig)
return retval
def fiducialregion(frame,psf):
"""
get the fiducial amplifier regions on the CCD pixel to fiber by wavelength space
args: frame: desispec.frame.Frame object
psf: desispec.psf.PSF like object
"""
from desispec.preproc import _parse_sec_keyword
startspec=0 #- will be None if don't have fibers on the right of the CCD.
endspec=499 #- will be None if don't have fibers on the right of the CCD
startwave0=0 #- lower index for the starting fiber
startwave1=0 #- lower index for the last fiber for the amp region
endwave0=frame.wave.shape[0] #- upper index for the starting fiber
endwave1=frame.wave.shape[0] #- upper index for the last fiber for that amp
pixboundary=[]
fidboundary=[]
for kk in ['1','2','3','4']: #- 4 amps
#- get the amp region in pix
ampboundary=_parse_sec_keyword(frame.meta["CCDSEC"+kk])
pixboundary.append(ampboundary)
for ispec in range(frame.flux.shape[0]):
if np.all(psf.x(ispec) > ampboundary[1].start):
startspec=ispec
#-cutting off wavelenth boundaries from startspec
yy=psf.y(ispec,frame.wave)
k=np.where(yy > ampboundary[0].start)[0]
startwave0=k[0]
yy=psf.y(ispec,frame.wave)
k=np.where(yy < ampboundary[0].stop)[0]
endwave0=k[-1]
break
else:
startspec=None
startwave0=None
endwave0=None
if startspec is not None:
for ispec in range(frame.flux.shape[0])[::-1]:
if np.all(psf.x(ispec) < ampboundary[1].stop):
endspec=ispec
#-cutting off wavelenth boundaries from startspec
yy=psf.y(ispec,frame.wave)
k=np.where(yy > ampboundary[0].start)[0]
startwave1=k[0]
yy=psf.y(ispec,frame.wave)
k=np.where(yy < ampboundary[0].stop)[0]
endwave1=k[-1]
break
else:
endspec=None
startwave1=None
endwave1=None
startwave=max(startwave0,startwave1)
endwave=min(endwave0,endwave1)
if endspec is not None:
endspec+=1 #- last entry exclusive in slice, so add 1
endwave+=1
fiducialb=(slice(startspec,endspec,None),slice(startwave,endwave,None)) #- Note: y,x --> spec, wavelength
fidboundary.append(fiducialb)
return pixboundary,fidboundary
def run_qa(self,
image,
camera,
expid,
paname=None,
amps=False,
url=None,
qafig=None):
retval = {}
retval["EXPID"] = expid
retval["ARM"] = camera[0]
retval["SPECTROGRAPH"] = int(camera[1])
retval["PANAME"] = paname
#retval["TIMESTAMP"]='{:%Y-%b-%d %H:%M:%S}'.format(datetime.datetime.now())
t = datetime.datetime.now()
retval["MJD"] = Time(t).mjd
#- get the counts over entire CCD
npix3sig = countpix(image.pix, nsig=3) #- above 3 sigma
npix100 = countpix(image.pix, ncounts=100) #- above 100 pixel count
npix500 = countpix(image.pix, ncounts=500) #- above 500 pixel count
#- get the counts for each amp
if amps:
npix3sig_amps = []
npix100_amps = []
npix500_amps = []
#- get amp boundary in pixels
from desispec.preproc import _parse_sec_keyword
for kk in ['1', '2', '3', '4']:
ampboundary = _parse_sec_keyword(image.meta["CCDSEC" + kk])
npix3sig = countpix(image.pix[ampboundary], nsig=3)
npix3sig_amps.append(npix3sig)
npix100 = countpix(image.pix[ampboundary], ncounts=100)
npix100_amps.append(npix100)
npix500 = countpix(image.pix[ampboundary], ncounts=500)
npix500_amps.append(npix500)
retval["VALUE"] = {
"NPIX3SIG": npix3sig,
"NPIX100": npix100,
"NPIX500": npix500,
"NPIX3SIG_AMP": npix3sig_amps,
"NPIX100_AMP": npix100_amps,
"NPIX500_AMP": npix500_amps
}
else:
retval["VALUE"] = {
"NPIX3SIG": npix3sig,
"NPIX100": npix100,
"NPIX500": npix500
}
if url is not None:
try:
import requests
response = requests.get(url)
#- Check if the api has json
api = response.json()
#- proceed with post
job = {
"name": "QL",
"status": 0,
"dictionary": retval
} #- QLF should disintegrate dictionary
response = requests.post(
api['job'], json=job, auth=("username", "password")
) #- username, password not real but placeholder here.
except:
log.info("Skipping HTTP post...")
if qafig is not None:
from desispec.qa.qa_plots_ql import plot_countpix
plot_countpix(retval, qafig)
log.info("Output QA fig %s" % qafig)
return retval
def fiducialregion(frame, psf):
"""
Get the fiducial amplifier regions on the CCD pixel to fiber by wavelength space
Args:
frame: desispec.frame.Frame object
psf: desispec.psf.PSF like object
"""
from desispec.preproc import _parse_sec_keyword
startspec = 0 #- will be None if don't have fibers on the right of the CCD.
endspec = 499 #- will be None if don't have fibers on the right of the CCD
startwave0 = 0 #- lower index for the starting fiber
startwave1 = 0 #- lower index for the last fiber for the amp region
endwave0 = frame.wave.shape[0] #- upper index for the starting fiber
endwave1 = frame.wave.shape[
0] #- upper index for the last fiber for that amp
pixboundary = []
fidboundary = []
#- Adding the min, max boundary individually for the benefit of dumping to yaml.
leftmax = 499 #- for amp 1 and 3
rightmin = 0 #- for amp 2 and 4
bottommax = frame.wave.shape[0] #- for amp 1 and 2
topmin = 0 #- for amp 3 and 4
#- Loop over each amp
for kk in ['1', '2', '3', '4']: #- 4 amps
#- get the amp region in pix
ampboundary = _parse_sec_keyword(frame.meta["CCDSEC" + kk])
pixboundary.append(ampboundary)
for ispec in range(frame.flux.shape[0]):
if np.all(psf.x(ispec) > ampboundary[1].start):
startspec = ispec
#-cutting off wavelenth boundaries from startspec
yy = psf.y(ispec, frame.wave)
k = np.where(yy > ampboundary[0].start)[0]
startwave0 = k[0]
yy = psf.y(ispec, frame.wave)
k = np.where(yy < ampboundary[0].stop)[0]
endwave0 = k[-1]
break
else:
startspec = None
startwave0 = None
endwave0 = None
if startspec is not None:
for ispec in range(frame.flux.shape[0])[::-1]:
if np.all(psf.x(ispec) < ampboundary[1].stop):
endspec = ispec
#-cutting off wavelenth boundaries from startspec
yy = psf.y(ispec, frame.wave)
k = np.where(yy > ampboundary[0].start)[0]
startwave1 = k[0]
yy = psf.y(ispec, frame.wave)
k = np.where(yy < ampboundary[0].stop)[0]
endwave1 = k[-1]
break
else:
endspec = None
startwave1 = None
endwave1 = None
if startwave0 is not None and startwave1 is not None:
startwave = max(startwave0, startwave1)
else:
startwave = None
if endwave0 is not None and endwave1 is not None:
endwave = min(endwave0, endwave1)
else:
endwave = None
if endspec is not None:
#endspec+=1 #- last entry exclusive in slice, so add 1
#endwave+=1
if endspec < leftmax:
leftmax = endspec
if startspec > rightmin:
rightmin = startspec
if endwave < bottommax:
bottommax = endwave
if startwave > topmin:
topmin = startwave
else:
rightmin = 0 #- Only if no spec in right side of CCD. passing 0 to encertain valid data type. Nontype throws a type error in yaml.dump.
#fiducialb=(slice(startspec,endspec,None),slice(startwave,endwave,None)) #- Note: y,x --> spec, wavelength
#fidboundary.append(fiducialb)
#- return pixboundary,fidboundary
return leftmax, rightmin, bottommax, topmin
def fiducialregion(frame, psf):
"""
get the fiducial amplifier regions on the CCD pixel to fiber by wavelength space
args: frame: desispec.frame.Frame object
psf: desispec.psf.PSF like object
"""
from desispec.preproc import _parse_sec_keyword
startspec = 0 #- will be None if don't have fibers on the right of the CCD.
endspec = 499 #- will be None if don't have fibers on the right of the CCD
startwave0 = 0 #- lower index for the starting fiber
startwave1 = 0 #- lower index for the last fiber for the amp region
endwave0 = frame.wave.shape[0] #- upper index for the starting fiber
endwave1 = frame.wave.shape[
0] #- upper index for the last fiber for that amp
pixboundary = []
fidboundary = []
for kk in ['1', '2', '3', '4']: #- 4 amps
#- get the amp region in pix
ampboundary = _parse_sec_keyword(frame.meta["CCDSEC" + kk])
pixboundary.append(ampboundary)
for ispec in range(frame.flux.shape[0]):
if np.all(psf.x(ispec) > ampboundary[1].start):
startspec = ispec
#-cutting off wavelenth boundaries from startspec
yy = psf.y(ispec, frame.wave)
k = np.where(yy > ampboundary[0].start)[0]
startwave0 = k[0]
yy = psf.y(ispec, frame.wave)
k = np.where(yy < ampboundary[0].stop)[0]
endwave0 = k[-1]
break
else:
startspec = None
startwave0 = None
endwave0 = None
if startspec is not None:
for ispec in range(frame.flux.shape[0])[::-1]:
if np.all(psf.x(ispec) < ampboundary[1].stop):
endspec = ispec
#-cutting off wavelenth boundaries from startspec
yy = psf.y(ispec, frame.wave)
k = np.where(yy > ampboundary[0].start)[0]
startwave1 = k[0]
yy = psf.y(ispec, frame.wave)
k = np.where(yy < ampboundary[0].stop)[0]
endwave1 = k[-1]
break
else:
endspec = None
startwave1 = None
endwave1 = None
startwave = max(startwave0, startwave1)
endwave = min(endwave0, endwave1)
if endspec is not None:
endspec += 1 #- last entry exclusive in slice, so add 1
endwave += 1
fiducialb = (slice(startspec, endspec, None),
slice(startwave, endwave,
None)) #- Note: y,x --> spec, wavelength
fidboundary.append(fiducialb)
return pixboundary, fidboundary
def setUp(self):
#- Create temporary calib directory
self.testDir = os.path.join(os.environ['HOME'], 'ql_test_io')
calibDir = os.path.join(self.testDir, 'ql_calib')
if not os.path.exists(calibDir): os.makedirs(calibDir)
#- Generate calib data
for camera in ['b0', 'r0', 'z0']:
#- Fiberflat has to exist but can be a dummpy file
filename = '{}/fiberflat-{}.fits'.format(calibDir, camera)
fx = open(filename, 'w')
fx.write('fiberflat file')
fx.close()
#- PSF has to be real file
psffile = '{}/psf-{}.fits'.format(calibDir, camera)
example_psf = resource_filename(
'desispec', 'test/data/ql/psf-{}.fits'.format(camera))
shutil.copy(example_psf, psffile)
#- Copy test calibration-data.yaml file
specdir = calibDir + "spec/sp0"
if not os.path.isdir(specdir):
os.makedirs(specdir)
for c in "brz":
shutil.copy(
resource_filename('desispec',
'test/data/ql/{}0.yaml'.format(c)),
os.path.join(specdir, "{}0.yaml".format(c)))
#- Set calibration environment variable
os.environ['DESI_SPECTRO_CALIB'] = calibDir
self.rawfile = os.path.join(self.testDir, 'test-raw-abcd.fits')
self.pixfile = os.path.join(self.testDir, 'test-pix-abcd.fits')
self.config = {}
#- rawimage
hdr = dict()
hdr['CAMERA'] = 'b0'
hdr['DATE-OBS'] = '2018-09-23T08:17:03.988'
#- Dimensions per amp, not full 4-quad CCD
ny = self.ny = 500
nx = self.nx = 400
noverscan = nover = 50
hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx + nover])
hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx + nover:nx + 2 * nover])
hdr['DATASEC2'] = xy2hdr(np.s_[0:ny,
nx + 2 * nover:nx + 2 * nover + nx])
hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx + nx])
hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nover])
hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx])
hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx])
hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + nover:nx + 2 * nover])
hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny + ny,
nx + 2 * nover:nx + 2 * nover + nx])
hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nx])
hdr['NIGHT'] = '20180923'
hdr['EXPID'] = 1
hdr['FLAVOR'] = 'dark'
rawimage = np.zeros((2 * ny, 2 * nx + 2 * noverscan))
offset = {'1': 100.0, '2': 100.5, '3': 50.3, '4': 200.4}
gain = {'1': 1.0, '2': 1.5, '3': 0.8, '4': 1.2}
rdnoise = {'1': 2.0, '2': 2.2, '3': 2.4, '4': 2.6}
quad = {
'1': np.s_[0:ny, 0:nx],
'2': np.s_[0:ny, nx:nx + nx],
'3': np.s_[ny:ny + ny, 0:nx],
'4': np.s_[ny:ny + ny, nx:nx + nx],
}
for amp in ('1', '2', '3', '4'):
hdr['GAIN' + amp] = gain[amp]
hdr['RDNOISE' + amp] = rdnoise[amp]
xy = _parse_sec_keyword(hdr['BIASSEC' + amp])
shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start]
rawimage[xy] += offset[amp]
rawimage[xy] += np.random.normal(scale=rdnoise[amp],
size=shape) / gain[amp]
xy = _parse_sec_keyword(hdr['DATASEC' + amp])
shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start]
rawimage[xy] += offset[amp]
rawimage[xy] += np.random.normal(scale=rdnoise[amp],
size=shape) / gain[amp]
#- raw data are integers, not floats
rawimg = rawimage.astype(np.int32)
self.expid = hdr["EXPID"]
self.camera = hdr["CAMERA"]
#- Confirm that all regions were correctly offset
assert not np.any(rawimage == 0.0)
hdr['DOSVER'] = 'SIM'
hdr['FEEVER'] = 'SIM'
hdr['DETECTOR'] = 'SIM'
desispec.io.write_raw(self.rawfile, rawimg, hdr, camera=self.camera)
self.rawimage = fits.open(self.rawfile)
def setUp(self):
self.calibfile = 'test-calib-askjapqwhezcpasehadfaqp.fits'
self.rawfile = 'test-raw-askjapqwhezcpasehadfaqp.fits'
self.pixfile = 'test-pix-askjapqwhezcpasehadfaqp.fits'
hdr = dict()
hdr['CAMERA'] = 'b0'
hdr['DATE-OBS'] = '2018-09-23T08:17:03.988'
#- [x,y] 1-indexed for FITS; in reality the amps will be symmetric
#- but the header definitions don't require that to make sure we are
#- getting dimensions correct
#- Dimensions per amp, not full 4-quad CCD
self.ny = ny = 500
self.nx = nx = 400
self.noverscan = nover = 50
#- BIASSEC = overscan region in raw image
#- DATASEC = data region in raw image
#- CCDSEC = where should this go in output
hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx+nover])
hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx+nover:nx+2*nover])
hdr['DATASEC2'] = xy2hdr(np.s_[0:ny, nx+2*nover:nx+2*nover+nx])
hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx+nx])
hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny+ny, nx:nx+nover])
hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny+ny, 0:nx])
hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny+ny, 0:nx])
hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny+ny, nx+nover:nx+2*nover])
hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny+ny, nx+2*nover:nx+2*nover+nx])
hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny+ny, nx:nx+nx])
hdr['NIGHT'] = '20150102'
hdr['EXPID'] = 1
self.header = hdr
self.rawimage = np.zeros((2*self.ny, 2*self.nx+2*self.noverscan))
self.offset = {'1':100.0, '2':100.5, '3':50.3, '4':200.4}
self.gain = {'1':1.0, '2':1.5, '3':0.8, '4':1.2}
self.rdnoise = {'1':2.0, '2':2.2, '3':2.4, '4':2.6}
self.quad = {
'1': np.s_[0:ny, 0:nx], '2': np.s_[0:ny, nx:nx+nx],
'3': np.s_[ny:ny+ny, 0:nx], '4': np.s_[ny:ny+ny, nx:nx+nx],
}
for amp in ('1', '2', '3', '4'):
self.header['GAIN'+amp] = self.gain[amp]
self.header['RDNOISE'+amp] = self.rdnoise[amp]
xy = _parse_sec_keyword(hdr['BIASSEC'+amp])
shape = [xy[0].stop-xy[0].start, xy[1].stop-xy[1].start]
self.rawimage[xy] += self.offset[amp]
self.rawimage[xy] += np.random.normal(scale=self.rdnoise[amp], size=shape)/self.gain[amp]
xy = _parse_sec_keyword(hdr['DATASEC'+amp])
shape = [xy[0].stop-xy[0].start, xy[1].stop-xy[1].start]
self.rawimage[xy] += self.offset[amp]
self.rawimage[xy] += np.random.normal(scale=self.rdnoise[amp], size=shape)/self.gain[amp]
#- raw data are integers, not floats
self.rawimage = self.rawimage.astype(np.int32)
#- Confirm that all regions were correctly offset
assert not np.any(self.rawimage == 0.0)
def setUp(self):
#- Create temporary calib directory
self.calibdir = os.path.join(os.environ['HOME'], 'preproc_unit_test')
if not os.path.exists(self.calibdir): os.makedirs(self.calibdir)
#- Copy test calibration-data.yaml file
specdir = os.path.join(self.calibdir, "spec/sp0")
if not os.path.isdir(specdir):
os.makedirs(specdir)
for c in "brz":
shutil.copy(
resource_filename('desispec',
'test/data/ql/{}0.yaml'.format(c)),
os.path.join(specdir, "{}0.yaml".format(c)))
#- Set calibration environment variable
os.environ["DESI_SPECTRO_CALIB"] = self.calibdir
self.calibfile = os.path.join(
self.calibdir, 'test-calib-askjapqwhezcpasehadfaqp.fits')
self.rawfile = os.path.join(self.calibdir,
'test-raw-askjapqwhezcpasehadfaqp.fits')
self.pixfile = os.path.join(self.calibdir,
'test-pix-askjapqwhezcpasehadfaqp.fits')
primary_hdr = dict()
primary_hdr['DATE-OBS'] = '2018-09-23T08:17:03.988'
primary_hdr['DOSVER'] = 'SIM' # ICS version
hdr = dict()
hdr['CAMERA'] = 'b0'
hdr['DETECTOR'] = 'SIM' # CCD chip identifier
hdr['FEEVER'] = 'SIM' # readout electronic
#- [x,y] 1-indexed for FITS; in reality the amps will be symmetric
#- but the header definitions don't require that to make sure we are
#- getting dimensions correct
#- Dimensions per amp, not full 4-quad CCD
self.ny = ny = 500
self.nx = nx = 400
self.noverscan = nover = 50
#- BIASSEC = overscan region in raw image
#- DATASEC = data region in raw image
#- CCDSEC = where should this go in output
hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx + nover])
hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx + nover:nx + 2 * nover])
hdr['DATASEC2'] = xy2hdr(np.s_[0:ny,
nx + 2 * nover:nx + 2 * nover + nx])
hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx + nx])
hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nover])
hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx])
hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx])
hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + nover:nx + 2 * nover])
hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny + ny,
nx + 2 * nover:nx + 2 * nover + nx])
hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nx])
hdr['NIGHT'] = '20150102'
hdr['EXPID'] = 1
# add to header the minimal set of keywords needed to
# identify the config in the ccd_calibration.yaml file
self.primary_header = primary_hdr
self.header = hdr
self.rawimage = np.zeros(
(2 * self.ny, 2 * self.nx + 2 * self.noverscan))
self.offset = {'1': 100.0, '2': 100.5, '3': 50.3, '4': 200.4}
self.gain = {'1': 1.0, '2': 1.5, '3': 0.8, '4': 1.2}
self.rdnoise = {'1': 2.0, '2': 2.2, '3': 2.4, '4': 2.6}
self.quad = {
'1': np.s_[0:ny, 0:nx],
'2': np.s_[0:ny, nx:nx + nx],
'3': np.s_[ny:ny + ny, 0:nx],
'4': np.s_[ny:ny + ny, nx:nx + nx],
}
for amp in ('1', '2', '3', '4'):
self.header['GAIN' + amp] = self.gain[amp]
self.header['RDNOISE' + amp] = self.rdnoise[amp]
xy = _parse_sec_keyword(hdr['BIASSEC' + amp])
shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start]
self.rawimage[xy] += self.offset[amp]
self.rawimage[xy] += np.random.normal(scale=self.rdnoise[amp],
size=shape) / self.gain[amp]
xy = _parse_sec_keyword(hdr['DATASEC' + amp])
shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start]
self.rawimage[xy] += self.offset[amp]
self.rawimage[xy] += np.random.normal(scale=self.rdnoise[amp],
size=shape) / self.gain[amp]
#- raw data are integers, not floats
self.rawimage = self.rawimage.astype(np.int32)
#- Confirm that all regions were correctly offset
assert not np.any(self.rawimage == 0.0)
def compute(args):
nbNeigh = args.nb_neighbours
cameras = args.cameras
sections = args.sections
assert len(args.infiles)%2. == 0.
### Where to save
dicCor = {}
for c in cameras:
dicCor[c] = {}
for i1,sec1 in enumerate(sections):
for sec2 in sections[:i1+1]:
dicCor[c][sec1+'_'+sec2] = sp.zeros( (2*nbNeigh+1,2*nbNeigh+1) )
### Compute
for p in range(int(len(args.infiles)//2.)):
p1 = args.infiles[2*p]
p2 = args.infiles[2*p+1]
h1 = fitsio.FITS(p1)
h2 = fitsio.FITS(p2)
for c in cameras:
### Read data
head1 = h1[c].read_header()
data1 = h1[c].read()
head2 = h2[c].read_header()
data2 = h2[c].read()
dic = {}
for sec in sections:
w1 = _parse_sec_keyword(head1['DATASEC'+sec])
w2 = _parse_sec_keyword(head2['DATASEC'+sec])
td1 = data1[w1].astype('float64')-data2[w2].astype('float64')
std1 = sp.std(td1)
if sec=='B':
td1 = td1[:,::-1]
elif sec=='C':
td1 = td1[::-1,:]
elif sec=='D':
td1 = td1[::-1,::-1]
mask = (td1<-5.*std1) | (td1>5.*std1)
td1 -= td1[~mask].mean()
td1[mask] = 0.
dic[sec] = td1.copy()
### Compute correlation coef
for i1,sec1 in enumerate(dic.keys()):
for sec2 in list(dic.keys())[:i1+1]:
print(p, c, sec1,sec2)
dicCor[c][sec1+'_'+sec2] += compute_correlation(dic[sec1],dic[sec2],nbNeigh)
h1.close()
h2.close()
### Write to ascii
for c in cameras:
for i1,sec1 in enumerate(sections):
for sec2 in sections[:i1+1]:
dicCor[c][sec1+'_'+sec2] /= len(args.infiles)/2.
sp.savetxt(args.outfile+'/correlation_{}_{}_{}.txt'.format(c,sec1,sec2),dicCor[c][sec1+'_'+sec2])
return
def testCalcXWSigma(self):
qa = QA.Calc_XWSigma('xwsigma', self.config)
#- create larger rawimage to incorporate sky peaks
hdr = dict()
hdr['CAMERA'] = 'z1'
hdr['DATE-OBS'] = '2018-09-23T08:17:03.988'
#- Dimensions per amp
ny = self.ny = 1800
nx = self.nx = 1800
noverscan = nover = 50
hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx + nover])
hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx + nover:nx + 2 * nover])
hdr['DATASEC2'] = xy2hdr(np.s_[0:ny,
nx + 2 * nover:nx + 2 * nover + nx])
hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx + nx])
hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nover])
hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx])
hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx])
hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + nover:nx + 2 * nover])
hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny + ny,
nx + 2 * nover:nx + 2 * nover + nx])
hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nx])
hdr['NIGHT'] = '20180923'
hdr['EXPID'] = 1
hdr['PROGRAM'] = 'dark'
hdr['FLAVOR'] = 'science'
rawimage = np.zeros((2 * ny, 2 * nx + 2 * noverscan))
offset = {'1': 100.0, '2': 100.5, '3': 50.3, '4': 200.4}
gain = {'1': 1.0, '2': 1.5, '3': 0.8, '4': 1.2}
rdnoise = {'1': 2.0, '2': 2.2, '3': 2.4, '4': 2.6}
quad = {
'1': np.s_[0:ny, 0:nx],
'2': np.s_[0:ny, nx:nx + nx],
'3': np.s_[ny:ny + ny, 0:nx],
'4': np.s_[ny:ny + ny, nx:nx + nx],
}
for amp in ('1', '2', '3', '4'):
hdr['GAIN' + amp] = gain[amp]
hdr['RDNOISE' + amp] = rdnoise[amp]
xy = _parse_sec_keyword(hdr['BIASSEC' + amp])
shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start]
rawimage[xy] += offset[amp]
rawimage[xy] += np.random.normal(scale=rdnoise[amp],
size=shape) / gain[amp]
xy = _parse_sec_keyword(hdr['DATASEC' + amp])
shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start]
rawimage[xy] += offset[amp]
rawimage[xy] += np.random.normal(scale=rdnoise[amp],
size=shape) / gain[amp]
#- set CCD parameters
self.ccdsec1 = hdr["CCDSEC1"]
self.ccdsec2 = hdr["CCDSEC2"]
self.ccdsec3 = hdr["CCDSEC3"]
self.ccdsec4 = hdr["CCDSEC4"]
#- raw data are integers, not floats
rawimg = rawimage.astype(np.int32)
self.expid = hdr["EXPID"]
self.camera = hdr["CAMERA"]
#- read psf, should use specter.PSF.load_psf instead of desispec.PSF(), otherwise need to create a psfboot somewhere.
psf = self.psf
#- make the test pixfile, fibermap file
img_pix = rawimg
img_ivar = np.ones_like(img_pix) / 3.0**2
img_mask = np.zeros(img_pix.shape, dtype=np.uint32)
img_mask[200] = 1
#- manually insert sky peaks for xwsigma
zpeaks = np.array([8401.5, 8432.4, 8467.5, 9479.4, 9505.6, 9521.8])
fibers = np.arange(5)
peak1 = psf.xy(fibers, zpeaks[0])
pix1 = np.rint(peak1)
peak2 = psf.xy(fibers, zpeaks[1])
pix2 = np.rint(peak2)
peak3 = psf.xy(fibers, zpeaks[2])
pix3 = np.rint(peak3)
peak4 = psf.xy(fibers, zpeaks[3])
pix4 = np.rint(peak4)
peak5 = psf.xy(fibers, zpeaks[4])
pix5 = np.rint(peak5)
peak6 = psf.xy(fibers, zpeaks[5])
pix6 = np.rint(peak6)
for i in range(len(fibers)):
img_pix[pix1[1][i]][pix1[0][i]] = 10000.
img_pix[pix1[1][i] + 1][pix1[0][i]] = 5000.
img_pix[pix1[1][i]][pix1[0][i] + 1] = 5000.
img_pix[pix1[1][i] - 1][pix1[0][i]] = 5000.
img_pix[pix1[1][i]][pix1[0][i] - 1] = 5000.
img_pix[pix1[1][i] + 2][pix1[0][i]] = 500.
img_pix[pix1[1][i]][pix1[0][i] + 2] = 500.
img_pix[pix1[1][i] - 2][pix1[0][i]] = 500.
img_pix[pix1[1][i]][pix1[0][i] - 2] = 500.
img_pix[pix1[1][i] + 3][pix1[0][i]] = 220.
img_pix[pix1[1][i]][pix1[0][i] + 3] = 220.
img_pix[pix1[1][i] - 3][pix1[0][i]] = 220.
img_pix[pix1[1][i]][pix1[0][i] - 3] = 220.
img_pix[pix2[1][i]][pix2[0][i]] = 10000.
img_pix[pix2[1][i] + 1][pix2[0][i]] = 5000.
img_pix[pix2[1][i]][pix2[0][i] + 1] = 5000.
img_pix[pix2[1][i] - 1][pix2[0][i]] = 5000.
img_pix[pix2[1][i]][pix2[0][i] - 1] = 5000.
img_pix[pix2[1][i] + 2][pix2[0][i]] = 500.
img_pix[pix2[1][i]][pix2[0][i] + 2] = 500.
img_pix[pix2[1][i] - 2][pix2[0][i]] = 500.
img_pix[pix2[1][i]][pix2[0][i] - 2] = 500.
img_pix[pix2[1][i] + 3][pix2[0][i]] = 220.
img_pix[pix2[1][i]][pix2[0][i] + 3] = 220.
img_pix[pix2[1][i] - 3][pix2[0][i]] = 220.
img_pix[pix2[1][i]][pix2[0][i] - 3] = 220.
img_pix[pix3[1][i]][pix3[0][i]] = 10000.
img_pix[pix3[1][i] + 1][pix3[0][i]] = 5000.
img_pix[pix3[1][i]][pix3[0][i] + 1] = 5000.
img_pix[pix3[1][i] - 1][pix3[0][i]] = 5000.
img_pix[pix3[1][i]][pix3[0][i] - 1] = 5000.
img_pix[pix3[1][i] + 2][pix3[0][i]] = 500.
img_pix[pix3[1][i]][pix3[0][i] + 2] = 500.
img_pix[pix3[1][i] - 2][pix3[0][i]] = 500.
img_pix[pix3[1][i]][pix3[0][i] - 2] = 500.
img_pix[pix3[1][i] + 3][pix3[0][i]] = 220.
img_pix[pix3[1][i]][pix3[0][i] + 3] = 220.
img_pix[pix3[1][i] - 3][pix3[0][i]] = 220.
img_pix[pix3[1][i]][pix3[0][i] - 3] = 220.
img_pix[pix4[1][i]][pix4[0][i]] = 10000.
img_pix[pix4[1][i] + 1][pix4[0][i]] = 5000.
img_pix[pix4[1][i]][pix4[0][i] + 1] = 5000.
img_pix[pix4[1][i] - 1][pix4[0][i]] = 5000.
img_pix[pix4[1][i]][pix4[0][i] - 1] = 5000.
img_pix[pix4[1][i] + 2][pix4[0][i]] = 500.
img_pix[pix4[1][i]][pix4[0][i] + 2] = 500.
img_pix[pix4[1][i] - 2][pix4[0][i]] = 500.
img_pix[pix4[1][i]][pix4[0][i] - 2] = 500.
img_pix[pix4[1][i] + 3][pix4[0][i]] = 220.
img_pix[pix4[1][i]][pix4[0][i] + 3] = 220.
img_pix[pix4[1][i] - 3][pix4[0][i]] = 220.
img_pix[pix4[1][i]][pix4[0][i] - 3] = 220.
img_pix[pix5[1][i]][pix5[0][i]] = 10000.
img_pix[pix5[1][i] + 1][pix5[0][i]] = 5000.
img_pix[pix5[1][i]][pix5[0][i] + 1] = 5000.
img_pix[pix5[1][i] - 1][pix5[0][i]] = 5000.
img_pix[pix5[1][i]][pix5[0][i] - 1] = 5000.
img_pix[pix5[1][i] + 2][pix5[0][i]] = 500.
img_pix[pix5[1][i]][pix5[0][i] + 2] = 500.
img_pix[pix5[1][i] - 2][pix5[0][i]] = 500.
img_pix[pix5[1][i]][pix5[0][i] - 2] = 500.
img_pix[pix5[1][i] + 3][pix5[0][i]] = 220.
img_pix[pix5[1][i]][pix5[0][i] + 3] = 220.
img_pix[pix5[1][i] - 3][pix5[0][i]] = 220.
img_pix[pix5[1][i]][pix5[0][i] - 3] = 220.
img_pix[pix6[1][i]][pix6[0][i]] = 10000.
img_pix[pix6[1][i] + 1][pix6[0][i]] = 5000.
img_pix[pix6[1][i]][pix6[0][i] + 1] = 5000.
img_pix[pix6[1][i] - 1][pix6[0][i]] = 5000.
img_pix[pix6[1][i]][pix6[0][i] - 1] = 5000.
img_pix[pix6[1][i] + 2][pix6[0][i]] = 500.
img_pix[pix6[1][i]][pix6[0][i] + 2] = 500.
img_pix[pix6[1][i] - 2][pix6[0][i]] = 500.
img_pix[pix6[1][i]][pix6[0][i] - 2] = 500.
img_pix[pix6[1][i] + 3][pix6[0][i]] = 220.
img_pix[pix6[1][i]][pix6[0][i] + 3] = 220.
img_pix[pix6[1][i] - 3][pix6[0][i]] = 220.
img_pix[pix6[1][i]][pix6[0][i] - 3] = 220.
self.image = desispec.image.Image(img_pix,
img_ivar,
img_mask,
camera='z1',
meta=hdr)
desispec.io.write_image(self.pixfile, self.image)
self.fibermap = desispec.io.empty_fibermap(5)
desispec.io.write_fibermap(self.fibermapfile, self.fibermap)
inp = self.image
qargs = {}
qargs["PSFFile"] = self.psf
qargs["FiberMap"] = self.fibermap
qargs["camera"] = self.camera
qargs["expid"] = self.expid
qargs["amps"] = False
qargs["paname"] = "abc"
resl = qa(inp, **qargs)
self.assertTrue(np.all(resl["METRICS"]["XSIGMA"]) > 0)
def setUp(self):
self.rawfile = 'test-raw-abcd.fits'
self.pixfile = 'test-pix-abcd.fits'
self.xwfile = 'test-xw-abcd.fits'
self.framefile = 'test-frame-abcd.fits'
self.fibermapfile = 'test-fibermap-abcd.fits'
self.skyfile = 'test-sky-abcd.fits'
self.qafile = 'test_qa.yaml'
self.qafig = 'test_qa.png'
#- use specter psf for this test
self.psffile = resource_filename('specter', 'test/t/psf-monospot.fits')
#self.psffile=os.environ['DESIMODEL']+'/data/specpsf/psf-b.fits'
self.config = {}
#- rawimage
hdr = dict()
hdr['CAMERA'] = 'z1'
hdr['DATE-OBS'] = '2018-09-23T08:17:03.988'
#- Dimensions per amp
ny = self.ny = 500
nx = self.nx = 400
noverscan = nover = 50
hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx + nover])
hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx + nover:nx + 2 * nover])
hdr['DATASEC2'] = xy2hdr(np.s_[0:ny,
nx + 2 * nover:nx + 2 * nover + nx])
hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx + nx])
hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nover])
hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx])
hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx])
hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + nover:nx + 2 * nover])
hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny + ny,
nx + 2 * nover:nx + 2 * nover + nx])
hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nx])
hdr['NIGHT'] = '20180923'
hdr['EXPID'] = 1
hdr['PROGRAM'] = 'dark'
hdr['FLAVOR'] = 'science'
hdr['EXPTIME'] = 100.0
rawimage = np.zeros((2 * ny, 2 * nx + 2 * noverscan))
offset = {'1': 100.0, '2': 100.5, '3': 50.3, '4': 200.4}
gain = {'1': 1.0, '2': 1.5, '3': 0.8, '4': 1.2}
rdnoise = {'1': 2.0, '2': 2.2, '3': 2.4, '4': 2.6}
quad = {
'1': np.s_[0:ny, 0:nx],
'2': np.s_[0:ny, nx:nx + nx],
'3': np.s_[ny:ny + ny, 0:nx],
'4': np.s_[ny:ny + ny, nx:nx + nx],
}
for amp in ('1', '2', '3', '4'):
hdr['GAIN' + amp] = gain[amp]
hdr['RDNOISE' + amp] = rdnoise[amp]
xy = _parse_sec_keyword(hdr['BIASSEC' + amp])
shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start]
rawimage[xy] += offset[amp]
rawimage[xy] += np.random.normal(scale=rdnoise[amp],
size=shape) / gain[amp]
xy = _parse_sec_keyword(hdr['DATASEC' + amp])
shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start]
rawimage[xy] += offset[amp]
rawimage[xy] += np.random.normal(scale=rdnoise[amp],
size=shape) / gain[amp]
#- set CCD parameters
self.ccdsec1 = hdr["CCDSEC1"]
self.ccdsec2 = hdr["CCDSEC2"]
self.ccdsec3 = hdr["CCDSEC3"]
self.ccdsec4 = hdr["CCDSEC4"]
#- raw data are integers, not floats
rawimg = rawimage.astype(np.int32)
self.expid = hdr["EXPID"]
self.camera = hdr["CAMERA"]
#- Confirm that all regions were correctly offset
assert not np.any(rawimage == 0.0)
#- write to the rawfile and read it in QA test
hdr['DOSVER'] = 'SIM'
hdr['FEEVER'] = 'SIM'
hdr['DETECTOR'] = 'SIM'
desispec.io.write_raw(self.rawfile, rawimg, hdr, camera=self.camera)
self.rawimage = fits.open(self.rawfile)
#- read psf, should use specter.PSF.load_psf instead of desispec.PSF(), otherwise need to create a psfboot somewhere.
psf = self.psf = load_psf(self.psffile)
#- make the test pixfile, fibermap file
img_pix = rawimg
img_ivar = np.ones_like(img_pix) / 3.0**2
img_mask = np.zeros(img_pix.shape, dtype=np.uint32)
img_mask[200] = 1
self.image = desispec.image.Image(img_pix,
img_ivar,
img_mask,
camera='z1',
meta=hdr)
desispec.io.write_image(self.pixfile, self.image)
self.fibermap = desispec.io.empty_fibermap(30)
self.fibermap['OBJTYPE'][::2] = 'ELG'
self.fibermap['OBJTYPE'][::3] = 'STD'
self.fibermap['OBJTYPE'][::5] = 'QSO'
self.fibermap['OBJTYPE'][::9] = 'LRG'
self.fibermap['OBJTYPE'][::7] = 'SKY'
#- add a filter and arbitrary magnitude
self.fibermap['MAG'][:29] = np.tile(np.random.uniform(18, 20, 29),
5).reshape(29,
5) #- Last fiber left
self.fibermap['FILTER'][:29] = np.tile(
['DECAM_R', '..', '..', '..', '..'], (29, 1)) #- last fiber left
desispec.io.write_fibermap(self.fibermapfile, self.fibermap)
#- make a test frame file
self.night = hdr['NIGHT']
self.nspec = nspec = 30
wave = np.arange(7600.0, 9800.0, 1.0) #- z channel
nwave = self.nwave = len(wave)
flux = np.random.uniform(size=(nspec, nwave)) + 100.
ivar = np.ones_like(flux)
resolution_data = np.ones((nspec, 13, nwave))
self.frame = desispec.frame.Frame(wave,
flux,
ivar,
resolution_data=resolution_data,
fibermap=self.fibermap)
self.frame.meta = dict(CAMERA=self.camera,
PROGRAM='dark',
FLAVOR='science',
NIGHT=self.night,
EXPID=self.expid,
EXPTIME=100,
CCDSEC1=self.ccdsec1,
CCDSEC2=self.ccdsec2,
CCDSEC3=self.ccdsec3,
CCDSEC4=self.ccdsec4)
desispec.io.write_frame(self.framefile, self.frame)
#- make a skymodel
sky = np.ones_like(self.frame.flux) * 0.5
skyivar = np.ones_like(sky)
self.mask = np.zeros(sky.shape, dtype=np.uint32)
self.skymodel = desispec.sky.SkyModel(wave, sky, skyivar, self.mask)
self.skyfile = desispec.io.write_sky(self.skyfile, self.skymodel)
#- Make a dummy boundary map for wavelength-flux in pixel space
self.map2pix = {}
self.map2pix["LEFT_MAX_FIBER"] = 14
self.map2pix["RIGHT_MIN_FIBER"] = 17
self.map2pix["BOTTOM_MAX_WAVE_INDEX"] = 900
self.map2pix["TOP_MIN_WAVE_INDEX"] = 1100
def setUp(self):
self.rawfile = 'test-raw-abcd.fits'
self.pixfile = 'test-pix-abcd.fits'
self.config = {}
#- rawimage
hdr = dict()
hdr['CAMERA'] = 'b1'
hdr['DATE-OBS'] = '2018-09-23T08:17:03.988'
#- Dimensions per amp, not full 4-quad CCD
ny = self.ny = 500
nx = self.nx = 400
noverscan = nover = 50
hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx + nover])
hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx + nover:nx + 2 * nover])
hdr['DATASEC2'] = xy2hdr(np.s_[0:ny,
nx + 2 * nover:nx + 2 * nover + nx])
hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx + nx])
hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nover])
hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx])
hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx])
hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + nover:nx + 2 * nover])
hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny + ny,
nx + 2 * nover:nx + 2 * nover + nx])
hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nx])
hdr['NIGHT'] = '20180923'
hdr['EXPID'] = 1
hdr['FLAVOR'] = 'dark'
rawimage = np.zeros((2 * ny, 2 * nx + 2 * noverscan))
offset = {'1': 100.0, '2': 100.5, '3': 50.3, '4': 200.4}
gain = {'1': 1.0, '2': 1.5, '3': 0.8, '4': 1.2}
rdnoise = {'1': 2.0, '2': 2.2, '3': 2.4, '4': 2.6}
quad = {
'1': np.s_[0:ny, 0:nx],
'2': np.s_[0:ny, nx:nx + nx],
'3': np.s_[ny:ny + ny, 0:nx],
'4': np.s_[ny:ny + ny, nx:nx + nx],
}
for amp in ('1', '2', '3', '4'):
hdr['GAIN' + amp] = gain[amp]
hdr['RDNOISE' + amp] = rdnoise[amp]
xy = _parse_sec_keyword(hdr['BIASSEC' + amp])
shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start]
rawimage[xy] += offset[amp]
rawimage[xy] += np.random.normal(scale=rdnoise[amp],
size=shape) / gain[amp]
xy = _parse_sec_keyword(hdr['DATASEC' + amp])
shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start]
rawimage[xy] += offset[amp]
rawimage[xy] += np.random.normal(scale=rdnoise[amp],
size=shape) / gain[amp]
#- raw data are integers, not floats
rawimg = rawimage.astype(np.int32)
self.expid = hdr["EXPID"]
self.camera = hdr["CAMERA"]
#- Confirm that all regions were correctly offset
assert not np.any(rawimage == 0.0)
hdr['DOSVER'] = 'SIM'
hdr['FEEVER'] = 'SIM'
hdr['DETECTOR'] = 'SIM'
desispec.io.write_raw(self.rawfile, rawimg, hdr, camera=self.camera)
self.rawimage = fits.open(self.rawfile)
def setUp(self):
self.calibfile = 'test-calib-askjapqwhezcpasehadfaqp.fits'
self.rawfile = 'test-raw-askjapqwhezcpasehadfaqp.fits'
self.pixfile = 'test-pix-askjapqwhezcpasehadfaqp.fits'
hdr = dict()
hdr['CAMERA'] = 'b0'
hdr['DATE-OBS'] = '2018-09-23T08:17:03.988'
#- [x,y] 1-indexed for FITS; in reality the amps will be symmetric
#- but the header definitions don't require that to make sure we are
#- getting dimensions correct
#- Dimensions per amp, not full 4-quad CCD
self.ny = ny = 500
self.nx = nx = 400
self.noverscan = nover = 50
#- BIASSEC = overscan region in raw image
#- DATASEC = data region in raw image
#- CCDSEC = where should this go in output
hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx + nover])
hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx + nover:nx + 2 * nover])
hdr['DATASEC2'] = xy2hdr(np.s_[0:ny,
nx + 2 * nover:nx + 2 * nover + nx])
hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx + nx])
hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nover])
hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx])
hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx])
hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + nover:nx + 2 * nover])
hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny + ny,
nx + 2 * nover:nx + 2 * nover + nx])
hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nx])
hdr['NIGHT'] = '20150102'
hdr['EXPID'] = 1
self.header = hdr
self.rawimage = np.zeros(
(2 * self.ny, 2 * self.nx + 2 * self.noverscan))
self.offset = {'1': 100.0, '2': 100.5, '3': 50.3, '4': 200.4}
self.gain = {'1': 1.0, '2': 1.5, '3': 0.8, '4': 1.2}
self.rdnoise = {'1': 2.0, '2': 2.2, '3': 2.4, '4': 2.6}
self.quad = {
'1': np.s_[0:ny, 0:nx],
'2': np.s_[0:ny, nx:nx + nx],
'3': np.s_[ny:ny + ny, 0:nx],
'4': np.s_[ny:ny + ny, nx:nx + nx],
}
for amp in ('1', '2', '3', '4'):
self.header['GAIN' + amp] = self.gain[amp]
self.header['RDNOISE' + amp] = self.rdnoise[amp]
xy = _parse_sec_keyword(hdr['BIASSEC' + amp])
shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start]
self.rawimage[xy] += self.offset[amp]
self.rawimage[xy] += np.random.normal(scale=self.rdnoise[amp],
size=shape) / self.gain[amp]
xy = _parse_sec_keyword(hdr['DATASEC' + amp])
shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start]
self.rawimage[xy] += self.offset[amp]
self.rawimage[xy] += np.random.normal(scale=self.rdnoise[amp],
size=shape) / self.gain[amp]
#- raw data are integers, not floats
self.rawimage = self.rawimage.astype(np.int32)
#- Confirm that all regions were correctly offset
assert not np.any(self.rawimage == 0.0)
tmp = sub[:,1:] - sub[:,:-1]
sub[:,1:] = tmp
sub[:,0] = 0
rms_scale = 1./np.sqrt(2.)
i=0
x=np.zeros(3+4*6).astype(float)
x[i] = expid ; i += 1
x[i] = mjdobs ; i += 1
x[i] = dateobs ; i += 1
for a,amp in enumerate(['A','B','C','D']) :
gain = 1.
if cfinder is not None and cfinder.haskey("GAIN"+amp) :
gain=cfinder.value("GAIN"+amp)
x[i] = mean(img[_parse_sec_keyword(header["BIASSEC"+amp])],gain=gain); i+=1
x[i] = rms_scale*rms(sub[_parse_sec_keyword(header["BIASSEC"+amp])],gain=gain); i+=1
if "ORSEC"+amp in header :
x[i] = mean(img[_parse_sec_keyword(header["ORSEC"+amp])],gain=gain); i+=1
x[i] = rms_scale*rms(sub[_parse_sec_keyword(header["ORSEC"+amp])],gain=gain); i+=1
else :
i += 2
x[i] = mean(img[_parse_sec_keyword(header["DATASEC"+amp])],gain=gain); i+=1
x[i] = rms_scale*rms(sub[_parse_sec_keyword(header["DATASEC"+amp])],gain=gain); i+=1
print("assuming gain= {:3.2f} for amp {}, ccd rms= {:3.2f}".format(gain,amp,x[i-1]))
sys.stdout.flush()
xx.append(x)
if args.outfile is not None :
xx=np.vstack(xx)
np.savetxt(args.outfile,xx,header=line)
def setUp(self):
#- use specter psf for this test
self.psffile=resource_filename('specter', 'test/t/psf-monospot.fits')
#self.psffile=os.environ['DESIMODEL']+'/data/specpsf/psf-b.fits'
self.config={"kwargs":{
"refKey":None,
"param":{},
"qso_resid":None
}}
#- rawimage
hdr = dict()
hdr['CAMERA'] = 'z1'
hdr['DATE-OBS'] = '2018-09-23T08:17:03.988'
hdr['PROGRAM'] = 'dark'
hdr['EXPTIME'] = 100
#- Dimensions per amp
ny = self.ny = 500
nx = self.nx = 400
noverscan = nover = 50
hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx+nover])
hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx+nover:nx+2*nover])
hdr['DATASEC2'] = xy2hdr(np.s_[0:ny, nx+2*nover:nx+2*nover+nx])
hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx+nx])
hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny+ny, nx:nx+nover])
hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny+ny, 0:nx])
hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny+ny, 0:nx])
hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny+ny, nx+nover:nx+2*nover])
hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny+ny, nx+2*nover:nx+2*nover+nx])
hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny+ny, nx:nx+nx])
hdr['NIGHT'] = '20180923'
hdr['EXPID'] = 1
hdr['PROGRAM'] = 'dark'
hdr['FLAVOR'] = 'science'
hdr['EXPTIME'] = 100.0
rawimage = np.zeros((2*ny, 2*nx+2*noverscan))
offset = {'1':100.0, '2':100.5, '3':50.3, '4':200.4}
gain = {'1':1.0, '2':1.5, '3':0.8, '4':1.2}
rdnoise = {'1':2.0, '2':2.2, '3':2.4, '4':2.6}
obsrdn = {'1':3.4, '2':3.3, '3':3.6, '4':3.3}
quad = {
'1': np.s_[0:ny, 0:nx], '2': np.s_[0:ny, nx:nx+nx],
'3': np.s_[ny:ny+ny, 0:nx], '4': np.s_[ny:ny+ny, nx:nx+nx],
}
for amp in ('1', '2', '3', '4'):
hdr['GAIN'+amp] = gain[amp]
hdr['RDNOISE'+amp] = rdnoise[amp]
hdr['OBSRDN'+amp] = obsrdn[amp]
xy = _parse_sec_keyword(hdr['BIASSEC'+amp])
shape = [xy[0].stop-xy[0].start, xy[1].stop-xy[1].start]
rawimage[xy] += offset[amp]
rawimage[xy] += np.random.normal(scale=rdnoise[amp], size=shape)/gain[amp]
xy = _parse_sec_keyword(hdr['DATASEC'+amp])
shape = [xy[0].stop-xy[0].start, xy[1].stop-xy[1].start]
rawimage[xy] += offset[amp]
rawimage[xy] += np.random.normal(scale=rdnoise[amp], size=shape)/gain[amp]
#- set CCD parameters
self.ccdsec1=hdr["CCDSEC1"]
self.ccdsec2=hdr["CCDSEC2"]
self.ccdsec3=hdr["CCDSEC3"]
self.ccdsec4=hdr["CCDSEC4"]
#- raw data are integers, not floats
rawimg = rawimage.astype(np.int32)
self.expid=hdr["EXPID"]
self.camera=hdr["CAMERA"]
#- Confirm that all regions were correctly offset
assert not np.any(rawimage == 0.0)
#- write to the rawfile and read it in QA test
hdr['DOSVER'] = 'SIM'
hdr['FEEVER'] = 'SIM'
hdr['DETECTOR'] = 'SIM'
desispec.io.write_raw(self.rawfile,rawimg,hdr,camera=self.camera)
self.rawimage=fits.open(self.rawfile)
#- read psf, should use specter.PSF.load_psf instead of desispec.PSF(), otherwise need to create a psfboot somewhere.
self.psf = load_psf(self.psffile)
#- make the test pixfile, fibermap file
img_pix = rawimg
img_ivar = np.ones_like(img_pix) / 3.0**2
img_mask = np.zeros(img_pix.shape, dtype=np.uint32)
img_mask[200] = 1
self.image = desispec.image.Image(img_pix, img_ivar, img_mask, camera='z1',meta=hdr)
desispec.io.write_image(self.pixfile, self.image)
#- Create a fibermap with purposefully overlapping targeting bits
n = 30
self.fibermap = desispec.io.empty_fibermap(n)
self.fibermap['OBJTYPE'][:] = 'TGT'
self.fibermap['DESI_TARGET'][::2] |= desi_mask.ELG
self.fibermap['DESI_TARGET'][::5] |= desi_mask.QSO
self.fibermap['DESI_TARGET'][::7] |= desi_mask.LRG
#- add some arbitrary fluxes
for key in ['FLUX_G', 'FLUX_R', 'FLUX_Z', 'FLUX_W1', 'FLUX_W2']:
self.fibermap[key] = 10**((22.5 - np.random.uniform(18, 21, size=n))/2.5)
#- Make some standards; these still have OBJTYPE = 'TGT'
ii = [6,18,29]
self.fibermap['DESI_TARGET'][ii] = desi_mask.STD_FAINT
#- set some targets to SKY
ii = self.skyfibers = [5,10,21]
self.fibermap['OBJTYPE'][ii] = 'SKY'
self.fibermap['DESI_TARGET'][ii] = desi_mask.SKY
for key in ['FLUX_G', 'FLUX_R', 'FLUX_Z', 'FLUX_W1', 'FLUX_W2']:
self.fibermap[key][ii] = np.random.normal(scale=100, size=len(ii))
desispec.io.write_fibermap(self.fibermapfile, self.fibermap)
#- make a test frame file
self.night=hdr['NIGHT']
self.nspec = nspec = 30
wave=np.arange(7600.0,9800.0,1.0) #- z channel
nwave = self.nwave = len(wave)
flux=np.random.uniform(size=(nspec,nwave))+100.
ivar=np.ones_like(flux)
resolution_data=np.ones((nspec,13,nwave))
self.frame=desispec.frame.Frame(wave,flux,ivar,resolution_data=resolution_data,fibermap=self.fibermap)
self.frame.meta = hdr
self.frame.meta['WAVESTEP']=0.5
desispec.io.write_frame(self.framefile, self.frame)
#- make a skymodel
sky=np.ones_like(self.frame.flux)*0.5
skyivar=np.ones_like(sky)
self.mask=np.zeros(sky.shape,dtype=np.uint32)
self.skymodel=desispec.sky.SkyModel(wave,sky,skyivar,self.mask)
self.skyfile=desispec.io.write_sky(self.skyfile,self.skymodel)
#- Make a dummy boundary map for wavelength-flux in pixel space
self.map2pix={}
self.map2pix["LEFT_MAX_FIBER"] = 14
self.map2pix["RIGHT_MIN_FIBER"] = 17
self.map2pix["BOTTOM_MAX_WAVE_INDEX"] = 900
self.map2pix["TOP_MIN_WAVE_INDEX"] = 1100
def setUp(self):
#- Create temporary calib directory
self.testDir = os.path.join(os.environ['HOME'], 'ql_test_io')
calibDir = os.path.join(self.testDir, 'ql_calib')
if not os.path.exists(calibDir): os.makedirs(calibDir)
#- Generate calib data
for camera in ['b0', 'r0', 'z0']:
#- Fiberflat has to exist but can be a dummpy file
filename = '{}/fiberflat-{}.fits'.format(calibDir, camera)
fx = open(filename, 'w'); fx.write('fiberflat file'); fx.close()
#- PSF has to be real file
psffile = '{}/psf-{}.fits'.format(calibDir, camera)
example_psf = resource_filename('desispec', 'test/data/ql/psf-{}.fits'.format(camera))
shutil.copy(example_psf, psffile)
#- Copy test calibration-data.yaml file
specdir=calibDir+"spec/sp0"
if not os.path.isdir(specdir) :
os.makedirs(specdir)
for c in "brz" :
shutil.copy(resource_filename('desispec', 'test/data/ql/{}0.yaml'.format(c)),os.path.join(specdir,"{}0.yaml".format(c)))
#- Set calibration environment variable
os.environ['DESI_SPECTRO_CALIB'] = calibDir
self.rawfile = os.path.join(self.testDir,'test-raw-abcd.fits')
self.pixfile = os.path.join(self.testDir,'test-pix-abcd.fits')
self.config={}
#- rawimage
hdr = dict()
hdr['CAMERA'] = 'b0'
hdr['DATE-OBS'] = '2018-09-23T08:17:03.988'
#- Dimensions per amp, not full 4-quad CCD
ny = self.ny = 500
nx = self.nx = 400
noverscan = nover = 50
hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx+nover])
hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx])
hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx+nover:nx+2*nover])
hdr['DATASEC2'] = xy2hdr(np.s_[0:ny, nx+2*nover:nx+2*nover+nx])
hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx+nx])
hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny+ny, nx:nx+nover])
hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny+ny, 0:nx])
hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny+ny, 0:nx])
hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny+ny, nx+nover:nx+2*nover])
hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny+ny, nx+2*nover:nx+2*nover+nx])
hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny+ny, nx:nx+nx])
hdr['NIGHT'] = '20180923'
hdr['EXPID'] = 1
hdr['FLAVOR']='dark'
rawimage = np.zeros((2*ny, 2*nx+2*noverscan))
offset = {'1':100.0, '2':100.5, '3':50.3, '4':200.4}
gain = {'1':1.0, '2':1.5, '3':0.8, '4':1.2}
rdnoise = {'1':2.0, '2':2.2, '3':2.4, '4':2.6}
quad = {
'1': np.s_[0:ny, 0:nx], '2': np.s_[0:ny, nx:nx+nx],
'3': np.s_[ny:ny+ny, 0:nx], '4': np.s_[ny:ny+ny, nx:nx+nx],
}
for amp in ('1', '2', '3', '4'):
hdr['GAIN'+amp] = gain[amp]
hdr['RDNOISE'+amp] = rdnoise[amp]
xy = _parse_sec_keyword(hdr['BIASSEC'+amp])
shape = [xy[0].stop-xy[0].start, xy[1].stop-xy[1].start]
rawimage[xy] += offset[amp]
rawimage[xy] += np.random.normal(scale=rdnoise[amp], size=shape)/gain[amp]
xy = _parse_sec_keyword(hdr['DATASEC'+amp])
shape = [xy[0].stop-xy[0].start, xy[1].stop-xy[1].start]
rawimage[xy] += offset[amp]
rawimage[xy] += np.random.normal(scale=rdnoise[amp], size=shape)/gain[amp]
#- raw data are integers, not floats
rawimg = rawimage.astype(np.int32)
self.expid=hdr["EXPID"]
self.camera=hdr["CAMERA"]
#- Confirm that all regions were correctly offset
assert not np.any(rawimage == 0.0)
hdr['DOSVER'] = 'SIM'
hdr['FEEVER'] = 'SIM'
hdr['DETECTOR'] = 'SIM'
desispec.io.write_raw(self.rawfile,rawimg,hdr,camera=self.camera)
self.rawimage=fits.open(self.rawfile)
