def test_pedantic(self): with self.assertRaises(ValueError): _parse_sec_keyword('blat') #- should log a warning about large readnoise rawimage = self.rawimage + np.random.normal(scale=2, size=self.rawimage.shape) image = preproc(rawimage, self.header) #- should log an error about huge readnoise rawimage = self.rawimage + np.random.normal(scale=10, size=self.rawimage.shape) image = preproc(rawimage, self.header) #- should log a warning about small readnoise rdnoise = 0.7 * np.mean(self.rdnoise.values()) rawimage = np.random.normal(scale=rdnoise, size=self.rawimage.shape) image = preproc(rawimage, self.header) #- should log a warning about tiny readnoise rdnoise = 0.01 * np.mean(self.rdnoise.values()) rawimage = np.random.normal(scale=rdnoise, size=self.rawimage.shape) image = preproc(rawimage, self.header) #- Missing expected RDNOISE keywords shouldn't be fatal hdr = self.header.copy() del hdr['RDNOISE1'] del hdr['RDNOISE2'] del hdr['RDNOISE3'] del hdr['RDNOISE4'] image = preproc(self.rawimage, hdr) #- Missing expected GAIN keywords should log error but not crash hdr = self.header.copy() del hdr['GAIN1'] del hdr['GAIN2'] del hdr['GAIN3'] del hdr['GAIN4'] image = preproc(self.rawimage, hdr)
def run_qa(self, image, camera, expid, paname=None, amps=False, url=None, qafig=None): retval = {} retval["EXPID"] = expid retval["ARM"] = camera[0] retval["SPECTROGRAPH"] = int(camera[1]) retval["PANAME"] = paname t = datetime.datetime.now() retval["MJD"] = Time(t).mjd bias_overscan = [] for kk in ['1', '2', '3', '4']: from desispec.preproc import _parse_sec_keyword sel = _parse_sec_keyword(image.meta['BIASSEC' + kk]) pix = image.pix[sel] #- Compute statistics of the bias region that only reject # the 0.5% of smallest and largest values. (from sdssproc) isort = np.sort(pix.ravel()) nn = isort.shape[0] bias = np.mean(isort[long(0.005 * nn):long(0.995 * nn)]) bias_overscan.append(bias) bias = np.mean(bias_overscan) if amps: bias_amps = np.array(bias_overscan) retval["VALUE"] = {'BIAS': bias, 'BIAS_AMP': bias_amps} else: retval["VALUE"] = {'BIAS': bias} #- http post if needed if url is not None: try: import requests response = requests.get(url) #- Check if the api has json api = response.json() #- proceed with post job = {"name": "QL", "status": 0, "dictionary": retval} response = requests.post(api['job'], json=job, auth=("username", "password")) except: log.info("Skipping HTTP post...") if qafig is not None: from desispec.qa.qa_plots_ql import plot_bias_overscan plot_bias_overscan(retval, qafig) log.info("Output QA fig %s" % qafig) return retval
def run_qa(self, image, camera, expid, paname=None, amps=False, url=None, qafig=None): retval = {} retval["EXPID"] = expid retval["ARM"] = camera[0] retval["SPECTROGRAPH"] = int(camera[1]) retval["PANAME"] = paname t = datetime.datetime.now() retval["MJD"] = Time(t).mjd rmsccd = getrms( image.pix ) #- should we add dark current and/or readnoise to this as well? if amps: rms_amps = [] #- get amp boundary in pixels from desispec.preproc import _parse_sec_keyword for kk in ['1', '2', '3', '4']: ampboundary = _parse_sec_keyword(image.meta["CCDSEC" + kk]) rmsamp = getrms(image.pix[ampboundary]) rms_amps.append(rmsamp) retval["VALUE"] = {"RMS": rmsccd, "RMS_AMP": np.array(rms_amps)} else: retval["VALUE"] = {"RMS": rmsccd} if url is not None: try: import requests response = requests.get(url) #- Check if the api has json api = response.json() #- proceed with post job = { "name": "QL", "status": 0, "dictionary": retval } #- QLF should disintegrate dictionary response = requests.post( api['job'], json=job, auth=("username", "password") ) #- username, password not real but placeholder here. except: log.info("Skipping HTTP post...") if qafig is not None: from desispec.qa.qa_plots_ql import plot_RMS plot_RMS(retval, qafig) log.info("Output QA fig %s" % qafig) return retval
def run_qa(self,image,camera,expid,paname=None,amps=False,url=None,qafig=None): retval={} retval["EXPID"]=expid retval["ARM"]=camera[0] retval["SPECTROGRAPH"]=int(camera[1]) retval["PANAME"]=paname #retval["TIMESTAMP"]='{:%Y-%b-%d %H:%M:%S}'.format(datetime.datetime.now()) t=datetime.datetime.now() retval["MJD"]=Time(t).mjd #- get the counts over entire CCD npix3sig=countpix(image.pix,nsig=3) #- above 3 sigma npix100=countpix(image.pix,ncounts=100) #- above 100 pixel count npix500=countpix(image.pix,ncounts=500) #- above 500 pixel count #- get the counts for each amp if amps: npix3sig_amps=[] npix100_amps=[] npix500_amps=[] #- get amp boundary in pixels from desispec.preproc import _parse_sec_keyword for kk in ['1','2','3','4']: ampboundary=_parse_sec_keyword(image.meta["CCDSEC"+kk]) npix3sig=countpix(image.pix[ampboundary],nsig=3) npix3sig_amps.append(npix3sig) npix100=countpix(image.pix[ampboundary],ncounts=100) npix100_amps.append(npix100) npix500=countpix(image.pix[ampboundary],ncounts=500) npix500_amps.append(npix500) retval["VALUE"]={"NPIX3SIG":npix3sig,"NPIX100":npix100,"NPIX500":npix500, "NPIX3SIG_AMP": npix3sig_amps, "NPIX100_AMP": npix100_amps,"NPIX500_AMP": npix500_amps} else: retval["VALUE"]={"NPIX3SIG":npix3sig,"NPIX100":npix100,"NPIX500":npix500} if url is not None: try: import requests response=requests.get(url) #- Check if the api has json api=response.json() #- proceed with post job={"name":"QL","status":0,"dictionary":retval} #- QLF should disintegrate dictionary response=requests.post(api['job'],json=job,auth=("username","password")) #- username, password not real but placeholder here. except: log.info("Skipping HTTP post...") if qafig is not None: from desispec.qa.qa_plots_ql import plot_countpix plot_countpix(retval,qafig) log.info("Output QA fig %s"%qafig) return retval
def ampregion(image): """ Get the pixel boundary regions for amps Args: image: desispec.image.Image object """ pixboundary = [] for kk in ['1', '2', '3', '4']: #- 4 amps #- get the amp region in pix ampboundary = _parse_sec_keyword(image.meta["CCDSEC" + kk]) pixboundary.append(ampboundary) return pixboundary
def ampregion(image): """ get the pixel boundary regions for amps args: image: desispec.image.Image object """ from desispec.preproc import _parse_sec_keyword pixboundary=[] for kk in ['1','2','3','4']: #- 4 amps #- get the amp region in pix ampboundary=_parse_sec_keyword(image.meta["CCDSEC"+kk]) pixboundary.append(ampboundary) return pixboundary
def run_qa(self,image,camera,expid,paname=None,amps=False,url=None,qafig=None): retval={} retval["EXPID"]=expid retval["ARM"]=camera[0] retval["SPECTROGRAPH"]=int(camera[1]) retval["PANAME"]=paname t=datetime.datetime.now() retval["MJD"]=Time(t).mjd bias_overscan=[] for kk in ['1','2','3','4']: from desispec.preproc import _parse_sec_keyword sel=_parse_sec_keyword(image.meta['BIASSEC'+kk]) pix=image.pix[sel] #- Compute statistics of the bias region that only reject # the 0.5% of smallest and largest values. (from sdssproc) isort=np.sort(pix.ravel()) nn=isort.shape[0] bias=np.mean(isort[long(0.005*nn) : long(0.995*nn)]) bias_overscan.append(bias) bias=np.mean(bias_overscan) if amps: bias_amps=np.array(bias_overscan) retval["VALUE"]={'BIAS':bias,'BIAS_AMP':bias_amps} else: retval["VALUE"]={'BIAS':bias} #- http post if needed if url is not None: try: import requests response=requests.get(url) #- Check if the api has json api=response.json() #- proceed with post job={"name":"QL","status":0,"dictionary":retval} response=requests.post(api['job'],json=job,auth=("username","password")) except: log.info("Skipping HTTP post...") if qafig is not None: from desispec.qa.qa_plots_ql import plot_bias_overscan plot_bias_overscan(retval,qafig) log.info("Output QA fig %s"%qafig) return retval
def run_qa(self,image,camera,expid,paname=None,amps=False,url=None,qafig=None): retval={} retval["EXPID"]=expid retval["ARM"]=camera[0] retval["SPECTROGRAPH"]=int(camera[1]) retval["PANAME"]=paname t=datetime.datetime.now() retval["MJD"]=Time(t).mjd rmsccd=getrms(image.pix) #- should we add dark current and/or readnoise to this as well? if amps: rms_amps=[] #- get amp boundary in pixels from desispec.preproc import _parse_sec_keyword for kk in ['1','2','3','4']: ampboundary=_parse_sec_keyword(image.meta["CCDSEC"+kk]) rmsamp=getrms(image.pix[ampboundary]) rms_amps.append(rmsamp) retval["VALUE"]={"RMS":rmsccd,"RMS_AMP":np.array(rms_amps)} else: retval["VALUE"]={"RMS":rmsccd} if url is not None: try: import requests response=requests.get(url) #- Check if the api has json api=response.json() #- proceed with post job={"name":"QL","status":0,"dictionary":retval} #- QLF should disintegrate dictionary response=requests.post(api['job'],json=job,auth=("username","password")) #- username, password not real but placeholder here. except: log.info("Skipping HTTP post...") if qafig is not None: from desispec.qa.qa_plots_ql import plot_RMS plot_RMS(retval,qafig) log.info("Output QA fig %s"%qafig) return retval
def fiducialregion(frame,psf): """ get the fiducial amplifier regions on the CCD pixel to fiber by wavelength space args: frame: desispec.frame.Frame object psf: desispec.psf.PSF like object """ from desispec.preproc import _parse_sec_keyword startspec=0 #- will be None if don't have fibers on the right of the CCD. endspec=499 #- will be None if don't have fibers on the right of the CCD startwave0=0 #- lower index for the starting fiber startwave1=0 #- lower index for the last fiber for the amp region endwave0=frame.wave.shape[0] #- upper index for the starting fiber endwave1=frame.wave.shape[0] #- upper index for the last fiber for that amp pixboundary=[] fidboundary=[] for kk in ['1','2','3','4']: #- 4 amps #- get the amp region in pix ampboundary=_parse_sec_keyword(frame.meta["CCDSEC"+kk]) pixboundary.append(ampboundary) for ispec in range(frame.flux.shape[0]): if np.all(psf.x(ispec) > ampboundary[1].start): startspec=ispec #-cutting off wavelenth boundaries from startspec yy=psf.y(ispec,frame.wave) k=np.where(yy > ampboundary[0].start)[0] startwave0=k[0] yy=psf.y(ispec,frame.wave) k=np.where(yy < ampboundary[0].stop)[0] endwave0=k[-1] break else: startspec=None startwave0=None endwave0=None if startspec is not None: for ispec in range(frame.flux.shape[0])[::-1]: if np.all(psf.x(ispec) < ampboundary[1].stop): endspec=ispec #-cutting off wavelenth boundaries from startspec yy=psf.y(ispec,frame.wave) k=np.where(yy > ampboundary[0].start)[0] startwave1=k[0] yy=psf.y(ispec,frame.wave) k=np.where(yy < ampboundary[0].stop)[0] endwave1=k[-1] break else: endspec=None startwave1=None endwave1=None startwave=max(startwave0,startwave1) endwave=min(endwave0,endwave1) if endspec is not None: endspec+=1 #- last entry exclusive in slice, so add 1 endwave+=1 fiducialb=(slice(startspec,endspec,None),slice(startwave,endwave,None)) #- Note: y,x --> spec, wavelength fidboundary.append(fiducialb) return pixboundary,fidboundary
def run_qa(self, image, camera, expid, paname=None, amps=False, url=None, qafig=None): retval = {} retval["EXPID"] = expid retval["ARM"] = camera[0] retval["SPECTROGRAPH"] = int(camera[1]) retval["PANAME"] = paname #retval["TIMESTAMP"]='{:%Y-%b-%d %H:%M:%S}'.format(datetime.datetime.now()) t = datetime.datetime.now() retval["MJD"] = Time(t).mjd #- get the counts over entire CCD npix3sig = countpix(image.pix, nsig=3) #- above 3 sigma npix100 = countpix(image.pix, ncounts=100) #- above 100 pixel count npix500 = countpix(image.pix, ncounts=500) #- above 500 pixel count #- get the counts for each amp if amps: npix3sig_amps = [] npix100_amps = [] npix500_amps = [] #- get amp boundary in pixels from desispec.preproc import _parse_sec_keyword for kk in ['1', '2', '3', '4']: ampboundary = _parse_sec_keyword(image.meta["CCDSEC" + kk]) npix3sig = countpix(image.pix[ampboundary], nsig=3) npix3sig_amps.append(npix3sig) npix100 = countpix(image.pix[ampboundary], ncounts=100) npix100_amps.append(npix100) npix500 = countpix(image.pix[ampboundary], ncounts=500) npix500_amps.append(npix500) retval["VALUE"] = { "NPIX3SIG": npix3sig, "NPIX100": npix100, "NPIX500": npix500, "NPIX3SIG_AMP": npix3sig_amps, "NPIX100_AMP": npix100_amps, "NPIX500_AMP": npix500_amps } else: retval["VALUE"] = { "NPIX3SIG": npix3sig, "NPIX100": npix100, "NPIX500": npix500 } if url is not None: try: import requests response = requests.get(url) #- Check if the api has json api = response.json() #- proceed with post job = { "name": "QL", "status": 0, "dictionary": retval } #- QLF should disintegrate dictionary response = requests.post( api['job'], json=job, auth=("username", "password") ) #- username, password not real but placeholder here. except: log.info("Skipping HTTP post...") if qafig is not None: from desispec.qa.qa_plots_ql import plot_countpix plot_countpix(retval, qafig) log.info("Output QA fig %s" % qafig) return retval
def fiducialregion(frame, psf): """ Get the fiducial amplifier regions on the CCD pixel to fiber by wavelength space Args: frame: desispec.frame.Frame object psf: desispec.psf.PSF like object """ from desispec.preproc import _parse_sec_keyword startspec = 0 #- will be None if don't have fibers on the right of the CCD. endspec = 499 #- will be None if don't have fibers on the right of the CCD startwave0 = 0 #- lower index for the starting fiber startwave1 = 0 #- lower index for the last fiber for the amp region endwave0 = frame.wave.shape[0] #- upper index for the starting fiber endwave1 = frame.wave.shape[ 0] #- upper index for the last fiber for that amp pixboundary = [] fidboundary = [] #- Adding the min, max boundary individually for the benefit of dumping to yaml. leftmax = 499 #- for amp 1 and 3 rightmin = 0 #- for amp 2 and 4 bottommax = frame.wave.shape[0] #- for amp 1 and 2 topmin = 0 #- for amp 3 and 4 #- Loop over each amp for kk in ['1', '2', '3', '4']: #- 4 amps #- get the amp region in pix ampboundary = _parse_sec_keyword(frame.meta["CCDSEC" + kk]) pixboundary.append(ampboundary) for ispec in range(frame.flux.shape[0]): if np.all(psf.x(ispec) > ampboundary[1].start): startspec = ispec #-cutting off wavelenth boundaries from startspec yy = psf.y(ispec, frame.wave) k = np.where(yy > ampboundary[0].start)[0] startwave0 = k[0] yy = psf.y(ispec, frame.wave) k = np.where(yy < ampboundary[0].stop)[0] endwave0 = k[-1] break else: startspec = None startwave0 = None endwave0 = None if startspec is not None: for ispec in range(frame.flux.shape[0])[::-1]: if np.all(psf.x(ispec) < ampboundary[1].stop): endspec = ispec #-cutting off wavelenth boundaries from startspec yy = psf.y(ispec, frame.wave) k = np.where(yy > ampboundary[0].start)[0] startwave1 = k[0] yy = psf.y(ispec, frame.wave) k = np.where(yy < ampboundary[0].stop)[0] endwave1 = k[-1] break else: endspec = None startwave1 = None endwave1 = None if startwave0 is not None and startwave1 is not None: startwave = max(startwave0, startwave1) else: startwave = None if endwave0 is not None and endwave1 is not None: endwave = min(endwave0, endwave1) else: endwave = None if endspec is not None: #endspec+=1 #- last entry exclusive in slice, so add 1 #endwave+=1 if endspec < leftmax: leftmax = endspec if startspec > rightmin: rightmin = startspec if endwave < bottommax: bottommax = endwave if startwave > topmin: topmin = startwave else: rightmin = 0 #- Only if no spec in right side of CCD. passing 0 to encertain valid data type. Nontype throws a type error in yaml.dump. #fiducialb=(slice(startspec,endspec,None),slice(startwave,endwave,None)) #- Note: y,x --> spec, wavelength #fidboundary.append(fiducialb) #- return pixboundary,fidboundary return leftmax, rightmin, bottommax, topmin
def fiducialregion(frame, psf): """ get the fiducial amplifier regions on the CCD pixel to fiber by wavelength space args: frame: desispec.frame.Frame object psf: desispec.psf.PSF like object """ from desispec.preproc import _parse_sec_keyword startspec = 0 #- will be None if don't have fibers on the right of the CCD. endspec = 499 #- will be None if don't have fibers on the right of the CCD startwave0 = 0 #- lower index for the starting fiber startwave1 = 0 #- lower index for the last fiber for the amp region endwave0 = frame.wave.shape[0] #- upper index for the starting fiber endwave1 = frame.wave.shape[ 0] #- upper index for the last fiber for that amp pixboundary = [] fidboundary = [] for kk in ['1', '2', '3', '4']: #- 4 amps #- get the amp region in pix ampboundary = _parse_sec_keyword(frame.meta["CCDSEC" + kk]) pixboundary.append(ampboundary) for ispec in range(frame.flux.shape[0]): if np.all(psf.x(ispec) > ampboundary[1].start): startspec = ispec #-cutting off wavelenth boundaries from startspec yy = psf.y(ispec, frame.wave) k = np.where(yy > ampboundary[0].start)[0] startwave0 = k[0] yy = psf.y(ispec, frame.wave) k = np.where(yy < ampboundary[0].stop)[0] endwave0 = k[-1] break else: startspec = None startwave0 = None endwave0 = None if startspec is not None: for ispec in range(frame.flux.shape[0])[::-1]: if np.all(psf.x(ispec) < ampboundary[1].stop): endspec = ispec #-cutting off wavelenth boundaries from startspec yy = psf.y(ispec, frame.wave) k = np.where(yy > ampboundary[0].start)[0] startwave1 = k[0] yy = psf.y(ispec, frame.wave) k = np.where(yy < ampboundary[0].stop)[0] endwave1 = k[-1] break else: endspec = None startwave1 = None endwave1 = None startwave = max(startwave0, startwave1) endwave = min(endwave0, endwave1) if endspec is not None: endspec += 1 #- last entry exclusive in slice, so add 1 endwave += 1 fiducialb = (slice(startspec, endspec, None), slice(startwave, endwave, None)) #- Note: y,x --> spec, wavelength fidboundary.append(fiducialb) return pixboundary, fidboundary
def setUp(self): #- Create temporary calib directory self.testDir = os.path.join(os.environ['HOME'], 'ql_test_io') calibDir = os.path.join(self.testDir, 'ql_calib') if not os.path.exists(calibDir): os.makedirs(calibDir) #- Generate calib data for camera in ['b0', 'r0', 'z0']: #- Fiberflat has to exist but can be a dummpy file filename = '{}/fiberflat-{}.fits'.format(calibDir, camera) fx = open(filename, 'w') fx.write('fiberflat file') fx.close() #- PSF has to be real file psffile = '{}/psf-{}.fits'.format(calibDir, camera) example_psf = resource_filename( 'desispec', 'test/data/ql/psf-{}.fits'.format(camera)) shutil.copy(example_psf, psffile) #- Copy test calibration-data.yaml file specdir = calibDir + "spec/sp0" if not os.path.isdir(specdir): os.makedirs(specdir) for c in "brz": shutil.copy( resource_filename('desispec', 'test/data/ql/{}0.yaml'.format(c)), os.path.join(specdir, "{}0.yaml".format(c))) #- Set calibration environment variable os.environ['DESI_SPECTRO_CALIB'] = calibDir self.rawfile = os.path.join(self.testDir, 'test-raw-abcd.fits') self.pixfile = os.path.join(self.testDir, 'test-pix-abcd.fits') self.config = {} #- rawimage hdr = dict() hdr['CAMERA'] = 'b0' hdr['DATE-OBS'] = '2018-09-23T08:17:03.988' #- Dimensions per amp, not full 4-quad CCD ny = self.ny = 500 nx = self.nx = 400 noverscan = nover = 50 hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx + nover]) hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx + nover:nx + 2 * nover]) hdr['DATASEC2'] = xy2hdr(np.s_[0:ny, nx + 2 * nover:nx + 2 * nover + nx]) hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx + nx]) hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nover]) hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx]) hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx]) hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + nover:nx + 2 * nover]) hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + 2 * nover:nx + 2 * nover + nx]) hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nx]) hdr['NIGHT'] = '20180923' hdr['EXPID'] = 1 hdr['FLAVOR'] = 'dark' rawimage = np.zeros((2 * ny, 2 * nx + 2 * noverscan)) offset = {'1': 100.0, '2': 100.5, '3': 50.3, '4': 200.4} gain = {'1': 1.0, '2': 1.5, '3': 0.8, '4': 1.2} rdnoise = {'1': 2.0, '2': 2.2, '3': 2.4, '4': 2.6} quad = { '1': np.s_[0:ny, 0:nx], '2': np.s_[0:ny, nx:nx + nx], '3': np.s_[ny:ny + ny, 0:nx], '4': np.s_[ny:ny + ny, nx:nx + nx], } for amp in ('1', '2', '3', '4'): hdr['GAIN' + amp] = gain[amp] hdr['RDNOISE' + amp] = rdnoise[amp] xy = _parse_sec_keyword(hdr['BIASSEC' + amp]) shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start] rawimage[xy] += offset[amp] rawimage[xy] += np.random.normal(scale=rdnoise[amp], size=shape) / gain[amp] xy = _parse_sec_keyword(hdr['DATASEC' + amp]) shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start] rawimage[xy] += offset[amp] rawimage[xy] += np.random.normal(scale=rdnoise[amp], size=shape) / gain[amp] #- raw data are integers, not floats rawimg = rawimage.astype(np.int32) self.expid = hdr["EXPID"] self.camera = hdr["CAMERA"] #- Confirm that all regions were correctly offset assert not np.any(rawimage == 0.0) hdr['DOSVER'] = 'SIM' hdr['FEEVER'] = 'SIM' hdr['DETECTOR'] = 'SIM' desispec.io.write_raw(self.rawfile, rawimg, hdr, camera=self.camera) self.rawimage = fits.open(self.rawfile)
def setUp(self): self.calibfile = 'test-calib-askjapqwhezcpasehadfaqp.fits' self.rawfile = 'test-raw-askjapqwhezcpasehadfaqp.fits' self.pixfile = 'test-pix-askjapqwhezcpasehadfaqp.fits' hdr = dict() hdr['CAMERA'] = 'b0' hdr['DATE-OBS'] = '2018-09-23T08:17:03.988' #- [x,y] 1-indexed for FITS; in reality the amps will be symmetric #- but the header definitions don't require that to make sure we are #- getting dimensions correct #- Dimensions per amp, not full 4-quad CCD self.ny = ny = 500 self.nx = nx = 400 self.noverscan = nover = 50 #- BIASSEC = overscan region in raw image #- DATASEC = data region in raw image #- CCDSEC = where should this go in output hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx+nover]) hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx+nover:nx+2*nover]) hdr['DATASEC2'] = xy2hdr(np.s_[0:ny, nx+2*nover:nx+2*nover+nx]) hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx+nx]) hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny+ny, nx:nx+nover]) hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny+ny, 0:nx]) hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny+ny, 0:nx]) hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny+ny, nx+nover:nx+2*nover]) hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny+ny, nx+2*nover:nx+2*nover+nx]) hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny+ny, nx:nx+nx]) hdr['NIGHT'] = '20150102' hdr['EXPID'] = 1 self.header = hdr self.rawimage = np.zeros((2*self.ny, 2*self.nx+2*self.noverscan)) self.offset = {'1':100.0, '2':100.5, '3':50.3, '4':200.4} self.gain = {'1':1.0, '2':1.5, '3':0.8, '4':1.2} self.rdnoise = {'1':2.0, '2':2.2, '3':2.4, '4':2.6} self.quad = { '1': np.s_[0:ny, 0:nx], '2': np.s_[0:ny, nx:nx+nx], '3': np.s_[ny:ny+ny, 0:nx], '4': np.s_[ny:ny+ny, nx:nx+nx], } for amp in ('1', '2', '3', '4'): self.header['GAIN'+amp] = self.gain[amp] self.header['RDNOISE'+amp] = self.rdnoise[amp] xy = _parse_sec_keyword(hdr['BIASSEC'+amp]) shape = [xy[0].stop-xy[0].start, xy[1].stop-xy[1].start] self.rawimage[xy] += self.offset[amp] self.rawimage[xy] += np.random.normal(scale=self.rdnoise[amp], size=shape)/self.gain[amp] xy = _parse_sec_keyword(hdr['DATASEC'+amp]) shape = [xy[0].stop-xy[0].start, xy[1].stop-xy[1].start] self.rawimage[xy] += self.offset[amp] self.rawimage[xy] += np.random.normal(scale=self.rdnoise[amp], size=shape)/self.gain[amp] #- raw data are integers, not floats self.rawimage = self.rawimage.astype(np.int32) #- Confirm that all regions were correctly offset assert not np.any(self.rawimage == 0.0)
def setUp(self): #- Create temporary calib directory self.calibdir = os.path.join(os.environ['HOME'], 'preproc_unit_test') if not os.path.exists(self.calibdir): os.makedirs(self.calibdir) #- Copy test calibration-data.yaml file specdir = os.path.join(self.calibdir, "spec/sp0") if not os.path.isdir(specdir): os.makedirs(specdir) for c in "brz": shutil.copy( resource_filename('desispec', 'test/data/ql/{}0.yaml'.format(c)), os.path.join(specdir, "{}0.yaml".format(c))) #- Set calibration environment variable os.environ["DESI_SPECTRO_CALIB"] = self.calibdir self.calibfile = os.path.join( self.calibdir, 'test-calib-askjapqwhezcpasehadfaqp.fits') self.rawfile = os.path.join(self.calibdir, 'test-raw-askjapqwhezcpasehadfaqp.fits') self.pixfile = os.path.join(self.calibdir, 'test-pix-askjapqwhezcpasehadfaqp.fits') primary_hdr = dict() primary_hdr['DATE-OBS'] = '2018-09-23T08:17:03.988' primary_hdr['DOSVER'] = 'SIM' # ICS version hdr = dict() hdr['CAMERA'] = 'b0' hdr['DETECTOR'] = 'SIM' # CCD chip identifier hdr['FEEVER'] = 'SIM' # readout electronic #- [x,y] 1-indexed for FITS; in reality the amps will be symmetric #- but the header definitions don't require that to make sure we are #- getting dimensions correct #- Dimensions per amp, not full 4-quad CCD self.ny = ny = 500 self.nx = nx = 400 self.noverscan = nover = 50 #- BIASSEC = overscan region in raw image #- DATASEC = data region in raw image #- CCDSEC = where should this go in output hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx + nover]) hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx + nover:nx + 2 * nover]) hdr['DATASEC2'] = xy2hdr(np.s_[0:ny, nx + 2 * nover:nx + 2 * nover + nx]) hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx + nx]) hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nover]) hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx]) hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx]) hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + nover:nx + 2 * nover]) hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + 2 * nover:nx + 2 * nover + nx]) hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nx]) hdr['NIGHT'] = '20150102' hdr['EXPID'] = 1 # add to header the minimal set of keywords needed to # identify the config in the ccd_calibration.yaml file self.primary_header = primary_hdr self.header = hdr self.rawimage = np.zeros( (2 * self.ny, 2 * self.nx + 2 * self.noverscan)) self.offset = {'1': 100.0, '2': 100.5, '3': 50.3, '4': 200.4} self.gain = {'1': 1.0, '2': 1.5, '3': 0.8, '4': 1.2} self.rdnoise = {'1': 2.0, '2': 2.2, '3': 2.4, '4': 2.6} self.quad = { '1': np.s_[0:ny, 0:nx], '2': np.s_[0:ny, nx:nx + nx], '3': np.s_[ny:ny + ny, 0:nx], '4': np.s_[ny:ny + ny, nx:nx + nx], } for amp in ('1', '2', '3', '4'): self.header['GAIN' + amp] = self.gain[amp] self.header['RDNOISE' + amp] = self.rdnoise[amp] xy = _parse_sec_keyword(hdr['BIASSEC' + amp]) shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start] self.rawimage[xy] += self.offset[amp] self.rawimage[xy] += np.random.normal(scale=self.rdnoise[amp], size=shape) / self.gain[amp] xy = _parse_sec_keyword(hdr['DATASEC' + amp]) shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start] self.rawimage[xy] += self.offset[amp] self.rawimage[xy] += np.random.normal(scale=self.rdnoise[amp], size=shape) / self.gain[amp] #- raw data are integers, not floats self.rawimage = self.rawimage.astype(np.int32) #- Confirm that all regions were correctly offset assert not np.any(self.rawimage == 0.0)
def compute(args): nbNeigh = args.nb_neighbours cameras = args.cameras sections = args.sections assert len(args.infiles)%2. == 0. ### Where to save dicCor = {} for c in cameras: dicCor[c] = {} for i1,sec1 in enumerate(sections): for sec2 in sections[:i1+1]: dicCor[c][sec1+'_'+sec2] = sp.zeros( (2*nbNeigh+1,2*nbNeigh+1) ) ### Compute for p in range(int(len(args.infiles)//2.)): p1 = args.infiles[2*p] p2 = args.infiles[2*p+1] h1 = fitsio.FITS(p1) h2 = fitsio.FITS(p2) for c in cameras: ### Read data head1 = h1[c].read_header() data1 = h1[c].read() head2 = h2[c].read_header() data2 = h2[c].read() dic = {} for sec in sections: w1 = _parse_sec_keyword(head1['DATASEC'+sec]) w2 = _parse_sec_keyword(head2['DATASEC'+sec]) td1 = data1[w1].astype('float64')-data2[w2].astype('float64') std1 = sp.std(td1) if sec=='B': td1 = td1[:,::-1] elif sec=='C': td1 = td1[::-1,:] elif sec=='D': td1 = td1[::-1,::-1] mask = (td1<-5.*std1) | (td1>5.*std1) td1 -= td1[~mask].mean() td1[mask] = 0. dic[sec] = td1.copy() ### Compute correlation coef for i1,sec1 in enumerate(dic.keys()): for sec2 in list(dic.keys())[:i1+1]: print(p, c, sec1,sec2) dicCor[c][sec1+'_'+sec2] += compute_correlation(dic[sec1],dic[sec2],nbNeigh) h1.close() h2.close() ### Write to ascii for c in cameras: for i1,sec1 in enumerate(sections): for sec2 in sections[:i1+1]: dicCor[c][sec1+'_'+sec2] /= len(args.infiles)/2. sp.savetxt(args.outfile+'/correlation_{}_{}_{}.txt'.format(c,sec1,sec2),dicCor[c][sec1+'_'+sec2]) return
def testCalcXWSigma(self): qa = QA.Calc_XWSigma('xwsigma', self.config) #- create larger rawimage to incorporate sky peaks hdr = dict() hdr['CAMERA'] = 'z1' hdr['DATE-OBS'] = '2018-09-23T08:17:03.988' #- Dimensions per amp ny = self.ny = 1800 nx = self.nx = 1800 noverscan = nover = 50 hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx + nover]) hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx + nover:nx + 2 * nover]) hdr['DATASEC2'] = xy2hdr(np.s_[0:ny, nx + 2 * nover:nx + 2 * nover + nx]) hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx + nx]) hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nover]) hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx]) hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx]) hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + nover:nx + 2 * nover]) hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + 2 * nover:nx + 2 * nover + nx]) hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nx]) hdr['NIGHT'] = '20180923' hdr['EXPID'] = 1 hdr['PROGRAM'] = 'dark' hdr['FLAVOR'] = 'science' rawimage = np.zeros((2 * ny, 2 * nx + 2 * noverscan)) offset = {'1': 100.0, '2': 100.5, '3': 50.3, '4': 200.4} gain = {'1': 1.0, '2': 1.5, '3': 0.8, '4': 1.2} rdnoise = {'1': 2.0, '2': 2.2, '3': 2.4, '4': 2.6} quad = { '1': np.s_[0:ny, 0:nx], '2': np.s_[0:ny, nx:nx + nx], '3': np.s_[ny:ny + ny, 0:nx], '4': np.s_[ny:ny + ny, nx:nx + nx], } for amp in ('1', '2', '3', '4'): hdr['GAIN' + amp] = gain[amp] hdr['RDNOISE' + amp] = rdnoise[amp] xy = _parse_sec_keyword(hdr['BIASSEC' + amp]) shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start] rawimage[xy] += offset[amp] rawimage[xy] += np.random.normal(scale=rdnoise[amp], size=shape) / gain[amp] xy = _parse_sec_keyword(hdr['DATASEC' + amp]) shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start] rawimage[xy] += offset[amp] rawimage[xy] += np.random.normal(scale=rdnoise[amp], size=shape) / gain[amp] #- set CCD parameters self.ccdsec1 = hdr["CCDSEC1"] self.ccdsec2 = hdr["CCDSEC2"] self.ccdsec3 = hdr["CCDSEC3"] self.ccdsec4 = hdr["CCDSEC4"] #- raw data are integers, not floats rawimg = rawimage.astype(np.int32) self.expid = hdr["EXPID"] self.camera = hdr["CAMERA"] #- read psf, should use specter.PSF.load_psf instead of desispec.PSF(), otherwise need to create a psfboot somewhere. psf = self.psf #- make the test pixfile, fibermap file img_pix = rawimg img_ivar = np.ones_like(img_pix) / 3.0**2 img_mask = np.zeros(img_pix.shape, dtype=np.uint32) img_mask[200] = 1 #- manually insert sky peaks for xwsigma zpeaks = np.array([8401.5, 8432.4, 8467.5, 9479.4, 9505.6, 9521.8]) fibers = np.arange(5) peak1 = psf.xy(fibers, zpeaks[0]) pix1 = np.rint(peak1) peak2 = psf.xy(fibers, zpeaks[1]) pix2 = np.rint(peak2) peak3 = psf.xy(fibers, zpeaks[2]) pix3 = np.rint(peak3) peak4 = psf.xy(fibers, zpeaks[3]) pix4 = np.rint(peak4) peak5 = psf.xy(fibers, zpeaks[4]) pix5 = np.rint(peak5) peak6 = psf.xy(fibers, zpeaks[5]) pix6 = np.rint(peak6) for i in range(len(fibers)): img_pix[pix1[1][i]][pix1[0][i]] = 10000. img_pix[pix1[1][i] + 1][pix1[0][i]] = 5000. img_pix[pix1[1][i]][pix1[0][i] + 1] = 5000. img_pix[pix1[1][i] - 1][pix1[0][i]] = 5000. img_pix[pix1[1][i]][pix1[0][i] - 1] = 5000. img_pix[pix1[1][i] + 2][pix1[0][i]] = 500. img_pix[pix1[1][i]][pix1[0][i] + 2] = 500. img_pix[pix1[1][i] - 2][pix1[0][i]] = 500. img_pix[pix1[1][i]][pix1[0][i] - 2] = 500. img_pix[pix1[1][i] + 3][pix1[0][i]] = 220. img_pix[pix1[1][i]][pix1[0][i] + 3] = 220. img_pix[pix1[1][i] - 3][pix1[0][i]] = 220. img_pix[pix1[1][i]][pix1[0][i] - 3] = 220. img_pix[pix2[1][i]][pix2[0][i]] = 10000. img_pix[pix2[1][i] + 1][pix2[0][i]] = 5000. img_pix[pix2[1][i]][pix2[0][i] + 1] = 5000. img_pix[pix2[1][i] - 1][pix2[0][i]] = 5000. img_pix[pix2[1][i]][pix2[0][i] - 1] = 5000. img_pix[pix2[1][i] + 2][pix2[0][i]] = 500. img_pix[pix2[1][i]][pix2[0][i] + 2] = 500. img_pix[pix2[1][i] - 2][pix2[0][i]] = 500. img_pix[pix2[1][i]][pix2[0][i] - 2] = 500. img_pix[pix2[1][i] + 3][pix2[0][i]] = 220. img_pix[pix2[1][i]][pix2[0][i] + 3] = 220. img_pix[pix2[1][i] - 3][pix2[0][i]] = 220. img_pix[pix2[1][i]][pix2[0][i] - 3] = 220. img_pix[pix3[1][i]][pix3[0][i]] = 10000. img_pix[pix3[1][i] + 1][pix3[0][i]] = 5000. img_pix[pix3[1][i]][pix3[0][i] + 1] = 5000. img_pix[pix3[1][i] - 1][pix3[0][i]] = 5000. img_pix[pix3[1][i]][pix3[0][i] - 1] = 5000. img_pix[pix3[1][i] + 2][pix3[0][i]] = 500. img_pix[pix3[1][i]][pix3[0][i] + 2] = 500. img_pix[pix3[1][i] - 2][pix3[0][i]] = 500. img_pix[pix3[1][i]][pix3[0][i] - 2] = 500. img_pix[pix3[1][i] + 3][pix3[0][i]] = 220. img_pix[pix3[1][i]][pix3[0][i] + 3] = 220. img_pix[pix3[1][i] - 3][pix3[0][i]] = 220. img_pix[pix3[1][i]][pix3[0][i] - 3] = 220. img_pix[pix4[1][i]][pix4[0][i]] = 10000. img_pix[pix4[1][i] + 1][pix4[0][i]] = 5000. img_pix[pix4[1][i]][pix4[0][i] + 1] = 5000. img_pix[pix4[1][i] - 1][pix4[0][i]] = 5000. img_pix[pix4[1][i]][pix4[0][i] - 1] = 5000. img_pix[pix4[1][i] + 2][pix4[0][i]] = 500. img_pix[pix4[1][i]][pix4[0][i] + 2] = 500. img_pix[pix4[1][i] - 2][pix4[0][i]] = 500. img_pix[pix4[1][i]][pix4[0][i] - 2] = 500. img_pix[pix4[1][i] + 3][pix4[0][i]] = 220. img_pix[pix4[1][i]][pix4[0][i] + 3] = 220. img_pix[pix4[1][i] - 3][pix4[0][i]] = 220. img_pix[pix4[1][i]][pix4[0][i] - 3] = 220. img_pix[pix5[1][i]][pix5[0][i]] = 10000. img_pix[pix5[1][i] + 1][pix5[0][i]] = 5000. img_pix[pix5[1][i]][pix5[0][i] + 1] = 5000. img_pix[pix5[1][i] - 1][pix5[0][i]] = 5000. img_pix[pix5[1][i]][pix5[0][i] - 1] = 5000. img_pix[pix5[1][i] + 2][pix5[0][i]] = 500. img_pix[pix5[1][i]][pix5[0][i] + 2] = 500. img_pix[pix5[1][i] - 2][pix5[0][i]] = 500. img_pix[pix5[1][i]][pix5[0][i] - 2] = 500. img_pix[pix5[1][i] + 3][pix5[0][i]] = 220. img_pix[pix5[1][i]][pix5[0][i] + 3] = 220. img_pix[pix5[1][i] - 3][pix5[0][i]] = 220. img_pix[pix5[1][i]][pix5[0][i] - 3] = 220. img_pix[pix6[1][i]][pix6[0][i]] = 10000. img_pix[pix6[1][i] + 1][pix6[0][i]] = 5000. img_pix[pix6[1][i]][pix6[0][i] + 1] = 5000. img_pix[pix6[1][i] - 1][pix6[0][i]] = 5000. img_pix[pix6[1][i]][pix6[0][i] - 1] = 5000. img_pix[pix6[1][i] + 2][pix6[0][i]] = 500. img_pix[pix6[1][i]][pix6[0][i] + 2] = 500. img_pix[pix6[1][i] - 2][pix6[0][i]] = 500. img_pix[pix6[1][i]][pix6[0][i] - 2] = 500. img_pix[pix6[1][i] + 3][pix6[0][i]] = 220. img_pix[pix6[1][i]][pix6[0][i] + 3] = 220. img_pix[pix6[1][i] - 3][pix6[0][i]] = 220. img_pix[pix6[1][i]][pix6[0][i] - 3] = 220. self.image = desispec.image.Image(img_pix, img_ivar, img_mask, camera='z1', meta=hdr) desispec.io.write_image(self.pixfile, self.image) self.fibermap = desispec.io.empty_fibermap(5) desispec.io.write_fibermap(self.fibermapfile, self.fibermap) inp = self.image qargs = {} qargs["PSFFile"] = self.psf qargs["FiberMap"] = self.fibermap qargs["camera"] = self.camera qargs["expid"] = self.expid qargs["amps"] = False qargs["paname"] = "abc" resl = qa(inp, **qargs) self.assertTrue(np.all(resl["METRICS"]["XSIGMA"]) > 0)
def setUp(self): self.rawfile = 'test-raw-abcd.fits' self.pixfile = 'test-pix-abcd.fits' self.xwfile = 'test-xw-abcd.fits' self.framefile = 'test-frame-abcd.fits' self.fibermapfile = 'test-fibermap-abcd.fits' self.skyfile = 'test-sky-abcd.fits' self.qafile = 'test_qa.yaml' self.qafig = 'test_qa.png' #- use specter psf for this test self.psffile = resource_filename('specter', 'test/t/psf-monospot.fits') #self.psffile=os.environ['DESIMODEL']+'/data/specpsf/psf-b.fits' self.config = {} #- rawimage hdr = dict() hdr['CAMERA'] = 'z1' hdr['DATE-OBS'] = '2018-09-23T08:17:03.988' #- Dimensions per amp ny = self.ny = 500 nx = self.nx = 400 noverscan = nover = 50 hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx + nover]) hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx + nover:nx + 2 * nover]) hdr['DATASEC2'] = xy2hdr(np.s_[0:ny, nx + 2 * nover:nx + 2 * nover + nx]) hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx + nx]) hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nover]) hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx]) hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx]) hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + nover:nx + 2 * nover]) hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + 2 * nover:nx + 2 * nover + nx]) hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nx]) hdr['NIGHT'] = '20180923' hdr['EXPID'] = 1 hdr['PROGRAM'] = 'dark' hdr['FLAVOR'] = 'science' hdr['EXPTIME'] = 100.0 rawimage = np.zeros((2 * ny, 2 * nx + 2 * noverscan)) offset = {'1': 100.0, '2': 100.5, '3': 50.3, '4': 200.4} gain = {'1': 1.0, '2': 1.5, '3': 0.8, '4': 1.2} rdnoise = {'1': 2.0, '2': 2.2, '3': 2.4, '4': 2.6} quad = { '1': np.s_[0:ny, 0:nx], '2': np.s_[0:ny, nx:nx + nx], '3': np.s_[ny:ny + ny, 0:nx], '4': np.s_[ny:ny + ny, nx:nx + nx], } for amp in ('1', '2', '3', '4'): hdr['GAIN' + amp] = gain[amp] hdr['RDNOISE' + amp] = rdnoise[amp] xy = _parse_sec_keyword(hdr['BIASSEC' + amp]) shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start] rawimage[xy] += offset[amp] rawimage[xy] += np.random.normal(scale=rdnoise[amp], size=shape) / gain[amp] xy = _parse_sec_keyword(hdr['DATASEC' + amp]) shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start] rawimage[xy] += offset[amp] rawimage[xy] += np.random.normal(scale=rdnoise[amp], size=shape) / gain[amp] #- set CCD parameters self.ccdsec1 = hdr["CCDSEC1"] self.ccdsec2 = hdr["CCDSEC2"] self.ccdsec3 = hdr["CCDSEC3"] self.ccdsec4 = hdr["CCDSEC4"] #- raw data are integers, not floats rawimg = rawimage.astype(np.int32) self.expid = hdr["EXPID"] self.camera = hdr["CAMERA"] #- Confirm that all regions were correctly offset assert not np.any(rawimage == 0.0) #- write to the rawfile and read it in QA test hdr['DOSVER'] = 'SIM' hdr['FEEVER'] = 'SIM' hdr['DETECTOR'] = 'SIM' desispec.io.write_raw(self.rawfile, rawimg, hdr, camera=self.camera) self.rawimage = fits.open(self.rawfile) #- read psf, should use specter.PSF.load_psf instead of desispec.PSF(), otherwise need to create a psfboot somewhere. psf = self.psf = load_psf(self.psffile) #- make the test pixfile, fibermap file img_pix = rawimg img_ivar = np.ones_like(img_pix) / 3.0**2 img_mask = np.zeros(img_pix.shape, dtype=np.uint32) img_mask[200] = 1 self.image = desispec.image.Image(img_pix, img_ivar, img_mask, camera='z1', meta=hdr) desispec.io.write_image(self.pixfile, self.image) self.fibermap = desispec.io.empty_fibermap(30) self.fibermap['OBJTYPE'][::2] = 'ELG' self.fibermap['OBJTYPE'][::3] = 'STD' self.fibermap['OBJTYPE'][::5] = 'QSO' self.fibermap['OBJTYPE'][::9] = 'LRG' self.fibermap['OBJTYPE'][::7] = 'SKY' #- add a filter and arbitrary magnitude self.fibermap['MAG'][:29] = np.tile(np.random.uniform(18, 20, 29), 5).reshape(29, 5) #- Last fiber left self.fibermap['FILTER'][:29] = np.tile( ['DECAM_R', '..', '..', '..', '..'], (29, 1)) #- last fiber left desispec.io.write_fibermap(self.fibermapfile, self.fibermap) #- make a test frame file self.night = hdr['NIGHT'] self.nspec = nspec = 30 wave = np.arange(7600.0, 9800.0, 1.0) #- z channel nwave = self.nwave = len(wave) flux = np.random.uniform(size=(nspec, nwave)) + 100. ivar = np.ones_like(flux) resolution_data = np.ones((nspec, 13, nwave)) self.frame = desispec.frame.Frame(wave, flux, ivar, resolution_data=resolution_data, fibermap=self.fibermap) self.frame.meta = dict(CAMERA=self.camera, PROGRAM='dark', FLAVOR='science', NIGHT=self.night, EXPID=self.expid, EXPTIME=100, CCDSEC1=self.ccdsec1, CCDSEC2=self.ccdsec2, CCDSEC3=self.ccdsec3, CCDSEC4=self.ccdsec4) desispec.io.write_frame(self.framefile, self.frame) #- make a skymodel sky = np.ones_like(self.frame.flux) * 0.5 skyivar = np.ones_like(sky) self.mask = np.zeros(sky.shape, dtype=np.uint32) self.skymodel = desispec.sky.SkyModel(wave, sky, skyivar, self.mask) self.skyfile = desispec.io.write_sky(self.skyfile, self.skymodel) #- Make a dummy boundary map for wavelength-flux in pixel space self.map2pix = {} self.map2pix["LEFT_MAX_FIBER"] = 14 self.map2pix["RIGHT_MIN_FIBER"] = 17 self.map2pix["BOTTOM_MAX_WAVE_INDEX"] = 900 self.map2pix["TOP_MIN_WAVE_INDEX"] = 1100
def setUp(self): self.rawfile = 'test-raw-abcd.fits' self.pixfile = 'test-pix-abcd.fits' self.config = {} #- rawimage hdr = dict() hdr['CAMERA'] = 'b1' hdr['DATE-OBS'] = '2018-09-23T08:17:03.988' #- Dimensions per amp, not full 4-quad CCD ny = self.ny = 500 nx = self.nx = 400 noverscan = nover = 50 hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx + nover]) hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx + nover:nx + 2 * nover]) hdr['DATASEC2'] = xy2hdr(np.s_[0:ny, nx + 2 * nover:nx + 2 * nover + nx]) hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx + nx]) hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nover]) hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx]) hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx]) hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + nover:nx + 2 * nover]) hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + 2 * nover:nx + 2 * nover + nx]) hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nx]) hdr['NIGHT'] = '20180923' hdr['EXPID'] = 1 hdr['FLAVOR'] = 'dark' rawimage = np.zeros((2 * ny, 2 * nx + 2 * noverscan)) offset = {'1': 100.0, '2': 100.5, '3': 50.3, '4': 200.4} gain = {'1': 1.0, '2': 1.5, '3': 0.8, '4': 1.2} rdnoise = {'1': 2.0, '2': 2.2, '3': 2.4, '4': 2.6} quad = { '1': np.s_[0:ny, 0:nx], '2': np.s_[0:ny, nx:nx + nx], '3': np.s_[ny:ny + ny, 0:nx], '4': np.s_[ny:ny + ny, nx:nx + nx], } for amp in ('1', '2', '3', '4'): hdr['GAIN' + amp] = gain[amp] hdr['RDNOISE' + amp] = rdnoise[amp] xy = _parse_sec_keyword(hdr['BIASSEC' + amp]) shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start] rawimage[xy] += offset[amp] rawimage[xy] += np.random.normal(scale=rdnoise[amp], size=shape) / gain[amp] xy = _parse_sec_keyword(hdr['DATASEC' + amp]) shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start] rawimage[xy] += offset[amp] rawimage[xy] += np.random.normal(scale=rdnoise[amp], size=shape) / gain[amp] #- raw data are integers, not floats rawimg = rawimage.astype(np.int32) self.expid = hdr["EXPID"] self.camera = hdr["CAMERA"] #- Confirm that all regions were correctly offset assert not np.any(rawimage == 0.0) hdr['DOSVER'] = 'SIM' hdr['FEEVER'] = 'SIM' hdr['DETECTOR'] = 'SIM' desispec.io.write_raw(self.rawfile, rawimg, hdr, camera=self.camera) self.rawimage = fits.open(self.rawfile)
def setUp(self): self.calibfile = 'test-calib-askjapqwhezcpasehadfaqp.fits' self.rawfile = 'test-raw-askjapqwhezcpasehadfaqp.fits' self.pixfile = 'test-pix-askjapqwhezcpasehadfaqp.fits' hdr = dict() hdr['CAMERA'] = 'b0' hdr['DATE-OBS'] = '2018-09-23T08:17:03.988' #- [x,y] 1-indexed for FITS; in reality the amps will be symmetric #- but the header definitions don't require that to make sure we are #- getting dimensions correct #- Dimensions per amp, not full 4-quad CCD self.ny = ny = 500 self.nx = nx = 400 self.noverscan = nover = 50 #- BIASSEC = overscan region in raw image #- DATASEC = data region in raw image #- CCDSEC = where should this go in output hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx + nover]) hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx + nover:nx + 2 * nover]) hdr['DATASEC2'] = xy2hdr(np.s_[0:ny, nx + 2 * nover:nx + 2 * nover + nx]) hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx + nx]) hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nover]) hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx]) hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny + ny, 0:nx]) hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + nover:nx + 2 * nover]) hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny + ny, nx + 2 * nover:nx + 2 * nover + nx]) hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny + ny, nx:nx + nx]) hdr['NIGHT'] = '20150102' hdr['EXPID'] = 1 self.header = hdr self.rawimage = np.zeros( (2 * self.ny, 2 * self.nx + 2 * self.noverscan)) self.offset = {'1': 100.0, '2': 100.5, '3': 50.3, '4': 200.4} self.gain = {'1': 1.0, '2': 1.5, '3': 0.8, '4': 1.2} self.rdnoise = {'1': 2.0, '2': 2.2, '3': 2.4, '4': 2.6} self.quad = { '1': np.s_[0:ny, 0:nx], '2': np.s_[0:ny, nx:nx + nx], '3': np.s_[ny:ny + ny, 0:nx], '4': np.s_[ny:ny + ny, nx:nx + nx], } for amp in ('1', '2', '3', '4'): self.header['GAIN' + amp] = self.gain[amp] self.header['RDNOISE' + amp] = self.rdnoise[amp] xy = _parse_sec_keyword(hdr['BIASSEC' + amp]) shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start] self.rawimage[xy] += self.offset[amp] self.rawimage[xy] += np.random.normal(scale=self.rdnoise[amp], size=shape) / self.gain[amp] xy = _parse_sec_keyword(hdr['DATASEC' + amp]) shape = [xy[0].stop - xy[0].start, xy[1].stop - xy[1].start] self.rawimage[xy] += self.offset[amp] self.rawimage[xy] += np.random.normal(scale=self.rdnoise[amp], size=shape) / self.gain[amp] #- raw data are integers, not floats self.rawimage = self.rawimage.astype(np.int32) #- Confirm that all regions were correctly offset assert not np.any(self.rawimage == 0.0)
tmp = sub[:,1:] - sub[:,:-1] sub[:,1:] = tmp sub[:,0] = 0 rms_scale = 1./np.sqrt(2.) i=0 x=np.zeros(3+4*6).astype(float) x[i] = expid ; i += 1 x[i] = mjdobs ; i += 1 x[i] = dateobs ; i += 1 for a,amp in enumerate(['A','B','C','D']) : gain = 1. if cfinder is not None and cfinder.haskey("GAIN"+amp) : gain=cfinder.value("GAIN"+amp) x[i] = mean(img[_parse_sec_keyword(header["BIASSEC"+amp])],gain=gain); i+=1 x[i] = rms_scale*rms(sub[_parse_sec_keyword(header["BIASSEC"+amp])],gain=gain); i+=1 if "ORSEC"+amp in header : x[i] = mean(img[_parse_sec_keyword(header["ORSEC"+amp])],gain=gain); i+=1 x[i] = rms_scale*rms(sub[_parse_sec_keyword(header["ORSEC"+amp])],gain=gain); i+=1 else : i += 2 x[i] = mean(img[_parse_sec_keyword(header["DATASEC"+amp])],gain=gain); i+=1 x[i] = rms_scale*rms(sub[_parse_sec_keyword(header["DATASEC"+amp])],gain=gain); i+=1 print("assuming gain= {:3.2f} for amp {}, ccd rms= {:3.2f}".format(gain,amp,x[i-1])) sys.stdout.flush() xx.append(x) if args.outfile is not None : xx=np.vstack(xx) np.savetxt(args.outfile,xx,header=line)
def setUp(self): #- use specter psf for this test self.psffile=resource_filename('specter', 'test/t/psf-monospot.fits') #self.psffile=os.environ['DESIMODEL']+'/data/specpsf/psf-b.fits' self.config={"kwargs":{ "refKey":None, "param":{}, "qso_resid":None }} #- rawimage hdr = dict() hdr['CAMERA'] = 'z1' hdr['DATE-OBS'] = '2018-09-23T08:17:03.988' hdr['PROGRAM'] = 'dark' hdr['EXPTIME'] = 100 #- Dimensions per amp ny = self.ny = 500 nx = self.nx = 400 noverscan = nover = 50 hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx+nover]) hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx+nover:nx+2*nover]) hdr['DATASEC2'] = xy2hdr(np.s_[0:ny, nx+2*nover:nx+2*nover+nx]) hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx+nx]) hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny+ny, nx:nx+nover]) hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny+ny, 0:nx]) hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny+ny, 0:nx]) hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny+ny, nx+nover:nx+2*nover]) hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny+ny, nx+2*nover:nx+2*nover+nx]) hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny+ny, nx:nx+nx]) hdr['NIGHT'] = '20180923' hdr['EXPID'] = 1 hdr['PROGRAM'] = 'dark' hdr['FLAVOR'] = 'science' hdr['EXPTIME'] = 100.0 rawimage = np.zeros((2*ny, 2*nx+2*noverscan)) offset = {'1':100.0, '2':100.5, '3':50.3, '4':200.4} gain = {'1':1.0, '2':1.5, '3':0.8, '4':1.2} rdnoise = {'1':2.0, '2':2.2, '3':2.4, '4':2.6} obsrdn = {'1':3.4, '2':3.3, '3':3.6, '4':3.3} quad = { '1': np.s_[0:ny, 0:nx], '2': np.s_[0:ny, nx:nx+nx], '3': np.s_[ny:ny+ny, 0:nx], '4': np.s_[ny:ny+ny, nx:nx+nx], } for amp in ('1', '2', '3', '4'): hdr['GAIN'+amp] = gain[amp] hdr['RDNOISE'+amp] = rdnoise[amp] hdr['OBSRDN'+amp] = obsrdn[amp] xy = _parse_sec_keyword(hdr['BIASSEC'+amp]) shape = [xy[0].stop-xy[0].start, xy[1].stop-xy[1].start] rawimage[xy] += offset[amp] rawimage[xy] += np.random.normal(scale=rdnoise[amp], size=shape)/gain[amp] xy = _parse_sec_keyword(hdr['DATASEC'+amp]) shape = [xy[0].stop-xy[0].start, xy[1].stop-xy[1].start] rawimage[xy] += offset[amp] rawimage[xy] += np.random.normal(scale=rdnoise[amp], size=shape)/gain[amp] #- set CCD parameters self.ccdsec1=hdr["CCDSEC1"] self.ccdsec2=hdr["CCDSEC2"] self.ccdsec3=hdr["CCDSEC3"] self.ccdsec4=hdr["CCDSEC4"] #- raw data are integers, not floats rawimg = rawimage.astype(np.int32) self.expid=hdr["EXPID"] self.camera=hdr["CAMERA"] #- Confirm that all regions were correctly offset assert not np.any(rawimage == 0.0) #- write to the rawfile and read it in QA test hdr['DOSVER'] = 'SIM' hdr['FEEVER'] = 'SIM' hdr['DETECTOR'] = 'SIM' desispec.io.write_raw(self.rawfile,rawimg,hdr,camera=self.camera) self.rawimage=fits.open(self.rawfile) #- read psf, should use specter.PSF.load_psf instead of desispec.PSF(), otherwise need to create a psfboot somewhere. self.psf = load_psf(self.psffile) #- make the test pixfile, fibermap file img_pix = rawimg img_ivar = np.ones_like(img_pix) / 3.0**2 img_mask = np.zeros(img_pix.shape, dtype=np.uint32) img_mask[200] = 1 self.image = desispec.image.Image(img_pix, img_ivar, img_mask, camera='z1',meta=hdr) desispec.io.write_image(self.pixfile, self.image) #- Create a fibermap with purposefully overlapping targeting bits n = 30 self.fibermap = desispec.io.empty_fibermap(n) self.fibermap['OBJTYPE'][:] = 'TGT' self.fibermap['DESI_TARGET'][::2] |= desi_mask.ELG self.fibermap['DESI_TARGET'][::5] |= desi_mask.QSO self.fibermap['DESI_TARGET'][::7] |= desi_mask.LRG #- add some arbitrary fluxes for key in ['FLUX_G', 'FLUX_R', 'FLUX_Z', 'FLUX_W1', 'FLUX_W2']: self.fibermap[key] = 10**((22.5 - np.random.uniform(18, 21, size=n))/2.5) #- Make some standards; these still have OBJTYPE = 'TGT' ii = [6,18,29] self.fibermap['DESI_TARGET'][ii] = desi_mask.STD_FAINT #- set some targets to SKY ii = self.skyfibers = [5,10,21] self.fibermap['OBJTYPE'][ii] = 'SKY' self.fibermap['DESI_TARGET'][ii] = desi_mask.SKY for key in ['FLUX_G', 'FLUX_R', 'FLUX_Z', 'FLUX_W1', 'FLUX_W2']: self.fibermap[key][ii] = np.random.normal(scale=100, size=len(ii)) desispec.io.write_fibermap(self.fibermapfile, self.fibermap) #- make a test frame file self.night=hdr['NIGHT'] self.nspec = nspec = 30 wave=np.arange(7600.0,9800.0,1.0) #- z channel nwave = self.nwave = len(wave) flux=np.random.uniform(size=(nspec,nwave))+100. ivar=np.ones_like(flux) resolution_data=np.ones((nspec,13,nwave)) self.frame=desispec.frame.Frame(wave,flux,ivar,resolution_data=resolution_data,fibermap=self.fibermap) self.frame.meta = hdr self.frame.meta['WAVESTEP']=0.5 desispec.io.write_frame(self.framefile, self.frame) #- make a skymodel sky=np.ones_like(self.frame.flux)*0.5 skyivar=np.ones_like(sky) self.mask=np.zeros(sky.shape,dtype=np.uint32) self.skymodel=desispec.sky.SkyModel(wave,sky,skyivar,self.mask) self.skyfile=desispec.io.write_sky(self.skyfile,self.skymodel) #- Make a dummy boundary map for wavelength-flux in pixel space self.map2pix={} self.map2pix["LEFT_MAX_FIBER"] = 14 self.map2pix["RIGHT_MIN_FIBER"] = 17 self.map2pix["BOTTOM_MAX_WAVE_INDEX"] = 900 self.map2pix["TOP_MIN_WAVE_INDEX"] = 1100
def setUp(self): #- Create temporary calib directory self.testDir = os.path.join(os.environ['HOME'], 'ql_test_io') calibDir = os.path.join(self.testDir, 'ql_calib') if not os.path.exists(calibDir): os.makedirs(calibDir) #- Generate calib data for camera in ['b0', 'r0', 'z0']: #- Fiberflat has to exist but can be a dummpy file filename = '{}/fiberflat-{}.fits'.format(calibDir, camera) fx = open(filename, 'w'); fx.write('fiberflat file'); fx.close() #- PSF has to be real file psffile = '{}/psf-{}.fits'.format(calibDir, camera) example_psf = resource_filename('desispec', 'test/data/ql/psf-{}.fits'.format(camera)) shutil.copy(example_psf, psffile) #- Copy test calibration-data.yaml file specdir=calibDir+"spec/sp0" if not os.path.isdir(specdir) : os.makedirs(specdir) for c in "brz" : shutil.copy(resource_filename('desispec', 'test/data/ql/{}0.yaml'.format(c)),os.path.join(specdir,"{}0.yaml".format(c))) #- Set calibration environment variable os.environ['DESI_SPECTRO_CALIB'] = calibDir self.rawfile = os.path.join(self.testDir,'test-raw-abcd.fits') self.pixfile = os.path.join(self.testDir,'test-pix-abcd.fits') self.config={} #- rawimage hdr = dict() hdr['CAMERA'] = 'b0' hdr['DATE-OBS'] = '2018-09-23T08:17:03.988' #- Dimensions per amp, not full 4-quad CCD ny = self.ny = 500 nx = self.nx = 400 noverscan = nover = 50 hdr['BIASSEC1'] = xy2hdr(np.s_[0:ny, nx:nx+nover]) hdr['DATASEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['CCDSEC1'] = xy2hdr(np.s_[0:ny, 0:nx]) hdr['BIASSEC2'] = xy2hdr(np.s_[0:ny, nx+nover:nx+2*nover]) hdr['DATASEC2'] = xy2hdr(np.s_[0:ny, nx+2*nover:nx+2*nover+nx]) hdr['CCDSEC2'] = xy2hdr(np.s_[0:ny, nx:nx+nx]) hdr['BIASSEC3'] = xy2hdr(np.s_[ny:ny+ny, nx:nx+nover]) hdr['DATASEC3'] = xy2hdr(np.s_[ny:ny+ny, 0:nx]) hdr['CCDSEC3'] = xy2hdr(np.s_[ny:ny+ny, 0:nx]) hdr['BIASSEC4'] = xy2hdr(np.s_[ny:ny+ny, nx+nover:nx+2*nover]) hdr['DATASEC4'] = xy2hdr(np.s_[ny:ny+ny, nx+2*nover:nx+2*nover+nx]) hdr['CCDSEC4'] = xy2hdr(np.s_[ny:ny+ny, nx:nx+nx]) hdr['NIGHT'] = '20180923' hdr['EXPID'] = 1 hdr['FLAVOR']='dark' rawimage = np.zeros((2*ny, 2*nx+2*noverscan)) offset = {'1':100.0, '2':100.5, '3':50.3, '4':200.4} gain = {'1':1.0, '2':1.5, '3':0.8, '4':1.2} rdnoise = {'1':2.0, '2':2.2, '3':2.4, '4':2.6} quad = { '1': np.s_[0:ny, 0:nx], '2': np.s_[0:ny, nx:nx+nx], '3': np.s_[ny:ny+ny, 0:nx], '4': np.s_[ny:ny+ny, nx:nx+nx], } for amp in ('1', '2', '3', '4'): hdr['GAIN'+amp] = gain[amp] hdr['RDNOISE'+amp] = rdnoise[amp] xy = _parse_sec_keyword(hdr['BIASSEC'+amp]) shape = [xy[0].stop-xy[0].start, xy[1].stop-xy[1].start] rawimage[xy] += offset[amp] rawimage[xy] += np.random.normal(scale=rdnoise[amp], size=shape)/gain[amp] xy = _parse_sec_keyword(hdr['DATASEC'+amp]) shape = [xy[0].stop-xy[0].start, xy[1].stop-xy[1].start] rawimage[xy] += offset[amp] rawimage[xy] += np.random.normal(scale=rdnoise[amp], size=shape)/gain[amp] #- raw data are integers, not floats rawimg = rawimage.astype(np.int32) self.expid=hdr["EXPID"] self.camera=hdr["CAMERA"] #- Confirm that all regions were correctly offset assert not np.any(rawimage == 0.0) hdr['DOSVER'] = 'SIM' hdr['FEEVER'] = 'SIM' hdr['DETECTOR'] = 'SIM' desispec.io.write_raw(self.rawfile,rawimg,hdr,camera=self.camera) self.rawimage=fits.open(self.rawfile)