def sens(model='apogee-dr14-giants.model', teff=4500, logg=3, mh=0., xh=False): ''' Plot sensitivities of spectra to changes in Cannon labels ''' model = tc.load_model(model) # get label names label_names = model.vectorizer.label_names # set default parameters if xh: d0 = mh else: d0 = 0. pars = [ teff, logg, mh, 0., mh, d0, d0, d0, d0, d0, d0, d0, d0, d0, d0, d0, d0, d0, d0, d0, d0, d0, d0, d0 ] #,d0] synth0 = model.predict(pars)[0] fig, ax = plots.multi(1, 1) axalt = ax.twinx() wave = 10.**(4.179 + 6.e-6 * np.arange(8575)) figs = [] for i, name in enumerate(label_names): print(name) delta = np.zeros(len(pars)) if i == 0: delta[i] += 100. else: delta[i] += 0.1 synth = model.predict(pars + delta)[0] ax.cla() axalt.cla() plots.plotl(ax, wave, synth - synth0, yr=[-0.10, 0.05]) ax.text(0.1, 0.9, name, transform=ax.transAxes) if name not in ['TEFF', 'LOGG', 'M_H', 'ALPHA_M']: elem = name.split('_')[0] print(elem, ' ', name) aspcap.elemsens([elem], plot=axalt, teff=teff, logg=logg, feh=mh, smooth=2, ylim=[0.05, -0.10]) file = name + '_sens.jpg' fig.savefig('plots/' + file) figs.append([file]) pdb.set_trace() #pdb.set_trace() html.htmltab(figs, file='plots/sens.html')
def compCframe(plate, frame, apred='test', ratio=True, rows=range(300), yr=None, hdu=1): load = apload.ApLoad(apred=apred) mjd = 55562 + int(frame // 10000) new = load.apCframe('M67', plate, mjd, frame) old = {} fig, ax = plots.multi(1, 3, hspace=0.001) x = np.arange(2048) for ichip, chip in enumerate(chips): old[chip] = fits.open(os.environ['APOGEE_REDUX'] + '/r8/apo25m/{:d}/{:d}/apCframe-{:s}-{:d}.fits'. format(plate, mjd, chip, frame)) for row in rows: if ratio: plots.plotl(ax[ichip], x, new[chip][hdu].data[row, :] / old[chip][hdu].data[row, :], yr=[0, 1.5]) else: plots.plotl(ax[ichip], x, new[chip][hdu].data[row, :], yr=yr) plots.plotl(ax[ichip], x, old[chip][hdu].data[row, :], yr=yr) plots.plotl(ax[ichip], x, new[chip][hdu].data[row, :] - old[chip][hdu].data[row, :], yr=yr)
def comp1d(frame, apred='test', rows=range(300)): load = apload.ApLoad(apred=apred) new = load.ap1D(frame) old = {} mjd = 55562 + int(frame // 10000) fig, ax = plots.multi(1, 3, hspace=0.001) x = np.arange(2048) for ichip, chip in enumerate(chips): old[chip] = fits.open( os.environ['APOGEE_REDUX'] + '/r8/red/{:d}/ap1D-{:s}-{:d}.fits'.format(mjd, chip, frame)) for row in rows: plots.plotl(ax[ichip], x, new[chip][1].data[row, :] / old[chip][1].data[row, :], yr=[0, 1.5])
def plot(ax, iso, x, y, xr=None, yr=None, color=None, dx=0., dy=0., isoadj=False, alpha=None): ''' plotting routine that handles tip of RGB ''' mdiff = iso['mini'][0:-1] - iso['mini'][1:] j = np.where(abs(mdiff) < 1.e-8)[0] if len(j) > 0: j = j[0] if isoadj: a = 116. b = 155. c = -22. dx = a + b * iso['feh'][0:j] + c * iso['logg'][0:j] line = plots.plotl(ax, iso[x][0:j] + dx, iso[y][0:j] + dy, xr=xr, yr=yr, color=color, alpha=alpha) plots.plotl(ax, iso[x][j + 1:], iso[y][j + 1:], color=line[0].get_color(), alpha=alpha) else: line = plots.plotl(ax, iso[x] + dx, iso[y] + dy, xr=xr, yr=yr, color=color, alpha=alpha) plt.draw()
def chip(a, row=150, ax=None, pixel=False, visit=False, color=None): """ Routine to plot 3 chips in 3 panels """ if ax is None: fig, ax = plots.multi(1, 3, hspace=0.3) if visit: rows = range(3) else: rows = [row, row, row] chips = ['a', 'b', 'c'] x = np.arange(a['a'][1].header['NAXIS1']) for ichip, chip in enumerate(chips): if pixel: plots.plotl(ax[ichip], x, a[chip][1].data[rows[ichip], :], color=color) else: plots.plotl(ax[ichip], a[chip][4].data[rows[ichip], :], a[chip][1].data[rows[ichip], :], color=color) try: return fig, ax except: return
def correct(field,libfile,plot=True,write=None,width=151) : """ Read raw FERRE files and create correction to normalization based on ratio """ lib=ferre.rdlibhead(libfile+'.hdr') out=ferre.read(field,libfile+'.hdr') nspec=out[1]['obs'].shape[0] norm=copy.copy(out[1]['obs']) if plot : fig,ax=plots.multi(1,3) for i in range(nspec) : p1=0 for ichip in range(3) : npix=lib[1][ichip]['NPIX'] obs=out[1]['obs'][i,p1:p1+npix] # replace bad pixels with median filtered value med=median_filter(obs,[5*width],mode='nearest') bd=np.where(obs < 0.01)[0] obs[bd] = med[bd] # get ratio of observed / model, make sure edge is reasonable number mdl=out[1]['mdl'][i,p1:p1+npix] ratio=obs/mdl ratio[0]=np.median(ratio[0:9]) ratio[-1]=np.median(ratio[-9:0]) corr=median_filter(obs/mdl,[width],mode='nearest') norm[i,p1:p1+npix] = corr if plot : ax[ichip].cla() plots.plotl(ax[ichip],np.arange(npix),obs,yr=[0.95,1.05],color='r') plots.plotl(ax[ichip],np.arange(npix),mdl,yr=[0.95,1.05],color='b') plots.plotl(ax[ichip],np.arange(npix),obs/mdl,yr=[0.95,1.05],color='g') plots.plotl(ax[ichip],np.arange(npix),corr,color='k',linewidth=3,yr=[0.95,1.05]) plt.draw() plt.show() p1+=npix if plot : pdb.set_trace() if write is not None: # make sure we have no zeros bd=np.where(norm.ravel() < 0.01)[0] norm.ravel()[bd]=1. ferre.writespec(write,norm) return norm
def plot(self, ax, **kwargs): for row in range(self.wave.shape[0]): gd = np.where(self.mask[row, :] == False)[0] plots.plotl(ax, self.wave[row, gd], self.data[row, gd], **kwargs)
def norm(w, coef, ichip): x = w - 16000. logflux = coef[0] * x**3 + coef[1] * x**2 + coef[2] * x if ichip == 0: logflux += coef[3] elif ichip == 2: logflux += coef[4] return 10.**logflux for row in rows[0:30:5]: for ichip, chip in enumerate(chips): x = b[chip][4].data[row, :] plots.plotl(ax[0], x, b[chip][1].data[row, :], color='k', semilogy=True) plots.plotl(ax[0], x, b[chip][1].data[row, :] / norm(x, w, ichip), color=colors[ichip], semilogy=True) r10 = apload.ApLoad(apred='r10') b = r10.ap1D(3190056) for row in rows[0:30:5]: plots.plotl(ax[1], b['c'][4].data[row, :], b['c'][1].data[row, :], color='b',
def test(planfile,grid='GKg',npiece=12,npca=75,runraw=True,runpca=True,fit=True, fast=False, niso=None, sns=[1000], rot=False) : """ Routine to set up and run a series of tests for a PCA library Includes comparison of raw and PCA spectra for a sample suite, and several FERRE runs to recover parameters for the sample suite Requires existing test.ipf file with sample suite parameters Args : planfile (str) : name of input plan file that includes filename npiece (int) : number of PCA pieces (for directory/file names) npca (int) : number of PCA pieces (for directory/file names) runraw (bool) : create the sample and input FERRE spectra (default=True) runpca (bool) : create the PCA-derived spectra (default=True) fit (bool) : do the raw-PCA FERRE runs and comparison (default=True) fit (bool) : submit the FERRE test runs to queue (default=True) fast (bool) : use the sdss-fast queue (default=False) """ # Read planfile and set output file name if not os.path.isfile(planfile): print('{:s} does not exist'.format(planfile)) return p=yanny.yanny(planfile,np=True) outfile=os.path.basename(p['name']) # create test directory try : os.mkdir('test') except : pass # create test sample for this grid if runraw: sample.sample('test/test',gridclass=grid,niso=niso) # raw and PCA library file root names flib='f_aps'+outfile plib='p_aps'+outfile+('_{:03d}_{:03d}').format(npiece,npca) # make raw directory and setup to produce test spectra prefix=liblink(flib,'test/raw/') try: os.remove('test/raw/test.ipf') except: pass os.symlink(prefix+'test/test_'+grid+'.ipf','test/raw/test.ipf') if runraw : l=ferre.rdlibhead('f_aps'+outfile+'.hdr')[0] ferre.writenml('test/raw/input.nml','test',l,nov=0,ncpus=1,f_access=1) mkslurm.write('ferre.x',outdir='test/raw/',runplans=False,cwd=os.getcwd()+'/test/raw',fast=fast) print('running ferre in raw to create spectra') subprocess.call(['test/raw/ferre.x'],shell=False) # create uncertainty spectra true=np.loadtxt('test/raw/test.mdl') ferre.writespec('test/raw/test.err',true*0.+0.001) else : true=np.loadtxt('test/raw/test.mdl') # add noise for sn in sns : if sn < 1000 : name = 'test/raw/testsn{:d}'.format(sn) if not os.path.isfile(name+'.mdl') : dim=true.shape truen=true.flatten()+np.random.normal(0.,1./sn,true.flatten().shape) truen=np.reshape(truen,(dim[0],dim[1])) ferre.writespec(name+'.mdl',truen) ferre.writespec(name+'.err',truen*0.+1./sn) # list of all the different tests and their input.nml files; latter are set below # first fdir is just for creating PCA root='test/pca_{:d}_{:d}'.format(npiece,npca) fdirs = [root+'/',root+'/algor3/',root+'/algor1/',root+'/algor1_12/',root+'/algor3_12/', root+'/algor3_001/',root+'/algor3_01/',root+'/algor3_1', root+'/algor3_12_01/',root+'/algor3_12_1', root+'/algor3_indi1234567/',root+'/algor3_indi3142567/', root+'/algor3_4d' ] # general setup for all FERRE directories using PCA library l=ferre.rdlibhead('p_aps'+outfile+('_{:03d}_{:03d}.hdr').format(npiece,npca))[0] print("set up...") for fdir in fdirs : # create library links and return relative directory print(fdir) prefix=liblink(plib,fdir) # create input ipf cmds=[] for sn in sns : if sn < 1000 : name = fdir+'/testsn{:d}'.format(sn) else : name = fdir+'/test' # create input ipf try: os.remove(name+'.ipf') except: pass os.symlink(prefix+'test/test_'+grid+'.ipf',name+'.ipf') # create obs file with true spectra try: os.remove(name+'.obs') except: pass os.symlink(prefix+'test/raw/'+os.path.basename(name)+'.mdl',name+'.obs') # create uncertainty file try: os.remove(name+'.err') except: pass os.symlink(prefix+'test/raw/'+os.path.basename(name)+'.err',name+'.err') cmds.extend(['cp '+os.path.basename(name)+'.nml input.nml','ferre.x']) # create the command file to run FERRE mkslurm.write(cmds,name='ferre.x',outdir=fdir+'/',runplans=False,cwd=os.getcwd()+'/'+fdir,time='48:00:00',fast=fast) # test-specific input.nml files ferre.writenml(root+'/input.nml','test',l,nov=0,ncpus=1) for sn in sns : if sn < 1000 : name = 'testsn{:d}'.format(sn) else : name = 'test' ferre.writenml(root+'/algor3/'+name+'.nml',name,l,algor=3,renorm=4,obscont=1,ncpus=32,init=1) ferre.writenml(root+'/algor1_12/'+name+'.nml',name,l,algor=1,renorm=4,obscont=1,ncpus=32,indini=[1,1,1,1,2,2,3],init=1) ferre.writenml(root+'/algor1/'+name+'.nml',name,l,algor=1,renorm=4,obscont=1,ncpus=32,init=1) ferre.writenml(root+'/algor3_12/'+name+'.nml',name,l,algor=3,renorm=4,obscont=1,ncpus=32,indini=[1,1,1,1,2,2,3],init=1) ferre.writenml(root+'/algor3_001/'+name+'.nml',name,l,algor=3,renorm=4,obscont=1,ncpus=32,stopcr=0.001,init=1) ferre.writenml(root+'/algor3_01/'+name+'.nml',name,l,algor=3,renorm=4,obscont=1,ncpus=32,stopcr=0.01,init=1) ferre.writenml(root+'/algor3_12_01/'+name+'.nml',name,l,algor=3,renorm=4,obscont=1,ncpus=32,stopcr=0.01,indini=[1,1,1,1,2,2,3],init=1) ferre.writenml(root+'/algor3_1/'+name+'.nml',name,l,algor=3,renorm=4,obscont=1,ncpus=32,stopcr=0.1,init=1) ferre.writenml(root+'/algor3_12_1/'+name+'.nml',name,l,algor=3,renorm=4,obscont=1,ncpus=32,stopcr=0.1,indini=[1,1,1,1,2,2,3],init=1) ferre.writenml(root+'/algor3_indi1234567/'+name+'.nml',name,l,algor=3,renorm=4,obscont=1,ncpus=32,indi=[1,2,3,4,5,6,7],init=1) ferre.writenml(root+'/algor3_indi3142567/'+name+'.nml',name,l,algor=3,renorm=4,obscont=1,ncpus=32,indi=[3,1,4,2,5,6,7],init=1) ferre.writenml(root+'/algor3_4d/'+name+'.nml',name,l,algor=3,renorm=4,obscont=1,ncpus=32,indv=[4,5,6,7],init=1) # produce PCA version of test spectra if runpca : print('running ferre in {:d}_{_d} to create spectra'.format(npiece,npca)) subprocess.call([root+'/ferre.x'],shell=False) pca=np.loadtxt(root+'/test.mdl') # histogram of ratio of pca to true print("making pca/raw comparison histogram ...") fig,ax=plots.multi(1,2) hist,bins=np.histogram((pca/true).flatten(),bins=np.linspace(0.9,1.1,4001)) plots.plotl(ax[0],np.linspace(0.8005,1.2,4000),hist/hist.sum(),semilogy=True,xt='pca/true') ax[1].hist(np.abs((pca-true).flatten()),bins=np.logspace(-7,3,50),histtype='step',normed=True,cumulative=True,color='k') ax[1].set_xlim(0.,0.01) ax[1].set_ylim(0.,1.0) ax[1].set_xlabel('|pca-true|') ax[1].set_ylabel('Cumulative fraction') fig.tight_layout() fig.savefig(root+'/'+outfile+'_pca.png') plt.close() # use the PCA libraries to fit raw spectra print("running/making plots...") tab=[] yt=[] # first fdir is just for creating PCA for fdir in fdirs[1:] : print(fdir) if fit : subprocess.call(['sbatch','ferre.x'],shell=False,cwd=fdir) xtab=[] xt=[] tmp='' for sn in sns : if sn < 1000 : name = fdir+'/testsn{:d}'.format(sn) else : name = fdir+'/test' try : sample.comp(name,hard=True,true='test/raw/test.ipf',rot=rot) dt=subprocess.check_output("grep ellapsed "+fdir+"/ferre.x.out | tail -1 | awk '{print $3}'",shell=True) except : print('failed comp: ',name) dt='-1.' xtab.extend(['../'+name+'.png','../'+name+'_2.png']) xt.extend(['S/N = '+str(sn),'S/N = '+str(sn)]) tmp=tmp+dt+',' tab.append(xtab) yt.append('<a href=../'+fdir+'>'+fdir+'</a><br>FERRE time: '+tmp+' s') header = '<h3>'+outfile+'</h3>\n' header = header + '<br> Test sample: <br><img src=test_'+grid+'.png width=40%> <img src=test_'+grid+'_2.png width=40%>' header = header + '<br> Histogram of pixel ratios between raw and PCA for test sample:<br>' header = header + '<img src=../'+root+'/'+outfile+'_pca.png width=40%>\n' html.htmltab(tab,file='test/test.html',header=header,ytitle=yt,xtitle=xt)
for iplt, vp in enumerate(vplist): a = fits.open('test' + t1 + suffix + '_lsfcombo5/new_ap00cp00np00vp' + vp + '.fits')[0] b = fits.open('test' + t2 + suffix + '_lsfcombo5/new_ap00cp00np00vp' + vp + '.fits')[0] # loop over 3 Teff, 3 logg, 3 [M/H] for i in range(3): for j in range(3): for k in range(3): if k == 0: color = 'r' elif k == 1: color = 'g' elif k == 2: color = 'b' plots.plotl(ax[iplt], wave_fits(a.header), a.data[i, j, k, :] / b.data[i, j, k, :], yr=[0.95, 1.05], color=color) ax[iplt].text(0.05, 0.9, r'$v_{micro}= $' + '{:3.1f}'.format(float(vp) / 10.), transform=ax[iplt].transAxes, va='top') fig.savefig('dw/' + t1 + '_' + t2 + suffix + '.png') plt.close(fig) x.append('dw/' + t1 + '_' + t2 + suffix + '.png') xlab.append('dw={:4.3f}'.format(float(t2) / 100.)) y.append(x) ylab.append(t1)
def plot(teff, logg, mh, meanfib, v, vr, fit1d, fit2d, xr=[5500, 3500], yr=[5, 0], vt='vmicro'): ''' auxiliary plotting routine for vmicro, vmacro plots ''' fig = plt.figure(figsize=(14, 8)) y, x = np.mgrid[yr[1]:yr[0]:200j, xr[1]:xr[0]:200j] ax = fig.add_subplot(3, 2, 1) plots.plotc(ax, teff, logg, 10.**v, xr=xr, yr=yr, zr=vr, xt='Teff', yt='log g', zt=vt, colorbar=True, size=15) ax.imshow(10.**fit2d(x, y), extent=[xr[1], xr[0], yr[1], yr[0]], aspect='auto', vmin=0, vmax=vr[1], origin='lower') ax = fig.add_subplot(3, 2, 2) plots.plotc(ax, teff, logg, 10.**v, xr=xr, yr=yr, zr=vr, xt='Teff', yt='log g', zt=vt, colorbar=True, size=15) ax.imshow(10.**fit1d(y), extent=[xr[1], xr[0], yr[1], yr[0]], aspect='auto', vmin=0, vmax=vr[1], origin='lower') ax = fig.add_subplot(3, 2, 3) plots.plotc(ax, logg, 10.**fit2d(teff, logg), mh, xr=[0, 5], yr=vr, zr=[-2.5, 0.5], size=10, colorbar=True, xt='log g', yt=vt, zt='[M/H]') ax = fig.add_subplot(3, 2, 4) plots.plotc(ax, logg, 10.**fit1d(logg), mh, xr=[0, 5], yr=vr, zr=[-2.5, 0.5], size=10, colorbar=True, xt='log g', yt=vt, zt='[M/H]') x = np.arange(0, 5, 0.01) plots.plotl(ax, x, 10.**(0.226 - 0.0228 * x + 0.0297 * x**2 - 0.0113 * x**3)) ax = fig.add_subplot(3, 2, 5) plots.plotc(ax, logg, 10.**v, mh, xr=[0, 5], yr=vr, zr=[-2.5, 0.5], size=10, colorbar=True, xt='log g', yt=vt, zt='[M/H]') ax = fig.add_subplot(3, 2, 6) plots.plotc(ax, logg, 10.**v, meanfib, xr=[0, 5], yr=vr, zr=[0, 300], size=10, colorbar=True, xt='log g', yt=vt, zt='<fiber>') plt.show()
def all(ymlfile, display=None, plot=None, verbose=True, clobber=True, wclobber=None, groups='all'): """ Reduce full night(s) of data given input configuration file """ # read input configuration file for reductions f = open(ymlfile, 'r') d = yaml.load(f, Loader=yaml.FullLoader) f.close() if type(groups) is not list: groups = [groups] # loop over multiple groups in input file for group in d['groups']: if 'skip' in group: if group['skip']: continue if groups[0] != 'all' and group['name'] not in groups: continue # clear displays if given if display is not None: display.clear() if plot is not None: plot.clf() # set up Reducer, Combiner, and output directory inst = group['inst'] print('Instrument: {:s}'.format(inst)) try: red = imred.Reducer(inst=group['inst'], dir=group['rawdir'], verbose=verbose, nfowler=group['nfowler']) except KeyError: red = imred.Reducer(inst=group['inst'], dir=group['rawdir'], verbose=verbose) reddir = group['reddir'] + '/' try: os.makedirs(reddir) except FileExistsError: pass #create superbiases if biases given if 'biases' in group: sbias = mkcal(group['biases'], 'bias', red, reddir, clobber=clobber, display=display) else: print('no bias frames given') sbias = None #create superdarks if darks given if 'darks' in group: sdark = mkcal(group['darks'], 'dark', red, reddir, clobber=clobber, display=display, sbias=sbias) else: print('no dark frames given') sdark = None #create superflats if darks given if 'flats' in group: sflat = mkcal(group['flats'], 'flat', red, reddir, clobber=clobber, display=display, sbias=sbias, sdark=sdark) else: print('no flat frames given') sflat = None # create wavecals if arcs given if 'arcs' in group: # existing trace template traces = pickle.load( open(ROOT + '/data/' + inst + '/' + inst + '_traces.pkl', 'rb')) if wclobber is None: wclobber = clobber wavedict = {} wavecals = group['arcs'] for wavecal in wavecals: print('create wavecal : {:s}'.format(wavecal['id'])) # existing wavecal template waves = pickle.load( open( ROOT + '/data/' + inst + '/' + wavecal['wref'] + '.pkl', 'rb')) if wclobber: make = True else: make = False try: waves_all = pickle.load( open(reddir + wavecal['id'] + '.pkl', 'rb')) except FileNotFoundError: make = True if make: # combine frames try: superbias = sbias[wavecal['bias']] except KeyError: superbias = None arcs = red.sum(wavecal['frames'], return_list=True, superbias=superbias, crbox=[5, 1], display=display) print(' extract wavecal') # loop over channels waves_all = [] for arc, wave, trace in zip(arcs, waves, traces): # loop over windows waves_channel = [] for iwind, (wcal, wtrace) in enumerate(zip(wave, trace)): try: file = wavecal['file'] except KeyError: file = None # extract and ID lines if wavecal['wavecal_type'] == 'echelle': arcec = wtrace.extract(arc, plot=display) wcal.identify(spectrum=arcec, rad=3, display=display, plot=plot, file=file) elif wavecal['wavecal_type'] == 'longslit': r0 = wtrace.rows[0] r1 = wtrace.rows[1] # 1d for inspection wtrace.pix0 += 30 arcec = wtrace.extract(arc, plot=display, rad=20) arcec.data = arcec.data - scipy.signal.medfilt( arcec.data, kernel_size=[1, 101]) wcal.identify(spectrum=arcec, rad=7, plot=plot, display=display, lags=range(-500, 500), file=file) wcal.fit() print("doing 2D wavecal...") arcec = wtrace.extract2d(arc) print(" remove continuum and smooth in rows") arcec.data = arcec.data - scipy.signal.medfilt( arcec.data, kernel_size=[1, 101]) # smooth vertically for better S/N, then sample accordingly image.smooth(arcec, [5, 1]) wcal.identify(spectrum=arcec, rad=3, display=display, plot=plot, nskip=5, lags=range(-50, 50)) wcal.fit() delattr(wcal, 'ax') delattr(wcal, 'fig') waves_channel.append(wcal) waves_all.append(waves_channel) if display is not None: display.clear() pickle.dump(waves_all, open(reddir + wavecal['id'] + '.pkl', 'wb')) if plot is not None: plot.clf() else: print(' already made!') wavedict[wavecal['id']] = waves_all else: print('no wavecal frames given') w = None # reduce objects if 'objects' in group: objects = group['objects'] if 'image' in objects: # images for obj in objects['image']: try: superbias = sbias[obj['bias']] except KeyError: superbias = None try: superdark = sdark[obj['dark']] except KeyError: superdark = None try: superflat = sflat[obj['flat']] except KeyError: superflat = None pdb.set_trace() for iframe, id in enumerate(obj['frames']): frames = red.reduce(id, superbias=superbias, superdark=superdark, superflat=superflat, scat=red.scat, return_list=True, crbox=red.crbox, display=display) elif 'extract1d' in objects: # 1D spectra traces = pickle.load( open(ROOT + '/data/' + inst + '/' + inst + '_traces.pkl', 'rb')) for obj in objects['extract1d']: try: superbias = sbias[obj['bias']] except KeyError: superbias = None try: superdark = sdark[obj['dark']] except KeyError: superdark = None try: superflat = sflat[obj['flat']] except KeyError: superflat = None waves_all = wavedict[obj['wavecal']] if obj['flat_type'] == '1d': print('extracting 1d flat') ecflat = [] for trace in traces: tmp = [] for wtrace in trace: shift = wtrace.find(superflat, plot=display) tmp.append( wtrace.extract(superflat, plot=display, medfilt=101)) ecflat.append(tmp) superflat = None # now frames for iframe, id in enumerate(obj['frames']): if display is not None: display.clear() print("extracting object {}".format(id)) frames = red.reduce(id, superbias=superbias, superdark=superdark, superflat=superflat, scat=red.scat, return_list=True, crbox=red.crbox, display=display) if 'skyframes' in obj: id = obj['skyframes'][iframe] skyframes = red.reduce(id, superbias=superbias, superdark=superdark, superflat=superflat, scat=red.scat, return_list=True, crbox=red.crbox, display=display) for iframe, (frame, skyframe) in enumerate( zip(frames, skyframes)): header = frame.header frames[iframe] = frame.subtract(skyframe) frames[iframe].header = header # extraction radius try: rad = obj['rad'] except KeyError: rad = None # retrace? try: retrace = obj['retrace'] except KeyError: retrace = True # initialize plots if plot is not None: plot.clf() ax = [] ax.append(plot.add_subplot(2, 1, 1)) ax.append(plot.add_subplot(2, 1, 2, sharex=ax[0])) plot.subplots_adjust(hspace=0.001) # loop over channels max = 0 for ichannel, (frame, wave, trace) in enumerate( zip(frames, waves_all, traces)): # loop over windows for iwind, (wcal, wtrace) in enumerate(zip(wave, trace)): if retrace: print(' retracing ....') shift = wtrace.retrace(frame, plot=display, thresh=10) else: shift = wtrace.find(frame, plot=display) ec = wtrace.extract(frame, plot=display, rad=rad) w = wcal.wave(image=np.array(ec.data.shape)) if obj['flat_type'] == '1d': header = ec.header ec = ec.divide(ecflat[ichannel][iwind]) ec.header = header if plot is not None: gd = np.where(ec.mask == False) med = np.median(ec.data[gd[0], gd[1]]) max = np.max([ max, scipy.signal.medfilt( ec.data, [1, 101]).max() ]) for row in range(ec.data.shape[0]): gd = np.where( ec.mask[row, :] == False)[0] plots.plotl(ax[0], w[row, gd], ec.data[row, gd], yr=[0, 1.2 * max], xt='Wavelength', yt='Flux') plots.plotl( ax[1], w[row, gd], ec.data[row, gd] / ec.uncertainty.array[row, gd], xt='Wavelength', yt='S/N') plot.suptitle(ec.header['OBJNAME']) plt.draw() plot.canvas.draw_idle() plt.pause(0.1) input(" hit a key to continue") #ec.write(reddir+ec.header['FILE'].replace('.fits','.ec.fits'),overwrite=True) comb = wcal.scomb(ec, 10.**np.arange(3.5, 4.0, 5.5e-6), average=False, usemask=False) plots.plotl(ax[0], 10.**np.arange(3.5, 4.0, 5.5e-6), comb.data, color='k') plots.plotl(ax[1], 10.**np.arange(3.5, 4.0, 5.5e-6), comb.data / comb.uncertainty.array, color='k') out = spectra.SpecData(ec, wave=w) out.write( reddir + ec.header['FILE'].replace('.fits', '.ec.fits')) out = spectra.SpecData(comb, wave=w) out.write(reddir + comb.header['FILE']) pdb.set_trace() elif 'extract2d' in objects: # 2D spectra print('extract2d')
def dr_compare(): # load the DRs, select stars with SN>150 dr12load = apload.ApLoad(dr='dr12') dr12 = dr12load.allStar()[1].data gd = np.where(dr12['SNR'] > 150)[0] dr12 = dr12[gd] dr13load = apload.ApLoad(dr='dr13') dr13 = dr13load.allStar()[1].data gd = np.where(dr13['SNR'] > 150)[0] dr13 = dr13[gd] dr14load = apload.ApLoad(dr='dr14') dr14 = dr14load.allStar()[1].data gd = np.where(dr14['SNR'] > 150)[0] dr14 = dr14[gd] c = apload.allStar()[3].data # match them m1a, m2a = match.match(dr12['APOGEE_ID'], dr13['APOGEE_ID']) m1b, m2b = match.match(dr12['APOGEE_ID'], dr14['APOGEE_ID']) m1c, m2c = match.match(dr13['APOGEE_ID'], dr14['APOGEE_ID']) # parameter figures figu, axu = plots.multi(3, 7, hspace=0.001, wspace=0.001) figc, axc = plots.multi(3, 7, hspace=0.001, wspace=0.001) tit = [ r'T$_{\rm eff}$', 'log g', r'V$_{\rm micro}$', '[M/H]', '[C/M]', '[N/M]', r'[$\alpha$/M]' ] for iparam in range(7): print(iparam) for iy, param in enumerate(['FPARAM', 'PARAM']): if iy == 0: ax = axu else: ax = axc yt = r'$\Delta$' + tit[iparam] if iparam == 6: xt = r'T$_{\rm eff}$' else: xt = None if iparam == 0: ax[iparam, 0].text(0.5, 1.0, 'DR13-DR12', transform=ax[iparam, 0].transAxes, ha='center', va='bottom') ax[iparam, 1].text(0.5, 1.0, 'DR14-DR12', transform=ax[iparam, 1].transAxes, ha='center', va='bottom') ax[iparam, 2].text(0.5, 1.0, 'DR14-DR13', transform=ax[iparam, 2].transAxes, ha='center', va='bottom') if iparam == 0: yr = [-300, 300] elif iparam == 1: yr = [-0.5, 0.5] else: yr = [-0.3, 0.3] xr = [3500, 6000] axim = plots.plotc(ax[iparam, 0], dr12['TEFF'][m1a], dr13[param][m2a, iparam] - dr12[param][m1a, iparam], dr12[param][m1a, 3], size=1, xr=xr, yr=yr, zr=[-1, 0.5], yt=yt, xt=xt, rasterized=True) plots.plotl(ax[iparam, 0], xr, [0., 0.], ls=':') plots.plotc(ax[iparam, 1], dr12['TEFF'][m1b], dr14[param][m2b, iparam] - dr12[param][m1b, iparam], dr12[param][m1b, 3], size=1, xr=xr, yr=yr, zr=[-1, 0.5], xt=xt, rasterized=True) plots.plotl(ax[iparam, 1], xr, [0., 0.], ls=':') plots.plotc(ax[iparam, 2], dr13['TEFF'][m1c], dr14[param][m2c, iparam] - dr13[param][m1c, iparam], dr13[param][m1c, 3], size=1, xr=xr, yr=yr, zr=[-1, 0.5], xt=xt, rasterized=True) plots.plotl(ax[iparam, 2], xr, [0., 0.], ls=':') for iax in range(3): ax[iparam, iax].tick_params(axis='both', labelsize=8) # add colorbar for fig in [figu, figc]: cbaxes = fig.add_axes([0.91, 0.1, 0.01, 0.8]) cb = plt.colorbar(axim, cax=cbaxes) cb.set_label('[M/H]') cbaxes.tick_params(axis='both', labelsize=8) figu.savefig('drcomp_uncal.pdf') figc.savefig('drcomp_cal.pdf') plots.close() # abundance figure fig, ax = plots.multi(3, 14, hspace=0.001, wspace=0.001, figsize=(8, 32)) for ielem, elem in enumerate([ 'C', 'N', 'O', 'Na', 'Mg', 'Al', 'Si', 'S', 'K', 'Ca', 'Ti', 'V', 'Mn', 'Ni' ]): print(elem) yt = r'$\Delta$' + elem if ielem == 13: xt = r'T$_{\rm eff}$' else: xt = None if ielem == 0: ax[ielem, 0].text(0.5, 1.0, 'DR13-DR12', transform=ax[ielem, 0].transAxes, ha='center', va='bottom') ax[ielem, 1].text(0.5, 1.0, 'DR14-DR12', transform=ax[ielem, 1].transAxes, ha='center', va='bottom') ax[ielem, 2].text(0.5, 1.0, 'DR14-DR13', transform=ax[ielem, 2].transAxes, ha='center', va='bottom') yr = [-0.5, 0.5] dr12elem = dr12[elem.upper() + '_H'][m1a] - dr12['FE_H'][m1a] dr13elem = dr13[elem.upper() + '_FE'][m2a] gd = np.where((dr12elem > -99) & (dr13elem > -99))[0] plots.plotc(ax[ielem, 0], dr12['TEFF'][m1a[gd]], dr13elem[gd] - dr12elem[gd], dr12['PARAM'][m1a[gd], 3], size=1, xr=[3500, 6000], yr=yr, zr=[-1, 0.5], yt=yt, xt=xt, nytick=5, rasterized=True) plots.plotl(ax[ielem, 0], xr, [0., 0.], ls=':') ax[ielem, 0].tick_params(axis='both', labelsize=8) dr12elem = dr12[elem.upper() + '_H'][m1b] - dr12['FE_H'][m1b] dr14elem = dr14[elem.upper() + '_FE'][m2b] gd = np.where((dr12elem > -99) & (dr14elem > -99))[0] plots.plotc(ax[ielem, 1], dr12['TEFF'][m1b[gd]], dr14elem[gd] - dr12elem[gd], dr12['PARAM'][m1b[gd], 3], size=1, xr=[3500, 6000], yr=yr, zr=[-1, 0.5], xt=xt, nytick=5, rasterized=True) plots.plotl(ax[ielem, 1], xr, [0., 0.], ls=':') ax[ielem, 1].tick_params(axis='both', labelsize=8) dr13elem = dr13[elem.upper() + '_FE'][m1c] dr14elem = dr14[elem.upper() + '_FE'][m2c] gd = np.where((dr13elem > -99) & (dr14elem > -99))[0] plots.plotc(ax[ielem, 2], dr13['TEFF'][m1c[gd]], dr14elem[gd] - dr13elem[gd], dr13['PARAM'][m1c[gd], 3], size=1, xr=[3500, 6000], yr=yr, zr=[-1, 0.5], xt=xt, nytick=5, rasterized=True) plots.plotl(ax[ielem, 2], xr, [0., 0.], ls=':') ax[ielem, 2].tick_params(axis='both', labelsize=8) cbaxes = fig.add_axes([0.91, 0.1, 0.01, 0.8]) cb = plt.colorbar(axim, cax=cbaxes) cb.set_label('[M/H]') cbaxes.tick_params(axis='both', labelsize=8) for item in (cbaxes.get_xticklabels() + cbaxes.get_yticklabels()): item.set_fontsize(8) fig.savefig('drcomp_elem.pdf')
print('y6: ') lsfsum(y6) y7 = load.apLSF(25560065)[chip] print('y7: ') lsfsum(y7) ii = 0 for ipar in range(26): iy = ipar % 3 ix = ipar // 3 pax[iy, ix].plot(y1[0].data[ii, :]) ii += 1 for icol in range(4): for fiber in [150]: col = 500 + icol * 500 n = y1[1].data.shape[0] plots.plotl(ax[ichip, icol], np.arange(21), y1[1].data[n / 2 - 10:n / 2 + 11, fiber, col]) n = y2[1].data.shape[0] plots.plotl(ax[ichip, icol], np.arange(21), y2[1].data[n / 2 - 10:n / 2 + 11, fiber, col]) n = y3[1].data.shape[0] plots.plotl(ax[ichip, icol], np.arange(21), y3[1].data[n / 2 - 10:n / 2 + 11, fiber, col]) n = y4[1].data.shape[0] plots.plotl(ax[ichip, icol], np.arange(21), y4[1].data[n / 2 - 10:n / 2 + 11, fiber, col]) n = y5[1].data.shape[0] plots.plotl(ax[ichip, icol], np.arange(21), y5[1].data[n / 2 - 10:n / 2 + 11, fiber, col]) n = y6[1].data.shape[0] plots.plotl(ax[ichip, icol], np.arange(21), y6[1].data[n / 2 - 10:n / 2 + 11, fiber, col])
def ghb(allstar,glatmin=30.,ebvmax=0.03,trange=[3750,5500],loggrange=[-1,6],mhrange=[-2.5,0.75],alpha=False,out='teffcomp',yr=[-500,500], calib=False,dr13=False,grid=None,cmap='rainbow') : """ Compares allstar ASPCPAP Teff with photometric Teff from GHB for sample of stars with GLAT>glatmin and SFD_EBV<ebvmax, does fits Args: allstar : allStar structure Keyword args: glatmin (float) : minimum GLAT for sample (default=30) ebvmax (float) : maximum SFD_EBV for sample (default=0.03) dwarf (bool) : use dwarfs and dwarf GHB (default = False) """ # select data to use if 'TARGFLAGS' in allstar.columns.names : badtarg = ['EMBEDDED','EXTENDED'] else : badtarg = None gd=apselect.select(allstar,badval=['STAR_BAD'],badtarg=badtarg,teff=trange,mh=mhrange,logg=loggrange,raw=True) allstar=allstar[gd] #if dr13 : # j=np.where((abs(allstar['GLAT'])>glatmin)&(allstar['SFD_EBV']<ebvmax))[0] #else : j=np.where((abs(allstar['GLAT'])>glatmin)&(allstar['SFD_EBV']>-0.01)&(allstar['SFD_EBV']<ebvmax)&(abs(allstar['J'])<90)&(abs(allstar['K'])<90))[0] # remove second gen GC stars #if not dr13 : gcstars = ascii.read(os.environ['APOGEE_DIR']+'/data/calib/gc_szabolcs.dat') bd=np.where(gcstars['pop'] != 1)[0] j = [x for x in j if allstar[x]['APOGEE_ID'] not in gcstars['id'][bd]] allstar=allstar[j] ghb,dtdjk=cte_ghb(allstar['J']-allstar['K'],allstar['FPARAM'][:,3],dwarf=False) ghb_dwarf,dtdjk_dwarf=cte_ghb(allstar['J']-allstar['K'],allstar['FPARAM'][:,3],dwarf=True) # use dwarf relation for dwarfs dw=np.where(allstar['FPARAM'][:,1] > 3.8)[0] ghb[dw]=ghb_dwarf[dw] dtdjk[dw]=dtdjk_dwarf[dw] gd=np.where(abs(allstar['FPARAM'][:,0]-ghb) < 500)[0] ghb=ghb[gd] dtdjk=dtdjk[gd] allstar=allstar[gd] print('Teff calibration, number of stars: ', len(allstar)) if calib : param='PARAM' teff=allstar[param][:,0] logg=allstar[param][:,1] mh=allstar[param][:,3] am=allstar[param][:,6] elif grid is None : param='FPARAM' teff=allstar[param][:,0] logg=allstar[param][:,1] mh=allstar[param][:,3] am=allstar[param][:,6] else : param='FPARAM_CLASS' teff=allstar[param][:,grid,0] logg=allstar[param][:,grid,1] mh=allstar[param][:,grid,3] am=allstar[param][:,grid,6] out=out+'_grid{:1d}'.format(grid) # plot Teff difference against metallicity, color-code by temperature fig,ax=plots.multi(1,1,hspace=0.001,wspace=0.001,figsize=(12,6)) xr=[-3.0,1.0] zr=trange if dr13: zr=[3500,5500] binsize=0.25 bins=np.arange(-2.5,0.75,binsize) # diff color-coded by gravity as f([M/H]) if alpha : plots.plotc(ax,mh,teff-ghb,am,zr=[-0.1,0.4],xr=xr,yr=yr,xt='[M/H]',yt='ASPCAP-photometric Teff',colorbar=True,zt=r'[$\alpha$/M]',rasterized=True,cmap=cmap) else : plots.plotc(ax,mh,teff-ghb,teff,xr=xr,yr=yr,xt='[M/H]',yt='ASPCAP-photometric Teff',colorbar=True,zt='$T_{eff}$',rasterized=True,zr=trange,cmap=cmap) mean=bindata(mh,teff-ghb,bins,median=False) if not dr13: plots.plotp(ax,bins+binsize/2.,mean,marker='o',size=40) mean=bindata(mh,teff-ghb,bins,median=True) if not dr13: plots.plotp(ax,bins+binsize/2.,mean,marker='o',size=40,color='b') ax.text(0.1,0.9,'E(B-V)<{:6.2f}'.format(ebvmax),transform=ax.transAxes) gd=np.where(np.isfinite(mean))[0] tefit = fit.fit1d(bins[gd]+binsize/2.,mean[gd],degree=2,reject=0) # 1D quadratic fit as a function of metallicity allfit = fit.fit1d(mh,teff-ghb,ydata=teff,degree=2,reject=0) fig2,ax2=plots.multi(1,1) tefit2 = fit.fit2d(mh,teff,teff-ghb,reject=0,plot=ax2,zr=[-500,200],xt='[M/H]',yt=['Teff'],zt='$\Delta Teff$') #pfit = fit.fit2d(allstar[param][:,3],allstar[param][:,0],allstar[param][:,0]-ghb,plot=ax[0,0],zr=[-500,200],xt='[M/H]',yt=['Teff'],zt='$\Delta Teff$') #ejk=np.clip(np.sqrt(allstar['J_ERR']**2+allstar['K_ERR']**2),0.,0.02) #errpar = err.errfit(teff,allstar['SNR'],mh,teff-tefit(mh)-ghb,title='Teff',out=out+'_phot',zr=[0,250],meanerr=abs(dtdjk)*ejk) errpar = err.errfit(teff,allstar['SNR'],mh,teff-tefit(mh)-ghb,title='Teff',out=out,zr=[0,150]) x=np.linspace(-3,1,200) rms = (teff-tefit(mh)-ghb).std() if dr13: plots.plotl(ax,x,-36.17+95.97*x-15.09*x**2,color='k') print(allfit) else : plots.plotl(ax,x,tefit(x),color='k') ax.text(0.98,0.9,'rms: {:6.1f}'.format(rms),transform=ax.transAxes,ha='right') cmap = matplotlib.cm.get_cmap(cmap) for t in np.arange(trange[0],trange[1],500.) : rgba=cmap((t-trange[0])/(trange[1]-trange[0])) y=x*0.+t plots.plotl(ax,x,tefit2(x,y),color=rgba) plots._data_x = mh plots._data_y = teff-ghb plots._data = allstar plots.event(fig) # separate fits for low/hi alpha/M if requested if alpha : gdlo=apselect.select(allstar,badval=['STAR_BAD'],teff=trange,mh=mhrange,logg=[0,3.8],alpha=[-0.1,0.1],raw=True) mean=bindata(mh[gdlo],teff[gdlo]-ghb[gdlo],bins) plots.plotp(ax,bins,mean,marker='o',size=40,color='g') tmpfit = fit.fit1d(mh[gdlo],teff[gdlo]-ghb[gdlo],ydata=teff[gdlo],degree=2) plots.plotl(ax,x,tmpfit(x)) print('low alpha: ', len(gdlo)) gdhi=apselect.select(allstar,badval=['STAR_BAD'],teff=trange,mh=mhrange,logg=[0,3.8],alpha=[0.1,0.5],raw=True) mean=bindata(mh[gdhi],teff[gdhi]-ghb[gdhi],bins) plots.plotp(ax,bins,mean,marker='o',size=40,color='b') tmpfit = fit.fit1d(mh[gdhi],teff[gdhi]-ghb[gdhi],ydata=teff[gdhi],degree=2) plots.plotl(ax,x,tmpfit(x)) print('hi alpha: ', len(gdhi)) fig.tight_layout() fig.savefig(out+'.png') plt.close() plt.rc('font',size=14) plt.rc('axes',titlesize=14) plt.rc('axes',labelsize=14) fig.savefig(out+'.pdf') plt.close() # auxiliary plots with different color-codings try: meanfib=allstar['MEANFIB'] except: meanfib=teff*0. fig,ax=plots.multi(2,2,hspace=0.001,wspace=0.001) plots.plotc(ax[0,0],mh,teff-ghb,logg,zr=[0,5],xr=xr,yr=yr,xt='[M/H]',yt='ASPCAP-photometric Teff',colorbar=True,zt='log g') plots.plotc(ax[0,1],mh,teff-ghb,meanfib,zr=[0,300],xr=xr,yr=yr,xt='[M/H]',yt='ASPCAP-photometric Teff',colorbar=True,zt='mean fiber') pfit = fit.fit1d(mh,teff-ghb,ydata=teff,plot=ax[1,0],zr=[-500,200],xt='[M/H]',yt='$\Delta Teff$',xr=[-2.7,0.9],yr=[3500,5000],colorbar=True,zt='Teff') pfit = fit.fit1d(teff,teff-ghb,ydata=mh,plot=ax[1,1],zr=[-500,200],xt='Teff',xr=trange,yr=[-2.5,0.5],colorbar=True,zt='[M/H]') fig.tight_layout() fig.savefig(out+'_b.png') plt.close() # do some test 2D and 1D fits and plots #fig,ax=plots.multi(2,2,hspace=0.5,wspace=0.001) #ax[0,1].xaxis.set_visible(False) #ax[0,1].yaxis.set_visible(False) #pfit = fit.fit2d(allstar[param][:,3],allstar[param][:,0],allstar[param][:,0]-ghb,plot=ax[0,0],zr=[-500,200],xt='[M/H]',yt=['Teff'],zt='$\Delta Teff$') #pfit = fit.fit1d(allstar[param][:,3],allstar[param][:,0]-ghb,ydata=allstar[param][:,0],plot=ax[1,0],zr=[-500,200],xt='[M/H]',yt='$\Delta Teff$',xr=[-2.7,0.9],yr=[3500,5000]) #pfit = fit.fit1d(allstar[param][:,0],allstar[param][:,0]-ghb,ydata=allstar[param][:,3],plot=ax[1,1],zr=[-500,200],xt='Teff',xr=[3900,5100],yr=[-2.5,0.5]) plt.draw() return {'caltemin': 3000., 'caltemax': 10000., 'temin' : trange[0], 'temax': trange[1], 'mhmin': mhrange[0], 'mhmax' : mhrange[1], 'par': tefit.parameters, 'rms' :rms, 'par2d': tefit2.parameters, 'errpar' : errpar}
def comp(plate=7267, mjd=56654, fiber=150, frame=10920059, field='M67'): r11 = apload.ApLoad(apred='r11') v = r11.apVisit(plate, mjd, fiber) a = r11.ap1D(frame) c = r11.apCframe(field, plate, mjd, frame) v14 = fits.open(os.environ['APOGEE_REDUX'] + '/r8/apo25m/{:d}/{:d}/apVisit-r8-{:d}-{:d}-{:03d}.fits'. format(plate, mjd, plate, mjd, fiber)) a14 = {} c14 = {} for chip in chips: a14[chip] = fits.open( os.environ['APOGEE_REDUX'] + '/r8/red/{:d}/ap1D-{:s}-{:d}.fits'.format(mjd, chip, frame)) c14[chip] = fits.open(os.environ['APOGEE_REDUX'] + '/r8/apo25m/{:d}/{:d}/apCframe-{:s}-{:08d}.fits'. format(plate, mjd, chip, frame)) fig, ax = plots.multi(1, 3, hspace=0.01) x = np.arange(4096) pixmask = bitmask.PixelBitMask() for ichip, chip in enumerate(chips): y = v[1].data[ichip, :] plots.plotl(ax[ichip], x, v[1].data[ichip, :] / v14[1].data[ichip, :]) bd = np.where(((v[3].data[ichip, :] & pixmask.badval()) > 0) | ( (v[3].data[ichip, :] & pixmask.getval('SIG_SKYLINE')) > 0))[0] y[bd] = np.nan plots.plotl(ax[ichip], x, y / v14[1].data[ichip, :]) fig, ax = plots.multi(3, 3, hspace=0.01) x = np.arange(2048) for ichip, chip in enumerate(chips): plots.plotl(ax[ichip, 0], x, c[chip][1].data[300 - fiber, :]) plots.plotl(ax[ichip, 0], x, c14[chip][1].data[300 - fiber, :]) plots.plotl( ax[ichip, 1], x, c[chip][1].data[300 - fiber, :] / c14[chip][1].data[300 - fiber]) plots.plotl( ax[ichip, 2], x, a[chip][1].data[300 - fiber, :] / a14[chip][1].data[300 - fiber])