def run_mjd5_yaml(yamlfile, logger=None): """ Run the MJD5 yaml file and create the relevant plan files. Parameters ---------- yamlfile : str Name of the MJD5 yaml file. Returns ------- planfiles : list of str List of the created plan file names. Examples -------- planfiles = mkplan.run_mjd5_yaml(yamlfile) By D.Nidever, Oct 2020 """ # Logger if logger is None: logger = dln.basiclogger() if os.path.exists(yamlfile) == False: raise ValueError(yamlfile + ' NOT FOUND') # Load the yaml file with open(yamlfile) as file: data = yaml.full_load(file) if type(data) is not list: data = [data] ndata = len(data) logger.info('Information for ' + str(ndata) + ' plan files') # Loop over the plan blocks and run mkplan() planfiles = [] for i in range(ndata): logger.info(' ') logger.info('Plan file ' + str(i + 1)) logger.info('------------') pargs = data[i] ims = pargs.pop('ims') plate = pargs.pop('plate') mjd = pargs.pop('mjd') psfid = pargs.pop('psfid') fluxid = pargs.pop('fluxid') planfile = mkplan(ims, plate, mjd, psfid, fluxid, **pargs, logger=logger) planfiles.append(planfile) return planfiles
def maskdiscrepant(spec,model,nsig=10,verbose=False,logger=None): """ Mask pixels that are discrepant when compared to a model. """ if logger is None: logger = dln.basiclogger() spec2 = spec.copy() wave = spec2.wave.copy().reshape(spec2.npix,spec2.norder) # make 2D flux = spec2.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D err = spec2.err.copy().reshape(spec2.npix,spec2.norder) # make 2D mask = spec2.mask.copy().reshape(spec2.npix,spec2.norder) # make 2D mflux = model.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D totnbd = 0 for o in range(spec2.norder): w = wave[:,o].copy() x = (w-np.median(w))/(np.max(w*0.5)-np.min(w*0.5)) # -1 to +1 y = flux[:,o].copy() m = mask[:,o].copy() my = mflux[:,o].copy() # Divide by median medy = np.nanmedian(y) y /= medy my /= medy # Perform sigma clipping out large positive outliers coef = dln.poly_fit(x,y,2,robust=True) sig = dln.mad(y-my) bd,nbd = dln.where( np.abs(y-my) > nsig*sig ) totnbd += nbd if nbd>0: flux[bd,o] = dln.poly(x[bd],coef)*medy err[bd,o] = 1e30 mask[bd,o] = True # Flatten to 1D if norder=1 if spec2.norder==1: flux = flux.flatten() err = err.flatten() mask = mask.flatten() # Stuff back in spec2.flux = flux spec2.err = err spec2.mask = mask if verbose is True: logger.info('Masked '+str(totnbd)+' discrepant pixels') return spec2
def queue_wait(queue,sleeptime=60,verbose=True,logger=None): """ Wait for the pbs queue to finish.""" if logger is None: logger = dln.basiclogger() # Wait for jobs to complete running = True while running: time.sleep(sleeptime) percent_complete = queue.get_percent_complete() if verbose==True: logger.info('percent complete = %d' % percent_complete) if percent_complete == 100: running = False
def maskoutliers(spec,nsig=5,verbose=False,logger=None): """ Mask large positive outliers and negative flux pixels in the spectrum. """ if logger is None: logger = dln.basiclogger() spec2 = spec.copy() wave = spec2.wave.copy().reshape(spec2.npix,spec2.norder) # make 2D flux = spec2.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D err = spec2.err.copy().reshape(spec2.npix,spec2.norder) # make 2D mask = spec2.mask.copy().reshape(spec2.npix,spec2.norder) # make 2D totnbd = 0 for o in range(spec2.norder): w = wave[:,o].copy() x = (w-np.median(w))/(np.max(w*0.5)-np.min(w*0.5)) # -1 to +1 y = flux[:,o].copy() m = mask[:,o].copy() # Divide by median medy = np.nanmedian(y) y /= medy # Perform sigma clipping out large positive outliers coef = dln.poly_fit(x,y,2,robust=True) sig = dln.mad(y-dln.poly(x,coef)) bd,nbd = dln.where( ((y-dln.poly(x,coef)) > nsig*sig) | (y<0)) totnbd += nbd if nbd>0: flux[bd,o] = dln.poly(x[bd],coef)*medy err[bd,o] = 1e30 mask[bd,o] = True # Flatten to 1D if norder=1 if spec2.norder==1: flux = flux.flatten() err = err.flatten() mask = mask.flatten() # Stuff back in spec2.flux = flux spec2.err = err spec2.mask = mask if verbose is True: logger.info('Masked '+str(totnbd)+' outlier or negative pixels') return spec2
def make_mjd5_yaml(mjd, apred, telescope, clobber=False, logger=None): """ Make a MJD5 yaml file that can be used to create plan files. Parameters ---------- mjd : int MJD number for this night. apred : str APOGEE reduction version. telescope : str APOGEE telescope: apo25m, apo1m, lco25m. clobber : bool Overwrite any existing files. Returns ------- out : list of dictionaries The list of dictionaries that contain the information needed to make the plan files. planfiles : list of str The names of the plan files that would be created. The yaml files are also written to disk in APOGEEREDUCEPLAN product directory. Examples -------- out,planfiles = mkplan.make_mjd5_yaml(57680,'t15','apo25m') By J.Holtzman, 2011 translated/rewritten, D.Nidever Oct2020 """ # Logger if logger is None: logger = dln.basiclogger() logger.info('Making MJD5.yaml file for MJD=' + str(mjd)) load = apload.ApLoad(apred=apred, telescope=telescope) datadir = { 'apo25m': os.environ['APOGEE_DATA_N'], 'apo1m': os.environ['APOGEE_DATA_N'], 'lco25m': os.environ['APOGEE_DATA_S'] }[telescope] observatory = telescope[0:3] # Output file/directory outfile = os.environ[ 'APOGEEREDUCEPLAN_DIR'] + '/yaml/' + telescope + '/' + telescope + '_' + str( mjd) + 'auto.yaml' if os.path.exists(os.path.dirname(outfile)) == False: os.makedirs(os.path.dirname(outfile)) # File already exists and clobber not set if os.path.exists(outfile) and clobber == False: logger.info(outfile + ' already EXISTS and clobber==False') return # Get the exposures and info about them info = getexpinfo(observatory, mjd) nfiles = len(info) logger.info(str(nfiles) + ' exposures found') # Print summary information about the data expindex = dln.create_index(info['exptype']) for i in range(len(expindex['value'])): logger.info(' ' + expindex['value'][i] + ': ' + str(expindex['num'][i])) objind, = np.where(info['exptype'] == 'OBJECT') if len(objind) > 0: plates = np.unique(info['plateid'][objind]) logger.info('Observations of ' + str(len(plates)) + ' plates') plateindex = dln.create_index(info['plateid'][objind]) for i in range(len(plateindex['value'])): logger.info(' ' + plateindex['value'][i] + ': ' + str(plateindex['num'][i])) # Scan through all files, accumulate IDs of the various types dark, cal, exp, sky, dome, calpsfid = [], [], [], [], None, None out, planfiles = [], [] for i in range(nfiles): # Load image number in variable according to exptype and nreads # discard images with nread<3 if info['nread'][i] < 3: logger.info(info['num'][i], ' has less than the required 3 reads') # Dark if (info['exptype'][i] == 'DARK') and info['nread'][i] >= 3: dark.append(int(info['num'][i])) # Internal flat # reduced only to 2D, hence treated like darks if (info['exptype'][i] == 'INTERNALFLAT') and info['nread'][i] >= 3: dark.append(int(info['num'][i])) # Dome flat if (info['exptype'][i] == 'QUARTZFLAT') and info['nread'][i] >= 3: cal.append(int(info['num'][i])) calpsfid = int(info['num'][i]) # Arc lamps if (info['exptype'][i] == 'ARCLAMP') and info['nread'][i] >= 3: cal.append(int(info['num'][i])) # Sky frame # identify sky frames as object frames with 10<nread<15 if (info['exptype'][i] == 'OBJECT') and (info['nread'][i] < 15 and info['nread'][i] > 10): sky.append(int(info['num'][i])) # Object exposure if (info['exptype'][i] == 'OBJECT') and info['nread'][i] > 15: exp.append(int(info['num'][i])) # Dome flat if (info['exptype'][i] == 'DOMEFLAT') and info['nread'][i] > 3: dome = int(info['num'][i]) # End of this plate block # if plateid changed or last exposure platechange = info['plateid'][i] != info['plateid'][np.minimum( i + 1, nfiles - 1)] if (platechange or i == nfiles - 1) and len(exp) > 0 and dome is not None: # Object plate visit objplan = { 'apred': str(apred), 'telescope': str(load.telescope), 'mjd': int(mjd), 'plate': int(info['plateid'][i]), 'psfid': dome, 'fluxid': dome, 'ims': exp } out.append(objplan) planfile = load.filename('Plan', plate=int(info['plateid'][i]), mjd=mjd) planfiles.append(planfile) exp = [] # Sky exposures # use same cals as for object if len(sky) > 0: skyplan = { 'apred': str(apred), 'telescope': str(load.telescope), 'mjd': int(mjd), 'plate': int(info['plateid'][i]), 'psfid': dome, 'fluxid': dome, 'ims': sky, 'sky': True } out.append(skyplan) skyplanfile = planfile.replace('.yaml', 'sky.yaml') planfiles.append(skyplanfile) sky = [] # Dark frame information cplate = '0000' if len(dark) > 0: darkplan = { 'apred': str(apred), 'telescope': str(load.telescope), 'mjd': int(mjd), 'plate': 0, 'psfid': 0, 'fluxid': 0, 'ims': dark, 'dark': True } out.append(darkplan) planfile = load.filename('DarkPlan', mjd=mjd) planfiles.append(planfile) # Calibration frame information if len(cal) > 0 and calpsfid is not None: calplan = { 'apred': str(apred), 'telescope': str(load.telescope), 'mjd': int(mjd), 'plate': 0, 'psfid': calpsfid, 'fluxid': calpsfid, 'ims': cal, 'cal': True } out.append(calplan) planfile = load.filename('CalPlan', mjd=mjd) planfiles.append(planfile) # Write out the MJD5 file if os.path.exists(outfile): os.remove(outfile) logger.info('Writing MJD5.yaml file to ' + outfile) with open(outfile, 'w') as file: dum = yaml.dump(out, file, default_flow_style=False, sort_keys=False) # Copy it to the non-"auto" version outfile2 = outfile.replace('auto', '') if os.path.exists(outfile2): os.remove(outfile2) shutil.copyfile(outfile, outfile2) return out, planfiles
def mkplan(ims, plate, mjd, psfid, fluxid, apred=None, telescope=None, cal=False, dark=False, sky=False, plugid=None, fixfiberid=None, stars=None, names=None, onem=False, hmags=None, mapper_data=None, suffix=None, ignore=False, test=False, logger=None): """ Makes plan files given input image numbers, MJD, psfid, fluxid includes options for dark frames, calibration frames, sky frames, ASDAF frames. This is called from the manually prepared MJD5.pro procedures Parameters ---------- ims : numpy int array List of array of exposure numbers to include in planfile. plate : int Plate number for this observation. mjd : int MJD number for this observation. psfid : int PSF cal exposure number. fluxid : int Flux cal frame exposure number. apred : str APOGEE reduction version. telescope : str APOGEE telescope. cal : bool, optional This is a calibration plan file. dark : bool, optional This is a dark sequence plan file. sky : bool, optional This is a sky flat sequence plan file. plugid : int, optional Base name of the plugmap filename. fixfiberid : int, optional Fiber fixing needed (1 or 2). stars : numpy int array, optional FiberID for apo1m or ASDAF observations. names : numpy int array, optional Name of the star for apo1m or ASDAF observations. onem : bool, optional This is for a apo1m observation. hmags : numpy float array, optional 2MASS H-magnitude for star in apo1m observation. mapper_data : str, optional Directory for the mapper data. suffix : str, optional Extra suffix to use (before the extension) on the planfile name. ignore : bool, optional Ignore warnings and continue. test : bool, optional Just a test. Returns ------- planfile : str The name of the plan file created This creates a planfile with the given inputs and places it in the appropriate place in the SDSS/APOGEE directory tree. For science visits this will be in $APOGEE_REDUX/{apred}/visit/{telescope}/{field}/{plate}/{mjd}/. Calibration, dark and sky plan files live in $APOGEE_REDUX/{apred}/cal/{instrument}/plan/{mjd}/ Examples -------- mkplan.mkplan(ims,plate,mjd,psfid,fluxid,apred=apred,telescope=telescope) By J.Holtzman, 2011? translated to python, D.Nidever Oct 2020 """ # Logger if logger is None: logger = dln.basiclogger() if apred is None: raise ValueError('apred must be input') if telescope is None: raise ValueError('telescope must be input') logger.info('Making plan for MJD: ' + str(mjd)) # Set up directories, plate, and MJD variables load = apload.ApLoad(apred=apred, telescope=telescope) caldir = os.environ['APOGEE_DRP_DIR'] + '/data/cal/' calfile = caldir + load.instrument + '.par' # Mapper directory if mapper_data is None: if load.instrument == 'apogee-n': mapper_data = os.environ['MAPPER_DATA_N'] else: mapper_data = os.environ['MAPPER_DATA_S'] # Planfile name and directory if cal == True: planfile = load.filename('CalPlan', mjd=mjd) elif dark == True: planfile = load.filename('DarkPlan', mjd=mjd) elif onem == True: planfile = load.filename('Plan', plate=plate, reduction=names[0], mjd=mjd) if suffix is not None: planfile = os.path.dirname(planfile) + '/' + os.path.splitext( os.path.basename(planfile, '.yaml'))[0] + suffix + '.yaml' else: planfile = load.filename('Plan', plate=plate, mjd=mjd) # Make sure the file ends with .yaml if planfile.endswith('.yaml') == False: planfile = planfile.replace(os.path.splitext(planfile)[-1], '.yaml') # TEMPORARY KLUDGE! outdir = os.path.dirname(planfile) + '/' if os.path.exists(outdir) == False: os.makedirs(outdir) # Get calibration files for this date if fixfiberid is not None: fix0 = fixfiberid else: fix0 = None caldata = mkcal.getcal(calfile, mjd) if fix0 is not None: caldata['fixfiber'] = fix0 # outplan plan file name if (stars is not None) & (onem == False): planfile = os.path.dirname(planfile) + '/' + os.path.basename( planfile, '.yaml') + 'star.yaml' else: if sky == True: planfile = os.path.dirname(planfile) + '/' + os.path.basename( planfile, '.yaml') + 'sky.yaml' if sky == True: logger.info('apdailycals') dailycals(lsfs=ims, psf=psfid, apred=apred, telescope=telescope) logger.info(planfile) # open plan file and write header if os.path.exists(planfile): os.remove(planfile) out = {} out['apogee_drp_ver'] = os.environ['APOGEE_DRP_VER'] out['telescope'] = telescope out['instrument'] = load.instrument out['plateid'] = plate out['mjd'] = mjd out['planfile'] = os.path.basename(planfile) out['logfile'] = 'apDiag-' + str(plate) + '-' + str(mjd) + '.log' out['plotfile'] = 'apDiag-' + str(plate) + '-' + str(mjd) + '.ps' # apred_vers keyword will override strict versioning using the plan file! out['apred_vers'] = apred # apo1m if onem == True: out['data_dir'] = datadir + '/' out['raw_dir'] = datadir + str(mjd) + '/' out['plate_dir'] = outdir out['star_dir'] = spectro_dir + '/fields/apo1m/' out['survey'] = 'apo1m' out['name'] = str(names[0]).strip() out['fiber'] = stars[0] if hmags is not None: out['hmag'] = hmags[0] out['telliter'] = 1 if suffix != '': out['mjdfrac'] = 1 # platetype if stars is not None: out['platetype'] = 'single' elif cal == True: out['platetype'] = 'cal' elif sky == True: out['platetype'] = 'sky' elif dark == True: out['platetype'] = 'dark' elif test == True: out['platetype'] = 'test' else: out['platetype'] = 'normal' # Note that q3fix is now done in ap3d.pro, not here!! if (mjd > 56930) & (mjd < 57600): out['q3fix'] = 1 sdss_path = path.Path() rawfile = sdss_path.full('apR', num=ims[0], chip='a', mjd=mjd) #rawfile = load.filename('R',chip='a',num=ims[0]) if os.path.exists(rawfile) == False: raise ValueError('Cannot find file ' + rawfile) head = fits.getheader(rawfile, 1) plateid = head['PLATEID'] if (ignore == False): if (plate != 0) & (plate != plateid): raise ValueError('plateid in header does not match plate!') # plugmap logger.info(str(plugid)) if plugid is None: rawfile = sdss_path.full('apR', num=ims[0], chip='a', mjd=mjd) #rawfile = load.filename('R',chip='a',num=ims[0]) if os.path.exists(rawfile) == True: head = fits.getheader(rawfile, 1) plugid = head['NAME'] if type(plugid) is not str: plugid = 'header' else: plugid = 'header' logger.info(str(ims[0])) logger.info(str(plugid)) if (cal == False) & (dark == False) & (onem == False): tmp = plugid.split('-') if os.path.exists(mapper_data + '/' + tmp[1] + '/plPlugMapM-' + plugid + '.par') == False: logger.info('Cannot find plugmap file ' + str(plugid)) #spawn,'"ls" '+mapper_data+'/'+tmp[1]+'/plPlugMapA*' if ignore is False: raise Exception if sky == False: logger.info('getting plate data') plug = platedata.getdata(plate, mjd, plugid=plugid, noobject=True, mapper_data=mapper_data, apred=apred, telescope=telescope) loc = plug['locationid'] spectro_dir = os.environ['APOGEE_REDUX'] + '/' + apred + '/' if os.path.exists(spectro_dir + 'fields/' + telescope + '/' + str(loc)) == False: os.makedirs(spectro_dir + 'fields/' + telescope + '/' + str(loc)) field, survey, program = apload.apfield(plate, plug['locationid']) out['survey'] = survey out['field'] = field with open( spectro_dir + 'fields/' + telescope + '/' + str(loc) + '/plan-' + str(loc) + '.lis', 'w+') as file: file.write(telescope + '/' + str(plate) + '/' + str(mjd) + '/' + os.path.basename(planfile)) file.close() out['plugmap'] = plugid # Calibration frames to use calnames = [ 'det', 'bpm', 'littrow', 'persist', 'persistmodel', 'dark', 'flat', 'sparse', 'fiber', 'badfiber', 'fixfiber', 'response', 'wave', 'lsf' ] for c in calnames: val = caldata[c] if str(val).isdigit(): val = int(val) out[c + 'id'] = val # We use multiwaveid for waveid waveid = caldata['multiwave'] if str(waveid).isdigit(): waveid = int(waveid) out['waveid'] = waveid # Input PSFID and FLUXID out['psfid'] = psfid out['fluxid'] = fluxid # object frames aplist = [] for i in range(len(ims)): aplist1 = {} if ims[i] > 0: if stars is not None: star = stars[i] name = names[i] else: star = -1 name = 'none' aplist1 = { 'plateid': plate, 'mjd': mjd, 'flavor': 'object', 'name': ims[i], 'single': star, 'singlename': name } aplist.append(aplist1) out['APEXP'] = aplist # Write to yaml file with open(planfile, 'w') as ofile: dum = yaml.dump(out, ofile, default_flow_style=False, sort_keys=False) os.chmod(planfile, 0o664) return planfile
def create_sumfiles(mjd5,apred,telescope,logger=None): """ Create allVisit/allStar files and summary of objects for this night.""" if logger is None: logger = dln.basiclogger() load = apload.ApLoad(apred=apred,telescope=telescope) # Start db session db = apogeedb.DBSession() # USE STAR_LATEST AND VISIT_LATEST "VIEWS" IN THE FUTURE! # Full allVisit and allStar files # apogee_id+apred_vers+telescope+starver uniquely identifies a particular star row # For each apogee_id+apred_vers+telescope we want the maximum starver # The subquery does this for us by grouping by apogee_id+apred_vers+telescope and # calculating the aggregate value MAX(starver). # We then select the particular row (with all columns) using apogee_id+apred_vers+telescope+starver # from this subquery. allstar = db.query(sql="select * from apogee_drp.star where (apogee_id, apred_vers, telescope, starver) in "+\ "(select apogee_id, apred_vers, telescope, max(starver) from apogee_drp.star where "+\ "apred_vers='"+apred+"' and telescope='"+telescope+"' group by apogee_id, apred_vers, telescope)") allstarfile = load.filename('allStar').replace('.fits','-'+telescope+'.fits') logger.info('Writing allStar file to '+allstarfile) if os.path.exists(os.path.dirname(allstarfile))==False: os.makedirs(os.path.dirname(allstarfile)) Table(allstar).write(allstarfile,overwrite=True) # allVisit # Same thing for visit except that we'll get the multiple visit rows returned for each unique star row # Get more info by joining with the visit table. vcols = ['apogee_id', 'target_id', 'apred_vers','file', 'uri', 'fiberid', 'plate', 'mjd', 'telescope', 'survey', 'field', 'programname', 'alt_id', 'location_id', 'ra', 'dec', 'glon', 'glat', 'j', 'j_err', 'h', 'h_err', 'k', 'k_err', 'src_h', 'wash_m', 'wash_m_err', 'wash_t2', 'wash_t2_err', 'ddo51', 'ddo51_err', 'irac_3_6', 'irac_3_6_err', 'irac_4_5', 'irac_4_5_err', 'irac_5_8', 'irac_5_8_err', 'irac_8_0', 'irac_8_0_err', 'wise_4_5', 'wise_4_5_err', 'targ_4_5', 'targ_4_5_err', 'wash_ddo51_giant_flag', 'wash_ddo51_star_flag', 'pmra', 'pmdec', 'pm_src', 'ak_targ', 'ak_targ_method', 'ak_wise', 'sfd_ebv', 'apogee_target1', 'apogee_target2', 'apogee_target3', 'apogee_target4', 'targflags', 'snr', 'starflag', 'starflags','dateobs','jd'] rvcols = ['starver', 'bc', 'vtype', 'vrel', 'vrelerr', 'vheliobary', 'chisq', 'rv_teff', 'rv_feh', 'rv_logg', 'xcorr_vrel', 'xcorr_vrelerr', 'xcorr_vheliobary', 'n_components', 'rv_components'] cols = ','.join('v.'+np.char.array(vcols)) +','+ ','.join('rv.'+np.char.array(rvcols)) allvisit = db.query(sql="select "+cols+" from apogee_drp.rv_visit as rv join apogee_drp.visit as v on rv.visit_pk=v.pk "+\ "where (rv.apogee_id, rv.apred_vers, rv.telescope, rv.starver) in "+\ "(select apogee_id, apred_vers, telescope, max(starver) from apogee_drp.rv_visit where "+\ "rv.apred_vers='"+apred+"' and rv.telescope='"+telescope+"' group by apogee_id, apred_vers, telescope)") allvisitfile = load.filename('allVisit').replace('.fits','-'+telescope+'.fits') logger.info('Writing allVisit file to '+allvisitfile) if os.path.exists(os.path.dirname(allvisitfile))==False: os.makedirs(os.path.dirname(allvisitfile)) Table(allvisit).write(allvisitfile,overwrite=True) # Nightly allVisit and allStar, allVisitMJD/allStarMJD gdstar, = np.where(allstar['starver']==str(mjd5)) allstarmjd = allstar[gdstar] gdvisit, = np.where(allvisit['mjd']==int(mjd5)) allvisitmjd = allvisit[gdvisit] # maybe in summary/MJD/ or qa/MJD/ ? #allstarmjdfile = load.filename('allStarMJD') allstarmjdfile = allstarfile.replace('allStar','allStarMJD').replace('.fits','-'+str(mjd5)+'.fits') logger.info('Writing Nightly allStarMJD file to '+allstarmjdfile) if os.path.exists(os.path.dirname(allstarmjdfile))==False: os.makedirs(os.path.dirname(allstarmjdfile)) Table(allstarmjd).write(allstarmjdfile,overwrite=True) #allvisitmjdfile = load.filename('allVisitMJD') allvisitmjdfile = allvisitfile.replace('allVisit','allVisitMJD').replace('.fits','-'+str(mjd5)+'.fits') logger.info('Writing Nightly allVisitMJD file to '+allvisitmjdfile) if os.path.exists(os.path.dirname(allvisitmjdfile))==False: os.makedirs(os.path.dirname(allvisitmjdfile)) Table(allvisitmjd).write(allvisitmjdfile,overwrite=True) db.close()
def check_rv(visits,pbskey,verbose=False,logger=None): """ Check that rv ran okay and load into database.""" if logger is None: logger = dln.basiclogger() db = apogeedb.DBSession() # open db session if verbose==True: logger.info('') logger.info('-------------------------------------') logger.info('Checking RV + Visit Combination runs') logger.info('=====================================') apred_vers = visits['apred_vers'][0] telescope = visits['telescope'][0] load = apload.ApLoad(apred=apred_vers,telescope=telescope) if verbose: logger.info('') logger.info('%d stars' % len(visits)) logger.info('apred_vers: %s' % apred_vers) logger.info('telescope: %s' % telescope) logger.info(' NUM APOGEE_ID HEALPIX NVISITS SUCCESS') # Loop over the stars nstars = len(visits) dtype = np.dtype([('apogee_id',(np.str,50)),('apred_vers',(np.str,20)),('telescope',(np.str,10)), ('healpix',int),('nvisits',int),('pbskey',(np.str,50)), ('file',(np.str,300)),('checktime',(np.str,100)),('success',bool)]) chkrv = np.zeros(nstars,dtype=dtype) chkrv['apred_vers'] = apred_vers chkrv['telescope'] = telescope chkrv['pbskey'] = pbskey for i,visit in enumerate(visits): starfilenover = load.filename('Star',obj=visit['apogee_id']) # add version number, should be MJD of of the latest visit stardir = os.path.dirname(starfilenover) starbase = os.path.splitext(os.path.basename(starfilenover))[0] starbase += '-'+str(visit['mjd']) # add star version starfile = stardir+'/'+starbase+'.fits' # Get nvisits for this star starvisits = db.query('visit',cols='*',where="apogee_id='"+visit['apogee_id']+"' and "+\ "telescope='"+telescope+"' and apred_vers='"+apred_vers+"'") chkrv['apogee_id'][i] = visit['apogee_id'] chkrv['healpix'][i] = apload.obj2healpix(visit['apogee_id']) chkrv['nvisits'][i] = len(starvisits) chkrv['file'][i] = starfile chkrv['checktime'][i] = str(datetime.now()) chkrv['success'][i] = os.path.exists(starfile) if verbose: logger.info('%5d %20s %8d %5d %9s' % (i+1,chkrv['apogee_id'][i],chkrv['healpix'][i], chkrv['nvisits'][i],chkrv['success'][i])) success, = np.where(chkrv['success']==True) logger.info('%d/%d succeeded' % (len(success),nstars)) # Inset into the database db.ingest('rv_status',chkrv) db.close() return chkrv
def check_apred(expinfo,planfiles,pbskey,verbose=False,logger=None): """ Check that apred ran okay and load into database.""" if logger is None: logger = dln.basiclogger() if verbose==True: logger.info('') logger.info('--------------------') logger.info('Checking APRED runs') logger.info('====================') # Loop over the planfiles nplanfiles = len(planfiles) for ip,pfile in enumerate(planfiles): planstr = plan.load(pfile,np=True) apred_vers = planstr['apred_vers'] telescope = planstr['telescope'] instrument = planstr['instrument'] platetype = planstr['platetype'] mjd = planstr['mjd'] plate = planstr['plateid'] expstr = planstr['APEXP'] nexp = len(expstr) load = apload.ApLoad(apred=apred_vers,telescope=telescope) # Science exposures if platetype=='normal': # Load the plugmap information plugmap = platedata.getdata(plate,mjd,apred_vers,telescope,plugid=planstr['plugmap']) fiberdata = plugmap['fiberdata'] # Exposure-level processing: ap3d, ap2d, apcframe dtype = np.dtype([('exposure_pk',int),('planfile',(np.str,300)),('apred_vers',(np.str,20)),('instrument',(np.str,20)), ('telescope',(np.str,10)),('platetype',(np.str,50)),('mjd',int),('plate',int), ('proctype',(np.str,30)),('pbskey',(np.str,50)),('checktime',(np.str,100)), ('num',int),('success',bool)]) chkexp = np.zeros(nexp*3,dtype=dtype) chkexp['planfile'] = pfile chkexp['apred_vers'] = apred_vers chkexp['instrument'] = instrument chkexp['telescope'] = telescope chkexp['platetype'] = platetype chkexp['mjd'] = mjd chkexp['plate'] = plate chkexp['proctype'] = 'AP3D' chkexp['pbskey'] = pbskey chkexp['success'] = False cnt = 0 for num in expstr['name']: ind, = np.where(expinfo['num']==num) exposure_pk = expinfo['pk'][ind[0]] # AP3D #----- chkexp['exposure_pk'][cnt] = exposure_pk chkexp['num'][cnt] = num chkexp['proctype'][cnt] = 'AP3D' base = load.filename('2D',num=num,mjd=mjd,chips=True) chfiles = [base.replace('2D-','2D-'+ch+'-') for ch in ['a','b','c']] exists = [os.path.exists(chf) for chf in chfiles] chkexp['checktime'][cnt] = str(datetime.now()) if np.sum(exists)==3: chkexp['success'][cnt] = True cnt += 1 # AP2D #----- chkexp['exposure_pk'][cnt] = exposure_pk chkexp['num'][cnt] = num chkexp['proctype'][cnt] = 'AP2D' base = load.filename('1D',num=num,mjd=mjd,chips=True) chfiles = [base.replace('1D-','1D-'+ch+'-') for ch in ['a','b','c']] exists = [os.path.exists(chf) for chf in chfiles] chkexp['checktime'][cnt] = str(datetime.now()) if np.sum(exists)==3: chkexp['success'][cnt] = True cnt += 1 # APCframe #--------- chkexp['exposure_pk'][cnt] = exposure_pk chkexp['num'][cnt] = num chkexp['proctype'][cnt] = 'APCFRAME' base = load.filename('Cframe',num=num,mjd=mjd,plate=plate,chips=True) chfiles = [base.replace('Cframe-','Cframe-'+ch+'-') for ch in ['a','b','c']] exists = [os.path.exists(chf) for chf in chfiles] chkexp['checktime'][cnt] = str(datetime.now()) if np.sum(exists)==3: chkexp['success'][cnt] = True cnt += 1 # AP1DVISIT # --------- # -apPlate # -apVisit files # -apVisitSum file dtypeap = np.dtype([('planfile',(np.str,300)),('logfile',(np.str,300)),('errfile',(np.str,300)), ('apred_vers',(np.str,20)),('instrument',(np.str,20)), ('telescope',(np.str,10)),('platetype',(np.str,50)),('mjd',int),('plate',int), ('nobj',int),('pbskey',(np.str,50)),('checktime',(np.str,100)),('ap3d_success',bool), ('ap3d_nexp_success',int),('ap2d_success',bool),('ap2d_nexp_success',int), ('apcframe_success',bool),('apcframe_nexp_success',int),('applate_success',bool), ('apvisit_success',bool),('apvisit_nobj_success',int), ('apvisitsum_success',bool),('success',bool)]) chkap = np.zeros(1,dtype=dtypeap) chkap['planfile'] = pfile chkap['logfile'] = pfile.replace('.yaml','_pbs.log') chkap['errfile'] = pfile.replace('.yaml','_pbs.err') chkap['apred_vers'] = apred_vers chkap['instrument'] = instrument chkap['telescope'] = telescope chkap['platetype'] = platetype chkap['mjd'] = mjd chkap['plate'] = plate chkap['nobj'] = np.sum(fiberdata['objtype']!='SKY') # stars and tellurics chkap['pbskey'] = pbskey chkap['checktime'] = str(datetime.now()) # ap3D, ap2D, apCframe success ind3d, = np.where((chkexp['num']==num) & (chkexp['proctype']=='AP3D')) chkap['ap3d_nexp_success'] = np.sum(chkexp['success'][ind3d]) chkap['ap3d_success'] = np.sum(chkexp['success'][ind3d])==nexp ind2d, = np.where((chkexp['num']==num) & (chkexp['proctype']=='AP2D')) chkap['ap2d_nexp_success'] = np.sum(chkexp['success'][ind2d]) chkap['ap2d_success'] = np.sum(chkexp['success'][ind2d])==nexp indcf, = np.where((chkexp['num']==num) & (chkexp['proctype']=='APCFRAME')) chkap['apcframe_nexp_success'] = np.sum(chkexp['success'][indcf]) chkap['apcframe_success'] = np.sum(chkexp['success'][indcf])==nexp # apPlate chkap['applate_success'] = False base = load.filename('Plate',plate=plate,mjd=mjd,chips=True) chfiles = [base.replace('Plate-','Plate-'+ch+'-') for ch in ['a','b','c']] exists = [os.path.exists(chf) for chf in chfiles] if np.sum(exists)==3: chkap['applate_success'] = True # apVisit base = load.filename('Visit',plate=plate,mjd=mjd,fiber=1) visitfiles = glob(base.replace('-001.fits','-???.fits')) nvisitfiles = len(visitfiles) chkap['apvisit_nobj_success'] = nvisitfiles chkap['apvisit_success'] = nvisitfiles==chkap['nobj'] apvisitsumfile = load.filename('VisitSum',plate=plate,mjd=mjd) chkap['apvisitsum_success'] = os.path.exists(apvisitsumfile) chkap['success'] = chkap['ap3d_success'][0] and chkap['ap2d_success'][0] and chkap['apcframe_success'][0] and \ chkap['applate_success'][0] and chkap['apvisit_success'][0] and chkap['apvisitsum_success'][0] if verbose: logger.info('') logger.info('%d/%d' % (ip+1,nplanfiles)) logger.info('planfile: '+pfile) logger.info('log/errfile: '+os.path.basename(chkap['logfile'][0])+', '+os.path.basename(chkap['errfile'][0])) logger.info('platetype: %s' % platetype) logger.info('mjd: %d' % mjd) logger.info('plate: %d' % plate) logger.info('nexp: %d' % nexp) logger.info('Nobj: %d' % chkap['nobj'][0]) logger.info('3D/2D/Cframe:') logger.info('Num EXPID NREAD 3D 2D Cframe') for k,num in enumerate(expstr['name']): ind, = np.where(expinfo['num']==num) ind3d, = np.where((chkexp['num']==num) & (chkexp['proctype']=='AP3D')) ind2d, = np.where((chkexp['num']==num) & (chkexp['proctype']=='AP2D')) indcf, = np.where((chkexp['num']==num) & (chkexp['proctype']=='APCFRAME')) logger.info('%2d %10d %4d %6s %6s %6s' % (k+1,chkexp['num'][ind3d],expinfo['nread'][ind], chkexp['success'][ind3d[0]],chkexp['success'][ind2d[0]], chkexp['success'][indcf[0]])) logger.info('apPlate files: %s ' % chkap['applate_success'][0]) logger.info('N apVisit files: %d ' % chkap['apvisit_nobj_success'][0]) logger.info('apVisitSum file: %s ' % chkap['apvisitsum_success'][0]) logger.info('apVisit success: %s ' % chkap['success'][0]) # Load into the database db = apogeedb.DBSession() db.ingest('exposure_status',chkexp) db.ingest('visit_status',chkap) db.close() #import pdb; pdb.set_trace() # Calibration exposures else: logger.info('calibration exposures') # STILL NEED TO DO THIS PART!!! #import pdb; pdb.set_trace() return chkexp,chkap
def getdata(refcat, minra, redo=False, silent=False, logger=None): """ Get reference catalog information from DL database Parameters ---------- cenra : float Central RA for the search. cendec : float Central DEC for the search. radius : float Search radius in degrees. refcat : table Reference catalog name (e.g. 2MASS, Gaia, etc.) version : str Version of NSC reduction.. saveref : bool, optional Save the output to SAVEFILE or a default filename. Default is False. savefile : str, optional The file to save to or search for existing catalog. silent : bool, optional Don't print anything to the screen. logger : logging object, optional Logging object to use for printing messages. Returns ------- ref : astropy table Search results from the reference catalog. Example ------- cat = getrefcat(cenra,cendec,radius,refcat,file=file,saveref=saveref) By D. Nidever Sep 2017 Translated to Python by D. Nidever, April 2022 """ count = 0 t0 = time.time() if logger is None: logger = dln.basiclogger() # Check that we have psql installed out = subprocess.check_output(['which', 'psql'], shell=False) if type(out) is bytes: out = out.decode() out = out.strip() if dln.size(out) > 1: out = out[0] if os.path.exists(out) == 0: raise ValueError('No PSQL found on this sytem.') # Temporary directory # /tmp is often small and get get fille dup dldir, mssdir, localdir = utils.rootdirs() # FLIP THIS AROUND, INPUT SHOULD BE THE "EASY" VERSION!!! refname = str(refcat).upper() if refname == 'II/312/AIS': refname = 'GALEX' elif refname == '2MASS-PSC': refname = 'TMASS' elif refname == '2MASS': refname = 'TMASS' elif refname == 'GAIA/GAIA': refname = 'GAIA' elif refname == 'Skymapper': refname = 'SKYMAPPER' elif refname == 'GLIMPSE': catname = 'II/293/glimpse' elif refname == 'SAGE': catname = 'II/305/archive' elif refname == 'ATLASREFCAT2': refname = 'ATLAS' ra0 = float(minra) ra1 = minra + 1.0 outdir = '/net/dl1/users/dnidever/nsc/refcatalogs/' + refname + '/' if os.path.exists(outdir) == False: os.makedirs(outdir) savefile = outdir + 'ref_%.6f_%.6f_%s.fits' % (ra0, ra1, refname) if os.path.exists(os.path.abspath(os.path.dirname(savefile))) == False: os.makedirs(os.path.abspath(os.path.dirname(savefile))) if silent == False: logger.info('Querying %s: %.6f <= RA < %.6f' % (refname, ra0, ra1)) # Loading previously loaded file if os.path.exists(savefile) and redo == False: logger.info(savefile + ' already exists and redo==False') return None # Do the Query #-------------- else: # Use DataLab database search #---------------------------- if refname in [ 'TMASS', 'GAIA', 'GAIADR2', 'GAIAEDR3', 'PS', 'SKYMAPPER', 'SKYMAPPERDR2', 'ALLWISE', 'ATLAS' ]: if refname == 'TMASS': tablename = 'twomass.psc' cols = 'designation,ra as raj2000,dec as dej2000,j_m as jmag,j_cmsig as e_jmag,h_m as hmag,h_cmsig as e_hmag,k_m as kmag,k_cmsig as e_kmag,ph_qual as qflg' ##server = 'gp04.datalab.noao.edu' #server = 'gp01.datalab.noirlab.edu' ##server = 'dldb1.sdm.noao.edu' server = 'db02.datalab.noirlab.edu' user = '******' racol = 'ra' deccol = 'dec' if refname == 'GAIA': tablename = 'gaia_dr1.gaia_source' cols = 'source_id as source,ra as ra_icrs,ra_error as e_ra_icrs,dec as de_icrs,dec_error as e_de_icrs,' cols += 'phot_g_mean_flux as fg,phot_g_mean_flux_error as e_fg,phot_g_mean_mag as gmag' #server = 'gp04.datalab.noirlab.edu' ##server = 'gp01.datalab.noao.edu' ##server = 'dldb1.sdm.noao.edu' server = 'db02.datalab.noirlab.edu' user = '******' if refname == 'GAIADR2': tablename = 'gaia_dr2.gaia_source' cols = 'source_id as source,ra,ra_error,dec,dec_error,pmra,pmra_error,pmdec,pmdec_error,phot_g_mean_flux as fg,phot_g_mean_flux_error as e_fg,' cols += 'phot_g_mean_mag as gmag,phot_bp_mean_mag as bp,phot_bp_mean_flux as fbp,phot_bp_mean_flux_error as e_fbp,' cols += 'phot_rp_mean_mag as rp,phot_rp_mean_flux as frp,phot_rp_mean_flux_error as e_frp' #server = 'gp04.datalab.noirlab.edu' ##server = 'gp01.datalab.noao.edu' server = 'db02.datalab.noirlab.edu' user = '******' if refname == 'GAIAEDR3': tablename = 'gaia_edr3.gaia_source' cols = 'source_id as source,ra,ra_error,dec,dec_error,pmra,pmra_error,pmdec,pmdec_error,phot_g_mean_flux as fg,phot_g_mean_flux_error as e_fg,' cols += 'phot_g_mean_mag as gmag,phot_bp_mean_mag as bp,phot_bp_mean_flux as fbp,phot_bp_mean_flux_error as e_fbp,' cols += 'phot_rp_mean_mag as rp,phot_rp_mean_flux as frp,phot_rp_mean_flux_error as e_frp' #server = 'gp04.datalab.noirlab.edu' ##server = 'gp01.datalab.noao.edu' server = 'db02.datalab.noirlab.edu' user = '******' if refname == 'PS': #tablename = 'cp_calib.ps1' tablename = 'public.ps1' cols = 'ra, dec, g as gmag, r as rmag, i as imag, z as zmag, y as ymag' ##server = 'gp02.datalab.noirlab.edu' #server = 'gp01.datalab.noirlab.edu' server = 'db02.datalab.noirlab.edu' user = '******' if refname == 'SKYMAPPER': tablename = 'skymapper_dr1.master' cols = 'raj2000 as ra, dej2000 as dec, u_psf as sm_umag, e_u_psf as e_sm_umag, g_psf as sm_gmag, e_g_psf as e_sm_gmag, r_psf as sm_rmag,' cols += 'e_r_psf as e_sm_rmag, i_psf as sm_imag,e_i_psf as e_sm_imag, z_psf as sm_zmag, e_z_psf as e_sm_zmag' #server = 'gp04.datalab.noirlab.edu' ##server = 'gp01.datalab.noao.edu' server = 'db02.datalab.noirlab.edu' user = '******' racol = 'raj2000' deccol = 'dej2000' if refname == 'SKYMAPPERDR2': tablename = 'skymapper_dr2.master' cols = 'raj2000 as ra, dej2000 as dec, u_psf as sm_umag, e_u_psf as e_sm_umag, g_psf as sm_gmag, e_g_psf as e_sm_gmag, r_psf as sm_rmag,' cols += 'e_r_psf as e_sm_rmag, i_psf as sm_imag,e_i_psf as e_sm_imag, z_psf as sm_zmag, e_z_psf as e_sm_zmag' #server = 'gp04.datalab.noirlab.edu' ##server = 'gp01.datalab.noao.edu' server = 'db02.datalab.noirlab.edu' user = '******' racol = 'raj2000' deccol = 'dej2000' if refname == 'ALLWISE': tablename = 'allwise.source' #cols = 'ra, dec, w1mdef as w1mag, w1sigmdef as e_w1mag, w2mdef as w2mag, w2sigmdef as e_w2mag' cols = 'ra, dec, w1mpro as w1mag, w1sigmpro as e_w1mag, w2mpro as w2mag, w2sigmpro as e_w2mag' #server = 'gp04.datalab.noao.edu' #server = 'gp01.datalab.noirlab.edu' server = 'db02.datalab.noirlab.edu' user = '******' if refname == 'ATLAS': tablename = 'atlasrefcat2' cols = 'objid,ra,dec,plx as parallax,dplx as parallax_error,pmra,dpmra as pmra_error,pmdec,dpmdec as pmdec_error,gaia,dgaia as gaiaerr,' cols += 'bp,dbp as bperr,rp,drp as rperr,teff,agaia,dupvar,ag,rp1,r1,r10,g as gmag,dg as gerr,gchi,gcontrib,' cols += 'r as rmag, dr as rerr,rchi,rcontrib,i as imag,di as ierr,ichi,icontrib,z as zmag,dz as zerr,zchi,zcontrib,nstat,' cols += 'j as jmag,dj as jerr,h as hmag,dh as herr,k as kmag,dk as kerr' server = 'gp10.datalab.noirlab.edu' user = '******' # Use Postgres command with q3c cone search refcattemp = savefile.replace('.fits', '.txt') cmd = "psql -h " + server + " -U " + user + " -d tapdb -w --pset footer -c 'SELECT " + cols + " FROM " + tablename if refname == 'SKYMAPPER' or refname == 'SKYMAPPERDR2': cmd += " WHERE raj2000 >= %.6f and raj2000 < %.6f'" % (ra0, ra1) else: cmd += " WHERE ra >= %.6f and ra < %.6f'" % (ra0, ra1) cmd += " > " + refcattemp dln.remove(refcattemp, allow=True) dln.remove(savefile, allow=True) out = subprocess.check_output(cmd, shell=True) # Check for empty query tlines = dln.readlines(refcattemp, nreadline=4) if len(tlines) < 4: if silent == False: logger.info('No Results') return [] # Load ASCII file and create the FITS file ref = ascii.read(refcattemp, data_start=3, delimiter='|') #ref = importascii(refcattemp,/header,delim='|',skipline=2,/silent) dln.remove(refcattemp, allow=True) # Fix 0.0 mags/errs in ATLAS if refname == 'ATLAS': magcols = [ 'gaia', 'bp', 'rp', 'gmag', 'rmag', 'imag', 'zmag', 'jmag', 'hmag', 'kmag' ] errcols = [ 'gaiaerr', 'bperr', 'rperr', 'gerr', 'rerr', 'ierr', 'zerr', 'jerr', 'herr', 'kerr' ] cols = ref.colnames # Set mags with 0.0 to 99.99 for j in range(len(magcols)): if magcols[j] in ref.colnames: bdmag = (ref[magcols[j]] <= 0.0) if np.sum(bdmag) > 0: ref[magcols[j]][bdmag] = 99.99 # Set errors with 0.0 to 9.99 for j in range(len(errcols)): if errcols[j] in ref.colnames: bderr = (ref[errcols[j]] <= 0.0) if np.sum(bderr) > 0: ref[errcols[j]][bderr] = 9.99 # Use astroquery vizier #---------------- # for low density with 2MASS/GAIA and always for GALEX and APASS else: # Use QUERYVIZIER for GALEX (python code has problems) #if refname == 'II/312/ais' or refname == 'GALEX': # if refcat eq 'APASS' then cfa=0 else cfa=1 ; cfa doesn't have APASS #cfa = 1 # problems with CDS VizieR and cfa has APASS now #if refcat == 'SAGE': # cfa = 0 if refname.upper() == 'GALEX': cols = [ 'RAJ2000', 'DEJ2000', 'FUVmag', 'e_FUVmag', 'NUVmag', 'e_NUVmag' ] catname = 'II/335/galex_ais' elif refname.upper() == 'GLIMPSE': # Only includes GLIMPSE I,II,3D cols = [ 'RAJ2000', 'DEJ2000', '_2MASS', '_3.6mag', 'e_3.6mag', '_4.5mag', 'e_4.5mag' ] catname = 'II/293/glimpse' elif refname.upper() == 'SAGE': cols = [ 'RAJ2000', 'DEJ2000', '__3.6_', 'e__3.6_', '__4.5_', 'e__4.5_' ] catname = 'II/305/catalog' Vizier.ROW_LIMIT = -1 Vizier.TIMEOUT = 1000000 Vizier.columns = cols result = Vizier.query_constraints(catalog=catname, RA='>=' + str(ra0) + ' & <' + str(ra1)) # Check for failure if len(result) == 0: if silent == False: logger.info('Failure or No Results') return [] ref = result[0] ref.meta['description'] = ref.meta['description'][0:50] #ref = QUERYVIZIER(refname,[cenra,cendec],radius*60,cfa=cfa,timeout=600,/silent) # Fix/homogenize the GAIA tags if refname == 'GAIA': nref = len(ref) orig = ref.copy() dt = [('source', int), ('ra_icrs', float), ('e_ra_icrs', float), ('de_icrs', float), ('e_de_icrs', float), ('fg', float), ('e_fg', float), ('gmag', float)] ref = np.zeros(nref, dtype=np.dtype(dt)) ref = Table(ref) for n in orig.colnames: ref[n] = orig[n] ref['fg'] = orig['_fg_'] ref['e_fg'] = orig['e__fg_'] ref['gmag'] = orig['_gmag_'] del orig # Fix/homogenize the 2MASS tags elif refname == 'TMASS': nref = len(ref) orig = ref.copy() dt = [('designation', (np.str, 50)), ('raj2000', float), ('dej2000', float), ('jmag', float), ('e_jmag', float), ('hmag', float), ('e_hmag', float), ('kmag', float), ('e_kmag', float), ('qflg', (np.str, 20))] ref = np.zeros(nref, dtype=np.dtype(dt)) for n in orig.colnames: ref[n] = orig[n] ref['designation'] = orig['_2mass'] del orig # Fix NANs in ALLWISE elif refname == 'ALLWISE': bd, = np.where(np.isfinite(ref['_3_6_']) == False) if len(bd) > 0: ref['_3_6_'][bd] = 99.99 ref['e__3_6_'][bd] = 9.99 bd, = np.where(np.isfinite(ref['_4_5_']) == False) if len(bd) > 0: ref['_4_5_'][bd] = 99.99 ref['e__4_5_'][bd] = 9.99 # Convert all mags and errmags to float32 for n in ref.colnames: if n.find('mag') > -1: ref[n] = ref[n].astype(np.float32) if n.find('e_') > -1 and n.find('mag') > -1: ref[n] = ref[n].astype(np.float32) # Lowercase column names for n in ref.colnames: ref[n].name = n.lower() # Convert raj2000/dej2000 to ra/dec if 'raj2000' in ref.colnames: ref['raj2000'].name = 'ra' if 'dej2000' in ref.colnames: ref['dej2000'].name = 'dec' # Save the file logger.info('Saving catalog to file ' + savefile) ref.write(savefile, overwrite=True) if silent == False: logger.info('%d sources found dt=%.1f sec.' % (len(ref), time.time() - t0)) return ref
def run(image,psfname='gaussian',iterdet=0,psfsubnei=False,psffitradius=None,fitradius=None,npsfpix=51, binned=False,lookup=False,lorder=0,psftrim=None,recenter=True,reject=False,apcorr=False, timestamp=False,verbose=False): """ Run PSF photometry on an image. Parameters ---------- image : string or CCDData object The input image to fit. This can be the filename or CCDData object. psfname : string, optional The name of the PSF type to use. The options are "gaussian", "moffat", "penny" and "gausspow". Default is "gaussian". iterdet : boolean, optional Number of iterations to use for detection. Default is iterdet=0, meaning detection is only performed once. psfsubnei : boolean, optional Subtract neighboring stars to PSF stars when generating the PSF. Default is False. psffitradius : float, optional The fitting readius when constructing the PSF (in pixels). By default the FWHM is used. fitradius: float, optional The fitting radius when fitting the PSF to the stars in the image (in pixels). By default the PSF FWHM is used. npsfpix : int, optional The size of the PSF footprint. Default is 51. binned : boolean, optional Use a binned model that integrates the analytical function across a pixel. Default is false. lookup : boolean, optional Use an empirical lookup table. Default is False. lorder : int, optional The order of the spatial variations (0=constant, 1=linear). Default is 0. psftrim: float, optional Trim the PSF size to a radius where "psftrim" fraction of flux is removed. Default is None. recenter : boolean, optional Allow the centroids to be fit. Default is True. reject : boolean, optional When constructin the PSF, reject PSF stars with high RMS values. Default is False. apcorr : boolean, optional Apply aperture correction. Default is False. timestamp : boolean, optional Add timestamp in verbose output (if verbose=True). Default is False. verbose : boolean, optional Verbose output to the screen. Default is False. Returns ------- cat : table The output table of best-fit PSF values for all of the sources. model : CCDData object The best-fitting model for the stars (without sky). sky : CCDData object The background sky image used for the image. psf : PSF object The best-fitting PSF model. Example ------- cat,model,sky,psf = prometheus.run(image,psfname='gaussian',verbose=True) """ # Set up the logger if timestamp and verbose: logger = dln.basiclogger() logger.handlers[0].setFormatter(logging.Formatter("%(asctime)s [%(levelname)-5.5s] %(message)s")) logger.handlers[0].setStream(sys.stdout) builtins.logger = logger # make it available globally across all modules start = time.time() print = utils.getprintfunc() # Get print function to be used locally, allows for easy logging # Load the file if isinstance(image,str): filename = image if verbose: print('Loading image from "'+filename+'"') image = CCDData.read(filename) if isinstance(image,CCDData) is False: raise ValueError('Input image must be a filename or CCDData object') residim = image.copy() # Processing steps #----------------- for niter in range(iterdet+1): if verbose and iterdet>0: print('--- Iteration = '+str(niter+1)+' ---') # 1) Detection #------------- if verbose: print('Step 1: Detection') objects = detection.detect(residim) objects['ndetiter'] = niter+1 if verbose: print(str(len(objects))+' objects detected') # 2) Aperture photometry #----------------------- if verbose: print('Step 2: Aperture photometry') objects = aperture.aperphot(residim,objects) nobjects = len(objects) # Bright and faint limit, use 5th and 95th percentile if niter==0: minmag, maxmag = np.sort(objects['mag_auto'])[[int(np.round(0.05*nobjects)),int(np.round(0.95*nobjects))]] if verbose: print('Min/Max mag: %5.2f, %5.2f' % (minmag,maxmag)) # 3) Construct the PSF #--------------------- # only on first iteration if niter==0: if verbose: print('Step 3: Construct the PSF') # 3a) Estimate FWHM #------------------ fwhm = utils.estimatefwhm(objects,verbose=verbose) # 3b) Pick PSF stars #------------------ psfobj = utils.pickpsfstars(objects,fwhm,verbose=verbose) # 3c) Construct the PSF iteratively #--------------------------------- # Make the initial PSF slightly elliptical so it's easier to fit the orientation if psfname.lower() != 'empirical': initpsf = models.psfmodel(psfname,[fwhm/2.35,0.9*fwhm/2.35,0.0],binned=binned,npix=npsfpix) else: initpsf = models.psfmodel(psfname,npix=npsfpix,imshape=image.shape,order=lorder) # run getpsf psf,psfpars,psfperror,psfcat = getpsf.getpsf(initpsf,image,psfobj,fitradius=psffitradius, lookup=lookup,lorder=lorder,subnei=psfsubnei, allcat=objects,reject=reject,verbose=(verbose>=2)) # Trim the PSF if psftrim is not None: oldnpix = psf.npix psf.trim(psftrim) if verbose: print('Trimming PSF size from '+str(oldnpix)+' to '+str(self.npix)) if verbose: print('Final PSF: '+str(psf)) gd, = np.where(psfcat['reject']==0) print('Median RMS: %.4f' % np.median(psfcat['rms'][gd])) # 4) Run on all sources #---------------------- # If niter>0, then use combined object catalog if iterdet>0: # Add detection # Combine objects catalogs if niter==0: allobjects = objects.copy() else: objects['id'] = np.arange(len(objects))+1+np.max(allobjects['id']) allobjects = vstack((allobjects,objects)) if 'group_id' in allobjects.keys(): allobjects.remove_column('group_id') else: allobjects = objects if verbose: print('Step 4: Get PSF photometry for all '+str(len(allobjects))+' objects') psfout,model,sky = allfit.fit(psf,image,allobjects,fitradius=fitradius, recenter=recenter,verbose=(verbose>=2)) # Construct residual image if iterdet>0: residim = image.copy() residim.data -= model.data # Combine aperture and PSF columns outobj = allobjects.copy() # rename some columns for clarity outobj['x'].name = 'xc' outobj['y'].name = 'yc' outobj['a'].name = 'asemi' outobj['b'].name = 'bsemi' outobj['flux'].name = 'sumflux' outobj.remove_columns(['cxx','cyy','cxy']) # copy over PSF output columns for n in psfout.columns: outobj[n] = psfout[n] outobj['psfamp'] = outobj['amp'].copy() outobj['amp_error'].name = 'psfamp_error' outobj['flux'].name = 'psfflux' outobj['flux_error'].name = 'psfflux_error' # change mag, magerr to psfmag, psfmag_error outobj['mag'].name = 'psfmag' outobj['mag_error'].name = 'psfmag_error' # put ID at the beginning cols = np.char.array(list(outobj.columns)) newcols = ['id']+list(cols[cols!='id']) outobj = outobj[newcols] # 5) Apply aperture correction #----------------------------- if apcorr: if verbose: print('Step 5: Applying aperture correction') outobj,grow,cgrow = aperture.apercorr(psf,image,outobj,psfcat,verbose=verbose) # Add exposure time correction exptime = image.header.get('exptime') if exptime is not None: if verbose: print('Applying correction for exposure time %.2f s' % exptime) outobj['psfmag'] += 2.5*np.log10(exptime) # Add coordinates if there's a WCS if image.wcs is not None: if image.wcs.has_celestial: if verbose: print('Adding RA/DEC coordinates to catalog') skyc = image.wcs.pixel_to_world(outobj['x'],outobj['y']) outobj['ra'] = skyc.ra outobj['dec'] = skyc.dec if verbose: print('dt = %.2f sec' % (time.time()-start)) # Breakdown logger if timestamp and verbose: del builtins.logger return outobj,model,sky,psf
def visitcomb(allvisit, starver, load=None, apred='r13', telescope='apo25m', nres=[5, 4.25, 3.5], bconly=False, plot=False, write=True, dorvfit=True, apstar_vers='stars', logger=None): """ Combine multiple visits with individual RVs to rest frame sum """ if logger is None: logger = dln.basiclogger() if load is None: load = apload.ApLoad(apred=apred, telescope=telescope) cspeed = 2.99792458e5 # speed of light in km/s logger.info('Doing visitcomb for {:s} '.format(allvisit['apogee_id'][0])) wnew = norm.apStarWave() nwave = len(wnew) nvisit = len(allvisit) # initialize array for stack of interpolated spectra zeros = np.zeros([nvisit, nwave]) izeros = np.zeros([nvisit, nwave], dtype=int) stack = apload.ApSpec(zeros, err=zeros.copy(), bitmask=izeros, cont=zeros.copy(), sky=zeros.copy(), skyerr=zeros.copy(), telluric=zeros.copy(), telerr=zeros.copy()) apogee_target1, apogee_target2, apogee_target3 = 0, 0, 0 apogee2_target1, apogee2_target2, apogee2_target3, apogee2_target4 = 0, 0, 0, 0 starflag, andflag = np.uint64(0), np.uint64(0) starmask = bitmask.StarBitMask() # Loop over each visit and interpolate to final wavelength grid if plot: fig, ax = plots.multi(1, 2, hspace=0.001) for i, visit in enumerate(allvisit): if bconly: vrel = -visit['bc'] else: vrel = visit['vrel'] # Skip if we don't have an RV if np.isfinite(vrel) is False: continue # Load the visit if load.telescope == 'apo1m': apvisit = load.apVisit1m(visit['plate'], visit['mjd'], visit['apogee_id'], load=True) else: apvisit = load.apVisit(int(visit['plate']), visit['mjd'], visit['fiberid'], load=True) pixelmask = bitmask.PixelBitMask() # Rest-frame wavelengths transformed to this visit spectra w = norm.apStarWave() * (1.0 + vrel / cspeed) # Loop over the chips for chip in range(3): # Get the pixel values to interpolate to pix = wave.wave2pix(w, apvisit.wave[chip, :]) gd, = np.where(np.isfinite(pix)) # Get a smoothed, filtered spectrum to use as replacement for bad values cont = gaussian_filter( median_filter(apvisit.flux[chip, :], [501], mode='reflect'), 100) errcont = gaussian_filter( median_filter(apvisit.flux[chip, :], [501], mode='reflect'), 100) bd, = np.where(apvisit.bitmask[chip, :] & pixelmask.badval()) if len(bd) > 0: apvisit.flux[chip, bd] = cont[bd] apvisit.err[chip, bd] = errcont[bd] # Load up quantity/error pairs for interpolation raw = [[apvisit.flux[chip, :], apvisit.err[chip, :]**2], [apvisit.sky[chip, :], apvisit.skyerr[chip, :]**2], [apvisit.telluric[chip, :], apvisit.telerr[chip, :]**2]] # Load up individual mask bits for ibit, name in enumerate(pixelmask.name): if name is not '' and len( np.where(apvisit.bitmask[chip, :] & 2**ibit)[0]) > 0: raw.append([ np.clip(apvisit.bitmask[chip, :] & 2**ibit, None, 1), None ]) # Do the sinc interpolation out = sincint.sincint(pix[gd], nres[chip], raw) # From output flux, get continuum to remove, so that all spectra are # on same scale. We'll later multiply in the median continuum flux = out[0][0] stack.cont[i, gd] = gaussian_filter( median_filter(flux, [501], mode='reflect'), 100) # Load interpolated spectra into output stack stack.flux[i, gd] = out[0][0] / stack.cont[i, gd] stack.err[i, gd] = out[0][1] / stack.cont[i, gd] stack.sky[i, gd] = out[1][0] stack.skyerr[i, gd] = out[1][1] stack.telluric[i, gd] = out[2][0] stack.telerr[i, gd] = out[2][1] # For mask, set bits where interpolated value is above some threshold # defined for each mask bit iout = 3 for ibit, name in enumerate(pixelmask.name): if name is not '' and len( np.where(apvisit.bitmask[chip, :] & 2**ibit)[0]) > 0: j = np.where( np.abs(out[iout][0]) > pixelmask.maskcontrib[ibit])[0] stack.bitmask[i, gd[j]] |= 2**ibit iout += 1 # Increase uncertainties for persistence pixels bd, = np.where( (stack.bitmask[i, :] & pixelmask.getval('PERSIST_HIGH')) > 0) if len(bd) > 0: stack.err[i, bd] *= np.sqrt(5) bd, = np.where(( (stack.bitmask[i, :] & pixelmask.getval('PERSIST_HIGH')) == 0) & ( (stack.bitmask[i, :] & pixelmask.getval('PERSIST_MED')) > 0)) if len(bd) > 0: stack.err[i, bd] *= np.sqrt(4) bd, = np.where( ((stack.bitmask[i, :] & pixelmask.getval('PERSIST_HIGH')) == 0) & ((stack.bitmask[i, :] & pixelmask.getval('PERSIST_MED')) == 0) & ((stack.bitmask[i, :] & pixelmask.getval('PERSIST_LOW')) > 0)) if len(bd) > 0: stack.err[i, bd] *= np.sqrt(3) bd, = np.where( (stack.bitmask[i, :] & pixelmask.getval('SIG_SKYLINE')) > 0) if len(bd) > 0: stack.err[i, bd] *= np.sqrt(100) if plot: ax[0].plot(norm.apStarWave(), stack.flux[i, :]) ax[1].plot(norm.apStarWave(), stack.flux[i, :] / stack.err[i, :]) plt.draw() pdb.set_trace() # Accumulate for header of combined frame. Turn off visit specific RV flags first visitflag = visit['starflag'] & ~starmask.getval( 'RV_REJECT') & ~starmask.getval('RV_SUSPECT') starflag |= visitflag andflag &= visitflag if visit['survey'] == 'apogee': apogee_target1 |= visit['apogee_target1'] apogee_target2 |= visit['apogee_target2'] apogee_target3 |= visit['apogee_target3'] elif visit['survey'].find('apogee2') >= 0: apogee2_target1 |= visit['apogee_target1'] apogee2_target2 |= visit['apogee_target2'] apogee2_target3 |= visit['apogee_target3'] try: apogee2_target4 |= visit['apogee_target4'] except: pass # MWM target flags? # Create final spectrum zeros = np.zeros([nvisit + 2, nwave]) izeros = np.zeros([nvisit + 2, nwave], dtype=int) apstar = apload.ApSpec(zeros, err=zeros.copy(), bitmask=izeros, wave=norm.apStarWave(), sky=zeros.copy(), skyerr=zeros.copy(), telluric=zeros.copy(), telerr=zeros.copy(), cont=zeros.copy(), template=zeros.copy()) apstar.header['CRVAL1'] = norm.logw0 apstar.header['CDELT1'] = norm.dlogw apstar.header['CRPIX1'] = 1 apstar.header['CTYPE1'] = ( 'LOG-LINEAR', 'Logarithmic wavelength scale in subsequent HDU') apstar.header['DC-FLAG'] = 1 # Pixel-by-pixel weighted average cont = np.median(stack.cont, axis=0) apstar.flux[0, :] = np.sum(stack.flux / stack.err**2, axis=0) / np.sum( 1. / stack.err**2, axis=0) * cont apstar.err[0, :] = np.sqrt(1. / np.sum(1. / stack.err**2, axis=0)) * cont apstar.bitmask[0, :] = np.bitwise_and.reduce(stack.bitmask, 0) apstar.cont[0, :] = cont # Individual visits apstar.flux[2:, :] = stack.flux * stack.cont apstar.err[2:, :] = stack.err * stack.cont apstar.bitmask[2:, :] = stack.bitmask apstar.sky[2:, :] = stack.sky apstar.skyerr[2:, :] = stack.skyerr apstar.telluric[2:, :] = stack.telluric apstar.telerr[2:, :] = stack.telerr # Populate header apstar.header['OBJID'] = (allvisit['apogee_id'][0], 'APOGEE object name') apstar.header['APRED'] = (apred, 'APOGEE reduction version') apstar.header['STARVER'] = (starver, 'apStar version') apstar.header['HEALPIX'] = (apload.obj2healpix(allvisit['apogee_id'][0]), 'HEALPix location') try: apstar.header['SNR'] = (np.nanmedian(apstar.flux / apstar.err), 'Median S/N per apStar pixel') except: apstar.header['SNR'] = (0., 'Median S/N per apStar pixel') apstar.header['RA'] = (allvisit['ra'].max(), 'right ascension, deg, J2000') apstar.header['DEC'] = (allvisit['dec'].max(), 'declination, deg, J2000') apstar.header['GLON'] = (allvisit['glon'].max(), 'Galactic longitude') apstar.header['GLAT'] = (allvisit['glat'].max(), 'Galactic latitude') apstar.header['J'] = (allvisit['j'].max(), '2MASS J magnitude') apstar.header['J_ERR'] = (allvisit['j_err'].max(), '2MASS J magnitude uncertainty') apstar.header['H'] = (allvisit['h'].max(), '2MASS H magnitude') apstar.header['H_ERR'] = (allvisit['h_err'].max(), '2MASS H magnitude uncertainty') apstar.header['K'] = (allvisit['k'].max(), '2MASS K magnitude') apstar.header['K_ERR'] = (allvisit['k_err'].max(), '2MASS K magnitude uncertainty') try: apstar.header['SRC_H'] = (allvisit['src_h'][0], 'source of H magnitude') except KeyError: pass keys = [ 'wash_m', 'wash_t2', 'ddo51', 'irac_3_6', 'irac_4_5', 'irac_5_8', 'wise_4_5', 'targ_4_5' ] for key in keys: try: apstar.header[key] = allvisit[key].max() except KeyError: pass apstar.header['AKTARG'] = (allvisit['ak_targ'].max(), 'Extinction used for targeting') apstar.header['AKMETHOD'] = (allvisit['ak_targ_method'][0], 'Extinction method using for targeting') apstar.header['AKWISE'] = (allvisit['ak_wise'].max(), 'WISE all-sky extinction') apstar.header['SFD_EBV'] = (allvisit['sfd_ebv'].max(), 'SFD E(B-V)') apstar.header['APTARG1'] = (apogee_target1, 'APOGEE_TARGET1 targeting flag') apstar.header['APTARG2'] = (apogee_target2, 'APOGEE_TARGET2 targeting flag') apstar.header['APTARG3'] = (apogee_target3, 'APOGEE_TARGET3 targeting flag') apstar.header['AP2TARG1'] = (apogee2_target1, 'APOGEE2_TARGET1 targeting flag') apstar.header['AP2TARG2'] = (apogee2_target2, 'APOGEE2_TARGET2 targeting flag') apstar.header['AP2TARG3'] = (apogee2_target3, 'APOGEE2_TARGET3 targeting flag') apstar.header['AP2TARG4'] = (apogee2_target4, 'APOGEE2_TARGET4 targeting flag') apstar.header['NVISITS'] = (len(allvisit), 'Number of visit spectra combined flag') apstar.header['STARFLAG'] = (starflag, 'bitwise OR of individual visit starflags') apstar.header['ANDFLAG'] = (andflag, 'bitwise AND of individual visit starflags') try: apstar.header['N_COMP'] = (allvisit['n_components'].max(), 'Maximum number of components in RV CCFs') except: pass apstar.header['VHBARY'] = ( (allvisit['vheliobary'] * allvisit['snr']).sum() / allvisit['snr'].sum(), 'S/N weighted mean barycentric RV') if len(allvisit) > 1: apstar.header['vscatter'] = (allvisit['vheliobary'].std(ddof=1), 'standard deviation of visit RVs') else: apstar.header['VSCATTER'] = (0., 'standard deviation of visit RVs') apstar.header['VERR'] = (0., 'unused') apstar.header['RV_TEFF'] = (allvisit['rv_teff'].max(), 'Effective temperature from RV fit') apstar.header['RV_LOGG'] = (allvisit['rv_logg'].max(), 'Surface gravity from RV fit') apstar.header['RV_FEH'] = (allvisit['rv_feh'].max(), 'Metallicity from RV fit') if len(allvisit) > 0: meanfib = (allvisit['fiberid'] * allvisit['snr']).sum() / allvisit['snr'].sum() else: meanfib = 999999. if len(allvisit) > 1: sigfib = allvisit['fiberid'].std(ddof=1) else: sigfib = 0. apstar.header['MEANFIB'] = (meanfib, 'S/N weighted mean fiber number') apstar.header['SIGFIB'] = ( sigfib, 'standard deviation (unweighted) of fiber number') apstar.header['NRES'] = ('{:5.2f}{:5.2f}{:5.2f}'.format(*nres), 'number of pixels/resolution used for sinc') # individual visit information in header for i0, visit in enumerate(allvisit): i = i0 + 1 apstar.header['SFILE{:d}'.format(i)] = ( visit['file'], ' Visit #{:d} spectrum file'.format(i)) apstar.header['DATE{:d}'.format(i)] = ( visit['dateobs'], 'DATE-OBS of visit {:d}'.format(i)) apstar.header['JD{:d}'.format(i)] = ( visit['jd'], 'Julian date of visit {:d}'.format(i)) # hjd = helio_jd(visitstr[i].jd-2400000.0,visitstr[i].ra,visitstr[i].dec) #apstar.header['HJD{:d}'.format(i)] = apstar.header['FIBER{:d}'.format(i)] = (visit['fiberid'], ' Fiber, visit {:d}'.format(i)) apstar.header['BC{:d}'.format(i)] = ( visit['bc'], ' Barycentric correction (km/s), visit {:d}'.format(i)) apstar.header['VRAD{:d}'.format(i)] = ( visit['vrel'], ' Doppler shift (km/s) of visit {:d}'.format(i)) #apstar.header['VERR%d'.format(i)] = apstar.header['VHBARY{:d}'.format(i)] = ( visit['vheliobary'], ' Barycentric velocity (km/s), visit {:d}'.format(i)) apstar.header['SNRVIS{:d}'.format(i)] = ( visit['snr'], ' Signal/Noise ratio, visit {:d}'.format(i)) apstar.header['FLAG{:d}'.format(i)] = ( visit['starflag'], ' STARFLAG for visit {:d}'.format(i)) apstar.header.insert('SFILE{:d}'.format(i), ('COMMENT', 'VISIT {:d} INFORMATION'.format(i))) # Do a RV fit just to get a template and normalized spectrum, for plotting if dorvfit: try: apstar.setmask(pixelmask.badval()) spec = doppler.Spec1D(apstar.flux[0, :], err=apstar.err[0, :], bitmask=apstar.bitmask[0, :], mask=apstar.mask[0, :], wave=apstar.wave, lsfpars=np.array([0]), lsfsigma=apstar.wave / 22500 / 2.354, instrument='APOGEE', filename=apstar.filename) out = doppler.rv.jointfit([spec], verbose=False, plot=False, tweak=False, maxvel=[-50, 50]) apstar.cont = out[3][0].flux apstar.template = out[2][0].flux except ValueError as err: logger.error('Exception raised in visitcomb RV for: ', apstar.header['FIELD'], apstar.header['OBJID']) logger.error("ValueError: {0}".format(err)) except RuntimeError as err: logger.error('Exception raised in visitcomb RV for: ', apstar.header['FIELD'], apstar.header['OBJID']) logger.error("Runtime error: {0}".format(err)) except: logger.error('Exception raised in visitcomb RV fit for: ', apstar.header['FIELD'], apstar.header['OBJID']) # Write the spectrum to file if write: outfilenover = load.filename('Star', obj=apstar.header['OBJID']) outdir = os.path.dirname(outfilenover) outbase = os.path.splitext(os.path.basename(outfilenover))[0] outbase += '-' + starver # add star version outfile = outdir + '/' + outbase + '.fits' if apstar_vers != 'stars': outfile = outfile.replace('/stars/', '/' + apstar_vers + '/') outdir = os.path.dirname(outfile) try: os.makedirs(os.path.dirname(outfile)) except: pass logger.info('Writing apStar file to ' + outfile) apstar.write(outfile) apstar.filename = outfile mwm_root = os.environ['MWM_ROOT'] apstar.uri = outfile[len(mwm_root) + 1:] # Create symlink no file with no version if os.path.exists(outfilenover) or os.path.islink(outfilenover): os.remove(outfilenover) os.symlink(os.path.basename(outfile), outfilenover) # relative path # Plot gd, = np.where( (apstar.bitmask[0, :] & (pixelmask.badval() | pixelmask.getval('SIG_SKYLINE'))) == 0) fig, ax = plots.multi(1, 3, hspace=0.001, figsize=(48, 6)) med = np.nanmedian(apstar.flux[0, :]) plots.plotl(ax[0], norm.apStarWave(), apstar.flux[0, :], color='k', yr=[0, 2 * med]) ax[0].plot(norm.apStarWave()[gd], apstar.flux[0, gd], color='g') ax[0].set_ylabel('Flux') try: ax[1].plot(norm.apStarWave()[gd], apstar.cont[gd], color='g') ax[1].set_ylabel('Normalized') ax[1].plot(norm.apStarWave(), apstar.template, color='r') except: pass plots.plotl(ax[2], norm.apStarWave(), apstar.flux[0, :] / apstar.err[0, :], yt='S/N') for i in range(3): ax[i].set_xlim(15100, 17000) ax[0].set_xlabel('Wavelength') fig.savefig(outdir + '/plots/' + outbase + '.png') # Plot if plot: ax[0].plot(norm.apStarWave(), apstar.flux, color='k') ax[1].plot(norm.apStarWave(), apstar.flux / apstar.err, color='k') plt.draw() pdb.set_trace() return apstar
def doppler_rv(star, apred, telescope, nres=[5, 4.25, 3.5], windows=None, tweak=False, clobber=False, verbose=False, plot=False, logger=None): """ Run Doppler on one star and perform visit combination. Parameters ---------- star : str The '2M' star name. apred : str APOGEE reduction version. telescope : str APOGEE telescope (apo25m, loc25m, apo1m). nres : array, optional Array of sinc widths. Default is nres=[5,4.25,3.5]. windows : array, optional Array of spectral windows to use. tweak : bool, optional Have Doppler tweak the continuum with the best-fit template. Default is False. clobber : bool, optional Overwrite any existing files (both RV and visit combined). verbose : bool, optional Verbose output to the screen. plot : bool, optional Make a plot of spectrum and best-fitting template. logger : logger, optional A logger for printed output. Returns ------- The program outputs Doppler files an apStar combined file to the appropriate place specified in the SDSS/APOGEE tree product. """ if logger is None: logger = dln.basiclogger() snmin = 3 apstar_vers = 'stars' logger.info( 'Running Doppler and performing visit combination for %s and telescope=%s' % (star, telescope)) # Get the visit files for this star and telescope db = apogeedb.DBSession() allvisits = db.query('visit', cols='*', where="apogee_id='" + star + "' and telescope='" + telescope + "' and apred_vers='" + apred + "'") db.close() nallvisits = len(allvisits) if nallvisits == 0: logger.info('No visit files found') return logger.info('%d visit file(s) found' % nallvisits) # Get the star version number # this is the largest MJD5 in the FULL list of visits starver = str(np.max(int(allvisits['mjd']))) logger.info('Version=' + starver) # Select good visit spectra starmask = bitmask.StarBitMask() gd, = np.where(((allvisits['starflag'] & starmask.badval()) == 0) & (allvisits['snr'] > snmin)) if len(gd) == 0: logger.info('No visits passed QA cuts') return logger.info('%d visit(s) passed QA cuts' % len(gd)) starvisits = Table(allvisits[gd]) nvisits = len(gd) # Change datatype of STARFLAG to 64-bit starvisits['starflag'] = starvisits['starflag'].astype(np.uint64) # Add STARVER starvisits['starver'] = starver # Output directory load = apload.ApLoad(apred=apred, telescope=telescope) starfile = load.filename('Star', obj=star) stardir = os.path.dirname(starfile) try: os.makedirs(os.path.dirname(stardir)) except FileExistsError: pass # Run Doppler with dorv() try: dopsumstr, dopvisitstr, gaussout = dorv(starvisits, starver, clobber=clobber, verbose=verbose, tweak=tweak, plot=plot, windows=windows, apstar_vers=apstar_vers, logger=logger) logger.info('Doppler completed successfully for {:s}'.format(star)) except: logger.info('Doppler failed for {:s}'.format(star)) raise # KLUDGE, rename vhelio->vheliobary if 'vhelio' in starvisits.colnames: starvisits['vhelio'].name = 'vheliobary' # First rename old visit RV tags and initialize new ones for col in [ 'vtype', 'vrel', 'vrelerr', 'vheliobary', 'bc', 'rv_teff', 'rv_logg', 'rv_feh', 'rv_carb', 'rv_alpha' ]: starvisits.rename_column(col, 'est' + col) if col == 'vtype': starvisits[col] = 0 else: starvisits[col] = np.nan for col in ['xcorr_vrel', 'xcorr_vrelerr', 'xcorr_vheliobary', 'bc']: starvisits[col] = np.nan # Add columns for RV components starvisits['n_components'] = -1 rv_components = Column(name='rv_components', dtype=float, shape=(3), length=len(starvisits)) starvisits.add_column(rv_components) # Now load the new ones with the dorv() output visits = [] ncomponents = 0 for i, (v, g) in enumerate(zip(dopvisitstr, gaussout)): # Match by filename components in case there was an error reading in doppler name = os.path.basename(v['filename']).replace('.fits', '').split('-') if telescope == 'apo1m': vind, = np.where( np.char.strip(starvisits['file']).astype(str) == os.path.basename(v['filename'].strip())) if len(vind) == 0: # special case for incremental release...yuck vind, = np.where( np.char.strip(starvisits['file']).astype(str) == os.path. basename(v['filename'].strip()).replace('-r13-', '-r12-')) else: vind, = np.where( (np.char.strip(starvisits['plate']).astype(str) == name[-3]) & (starvisits['mjd'] == int(name[-2])) & (starvisits['fiberid'] == int(name[-1]))) if len(vind) > 0: vind = vind[0] else: continue visits.append(vind) starvisits[vind]['vrel'] = v['vrel'] starvisits[vind]['vrelerr'] = v['vrelerr'] starvisits[vind]['vheliobary'] = v['vhelio'] starvisits[vind]['xcorr_vrel'] = v['xcorr_vrel'] starvisits[vind]['xcorr_vrelerr'] = v['xcorr_vrelerr'] starvisits[vind]['xcorr_vheliobary'] = v['xcorr_vhelio'] starvisits[vind]['bc'] = v['bc'] starvisits[vind]['rv_teff'] = v['teff'] starvisits[vind]['rv_logg'] = v['logg'] starvisits[vind]['rv_feh'] = v['feh'] if g is None: starvisits[vind]['n_components'] = 0 else: starvisits[vind]['n_components'] = g['N_components'] if starvisits[vind]['n_components'] > 1: starvisits[vind]['starflag'] |= starmask.getval('MULTIPLE_SUSPECT') n = len(g['best_fit_parameters']) // 3 gd, = np.where(np.array(g['best_fit_parameters'])[0:n] > 0) rv_comp = np.array(g['best_fit_parameters'])[2 * n + gd] n_rv_comp = np.min([3, len(rv_comp)]) starvisits[vind]['rv_components'][0:n_rv_comp] = rv_comp[ 0:n_rv_comp] # flag visits with suspect RVs if starvisits[vind]['rv_teff'] < 6000: bd_diff = 10 else: bd_diff = 50. if (np.abs(starvisits[vind]['vheliobary'] - starvisits[vind]['xcorr_vheliobary']) > bd_diff): starvisits[vind]['starflag'] |= starmask.getval('RV_REJECT') elif (np.abs(starvisits[vind]['vheliobary'] - starvisits[vind]['xcorr_vheliobary']) > 0): starvisits[vind]['starflag'] |= starmask.getval('RV_SUSPECT') # Get the good visits if len(visits) > 0: visits = np.array(visits) gdrv, = np.where((starvisits[visits]['starflag'] & starmask.getval('RV_REJECT')) == 0) # Do the visit combination if len(gdrv) > 0: apstar = visitcomb(starvisits[visits[gdrv]], starver, load=load, apstar_vers=apstar_vers, apred=apred, nres=nres, logger=logger) else: logger.info('No good visits for ' + star) raise # Load information into the database apstar.header['MJDBEG'] = (np.max(int(allvisits['mjd'])), 'Beginning MJD') apstar.header['MJDEND'] = (np.max(int(allvisits['mjd'])), 'Ending MJD') dbingest(apstar, starvisits[visits[gdrv]]) return
def dorv(allvisit, starver, obj=None, telescope=None, apred=None, clobber=False, verbose=False, tweak=False, plot=False, windows=None, apstar_vers='stars', logger=None): """ Do the Doppler rv jointfit from list of files """ if logger is None: logger = dln.basiclogger() if tweak == True: suffix = '_tweak' else: suffix = '_out' if obj is None: obj = str(allvisit['apogee_id'][0]) if type(obj) is not str: obj = obj.decode('UTF-8') if apred is None: apred = str(allvisit['apred_vers'][0]) if telescope is None: telescope = str(allvisit['telescope'][0]) load = apload.ApLoad(apred=apred, telescope=telescope) outfile = load.filename('Star', obj=obj) outdir = os.path.dirname(outfile) outbase = os.path.splitext(os.path.basename(outfile))[0] outbase += '-' + starver # add star version if os.path.exists(outdir) == False: os.makedirs(outdir) if apstar_vers != 'stars': outdir = outdir.replace('/stars/', '/' + apstar_vers + '/') if os.path.exists(outdir + '/' + outbase + suffix + '_doppler.pkl') and not clobber: logger.info(obj + ' already done') fp = open(outdir + '/' + outbase + suffix + '_doppler.pkl', 'rb') try: out = pickle.load(fp) sumstr, finalstr, bmodel, specmlist, gout = out fp.close() return sumstr, finalstr, gout except: logger.warning('error loading: ' + outbase + suffix + '_doppler.pkl') #pass speclist = [] pixelmask = bitmask.PixelBitMask() badval = pixelmask.badval() | pixelmask.getval( 'SIG_SKYLINE') | pixelmask.getval('LITTROW_GHOST') # If we have a significant number of low S/N visits, combine first using # barycentric correction only, use that to get an estimate of systemic # velocity, then do RV determination restricting RVs to within 50 km/s # of estimate. This seems to help significant for faint visits lowsnr_visits, = np.where(allvisit['snr'] < 10) if (len(lowsnr_visits) > 1) & (len(lowsnr_visits) / len(allvisit) > 0.1): try: apstar_bc = visitcomb(allvisit, starver, bconly=True, load=load, write=False, dorvfit=False, apstar_vers=apstar_vers) apstar_bc.setmask(badval) spec = doppler.Spec1D(apstar_bc.flux[0, :], err=apstar_bc.err[0, :], bitmask=apstar_bc.bitmask[0, :], mask=apstar_bc.mask[0, :], wave=apstar_bc.wave, lsfpars=np.array([0]), lsfsigma=apstar_bc.wave / 22500 / 2.354, instrument='APOGEE', filename=apstar_bc.filename) logger.info('Lots of low-S/N visits. Running BC jointfit for :', obj) out = doppler.rv.jointfit([spec], verbose=verbose, plot=plot, tweak=tweak, maxvel=[-500, 500]) rvrange = [out[1][0]['vrel'] - 50, out[1][0]['vrel'] + 50] except: logger.info(' BC jointfit failed') rvrange = [-500, 500] elif allvisit['h'].max() > 13.5: # If it's faint, restrict to +/- 500 km/s rvrange = [-500, 500] else: # Otherwise, restrict to +/ 1000 km/s rvrange = [-1000, 1000] # Loop over visits for i in range(len(allvisit)): # Load all of the visits into doppler Spec1D objects if load.telescope == 'apo1m': visitfile = load.allfile('Visit', plate=allvisit['plate'][i], mjd=allvisit['mjd'][i], reduction=allvisit['apogee_id'][i], field=allvisit['field'][i]) else: visitfile = load.allfile('Visit', plate=int(allvisit['plate'][i]), mjd=allvisit['mjd'][i], fiber=allvisit['fiberid'][i], field=allvisit['field'][i]) spec = doppler.read(visitfile, badval=badval) if windows is not None: # If we have spectral windows to mask, do so here for ichip in range(3): mask = np.full_like(spec.mask[:, ichip], True) gd = [] for window in windows: gd.extend( np.where((spec.wave[:, ichip] > window[0]) & (spec.wave[:, ichip] < window[1]))[0]) mask[gd] = False spec.mask[:, ichip] |= mask if spec is not None: speclist.append(spec) if len(speclist) == 0: raise Exception('No visit spectra loaded') # Now do the Doppler jointfit to get RVs # Dump empty pickle to stand in case of failure (to prevent redo if not clobber) try: # Dump empty pickle to stand in case of failure (to prevent redo if not clobber) fp = open(outdir + '/' + outbase + suffix + '_doppler.pkl', 'wb') pickle.dump(None, fp) fp.close() logger.info( 'Running Doppler jointfit for: {:s} rvrange:[{:.1f},{:.1f}] nvisits: {:d}' .format(obj, *rvrange, len(speclist))) sumstr, finalstr, bmodel, specmlist, dt = doppler.rv.jointfit( speclist, maxvel=rvrange, verbose=verbose, plot=plot, saveplot=plot, outdir=outdir + '/', tweak=tweak) logger.info('Running CCF decomposition for: ' + obj) gout = gauss_decomp(finalstr, phase='two', filt=True) fp = open(outdir + '/' + outbase + suffix + '_doppler.pkl', 'wb') pickle.dump([sumstr, finalstr, bmodel, specmlist, gout], fp) fp.close() # Making plots logger.info('Making plots for :' + obj + ' ' + outdir) try: os.makedirs(outdir + '/plots/') except: pass dop_plot(outdir + '/plots/', outbase, [sumstr, finalstr, bmodel, specmlist], decomp=gout) except KeyboardInterrupt: raise except ValueError as err: logger.error('Exception raised in dorv for: ' + obj) logger.error("ValueError: {0}".format(err)) return except RuntimeError as err: logger.error('Exception raised in dorv for: ' + obj) logger.error("Runtime error: {0}".format(err)) return except: raise logger.error('Exception raised in dorv for: ', field, obj) return # Return summary RV info, visit RV info, decomp info return sumstr, finalstr, gout