def run_mjd5_yaml(yamlfile, logger=None):
"""
Run the MJD5 yaml file and create the relevant plan files.
Parameters
----------
yamlfile : str
Name of the MJD5 yaml file.
Returns
-------
planfiles : list of str
List of the created plan file names.
Examples
--------
planfiles = mkplan.run_mjd5_yaml(yamlfile)
By D.Nidever, Oct 2020
"""
# Logger
if logger is None: logger = dln.basiclogger()
if os.path.exists(yamlfile) == False:
raise ValueError(yamlfile + ' NOT FOUND')
# Load the yaml file
with open(yamlfile) as file:
data = yaml.full_load(file)
if type(data) is not list:
data = [data]
ndata = len(data)
logger.info('Information for ' + str(ndata) + ' plan files')
# Loop over the plan blocks and run mkplan()
planfiles = []
for i in range(ndata):
logger.info(' ')
logger.info('Plan file ' + str(i + 1))
logger.info('------------')
pargs = data[i]
ims = pargs.pop('ims')
plate = pargs.pop('plate')
mjd = pargs.pop('mjd')
psfid = pargs.pop('psfid')
fluxid = pargs.pop('fluxid')
planfile = mkplan(ims,
plate,
mjd,
psfid,
fluxid,
**pargs,
logger=logger)
planfiles.append(planfile)
return planfiles
def maskdiscrepant(spec,model,nsig=10,verbose=False,logger=None):
"""
Mask pixels that are discrepant when compared to a model.
"""
if logger is None:
logger = dln.basiclogger()
spec2 = spec.copy()
wave = spec2.wave.copy().reshape(spec2.npix,spec2.norder) # make 2D
flux = spec2.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D
err = spec2.err.copy().reshape(spec2.npix,spec2.norder) # make 2D
mask = spec2.mask.copy().reshape(spec2.npix,spec2.norder) # make 2D
mflux = model.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D
totnbd = 0
for o in range(spec2.norder):
w = wave[:,o].copy()
x = (w-np.median(w))/(np.max(w*0.5)-np.min(w*0.5)) # -1 to +1
y = flux[:,o].copy()
m = mask[:,o].copy()
my = mflux[:,o].copy()
# Divide by median
medy = np.nanmedian(y)
y /= medy
my /= medy
# Perform sigma clipping out large positive outliers
coef = dln.poly_fit(x,y,2,robust=True)
sig = dln.mad(y-my)
bd,nbd = dln.where( np.abs(y-my) > nsig*sig )
totnbd += nbd
if nbd>0:
flux[bd,o] = dln.poly(x[bd],coef)*medy
err[bd,o] = 1e30
mask[bd,o] = True
# Flatten to 1D if norder=1
if spec2.norder==1:
flux = flux.flatten()
err = err.flatten()
mask = mask.flatten()
# Stuff back in
spec2.flux = flux
spec2.err = err
spec2.mask = mask
if verbose is True: logger.info('Masked '+str(totnbd)+' discrepant pixels')
return spec2
def queue_wait(queue,sleeptime=60,verbose=True,logger=None):
""" Wait for the pbs queue to finish."""
if logger is None:
logger = dln.basiclogger()
# Wait for jobs to complete
running = True
while running:
time.sleep(sleeptime)
percent_complete = queue.get_percent_complete()
if verbose==True:
logger.info('percent complete = %d' % percent_complete)
if percent_complete == 100:
running = False
def maskoutliers(spec,nsig=5,verbose=False,logger=None):
"""
Mask large positive outliers and negative flux pixels in the spectrum.
"""
if logger is None:
logger = dln.basiclogger()
spec2 = spec.copy()
wave = spec2.wave.copy().reshape(spec2.npix,spec2.norder) # make 2D
flux = spec2.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D
err = spec2.err.copy().reshape(spec2.npix,spec2.norder) # make 2D
mask = spec2.mask.copy().reshape(spec2.npix,spec2.norder) # make 2D
totnbd = 0
for o in range(spec2.norder):
w = wave[:,o].copy()
x = (w-np.median(w))/(np.max(w*0.5)-np.min(w*0.5)) # -1 to +1
y = flux[:,o].copy()
m = mask[:,o].copy()
# Divide by median
medy = np.nanmedian(y)
y /= medy
# Perform sigma clipping out large positive outliers
coef = dln.poly_fit(x,y,2,robust=True)
sig = dln.mad(y-dln.poly(x,coef))
bd,nbd = dln.where( ((y-dln.poly(x,coef)) > nsig*sig) | (y<0))
totnbd += nbd
if nbd>0:
flux[bd,o] = dln.poly(x[bd],coef)*medy
err[bd,o] = 1e30
mask[bd,o] = True
# Flatten to 1D if norder=1
if spec2.norder==1:
flux = flux.flatten()
err = err.flatten()
mask = mask.flatten()
# Stuff back in
spec2.flux = flux
spec2.err = err
spec2.mask = mask
if verbose is True: logger.info('Masked '+str(totnbd)+' outlier or negative pixels')
return spec2
def make_mjd5_yaml(mjd, apred, telescope, clobber=False, logger=None):
"""
Make a MJD5 yaml file that can be used to create plan files.
Parameters
----------
mjd : int
MJD number for this night.
apred : str
APOGEE reduction version.
telescope : str
APOGEE telescope: apo25m, apo1m, lco25m.
clobber : bool
Overwrite any existing files.
Returns
-------
out : list of dictionaries
The list of dictionaries that contain the information needed to
make the plan files.
planfiles : list of str
The names of the plan files that would be created.
The yaml files are also written to disk in APOGEEREDUCEPLAN product
directory.
Examples
--------
out,planfiles = mkplan.make_mjd5_yaml(57680,'t15','apo25m')
By J.Holtzman, 2011
translated/rewritten, D.Nidever Oct2020
"""
# Logger
if logger is None: logger = dln.basiclogger()
logger.info('Making MJD5.yaml file for MJD=' + str(mjd))
load = apload.ApLoad(apred=apred, telescope=telescope)
datadir = {
'apo25m': os.environ['APOGEE_DATA_N'],
'apo1m': os.environ['APOGEE_DATA_N'],
'lco25m': os.environ['APOGEE_DATA_S']
}[telescope]
observatory = telescope[0:3]
# Output file/directory
outfile = os.environ[
'APOGEEREDUCEPLAN_DIR'] + '/yaml/' + telescope + '/' + telescope + '_' + str(
mjd) + 'auto.yaml'
if os.path.exists(os.path.dirname(outfile)) == False:
os.makedirs(os.path.dirname(outfile))
# File already exists and clobber not set
if os.path.exists(outfile) and clobber == False:
logger.info(outfile + ' already EXISTS and clobber==False')
return
# Get the exposures and info about them
info = getexpinfo(observatory, mjd)
nfiles = len(info)
logger.info(str(nfiles) + ' exposures found')
# Print summary information about the data
expindex = dln.create_index(info['exptype'])
for i in range(len(expindex['value'])):
logger.info(' ' + expindex['value'][i] + ': ' +
str(expindex['num'][i]))
objind, = np.where(info['exptype'] == 'OBJECT')
if len(objind) > 0:
plates = np.unique(info['plateid'][objind])
logger.info('Observations of ' + str(len(plates)) + ' plates')
plateindex = dln.create_index(info['plateid'][objind])
for i in range(len(plateindex['value'])):
logger.info(' ' + plateindex['value'][i] + ': ' +
str(plateindex['num'][i]))
# Scan through all files, accumulate IDs of the various types
dark, cal, exp, sky, dome, calpsfid = [], [], [], [], None, None
out, planfiles = [], []
for i in range(nfiles):
# Load image number in variable according to exptype and nreads
# discard images with nread<3
if info['nread'][i] < 3:
logger.info(info['num'][i], ' has less than the required 3 reads')
# Dark
if (info['exptype'][i] == 'DARK') and info['nread'][i] >= 3:
dark.append(int(info['num'][i]))
# Internal flat
# reduced only to 2D, hence treated like darks
if (info['exptype'][i] == 'INTERNALFLAT') and info['nread'][i] >= 3:
dark.append(int(info['num'][i]))
# Dome flat
if (info['exptype'][i] == 'QUARTZFLAT') and info['nread'][i] >= 3:
cal.append(int(info['num'][i]))
calpsfid = int(info['num'][i])
# Arc lamps
if (info['exptype'][i] == 'ARCLAMP') and info['nread'][i] >= 3:
cal.append(int(info['num'][i]))
# Sky frame
# identify sky frames as object frames with 10<nread<15
if (info['exptype'][i] == 'OBJECT') and (info['nread'][i] < 15
and info['nread'][i] > 10):
sky.append(int(info['num'][i]))
# Object exposure
if (info['exptype'][i] == 'OBJECT') and info['nread'][i] > 15:
exp.append(int(info['num'][i]))
# Dome flat
if (info['exptype'][i] == 'DOMEFLAT') and info['nread'][i] > 3:
dome = int(info['num'][i])
# End of this plate block
# if plateid changed or last exposure
platechange = info['plateid'][i] != info['plateid'][np.minimum(
i + 1, nfiles - 1)]
if (platechange
or i == nfiles - 1) and len(exp) > 0 and dome is not None:
# Object plate visit
objplan = {
'apred': str(apred),
'telescope': str(load.telescope),
'mjd': int(mjd),
'plate': int(info['plateid'][i]),
'psfid': dome,
'fluxid': dome,
'ims': exp
}
out.append(objplan)
planfile = load.filename('Plan',
plate=int(info['plateid'][i]),
mjd=mjd)
planfiles.append(planfile)
exp = []
# Sky exposures
# use same cals as for object
if len(sky) > 0:
skyplan = {
'apred': str(apred),
'telescope': str(load.telescope),
'mjd': int(mjd),
'plate': int(info['plateid'][i]),
'psfid': dome,
'fluxid': dome,
'ims': sky,
'sky': True
}
out.append(skyplan)
skyplanfile = planfile.replace('.yaml', 'sky.yaml')
planfiles.append(skyplanfile)
sky = []
# Dark frame information
cplate = '0000'
if len(dark) > 0:
darkplan = {
'apred': str(apred),
'telescope': str(load.telescope),
'mjd': int(mjd),
'plate': 0,
'psfid': 0,
'fluxid': 0,
'ims': dark,
'dark': True
}
out.append(darkplan)
planfile = load.filename('DarkPlan', mjd=mjd)
planfiles.append(planfile)
# Calibration frame information
if len(cal) > 0 and calpsfid is not None:
calplan = {
'apred': str(apred),
'telescope': str(load.telescope),
'mjd': int(mjd),
'plate': 0,
'psfid': calpsfid,
'fluxid': calpsfid,
'ims': cal,
'cal': True
}
out.append(calplan)
planfile = load.filename('CalPlan', mjd=mjd)
planfiles.append(planfile)
# Write out the MJD5 file
if os.path.exists(outfile): os.remove(outfile)
logger.info('Writing MJD5.yaml file to ' + outfile)
with open(outfile, 'w') as file:
dum = yaml.dump(out, file, default_flow_style=False, sort_keys=False)
# Copy it to the non-"auto" version
outfile2 = outfile.replace('auto', '')
if os.path.exists(outfile2): os.remove(outfile2)
shutil.copyfile(outfile, outfile2)
return out, planfiles
def mkplan(ims,
plate,
mjd,
psfid,
fluxid,
apred=None,
telescope=None,
cal=False,
dark=False,
sky=False,
plugid=None,
fixfiberid=None,
stars=None,
names=None,
onem=False,
hmags=None,
mapper_data=None,
suffix=None,
ignore=False,
test=False,
logger=None):
"""
Makes plan files given input image numbers, MJD, psfid, fluxid
includes options for dark frames, calibration frames, sky frames,
ASDAF frames. This is called from the manually prepared MJD5.pro
procedures
Parameters
----------
ims : numpy int array
List of array of exposure numbers to include in planfile.
plate : int
Plate number for this observation.
mjd : int
MJD number for this observation.
psfid : int
PSF cal exposure number.
fluxid : int
Flux cal frame exposure number.
apred : str
APOGEE reduction version.
telescope : str
APOGEE telescope.
cal : bool, optional
This is a calibration plan file.
dark : bool, optional
This is a dark sequence plan file.
sky : bool, optional
This is a sky flat sequence plan file.
plugid : int, optional
Base name of the plugmap filename.
fixfiberid : int, optional
Fiber fixing needed (1 or 2).
stars : numpy int array, optional
FiberID for apo1m or ASDAF observations.
names : numpy int array, optional
Name of the star for apo1m or ASDAF observations.
onem : bool, optional
This is for a apo1m observation.
hmags : numpy float array, optional
2MASS H-magnitude for star in apo1m observation.
mapper_data : str, optional
Directory for the mapper data.
suffix : str, optional
Extra suffix to use (before the extension) on the planfile name.
ignore : bool, optional
Ignore warnings and continue.
test : bool, optional
Just a test.
Returns
-------
planfile : str
The name of the plan file created
This creates a planfile with the given inputs and places it in
the appropriate place in the SDSS/APOGEE directory tree. For
science visits this will be in $APOGEE_REDUX/{apred}/visit/{telescope}/{field}/{plate}/{mjd}/.
Calibration, dark and sky plan files live in
$APOGEE_REDUX/{apred}/cal/{instrument}/plan/{mjd}/
Examples
--------
mkplan.mkplan(ims,plate,mjd,psfid,fluxid,apred=apred,telescope=telescope)
By J.Holtzman, 2011?
translated to python, D.Nidever Oct 2020
"""
# Logger
if logger is None: logger = dln.basiclogger()
if apred is None:
raise ValueError('apred must be input')
if telescope is None:
raise ValueError('telescope must be input')
logger.info('Making plan for MJD: ' + str(mjd))
# Set up directories, plate, and MJD variables
load = apload.ApLoad(apred=apred, telescope=telescope)
caldir = os.environ['APOGEE_DRP_DIR'] + '/data/cal/'
calfile = caldir + load.instrument + '.par'
# Mapper directory
if mapper_data is None:
if load.instrument == 'apogee-n':
mapper_data = os.environ['MAPPER_DATA_N']
else:
mapper_data = os.environ['MAPPER_DATA_S']
# Planfile name and directory
if cal == True:
planfile = load.filename('CalPlan', mjd=mjd)
elif dark == True:
planfile = load.filename('DarkPlan', mjd=mjd)
elif onem == True:
planfile = load.filename('Plan',
plate=plate,
reduction=names[0],
mjd=mjd)
if suffix is not None:
planfile = os.path.dirname(planfile) + '/' + os.path.splitext(
os.path.basename(planfile, '.yaml'))[0] + suffix + '.yaml'
else:
planfile = load.filename('Plan', plate=plate, mjd=mjd)
# Make sure the file ends with .yaml
if planfile.endswith('.yaml') == False:
planfile = planfile.replace(os.path.splitext(planfile)[-1],
'.yaml') # TEMPORARY KLUDGE!
outdir = os.path.dirname(planfile) + '/'
if os.path.exists(outdir) == False:
os.makedirs(outdir)
# Get calibration files for this date
if fixfiberid is not None:
fix0 = fixfiberid
else:
fix0 = None
caldata = mkcal.getcal(calfile, mjd)
if fix0 is not None:
caldata['fixfiber'] = fix0
# outplan plan file name
if (stars is not None) & (onem == False):
planfile = os.path.dirname(planfile) + '/' + os.path.basename(
planfile, '.yaml') + 'star.yaml'
else:
if sky == True:
planfile = os.path.dirname(planfile) + '/' + os.path.basename(
planfile, '.yaml') + 'sky.yaml'
if sky == True:
logger.info('apdailycals')
dailycals(lsfs=ims, psf=psfid, apred=apred, telescope=telescope)
logger.info(planfile)
# open plan file and write header
if os.path.exists(planfile): os.remove(planfile)
out = {}
out['apogee_drp_ver'] = os.environ['APOGEE_DRP_VER']
out['telescope'] = telescope
out['instrument'] = load.instrument
out['plateid'] = plate
out['mjd'] = mjd
out['planfile'] = os.path.basename(planfile)
out['logfile'] = 'apDiag-' + str(plate) + '-' + str(mjd) + '.log'
out['plotfile'] = 'apDiag-' + str(plate) + '-' + str(mjd) + '.ps'
# apred_vers keyword will override strict versioning using the plan file!
out['apred_vers'] = apred
# apo1m
if onem == True:
out['data_dir'] = datadir + '/'
out['raw_dir'] = datadir + str(mjd) + '/'
out['plate_dir'] = outdir
out['star_dir'] = spectro_dir + '/fields/apo1m/'
out['survey'] = 'apo1m'
out['name'] = str(names[0]).strip()
out['fiber'] = stars[0]
if hmags is not None:
out['hmag'] = hmags[0]
out['telliter'] = 1
if suffix != '':
out['mjdfrac'] = 1
# platetype
if stars is not None:
out['platetype'] = 'single'
elif cal == True:
out['platetype'] = 'cal'
elif sky == True:
out['platetype'] = 'sky'
elif dark == True:
out['platetype'] = 'dark'
elif test == True:
out['platetype'] = 'test'
else:
out['platetype'] = 'normal'
# Note that q3fix is now done in ap3d.pro, not here!!
if (mjd > 56930) & (mjd < 57600):
out['q3fix'] = 1
sdss_path = path.Path()
rawfile = sdss_path.full('apR', num=ims[0], chip='a', mjd=mjd)
#rawfile = load.filename('R',chip='a',num=ims[0])
if os.path.exists(rawfile) == False:
raise ValueError('Cannot find file ' + rawfile)
head = fits.getheader(rawfile, 1)
plateid = head['PLATEID']
if (ignore == False):
if (plate != 0) & (plate != plateid):
raise ValueError('plateid in header does not match plate!')
# plugmap
logger.info(str(plugid))
if plugid is None:
rawfile = sdss_path.full('apR', num=ims[0], chip='a', mjd=mjd)
#rawfile = load.filename('R',chip='a',num=ims[0])
if os.path.exists(rawfile) == True:
head = fits.getheader(rawfile, 1)
plugid = head['NAME']
if type(plugid) is not str:
plugid = 'header'
else:
plugid = 'header'
logger.info(str(ims[0]))
logger.info(str(plugid))
if (cal == False) & (dark == False) & (onem == False):
tmp = plugid.split('-')
if os.path.exists(mapper_data + '/' + tmp[1] + '/plPlugMapM-' +
plugid + '.par') == False:
logger.info('Cannot find plugmap file ' + str(plugid))
#spawn,'"ls" '+mapper_data+'/'+tmp[1]+'/plPlugMapA*'
if ignore is False:
raise Exception
if sky == False:
logger.info('getting plate data')
plug = platedata.getdata(plate,
mjd,
plugid=plugid,
noobject=True,
mapper_data=mapper_data,
apred=apred,
telescope=telescope)
loc = plug['locationid']
spectro_dir = os.environ['APOGEE_REDUX'] + '/' + apred + '/'
if os.path.exists(spectro_dir + 'fields/' + telescope + '/' +
str(loc)) == False:
os.makedirs(spectro_dir + 'fields/' + telescope + '/' +
str(loc))
field, survey, program = apload.apfield(plate, plug['locationid'])
out['survey'] = survey
out['field'] = field
with open(
spectro_dir + 'fields/' + telescope + '/' + str(loc) +
'/plan-' + str(loc) + '.lis', 'w+') as file:
file.write(telescope + '/' + str(plate) + '/' + str(mjd) +
'/' + os.path.basename(planfile))
file.close()
out['plugmap'] = plugid
# Calibration frames to use
calnames = [
'det', 'bpm', 'littrow', 'persist', 'persistmodel', 'dark', 'flat',
'sparse', 'fiber', 'badfiber', 'fixfiber', 'response', 'wave', 'lsf'
]
for c in calnames:
val = caldata[c]
if str(val).isdigit(): val = int(val)
out[c + 'id'] = val
# We use multiwaveid for waveid
waveid = caldata['multiwave']
if str(waveid).isdigit(): waveid = int(waveid)
out['waveid'] = waveid
# Input PSFID and FLUXID
out['psfid'] = psfid
out['fluxid'] = fluxid
# object frames
aplist = []
for i in range(len(ims)):
aplist1 = {}
if ims[i] > 0:
if stars is not None:
star = stars[i]
name = names[i]
else:
star = -1
name = 'none'
aplist1 = {
'plateid': plate,
'mjd': mjd,
'flavor': 'object',
'name': ims[i],
'single': star,
'singlename': name
}
aplist.append(aplist1)
out['APEXP'] = aplist
# Write to yaml file
with open(planfile, 'w') as ofile:
dum = yaml.dump(out, ofile, default_flow_style=False, sort_keys=False)
os.chmod(planfile, 0o664)
return planfile
def create_sumfiles(mjd5,apred,telescope,logger=None):
""" Create allVisit/allStar files and summary of objects for this night."""
if logger is None:
logger = dln.basiclogger()
load = apload.ApLoad(apred=apred,telescope=telescope)
# Start db session
db = apogeedb.DBSession()
# USE STAR_LATEST AND VISIT_LATEST "VIEWS" IN THE FUTURE!
# Full allVisit and allStar files
# apogee_id+apred_vers+telescope+starver uniquely identifies a particular star row
# For each apogee_id+apred_vers+telescope we want the maximum starver
# The subquery does this for us by grouping by apogee_id+apred_vers+telescope and
# calculating the aggregate value MAX(starver).
# We then select the particular row (with all columns) using apogee_id+apred_vers+telescope+starver
# from this subquery.
allstar = db.query(sql="select * from apogee_drp.star where (apogee_id, apred_vers, telescope, starver) in "+\
"(select apogee_id, apred_vers, telescope, max(starver) from apogee_drp.star where "+\
"apred_vers='"+apred+"' and telescope='"+telescope+"' group by apogee_id, apred_vers, telescope)")
allstarfile = load.filename('allStar').replace('.fits','-'+telescope+'.fits')
logger.info('Writing allStar file to '+allstarfile)
if os.path.exists(os.path.dirname(allstarfile))==False:
os.makedirs(os.path.dirname(allstarfile))
Table(allstar).write(allstarfile,overwrite=True)
# allVisit
# Same thing for visit except that we'll get the multiple visit rows returned for each unique star row
# Get more info by joining with the visit table.
vcols = ['apogee_id', 'target_id', 'apred_vers','file', 'uri', 'fiberid', 'plate', 'mjd', 'telescope', 'survey',
'field', 'programname', 'alt_id', 'location_id', 'ra', 'dec', 'glon', 'glat', 'j', 'j_err', 'h',
'h_err', 'k', 'k_err', 'src_h', 'wash_m', 'wash_m_err', 'wash_t2', 'wash_t2_err', 'ddo51',
'ddo51_err', 'irac_3_6', 'irac_3_6_err', 'irac_4_5', 'irac_4_5_err', 'irac_5_8', 'irac_5_8_err',
'irac_8_0', 'irac_8_0_err', 'wise_4_5', 'wise_4_5_err', 'targ_4_5', 'targ_4_5_err',
'wash_ddo51_giant_flag', 'wash_ddo51_star_flag', 'pmra', 'pmdec', 'pm_src', 'ak_targ',
'ak_targ_method', 'ak_wise', 'sfd_ebv', 'apogee_target1', 'apogee_target2', 'apogee_target3',
'apogee_target4', 'targflags', 'snr', 'starflag', 'starflags','dateobs','jd']
rvcols = ['starver', 'bc', 'vtype', 'vrel', 'vrelerr', 'vheliobary', 'chisq', 'rv_teff', 'rv_feh',
'rv_logg', 'xcorr_vrel', 'xcorr_vrelerr', 'xcorr_vheliobary', 'n_components', 'rv_components']
cols = ','.join('v.'+np.char.array(vcols)) +','+ ','.join('rv.'+np.char.array(rvcols))
allvisit = db.query(sql="select "+cols+" from apogee_drp.rv_visit as rv join apogee_drp.visit as v on rv.visit_pk=v.pk "+\
"where (rv.apogee_id, rv.apred_vers, rv.telescope, rv.starver) in "+\
"(select apogee_id, apred_vers, telescope, max(starver) from apogee_drp.rv_visit where "+\
"rv.apred_vers='"+apred+"' and rv.telescope='"+telescope+"' group by apogee_id, apred_vers, telescope)")
allvisitfile = load.filename('allVisit').replace('.fits','-'+telescope+'.fits')
logger.info('Writing allVisit file to '+allvisitfile)
if os.path.exists(os.path.dirname(allvisitfile))==False:
os.makedirs(os.path.dirname(allvisitfile))
Table(allvisit).write(allvisitfile,overwrite=True)
# Nightly allVisit and allStar, allVisitMJD/allStarMJD
gdstar, = np.where(allstar['starver']==str(mjd5))
allstarmjd = allstar[gdstar]
gdvisit, = np.where(allvisit['mjd']==int(mjd5))
allvisitmjd = allvisit[gdvisit]
# maybe in summary/MJD/ or qa/MJD/ ?
#allstarmjdfile = load.filename('allStarMJD')
allstarmjdfile = allstarfile.replace('allStar','allStarMJD').replace('.fits','-'+str(mjd5)+'.fits')
logger.info('Writing Nightly allStarMJD file to '+allstarmjdfile)
if os.path.exists(os.path.dirname(allstarmjdfile))==False:
os.makedirs(os.path.dirname(allstarmjdfile))
Table(allstarmjd).write(allstarmjdfile,overwrite=True)
#allvisitmjdfile = load.filename('allVisitMJD')
allvisitmjdfile = allvisitfile.replace('allVisit','allVisitMJD').replace('.fits','-'+str(mjd5)+'.fits')
logger.info('Writing Nightly allVisitMJD file to '+allvisitmjdfile)
if os.path.exists(os.path.dirname(allvisitmjdfile))==False:
os.makedirs(os.path.dirname(allvisitmjdfile))
Table(allvisitmjd).write(allvisitmjdfile,overwrite=True)
db.close()
def check_rv(visits,pbskey,verbose=False,logger=None):
""" Check that rv ran okay and load into database."""
if logger is None:
logger = dln.basiclogger()
db = apogeedb.DBSession() # open db session
if verbose==True:
logger.info('')
logger.info('-------------------------------------')
logger.info('Checking RV + Visit Combination runs')
logger.info('=====================================')
apred_vers = visits['apred_vers'][0]
telescope = visits['telescope'][0]
load = apload.ApLoad(apred=apred_vers,telescope=telescope)
if verbose:
logger.info('')
logger.info('%d stars' % len(visits))
logger.info('apred_vers: %s' % apred_vers)
logger.info('telescope: %s' % telescope)
logger.info(' NUM APOGEE_ID HEALPIX NVISITS SUCCESS')
# Loop over the stars
nstars = len(visits)
dtype = np.dtype([('apogee_id',(np.str,50)),('apred_vers',(np.str,20)),('telescope',(np.str,10)),
('healpix',int),('nvisits',int),('pbskey',(np.str,50)),
('file',(np.str,300)),('checktime',(np.str,100)),('success',bool)])
chkrv = np.zeros(nstars,dtype=dtype)
chkrv['apred_vers'] = apred_vers
chkrv['telescope'] = telescope
chkrv['pbskey'] = pbskey
for i,visit in enumerate(visits):
starfilenover = load.filename('Star',obj=visit['apogee_id'])
# add version number, should be MJD of of the latest visit
stardir = os.path.dirname(starfilenover)
starbase = os.path.splitext(os.path.basename(starfilenover))[0]
starbase += '-'+str(visit['mjd']) # add star version
starfile = stardir+'/'+starbase+'.fits'
# Get nvisits for this star
starvisits = db.query('visit',cols='*',where="apogee_id='"+visit['apogee_id']+"' and "+\
"telescope='"+telescope+"' and apred_vers='"+apred_vers+"'")
chkrv['apogee_id'][i] = visit['apogee_id']
chkrv['healpix'][i] = apload.obj2healpix(visit['apogee_id'])
chkrv['nvisits'][i] = len(starvisits)
chkrv['file'][i] = starfile
chkrv['checktime'][i] = str(datetime.now())
chkrv['success'][i] = os.path.exists(starfile)
if verbose:
logger.info('%5d %20s %8d %5d %9s' % (i+1,chkrv['apogee_id'][i],chkrv['healpix'][i],
chkrv['nvisits'][i],chkrv['success'][i]))
success, = np.where(chkrv['success']==True)
logger.info('%d/%d succeeded' % (len(success),nstars))
# Inset into the database
db.ingest('rv_status',chkrv)
db.close()
return chkrv
def check_apred(expinfo,planfiles,pbskey,verbose=False,logger=None):
""" Check that apred ran okay and load into database."""
if logger is None:
logger = dln.basiclogger()
if verbose==True:
logger.info('')
logger.info('--------------------')
logger.info('Checking APRED runs')
logger.info('====================')
# Loop over the planfiles
nplanfiles = len(planfiles)
for ip,pfile in enumerate(planfiles):
planstr = plan.load(pfile,np=True)
apred_vers = planstr['apred_vers']
telescope = planstr['telescope']
instrument = planstr['instrument']
platetype = planstr['platetype']
mjd = planstr['mjd']
plate = planstr['plateid']
expstr = planstr['APEXP']
nexp = len(expstr)
load = apload.ApLoad(apred=apred_vers,telescope=telescope)
# Science exposures
if platetype=='normal':
# Load the plugmap information
plugmap = platedata.getdata(plate,mjd,apred_vers,telescope,plugid=planstr['plugmap'])
fiberdata = plugmap['fiberdata']
# Exposure-level processing: ap3d, ap2d, apcframe
dtype = np.dtype([('exposure_pk',int),('planfile',(np.str,300)),('apred_vers',(np.str,20)),('instrument',(np.str,20)),
('telescope',(np.str,10)),('platetype',(np.str,50)),('mjd',int),('plate',int),
('proctype',(np.str,30)),('pbskey',(np.str,50)),('checktime',(np.str,100)),
('num',int),('success',bool)])
chkexp = np.zeros(nexp*3,dtype=dtype)
chkexp['planfile'] = pfile
chkexp['apred_vers'] = apred_vers
chkexp['instrument'] = instrument
chkexp['telescope'] = telescope
chkexp['platetype'] = platetype
chkexp['mjd'] = mjd
chkexp['plate'] = plate
chkexp['proctype'] = 'AP3D'
chkexp['pbskey'] = pbskey
chkexp['success'] = False
cnt = 0
for num in expstr['name']:
ind, = np.where(expinfo['num']==num)
exposure_pk = expinfo['pk'][ind[0]]
# AP3D
#-----
chkexp['exposure_pk'][cnt] = exposure_pk
chkexp['num'][cnt] = num
chkexp['proctype'][cnt] = 'AP3D'
base = load.filename('2D',num=num,mjd=mjd,chips=True)
chfiles = [base.replace('2D-','2D-'+ch+'-') for ch in ['a','b','c']]
exists = [os.path.exists(chf) for chf in chfiles]
chkexp['checktime'][cnt] = str(datetime.now())
if np.sum(exists)==3:
chkexp['success'][cnt] = True
cnt += 1
# AP2D
#-----
chkexp['exposure_pk'][cnt] = exposure_pk
chkexp['num'][cnt] = num
chkexp['proctype'][cnt] = 'AP2D'
base = load.filename('1D',num=num,mjd=mjd,chips=True)
chfiles = [base.replace('1D-','1D-'+ch+'-') for ch in ['a','b','c']]
exists = [os.path.exists(chf) for chf in chfiles]
chkexp['checktime'][cnt] = str(datetime.now())
if np.sum(exists)==3:
chkexp['success'][cnt] = True
cnt += 1
# APCframe
#---------
chkexp['exposure_pk'][cnt] = exposure_pk
chkexp['num'][cnt] = num
chkexp['proctype'][cnt] = 'APCFRAME'
base = load.filename('Cframe',num=num,mjd=mjd,plate=plate,chips=True)
chfiles = [base.replace('Cframe-','Cframe-'+ch+'-') for ch in ['a','b','c']]
exists = [os.path.exists(chf) for chf in chfiles]
chkexp['checktime'][cnt] = str(datetime.now())
if np.sum(exists)==3:
chkexp['success'][cnt] = True
cnt += 1
# AP1DVISIT
# ---------
# -apPlate
# -apVisit files
# -apVisitSum file
dtypeap = np.dtype([('planfile',(np.str,300)),('logfile',(np.str,300)),('errfile',(np.str,300)),
('apred_vers',(np.str,20)),('instrument',(np.str,20)),
('telescope',(np.str,10)),('platetype',(np.str,50)),('mjd',int),('plate',int),
('nobj',int),('pbskey',(np.str,50)),('checktime',(np.str,100)),('ap3d_success',bool),
('ap3d_nexp_success',int),('ap2d_success',bool),('ap2d_nexp_success',int),
('apcframe_success',bool),('apcframe_nexp_success',int),('applate_success',bool),
('apvisit_success',bool),('apvisit_nobj_success',int),
('apvisitsum_success',bool),('success',bool)])
chkap = np.zeros(1,dtype=dtypeap)
chkap['planfile'] = pfile
chkap['logfile'] = pfile.replace('.yaml','_pbs.log')
chkap['errfile'] = pfile.replace('.yaml','_pbs.err')
chkap['apred_vers'] = apred_vers
chkap['instrument'] = instrument
chkap['telescope'] = telescope
chkap['platetype'] = platetype
chkap['mjd'] = mjd
chkap['plate'] = plate
chkap['nobj'] = np.sum(fiberdata['objtype']!='SKY') # stars and tellurics
chkap['pbskey'] = pbskey
chkap['checktime'] = str(datetime.now())
# ap3D, ap2D, apCframe success
ind3d, = np.where((chkexp['num']==num) & (chkexp['proctype']=='AP3D'))
chkap['ap3d_nexp_success'] = np.sum(chkexp['success'][ind3d])
chkap['ap3d_success'] = np.sum(chkexp['success'][ind3d])==nexp
ind2d, = np.where((chkexp['num']==num) & (chkexp['proctype']=='AP2D'))
chkap['ap2d_nexp_success'] = np.sum(chkexp['success'][ind2d])
chkap['ap2d_success'] = np.sum(chkexp['success'][ind2d])==nexp
indcf, = np.where((chkexp['num']==num) & (chkexp['proctype']=='APCFRAME'))
chkap['apcframe_nexp_success'] = np.sum(chkexp['success'][indcf])
chkap['apcframe_success'] = np.sum(chkexp['success'][indcf])==nexp
# apPlate
chkap['applate_success'] = False
base = load.filename('Plate',plate=plate,mjd=mjd,chips=True)
chfiles = [base.replace('Plate-','Plate-'+ch+'-') for ch in ['a','b','c']]
exists = [os.path.exists(chf) for chf in chfiles]
if np.sum(exists)==3:
chkap['applate_success'] = True
# apVisit
base = load.filename('Visit',plate=plate,mjd=mjd,fiber=1)
visitfiles = glob(base.replace('-001.fits','-???.fits'))
nvisitfiles = len(visitfiles)
chkap['apvisit_nobj_success'] = nvisitfiles
chkap['apvisit_success'] = nvisitfiles==chkap['nobj']
apvisitsumfile = load.filename('VisitSum',plate=plate,mjd=mjd)
chkap['apvisitsum_success'] = os.path.exists(apvisitsumfile)
chkap['success'] = chkap['ap3d_success'][0] and chkap['ap2d_success'][0] and chkap['apcframe_success'][0] and \
chkap['applate_success'][0] and chkap['apvisit_success'][0] and chkap['apvisitsum_success'][0]
if verbose:
logger.info('')
logger.info('%d/%d' % (ip+1,nplanfiles))
logger.info('planfile: '+pfile)
logger.info('log/errfile: '+os.path.basename(chkap['logfile'][0])+', '+os.path.basename(chkap['errfile'][0]))
logger.info('platetype: %s' % platetype)
logger.info('mjd: %d' % mjd)
logger.info('plate: %d' % plate)
logger.info('nexp: %d' % nexp)
logger.info('Nobj: %d' % chkap['nobj'][0])
logger.info('3D/2D/Cframe:')
logger.info('Num EXPID NREAD 3D 2D Cframe')
for k,num in enumerate(expstr['name']):
ind, = np.where(expinfo['num']==num)
ind3d, = np.where((chkexp['num']==num) & (chkexp['proctype']=='AP3D'))
ind2d, = np.where((chkexp['num']==num) & (chkexp['proctype']=='AP2D'))
indcf, = np.where((chkexp['num']==num) & (chkexp['proctype']=='APCFRAME'))
logger.info('%2d %10d %4d %6s %6s %6s' % (k+1,chkexp['num'][ind3d],expinfo['nread'][ind],
chkexp['success'][ind3d[0]],chkexp['success'][ind2d[0]],
chkexp['success'][indcf[0]]))
logger.info('apPlate files: %s ' % chkap['applate_success'][0])
logger.info('N apVisit files: %d ' % chkap['apvisit_nobj_success'][0])
logger.info('apVisitSum file: %s ' % chkap['apvisitsum_success'][0])
logger.info('apVisit success: %s ' % chkap['success'][0])
# Load into the database
db = apogeedb.DBSession()
db.ingest('exposure_status',chkexp)
db.ingest('visit_status',chkap)
db.close()
#import pdb; pdb.set_trace()
# Calibration exposures
else:
logger.info('calibration exposures')
# STILL NEED TO DO THIS PART!!!
#import pdb; pdb.set_trace()
return chkexp,chkap
def getdata(refcat, minra, redo=False, silent=False, logger=None):
"""
Get reference catalog information from DL database
Parameters
----------
cenra : float
Central RA for the search.
cendec : float
Central DEC for the search.
radius : float
Search radius in degrees.
refcat : table
Reference catalog name (e.g. 2MASS, Gaia, etc.)
version : str
Version of NSC reduction..
saveref : bool, optional
Save the output to SAVEFILE or a default filename. Default is False.
savefile : str, optional
The file to save to or search for existing catalog.
silent : bool, optional
Don't print anything to the screen.
logger : logging object, optional
Logging object to use for printing messages.
Returns
-------
ref : astropy table
Search results from the reference catalog.
Example
-------
cat = getrefcat(cenra,cendec,radius,refcat,file=file,saveref=saveref)
By D. Nidever Sep 2017
Translated to Python by D. Nidever, April 2022
"""
count = 0
t0 = time.time()
if logger is None:
logger = dln.basiclogger()
# Check that we have psql installed
out = subprocess.check_output(['which', 'psql'], shell=False)
if type(out) is bytes:
out = out.decode()
out = out.strip()
if dln.size(out) > 1:
out = out[0]
if os.path.exists(out) == 0:
raise ValueError('No PSQL found on this sytem.')
# Temporary directory
# /tmp is often small and get get fille dup
dldir, mssdir, localdir = utils.rootdirs()
# FLIP THIS AROUND, INPUT SHOULD BE THE "EASY" VERSION!!!
refname = str(refcat).upper()
if refname == 'II/312/AIS':
refname = 'GALEX'
elif refname == '2MASS-PSC':
refname = 'TMASS'
elif refname == '2MASS':
refname = 'TMASS'
elif refname == 'GAIA/GAIA':
refname = 'GAIA'
elif refname == 'Skymapper':
refname = 'SKYMAPPER'
elif refname == 'GLIMPSE':
catname = 'II/293/glimpse'
elif refname == 'SAGE':
catname = 'II/305/archive'
elif refname == 'ATLASREFCAT2':
refname = 'ATLAS'
ra0 = float(minra)
ra1 = minra + 1.0
outdir = '/net/dl1/users/dnidever/nsc/refcatalogs/' + refname + '/'
if os.path.exists(outdir) == False:
os.makedirs(outdir)
savefile = outdir + 'ref_%.6f_%.6f_%s.fits' % (ra0, ra1, refname)
if os.path.exists(os.path.abspath(os.path.dirname(savefile))) == False:
os.makedirs(os.path.abspath(os.path.dirname(savefile)))
if silent == False:
logger.info('Querying %s: %.6f <= RA < %.6f' % (refname, ra0, ra1))
# Loading previously loaded file
if os.path.exists(savefile) and redo == False:
logger.info(savefile + ' already exists and redo==False')
return None
# Do the Query
#--------------
else:
# Use DataLab database search
#----------------------------
if refname in [
'TMASS', 'GAIA', 'GAIADR2', 'GAIAEDR3', 'PS', 'SKYMAPPER',
'SKYMAPPERDR2', 'ALLWISE', 'ATLAS'
]:
if refname == 'TMASS':
tablename = 'twomass.psc'
cols = 'designation,ra as raj2000,dec as dej2000,j_m as jmag,j_cmsig as e_jmag,h_m as hmag,h_cmsig as e_hmag,k_m as kmag,k_cmsig as e_kmag,ph_qual as qflg'
##server = 'gp04.datalab.noao.edu'
#server = 'gp01.datalab.noirlab.edu'
##server = 'dldb1.sdm.noao.edu'
server = 'db02.datalab.noirlab.edu'
user = '******'
racol = 'ra'
deccol = 'dec'
if refname == 'GAIA':
tablename = 'gaia_dr1.gaia_source'
cols = 'source_id as source,ra as ra_icrs,ra_error as e_ra_icrs,dec as de_icrs,dec_error as e_de_icrs,'
cols += 'phot_g_mean_flux as fg,phot_g_mean_flux_error as e_fg,phot_g_mean_mag as gmag'
#server = 'gp04.datalab.noirlab.edu'
##server = 'gp01.datalab.noao.edu'
##server = 'dldb1.sdm.noao.edu'
server = 'db02.datalab.noirlab.edu'
user = '******'
if refname == 'GAIADR2':
tablename = 'gaia_dr2.gaia_source'
cols = 'source_id as source,ra,ra_error,dec,dec_error,pmra,pmra_error,pmdec,pmdec_error,phot_g_mean_flux as fg,phot_g_mean_flux_error as e_fg,'
cols += 'phot_g_mean_mag as gmag,phot_bp_mean_mag as bp,phot_bp_mean_flux as fbp,phot_bp_mean_flux_error as e_fbp,'
cols += 'phot_rp_mean_mag as rp,phot_rp_mean_flux as frp,phot_rp_mean_flux_error as e_frp'
#server = 'gp04.datalab.noirlab.edu'
##server = 'gp01.datalab.noao.edu'
server = 'db02.datalab.noirlab.edu'
user = '******'
if refname == 'GAIAEDR3':
tablename = 'gaia_edr3.gaia_source'
cols = 'source_id as source,ra,ra_error,dec,dec_error,pmra,pmra_error,pmdec,pmdec_error,phot_g_mean_flux as fg,phot_g_mean_flux_error as e_fg,'
cols += 'phot_g_mean_mag as gmag,phot_bp_mean_mag as bp,phot_bp_mean_flux as fbp,phot_bp_mean_flux_error as e_fbp,'
cols += 'phot_rp_mean_mag as rp,phot_rp_mean_flux as frp,phot_rp_mean_flux_error as e_frp'
#server = 'gp04.datalab.noirlab.edu'
##server = 'gp01.datalab.noao.edu'
server = 'db02.datalab.noirlab.edu'
user = '******'
if refname == 'PS':
#tablename = 'cp_calib.ps1'
tablename = 'public.ps1'
cols = 'ra, dec, g as gmag, r as rmag, i as imag, z as zmag, y as ymag'
##server = 'gp02.datalab.noirlab.edu'
#server = 'gp01.datalab.noirlab.edu'
server = 'db02.datalab.noirlab.edu'
user = '******'
if refname == 'SKYMAPPER':
tablename = 'skymapper_dr1.master'
cols = 'raj2000 as ra, dej2000 as dec, u_psf as sm_umag, e_u_psf as e_sm_umag, g_psf as sm_gmag, e_g_psf as e_sm_gmag, r_psf as sm_rmag,'
cols += 'e_r_psf as e_sm_rmag, i_psf as sm_imag,e_i_psf as e_sm_imag, z_psf as sm_zmag, e_z_psf as e_sm_zmag'
#server = 'gp04.datalab.noirlab.edu'
##server = 'gp01.datalab.noao.edu'
server = 'db02.datalab.noirlab.edu'
user = '******'
racol = 'raj2000'
deccol = 'dej2000'
if refname == 'SKYMAPPERDR2':
tablename = 'skymapper_dr2.master'
cols = 'raj2000 as ra, dej2000 as dec, u_psf as sm_umag, e_u_psf as e_sm_umag, g_psf as sm_gmag, e_g_psf as e_sm_gmag, r_psf as sm_rmag,'
cols += 'e_r_psf as e_sm_rmag, i_psf as sm_imag,e_i_psf as e_sm_imag, z_psf as sm_zmag, e_z_psf as e_sm_zmag'
#server = 'gp04.datalab.noirlab.edu'
##server = 'gp01.datalab.noao.edu'
server = 'db02.datalab.noirlab.edu'
user = '******'
racol = 'raj2000'
deccol = 'dej2000'
if refname == 'ALLWISE':
tablename = 'allwise.source'
#cols = 'ra, dec, w1mdef as w1mag, w1sigmdef as e_w1mag, w2mdef as w2mag, w2sigmdef as e_w2mag'
cols = 'ra, dec, w1mpro as w1mag, w1sigmpro as e_w1mag, w2mpro as w2mag, w2sigmpro as e_w2mag'
#server = 'gp04.datalab.noao.edu'
#server = 'gp01.datalab.noirlab.edu'
server = 'db02.datalab.noirlab.edu'
user = '******'
if refname == 'ATLAS':
tablename = 'atlasrefcat2'
cols = 'objid,ra,dec,plx as parallax,dplx as parallax_error,pmra,dpmra as pmra_error,pmdec,dpmdec as pmdec_error,gaia,dgaia as gaiaerr,'
cols += 'bp,dbp as bperr,rp,drp as rperr,teff,agaia,dupvar,ag,rp1,r1,r10,g as gmag,dg as gerr,gchi,gcontrib,'
cols += 'r as rmag, dr as rerr,rchi,rcontrib,i as imag,di as ierr,ichi,icontrib,z as zmag,dz as zerr,zchi,zcontrib,nstat,'
cols += 'j as jmag,dj as jerr,h as hmag,dh as herr,k as kmag,dk as kerr'
server = 'gp10.datalab.noirlab.edu'
user = '******'
# Use Postgres command with q3c cone search
refcattemp = savefile.replace('.fits', '.txt')
cmd = "psql -h " + server + " -U " + user + " -d tapdb -w --pset footer -c 'SELECT " + cols + " FROM " + tablename
if refname == 'SKYMAPPER' or refname == 'SKYMAPPERDR2':
cmd += " WHERE raj2000 >= %.6f and raj2000 < %.6f'" % (ra0,
ra1)
else:
cmd += " WHERE ra >= %.6f and ra < %.6f'" % (ra0, ra1)
cmd += " > " + refcattemp
dln.remove(refcattemp, allow=True)
dln.remove(savefile, allow=True)
out = subprocess.check_output(cmd, shell=True)
# Check for empty query
tlines = dln.readlines(refcattemp, nreadline=4)
if len(tlines) < 4:
if silent == False:
logger.info('No Results')
return []
# Load ASCII file and create the FITS file
ref = ascii.read(refcattemp, data_start=3, delimiter='|')
#ref = importascii(refcattemp,/header,delim='|',skipline=2,/silent)
dln.remove(refcattemp, allow=True)
# Fix 0.0 mags/errs in ATLAS
if refname == 'ATLAS':
magcols = [
'gaia', 'bp', 'rp', 'gmag', 'rmag', 'imag', 'zmag', 'jmag',
'hmag', 'kmag'
]
errcols = [
'gaiaerr', 'bperr', 'rperr', 'gerr', 'rerr', 'ierr',
'zerr', 'jerr', 'herr', 'kerr'
]
cols = ref.colnames
# Set mags with 0.0 to 99.99
for j in range(len(magcols)):
if magcols[j] in ref.colnames:
bdmag = (ref[magcols[j]] <= 0.0)
if np.sum(bdmag) > 0:
ref[magcols[j]][bdmag] = 99.99
# Set errors with 0.0 to 9.99
for j in range(len(errcols)):
if errcols[j] in ref.colnames:
bderr = (ref[errcols[j]] <= 0.0)
if np.sum(bderr) > 0:
ref[errcols[j]][bderr] = 9.99
# Use astroquery vizier
#----------------
# for low density with 2MASS/GAIA and always for GALEX and APASS
else:
# Use QUERYVIZIER for GALEX (python code has problems)
#if refname == 'II/312/ais' or refname == 'GALEX':
# if refcat eq 'APASS' then cfa=0 else cfa=1 ; cfa doesn't have APASS
#cfa = 1 # problems with CDS VizieR and cfa has APASS now
#if refcat == 'SAGE':
# cfa = 0
if refname.upper() == 'GALEX':
cols = [
'RAJ2000', 'DEJ2000', 'FUVmag', 'e_FUVmag', 'NUVmag',
'e_NUVmag'
]
catname = 'II/335/galex_ais'
elif refname.upper() == 'GLIMPSE':
# Only includes GLIMPSE I,II,3D
cols = [
'RAJ2000', 'DEJ2000', '_2MASS', '_3.6mag', 'e_3.6mag',
'_4.5mag', 'e_4.5mag'
]
catname = 'II/293/glimpse'
elif refname.upper() == 'SAGE':
cols = [
'RAJ2000', 'DEJ2000', '__3.6_', 'e__3.6_', '__4.5_',
'e__4.5_'
]
catname = 'II/305/catalog'
Vizier.ROW_LIMIT = -1
Vizier.TIMEOUT = 1000000
Vizier.columns = cols
result = Vizier.query_constraints(catalog=catname,
RA='>=' + str(ra0) + ' & <' +
str(ra1))
# Check for failure
if len(result) == 0:
if silent == False:
logger.info('Failure or No Results')
return []
ref = result[0]
ref.meta['description'] = ref.meta['description'][0:50]
#ref = QUERYVIZIER(refname,[cenra,cendec],radius*60,cfa=cfa,timeout=600,/silent)
# Fix/homogenize the GAIA tags
if refname == 'GAIA':
nref = len(ref)
orig = ref.copy()
dt = [('source', int), ('ra_icrs', float),
('e_ra_icrs', float), ('de_icrs', float),
('e_de_icrs', float), ('fg', float), ('e_fg', float),
('gmag', float)]
ref = np.zeros(nref, dtype=np.dtype(dt))
ref = Table(ref)
for n in orig.colnames:
ref[n] = orig[n]
ref['fg'] = orig['_fg_']
ref['e_fg'] = orig['e__fg_']
ref['gmag'] = orig['_gmag_']
del orig
# Fix/homogenize the 2MASS tags
elif refname == 'TMASS':
nref = len(ref)
orig = ref.copy()
dt = [('designation', (np.str, 50)), ('raj2000', float),
('dej2000', float), ('jmag', float), ('e_jmag', float),
('hmag', float), ('e_hmag', float), ('kmag', float),
('e_kmag', float), ('qflg', (np.str, 20))]
ref = np.zeros(nref, dtype=np.dtype(dt))
for n in orig.colnames:
ref[n] = orig[n]
ref['designation'] = orig['_2mass']
del orig
# Fix NANs in ALLWISE
elif refname == 'ALLWISE':
bd, = np.where(np.isfinite(ref['_3_6_']) == False)
if len(bd) > 0:
ref['_3_6_'][bd] = 99.99
ref['e__3_6_'][bd] = 9.99
bd, = np.where(np.isfinite(ref['_4_5_']) == False)
if len(bd) > 0:
ref['_4_5_'][bd] = 99.99
ref['e__4_5_'][bd] = 9.99
# Convert all mags and errmags to float32
for n in ref.colnames:
if n.find('mag') > -1:
ref[n] = ref[n].astype(np.float32)
if n.find('e_') > -1 and n.find('mag') > -1:
ref[n] = ref[n].astype(np.float32)
# Lowercase column names
for n in ref.colnames:
ref[n].name = n.lower()
# Convert raj2000/dej2000 to ra/dec
if 'raj2000' in ref.colnames:
ref['raj2000'].name = 'ra'
if 'dej2000' in ref.colnames:
ref['dej2000'].name = 'dec'
# Save the file
logger.info('Saving catalog to file ' + savefile)
ref.write(savefile, overwrite=True)
if silent == False:
logger.info('%d sources found dt=%.1f sec.' %
(len(ref), time.time() - t0))
return ref
def run(image,psfname='gaussian',iterdet=0,psfsubnei=False,psffitradius=None,fitradius=None,npsfpix=51,
binned=False,lookup=False,lorder=0,psftrim=None,recenter=True,reject=False,apcorr=False,
timestamp=False,verbose=False):
"""
Run PSF photometry on an image.
Parameters
----------
image : string or CCDData object
The input image to fit. This can be the filename or CCDData object.
psfname : string, optional
The name of the PSF type to use. The options are "gaussian", "moffat",
"penny" and "gausspow". Default is "gaussian".
iterdet : boolean, optional
Number of iterations to use for detection. Default is iterdet=0, meaning
detection is only performed once.
psfsubnei : boolean, optional
Subtract neighboring stars to PSF stars when generating the PSF. Default is False.
psffitradius : float, optional
The fitting readius when constructing the PSF (in pixels). By default
the FWHM is used.
fitradius: float, optional
The fitting radius when fitting the PSF to the stars in the image (in pixels).
By default the PSF FWHM is used.
npsfpix : int, optional
The size of the PSF footprint. Default is 51.
binned : boolean, optional
Use a binned model that integrates the analytical function across a pixel.
Default is false.
lookup : boolean, optional
Use an empirical lookup table. Default is False.
lorder : int, optional
The order of the spatial variations (0=constant, 1=linear). Default is 0.
psftrim: float, optional
Trim the PSF size to a radius where "psftrim" fraction of flux is removed. Default is None.
recenter : boolean, optional
Allow the centroids to be fit. Default is True.
reject : boolean, optional
When constructin the PSF, reject PSF stars with high RMS values. Default is False.
apcorr : boolean, optional
Apply aperture correction. Default is False.
timestamp : boolean, optional
Add timestamp in verbose output (if verbose=True). Default is False.
verbose : boolean, optional
Verbose output to the screen. Default is False.
Returns
-------
cat : table
The output table of best-fit PSF values for all of the sources.
model : CCDData object
The best-fitting model for the stars (without sky).
sky : CCDData object
The background sky image used for the image.
psf : PSF object
The best-fitting PSF model.
Example
-------
cat,model,sky,psf = prometheus.run(image,psfname='gaussian',verbose=True)
"""
# Set up the logger
if timestamp and verbose:
logger = dln.basiclogger()
logger.handlers[0].setFormatter(logging.Formatter("%(asctime)s [%(levelname)-5.5s] %(message)s"))
logger.handlers[0].setStream(sys.stdout)
builtins.logger = logger # make it available globally across all modules
start = time.time()
print = utils.getprintfunc() # Get print function to be used locally, allows for easy logging
# Load the file
if isinstance(image,str):
filename = image
if verbose:
print('Loading image from "'+filename+'"')
image = CCDData.read(filename)
if isinstance(image,CCDData) is False:
raise ValueError('Input image must be a filename or CCDData object')
residim = image.copy()
# Processing steps
#-----------------
for niter in range(iterdet+1):
if verbose and iterdet>0:
print('--- Iteration = '+str(niter+1)+' ---')
# 1) Detection
#-------------
if verbose:
print('Step 1: Detection')
objects = detection.detect(residim)
objects['ndetiter'] = niter+1
if verbose:
print(str(len(objects))+' objects detected')
# 2) Aperture photometry
#-----------------------
if verbose:
print('Step 2: Aperture photometry')
objects = aperture.aperphot(residim,objects)
nobjects = len(objects)
# Bright and faint limit, use 5th and 95th percentile
if niter==0:
minmag, maxmag = np.sort(objects['mag_auto'])[[int(np.round(0.05*nobjects)),int(np.round(0.95*nobjects))]]
if verbose:
print('Min/Max mag: %5.2f, %5.2f' % (minmag,maxmag))
# 3) Construct the PSF
#---------------------
# only on first iteration
if niter==0:
if verbose:
print('Step 3: Construct the PSF')
# 3a) Estimate FWHM
#------------------
fwhm = utils.estimatefwhm(objects,verbose=verbose)
# 3b) Pick PSF stars
#------------------
psfobj = utils.pickpsfstars(objects,fwhm,verbose=verbose)
# 3c) Construct the PSF iteratively
#---------------------------------
# Make the initial PSF slightly elliptical so it's easier to fit the orientation
if psfname.lower() != 'empirical':
initpsf = models.psfmodel(psfname,[fwhm/2.35,0.9*fwhm/2.35,0.0],binned=binned,npix=npsfpix)
else:
initpsf = models.psfmodel(psfname,npix=npsfpix,imshape=image.shape,order=lorder)
# run getpsf
psf,psfpars,psfperror,psfcat = getpsf.getpsf(initpsf,image,psfobj,fitradius=psffitradius,
lookup=lookup,lorder=lorder,subnei=psfsubnei,
allcat=objects,reject=reject,verbose=(verbose>=2))
# Trim the PSF
if psftrim is not None:
oldnpix = psf.npix
psf.trim(psftrim)
if verbose:
print('Trimming PSF size from '+str(oldnpix)+' to '+str(self.npix))
if verbose:
print('Final PSF: '+str(psf))
gd, = np.where(psfcat['reject']==0)
print('Median RMS: %.4f' % np.median(psfcat['rms'][gd]))
# 4) Run on all sources
#----------------------
# If niter>0, then use combined object catalog
if iterdet>0:
# Add detection
# Combine objects catalogs
if niter==0:
allobjects = objects.copy()
else:
objects['id'] = np.arange(len(objects))+1+np.max(allobjects['id'])
allobjects = vstack((allobjects,objects))
if 'group_id' in allobjects.keys():
allobjects.remove_column('group_id')
else:
allobjects = objects
if verbose:
print('Step 4: Get PSF photometry for all '+str(len(allobjects))+' objects')
psfout,model,sky = allfit.fit(psf,image,allobjects,fitradius=fitradius,
recenter=recenter,verbose=(verbose>=2))
# Construct residual image
if iterdet>0:
residim = image.copy()
residim.data -= model.data
# Combine aperture and PSF columns
outobj = allobjects.copy()
# rename some columns for clarity
outobj['x'].name = 'xc'
outobj['y'].name = 'yc'
outobj['a'].name = 'asemi'
outobj['b'].name = 'bsemi'
outobj['flux'].name = 'sumflux'
outobj.remove_columns(['cxx','cyy','cxy'])
# copy over PSF output columns
for n in psfout.columns:
outobj[n] = psfout[n]
outobj['psfamp'] = outobj['amp'].copy()
outobj['amp_error'].name = 'psfamp_error'
outobj['flux'].name = 'psfflux'
outobj['flux_error'].name = 'psfflux_error'
# change mag, magerr to psfmag, psfmag_error
outobj['mag'].name = 'psfmag'
outobj['mag_error'].name = 'psfmag_error'
# put ID at the beginning
cols = np.char.array(list(outobj.columns))
newcols = ['id']+list(cols[cols!='id'])
outobj = outobj[newcols]
# 5) Apply aperture correction
#-----------------------------
if apcorr:
if verbose:
print('Step 5: Applying aperture correction')
outobj,grow,cgrow = aperture.apercorr(psf,image,outobj,psfcat,verbose=verbose)
# Add exposure time correction
exptime = image.header.get('exptime')
if exptime is not None:
if verbose:
print('Applying correction for exposure time %.2f s' % exptime)
outobj['psfmag'] += 2.5*np.log10(exptime)
# Add coordinates if there's a WCS
if image.wcs is not None:
if image.wcs.has_celestial:
if verbose:
print('Adding RA/DEC coordinates to catalog')
skyc = image.wcs.pixel_to_world(outobj['x'],outobj['y'])
outobj['ra'] = skyc.ra
outobj['dec'] = skyc.dec
if verbose:
print('dt = %.2f sec' % (time.time()-start))
# Breakdown logger
if timestamp and verbose:
del builtins.logger
return outobj,model,sky,psf
def visitcomb(allvisit,
starver,
load=None,
apred='r13',
telescope='apo25m',
nres=[5, 4.25, 3.5],
bconly=False,
plot=False,
write=True,
dorvfit=True,
apstar_vers='stars',
logger=None):
""" Combine multiple visits with individual RVs to rest frame sum
"""
if logger is None:
logger = dln.basiclogger()
if load is None: load = apload.ApLoad(apred=apred, telescope=telescope)
cspeed = 2.99792458e5 # speed of light in km/s
logger.info('Doing visitcomb for {:s} '.format(allvisit['apogee_id'][0]))
wnew = norm.apStarWave()
nwave = len(wnew)
nvisit = len(allvisit)
# initialize array for stack of interpolated spectra
zeros = np.zeros([nvisit, nwave])
izeros = np.zeros([nvisit, nwave], dtype=int)
stack = apload.ApSpec(zeros,
err=zeros.copy(),
bitmask=izeros,
cont=zeros.copy(),
sky=zeros.copy(),
skyerr=zeros.copy(),
telluric=zeros.copy(),
telerr=zeros.copy())
apogee_target1, apogee_target2, apogee_target3 = 0, 0, 0
apogee2_target1, apogee2_target2, apogee2_target3, apogee2_target4 = 0, 0, 0, 0
starflag, andflag = np.uint64(0), np.uint64(0)
starmask = bitmask.StarBitMask()
# Loop over each visit and interpolate to final wavelength grid
if plot: fig, ax = plots.multi(1, 2, hspace=0.001)
for i, visit in enumerate(allvisit):
if bconly: vrel = -visit['bc']
else: vrel = visit['vrel']
# Skip if we don't have an RV
if np.isfinite(vrel) is False: continue
# Load the visit
if load.telescope == 'apo1m':
apvisit = load.apVisit1m(visit['plate'],
visit['mjd'],
visit['apogee_id'],
load=True)
else:
apvisit = load.apVisit(int(visit['plate']),
visit['mjd'],
visit['fiberid'],
load=True)
pixelmask = bitmask.PixelBitMask()
# Rest-frame wavelengths transformed to this visit spectra
w = norm.apStarWave() * (1.0 + vrel / cspeed)
# Loop over the chips
for chip in range(3):
# Get the pixel values to interpolate to
pix = wave.wave2pix(w, apvisit.wave[chip, :])
gd, = np.where(np.isfinite(pix))
# Get a smoothed, filtered spectrum to use as replacement for bad values
cont = gaussian_filter(
median_filter(apvisit.flux[chip, :], [501], mode='reflect'),
100)
errcont = gaussian_filter(
median_filter(apvisit.flux[chip, :], [501], mode='reflect'),
100)
bd, = np.where(apvisit.bitmask[chip, :] & pixelmask.badval())
if len(bd) > 0:
apvisit.flux[chip, bd] = cont[bd]
apvisit.err[chip, bd] = errcont[bd]
# Load up quantity/error pairs for interpolation
raw = [[apvisit.flux[chip, :], apvisit.err[chip, :]**2],
[apvisit.sky[chip, :], apvisit.skyerr[chip, :]**2],
[apvisit.telluric[chip, :], apvisit.telerr[chip, :]**2]]
# Load up individual mask bits
for ibit, name in enumerate(pixelmask.name):
if name is not '' and len(
np.where(apvisit.bitmask[chip, :] & 2**ibit)[0]) > 0:
raw.append([
np.clip(apvisit.bitmask[chip, :] & 2**ibit, None, 1),
None
])
# Do the sinc interpolation
out = sincint.sincint(pix[gd], nres[chip], raw)
# From output flux, get continuum to remove, so that all spectra are
# on same scale. We'll later multiply in the median continuum
flux = out[0][0]
stack.cont[i, gd] = gaussian_filter(
median_filter(flux, [501], mode='reflect'), 100)
# Load interpolated spectra into output stack
stack.flux[i, gd] = out[0][0] / stack.cont[i, gd]
stack.err[i, gd] = out[0][1] / stack.cont[i, gd]
stack.sky[i, gd] = out[1][0]
stack.skyerr[i, gd] = out[1][1]
stack.telluric[i, gd] = out[2][0]
stack.telerr[i, gd] = out[2][1]
# For mask, set bits where interpolated value is above some threshold
# defined for each mask bit
iout = 3
for ibit, name in enumerate(pixelmask.name):
if name is not '' and len(
np.where(apvisit.bitmask[chip, :] & 2**ibit)[0]) > 0:
j = np.where(
np.abs(out[iout][0]) > pixelmask.maskcontrib[ibit])[0]
stack.bitmask[i, gd[j]] |= 2**ibit
iout += 1
# Increase uncertainties for persistence pixels
bd, = np.where(
(stack.bitmask[i, :] & pixelmask.getval('PERSIST_HIGH')) > 0)
if len(bd) > 0: stack.err[i, bd] *= np.sqrt(5)
bd, = np.where((
(stack.bitmask[i, :] & pixelmask.getval('PERSIST_HIGH')) == 0) & (
(stack.bitmask[i, :] & pixelmask.getval('PERSIST_MED')) > 0))
if len(bd) > 0: stack.err[i, bd] *= np.sqrt(4)
bd, = np.where(
((stack.bitmask[i, :] & pixelmask.getval('PERSIST_HIGH')) == 0)
& ((stack.bitmask[i, :] & pixelmask.getval('PERSIST_MED')) == 0)
& ((stack.bitmask[i, :] & pixelmask.getval('PERSIST_LOW')) > 0))
if len(bd) > 0: stack.err[i, bd] *= np.sqrt(3)
bd, = np.where(
(stack.bitmask[i, :] & pixelmask.getval('SIG_SKYLINE')) > 0)
if len(bd) > 0: stack.err[i, bd] *= np.sqrt(100)
if plot:
ax[0].plot(norm.apStarWave(), stack.flux[i, :])
ax[1].plot(norm.apStarWave(), stack.flux[i, :] / stack.err[i, :])
plt.draw()
pdb.set_trace()
# Accumulate for header of combined frame. Turn off visit specific RV flags first
visitflag = visit['starflag'] & ~starmask.getval(
'RV_REJECT') & ~starmask.getval('RV_SUSPECT')
starflag |= visitflag
andflag &= visitflag
if visit['survey'] == 'apogee':
apogee_target1 |= visit['apogee_target1']
apogee_target2 |= visit['apogee_target2']
apogee_target3 |= visit['apogee_target3']
elif visit['survey'].find('apogee2') >= 0:
apogee2_target1 |= visit['apogee_target1']
apogee2_target2 |= visit['apogee_target2']
apogee2_target3 |= visit['apogee_target3']
try:
apogee2_target4 |= visit['apogee_target4']
except:
pass
# MWM target flags?
# Create final spectrum
zeros = np.zeros([nvisit + 2, nwave])
izeros = np.zeros([nvisit + 2, nwave], dtype=int)
apstar = apload.ApSpec(zeros,
err=zeros.copy(),
bitmask=izeros,
wave=norm.apStarWave(),
sky=zeros.copy(),
skyerr=zeros.copy(),
telluric=zeros.copy(),
telerr=zeros.copy(),
cont=zeros.copy(),
template=zeros.copy())
apstar.header['CRVAL1'] = norm.logw0
apstar.header['CDELT1'] = norm.dlogw
apstar.header['CRPIX1'] = 1
apstar.header['CTYPE1'] = (
'LOG-LINEAR', 'Logarithmic wavelength scale in subsequent HDU')
apstar.header['DC-FLAG'] = 1
# Pixel-by-pixel weighted average
cont = np.median(stack.cont, axis=0)
apstar.flux[0, :] = np.sum(stack.flux / stack.err**2, axis=0) / np.sum(
1. / stack.err**2, axis=0) * cont
apstar.err[0, :] = np.sqrt(1. / np.sum(1. / stack.err**2, axis=0)) * cont
apstar.bitmask[0, :] = np.bitwise_and.reduce(stack.bitmask, 0)
apstar.cont[0, :] = cont
# Individual visits
apstar.flux[2:, :] = stack.flux * stack.cont
apstar.err[2:, :] = stack.err * stack.cont
apstar.bitmask[2:, :] = stack.bitmask
apstar.sky[2:, :] = stack.sky
apstar.skyerr[2:, :] = stack.skyerr
apstar.telluric[2:, :] = stack.telluric
apstar.telerr[2:, :] = stack.telerr
# Populate header
apstar.header['OBJID'] = (allvisit['apogee_id'][0], 'APOGEE object name')
apstar.header['APRED'] = (apred, 'APOGEE reduction version')
apstar.header['STARVER'] = (starver, 'apStar version')
apstar.header['HEALPIX'] = (apload.obj2healpix(allvisit['apogee_id'][0]),
'HEALPix location')
try:
apstar.header['SNR'] = (np.nanmedian(apstar.flux / apstar.err),
'Median S/N per apStar pixel')
except:
apstar.header['SNR'] = (0., 'Median S/N per apStar pixel')
apstar.header['RA'] = (allvisit['ra'].max(), 'right ascension, deg, J2000')
apstar.header['DEC'] = (allvisit['dec'].max(), 'declination, deg, J2000')
apstar.header['GLON'] = (allvisit['glon'].max(), 'Galactic longitude')
apstar.header['GLAT'] = (allvisit['glat'].max(), 'Galactic latitude')
apstar.header['J'] = (allvisit['j'].max(), '2MASS J magnitude')
apstar.header['J_ERR'] = (allvisit['j_err'].max(),
'2MASS J magnitude uncertainty')
apstar.header['H'] = (allvisit['h'].max(), '2MASS H magnitude')
apstar.header['H_ERR'] = (allvisit['h_err'].max(),
'2MASS H magnitude uncertainty')
apstar.header['K'] = (allvisit['k'].max(), '2MASS K magnitude')
apstar.header['K_ERR'] = (allvisit['k_err'].max(),
'2MASS K magnitude uncertainty')
try:
apstar.header['SRC_H'] = (allvisit['src_h'][0],
'source of H magnitude')
except KeyError:
pass
keys = [
'wash_m', 'wash_t2', 'ddo51', 'irac_3_6', 'irac_4_5', 'irac_5_8',
'wise_4_5', 'targ_4_5'
]
for key in keys:
try:
apstar.header[key] = allvisit[key].max()
except KeyError:
pass
apstar.header['AKTARG'] = (allvisit['ak_targ'].max(),
'Extinction used for targeting')
apstar.header['AKMETHOD'] = (allvisit['ak_targ_method'][0],
'Extinction method using for targeting')
apstar.header['AKWISE'] = (allvisit['ak_wise'].max(),
'WISE all-sky extinction')
apstar.header['SFD_EBV'] = (allvisit['sfd_ebv'].max(), 'SFD E(B-V)')
apstar.header['APTARG1'] = (apogee_target1,
'APOGEE_TARGET1 targeting flag')
apstar.header['APTARG2'] = (apogee_target2,
'APOGEE_TARGET2 targeting flag')
apstar.header['APTARG3'] = (apogee_target3,
'APOGEE_TARGET3 targeting flag')
apstar.header['AP2TARG1'] = (apogee2_target1,
'APOGEE2_TARGET1 targeting flag')
apstar.header['AP2TARG2'] = (apogee2_target2,
'APOGEE2_TARGET2 targeting flag')
apstar.header['AP2TARG3'] = (apogee2_target3,
'APOGEE2_TARGET3 targeting flag')
apstar.header['AP2TARG4'] = (apogee2_target4,
'APOGEE2_TARGET4 targeting flag')
apstar.header['NVISITS'] = (len(allvisit),
'Number of visit spectra combined flag')
apstar.header['STARFLAG'] = (starflag,
'bitwise OR of individual visit starflags')
apstar.header['ANDFLAG'] = (andflag,
'bitwise AND of individual visit starflags')
try:
apstar.header['N_COMP'] = (allvisit['n_components'].max(),
'Maximum number of components in RV CCFs')
except:
pass
apstar.header['VHBARY'] = (
(allvisit['vheliobary'] * allvisit['snr']).sum() /
allvisit['snr'].sum(), 'S/N weighted mean barycentric RV')
if len(allvisit) > 1:
apstar.header['vscatter'] = (allvisit['vheliobary'].std(ddof=1),
'standard deviation of visit RVs')
else:
apstar.header['VSCATTER'] = (0., 'standard deviation of visit RVs')
apstar.header['VERR'] = (0., 'unused')
apstar.header['RV_TEFF'] = (allvisit['rv_teff'].max(),
'Effective temperature from RV fit')
apstar.header['RV_LOGG'] = (allvisit['rv_logg'].max(),
'Surface gravity from RV fit')
apstar.header['RV_FEH'] = (allvisit['rv_feh'].max(),
'Metallicity from RV fit')
if len(allvisit) > 0:
meanfib = (allvisit['fiberid'] *
allvisit['snr']).sum() / allvisit['snr'].sum()
else:
meanfib = 999999.
if len(allvisit) > 1: sigfib = allvisit['fiberid'].std(ddof=1)
else: sigfib = 0.
apstar.header['MEANFIB'] = (meanfib, 'S/N weighted mean fiber number')
apstar.header['SIGFIB'] = (
sigfib, 'standard deviation (unweighted) of fiber number')
apstar.header['NRES'] = ('{:5.2f}{:5.2f}{:5.2f}'.format(*nres),
'number of pixels/resolution used for sinc')
# individual visit information in header
for i0, visit in enumerate(allvisit):
i = i0 + 1
apstar.header['SFILE{:d}'.format(i)] = (
visit['file'], ' Visit #{:d} spectrum file'.format(i))
apstar.header['DATE{:d}'.format(i)] = (
visit['dateobs'], 'DATE-OBS of visit {:d}'.format(i))
apstar.header['JD{:d}'.format(i)] = (
visit['jd'], 'Julian date of visit {:d}'.format(i))
# hjd = helio_jd(visitstr[i].jd-2400000.0,visitstr[i].ra,visitstr[i].dec)
#apstar.header['HJD{:d}'.format(i)] =
apstar.header['FIBER{:d}'.format(i)] = (visit['fiberid'],
' Fiber, visit {:d}'.format(i))
apstar.header['BC{:d}'.format(i)] = (
visit['bc'],
' Barycentric correction (km/s), visit {:d}'.format(i))
apstar.header['VRAD{:d}'.format(i)] = (
visit['vrel'], ' Doppler shift (km/s) of visit {:d}'.format(i))
#apstar.header['VERR%d'.format(i)] =
apstar.header['VHBARY{:d}'.format(i)] = (
visit['vheliobary'],
' Barycentric velocity (km/s), visit {:d}'.format(i))
apstar.header['SNRVIS{:d}'.format(i)] = (
visit['snr'], ' Signal/Noise ratio, visit {:d}'.format(i))
apstar.header['FLAG{:d}'.format(i)] = (
visit['starflag'], ' STARFLAG for visit {:d}'.format(i))
apstar.header.insert('SFILE{:d}'.format(i),
('COMMENT', 'VISIT {:d} INFORMATION'.format(i)))
# Do a RV fit just to get a template and normalized spectrum, for plotting
if dorvfit:
try:
apstar.setmask(pixelmask.badval())
spec = doppler.Spec1D(apstar.flux[0, :],
err=apstar.err[0, :],
bitmask=apstar.bitmask[0, :],
mask=apstar.mask[0, :],
wave=apstar.wave,
lsfpars=np.array([0]),
lsfsigma=apstar.wave / 22500 / 2.354,
instrument='APOGEE',
filename=apstar.filename)
out = doppler.rv.jointfit([spec],
verbose=False,
plot=False,
tweak=False,
maxvel=[-50, 50])
apstar.cont = out[3][0].flux
apstar.template = out[2][0].flux
except ValueError as err:
logger.error('Exception raised in visitcomb RV for: ',
apstar.header['FIELD'], apstar.header['OBJID'])
logger.error("ValueError: {0}".format(err))
except RuntimeError as err:
logger.error('Exception raised in visitcomb RV for: ',
apstar.header['FIELD'], apstar.header['OBJID'])
logger.error("Runtime error: {0}".format(err))
except:
logger.error('Exception raised in visitcomb RV fit for: ',
apstar.header['FIELD'], apstar.header['OBJID'])
# Write the spectrum to file
if write:
outfilenover = load.filename('Star', obj=apstar.header['OBJID'])
outdir = os.path.dirname(outfilenover)
outbase = os.path.splitext(os.path.basename(outfilenover))[0]
outbase += '-' + starver # add star version
outfile = outdir + '/' + outbase + '.fits'
if apstar_vers != 'stars':
outfile = outfile.replace('/stars/', '/' + apstar_vers + '/')
outdir = os.path.dirname(outfile)
try:
os.makedirs(os.path.dirname(outfile))
except:
pass
logger.info('Writing apStar file to ' + outfile)
apstar.write(outfile)
apstar.filename = outfile
mwm_root = os.environ['MWM_ROOT']
apstar.uri = outfile[len(mwm_root) + 1:]
# Create symlink no file with no version
if os.path.exists(outfilenover) or os.path.islink(outfilenover):
os.remove(outfilenover)
os.symlink(os.path.basename(outfile), outfilenover) # relative path
# Plot
gd, = np.where(
(apstar.bitmask[0, :]
& (pixelmask.badval() | pixelmask.getval('SIG_SKYLINE'))) == 0)
fig, ax = plots.multi(1, 3, hspace=0.001, figsize=(48, 6))
med = np.nanmedian(apstar.flux[0, :])
plots.plotl(ax[0],
norm.apStarWave(),
apstar.flux[0, :],
color='k',
yr=[0, 2 * med])
ax[0].plot(norm.apStarWave()[gd], apstar.flux[0, gd], color='g')
ax[0].set_ylabel('Flux')
try:
ax[1].plot(norm.apStarWave()[gd], apstar.cont[gd], color='g')
ax[1].set_ylabel('Normalized')
ax[1].plot(norm.apStarWave(), apstar.template, color='r')
except:
pass
plots.plotl(ax[2],
norm.apStarWave(),
apstar.flux[0, :] / apstar.err[0, :],
yt='S/N')
for i in range(3):
ax[i].set_xlim(15100, 17000)
ax[0].set_xlabel('Wavelength')
fig.savefig(outdir + '/plots/' + outbase + '.png')
# Plot
if plot:
ax[0].plot(norm.apStarWave(), apstar.flux, color='k')
ax[1].plot(norm.apStarWave(), apstar.flux / apstar.err, color='k')
plt.draw()
pdb.set_trace()
return apstar
def doppler_rv(star,
apred,
telescope,
nres=[5, 4.25, 3.5],
windows=None,
tweak=False,
clobber=False,
verbose=False,
plot=False,
logger=None):
"""
Run Doppler on one star and perform visit combination.
Parameters
----------
star : str
The '2M' star name.
apred : str
APOGEE reduction version.
telescope : str
APOGEE telescope (apo25m, loc25m, apo1m).
nres : array, optional
Array of sinc widths. Default is nres=[5,4.25,3.5].
windows : array, optional
Array of spectral windows to use.
tweak : bool, optional
Have Doppler tweak the continuum with the best-fit template. Default is False.
clobber : bool, optional
Overwrite any existing files (both RV and visit combined).
verbose : bool, optional
Verbose output to the screen.
plot : bool, optional
Make a plot of spectrum and best-fitting template.
logger : logger, optional
A logger for printed output.
Returns
-------
The program outputs Doppler files an apStar combined file to the appropriate
place specified in the SDSS/APOGEE tree product.
"""
if logger is None:
logger = dln.basiclogger()
snmin = 3
apstar_vers = 'stars'
logger.info(
'Running Doppler and performing visit combination for %s and telescope=%s'
% (star, telescope))
# Get the visit files for this star and telescope
db = apogeedb.DBSession()
allvisits = db.query('visit',
cols='*',
where="apogee_id='" + star + "' and telescope='" +
telescope + "' and apred_vers='" + apred + "'")
db.close()
nallvisits = len(allvisits)
if nallvisits == 0:
logger.info('No visit files found')
return
logger.info('%d visit file(s) found' % nallvisits)
# Get the star version number
# this is the largest MJD5 in the FULL list of visits
starver = str(np.max(int(allvisits['mjd'])))
logger.info('Version=' + starver)
# Select good visit spectra
starmask = bitmask.StarBitMask()
gd, = np.where(((allvisits['starflag'] & starmask.badval()) == 0)
& (allvisits['snr'] > snmin))
if len(gd) == 0:
logger.info('No visits passed QA cuts')
return
logger.info('%d visit(s) passed QA cuts' % len(gd))
starvisits = Table(allvisits[gd])
nvisits = len(gd)
# Change datatype of STARFLAG to 64-bit
starvisits['starflag'] = starvisits['starflag'].astype(np.uint64)
# Add STARVER
starvisits['starver'] = starver
# Output directory
load = apload.ApLoad(apred=apred, telescope=telescope)
starfile = load.filename('Star', obj=star)
stardir = os.path.dirname(starfile)
try:
os.makedirs(os.path.dirname(stardir))
except FileExistsError:
pass
# Run Doppler with dorv()
try:
dopsumstr, dopvisitstr, gaussout = dorv(starvisits,
starver,
clobber=clobber,
verbose=verbose,
tweak=tweak,
plot=plot,
windows=windows,
apstar_vers=apstar_vers,
logger=logger)
logger.info('Doppler completed successfully for {:s}'.format(star))
except:
logger.info('Doppler failed for {:s}'.format(star))
raise
# KLUDGE, rename vhelio->vheliobary
if 'vhelio' in starvisits.colnames:
starvisits['vhelio'].name = 'vheliobary'
# First rename old visit RV tags and initialize new ones
for col in [
'vtype', 'vrel', 'vrelerr', 'vheliobary', 'bc', 'rv_teff',
'rv_logg', 'rv_feh', 'rv_carb', 'rv_alpha'
]:
starvisits.rename_column(col, 'est' + col)
if col == 'vtype':
starvisits[col] = 0
else:
starvisits[col] = np.nan
for col in ['xcorr_vrel', 'xcorr_vrelerr', 'xcorr_vheliobary', 'bc']:
starvisits[col] = np.nan
# Add columns for RV components
starvisits['n_components'] = -1
rv_components = Column(name='rv_components',
dtype=float,
shape=(3),
length=len(starvisits))
starvisits.add_column(rv_components)
# Now load the new ones with the dorv() output
visits = []
ncomponents = 0
for i, (v, g) in enumerate(zip(dopvisitstr, gaussout)):
# Match by filename components in case there was an error reading in doppler
name = os.path.basename(v['filename']).replace('.fits', '').split('-')
if telescope == 'apo1m':
vind, = np.where(
np.char.strip(starvisits['file']).astype(str) ==
os.path.basename(v['filename'].strip()))
if len(vind) == 0:
# special case for incremental release...yuck
vind, = np.where(
np.char.strip(starvisits['file']).astype(str) == os.path.
basename(v['filename'].strip()).replace('-r13-', '-r12-'))
else:
vind, = np.where(
(np.char.strip(starvisits['plate']).astype(str) == name[-3])
& (starvisits['mjd'] == int(name[-2]))
& (starvisits['fiberid'] == int(name[-1])))
if len(vind) > 0:
vind = vind[0]
else:
continue
visits.append(vind)
starvisits[vind]['vrel'] = v['vrel']
starvisits[vind]['vrelerr'] = v['vrelerr']
starvisits[vind]['vheliobary'] = v['vhelio']
starvisits[vind]['xcorr_vrel'] = v['xcorr_vrel']
starvisits[vind]['xcorr_vrelerr'] = v['xcorr_vrelerr']
starvisits[vind]['xcorr_vheliobary'] = v['xcorr_vhelio']
starvisits[vind]['bc'] = v['bc']
starvisits[vind]['rv_teff'] = v['teff']
starvisits[vind]['rv_logg'] = v['logg']
starvisits[vind]['rv_feh'] = v['feh']
if g is None:
starvisits[vind]['n_components'] = 0
else:
starvisits[vind]['n_components'] = g['N_components']
if starvisits[vind]['n_components'] > 1:
starvisits[vind]['starflag'] |= starmask.getval('MULTIPLE_SUSPECT')
n = len(g['best_fit_parameters']) // 3
gd, = np.where(np.array(g['best_fit_parameters'])[0:n] > 0)
rv_comp = np.array(g['best_fit_parameters'])[2 * n + gd]
n_rv_comp = np.min([3, len(rv_comp)])
starvisits[vind]['rv_components'][0:n_rv_comp] = rv_comp[
0:n_rv_comp]
# flag visits with suspect RVs
if starvisits[vind]['rv_teff'] < 6000:
bd_diff = 10
else:
bd_diff = 50.
if (np.abs(starvisits[vind]['vheliobary'] -
starvisits[vind]['xcorr_vheliobary']) > bd_diff):
starvisits[vind]['starflag'] |= starmask.getval('RV_REJECT')
elif (np.abs(starvisits[vind]['vheliobary'] -
starvisits[vind]['xcorr_vheliobary']) > 0):
starvisits[vind]['starflag'] |= starmask.getval('RV_SUSPECT')
# Get the good visits
if len(visits) > 0:
visits = np.array(visits)
gdrv, = np.where((starvisits[visits]['starflag']
& starmask.getval('RV_REJECT')) == 0)
# Do the visit combination
if len(gdrv) > 0:
apstar = visitcomb(starvisits[visits[gdrv]],
starver,
load=load,
apstar_vers=apstar_vers,
apred=apred,
nres=nres,
logger=logger)
else:
logger.info('No good visits for ' + star)
raise
# Load information into the database
apstar.header['MJDBEG'] = (np.max(int(allvisits['mjd'])), 'Beginning MJD')
apstar.header['MJDEND'] = (np.max(int(allvisits['mjd'])), 'Ending MJD')
dbingest(apstar, starvisits[visits[gdrv]])
return
def dorv(allvisit,
starver,
obj=None,
telescope=None,
apred=None,
clobber=False,
verbose=False,
tweak=False,
plot=False,
windows=None,
apstar_vers='stars',
logger=None):
""" Do the Doppler rv jointfit from list of files
"""
if logger is None:
logger = dln.basiclogger()
if tweak == True:
suffix = '_tweak'
else:
suffix = '_out'
if obj is None:
obj = str(allvisit['apogee_id'][0])
if type(obj) is not str:
obj = obj.decode('UTF-8')
if apred is None:
apred = str(allvisit['apred_vers'][0])
if telescope is None:
telescope = str(allvisit['telescope'][0])
load = apload.ApLoad(apred=apred, telescope=telescope)
outfile = load.filename('Star', obj=obj)
outdir = os.path.dirname(outfile)
outbase = os.path.splitext(os.path.basename(outfile))[0]
outbase += '-' + starver # add star version
if os.path.exists(outdir) == False:
os.makedirs(outdir)
if apstar_vers != 'stars':
outdir = outdir.replace('/stars/', '/' + apstar_vers + '/')
if os.path.exists(outdir + '/' + outbase + suffix +
'_doppler.pkl') and not clobber:
logger.info(obj + ' already done')
fp = open(outdir + '/' + outbase + suffix + '_doppler.pkl', 'rb')
try:
out = pickle.load(fp)
sumstr, finalstr, bmodel, specmlist, gout = out
fp.close()
return sumstr, finalstr, gout
except:
logger.warning('error loading: ' + outbase + suffix +
'_doppler.pkl')
#pass
speclist = []
pixelmask = bitmask.PixelBitMask()
badval = pixelmask.badval() | pixelmask.getval(
'SIG_SKYLINE') | pixelmask.getval('LITTROW_GHOST')
# If we have a significant number of low S/N visits, combine first using
# barycentric correction only, use that to get an estimate of systemic
# velocity, then do RV determination restricting RVs to within 50 km/s
# of estimate. This seems to help significant for faint visits
lowsnr_visits, = np.where(allvisit['snr'] < 10)
if (len(lowsnr_visits) > 1) & (len(lowsnr_visits) / len(allvisit) > 0.1):
try:
apstar_bc = visitcomb(allvisit,
starver,
bconly=True,
load=load,
write=False,
dorvfit=False,
apstar_vers=apstar_vers)
apstar_bc.setmask(badval)
spec = doppler.Spec1D(apstar_bc.flux[0, :],
err=apstar_bc.err[0, :],
bitmask=apstar_bc.bitmask[0, :],
mask=apstar_bc.mask[0, :],
wave=apstar_bc.wave,
lsfpars=np.array([0]),
lsfsigma=apstar_bc.wave / 22500 / 2.354,
instrument='APOGEE',
filename=apstar_bc.filename)
logger.info('Lots of low-S/N visits. Running BC jointfit for :',
obj)
out = doppler.rv.jointfit([spec],
verbose=verbose,
plot=plot,
tweak=tweak,
maxvel=[-500, 500])
rvrange = [out[1][0]['vrel'] - 50, out[1][0]['vrel'] + 50]
except:
logger.info(' BC jointfit failed')
rvrange = [-500, 500]
elif allvisit['h'].max() > 13.5:
# If it's faint, restrict to +/- 500 km/s
rvrange = [-500, 500]
else:
# Otherwise, restrict to +/ 1000 km/s
rvrange = [-1000, 1000]
# Loop over visits
for i in range(len(allvisit)):
# Load all of the visits into doppler Spec1D objects
if load.telescope == 'apo1m':
visitfile = load.allfile('Visit',
plate=allvisit['plate'][i],
mjd=allvisit['mjd'][i],
reduction=allvisit['apogee_id'][i],
field=allvisit['field'][i])
else:
visitfile = load.allfile('Visit',
plate=int(allvisit['plate'][i]),
mjd=allvisit['mjd'][i],
fiber=allvisit['fiberid'][i],
field=allvisit['field'][i])
spec = doppler.read(visitfile, badval=badval)
if windows is not None:
# If we have spectral windows to mask, do so here
for ichip in range(3):
mask = np.full_like(spec.mask[:, ichip], True)
gd = []
for window in windows:
gd.extend(
np.where((spec.wave[:, ichip] > window[0])
& (spec.wave[:, ichip] < window[1]))[0])
mask[gd] = False
spec.mask[:, ichip] |= mask
if spec is not None: speclist.append(spec)
if len(speclist) == 0:
raise Exception('No visit spectra loaded')
# Now do the Doppler jointfit to get RVs
# Dump empty pickle to stand in case of failure (to prevent redo if not clobber)
try:
# Dump empty pickle to stand in case of failure (to prevent redo if not clobber)
fp = open(outdir + '/' + outbase + suffix + '_doppler.pkl', 'wb')
pickle.dump(None, fp)
fp.close()
logger.info(
'Running Doppler jointfit for: {:s} rvrange:[{:.1f},{:.1f}] nvisits: {:d}'
.format(obj, *rvrange, len(speclist)))
sumstr, finalstr, bmodel, specmlist, dt = doppler.rv.jointfit(
speclist,
maxvel=rvrange,
verbose=verbose,
plot=plot,
saveplot=plot,
outdir=outdir + '/',
tweak=tweak)
logger.info('Running CCF decomposition for: ' + obj)
gout = gauss_decomp(finalstr, phase='two', filt=True)
fp = open(outdir + '/' + outbase + suffix + '_doppler.pkl', 'wb')
pickle.dump([sumstr, finalstr, bmodel, specmlist, gout], fp)
fp.close()
# Making plots
logger.info('Making plots for :' + obj + ' ' + outdir)
try:
os.makedirs(outdir + '/plots/')
except:
pass
dop_plot(outdir + '/plots/',
outbase, [sumstr, finalstr, bmodel, specmlist],
decomp=gout)
except KeyboardInterrupt:
raise
except ValueError as err:
logger.error('Exception raised in dorv for: ' + obj)
logger.error("ValueError: {0}".format(err))
return
except RuntimeError as err:
logger.error('Exception raised in dorv for: ' + obj)
logger.error("Runtime error: {0}".format(err))
return
except:
raise
logger.error('Exception raised in dorv for: ', field, obj)
return
# Return summary RV info, visit RV info, decomp info
return sumstr, finalstr, gout