def add_plugmap_info(plate_info, plugmap_keys):
import yanny
plate = plate_info['plate']
mjd = plate_info['mjd']
plan_name = os.path.join(plate_info['bossdir'], str(plate), 'spPlancomb-%s-%s.par' % (plate, mjd))
plan = yanny.yanny(plan_name)
# look up plugmap filename
unique_mapnames = set(zip(plan['SPEXP']['mjd'], plan['SPEXP']['mapname']))
(mapmjd, mapname) = unique_mapnames.pop()
plugmap_name = os.path.join(plate_info['speclog'], str(mapmjd), 'plPlugMapM-%s.par' % mapname)
plugmap = yanny.yanny(plugmap_name)
plate_info['mapmjd'] = mapmjd
plate_info['mapname'] = mapname
# Process plugmap header keywords
for keyword in plugmap_keys:
plate_info[keyword] = plugmap[keyword]
design_ra = Angle(float(plugmap['raCen']), unit=u.degree)
design_dec = Angle(float(plugmap['decCen']), unit=u.degree)
design_ha = Angle(float(plugmap['haMin']), unit=u.degree)
design_alt, design_az = equatorial_to_horizontal(design_ra, design_dec, apolat, design_ha)
plate_info['design_ra'] = design_ra.to(u.degree).value
plate_info['design_dec'] = design_dec.to(u.degree).value
plate_info['design_ha'] = design_ha.to(u.degree).value
plate_info['design_alt'] = design_alt.to(u.degree).value
plate_info['design_az'] = design_az.to(u.degree).value
def add_plugmap_info(plate_info, plugmap_keys):
import yanny
plate = plate_info['plate']
mjd = plate_info['mjd']
plan_name = os.path.join(plate_info['bossdir'], str(plate),
'spPlancomb-%s-%s.par' % (plate, mjd))
plan = yanny.yanny(plan_name)
# look up plugmap filename
unique_mapnames = set(zip(plan['SPEXP']['mjd'], plan['SPEXP']['mapname']))
(mapmjd, mapname) = unique_mapnames.pop()
plugmap_name = os.path.join(plate_info['speclog'], str(mapmjd),
'plPlugMapM-%s.par' % mapname)
plugmap = yanny.yanny(plugmap_name)
plate_info['mapmjd'] = mapmjd
plate_info['mapname'] = mapname
# Process plugmap header keywords
for keyword in plugmap_keys:
plate_info[keyword] = plugmap[keyword]
design_ra = Angle(float(plugmap['raCen']), unit=u.degree)
design_dec = Angle(float(plugmap['decCen']), unit=u.degree)
design_ha = Angle(float(plugmap['haMin']), unit=u.degree)
design_alt, design_az = equatorial_to_horizontal(design_ra, design_dec,
apolat, design_ha)
plate_info['design_ra'] = design_ra.to(u.degree).value
plate_info['design_dec'] = design_dec.to(u.degree).value
plate_info['design_ha'] = design_ha.to(u.degree).value
plate_info['design_alt'] = design_alt.to(u.degree).value
plate_info['design_az'] = design_az.to(u.degree).value
def write_gbu(parfile):
backupfile = re.sub(r"\.drp\.par", r".gbu_backup", parfile)
par = yanny(parfile, np=True)
with open(backupfile, "w") as f:
f.write("#{:2}{:>4}{:>4}\n".format("ap", "ur", "az"))
for ap in par["APINFO"]:
f.write("{:3n}{:4n}{:4n}\n".format(ap["ap"], ap["ur_gbu"], ap["az_gbu"]))
def plot_yanny_on_grid(parfile, ax, band1, band2):
par = yanny(parfile,np=True)
scat = ax.scatter(par['APINFO'][band1], par['APINFO'][band2],
c=np.abs(par['APINFO']['z']), s=40, linewidths=0,
alpha=0.7, cmap=plt.cm.gnuplot2)
return scat
def makeHeaderFixFile(mjd, exp, status, hdrdir):
''' Write or add to a header fix file in mangacore'''
# Make full file path name
hdrfile = os.path.join(hdrdir, 'sdHdrFix-{0}.par'.format(mjd))
# Create header directory if it doesn't exist
dir_exists = os.path.isdir(hdrdir)
if dir_exists:
print('Header fix directory exists.')
else:
print('Header fix does not directory exist. Creating it.')
os.makedirs(hdrdir)
# Make new data
newdata = {'fileroot':[],'keyword':[],'value':[]}
newdata['fileroot'].append('sdR-??-00{0}'.format(exp))
newdata['keyword'].append('QUALITY')
newdata['value'].append('{0}'.format(status.upper()))
order=['fileroot','keyword','value']
newtable = Table(newdata,dtype=['S15','S15','S15'])
newtable = newtable[order]
# Append or write to file
file_exists = os.path.isfile(hdrfile)
if file_exists:
print('Header fix file exists. Updating file and overwriting.')
# Read header fix file
hdrfix = yanny.yanny(filename=hdrfile, np=True)
oldtable = Table(hdrfix['OPHDRFIX'],dtype=['S15','S15','S15'])
oldtable = oldtable[order]
# Check if new entry inside old
newinold = newtable['fileroot'] == oldtable['fileroot']
# Update rows
if not any(newinold):
# add new row if it doesn't exist
[oldtable.add_row(row) for row in newtable]
else:
# update the row
fileindex = np.where(newinold)[0]
for row in newtable:
if row['value'] in ['GOOD','OVERRIDE GOOD']:
oldtable.remove_row(fileindex[0])
else:
oldtable[fileindex] = row
yanny.write_ndarray_to_yanny(hdrfile,np.array(oldtable),structname='OPHDRFIX',overwrite=True)
else:
print('No header fix file found. Creating it')
yanny.write_ndarray_to_yanny(hdrfile,np.array(newtable),structname='OPHDRFIX')
return hdrfile, dir_exists, file_exists
def swap_plugmap_stds(speclog_from, speclog_to, plate, night, mapname,
fiberids, dry_run, clobber, verbose):
# build path to plPlugMagM for unique mapname
plugmap_name = os.path.join(speclog_from, night,
'plPlugMapM-%s.par' % mapname)
new_plugmap_name = os.path.join(speclog_to, night,
'plPlugMapM-%s.par' % mapname)
# Check to see if file exists already
# yanny will not overwrite files so remove this by hand, in case we are running
if os.path.isfile(new_plugmap_name):
if clobber:
os.remove(new_plugmap_name)
else:
print 'Modified plugmap file already exists: %s' % new_plugmap_name
print ' use --clobber option to overwrite...'
return -1
# read the plugmap file
try:
plugmap = yanny.yanny(plugmap_name)
except:
print 'Could not open plugmap file: %s' % plugmap_name
return -1
plugmap_objs = plugmap['PLUGMAPOBJ']
# hide the 'standard' boss spectrophoto standards
boss_stds = np.where(
np.array(plugmap_objs['objType']) == 'SPECTROPHOTO_STD')[0]
if verbose and len(boss_stds) != 20:
print 'Found %d boss standards on plate %s (expected 20)' % (
len(boss_stds), plate)
for i in boss_stds:
plugmap_objs['objType'][i] = 'NA'
# set the ancillary targets' objType as "SPECTROPHOTO_STD" by iterating
# over plugmap objects and matching fiberids to ancillary target fiberids
# also make sure to check holetype is correct (guide stars and light traps have
# overlapping fiberids with the boss 'science' targets)
nblue_stds = 0
for i in range(len(plugmap_objs['objType'])):
if str(plugmap_objs['fiberId'][i]) in set(
fiberids) and plugmap_objs['holeType'][i] == 'OBJECT':
# We found a blue standard, change its OBJTYPE
plugmap_objs['objType'][i] = 'SPECTROPHOTO_STD'
nblue_stds += 1
if verbose and nblue_stds != len(fiberids):
print 'Found %d blue standards on plate %s (expected %d)' % (
nblue_stds, plate, len(fiberids))
# save the plug map for this night's observation
if verbose:
print 'Saving modified plugmap file: %s' % new_plugmap_name
plugmap.set_filename(new_plugmap_name)
if not dry_run:
plugmap.write()
def getExposureIDs(plate,mjd,fiber='',camera='',verbose=False):
plan2dFilename = os.path.join(BSR,redVersion,plate,'spPlan2d-%s-%s.par'%(plate,mjd))
plan2d = yanny.yanny(plan2dFilename)
spexp = plan2d['SPEXP']
exposureIDs = []
for i in range(len(spexp['flavor'])):
if spexp['flavor'][i] == 'science':
if camera == '' and fiber == '':
for c in range(4):
exposureIDs.append('-'.join(spexp['name'][i][c].split('-')[1:3]).split('.')[0])
else:
exposureIDs.append('-'.join(spexp['name'][i][0].split('-')[1:3]).split('.')[0])
return exposureIDs
def getExposureIDs(plate, mjd, fiber='', camera='', verbose=False):
plan2dFilename = os.path.join(BSR, redVersion, plate,
'spPlan2d-%s-%s.par' % (plate, mjd))
plan2d = yanny.yanny(plan2dFilename)
spexp = plan2d['SPEXP']
exposureIDs = []
for i in range(len(spexp['flavor'])):
if spexp['flavor'][i] == 'science':
if camera == '' and fiber == '':
for c in range(4):
exposureIDs.append('-'.join(
spexp['name'][i][c].split('-')[1:3]).split('.')[0])
else:
exposureIDs.append('-'.join(
spexp['name'][i][0].split('-')[1:3]).split('.')[0])
return exposureIDs
def swap_plugmap_stds(speclog_from, speclog_to, plate, night, mapname, fiberids, dry_run, clobber, verbose):
# build path to plPlugMagM for unique mapname
plugmap_name = os.path.join(speclog_from, night, 'plPlugMapM-%s.par' % mapname)
new_plugmap_name = os.path.join(speclog_to, night, 'plPlugMapM-%s.par' % mapname)
# Check to see if file exists already
# yanny will not overwrite files so remove this by hand, in case we are running
if os.path.isfile(new_plugmap_name):
if clobber:
os.remove(new_plugmap_name)
else:
print 'Modified plugmap file already exists: %s'% new_plugmap_name
print ' use --clobber option to overwrite...'
return -1
# read the plugmap file
try:
plugmap = yanny.yanny(plugmap_name)
except:
print 'Could not open plugmap file: %s' % plugmap_name
return -1
plugmap_objs = plugmap['PLUGMAPOBJ']
# hide the 'standard' boss spectrophoto standards
boss_stds = np.where(np.array(plugmap_objs['objType']) == 'SPECTROPHOTO_STD')[0]
if verbose and len(boss_stds) != 20:
print 'Found %d boss standards on plate %s (expected 20)' % (len(boss_stds), plate)
for i in boss_stds:
plugmap_objs['objType'][i] = 'NA'
# set the ancillary targets' objType as "SPECTROPHOTO_STD" by iterating
# over plugmap objects and matching fiberids to ancillary target fiberids
# also make sure to check holetype is correct (guide stars and light traps have
# overlapping fiberids with the boss 'science' targets)
nblue_stds = 0
for i in range(len(plugmap_objs['objType'])):
if str(plugmap_objs['fiberId'][i]) in set(fiberids) and plugmap_objs['holeType'][i] == 'OBJECT':
# We found a blue standard, change its OBJTYPE
plugmap_objs['objType'][i] = 'SPECTROPHOTO_STD'
nblue_stds += 1
if verbose and nblue_stds != len(fiberids):
print 'Found %d blue standards on plate %s (expected %d)' % (nblue_stds, plate, len(fiberids))
# save the plug map for this night's observation
if verbose:
print 'Saving modified plugmap file: %s' % new_plugmap_name
plugmap.set_filename(new_plugmap_name)
if not dry_run:
plugmap.write()
def __init__(self, fscan, verbose=False):
assert isinstance(verbose, bool)
self.verbose = verbose
if isinstance(fscan, str):
self.data = yanny.yanny(fscan, np=True)
self.filename = os.path.basename(fscan)
self.dirname = os.path.dirname(fscan)
elif isinstance(fscan, yanny.yanny):
self.data = fscan
self.filename = os.path.basename(fscan.filename)
self.dirname = os.path.dirname(fscan.filename)
else:
raise TypeError('fscan of type {0} must be a string or a '
'yanny object'.format(type(fscan)))
self._initGuideNums()
self._initGuide()
def load_filters(**kwargs):
"""
Load filter information from a list of files.
Filter files should contain one structure with columns 'lambda' &
'pass'. The name of the table can be arbitrary though.
"""
if 'filterlist' not in kwargs:
raise TypeError('Invalid keyword arguments passed to load_filters')
if 'filterpath' not in kwargs:
kwargs['filterpath'] = os.getenv('KCORRECT_DIR')+'/data/filters'
filterdata = {'lambda':[], 'pass':[]}
for fil in kwargs['filterlist']:
f = yanny.yanny(kwargs['filterpath']+'/'+fil)
if str(f) == '':
raise IOError("Could not read %s" % (kwargs['filterpath']+'/'+fil))
t = f.tables()
filterdata['lambda'].append(pylab.array(f[t[0]]['lambda']))
filterdata['pass'].append(pylab.array(f[t[0]]['pass']))
return filterdata
def fromDatabase(cls, pk):
"""Returns a `PlPlugMapMFile` from the DB.
Returns the fscan corresponging to the plPlugMapM entry with primary
key `pk`.
"""
plPlugMapM = getPlPlugMapM(pk)
try:
yy = yanny.yanny(string=str(plPlugMapM.file), np=True)
except:
raise PlPlugMapMFileException(
'problem found converting blob to Yanny file '
'for PlPlugMapM.pk={0}'.format(pk))
plFile = PlPlugMapMFile.__new__(cls)
plFile.__init__(yy)
plFile.filename = plPlugMapM.filename
return plFile
def load_filters(**kwargs):
"""
Load filter information from a list of files.
Filter files should contain one structure with columns 'lambda' &
'pass'. The name of the table can be arbitrary though.
"""
if 'filterlist' not in kwargs:
raise TypeError('Invalid keyword arguments passed to load_filters')
if 'filterpath' not in kwargs:
kwargs['filterpath'] = os.getenv('KCORRECT_DIR') + '/data/filters'
filterdata = {'lambda': [], 'pass': []}
for fil in kwargs['filterlist']:
f = yanny.yanny(kwargs['filterpath'] + '/' + fil)
if str(f) == '':
raise IOError("Could not read %s" %
(kwargs['filterpath'] + '/' + fil))
t = f.tables()
filterdata['lambda'].append(pylab.array(f[t[0]]['lambda']))
filterdata['pass'].append(pylab.array(f[t[0]]['pass']))
return filterdata
def update_gbu(parfile, method, ap, gbu):
if method not in ["ur", "at", "az"]:
print "WARNING: Method not recognized.\nValid methods are ur, az, at"
return
par = yanny(parfile, np=True)
with open(parfile, "r") as f:
lines = f.readlines()
header = [l for l in lines if l[0] == "#"]
try:
par["APINFO"]["{}_gbu".format(method)][ap - 1] = gbu
except IndexError:
print "WARNING: Could not find ap {} in {}".format(ap, parfile)
return
with open(parfile, "w") as p:
p.writelines(header)
par.write(p)
return
def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v","--verbose", action = "store_true",
help="provide more verbose output")
parser.add_argument("-i","--input", type=str, default="",
help="input plate, mjd list")
parser.add_argument("--clobber", action="store_true",
help="overwrite existing files")
parser.add_argument("--dry-run", action="store_true",
help="dry run, no copies")
parser.add_argument("--speclog-from", type=str, default="/home/boss/products/NULL/speclog/trunk",
help="source speclog directory")
parser.add_argument("--bsr-run2d", type=str, default="/clusterfs/riemann/raid008/bosswork/boss/spectro/redux/v5_7_0",
help="path to reduction to mirror")
parser.add_argument("--speclog-to", type=str, default="/clusterfs/riemann/raid008/bosswork/boss/spectro/redux/test/dmargala/speclog",
help="destination speclog directory to write modified plugmap files to")
parser.add_argument("--work-dir", type=str, default="",
help="Working directory")
args = parser.parse_args()
# Read list of plate, mjd pairs to process
plate_mjd_file = open(args.input, 'r')
plate_mjd_list = []
for line in plate_mjd_file:
plate, mjd = line.split()
plate_mjd_list.append((plate, mjd))
if args.verbose:
print 'Read %d plate-mjd pairs from file: %s' % (len(plate_mjd_list), args.input)
print 'Mirroring reduction at: %s' % args.bsr_run2d
print 'Creating modified speclog with with blue standards and spectrophoto standards swapped...'
print ' source speclog: %s' % args.speclog_from
print ' new speclog: %s' % args.speclog_to
print ''
def copy_speclog_file(name):
from_name = os.path.join(args.speclog_from, name)
to_name = os.path.join(args.speclog_to, name)
if os.path.isfile(from_name):
if not args.clobber and os.path.isfile(to_name) and not args.dry_run:
raise ValueError('Destination file already exists: %s' % to_name)
if args.verbose:
print os.path.join(to_name)
if not args.dry_run:
shutil.copy(from_name, to_name)
else:
print 'Warning: Source file does not exist: %s' % from_name
copy_speclog_file('spPlateList.par')
# copy opfiles
if not os.path.exists(os.path.join(args.speclog_to, 'opfiles')):
os.makedirs(os.path.join(args.speclog_to, 'opfiles'))
opfiles = ['spPlateMinSN2.par', 'spPlateZrange.par', 'spPlateList.par']
for opfile in opfiles:
copy_speclog_file(os.path.join('opfiles', opfile))
for plate_mjd_pair in plate_mjd_list:
(plate, mjd) = plate_mjd_pair
# read blue standards list
target_file = os.path.join(args.work_dir, plate, 'blue-stds-%s-%s.txt' % plate_mjd_pair)
if args.verbose:
print ''
print 'Reading targets list: %s' % target_file
fiberids = []
with open(target_file,'r') as targetlist:
for line in targetlist:
plate_mjd_fiber = line.strip().split('-')
fiberids.append(plate_mjd_fiber[2])
# build path to spPlan for current plate-mjd
plan_name = os.path.join(args.bsr_run2d, plate, 'spPlancomb-%s-%s.par' % plate_mjd_pair)
# read spPlan file
plan = yanny.yanny(plan_name)
# get mjd list of observations for this plate-mjd
unique_mapnames = set(zip(plan['SPEXP']['mjd'], plan['SPEXP']['mapname']))
# The plugmaps are saved in files corresponding to the actual
# observation nights, not necessarily the last night (which is part of the target identifier)
for night, mapname in unique_mapnames:
night = str(night)
if args.verbose:
print '... processing night, mapname: %s, %s' % (night, mapname)
if not os.path.exists(os.path.join(args.speclog_to, night)):
os.makedirs(os.path.join(args.speclog_to, night))
# copy sdHdrFix file over
copy_speclog_file(os.path.join(night, 'sdHdrFix-%s.par' % night))
# copy guiderMon file over
copy_speclog_file(os.path.join(night, 'guiderMon-%s.par' % night))
# copy plPlugMapM file and swap standards' OBJTYPE
swap_plugmap_stds(args.speclog_from, args.speclog_to, plate, night, mapname, fiberids, args.dry_run, args.clobber, args.verbose)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v",
"--verbose",
action="store_true",
help="provide more verbose output")
parser.add_argument("-i",
"--input",
type=str,
default="",
help="input plate, mjd list")
parser.add_argument("--clobber",
action="store_true",
help="overwrite existing files")
parser.add_argument("--dry-run",
action="store_true",
help="dry run, no copies")
parser.add_argument("--speclog-from",
type=str,
default="/home/boss/products/NULL/speclog/trunk",
help="source speclog directory")
parser.add_argument(
"--bsr-run2d",
type=str,
default="/clusterfs/riemann/raid008/bosswork/boss/spectro/redux/v5_7_0",
help="path to reduction to mirror")
parser.add_argument(
"--speclog-to",
type=str,
default=
"/clusterfs/riemann/raid008/bosswork/boss/spectro/redux/test/dmargala/speclog",
help="destination speclog directory to write modified plugmap files to"
)
parser.add_argument("--work-dir",
type=str,
default="",
help="Working directory")
args = parser.parse_args()
# Read list of plate, mjd pairs to process
plate_mjd_file = open(args.input, 'r')
plate_mjd_list = []
for line in plate_mjd_file:
plate, mjd = line.split()
plate_mjd_list.append((plate, mjd))
if args.verbose:
print 'Read %d plate-mjd pairs from file: %s' % (len(plate_mjd_list),
args.input)
print 'Mirroring reduction at: %s' % args.bsr_run2d
print 'Creating modified speclog with with blue standards and spectrophoto standards swapped...'
print ' source speclog: %s' % args.speclog_from
print ' new speclog: %s' % args.speclog_to
print ''
def copy_speclog_file(name):
from_name = os.path.join(args.speclog_from, name)
to_name = os.path.join(args.speclog_to, name)
if os.path.isfile(from_name):
if not args.clobber and os.path.isfile(
to_name) and not args.dry_run:
raise ValueError('Destination file already exists: %s' %
to_name)
if args.verbose:
print os.path.join(to_name)
if not args.dry_run:
shutil.copy(from_name, to_name)
else:
print 'Warning: Source file does not exist: %s' % from_name
copy_speclog_file('spPlateList.par')
# copy opfiles
if not os.path.exists(os.path.join(args.speclog_to, 'opfiles')):
os.makedirs(os.path.join(args.speclog_to, 'opfiles'))
opfiles = ['spPlateMinSN2.par', 'spPlateZrange.par', 'spPlateList.par']
for opfile in opfiles:
copy_speclog_file(os.path.join('opfiles', opfile))
for plate_mjd_pair in plate_mjd_list:
(plate, mjd) = plate_mjd_pair
# read blue standards list
target_file = os.path.join(args.work_dir, plate,
'blue-stds-%s-%s.txt' % plate_mjd_pair)
if args.verbose:
print ''
print 'Reading targets list: %s' % target_file
fiberids = []
with open(target_file, 'r') as targetlist:
for line in targetlist:
plate_mjd_fiber = line.strip().split('-')
fiberids.append(plate_mjd_fiber[2])
# build path to spPlan for current plate-mjd
plan_name = os.path.join(args.bsr_run2d, plate,
'spPlancomb-%s-%s.par' % plate_mjd_pair)
# read spPlan file
plan = yanny.yanny(plan_name)
# get mjd list of observations for this plate-mjd
unique_mapnames = set(
zip(plan['SPEXP']['mjd'], plan['SPEXP']['mapname']))
# The plugmaps are saved in files corresponding to the actual
# observation nights, not necessarily the last night (which is part of the target identifier)
for night, mapname in unique_mapnames:
night = str(night)
if args.verbose:
print '... processing night, mapname: %s, %s' % (night,
mapname)
if not os.path.exists(os.path.join(args.speclog_to, night)):
os.makedirs(os.path.join(args.speclog_to, night))
# copy sdHdrFix file over
copy_speclog_file(os.path.join(night, 'sdHdrFix-%s.par' % night))
# copy guiderMon file over
copy_speclog_file(os.path.join(night, 'guiderMon-%s.par' % night))
# copy plPlugMapM file and swap standards' OBJTYPE
swap_plugmap_stds(args.speclog_from, args.speclog_to, plate, night,
mapname, fiberids, args.dry_run, args.clobber,
args.verbose)
def create_yanny(pointing, output, indexfile=None):
numaps = get_numaps_loc(pointing)
data = np.zeros(
(numaps,),
dtype=[
("ap", "i"),
("apsize", "f4"),
("vdisp", "f4"),
("r", "f4"),
("z", "f4"),
("SN", "f4"),
("Hb", "f4"),
("HdA", "f4"),
("HgA", "f4"),
("HdF", "f4"),
("HgF", "f4"),
("Fe", "f4"),
("MgFe", "f4"),
("ur_gbu", "i"),
("ur_age", "f4"),
("ur_Av", "f4"),
("ur_Z", "f4"),
("ur_chisq", "f4"),
("ur_redchi", "f4"),
("ur_bluechi", "f4"),
("at_gbu", "i"),
("at_age", "f4"),
("at_Av", "f4"),
("at_Z", "f4"),
("at_chisq", "f4"),
("at_redchi", "f4"),
("at_bluechi", "f4"),
("az_gbu", "i"),
("az_age", "f4"),
("az_Av", "f4"),
("az_Z", "f4"),
("az_chisq", "f4"),
("az_redchi", "f4"),
("az_bluechi", "f4"),
],
)
data = get_basics_loc(pointing, data)
data = get_index(pointing, data, datfile=indexfile)
# data = get_unreg(pointing, data)
# data = get_alltau(pointing, data)
# data = get_allz(pointing, data)
par = yanny(filename=None, np=True, debug=False)
struct = par.dtype_to_struct(data.dtype, structname="APINFO", enums=dict())
par["symbols"]["struct"] = struct["struct"]
par["symbols"]["enum"] = struct["enum"]
par["symbols"]["APINFO".upper()] = struct["APINFO".upper()]
par["APINFO"] = data
par["pointing"] = pointing
par["numaps"] = numaps
with open(output, "w") as f:
f.write(
"""# NGC 891 aperture file for pointing {}
# File generated on {}
#
# Uses SDSS-style Yanny .par formatting
#
# This file contains a collection of all of the important parameters derived
# from full spectrum fitting of our NGC 891 data. The various fitting methods
# are described below:
#
# unreg:
# The OG. Each age is forced to have a single metallicity (found as a
# separate step) and there is a single extinction value for all the SSPs
# in an aperture.
#
# all_tau:
# Each age is forced to have a single metallicity (found as a
# separate step), but each age is allowed to have a different A_V value
# (limited by 0 < A_V < 20).
#
# all_Z2:
# Each age can have an arbitrary number of metallicities with
# arbitrary weights. There is a single extinction value for all the SSPs
# in an aperture.
#
# The data structure is as follows:
#
# Header keywords:
# pointing: Which pointing's data is contained in this file
# numaps: The number of binned apertures contained in this pointing
#
# Data table entries:
# 1) ap: The aperture number.
# 2) apsize: The size of the fibers combined into the aperture, in arcsec.
# Note that this is NOT the size of the final aperture.
# 3) vdisp: The resolution (in km/s) of the aperture. Different fiber sizes
# produce different resolutions.
# 4) r: Radius (in kpc) of aperture from the center of the galaxy
# 5) z: Height (in kpc) of aperture above the midplane of the galaxy
# 6) SN: Final signal to noise ratio of the binned aperture
# 7) ID_age: Age from LICK index matching with BC03 models
# 8) ID_Z: Metallicity from LICK index matching with BC03 models
# 9) ur_gbu: Unreg quality flag. 0=good, 1=bad fit, 2=ugly for other reason
# 10) ur_age: MLWA from unreg fit, in Gyr
# 11) ur_Av: Av from unreg fit
# 12) ur_Z: Best metallicity (in chisq sense) of unreg fit
# 13) ur_chisq: Full, reduced chisq value from best unreg fit
# 14) ur_redchi: Reduced chisq for 5250 <= lambda <= 6800 from unreg
# 15) ur_bluechi: Reduced chisq for 3750 <= lambda < 5250 from unreg
# 16) at_gbu: all_tau quality flag. 0=good, 1=bad fit, 2=ugly for other reason
# 17) at_age: MLWA from all_tau fit, in Gyr
# 18) at_Av: MLWAv all_tau from fit
# 19) at_Z: Best metallicity (in chisq sense) of all_tau fit
# 20) at_chisq: Full, reduced chisq value from best all_tau fit
# 21) at_redchi: Reduced chisq for 5250 <= lambda <= 6800 from all_tau fit
# 22) at_bluechi: Reduced chisq for 3750 <= lambda < 5250 from all_tau fit
# 23) az_gbu: all_Z2 quality flag. 0=good, 1=bad fit, 2=ugly for other reason
# 24) az_age: MLWA from all_Z2 fit, in Gyr
# 25) az_Av: Av from all_Z2 fit
# 26) az_Z: MLWZ from all_Z2 fit
# 27) az_chisq: Full, reduced chisq value from best all_Z2 fit
# 28) az_redchi: Reduced chisq for 5250 <= lambda <= 6800 from all_Z2 fit
# 29) az_bluechi: Reduced chisq for 3750 <= lambda < 5250 from all_Z2 fit
#
""".format(
pointing, time.asctime()
)
)
par.write(f)
return
import matplotlib.pyplot as plt
import matplotlib.font_manager
from matplotlib import rc
import scipy.optimize as optimize
from scipy import interpolate
import matplotlib.pylab as py
import yanny as yan
import sys
rc('text', usetex=True)
rc('font', family='serif')
sdss_run = int(sys.argv[1])
nran = int(sys.argv[2])
nrsk = int(sys.argv[3])
par = yan.yanny(
'/home/users/hahn/qf_environment/zerod_environment_parameters.par',
np=True)
rad = str(par['PARAMETERS']['cylrad'][sdss_run - 1])
height = str(par['PARAMETERS']['cylheight'][sdss_run - 1])
thresh = str(par['PARAMETERS']['thresh'][sdss_run - 1])
nbin = str(par['PARAMETERS']['nbin'][sdss_run - 1])
vmax_avail_dir = '/global/data/scr/chh327/primus/data/vmax_avail/'
ransack_dir = '/global/data/scr/chh327/primus/data/ransack/'
envcount_dir = '/global/data/scr/chh327/primus/data/envcount/'
random_vmax_name = 'vmax_avail_cylr' + rad + 'h' + height + '_thresh' + thresh + '_nbin' + nbin + '_sdss_ran' + str(
nran) + '_rsk' + str(nrsk) + '.fits'
random_vmax_data = fits.open(vmax_avail_dir + random_vmax_name)
random_vmax_z = random_vmax_data[1].data.field('z')
#redshift = ["$0.6<z<0.8$","$0.4<z<0.6$","$0.2<z<0.4$"]
#color = ['b','m','r']
sfq = ['active', 'quiescent']
sfqtitle = ['Star Forming', 'Quiescent','Quiescent Fraction']
markers = ['o', 's', '^']
smf_mass = []
sf = []
q = []
sferr = []
qerr = []
rc('text', usetex=True)
rc('font', family='serif')
primus_run = int(sys.argv[1])
par = yan.yanny('/home/users/hahn/qf_environment/zerod_environment_parameters.par',np=True)
rad = str(par['PARAMETERS']['cylrad'][primus_run-1])
height = str(par['PARAMETERS']['cylheight'][primus_run-1])
thresh = str(par['PARAMETERS']['thresh'][primus_run-1])
nbin = str(par['PARAMETERS']['nbin'][primus_run-1])
sdss_run = int(sys.argv[2])
par = yan.yanny('/home/users/hahn/qf_environment/zerod_environment_parameters.par',np=True)
sdss_rad = str(par['PARAMETERS']['cylrad'][sdss_run-1])
sdss_height = str(par['PARAMETERS']['cylheight'][sdss_run-1])
sdss_thresh = str(par['PARAMETERS']['thresh'][sdss_run-1])
sdss_nbin = str(par['PARAMETERS']['nbin'][sdss_run-1])
if (sys.argv[3]=='2envbin'):
markers = ['o', 's', '^']
redshift = ["$0.8<z<1.0$","$0.6<z<0.8$","$0.4<z<0.6$","$0.2<z<0.4$"]
mass = []
sf = []
q = []
hist_sf = []
hist_q = []
hist_sferr = []
hist_qerr = []
sferr = []
qerr = []
rc('text', usetex=True)
rc('font', family='serif')
par = yan.yanny('/home/users/hahn/primus/pro/spec0d/envcount/zerod_environment_parameters.par',np=True)
run = int(raw_input('Run: '))
rad = str(par['PARAMETERS']['cylrad'][run-1])
height = str(par['PARAMETERS']['cylheight'][run-1])
thresh = str(par['PARAMETERS']['thresh'][run-1])
nbin = str(par['PARAMETERS']['nbin'][run-1])
fig=plt.figure(1,figsize=(10,10))
for i in range(len(sfq)):
for ii in range(len(ebin)):
ax = fig.add_subplot(3, 3, 3*i + ii + 1)
for iii in range(len(zbin)):
fname = 'smf_primus_cylr'+rad+'h'+height+'_thresh'+thresh+'_nbin'+nbin+'_lit_'+sfq[i]+'_'+ebin[ii]+'_'+zbin[iii]+'.dat'
print fname
data=np.loadtxt(smfdir+fname)
plateListHDUList = pyfits.open(bossPlatelistFile)
done_boss_plates = []
for ind in range(len(plateListHDUList[1].data.field('STATUS1D'))):
run1dst = plateListHDUList[1].data.field('STATUS1D')[ind].lower() == 'done'
run2dst = plateListHDUList[1].data.field('STATUS2D')[ind].lower() == 'done'
combinest = plateListHDUList[1].data.field('STATUSCOMBINE')[ind].lower() == 'done'
is_boss = plateListHDUList[1].data.field('SURVEY')[ind].lower() == 'boss'
if run1dst and run2dst and combinest and is_boss:
done_boss_plates.append(plateListHDUList[1].data.field('PLATE')[ind])
else:
print "Plate: %i not done" % plateListHDUList[1].data.field('PLATE')[ind]
# Read plateplans from file.
platePlansYanny = yanny.yanny(platePlansFile)
platePlans = platePlansYanny.list_of_dicts('PLATEPLANS') # returns an array of dictionary objects
boss_plates = []
boss_not_done = []
sdss_plates = []
for pp in platePlans:
if pp['survey'].lower() == "sdss" or "segue" in pp['survey'].lower():
sdss_plates.append(((pp['raCen']+75)%360, pp['decCen']))
elif pp['survey'].lower() == 'boss':
if pp['plateid'] in done_boss_plates:
boss_plates.append(((pp['raCen']+75)%360, pp['decCen']))
else:
boss_not_done.append(((pp['raCen']+75)%360, pp['decCen']))
"plateHoles-{0:06d}.par".format(plate))
if os.path.isfile(plateHoleFilename) == False:
print "plateHole file for plate {0} does not exist, downloading...".format(
plate)
url = "https://svn.sdss.org/public/data/sdss/platelist/trunk/plates/00{id}XX/{plate:06d}/plateHoles-{plate:06d}.par".format(
id=str(plate)[0:2], plate=plate)
#print url
plateHoleURL = urllib2.urlopen(url)
plateHoleFile = open(plateHoleFilename, "w")
data = plateHoleURL.read(40)
plateHoleFile.write(data)
HAData = data.split('\n')[1]
HADes = n.float(HAData[3:8])
elif os.path.isfile(plateHoleFilename) == True:
print "plateHole file for plate {0} exists, opening...".format(plate)
plateHoleFile = y.yanny(plateHoleFilename, np=True)
# read the designed hourangle (in degrees)
HADes = n.float(plateHoleFile["ha"][0:5])
# change HA to be in range [-180,180] if needed:
if HADes > 180:
HADes = HADes - 360.
elif HADes < -180:
HADes = HAdes + 360.
# calculate design altitude
AltDes = RadToDeg(
n.arcsin(
n.sin(DegToRad(h0["PLUG_DEC"])) * n.sin(DegToRad(lat)) +
n.cos(DegToRad(h0["PLUG_DEC"])) * n.cos(DegToRad(lat)) *
n.cos(DegToRad(HADes))))
print "Des hourangle: {0:-10.2f} Des altitude: {1:-10.2f}".format(
HADes, AltDes)
def height_summary(output, field, label='', title='', log=False):
# Designed to work with drpall files, not fits
meanbase = output.split('.pdf')[0].split('_height')[0]
drpdir = '/d/monk/eigenbrot/WIYN/14B-0456/anal/drpall'
ax = plt.figure().add_subplot(111)
symblist = ["o","^","v","s","*","x"]
colorlist = ['b','c','g','y','m','r']
plist = [6,3,4,2,1,5]
rlist = [-10.0, -6.4, -2.3, 0.8, 4.4, 8.1]
binidxs = [[0,1,2,3,4,5],
[2,3],
[1,4],
[0,5],
[0,1,4,5]]
bigd = np.zeros((6,3,11))
for i, p in enumerate(plist):
parfile = glob('{}/NGC_891_P{}.drp.par'.format(drpdir,p))[0]
print parfile
par = yanny(parfile,np=True,debug=False)
zs = par['APINFO']['z']
zs= np.abs(zs)
try:
value = par['APINFO'][field]
except ValueError:
print 'WARNING: Could not find {} field in {}'.format(field,parfile)
return
if log:
value = np.log10(value)
unqz = np.unique(zs) #also sorts. Nice!
abcissa = np.zeros(unqz.size)
ordinate = np.zeros(unqz.size)
err = np.zeros(unqz.size)
for j, z in enumerate(unqz):
idx = np.where(zs == unqz[j])[0]
abcissa[j] = unqz[j]
ordinate[j] = np.mean(value[idx])
#err[j] = ordinate[j]*np.sqrt(1./idx.size)
err[j] = np.std(value[zs == unqz[j]])/np.sqrt(idx.size)
ax.errorbar(abcissa,ordinate,yerr=err,
color=colorlist[i],marker=symblist[i],ls='',
elinewidth=0.4,
label=str(rlist[i]))
bigd[i,0,:] = abcissa
bigd[i,1,:] = ordinate
bigd[i,2,:] = err
# num, z_edge = np.histogram(bigd[:,0,:].flatten(),11,normed=False)
# total, z_edge2 = np.histogram(bigd[:,0,:].flatten(),11,normed=False,
# weights=bigd[:,1,:].flatten())
# print z_edge - z_edge2
# bigz = (z_edge[1:] + z_edge[:-1])*0.5
bigz2 = np.linspace(np.min(bigd[:,0,:]),np.max(bigd[:,0,:]),13)
bigI = np.zeros((6,2,13))
for k in range(len(plist)):
print np.all(np.diff(bigd[k,0,:]) > 0)
bigI[k,0,:] = np.interp(bigz2,bigd[k,0,:],bigd[k,1,:])
bigI[k,1,:] = np.interp(bigz2,bigd[k,0,:],bigd[k,2,:])
bigval = np.mean(bigI[:,0,:],axis=0)
bigerr = np.sqrt(
np.nansum(bigI[:,1,:]*(bigI[:,0,:] - bigval)**2,axis=0)/
((5/6.)*np.nansum(bigI[:,1,:],axis=0)))
for l, b in enumerate(binidxs):
binmean = np.mean(bigI[b,0,:],axis=0)
binerr = np.sqrt(
np.nansum(bigI[b,1,:]*(bigI[b,0,:] - binmean)**2,axis=0)/
((len(b)-1.0)/len(b) * np.nansum(bigI[b,1,:],axis=0)))
with open('{}_bin{}_means.dat'.format(meanbase,l),'w') as f:
f.write('# Bin {}: r = {} kpc\n'.\
format(l,np.array(rlist)[None,b].flatten()))
f.write(str('#{:>12}'+'{:>13}'*2+'\n').\
format('height',
field,
'{} err'.format(field)))
for Z, V, E in zip(bigz2,binmean,binerr):
f.write(str('{:13.4f}'*3+'\n').format(Z,V,E))
# ax.plot(bigz,total/num,'k-')
ax.plot(bigz2,bigval,'b-')
ax.fill_between(bigz2,bigval-bigerr,bigval+bigerr,alpha=0.2)
ax.set_xlabel('z [kpc]')
ax.set_ylabel(label)
ax.set_title(title)
ax.legend(loc=0,numpoints=1,title='radius [kpc]',fontsize=10)
pp = PDF(output)
pp.savefig(ax.figure)
pp.close()
plt.close(ax.figure)
return
import numpy as np
import pylab as py
import pyfits as fits
from matplotlib import rc
import yanny as yan
import sys
rc('text', usetex=True)
rc('font', family='serif')
run = int(sys.argv[1])
par = yan.yanny('/home/users/hahn/research/pro/blanton/qf_environment/zerod_environment_parameters.par',np=True)
rad = str(par['PARAMETERS']['cylrad'][run-1])
height = str(par['PARAMETERS']['cylheight'][run-1])
thresh = str(par['PARAMETERS']['thresh'][run-1])
highenvthresh = par['PARAMETERS']['highenvthresh'][run-1]
lowenvthresh = par['PARAMETERS']['lowenvthresh'][run-1]
sdss_run = int(sys.argv[2])
sdss_envfile = 'EDP-sdss-z00375_0145-numden.fits'
envfile = 'EDP-primus-z0210-numden.fits'
if (sys.argv[3] == 'lit'):
litsuffix = '_lit'
else:
litsuffix = ''
zbin = [0.1, 0.3, 0.5, 0.7, 0.9]
zbin_labels = ['SDSS $0.0375 - 0.145$','PRIMUS $0.2 - 0.4$', 'PRIMUS $0.4 - 0.6$', 'PRIMUS $0.6 - 0.8$', 'PRIMUS $0.8 - 1.0$']
sf_q = ['active', 'quiescent']
