def coaddDiffBundle(mapfiles, output_name, pad_to=None, overwrite=True):
coadd = files.read(mapfiles[0])
plus = 1
minus = 0
for i, mfile in enumerate(mapfiles[1:]):
m = files.read(mfile)
if i % 2 == 0:
coadd += -1 * m # subtraction works only on weighted regions
else:
coadd += m
# pad the map
if pad_to is not None:
args = coadd.initargs(noshape=True)
weight_full = np.zeros(pad_to)
map_full = np.zeros(pad_to)
pad = [(pt - ms) / 2 for pt, ms in zip(pad_to, coadd.shape)]
weight_full[pad[0]:pad[0] + coadd.shape[0],
pad[1]:pad[1] + coadd.shape[1]] = weight_map
map_full[pad[0]:pad[0] + coadd.shape[0],
pad[1]:pad[1] + coadd.shape[1]] = sum_map
coadd = ObservedSky(map=map_full, weight=weight_full)
coadd.writeToHDF5(output_name, overwrite=overwrite)
info = open(output_name + '.info', 'w')
info.write('%d\n' % len(mapfiles))
info.write('\n'.join(mapfiles))
info.close()
def coadd(mapfiles, pol = True):
coadd = None
for mf in mapfiles:
if coadd is None:
coadd = files.read(mf)
else:
coadd += files.read(mf)
return coadd
def quickCheck(auxfiles):
for f in auxfiles:
aux = files.read(f)
try:
loadDataForAuxFile(aux)
except KeyError:
print '\t\t\t' + f
def loadDataForAuxFile(aux):
if isinstance(aux, str):
aux = files.read(aux)
data = SPTDataReader(aux.header.start_date,
aux.header.stop_date,
quiet=True)
data.readData()
return data
def plotCmap(mapfile, plot_dir):
m = files.read(mapfile)
f = pl.figure()
pl.imshow(m.Map.real_map_T, vmin=-5e-3, vmax=5e-3, cmap=matplotlib.cm.gray)
pl.colorbar()
name = '.'.join(path.basename(mapfile).split('.')[:-1])
pl.savefig(path.join(plot_dir, name + '.png'))
pl.close('all')
def getMap(fname):
dirname, bname = os.path.split(fname)
mapname = bname[:-14] + '.h5'
if dirname.endswith('maps') or 'bundle' in dirname:
map_dir = dirname
else:
map_dir = os.path.join(os.path.dirname(dirname), 'maps')
return files.read(os.path.join(map_dir, mapname))
def getNoise(mapfiles, noise_ell=[5500, 6500]):
noise = np.zeros(np.shape(mapfiles))
for i, mf in enumerate(mapfiles):
m = files.read(mf)
noise[i] = calculateNoise(m.removeWeight(),
ell_range=noise_ell,
return_noise=True,
quiet=True)[0]
del m
return noise
def coadd(directory, band):
mapfiles = glob.glob(os.path.join(directory, '*%sghz.h5' %band))
coadd = None
for mf in mapfiles:
_map = files.read(mf)
if coadd is None:
coadd = _map
else:
coadd += _map
return coadd
def idf_to_map_half(outdir):
try:
os.makedirs(os.path.join(outdir, 'left'))
os.makedirs(os.path.join(outdir, 'right'))
except OSError:
pass
map_args = {
'good_bolos': [
'optical_bolometers',
'no_c4',
'bolometer_flags',
'has_pointing',
'has_polcal',
'timestream_flags',
# 'elnod',
# 'calibrator',
'good_timestream_weight',
],
'reso_arcmin':
30,
'proj':
1,
'map_shape': (45, 170),
'map_center': [9 * 15, -51],
'timestream_filtering': {
'poly_order': 0,
}
}
bands = [90, 150]
dirs = ['half1', 'half2']
for b in bands:
idfs = []
for d in dirs:
idfs.append(
files.read(
glob.glob(os.path.join(outdir, d,
'*{:03d}*.h5'.format(b))))[0])
maps_left = [
cMapping(i, doreal=True, use_leftgoing=True, **map_args)[str(b)]
for i in idfs
]
maps_right = [
cMapping(i, doreal=True, use_leftgoing=False, **map_args)[str(b)]
for i in idfs
]
left = np.sum(maps_left)
right = np.sum(maps_right)
left.writeToHDF5(os.path.join(outdir, 'left',
'sunop_map_{:03d}ghz.h5'.format(b)),
overwrite=True)
right.writeToHDF5(os.path.join(outdir, 'right',
'sunop_map_{:03d}ghz.h5'.format(b)),
overwrite=True)
return left, right
def plot(mapfile, plot_dir):
print mapfile
m = files.read(mapfile)
m.removeWeight()
f = pl.figure()
# m.plot('T', bw = True, vmin = -1e-3, vmax = 1e-3, figure = 1)
m.plot('T', bw=True, weight=True, figure=1)
name = '.'.join(path.basename(mapfile).split('.')[:-1])
f.set_size_inches(12.15, 6.)
pl.savefig(path.join(plot_dir, name + '.png'))
pl.close('all')
def map_from_many_idfs(idf_list, outdir):
map_args = {
'good_bolos': [
'optical_bolometers',
'no_c4',
'bolometer_flags',
'has_pointing',
'has_polcal',
'timestream_flags',
# 'elnod',
# 'calibrator',
'good_timestream_weight',
],
'reso_arcmin':
30,
'proj':
1,
'map_shape': (45, 170),
'map_center': [9 * 15, -51],
'timestream_filtering': {
'poly_order': 4,
'ell_lowpass': 360
}
}
glitchfinder_args = {
'timestream_ids_to_check': None,
'ps_list': "",
'verbose': True
}
cutter = cuts.Cutter()
idfs = []
for i_file in idf_list:
idf = files.read(i_file)
cutter.flagTimestreamJumps(idf, **glitchfinder_args)
idfs.append(idf)
maps_left = [
cMapping(i, doreal=True, use_leftgoing=True,
**map_args)[str(idfs[0].band)] for i in idfs
]
maps_right = [
cMapping(i, doreal=True, use_leftgoing=False,
**map_args)[str(idfs[0].band)] for i in idfs
]
left = np.sum(maps_left)
right = np.sum(maps_right)
left.writeToHDF5(os.path.join(
outdir, 'left', 'sunop_map_{:03d}ghz.h5'.format(idfs[0].band)),
overwrite=True)
right.writeToHDF5(os.path.join(
outdir, 'right', 'sunop_map_{:03d}ghz.h5'.format(idfs[0].band)),
overwrite=True)
return left, right
def coaddBundle(mapfiles, output_name, pad_to=None, overwrite=True):
coadd = files.read(mapfiles[0])
for mfile in mapfiles[1:]:
m = files.read(mfile)
coadd += m
# pad the map
if pad_to is not None:
args = coadd.initargs(noshape=True)
weight_full = np.zeros(pad_to)
map_full = np.zeros(pad_to)
pad = [(pt - ms) / 2 for pt, ms in zip(pad_to, coadd.shape)]
weight_full[pad[0]:pad[0] + coadd.shape[0],
pad[1]:pad[1] + coadd.shape[1]] = weight_map
map_full[pad[0]:pad[0] + coadd.shape[0],
pad[1]:pad[1] + coadd.shape[1]] = sum_map
coadd = ObservedSky(map=map_full, weight=weight_full)
coadd.writeToHDF5(output_name, overwrite=overwrite)
info = open(output_name + '.info', 'w')
info.write('%d\n' % len(mapfiles))
info.write('\n'.join(mapfiles))
info.close()
def getLTFromRA(mapfile, thresh=.3, min_offset=100):
# look at the mean x-position of good pixels
try:
map = files.read(mapfile)
except IOError:
return None
xpix = map.shape[1]
goodpix = np.where(map.weight > thresh * np.median(map.weight))
mean_x = np.mean(goodpix[1])
if mean_x < xpix / 2 - min_offset:
return 'L'
if mean_x > xpix / 2 + min_offset:
return 'T'
else:
return 'F'
def checkOb(obfile, check_field, lock=None):
# return True if the ob is bad
try:
ob = files.read(obfile)
except ValueError:
return False
unique_vals = np.unique(ob.observation[check_field])
unique_frac = float(len(unique_vals)) / len(ob.observation[check_field])
if unique_frac < 1.2:
lock.acquire()
res_str = ob.from_filename + ' ' + str(unique_frac)
# for v in unique_vals:
# res_str += ', ' + str(v)
print res_str
lock.release()
return True
def carrierCheck(auxfiles, carrier_filename, read_filename):
carrier_out = open(carrier_filename, 'w')
read_out = open(read_filename, 'w')
passed_out = open('/home/ndhuang/code/auxchecker/passed_carrier.txt', 'w')
for af in auxfiles:
aux = files.read(af)
try:
data = loadDataForAuxFile(aux)
except:
read_out.write(af + '\n')
continue
carrier_frac = singleCarrierCheck(data)
carrier_out.write(af + ' %f\n' % carrier_frac)
carrier.close()
read_out.close()
passed_out.close()
def noise_diff(band):
directory = '/mnt/rbfa/ndhuang/fast_500d_map/run3/map_ra0hec-57.5/ra0hdec-57.5/coadds/'
mapfiles = glob.glob(os.path.join(directory, '*%sghz.h5' %band))
# ensure an even number of maps
if len(mapfiles) %2 != 0:
mapfiles = mapfiles[1:]
coadd = None
for i, mf in enumerate(mapfiles):
_map = files.read(mf)
if coadd is None:
coadd = _map
else:
try:
if i %2 == 0:
coadd += _map
else:
coadd -= _map
except ValueError:
print mf
pass
return coadd
import matplotlib
matplotlib.use('Agg')
import glob
from matplotlib import pyplot as pl, cm
from sptpol_software.util import files
import numpy as np
dir = '/data39/ndhuang/clusters/ra23h30dec-55/run2/bundles2/150/bundle*.fits'
mask = '/data39/ndhuang/clusters/ra23h30dec-55/run2/bundles1/150/mask.fits'
mask = files.read(mask)
mapfiles = glob.glob(dir)
fig = pl.figure()
for mf in mapfiles:
map = files.read(mf)
pl.imshow(map.map.map * mask.masks.apod_mask, cmap = cm.gray, vmin = -5e-4, vmax = 5e-4)
pl.colorbar()
pl.savefig(mf.strip('.fits') + '.png')
fig.clf()
# plot a map with ringing in it
import glob, os
import numpy as np
from sptpol_software.util import files
directory = '/data22/acrites/coadds_cuts_test/test11_20130503/'
mapfiles = glob.glob(os.path.join(directory, 'smap*.h5'))
m = files.read(mapfiles[0])
weight_map = m.getTOnly().weight
tcoadd_map = m.getTOnly().map
for mf in mapfiles[1:]:
m = files.read(mf)
t_map = m.getTOnly()
tcoadd_map += t_map.map
weight_map += t_map.weight
inds = np.nonzero(weight_map)
tcoadd_map[inds] /= weight_map[inds]
inds = np.nonzero(weight_map == 0)
tcoadd_map[inds] = 0
np.save('ringy_coadd', tcoadd_map)
np.save('ringy_weight', weight_map)
def makeBundle(mapfiles):
bundle = files.read(mapfiles[0])
for mf in mapfiles[1:]:
bundles += files.read(mf)
import sys, glob, os
from sptpol_software.util import files
if __name__ == '__main__':
mapdir = sys.argv[1]
mapfiles_left = glob.glob(os.path.join(mapdir, 'left', '*.h5'))
coadd_dir = os.path.join(mapdir, 'coadds')
if not os.path.exists(coadd_dir):
os.makedirs(coadd_dir)
for mf_l in mapfiles_left:
mapname = os.path.basename(mf_l)
mf_r = os.path.join(mapdir, 'right', mapname)
if not os.path.exists(mf_r):
continue
map_l = files.read(mf_l)
map_r = files.read(mf_r)
try:
full_map = map_l + map_r
except ValueError, err:
print mapname
import IPython
IPython.embed()
full_map.writeToHDF5(os.path.join(coadd_dir, mapname))
from sptpol_software.util import files
import glob
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as pl
mapfiles = glob.glob(
'/data/sptdat/auxdata/map/autoprocessed_maps/ra5hdec-35/coadds/left/*150ghz.h5'
)
mapfiles = sorted(mapfiles)
for i, mf in enumerate(mapfiles):
print mf
mapp = files.read(mf)
mapp.removeWeight()
fig = pl.figure()
mapp.plot('T', bw=True, vmin=-5e-4, vmax=5e-4, figure=1)
dpi = fig.get_dpi()
fig.set_size_inches((1920. / dpi, 1200. / dpi))
fig.tight_layout()
pl.savefig('/home/ndhuang/plots/pretty_maps/%d.png' % i)
pl.close('all')
def analyzeData(time_interval, idf_band=None):
"""
Nearly the generic function, but we return the result of the analysis function instead of
the data object that we read out.
"""
# This is a bit of a kludge to let us analyze both 150 GHz and 90 GHz IDFs with the same script.
# When the autoprocessor calls analyzeData, we'll get the default argument idf_band=None.
# This bit of code will separately run the 150 GHz and 90 GHz mapmaking, then combine the
# results and return them.
if read_from_idf and idf_band is None:
if do_left_right:
# Do mapmaking for each of the bands in turn.
analysis_result150, analysis_result150_left, analysis_result150_right = analyzeData(time_interval, idf_band=150)
analysis_result90, analysis_result90_left, analysis_result90_right = analyzeData(time_interval, idf_band=90)
# Combine the dictionaries of maps.
analysis_result150.update(analysis_result90)
analysis_result150_left.update(analysis_result90_left)
analysis_result150_right.update(analysis_result90_right)
# Return the combined results.
return analysis_result150, analysis_result150_left, analysis_result150_right
else:
# Do mapmaking for each of the bands in turn.
analysis_result150 = analyzeData(time_interval, idf_band=150)
analysis_result90 = analyzeData(time_interval, idf_band=90)
# Combine the dictionaries of maps.
analysis_result150.update(analysis_result90)
# Return the combined results.
return analysis_result150
# Check if we're reading from an IDF or going directly from archive files.
if read_from_idf:
if my_name == 'ra0hdec-57.5':
print '2013 field...'
idf_filename = os.path.join(directories['idf_dir'],'data','%s_idf_%s_%03ighz.h5' % (my_name, time_interval[0].file, idf_band))
else:
idf_filename = os.path.join(directories['idf_dir'],my_name,'data','%s_idf_%s_%03ighz.h5' % (my_name, time_interval[0].file, idf_band))
data = read(idf_filename, timeit=True)
if not data:
raise ValueError("I couldn't find an IDF for the %s observation taken at %s. (No file at %s .)"
% (my_name, time_interval[0].archive, idf_filename))
# test if data is in ra range you want
if ra_cut:
if (data.observation.mean_ra < ra_cut[0]) or (data.observation.mean_ra > ra_cut[1]):
raise ValueError("Skipping this IDF b/c wrong RA range: %s observation taken at %s. (file at %s .)"
% (my_name, time_interval[0].archive, idf_filename))
data._readConfigs() # We need the auxdata_keys information.
# The IDF generator may have flagged some scans in some timestreams as bad.
# With such a huge source, the RMS flagger's output is suspect (in fact, very
# bad over the source), so remove all timestream flags.
for scan in data.scan:
scan.is_bad_channel[:]=False
#Correct pointing after the fact.
if correct_pointing:
op.applyOfflinePointing(data, model='SPT', overwrite_global=True,
flags=flags, overwrite=True)
#Grab the thermometry and metrology data.
thermo_data = {'tracker.encoder_off':np.median(data.antenna.track_enc_off, axis=1), \
'tracker.horiz_mount':np.median(data.antenna.track_hor_mnt, axis=1), \
'tracker.horiz_off':np.median(data.antenna.track_hor_off, axis=1), \
#'tracker.tilt_xy_avg':np.median(data.antenna.track_tilt_xy_avg, axis=1), \
'tracker.linear_sensor_avg':np.median(data.antenna.track_linsens_avg, axis=1), \
'scu.temp':np.median(data.antenna.scu_temp, axis=1),
'scu.benchoff':data.antenna.scu_benchoff[:,0],
'observation.temp_avg':data.observation.temp_avg,
'observation.pressure_avg':data.observation.pressure_avg,
'observation.mean_az':data.observation.mean_az,
'observation.mean_el':data.observation.mean_el,
'observation.wind_dir_avg':data.observation.wind_dir_avg,
'observation.wind_speed_avg':data.observation.wind_speed_avg
}
this_filename = "%s_%s_thermolinear.pkl" % (my_name, time_interval[0].file)
filename_out_dir = os.path.join(directories['output_dir'], my_name, subdirectory_name)
filename = os.path.join(filename_out_dir, this_filename)
pk.dump(thermo_data, open(filename, 'w'))
else:
data = SPTDataReader(time_interval[0], time_interval[1],
experiment=experiment,
master_configfile=master_config)
data.readData(obstype=my_name, **readout_kwargs)
#Correct pointing after the fact.
if correct_pointing:
op.applyOfflinePointing(data, model='SPT', overwrite_global=True,
flags=flags, overwrite=True)
#Grab the thermometry and metrology data.
thermo_data = {'tracker.encoder_off':np.median(data.antenna.track_enc_off, axis=1), \
'tracker.horiz_mount':np.median(data.antenna.track_hor_mnt, axis=1), \
'tracker.horiz_off':np.median(data.antenna.track_hor_off, axis=1), \
#'tracker.tilt_xy_avg':np.median(data.antenna.track_tilt_xy_avg, axis=1), \
'tracker.linear_sensor_avg':np.median(data.antenna.track_linsens_avg, axis=1), \
'scu.temp':np.median(data.antenna.scu_temp, axis=1),
'scu.benchoff':data.antenna.scu_benchoff[:,0],
'observation.temp_avg':data.observation.temp_avg,
'observation.pressure_avg':data.observation.pressure_avg,
'observation.mean_az':data.observation.mean_az,
'observation.mean_el':data.observation.mean_el,
'observation.wind_dir_avg':data.observation.wind_dir_avg,
'observation.wind_speed_avg':data.observation.wind_speed_avg
}
this_filename = "%s_%s_thermolinear.pkl" % (my_name, time_interval[0].file)
filename_out_dir = os.path.join(directories['output_dir'], my_name, subdirectory_name)
filename = os.path.join(filename_out_dir, this_filename)
pk.dump(thermo_data, open(filename, 'w'))
if mask_radius:
if force_source_location:
ra, dec = force_source_location[0], force_source_location[1]
else:
# Find the location of the source. Do a linear fit subtraction now. Make an IDF
# for the sourcefinding, as a quick and easy way to be able to do light filtering
# on the sourcefinding map without filtering the real data.
temp_idf = (data.copy() if read_from_idf else mapidf.MapIDF(data))
finder_map = quicklook.quickmap(temp_idf, good_bolos=['flagged','pointing'],
reso_arcmin=0.25,
proj=5,
map_shape=[0.8,1.],
map_center=map_center,
t_only=True,
inverse_noise_weighted_map=True,
timestream_filtering={'poly_order':1,
'dynamic_source_mask':True,
'outlier_mask_nsigma':3.0})
temp_idf = None # Discard the temporary IDF now that we're done with it.
# Discard the map we're not creating. We only need one of them.
finder_map = finder_map[str(idf_band)] if idf_band is not None else finder_map['150']
# Find the RA and dec of the maximum point in the map.
ra, dec = finder_map.pix2Ang(np.unravel_index(np.argmax(np.abs(finder_map.map)), finder_map.shape))
# Make a temporary pointsource config file, and write it in the arguments
# to be passed to quickmap.
ptsrc_configfile = tempfile.NamedTemporaryFile(mode='w', suffix='.txt')
ptsrc_configfile.write('1 %f %f %f' % (ra, dec, mask_radius))
ptsrc_configfile.flush()
print " Writing temporary pointsource config file, %s . Contents:" % ptsrc_configfile.name
print '1 %f %f %f\n' % (ra, dec, mask_radius)
analysis_function_kwargs['timestream_filtering']['pointsource_file'] = ptsrc_configfile.name
for func, kwargs in zip(preprocessing_function, preprocessing_function_kwargs):
func(data, **kwargs)
print " Processing sum map.\n"
print " analysis_function_kwargs : %s" % str(analysis_function_kwargs)
analysis_result = analysis_function(data, **analysis_function_kwargs)
if do_left_right:
# Now we've done filtering once, no need to do it again.
lr_analysis_function_kwargs = analysis_function_kwargs.copy() # Don't alter the original kwargs!
lr_analysis_function_kwargs['timestream_filtering'] = {}
print " Processing leftgoing map.\n"
analysis_result_left = analysis_function(data, use_leftgoing=True, **lr_analysis_function_kwargs)
print " Processing rightgoing map.\n"
analysis_result_right = analysis_function(data, use_leftgoing=False, **lr_analysis_function_kwargs)
if mask_radius:
ptsrc_configfile.close()
if read_from_idf:
# If this is an IDF, it's only got one band. Delete maps from the other(s) so we
# don't save empty maps.
if str(data.band) not in analysis_result:
raise RuntimeError("I was expecting to see %s in the output maps, but it's not there!" % str(data.band))
for band in analysis_result.keys():
if band!=str(data.band):
del analysis_result[band]
if do_left_right:
del analysis_result_left[band]
del analysis_result_right[band]
if do_left_right:
return analysis_result, analysis_result_left, analysis_result_right
else:
return analysis_result
import os
import cPickle as pickle
import numpy as np
from sptpol_software.util import files
def readRunlist(runlist):
runlist = open(runlist, 'r')
mapfiles = runlist.readlines()
i = 0
while i < len(mapfiles):
mapfiles[i] = mapfiles[i].strip()
if not os.path.exists(mapfiles[i]):
mapfiles.pop(i)
else:
i += 1
return mapfiles
if __name__ == '__main__':
mapfiles = readRunlist('/mnt/rbfa/ndhuang/maps/clusters/ra1hdec-35/150ghz_runlist.txt')
idf_dir = '/data/ndhuang/auxdata/idf/ra1hdec-35/data/'
T = np.zeros(len(mapfiles))
for i, mf in enumerate(mapfiles):
filename = os.path.basename(mf)
idf_name = filename.replace('clustermap_', '').replace('5_2', '5_idf_2', 1)
idf = files.read(os.path.join(idf_dir, idf_name))
T[i] = idf.observation.temp_avg
f = open('/mnt/rbfa/ndhuang/maps/clusters/ra1hdec-35/t_avg', 'w')
pickle.dump([T, mapfiles], f)
f.close()
def __init__(self,
field,
band,
plotname,
clusterdir='/mnt/rbfa/ndhuang/maps/clusters',
v=1e-3,
maskfile=None,
radec0=None):
self.proj = 0
self.reso = .25
# parse field center from field name
if radec0 is not None:
self.radec0 = radec0
elif field in centers.keys():
self.radec0 = centers[field]
else:
match = re.match('ra(\d+)h(\d*)dec([0-9\-]+)', field)
ra = 15 * float(match.group(1))
if len(match.group(2)) > 0:
ra += float(match.group(2)) / 60 * 15
dec = float(match.group(3))
self.radec0 = [ra, dec]
# grab the coadd and expand it
try:
coadd = files.read(
os.path.join(clusterdir, field, '%03d_coadd.fits' % band))
except IOError:
coadd = files.read(
os.path.join(clusterdir, field, '%03dghz_coadd.fits' % band))
self.map = coadd.coadd.map
self.map_shape = np.shape(self.map)
# print self.map_shape
# self.resizeMap(2)
self.contour = np.zeros(self.map_shape, dtype=int)
self.fig = pl.figure(figsize=(9, 9), dpi=3000)
pl.imshow(self.map,
cmap=cm.gray,
vmin=-v,
vmax=v,
interpolation='None',
figure=self.fig)
sig, ra, dec, rad = \
np.loadtxt(os.path.join(clusterdir, field, 'cluster_out',
'new', field + '_3sigma_clusters.txt'),
dtype = float, skiprows = 1, unpack = True)
i = 0
while sig[i] >= 4.5:
# self.circleCluster([ra[i], dec[i]], rad[i])
if sig[i] > 5:
color = 'red'
else:
color = 'blue'
self.addClusterToContour([ra[i], dec[i]], rad[i], color)
i += 1
# self.fig.set_size_inches(9, 9)
# pl.contour(self.contour, colors = 'red', levels = [0], linewidth = .001, aa = False)
# hot = np.argwhere(self.contour)
# for points in hot:
# pl.plot(points[1], points[0], 'r.', markersize = .1, mew = 0, mec = 'r', mfc = 'r', aa = True)
if maskfile is not None:
mask = masks.srcMaskFromfile(maskfile, self.reso, self.map_shape,
self.proj, self.radec0)
pl.contour(mask, levels=[.9], colors='yellow', linewidth=.15)
pl.savefig(os.path.join('/home/ndhuang/plots/clusters/',
plotname + '.png'),
bbox_inches='tight',
pad_inches=0,
dpi=1000)
print i
import cPickle as pickle
import IPython
import glob
from sptpol_software.util import files
import numpy as np
mask = files.read(
'/data39/ndhuang/clusters/ra23h30dec-55/run2/bundles1/150/mask.fits')
mask = mask.masks.apod_mask
mask = mask[348:3748, 348:3748]
bundle_dir = '/data39/ndhuang/clusters/ra23h30dec-55/run1/bundles/abbys_bundles/bundle*.fits'
# bundle_dir = 'bundles/150/bundle*.fits'
bundles = glob.glob(bundle_dir)
noise = np.zeros(50)
weight = np.zeros(50)
bundle_pairs = ['' for b in noise]
i = 0
while len(bundles) > 0:
bf1 = bundles.pop(np.random.randint(0, len(bundles)))
bf2 = bundles.pop(np.random.randint(0, len(bundles)))
bundle_pairs[i] = bf1 + ', ' + bf2
b1 = files.read(bf1)
b2 = files.read(bf2)
diff = (b1.map.map - b2.map.map) * mask
good = np.nonzero(mask)
noise[i] = np.std(diff[good])
weight[i] = np.std((b1.weight.weight + b1.weight.weight)[good])
print bundle_pairs[i], noise[i], weight[i]
i += 1
f = open('bundle_pairs.pkl', 'w')
import numpy as np
from sptpol_software.observation.receiver import Receiver
from sptpol_software.util import files
boards = ['120', '122', '144', '150']
rec = Receiver()
bolo_inds = np.zeros(24 * len(boards) * 2, dtype=int)
print np.shape(bolo_inds)
i = 0
for b in boards:
board = rec.devices.mux_boards[b]
for squid in board.getSQUIDs():
squid = board[squid]
for bolo in squid.getBolosOrderedByFrequency():
bolo_inds[i] = bolo.index
i += 1
bolo_inds = bolo_inds[np.nonzero(bolo_inds)[0]]
source_dir = '/data/sptdat/auxdata/source/'
pre_source = 'source_rcw38_20131111_011216.fits'
post_source = 'source_rcw38_20131213_102250.fits'
pre = files.read(source_dir + pre_source)
post = files.read(source_dir + post_source)
deltax = pre.observation.bolo_x_offset - post.observation.bolo_x_offset
deltay = pre.observation.bolo_y_offset - post.observation.bolo_y_offset
parser = argparse.ArgumentParser()
parser.add_argument('psddir', type=str)
parser.add_argument('--band', default=None, type=int)
parser.add_argument('--plot-dir', default='', dest='plotdir')
parser.add_argument('--tex-file', default=None, type=str)
args = parser.parse_args()
if args.band is None:
args.band = [90, 150]
else:
args.band = [args.band]
if args.tex_file is not None:
texfile = open(args.tex_file, 'a')
texfile.write('\\clearpage\n')
else:
texfile = None
for band in args.band:
band_name = '%03dghz' % band
psd = files.read(os.path.join(args.psddir, band_name + '_psd.fits'))
ell = np.fft.fftfreq(np.shape(psd.psd.psd)[0], np.pi /
(4 * 60 * 180)) * 2 * np.pi
psd = properPSD(psd.psd.psd, ell)
tex_2d = plot2d(psd, ell, band, args.plotdir)
tex_1d = plot1d(psd, ell, band, args.plotdir)
if texfile is not None:
texfile.write(tex_2d + '\n')
texfile.write(tex_1d + '\n')
if texfile is not None:
texfile.close()
def consolidate_source_scan(start_date=None, stop_date=None, fitsdir='/data/sptdat/auxdata/', \
sources_to_use=['rcw38','mat5a'], filename=None, savename=None, doall=True, dosave=True):
"""
Translated from RK's original consolidate_source_scan.pro.
The purpose of this program is to consolidate the information contained in the
source_scan fits files which is relevant to pointing. The relevant info is stuffed
into a dictionary and saved as a pkl file.
===Things that need to be saved in this pickle file===
XDEG, YDEG, AMP, DC, ID
STARTTIME,STARTTIME_MJD, STOPTIME, STOPTIME_MJD, MEAN_MJD
SOURCE
FILENAME
TEMP_AVG, WIND_SPEED_AVG, WID_DIR_AVG, PRESSURE_AVG, TIPPER_TAU_MEAN
AZ_ACTUAL, EL_ACTUAL
- MEAN_AZ, MEAN_EL
WNOISE, ELNOD_RESPONSE, ELNOD_SIGMA, CAL_RESPONSE, CAL_SIGMA
LINEAR_SENSOR data (DET, DEL, DAZ, L1, L2, R1, R2)
NSCANS, NFRAMES, NSAMPLES
STRUCTURE THERMOMETRY
===
Originally created by RK, 24 Sep 2008 in IDL.
Translated to python by JWH, Oct 2012.
"""
nbolo = 1599
# Grab the filenames. Convert name strings to dates, and sort the filenames by date.
filelist = np.array(tools.flatten([[glob(os.path.join(fitsdir, 'source/*'+srcname+'*.fits')) for srcname in sources_to_use], \
[glob(os.path.join(fitsdir, 'source_vfast/*'+srcname+'*.fits')) for srcname in sources_to_use]]))
filelist_times = time.filenameToTime(filelist)
time_argsort = np.argsort(filelist_times)
filelist = filelist[time_argsort]
filelist_times = filelist_times[time_argsort]
if start_date == None:
start_date = filelist_times[0]
else:
start_date = time.SptDatetime(start_date)
if stop_date == None:
stop_date = filelist_times[-1]
else:
stop_date = time.SptDatetime(stop_date)
#Figure out which filenames to read given the start and stop dates.
times_to_use = np.array(map(lambda x: start_date <= x <= stop_date, filelist_times))
print 'Consolidating source scans between ', start_date, ' and ', stop_date, '.'
if not times_to_use.any():
print 'There are no source scans in that date range...'
return
else:
print 'There are %d source observations in that date range.' % np.sum(times_to_use)
filelist = filelist[times_to_use]
nlist = len(filelist)
#Record the system time
clocka = comptime()
#Restore the old dictionary if you gave filename to the function.
if filename == None:
filename = '/home/jhenning/sptpol_code/sptpol_software/analysis/pointing/source_scan_structure.pkl'
if savename == None:
savename = filename
if doall == False:
sold = pk.load(open(filename, 'r'))
#Create an empty list of dictionaries to fill with the data that we need.
s = []
for i in range(nlist):
s.append({'xdeg':np.zeros(nbolo), 'ydeg':np.zeros(nbolo),
'amp':np.zeros(nbolo), 'dc':np.zeros(nbolo), 'id':np.zeros(nbolo, dtype=np.int16),
'starttime':'', 'starttime_mjd':0.0,
'stoptime':'', 'stoptime_mjd':0.0, 'mean_mjd':0.0,
'source':'', 'filename':'',
'temp_avg':0.0, 'wind_speed_avg':0.0, 'wind_dir_avg':0.0, 'pressure_avg':0.0,
'tipper_tau_mean':0.0,
'az_actual':np.zeros(nbolo), 'el_actual':np.zeros(nbolo),
'mean_az':0.0, 'mean_el':0.0,
'focus_position':np.zeros(6),
'wnoise':np.zeros(nbolo), 'elnod_response':np.zeros(nbolo), 'elnod_sigma':np.zeros(nbolo),
'cal_response':np.zeros(nbolo), 'cal_sigma':np.zeros(nbolo),
'med_daz':0.0, 'med_del':0.0, 'med_det':0.0,
'med_l1':0.0, 'med_l2':0.0, 'med_r1':0.0, 'med_r2':0.0,
'nscans':0.0, 'nframes':0.0, 'nsamples':0.0,
'med_scu_temp':np.zeros(60), 'mean_scu_temp':np.zeros(60)})
#Now loop over each source scan and grab each of these data.
nnew=0
for i in range(nlist):
timeb = comptime()
print str(i), '/', str(nlist-1)
print '...checking ', filelist[i]
#Check to see if the old dictionary list has this file's information.
if doall == True: pass
else:
wh_already = np.where(filelist[i] == np.array([item['filename'] for item in sold]))[0]
if len(wh_already) > 1:
print 'This file is in the source scan summary more than once!... '
print 'Pausing so you can do something about it.'
raw_input('Press ENTER to continue...')
if len(wh_already)==1:
print 'This file has already been incorporated into the source scan summary... '
print 'Packing old results and moving to next source scan.'
#Stuff the new dictionary with the old information.
s[i] = sold[wh_already[0]]
continue
#Okay, let's start grabbing information from new source scans.
nnew += 1
data = 0
print '...reading FITS file...'
print '...loading ', filelist[i]
data = files.read(filelist[i])
#nframes = data.observation.n_frames
#nscans = data.observation.n_scans
#nsamples = data.observation.n_samples
#For now, load in the data from the archive for the observation time to grab
#raw pointing information etc. Eventually, these should be saved with the source fits.
#extra = SPTDataReader(start_date=time.SptDatetime(data.header.start_date), \
# stop_date = time.SptDatetime(data.header.stop_date))
#extra.readData(verbose=False, timestream_units='watts', correct_global_pointing=False)
#Double-check that the data has as observation structure.
if hasattr(data, 'observation') == False:
#print " Missing an observation structure! Skipping observation %s." % data.header.start_date.arc
print " Missing an observation structure! Skipping observation %s." % data.from_filename
nnew -= 1
continue
#Fill in auxiliary data for this scan.
#Creat an SPTDataReader object to get ancillary information about the observation.
da = SPTDataReader(start_date=data.header.start_date,
stop_date=data.header.stop_date, quiet=True)
data.auxdata_keys = da.auxdata_keys
data.rec = da.rec
data.cuts = da.cuts
data.telescope = da.telescope
data.config = da.config
try:
#Let's not read in noise data.
#files.fillAuxiliaryData(data, obs_types=['calibrator','elnod','noise'],
files.fillAuxiliaryData(data, obs_types=['calibrator','elnod'],
same_fridge_cycle=True, overwrite=False)
except Exception, err:
print " Couldn't copy in auxiliary data: %s" % str(err)
nnew -= 1
continue
#Finally, let's start pulling out some of the values we're looking for.
xdeg = data.observation.bolo_x_offset
ydeg = data.observation.bolo_y_offset
bolo_id = data.observation.bolo_id_index_in_arc_file
starttime = data.observation.start_time
stoptime = data.observation.stop_time
starttime_mjd = time.SptDatetime(starttime).mjd
stoptime_mjd = time.SptDatetime(stoptime).mjd
mean_mjd = np.mean([starttime_mjd, stoptime_mjd])
#Is this a valid source scan?
wh = np.where(xdeg == xdeg)
nwh = len(wh)
if nwh == 0:
print " Not a valid source scan. Skipping observation %s." % data.header.start_date.arc
nnew -= 1
continue
#Make sure calibrator and noise auxiliary info is present.
#Be sure to skip nan's otherwise np.std() will return nan no matter how many
#good bolo data there are.
try:
#if ( np.std(data.observation.bolo_cal_response[np.isfinite(data.observation.bolo_cal_response)]) == 0. or
# np.std(data.observation.bolo_psd_white_noise_level[np.isfinite(data.observation.bolo_psd_white_noise_level)] == 0.0)):
if ( np.std(data.observation.bolo_cal_response[np.isfinite(data.observation.bolo_cal_response)]) == 0.):
#print " Missing calibrator or noise data. Skipping this observation."
print " Missing calibrator data. Not worrying about noise... Skipping this observation."
nnew -= 1
continue
except AttributeError, err:
# If we couldn't get calibrator and/or noise data, skip this observation.
print err
nnew -= 1
continue
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('files', nargs = '+', type = str)
parser.add_argument('--field', type = str, default = None)
parser.add_argument('--coadd', action = 'store_true',
help = "Coadd maps with good pointing")
args = parser.parse_args()
if args.field is not None:
import field_centers
radec0_nom = field_centers.centers[args.field]
pting = [[] for f in args.files]
times = [[] for f in args.files]
for i, f in enumerate(args.files):
pting[i] = files.read(f)
times[i] = fnameToTime(f)
times = np.asarray(times)
mddec = grabField(pting, 'mean_ddec')
mdra = grabField(pting, 'mean_dra')
sddec = grabField(pting, 'sigma_ddec')
sdra = grabField(pting, 'sigma_dra')
radec0 = grabField(pting, 'radec0_corr')
to_arr = lambda x: np.asarray([x]).flatten()
dra = grabField(pting, 'dra', converter = to_arr)
nsrc = np.asarray([len(r) for r in dra])
if args.coadd:
good = np.where((nsrc > 1) & (abs(sdra) < .1))[0]
coadd = getMap(args.files[good[0]])
y0, x0 = ang2Pix(np.transpose(radec0), coadd.center, .25, coadd.shape,
import glob, sys
from os import path
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as pl
from sptpol_software.util import files
#map_dir = '/data/sptdat/auxdata/map/autoprocessed_maps/ra3hdec-25/coadds/left/'
plot_dir = '/data/ndhuang/plots/ra3hdec-25_monitoring'
if __name__ == '__main__':
map_dir = sys.argv[1]
mapfiles = glob.glob(path.join(map_dir, '*.h5'))
for mf in mapfiles:
m = files.read(mf)
m.removeWeight()
f = pl.figure()
m.plot('T', bw=True, vmin=-2e-3, vmax=2e-3, figure=1)
name = path.basename(mf).split('.')[0]
f.set_size_inches(18, 12.15)
pl.savefig(path.join(plot_dir, name + '.png'))
pl.close('all')
bundles = [x for x in sorted(gl('/data57/jhenning/ra0hdec-57p5_V3/20160518/bundles/150/lowell_ratio_1p0/all/*.h5')) if '.ffts.' not in x]
pos = bundles[::2]
neg = bundles[1::2]
D = files.read('/data57/jhenning/ra0hdec-57p5_V3/20160518/bundles/150/lowell_ratio_1p0/all/coadd_halfASubhalfB.h5').removeWeight().flattenPol()
MAN = maps.MapAnalyzer(delta_ell=delta_ell,set_special_bb_bins=False)
mask = files.read('/data57/jhenning/ra0hdec-57p5_V3/20160518/bundles/150/lowell_ratio_1p0/all/total_mask_apod15.0arcmin_ptsrc10.0arcmin.pkl')
fft_shape = np.array([int(2*np.pi/(D['T'].reso_rad *delta_ell))] * 2)
noise_Cls = MAN.calculateCls(D, sky_window_func=mask, maps_to_cross=['T','E','B'], fft_shape=fft_shape, calculate_dls=False)
"""
#noise_cls_file = 'data/SPTpol_maps/noise_cls.pkl.gz_v_JT_20161004'
noise_cls_file = 'data/SPTpol_maps/noise_cls.pkl.gz_v_JT_20161005'
if not local:
folderpath = '/data57/jhenning/ra0hdec-57p5_V3/20160518/bundles/150/lowell_ratio_1p0/all'
D = files.read('%s/coadd_halfASubhalfB.h5' %(folderpath)).removeWeight().flattenPol()
delta_ell=5.
MAN = maps.MapAnalyzer(delta_ell=delta_ell,set_special_bb_bins=False)
mask = files.read('%s/total_mask_apod15.0arcmin_ptsrc10.0arcmin.pkl' %(folderpath))
fft_shape = np.array([int(2*np.pi/(D['T'].reso_rad *delta_ell))] * 2)
#noise_Cls = MAN.calculateCls(D, sky_window_func=mask, maps_to_cross=['T','E','B'], fft_shape=fft_shape, calculate_dls=False)
noise_Cls = MAN.calculateCls(D, sky_window_func=mask, maps_to_cross=['T','Q','U'], fft_shape=fft_shape, calculate_dls=False)
#get ells
binned_ells = noise_Cls.ell.TT
#ell_bins = noise_Cls.ell_bins.TT
ells_equal = np.allclose(noise_Cls.ell.TT, noise_Cls.ell.QQ) #assert statement
if not ells_equal:
print 'TT and QQ ells are different'
