def __init__(self, filename):
"""Initializes a config file if does not exist. If exists, uses
it to validate the file, and setup default initial parameters"""
self.cfg_spec = ConfigObj(config_spec_text.splitlines(),
list_values=False)
self.cfg_filename = filename
valid = Validator()
if not os.path.exists(self.cfg_filename):
#no .dreampyrc file found
cfg = ConfigObj(configspec=self.cfg_spec,
stringify=True,
list_values=True)
cfg.filename = self.cfg_filename
test = cfg.validate(valid, copy=True)
cfg.write()
self.cfg = ConfigObj(self.cfg_filename, configspec=self.cfg_spec)
rtn = self.cfg.validate(valid, preserve_errors=True)
if isinstance(rtn, bool) and rtn:
logger.info("Config file validated")
self.tested = True
else:
self.tested = False
res = flatten_errors(self.cfg, rtn)
self.errortxt = ''
for row in res:
self.errortxt += 'In Section %s, key %s has error: %s' % (
row[0], row[1], row[2])
logger.error(self.errortxt)
def process_diagnostic_calibration(self):
#14 subscans
logger.info('Processing diagnostic calibration')
hot_ratio = self._get_ratio(scans=(2, 3, 4, 5))
hotacf = self._get_accdata(self._get_filename_for_scan(1))
sky_ratio = self._get_ratio(scans=(7, 8, 9, 10))
skyacf = self._get_accdata(self._get_filename_for_scan(6))
self.hotspec = numpy.ma.zeros(hotacf.shape, dtype='float')
self.skyspec = numpy.ma.zeros(hotacf.shape, dtype='float')
self.Tsys = numpy.ma.zeros(hotacf.shape, dtype='float')
for board in self.header.BoardNumber:
badlags = get_bad_lags(self.chassis, board)
self.hotspec[board, :] = makespectrum(
hotacf[board, :],
ratio=hot_ratio[board, :],
badlags=badlags,
corr_cal=self.ccal[self.header.SlotNumber[board]])
self.skyspec[board, :] = makespectrum(
skyacf[board, :],
ratio=sky_ratio[board, :],
badlags=badlags,
corr_cal=self.ccal[self.header.SlotNumber[board]])
self.Tsys[board, :] = self.Tamb * self.skyspec[board, :] / (
self.hotspec[board, :] - self.skyspec[board, :])
if self.corr_linear:
scale = self.get_scaling(board)
self.Tsys[board, :] = self.Tsys[board, :] * scale
self.g_norm = self._get_gnorm(scans=(11, 12, 13, 14))
def first_time_setup():
"""
Check to see if the user has setup their
environment correctly
"""
#dreamdata = '/usr/share/dreampy'
#if os.environ.has_key("DREAMDATA"):
# if os.path.exists(os.environ["DREAMDATA"]):
# dreamdata = os.environ["DREAMDATA"]
# set up user space
userdir = os.environ["HOME"] + os.path.sep + ".dreampy"
if not os.path.exists(userdir):
print('First time DREAMPY use. Setting up ~/.dreampy')
os.mkdir(userdir)
#shutil.copyfile(dreamdata+"/data/ipythonrc-dreampy", userdir+"/ipythonrc-dreampy")
#for fname in ('ipythonrc-dreampy', 'ipy_user_conf.py'):
# shutil.copyfile(os.path.join(dreamdata, fname),
# os.path.join(userdir, fname))
f = open(userdir + "/ipythonrc", "w")
f.close()
else:
if not os.path.exists(os.path.join(userdir, 'upgraded')):
logger.info("Upgrading ipy_user_conf.py")
#shutil.copyfile(os.path.join(dreamdata, 'ipy_user_conf.py'),
# os.path.join(userdir, 'ipy_user_conf.py'))
f = open(os.path.join(userdir, 'upgraded'), "w")
f.close()
def get_configobj():
"""Return the default configuration object in the rc file"""
fname = _dreampy_fname()
if fname is None or not os.path.exists(fname):
logger.info("could not find rc file; creating file with defaults")
home = os.environ['HOME']
fname = os.path.join(home, '.dreampy', 'dreampyrc')
return Configuration(fname)
def bad_lags_acf(acf, chassis, board):
badlags = get_bad_lags(chassis, board)
acf = acf.copy()
if badlags:
logger.info('Setting bad lags for chassis %d board %d to %s' %
(chassis, board, badlags))
for blag in badlags:
acf[blag] = numpy.nan
return acf
def average_all_hdus(self, weight='sigma', threshold_sigma=None):
"""Given a list of hdus in a netCDF4 file this method
will average all the scans and returns a representative group
and new hdu with the average of all hdus"""
avghdu = RedshiftScan(data=self.hdu.data,
header=self.hdu.header,
filename=self.filename)
avggroup = self.groups[self.groups.keys()[0]]
firsthdu = self._get_group_hdu(self.groups.keys()[0])
frequencies = firsthdu._get_frequencies()
spec = firsthdu._get_spectrum()
avgspectrum = numpy.zeros((1, spec.shape[1], spec.shape[2]),
dtype=spec.dtype)
weights = {}
tsys = {}
tint = 0.0
for board in range(frequencies.shape[0]):
weights[board] = 0.0
tsys[board] = 0.0
for gname in self.groups.keys():
hdu = self._get_group_hdu(gname)
spec = hdu._get_spectrum()
htsys = hdu._get_tsys()
logger.info("Processing hdu in %s" % gname)
count = 0
for board in range(frequencies.shape[0]):
wt = hdu._calc_weight(0, board, weight=weight)
if weight == 'sigma' and threshold_sigma is not None:
#print gname, board, wt, threshold_sigma
if 1 / numpy.sqrt(wt) < threshold_sigma:
avgspectrum[0, board, :] += spec[0, board, :] * wt
tsys[board] += htsys[board] * wt
weights[board] += wt
count += 1
else:
logger.info(
"Rejecting Board %d group %s due to sigma threshold"
% (board, gname))
else:
avgspectrum[0, board, :] += spec[0, board, :] * wt
tsys[board] += htsys[board] * wt
weights[board] += wt
count += 1
if count != 0:
tint += hdu._get_tint()
for board in range(frequencies.shape[0]):
avgspectrum[0, board, :] /= weights[board]
tsys[board] /= weights[board]
firsthdu.spectrum = avgspectrum
firsthdu.frequencies = frequencies
firsthdu.tsys = tsys
firsthdu.tint = tint
return avggroup, firsthdu
def process_cross_scan(
self,
stepsize=1.0, # in arcseconds
apply_Tsys=False,
Tsys=[],
baseline=True,
):
"""
Cross scan
If apply_Tsys is True,
pass a list or dictionary for Tsys with
Tsys values.
If baseline is True, removes the off-source value
from the overall measurements
"""
if self.header.get('Dcs.ObsPgm') != 'CrossScan':
logger.error("Not a cross-scan datafile")
return
self.numpixels = self.data.BasebandLevel.shape[1]
logger.info(
"Processing IFProc Continuum CrossScan Observation with ObsNum %d"
% int(self.header.get('Dcs.ObsNum')))
ind = numpy.where(self.data.BufPos == 0)
self.off_source = {}
for pixel in range(self.numpixels):
self.off_source[pixel] = numpy.histogram(
self.data.BasebandLevel[ind, pixel].flatten())[1][:4].mean()
if self.header.get('CrossScan.MapCoord') == 'Az':
xpos = numpy.degrees(self.data.TelAzMap[ind]) * 3600.
else:
xpos = numpy.degrees(self.data.TelElMap[ind]) * 3600.
mapsize = 2 * int(
((xpos[-1] - xpos[0]) * 0.95) / 2.) # slightly smaller
numsteps = int(mapsize / stepsize + 1)
self.grid = numpy.linspace(-mapsize / 2, mapsize / 2, numsteps)
self.continuum = numpy.zeros((self.grid.size, self.numpixels),
dtype=self.data.BasebandLevel.dtype)
cont = {}
if baseline:
for pixel in range(self.numpixels):
cont[pixel] = (self.data.BasebandLevel[ind, pixel] -
self.off_source[pixel]).flatten()
else:
for pixel in range(self.numpixels):
cont[pixel] = self.data.BasebandLevel[ind, pixel].flatten()
for pixel in range(self.numpixels):
f = interp1d(xpos, cont[pixel])
self.continuum[:, pixel] = f(self.grid)
return self.continuum
def save_hdu(self, group, hdu, groupname=None):
"""Saves a group and associated hdu in the opened
netcdf file"""
if groupname is None:
groupname = "%s_%d_%d_%d_C%d" % (
hdu.header.utdate().strftime('%Y-%m-%d'),
int(hdu.header.ObsNum),
int(hdu.header.SubObsNum),
int(hdu.header.ScanNum),
int(hdu.header.ChassisNumber),
)
logger.info("Creating Groupname %s" % groupname)
outgroup = self.createGroup(groupname)
self.copy_dimensions(group, hdu, outgroup)
self.copy_header(group, hdu, outgroup)
self.copy_data(group, hdu, outgroup)
outgroup.sync()
self.sync()
def _get_single_file_ratio(self, nc, scans=(2, 3, 4, 5)):
W = self._get_single_file_accdata(nc, scans[0])
X = self._get_single_file_accdata(nc, scans[1])
Y = self._get_single_file_accdata(nc, scans[2])
Z = self._get_single_file_accdata(nc, scans[3])
ratio = numpy.ma.zeros(W.shape, dtype='float')
for board in self.header.BoardNumber:
logger.info('Processing ratio for board %d' % board)
ratio[board, :] = ratio_blank(bad_lags_acf(W[board, :],
self.chassis, board),
bad_lags_acf(X[board, :],
self.chassis, board),
bad_lags_acf(Y[board, :],
self.chassis, board),
bad_lags_acf(Z[board, :],
self.chassis, board),
scale_factor=0.65e6)
return ratio
def process_astronomical_calibration(self, nc):
#6 subscans
if self.SingleFileCal:
logger.info(
'Processing astronomical calibration using SingleFileCal')
else:
logger.info('Processing astronomical calibration using 6 files')
if self.SingleFileCal:
hotacf = self._get_single_file_accdata(nc, 0)
skyacf = self._get_single_file_accdata(nc, 1)
else:
hotacf = self._get_accdata(self._get_filename_for_scan(1))
skyacf = self._get_accdata(self._get_filename_for_scan(2))
#print(type(hotacf))
#print(hotacf)
self.hotspec = numpy.ma.zeros(hotacf.shape, dtype='float')
self.skyspec = numpy.ma.zeros(hotacf.shape, dtype='float')
self.Tsys = numpy.ma.zeros(hotacf.shape, dtype='float')
for board in self.header.BoardNumber:
badlags = get_bad_lags(self.chassis, board)
self.hotspec[board, :] = makespectrum(
hotacf[board, :],
badlags=badlags,
corr_cal=self.ccal[self.header.SlotNumber[board]])
#print(self.hotspec[board, :])
self.skyspec[board, :] = makespectrum(
skyacf[board, :],
badlags=badlags,
corr_cal=self.ccal[self.header.SlotNumber[board]])
self.Tsys[board, :] = self.Tamb * self.skyspec[board, :] / (
self.hotspec[board, :] - self.skyspec[board, :])
#print(self.Tsys[board, :])
#print(self.Tsys[board, :].shape)
if self.corr_linear:
scale = self.get_scaling(board)
self.Tsys[board, :] = self.Tsys[board, :] * scale
if self.SingleFileCal:
self.skyratio = self._get_single_file_ratio(nc, (2, 3, 4, 5))
self.g_norm = self._get_single_file_gnorm(nc, (2, 3, 4, 5))
nc.nc.close()
else:
self.skyratio = self._get_ratio(scans=(3, 4, 5, 6))
self.g_norm = self._get_gnorm(scans=(3, 4, 5, 6))
def set_bad_lags():
"""
Interactive set_bad_lags for redshift boards
"""
chassis = int(input("Enter Chassis number (one of 0,1,2 or 3) > "))
board = int(input("Enter Board number (one of 0,1,2,3,4 or 5) > "))
print("Current bad lags for chassis %d and board %d : %s" %
(chassis, board, get_bad_lags(chassis, board)))
blags_str = input(
"Enter bad lags for Chassis %d, board %d in comma-separated form > " %
(chassis, board))
if blags_str:
blags = list(map(int, blags_str.split(',')))
else:
blags = ''
_set_bad_lags(chassis, board, blags)
logger.info(
"Bad Lags for chassis %d board %d set to %s" %
(chassis, board,
dreampyParams['redshiftchassis']['bad_lags%d' % chassis][board]))
def append_observation(self, ncout, groupname=None):
"""This method will append the current netCDF3 based single
observation to the output ncout netCDF4 file.
It is assumed that ncout is an open writeable netCDF4 file.
A new rootgroup will be created in ncout for each netCDF3
input observation"""
if groupname is None:
groupname = "%s_%d_%d_%d_C%d" % (
self.hdu.header.utdate().strftime('%Y-%m-%d'),
int(self.hdu.header.ObsNum),
int(self.hdu.header.SubObsNum),
int(self.hdu.header.ScanNum),
int(self.hdu.header.ChassisNumber),
)
logger.info("Creating Groupname %s" % groupname)
group = ncout.createGroup(groupname)
self.copy_dimensions(self, self.hdu, group)
self.copy_header(self, self.hdu, group)
self.copy_data(self, self.hdu, group)
group.sync()
ncout.sync()
def corr_cal_setup():
# Allow user defined data location
dreamdata = '/usr/share/dreampy'
default_corr_cal_dir = '/raw/rsr/cal'
#if os.environ.has_key('CORR_CAL_DIR'):
if 'CORR_CAL_DIR' in os.environ:
corr_cal_dir = os.environ['CORR_CAL_DIR']
if not os.path.exists(corr_cal_dir):
logger.info(
"corr_cal_dir %s does not exist. Setting to current directory %s. Please check environment variable CORR_CAL_DIR"
% (corr_cal_dir, os.curdir))
if os.path.exists(default_corr_cal_dir):
corr_cal_dir = default_corr_cal_dir
else:
corr_cal_dir = os.curdir
else:
logger.info("Setting corr_cal_dir to directory %s" % corr_cal_dir)
else:
if os.path.exists(default_corr_cal_dir):
corr_cal_dir = default_corr_cal_dir
else:
corr_cal_dir = os.curdir
logger.info("Setting corr_cal_dir to current directory %s" %
corr_cal_dir)
# remove from namespace
#if dreamdata:
# del dreamdata
#del userdir, shutil, platform
return corr_cal_dir
def _get_single_file_gnorm(self, nc, scans=(2, 3, 4, 5)):
W = self._get_single_file_accdata(nc, scans[0])
X = self._get_single_file_accdata(nc, scans[1])
Y = self._get_single_file_accdata(nc, scans[2])
Z = self._get_single_file_accdata(nc, scans[3])
g_norm = numpy.ma.zeros(W.shape, dtype='float')
for board in self.header.BoardNumber:
logger.info('Processing g_norm for board %d' % board)
g_norm[board, :] = get_gnorm(bad_lags_acf(W[board, :],
self.chassis, board),
bad_lags_acf(X[board, :],
self.chassis, board),
bad_lags_acf(Y[board, :],
self.chassis, board),
bad_lags_acf(Z[board, :],
self.chassis, board),
scale_factor=1.3e6)
# hack to fix faulty board 1 chassis 0 blanking
if int(self.header.ChassisNumber) == 0:
if self.header.utdate().date() > datetime.date(2015, 12, 1):
g_norm[1, :] = g_norm[2, :]
return g_norm
def savefig(self,
filename,
dpi=None,
facecolor='w',
edgecolor='w',
orientation='portrait',
format='png',
transparent=False,
bbox_inches=None,
pad_inches=0.1):
"""
saves the figure into a file
"""
logger.info("Saving figure to file %s" % filename)
self.plotobj.savefig(filename,
dpi=dpi,
facecolor=facecolor,
edgecolor=edgecolor,
orientation=orientation,
format=format,
transparent=transparent,
bbox_inches=bbox_inches,
pad_inches=pad_inches)
def average_all_hdus_compspectrum(self,
weight='sigma',
threshold_sigma=None):
"""Given a list of hdus in a netCDF4 file this method
will average all the compspectrum scans and returns a representative group
and new hdu with the average of all hdus"""
avghdu = RedshiftScan(data=self.hdu.data,
header=self.hdu.header,
filename=self.filename)
avggroup = self.groups[self.groups.keys()[0]]
firsthdu = self._get_group_hdu(self.groups.keys()[0])
compfreq = firsthdu._get_comp_freq()
spec = firsthdu._get_comp_spectrum()
avgspectrum = numpy.zeros((1, spec.shape[1]), dtype=spec.dtype)
weights = 0.0
for gname in self.groups.keys():
logger.info("Processing hdu in %s" % gname)
hdu = self._get_group_hdu(gname)
spec = hdu._get_comp_spectrum()
wt = hdu._calc_weight_comp(0, weight=weight)
if weight == 'sigma' and threshold_sigma is not None:
print(gname, wt, threshold_sigma)
if 1 / numpy.sqrt(wt) < threshold_sigma:
avgspectrum[0, :] += spec[0, :] * wt
weights += wt
else:
logger.info("Rejecting group %s due to sigma threshold" %
gname)
else:
avgspectrum[0, :] += spec[0, :] * wt
weights += wt
avgspectrum[0, :] /= weights
firsthdu.compspectrum = avgspectrum
firsthdu.compfreq = compfreq
return avggroup, firsthdu
def _save_defaults():
logger.info('Saving dreampy defaults')
if cfgobj:
cfgobj.save_config(dreampyParams)
def get_corr_cal_matrix(header, slotno=None):
"""Given a netCDF based header value, uses the UTDate
in the header to determine from the corr_cal database which
corr_cal file to read, and reads it and returns the corr_cal
matrix.
If slotno is given it can either be a single integer or a list of
slotnumbers. Returned value is a dictionary of corr_cal values
with slotnumbers as keys. If slotno is None, all slots in the header
are returned.
"""
#from dreampy.redshift.netcdf.redshift_corrcal_netcdf_file import RedshiftCorrCalFile
if slotno is None:
slotno = header.SlotNumber
elif type(slotno) == types.IntType:
slotno = [slotno]
elif type(slotno) in (numpy.ndarray, types.ListType, type.TupleType):
slotno = slotno
else:
#something wrong
slotno = header.SlotNumber
chassis = int(header.ChassisNumber)
ccaldic = {}
corr_cal_dir = os.environ.get('CORR_CAL_DIR', '/raw/rsr/cal')
for slot in slotno:
try:
ccal = CorrCalTable.objects.filter(
chassis=chassis, slotno=slot,
obsdate__lt=header.utdate()).latest('obsdate')
ccal_filename = ccal.filename
except:
#cannot reach the internet, or the database
glb = glob.glob(
os.path.join(
corr_cal_dir,
'corr_cal_%s_%s.nc' % (int(header.CorrCalID), chassis)))
if glb:
ccal_filename = os.path.basename(glb[0])
else:
raise LMTRedshiftError(
"get_corr_cal_matrix",
"Some error in getting corr_cal table information")
fname = os.path.join(corr_cal_dir, os.path.basename(ccal_filename))
logger.info("Corr Cal file for Slot no %d is %s" % (slot, fname))
if fname in corr_cal and slot in corr_cal[fname]:
ccaldic[slot] = corr_cal[fname][slot]
logger.info(
"Re-reading Corr Cal from memory for Slot no %d for %s" %
(slot, fname))
else:
if not fname in corr_cal:
corr_cal[fname] = {}
if not slot in corr_cal[fname]:
if os.path.exists(fname):
logger.info(
"Reading first corr cal for slot no %d for %s" %
(slot, fname))
ccalnc = RedshiftCorrCalFile(fname)
idx = int(numpy.where(ccalnc.hdu.header.slotno == slot)[0])
ccaldic[slot] = ccalnc.hdu.data.corr_cal[idx, :, :]
corr_cal[fname][slot] = ccalnc.hdu.data.corr_cal[idx, :, :]
else:
raise LMTRedshiftError(
"get_corr_cal_matrix",
"Corr cal file %s does not exist" % fname)
return ccaldic
def process_map(self,
remove_offset=False,
numoff=20,
scigrid=False,
**kwargs):
"""
processes Map Obspgm that has been made using compressed
continuum mode. Uses a regridding algorithm
and uses some kwargs arguments to derive output
grid size and sampling
"""
if self.header.ObsPgm not in ('Map', 'Lissajous'):
logger.error("Not a Map datafile")
return
else:
maptype = self.header.ObsPgm
logger.info(
"Processing MSIP 1mm Continuum Map data and regridding for Observation with ObsNum %d"
% int(self.header.ObsNum))
self.numpixels = self.data.BasebandLevel.shape[1]
xlength = numpy.degrees(self.header.get(
'%s.XLength' % maptype)) * 3600.0
ylength = numpy.degrees(self.header.get(
'%s.YLength' % maptype)) * 3600.0
if maptype == 'Lissajous':
xlength = xlength / numpy.cos(numpy.radians(45))
xlength = ylength / numpy.cos(numpy.radians(45))
ind = numpy.where(self.data.BufPos == 0)
xpos = numpy.degrees(self.data.TelAzMap[ind]) * 3600.
ypos = numpy.degrees(self.data.TelElMap[ind]) * 3600.
rows = self.header.get('Map.RowsPerScan')
z = {}
self.off_source = {}
for chan in range(self.numpixels):
z[chan] = self.data.BasebandLevel[ind, chan].flatten()
self.off_source[chan] = numpy.histogram(
self.data.BasebandLevel[ind, chan].flatten())[1][:4].mean()
if remove_offset:
z[chan] = z[chan] - self.off_source[chan]
print(z[chan].shape)
ramp = kwargs.get('ramp', 5.)
numpoints = kwargs.get('numpoints', 100)
numypoints = kwargs.get('numypoints', 100)
xlength = xlength * (1. - ramp / 100.)
ylength = ylength * (1. - ramp / 100.)
ind = numpy.logical_and(xpos > -xlength / 2., xpos < xlength / 2.)
xpos, ypos = xpos[ind], ypos[ind]
# add a tiny random number to stop griddata from crashing when two pixels are same
xpos = xpos + numpy.random.random(xpos.size) * 1e-6
ypos = ypos + numpy.random.random(ypos.size) * 1e-6
for chan in range(self.numpixels):
z[chan] = z[chan][ind]
ind = numpy.logical_and(ypos > -ylength / 2., ypos < ylength / 2.)
xpos, ypos = xpos[ind], ypos[ind]
for chan in range(self.numpixels):
z[chan] = z[chan][ind]
self.xi = numpy.linspace(-xlength / 2, xlength / 2, numpoints)
self.yi = numpy.linspace(-ylength / 2, ylength / 2, numypoints)
print("Making %d x %d map" % (numpoints, numypoints))
#self.z = z
#self.xpos = xpos
#self.ypos = ypos
self.BeamMap = numpy.zeros(
(self.yi.size, self.xi.size, self.numpixels))
for chan in range(self.numpixels):
#self.BeamMap[chan] = numpy.zeros((self.yi.size, self.xi.size),
# dtype='float')
if scigrid:
self.BeamMap[:, :, chan] = scipy.interpolate.griddata(
xpos, ypos, z[chan], (self.xi, self.yi), method='cubic')
else:
self.BeamMap[:, :, chan] = griddata(xpos, ypos, z[chan],
self.xi, self.yi)
def delete_event(self, widget, event, data=None):
#return True means will not generate destroy event next
#print "Type Ctrl-D or exit() to exit from main terminal window"
logger.info("Killing plot window")
self.alive = False
return False
def _save_defaults():
logger.info('Saving dreampy defaults in ~/.dreamy/') # @todo
if cfgobj:
cfgobj.save_config(dreampyParams)
def version():
logger.info("dreampy %s(%s)" % (__version__, __date__))
