def action(self, Data):
params = self.params
data = Data.data
n_pol = data.shape[1]
n_cal = data.shape[2]
n_chan = data.shape[3]
# Figure out the data base key for this file.
file_ind = self.file_ind
file_middle = self.params['file_middles'][file_ind]
key = ts_measure.get_key(file_middle)
# Get the eigenvectors.
# Shape is (n_bands, n_pols, n_cals, n_chan, n_chan).
evects = self.foreground_db[key + '.vects']
if evects.shape[1:] != (n_pol, n_cal, n_chan, n_chan):
raise ce.DataError("Data and foreground array shapes don't match.")
# Shape is (n_bands, n_chan).
freq = self.foreground_db[key + '.freq']
# Figure out which band the current data block is from.
Data.calc_freq()
this_freq = Data.freq
band_ind = -1
for ii in range(freq.shape[0]):
if np.allclose(this_freq, freq[ii, :]):
band_ind = ii
if band_ind == -1:
raise ce.DataError("Frequency axis does not match any IF.")
evects = evects[ii, ...]
# Subtract the time mean.
data[...] -= ma.mean(data, 0)
# Now loop through the polarizations and subtract out the appropiate
# foreground vectors.
for ii in range(n_pol):
for jj in range(n_cal):
for kk in range(params['n_modes_subtract']):
# Get the relevant data.
vector = evects[ii, jj, :, -1 - kk]
this_data = data[:, ii, jj, :]
this_mask = np.logical_not(ma.getmaskarray(this_data))
filled_data = this_data.filled(0)
# Calculate the amplitude of the foregrounds at each time,
# properly treating the mask.
amps = np.sum(filled_data * vector, -1)
norms = np.sum(this_mask * vector**2, -1)
bad_times = norms < 1.e-2
norms[bad_times] = 1.
amps[bad_times] = 0
amps /= norms
# Now subtract the forgrounds out.
this_data[...] -= amps[:, None] * vector
this_data[bad_times, :] = ma.masked
Data.add_history('Subtracted foreground modes.',
("Subtracted %d modes." % params['n_modes_subtract']))
return Data
def _add_single_block(self, Block) :
"""Adds all the data in a DataBlock Object to the Writer such that it
can be written to a fits file eventually."""
Block.verify()
# Merge the histories
if self.first_block_added :
self.history = db.History(Block.history)
else :
self.history = db.merge_histories(self.history, Block)
# Some dimensioning and such
dims = tuple(Block.dims)
n_records = dims[0]*dims[1]*dims[2]
block_shape = dims[0:-1]
# For now automatically determine the format for the data field.
data_format = str(dims[-1]) + 'E'
if self.first_block_added :
self.data_format = data_format
elif self.data_format != data_format :
raise ce.DataError('Data shape miss match: freq axis must be same'
' length for all DataBlocks added to Wirter.')
# Copy the reshaped data from the DataBlock
data = sp.array(ma.filled(Block.data, float('nan')))
if self.first_block_added :
self.data = data.reshape((n_records, dims[3]))
else :
self.data = sp.concatenate((self.data, data.reshape((
n_records, dims[3]))), axis=0)
# Now get all stored fields for writing out.
for field, axes in Block.field_axes.iteritems() :
# Need to expand the field data to the full ntimes x npol x ncal
# length (with lots of repitition). We will use np broadcasting.
broadcast_shape = [1,1,1]
for axis in axes :
axis_ind = list(Block.axes).index(axis)
broadcast_shape[axis_ind] = dims[axis_ind]
# Allowcate memory for the new full field.
data_type = Block.field[field].dtype
field_data = sp.empty(block_shape, dtype=data_type)
# Copy data with the entries, expanding dummy axes.
field_data[:,:,:] = sp.reshape(Block.field[field],
broadcast_shape)
if self.first_block_added :
self.field[field] = field_data.reshape(n_records)
self.formats[field] = Block.field_formats[field]
else :
self.field[field] = sp.concatenate((self.field[field],
field_data.reshape(n_records)), axis=0)
if self.formats[field] != Block.field_formats[field] :
raise ce.DataError('Format miss match in added data blocks'
' and field: ' + field)
self.first_block_added = False
def get_scan_IF_inds(self, scan_ind, IF_ind) :
"""Gets the record indices of the fits file that correspond to the
given scan and IF.
Note that the scans are numbered with 0 corresponding to the first scan
in the file i.e., it is not the session scan number."""
# TODO: Should check valid scan IF, and raise value errors as apropriate
thescan = self.scan_set[scan_ind]
theIF = self.IF_set[IF_ind]
# Find all the records that correspond to this IF and this scan.
# These indicies *should now be ordered in time, cal (on off)
# and in polarization, once the IF is isolated.
(inds_sif,) = sp.where(sp.logical_and(self._IFs_all==theIF,
self._scans_all==thescan))
ncal = len(sp.unique(self.fitsdata.field('CAL')[inds_sif]))
npol = len(sp.unique(self.fitsdata.field('CRVAL4')[inds_sif]))
# Reform to organize by pol, cal, etc.
ntimes = len(inds_sif)//npol//ncal
inds_sif = sp.reshape(inds_sif, (ntimes, npol, ncal))
if self.verify_ordering > 0:
# We expect noise cal to be on for every second record.
for thecal in range(ncal) :
tmp = sp.unique(self.fitsdata.field('CAL')[inds_sif[:,:,thecal]])
if len(tmp) > 1 :
raise ce.DataError("Calibration (ON/OFF) not in "
"perfect order in file: "+self.fname)
# Polarization should cycle through 4 modes (-5,-7,-8,-6)
for thepol in range(npol) :
tmp = sp.unique(self.fitsdata.field('CRVAL4')
[inds_sif[:,thepol,:]])
if len(tmp) > 1 :
raise ce.DataError("Polarizations not in perfect order in "
"file: "+self.fname)
# We expect the entries to be sorted in time and for time to not
# change across pol and cal.
lastLST = 0
for ii in range(ntimes) :
# Sometimes won't have the LST.
try :
thisLST = self.fitsdata.field('LST')[inds_sif[ii,0,0]]
# If 'LST' is missing raises a KeyError in later versions of
# pyfits, and a NameError in earlier ones.
except (KeyError, NameError) :
break
if not (sp.allclose(self.fitsdata.field('LST')
[inds_sif[ii,:,:]] - thisLST, 0)) :
raise ce.DataError("LST change across cal or pol in "
"file: " + self.fname)
return inds_sif
def verify(self) :
"""Verifies that all the data is consistant.
This method should be run every time you muck around with the data
and field entries. It simply checks that all the data is consistant
(axes, lengths etc.).
Note that even if you know that your DataBlock will pass the verify,
you still need to verify as this tells the DataBlock that you are done
messing with the data. It then sets some internal variables.
"""
if self.data_set :
self.dims = sp.shape(self.data)
else :
raise RunTimeError('Data needs to be set before running verify()')
# Will delete these keys if they are found in 'field', then see if any
# are left over.
axes_keys = self.field_axes.keys()
format_keys = self.field_formats.keys()
for field_name in self.field.iterkeys() :
# Check for keys in fields and not in field_axes, the oposite is
# done outside this loop.
if ((not self.field_axes.has_key(field_name)) or
(not self.field_formats.has_key(field_name))) :
raise ce.DataError("Dictionaries 'field', 'field_axes' and "
"field_formats must have the same keys.")
axes_keys.remove(field_name)
format_keys.remove(field_name)
# Check all the axes
axes = self.field_axes[field_name] # for saving keystrokes only
self._verify_single_axis_names(axes)
# Check the shape.
field_data_shape = sp.shape(self.field[field_name])
for ii in range(len(axes)) :
axis_ind = list(self.axes).index(axes[ii])
if field_data_shape[ii] != self.dims[axis_ind] :
raise ce.DataError("The shape of the data in one of the "
"fields is incompatible with the shape "
"of the main data. field: "+field_name)
# Check the format string.
# TODO: This should do something better than just check that there
# is a string.
if not type(self.field_formats[field_name]) is str :
print type(self.field_formats[field_name])
print self.field_formats[field_name]
raise ce.DataError("The field_format must be type str. field: "
+ field_name)
# The opposite of the first check in the loop.
if len(axes_keys) or len(format_keys) :
raise ce.DataError("Dictionaries 'field', 'field_axes' and "
"field_formats must have the same keys.")
def read_n_numbers(f, arr, dt, n, row_len, row_pos):
'''Read n numbers from an open file stream, f, and add them to the next
available row, row_pos, in the 2D array, arr. The data type, dt, is needed
to convert from bytes to usable numbers. row_len and row_pos are needed
so that only the upper triangular part of a row is added to arr.'''
bytes_per_num = 0
# Assign how long a number is in memory.
if ((dt.name == 'int64') or (dt.name == 'float64')):
bytes_per_num = 8
if ((dt.name == 'int32') or (dt.name == 'float32')):
bytes_per_num = 4
# Check if possible to run.
if (bytes_per_num == 0):
msg = "%s is not a supported number type right now." % (dt.name)
raise ce.DataError(msg)
# Read in n numbers.
raw_nums = f.read(bytes_per_num * n)
# Convert numbers in bytes to useable numbers all at once.
new_slice = np.fromstring(raw_nums, dtype=dt, count=n)
# n is a multiple of rows.
rows_to_read = int(n / row_len)
# Copy each row from the buffer (in memory) to the array (in memory).
# We are copying in only the upper triangular part of the array so
# we must only write in the appropriate values.
for i in range(rows_to_read):
# Every row down the matrix, we want to ignore one more number
# at the beginning of a row.
arr[row_pos+i,row_pos+i:row_len] = \
new_slice[i*row_len+row_pos+i:i*row_len+row_len]
def hanning_smooth(Data):
"""Perform Hanning smoothing.
This function accepts a DataBlock class and returns nothing. It changes
the data attribute of the passed Data.
"""
# Perform Hanning smoothing frequency space convolution.
# Masked array automatically flags data adjacent to masked
# data.
if len(Data.axes) != 4 or Data.axes[-1] != 'freq':
raise ce.DataError(
'Hanning smoothing expects data be 4D with frequency '
'being the last axis.')
# Channels adjacent to masked data automatically masked.
Data.data[:, :, :, 1:-1] = (0.25 * Data.data[:, :, :, :-2] +
0.50 * Data.data[:, :, :, 1:-1] +
0.25 * Data.data[:, :, :, 2:])
# Below copiled c code replaces the above numpy code. Speed up of ~20.
# Not working.
#data = Data.data.data
#mask = Data.data.mask
#nt, npol, ncal, nf = data.shape
#code = """
# #line 43 "hanning.py"
# double this_data, last_data;
# int this_mask, last_mask, ind;
#
# for (int tii=0; tii<nt; tii++) {
# ind = tii*npol*ncal*nf;
# for (int pjj=0; pjj<npol; pjj++) {
# ind = ind + pjj*ncal*nf;
# for (int ckk=0; ckk<ncal; ckk++) {
# ind = ind + ckk*nf;
# last_data = data[ind];
# last_mask = mask[ind];
# for (int fmm=1; fmm<nf-1; fmm++) {
# this_data = data[ind+fmm];
# this_mask = mask[ind+fmm];
# if (last_mask || mask[ind+fmm+1])
# mask[ind+fmm] = 1;
# else if (!this_mask)
# data[ind+fmm] = 0.25*last_data + 0.5*this_data
# + 0.25*data[ind+fmm+1];
# last_data = this_data;
# last_mask = this_mask;
# }
# }
# }
# }
# """
#weave.inline(code, ['data', 'mask', 'nt', 'npol', 'ncal', 'nf'])
# End points where not smoothed, mask them as bad data.
Data.data[:, :, :, 0] = ma.masked
Data.data[:, :, :, -1] = ma.masked
def verify(self):
"""Mostly the same as BaseData.verify, but with added checks.
"""
for field_axes in self.field_axes.itervalues():
# The fields in the map must be stored in the header (not the data)
# and as such, they must be 0 dimensional (scalars).
if len(field_axes) != 0:
raise ce.DataError("Maps can only have scalar fields.")
# Now call verify from the base class.
base_data.BaseData.verify(self)
def __init__(self, parameter_file_or_dict=None, feedback=2):
# Call the base_single init.
base_single.BaseSingle.__init__(self, parameter_file_or_dict, feedback)
# Read in calibration table.
file_names = self.params['cal_temperature_files']
if len(file_names) > 1:
raise NotImplementedError('Can use a single calibration file'
' at a time.')
Reader = fitsGBT.Reader(file_names[0], feedback=self.feedback)
self.CalData = Reader.read([], [])
if type(self.CalData) is tuple:
raise ce.DataError('Expected calibration file to have only a'
' single scan and IF.')
def merge(self, *args) :
"""Merge this History object with ones passed to this function."""
for obj in args :
if hasattr(obj, 'history') :
thishistory = obj.history
else :
thishistory = obj
for entry, details in thishistory.iteritems() :
for detail in details :
try :
if not detail in self[entry] :
self[entry] = self[entry] + (detail, )
except KeyError :
raise ce.DataError("Histories to be merged must have"
" identical keys.")
def execute(self, nprocesses):
params = self.params
# Make parent directory and write parameter file.
kiyopy.utils.mkparents(params['output_root'])
parse_ini.write_params(params,
params['output_root'] + 'params.ini',
prefix=prefix)
in_root = params['input_root']
# Figure out what the band names are.
bands = params['bands']
if not bands:
map_files = glob.glob(in_root + pol_str + "_*.npy")
bands = []
root_len = len(in_root)
for file_name in map_files:
bands.append(file_name[root_len:-4])
# Loop over polarizations.
for pol_str in params['polarizations']:
# Read in all the maps to be glued.
maps = []
for band in bands:
band_map_fname = (in_root + pol_str + "_" + repr(band) +
'.npy')
if self.feedback > 1:
print "Read using map: " + band_map_fname
if params['mat_diag']:
if self.feedback > 1:
print "Treating as a matrix, getting diagonal."
band_map = al.open_memmap(band_map_fname, mode='r')
band_map = al.make_mat(band_map)
band_map = band_map.mat_diag()
else:
band_map = al.load(band_map_fname)
band_map = al.make_vect(band_map)
if band_map.axes != ('freq', 'ra', 'dec'):
msg = ("Expeced maps to have axes ('freq',"
"'ra', 'dec'), but it has axes: " +
str(band_map.axes))
raise ce.DataError(msg)
maps.append(band_map)
# Now glue them together.
out_map = glue(maps)
out_fname = (params['output_root'] + pol_str + "_" + "all" +
'.npy')
if self.feedback > 1:
print "Writing glued map to: " + out_fname
al.save(out_fname, out_map)
def __init__(self, fname, feedback=2, checking=1, memmap=False) :
"""Init script for the fitsGBT Reader class.
The reader is initialised with the fits file name to be read.
Optionally, one can supply up to two integers: feedback (default 2)
indicating how much junk to print, and checking (default 1) specifying
to what level the input file is checked for sensible data.
"""
self.feedback = feedback
self.checking = checking
if checking > 0 :
self.verify_ordering = 1
else :
self.verify_ordering = 0
self.fname = fname
# The passed file name is assumed to be a GBT spectrometer fits file.
self.hdulist = pyfits.open(self.fname, 'readonly', memmap=memmap)
if len(self.hdulist) < 2 :
raise ce.DataError("File missing data extension")
if self.feedback > 0 :
print "Opened GBT fits file: ", ku.abbreviate_file_path(fname)
# Separate in to the useful sub objects. These assignments are all
# done by reference, so this is efficient.
self.fitsdata = self.hdulist[1].data
# The records in fitsdata are not guaranteed to be in proper order.
# Mostly the IFs are all out of whack. However, once you isolate an
# IF everything should be well ordered.
# Get the scans and IF of all records. This is later used to isolate a
# single IF and scan. Also get the set of unique IFs and scans, so we
# know what is in the file.
self._scans_all = self.fitsdata.field('SCAN')
self.scan_set = sp.unique(self._scans_all)
# Sort scans for reapeatable ordering.
self.scan_set.sort()
self._IFs_all = self.fitsdata.field('CRVAL1')/1E6 # MHz
# Round the frequencies as we only need to tell the difference between
# one IF and the other.
self._IFs_all = self._IFs_all.round(0)
self.IF_set = sp.unique(self._IFs_all)
self.IF_set.sort()
def set_field(self, field_name, field_data, axis_names=(), format=None) :
"""Set field data to be stored.
Note that these operation can also be done by accessing the 'field' and
'field_axes' dictionaries directly, but using this function combines a
few operations that go together. It also does some sanity checks.
Using this function is safer.
Arguments are the field name (like 'CRVAL2', or 'SCAN'), field data
(numpy array or appropriate length according to axis_names), axis_names
(tuple of names like ('time', ) or ('pol',) or simply () for 0D data),
and finally a fits format string (like '1E' or '10A', see fits
documentation).
"""
field_data = sp.array(field_data)
if type(axis_names) is str :
a_names = (axis_names,)
else :
a_names = axis_names
if not format:
if field_data.dtype == sp.float64:
format = 'D'
elif field_data.dtype == sp.float32:
format = 'E'
elif field_data.dtype == sp.int16:
format = 'I'
elif field_data.dtype == sp.int32:
format = 'J'
elif field_data.dtype == sp.int64:
format = 'K'
elif field_data.dtype == sp.complex64:
format = 'C'
elif field_data.dtype == sp.complex128:
format = 'M'
else:
raise ce.DataError("dtype not understood.")
self._verify_single_axis_names(a_names)
self.field[field_name] = sp.array(field_data)
self.field_axes[field_name] = tuple(a_names)
self.field_formats[field_name] = str(format)
def write(Maps, file_name, feedback=2):
"""Write a map to fits file.
Map should be a map_data.MapData object.
"""
# If only a single Map was passed, make it iterable.
if not hasattr(Maps, '__iter__'):
Maps = (Maps, )
# First create a primary with the history and such:
prihdu = pyfits.PrimaryHDU()
# Add history to the primary.
fname_abbr = ku.abbreviate_file_path(file_name)
# Add final history entry and store in the primary header.
history = base_data.merge_histories(*Maps)
history.add('Written to file.', 'File name: ' + fname_abbr)
bf.write_history_header(prihdu.header, history)
list_of_hdus = [prihdu]
for ii, Map in enumerate(Maps):
# Creat an image HDU.
map = sp.array(ma.filled(Map.data, float('nan')))
imhdu = pyfits.ImageHDU(sp.swapaxes(map, 0, 2), name='MAP%d' % ii)
# Add extra data to the HDU
for key in Map.field.iterkeys():
if Map.field_axes[key] != ():
raise ce.DataError('Only 0D data can be written to a Fits Map '
'Header.')
card = pyfits.Card(key, Map.field[key].item())
imhdu.header.ascardlist().append(card)
list_of_hdus.append(imhdu)
# Creat the HDU list and write to file.
hdulist = pyfits.HDUList(list_of_hdus)
hdulist.writeto(file_name, clobber=True)
if feedback > 0:
print 'Wrote data to file: ' + fname_abbr
def combine(Data, weights=(0.5, 0.5), sub_mean=True, average_cals=True):
"""
"""
if average_cals:
on_ind = 0
off_ind = 1
if (Data.field['CAL'][on_ind] != 'T'
or Data.field['CAL'][off_ind] != 'F'):
raise ce.DataError('Cal states not in expected order.')
# If giving equal weights, use mean such that flags aren't propagated.
newdata = (Data.data[:, :, [0], :] * weights[0] +
Data.data[:, :, [1], :] * weights[1])
Data.set_data(newdata)
Data.field['CAL'] = sp.array(['A']) # For averaged.
if Data.field.has_key('EXPOSURE'):
Data.field['EXPOSURE'] = (
Data.field['EXPOSURE'][:, [0]] * weights[0] +
Data.field['EXPOSURE'][:, [1]] * weights[1])
# Subtract the time median if desired.
if sub_mean:
Data.data -= ma.mean(Data.data, 0)
def calibrate_pol(Data, m_total, flux_status):
"""Subtracts a Map out of Data."""
# Data is a DataBlock object. It holds everything you need to know about
# the data in a single scan and IF. You should get to know them very well.
# Data.data is a numpy masked array (see numpy documentation) and holds the
# acctual data. It is a 4 dimensional array. The demensions are (in
# order): (time, pol, cal, freq). Each dimension can be any length which
# you can figure out by looking at Data.dims = sp.shape(Data.data).
# Data.field is a python dictionary that holds all the other data that you
# might care about from the origional fits file. For instance,
# Data.field['CAL'] is an array with length dims[2]. It normally has
# values ['T', 'F']. Data.field['CRVAL4'] tells you about the polarization
# axis of Data.data. By SDfits convension each polarization is represented
# by an integer: 1=I, 2=Q, 3=U, 4=V, -5=XX, -6=YY, -7=XY, -8=YX.
# Also this depends on having the polarizations rotated correctly to IQUV.
# Some code to do this has been hacked together in the rotate_pol module,
# but I don't trust it yet.
# Some dimension checks.
# We expect 4 polarizations.
if not Data.dims[1] == 4:
raise ce.DataError('Require 4 polarizations.')
# We expect polarizations to be in order IQUV.
if (Data.field['CRVAL4'][0] != 1 or Data.field['CRVAL4'][1] != 2
or Data.field['CRVAL4'][2] != 3 or Data.field['CRVAL4'][3] != 4):
raise ce.DataError('Expected the polarization basis to be IQUV.')
# A useful function that might need:
Data.calc_freq()
# Now data has an atribute Data.freq which is an array that gives the
# frequency along the last axis.
# Data.field['CRVAL1'] is center frequency in Hz.
# Data.data 4 dim array 2nd index polarization, 4th index frequency.
# Need to get parallactic angle:
Data.calc_PA()
# This gives an array (Data.PA) of PA values of length = time dim.
for time_index in range(0, Data.dims[0]):
#Generate a sky matrix for this time index:
m_sky = sp.zeros((4, 4))
m_sky[0, 0] = 1
m_sky[1, 1] = ma.cos(2 * Data.PA[time_index] * sp.pi / 180)
m_sky[1, 2] = ma.sin(2 * Data.PA[time_index] * sp.pi / 180)
m_sky[2, 1] = -ma.sin(2 * Data.PA[time_index] * sp.pi / 180)
m_sky[2, 2] = ma.cos(2 * Data.PA[time_index] * sp.pi / 180)
m_sky[3, 3] = 1
M_sky = sp.mat(m_sky)
M_sky = M_sky.I
# print M_sky
for cal_index in range(0, Data.dims[2]):
# Determines the Mueller Matrix to use
for freq in range(0, Data.dims[3]):
# Tells which mueller matrix to use.
freq_limit = len(m_total[0, 0, :])
frequency = int(Data.freq[freq] / 1000)
# print frequency
bin = int((900000 - frequency) * freq_limit / 200000)
# print bin
# if freq_limit == 200:
# bin = 900-frequency
#Not setup to work with spectrometer data.
# elif freq_limit == 260:
# bin = 929-frequency
# print bin
# Converts files into matrix format
STOKES = Data.data[time_index, :, cal_index, freq]
# print STOKES
MUELLER = sp.mat(m_total[:, :, bin])
# print MUELLER
# Next there is a matrix multiplication that will generate
# a new set of stokes values.
stokesmod = STOKES
if flux_status == 'False':
stokesmod = np.dot(MUELLER, stokesmod)
stokesmod = np.dot(M_sky, stokesmod)
# You always want to include the M_sky matrix transformation, but you if you just want the flux cal, coment out the MUELLER, STOKES dot product above and include the flux multiplication below instead.
if flux_status == 'True':
stokesmod[0] = stokesmod[0] * MUELLER[0, 0]
stokesmod[1] = stokesmod[1] * MUELLER[0, 0]
stokesmod[2] = stokesmod[2] * MUELLER[0, 0]
stokesmod[3] = stokesmod[3] * MUELLER[0, 0]
# print stokesmod
for i in range(0, Data.dims[1]):
Data.data[time_index, i, cal_index, freq] = stokesmod[i]
def glue(maps):
"""Function that glues a set of maps together into one map.
Gluing is done along frequency axis
"""
# Check the non-frequency axis to make sure all the maps are the same.
angular_shape = maps[0].shape[1:]
ra_centre = maps[0].info["ra_centre"]
dec_centre = maps[0].info["dec_centre"]
ra_delta = maps[0].info["ra_delta"]
dec_delta = maps[0].info["dec_delta"]
for map in maps:
if map.shape[1:] != angular_shape:
msg = ("Angular shapes don't match up. " + str(map.shape[1:]) +
" vs " + str(angular_shape))
raise ce.DataError(msg)
if (not np.allclose(map.info["ra_centre"], ra_centre)
or not np.allclose(map.info["dec_centre"], dec_centre)
or not np.allclose(map.info["ra_delta"], ra_delta)
or not np.allclose(map.info["dec_delta"], dec_delta)):
msg = "Angular information doesn't match up."
raise ce.DataError(msg)
# Now check that the frequecy axes all have the same spacing and get the
# centres.
delta = maps[0].info["freq_delta"]
centres = [] # In units of delta.
# Size of the frequency axis.
size = 0
for map in maps:
if not np.allclose(map.info["freq_delta"], delta):
msg = "Frequency spacing not all the same."
raise ce.DataError(msg)
centres.append(map.info["freq_centre"] / delta)
size += map.shape[0]
# Sort the map by thier centre.
inds = np.argsort(centres)
# Allocate memory for the output map.
out_map = np.empty((size, ) + angular_shape, dtype=float)
out_map = al.make_vect(out_map, axis_names=('freq', 'ra', 'dec'))
out_map.copy_axis_info(maps[0])
# Loop over the maps and put them in the larger map.
so_far = 0 # frequency axis used so far.
for index in inds:
map = maps[index]
s = map.shape[0]
freq = map.get_axis('freq')
# Check if the centre of the total band is in this map and if so, set
# the centre.
if size // 2 >= so_far and size // 2 < so_far + s:
centre = freq[size // 2 - so_far]
out_map.set_axis_info('freq', centre, delta)
# Check that the space between the last frequency of the last map and
# the first frequency of the first map is delta.
if so_far > 0:
if not np.allclose(freq[0], last_freq + delta):
raise ce.DataError("Bands don't line up correctly.")
last_freq = freq[-1]
# Finally copy the map into the conglomerate map.
out_map[so_far:so_far + s, :, :] = map
so_far += s
if so_far != size:
raise RuntimeError("Something went horrible wrong.")
return out_map
def scale_by_cal(Data,
scale_t_ave=True,
scale_f_ave=False,
sub_med=False,
scale_f_ave_mod=False,
rotate=False):
"""Puts all data in units of the cal temperature.
Data is put into units of the cal temperature, thus removing dependence on
the gain. This can be done by dividing by the time average of the cal
(scale_t_ave=True, Default) thus removing dependence on the frequency-
dependant gain. Alternatively, you can scale by the frequency average to
remove the time-dependent gain (scale_f_ave=True). Data is then in units of
the frequency averaged cal temperture. You can also do both (recommended).
After some scaling the data ends up in units of the cal temperture as a
funciton of frequency.
Optionally you can also subtract the time average of the data off here
(subtract_time_median), since you might be done with the cal information at
this point.
"""
on_ind = 0
off_ind = 1
if (Data.field['CAL'][on_ind] != 'T' or Data.field['CAL'][off_ind] != 'F'):
raise ce.DataError('Cal states not in expected order.')
if tuple(Data.field['CRVAL4']) == (-5, -7, -8, -6):
# Here we check the polarizations and cal indicies
xx_ind = 0
yy_ind = 3
xy_inds = [1, 2]
# A bunch of calculations used to test phase closure. Not acctually
# relevant to what is being done here.
#a = (Data.data[5, xy_inds, on_ind, 15:20]
# - Data.data[5, xy_inds, off_ind, 15:20])
#a /= sp.sqrt( Data.data[5, xx_ind, on_ind, 15:20]
# - Data.data[5, xx_ind, off_ind, 15:20])
#a /= sp.sqrt( Data.data[5, yy_ind, on_ind, 15:20]
# - Data.data[5, yy_ind, off_ind, 15:20])
#print a[0,:]**2 + a[1,:]**2
diff_xx = Data.data[:, xx_ind, on_ind, :] - Data.data[:, xx_ind,
off_ind, :]
diff_yy = Data.data[:, yy_ind, on_ind, :] - Data.data[:, yy_ind,
off_ind, :]
if scale_t_ave:
# Find the cal means (in time) and scale by them.
# Means work much better than medians. Medians seems to bias the
# result by up to 10%. This seems to be discretization noise. Cal
# switches fast enough that we shouldn't need this anyway.
cal_tmed_xx = ma.mean(diff_xx, 0)
cal_tmed_yy = ma.mean(diff_yy, 0)
cal_tmed_xx[sp.logical_or(cal_tmed_xx <= 0,
cal_tmed_yy <= 0)] = ma.masked
cal_tmed_yy[cal_tmed_xx.mask] = ma.masked
Data.data[:, xx_ind, :, :] /= cal_tmed_xx
Data.data[:, yy_ind, :, :] /= cal_tmed_yy
Data.data[:, xy_inds, :, :] /= ma.sqrt(cal_tmed_yy * cal_tmed_xx)
if scale_f_ave:
# The frequency gains have have systematic structure to them,
# they are not by any approximation gaussian distributed. Use
# means, not medians across frequency.
operation = ma.mean
cal_fmea_xx = operation(diff_xx, -1)
cal_fmea_yy = operation(diff_yy, -1)
# Flag data with wierd cal power. Still Experimental.
cal_fmea_xx[sp.logical_or(cal_fmea_xx <= 0,
cal_fmea_yy <= 0)] = ma.masked
cal_fmea_yy[cal_fmea_xx.mask] = ma.masked
cal_xx = ma.mean(cal_fmea_xx)
cal_yy = ma.mean(cal_fmea_yy)
cal_fmea_xx[sp.logical_or(
abs(cal_fmea_xx.anom()) >= 0.1 * cal_xx,
abs(cal_fmea_yy.anom()) >= 0.1 * cal_yy)] = ma.masked
cal_fmea_yy[cal_fmea_xx.mask] = ma.masked
ntime = len(cal_fmea_xx)
cal_fmea_xx.shape = (ntime, 1, 1)
cal_fmea_yy.shape = (ntime, 1, 1)
Data.data[:, xx_ind, :, :] /= cal_fmea_xx
Data.data[:, yy_ind, :, :] /= cal_fmea_yy
cal_fmea_xx.shape = (ntime, 1, 1, 1)
cal_fmea_yy.shape = (ntime, 1, 1, 1)
Data.data[:, xy_inds, :, :] /= ma.sqrt(cal_fmea_yy * cal_fmea_xx)
if scale_f_ave_mod:
# The frequency gains have have systematic structure to them,
# they are not by any approximation gaussian distributed. Use
# means, not medians across frequency.
operation = ma.mean
cal_fmea_xx = operation(diff_xx, -1)
cal_fmea_yy = operation(diff_yy, -1)
cal_fmea_xx_off = operation(Data.data[:, xx_ind, off_ind, :], -1)
cal_fmea_yy_off = operation(Data.data[:, yy_ind, off_ind, :], -1)
sys_xx = cal_fmea_xx_off / cal_fmea_xx
sys_yy = cal_fmea_yy_off / cal_fmea_yy
percent_ok = 0.03
sys_xx_tmed = ma.median(sys_xx)
sys_yy_tmed = ma.median(sys_yy)
maskbad_xx = (sys_xx > sys_xx_tmed + sys_xx_tmed * percent_ok) | (
sys_xx < sys_xx_tmed - sys_xx_tmed * percent_ok)
maskbad_yy = (sys_yy > sys_yy_tmed + sys_yy_tmed * percent_ok) | (
sys_yy < sys_yy_tmed - sys_yy_tmed * percent_ok)
cal_fmea_xx[sp.logical_or(cal_fmea_xx <= 0,
cal_fmea_yy <= 0)] = ma.masked
cal_fmea_yy[cal_fmea_xx.mask] = ma.masked
cal_fmea_xx[maskbad_xx] = ma.masked
cal_fmea_yy[maskbad_yy] = ma.masked
cal_xx = ma.mean(cal_fmea_xx)
cal_yy = ma.mean(cal_fmea_yy)
ntime = len(cal_fmea_xx)
cal_fmea_xx.shape = (ntime, 1, 1)
cal_fmea_yy.shape = (ntime, 1, 1)
Data.data[:, xx_ind, :, :] /= cal_fmea_xx
Data.data[:, yy_ind, :, :] /= cal_fmea_yy
cal_fmea_xx.shape = (ntime, 1, 1, 1)
cal_fmea_yy.shape = (ntime, 1, 1, 1)
Data.data[:, xy_inds, :, :] /= ma.sqrt(cal_fmea_yy * cal_fmea_xx)
if scale_f_ave and scale_t_ave:
# We have devided out t_cal twice so we need to put one factor back
# in.
cal_xx = operation(cal_tmed_xx)
cal_yy = operation(cal_tmed_yy)
Data.data[:, xx_ind, :, :] *= cal_xx
Data.data[:, yy_ind, :, :] *= cal_yy
Data.data[:, xy_inds, :, :] *= ma.sqrt(cal_yy * cal_xx)
if scale_f_ave_mod and scale_t_ave:
#Same divide out twice problem.
cal_xx = operation(cal_tmed_xx)
cal_yy = operation(cal_tmed_yy)
Data.data[:, xx_ind, :, :] *= cal_xxcal_imag_mean
Data.data[:, yy_ind, :, :] *= cal_yy
Data.data[:, xy_inds, :, :] *= ma.sqrt(cal_yy * cal_xx)
if scale_f_ave and scale_f_ave_mod:
raise ce.DataError("time averaging twice")
if rotate:
# Define the differential cal phase to be zero and rotate all data
# such that this is true.
cal_real_mean = ma.mean(
Data.data[:, 1, 0, :] - Data.data[:, 1, 1, :], 0)
cal_imag_mean = ma.mean(
Data.data[:, 2, 0, :] - Data.data[:, 2, 1, :], 0)
# Get the cal phase angle as a function of frequency.
cal_phase = -ma.arctan2(cal_imag_mean, cal_real_mean)
# Rotate such that the cal phase is zero. Imperative to have a
# temporary variable.
New_data_real = (ma.cos(cal_phase) * Data.data[:, 1, :, :] -
ma.sin(cal_phase) * Data.data[:, 2, :, :])
New_data_imag = (ma.sin(cal_phase) * Data.data[:, 1, :, :] +
ma.cos(cal_phase) * Data.data[:, 2, :, :])
Data.data[:, 1, :, :] = New_data_real
Data.data[:, 2, :, :] = New_data_imag
elif tuple(Data.field['CRVAL4']) == (1, 2, 3, 4):
# For the shot term, just devide everything by on-off in I.
I_ind = 0
cal_I_t = Data.data[:, I_ind, on_ind, :] - Data.data[:, I_ind,
off_ind, :]
cal_I = ma.mean(cal_I_t, 0)
Data.data /= cal_I
else:
raise ce.DataError("Unsupported polarization states.")
# Subtract the time median if desired.
if sub_med:
Data.data -= ma.median(Data.data, 0)
def execute(self, nprocesses=1) :
params = self.params
kiyopy.utils.mkparents(params['output_root'] +
params['output_filename'])
parse_ini.write_params(params, params['output_root'] + 'params.ini',
prefix=prefix)
file_middles = params['file_middles']
# Store the correlation and normalization session by session in a
# dictionary.
corr_dict = {}
norm_dict = {}
# Also store the frequency axis.
freq_dict = {}
# Finally, a gain dictionary.
gain_dict = {}
# Loop though all the files and accumulate the correlation and the
# normalization. Files in the same session are summed together.
n_new = nprocesses-1 # How many new processes to spawn at once.
n_files = len(file_middles)
if n_new > 0:
# Multiprocessed version.
process_list = range(n_new)
pipe_list = range(n_new)
for ii in xrange(n_files + n_new) :
if ii >= n_new :
# First end a process before starting a new one.
key, corr, norm, freq = pipe_list[ii%n_new].recv()
if corr_dict.has_key(key):
if corr_dict[key].shape != corr.shape:
msg = ("All data needs to have the same band and"
"polarization structure.")
raise ce.DataError(msg)
corr_dict[key] += corr
norm_dict[key] += norm
if not np.allclose(freq_dict[key], freq):
raise ce.DataError("Frequency structure not "
"consistant.")
else:
corr_dict[key] = corr
norm_dict[key] = norm
freq_dict[key] = freq
if ii < n_files :
# Start a new process.
Here, Far = mp.Pipe()
pipe_list[ii%n_new] = Here
process_list[ii%n_new] = mp.Process(
target=self.process_file, args=(file_middles[ii], Far))
process_list[ii%n_new].start()
else:
# Single process.
for middle in file_middles:
key, corr, norm, freq = self.process_file(middle)
if corr_dict.has_key(key):
if corr_dict[key].shape != corr.shape:
msg = ("All data needs to have the same band and"
"polarization structure.")
raise ce.DataError(msg)
corr_dict[key] += corr
norm_dict[key] += norm
if not np.allclose(freq_dict[key], freq):
raise ce.DataError("Frequency structure not consistant.")
else:
corr_dict[key] = corr
norm_dict[key] = norm
freq_dict[key] = freq
# Now that we have the correlation summed for all files in each
# session, normalize it and store it as an output.
output_fname = params['output_root'] + params["output_filename"]
out_db = shelve.open(output_fname)
# Loop through all the data divisions and processes them one at a
# time.
for key in corr_dict.iterkeys():
corr = corr_dict[key]
norm = norm_dict[key]
# Normalize.
corr[norm==0] = 1
norm[norm==0] = 1
gains = corr / norm
gain_dict[key] = gains
#plt.figure()
#if params['diff_gain_cal_only']:
# plt.plot(freq_dict[key][0,:],
# (gains[0,0,0,:] + gains[0,0,1,:])/2., '.')
# plt.plot(freq_dict[key][0,:],
# (gains[0,3,0,:] + gains[0,3,1,:])/2., '.')
#else:
# plt.plot(freq_dict[key][0,:], gains[0,0,0,:], '.')
#plt.title(key)
#plt.xlabel('Frequency (Hz)')
#plt.ylabel('Correlation amplitude')
out_db[key + '.gains'] = gains
out_db[key + '.freq'] = freq_dict[key]
#if not params['diff_gain_cal_only']:
# plt.show()
out_db.close()
#### Apply the calibration to the data. ####
for middle in file_middles:
key = get_key(middle)
gain = gain_dict[key]
freq = freq_dict[key]
self.calibrate_file(middle, gain, freq)
def full_calculation(self, chan_modes_subtract=None, n_poly_subtract=0):
# Some set up.
params = self.params
deconvolve = params['deconvolve']
kiyopy.utils.mkparents(params['output_root'])
parse_ini.write_params(params,
params['output_root'] + 'params.ini',
prefix=prefix)
self.n_time = params["n_time_bins"]
n_files = len(params["file_middles"])
# Loop over files to process.
first_iteration = True
for file_middle in params['file_middles']:
# Get the data.
full_data, mask, this_dt, full_mean = self.get_data(file_middle)
if first_iteration:
self.n_time = full_data.shape[0]
n_chan = full_data.shape[-1]
dt = this_dt
elif not sp.allclose(dt, this_dt, rtol=0.001):
msg = "Files have different time samplings."
raise ce.DataError(msg)
# Subtract out any channel modes passed in.
if not (chan_modes_subtract is None):
for v in chan_modes_subtract:
full_data -= v * sp.sum(v * full_data, -1)[:, None]
# Subtract out polynomials from each channel if desired.
if first_iteration:
# Generate basis polynomials.
basis_poly = sp.empty((n_poly_subtract, self.n_time))
time_scaled = ((sp.arange(self.n_time, dtype=float) * 2 -
self.n_time + 1.0) / self.n_time)
for ii in range(n_poly_subtract):
#tmp_poly = scipy.special.eval_chebyu(ii, time_scaled)
tmp_poly = sp.cos(sp.pi * ii * time_scaled)
tmp_poly *= 1.0 / sp.sqrt(sp.sum(tmp_poly**2))
basis_poly[ii, :] = tmp_poly
# Allocate memory to hold the amplitude spectrum.
poly_spectrum = sp.zeros((n_poly_subtract, n_chan),
dtype=float)
# Fit for the polynomials in each channel.
for ii in range(n_chan):
weighted_poly = basis_poly * mask[:, 0, 0, ii]
poly_corr = sp.sum(full_data[:, 0, 0, ii] * basis_poly[:, :],
-1)
poly_covar = sp.sum(
weighted_poly[:, None, :] * basis_poly[None, :, :], -1)
if n_poly_subtract:
poly_amps = linalg.solve(poly_covar,
poly_corr,
sym_pos=True,
overwrite_a=True,
overwrite_b=True)
poly_spectrum[:, ii] += poly_amps**2
# Calculate the raw power spectrum.
power_mat, window_function = calculate_full_power_mat(
full_data, mask, deconvolve=deconvolve)
# Get rid of the extra cal and polarization axes.
power_mat = power_mat[:, 0, 0, :, :]
window_function = window_function[:, 0, 0, :, :]
full_mean = full_mean[0, 0, :]
# TODO: Figure out a better way to deal with this (only drop
# affected frequencies).
#if sp.any(sp.allclose(mask[:,0,0,:], 0.0, 0)):
# n_files -= 1
# continue
# Format the power spectrum.
power_mat = prune_power(power_mat, 0)
power_mat = make_power_physical_units(power_mat, this_dt)
# TODO In the future the thermal expectation could include
# polarization factors (be 'I' aware) and cal factors.
thermal_expectation = (full_mean / sp.sqrt(this_dt) /
sp.sqrt(abs(self.chan_width)) /
sp.sqrt(1.0 / 2.0 / this_dt))
if params['norm_to_thermal']:
power_mat /= (thermal_expectation[:, None] *
thermal_expectation[None, :])
# Combine across files.
if first_iteration:
total_power_mat = power_mat
total_thermal_expectation = thermal_expectation
else:
total_power_mat += power_mat
total_thermal_expectation += thermal_expectation
first_iteration = False
if not hasattr(self, 'frequency'):
self.frequency = ps_freq_axis(dt, self.n_time)
if n_files > 0:
total_power_mat /= n_files
total_thermal_expectation / n_files
poly_spectrum /= n_files
if n_poly_subtract:
return total_power_mat, total_thermal_expectation, poly_spectrum
else:
return total_power_mat, total_thermal_expectation
def filter_cal_scale(Data,
size,
filter_type='rectangular',
filter_size_units='bins'):
"""Estimates Achromatic gain fluctuations and scales by them.
"""
on_ind = 0
off_ind = 1
n_time = Data.data.shape[0]
if (Data.field['CAL'][on_ind] != 'T' or Data.field['CAL'][off_ind] != 'F'):
raise ce.DataError('Cal states not in expected order.')
if tuple(Data.field['CRVAL4']) == (-5, -7, -8, -6):
# Here we check the polarizations and cal indicies
xx_ind = 0
yy_ind = 3
xy_inds = [1, 2]
diff_xx = Data.data[:, xx_ind, on_ind, :] - Data.data[:, xx_ind,
off_ind, :]
diff_yy = Data.data[:, yy_ind, on_ind, :] - Data.data[:, yy_ind,
off_ind, :]
cal_xx = ma.mean(diff_xx, -1)
cal_yy = ma.mean(diff_yy, -1)
mask = ma.getmaskarray(Data.data)
#mask = sp.any(sp.any(mask, 1), 1)
# XXX Wrong, Just needs to be factorizable. Need to devellop and
# algorithem for ensuring this, but for now, just use this. This
# shouldn't be too bad for the current RFI flagging algorithm (Apr
# 2012).
time_mask = sp.any(sp.any(sp.any(mask, 1), 1), 1)
# Sanity check the masks.
if not (sp.all(time_mask[ma.getmaskarray(cal_xx)])
and sp.all(time_mask[ma.getmaskarray(cal_yy)])):
msg = "Doesn't make since, this should always be true."
raise RuntimeError(msg)
# Convert to normal arrays.
cal_xx = cal_xx.filled(0)
cal_yy = cal_yy.filled(0)
# XXX
#Data.calc_time()
#time_unmask = sp.logical_not(time_mask)
#plt.plot(Data.time[time_unmask], cal_xx[time_unmask])
# Now set up the filter.
if filter_type == 'rectangular':
if filter_size_units == 'bins':
n_bins_filter = int(size)
if n_bins_filter % 2 == 0:
raise ValueError("Rectangular filter should have an odd"
"number of bins.")
elif filter_size_units == 'seconds':
Data.calc_time()
dt = abs(sp.mean(sp.diff(Data.time)))
n_bins_filter = size / dt
# Round to the nearest odd number.
n_bins_filter = 2 * int(round(n_bins_filter / 2. - 0.5)) + 1
else:
msg = 'Filter unit type unsupported.'
raise ValueError(msg)
kernal = sp.ones(n_bins_filter) / n_bins_filter
else:
msg = 'Filter type unsupported.'
raise ValueError(msg)
# Now that we know the kernal size, figure out what elements will
# be newly masked by the smoothing.
half_width = n_bins_filter // 2
old_mask = time_mask.copy()
for ii in range(n_time):
if old_mask[ii]:
time_mask[ii - half_width:ii + half_width + 1] = True
# Also mask the edges.
time_mask[:half_width] = True
time_mask[-half_width:] = True
# Now acctually do the convolution.
cal_xx = signal.convolve(cal_xx, kernal, mode='same')
cal_yy = signal.convolve(cal_yy, kernal, mode='same')
# XXX
#Data.calc_time()
#time_unmask = sp.logical_not(time_mask)
#plt.plot(Data.time[time_unmask], cal_xx[time_unmask])
#plt.plot(Data.time, time_mask)
plt.show()
# Replace invalid entries with unity (They get masked later anyway).
cal_xx[time_mask] = 1.
cal_yy[time_mask] = 1.
# Calibrate and apply mask.
Data.data[:, xx_ind, :, :] /= cal_xx[:, None, None]
Data.data[:, yy_ind, :, :] /= cal_yy[:, None, None]
cross_cal = sp.sqrt(cal_xx * cal_yy)
Data.data[:, xy_inds, :, :] /= cross_cal[:, None, None, None]
# Apply the mask.
Data.data[time_mask, ...] = ma.masked
#elif tuple(Data.field['CRVAL4']) == (1, 2, 3, 4) :
else:
raise ce.DataError("Unsupported polarization states.")
def execute(self, nprocesses=1):
"""Worker funciton."""
params = self.params
# Make parent directory and write parameter file.
kiyopy.utils.mkparents(params['output_root'])
parse_ini.write_params(params,
params['output_root'] + 'params.ini',
prefix=prefix)
save_noise_diag = params['save_noise_diag']
in_root = params['input_root']
all_out_fname_list = []
all_in_fname_list = []
# Figure out what the band names are.
bands = params['bands']
if not bands:
map_files = glob.glob(in_root + 'dirty_map_' + pol_str + "_*.npy")
bands = []
root_len = len(in_root + 'dirty_map_')
for file_name in map_files:
bands.append(file_name[root_len:-4])
# Loop over files to process.
for pol_str in params['polarizations']:
for band in bands:
if band == -1:
band_str = ''
else:
band_str = "_" + repr(band)
dmap_fname = (in_root + 'dirty_map_' + pol_str + band_str +
'.npy')
all_in_fname_list.append(
kiyopy.utils.abbreviate_file_path(dmap_fname))
# Load the dirty map and the noise matrix.
dirty_map = algebra.load(dmap_fname)
dirty_map = algebra.make_vect(dirty_map)
if dirty_map.axes != ('freq', 'ra', 'dec'):
msg = ("Expeced dirty map to have axes ('freq',"
"'ra', 'dec'), but it has axes: " +
str(dirty_map.axes))
raise ce.DataError(msg)
shape = dirty_map.shape
# Initialize the clean map.
clean_map = algebra.info_array(sp.zeros(dirty_map.shape))
clean_map.info = dict(dirty_map.info)
clean_map = algebra.make_vect(clean_map)
# If needed, initialize a map for the noise diagonal.
if save_noise_diag:
noise_diag = algebra.zeros_like(clean_map)
if params["from_eig"]:
# Solving from eigen decomposition of the noise instead of
# the noise itself.
# Load in the decomposition.
evects_fname = (in_root + 'noise_evects_' + pol_str +
+band_str + '.npy')
if self.feedback > 1:
print "Using dirty map: " + dmap_fname
print "Using eigenvectors: " + evects_fname
evects = algebra.open_memmap(evects_fname, 'r')
evects = algebra.make_mat(evects)
evals_inv_fname = (in_root + 'noise_evalsinv_' + pol_str +
"_" + repr(band) + '.npy')
evals_inv = algebra.load(evals_inv_fname)
evals_inv = algebra.make_mat(evals_inv)
# Solve for the map.
if params["save_noise_diag"]:
clean_map, noise_diag = solve_from_eig(
evals_inv, evects, dirty_map, True, self.feedback)
else:
clean_map = solve_from_eig(evals_inv, evects,
dirty_map, False,
self.feedback)
# Delete the eigen vectors to recover memory.
del evects
else:
# Solving from the noise.
noise_fname = (in_root + 'noise_inv_' + pol_str +
band_str + '.npy')
if self.feedback > 1:
print "Using dirty map: " + dmap_fname
print "Using noise inverse: " + noise_fname
all_in_fname_list.append(
kiyopy.utils.abbreviate_file_path(noise_fname))
noise_inv = algebra.open_memmap(noise_fname, 'r')
noise_inv = algebra.make_mat(noise_inv)
# Two cases for the noise. If its the same shape as the map
# then the noise is diagonal. Otherwise, it should be
# block diagonal in frequency.
if noise_inv.ndim == 3:
if noise_inv.axes != ('freq', 'ra', 'dec'):
msg = ("Expeced noise matrix to have axes "
"('freq', 'ra', 'dec'), but it has: " +
str(noise_inv.axes))
raise ce.DataError(msg)
# Noise inverse can fit in memory, so copy it.
noise_inv_memory = sp.array(noise_inv, copy=True)
# Find the non-singular (covered) pixels.
max_information = noise_inv_memory.max()
good_data = noise_inv_memory < 1.0e-10 * max_information
# Make the clean map.
clean_map[good_data] = (dirty_map[good_data] /
noise_inv_memory[good_data])
if save_noise_diag:
noise_diag[good_data] = \
1/noise_inv_memory[good_data]
elif noise_inv.ndim == 5:
if noise_inv.axes != ('freq', 'ra', 'dec', 'ra',
'dec'):
msg = ("Expeced noise matrix to have axes "
"('freq', 'ra', 'dec', 'ra', 'dec'), "
"but it has: " + str(noise_inv.axes))
raise ce.DataError(msg)
# Arrange the dirty map as a vector.
dirty_map_vect = sp.array(dirty_map) # A view.
dirty_map_vect.shape = (shape[0], shape[1] * shape[2])
frequencies = dirty_map.get_axis('freq') / 1.0e6
# Allowcate memory only once.
noise_inv_freq = sp.empty(
(shape[1], shape[2], shape[1], shape[2]),
dtype=float)
if self.feedback > 1:
print "Inverting noise matrix."
# Block diagonal in frequency so loop over frequencies.
for ii in xrange(dirty_map.shape[0]):
if self.feedback > 1:
print "Frequency: ", "%5.1f" % (
frequencies[ii]),
if self.feedback > 2:
print ", start mmap read:",
sys.stdout.flush()
noise_inv_freq[...] = noise_inv[ii, ...]
if self.feedback > 2:
print "done, start eig:",
sys.stdout.flush()
noise_inv_freq.shape = (shape[1] * shape[2],
shape[1] * shape[2])
# Solve the map making equation by diagonalization.
noise_inv_diag, Rot = sp.linalg.eigh(
noise_inv_freq, overwrite_a=True)
if self.feedback > 2:
print "done",
map_rotated = sp.dot(Rot.T, dirty_map_vect[ii])
# Zero out infinite noise modes.
bad_modes = (noise_inv_diag <
1.0e-5 * noise_inv_diag.max())
if self.feedback > 1:
print ", discarded: ",
print "%4.1f" % (100.0 * sp.sum(bad_modes) /
bad_modes.size),
print "% of modes",
if self.feedback > 2:
print ", start rotations:",
sys.stdout.flush()
map_rotated[bad_modes] = 0.
noise_inv_diag[bad_modes] = 1.0
# Solve for the clean map and rotate back.
map_rotated /= noise_inv_diag
map = sp.dot(Rot, map_rotated)
if self.feedback > 2:
print "done",
sys.stdout.flush()
# Fill the clean array.
map.shape = (shape[1], shape[2])
clean_map[ii, ...] = map
if save_noise_diag:
# Using C = R Lambda R^T
# where Lambda = diag(1/noise_inv_diag).
temp_noise_diag = 1 / noise_inv_diag
temp_noise_diag[bad_modes] = 0
# Multiply R by the diagonal eigenvalue matrix.
# Broadcasting does equivalent of mult by diag
# matrix.
temp_mat = Rot * temp_noise_diag
# Multiply by R^T, but only calculate the
# diagonal elements.
for jj in range(shape[1] * shape[2]):
temp_noise_diag[jj] = sp.dot(
temp_mat[jj, :], Rot[jj, :])
temp_noise_diag.shape = (shape[1], shape[2])
noise_diag[ii, ...] = temp_noise_diag
# Return workspace memory to origional shape.
noise_inv_freq.shape = (shape[1], shape[2],
shape[1], shape[2])
if self.feedback > 1:
print ""
sys.stdout.flush()
elif noise_inv.ndim == 6:
if save_noise_diag:
# OLD WAY.
#clean_map, noise_diag, chol = solve(noise_inv,
# dirty_map, True, feedback=self.feedback)
# NEW WAY.
clean_map, noise_diag, noise_inv_diag, chol = \
solve(noise_fname, noise_inv, dirty_map,
True, feedback=self.feedback)
else:
# OLD WAY.
#clean_map, chol = solve(noise_inv, dirty_map,
# False, feedback=self.feedback)
# NEW WAY.
clean_map, noise_inv_diag, chol = \
solve(noise_fname, noise_inv, dirty_map,
False, feedback=self.feedback)
if params['save_cholesky']:
chol_fname = (params['output_root'] + 'chol_' +
pol_str + band_str + '.npy')
sp.save(chol_fname, chol)
if params['save_noise_inv_diag']:
noise_inv_diag_fname = (params['output_root'] +
'noise_inv_diag_' +
pol_str + band_str +
'.npy')
algebra.save(noise_inv_diag_fname, noise_inv_diag)
# Delete the cholesky to recover memory.
del chol
else:
raise ce.DataError("Noise matrix has bad shape.")
# In all cases delete the noise object to recover memeory.
del noise_inv
# Write the clean map to file.
out_fname = (params['output_root'] + 'clean_map_' + pol_str +
band_str + '.npy')
if self.feedback > 1:
print "Writing clean map to: " + out_fname
algebra.save(out_fname, clean_map)
all_out_fname_list.append(
kiyopy.utils.abbreviate_file_path(out_fname))
if save_noise_diag:
noise_diag_fname = (params['output_root'] + 'noise_diag_' +
pol_str + band_str + '.npy')
algebra.save(noise_diag_fname, noise_diag)
all_out_fname_list.append(
kiyopy.utils.abbreviate_file_path(noise_diag_fname))
# Check the clean map for faileur.
if not sp.alltrue(sp.isfinite(clean_map)):
n_bad = sp.sum(sp.logical_not(sp.isfinite(clean_map)))
msg = ("Non finite entries found in clean map. Solve"
" failed. %d out of %d entries bad" %
(n_bad, clean_map.size))
raise RuntimeError(msg)
def up_tri_copy_from_file(filename):
'''Return the upper triagular part of the array in filename. The file
must be a binary .npy file.'''
f = open(filename, 'rb')
# The first 6 bytes are a magic string: exactly "\x93NUMPY".
numpy_string = f.read(6)
# The next 1 byte is an unsigned byte: the major version number
# of the file format, e.g. \x01.
major_ver = ord(f.read(1))
# The next 1 byte is an unsigned byte: the minor version number
# of the file format, e.g. \x00.
minor_ver = ord(f.read(1))
# Check that the version of the file used is what this code can handle.
if ((numpy_string != "\x93NUMPY") or (major_ver != 1) or (minor_ver != 0)):
msg = "Array can only be read from a '\93NUMPY' version 1.0 .npy file "
msg += "not %s %d.%d" % (numpy_string, major_ver, minor_ver)
raise ce.DataError(msg)
# The next 2 bytes form a little-endian unsigned short int: the
# length of the header data HEADER_LEN.
byte2, byte1 = f.read(2)
# Get the value from a short int (2 bytes) to decimal.
header_len = (16**2) * ord(byte1) + ord(byte2)
# The next HEADER_LEN bytes form the header data describing the
# array's format. It is an ASCII string which contains a Python
# literal expression of a dictionary. It is terminated by a newline
# ('\n') and padded with spaces ('\x20') to make the total length of
# the magic string + 4 + HEADER_LEN be evenly divisible by 16 for
# alignment purposes.
raw_dict = f.read(header_len)
# Take off trailing uselessness.
raw_dict = raw_dict.rstrip('\n')
raw_dict = raw_dict.rstrip()
header_dict = ev(raw_dict)
# Check that it is possible to read array.
# "fortran_order" : bool
# Whether the array data is Fortran-contiguous or not.
if (header_dict['fortran_order']):
msg = "Array must be in C order, not fortran order."
raise ce.DataError(msg)
# "shape" : tuple of int
# The shape of the array.
arr_shape = header_dict['shape']
# "descr" : dtype.descr
# An object that can be passed as an argument to the
# numpy.dtype() constructor to create the array's dtype.
dt = np.dtype(header_dict['descr'])
# Where to save all the data. Note this does not make a new array in memory.
num_nums = np.product(arr_shape)
# Assume array is square
row_len = int(np.sqrt(num_nums))
num_rows = row_len
loaded_arr = np.empty((row_len, row_len))
print "Rows copied:"
# Load the array reading n numbers at a time. Reading one number at
# a time is best for memory but requires too many disk seeks for a
# memory map. Loading too many numbers at once requires too much memory,
# so choose a happy medium.
# Assume array is square. Reads n rows of numbers at a time.
rows_to_read = 1000
n = rows_to_read * row_len
row_pos = 0
while (row_pos < num_rows):
# n may not divide evenly into the total number of numbers.
# This statement can only be True at the end of the array.
if ((num_rows - row_pos) < rows_to_read):
rows_to_read = num_rows - row_pos
n = rows_to_read * row_len
read_n_numbers(f, loaded_arr, dt, n, row_len, row_pos)
# Changing row_pos in the function called does not change it here.
row_pos += rows_to_read
sys.stderr.write(str(row_pos) + ' ')
print
f.close()
return loaded_arr
def stitch(blocks):
"""Stitches to gether data blocks from different frequency windows.
Accepts a tuple of Data Block objects to be stitched together.
"""
# Sort data blocks by frequeny.
try:
blocks = sorted(blocks, key=lambda Data: -1 * Data.field['CRVAL1'])
except KeyError:
raise ce.DataError('All blocks must have frequency axis'
' information.')
# Stitched data starts life as a copy of one of the old data blocks.
OutData = copy.deepcopy(blocks[0])
# First make sure all the data is compatible.
for Data in blocks:
# Make sure the data we need is here.
if not (Data.field.has_key('CRVAL1') and Data.field.has_key('CDELT1')
and Data.field.has_key('CRPIX1')):
raise ce.DataError('All blocks must have frequency axis'
' information.')
# Make sure all the data not involved in the stitching is the same.
# For now enforce that CDELT1 be the same for all IFs.
for key, field_data in OutData.field.iteritems():
if not key in ('CRVAL1', 'CRPIX1', 'OBSFREQ', 'RESTFREQ'):
if not Data.field.has_key(key):
raise ce.DataError('All blocks must have the same data '
'fields.')
# Treat strings differently.
if OutData.field_formats[key][-1] != 'A':
if not sp.allclose(field_data, Data.field[key]):
raise ce.DataError('All blocks to be stitched must '
'have matching data fields execpt '
'for frequency axis information.')
else:
if not sp.alltrue(field_data == Data.field[key]):
raise ce.DataError('All blocks to be stitched must '
'have matching data fields execpt '
'for frequency axis information.')
# For now assume that the frequencies are reversed ordered.
if OutData.field['CDELT1'] >= 0:
raise NotImplementedError('Expected frequency steps to be negitive.')
delt = abs(OutData.field['CDELT1'])
# Loop over data and stitch.
for Data in blocks[1:]:
# Get the freqeuncy axes
OutData.calc_freq()
Data.calc_freq()
n_over = list(Data.freq >= OutData.freq[-1]).count(True)
if n_over == 0:
raise ce.DataError('Frequency windows do not overlap.')
# Use mean, not sum, to normalize in case of flagged data.
factor = ma.mean(OutData.data[:, :, :, -2 * n_over // 3:-n_over // 3],
axis=3)
factor /= ma.mean(Data.data[:, :, :, n_over // 3:2 * n_over // 3],
axis=3)
Data.data *= factor[:, :, :, sp.newaxis]
OutData.set_data(
ma.concatenate((OutData.data[:, :, :, :-n_over // 2],
Data.data[:, :, :, n_over // 2:]),
axis=3))
OutData.calc_freq()
OutData.set_field('BANDWID',
abs(OutData.freq[0] - OutData.freq[-1]), (),
format='D')
return OutData
def apply(self, alg_ob, mode="constant", cval=0, right_apply=False):
"""Apply the beam, as a linear operator, to vector or matrix.
This operation is equivalent to matrix multiplication by the beam
matrix. The matrix multiplication can be performed with the beam
matrix on either the left or the right. The beam matrix is a symmetric
matrix.
Parameters
----------
alg_ob: `vect` or `mat` subclass.
Object to which the beam will be applied. This object must live in
map space. If it is a vect, alg_ob.axes must be ('freq', 'ra',
'dec'). If it is a mat, and `right_apply` is False, then
alg_ob.row_names() must return ('freq', 'ra', 'dec'). If
`right_apply` is True, then alg_ob.col_names() should return
this tuple. Also, the meta data for these three axis must be set
(see `algebra.alg_object.set_axis_info`).
wrap: bool
If tests True, use periodic boundary conditions for the
convolution. Otherwise zero pad (default).
right_apply: bool
Whether to apply the beam operator with the from the left (False,
default) or from the right (True). If `alg_ob` is a vect subclass,
this has no effect (because the beam matrix is symmetric).
Returns
-------
out: `vect` or `mat` subclass same shape as `alg_ob`.
Convolved map or matrix.
Notes
-----
The missing feature here is the ability to preallocate memory or to be
able to overwrite the input alg_ob in place. This will be important
for large matrices that we can only hold in memory one at a time.
Also this would be pretty easy to implement.
"""
if ((not 'freq' in alg_ob.axes)
or (not 'ra' in alg_ob.axes)
or (not 'dec' in alg_ob.axes)):
raise ce.DataError("Beam operation only works in frequency, "
"ra, dec, coords.")
out = algebra.zeros_like(alg_ob)
# Figure out the pixel sizes (in real degrees).
dra = abs(alg_ob.info['ra_delta'])
dra /= sp.cos(alg_ob.info['dec_centre'] * sp.pi / 180.)
ddec = abs(alg_ob.info['dec_delta'])
# Loop over frequencies and do convolution one frequency at a time.
freq_array = alg_ob.get_axis('freq')
for ii, freq in enumerate(freq_array):
width = self.kernel_size(freq)
# Make sure the dimensions are an odd number of pixels.
nkx = width // abs(dra)
if nkx % 2 == 0:
nkx += 1
nky = width // abs(ddec)
if nky % 2 == 0:
nky += 1
# Calculate kernel lags.
lagsx = (sp.arange(nkx, dtype=float) - (nkx - 1) // 2) * dra
lagsy = (sp.arange(nky, dtype=float) - (nky - 1) // 2) * ddec
lags_sq = lagsx[:, None] ** 2. + lagsy[None, :] ** 2.
kernel = dra * ddec * self.beam_function(lags_sq, freq,
squared_delta=True)
if isinstance(alg_ob, algebra.vect):
if alg_ob.axes != ('freq', 'ra', 'dec'):
raise ce.DataError("Vector axis names must be exactly "
"('freq', 'ra', 'dec')")
convolve(alg_ob[ii, ...], kernel, output=out[ii], mode=mode,
cval=cval)
elif isinstance(alg_ob, algebra.mat):
# If applying from the left, loop over columns and convolve
# over rows. If applying from the right, do the opposite.
if right_apply:
if alg_ob.col_names() != ('freq', 'ra', 'dec'):
raise ce.DataError("Matrix column axis names must be "
"exactly ('freq', 'ra', 'dec')")
iterator = alg_ob.iter_row_index()
else:
if alg_ob.row_names() != ('freq', 'ra', 'dec'):
raise ce.DataError("Matrix row axis names must be "
"exactly ('freq', 'ra', 'dec')")
iterator = alg_ob.iter_col_index()
for index in iterator:
sub_mat = alg_ob[index]
# Pick out this frequency.
sub_mat = sub_mat[ii, ...]
# make a view of the output array.
sub_out = out[index]
sub_out = sub_out[ii, ...]
convolve(sub_mat, kernel, sub_out, mode=mode, cval=cval)
return out
def execute(self, nprocesses=1):
params = self.params
kiyopy.utils.mkparents(params['output_root'] +
params['output_filename'])
parse_ini.write_params(params,
params['output_root'] + 'params.ini',
prefix=prefix)
file_middles = params['file_middles']
# Store the covariance and counts session by session in a dictionary.
covar_dict = {}
counts_dict = {}
# Also store the frequency axis.
freq_dict = {}
# Loop though all the files and accumulate the covariance and the
# counts. Files in the same session are summed together.
for middle in file_middles:
key, covar, counts, freq = self.process_file(middle)
if covar_dict.has_key(key):
if covar_dict[key].shape != covar.shape:
msg = ("All data needs to have the same band and"
"polarization structure.")
raise ce.DataError(msg)
covar_dict[key] += covar
counts_dict[key] += counts
if not np.allclose(freq_dict[key], freq):
raise ce.DataError("Frequency structure not consistant.")
else:
covar_dict[key] = covar
counts_dict[key] = counts
freq_dict[key] = freq
# Now that we have the covariance, factor it into eigen-vectors and
# store it in a data base.
output_fname = params['output_root'] + params["output_filename"]
out_db = shelve.open(output_fname)
# Loop through all the data divisions and processes them one at a
# time.
for key in covar_dict.iterkeys():
covar = covar_dict[key]
counts = counts_dict[key]
# Normalize.
counts[counts == 0] = 1
covar /= counts
# Loop to each matrix, decompose it and save it.
eigen_vects = np.empty_like(covar)
for band_ii in range(covar.shape[0]):
for pol_jj in range(covar.shape[1]):
for cal_kk in range(covar.shape[2]):
# Factor
h, v = linalg.eigh(covar[band_ii, pol_jj, cal_kk])
eigen_vects[band_ii, pol_jj, cal_kk] = v
#plt.semilogy(h, '.')
#plt.figure()
#for ii in range(1,5):
# plt.plot(v[:,-ii])
#plt.show()
out_db[key + '.vects'] = eigen_vects
out_db[key + '.freq'] = freq_dict[key]
if params['n_modes_removed']:
for middle in file_middles:
key = get_key(middle)
modes = out_db[key + '.vects']
modes = modes[:, :, :, :, -params['n_modes_removed']:]
self.clean_file(middle, modes)
# Close the data base.
out_db.close()
def measure_noise_parameters(Blocks,
parameters,
split_scans=False,
plots=False):
"""Given a set of data blocks, measure noise parameters.
Measurement done for all polarizations but only the first cal state.
"""
# Initialize the output.
out_parameters = {}
if set(parameters) == {"channel_var"}:
# Calculate the full correlated power spectrum.
power_mat, window_function, dt, channel_means = npow.full_power_diag(
Blocks,
window="hanning",
deconvolve=False,
n_time=-1.05,
normalize=False,
split_scans=split_scans,
subtract_slope=True)
# This shouldn't be nessisary, since I've tried to keep things finite in
# the above function. However, leave it in for now just in case.
if not sp.alltrue(sp.isfinite(power_mat)):
msg = ("Non finite power spectrum calculated. Offending data in "
"file starting with scan %d." % (Blocks[0].field['SCAN']))
raise ce.DataError(msg)
# Get frequency axis and do unit conversions.
n_time = power_mat.shape[0]
n_chan = power_mat.shape[-1]
frequency = npow.ps_freq_axis(dt, n_time)
power_mat = npow.prune_power(power_mat, 0)
power_mat = npow.make_power_physical_units(power_mat, dt)
# Discard the mean mode.
frequency = frequency[1:]
power_mat = power_mat[1:, ...]
n_f = len(frequency)
# Loop over polarizations.
cal_ind = 0
n_pols = power_mat.shape[1]
for ii in range(n_pols):
this_pol_power = power_mat[:, ii, cal_ind, :]
this_pol_window = window_function[:, ii, cal_ind, :]
this_pol = Blocks[0].field['CRVAL4'][ii]
this_pol_parameters = {}
# If we are plotting, activate the subplot for this polarization and
# this band.
if plots:
h = plt.gcf()
# The last subplot should be hidden in the figure object.
current_subplot = h.current_subplot
current_subplot = current_subplot[:2] + (current_subplot[2] +
1, )
h.current_subplot = current_subplot
# Now figure out what we want to measure and measure it.
if "channel_var" in parameters:
power_diag = this_pol_power.view()
window_function_diag = this_pol_window.view()
# Integral of the power spectrum from -BW to BW.
channel_var = sp.mean(power_diag, 0) / dt
# Normalize for the window.
norms = sp.mean(window_function_diag, 0).real
bad_inds = norms < 10. / n_time
norms[bad_inds] = 1
channel_var /= norms
# If a channel is completly masked Deweight it by giving a high
# variance
channel_var[bad_inds] = T_infinity**2
this_pol_parameters["channel_var"] = channel_var
out_parameters[this_pol] = this_pol_parameters
return out_parameters
else:
# Calculate the full correlated power spectrum.
power_mat, window_function, dt, channel_means = npow.full_power_mat(
Blocks,
window="hanning",
deconvolve=False,
n_time=-1.05,
normalize=False,
split_scans=split_scans,
subtract_slope=True)
# This shouldn't be nessisary, since I've tried to keep things finite in
# the above function. However, leave it in for now just in case.
if not sp.alltrue(sp.isfinite(power_mat)):
msg = ("Non finite power spectrum calculated. Offending data in "
"file starting with scan %d." % (Blocks[0].field['SCAN']))
raise ce.DataError(msg)
# Get frequency axis and do unit conversions.
n_time = power_mat.shape[0]
n_chan = power_mat.shape[-1]
frequency = npow.ps_freq_axis(dt, n_time)
power_mat = npow.prune_power(power_mat, 0)
power_mat = npow.make_power_physical_units(power_mat, dt)
# Discard the mean mode.
frequency = frequency[1:]
power_mat = power_mat[1:, ...]
n_f = len(frequency)
# Loop over polarizations.
cal_ind = 0
n_pols = power_mat.shape[1]
for ii in range(n_pols):
this_pol_power = power_mat[:, ii, cal_ind, :, :]
this_pol_window = window_function[:, ii, cal_ind, :, :]
this_pol = Blocks[0].field['CRVAL4'][ii]
this_pol_parameters = {}
# If we are plotting, activate the subplot for this polarization and
# this band.
if plots:
h = plt.gcf()
# The last subplot should be hidden in the figure object.
current_subplot = h.current_subplot
current_subplot = current_subplot[:2] + (current_subplot[2] +
1, )
h.current_subplot = current_subplot
# Now figure out what we want to measure and measure it.
if "channel_var" in parameters:
power_diag = this_pol_power.view()
power_diag.shape = (n_f, n_chan**2)
power_diag = power_diag[:, ::n_chan + 1].real
window_function_diag = this_pol_window.view()
window_function_diag.shape = (n_time, n_chan**2)
window_function_diag = window_function_diag[:, ::n_chan + 1]
# Integral of the power spectrum from -BW to BW.
channel_var = sp.mean(power_diag, 0) / dt
# Normalize for the window.
norms = sp.mean(window_function_diag, 0).real
bad_inds = norms < 10. / n_time
norms[bad_inds] = 1
channel_var /= norms
# If a channel is completly masked Deweight it by giving a high
# variance
channel_var[bad_inds] = T_infinity**2
this_pol_parameters["channel_var"] = channel_var
for noise_model in parameters:
if noise_model[:18] == "freq_modes_over_f_":
n_modes = int(noise_model[18:])
this_pol_parameters[noise_model] = \
get_freq_modes_over_f(this_pol_power, this_pol_window,
frequency, n_modes, plots=plots)
out_parameters[this_pol] = this_pol_parameters
return out_parameters
def multiply_by_cal(Data, CalData):
"""Function scales data by the noise cal temperature.
"""
# For now we just assume that the cal and polarizations are arranged in a
# certain way and then check to make sure we are right.
calibrate_to_I = False
if tuple(Data.field['CRVAL4']) == (-5, -7, -8, -6):
xx_ind = 0
yy_ind = 3
xy_inds = [1, 2]
elif tuple(Data.field['CRVAL4']) == (1, 2, 3, 4):
# This is a hack. Completly temporairy.
calibrate_to_I = True
else:
raise ce.DataError('Polarization types not as expected in data.')
cal_xx_ind = 0
cal_yy_ind = 1
if (CalData.field['CRVAL4'][cal_xx_ind] != -5
or CalData.field['CRVAL4'][cal_yy_ind] != -6):
raise ce.DataError('Polarization types not as expected in cal.')
# Cal should only have 1 time, 1 cal state and 2 polarizations.
if CalData.dims[:3] != (1, 2, 1):
raise ce.DataError('Cal temperature data has wrong dimensions.')
# Cal state should be special state 'R'.
if CalData.field['CAL'][0] != 'R':
raise ce.DataError("Cal state in cal temperture data should be "
"'R'.")
# Bring the Cal data to the same frequencies as the other data.
Data.calc_freq()
CalData.calc_freq()
if sp.allclose(Data.freq, CalData.freq):
cdata = CalData.data
elif abs(Data.field['CDELT1']) <= abs(CalData.field['CDELT1']):
calfunc = interpolate.interp1d(CalData.freq,
CalData.data,
fill_value=sp.nan,
bounds_error=False)
cdata = ma.array(calfunc(Data.freq))
cdata[sp.logical_not(sp.isfinite(cdata))] = ma.masked
else:
nf = len(Data.freq)
width = abs(Data.field['CDELT1'])
cdata = ma.empty((1, 2, 1, nf))
for find in range(nf):
f = Data.freq[find]
inds, = sp.where(
sp.logical_and(CalData.freq >= f - width / 2.0,
CalData.freq < f + width / 2.0))
cdata[:, :, :, find] = ma.mean(CalData.data[:, :, :, inds], 3)
if calibrate_to_I:
Data.data *= (cdata[0, cal_xx_ind, 0, :] +
cdata[0, cal_yy_ind, 0, :]) / 2.0
else:
# Loop over times and cal and scale each polarization appropriately.
for tind in range(Data.dims[0]):
for cind in range(Data.dims[2]):
Data.data[tind, xx_ind, cind, :] *= cdata[0, cal_xx_ind, 0, :]
Data.data[tind, yy_ind, cind, :] *= cdata[0, cal_yy_ind, 0, :]
Data.data[tind, xy_inds, cind, :] *= ma.sqrt(
cdata[0, cal_yy_ind, 0, :] * cdata[0, cal_xx_ind, 0, :])
def calibrate_pol(Data, m_total, RM_dir, R_to_sky, DP_correct, RM_correct):
"""Subtracts a Map out of Data."""
# Data is a DataBlock object. It holds everything you need to know about
# the data in a single scan and IF. You should get to know them very well.
# Data.data is a numpy masked array (see numpy documentation) and holds the
# acctual data. It is a 4 dimensional array. The demensions are (in
# order): (time, pol, cal, freq). Each dimension can be any length which
# you can figure out by looking at Data.dims = sp.shape(Data.data).
# Data.field is a python dictionary that holds all the other data that you
# might care about from the origional fits file. For instance,
# Data.field['CAL'] is an array with length dims[2]. It normally has
# values ['T', 'F']. Data.field['CRVAL4'] tells you about the polarization
# axis of Data.data. By SDfits convension each polarization is represented
# by an integer: 1=I, 2=Q, 3=U, 4=V, -5=XX, -6=YY, -7=XY, -8=YX.
# Also this depends on having the polarizations rotated correctly to IQUV.
# Some code to do this has been hacked together in the rotate_pol module,
# but I don't trust it yet.
# Some dimension checks.
# We expect 4 polarizations.
if not Data.dims[1] == 4:
raise ce.DataError('Require 4 polarizations.')
# We expect polarizations to be in order IQUV.
if (Data.field['CRVAL4'][0] != -5 or Data.field['CRVAL4'][1] != -7
or Data.field['CRVAL4'][2] != -8 or Data.field['CRVAL4'][3] != -6):
raise ce.DataError('Expected the polarization basis to be XY.')
# A useful function that might need:
Data.calc_freq()
# Now data has an atribute Data.freq which is an array that gives the
# frequency along the last axis.
# Data.field['CRVAL1'] is center frequency in Hz.
# Data.data 4 dim array 2nd index polarization, 4th index frequency.
# Need to get parallactic angle:
Data.calc_PA()
# This gives an array (Data.PA) of PA values of length = time dim.
# print Data.dims[0]
# This segment of the code is for Rotation Measure Component
# Since the RM Tables have half hour time divisions and scans are shorter, we can do 1 selection.
if RM_correct == True:
Comp_Time = 0.0
Full_date = Data.field['DATE-OBS'][Data.dims[0] / 2]
Date = Full_date.split('T')[0]
Year = Date.split('-')[0]
Month = Date.split('-')[1]
Day = Date.split('-')[2]
Full_time = Full_date.split('T')[1]
Hour = Full_time.split(':')[0]
Min = Full_time.split(':')[1]
Sec = Full_time.split(':')[2]
if int(Min) <= 15:
Comp_Time = float(Hour) + 0.0
elif int(Min) <= 45:
Comp_Time = float(Hour) + 0.5
else:
Comp_Time = float(Hour) + 1.0
#Victor's tables have time in format Hour (xx.xx), Az (deg), El (deg), RM
# Angle phi = RM*(wavelength)^2 where phi is in radians and wavelength is in meters
RM_file_name = RM_dir + Year + Month + Day + '_RM.txt'
RM_data = np.loadtxt(RM_file_name)
RA_RM = sp.zeros(len(RM_data[:, 0]))
DEC_RM = sp.zeros(len(RM_data[:, 0]))
for i in range(0, len(RM_data[:, 0])):
RM_Hr = int(RM_data[i, 0])
if RM_data[i, 0] % 1 == 0:
RM_Min = '00'
minutes = 0.0
else:
RM_Min = '30'
minutes = 0.5
Test = float(RM_Hr) + minutes
if str(Comp_Time) == str(Test):
UT_RM = Year + '-' + Month + '-' + Day + 'T' + str(
RM_Hr) + ':' + RM_Min + ':00.00'
EL_RM = RM_data[i, 2]
AZ_RM = RM_data[i, 1]
RA_RM[i], DEC_RM[i] = utils.elaz2radecGBT(EL_RM, AZ_RM, UT_RM)
#Now have tables of RA/DEC to compare to actual RA/DEC
RM = 0
#This segment of the code is for Differential Phase Correction Generation
# Can determine the differential phase prior to the loop:
if DP_correct == True:
# Set up a table of data to examine (calon-caloff to get Tcal)
Tcal = ma.mean(Data.data[:, :, 0, :], axis=0) - ma.mean(
Data.data[:, :, 1, :], axis=0)
# Tcal = ma.mean(Data.data[:,:,0,:]-Data.data[:,:,1,:],axis=0)
# This version was if we arbitrarily set 4 possible phases and found closest match. There seems to be
# enough variability in the phase within the four categories that this doesn't quite work.
# Randomly pick frequency bin near one of the zero crossings to compare U's
# U_test = Tcal[1,230]/sp.sqrt(Tcal[1,230]**2+Tcal[2,230]**2)
# print Tcal[:,191]
# print Tcal[1,:]
# U_test = Tcal[1,191]
# print U_test
# chi_sq =sp.zeros(4)
# dp_dat = sp.zeros((4,2))
# dp_dat[0] = [0.1354,2.341]
# dp_dat[1] = [0.0723, 2.4575]
# dp_dat[1] = [0.0730,2.611] #calculated specifically for sess 81
# dp_dat[2] = [0.1029,0.045]
# dp_dat[3] = [0,0]
# dp_dat[3] = [0.1669,5.609] # giving problems because closer for sess 81 at given freq
# min = 10
# val = 5
# for i in range(0,4):
# chi_sq[i] = U_test-sp.cos(dp_dat[i,0]*Data.freq[230]/1000000+dp_dat[i,1])/sp.sqrt(Tcal[1,230]**2+Tcal[2,230]**2)
# print sp.cos(dp_dat[i,0]*Data.freq[191]/1000000+dp_dat[i,1])
# chi_sq[i] = U_test-sp.cos(dp_dat[i,0]*Data.freq[191]/1000000+dp_dat[i,1])
# if abs(chi_sq[i]) < min:
# min = abs(chi_sq[i])
# val = i
# val tells which of the correction functions to use.
# print chi_sq
# print val
# print Data.freq[191]
# Alternate code for solving differential phase for each scan.
fitfunc = lambda p, x: sp.cos(p[0] * x + p[1])
errfunc = lambda p, x, y: fitfunc(p, x) - y
freqs = sp.zeros(Data.dims[3])
U_data = sp.zeros(Data.dims[3])
V_data = sp.zeros(Data.dims[3])
R_data = sp.zeros(Data.dims[3])
for i in range(0, Data.dims[3]):
freqs[i] = Data.freq[Data.dims[3] - i - 1] / 1000000
U_data[i] = Tcal[1, Data.dims[3] - i - 1]
V_data[i] = Tcal[2, Data.dims[3] - i - 1]
R_data[i] = U_data[i] / sp.sqrt(U_data[i]**2 + V_data[i]**2)
# print np.any(np.isnan(R_data))
# print np.any(np.isinf(R_data))
# print freqs
for j in range(0, Data.dims[3]):
if int(freqs[j]) == 710:
mask_num = j
if int(freqs[j]) == 740:
mask_num2 = j
Datain = R_data[mask_num:mask_num2]
fin = freqs[mask_num:mask_num2]
bad_pts = np.logical_or(np.isnan(Datain), np.isinf(Datain))
good_ind = np.where(np.logical_not(bad_pts))
Datain = Datain[good_ind]
fin = fin[good_ind]
R0 = [0.18, 1.0]
if len(good_ind[0]) > 1:
# print good_ind[0]
R, success = optimize.leastsq(errfunc,
R0[:],
args=(fin, Datain),
maxfev=10000)
R[1] = R[1] % (2 * sp.pi)
print R
else:
R = [0.0, 0.0]
print "Not able to resolve a noise cal phase, setting phase to zero."
# This starts the actual data processing for the given scan
for time_index in range(0, Data.dims[0]):
# Extra data needed for Rotation Measure Correction
if RM_correct == True:
RA = Data.field['CRVAL2'][time_index]
DEC = Data.field['CRVAL3'][time_index]
# print RA
# print DEC
RM = 0
valid = []
for i in range(0, len(RA_RM)):
if RA_RM[i] != 0:
if abs(RA - RA_RM[i]) < 10.0:
if abs(DEC - DEC_RM[i]) < 10.0:
RM = RM_data[i, 3]
valid.append(i)
RA_M = 10.0
DEC_M = 10.0
for j in range(0, len(valid)):
if abs(RA - RA_RM[valid[j]]) < RA_M:
if abs(DEC - DEC_RM[valid[j]]) < DEC_M:
RM = RM_data[valid[j], 3]
# print RM
#Generate a sky matrix for this time index (assumes a XY basis):
m_sky = sp.zeros((4, 4))
m_sky[0, 0] = 0.5 * (1 + ma.cos(2 * Data.PA[time_index] * sp.pi / 180))
m_sky[0, 1] = -ma.sin(2 * Data.PA[time_index] * sp.pi / 180)
m_sky[0, 3] = 0.5 * (1 - ma.cos(2 * Data.PA[time_index] * sp.pi / 180))
m_sky[1, 0] = 0.5 * ma.sin(2 * Data.PA[time_index] * sp.pi / 180)
m_sky[1, 1] = ma.cos(2 * Data.PA[time_index] * sp.pi / 180)
m_sky[1, 3] = -0.5 * ma.sin(2 * Data.PA[time_index] * sp.pi / 180)
m_sky[2, 2] = 1
m_sky[3, 0] = 0.5 * (1 - ma.cos(2 * Data.PA[time_index] * sp.pi / 180))
m_sky[3, 1] = ma.sin(2 * Data.PA[time_index] * sp.pi / 180)
m_sky[3, 3] = 0.5 * (1 + ma.cos(2 * Data.PA[time_index] * sp.pi / 180))
M_sky = sp.mat(m_sky)
M_sky = M_sky.I
# print M_sky
# for cal_index in range(0,Data.dims[2]):
for cal_index in range(0, 2):
# Determines the Gains to use
for freq in range(0, Data.dims[3]):
# Tells which mueller matrix to use.
freq_limit = len(m_total[0, :])
frequency = int(Data.freq[freq] / 1000)
# print frequency
bin = int((900000 - frequency) * freq_limit / 200000)
# print bin
# if freq_limit == 200:
# bin = 900-frequency
#Not setup to work with spectrometer data.
# elif freq_limit == 260:
# bin = 929-frequency
# print bin
#Generate a sky matrix for this time index:
#With faraday rotation sky matrix now frequency dependent
if RM_correct == True:
wavelength = 300000.0 / float(
frequency
) # should be in meters given that frequency is in kHz
# print wavelength
Phi = RM * wavelength * wavelength
# print Phi
m_sky = sp.zeros((4, 4))
m_sky[0, 0] = 0.5 * (
1 +
ma.cos(2 * Data.PA[time_index] * sp.pi / 180 + Phi))
m_sky[0,
1] = -ma.sin(2 * Data.PA[time_index] * sp.pi / 180 +
Phi)
m_sky[0, 3] = 0.5 * (
1 -
ma.cos(2 * Data.PA[time_index] * sp.pi / 180 + Phi))
m_sky[1, 0] = 0.5 * ma.sin(2 * Data.PA[time_index] *
sp.pi / 180 + Phi)
m_sky[1,
1] = ma.cos(2 * Data.PA[time_index] * sp.pi / 180 +
Phi)
m_sky[1, 3] = -0.5 * ma.sin(2 * Data.PA[time_index] *
sp.pi / 180 + Phi)
m_sky[2, 2] = 1
m_sky[3, 0] = 0.5 * (
1 -
ma.cos(2 * Data.PA[time_index] * sp.pi / 180 + Phi))
m_sky[3,
1] = ma.sin(2 * Data.PA[time_index] * sp.pi / 180 +
Phi)
m_sky[3, 3] = 0.5 * (
1 +
ma.cos(2 * Data.PA[time_index] * sp.pi / 180 + Phi))
M_sky = sp.mat(m_sky)
M_sky = M_sky.I
# print M_sky
# Converts files into vector format
XY_params = Data.data[time_index, :, cal_index, freq]
# Next there is a matrix multiplication that will generate
# a new set of xy values. (Differential gain correction)
XY_params[0] = XY_params[0] * m_total[0, bin]
XY_params[3] = XY_params[3] * m_total[1, bin]
XY_params[1] = XY_params[1] * sp.sqrt(
m_total[0, bin] * m_total[1, bin])
XY_params[2] = XY_params[2] * sp.sqrt(
m_total[0, bin] * m_total[1, bin])
# Add in correction for differential phase
if DP_correct == True:
XY_params[1] = XY_params[1] * sp.cos(
R[0] * frequency / 1000 + R[1]
) - XY_params[2] * sp.sin(R[0] * frequency / 1000 + R[1])
XY_params[2] = XY_params[1] * sp.sin(
R[0] * frequency / 1000 + R[1]
) + XY_params[2] * sp.cos(R[0] * frequency / 1000 + R[1])
#Rotate to sky coordinates (and RM correct if set)
if R_to_sky == True:
XY_params = np.dot(M_sky, XY_params)
#Write corrected data to the new file.
for i in range(0, Data.dims[1]):
Data.data[time_index, i, cal_index, freq] = XY_params[i]
def process_file(self, file_ind):
params = self.params
file_middle = params['file_middles'][file_ind]
input_fname = (params['input_root'] + file_middle +
params['input_end'])
sub_input_fname = (params['subtracted_input_root'] + file_middle +
params['input_end'])
output_fname = (params['output_root'] + file_middle +
params['output_end'])
sub_output_fname = (params['subtracted_output_root'] + file_middle +
params['output_end'])
Writer = fitsGBT.Writer(feedback=self.feedback)
SubWriter = fitsGBT.Writer(feedback=self.feedback)
# Read in the data, and loop over data blocks.
Reader = fitsGBT.Reader(input_fname, feedback=self.feedback)
SubReader = fitsGBT.Reader(sub_input_fname, feedback=self.feedback)
if (sp.any(Reader.scan_set != SubReader.scan_set)
or sp.any(Reader.IF_set != SubReader.IF_set)):
raise ce.DataError("IFs and scans don't match signal subtracted"
" data.")
# Get the number of scans if asked for all of them.
scan_inds = params['scans']
if len(scan_inds) == 0 or scan_inds is None:
scan_inds = range(len(Reader.scan_set))
if_inds = params['IFs']
if len(if_inds) == 0 or scan_inds is None:
if_inds = range(len(Reader.IF_set))
if self.feedback > 1:
print "New flags each block:",
# Loop over scans and IFs
for thisscan in scan_inds:
for thisIF in if_inds:
Data = Reader.read(thisscan, thisIF)
SubData = SubReader.read(thisscan, thisIF)
# Make sure they have agreeing masks to start.
SubData.data[ma.getmaskarray(Data.data)] = ma.masked
Data.data[ma.getmaskarray(SubData.data)] = ma.masked
# Get initial number of flags.
n_flags = ma.count_masked(Data.data)
# Now do the flagging.
flag(Data, SubData, params['thres'],
params['max_noise_factor'],
params['smooth_modes_subtract'], params['filter_type'])
Data.add_history(
"Reflaged for outliers.",
("Used file: " +
utils.abbreviate_file_path(sub_input_fname), ))
SubData.add_history("Reflaged for outliers.")
Writer.add_data(Data)
SubWriter.add_data(SubData)
# Report the number of new flags.
n_flags = ma.count_masked(Data.data) - n_flags
if self.feedback > 1:
print n_flags,
if self.feedback > 1:
print ''
# Finally write the data back to file.
utils.mkparents(output_fname)
utils.mkparents(sub_output_fname)
Writer.write(output_fname)
SubWriter.write(sub_output_fname)
def make_masked_time_stream(Blocks,
ntime=None,
window=None,
return_means=False,
subtract_slope=False):
"""Converts Data Blocks into a single uniformly sampled time stream.
Also produces the mask giving whether elements are valid entries or came
from a zero pad. This produes the required inputs for calculating a
windowed power spectrum.
Parameters
----------
Blocks : tuple of DataBlock objects.
ntime : int
Total number of time bins in output arrays. If shorter than required
extra data is truncated. If longer, extra data is masked. Default is
to use exactly the number that fits all the data. Set to a negitive
factor to zero pad to a power of 2 and by at least at least the factor.
window : string or tuple
Type of window to apply to each DataBlock. Valid options are the valid
arguments to scipy.signal.get_window(). By default, don't window.
return_means : bool
Whether to return an array of the channed means.
subtract_slope : bool
Whether to subtract a linear function of time from each channel.
Returns
-------
time_stream : array
All the data in `Blocks` but concatenated along the time axis and
padded with zeros such that the time axis is uniformly sampled and
uninterupted.
mask : array same shape as `time_stream`
1.0 if data in the correspoonding `time_stream` element is filled
and 0 if the data was missing. This is like a window where
time_stream = mask*real_data.
dt : float
The time step of the returned time stream.
means : array (optional)
The mean from each channel.
"""
# Shape of all axes except the time axis.
back_shape = Blocks[0].dims[1:]
# Get the time sample spacing.
Blocks[0].calc_time()
dt = abs(sp.mean(sp.diff(Blocks[0].time)))
# Find the beginning and the end of the time axis by looping through
# blocks.
# Also get the time axis and the mask
# for calculating basis polynomials.
unmask = sp.zeros((0, ) + back_shape, dtype=bool)
time = sp.zeros((0, ), dtype=float)
start_ind = []
min_time = float('inf')
max_time = 0.0
#mean_time = 0.0
#n_data_times = 0
for Data in Blocks:
Data.calc_time()
start_ind.append(len(time))
time = sp.concatenate((time, Data.time))
this_unmask = sp.logical_not(ma.getmaskarray(Data.data))
unmask = sp.concatenate((unmask, this_unmask), 0)
# Often the start or the end of a scan is completly masked. Make sure
# we don't start till the first unmasked time and end at the last
# unmasked time.
time_unmask = sp.alltrue(ma.getmaskarray(Data.data), -1)
time_unmask = sp.alltrue(time_unmask, -1)
time_unmask = sp.alltrue(time_unmask, -1)
if sp.alltrue(time_unmask):
continue
time_unmask = sp.logical_not(time_unmask)
min_time = min(min_time, min(Data.time[time_unmask]))
max_time = max(min_time, max(Data.time[time_unmask]))
#mean_time += sp.sum(Data.time[time_unmask])
#n_data_times += len(Data.time[time_unmask])
# Ensure that the time sampling is uniform.
if not (sp.allclose(abs(sp.diff(Data.time)), dt, rtol=0.1) and
sp.allclose(abs(sp.mean(sp.diff(Data.time))), dt, rtol=0.001)):
msg = ("Time sampling not uniformly spaced or Data Blocks don't "
"agree on sampling.")
raise ce.DataError(msg)
# Ensure the shapes are right.
if Data.dims[1:] != back_shape:
msg = ("All data blocks must have the same shape except the time "
"axis.")
raise ce.DataError(msg)
# Now calculate basis polynomials for the mean mode and the slope mode.
polys = misc.ortho_poly(time[:, None, None, None], 2, unmask, 0)
#mean_time /= n_data_times
#if n_data_times == 0:
# n_data_times = 1
# Very important to subtract the mean out of the signal, otherwise the
# window coupling to the mean (0) mode will dominate everything. Can also
# optionally take out a slope.
# Old algorithm.
#total_sum = 0.0
#total_counts = 0
#total_slope = 0.0
#time_norm = 0.0
#for Data in Blocks:
# total_sum += sp.sum(Data.data.filled(0), 0)
# total_counts += ma.count(Data.data, 0)
# total_slope += sp.sum(Data.data.filled(0)
# * (Data.time[:,None,None,None] - mean_time), 0)
# time_norm += sp.sum(sp.logical_not(ma.getmaskarray(Data.data))
# * (Data.time[:,None,None,None] - mean_time)**2, 0)
#total_counts[total_counts == 0] = 1
#time_norm[time_norm == 0.0] = 1
#total_mean = total_sum / total_counts
#total_slope /= time_norm
# New algorithm.
mean_amp = 0
slope_amp = 0
for ii, Data in enumerate(Blocks):
si = start_ind[ii]
this_nt = Data.dims[0]
data = Data.data.filled(0)
mean_amp += sp.sum(
data * unmask[si:si + this_nt, ...] *
polys[0, si:si + this_nt, ...], 0)
slope_amp += sp.sum(
data * unmask[si:si + this_nt, ...] *
polys[1, si:si + this_nt, ...], 0)
polys[0, ...] *= mean_amp
polys[1, ...] *= slope_amp
# Calculate the time axis.
if min_time > max_time:
min_time = 0
max_time = 6 * dt
if not ntime:
ntime = (max_time - min_time) // dt + 1
elif ntime < 0:
# 0 pad by a factor of at least -ntime, but at most 10% more than this.
time_min = -ntime * (max_time - min_time) / dt
n_block = 1
while n_block < time_min / 20.0:
n_block *= 2
ntime = (time_min // n_block + 1) * n_block
time = sp.arange(ntime) * dt + min_time
# Allowcate memory for the outputs.
time_stream = sp.zeros((ntime, ) + back_shape, dtype=float)
mask = sp.zeros((ntime, ) + back_shape, dtype=sp.float32)
# Loop over all times and fill in the arrays.
for ii, Data in enumerate(Blocks):
this_nt = Data.dims[0]
si = start_ind[ii]
# Subtract the mean calculated above.
this_data = Data.data.copy()
this_data -= polys[0, si:si + this_nt, ...]
# If desired, subtract of the linear function of time.
if subtract_slope:
#this_data -= (total_slope
# * (Data.time[:,None,None,None] - mean_time))
this_data -= polys[1, si:si + this_nt, ...]
# Find the first and last unmasked times.
time_unmask = sp.alltrue(ma.getmaskarray(this_data), -1)
time_unmask = sp.alltrue(time_unmask, -1)
time_unmask = sp.alltrue(time_unmask, -1)
if sp.alltrue(time_unmask):
continue
time_unmask = sp.logical_not(time_unmask)
unmasked_ind, = sp.where(time_unmask)
first_ind = min(unmasked_ind)
last_ind = max(unmasked_ind)
# Ensure that the time sampling is uniform.
if not (sp.allclose(abs(sp.diff(Data.time)), dt, rtol=0.1) and
sp.allclose(abs(sp.mean(sp.diff(Data.time))), dt, rtol=0.001)):
msg = ("Time sampling not uniformly spaced or Data Blocks don't "
"agree on sampling.")
raise ce.DataError(msg)
# Ensure the shapes are right.
if Data.dims[1:] != back_shape:
msg = ("All data blocks must have the same shape except the time "
"axis.")
# Apply an offset to the time in case the start of the Data Block
# doesn't line up with the time array perfectly.
offset = (time[sp.argmin(abs(time - Data.time[first_ind]))] -
Data.time[first_ind])
# Generate window function.
if window:
window_function = sig.get_window(window, last_ind - first_ind + 1)
for ii in range(first_ind, last_ind + 1):
ind = sp.argmin(abs(time - (Data.time[ii] + offset)))
if abs(time[ind] - (Data.time[ii])) < 0.5 * dt:
if sp.any(mask[ind, ...]):
msg = "Overlapping times in Data Blocks."
raise ce.DataError(msg)
if window:
window_value = window_function[ii - first_ind]
else:
window_value = 1.0
time_stream[ind, ...] = (window_value *
this_data[ii, ...].filled(0.0))
mask[ind, ...] = window_value * sp.logical_not(
ma.getmaskarray(this_data)[ii, ...])
if return_means:
return time_stream, mask, dt, polys[0, 0, ...]
else:
return time_stream, mask, dt
