def test_known(self):
dims = self.Blocks[0].dims
data = ma.ones(dims) * sp.arange(1, dims[-1] + 1)
data[0:-1:2, 0, 0, :] = 2 * sp.arange(1, dims[-1] + 1)
for Data in self.Blocks:
Data.data = data
var = tools.calc_time_var_file(self.Blocks, 0, 0)
self.assertTrue(sp.allclose(var.filled(), sp.arange(1, dims[-1] + 1) ** 2 / 4.0))
def test_known2(self):
dims = self.Blocks[0].dims
data = ma.ones(dims) * sp.arange(1, dims[-1] + 1)
ii = 0.0
for Data in self.Blocks:
ii += 1.0
Data.data = data * ii
var = tools.calc_time_var_file(self.Blocks, 0, 0)
self.assertTrue(sp.allclose(var.filled(), sp.arange(1, dims[-1] + 1) ** 2 / 4.0))
def test_known(self):
dims = self.Blocks[0].dims
data = ma.ones(dims) * sp.arange(1, dims[-1] + 1)
data[0:-1:2, 0, 0, :] = 2 * sp.arange(1, dims[-1] + 1)
for Data in self.Blocks:
Data.data = data
var = tools.calc_time_var_file(self.Blocks, 0, 0)
self.assertTrue(
sp.allclose(var.filled(),
sp.arange(1, dims[-1] + 1)**2 / 4.0))
def test_known2(self):
dims = self.Blocks[0].dims
data = ma.ones(dims) * sp.arange(1, dims[-1] + 1)
ii = 0.0
for Data in self.Blocks:
ii += 1.0
Data.data = data * ii
var = tools.calc_time_var_file(self.Blocks, 0, 0)
self.assertTrue(
sp.allclose(var.filled(),
sp.arange(1, dims[-1] + 1)**2 / 4.0))
def test_known_with_masked(self):
dims = self.Blocks[0].dims
data = ma.ones(dims) * sp.arange(1, dims[-1] + 1)
data[0:-1:2, 0, 0, :] = 2 * sp.arange(1, dims[-1] + 1)
for Data in self.Blocks:
Data.data = data
self.Blocks[0].data[2, 0, 0, 43] = ma.masked
self.Blocks[0].data[7, 0, 0, 43] = ma.masked
self.Blocks[1].data[4, 0, 0, 43] = ma.masked
self.Blocks[1].data[3, 0, 0, 43] = ma.masked
self.Blocks[0].data[:, 0, 0, 103] = ma.masked
self.Blocks[1].data[:, 0, 0, 103] = ma.masked
self.Blocks[0].data[:, 0, 0, 554] = ma.masked
self.Blocks[1].data[1:, 0, 0, 554] = ma.masked
var = tools.calc_time_var_file(self.Blocks, 0, 0)
expected = ma.arange(1, dims[-1] + 1) ** 2 / 4.0
expected[103] = ma.masked
expected[554] = ma.masked
self.assertTrue(sp.allclose(var.filled(-1), expected.filled(-1)))
def test_known_with_masked(self):
dims = self.Blocks[0].dims
data = ma.ones(dims) * sp.arange(1, dims[-1] + 1)
data[0:-1:2, 0, 0, :] = 2 * sp.arange(1, dims[-1] + 1)
for Data in self.Blocks:
Data.data = data
self.Blocks[0].data[2, 0, 0, 43] = ma.masked
self.Blocks[0].data[7, 0, 0, 43] = ma.masked
self.Blocks[1].data[4, 0, 0, 43] = ma.masked
self.Blocks[1].data[3, 0, 0, 43] = ma.masked
self.Blocks[0].data[:, 0, 0, 103] = ma.masked
self.Blocks[1].data[:, 0, 0, 103] = ma.masked
self.Blocks[0].data[:, 0, 0, 554] = ma.masked
self.Blocks[1].data[1:, 0, 0, 554] = ma.masked
var = tools.calc_time_var_file(self.Blocks, 0, 0)
expected = ma.arange(1, dims[-1] + 1)**2 / 4.0
expected[103] = ma.masked
expected[554] = ma.masked
self.assertTrue(sp.allclose(var.filled(-1), expected.filled(-1)))
def execute(self, nprocesses=1):
"""Function that acctually does the work.
The nprocesses parameter does not do anything yet. It is just there
for compatibility with the pipeline manager.
"""
params = self.params
kiyopy.utils.mkparents(params["output_root"])
parse_ini.write_params(params, params["output_root"] + "params.ini", prefix="mm_")
# Rename some commonly used parameters.
map_shape = params["map_shape"]
spacing = params["pixel_spacing"]
algorithm = params["noise_model"]
noise_root = params["noise_parameters_input_root"]
ra_spacing = -spacing / sp.cos(params["field_centre"][1] * sp.pi / 180.0)
if not algorithm in ("grid", "diag_file", "disjoint_scans"):
raise ValueError("Invalid noise model: " + algorithm)
if len(params["IFs"]) != 1:
raise ce.FileParameterTypeError("Can only process a single IF.")
# Set up to iterate over the pol states.
npol = 2 # This will be reset when we read the first data block.
pol_ind = 0
all_file_names = []
while pol_ind < npol:
# Flag for the first block processed (will allowcate memory on the
# first iteration).
first_block = True
# Loop over the files to process.
try:
for file_middle in params["file_middles"]:
input_fname = params["input_root"] + file_middle + params["input_end"]
# Read in the data, and loop over data blocks.
Reader = fitsGBT.Reader(input_fname, feedback=self.feedback)
Blocks = Reader.read(params["scans"], params["IFs"])
# Calculate the time varience at each frequency. This will
# be used as weights in most algorithms.
if not algorithm == "grid":
if not noise_root == "None":
# We have measured variance.
noise_pars = sp.load(noise_root + file_middle + ".npy")
var = noise_pars[params["IFs"][0], pol_ind, 0, :]
else:
# We need to measure the variance.
var = tools.calc_time_var_file(Blocks, pol_ind, 0)
# Convert from masked array to array.
var = var.filled(9999.0)
else:
var = 1.0
weight = 1 / var
for Data in Blocks:
dims = Data.dims
# On first pass set up the map parameters.
if first_block:
shape = map_shape + (dims[-1],)
Data.calc_freq()
centre_freq = Data.freq[dims[-1] // 2]
delta_freq = Data.field["CDELT1"]
if pol_ind == 0:
# Figure out the length of the polarization
# loop.
npol = dims[1]
# Accumulate the data history.
history = hist.History(Data.history)
# Get the current polarization integer.
this_pol = Data.field["CRVAL4"][pol_ind]
# Check that we even want to make a dirty map for
# this polarization.
if (not utils.polint2str(this_pol) in params["polarizations"]) and params["polarizations"]:
# Break to the end of the polarization loop.
raise ce.NextIteration()
# Allowcate memory for the map.
map_data = sp.zeros(shape, dtype=float)
map_data = algebra.make_vect(map_data, axis_names=("ra", "dec", "freq"))
# Allowcate memory for the inverse map noise.
if algorithm in ("grid", "diag_file"):
noise_inv = sp.zeros(shape, dtype=float)
noise_inv = algebra.make_mat(
noise_inv, axis_names=("ra", "dec", "freq"), row_axes=(0, 1, 2), col_axes=(0, 1, 2)
)
elif algorithm in ("disjoint_scans", "ds_grad"):
# At each frequency use full N^2 noise matrix,
# but assume each frequency has uncorrelated
# noise. This is a big matrix so make sure it
# is reasonable.
size = shape[0] ^ 2 * shape[1] ^ 2 * shape[2]
if size > 4e9: # 16 GB
raise RunTimeError("Map size too big. " "Asked for a lot " "of memory.")
noise_inv = sp.zeros(shape[0:2] + shape, dtype=sp.float32)
noise_inv = algebra.make_mat(
noise_inv,
axis_names=("ra", "dec", "ra", "dec", "freq"),
row_axes=(0, 1, 4),
col_axes=(2, 3, 4),
)
# Allowcate memory for temporary data. Hold the
# number of times each pixel in this scan is
# hit. Factor of 2 longer in time in case some
# scans are longer than first block (guppi).
pixel_hits = sp.empty((2 * dims[0], dims[-1]))
first_block = False
else:
if pol_ind == 0:
history.merge(Data)
# Figure out the pointing pixel index and the frequency
# indicies.
Data.calc_pointing()
ra_inds = tools.calc_inds(Data.ra, params["field_centre"][0], shape[0], ra_spacing)
dec_inds = tools.calc_inds(
Data.dec, params["field_centre"][1], shape[1], params["pixel_spacing"]
)
data = Data.data[:, pol_ind, 0, :]
if algorithm in ("grid", "diag_file"):
add_data_2_map(data, ra_inds, dec_inds, map_data, noise_inv, weight)
elif algorithm in ("disjoint_scans",):
add_data_2_map(data - ma.mean(data, 0), ra_inds, dec_inds, map_data, None, weight)
pixel_hits[:] = 0
pixel_list = pixel_counts(data, ra_inds, dec_inds, pixel_hits, map_shape=shape[0:2])
add_scan_noise(pixel_list, pixel_hits, var, noise_inv)
# End Blocks for loop.
# End file name for loop.
# Now write the dirty maps out for this polarization.
# Use memmaps for this since we want to reorganize data
# and write at the same time.
# New maps will have the frequency axis as slowly varying, for
# future efficiency.
map_file_name = params["output_root"] + "dirty_map_" + utils.polint2str(this_pol) + ".npy"
mfile = algebra.open_memmap(map_file_name, mode="w+", shape=(shape[2],) + shape[:2])
map_mem = algebra.make_vect(mfile, axis_names=("freq", "ra", "dec"))
# And the noise matrix.
noise_file_name = params["output_root"] + "noise_inv_" + utils.polint2str(this_pol) + ".npy"
if algorithm in ("disjoint_scans", "ds_grad"):
mfile = algebra.open_memmap(noise_file_name, mode="w+", shape=(shape[2],) + shape[:2] * 2)
noise_mem = algebra.make_mat(
mfile, axis_names=("freq", "ra", "dec", "ra", "dec"), row_axes=(0, 1, 2), col_axes=(0, 3, 4)
)
else:
mfile = algebra.open_memmap(noise_file_name, mode="w+", shape=(shape[2],) + shape[:2])
noise_mem = algebra.make_mat(
mfile, axis_names=("freq", "ra", "dec"), row_axes=(0, 1, 2), col_axes=(0, 1, 2)
)
# Give the data arrays axis information.
map_mem.set_axis_info("freq", centre_freq, delta_freq)
map_mem.set_axis_info("ra", params["field_centre"][0], ra_spacing)
map_mem.set_axis_info("dec", params["field_centre"][1], params["pixel_spacing"])
noise_mem.set_axis_info("freq", centre_freq, delta_freq)
noise_mem.set_axis_info("ra", params["field_centre"][0], ra_spacing)
noise_mem.set_axis_info("dec", params["field_centre"][1], params["pixel_spacing"])
# Copy the data to the memory maps after rearranging.
# The roll_axis should return a view, so this should
# be memory efficient.
map_mem[...] = sp.rollaxis(map_data, -1)
noise_mem[...] = sp.rollaxis(noise_inv, -1)
# Free up all that memory and flush memory maps to file.
del mfile, map_mem, noise_mem, map_data, noise_inv
# Save the file names for the history.
all_file_names.append(kiyopy.utils.abbreviate_file_path(map_file_name))
all_file_names.append(kiyopy.utils.abbreviate_file_path(noise_file_name))
except ce.NextIteration:
pass
pol_ind += 1
# End polarization for loop.
history.add("Made dirty map.", all_file_names)
h_file_name = params["output_root"] + "history.hist"
history.write(h_file_name)
