def test_circle(self):
centre = 4.6
shape = 19
spacing = 1.3
bins = tools.calc_bins(centre, shape, spacing, edge="left") + 0.1 * spacing
inds = tools.calc_inds(bins, centre, shape, spacing)
self.assertTrue(sp.allclose(inds, sp.arange(int(shape))))
spacing = -0.35
bins = tools.calc_bins(centre, shape, spacing, edge="left") + 0.1 * spacing
inds = tools.calc_inds(bins, centre, shape, spacing)
self.assertTrue(sp.allclose(inds, sp.arange(int(shape))))
def test_circle(self):
centre = 4.6
shape = 19
spacing = 1.3
bins = tools.calc_bins(centre, shape, spacing,
edge='left') + 0.1 * spacing
inds = tools.calc_inds(bins, centre, shape, spacing)
self.assertTrue(sp.allclose(inds, sp.arange(int(shape))))
spacing = -0.35
bins = tools.calc_bins(centre, shape, spacing,
edge='left') + 0.1 * spacing
inds = tools.calc_inds(bins, centre, shape, spacing)
self.assertTrue(sp.allclose(inds, sp.arange(int(shape))))
def setUp(self) :
self.shape = (5, 5, 20)
self.ntime = 15
self.map = ma.zeros(self.shape, dtype=float)
self.noise_inv = sp.zeros(self.shape, dtype=float)
self.data = ma.ones((self.ntime, self.shape[-1]))
self.centre = (90.0,0.0,100.0)
self.spacing = 1.0
self.ra = 0.2*(sp.arange(self.ntime)-self.ntime/2.0) + self.centre[0]
self.dec = 0.2*(sp.arange(self.ntime)-self.ntime/2.0) + self.centre[1]
self.ra_inds = tools.calc_inds(self.ra, self.centre[0], self.shape[0],
self.spacing)
self.dec_inds = tools.calc_inds(self.dec, self.centre[1],
self.shape[1], self.spacing)
def test_calc_inds(self):
pointing = [4.001, 3.999, -0.999, 8.999]
centre = 4.0
shape = 10
inds = tools.calc_inds(pointing, centre, shape)
self.assertEqual(inds[0], 5)
self.assertEqual(inds[1], 5)
self.assertEqual(inds[2], 0)
self.assertEqual(inds[3], 10)
shape = 5
spacing = 2.0
inds = tools.calc_inds(pointing, centre, shape, spacing)
self.assertEqual(inds[0], 2)
self.assertEqual(inds[1], 2)
self.assertEqual(inds[2], 0)
self.assertEqual(inds[3], 4)
def test_calc_inds(self):
pointing = [4.001, 3.999, -0.999, 8.999]
centre = 4.
shape = 10
inds = tools.calc_inds(pointing, centre, shape)
self.assertEqual(inds[0], 5)
self.assertEqual(inds[1], 5)
self.assertEqual(inds[2], 0)
self.assertEqual(inds[3], 10)
shape = 5
spacing = 2.0
inds = tools.calc_inds(pointing, centre, shape, spacing)
self.assertEqual(inds[0], 2)
self.assertEqual(inds[1], 2)
self.assertEqual(inds[2], 0)
self.assertEqual(inds[3], 4)
def test_calc_inds_neg(self):
"""Needs to work for negitive spacing."""
pointing = [2.001, 1.999, 0.001, 3.49]
centre = 2.0
shape = 4
spacing = -1
inds = tools.calc_inds(pointing, centre, shape, spacing)
self.assertEqual(inds[0], 2)
self.assertEqual(inds[1], 2)
self.assertEqual(inds[2], 4)
self.assertEqual(inds[3], 1)
def test_calc_inds_neg(self):
"""Needs to work for negitive spacing."""
pointing = [2.001, 1.999, 0.001, 3.49]
centre = 2.
shape = 4
spacing = -1
inds = tools.calc_inds(pointing, centre, shape, spacing)
self.assertEqual(inds[0], 2)
self.assertEqual(inds[1], 2)
self.assertEqual(inds[2], 4)
self.assertEqual(inds[3], 1)
def build_hitmap(file_middles, input_end, output_root, scans, IFs,
field_centre, map_shape, pixel_spacing):
# Rename some commonly used parameters.
ra_spacing = -spacing / sp.cos(field_centre[1] * sp.pi / 180.)
if len(IFs) != 1:
raise ce.FileParameterTypeError('Can only process a single IF.')
all_file_names = []
# Flag for the first block processed (will allowcate memory on the
# first iteration).
first_block = True
# Loop over the files to process.
try:
for file_middle in file_middles:
input_fname = (input_root + file_middle + input_end)
# Read in the data, and loop over data blocks.
Reader = fitsGBT.Reader(input_fname, feedback=feedback)
Blocks = Reader.read(scans, IFs)
# Calculate the time varience at each frequency. This will
# be used as weights in most algorithms.
for Data in Blocks:
dims = Data.dims
# On first pass set up the map parameters.
if first_block:
shape = map_shape + (dims[-1], )
Data.calc_freq()
centre_freq = Data.freq[dims[-1] // 2]
delta_freq = Data.field['CDELT1']
# Allocate memory for the map.
map_data = sp.zeros(shape, dtype=float)
map_data = algebra.make_vect(map_data,
axis_names=('ra', 'dec',
'freq'))
first_block = False
# Figure out the pointing pixel index and the frequency
# indicies.
Data.calc_pointing()
ra_inds = tools.calc_inds(Data.ra, field_centre[0], shape[0],
ra_spacing)
dec_inds = tools.calc_inds(Data.dec, field_centre[1], shape[1],
pixel_spacing)
data = Data.data[:, pol_ind, 0, :]
pixel_hits[:] = 0
pixel_list = pixel_counts(data,
ra_inds,
dec_inds,
pixel_hits,
map_shape=shape[0:2])
# End Blocks for loop.
# Free up all that memory and flush memory maps to file.
del map_data
# Save the file names for the history.
all_file_names.append(
kiyopy.utils.abbreviate_file_path(map_file_name))
except ce.NextIteration:
pass
# End polarization for loop.
history.add("Made dirty map.", all_file_names)
h_file_name = (output_root + 'history.hist')
history.write(h_file_name)
def execute(self, nprocesses=1):
"""Function that acctually does the work.
The nprocesses parameter does not do anything yet. It is just there
for compatibility with the pipeline manager.
"""
params = self.params
kiyopy.utils.mkparents(params["output_root"])
parse_ini.write_params(params, params["output_root"] + "params.ini", prefix="mm_")
# Rename some commonly used parameters.
map_shape = params["map_shape"]
spacing = params["pixel_spacing"]
algorithm = params["noise_model"]
noise_root = params["noise_parameters_input_root"]
ra_spacing = -spacing / sp.cos(params["field_centre"][1] * sp.pi / 180.0)
if not algorithm in ("grid", "diag_file", "disjoint_scans"):
raise ValueError("Invalid noise model: " + algorithm)
if len(params["IFs"]) != 1:
raise ce.FileParameterTypeError("Can only process a single IF.")
# Set up to iterate over the pol states.
npol = 2 # This will be reset when we read the first data block.
pol_ind = 0
all_file_names = []
while pol_ind < npol:
# Flag for the first block processed (will allowcate memory on the
# first iteration).
first_block = True
# Loop over the files to process.
try:
for file_middle in params["file_middles"]:
input_fname = params["input_root"] + file_middle + params["input_end"]
# Read in the data, and loop over data blocks.
Reader = fitsGBT.Reader(input_fname, feedback=self.feedback)
Blocks = Reader.read(params["scans"], params["IFs"])
# Calculate the time varience at each frequency. This will
# be used as weights in most algorithms.
if not algorithm == "grid":
if not noise_root == "None":
# We have measured variance.
noise_pars = sp.load(noise_root + file_middle + ".npy")
var = noise_pars[params["IFs"][0], pol_ind, 0, :]
else:
# We need to measure the variance.
var = tools.calc_time_var_file(Blocks, pol_ind, 0)
# Convert from masked array to array.
var = var.filled(9999.0)
else:
var = 1.0
weight = 1 / var
for Data in Blocks:
dims = Data.dims
# On first pass set up the map parameters.
if first_block:
shape = map_shape + (dims[-1],)
Data.calc_freq()
centre_freq = Data.freq[dims[-1] // 2]
delta_freq = Data.field["CDELT1"]
if pol_ind == 0:
# Figure out the length of the polarization
# loop.
npol = dims[1]
# Accumulate the data history.
history = hist.History(Data.history)
# Get the current polarization integer.
this_pol = Data.field["CRVAL4"][pol_ind]
# Check that we even want to make a dirty map for
# this polarization.
if (not utils.polint2str(this_pol) in params["polarizations"]) and params["polarizations"]:
# Break to the end of the polarization loop.
raise ce.NextIteration()
# Allowcate memory for the map.
map_data = sp.zeros(shape, dtype=float)
map_data = algebra.make_vect(map_data, axis_names=("ra", "dec", "freq"))
# Allowcate memory for the inverse map noise.
if algorithm in ("grid", "diag_file"):
noise_inv = sp.zeros(shape, dtype=float)
noise_inv = algebra.make_mat(
noise_inv, axis_names=("ra", "dec", "freq"), row_axes=(0, 1, 2), col_axes=(0, 1, 2)
)
elif algorithm in ("disjoint_scans", "ds_grad"):
# At each frequency use full N^2 noise matrix,
# but assume each frequency has uncorrelated
# noise. This is a big matrix so make sure it
# is reasonable.
size = shape[0] ^ 2 * shape[1] ^ 2 * shape[2]
if size > 4e9: # 16 GB
raise RunTimeError("Map size too big. " "Asked for a lot " "of memory.")
noise_inv = sp.zeros(shape[0:2] + shape, dtype=sp.float32)
noise_inv = algebra.make_mat(
noise_inv,
axis_names=("ra", "dec", "ra", "dec", "freq"),
row_axes=(0, 1, 4),
col_axes=(2, 3, 4),
)
# Allowcate memory for temporary data. Hold the
# number of times each pixel in this scan is
# hit. Factor of 2 longer in time in case some
# scans are longer than first block (guppi).
pixel_hits = sp.empty((2 * dims[0], dims[-1]))
first_block = False
else:
if pol_ind == 0:
history.merge(Data)
# Figure out the pointing pixel index and the frequency
# indicies.
Data.calc_pointing()
ra_inds = tools.calc_inds(Data.ra, params["field_centre"][0], shape[0], ra_spacing)
dec_inds = tools.calc_inds(
Data.dec, params["field_centre"][1], shape[1], params["pixel_spacing"]
)
data = Data.data[:, pol_ind, 0, :]
if algorithm in ("grid", "diag_file"):
add_data_2_map(data, ra_inds, dec_inds, map_data, noise_inv, weight)
elif algorithm in ("disjoint_scans",):
add_data_2_map(data - ma.mean(data, 0), ra_inds, dec_inds, map_data, None, weight)
pixel_hits[:] = 0
pixel_list = pixel_counts(data, ra_inds, dec_inds, pixel_hits, map_shape=shape[0:2])
add_scan_noise(pixel_list, pixel_hits, var, noise_inv)
# End Blocks for loop.
# End file name for loop.
# Now write the dirty maps out for this polarization.
# Use memmaps for this since we want to reorganize data
# and write at the same time.
# New maps will have the frequency axis as slowly varying, for
# future efficiency.
map_file_name = params["output_root"] + "dirty_map_" + utils.polint2str(this_pol) + ".npy"
mfile = algebra.open_memmap(map_file_name, mode="w+", shape=(shape[2],) + shape[:2])
map_mem = algebra.make_vect(mfile, axis_names=("freq", "ra", "dec"))
# And the noise matrix.
noise_file_name = params["output_root"] + "noise_inv_" + utils.polint2str(this_pol) + ".npy"
if algorithm in ("disjoint_scans", "ds_grad"):
mfile = algebra.open_memmap(noise_file_name, mode="w+", shape=(shape[2],) + shape[:2] * 2)
noise_mem = algebra.make_mat(
mfile, axis_names=("freq", "ra", "dec", "ra", "dec"), row_axes=(0, 1, 2), col_axes=(0, 3, 4)
)
else:
mfile = algebra.open_memmap(noise_file_name, mode="w+", shape=(shape[2],) + shape[:2])
noise_mem = algebra.make_mat(
mfile, axis_names=("freq", "ra", "dec"), row_axes=(0, 1, 2), col_axes=(0, 1, 2)
)
# Give the data arrays axis information.
map_mem.set_axis_info("freq", centre_freq, delta_freq)
map_mem.set_axis_info("ra", params["field_centre"][0], ra_spacing)
map_mem.set_axis_info("dec", params["field_centre"][1], params["pixel_spacing"])
noise_mem.set_axis_info("freq", centre_freq, delta_freq)
noise_mem.set_axis_info("ra", params["field_centre"][0], ra_spacing)
noise_mem.set_axis_info("dec", params["field_centre"][1], params["pixel_spacing"])
# Copy the data to the memory maps after rearranging.
# The roll_axis should return a view, so this should
# be memory efficient.
map_mem[...] = sp.rollaxis(map_data, -1)
noise_mem[...] = sp.rollaxis(noise_inv, -1)
# Free up all that memory and flush memory maps to file.
del mfile, map_mem, noise_mem, map_data, noise_inv
# Save the file names for the history.
all_file_names.append(kiyopy.utils.abbreviate_file_path(map_file_name))
all_file_names.append(kiyopy.utils.abbreviate_file_path(noise_file_name))
except ce.NextIteration:
pass
pol_ind += 1
# End polarization for loop.
history.add("Made dirty map.", all_file_names)
h_file_name = params["output_root"] + "history.hist"
history.write(h_file_name)
def build_hitmap(file_middles, input_end, output_root, scans, IFs,
field_centre, map_shape, pixel_spacing):
# Rename some commonly used parameters.
ra_spacing = -spacing/sp.cos(field_centre[1]*sp.pi/180.)
if len(IFs) != 1:
raise ce.FileParameterTypeError('Can only process a single IF.')
all_file_names = []
# Flag for the first block processed (will allowcate memory on the
# first iteration).
first_block = True
# Loop over the files to process.
try:
for file_middle in file_middles:
input_fname = (input_root + file_middle +
input_end)
# Read in the data, and loop over data blocks.
Reader = fitsGBT.Reader(input_fname, feedback=feedback)
Blocks = Reader.read(scans, IFs)
# Calculate the time varience at each frequency. This will
# be used as weights in most algorithms.
for Data in Blocks:
dims = Data.dims
# On first pass set up the map parameters.
if first_block:
shape = map_shape + (dims[-1],)
Data.calc_freq()
centre_freq = Data.freq[dims[-1]//2]
delta_freq = Data.field['CDELT1']
# Allocate memory for the map.
map_data = sp.zeros(shape, dtype=float)
map_data = algebra.make_vect(map_data,
axis_names=('ra', 'dec', 'freq'))
first_block=False
# Figure out the pointing pixel index and the frequency
# indicies.
Data.calc_pointing()
ra_inds = tools.calc_inds(Data.ra,
field_centre[0], shape[0],
ra_spacing)
dec_inds = tools.calc_inds(Data.dec,
field_centre[1],
shape[1], pixel_spacing)
data = Data.data[:,pol_ind,0,:]
pixel_hits[:] = 0
pixel_list = pixel_counts(data, ra_inds, dec_inds,
pixel_hits, map_shape=shape[0:2])
# End Blocks for loop.
# Free up all that memory and flush memory maps to file.
del map_data
# Save the file names for the history.
all_file_names.append(kiyopy.utils.abbreviate_file_path(
map_file_name))
except ce.NextIteration:
pass
# End polarization for loop.
history.add("Made dirty map.", all_file_names)
h_file_name = (output_root + 'history.hist')
history.write(h_file_name)
