def proper_fits_shapes(qname, uname, frequencyname):
"""
Verify that the Q and U FITS cubes and the file with frequency
data have compatible shapes. *qname* and *uname* are the filenames
of the Q and U FITS files, respectively; *frequencyname* is the
filename of the frequency file.
Returns True if all is well, raises ShapeError otherwise.
"""
frequencies = parse_frequency_file(frequencyname)
q_h = fits.get_header(qname)
u_h = fits.get_header(uname)
errors = []
for name, hdr in [(qname, q_h), (uname, u_h)]:
if hdr['NAXIS'] != 3:
errors.append('number of axes in %s is %d, not 3' %
(name, hdr['NAXIS']))
for axis in ['NAXIS1', 'NAXIS2', 'NAXIS3']:
if q_h[axis] != u_h[axis]:
errors.append('%s in %s (%d) not equal to %s in %s (%d)' %
(axis, qname, q_h[axis], axis, uname, u_h[axis]))
if q_h['NAXIS3'] != len(frequencies):
msg = ('frames in %r and %r (%d) not the same as in %r (%d)' %
(qname, uname, q_h['NAXIS3'], frequencyname, len(frequencies)))
errors.append(msg)
if len(errors) > 0:
raise ShapeError('\n'.join(errors))
return True
def correct_and_average_cubes(p_out_name,
q_out_name,
u_out_name,
weights_name,
qu_input_names,
freq_hz,
force_overwrite=False,
rm_to_remove=None,
ignore_frames=None):
r'''
TBD
'''
logging.info('correct_and_average_cubes()')
q_header = fits.get_header(qu_input_names[0][0])
u_header = fits.get_header(qu_input_names[0][1])
qu_iterators = [[fits.image_frames(name) for name in qu_name]
for qu_name in qu_input_names]
flat_qu_iterators = [
item for sub_list in qu_iterators for item in sub_list
]
if rm_to_remove is None:
rm_to_remove = zeros(len(qu_input_names), dtype=float64)
if ignore_frames is None:
ignore_frames = [[]] * len(qu_input_names)
p_out_hdr, q_out_hdr, u_out_hdr = output_pqu_headers(q_header)
logging.info('P header:\n%r', p_out_hdr)
p_out, q_out, u_out = [
fits.streaming_output_hdu(fits_name, header, force_overwrite)
for fits_name, header in [(
p_out_name, p_out_hdr), (q_out_name,
q_out_hdr), (u_out_name, u_out_hdr)]
]
weights = []
for channel, freq_qu_list in enumerate(izip(freq_hz, *flat_qu_iterators)):
freq = freq_qu_list[0]
logging.info('Channel %r: %.3f MHz', channel, freq / 1e6)
wl2 = wavelength_squared_m2_from_freq_hz(freq)
q_frames = array(freq_qu_list[1::2], dtype=float32)
u_frames = array(freq_qu_list[2::2], dtype=float32)
mask = 1.0 - array([channel in ignore for ignore in ignore_frames])
phasors = exp(-2.j * array(rm_to_remove) * wl2) * mask
corr_p_frames = (q_frames + 1j * u_frames) * phasors[:, newaxis,
newaxis]
if mask.sum() == 0.0:
p_final = corr_p_frames.sum(axis=0) * 0.0
else:
p_final = corr_p_frames.sum(axis=0) / float(mask.sum())
weights.append(mask.sum())
p_out.write(array(abs(p_final), dtype=float32))
q_out.write(array(p_final.real, dtype=float32))
u_out.write(array(p_final.imag, dtype=float32))
p_out.close()
q_out.close()
u_out.close()
with open(weights_name, 'w') as w_out:
w_out.write('\n'.join([str(w) for w in weights]))
logging.info('Done correcting')
def correct_and_average_cubes(p_out_name, q_out_name, u_out_name,
weights_name,
qu_input_names,
freq_hz,
force_overwrite=False,
rm_to_remove=None,
ignore_frames=None):
r'''
TBD
'''
logging.info('correct_and_average_cubes()')
q_header = fits.get_header(qu_input_names[0][0])
u_header = fits.get_header(qu_input_names[0][1])
qu_iterators = [[fits.image_frames(name) for name in qu_name]
for qu_name in qu_input_names]
flat_qu_iterators = [item for sub_list in qu_iterators
for item in sub_list]
if rm_to_remove is None:
rm_to_remove = zeros(len(qu_input_names), dtype=float64)
if ignore_frames is None:
ignore_frames = [[]]*len(qu_input_names)
p_out_hdr, q_out_hdr, u_out_hdr = output_pqu_headers(q_header)
logging.info('P header:\n%r', p_out_hdr)
p_out, q_out, u_out = [fits.streaming_output_hdu(fits_name,
header,
force_overwrite)
for fits_name, header in [(p_out_name, p_out_hdr),
(q_out_name, q_out_hdr),
(u_out_name, u_out_hdr)]]
weights = []
for channel, freq_qu_list in enumerate(izip(freq_hz, *flat_qu_iterators)):
freq = freq_qu_list[0]
logging.info('Channel %r: %.3f MHz', channel, freq/1e6)
wl2 = wavelength_squared_m2_from_freq_hz(freq)
q_frames = array(freq_qu_list[1::2], dtype=float32)
u_frames = array(freq_qu_list[2::2], dtype=float32)
mask = 1.0 - array([channel in ignore for ignore in ignore_frames])
phasors = exp(-2.j*array(rm_to_remove)*wl2)*mask
corr_p_frames = (q_frames +1j*u_frames)*phasors[:, newaxis, newaxis]
if mask.sum() == 0.0:
p_final = corr_p_frames.sum(axis=0)*0.0
else:
p_final = corr_p_frames.sum(axis=0)/float(mask.sum())
weights.append(mask.sum())
p_out.write(array(abs(p_final), dtype=float32))
q_out.write(array(p_final.real, dtype=float32))
u_out.write(array(p_final.imag, dtype=float32))
p_out.close()
q_out.close()
u_out.close()
with open(weights_name, 'w') as w_out:
w_out.write('\n'.join([str(w) for w in weights]))
logging.info('Done correcting')
def mean_psf(psf_name, frequencies, output_fits_name, force_overwrite,
max_mem_gb=2.0, bad_frames=None):
logging.info('mean_psf()')
psf_header = fits.get_header(psf_name)
pixels_per_frame = psf_header['NAXIS1']*psf_header['NAXIS2']
max_mem_bytes = max_mem_gb*1024**3
bytes_per_input_pixel = psf_header['BITPIX']/8.0
max_input_pixels = max_mem_bytes/bytes_per_input_pixel
block_length = int(floor(max_input_pixels/pixels_per_frame/4.0))
num_blocks = int(floor(psf_header['NAXIS3']/block_length))
if psf_header['NAXIS3'] % block_length > 0:
num_blocks += 1
output_header = psf_header
nfreq = len(frequencies)
psf_frames = fits.image_frames(psf_name)
frame_id = 0
skipped_frames = 0
sum_psf = zeros((psf_header['NAXIS2'], psf_header['NAXIS1']), dtype=float32)
sum_freq_hz = 0.0
for freq_hz, psf in izip(frequencies, psf_frames):
if bad_frames is not None and frame_id in bad_frames:
logging.warn('skipping frame % d: in bad frame list.', frame_id)
skipped_frames += 1
else:
logging.info('processing frame '+str(frame_id+1)+'/'+str(nfreq))
sum_psf += psf
sum_freq_hz += freq_hz
gc.collect()
frame_id += 1
frames_added = nfreq - skipped_frames
mean_psf_frame = sum_psf/frames_added
mean_freq_hz = sum_freq_hz/frames_added
fits.write_cube(output_fits_name, output_header, mean_psf_frame, force_overwrite)
logging.info('Done')
def rmsynthesis_dirty_lowmem_mp(qname, uname, q_factor, u_factor,
frequencies, phi_array):
"""
Perform an RM synthesis on Q and U image cubes with given
frequencies. The rmcube that is returned is complex valued and has
a frame for every value in phi_array. It is assumed that the
dimensions of qcube, ucube, and frequencies have been verified
before with the help of the proper_fits_shapes() function. The
polarization vectors are derotated to the average lambda^2.
Uses the multiprocessing module to speed up the calculations.
"""
num_workers = mp.cpu_count()-1
wl2 = wavelength_squared_m2_from_freq_hz(frequencies)
qheader = fits.get_header(qname)
wl2_0 = wl2.mean()
nfreq = len(frequencies)
q_frames = fits.image_frames(qname)
u_frames = fits.image_frames(uname)
frame_id = 0
wl2_norm = wl2 - wl2_0
phi_arrays = array_split(phi_array, num_workers)
frame_shape = (qheader['NAXIS2'], qheader['NAXIS1'])
mp_shared_p_complex = mp.Array(ctypes.c_float,
frame_shape[0]*frame_shape[1]*2)
p_complex = frombuffer(mp_shared_p_complex.get_obj(),
dtype=complex64).reshape(frame_shape)
queues = [mp.JoinableQueue() for _ in range(num_workers)]
workers = [mp.Process(target=rmsynthesis_worker,
args=(queue, mp_shared_p_complex,
frame_shape, phi))
for queue, phi in zip(queues, phi_arrays)]
for worker in workers:
worker.daemon = True
worker.start()
for q_frame, u_frame in izip(q_frames, u_frames):
logging.info('processing frame '+str(frame_id+1)+'/'+str(nfreq))
p_complex[:, :] = q_frame*q_factor + 1.0j*u_frame*u_factor
wl2_frame = wl2_norm[frame_id]
for queue in queues:
queue.put(wl2_frame)
for queue in queues:
queue.join()
frame_id += 1
gc.collect()
for queue in queues:
queue.put(None)
logging.info('Collecting partial results')
partial_rmcubes = [queue.get() for queue in queues]
rmcube = concatenate(partial_rmcubes)/nfreq
for worker in workers:
worker.join()
gc.collect()
return rmcube
def average_psf_cubes(psf_out_name,
weights_name,
psf_input_names,
force_overwrite=False,
ignore_frames=None):
r'''
TBD
'''
logging.info('correct_and_average_cubes()')
psf_header = fits.get_header(psf_input_names[0])
psf_iterators = [fits.image_frames(name) for name in psf_input_names]
if ignore_frames is None:
ignore_frames = [[]]*len(qu_input_names)
psf_out_hdr = psf_header
logging.info('PSF header:\n%r', psf_out_hdr)
psf_out = fits.streaming_output_hdu(psf_out_name, psf_header, force_overwrite)
weights = []
for channel, frames in enumerate(izip(*psf_iterators)):
logging.info('Channel %r', channel)
psf_frames = array(frames, dtype=float32)
mask = 1.0 - array([channel in ignore for ignore in ignore_frames])
masked_psf_frames = psf_frames*mask[:, newaxis, newaxis]
if mask.sum() == 0.0:
psf_final = psf_frames[0,:,:]*0.0
else:
psf_final = psf_frames.sum(axis=0)/float(mask.sum())
weights.append(mask.sum())
psf_out.write(array(psf_final, dtype=float32))
psf_out.close()
with open(weights_name, 'w') as w_out:
w_out.write('\n'.join([str(w) for w in weights]))
logging.info('Done averaging')
def rmsynthesis_crosscorr_dirty_lowmem_main(q_template_name, u_template_name,
q_name, u_name,
q_factor, u_factor,
output_dir, freq_hz, phi_rad_m2,
force_overwrite, max_mem_gb=2.0,
bad_frames=None):
r'''
'''
logging.info('rmsynthesis_crosscorr_dirty_lowmem_main()')
q_header = fits.get_header(q_name)
pixels_per_frame = q_header['NAXIS1']*q_header['NAXIS2']
max_mem_bytes = max_mem_gb*1024**3
bytes_per_output_pixel = 4
max_output_pixels = max_mem_bytes/bytes_per_output_pixel
# 7 = (re, im) + p_out + q_out + u_out
block_length = int(floor(max_output_pixels/pixels_per_frame/9.0))
num_blocks = int(floor(len(phi_rad_m2)/block_length))
if len(phi_rad_m2) % block_length > 0:
num_blocks += 1
output_header = add_phi_to_fits_header(q_header.copy(), phi_rad_m2)
p_out_name, q_out_name, u_out_name = output_pqu_fits_names(output_dir)
p_out_hdr, q_out_hdr, u_out_hdr = output_pqu_headers(output_header)
p_out, q_out, u_out = [fits.streaming_output_hdu(fits_name,
header,
force_overwrite)
for fits_name, header in [(p_out_name, p_out_hdr),
(q_out_name, q_out_hdr),
(u_out_name, u_out_hdr)]]
try:
logging.info('Computing cross correlation with templates\n - Q:%s\n - U:%s',
q_template_name, u_template_name)
for block in range(num_blocks):
phi = phi_rad_m2[block*block_length:(block+1)*block_length]
logging.info('Processing block %d / %d', block + 1, num_blocks)
logging.info('Phi in [%.1f, %.1f]', phi[0], phi[-1])
rmcube = rmsynthesis_crosscorr_dirty_lowmem(
q_template_name, u_template_name,
q_name, u_name,
q_factor, u_factor,
freq_hz, phi,
bad_frames=bad_frames)
logging.info('Saving data')
p_out.write(absolute(rmcube))
q_out.write(rmcube.real)
u_out.write(rmcube.imag)
finally:
p_out.close()
q_out.close()
u_out.close()
logging.info('Done')
def rmsynthesis_crosscorr_dirty_lowmem(q_template_name, u_template_name,
qname, uname, q_factor, u_factor,
frequencies, phi_array,
bad_frames=None):
"""Perform a cross correlation in Faraday space by multiplying QU
frames with template QU frames, and performing an RM synthesis on
the resulting Q and U image cubes with given frequencies. The
rmcube that is returned is complex valued and has a frame for
every value in phi_array. It is assumed that the dimensions of
qcube, ucube, and frequencies have been verified before with the
help of the proper_fits_shapes() function. The polarization
vectors are derotated to the average lambda^2.
"""
wl2 = wavelength_squared_m2_from_freq_hz(frequencies)
qheader = fits.get_header(qname)
rmcube = zeros((len(phi_array), qheader['NAXIS2'], qheader['NAXIS1']),
dtype=complex64)
wl2_0 = wl2.mean()
nfreq = len(frequencies)
q_template_frames = fits.image_frames(q_template_name)
u_template_frames = fits.image_frames(u_template_name)
q_frames = fits.image_frames(qname)
u_frames = fits.image_frames(uname)
frame_id = 0
skipped_frames = 0
wl2_norm = wl2 - wl2_0
for q_temp, u_temp, q_frame, u_frame in izip(q_template_frames, u_template_frames, q_frames, u_frames):
if bad_frames is not None and frame_id in bad_frames:
logging.warn('skipping frame % d: in bad frame list.', frame_id)
skipped_frames += 1
else:
logging.info('processing frame '+str(frame_id+1)+'/'+str(nfreq))
template_complex = q_temp*q_factor +1.0j*u_temp*u_factor
p_complex = (q_frame*q_factor + 1.0j*u_frame*u_factor)*template_complex.conj()
wl2_frame = wl2_norm[frame_id]
phases = phases_lambda2_to_phi(wl2_frame, phi_array)
for frame, phase in enumerate(phases):
rmcube[frame, :, :] += p_complex*phase
gc.collect()
frame_id += 1
frames_added = nfreq - skipped_frames
return rmcube/float(frames_added)
def average_psf_cubes(psf_out_name,
weights_name,
psf_input_names,
force_overwrite=False,
ignore_frames=None):
r'''
TBD
'''
logging.info('correct_and_average_cubes()')
psf_header = fits.get_header(psf_input_names[0])
psf_iterators = [fits.image_frames(name) for name in psf_input_names]
if ignore_frames is None:
ignore_frames = [[]] * len(qu_input_names)
psf_out_hdr = psf_header
logging.info('PSF header:\n%r', psf_out_hdr)
psf_out = fits.streaming_output_hdu(psf_out_name, psf_header,
force_overwrite)
weights = []
for channel, frames in enumerate(izip(*psf_iterators)):
logging.info('Channel %r', channel)
psf_frames = array(frames, dtype=float32)
mask = 1.0 - array([channel in ignore for ignore in ignore_frames])
masked_psf_frames = psf_frames * mask[:, newaxis, newaxis]
if mask.sum() == 0.0:
psf_final = psf_frames[0, :, :] * 0.0
else:
psf_final = psf_frames.sum(axis=0) / float(mask.sum())
weights.append(mask.sum())
psf_out.write(array(psf_final, dtype=float32))
psf_out.close()
with open(weights_name, 'w') as w_out:
w_out.write('\n'.join([str(w) for w in weights]))
logging.info('Done averaging')