def calc_isum(self, image):
"""
Function to calculate the sum of the values of the pixels in an image
image (string): The name of the image file. Must be in MIRIAD-format
returns (float): the sum of the pxiels in the image
"""
fits = lib.miriad('fits')
fits.op = 'xyout'
fits.in_ = image
fits.out = image + '.fits'
fits.go()
image_data = pyfits.open(image + '.fits') # Open the image
data = image_data[0].data
isum = np.nansum(data) # Get the maximum
image_data.close() # Close the image
subs_managefiles.director(self, 'rm', image + '.fits')
return isum
def getimagestats(self, image):
"""
Subroutine to calculate the min, max and rms of an image
image (string): The absolute path to the image file.
returns (numpy array): The min, max and rms of the image
"""
setinit.setinitdirs(self)
char_set = string.ascii_uppercase + string.digits
if os.path.isdir(image) or os.path.isfile(image):
if os.path.isdir(image):
temp_string = ''.join(random.sample(char_set * 8, 8))
fits = lib.miriad('fits')
fits.op = 'xyout'
fits.in_ = image
with tempfile.TemporaryDirectory() as tempdir:
fits.out = tempdir + '/' + temp_string + '.fits'
fits.go()
image_data = pyfits.open(tempdir + '/' + temp_string + '.fits')
elif os.path.isfile(image):
image_data = pyfits.open(image)
else:
error = 'Image format not supported. Only MIRIAD and FITS formats are supported!'
logger.error(error)
raise ApercalException(error)
data = image_data[0].data
imagestats = np.full(3, np.nan)
if data.shape[-3] == 2:
imagestats[0] = np.nanmin(
data[0, 0, :, :]) # Get the maxmimum of the image
imagestats[1] = np.nanmax(
data[0, 0, :, :]) # Get the minimum of the image
imagestats[2] = np.nanstd(
data[0, 0, :, :]) # Get the standard deviation
else:
imagestats[0] = np.nanmin(data) # Get the maxmimum of the image
imagestats[1] = np.nanmax(data) # Get the minimum of the image
imagestats[2] = np.nanstd(data) # Get the standard deviation
image_data.close() # Close the image
else:
error = 'Image does not seem to exist!'
logger.error(error)
raise ApercalException(error)
return imagestats
def imagetofits(self, mirimage, fitsimage, remove=True):
"""
Converts a MIRIAD image to a FITS image
mirimage: The MIRIAD image to convert
fitsimage: The converted FITS image
"""
subs_setinit.setinitdirs(self)
fits = lib.miriad('fits')
fits.op = 'xyout'
fits.in_ = mirimage
fits.out = fitsimage
fits.go()
if os.path.isfile(fitsimage) and remove:
director(self, 'rm', mirimage)
elif os.path.isfile(fitsimage) and remove == False:
pass
else:
pass
def get_gains(file_):
"""
Function to create a complex python array of amplitude and phase gains from a dataset
file_ (str): u,v file with the bandpass calibration
return(array, array): an array with the amplitude and phase gains for each antenna and solution interval, a
datetime array with the actual solution timesteps
"""
char_set = string.ascii_uppercase + string.digits # Create a charset for random gain log file generation
tempdir = subs.managetmp.manage_tempdir('mirlog')
gains_string = ''.join(random.sample(char_set * 8, 8))
gpplt = lib.miriad('gpplt')
gpplt.vis = file_
gpplt.log = tempdir + '/' + gains_string
gpplt.options = 'gains'
gpplt.yaxis = 'amp'
cmd = gpplt.go()
check_table(tempdir + '/' + gains_string, 'gains')
obsdate = cmd[3].split(' ')[4][0:7]
starttime = datetime.datetime(
int('20' + obsdate[0:2]),
int(time.strptime(obsdate[2:5], '%b').tm_mon), int(obsdate[5:7]))
s = Table.read(tempdir + '/' + gains_string, format='ascii')
gpplt.yaxis = 'phase'
gpplt.go()
check_table(tempdir + '/' + gains_string, 'gains')
t = Table.read(tempdir + '/' + gains_string, format='ascii')
days = np.array(t['col1'])
times = np.array(t['col2'])
nint = len(days)
nant = int(len(t[0]) - 2)
gain_array = np.zeros((nant, nint, 2))
for ant in range(nant):
gain_array[ant, :, 0] = s['col' + str(ant + 3)]
# gain_array[ant, :, 1] = t['col' + str(ant + 3)]
gain_array[ant, :, 1] = np.unwrap(t['col' + str(ant + 3)], discont=90)
time_array = [
starttime + datetime.timedelta(days=int(days[step]),
hours=int(times[step][0:2]),
minutes=int(times[step][3:5]),
seconds=int(times[step][6:8]))
for step in range(nint)
]
return gain_array, time_array
def get_ndims(file_):
"""
Wrapper funtion to get the dimension of the selfcal gain file
file_ (str): u,v file with the gain calibration
returns (int, int, int): Number of antennas, number of frequency bins, number of time intervals
"""
char_set = string.ascii_uppercase + string.digits # Create a charset for random gain log file generation
tempdir = subs.managetmp.manage_tempdir('mirlog')
subs.managetmp.clean_tempdir('mirlog')
gains_string = ''.join(random.sample(char_set * 8, 8))
gpplt = lib.miriad('gpplt')
gpplt.vis = file_
gpplt.log = tempdir + '/' + gains_string
gpplt.yaxis = 'amp'
cmd = gpplt.go()
nants = get_nants(tempdir + '/' + gains_string)
nbins = get_nbins(tempdir + '/' + gains_string)
nsols = get_nsols(tempdir + '/' + gains_string)
return nants, nbins, nsols
def fixheader(beamname, beamdir):
"""
Generation of Gaussian beam maps results in the Gaussian
parameters included as the "beam"
This function removes those values
Args:
beamname (str): name of beam map
beamdir (str): location of beam map
"""
# load function to delete header information
delhd = lib.miriad('delhd')
# delete bmaj
delhd.in_ = '{0}/{1}/bmaj'.format(beamdir, beamname)
delhd.go()
# delete bmin
delhd.in_ = '{0}/{1}/bmin'.format(beamdir, beamname)
delhd.go()
# delete bmpa
delhd.in_ = '{0}/{1}/bpa'.format(beamdir, beamname)
delhd.go()
def get_phases(file_):
"""
Function to create a python array of selfcal phase gains from a dataset
file_ (str): u,v file with the gain calibration
return(array, array): an array with the phase gains for each antenna, frequency bin and solution interval, a
datetime array with the actual solution timesteps
"""
char_set = string.ascii_uppercase + string.digits # Create a charset for random gain log file generation
tempdir = subs.managetmp.manage_tempdir('mirlog')
subs.managetmp.clean_tempdir('mirlog')
gains_string = ''.join(random.sample(char_set * 8, 8))
gpplt = lib.miriad('gpplt')
gpplt.vis = file_
gpplt.log = tempdir + '/' + gains_string
gpplt.yaxis = 'phase'
cmd = gpplt.go()
nants = get_nants(tempdir + '/' + gains_string)
nbins = get_nbins(tempdir + '/' + gains_string)
nsols = get_nsols(tempdir + '/' + gains_string)
reformat_table(tempdir + '/' + gains_string)
obsdate = cmd[3].split(' ')[4][0:7]
starttime = datetime.datetime(
int('20' + obsdate[0:2]),
int(time.strptime(obsdate[2:5], '%b').tm_mon), int(obsdate[5:7]))
s = Table.read(tempdir + '/' + gains_string, format='ascii')
days = np.array(s['col1'][0:nsols])
times = np.array(s['col2'][0:nsols])
time_array = [
starttime + datetime.timedelta(days=int(days[step]),
hours=int(times[step][0:2]),
minutes=int(times[step][3:5]),
seconds=int(times[step][6:8]))
for step in range(nsols)
]
gain_array = np.zeros([nants, nbins, nsols])
for ant in range(nants):
gain_array[ant, :, :] = np.reshape(s['col' + str(ant + 3)][nsols:],
(1, nbins, nsols))
return gain_array, time_array
def beam_cutoff(beamname, tmpbeamname, beamdir, cutoff):
"""
Function to apply a beam cutoff value
Args:
beamname (str): final name for beam map
tmpbeamname (str): name of temporary map, to be deleted
beamdir (str): output directory for beam map
cutoff (float): relative level at which beam map should be cut off
"""
# load maths
maths = lib.miriad('maths')
# expression is just beam map to have cutoff value applied
# Use brackets to guard against formatting issues
maths.exp = "'<{0}/{1}>'".format(beamdir, tmpbeamname)
# Apply cutoff value as mask, using miriad formatting, brackets around image name
maths.mask = "'<{0}/{1}>.gt.{2}'".format(beamdir, tmpbeamname, cutoff)
maths.out = '{0}/{1}'.format(beamdir, beamname)
# run
maths.go()
# clean up temp file
os.system('rm -rf {0}/{1}'.format(beamdir, tmpbeamname))
def get_delays(file_):
"""
Function to create a numpy array with the antenna delays for each solution interval
file_ (str): u,v file with the bandpass calibration
return(array, array): an array with the delays for each antenna and solution interval in nsec, a datetime array
with the actual solution timesteps
"""
char_set = string.ascii_uppercase + string.digits # Create a charset for random gain log file_ generation
tempdir = subs.managetmp.manage_tempdir('mirlog')
# tempdir = os.path.expanduser('~') + '/apercal/temp/mirlog'
gains_string = ''.join(random.sample(char_set * 8, 8))
gpplt = lib.miriad('gpplt')
gpplt.vis = file_
gpplt.log = tempdir + '/' + gains_string
gpplt.options = 'delays'
cmd = gpplt.go()
reformat_table(tempdir + '/' + gains_string)
obsdate = cmd[3].split(' ')[4][0:7]
starttime = datetime.datetime(
int('20' + obsdate[0:2]),
int(time.strptime(obsdate[2:5], '%b').tm_mon), int(obsdate[5:7]))
t = Table.read(tempdir + '/' + gains_string, format='ascii')
days = np.array(t['col1'])
times = np.array(t['col2'])
nint = len(days)
nant = int(len(t[0]) - 2)
delay_array = np.zeros((nant, nint))
for ant in range(nant):
delay_array[ant, :] = t['col' + str(ant + 3)]
time_array = [
starttime + datetime.timedelta(days=int(days[step]),
hours=int(times[step][0:2]),
minutes=int(times[step][3:5]),
seconds=int(times[step][6:8]))
for step in range(nint)
]
return delay_array, time_array
def getmaskstats(self, image, size):
"""
Subroutine to calculate the number of pixels in a mask and its percentage of the full image
image (string): The absolute path to the image file.
size (int): Number of pixels along an axis of the original image. Assumes square images.
returns (numpy array): The number of pixels and their percentage of the full image
"""
setinit.setinitdirs(self)
char_set = string.ascii_uppercase + string.digits
if os.path.isdir(image) or os.path.isfile(image):
if os.path.isdir(image):
temp_string = ''.join(random.sample(char_set * 8, 8))
fits = lib.miriad('fits')
fits.op = 'xyout'
fits.in_ = image
with tempfile.TemporaryDirectory() as tempdir:
fits.out = tempdir + '/' + temp_string + '.fits'
fits.go()
mask_data = pyfits.open(tempdir + '/' + temp_string + '.fits')
elif os.path.isfile(image):
mask_data = pyfits.open(image)
else:
error = 'Image format not supported. Only MIRIAD and FITS formats are supported!'
logger.error(error)
raise ApercalException(error)
data = mask_data[0].data
maskstats = np.full(2, np.nan)
maskstats[0] = np.count_nonzero(~np.isnan(data))
maskstats[1] = maskstats[0] / (size**2)
mask_data.close()
else:
error = 'Image does not seem to exist!'
logger.error(error)
raise ApercalException(error)
return maskstats
def create_mask(self, image, mask, threshold, theoretical_noise, beampars=None, rms_map=None):
"""
Creates a mask from an image using pybdsf
image (string): Input image to use in MIRIAD format
mask (string): Output mask image in MIRIAD format
threshold (float): Threshold in Jy to use
theoretical_noise (float): Theoretical noise for calculating the adaptive threshold parameter inside pybdsf
"""
convim.mirtofits(image, image + '.fits')
bdsf_threshold = threshold / theoretical_noise
if beampars:
# bdsf.process_image(image + '.fits', stop_at='isl', thresh_isl=bdsf_threshold, beam=beampars, adaptive_rms_box=True, rms_map=rms_map).export_image(outfile=mask + '.fits', img_format='fits', img_type='island_mask', pad_image=True)
bdsf.process_image(image + '.fits', stop_at='isl', thresh_isl=bdsf_threshold, beam=beampars, adaptive_rms_box=True, rms_map=False, rms_value=theoretical_noise).export_image(outfile=mask + '.fits', img_format='fits', img_type='island_mask', pad_image=True)
else:
bdsf.process_image(image + '.fits', stop_at='isl', thresh_isl=bdsf_threshold, adaptive_rms_box=True, rms_map=False, rms_value=theoretical_noise).export_image(outfile=mask + '.fits', img_format='fits', img_type='island_mask', pad_image=True)
if os.path.isfile(mask + '.fits'):
# Add a random number to the masks to make it viewable in kvis
fitsmask = pyfits.open(mask + '.fits')
fitsmask_data = fitsmask[0].data
fitsmask_hdr = fitsmask[0].header
rand_array = np.random.rand(int(fitsmask_hdr['NAXIS1']), int(fitsmask_hdr['NAXIS2']))
fitsmask[0].data = np.multiply(rand_array, fitsmask_data)
fitsmask.writeto(mask + '.fits', clobber=True)
# Convert mask to MIRIAD and generate a usable one for MIRIAD
convim.fitstomir(mask + '.fits', mask + '_pybdsf')
maths = lib.miriad('maths')
maths.out = mask
maths.exp = '"<' + mask + '_pybdsf>"'
maths.mask = '"<' + mask + '_pybdsf>.gt.0' + '"'
maths.go()
managefiles.director(self, 'rm', image + '.fits.pybdsf.log')
managefiles.director(self, 'rm', image + '.fits')
managefiles.director(self, 'rm', mask + '.fits')
managefiles.director(self, 'rm', mask + '_pybdsf')
else:
pass
def make_gaussian_beam(beamdir, beamoutname, bm_size, cell, fwhm, cutoff):
"""
Function to create Gaussian beam
Args:
beamdir (str): destination directory for beam map
beamoutname (str): name for beam map, based on beam number
bm_size (integer): number of pixels for reference map
cell (float): cell size of a pixel in arcsec
fwhm (float): FWHM of Gaussian in arcsec
cutoff (float): Relative level to cut beam off at
"""
# use a temp beam name because you have to apply a cutoff and then clean up
tmpbeamname = beamoutname + '_tmp'
# set peak level to 1 and PA = 0, plus arbitrary reference pixel
pk = 1.
bpa = 0.
pos = [0., 60.]
# set up imgen parameters
imgen = lib.miriad('imgen')
imgen.out = os.path.join(beamdir, tmpbeamname)
imgen.imsize = bm_size
imgen.cell = cell
imgen.object = 'gaussian'
imgen.spar = '{0},{1},{2},{3},{4},{5}'.format(str(pk), str(0), str(0),
str(fwhm), str(fwhm),
str(bpa))
imgen.radec = '{0},{1}'.format(str(pos[0]), str(pos[1]))
# create image
imgen.go()
# apply beam cutoff
beam_cutoff(beamoutname, tmpbeamname, beamdir, cutoff)
# fix header
fixheader(beamoutname, beamdir)
def getmodelstats(self, image):
"""
Subroutine to calculate the number of clean components and their flux
image (string): The absolute path to the image file.
returns (numpy array): The number of pixels with clean components and their summed flux in Jy
"""
setinit.setinitdirs(self)
char_set = string.ascii_uppercase + string.digits
if os.path.isdir(image) or os.path.isfile(image):
if os.path.isdir(image):
temp_string = ''.join(random.sample(char_set * 8, 8))
fits = lib.miriad('fits')
fits.op = 'xyout'
fits.in_ = image
with tempfile.TemporaryDirectory() as tempdir:
fits.out = tempdir + '/' + temp_string + '.fits'
fits.go()
model_data = pyfits.open(tempdir + '/' + temp_string + '.fits')
elif os.path.isfile(image):
model_data = pyfits.open(image)
else:
error = 'Image format not supported. Only MIRIAD and FITS formats are supported!'
logger.error(error)
raise ApercalException(error)
data = model_data[0].data[:, 0, :, :]
modelstats = np.full(2, np.nan)
modelstats[0] = np.count_nonzero(data)
modelstats[1] = np.sum(data)
model_data.close()
else:
error = 'Image does not seem to exist!'
logger.error(error)
raise ApercalException(error)
return modelstats
def splitdata(self):
"""
Applies calibrator corrections to data, splits the data into chunks in frequency and bins it to the given
frequency resolution for the self-calibration
"""
if self.selfcal_splitdata:
subs_setinit.setinitdirs(self)
subs_setinit.setdatasetnamestomiriad(self)
subs_managefiles.director(self, 'ch', self.selfcaldir)
logger.info(' Splitting of target data into individual frequency chunks started')
if os.path.exists(self.selfcaldir + '/' + self.target):
logger.info('Calibrator corrections already seem to have been applied #')
else:
logger.info('Applying calibrator solutions to target data before averaging #')
uvaver = lib.miriad('uvaver')
uvaver.vis = self.crosscaldir + '/' + self.target
uvaver.out = self.selfcaldir + '/' + self.target
uvaver.go()
logger.info('Calibrator solutions to target data applied #')
if self.selfcal_flagantenna != '':
uvflag = lib.miriad('uvflag')
uvflag.vis = self.selfcaldir + '/' + self.target
uvflag.flagval = 'flag'
uvflag.select = 'antenna(' + str(self.selfcal_flagantenna) + ')'
uvflag.go()
else:
pass
try:
uv = aipy.miriad.UV(self.selfcaldir + '/' + self.target)
except RuntimeError:
raise ApercalException(' No data in your selfcal directory!')
try:
nsubband = len(uv['nschan']) # Number of subbands in data
except TypeError:
nsubband = 1 # Only one subband in data since exception was triggered
logger.info('Found ' + str(nsubband) + ' subband(s) in target data #')
counter = 0 # Counter for naming the chunks and directories
for subband in range(nsubband):
logger.info('Started splitting of subband ' + str(subband) + ' #')
if nsubband == 1:
numchan = uv['nschan']
finc = np.fabs(uv['sdf'])
else:
numchan = uv['nschan'][subband] # Number of channels per subband
finc = np.fabs(uv['sdf'][subband]) # Frequency increment for each channel
subband_bw = numchan * finc # Bandwidth of one subband
subband_chunks = round(subband_bw / self.selfcal_splitdata_chunkbandwidth)
# Round to the closest power of 2 for frequency chunks with the same bandwidth over the frequency
# range of a subband
subband_chunks = int(np.power(2, np.ceil(np.log(subband_chunks) / np.log(2))))
if subband_chunks == 0:
subband_chunks = 1
chunkbandwidth = (numchan / subband_chunks) * finc
logger.info('Adjusting chunk size to ' + str(
chunkbandwidth) + ' GHz for regular gridding of the data chunks over frequency #')
for chunk in range(subband_chunks):
logger.info(
'Starting splitting of data chunk ' + str(chunk) + ' for subband ' + str(subband) + ' #')
binchan = round(
self.selfcal_splitdata_channelbandwidth / finc) # Number of channels per frequency bin
chan_per_chunk = numchan / subband_chunks
if chan_per_chunk % binchan == 0: # Check if the freqeuncy bin exactly fits
logger.info('Using frequency binning of ' + str(
self.selfcal_splitdata_channelbandwidth) + ' for all subbands #')
else:
# Increase the frequency bin to keep a regular grid for the chunks
while chan_per_chunk % binchan != 0:
binchan = binchan + 1
else:
# Check if the calculated bin is not larger than the subband channel number
if chan_per_chunk >= binchan:
pass
else:
# Set the frequency bin to the number of channels in the chunk of the subband
binchan = chan_per_chunk
logger.info('Increasing frequency bin of data chunk ' + str(
chunk) + ' to keep bandwidth of chunks equal over the whole bandwidth #')
logger.info('New frequency bin is ' + str(binchan * finc) + ' GHz #')
nchan = int(chan_per_chunk / binchan) # Total number of output channels per chunk
start = 1 + chunk * chan_per_chunk
width = int(binchan)
step = int(width)
subs_managefiles.director(self, 'mk', self.selfcaldir + '/' + str(counter).zfill(2))
uvaver = lib.miriad('uvaver')
uvaver.vis = self.selfcaldir + '/' + self.target
uvaver.out = self.selfcaldir + '/' + str(counter).zfill(2) + '/' + str(counter).zfill(2) + '.mir'
uvaver.select = "'" + 'window(' + str(subband + 1) + ')' + "'"
uvaver.line = "'" + 'channel,' + str(nchan) + ',' + str(start) + ',' + str(width) + ',' + str(
step) + "'"
uvaver.go()
counter = counter + 1
logger.info('Splitting of data chunk ' + str(chunk) + ' for subband ' + str(subband) + ' done #')
logger.info('Splitting of data for subband ' + str(subband) + ' done #')
logger.info(' Splitting of target data into individual frequency chunks done')
def run_continuum_minoriteration(self, chunk, majc, minc, drmin, theoretical_noise_threshold):
"""
Does a selfcal minor iteration for the standard mode
chunk: The frequency chunk to image and calibrate
maj: Current major iteration
min: Current minor iteration
drmin: maximum dynamic range for minor iteration
theoretical_noise_threshold: calculated theoretical noise threshold
"""
subs_setinit.setinitdirs(self)
subs_setinit.setdatasetnamestomiriad(self)
logger.info('Minor self-calibration cycle ' + str(minc) + ' for frequency chunk ' + chunk + ' started #')
if minc == 0:
invert = lib.miriad('invert') # Create the dirty image
invert.vis = chunk + '.mir'
invert.map = str(majc).zfill(2) + '/map_' + str(minc).zfill(2)
invert.beam = str(majc).zfill(2) + '/beam_' + str(minc).zfill(2)
invert.imsize = self.selfcal_image_imsize
invert.cell = self.selfcal_image_cellsize
invert.stokes = 'ii'
invert.options = 'mfs,double'
invert.slop = 1
invert.robust = -2
invert.go()
imax = self.calc_imax(str(majc).zfill(2) + '/map_' + str(minc).zfill(2))
noise_threshold = self.calc_noise_threshold(imax, minc, majc)
dynamic_range_threshold = calc_dynamic_range_threshold(imax, drmin,
self.selfcal_standard_minorcycle0_dr)
mask_threshold, mask_threshold_type = calc_mask_threshold(theoretical_noise_threshold, noise_threshold,
dynamic_range_threshold)
logger.info('Mask threshold for major/minor cycle ' + str(majc) + '/' + str(minc) + ' set to ' + str(
mask_threshold) + ' Jy/beam #')
logger.info('Mask threshold set by ' + str(mask_threshold_type) + ' #')
if majc == 0:
maths = lib.miriad('maths')
maths.out = str(majc).zfill(2) + '/mask_' + str(minc).zfill(2)
maths.exp = '"<' + str(majc).zfill(2) + '/map_' + str(minc).zfill(2) + '>"'
maths.mask = '"<' + str(majc).zfill(2) + '/map_' + str(minc).zfill(2) + '>.gt.' + str(
mask_threshold) + '"'
maths.go()
logger.info('Mask with threshold ' + str(mask_threshold) + ' Jy/beam created #')
else:
subs_managefiles.director(self, 'cp', str(majc).zfill(2) + '/mask_' + str(minc).zfill(2),
file_=str(majc - 1).zfill(2) + '/mask_' + str(
self.selfcal_standard_minorcycle - 1).zfill(2))
logger.info('Mask from last minor iteration of last major cycle copied #')
clean_cutoff = self.calc_clean_cutoff(mask_threshold)
logger.info('Clean threshold at major/minor cycle ' + str(majc) + '/' + str(minc) + ' was set to ' + str(
clean_cutoff) + ' Jy/beam #')
clean = lib.miriad('clean') # Clean the image down to the calculated threshold
clean.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2)
clean.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2)
clean.out = str(majc).zfill(2) + '/model_' + str(minc).zfill(2)
clean.cutoff = clean_cutoff
clean.niters = 1000000
clean.region = '"' + 'mask(' + str(majc).zfill(2) + '/mask_' + str(minc).zfill(2) + ')' + '"'
clean.go()
logger.info('Major/minor cycle ' + str(majc) + '/' + str(minc) + ' cleaning done #')
restor = lib.miriad('restor')
restor.model = str(majc).zfill(2) + '/model_' + str(minc).zfill(2)
restor.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2)
restor.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2)
restor.out = str(majc).zfill(2) + '/image_' + str(minc).zfill(2)
restor.mode = 'clean'
restor.go() # Create the cleaned image
logger.info('Cleaned image for major/minor cycle ' + str(majc) + '/' + str(minc) + ' created #')
restor.mode = 'residual'
restor.out = str(majc).zfill(2) + '/residual_' + str(minc).zfill(2)
restor.go()
logger.info('Residual image for major/minor cycle ' + str(majc) + '/' + str(minc) + ' created #')
logger.info('Peak of the residual image is ' + str(
self.calc_imax(str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) + ' Jy/beam #')
logger.info('RMS of the residual image is ' + str(
self.calc_irms(str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) + ' Jy/beam #')
else:
imax = self.calc_imax(str(majc).zfill(2) + '/map_' + str(0).zfill(2))
noise_threshold = self.calc_noise_threshold(imax, minc, majc)
dynamic_range_threshold = calc_dynamic_range_threshold(imax, drmin,
self.selfcal_standard_minorcycle0_dr)
mask_threshold, mask_threshold_type = calc_mask_threshold(theoretical_noise_threshold, noise_threshold,
dynamic_range_threshold)
logger.info('Mask threshold for major/minor cycle ' + str(majc) + '/' + str(minc) + ' set to ' + str(
mask_threshold) + ' Jy/beam #')
logger.info('Mask threshold set by ' + str(mask_threshold_type) + ' #')
maths = lib.miriad('maths')
maths.out = str(majc).zfill(2) + '/mask_' + str(minc).zfill(2)
maths.exp = '"<' + str(majc).zfill(2) + '/image_' + str(minc - 1).zfill(2) + '>"'
maths.mask = '"<' + str(majc).zfill(2) + '/image_' + str(minc - 1).zfill(2) + '>.gt.' + str(
mask_threshold) + '"'
maths.go()
logger.info('Mask with threshold ' + str(mask_threshold) + ' Jy/beam created #')
clean_cutoff = self.calc_clean_cutoff(mask_threshold)
logger.info('Clean threshold at major/minor cycle ' + str(majc) + '/' + str(minc) + ' was set to ' + str(
clean_cutoff) + ' Jy/beam #')
clean = lib.miriad('clean') # Clean the image down to the calculated threshold
clean.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2)
clean.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2)
clean.model = str(majc).zfill(2) + '/model_' + str(minc - 1).zfill(2)
clean.out = str(majc).zfill(2) + '/model_' + str(minc).zfill(2)
clean.cutoff = clean_cutoff
clean.niters = 1000000
clean.region = '"' + 'mask(' + str(majc).zfill(2) + '/mask_' + str(minc).zfill(2) + ')' + '"'
clean.go()
logger.info('Major/minor cycle ' + str(majc) + '/' + str(minc) + ' cleaning done #')
restor = lib.miriad('restor')
restor.model = str(majc).zfill(2) + '/model_' + str(minc).zfill(2)
restor.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2)
restor.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2)
restor.out = str(majc).zfill(2) + '/image_' + str(minc).zfill(2)
restor.mode = 'clean'
restor.go() # Create the cleaned image
logger.info('Cleaned image for major/minor cycle ' + str(majc) + '/' + str(minc) + ' created #')
restor.mode = 'residual'
restor.out = str(majc).zfill(2) + '/residual_' + str(minc).zfill(2)
restor.go()
logger.info('Residual image for major/minor cycle ' + str(majc) + '/' + str(minc) + ' created #')
logger.info('Peak of the residual image is ' + str(
self.calc_imax(str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) + ' Jy/beam #')
logger.info('RMS of the residual image is ' + str(
self.calc_irms(str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) + ' Jy/beam #')
logger.info('Minor self-calibration cycle ' + str(minc) + ' for frequency chunk ' + chunk + ' finished #')
def selfcal_standard(self):
"""
Executes the standard method of self-calibration with the given parameters
"""
subs_setinit.setinitdirs(self)
subs_setinit.setdatasetnamestomiriad(self)
logger.info(' Starting standard self calibration routine')
subs_managefiles.director(self, 'ch', self.selfcaldir)
for chunk in self.list_chunks():
logger.info('Starting standard self-calibration routine on frequency chunk ' + chunk + ' #')
subs_managefiles.director(self, 'ch', self.selfcaldir + '/' + chunk)
if os.path.isfile(self.selfcaldir + '/' + chunk + '/' + chunk + '.mir/visdata'):
theoretical_noise = calc_theoretical_noise(self.selfcaldir + '/' + chunk + '/' + chunk + '.mir')
logger.info('Theoretical noise for chunk ' + chunk + ' is ' + str(theoretical_noise) + ' Jy/beam #')
theoretical_noise_threshold = self.calc_theoretical_noise_threshold(theoretical_noise)
logger.info('Your theoretical noise threshold will be ' + str(
self.selfcal_standard_nsigma) + ' times the theoretical noise corresponding to ' + str(
theoretical_noise_threshold) + ' Jy/beam #')
dr_list = calc_dr_maj(self.selfcal_standard_drinit, self.selfcal_standard_dr0,
self.selfcal_standard_majorcycle, self.selfcal_standard_majorcycle_function)
logger.info(
'Your dynamic range limits are set to ' + str(dr_list) + ' for the major self-calibration cycles #')
for majc in range(self.selfcal_standard_majorcycle):
logger.info(
'Major self-calibration cycle ' + str(majc) + ' for frequency chunk ' + chunk + ' started #')
subs_managefiles.director(self, 'mk', self.selfcaldir + '/' + str(chunk) + '/' + str(majc).zfill(2))
# Calculate the dynamic ranges during minor cycles
dr_minlist = self.calc_dr_min(dr_list, majc, self.selfcal_standard_minorcycle,
self.selfcal_standard_minorcycle_function)
logger.info('The minor cycle dynamic range limits for major cycle ' + str(majc) + ' are ' + str(
dr_minlist) + ' #')
for minc in range(self.selfcal_standard_minorcycle):
try:
self.run_continuum_minoriteration(chunk, majc, minc, dr_minlist[minc],
theoretical_noise_threshold)
except Exception:
logger.warning('Chunk ' + chunk + ' does not seem to contain data to image #')
break
try:
logger.info('Doing self-calibration with uvmin=' + str(
self.selfcal_standard_uvmin[majc]) + ', uvmax=' + str(
self.selfcal_standard_uvmax[majc]) + ', solution interval=' + str(
self.selfcal_standard_solint[majc]) + ' minutes for major cycle ' + str(majc).zfill(
2) + ' #')
selfcal = lib.miriad('selfcal')
selfcal.vis = chunk + '.mir'
selfcal.select = '"' + 'uvrange(' + str(self.selfcal_standard_uvmin[majc]) + ',' + str(
self.selfcal_standard_uvmax[majc]) + ')"'
selfcal.model = str(majc).zfill(2) + '/model_' + str(minc).zfill(2)
selfcal.interval = self.selfcal_standard_solint[majc]
# Choose reference antenna if given
if self.selfcal_refant == '':
pass
else:
selfcal.refant = self.selfcal_refant
# Enable amplitude calibration if triggered
if not self.selfcal_standard_amp: # See if we want to do amplitude calibration
selfcal.options = 'mfs,phase'
elif self.selfcal_standard_amp:
selfcal.options = 'mfs,amp'
elif self.selfcal_standard_amp == 'auto':
modelflux = self.calc_isum(str(majc).zfill(2) + '/model_' + str(minc).zfill(2))
if modelflux >= self.selfcal_standard_amp_auto_limit:
logger.info(
'Flux of clean model is ' + str(modelflux) + ' Jy. Doing amplitude calibration. #')
selfcal.options = 'mfs,amp'
else:
selfcal.options = 'mfs,phase'
if self.selfcal_standard_nfbin >= 1:
selfcal.nfbin = self.selfcal_standard_nfbin
selfcal.go()
logger.info('Major self-calibration cycle ' + str(
majc) + ' for frequency chunk ' + chunk + ' finished #')
except Exception:
logger.warning(
'Model for self-calibration not found. No further calibration on this chunk possible!')
break
logger.info('Standard self-calibration routine for chunk ' + chunk + ' finished #')
else:
logger.warning('No data in chunk ' + chunk + '. Maybe all data is flagged? #')
logger.info(' Standard self calibration routine finished')
def parametric(self):
"""
Parametric self calibration using an NVSS/FIRST skymodel and calculating spectral indices by source matching
with WENSS.
"""
if self.selfcal_parametric:
subs_setinit.setinitdirs(self)
subs_setinit.setdatasetnamestomiriad(self)
logger.info(' Doing parametric self calibration')
subs_managefiles.director(self, 'ch', self.selfcaldir)
for chunk in self.list_chunks():
logger.info('Starting parametric self calibration routine on chunk ' + chunk + ' #')
subs_managefiles.director(self, 'ch', self.selfcaldir + '/' + chunk)
subs_managefiles.director(self, 'mk', self.selfcaldir + '/' + chunk + '/' + 'pm')
parametric_textfile = lsm.lsm_model(chunk + '.mir', self.selfcal_parametric_skymodel_radius,
self.selfcal_parametric_skymodel_cutoff,
self.selfcal_parametric_skymodel_distance)
lsm.write_model(self.selfcaldir + '/' + chunk + '/' + 'pm' + '/model.txt', parametric_textfile)
logger.info('Creating model from textfile model.txt for chunk ' + chunk + ' #')
uv = aipy.miriad.UV(self.selfcaldir + '/' + chunk + '/' + chunk + '.mir')
freq = uv['sfreq']
uvmodel = lib.miriad('uvmodel')
uvmodel.vis = chunk + '.mir'
parametric_modelfile = open(self.selfcaldir + '/' + str(chunk) + '/' + 'pm' + '/model.txt', 'r')
for n, source in enumerate(parametric_modelfile.readlines()):
if n == 0:
uvmodel.options = 'replace,mfs'
else:
uvmodel.options = 'add,mfs'
uvmodel.offset = source.split(',')[0] + ',' + source.split(',')[1]
uvmodel.flux = source.split(',')[2] + ',i,' + str(freq) + ',' + source.split(',')[4].rstrip(
'\n') + ',0,0'
uvmodel.out = 'pm/tmp' + str(n)
uvmodel.go()
uvmodel.vis = uvmodel.out
subs_managefiles.director(self, 'rn', 'pm/model', uvmodel.out) # Rename the last modelfile to model
subs_managefiles.director(self, 'rm', 'pm/tmp*') # Remove all the obsolete modelfiles
logger.info('Doing parametric self-calibration on chunk {} with solution interval {} min'
'and uvrange limits of {}~{} klambda #'.format(chunk, self.selfcal_parametric_solint,
self.selfcal_parametric_uvmin,
self.selfcal_parametric_uvmax))
selfcal = lib.miriad('selfcal')
selfcal.vis = chunk + '.mir'
selfcal.model = 'pm/model'
selfcal.interval = self.selfcal_parametric_solint
selfcal.select = "'" + 'uvrange(' + str(self.selfcal_parametric_uvmin) + ',' + str(
self.selfcal_parametric_uvmax) + ')' + "'"
# Choose reference antenna if given
if self.selfcal_refant == '':
pass
else:
selfcal.refant = self.selfcal_refant
# Do amplitude calibration if wanted
if self.selfcal_parametric_amp:
selfcal.options = 'mfs,amp'
else:
selfcal.options = 'mfs'
selfcal.go()
logger.info('Parametric self calibration routine on chunk ' + chunk + ' done! #')
logger.info(' Parametric self calibration done')
else:
logger.info(' Parametric self calibration disabled')
def convert_selfcaluv2uvfits(self):
"""
Looks for the last self-calibrated uv-fits file, copies its gains over to the original file, applies them and coverts to UVFITS format
"""
subs_setinit.setinitdirs(self)
sbeam = 'selfcal_B' + str(self.beam).zfill(2)
tbeam = 'transfer_B' + str(self.beam).zfill(2)
# Create the parameters for the parameter file for the conversion of the UVFITS-files
transfertargetbeamsselfcaluv2uvfitsstatus = get_param_def(
self, tbeam + '_targetbeams_selfcaluv2uvfits_status', False)
if self.transfer_convert_selfcaluv2uvfits:
subs_setinit.setinitdirs(self)
subs_setinit.setdatasetnamestomiriad(self)
subs_managefiles.director(self,
'ch',
self.transferdir,
verbose=True)
if not transfertargetbeamsselfcaluv2uvfitsstatus:
# Get the status of the selfcal for the specified beam
selfcaltargetbeamsphasestatus = get_param_def(
self, sbeam + '_targetbeams_phase_status', False)
selfcaltargetbeamsampstatus = get_param_def(
self, sbeam + '_targetbeams_amp_status', False)
datasetname_amp = os.path.join(
self.selfcaldir, self.target).rstrip('.mir') + '_amp.mir'
datasetname_phase = os.path.join(self.selfcaldir, self.target)
logger.debug("Setting amplitude selfcal file name: {}".format(
datasetname_amp))
logger.debug("Setting phase selfcal file name: {}".format(
datasetname_phase))
# datasetname_amp = self.get_target_path().rstrip('.mir') + '_amp.mir'
# datasetname_phase = self.get_target_path()
if os.path.isdir(
datasetname_amp) and selfcaltargetbeamsampstatus:
logger.info('Beam ' + self.beam +
': Using amplitude self-calibrated dataset!')
dataset = datasetname_amp
elif os.path.isdir(
datasetname_phase) and selfcaltargetbeamsphasestatus:
logger.info(
'Beam ' + self.beam +
': Using phase self-calibrated dataset. Amplitude calibration was not successful or not wanted!'
)
dataset = datasetname_phase
else:
dataset = None
if dataset is not None:
# Copy the raw dataset to the transfer directory
subs_managefiles.director(
self,
'cp',
self.transferdir + '/' + self.target,
file_=self.crosscaldir + '/' + self.target)
if selfcaltargetbeamsampstatus:
gpcopy = lib.miriad('gpcopy')
gpcopy.vis = datasetname_phase
gpcopy.out = self.transferdir + '/' + self.target
gpcopy.go()
uvaver = lib.miriad('uvaver')
uvaver.vis = self.transferdir + '/' + self.target
uvaver.out = self.transferdir + '/' + \
self.target.rstrip('.mir') + '_phase.mir'
uvaver.go()
gpcopy = lib.miriad('gpcopy')
gpcopy.vis = datasetname_amp
gpcopy.out = self.transferdir + '/' + \
self.target.rstrip('.mir') + '_phase.mir'
gpcopy.go()
fits = lib.miriad('fits')
fits.op = 'uvout'
fits.in_ = self.transferdir + '/' + \
self.target.rstrip('.mir') + '_phase.mir'
fits.out = self.transferdir + '/' + \
self.target.rstrip('.mir') + '.UVFITS'
fits.go()
if os.path.isfile(self.transferdir + '/' +
self.target.rstrip('.mir') +
'.UVFITS'):
subs_managefiles.director(
self, 'rm',
self.transferdir + '/' + self.target)
subs_managefiles.director(
self, 'rm', self.transferdir + '/' +
self.target.rstrip('.mir') + '_phase.mir')
transfertargetbeamsselfcaluv2uvfitsstatus = True
else:
logger.error(
'Beam ' + self.beam +
': Conversion was not successful. No UVFITS-file generated!'
)
transfertargetbeamsselfcaluv2uvfitsstatus = False
elif selfcaltargetbeamsphasestatus:
gpcopy = lib.miriad('gpcopy')
gpcopy.vis = datasetname_phase
gpcopy.out = self.transferdir + '/' + self.target
gpcopy.go()
fits = lib.miriad('fits')
fits.op = 'uvout'
fits.in_ = self.transferdir + '/' + self.target
fits.out = self.transferdir + '/' + \
self.target.rstrip('.mir') + '.UVFITS'
fits.go()
if os.path.isfile(self.transferdir + '/' +
self.target.rstrip('.mir') +
'.UVFITS'):
subs_managefiles.director(
self, 'rm',
self.transferdir + '/' + self.target)
transfertargetbeamsselfcaluv2uvfitsstatus = True
else:
logger.error(
'Beam ' + self.beam +
': Conversion was not successful. No UVFITS-file generated!'
)
transfertargetbeamsselfcaluv2uvfitsstatus = False
else:
logger.error(
'Beam ' + self.beam +
': Self-calibration was not successful. No conversion to UVFITS-format possible!'
)
transfertargetbeamsselfcaluv2uvfitsstatus = False
else:
logger.info(
'Beam ' + self.beam +
': Conversion of final calibrated data to UVFITS-format already successfully executed!'
)
else:
logger.info(
'Beam ' + self.beam +
': Conversion of final calibrated data to UVFITS-format not selected!'
)
# Save the derived parameters to the parameter file
subs_param.add_param(self,
tbeam + '_targetbeams_selfcaluv2uvfits_status',
transfertargetbeamsselfcaluv2uvfitsstatus)
items = os.listdir('/data/apertif/'+str(data_dir)+'/mosaic/cont_tmp/')
def convert_fits():
for i in range(len(items)):
fits = lib.miriad('fits')
fits.in_ = '/data/apertif/'+str(data_dir)+'/mosaic/cont_tmp/'+str(items[i])
fits.out= '/data/apertif/'+str(data_dir)+'/mosaic/cont_tmp/'+str(items[i][:-5])+'.mir'
fits.op = 'xyin'
fits.go()
convert_fits()
#-----------------------------------
# create mosaic image with linmos
linmos = lib.miriad('linmos')
linmos.in_ = '/data/apertif/'+str(data_dir)+'/mosaic/cont_tmp/image_mf_*.mir'
linmos.out = '/data/apertif/'+str(data_dir)+'/mosaic/'+str(data_dir)+'_mosaic_image'
linmos.go()
#-----------------------------------
# convert mosaic miriad image into fits file
fits = lib.miriad('fits')
fits.in_ = '/data/apertif/'+str(data_dir)+'/mosaic/'+str(data_dir)+'_mosaic_image'
fits.op = 'xyout'
fits.out = '/data/apertif/'+str(data_dir)+'/mosaic/'+str(data_dir)+'_mosaic_image.fits'
fits.go()
#-------------------------------------
#clen up temporary files
def regrid_pb(self):
"""
Convert PB image to miriad in working directory
Regrid image
"""
#first get PB image and write to miriad in workingdir
#naming convention might depend on type of PB image
#drift, Alexander's method, etc
#check if drift
if len(self.pbname) == 6:
#YYMMDD name used for driftscans
#pbfits = os.path.join(self.pbdir,"{0}_{1}_I_model_reg.fits".
# format(self.pbname,self.beam))
pbfits = os.path.join(
self.pbdir,
"{0}_{1}_I_model.fits".format(self.pbname, self.beam))
#may have to add a check about string length, but wait until it's a problem
#check if gpall
if self.pbname == 'gpall':
pbfits = os.path.join(self.pbdir,
"pb_{0}_gpall.fits".format(self.beam))
if self.pbname == 'norm':
pbfits = os.path.join(self.pbdir,
"{0}_norm.fits".format(self.beam))
if self.pbname == 'nonorm':
pbfits = os.path.join(self.pbdir,
"{0}_nonorm.fits".format(self.beam))
if self.pbname == 'gp_avg_norm':
pbfits = os.path.join(self.pbdir,
"{0}_gp_avg_norm.fits".format(self.beam))
if self.pbname == 'gp_avg_orig':
pbfits = os.path.join(self.pbdir,
"{0}_gp_avg_orig.fits".format(self.beam))
#having defined pbfits above,
#check it exists, retrieve to miriad in workingdir
#also check that output doesn't exist
pbim = os.path.join(self.workingdir, "pb")
if (os.path.exists(pbfits) and not os.path.isdir(pbim)):
#get pbimage to miriad
fits = lib.miriad('fits')
fits.op = 'xyin'
fits.in_ = pbfits
fits.out = pbim
try:
fits.go()
except Exception as e:
self.status = False
print(("Conversion to miriad image failed "
"for primary beam {0}").format(self.beam))
print(e)
#this seems to then have regrid do nothing
#but it lets me run code and can update/fix this later
#get projection center of continuum image for regridding
#and update center of PB image manually
#is this the right thing to do?
#check w/ Thijs & Helga.....
if (os.path.isdir(self.smimpath) and os.path.isdir(pbim)):
#get center values
gethd = lib.miriad('gethd')
gethd.in_ = os.path.join(self.smimpath, 'crval1')
#gethd.format = 'hms'
ra_ref = gethd.go()
gethd.in_ = os.path.join(self.smimpath, 'crval2')
#gethd.format = 'dms'
dec_ref = gethd.go()
#update center
puthd = lib.miriad('puthd')
puthd.in_ = os.path.join(pbim, 'crval1')
puthd.value = ra_ref[0]
puthd.type = 'double'
puthd.go()
puthd.in_ = os.path.join(pbim, 'crval2')
puthd.value = dec_ref[0]
puthd.type = 'double'
puthd.go()
#keep ra & dec in object; will need later
self.ra = ra_ref[0] * u.rad
self.dec = dec_ref[0] * u.rad
else:
#getting projection center of (smoothed) image failed
self.status = False
print(
("Updating projection center from beam {0}, "
"based on taskid {1}, failed").format(self.beam, self.taskid))
#regrid image
#define out put, pbr - PB R egridded
#check pbim exists for regridding
#plus smoothed path for template
#also that output doesn't exist
if (os.path.isdir(pbim) and os.path.isdir(self.smimpath)
and not os.path.isdir(self.pbpath)):
regrid = lib.miriad('regrid')
regrid.in_ = pbim
regrid.out = self.pbpath
regrid.tin = self.smimpath
regrid.axes = '1,2'
#what if I try options = offset
#then maybe keep image values but on right grid
#regrid.options = 'offset'
#then my image is resize so things don't work later
#what if I add desc value?
#use helper function to get values
###ra_vals = self.get_hdr('ra',self.smimpath)
###dec_vals = self.get_hdr('dec',self.smimpath)
###desc = "{0},{1},{2},{3},{4},{5},{6},{7}".format(
### ra_vals[0],ra_vals[1],ra_vals[2],ra_vals[3],
### dec_vals[0],dec_vals[1],dec_vals[2],dec_vals[3])
try:
regrid.go()
except Exception as e:
self.status = False
print(("Regridding failed for primary beam {0},"
"reference taskid {1}").format(self.beam, self.taskid))
print(e)
else:
self.status = False
print(("Either primary beam or smoothed continuum "
"image missing for beam {0}, taskid{1}").format(
self.beam, self.taskid))
#write regridded pb to fits; need later
if os.path.isdir(self.pbpath):
fits = lib.miriad('fits')
fits.in_ = self.pbpath
fits.out = self.pbrfits
fits.op = 'xyout'
try:
fits.go()
print("Writing {0} to {1}".format(self.pbpath, self.pbrfits))
except Exception as e:
print("Writing {0} failed".format(self.pbrfits))
print(e)
def ms2miriad(self):
"""
Converts the data from MS to MIRIAD format via UVFITS using drivecasa. Does it for the flux calibrator,
polarisation calibrator, and target field independently.
"""
subs_setinit.setinitdirs(self)
ccalbeam = 'ccal_B' + str(self.beam).zfill(2)
cbeam = 'convert_B' + str(self.beam).zfill(2)
# Read the parameters from crosscal
# and check before doing anything
# Status of the solution transfer for the target, flux calibrator and polarisation calibrator
ccal_targetbeams_transfer = get_param_def(
self, ccalbeam + '_targetbeams_transfer', False)
ccal_calibration_calibrator_finished = get_param_def(
self, ccalbeam + '_calibration_calibrator_finished', False)
if not ccal_calibration_calibrator_finished:
error = "Beam {}: Will not convert files to miriad format because cross-calibration failed.".format(
str(self.beam).zfill(2))
logger.error(error)
raise ApercalException(error)
elif not ccal_targetbeams_transfer:
error = "Beam {}: Will not convert files to miriad format because cross-calibration solutions were not successfully applied to target.".format(
str(self.beam).zfill(2))
logger.error(error)
raise ApercalException(error)
# Create the parameters for the parameter file for converting from MS to UVFITS format
# Flux calibrator MS dataset available?
convertfluxcalmsavailable = get_param_def(
self, cbeam + '_fluxcal_MSavailable', False)
# Polarised calibrator MS dataset available?
convertpolcalmsavailable = get_param_def(self,
cbeam + '_polcal_MSavailable',
False)
# Target beam MS dataset available?
converttargetbeamsmsavailable = get_param_def(
self, cbeam + '_targetbeams_MSavailable', False)
# Flux calibrator MS dataset converted to UVFITS?
convertfluxcalms2uvfits = get_param_def(self,
cbeam + '_fluxcal_MS2UVFITS',
False)
# Polarised calibrator MS dataset converted to UVFITS?
convertpolcalms2uvfits = get_param_def(self,
cbeam + '_polcal_MS2UVFITS',
False)
# Target beam MS dataset converted to UVFITS?
converttargetbeamsms2uvfits = get_param_def(
self, cbeam + '_targetbeams_MS2UVFITS', False)
# Flux calibrator UVFITS dataset available?
convertfluxcaluvfitsavailable = get_param_def(
self, cbeam + '_fluxcal_UVFITSavailable', False)
# Polarised calibrator UVFITS dataset available?
convertpolcaluvfitsavailable = get_param_def(
self, cbeam + '_polcal_UVFITSavailable', False)
# Target beam UVFITS dataset available?
converttargetbeamsuvfitsavailable = get_param_def(
self, cbeam + '_targetbeams_UVFITSavailable', False)
# Flux calibrator UVFITS dataset converted to MIRIAD?
convertfluxcaluvfits2miriad = get_param_def(
self, cbeam + '_fluxcal_UVFITS2MIRIAD', False)
# Polarised calibrator UVFITS dataset converted to MIRIAD?
convertpolcaluvfits2miriad = get_param_def(
self, cbeam + '_polcal_UVFITS2MIRIAD', False)
# Target beam UVFITS dataset converted to MIRIAD?
converttargetbeamsuvfits2miriad = get_param_def(
self, cbeam + '_targetbeams_UVFITS2MIRIAD', False)
# Check which datasets are available in MS format #
if self.fluxcal != '':
convertfluxcalmsavailable = path.isdir(self.get_fluxcal_path())
else:
logger.warning(
'Beam ' + self.beam +
': Flux calibrator dataset not specified. Cannot convert flux calibrator!'
)
if self.polcal != '':
convertpolcalmsavailable = path.isdir(self.get_polcal_path())
else:
logger.warning(
'Beam ' + self.beam +
': Polarised calibrator dataset not specified. Cannot convert polarised calibrator!'
)
if self.target != '':
converttargetbeamsmsavailable = path.isdir(self.get_target_path())
else:
logger.warning(
'Beam ' + self.beam +
': Target beam dataset not specified. Cannot convert target beams!'
)
# Save the derived parameters for the availability to the parameter file
subs_param.add_param(self, cbeam + '_fluxcal_MSavailable',
convertfluxcalmsavailable)
subs_param.add_param(self, cbeam + '_polcal_MSavailable',
convertpolcalmsavailable)
subs_param.add_param(self, cbeam + '_targetbeams_MSavailable',
converttargetbeamsmsavailable)
# Convert the flux calibrator
if self.convert_fluxcal:
if self.fluxcal != '':
if not convertfluxcaluvfits2miriad:
if convertfluxcalmsavailable:
logger.debug(
'Beam ' + self.beam +
': Converting flux calibrator dataset from MS to UVFITS format.'
)
subs_managefiles.director(
self,
'mk',
self.get_crosscalsubdir_path(),
verbose=False)
fluxcal_ms = self.get_fluxcal_path()
# convert only if corrected data column exists
if subs_msutils.has_correcteddata(fluxcal_ms):
datacolumn = "corrected"
fluxcal_fits = mspath_to_fitspath(
self.get_crosscalsubdir_path(), fluxcal_ms)
fc_convert = exportuvfits_cmd.format(
vis=self.get_fluxcal_path(),
fits=fluxcal_fits,
datacolumn=datacolumn)
lib.run_casa([fc_convert], timeout=3600)
if path.isfile(fluxcal_fits):
convertfluxcalms2uvfits = True
logger.info(
'Beam ' + self.beam +
': Converted flux calibrator dataset from MS to UVFITS format!'
)
else:
convertfluxcalms2uvfits = False
logger.warning(
'Beam ' + self.beam +
': Could not convert flux calibrator dataset {} '
'from MS to UVFITS format!'.format(
fluxcal_fits))
else:
logger.warning(
'Beam ' + self.beam +
': Flux calibrator does not have a corrected_data column! Not '
'converting flux calibrator dataset!')
else:
logger.warning(
'Beam ' + self.beam +
': Flux calibrator dataset {} not available!'.
format(self.get_fluxcal_path()))
else:
logger.info(
'Beam ' + self.beam +
': Flux calibrator dataset was already converted from MS to UVFITS format'
)
else:
logger.warning(
'Beam ' + self.beam +
': Flux calibrator dataset not specified. Cannot convert flux calibrator!'
)
else:
logger.warning('Beam ' + self.beam +
': Not converting flux calibrator dataset!')
# Convert the polarised calibrator
if self.convert_polcal:
if self.polcal != '':
if not convertpolcaluvfits2miriad:
if convertpolcalmsavailable:
logger.debug(
'Beam ' + self.beam +
': Converting polarised calibrator dataset from MS to UVFITS format.'
)
subs_managefiles.director(
self,
'mk',
self.get_crosscalsubdir_path(),
verbose=False)
polcal_ms = self.get_polcal_path()
# convert only if corrected data column exists
if subs_msutils.has_correcteddata(polcal_ms):
datacolumn = "corrected"
polcal_fits = mspath_to_fitspath(
self.get_crosscalsubdir_path(), polcal_ms)
pc_convert = exportuvfits_cmd.format(
vis=polcal_ms,
fits=polcal_fits,
datacolumn=datacolumn)
lib.run_casa([pc_convert], timeout=3600)
if path.isfile(polcal_fits):
convertpolcalms2uvfits = True
logger.info(
'Beam ' + self.beam +
': Converted polarised calibrator dataset from MS to UVFITS format!'
)
else:
convertpolcalms2uvfits = False
logger.warning(
'Beam ' + self.beam +
': Could not convert polarised calibrator dataset from MS to UVFITS format!'
)
else:
logger.warning(
'Beam ' + self.beam +
': Polarised calibrator does not have a corrected_data column! Not '
'converting polarised calibrator dataset!')
else:
logger.warning(
'Beam ' + self.beam +
': Polarised calibrator dataset not available!')
else:
logger.info(
'Beam ' + self.beam +
': Polarised calibrator dataset was already converted from MS to UVFITS format'
)
else:
logger.warning(
'Beam ' + self.beam +
': Polarised calibrator dataset not specified. Cannot convert polarised calibrator!'
)
else:
logger.warning('Beam ' + self.beam +
': Not converting polarised calibrator dataset!')
# Convert the target beams
if self.convert_target:
if self.target != '':
logger.info(
'Beam ' + self.beam +
': Converting target beam dataset from MS to UVFITS format.'
)
if not converttargetbeamsuvfits2miriad:
if converttargetbeamsmsavailable:
subs_managefiles.director(
self,
'mk',
self.get_crosscalsubdir_path(),
verbose=False)
target_ms = self.get_target_path()
target_fits = mspath_to_fitspath(
self.get_crosscalsubdir_path(), target_ms)
# only convert if corrected data column exists
if subs_msutils.has_correcteddata(target_ms):
datacolumn = "corrected"
tg_convert = exportuvfits_cmd.format(
vis=target_ms,
fits=target_fits,
datacolumn=datacolumn)
lib.run_casa([tg_convert], timeout=10000)
if path.isfile(target_fits):
converttargetbeamsms2uvfits = True
logger.debug(
'Beam ' + self.beam +
': Converted dataset of target beam from MS to UVFITS format!'
)
else:
converttargetbeamsms2uvfits = False
logger.warning(
'Beam ' + self.beam +
': Could not convert dataset for target beam from MS to UVFITS format!'
)
else:
logger.warning(
'Beam ' + self.beam +
': Target beam dataset does not have a corrected_data column! Not '
'converting target beam dataset!')
else:
logger.warning('Beam ' + self.beam +
': Target beam dataset not available!')
else:
logger.info('Beam ' + self.beam +
': Target beam dataset was already '
'converted from MS to UVFITS format')
else:
logger.warning(
'Beam ' + self.beam +
': Target beam dataset not specified. Cannot convert target beam dataset!'
)
else:
logger.warning('Beam ' + self.beam +
': Not converting target beam dataset!')
# Save the derived parameters for the MS to UVFITS conversion to the parameter file
subs_param.add_param(self, cbeam + '_fluxcal_MS2UVFITS',
convertfluxcalms2uvfits)
subs_param.add_param(self, cbeam + '_polcal_MS2UVFITS',
convertpolcalms2uvfits)
subs_param.add_param(self, cbeam + '_targetbeams_MS2UVFITS',
converttargetbeamsms2uvfits)
# Check which datasets are available in UVFITS format #
if self.fluxcal != '':
crosscal_fluxcal = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.fluxcal)
convertfluxcaluvfitsavailable = path.isfile(crosscal_fluxcal)
else:
logger.warning(
'Beam ' + self.beam +
': Flux calibrator dataset not specified. Cannot convert flux calibrator!'
)
if self.polcal != '':
crosscal_polcal = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.polcal)
convertpolcaluvfitsavailable = path.isfile(crosscal_polcal)
else:
logger.warning(
'Beam ' + self.beam +
': Polarised calibrator dataset not specified. Cannot convert polarised calibrator!'
)
if self.target != '':
crosscal_target = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.target)
converttargetbeamsuvfitsavailable = path.isfile(crosscal_target)
else:
logger.warning(
'Beam ' + self.beam +
': Target beam dataset not specified. Cannot convert target beam!'
)
# Save the derived parameters for the availability to the parameter file
subs_param.add_param(self, cbeam + '_fluxcal_UVFITSavailable',
convertfluxcaluvfitsavailable)
subs_param.add_param(self, cbeam + '_polcal_UVFITSavailable',
convertpolcaluvfitsavailable)
subs_param.add_param(self, cbeam + '_targetbeams_UVFITSavailable',
converttargetbeamsuvfitsavailable)
# Convert the available UVFITS-datasets to MIRIAD format #
# Convert the flux calibrator
if self.convert_fluxcal:
if self.fluxcal != '':
if not convertfluxcaluvfits2miriad:
if convertfluxcaluvfitsavailable:
logger.debug(
'Beam ' + self.beam +
': Converting flux calibrator dataset from UVFITS to MIRIAD format.'
)
subs_managefiles.director(
self,
'ch',
self.get_crosscalsubdir_path(),
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.fluxcal)
fits.out = mspath_to_fitspath(
self.get_crosscalsubdir_path(),
self.fluxcal,
ext='mir')
fits.go()
if path.isdir(fits.out):
convertfluxcaluvfits2miriad = True
logger.info(
'Beam ' + self.beam +
': Converted flux calibrator dataset from UVFITS to MIRIAD format!'
)
else:
convertfluxcaluvfits2miriad = False
logger.warning(
'Beam ' + self.beam +
': Could not convert flux calibrator dataset {} from UVFITS to '
'MIRIAD format!'.format(fits.out))
else:
logger.warning(
'Beam ' + self.beam +
': Flux calibrator dataset not available!')
else:
logger.info(
'Beam ' + self.beam +
': Flux calibrator dataset was already converted from UVFITS to MIRIAD format'
)
else:
logger.warning(
'Beam ' + self.beam +
': Flux calibrator dataset not specified. Cannot convert flux calibrator!'
)
else:
logger.warning('Beam ' + self.beam +
': Not converting flux calibrator dataset!')
# Convert the polarised calibrator
if self.convert_polcal:
if self.polcal != '':
if not convertpolcaluvfits2miriad:
if convertpolcaluvfitsavailable:
logger.debug(
'Beam ' + self.beam +
': Converting polarised calibrator dataset from UVFITS to MIRIAD format.'
)
subs_managefiles.director(
self,
'ch',
self.get_crosscalsubdir_path(),
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.polcal)
fits.out = mspath_to_fitspath(
self.get_crosscalsubdir_path(),
self.polcal,
ext='mir')
fits.go()
if path.isdir(fits.out):
convertpolcaluvfits2miriad = True
logger.info(
'Beam ' + self.beam +
': Converted polarised calibrator dataset from UVFITS to MIRIAD format!'
)
else:
convertpolcaluvfits2miriad = False
logger.warning(
'Beam ' + self.beam +
': Could not convert polarised calibrator dataset from UVFITS to MIRIAD format!'
)
else:
logger.warning(
'Beam ' + self.beam +
': Polarised calibrator dataset not available!')
else:
logger.info(
'Beam ' + self.beam +
': Polarised calibrator dataset was already converted from UVFITS to MIRIAD format'
)
else:
logger.warning(
'Beam ' + self.beam +
': Polarised calibrator dataset not specified. Cannot convert polarised calibrator!'
)
else:
logger.warning('Beam ' + self.beam +
': Not converting polarised calibrator dataset!')
# Convert the target beams
if self.convert_target:
if self.target != '':
logger.info(
'Beam ' + self.beam +
': Converting target beam dataset from UVFITS to MIRIAD format.'
)
if not converttargetbeamsuvfits2miriad:
if converttargetbeamsuvfitsavailable:
subs_managefiles.director(
self,
'ch',
self.get_crosscalsubdir_path(),
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.target)
fits.out = mspath_to_fitspath(
self.get_crosscalsubdir_path(),
self.target,
ext='mir')
fits.go()
if path.isdir(fits.out):
converttargetbeamsuvfits2miriad = True
logger.debug(
'Beam ' + self.beam +
': Converted target beam dataset from '
'UVFITS to MIRIAD format!')
else:
converttargetbeamsuvfits2miriad = False
logger.warning(
'Beam ' + self.beam +
': Could not convert target beam dataset '
'{} from UVFITS to MIRIAD format!'.format(
fits.out))
else:
logger.warning('Beam ' + self.beam +
': Target beam dataset not available!')
else:
logger.info('Beam ' + self.beam +
': Target beam dataset was already converted '
'from MS to UVFITS format')
else:
logger.warning(
'Beam ' + self.beam +
': Target beam dataset not specified. Cannot convert target beam datasets!'
)
else:
logger.warning('Beam ' + self.beam +
': Not converting target beam dataset!')
# Save the derived parameters for the MS to UVFITS conversion to the parameter file
subs_param.add_param(self, cbeam + '_fluxcal_UVFITS2MIRIAD',
convertfluxcaluvfits2miriad)
subs_param.add_param(self, cbeam + '_polcal_UVFITS2MIRIAD',
convertpolcaluvfits2miriad)
subs_param.add_param(self, cbeam + '_targetbeams_UVFITS2MIRIAD',
converttargetbeamsuvfits2miriad)
if self.convert_averagems and self.subdirification:
logger.info('Beam ' + self.beam +
': Averaging down target measurement set')
average_cmd = 'mstransform(vis="{vis}", outputvis="{outputvis}", chanaverage=True, chanbin=64)'
vis = self.get_target_path()
outputvis = vis.replace(".MS", "_avg.MS")
lib.run_casa([average_cmd.format(vis=vis, outputvis=outputvis)],
timeout=10000)
# Remove measurement sets if wanted
if self.convert_removems and self.subdirification:
logger.info('Beam ' + self.beam + ': Removing measurement sets')
vis = self.get_target_path()
if path.exists(vis):
subs_managefiles.director(self, 'rm', vis)
# Remove the UVFITS files if wanted
if self.convert_removeuvfits and self.subdirification:
logger.info('Beam ' + self.beam + ': Removing all UVFITS files')
if self.fluxcal != '' and path.exists(
mspath_to_fitspath(
self.get_crosscalsubdir_path(),
self.fluxcal)) and convertfluxcalms2uvfits:
subs_managefiles.director(
self, 'rm',
mspath_to_fitspath(self.get_crosscalsubdir_path(),
self.fluxcal))
logger.info('Beam ' + self.beam +
': Removed fluxcal UVFITS files')
else:
logger.warning(
'Beam ' + self.beam +
': No fluxcal UVFITS file available for removing')
if self.polcal != '' and path.exists(
mspath_to_fitspath(
self.get_crosscalsubdir_path(),
self.polcal)) and convertpolcalms2uvfits:
subs_managefiles.director(
self, 'rm',
mspath_to_fitspath(self.get_crosscalsubdir_path(),
self.polcal))
logger.info('Beam ' + self.beam +
': Removed polcal UVFITS files')
else:
logger.warning(
'Beam ' + self.beam +
': No polcal UVFITS file available for removing')
if self.target != '' and path.exists(
mspath_to_fitspath(
self.get_crosscalsubdir_path(),
self.target)) and convertfluxcalms2uvfits:
subs_managefiles.director(
self, 'rm',
mspath_to_fitspath(self.get_crosscalsubdir_path(),
self.target))
logger.info('Beam ' + self.beam +
': Removed target UVFITS files')
else:
logger.warning(
'Beam ' + self.beam +
': No target UVFITS file available for removing')
def regrid_in_miriad(taskid, image_name, hdu_image, b, c):
"""
Find appropriate beam model and set center to center of image.
Rescale the beam model to appropriate size for the center frequency of the cube.
Regrid the beam model image in miriad to the HI image.
Expand beam model imagine in 3D.
"""
# Change the reference pixel of beam model to reference pixel of image to correct
cb_model = beam_lookup.model_lookup2(taskid, b)
hdulist_cb = pyfits.open(cb_model)
hdulist_cb[0].header['CRVAL1'] = hdu_image[0].header['CRVAL1']
hdulist_cb[0].header['CRVAL2'] = hdu_image[0].header['CRVAL2']
# Rescale to appropriate frequency. This should work for either drift scans or Gaussian regression (only tested on latter):
avg_cube_freq = (
hdu_image[0].header['CRVAL3'] +
hdu_image[0].header['CDELT3'] * hdu_image[0].data.shape[0]) * u.Hz
hdulist_cb[0].header['CDELT1'] = (hdulist_cb[0].header['CDELT1'] *
get_cb_model_freq().to(u.Hz) /
avg_cube_freq).value
hdulist_cb[0].header['CDELT2'] = (hdulist_cb[0].header['CDELT2'] *
get_cb_model_freq().to(u.Hz) /
avg_cube_freq).value
cb2d_name = 'temp_b{}_c{}_cb-2d.fits'.format(b, c)
hdulist_cb.writeto(cb2d_name)
hdulist_cb.close()
print('\tRegridding in miriad using model {}'.format(cb_model))
fits = lib.miriad('fits')
regrid = lib.miriad('regrid')
# Convert files to miriad:
fits.in_ = image_name
fits.out = '{}.mir'.format(image_name[:-5])
fits.op = 'xyin'
fits.go()
fits.in_ = cb2d_name
fits.out = '{}.mir'.format(cb2d_name[:-5])
fits.op = 'xyin'
fits.go()
# Regrid beam image
regrid.in_ = '{}.mir'.format(cb2d_name[:-5])
regrid.out = '{}_rgrid.mir'.format(cb2d_name[:-5])
regrid.tin = '{}.mir'.format(image_name[:-5])
regrid.axes = '1,2'
regrid.go()
# Convert regrided beam image to fits
fits.in_ = '{}_rgrid.mir'.format(cb2d_name[:-5])
fits.out = '{}_rgrid.fits'.format(cb2d_name[:-5])
fits.op = 'xyout'
fits.go()
# Make cb 3D and save as FITS:
hdu_cb = pyfits.open('{}_rgrid.fits'.format(cb2d_name[:-5]))
d_new = np.ones((hdu_image[0].header['NAXIS3'], hdu_cb[0].header['NAXIS2'],
hdu_cb[0].header['NAXIS2']))
d_beam_cube = d_new * hdu_cb[0].data
hdu_cb[0].data = np.float32(d_beam_cube)
print('\tWriting compound beam cube.')
hdu_cb.writeto('{}_cb.fits'.format(image_name[:-5]))
hdu_cb.close()
# Clean up the extra Miriad & 2D cb files
os.system('rm -rf {}*.mir'.format(image_name[:-5]))
os.system('rm -rf {}*'.format(cb2d_name[:-5]))
def run2(i):
global new_cleancube_data, new_modelcube_data, new_residualcube_data
try:
for name in [
'map_{:02}'.format(minc), 'beam_{:02}'.format(minc),
'mask_{:02}'.format(minc)
]:
imsub = lib.miriad('imsub')
imsub.in_ = name
imsub.out = name + "_" + str(chan[i]).zfill(4)
imsub.region = '"images({})"'.format(chan[i] + 1)
imsub.go()
# print("[CLEAN2] Cleaning HI emission using SoFiA mask for Sources {}.".format(args.sources))
klean.map = 'map_{:02}_{:04}'.format(minc, chan[i])
klean.beam = 'beam_{:02}_{:04}'.format(minc, chan[i])
klean.out = 'model_{:02}_{:04}'.format(minc + 1, chan[i])
klean.cutoff = lineimagestats[2] * 0.5
klean.niters = 10000
klean.region = '"mask(mask_{:02}_{:04}/)"'.format(minc, chan[i])
klean.go()
# print("[CLEAN2] Restoring line cube.")
restor.model = 'model_{:02}_{:04}'.format(minc + 1, chan[i])
restor.beam = 'beam_{:02}_{:04}'.format(minc, chan[i])
restor.map = 'map_{:02}_{:04}'.format(minc, chan[i])
restor.out = 'image_{:02}_{:04}'.format(minc + 1, chan[i])
restor.mode = 'clean'
restor.go()
# print("[CLEAN2] Making residual cube.")
out_array = ['image_{:02}_{:04}'.format(minc + 1, chan[i])]
if args.all:
restor.mode = 'residual' # Create the residual image
restor.out = loc + 'residual_{:02}_{:04}'.format(minc + 1, chan[i])
restor.go()
out_array = [
'model_{:02}_{:04}'.format(minc + 1, chan[i]),
'image_{:02}_{:04}'.format(minc + 1, chan[i]),
'residual_{:02}_{:04}'.format(minc + 1, chan[i])
]
for name in out_array:
fits.op = 'xyout'
fits.in_ = name
fits.out = name + '.fits'
fits.go()
new_cleancube_data[chan[i], :, :] = pyfits.getdata(
'image_{:02}_{:04}.fits'.format(minc + 1, chan[i]))
if args.all:
new_modelcube_data[chan[i], :, :] = pyfits.getdata(
'model_{:02}_{:04}.fits'.format(minc + 1, chan[i]))
new_residualcube_data[chan[i], :, :] = pyfits.getdata(
'residual_{:02}_{:04}.fits'.format(minc + 1, chan[i]))
except RuntimeError:
print(
"Channel {} probably all blank and klean failed. Continue to next."
.format(chan[i]))
return 'OK'
def flagline(self):
"""
Creates an image cube of the different chunks and measures the rms in each channel. All channels with an rms
outside of a given sigma interval are flagged in the continuum calibration, but are still used for line imaging.
"""
if self.selfcal_flagline:
subs_setinit.setinitdirs(self)
subs_setinit.setdatasetnamestomiriad(self)
logger.info(' Automatic flagging of HI-line/RFI started')
subs_managefiles.director(self, 'ch', self.selfcaldir)
for chunk in self.list_chunks():
subs_managefiles.director(self, 'ch',
self.selfcaldir + '/' + str(chunk))
logger.info('Looking through data chunk ' + str(chunk) + ' #')
invert = lib.miriad('invert')
invert.vis = chunk + '.mir'
invert.map = 'map'
invert.beam = 'beam'
invert.imsize = self.selfcal_image_imsize
invert.cell = self.selfcal_image_cellsize
invert.stokes = 'ii'
invert.slop = 1
invert.go()
if os.path.exists('map'):
fits = lib.miriad('fits')
fits.in_ = 'map'
fits.op = 'xyout'
fits.out = 'map.fits'
fits.go()
cube = pyfits.open('map.fits')
data = cube[0].data
std = np.nanstd(data, axis=(0, 2, 3))
median = np.median(std)
stdall = np.nanstd(std)
diff = std - median
detections = np.where(
np.abs(self.selfcal_flagline_sigma * diff) > stdall)[0]
if len(detections) > 0:
logger.info('Found high noise in channel(s) ' +
str(detections).lstrip('[').rstrip(']') +
' #')
for d in detections:
uvflag = lib.miriad('uvflag')
uvflag.vis = chunk + '.mir'
uvflag.flagval = 'flag'
uvflag.line = "'" + 'channel,1,' + str(d + 1) + "'"
uvflag.go()
logger.info('Flagged channel(s) ' +
str(detections).lstrip('[').rstrip(']') +
' in data chunk ' + str(chunk) + ' #')
else:
logger.info('No high noise found in data chunk ' +
str(chunk) + ' #')
subs_managefiles.director(
self, 'rm',
self.selfcaldir + '/' + str(chunk) + '/' + 'map')
subs_managefiles.director(
self, 'rm',
self.selfcaldir + '/' + str(chunk) + '/' + 'map.fits')
subs_managefiles.director(
self, 'rm',
self.selfcaldir + '/' + str(chunk) + '/' + 'beam')
else:
logger.info(' No data in chunk ' + str(chunk) + '!')
logger.info(' Automatic flagging of HI-line/RFI done')
def get_measured_beam_maps(beam, beam_map_input_path, beam_map_output_path,
beam_map_output_name, cutoff):
"""
Function to create beam map from drift scans
Args:
input:
beam = beam number (int, e.g. 01)
image_path = path to mir image
beam_map_path = path to the directory with measured beam maps (/data5/apertif/driftscans/fits_files/191023/)
output_name = output name of regridded beam map
"""
work_dir = os.getcwd()
# file name of the beam model fits file which will be copied
input_beam_model = os.path.join(
beam_map_input_path,
"{0}_{1}_I_model.fits".format(beam_map_input_path.split("/")[5], beam))
# file name of the fits file after copying
temp_beam_model_name = os.path.join(
beam_map_output_path, 'beam_model_{0}_temp.fits'.format(beam))
# file name of the miriad file before applying the cutoff
temp_beam_map_output_name = beam_map_output_name.replace(
".map", "_temp.map")
# copy beam model to work directory
logger.debug("Copying beam model of beam {0} ({1} -> {2})".format(
beam, input_beam_model, temp_beam_model_name))
# copy_cmd = 'cp -r {0} {1}'.format(input_beam_model, temp_beam_model_name)
# copy_cmd = 'cp -r {0}191023_{1}_I_model.fits {2}'.format(
# beam_map_input_path, beam, temp_beam_model_name)
# logger.debug(copy_cmd)
# switched to python command
shutil.copy2(input_beam_model, temp_beam_model_name)
# os.system(copy_cmd)
# convert beam model to mir file
# but only if it does not exists
# if os.path.isdir('./beam_model_{0}_temp.mir'.format(beam)) == False:
if os.path.isdir(os.path.join(beam_map_output_path,
beam_map_output_name)) is False:
logger.debug("Converting beam model of beam {}".format(beam))
fits = lib.miriad('fits')
# fits.in_ = './beam_model_{0}_temp.fits'.format(beam)
fits.in_ = temp_beam_model_name
fits.op = 'xyin'
# fits.out = './beam_model_{0}_temp.mir'.format(beam)
fits.out = os.path.join(beam_map_output_path,
temp_beam_map_output_name)
fits.go()
# now apply the cutoff
logger.debug("Applying cutoff of {0} to beam model of beam {1}".format(
cutoff, beam))
beam_cutoff(beam_map_output_name, temp_beam_map_output_name,
beam_map_output_path, cutoff)
# if os.path.isdir('{}.mir'.format(image_path[:-5])) == False:
# fits = lib.miriad('fits')
# fits.in_ = image_path
# fits.op = 'xyin'
# fits.out = '{}.mir'.format(image_path[:-5])
# fits.go()
# replace centre of beam_model
# gethd = lib.miriad('gethd')
# gethd.in_ = '{0}.mir/crval1'.format(image_path[:-5])
# ra_ref=gethd.go()
# gethd.in_ = '{0}.mir/crval2'.format(image_path[:-5])
# dec_ref=gethd.go()
# print(ra_ref, dec_ref)
# puthd = lib.miriad('puthd')
# puthd.in_ = './beam_model_{0}_temp.mir/crval1'.format(beam)
# puthd.value = ra_ref[0]
# puthd.type = 'double'
# puthd.go()
# puthd.in_ = './beam_model_{0}_temp.mir/crval2'.format(beam)
# puthd.value = dec_ref[0]
# puthd.type = 'double'
# puthd.go()
# # regrid beam model
# if os.path.isdir('./beam_model_{0}.mir'.format(beam)) == False:
# regrid = lib.miriad('regrid')
# regrid.in_ = './beam_model_{0}_temp.mir'.format(beam)
# regrid.out = './{0}_{1}.mir'.format(output_name, beam)
# regrid.axes = '1,2'
# regrid.tin = '{0}.mir'.format(image_path[:-5])
# regrid.go()
# print('DONE')
# clean temp file
logger.debug("Removing fits file of beam model of beam {}".format(beam))
os.remove(temp_beam_model_name)
def ms2miriad(self):
"""
Converts the data from MS to MIRIAD format via UVFITS using drivecasa. Does it for the flux calibrator,
polarisation calibrator, and target field independently.
"""
subs_setinit.setinitdirs(self)
nbeams = 37
# Create the parameters for the parameter file for converting from MS to UVFITS format
# Flux calibrator MS dataset available?
convertfluxcalmsavailable = get_param_def(self, 'convert_fluxcal_MSavailable', False)
# Polarised calibrator MS dataset available?
convertpolcalmsavailable = get_param_def(self, 'convert_polcal_MSavailable', False)
# Target beam MS dataset available?
converttargetbeamsmsavailable = get_param_def(self, 'convert_targetbeams_MSavailable', np.full(nbeams, False))
# Flux calibrator MS dataset converted to UVFITS?
convertfluxcalms2uvfits = get_param_def(self, 'convert_fluxcal_MS2UVFITS', False)
# Polarised calibrator MS dataset converted to UVFITS?
convertpolcalms2uvfits = get_param_def(self, 'convert_polcal_MS2UVFITS', False)
# Target beam MS dataset converted to UVFITS?
converttargetbeamsms2uvfits = get_param_def(self, 'convert_targetbeams_MS2UVFITS', np.full(nbeams, False))
# Flux calibrator UVFITS dataset available?
convertfluxcaluvfitsavailable = get_param_def(self, 'convert_fluxcal_UVFITSavailable', False)
# Polarised calibrator UVFITS dataset available?
convertpolcaluvfitsavailable = get_param_def(self, 'convert_polcal_UVFITSavailable', False)
# Target beam UVFITS dataset available?
converttargetbeamsuvfitsavailable = get_param_def(self, 'convert_targetbeams_UVFITSavailable',
np.full(nbeams, False))
# Flux calibrator UVFITS dataset converted to MIRIAD?
convertfluxcaluvfits2miriad = get_param_def(self, 'convert_fluxcal_UVFITS2MIRIAD', False)
# Polarised calibrator UVFITS dataset converted to MIRIAD?
convertpolcaluvfits2miriad = get_param_def(self, 'convert_polcal_UVFITS2MIRIAD', False)
# Target beam UVFITS dataset converted to MIRIAD?
converttargetbeamsuvfits2miriad = get_param_def(self, 'convert_targetbeams_UVFITS2MIRIAD',
np.full(nbeams, False))
# Check which datasets are available in MS format #
if self.fluxcal != '':
convertfluxcalmsavailable = os.path.isdir(self.basedir + '00' + '/' + self.rawsubdir + '/' + self.fluxcal)
else:
logger.warning('Flux calibrator dataset not specified. Cannot convert flux calibrator!')
if self.polcal != '':
convertpolcalmsavailable = os.path.isdir(self.basedir + '00' + '/' + self.rawsubdir + '/' + self.polcal)
else:
logger.warning('Polarised calibrator dataset not specified. Cannot convert polarised calibrator!')
if self.target != '':
for b in range(nbeams):
converttargetbeamsmsavailable[b] = os.path.isdir(
self.basedir + str(b).zfill(2) + '/' + self.rawsubdir + '/' + self.target)
else:
logger.warning('Target beam dataset not specified. Cannot convert target beams!')
# Save the derived parameters for the availability to the parameter file
subs_param.add_param(self, 'convert_fluxcal_MSavailable', convertfluxcalmsavailable)
subs_param.add_param(self, 'convert_polcal_MSavailable', convertpolcalmsavailable)
subs_param.add_param(self, 'convert_targetbeams_MSavailable', converttargetbeamsmsavailable)
# Convert the available MS-datasets to UVFITS #
raw_convert_cmd = 'exportuvfits(vis="{basedir}00/{rawsubdir}/{cal}", ' \
'fitsfile="{basedir}00/{crosscalsubdir}/{calbase}UVFITS", datacolumn="{datacolumn}", ' \
'combinespw=True, padwithflags=True, multisource=True, writestation=True)'
# Convert the flux calibrator
if self.convert_fluxcal:
if self.fluxcal != '':
if not convertfluxcaluvfits2miriad:
if convertfluxcalmsavailable:
logger.debug('Converting flux calibrator dataset from MS to UVFITS format.')
subs_managefiles.director(self, 'mk', self.basedir + '00' + '/' + self.crosscalsubdir,
verbose=False)
path = self.basedir + '00' + '/' + self.rawsubdir + '/' + self.fluxcal
if subs_msutils.has_correcteddata(path):
fc_convert = raw_convert_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir,
cal=self.fluxcal, calbase=self.fluxcal[:-2],
crosscalsubdir=self.crosscalsubdir,
datacolumn="corrected")
else:
fc_convert = raw_convert_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir,
cal=self.fluxcal, calbase=self.fluxcal[:-2],
crosscalsubdir=self.crosscalsubdir,
datacolumn="data")
logger.warning('Flux calibrator does not have a corrected_data column! Using uncorrected'
'data for conversion!')
lib.run_casa([fc_convert], timeout=3600)
if os.path.isfile(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip(
'MS') + 'UVFITS'):
convertfluxcalms2uvfits = True
logger.info('Converted flux calibrator dataset from MS to UVFITS format!')
else:
convertfluxcalms2uvfits = False
logger.warning('Could not convert flux calibrator dataset from MS to UVFITS format!')
else:
logger.warning('Flux calibrator dataset not available!')
else:
logger.info('Flux calibrator dataset was already converted from MS to UVFITS format')
else:
logger.warning('Flux calibrator dataset not specified. Cannot convert flux calibrator!')
else:
logger.warning('Not converting flux calibrator dataset!')
# Convert the polarised calibrator
if self.convert_polcal:
if self.polcal != '':
if not convertpolcaluvfits2miriad:
if convertpolcalmsavailable:
logger.debug('Converting polarised calibrator dataset from MS to UVFITS format.')
subs_managefiles.director(self, 'mk', self.basedir + '00' + '/' + self.crosscalsubdir,
verbose=False)
path = self.basedir + '00' + '/' + self.rawsubdir + '/' + self.polcal
if subs_msutils.has_correcteddata(path):
pc_convert = raw_convert_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir,
cal=self.fluxcal, calbase=self.polcal[:-2],
crosscalsubdir=self.crosscalsubdir,
datacolumn="corrected")
else:
pc_convert = raw_convert_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir,
cal=self.fluxcal, calbase=self.polcal[:-2],
crosscalsubdir=self.crosscalsubdir,
datacolumn="data")
logger.warning('Polarised calibrator does not have a corrected_data column! Using'
'uncorrected data for conversion!')
lib.run_casa([pc_convert], timeout=3600)
if os.path.isfile(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip(
'MS') + 'UVFITS'):
convertpolcalms2uvfits = True
logger.info('Converted polarised calibrator dataset from MS to UVFITS format!')
else:
convertpolcalms2uvfits = False
logger.warning('Could not convert polarised calibrator dataset from MS to UVFITS format!')
else:
logger.warning('Polarised calibrator dataset not available!')
else:
logger.info('Polarised calibrator dataset was already converted from MS to UVFITS format')
else:
logger.warning('Polarised calibrator dataset not specified. Cannot convert polarised calibrator!')
else:
logger.warning('Not converting polarised calibrator dataset!')
# Convert the target beams
if self.convert_target:
if self.target != '':
logger.info('Converting target beam datasets from MS to UVFITS format.')
if self.convert_targetbeams == 'all':
datasets = glob.glob(self.basedir + '[0-9][0-9]' + '/' + self.rawsubdir + '/' + self.target)
logger.debug('Converting all available target beam datasets')
else:
beams = self.convert_targetbeams.split(",")
datasets = [self.basedir + str(b).zfill(2) + '/' + self.rawsubdir + '/' + self.target for b in
beams]
logger.debug('Converting all selected target beam datasets')
for vis in datasets:
if not converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])]:
if converttargetbeamsmsavailable[int(vis.split('/')[-3])]:
subs_managefiles.director(self, 'mk',
self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir,
verbose=False)
beam_dataset = vis.split('/')[-3]
raw_tg_cmd = 'exportuvfits(vis="{basedir}{beam_dataset}/{rawsubdir}/{target}", ' \
'fitsfile="{basedir}{beam_dataset}/{crosscalsubdir}/{targetbase}UVFITS", ' \
'datacolumn="{datacolumn}", combinespw=True, padwithflags=True, ' \
'multisource=True, writestation=True)'
path = self.basedir + beam_dataset + '/' + self.rawsubdir + '/' + self.target
if subs_msutils.has_correcteddata(path):
datacolumn = "corrected"
else:
datacolumn = "data"
logger.warning('Target beam dataset {} does not have a corrected_data column! Using '
'uncorrected data for conversion!'.format(beam_dataset))
tg_convert = raw_tg_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir,
target=self.target, crosscalsubdir=self.crosscalsubdir,
targetbase=self.target.rstrip('MS'), datacolumn=datacolumn,
beam_dataset=beam_dataset)
lib.run_casa([tg_convert], timeout=7200)
if os.path.isfile(self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' +
self.target.rstrip('MS') + 'UVFITS'):
converttargetbeamsms2uvfits[int(vis.split('/')[-3])] = True
logger.debug('Converted dataset of target beam ' + vis.split('/')[
-3] + ' from MS to UVFITS format!')
else:
converttargetbeamsms2uvfits[int(vis.split('/')[-3])] = False
logger.warning('Could not convert dataset for target beam ' + vis.split('/')[
-3] + ' from MS to UVFITS format!')
else:
logger.warning('Dataset for target beam ' + vis.split('/')[-3] + ' not available!')
else:
logger.info('Dataset for target beam ' + vis.split('/')[
-3] + ' was already converted from MS to UVFITS format')
else:
logger.warning('Target beam dataset(s) not specified. Cannot convert target beam datasets!')
else:
logger.warning('Not converting target beam dataset(s)!')
# Save the derived parameters for the MS to UVFITS conversion to the parameter file
subs_param.add_param(self, 'convert_fluxcal_MS2UVFITS', convertfluxcalms2uvfits)
subs_param.add_param(self, 'convert_polcal_MS2UVFITS', convertpolcalms2uvfits)
subs_param.add_param(self, 'convert_targetbeams_MS2UVFITS', converttargetbeamsms2uvfits)
# Check which datasets are available in UVFITS format #
if self.fluxcal != '':
convertfluxcaluvfitsavailable = os.path.isfile(
self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip('MS') + 'UVFITS')
else:
logger.warning('Flux calibrator dataset not specified. Cannot convert flux calibrator!')
if self.polcal != '':
convertpolcaluvfitsavailable = os.path.isfile(
self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip('MS') + 'UVFITS')
else:
logger.warning('Polarised calibrator dataset not specified. Cannot convert polarised calibrator!')
if self.target != '':
for b in range(nbeams):
converttargetbeamsuvfitsavailable[b] = os.path.isfile(
self.basedir + str(b).zfill(2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip(
'MS') + 'UVFITS')
else:
logger.warning('Target beam dataset not specified. Cannot convert target beams!')
# Save the derived parameters for the availability to the parameter file
subs_param.add_param(self, 'convert_fluxcal_UVFITSavailable', convertfluxcaluvfitsavailable)
subs_param.add_param(self, 'convert_polcal_UVFITSavailable', convertpolcaluvfitsavailable)
subs_param.add_param(self, 'convert_targetbeams_UVFITSavailable', converttargetbeamsuvfitsavailable)
# Convert the available UVFITS-datasets to MIRIAD format #
# Convert the flux calibrator
if self.convert_fluxcal:
if self.fluxcal != '':
if not convertfluxcaluvfits2miriad:
if convertfluxcaluvfitsavailable:
logger.debug('Converting flux calibrator dataset from UVFITS to MIRIAD format.')
subs_managefiles.director(self, 'ch', self.basedir + '00' + '/' + self.crosscalsubdir,
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip(
'MS') + 'UVFITS'
fits.out = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip(
'MS') + 'mir'
fits.go()
if os.path.isdir(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip(
'MS') + 'mir'):
convertfluxcaluvfits2miriad = True
logger.info('Converted flux calibrator dataset from UVFITS to MIRIAD format!')
else:
convertfluxcaluvfits2miriad = False
logger.warning('Could not convert flux calibrator dataset from UVFITS to MIRIAD format!')
else:
logger.warning('Flux calibrator dataset not available!')
else:
logger.info('Flux calibrator dataset was already converted from UVFITS to MIRIAD format')
else:
logger.warning('Flux calibrator dataset not specified. Cannot convert flux calibrator!')
else:
logger.warning('Not converting flux calibrator dataset!')
# Convert the polarised calibrator
if self.convert_polcal:
if self.polcal != '':
if not convertpolcaluvfits2miriad:
if convertpolcaluvfitsavailable:
logger.debug('Converting polarised calibrator dataset from UVFITS to MIRIAD format.')
subs_managefiles.director(self, 'ch', self.basedir + '00' + '/' + self.crosscalsubdir,
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip(
'MS') + 'UVFITS'
fits.out = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip(
'MS') + 'mir'
fits.go()
if os.path.isdir(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip(
'MS') + 'mir'):
convertpolcaluvfits2miriad = True
logger.info('Converted polarised calibrator dataset from UVFITS to MIRIAD format!')
else:
convertpolcaluvfits2miriad = False
logger.warning(
'Could not convert polarised calibrator dataset from UVFITS to MIRIAD format!')
else:
logger.warning('Polarised calibrator dataset not available!')
else:
logger.info('Polarised calibrator dataset was already converted from UVFITS to MIRIAD format')
else:
logger.warning('Polarised calibrator dataset not specified. Cannot convert polarised calibrator!')
else:
logger.warning('Not converting polarised calibrator dataset!')
# Convert the target beams
if self.convert_target:
if self.target != '':
logger.info('Converting target beam datasets from UVFITS to MIRIAD format.')
if self.convert_targetbeams == 'all':
datasets = glob.glob(
self.basedir + '[0-9][0-9]' + '/' + self.crosscalsubdir + '/' + self.target.rstrip(
'MS') + 'UVFITS')
logger.debug('Converting all available target beam datasets')
else:
beams = self.convert_targetbeams.split(",")
datasets = [self.basedir + str(b).zfill(2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip(
'MS') + 'UVFITS' for b in beams]
logger.debug('Converting all selected target beam datasets')
for vis in datasets:
if not converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])]:
if converttargetbeamsuvfitsavailable[int(vis.split('/')[-3])]:
subs_managefiles.director(self, 'ch',
self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir,
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = self.basedir + vis.split('/')[
-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS'
fits.out = self.basedir + vis.split('/')[
-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'mir'
fits.go()
if os.path.isdir(self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'mir'):
converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])] = True
logger.debug('Converted dataset of target beam ' + vis.split('/')[
-3] + ' from UVFITS to MIRIAD format!')
else:
converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])] = False
logger.warning('Could not convert dataset for target beam ' + vis.split('/')[
-3] + ' from UVFITS to MIRIAD format!')
else:
logger.warning('Dataset for target beam ' + vis.split('/')[-3] + ' not available!')
else:
logger.info('Dataset for target beam ' + vis.split('/')[
-3] + ' was already converted from MS to UVFITS format')
else:
logger.warning('Target beam dataset(s) not specified. Cannot convert target beam datasets!')
else:
logger.warning('Not converting target beam dataset(s)!')
# Save the derived parameters for the MS to UVFITS conversion to the parameter file
subs_param.add_param(self, 'convert_fluxcal_UVFITS2MIRIAD', convertfluxcaluvfits2miriad)
subs_param.add_param(self, 'convert_polcal_UVFITS2MIRIAD', convertpolcaluvfits2miriad)
subs_param.add_param(self, 'convert_targetbeams_UVFITS2MIRIAD', converttargetbeamsuvfits2miriad)
# Remove the UVFITS files if wanted #
if self.convert_removeuvfits:
logger.info('Removing all UVFITS files')
subs_managefiles.director(self, 'rm',
self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip(
'MS') + 'UVFITS')
subs_managefiles.director(self, 'rm',
self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip(
'MS') + 'UVFITS')
for beam in range(nbeams):
if os.path.isdir(self.basedir + str(beam).zfill(2) + '/' + self.crosscalsubdir):
subs_managefiles.director(self, 'rm', self.basedir + str(beam).zfill(
2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS')
else:
pass
def ms2miriad(self):
"""
Converts the data from MS to MIRIAD format via UVFITS using drivecasa. Does it for the flux calibrator,
polarisation calibrator, and target field independently.
"""
subs_setinit.setinitdirs(self)
nbeams = 37
# Create the parameters for the parameter file for converting from MS to UVFITS format
# Flux calibrator MS dataset available?
convertfluxcalmsavailable = get_param_def(
self, 'convert_fluxcal_MSavailable', False)
# Polarised calibrator MS dataset available?
convertpolcalmsavailable = get_param_def(self,
'convert_polcal_MSavailable',
False)
# Target beam MS dataset available?
converttargetbeamsmsavailable = get_param_def(
self, 'convert_targetbeams_MSavailable', np.full(nbeams, False))
# Flux calibrator MS dataset converted to UVFITS?
convertfluxcalms2uvfits = get_param_def(self,
'convert_fluxcal_MS2UVFITS',
False)
# Polarised calibrator MS dataset converted to UVFITS?
convertpolcalms2uvfits = get_param_def(self,
'convert_polcal_MS2UVFITS',
False)
# Target beam MS dataset converted to UVFITS?
converttargetbeamsms2uvfits = get_param_def(
self, 'convert_targetbeams_MS2UVFITS', np.full(nbeams, False))
# Flux calibrator UVFITS dataset available?
convertfluxcaluvfitsavailable = get_param_def(
self, 'convert_fluxcal_UVFITSavailable', False)
# Polarised calibrator UVFITS dataset available?
convertpolcaluvfitsavailable = get_param_def(
self, 'convert_polcal_UVFITSavailable', False)
# Target beam UVFITS dataset available?
converttargetbeamsuvfitsavailable = get_param_def(
self, 'convert_targetbeams_UVFITSavailable',
np.full(nbeams, False))
# Flux calibrator UVFITS dataset converted to MIRIAD?
convertfluxcaluvfits2miriad = get_param_def(
self, 'convert_fluxcal_UVFITS2MIRIAD', False)
# Polarised calibrator UVFITS dataset converted to MIRIAD?
convertpolcaluvfits2miriad = get_param_def(
self, 'convert_polcal_UVFITS2MIRIAD', False)
# Target beam UVFITS dataset converted to MIRIAD?
converttargetbeamsuvfits2miriad = get_param_def(
self, 'convert_targetbeams_UVFITS2MIRIAD', np.full(nbeams, False))
# Check which datasets are available in MS format #
if self.fluxcal != '':
convertfluxcalmsavailable = path.isdir(self.get_fluxcal_path())
else:
logger.warning(
'Flux calibrator dataset not specified. Cannot convert flux calibrator!'
)
if self.polcal != '':
convertpolcalmsavailable = path.isdir(self.get_polcal_path())
else:
logger.warning(
'Polarised calibrator dataset not specified. Cannot convert polarised calibrator!'
)
if self.target != '':
for b in range(nbeams):
converttargetbeamsmsavailable[b] = path.isdir(
self.get_target_path(str(b).zfill(2)))
else:
logger.warning(
'Target beam dataset not specified. Cannot convert target beams!'
)
# Save the derived parameters for the availability to the parameter file
subs_param.add_param(self, 'convert_fluxcal_MSavailable',
convertfluxcalmsavailable)
subs_param.add_param(self, 'convert_polcal_MSavailable',
convertpolcalmsavailable)
subs_param.add_param(self, 'convert_targetbeams_MSavailable',
converttargetbeamsmsavailable)
# Convert the flux calibrator
if self.convert_fluxcal:
if self.fluxcal != '':
if not convertfluxcaluvfits2miriad:
if convertfluxcalmsavailable:
logger.debug(
'Converting flux calibrator dataset from MS to UVFITS format.'
)
subs_managefiles.director(
self,
'mk',
self.get_crosscalsubdir_path(),
verbose=False)
fluxcal_ms = self.get_fluxcal_path()
if subs_msutils.has_correcteddata(fluxcal_ms):
datacolumn = "corrected"
else:
datacolumn = "data"
logger.warning(
'Flux calibrator does not have a corrected_data column! Using uncorrected'
'data for conversion!')
fluxcal_fits = mspath_to_fitspath(
self.get_crosscalsubdir_path(), fluxcal_ms)
fc_convert = exportuvfits_cmd.format(
vis=self.get_fluxcal_path(),
fits=fluxcal_fits,
datacolumn=datacolumn)
lib.run_casa([fc_convert], timeout=3600)
if path.isfile(fluxcal_fits):
convertfluxcalms2uvfits = True
logger.info(
'Converted flux calibrator dataset from MS to UVFITS format!'
)
else:
convertfluxcalms2uvfits = False
logger.warning(
'Could not convert flux calibrator dataset from MS to UVFITS format!'
)
else:
logger.warning(
'Flux calibrator dataset not available!')
else:
logger.info(
'Flux calibrator dataset was already converted from MS to UVFITS format'
)
else:
logger.warning(
'Flux calibrator dataset not specified. Cannot convert flux calibrator!'
)
else:
logger.warning('Not converting flux calibrator dataset!')
# Convert the polarised calibrator
if self.convert_polcal:
if self.polcal != '':
if not convertpolcaluvfits2miriad:
if convertpolcalmsavailable:
logger.debug(
'Converting polarised calibrator dataset from MS to UVFITS format.'
)
subs_managefiles.director(
self,
'mk',
self.get_crosscalsubdir_path(),
verbose=False)
polcal_ms = self.get_polcal_path()
if subs_msutils.has_correcteddata(polcal_ms):
datacolumn = "corrected"
else:
datacolumn = "data"
logger.warning(
'Polarised calibrator does not have a corrected_data column! Using'
'uncorrected data for conversion!')
polcal_fits = mspath_to_fitspath(
self.get_crosscalsubdir_path(), polcal_ms)
pc_convert = exportuvfits_cmd.format(
vis=polcal_ms,
fits=polcal_fits,
datacolumn=datacolumn)
lib.run_casa([pc_convert], timeout=3600)
if path.isfile(polcal_fits):
convertpolcalms2uvfits = True
logger.info(
'Converted polarised calibrator dataset from MS to UVFITS format!'
)
else:
convertpolcalms2uvfits = False
logger.warning(
'Could not convert polarised calibrator dataset from MS to UVFITS format!'
)
else:
logger.warning(
'Polarised calibrator dataset not available!')
else:
logger.info(
'Polarised calibrator dataset was already converted from MS to UVFITS format'
)
else:
logger.warning(
'Polarised calibrator dataset not specified. Cannot convert polarised calibrator!'
)
else:
logger.warning('Not converting polarised calibrator dataset!')
# Convert the target beams
if self.convert_target:
if self.target != '':
logger.info(
'Converting target beam datasets from MS to UVFITS format.'
)
if self.convert_targetbeams == 'all':
datasets = self.get_datasets()
logger.debug(
'Converting all available target beam datasets')
else:
beams = self.convert_targetbeams.split(",")
datasets = self.get_datasets(beams)
logger.debug(
'Converting all selected target beam datasets')
for vis, beam in datasets:
if not converttargetbeamsuvfits2miriad[int(beam)]:
if converttargetbeamsmsavailable[int(beam)]:
subs_managefiles.director(
self,
'mk',
self.get_crosscalsubdir_path(beam),
verbose=False)
target_ms = self.get_target_path(beam)
target_fits = mspath_to_fitspath(
self.get_crosscalsubdir_path(beam), target_ms)
if subs_msutils.has_correcteddata(target_ms):
datacolumn = "corrected"
else:
datacolumn = "data"
logger.warning(
'Target beam dataset {} does not have a corrected_data column! Using '
'uncorrected data for conversion!'.format(
beam))
cmd = exportuvfits_cmd.format(
vis=target_ms,
fits=target_fits,
beam=beam,
datacolumn=datacolumn)
lib.run_casa([cmd], timeout=7200)
if path.isfile(target_fits):
converttargetbeamsms2uvfits[int(beam)] = True
logger.debug(
'Converted dataset of target beam '
'l{} from MS to UVFITS format!'.format(
beam))
else:
converttargetbeamsms2uvfits[int(beam)] = False
logger.warning(
'Could not convert dataset for target beam '
'{} from MS to UVFITS format!'.format(
beam))
else:
logger.warning(
'Dataset for target beam {} not available!'.
format(beam))
else:
logger.info(
'Dataset for target beam {} was already '
'converted from MS to UVFITS format'.format(beam))
else:
logger.warning(
'Target beam dataset(s) not specified. Cannot convert target beam datasets!'
)
else:
logger.warning('Not converting target beam dataset(s)!')
# Save the derived parameters for the MS to UVFITS conversion to the parameter file
subs_param.add_param(self, 'convert_fluxcal_MS2UVFITS',
convertfluxcalms2uvfits)
subs_param.add_param(self, 'convert_polcal_MS2UVFITS',
convertpolcalms2uvfits)
subs_param.add_param(self, 'convert_targetbeams_MS2UVFITS',
converttargetbeamsms2uvfits)
# Check which datasets are available in UVFITS format #
if self.fluxcal != '':
crosscal_fluxcal = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.fluxcal)
convertfluxcaluvfitsavailable = path.isfile(crosscal_fluxcal)
else:
logger.warning(
'Flux calibrator dataset not specified. Cannot convert flux calibrator!'
)
if self.polcal != '':
crosscal_polcal = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.polcal)
convertpolcaluvfitsavailable = path.isfile(crosscal_polcal)
else:
logger.warning(
'Polarised calibrator dataset not specified. Cannot convert polarised calibrator!'
)
if self.target != '':
for b in range(nbeams):
b_formatted = str(b).zfill(2)
converttargetbeamsuvfitsavailable[b] = path.isfile(
mspath_to_fitspath(
self.get_crosscalsubdir_path(b_formatted),
self.target))
else:
logger.warning(
'Target beam dataset not specified. Cannot convert target beams!'
)
# Save the derived parameters for the availability to the parameter file
subs_param.add_param(self, 'convert_fluxcal_UVFITSavailable',
convertfluxcaluvfitsavailable)
subs_param.add_param(self, 'convert_polcal_UVFITSavailable',
convertpolcaluvfitsavailable)
subs_param.add_param(self, 'convert_targetbeams_UVFITSavailable',
converttargetbeamsuvfitsavailable)
# Convert the available UVFITS-datasets to MIRIAD format #
# Convert the flux calibrator
if self.convert_fluxcal:
if self.fluxcal != '':
if not convertfluxcaluvfits2miriad:
if convertfluxcaluvfitsavailable:
logger.debug(
'Converting flux calibrator dataset from UVFITS to MIRIAD format.'
)
subs_managefiles.director(
self,
'ch',
self.get_crosscalsubdir_path(),
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.fluxcal)
fits.out = mspath_to_fitspath(
self.get_crosscalsubdir_path(),
self.fluxcal,
ext='mir')
fits.go()
if path.isdir(fits.out):
convertfluxcaluvfits2miriad = True
logger.info(
'Converted flux calibrator dataset from UVFITS to MIRIAD format!'
)
else:
convertfluxcaluvfits2miriad = False
logger.warning(
'Could not convert flux calibrator dataset from UVFITS to MIRIAD format!'
)
else:
logger.warning(
'Flux calibrator dataset not available!')
else:
logger.info(
'Flux calibrator dataset was already converted from UVFITS to MIRIAD format'
)
else:
logger.warning(
'Flux calibrator dataset not specified. Cannot convert flux calibrator!'
)
else:
logger.warning('Not converting flux calibrator dataset!')
# Convert the polarised calibrator
if self.convert_polcal:
if self.polcal != '':
if not convertpolcaluvfits2miriad:
if convertpolcaluvfitsavailable:
logger.debug(
'Converting polarised calibrator dataset from UVFITS to MIRIAD format.'
)
subs_managefiles.director(
self,
'ch',
self.get_crosscalsubdir_path(),
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.polcal)
fits.out = mspath_to_fitspath(
self.get_crosscalsubdir_path(),
self.polcal,
ext='mir')
fits.go()
if path.isdir(fits.out):
convertpolcaluvfits2miriad = True
logger.info(
'Converted polarised calibrator dataset from UVFITS to MIRIAD format!'
)
else:
convertpolcaluvfits2miriad = False
logger.warning(
'Could not convert polarised calibrator dataset from UVFITS to MIRIAD format!'
)
else:
logger.warning(
'Polarised calibrator dataset not available!')
else:
logger.info(
'Polarised calibrator dataset was already converted from UVFITS to MIRIAD format'
)
else:
logger.warning(
'Polarised calibrator dataset not specified. Cannot convert polarised calibrator!'
)
else:
logger.warning('Not converting polarised calibrator dataset!')
# Convert the target beams
if self.convert_target:
if self.target != '':
logger.info(
'Converting target beam datasets from UVFITS to MIRIAD format.'
)
if self.convert_targetbeams == 'all':
datasets = glob.glob(
mspath_to_fitspath(
self.get_crosscalsubdir_path('[0-9][0-9]'),
self.get_target_path()))
logger.debug(
'Converting all available target beam datasets')
else:
beams = self.convert_targetbeams.split(",")
datasets = [
mspath_to_fitspath(
self.get_crosscalsubdir_path(str(b).zfill(2)),
self.target) for b in beams
]
logger.debug(
'Converting all selected target beam datasets')
for vis in datasets:
beam = vis.split('/')[-3]
if not converttargetbeamsuvfits2miriad[int(beam)]:
if converttargetbeamsuvfitsavailable[int(beam)]:
subs_managefiles.director(
self,
'ch',
self.get_crosscalsubdir_path(beam),
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = mspath_to_fitspath(
self.get_crosscalsubdir_path(beam),
self.target)
fits.out = mspath_to_fitspath(
self.get_crosscalsubdir_path(beam),
self.target,
ext='mir')
fits.go()
if path.isdir(fits.out):
converttargetbeamsuvfits2miriad[int(
beam)] = True
logger.debug(
'Converted dataset of target beam {} from '
'UVFITS to MIRIAD format!'.format(beam))
else:
converttargetbeamsuvfits2miriad[int(
beam)] = False
logger.warning(
'Could not convert dataset for target beam '
'{} from UVFITS to MIRIAD format!'.format(
beam))
else:
logger.warning(
'Dataset for target beam {} not available!'.
format(beam))
else:
logger.info(
'Dataset for target beam {} was already converted '
'from MS to UVFITS format'.format(beam))
else:
logger.warning(
'Target beam dataset(s) not specified. Cannot convert target beam datasets!'
)
else:
logger.warning('Not converting target beam dataset(s)!')
# Save the derived parameters for the MS to UVFITS conversion to the parameter file
subs_param.add_param(self, 'convert_fluxcal_UVFITS2MIRIAD',
convertfluxcaluvfits2miriad)
subs_param.add_param(self, 'convert_polcal_UVFITS2MIRIAD',
convertpolcaluvfits2miriad)
subs_param.add_param(self, 'convert_targetbeams_UVFITS2MIRIAD',
converttargetbeamsuvfits2miriad)
# Remove the UVFITS files if wanted #
if self.convert_removeuvfits:
logger.info('Removing all UVFITS files')
subs_managefiles.director(
self, 'rm',
mspath_to_fitspath(self.get_crosscalsubdir_path(),
self.fluxcal))
subs_managefiles.director(
self, 'rm',
mspath_to_fitspath(self.get_crosscalsubdir_path(),
self.polcal))
for beam in range(nbeams):
basedir = self.get_crosscalsubdir_path(str(beam).zfill(2))
if path.isdir(basedir):
subs_managefiles.director(self, 'rm', basedir, self.target)
def mosaic_continuum_mf(self):
"""Looks for all available stacked continuum images and mosaics them into one large image."""
subs_setinit.setinitdirs(self)
subs_setinit.setdatasetnamestomiriad(self)
##########################################################################################################
# Check if the parameter is already in the parameter file and load it otherwise create the needed arrays #
##########################################################################################################
mosaiccontinuummfstatus = get_param_def(
self, 'mosaic_continuum_mf_status',
False) # Status of the continuum mf mosaic
mosaiccontinuummfcontinuumstatus = get_param_def(
self, 'mosaic_continuum_mf_continuumstatus',
np.full(self.NBEAMS, False)) # Status of the continuum imaging
mosaiccontinuummfcopystatus = get_param_def(
self, 'mosaic_continuum_mf_copystatus',
np.full(self.NBEAMS, False)) # Status of the copy of the images
mosaiccontinuummfconvolstatus = get_param_def(
self, 'mosaic_continuum_mf_convolstatus',
np.full(self.NBEAMS, False)) # Status of the convolved images
mosaiccontinuummfcontinuumbeamparams = get_param_def(
self, 'mosaic_continuum_mf_continuumbeamparams',
np.full((self.NBEAMS, 3),
np.nan)) # Beam sizes of the input images
mosaiccontinuummfcontinuumimagestats = get_param_def(
self, 'mosaic_continuum_mf_continuumimagestats',
np.full((self.NBEAMS, 3),
np.nan)) # Image statistics of the input images
# Start the mosaicking of the stacked continuum images
if self.mosaic_continuum_mf:
subs_setinit.setinitdirs(self)
subs_setinit.setdatasetnamestomiriad(self)
subs_managefiles.director(self, 'ch', self.mosdir + '/continuum')
if not mosaiccontinuummfstatus:
logger.info('Mosaicking multi-frequency continuum images')
# Acquire the results and statistics from continuum mf imaging
for b in range(self.NBEAMS):
mosaiccontinuummfcontinuumstatus[b] = get_param_def(
self, 'continuum_B' + str(b).zfill(2) +
'_targetbeams_mf_status', False)
if mosaiccontinuummfcontinuumstatus[b]:
finalminor = get_param_def(
self, 'continuum_B' + str(b).zfill(2) +
'_targetbeams_mf_final_minorcycle', np.nan)
subs_managefiles.director(
self,
'cp',
str(b).zfill(2) + '.fits',
file_=self.basedir + str(b).zfill(2) + '/' +
self.contsubdir + '/' + 'image_mf_' +
str(finalminor).zfill(2) + '.fits')
if os.path.isfile(str(b).zfill(2) + '.fits'):
mosaiccontinuummfcopystatus[b] = True
subs_convim.fitstomir(
str(b).zfill(2) + '.fits',
str(b).zfill(2))
subs_managefiles.director(
self, 'rm',
str(b).zfill(2) + '.fits')
else:
mosaiccontinuummfcopystatus[b] = False
logger.warning('Beam ' + str(b).zfill(2) +
' was not copied successfully!')
# Copy the images over to the mosaic directory
for b in range(self.NBEAMS):
if mosaiccontinuummfcontinuumstatus[
b] and mosaiccontinuummfcopystatus[b]:
# Get the image beam parameters and the image statistics
mosaiccontinuummfcontinuumimagestats[
b, :] = subs_imstats.getimagestats(
self,
str(b).zfill(2))
mosaiccontinuummfcontinuumbeamparams[
b, :] = subs_readmirhead.getbeamimage(
str(b).zfill(2))
else:
logger.warning(
'Skipping Beam ' + str(b).zfill(2) +
'! Continuum mf-imaging was not successful or continuum image not available!'
)
# Calculate the synthesised beam and reject outliers (algorithm needs to be updated)
rejbeams, beamparams = subs_combim.calc_synbeam(
mosaiccontinuummfcontinuumbeamparams)
# Convolve all the images to the calculated beam
for b in range(self.NBEAMS):
if mosaiccontinuummfcontinuumstatus[
b] and mosaiccontinuummfcopystatus[b]:
try:
convol = lib.miriad('convol')
convol.map = str(b).zfill(2)
convol.fwhm = str(beamparams[0]) + ',' + str(
beamparams[1])
convol.pa = str(beamparams[2])
convol.options = 'final'
convol.out = str(b).zfill(2) + '_cv'
convol.go()
if os.path.isdir(str(b).zfill(2) + '_cv'):
mosaiccontinuummfconvolstatus[b] = True
else:
mosaiccontinuummfconvolstatus[b] = False
logger.warning(
'Beam ' + str(b).zfill(2) +
' could not be convolved to the calculated beam size! File not there!'
)
except:
mosaiccontinuummfconvolstatus[b] = False
logger.warning(
'Beam ' + str(b).zfill(2) +
' could not be convolved to the calculated beam size!'
)
# Combine all the images using linmos (needs to be updated with proper primary beam model)
linmosimages = ''
linmosrms = ''
for b in range(self.NBEAMS):
if mosaiccontinuummfcontinuumstatus[
b] and mosaiccontinuummfcopystatus[
b] and mosaiccontinuummfconvolstatus[b]:
linmosimages = linmosimages + str(b).zfill(2) + '_cv,'
linmosrms = linmosrms + str(
subs_imstats.getimagestats(
self,
str(b).zfill(2) + '_cv')[2]) + ','
linmos = lib.miriad('linmos')
linmos.in_ = linmosimages.rstrip(',')
linmos.rms = linmosrms.rstrip(',')
linmos.out = self.target.rstrip('.MS') + '_mf'
linmos.go()
if os.path.isdir(self.target.rstrip('.MS') + '_mf'):
mosaiccontinuummfstatus = True
subs_convim.mirtofits(
self.target.rstrip('.MS') + '_mf',
self.target.rstrip('.MS') + '_mf.fits')
logger.info(
'Mosaicking of multi-frequency image successful!')
else:
mosaiccontinuummfstatus = False
logger.error(
'Multi-freqeuncy mosaic was not created successfully!')
else:
mosaiccontinuummfstatus = True
logger.info(
'Multi-frequency continuum mosaic was already successfully created!'
)
# Save the derived parameters to the parameter file
subs_param.add_param(self, 'mosaic_continuum_mf_status',
mosaiccontinuummfstatus)
subs_param.add_param(self, 'mosaic_continuum_mf_continuumstatus',
mosaiccontinuummfcontinuumstatus)
subs_param.add_param(self, 'mosaic_continuum_mf_copystatus',
mosaiccontinuummfcopystatus)
subs_param.add_param(self, 'mosaic_continuum_mf_convolstatus',
mosaiccontinuummfconvolstatus)
subs_param.add_param(self, 'mosaic_continuum_mf_continuumbeamparams',
mosaiccontinuummfcontinuumbeamparams)
subs_param.add_param(self, 'mosaic_continuum_mf_continuumimagestats',
mosaiccontinuummfcontinuumimagestats)
# Output what exactly is being used to clean the data
print("\t{}".format(maskfits))
# Edit mask cube to trick Miriad into using the whole volume.
m = pyfits.open(maskfits, mode='update')
m[0].data[0, 0, 0] = -1
m[0].data[-1, -1, -1] = -1
m[0].scale('int16')
m.flush()
# Delete any pre-existing Miriad files.
os.system('rm -rf model_* beam_* map_* image_* mask_* residual_*')
print("[CLEAN] Reading in FITS files, making Miriad mask.")
fits = lib.miriad('fits')
fits.op = 'xyin'
if args.nospline:
fits.in_ = line_cube
else:
fits.in_ = new_splinefits
fits.out = 'map_00'
fits.go()
if not os.path.isfile(beam_cube):
print("[CLEAN] Retrieving synthesized beam cube from ALTA.")
os.system('iget {}{}_AP_B0{:02}/HI_beam_cube{}.fits {}'.format(
alta_dir, taskid, b, c, loc))
fits.in_ = beam_cube
fits.out = 'beam_00'
fits.go()
def get_cont_image(self):
"""
Get continuum image specified by taskid, beam
Want to retrieve from ALTA
Can use mosaic module in Apercal as a guide to how to do this
Convert to miriad format
"""
#directory for beam/taskid
altadir = ("/altaZone/archive/apertif_main/"
"visibilities_default/{0}_AP_B{1}").format(
self.taskid, self.beam.zfill(3))
#check if beam/taskid available in ALTA
if self.check_alta_path(altadir) == 0:
#found beam so can continue
#have to find continuum image name since it can be different
fits_name = ''
alta_fits_path = ''
for k in range(10):
fits_name = "image_mf_{0:02d}.fits".format(k)
alta_fits_path = os.path.join(altadir, fits_name)
if self.check_alta_path(alta_fits_path) == 0:
break
else:
# make empty again when no image was found
fits_name = ''
continue
if fits_name == '':
#failed to find image. Note this and print warning
self.status = False
("No continuum image found on ALTA for "
"beam {0} of taskid {1}").format(self.beam, self.taskid)
else:
#create working directory
if not os.path.exists(self.workingdir):
os.makedirs(self.workingdir)
#check if file is already there
self.fitspath = os.path.join(self.workingdir, fits_name)
print(self.fitspath)
if not os.path.exists(self.fitspath):
#copy continuum image to this directory
return_msg = self.get_data_from_alta(
alta_fits_path, self.fitspath)
if return_msg != 0.0:
print(("Failed to retrieve continuum "
"image for beam {0} of taskid {1}").format(
self.beam, self.taskid))
self.status = False
#now, as long as status is good, convert to miriad
#change to check fitspath exist
#setting status but not really using now
if os.path.exists(self.fitspath):
#check that miriad image doesn't already exist and do conversion
if not os.path.isdir(self.impath):
fits = lib.miriad('fits')
fits.op = 'xyin'
fits.in_ = self.fitspath
fits.out = self.impath
try:
fits.go()
except Exception as e:
self.status = False
print(("Conversion to miriad image failed "
"for beam {0} of taskid {1}").format(
self.beam, self.taskid))
print(e)
