def fieldflux(infile):
invert = lib.miriad('invert')
invert.vis = infile
invert.map = 'fluxmap'
invert.beam = 'fluxbeam'
invert.imsize = 2049
invert.cell = 3
invert.stokes = 'ii'
invert.options = 'mfs'
invert.robust = 0
invert.slop = 1
invert.go()
clean = lib.miriad('clean')
clean.map = invert.map
clean.beam = invert.beam
clean.out = 'fluxmodel'
clean.niters = 10000
clean.go()
fits = lib.miriad('fits')
fits.in_ = clean.out
fits.op = 'xyout'
fits.out = clean.out + '.fits'
fits.go()
pyfile = pyfits.open(fits.out)
image = pyfile[0].data[0][0]
pyfile.close()
intflux = np.sum(image)
os.system('rm -rf flux*')
return intflux
def create_parametric_mask(self, dataset, radius, cutoff, cat, outputdir):
'''
Creates a parametric mask using a model from an input catalogue.
dataset (string): The dataset to get the coordiantes for the model from.
radius (float): The radius around the pointing centre of the input dataset to consider sources in in deg.
cutoff (float): The apparent flux percentage to consider sources from 0.0 accounts for no sources, 1.0 for all sources in the catalogue within the search radius of the target field.
cat (string): The catalogue to search sources in. Possible options are 'NVSS', 'FIRST', and 'WENSS'.
outputdir (string): The output directory to create the MIRIAD mask file in. The file is named mask.
'''
lsm.write_mask(outputdir + '/mask.txt',
lsm.lsm_mask(dataset, radius, cutoff, cat))
mskfile = open(outputdir + '/mask.txt', 'r')
object = mskfile.readline().rstrip('\n')
spar = mskfile.readline()
mskfile.close()
imgen = lib.miriad('imgen')
imgen.imsize = self.selfcal_image_imsize
imgen.cell = self.selfcal_image_cellsize
imgen.object = object
imgen.spar = spar
imgen.out = outputdir + '/imgen'
imgen.go()
maths = lib.miriad('maths')
maths.exp = '"<' + outputdir + '/imgen' + '>"'
maths.mask = '"<' + outputdir + '/imgen>.gt.1e-6' + '"'
maths.out = outputdir + '/mask'
maths.go()
subs.managefiles.director(self, 'rm', outputdir + '/imgen')
self.single_mskcutoff = 1e-6
subs.managefiles.director(self, 'rm', outputdir + '/mask.txt')
def cal_mult_bp(scanlist,
cfgfile,
basedir,
sourcename,
gainint='60',
gapint='60',
bpint='60'):
#now do the bandpass solutions - the point of all of this!
#define intervals by default
#numbers don't matter overly much - will be full solution of short scan
#sourcename is important! Make sure it's a recognized calibrator format!!!!
#read cfgfile, not sure this will actually matter here since I set basedir manually
ccal = apercal.ccal(cfgfile)
#set-up utilities for bandpass calibration
mfcal = lib.miriad('mfcal')
# Comment the next line out if you don't want to solve for delays
mfcal.options = 'delay'
mfcal.stokes = 'XX'
mfcal.interval = gainint + ',' + gapint + ',' + bpint
mfcal.tol = 0.1 # Set the tolerance a bit lower. Otherwise mfcal takes a long time to finsh
#IMPORTANT! source names have beam number, need to make sure miriad canrecongize
#these parts are global, and I do specific scans below
puthd = lib.miriad('puthd')
puthd.value = sourcename
#Now iterate and
# Execute the bandpass calibration
#reading into different arrays
#now iterate through scans and do calibration
for i, scan in enumerate(scanlist):
print i, scan
ccal.basedir = basedir + '_' + str(i) + '/'
ccal.crosscaldir = ccal.basedir + '00/crosscal/'
#print ccal.basedir
#print ccal.fluxcal
#print ccal.crosscaldir
puthdstring = ccal.crosscaldir + sourcename + '.mir/source'
#print puthdstring
puthd.in_ = puthdstring #extra underscore because python has a global in
puthd.go()
#convert.show()
#print ccal.crosscaldir
if os.path.exists(ccal.crosscaldir):
print 'doing calibration'
mfcal.vis = ccal.crosscaldir + ccal.fluxcal
mfcal.go()
def getimagestats(self, image):
'''
Subroutine to calculate the max,min and rms of an image
image (string): The absolute path to the image file.
returns (numpy array): The min, max and rms of the image
'''
subs.setinit.setinitdirs(self)
char_set = string.ascii_uppercase + string.digits
if os.path.isdir(image) or os.path.isfile(image):
if os.path.isdir(image):
tempdir = subs.managetmp.manage_tempdir('images')
temp_string = ''.join(random.sample(char_set * 8, 8))
fits = lib.miriad('fits')
fits.op = 'xyout'
fits.in_ = image
fits.out = tempdir + '/' + temp_string + '.fits'
fits.go()
image_data = pyfits.open(tempdir + '/' + temp_string + '.fits')
elif os.path.isfile(image):
image_data = pyfits.open(image)
else:
print(
'Image format not supported. Only MIRIAD and FITS formats are supported!'
)
data = image_data[0].data
imagestats = np.full((3), np.nan)
imagestats[0] = np.nanmin(data) # Get the maxmimum of the image
imagestats[1] = np.nanmax(data) # Get the minimum of the image
imagestats[2] = np.nanstd(data) # Get the standard deviation
image_data.close() # Close the image
subs.managetmp.clean_tempdir('images')
else:
print('Image does not seem to exist!')
return imagestats
def calc_theoretical_noise(self, dataset):
'''
Calculate the theoretical rms of a given dataset
dataset (string): The input dataset to calculate the theoretical rms from
returns (float): The theoretical rms of the input dataset as a float
'''
uv = aipy.miriad.UV(dataset)
obsrms = lib.miriad('obsrms')
try:
tsys = np.median(uv['systemp'])
if np.isnan(tsys):
obsrms.tsys = 30.0
else:
obsrms.tsys = tsys
except KeyError:
obsrms.tsys = 30.0
obsrms.jyperk = uv['jyperk']
obsrms.antdiam = 25
obsrms.freq = uv['sfreq']
obsrms.theta = 15
obsrms.nants = uv['nants']
obsrms.bw = np.abs(uv['sdf'] * uv['nschan']) * 1000.0
obsrms.inttime = 12.0 * 60.0
obsrms.coreta = 0.88
theorms = float(obsrms.go()[-1].split()[3]) / 1000.0
return theorms
def get_bp(file):
'''
Function to create a python array from a bandpass calibrated dataset to analyse
file (str): u,v file with the bandpass calibration
return(array, array): The bandpass array in the following order (antenna, frequencies, solution intervals) and a list of the frequencies
'''
char_set = string.ascii_uppercase + string.digits # Create a charset for random gain log file generation
tempdir = subs.managetmp.manage_tempdir('mirlog')
# tempdir = os.path.expanduser('~') + '/apercal/temp/mirlog'
bp_string = ''.join(random.sample(char_set * 8, 8))
gpplt = lib.miriad('gpplt')
gpplt.vis = file
gpplt.log = tempdir + '/' + bp_string
gpplt.options = 'bandpass'
gpplt.go()
check_table(tempdir + '/' + bp_string, 'bp')
t = Table.read(tempdir + '/' + bp_string, format='ascii')
freqs = np.array(np.unique(t['col1']))
nfreq = len(freqs)
nint = len(t['col1']) / nfreq
nant = int(len(t[0]) - 1)
bp_array = np.zeros((nant, nfreq, nint))
for ant in range(nant):
bp_array[ant, :, :] = np.swapaxes(
t['col' + str(ant + 2)].reshape(nint, nfreq), 0, 1)
return bp_array, freqs
def theostats(infile):
'''
theostats: Calculates the theoretical noise of an observation using the MIRIAD task obsrms
infile: The input file to calculate the noise for
return: The theoretical rms
'''
obsrms = lib.miriad('obsrms')
theorms = 0.00004
return theorms
def transfer_to_target(self):
'''
Transfers the gains of the calibrators to the target field. Automatically checks if polarisation calibration has been done.
'''
if self.crosscal_transfer_to_target:
subs.setinit.setinitdirs(self)
subs.setinit.setdatasetnamestomiriad(self)
subs.managefiles.director(self, 'ch', self.crosscaldir)
self.logger.info(
'### Copying calibrator solutions to target dataset ###')
gpcopy = lib.miriad('gpcopy')
if os.path.isfile(self.crosscaldir + '/' + self.polcal + '/' +
'bandpass') and os.path.isfile(
self.crosscaldir + '/' + self.polcal + '/' +
'gains') and os.path.isfile(
self.crosscaldir + '/' + self.polcal +
'/' + 'leakage'):
gpcopy.vis = self.polcal
self.logger.info(
'# Copying calibrator solutions (bandpass, gains, leakage, angle) from polarised calibrator #'
)
elif os.path.isfile(self.crosscaldir + '/' + self.fluxcal + '/' +
'bandpass') and os.path.isfile(
self.crosscaldir + '/' + self.fluxcal +
'/' + 'gains'):
gpcopy.vis = self.fluxcal
self.logger.info(
'# Copying calibrator solutions (bandpass, gains) from flux calibrator #'
)
self.logger.info(
'# Polarisation calibration solutions (leakage, angle) not found #'
)
else:
self.logger.error(
'# No calibrator solutions found! Exiting! #')
sys.exit(1)
datasets = glob.glob('../../*')
self.logger.info('# Copying calibrator solutions to ' +
str(len(datasets)) + ' beams! #')
for n, ds in enumerate(datasets):
if os.path.isfile(ds + '/' + self.crosscalsubdir + '/' +
self.target + '/visdata'):
gpcopy.out = ds + '/' + self.crosscalsubdir + '/' + self.target
gpcopy.options = 'relax'
gpcopy.go()
self.logger.info('# Calibrator solutions copied to beam ' +
str(n).zfill(2) + '! #')
else:
self.logger.warning('# Beam ' + str(n).zfill(2) +
' does not seem to contain data! #')
self.logger.info(
'### All solutions copied to target data set(s) ###')
else:
self.logger.info(
'### No copying of calibrator solutions to target data done! ###'
)
def resistats(infile):
fits = lib.miriad('fits')
fits.in_ = 'residual'
fits.op = 'xyout'
fits.out = 'rmsresidual' + '.fits'
fits.go()
pyfile = pyfits.open(fits.out)
image = pyfile[0].data[0][0]
pyfile.close()
resimax = np.amax(image)
resirms = np.std(image)
os.system('rm -rf rms*')
return resirms, resimax
def imagetofits(self, mirimage, fitsimage):
'''
Converts a MIRIAD image to a FITS image
mirimage: The MIRIAD image to convert
fitsimage: The converted FITS image
'''
subs.setinit.setinitdirs(self)
fits = lib.miriad('fits')
fits.op = 'xyout'
fits.in_ = mirimage
fits.out = fitsimage
fits.go()
if os.path.isfile(fitsimage):
director(self, 'rm', mirimage)
def get_gains(self, chunk, yaxis):
'''
Function to write and return the plot txt file
chunk (int): The chunk to return the arrays for.
yaxis (string): The axis you want to show the gains for.
return (array, array): The array with the timestep information, the array with the gains for all antennas of a chunk.
'''
subs.setinit.setinitdirs(self)
char_set = string.ascii_uppercase + string.digits # Create a charset for random gain log file generation
self.tempdir = os.path.expanduser('~') + '/apercal/temp/iaplot'
gpplt = lib.miriad('gpplt')
gpplt.vis = self.selfcaldir + '/' + str(chunk).zfill(2) + '/' + str(
chunk).zfill(2) + '.mir'
gpplt.log = self.tempdir + '/' + ''.join(random.sample(
char_set * 8, 8)) + '.txt'
gpplt.yaxis = yaxis
gpplt.options = 'gains'
gpplt.go()
gainstring = ''
with open(gpplt.log, 'r') as gaintxt:
gainlines = gaintxt.readlines()
nants = int(gainlines[3][-3:-1])
obsdate = gainlines[1][-19:-12]
gainlines = gainlines[4:]
joiner = range(0, len(gainlines), int(m.ceil(nants / 6) + 1))
for num, line in enumerate(gainlines):
if num in joiner:
joinedline = re.sub('\s+', ' ', ''.join(
gainlines[num:num + 3])).strip() + '\n'
joinedline = re.sub(':', ' ', joinedline)
gainstring = gainstring + joinedline
gainfile = self.tempdir + '/' + ''.join(random.sample(char_set * 8,
8)) + '.txt'
with open(gainfile, 'w') as gainreformated:
gainreformated.write(gainstring)
with open(gainfile, 'r') as gainarrayfile:
gainarray = np.loadtxt(gainarrayfile)
starttime = datetime.datetime(
int('20' + obsdate[0:2]),
int(time.strptime(obsdate[2:5], '%b').tm_mon), int(obsdate[5:7]))
timearray = [
starttime + datetime.timedelta(days=gainarray[gain, 0],
hours=gainarray[gain, 1],
minutes=gainarray[gain, 2],
seconds=gainarray[gain, 3])
for gain in range(len(gainarray))
]
gainarray = gainarray[:, 4:]
return timearray, gainarray
def imstats(infile, stokes):
invert = lib.miriad('invert')
invert.vis = infile
invert.map = 'rmsmap'
invert.beam = 'rmsbeam'
invert.imsize = 2049
invert.cell = 3
invert.stokes = stokes
invert.options = 'mfs'
invert.robust = 0
invert.slop = 1
invert.go()
fits = lib.miriad('fits')
fits.in_ = invert.map
fits.op = 'xyout'
fits.out = invert.map + '.fits'
fits.go()
pyfile = pyfits.open(fits.out)
image = pyfile[0].data[0][0]
pyfile.close()
inimax = np.amax(image)
inirms = np.std(image)
os.system('rm -rf rms*')
return inimax, inirms
def calc_isum(self, image):
'''
Function to calculate the sum of the values of the pixels in an image
image (string): The name of the image file. Must be in MIRIAD-format
returns (float): the sum of the pxiels in the image
'''
fits = lib.miriad('fits')
fits.op = 'xyout'
fits.in_ = image
fits.out = image + '.fits'
fits.go()
image_data = pyfits.open(image + '.fits') # Open the image
data = image_data[0].data
isum = np.nansum(data) # Get the maximum
image_data.close() # Close the image
subs.managefiles.director(self, 'rm', image + '.fits')
return isum
def get_gains(file):
'''
Function to create a complex python array of amplitude and phase gains from a dataset
file (str): u,v file with the bandpass calibration
return(array, array): an array with the amplitude and phase gains for each antenna and solution interval, a datetime array with the actual solution timesteps
'''
char_set = string.ascii_uppercase + string.digits # Create a charset for random gain log file generation
tempdir = subs.managetmp.manage_tempdir('mirlog')
gains_string = ''.join(random.sample(char_set * 8, 8))
gpplt = lib.miriad('gpplt')
gpplt.vis = file
gpplt.log = tempdir + '/' + gains_string
gpplt.options = 'gains'
gpplt.yaxis = 'amp'
cmd = gpplt.go()
check_table(tempdir + '/' + gains_string, 'gains')
obsdate = cmd[3].split(' ')[4][0:7]
starttime = datetime.datetime(
int('20' + obsdate[0:2]),
int(time.strptime(obsdate[2:5], '%b').tm_mon), int(obsdate[5:7]))
s = Table.read(tempdir + '/' + gains_string, format='ascii')
gpplt.yaxis = 'phase'
gpplt.go()
check_table(tempdir + '/' + gains_string, 'gains')
t = Table.read(tempdir + '/' + gains_string, format='ascii')
days = np.array(t['col1'])
times = np.array(t['col2'])
nint = len(days)
nant = int(len(t[0]) - 2)
gain_array = np.zeros((nant, nint, 2))
for ant in range(nant):
gain_array[ant, :, 0] = s['col' + str(ant + 3)]
# gain_array[ant, :, 1] = t['col' + str(ant + 3)]
gain_array[ant, :, 1] = np.unwrap(t['col' + str(ant + 3)], discont=90)
time_array = [
starttime + datetime.timedelta(days=int(days[step]),
hours=int(times[step][0:2]),
minutes=int(times[step][3:5]),
seconds=int(times[step][6:8]))
for step in range(nint)
]
return gain_array, time_array
def do_bp(cfgfile, inttime='30'):
#function to do the bandpass solution, for given interval
#inttime is the solution interval for bandpass, in minutes
ccal = apercal.ccal(cfgfile)
#change to working directory
#workaround for long path names
ccal.director('ch', ccal.crosscaldir)
gainint = inttime
gapint = inttime
bpint = inttime
mfcal = lib.miriad('mfcal')
mfcal.vis = ccal.fluxcal
print mfcal.vis
#print mfcal.vis
# Comment the next line out if you don't want to solve for delays
mfcal.options = 'delay'
mfcal.stokes = 'XX'
mfcal.interval = gainint + ',' + gapint + ',' + bpint
mfcal.tol = 0.1 # Set the tolerance a bit lower. Otherwise mfcal takes a long time to finsh
mfcal.go()
print 'calibration finished'
def bandpass(self):
'''
Calibrates the bandpass for the flux calibrator using mfcal in MIRIAD.
'''
if self.crosscal_bandpass:
subs.setinit.setinitdirs(self)
subs.setinit.setdatasetnamestomiriad(self)
subs.managefiles.director(self, 'ch', self.crosscaldir)
self.logger.info(
'### Bandpass calibration on the flux calibrator data started ###'
)
mfcal = lib.miriad('mfcal')
mfcal.vis = self.fluxcal
mfcal.stokes = 'ii'
if self.crosscal_delay:
mfcal.options = 'delay'
else:
pass
mfcal.interval = 1000
mfcal.go()
self.logger.info(
'### Bandpass calibration on the flux calibrator data done ###'
)
def get_delays(file):
'''
Function to create a numpy array with the antenna delays for each solution interval
file (str): u,v file with the bandpass calibration
return(array, array): an array with the delays for each antenna and solution interval in nsec, a datetime array with the actual solution timesteps
'''
char_set = string.ascii_uppercase + string.digits # Create a charset for random gain log file generation
tempdir = subs.managetmp.manage_tempdir('mirlog')
# tempdir = os.path.expanduser('~') + '/apercal/temp/mirlog'
gains_string = ''.join(random.sample(char_set * 8, 8))
gpplt = lib.miriad('gpplt')
gpplt.vis = file
gpplt.log = tempdir + '/' + gains_string
gpplt.options = 'delays'
cmd = gpplt.go()
check_table(tempdir + '/' + gains_string, 'delays')
obsdate = cmd[3].split(' ')[4][0:7]
starttime = datetime.datetime(
int('20' + obsdate[0:2]),
int(time.strptime(obsdate[2:5], '%b').tm_mon), int(obsdate[5:7]))
t = Table.read(tempdir + '/' + gains_string, format='ascii')
days = np.array(t['col1'])
times = np.array(t['col2'])
nint = len(days)
nant = int(len(t[0]) - 2)
delay_array = np.zeros((nant, nint))
for ant in range(nant):
delay_array[ant, :] = t['col' + str(ant + 3)]
time_array = [
starttime + datetime.timedelta(days=int(days[step]),
hours=int(times[step][0:2]),
minutes=int(times[step][3:5]),
seconds=int(times[step][6:8]))
for step in range(nint)
]
return delay_array, time_array
def find_src_imsad(self):
'''
Finds sources in continuum image above a threshold in flux specified by the user
IN
Continuum image: located in contdir
IN cfga
abs_ex_imsad_clip: sets flux threshold in Jy
abs_ex_imsad_region: xmin,xmax,ymin,ymax
defines regions where to search for sources
OUT
cont_src_imsad.txt: table with found sources
stored in contdir
'''
self.logger.info('### Find continuum sources ###')
os.chdir(self.contdir)
if os.path.exists(self.cont_im_mir) == False:
fits = lib.miriad('fits')
fits.op = 'xyin'
fits.in_ = self.cont_im
fits.out = self.cont_im_mir
fits.go(rmfiles=True)
if os.path.exists(self.cont_im) == False:
fits = lib.miriad('fits')
fits.op = 'xyout'
fits.in_ = self.cont_im_mir
fits.out = self.cont_im
fits.go(rmfiles=True)
imsad = lib.miriad('imsad')
imsad.in_ = self.cont_im_mir
imsad.out = self.src_imsad_out
imsad.clip = self.abs_ex_imsad_clip
#imsad.region = 'boxes\('+self.abs_ex_imsad_region+'\)'
imsad.options = self.abs_ex_imsad_options
imsad.go(rmfiles=True)
#modify output of imsad for module load_src_csv
src_list = open(self.src_imsad_out, 'r')
lines = src_list.readlines()
len_lines = len(lines)
ra_tmp = []
dec_tmp = []
peak_tmp = []
for i in xrange(0, len_lines):
lines[i] = lines[i].strip()
tmp = lines[i].split(' ')
ra_tmp.append(str(tmp[3]))
dec_tmp.append(str(tmp[4]))
peak_tmp.append(str(tmp[6]))
ra_tmp = np.array(ra_tmp)
dec_tmp = np.array(dec_tmp)
peak_tmp = np.array(peak_tmp)
#ID
ids = np.array(np.arange(1, len_lines + 1, 1), dtype=str)
#J2000
#convert ra
ra_vec = []
for i in xrange(0, len_lines):
line = ra_tmp[i].split(':')
last_dig = int(round(float(line[2]), 0))
if last_dig < 10:
last_dig = '0' + str(last_dig)
ra_vec.append(line[0] + line[1] + str(last_dig))
#convert dec
dec_vec = []
dec_coord = []
for i in xrange(0, len_lines):
line = dec_tmp[i].split(':')
last_dig = int(round(float(line[2]), 0))
first_dig = line[0].split('+')
dec_vec.append(first_dig[2] + line[1] + str(last_dig))
dec_coord.append('+' + first_dig[2] + ':' + line[1] + ':' +
str(last_dig))
J2000_tmp = np.array([a + '+' + b for a, b in zip(ra_vec, dec_vec)])
dec_coord = np.array(dec_coord)
### Find pixels of sources in continuum image
#open continuum image
hdulist = pyfits.open(self.cont_im) # read input
# read data and header
#what follows works for wcs, but can be written better
prihdr = hdulist[0].header
if prihdr['NAXIS'] == 4:
del prihdr['CTYPE4']
del prihdr['CDELT4']
del prihdr['CRVAL4']
del prihdr['CRPIX4']
del prihdr['CTYPE3']
del prihdr['CDELT3']
del prihdr['CRVAL3']
del prihdr['CRPIX3']
del prihdr['NAXIS3']
del prihdr['NAXIS']
prihdr['NAXIS'] = 2
#load wcs system
w = wcs.WCS(prihdr)
self.pixels_cont = np.zeros([ra_tmp.shape[0], 2])
ra_list_tmp = np.zeros([ra_tmp.shape[0]])
for i in xrange(0, ra_tmp.shape[0]):
ra_deg_tmp = self.ra2deg(ra_tmp[i])
dec_deg_tmp = self.dec2deg(dec_coord[i])
px, py = w.wcs_world2pix(ra_deg_tmp, dec_deg_tmp, 0)
self.pixels_cont[i, 0] = str(round(px, 0))
self.pixels_cont[i, 1] = str(round(py, 0))
#make csv file with ID, J2000, Ra, Dec, Pix_y, Pix_y, Peak[Jy]
tot = np.column_stack(
(ids, J2000_tmp, ra_tmp, dec_coord, self.pixels_cont[:, 0],
self.pixels_cont[:, 1], peak_tmp))
self.logger.info('# Continuum sources found. #')
self.write_src_csv(tot)
self.logger.info('### Continuum sources found ###')
return 0
def selfcal_standard(self):
'''
Executes the standard method of self-calibration with the given parameters
'''
subs.setinit.setinitdirs(self)
subs.setinit.setdatasetnamestomiriad(self)
self.logger.info('### Starting standard self calibration routine ###')
subs.managefiles.director(self, 'ch', self.selfcaldir)
for chunk in self.list_chunks():
self.logger.info(
'# Starting standard self-calibration routine on frequency chunk '
+ chunk + ' #')
subs.managefiles.director(self, 'ch',
self.selfcaldir + '/' + chunk)
if os.path.isfile(self.selfcaldir + '/' + chunk + '/' + chunk +
'.mir/visdata'):
theoretical_noise = self.calc_theoretical_noise(
self.selfcaldir + '/' + chunk + '/' + chunk + '.mir')
self.logger.info('# Theoretical noise for chunk ' + chunk +
' is ' + str(theoretical_noise) +
' Jy/beam #')
theoretical_noise_threshold = self.calc_theoretical_noise_threshold(
theoretical_noise)
self.logger.info(
'# Your theoretical noise threshold will be ' +
str(self.selfcal_standard_nsigma) +
' times the theoretical noise corresponding to ' +
str(theoretical_noise_threshold) + ' Jy/beam #')
dr_list = self.calc_dr_maj(
self.selfcal_standard_drinit, self.selfcal_standard_dr0,
self.selfcal_standard_majorcycle,
self.selfcal_standard_majorcycle_function)
self.logger.info('# Your dynamic range limits are set to ' +
str(dr_list) +
' for the major self-calibration cycles #')
for majc in range(self.selfcal_standard_majorcycle):
self.logger.info('# Major self-calibration cycle ' +
str(majc) + ' for frequency chunk ' +
chunk + ' started #')
subs.managefiles.director(
self, 'mk', self.selfcaldir + '/' + str(chunk) + '/' +
str(majc).zfill(2))
dr_minlist = self.calc_dr_min(
dr_list, majc, self.selfcal_standard_minorcycle,
self.selfcal_standard_minorcycle_function
) # Calculate the dynamic ranges during minor cycles
self.logger.info(
'# The minor cycle dynamic range limits for major cycle '
+ str(majc) + ' are ' + str(dr_minlist) + ' #')
for minc in range(self.selfcal_standard_minorcycle):
try:
self.run_continuum_minoriteration(
chunk, majc, minc, dr_minlist[minc],
theoretical_noise_threshold)
except:
self.logger.warning(
'# Chunk ' + chunk +
' does not seem to contain data to image #')
break
try:
self.logger.info(
'# Doing self-calibration with uvmin=' +
str(self.selfcal_standard_uvmin[majc]) +
', uvmax=' +
str(self.selfcal_standard_uvmax[majc]) +
', solution interval=' +
str(self.selfcal_standard_solint[majc]) +
' minutes for major cycle ' + str(majc).zfill(2) +
' #')
selfcal = lib.miriad('selfcal')
selfcal.vis = chunk + '.mir'
selfcal.select = '"' + 'uvrange(' + str(
self.selfcal_standard_uvmin[majc]) + ',' + str(
self.selfcal_standard_uvmax[majc]) + ')"'
selfcal.model = str(majc).zfill(2) + '/model_' + str(
minc).zfill(2)
selfcal.interval = self.selfcal_standard_solint[majc]
# Choose reference antenna if given
if self.selfcal_refant == '':
pass
else:
selfcal.refant = self.selfcal_refant
# Enable amplitude calibration if triggered
if self.selfcal_standard_amp == False: # See if we want to do amplitude calibration
selfcal.options = 'mfs,phase'
elif self.selfcal_standard_amp == True:
selfcal.options = 'mfs,amp'
elif self.selfcal_standard_amp == 'auto':
modelflux = self.calc_isum(
str(majc).zfill(2) + '/model_' +
str(minc).zfill(2))
if modelflux >= self.selfcal_standard_amp_auto_limit:
self.logger.info(
'# Flux of clean model is ' +
str(modelflux) +
' Jy. Doing amplitude calibration. #')
selfcal.options = 'mfs,amp'
else:
selfcal.options = 'mfs,phase'
if self.selfcal_standard_nfbin >= 1:
selfcal.nfbin = self.selfcal_standard_nfbin
selfcal.go()
self.logger.info('# Major self-calibration cycle ' +
str(majc) + ' for frequency chunk ' +
chunk + ' finished #')
except:
self.logger.warning(
'# Model for self-calibration not found. No further calibration on this chunk possible!'
)
break
self.logger.info(
'# Standard self-calibration routine for chunk ' + chunk +
' finished #')
else:
self.logger.warning('# No data in chunk ' + chunk +
'. Maybe all data is flagged? #')
self.logger.info('### Standard self calibration routine finished ###')
def show_image(self, beam=None, step=None, chunk=None, major=None, minor=None, finaltype=None, type=None, imin=None, imax=None):
'''
Function to show an image in the notebook
beam (string): The beam number to show the image for. 'iterate' will iterate over beams.
step (string): The step of the pipeline to show the image for. No default. selfcal and final are allowed.
chunk (string): The frequency chunk to show the image for. 'iterate' will iterate over chunks. No default.
major(string or int): Major iteration of clean to display. Default is last iteration. pm is also possible. Only for selfcal, not for final step. 'iterate' will iterate over images for major cycles.
minor (string or int): Minor iteration of clean to display. Default is last iteration. 'iterate' will iterate over images for minor cycles.
finaltype(string): Type of final image to show. mf and stack are possible.
type(string): Type of image to show. mask, beam, image, residual, and in case of step=final final are possible. 'all' will show image, mask, residual, and model in one plot.
imin(float): Minimum cutoff in the image.
imax(float): Maximum cutoff in the image.
return (string): String with a path and image name to use for displaying.
'''
subs.setinit.setinitdirs(self)
self.manage_tempdir() # Check if the temporary directory exists and if not create it
self.clean_tempdir() # Remove any temorary files from the temporary directory
char_set = string.ascii_uppercase + string.digits # Create a charset for random image name generation
if any(it == 'iterate' for it in [beam, chunk, major, minor]):
if beam == 'iterate':
print('### Not implemented yet ###')
elif chunk == 'iterate':
imagelist = glob.glob(self.selfcaldir + '/*/' + str(major).zfill(2) + '/' + str(type) + '_' + str(minor).zfill(2))
plottext = 'Frequency \nchunk'
elif major == 'iterate':
imagelist = glob.glob(self.selfcaldir + '/' + str(chunk).zfill(2) + '/*/' + str(type) + '_' + str(minor).zfill(2))
plottext = 'Major \ncycle'
elif minor == 'iterate':
imagelist = glob.glob(self.selfcaldir + '/' + str(chunk).zfill(2) + '/' + str(major).zfill(2) + '/' + str(type) + '_*')
plottext = 'Minor \ncycle'
images = []
for image in imagelist:
self.fitsimage = self.tempdir + '/' + ''.join(random.sample(char_set * 8, 8)) + '.fits'
fits = lib.miriad('fits')
fits.op = 'xyout'
fits.in_ = image
fits.out = self.fitsimage
fits.go()
images.append(maputils.FITSimage(self.fitsimage))
# Do the autoscaling of the image contrast
if imin == None or imax == None:
imagedata = pyfits.open(self.fitsimage)[0].data
imagestd = np.nanstd(imagedata)
if imin == None:
imin = -1*imagestd
else:
pass
if imax == None:
imax = 3*imagestd
else:
pass
else:
pass
print('Minimum/maximum colour range for image: ' + str(imin) + '/' + str(imax))
# Draw the plot
fig = plt.figure(figsize=(12, 10))
frame = fig.add_axes([0.1, 0.15, 0.85, 0.8])
frame.set_title(str(type).capitalize())
# Add the slider for selecting the image
slider_ax = fig.add_axes([0.103, 0.05, 0.672, 0.03])
slider = Slider(slider_ax, plottext, 0, len(images)-1, valinit = 0, valfmt='%1.0f', dragging=False, color='black')
# Define the attributes for the plot
image_object = images[int(round(slider.val))].Annotatedimage(frame, cmap='gist_gray', clipmin=imin, clipmax=imax)
image_object.Image()
image_object.Graticule()
image_object.Colorbar()
image_object.interact_toolbarinfo()
image_object.interact_imagecolors()
image_object.plot()
fig.canvas.draw()
# Update the plot if the slider is clicked
def slider_on_changed(val):
image_object = images[int(round(slider.val))].Annotatedimage(frame, cmap='gist_gray', clipmin=imin, clipmax=imax)
image_object.Image()
image_object.interact_toolbarinfo()
image_object.interact_imagecolors()
image_object.plot()
fig.canvas.draw()
slider.on_changed(slider_on_changed)
plt.show()
else:
if type == 'all': # Make a 2x2 plot of image, residual, mask, and model of a cycle
rawimage = self.get_image(beam, step, chunk, major, minor, finaltype, 'image')
rawresidual = self.get_image(beam, step, chunk, major, minor, finaltype, 'residual')
rawmask = self.get_image(beam, step, chunk, major, minor, finaltype, 'mask')
rawmodel = self.get_image(beam, step, chunk, major, minor, finaltype, 'model')
self.fitsimage = self.tempdir + '/' + ''.join(random.sample(char_set * 8, 8)) + '.fits'
self.fitsresidual = self.tempdir + '/' + ''.join(random.sample(char_set * 8, 8)) + '.fits'
self.fitsmask = self.tempdir + '/' + ''.join(random.sample(char_set * 8, 8)) + '.fits'
self.fitsmodel = self.tempdir + '/' + ''.join(random.sample(char_set * 8, 8)) + '.fits'
fits = lib.miriad('fits')
fits.op = 'xyout'
fits.in_ = rawimage
fits.out = self.fitsimage
fits.go()
fits.in_ = rawresidual
fits.out = self.fitsresidual
fits.go()
fits.in_ = rawmask
fits.out = self.fitsmask
fits.go()
regrid = lib.miriad('regrid')
regrid.in_ = rawmodel
tempregrid = self.tempdir + '/' + ''.join(random.sample(char_set * 8, 8))
regrid.out = tempregrid
regrid.axes = '1,2'
regrid.tin = rawimage
regrid.go()
fits.in_ = tempregrid
fits.out = self.fitsmodel
fits.go()
im = maputils.FITSimage(self.fitsimage)
imdata = pyfits.open(self.fitsimage)[0].data
imstd = np.nanstd(imdata)
re = maputils.FITSimage(self.fitsresidual)
redata = pyfits.open(self.fitsresidual)[0].data
restd = np.nanstd(redata)
ma = maputils.FITSimage(self.fitsmask)
madata = pyfits.open(self.fitsmask)[0].data
mastd = np.nanstd(madata)
mo = maputils.FITSimage(self.fitsmodel)
modata = pyfits.open(self.fitsmodel)[0].data
mostd = np.nanstd(modata)
fig = plt.figure(figsize=(12, 10))
frame1 = fig.add_axes([0.1,0.58,0.4,0.4])
frame1.set_title('Image')
frame2 = fig.add_axes([0.59,0.58,0.4,0.4])
frame2.set_title('Residual')
frame3 = fig.add_axes([0.1, 0.12, 0.4, 0.4])
frame3.set_title('Mask')
frame4 = fig.add_axes([0.59, 0.12, 0.4, 0.4])
frame4.set_title('Model')
annim1 = im.Annotatedimage(frame1, cmap='gist_gray')
annim2 = re.Annotatedimage(frame2, cmap='gist_gray')
annim3 = ma.Annotatedimage(frame3, cmap='gist_gray')
annim4 = mo.Annotatedimage(frame4, cmap='gist_gray')
annim1.Image(); annim2.Image(); annim3.Image(); annim4.Image()
if imin == None: # Set the displayed colour range if not set
immin, remin, mamin, momin = -1*imstd, -1*restd, -1*mastd, -1*mostd
else:
immin, remin, mamin, momin = imin, imin, imin, imin
if imax == None:
immax, remax, mamax, momax = 3*imstd, 3*restd, mastd, mostd
else:
immax, remax, mamax, momax = imax, imax, imax, imax
print('Minimum colour range for image: ' + str(immin) + ' residual: ' + str(remin) + ' mask: ' + str(mamin) + ' model: ' + str(momin))
print('Maximum colour range for image: ' + str(immax) + ' residual: ' + str(remax) + ' mask: ' + str(mamax) + ' model: ' + str(momax))
annim1.set_norm(clipmin=immin, clipmax=immax); annim2.set_norm(clipmin=remin, clipmax=remax); annim3.set_norm(clipmin=0.0, clipmax=mamax); annim4.set_norm(clipmin=0.0, clipmax=momax)
annim1.Colorbar(); annim2.Colorbar(); annim3.Colorbar(); annim4.Colorbar()
annim1.Graticule(); annim2.Graticule(); annim3.Graticule(); annim4.Graticule()
annim1.interact_toolbarinfo(); annim2.interact_toolbarinfo(); annim3.interact_toolbarinfo(); annim4.interact_toolbarinfo()
annim1.interact_imagecolors(); annim2.interact_imagecolors(); annim3.interact_imagecolors(); annim4.interact_imagecolors()
tdict = dict(color='g', fontsize=10, va='bottom', ha='left')
fig.text(0.01, 0.01, annim1.get_colornavigation_info(), tdict)
maputils.showall()
else: # Make a simple plot of one specified image
rawimage = self.get_image(beam, step, chunk, major, minor, finaltype, type)
self.fitsimage = self.tempdir + '/' + ''.join(random.sample(char_set * 8, 8)) + '.fits'
fits = lib.miriad('fits')
fits.op = 'xyout'
fits.in_ = rawimage
fits.out = self.fitsimage
fits.go()
f = maputils.FITSimage(self.fitsimage)
imdata = pyfits.open(self.fitsimage)[0].data
imstd = np.nanstd(imdata)
fig = plt.figure(figsize=(12,10))
frame = fig.add_axes([0.1, 0.15, 0.85, 0.8])
annim = f.Annotatedimage(frame, cmap='gist_gray')
annim.Image()
if imin == None: # Set the displayed colour range if not set
imin = -1*imstd
if imax == None:
imax = 3*imstd
print('Minimum/maximum colour range for image: ' + str(imin) + '/' + str(imax))
annim.set_norm(clipmin=imin,clipmax=imax)
annim.Colorbar()
annim.Graticule()
annim.interact_toolbarinfo()
annim.interact_imagecolors()
units = 'unknown'
if 'BUNIT' in f.hdr:
units = f.hdr['BUNIT']
helptext = "File: [%s] Data units: [%s]\n" % (rawimage, units)
helptext += annim.get_colornavigation_info()
tdict = dict(color='g', fontsize=10, va='bottom', ha='left')
fig.text(0.01, 0.01, helptext, tdict)
maputils.showall()
def convert_lineuv2uvfits(self):
'''
Looks for all calibrated datasets created by the line module, combines the chunks of individual beams and converts them to UVFITS format
'''
subs.setinit.setinitdirs(self)
subs.managefiles.director(self, 'ch', self.transferdir, verbose=False)
beamlist = sorted(glob.glob(self.basedir + '[0-9][0-9]'))
beamnames = [beam.split('/')[-1] for beam in beamlist]
subs.param.add_param(self, 'transfer_input_beams', beamnames)
# transferstatusarray = np.full((len(beamnames, 2), np.False))
for b, beam in enumerate(beamlist):
uvgluestatusarray = np.full((len(beamnames)), False)
uvfitsstatusarray = np.full((len(beamnames)), False)
if os.path.isfile(
self.target.rstrip('.mir') + '_B' + beam.split('/')[-1] +
'.UVFITS'):
self.logger.warning('# UVFITS file for beam ' +
beam.split('/')[-1] + ' already exists! #')
else:
chunklist = sorted(
glob.glob(beam + '/' + self.linesubdir + '/' +
'[0-9][0-9]/[0-9][0-9]' + '.mir'))
chunknames = [chunk.split('/')[-2] for chunk in chunklist]
subs.param.add_param(
self,
'transfer_input_beam_' + str(beamnames[b]) + '_chunks',
chunknames)
uvcatstatusarray = np.full((len(chunknames)), False)
self.logger.debug(
'# Starting combination of frequency chunks for beam ' +
beam.split('/')[-1] + ' #')
for c, chunk in enumerate(chunklist):
uvcat = lib.miriad('uvcat')
uvcat.vis = chunk
uvcat.out = self.transferdir + '/' + 'B' + beam.split(
'/')[-1] + '_' + str(c + 1)
uvcat.go()
if os.path.isdir(
self.transferdir + '/' + 'B' +
beam.split('/')[-1] + '_' + str(c + 1)
): # Check if file has been copied successfully
self.logger.debug('# Chunk ' + str(chunk).zfill(2) +
' for beam ' +
str(beam.split('/')[-1]) +
' copied successfully! #')
uvcatstatusarray[c] = True
else:
self.logger.warning('# Chunk ' + str(chunk).zfill(2) +
' for beam ' +
str(beam.split('/')[-1]) +
' NOT copied successfully! #')
uvcatstatusarray[c] = False
subs.param.add_param(
self, 'transfer_input_beam_' + str(beamnames[b]) +
'_copy_status', uvcatstatusarray)
uvglue = lib.miriad('uvglue')
uvglue.vis = 'B' + beam.split('/')[-1]
uvglue.nfiles = len(chunklist)
uvglue.out = self.target.rstrip('.mir') + '_B' + beam.split(
'/')[-1] + '.mir'
uvglue.go()
if os.path.isdir(
self.target.rstrip('.mir') + '_B' +
beam.split('/')[-1] + '.mir'):
self.logger.debug(
'# Combination of frequency chunks for beam ' +
beam.split('/')[-1] + ' successful! #')
subs.managefiles.director(self, 'rm',
'B' + beam.split('/')[-1] + '*')
uvgluestatusarray[b] = True
else:
self.logger.warning(
'# Combination of frequency chunks for beam ' +
beam.split('/')[-1] + ' not successful! #')
uvgluestatusarray[b] = False
fits = lib.miriad('fits')
fits.op = 'uvout'
fits.in_ = self.target.rstrip('.mir') + '_B' + beam.split(
'/')[-1] + '.mir'
fits.out = self.target.rstrip('.mir') + '_B' + beam.split(
'/')[-1] + '.UVFITS'
fits.go()
if os.path.isfile(
self.target.rstrip('.mir') + '_B' +
beam.split('/')[-1] + '.UVFITS'):
self.logger.debug(
'# Conversion of MIRIAD file to UVFITS for beam ' +
beam.split('/')[-1] + ' successful! #')
subs.managefiles.director(
self, 'rm',
self.target.rstrip('.mir') + '_B' +
beam.split('/')[-1] + '.mir')
uvfitsstatusarray[b] = True
else:
self.logger.warning(
'# Conversion of MIRIAD file to UVFITS for beam ' +
beam.split('/')[-1] + ' NOT successful! #')
uvfitsstatusarray[b] = False
subs.param.add_param(self, 'transfer_input_beams_uvglue',
uvgluestatusarray)
subs.param.add_param(self, 'transfer_input_beams_uvfits',
uvfitsstatusarray)
def spec_ex(self):
'''
Extract spectrum at the coordinates of each source found by find_src_imsad
IN
Arrays stored in memory by load_src_csv
IN cfga
abs_ex_spec_format: .csv or .fits
OUT
Spectrum in .csv or .fits file format stored in abs/spec/ folder
New line in abs_table.txt. Each line has the following information:
Obs_ID, Beam, Source_ID, Ra, Dec, Peak Flux [Jy], r.m.s. spectrum
'''
if self.abs_ex_spec_ex == True:
self.logger.info(
'### Extract spectra from the position of the peak of each continuum source ###'
)
os.chdir(self.linedir)
print os.getcwd()
if os.path.exists(self.data_cube) == False:
fits = lib.miriad('fits')
fits.op = 'xyout'
fits.in_ = self.data_cube_mir
fits.out = self.data_cube
fits.go(rmfiles=True)
#open file
hdulist = pyfits.open(self.data_cube) # read input
# read data and header
scidata = hdulist[0].data
# reduce dimensions of the input cube if it his 4-dim
sci = np.squeeze(scidata)
prihdr = hdulist[0].header
if (self.abs_ex_cube_zunit == 'frequency'
or self.abs_ex_cube_zunit == 'freq'
or self.abs_ex_cube_zunit == 'fr'):
#convert frequencies in velocities
freq = (np.linspace(1, sci.shape[0], sci.shape[0]) -
prihdr['CRPIX3']) * prihdr['CDELT3'] + prihdr['CRVAL3']
v = C * ((HI - freq) / freq)
freq0 = prihdr['CRVAL3']
freq_del = prihdr['CDELT3']
elif (self.abs_ex_cube_zunit == 'velocity'
or self.abs_ex_cube_zunit == 'vel'
or self.abs_ex_cube_zunit == 'v'):
v = (np.linspace(1, sci.shape[0], sci.shape[0]) -
prihdr['CRPIX3']) * prihdr['CDELT3'] + prihdr['CRVAL3']
v /= 1e3
freq = (C * HI) / (v + C)
freq0 = (C * HI) / (prihdr['CRVAL3'] / 1e3 + C)
freq_del = (freq0 - freq[-1]) / len(freq)
#v_wrt_sys.append( C * ((HI - v_sys[j]) )
#find pixels where to extract the spectra
self.coord_to_pix()
# Load the list of coordinates in pixels
self.abs_mean_rms = np.zeros(self.pixels.shape[0])
self.abs_los_rms = np.zeros(self.pixels.shape[0])
self.tau_los_rms = np.zeros(self.pixels.shape[0])
for i in xrange(0, self.pixels.shape[0]):
# extract spectrum from each line of sight
flux = np.zeros(freq.shape[0])
madfm = np.zeros(freq.shape[0])
pix_x_or = int(self.pixels[i][0])
pix_y_or = int(self.pixels[i][1])
if (0 < pix_x_or < prihdr['NAXIS1']
and 0 < pix_y_or < prihdr['NAXIS2']):
for j in xrange(0, prihdr['NAXIS3']):
#correct for chromatic aberration
if self.abs_ex_chrom_aber == True:
#depending if the cube is in velocity or frequency ?
if freq_del <= 0.:
scale = (freq0 + j * freq_del) / freq0
elif freq_del >= 0:
scale = (freq0 - j * freq_del) / freq0
pix_x = (pix_x_or - prihdr['CRPIX1']
) * scale + prihdr['CRPIX1']
pix_y = (pix_y_or - prihdr['CRPIX2']
) * scale + prihdr['CRPIX2']
pix_x = int(round(pix_x, 0))
pix_y = int(round(pix_y, 0))
else:
pix_x = pix_x_or
pix_y = pix_y_or
if (0 < pix_x < prihdr['NAXIS1']
and 0 < pix_y < prihdr['NAXIS2']):
flux[j] = sci[j, pix_y, pix_x]
else:
flux[j] = 0.0
# determine the noise of the spectrum [Whiting 2012 et al.] in each channel
# MADMF: median absolute deviation from the median
# extract a region were to determine the noise: A BOX around the l.o.s.
if (pix_x + 10 < prihdr['NAXIS1']
and pix_y + 5 < prihdr['NAXIS2']
and pix_y - 5 > 0):
rms = np.nanmedian(sci[j, pix_x + 5:pix_x + 10,
pix_y - 5:pix_y + 5])
med2 = abs(sci[j, pix_x, pix_y] - rms)
madfm[j] = np.nanmedian(med2) / 0.6744888
else:
madfm[j] = 0.0
self.abs_mean_rms[i] = np.nanmean(madfm)
# measure noise in the spectrum outside of the line
end_spec = float(sci.shape[0])
end_spec_th = int(end_spec / 3.)
end_spec = int(end_spec)
#print self.src_id[i], self.J2000_name[i], pix_x_or, pix_y_or, pix_x, pix_y
tau = self.optical_depth(flux, self.peak_flux[i])
tau_noise = self.optical_depth(madfm, self.peak_flux[i])
#convert flux in optical depth
self.abs_los_rms[i] = (
np.std(flux[0:end_spec_th]) +
np.std(flux[end_spec - end_spec_th:end_spec])) / 2.
self.tau_los_rms[i] = self.optical_depth(
self.abs_los_rms[i], self.peak_flux[i])
self.logger.info('# Extracted spectrum of source ' +
self.src_id[i] + ' ' +
self.J2000_name[i] + ' #')
#write spectrum
tot = np.column_stack(
(freq, v, flux, madfm, tau, tau_noise))
out_spec = str(self.specdir + self.src_id[i] + '_' +
self.J2000_name[i])
#save spectrum
if self.abs_ex_spec_format == 'csv':
self.write_spec_csv(tot, out_spec)
elif self.abs_ex_spec_format == 'fits':
self.write_spec_fitstab(tot, out_spec)
else:
self.write_spec_csv(tot, out_spec)
self.write_spec_fitstab(tot, out_spec)
else:
self.logger.warning(
'# Source #' + self.src_id[i] +
' lies outside the fov of the data cube #')
pass
self.logger.info('### End of spectrum extraction ###')
return 0
def flagline(self):
'''
Creates an image cube of the different chunks and measures the rms in each channel. All channels with an rms outside of a given sigma interval are flagged in the continuum calibration, but are still used for line imaging.
'''
if self.selfcal_flagline:
subs.setinit.setinitdirs(self)
subs.setinit.setdatasetnamestomiriad(self)
self.logger.info(
'### Automatic flagging of HI-line/RFI started ###')
subs.managefiles.director(self, 'ch', self.selfcaldir)
for chunk in self.list_chunks():
subs.managefiles.director(self, 'ch',
self.selfcaldir + '/' + str(chunk))
self.logger.info('# Looking through data chunk ' + str(chunk) +
' #')
invert = lib.miriad('invert')
invert.vis = chunk + '.mir'
invert.map = 'map'
invert.beam = 'beam'
invert.imsize = self.selfcal_image_imsize
invert.cell = self.selfcal_image_cellsize
invert.stokes = 'ii'
invert.slop = 1
invert.go()
if os.path.exists('map'):
fits = lib.miriad('fits')
fits.in_ = 'map'
fits.op = 'xyout'
fits.out = 'map.fits'
fits.go()
cube = pyfits.open('map.fits')
data = cube[0].data
std = np.nanstd(data, axis=(0, 2, 3))
median = np.median(std)
stdall = np.nanstd(std)
diff = std - median
detections = np.where(
np.abs(self.selfcal_flagline_sigma * diff) > stdall)[0]
if len(detections) > 0:
self.logger.info(
'# Found high noise in channel(s) ' +
str(detections).lstrip('[').rstrip(']') + ' #')
for d in detections:
uvflag = lib.miriad('uvflag')
uvflag.vis = chunk + '.mir'
uvflag.flagval = 'flag'
uvflag.line = "'" + 'channel,1,' + str(d + 1) + "'"
uvflag.go()
self.logger.info(
'# Flagged channel(s) ' +
str(detections).lstrip('[').rstrip(']') +
' in data chunk ' + str(chunk) + ' #')
else:
self.logger.info(
'# No high noise found in data chunk ' +
str(chunk) + ' #')
subs.managefiles.director(
self, 'rm',
self.selfcaldir + '/' + str(chunk) + '/' + 'map')
subs.managefiles.director(
self, 'rm',
self.selfcaldir + '/' + str(chunk) + '/' + 'map.fits')
subs.managefiles.director(
self, 'rm',
self.selfcaldir + '/' + str(chunk) + '/' + 'beam')
else:
self.logger.info('### No data in chunk ' + str(chunk) +
'! ###')
self.logger.info('### Automatic flagging of HI-line/RFI done ###')
def run_continuum_minoriteration(self, chunk, majc, minc, drmin,
theoretical_noise_threshold):
'''
Does a selfcal minor iteration for the standard mode
chunk: The frequency chunk to image and calibrate
maj: Current major iteration
min: Current minor iteration
drmin: maximum dynamic range for minor iteration
theoretical_noise_threshold: calculated theoretical noise threshold
'''
subs.setinit.setinitdirs(self)
subs.setinit.setdatasetnamestomiriad(self)
self.logger.info('# Minor self-calibration cycle ' + str(minc) +
' for frequency chunk ' + chunk + ' started #')
if minc == 0:
invert = lib.miriad('invert') # Create the dirty image
invert.vis = chunk + '.mir'
invert.map = str(majc).zfill(2) + '/map_' + str(minc).zfill(2)
invert.beam = str(majc).zfill(2) + '/beam_' + str(minc).zfill(2)
invert.imsize = self.selfcal_image_imsize
invert.cell = self.selfcal_image_cellsize
invert.stokes = 'ii'
invert.options = 'mfs,double'
invert.slop = 1
invert.robust = -2
invert.go()
imax = self.calc_imax(
str(majc).zfill(2) + '/map_' + str(minc).zfill(2))
noise_threshold = self.calc_noise_threshold(imax, minc, majc)
dynamic_range_threshold = self.calc_dynamic_range_threshold(
imax, drmin, self.selfcal_standard_minorcycle0_dr)
mask_threshold, mask_threshold_type = self.calc_mask_threshold(
theoretical_noise_threshold, noise_threshold,
dynamic_range_threshold)
self.logger.info('# Mask threshold for major/minor cycle ' +
str(majc) + '/' + str(minc) + ' set to ' +
str(mask_threshold) + ' Jy/beam #')
self.logger.info('# Mask threshold set by ' +
str(mask_threshold_type) + ' #')
if majc == 0:
maths = lib.miriad('maths')
maths.out = str(majc).zfill(2) + '/mask_' + str(minc).zfill(2)
maths.exp = '"<' + str(majc).zfill(2) + '/map_' + str(
minc).zfill(2) + '>"'
maths.mask = '"<' + str(majc).zfill(2) + '/map_' + str(
minc).zfill(2) + '>.gt.' + str(mask_threshold) + '"'
maths.go()
self.logger.info('# Mask with threshold ' +
str(mask_threshold) + ' Jy/beam created #')
else:
subs.managefiles.director(
self,
'cp',
str(majc).zfill(2) + '/mask_' + str(minc).zfill(2),
file=str(majc - 1).zfill(2) + '/mask_' +
str(self.selfcal_standard_minorcycle - 1).zfill(2))
self.logger.info(
'# Mask from last minor iteration of last major cycle copied #'
)
clean_cutoff = self.calc_clean_cutoff(mask_threshold)
self.logger.info('# Clean threshold at major/minor cycle ' +
str(majc) + '/' + str(minc) + ' was set to ' +
str(clean_cutoff) + ' Jy/beam #')
clean = lib.miriad(
'clean') # Clean the image down to the calculated threshold
clean.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2)
clean.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2)
clean.out = str(majc).zfill(2) + '/model_' + str(minc).zfill(2)
clean.cutoff = clean_cutoff
clean.niters = 1000000
clean.region = '"' + 'mask(' + str(majc).zfill(2) + '/mask_' + str(
minc).zfill(2) + ')' + '"'
clean.go()
self.logger.info('# Major/minor cycle ' + str(majc) + '/' +
str(minc) + ' cleaning done #')
restor = lib.miriad('restor')
restor.model = str(majc).zfill(2) + '/model_' + str(minc).zfill(2)
restor.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2)
restor.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2)
restor.out = str(majc).zfill(2) + '/image_' + str(minc).zfill(2)
restor.mode = 'clean'
restor.go() # Create the cleaned image
self.logger.info('# Cleaned image for major/minor cycle ' +
str(majc) + '/' + str(minc) + ' created #')
restor.mode = 'residual'
restor.out = str(majc).zfill(2) + '/residual_' + str(minc).zfill(2)
restor.go()
self.logger.info('# Residual image for major/minor cycle ' +
str(majc) + '/' + str(minc) + ' created #')
self.logger.info('# Peak of the residual image is ' + str(
self.calc_imax(
str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) +
' Jy/beam #')
self.logger.info('# RMS of the residual image is ' + str(
self.calc_irms(
str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) +
' Jy/beam #')
else:
imax = self.calc_imax(
str(majc).zfill(2) + '/map_' + str(0).zfill(2))
noise_threshold = self.calc_noise_threshold(imax, minc, majc)
dynamic_range_threshold = self.calc_dynamic_range_threshold(
imax, drmin, self.selfcal_standard_minorcycle0_dr)
mask_threshold, mask_threshold_type = self.calc_mask_threshold(
theoretical_noise_threshold, noise_threshold,
dynamic_range_threshold)
self.logger.info('# Mask threshold for major/minor cycle ' +
str(majc) + '/' + str(minc) + ' set to ' +
str(mask_threshold) + ' Jy/beam #')
self.logger.info('# Mask threshold set by ' +
str(mask_threshold_type) + ' #')
maths = lib.miriad('maths')
maths.out = str(majc).zfill(2) + '/mask_' + str(minc).zfill(2)
maths.exp = '"<' + str(majc).zfill(2) + '/image_' + str(
minc - 1).zfill(2) + '>"'
maths.mask = '"<' + str(majc).zfill(2) + '/image_' + str(
minc - 1).zfill(2) + '>.gt.' + str(mask_threshold) + '"'
maths.go()
self.logger.info('# Mask with threshold ' + str(mask_threshold) +
' Jy/beam created #')
clean_cutoff = self.calc_clean_cutoff(mask_threshold)
self.logger.info('# Clean threshold at major/minor cycle ' +
str(majc) + '/' + str(minc) + ' was set to ' +
str(clean_cutoff) + ' Jy/beam #')
clean = lib.miriad(
'clean') # Clean the image down to the calculated threshold
clean.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2)
clean.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2)
clean.model = str(majc).zfill(2) + '/model_' + str(minc -
1).zfill(2)
clean.out = str(majc).zfill(2) + '/model_' + str(minc).zfill(2)
clean.cutoff = clean_cutoff
clean.niters = 1000000
clean.region = '"' + 'mask(' + str(majc).zfill(2) + '/mask_' + str(
minc).zfill(2) + ')' + '"'
clean.go()
self.logger.info('# Major/minor cycle ' + str(majc) + '/' +
str(minc) + ' cleaning done #')
restor = lib.miriad('restor')
restor.model = str(majc).zfill(2) + '/model_' + str(minc).zfill(2)
restor.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2)
restor.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2)
restor.out = str(majc).zfill(2) + '/image_' + str(minc).zfill(2)
restor.mode = 'clean'
restor.go() # Create the cleaned image
self.logger.info('# Cleaned image for major/minor cycle ' +
str(majc) + '/' + str(minc) + ' created #')
restor.mode = 'residual'
restor.out = str(majc).zfill(2) + '/residual_' + str(minc).zfill(2)
restor.go()
self.logger.info('# Residual image for major/minor cycle ' +
str(majc) + '/' + str(minc) + ' created #')
self.logger.info('# Peak of the residual image is ' + str(
self.calc_imax(
str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) +
' Jy/beam #')
self.logger.info('# RMS of the residual image is ' + str(
self.calc_irms(
str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) +
' Jy/beam #')
self.logger.info('# Minor self-calibration cycle ' + str(minc) +
' for frequency chunk ' + chunk + ' finished #')
def ms2miriad(self):
'''
Converts the data from MS to MIRIAD format via UVFITS using drivecasa. Does it for the flux calibrator, polarisation calibrator, and target field independently.
'''
subs.setinit.setinitdirs(self)
beams = 37
# Create the parameters for the parameter file for converting from MS to UVFITS format
convertfluxcalmsavailable = get_param_def(self, 'convert_fluxcal_MSavailable', False ) # Flux calibrator MS dataset available?
convertpolcalmsavailable = get_param_def(self, 'convert_polcal_MSavailable', False ) # Polarised calibrator MS dataset available?
converttargetbeamsmsavailable = get_param_def(self, 'convert_targetbeams_MSavailable', np.full((beams), False) ) # Target beam MS dataset available?
convertfluxcalms2uvfits = get_param_def(self, 'convert_fluxcal_MS2UVFITS', False ) # Flux calibrator MS dataset converted to UVFITS?
convertpolcalms2uvfits = get_param_def(self, 'convert_polcal_MS2UVFITS', False ) # Polarised calibrator MS dataset converted to UVFITS?
converttargetbeamsms2uvfits = get_param_def(self, 'convert_targetbeams_MS2UVFITS', np.full((beams), False) ) # Target beam MS dataset converted to UVFITS?
convertfluxcaluvfitsavailable = get_param_def(self, 'convert_fluxcal_UVFITSavailable', False ) # Flux calibrator UVFITS dataset available?
convertpolcaluvfitsavailable = get_param_def(self, 'convert_polcal_UVFITSavailable', False ) # Polarised calibrator UVFITS dataset available?
converttargetbeamsuvfitsavailable = get_param_def(self, 'convert_targetbeams_UVFITSavailable', np.full((beams), False) ) # Target beam UVFITS dataset available?
convertfluxcaluvfits2miriad = get_param_def(self, 'convert_fluxcal_UVFITS2MIRIAD', False ) # Flux calibrator UVFITS dataset converted to MIRIAD?
convertpolcaluvfits2miriad = get_param_def(self, 'convert_polcal_UVFITS2MIRIAD', False ) # Polarised calibrator UVFITS dataset converted to MIRIAD?
converttargetbeamsuvfits2miriad = get_param_def(self, 'convert_targetbeams_UVFITS2MIRIAD', np.full((beams), False) ) # Target beam UVFITS dataset converted to MIRIAD?
###################################################
# Check which datasets are available in MS format #
###################################################
if self.fluxcal != '':
convertfluxcalmsavailable = os.path.isdir(self.basedir + '00' + '/' + self.rawsubdir + '/' + self.fluxcal)
else:
self.logger.warning('# Flux calibrator dataset not specified. Cannot convert flux calibrator! #')
if self.polcal != '':
convertpolcalmsavailable = os.path.isdir(self.basedir + '00' + '/' + self.rawsubdir + '/' + self.polcal)
else:
self.logger.warning('# Polarised calibrator dataset not specified. Cannot convert polarised calibrator! #')
if self.target != '':
for b in range(beams):
converttargetbeamsmsavailable[b] = os.path.isdir(self.basedir + str(b).zfill(2) + '/' + self.rawsubdir + '/' + self.target)
else:
self.logger.warning('# Target beam dataset not specified. Cannot convert target beams! #')
# Save the derived parameters for the availability to the parameter file
subs.param.add_param(self, 'convert_fluxcal_MSavailable', convertfluxcalmsavailable)
subs.param.add_param(self, 'convert_polcal_MSavailable', convertpolcalmsavailable)
subs.param.add_param(self, 'convert_targetbeams_MSavailable', converttargetbeamsmsavailable)
###############################################
# Convert the available MS-datasets to UVFITS #
###############################################
# Convert the flux calibrator
if self.convert_fluxcal:
if self.fluxcal != '':
if convertfluxcaluvfits2miriad == False:
if convertfluxcalmsavailable:
self.logger.debug('# Converting flux calibrator dataset from MS to UVFITS format. #')
subs.managefiles.director(self, 'mk', self.basedir + '00' + '/' + self.crosscalsubdir, verbose=False)
fc_convert = 'exportuvfits(vis="' + self.basedir + '00' + '/' + self.rawsubdir + '/' + self.fluxcal + '", fitsfile="' + self.basedir + '00' + '/' + self.crosscalsubdir + '/' + str(self.fluxcal).rstrip('MS') + 'UVFITS' + '", datacolumn="data", combinespw=True, padwithflags=True, multisource=True, writestation=True)'
casacmd = [fc_convert]
casa = drivecasa.Casapy()
casa.run_script(casacmd, raise_on_severe=False, timeout=3600)
if os.path.isfile( self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip('MS') + 'UVFITS'):
convertfluxcalms2uvfits = True
self.logger.info('# Converted flux calibrator dataset from MS to UVFITS format! #')
else:
convertfluxcalms2uvfits = False
self.logger.warning('# Could not convert flux calibrator dataset from MS to UVFITS format! #')
else:
self.logger.warning('# Flux calibrator dataset not available! #')
else:
self.logger.info('# Flux calibrator dataset was already converted from MS to UVFITS format #')
else:
self.logger.warning('# Flux calibrator dataset not specified. Cannot convert flux calibrator! #')
else:
self.logger.warning('# Not converting flux calibrator dataset! #')
# Convert the polarised calibrator
if self.convert_polcal:
if self.polcal != '':
if convertpolcaluvfits2miriad == False:
if convertpolcalmsavailable:
self.logger.debug('# Converting polarised calibrator dataset from MS to UVFITS format. #')
subs.managefiles.director(self, 'mk', self.basedir + '00' + '/' + self.crosscalsubdir, verbose=False)
pc_convert = 'exportuvfits(vis="' + self.basedir + '00' + '/' + self.rawsubdir + '/' + self.polcal + '", fitsfile="' + self.basedir + '00' + '/' + self.crosscalsubdir + '/' + str(self.polcal).rstrip('MS') + 'UVFITS' + '", datacolumn="data", combinespw=True, padwithflags=True, multisource=True, writestation=True)'
casacmd = [pc_convert]
casa = drivecasa.Casapy()
casa.run_script(casacmd, raise_on_severe=False, timeout=3600)
if os.path.isfile( self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip('MS') + 'UVFITS'):
convertpolcalms2uvfits = True
self.logger.info('# Converted polarised calibrator dataset from MS to UVFITS format! #')
else:
convertpolcalms2uvfits = False
self.logger.warning('# Could not convert polarised calibrator dataset from MS to UVFITS format! #')
else:
self.logger.warning('# Polarised calibrator dataset not available! #')
else:
self.logger.info('# Polarised calibrator dataset was already converted from MS to UVFITS format #')
else:
self.logger.warning('# Polarised calibrator dataset not specified. Cannot convert polarised calibrator! #')
else:
self.logger.warning('# Not converting polarised calibrator dataset! #')
# Convert the target beams
if self.convert_target:
if self.target != '':
self.logger.info('# Converting target beam datasets from MS to UVFITS format. #')
if self.convert_targetbeams == 'all':
datasets = glob.glob(self.basedir + '[0-9][0-9]' + '/' + self.rawsubdir + '/' + self.target)
self.logger.debug('# Converting all available target beam datasets #')
else:
beams = self.convert_targetbeams.split(",")
datasets = [self.basedir + str(b).zfill(2) + '/' + self.rawsubdir + '/' + self.target for b in beams]
self.logger.debug('# Converting all selected target beam datasets #')
for vis in datasets:
if converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])] == False:
if converttargetbeamsmsavailable[int(vis.split('/')[-3])]:
subs.managefiles.director(self, 'mk', self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir, verbose=False)
tg_convert = 'exportuvfits(vis="' + self.basedir + vis.split('/')[-3] + '/' + self.rawsubdir + '/' + self.target + '", fitsfile="' + self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS' + '", datacolumn="data", combinespw=True, padwithflags=True, multisource=True, writestation=True)'
casacmd = [tg_convert]
casa = drivecasa.Casapy()
casa.run_script(casacmd, raise_on_severe=False, timeout=7200)
if os.path.isfile( self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS'):
converttargetbeamsms2uvfits[int(vis.split('/')[-3])] = True
self.logger.debug('# Converted dataset of target beam ' + vis.split('/')[-3] + ' from MS to UVFITS format! #')
else:
converttargetbeamsms2uvfits[int(vis.split('/')[-3])] = False
self.logger.warning('# Could not convert dataset for target beam ' + vis.split('/')[-3] + ' from MS to UVFITS format! #')
else:
self.logger.warning('# Dataset for target beam ' + vis.split('/')[-3] + ' not available! #')
else:
self.logger.info('# Dataset for target beam ' + vis.split('/')[-3] + ' was already converted from MS to UVFITS format #')
else:
self.logger.warning('# Target beam dataset(s) not specified. Cannot convert target beam datasets! #')
else:
self.logger.warning('# Not converting target beam dataset(s)! #')
# Save the derived parameters for the MS to UVFITS conversion to the parameter file
subs.param.add_param(self, 'convert_fluxcal_MS2UVFITS', convertfluxcalms2uvfits)
subs.param.add_param(self, 'convert_polcal_MS2UVFITS', convertpolcalms2uvfits)
subs.param.add_param(self, 'convert_targetbeams_MS2UVFITS', converttargetbeamsms2uvfits)
#######################################################
# Check which datasets are available in UVFITS format #
#######################################################
if self.fluxcal != '':
convertfluxcaluvfitsavailable = os.path.isfile(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip('MS') + 'UVFITS')
else:
self.logger.warning('# Flux calibrator dataset not specified. Cannot convert flux calibrator! #')
if self.polcal != '':
convertpolcaluvfitsavailable = os.path.isfile(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip('MS') + 'UVFITS')
else:
self.logger.warning('# Polarised calibrator dataset not specified. Cannot convert polarised calibrator! #')
if self.target != '':
for b in range(beams):
converttargetbeamsuvfitsavailable[b] = os.path.isfile(self.basedir + str(b).zfill(2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS')
else:
self.logger.warning('# Target beam dataset not specified. Cannot convert target beams! #')
# Save the derived parameters for the availability to the parameter file
subs.param.add_param(self, 'convert_fluxcal_UVFITSavailable', convertfluxcaluvfitsavailable)
subs.param.add_param(self, 'convert_polcal_UVFITSavailable', convertpolcaluvfitsavailable)
subs.param.add_param(self, 'convert_targetbeams_UVFITSavailable', converttargetbeamsuvfitsavailable)
##########################################################
# Convert the available UVFITS-datasets to MIRIAD format #
##########################################################
# Convert the flux calibrator
if self.convert_fluxcal:
if self.fluxcal != '':
if convertfluxcaluvfits2miriad == False:
if convertfluxcaluvfitsavailable:
self.logger.debug('# Converting flux calibrator dataset from UVFITS to MIRIAD format. #')
subs.managefiles.director(self, 'ch', self.basedir + '00' + '/' + self.crosscalsubdir, verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip('MS') + 'UVFITS'
fits.out = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip('MS') + 'mir'
fits.go()
if os.path.isdir( self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip('MS') + 'mir'):
convertfluxcaluvfits2miriad = True
self.logger.info('# Converted flux calibrator dataset from UVFITS to MIRIAD format! #')
else:
convertfluxcaluvfits2miriad = False
self.logger.warning('# Could not convert flux calibrator dataset from UVFITS to MIRIAD format! #')
else:
self.logger.warning('# Flux calibrator dataset not available! #')
else:
self.logger.info('# Flux calibrator dataset was already converted from UVFITS to MIRIAD format #')
else:
self.logger.warning('# Flux calibrator dataset not specified. Cannot convert flux calibrator! #')
else:
self.logger.warning('# Not converting flux calibrator dataset! #')
# Convert the polarised calibrator
if self.convert_polcal:
if self.polcal != '':
if convertpolcaluvfits2miriad == False:
if convertpolcaluvfitsavailable:
self.logger.debug('# Converting polarised calibrator dataset from UVFITS to MIRIAD format. #')
subs.managefiles.director(self, 'ch',self.basedir + '00' + '/' + self.crosscalsubdir, verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip('MS') + 'UVFITS'
fits.out = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip('MS') + 'mir'
fits.go()
if os.path.isdir(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip('MS') + 'mir'):
convertpolcaluvfits2miriad = True
self.logger.info('# Converted polarised calibrator dataset from UVFITS to MIRIAD format! #')
else:
convertpolcaluvfits2miriad = False
self.logger.warning('# Could not convert polarised calibrator dataset from UVFITS to MIRIAD format! #')
else:
self.logger.warning('# Polarised calibrator dataset not available! #')
else:
self.logger.info('# Polarised calibrator dataset was already converted from UVFITS to MIRIAD format #')
else:
self.logger.warning('# Polarised calibrator dataset not specified. Cannot convert polarised calibrator! #')
else:
self.logger.warning('# Not converting polarised calibrator dataset! #')
# Convert the target beams
if self.convert_target:
if self.target != '':
self.logger.info('# Converting target beam datasets from UVFITS to MIRIAD format. #')
if self.convert_targetbeams == 'all':
datasets = glob.glob(self.basedir + '[0-9][0-9]' + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS')
self.logger.debug('# Converting all available target beam datasets #')
else:
beams = self.convert_targetbeams.split(",")
datasets = [self.basedir + str(b).zfill(2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS' for b in beams]
self.logger.debug('# Converting all selected target beam datasets #')
for vis in datasets:
if converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])] == False:
if converttargetbeamsuvfitsavailable[int(vis.split('/')[-3])]:
subs.managefiles.director(self, 'ch', self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir, verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS'
fits.out = self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'mir'
fits.go()
if os.path.isdir( self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'mir'):
converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])] = True
self.logger.debug('# Converted dataset of target beam ' + vis.split('/')[-3] + ' from UVFITS to MIRIAD format! #')
else:
converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])] = False
self.logger.warning('# Could not convert dataset for target beam ' + vis.split('/')[-3] + ' from UVFITS to MIRIAD format! #')
else:
self.logger.warning('# Dataset for target beam ' + vis.split('/')[-3] + ' not available! #')
else:
self.logger.info('# Dataset for target beam ' + vis.split('/')[-3] + ' was already converted from MS to UVFITS format #')
else:
self.logger.warning('# Target beam dataset(s) not specified. Cannot convert target beam datasets! #')
else:
self.logger.warning('# Not converting target beam dataset(s)! #')
# Save the derived parameters for the MS to UVFITS conversion to the parameter file
subs.param.add_param(self, 'convert_fluxcal_UVFITS2MIRIAD', convertfluxcaluvfits2miriad)
subs.param.add_param(self, 'convert_polcal_UVFITS2MIRIAD', convertpolcaluvfits2miriad)
subs.param.add_param(self, 'convert_targetbeams_UVFITS2MIRIAD', converttargetbeamsuvfits2miriad)
#####################################
# Remove the UVFITS files if wanted #
#####################################
if self.convert_removeuvfits:
self.logger.info('# Removing all UVFITS files #')
subs.managefiles.director(self, 'rm', self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip('MS') + 'UVFITS')
subs.managefiles.director(self, 'rm', self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip('MS') + 'UVFITS')
for beam in range(beams):
if os.path.isdir(self.basedir + str(beam).zfill(2) + '/' + self.crosscalsubdir):
subs.managefiles.director(self, 'rm', self.basedir + str(beam).zfill(2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS')
else:
pass
def polarisation(self):
'''
Derives the polarisation corrections (leakage, angle) from the polarised calibrator. Uses the bandpass from the bandpass calibrator. Does not account for freqeuncy dependent solutions at the moment.
'''
if self.crosscal_polarisation:
subs.setinit.setinitdirs(self)
subs.setinit.setdatasetnamestomiriad(self)
subs.managefiles.director(self, 'ch', self.crosscaldir)
self.logger.info(
'### Polarisation calibration on the polarised calibrator data started ###'
)
if os.path.isfile(self.crosscaldir + '/' + self.fluxcal + '/' +
'bandpass'):
self.logger.info(
'# Bandpass solutions in flux calibrator data found. Using them! #'
)
gpcopy = lib.miriad('gpcopy')
gpcopy.vis = self.fluxcal
gpcopy.out = self.polcal
gpcopy.mode = 'copy'
gpcopy.options = 'nopol,relax'
gpcopy.go()
self.logger.info(
'# Bandpass from flux calibrator data copied to polarised calibrator data #'
)
gpcal = lib.miriad('gpcal')
gpcal.vis = self.polcal
# uv = aipy.miriad.UV(self.polcal)
# nchan = uv['nchan']
# gpcal.nfbin = round(nchan / self.crosscal_polarisation_nchan)
gpcal.options = 'xyvary,linear'
gpcal.go()
self.logger.info(
'# Solved for polarisation leakage and angle on polarised calibrator #'
)
else:
self.logger.info(
'# Bandpass solutions from flux calibrator not found #')
self.logger.info(
'# Deriving bandpass from polarised calibrator using mfcal #'
)
mfcal = lib.miriad('mfcal')
mfcal.vis = self.polcal
mfcal.go()
self.logger.info(
'# Bandpass solutions from polarised calibrator derived #')
self.logger.info(
'# Continuing with polarisation calibration (leakage, angle) from polarised calibrator data #'
)
gpcal = lib.miriad('gpcal')
gpcal.vis = self.polcal
# uv = aipy.miriad.UV(self.polcal)
# nchan = uv['nchan']
# gpcal.nfbin = round(nchan / self.crosscal_polarisation_nchan)
gpcal.options = 'xyvary,linear'
gpcal.go()
self.logger.info(
'# Solved for polarisation leakage and angle on polarised calibrator #'
)
self.logger.info(
'### Polarisation calibration on the polarised calibrator data done ###'
)
else:
self.logger.info('### No polarisation calibration done! ###')
#Will create almost empty configuration files for both
cfg1 = '/home/adams/apertif/AAF/180216005.cfg'
cfg2 = '/home/adams/apertif/AAF/180302009.cfg'
#Now in miriad need to use uvimage to create image cube
#Want to limit number of channels
#Want to include near 9330, 20 subbands
#Subbands start at multiples of 64
#take subbands 130-149
#channels 8320, 9599
#move to directory where data is
ccal = apercal.ccal(cfg1)
ccal.director('ch', ccal.crosscaldir)
uvimage = lib.miriad('uvimage')
uvimage.vis = ccal.target + 'mir' #uvaver.out
uvimage.view = 'amplitude'
targetname = ccal.target
uvimage.out = targetname + '_amp_freq.im'
#still too much data so also select a time range
#parentheses in command so need double quote
uvimage.select = "'-auto,time(11:00:00,12:00:00)'"
uvimage.line = 'channel,1280,8320,1'
uvimage.options = 'freq'
uvimage.mode = 3
uvimage.go()
#write to fits file
fits = lib.miriad('fits')
fits.in_ = uvimage.out
def splitdata(self):
'''
Applies calibrator corrections to data, splits the data into chunks in frequency and bins it to the given frequency resolution for the self-calibration
'''
if self.splitdata:
subs.setinit.setinitdirs(self)
subs.setinit.setdatasetnamestomiriad(self)
subs.managefiles.director(self, 'ch', self.selfcaldir)
self.logger.info(
'### Splitting of target data into individual frequency chunks started ###'
)
if os.path.isfile(self.selfcaldir + '/' + self.target):
self.logger.info(
'# Calibrator corrections already seem to have been applied #'
)
else:
self.logger.info(
'# Applying calibrator solutions to target data before averaging #'
)
uvaver = lib.miriad('uvaver')
uvaver.vis = self.crosscaldir + '/' + self.target
uvaver.out = self.selfcaldir + '/' + self.target
uvaver.go()
self.logger.info(
'# Calibrator solutions to target data applied #')
if self.selfcal_flagantenna != '':
uvflag = lib.miriad('uvflag')
uvflag.vis = self.selfcaldir + '/' + self.target
uvflag.flagval = 'flag'
uvflag.select = 'antenna(' + str(
self.selfcal_flagantenna) + ')'
uvflag.go()
else:
pass
try:
uv = aipy.miriad.UV(self.selfcaldir + '/' + self.target)
except RuntimeError:
self.logger.error(
'### No data in your crosscal directory! Exiting pipeline! ###'
)
sys.exit(1)
try:
nsubband = len(uv['nschan']) # Number of subbands in data
except TypeError:
nsubband = 1 # Only one subband in data since exception was triggered
self.logger.info('# Found ' + str(nsubband) +
' subband(s) in target data #')
counter = 0 # Counter for naming the chunks and directories
for subband in range(nsubband):
self.logger.info('# Started splitting of subband ' +
str(subband) + ' #')
if nsubband == 1:
numchan = uv['nschan']
finc = np.fabs(uv['sdf'])
else:
numchan = uv['nschan'][
subband] # Number of channels per subband
finc = np.fabs(
uv['sdf']
[subband]) # Frequency increment for each channel
subband_bw = numchan * finc # Bandwidth of one subband
subband_chunks = round(subband_bw /
self.selfcal_splitdata_chunkbandwidth)
subband_chunks = int(
np.power(2, np.ceil(np.log(subband_chunks) / np.log(2)))
) # Round to the closest power of 2 for frequency chunks with the same bandwidth over the frequency range of a subband
if subband_chunks == 0:
subband_chunks = 1
chunkbandwidth = (numchan / subband_chunks) * finc
self.logger.info(
'# Adjusting chunk size to ' + str(chunkbandwidth) +
' GHz for regular gridding of the data chunks over frequency #'
)
for chunk in range(subband_chunks):
self.logger.info('# Starting splitting of data chunk ' +
str(chunk) + ' for subband ' +
str(subband) + ' #')
binchan = round(
self.selfcal_splitdata_channelbandwidth /
finc) # Number of channels per frequency bin
chan_per_chunk = numchan / subband_chunks
if chan_per_chunk % binchan == 0: # Check if the freqeuncy bin exactly fits
self.logger.info(
'# Using frequency binning of ' +
str(self.selfcal_splitdata_channelbandwidth) +
' for all subbands #')
else:
while chan_per_chunk % binchan != 0: # Increase the frequency bin to keep a regular grid for the chunks
binchan = binchan + 1
else:
if chan_per_chunk >= binchan: # Check if the calculated bin is not larger than the subband channel number
pass
else:
binchan = chan_per_chunk # Set the frequency bin to the number of channels in the chunk of the subband
self.logger.info(
'# Increasing frequency bin of data chunk ' +
str(chunk) +
' to keep bandwidth of chunks equal over the whole bandwidth #'
)
self.logger.info('# New frequency bin is ' +
str(binchan * finc) + ' GHz #')
nchan = int(
chan_per_chunk /
binchan) # Total number of output channels per chunk
start = 1 + chunk * chan_per_chunk
width = int(binchan)
step = int(width)
subs.managefiles.director(
self, 'mk',
self.selfcaldir + '/' + str(counter).zfill(2))
uvaver = lib.miriad('uvaver')
uvaver.vis = self.selfcaldir + '/' + self.target
uvaver.out = self.selfcaldir + '/' + str(counter).zfill(
2) + '/' + str(counter).zfill(2) + '.mir'
uvaver.select = "'" + 'window(' + str(subband +
1) + ')' + "'"
uvaver.line = "'" + 'channel,' + str(nchan) + ',' + str(
start) + ',' + str(width) + ',' + str(step) + "'"
uvaver.go()
counter = counter + 1
self.logger.info('# Splitting of data chunk ' +
str(chunk) + ' for subband ' +
str(subband) + ' done #')
self.logger.info('# Splitting of data for subband ' +
str(subband) + ' done #')
self.logger.info(
'### Splitting of target data into individual frequency chunks done ###'
)
def parametric(self):
'''
Parametric self calibration using an NVSS/FIRST skymodel and calculating spectral indices by source matching with WENSS.
'''
if self.selfcal_parametric:
subs.setinit.setinitdirs(self)
subs.setinit.setdatasetnamestomiriad(self)
self.logger.info('### Doing parametric self calibration ###')
subs.managefiles.director(self, 'ch', self.selfcaldir)
for chunk in self.list_chunks():
self.logger.info(
'# Starting parametric self calibration routine on chunk '
+ chunk + ' #')
subs.managefiles.director(self, 'ch',
self.selfcaldir + '/' + chunk)
subs.managefiles.director(
self, 'mk', self.selfcaldir + '/' + chunk + '/' + 'pm')
parametric_textfile = lsm.lsm_model(
chunk + '.mir', self.selfcal_parametric_skymodel_radius,
self.selfcal_parametric_skymodel_cutoff,
self.selfcal_parametric_skymodel_distance)
lsm.write_model(
self.selfcaldir + '/' + chunk + '/' + 'pm' + '/model.txt',
parametric_textfile)
self.logger.info(
'# Creating model from textfile model.txt for chunk ' +
chunk + ' #')
uv = aipy.miriad.UV(self.selfcaldir + '/' + chunk + '/' +
chunk + '.mir')
freq = uv['sfreq']
uvmodel = lib.miriad('uvmodel')
uvmodel.vis = chunk + '.mir'
parametric_modelfile = open(
self.selfcaldir + '/' + str(chunk) + '/' + 'pm' +
'/model.txt', 'r')
for n, source in enumerate(parametric_modelfile.readlines()):
if n == 0:
uvmodel.options = 'replace,mfs'
else:
uvmodel.options = 'add,mfs'
uvmodel.offset = source.split(',')[0] + ',' + source.split(
',')[1]
uvmodel.flux = source.split(',')[2] + ',i,' + str(
freq) + ',' + source.split(',')[4].rstrip(
'\n') + ',0,0'
uvmodel.out = 'pm/tmp' + str(n)
uvmodel.go()
uvmodel.vis = uvmodel.out
subs.managefiles.director(
self, 'rn', 'pm/model',
uvmodel.out) # Rename the last modelfile to model
subs.managefiles.director(
self, 'rm',
'pm/tmp*') # Remove all the obsolete modelfiles
self.logger.info(
'# Doing parametric self-calibration on chunk ' + chunk +
' with solution interval ' +
str(self.selfcal_parametric_solint) +
' min and uvrange limits of ' +
str(self.selfcal_parametric_uvmin) + '~' +
str(self.selfcal_parametric_uvmax) + ' klambda #')
selfcal = lib.miriad('selfcal')
selfcal.vis = chunk + '.mir'
selfcal.model = 'pm/model'
selfcal.interval = self.selfcal_parametric_solint
selfcal.select = "'" + 'uvrange(' + str(
self.selfcal_parametric_uvmin) + ',' + str(
self.selfcal_parametric_uvmax) + ')' + "'"
# Choose reference antenna if given
if self.selfcal_refant == '':
pass
else:
selfcal.refant = self.selfcal_refant
# Do amplitude calibration if wanted
if self.selfcal_parametric_amp:
selfcal.options = 'mfs,amp'
else:
selfcal.options = 'mfs'
selfcal.go()
self.logger.info(
'# Parametric self calibration routine on chunk ' + chunk +
' done! #')
self.logger.info('### Parametric self calibration done ###')
else:
self.logger.info('### Parametric self calibration disabled ###')
