def subtract(active_ms, modelimg, wprojplanes=0):
"""General function to call the necessary steps to subtract point sources
the modelimg must have only point sources one wants to keep into the field.
second_ms_path: MS with calibrated data in DATA
modelimg: model of the whole sky (array of tt)
wprojplanes: number of w-projection planes, if 0 a direct ft() will be used (best for small field)
"""
if wprojplanes == 0:
ft(vis=active_ms, model=modelimg, nterms=len(modelimg), usescratch=True)
else:
ftw(vis=active_ms, model=modelimg, nterms=len(modelimg), wprojplanes=wprojplanes, usescratch=True)
uvsub(vis=active_ms)
def subtract(active_ms, modelimg, wprojplanes=0):
"""General function to call the necessary steps to subtract point sources
the modelimg must have only point sources one wants to keep into the field.
second_ms_path: MS with calibrated data in DATA
modelimg: model of the whole sky (array of tt)
wprojplanes: number of w-projection planes, if 0 a direct ft() will be used (best for small field)
"""
if wprojplanes == 0:
ft(vis=active_ms,
model=modelimg,
nterms=len(modelimg),
usescratch=True)
else:
ftw(vis=active_ms,
model=modelimg,
nterms=len(modelimg),
wprojplanes=wprojplanes,
usescratch=True)
uvsub(vis=active_ms)
def peel(residualMS, target_mset, modelimg, current_peel_directory, solint_data, prev_rms):
if not os.path.isdir(current_peel_directory+'/cal'):
os.makedirs(current_peel_directory+'/cal')
if not os.path.isdir(current_peel_directory+'/img'):
os.makedirs(current_peel_directory+'/img')
if not os.path.isdir(current_peel_directory+'/plot'):
os.makedirs(current_peel_directory+'/plot')
# Declare some needed parameters
solint = solint_data['solint']
number_of_integrations = solint_data['numint']
min_SNR_per_solution_interval = solint_data['minsnr']
fudge_factor = solint_data['ff']
integration_time = solint_data['inttime']
# Set the peel region
region = current_peel_directory+'/region.crtf'
# Obtain directions for later
orig_center = target_mset.get_direction_from_tgt_field_id(0)
ra = orig_center['m0']['value'] # NOTE I assume radians
dec = orig_center['m1']['value']
orig_center_string = 'J2000 '+str(ra)+'rad '+str(dec)+'rad'
ia.open(modelimg[0])
reg = rg.fromtextfile(filename=region, shape=ia.shape(), csys=ia.coordsys().torecord())
peel_source_center = reg['center']
ra_p = peel_source_center['*1']['value']
dec_p = peel_source_center['*2']['value']
peel_source_center_string = 'J2000 '+str(ra_p)+'rad '+str(dec_p)+'rad'
# Cutout everything except the peel source from the model image
peel_source_model = extrModel(modelimg, region, 0)
# Remove the on-the-fly model in the header (we only use the MODEL_DATA
# column).
delmod(vis=residualMS.file_path)
# Add this source to the residual image
ftw(vis=residualMS.file_path, model=peel_source_model,
nterms=len(peel_source_model), usescratch=True, wprojplanes=512)
uvsub(vis=residualMS.file_path, reverse = True)
# Split off the data set in order to make sure that the DATA column
# holds the residual map + recently added peel source
peelMSfilepath = current_peel_directory+'/img/'+residualMS.ms_name+'_and_peelsource.ms'
split(residualMS.file_path, peelMSfilepath)
peelMS = TableObjects.MSObj(peelMSfilepath)
# The cutout is smaller than the original image and placed outside the
# center. This means that a phase shift has to be applied.
# Shift the phase center to the peel source position
fixvis(vis=peelMS.file_path, outputvis=peelMS.file_path,
phasecenter = peel_source_center_string)
# Get some values for clean
shape, cell = findShape(peel_source_model[0])
# Determine the best reference antenna
refAntObj = AntennaObjects.RefAntHeuristics(vis=peelMS.file_path,
geometry=True, flagging=True)
refAnt = refAntObj.calculate()[0]
# In order for plotcal to work a symbolic link is needed.
# Plotcal assumes the measurement set is in the same directory
# as the cal table.
syscommand = 'ln -s '+peelMS.file_path+' '+current_peel_directory+\
'/cal/'+peelMS.ms_name+'.ms'
os.system(syscommand)
# Use the model from this clean session for the first selfcal cycle
delmod(vis=peelMS.file_path)
ft(vis=peelMS.file_path, model=peel_source_model,
nterms=len(peel_source_model), usescratch=True)
# Start the selfcalibration
numberofcycles = 10
usedcycles = -1
for i in xrange(numberofcycles):
# Create new directories
if not os.path.isdir(current_peel_directory+'/img/cycle'+str(i)):
os.makedirs(current_peel_directory+'/img/cycle'+str(i))
if not os.path.isdir(current_peel_directory+'/plot/cycle'+str(i)):
os.makedirs(current_peel_directory+'/plot/cycle'+str(i))
# Do selfcal
gaincal(vis=peelMS.file_path, caltable=current_peel_directory+\
'/cal/cycle'+str(i)+'.Gp', solint=solint, minsnr=0.01, selectdata = True,
uvrange='>50m', refant=refAnt, calmode='p')
Gp = TableObjects.STObj(current_peel_directory+'/cal/cycle'+str(i)+'.Gp')
Gp.plot(current_peel_directory+'/plot/cycle'+str(i), phase_only=True)
applycal(vis=peelMS.file_path, gaintable=Gp.file_path,
calwt=False, flagbackup=False)
# Image the result
parms = {'vis':peelMS.file_path, 'imagename':current_peel_directory+\
'/img/cycle'+str(i)+'/im',
'gridmode':'widefield', 'wprojplanes':512, 'mode':'mfs',
'nterms':2, 'niter':10000, 'gain':0.1, 'psfmode':'clark',
'imagermode':'csclean', 'imsize':[shape], 'cell':cell,
'weighting':'briggs', 'robust':rob, 'usescratch':True,
'mask':''}
utils.cleanmaskclean(parms)
# Get the flux and rms from this image, this is needed to compare different
# cycles. "< 1" is to invert the mask, so that the rms is caculated
# only for the background. The syntax is very weird, see:
# https://casa.nrao.edu/aips2_docs/notes/223/index.shtml
mask = '"'+current_peel_directory+'/img/cycle'+str(i)+'/im.newmask'+'"'+' < 1'
rms = imstat(imagename=current_peel_directory+'/img/cycle'+str(i)+'/im-masked.image.tt0',
mask=mask)['rms'][0]
fit = imfit(imagename=current_peel_directory+'/img/cycle'+str(i)+'/im-masked.image.tt0')
print 'I ASSUMED THAT COMPONENT0 IS OUR PEEL SOURCE (BRIGHTEST)'
print fit
tflux = fit['results']['component0']['flux']['value'][0] # Jy, assume component0 is the brightest
# Do a check and update solint
if rms < 1.1*prev_rms:# or i < 2:
print '---'
print '---'
noise_per_interval = rms * (number_of_integrations)**0.5
snr_per_interval = tflux/noise_per_interval
time_steps_needed = ((min_SNR_per_solution_interval/fudge_factor)/snr_per_interval)**2.0 # SNR grows with sqrt(time)
time_needed_in_sec = integration_time*time_steps_needed
solint = '%.1fs' % time_needed_in_sec
print solint
print '---'
print '---'
# Set the new model into the MODEL_DATA column (only select region)
new_model = [current_peel_directory+'/img/cycle'+str(i)+'/im-masked.model.tt0',
current_peel_directory+'/img/cycle'+str(i)+'/im-masked.model.tt1']
updated_peel_source_model = extrModel(new_model, region, i+1)
ft(vis=peelMS.file_path, model=updated_peel_source_model, nterms=len(updated_peel_source_model),
usescratch=True)
usedcycles = i
prev_rms = rms
else:
# No improvement and no need to proceed with further cycles
break
# Now, after enough self-cal cycles, we have a solution table and model for
# this peel source. Subtract this new model in order to get an updated
# residual measurement set.
if usedcycles == -1:
# No improvent at all
best_updated_model = peel_source_model
else:
best_updated_model = [current_peel_directory+'/img/cycle'+str(usedcycles)+'/im-masked.model.tt0',
current_peel_directory+'/img/cycle'+str(usedcycles)+'/im-masked.model.tt1']
delmod(vis=peelMS.file_path)
subtract(peelMS.file_path, best_updated_model)
# Split off the data set in order to make sure that the DATA column
# only holds the residual map
updatedResMSfilepath = current_peel_directory+'/updated_residual.ms'
split(peelMS.file_path, updatedResMSfilepath)
updatedResidualMS = TableObjects.MSObj(updatedResMSfilepath)
if not usedcycles == -1:
# Apply the inverse phase solutions for this particular peel source
allTimestamps = updatedResidualMS.get_all_available_timestamps()
Gp = TableObjects.STObj(current_peel_directory+'/cal/cycle'+str(usedcycles)+'.Gp')
print '-- refant --'
print refAnt
Gp_reref_path = Gp.re_reference_table(refant=1)
Gp_reref = TableObjects.STObj(Gp_reref_path)
Gp_normref_path = Gp_reref.normalize_reference_antenna()
Gp_normref = TableObjects.STObj(Gp_normref_path)
Gp_resamp_path = Gp_normref.resample_solutions(allTimestamps, interp_type = 'linear')
Gp_resamp = TableObjects.STObj(Gp_resamp_path)
Gp_resamp.plot(current_peel_directory+'/plot/cycle'+str(usedcycles)+'/resamp', phase_only=True)
Gp_inv_path = Gp_resamp.invert_table()
applycal(vis=updatedResidualMS.file_path, gaintable=Gp_inv_path, calwt=False,
flagbackup=False)
# Set the phase center again to the initial position
fixvis(vis=updatedResidualMS.file_path, outputvis=updatedResidualMS.file_path,
phasecenter = orig_center_string)
# CHECK IF EVERYTHING WORKS
parms = {'vis':updatedResidualMS.file_path, 'imagename':current_peel_directory+'/updated_res',
'gridmode':'widefield', 'wprojplanes':512, 'mode':'mfs',
'nterms':2, 'niter':10000, 'gain':0.1, 'psfmode':'clark',
'imagermode':'csclean', 'imsize':[5000], 'cell':cell,
'weighting':'briggs', 'robust':rob, 'usescratch':True,
'mask':''}
utils.cleanmaskclean(parms, makemask = False)
exportfits(imagename=current_peel_directory+'/updated_res.image.tt0',
fitsimage=current_peel_directory+'/updated_res.image.tt0.fits',
history=False, overwrite=True)
residualrms = imstat(imagename=current_peel_directory+'/updated_res.image.tt0')['rms'][0]
print '--'
print '--'
print '--'
print 'residual rms: ', residualrms
print '--'
print '--'
print '--'
return updatedResidualMS
def peel(residualMS, target_mset, modelimg, current_peel_directory,
solint_data, prev_rms):
if not os.path.isdir(current_peel_directory + '/cal'):
os.makedirs(current_peel_directory + '/cal')
if not os.path.isdir(current_peel_directory + '/img'):
os.makedirs(current_peel_directory + '/img')
if not os.path.isdir(current_peel_directory + '/plot'):
os.makedirs(current_peel_directory + '/plot')
# Declare some needed parameters
solint = solint_data['solint']
number_of_integrations = solint_data['numint']
min_SNR_per_solution_interval = solint_data['minsnr']
fudge_factor = solint_data['ff']
integration_time = solint_data['inttime']
# Set the peel region
region = current_peel_directory + '/region.crtf'
# Obtain directions for later
orig_center = target_mset.get_direction_from_tgt_field_id(0)
ra = orig_center['m0']['value'] # NOTE I assume radians
dec = orig_center['m1']['value']
orig_center_string = 'J2000 ' + str(ra) + 'rad ' + str(dec) + 'rad'
ia.open(modelimg[0])
reg = rg.fromtextfile(filename=region,
shape=ia.shape(),
csys=ia.coordsys().torecord())
peel_source_center = reg['center']
ra_p = peel_source_center['*1']['value']
dec_p = peel_source_center['*2']['value']
peel_source_center_string = 'J2000 ' + str(ra_p) + 'rad ' + str(
dec_p) + 'rad'
# Cutout everything except the peel source from the model image
peel_source_model = extrModel(modelimg, region, 0)
# Remove the on-the-fly model in the header (we only use the MODEL_DATA
# column).
delmod(vis=residualMS.file_path)
# Add this source to the residual image
ftw(vis=residualMS.file_path,
model=peel_source_model,
nterms=len(peel_source_model),
usescratch=True,
wprojplanes=512)
uvsub(vis=residualMS.file_path, reverse=True)
# Split off the data set in order to make sure that the DATA column
# holds the residual map + recently added peel source
peelMSfilepath = current_peel_directory + '/img/' + residualMS.ms_name + '_and_peelsource.ms'
split(residualMS.file_path, peelMSfilepath)
peelMS = TableObjects.MSObj(peelMSfilepath)
# The cutout is smaller than the original image and placed outside the
# center. This means that a phase shift has to be applied.
# Shift the phase center to the peel source position
fixvis(vis=peelMS.file_path,
outputvis=peelMS.file_path,
phasecenter=peel_source_center_string)
# Get some values for clean
shape, cell = findShape(peel_source_model[0])
# Determine the best reference antenna
refAntObj = AntennaObjects.RefAntHeuristics(vis=peelMS.file_path,
geometry=True,
flagging=True)
refAnt = refAntObj.calculate()[0]
# In order for plotcal to work a symbolic link is needed.
# Plotcal assumes the measurement set is in the same directory
# as the cal table.
syscommand = 'ln -s '+peelMS.file_path+' '+current_peel_directory+\
'/cal/'+peelMS.ms_name+'.ms'
os.system(syscommand)
# Use the model from this clean session for the first selfcal cycle
delmod(vis=peelMS.file_path)
ft(vis=peelMS.file_path,
model=peel_source_model,
nterms=len(peel_source_model),
usescratch=True)
# Start the selfcalibration
numberofcycles = 10
usedcycles = -1
for i in xrange(numberofcycles):
# Create new directories
if not os.path.isdir(current_peel_directory + '/img/cycle' + str(i)):
os.makedirs(current_peel_directory + '/img/cycle' + str(i))
if not os.path.isdir(current_peel_directory + '/plot/cycle' + str(i)):
os.makedirs(current_peel_directory + '/plot/cycle' + str(i))
# Do selfcal
gaincal(vis=peelMS.file_path, caltable=current_peel_directory+\
'/cal/cycle'+str(i)+'.Gp', solint=solint, minsnr=0.01, selectdata = True,
uvrange='>50m', refant=refAnt, calmode='p')
Gp = TableObjects.STObj(current_peel_directory + '/cal/cycle' +
str(i) + '.Gp')
Gp.plot(current_peel_directory + '/plot/cycle' + str(i),
phase_only=True)
applycal(vis=peelMS.file_path,
gaintable=Gp.file_path,
calwt=False,
flagbackup=False)
# Image the result
parms = {'vis':peelMS.file_path, 'imagename':current_peel_directory+\
'/img/cycle'+str(i)+'/im',
'gridmode':'widefield', 'wprojplanes':512, 'mode':'mfs',
'nterms':2, 'niter':10000, 'gain':0.1, 'psfmode':'clark',
'imagermode':'csclean', 'imsize':[shape], 'cell':cell,
'weighting':'briggs', 'robust':rob, 'usescratch':True,
'mask':''}
utils.cleanmaskclean(parms)
# Get the flux and rms from this image, this is needed to compare different
# cycles. "< 1" is to invert the mask, so that the rms is caculated
# only for the background. The syntax is very weird, see:
# https://casa.nrao.edu/aips2_docs/notes/223/index.shtml
mask = '"' + current_peel_directory + '/img/cycle' + str(
i) + '/im.newmask' + '"' + ' < 1'
rms = imstat(imagename=current_peel_directory + '/img/cycle' + str(i) +
'/im-masked.image.tt0',
mask=mask)['rms'][0]
fit = imfit(imagename=current_peel_directory + '/img/cycle' + str(i) +
'/im-masked.image.tt0')
print 'I ASSUMED THAT COMPONENT0 IS OUR PEEL SOURCE (BRIGHTEST)'
print fit
tflux = fit['results']['component0']['flux']['value'][
0] # Jy, assume component0 is the brightest
# Do a check and update solint
if rms < 1.1 * prev_rms: # or i < 2:
print '---'
print '---'
noise_per_interval = rms * (number_of_integrations)**0.5
snr_per_interval = tflux / noise_per_interval
time_steps_needed = (
(min_SNR_per_solution_interval / fudge_factor) /
snr_per_interval)**2.0 # SNR grows with sqrt(time)
time_needed_in_sec = integration_time * time_steps_needed
solint = '%.1fs' % time_needed_in_sec
print solint
print '---'
print '---'
# Set the new model into the MODEL_DATA column (only select region)
new_model = [
current_peel_directory + '/img/cycle' + str(i) +
'/im-masked.model.tt0', current_peel_directory + '/img/cycle' +
str(i) + '/im-masked.model.tt1'
]
updated_peel_source_model = extrModel(new_model, region, i + 1)
ft(vis=peelMS.file_path,
model=updated_peel_source_model,
nterms=len(updated_peel_source_model),
usescratch=True)
usedcycles = i
prev_rms = rms
else:
# No improvement and no need to proceed with further cycles
break
# Now, after enough self-cal cycles, we have a solution table and model for
# this peel source. Subtract this new model in order to get an updated
# residual measurement set.
if usedcycles == -1:
# No improvent at all
best_updated_model = peel_source_model
else:
best_updated_model = [
current_peel_directory + '/img/cycle' + str(usedcycles) +
'/im-masked.model.tt0', current_peel_directory + '/img/cycle' +
str(usedcycles) + '/im-masked.model.tt1'
]
delmod(vis=peelMS.file_path)
subtract(peelMS.file_path, best_updated_model)
# Split off the data set in order to make sure that the DATA column
# only holds the residual map
updatedResMSfilepath = current_peel_directory + '/updated_residual.ms'
split(peelMS.file_path, updatedResMSfilepath)
updatedResidualMS = TableObjects.MSObj(updatedResMSfilepath)
if not usedcycles == -1:
# Apply the inverse phase solutions for this particular peel source
allTimestamps = updatedResidualMS.get_all_available_timestamps()
Gp = TableObjects.STObj(current_peel_directory + '/cal/cycle' +
str(usedcycles) + '.Gp')
print '-- refant --'
print refAnt
Gp_reref_path = Gp.re_reference_table(refant=1)
Gp_reref = TableObjects.STObj(Gp_reref_path)
Gp_normref_path = Gp_reref.normalize_reference_antenna()
Gp_normref = TableObjects.STObj(Gp_normref_path)
Gp_resamp_path = Gp_normref.resample_solutions(allTimestamps,
interp_type='linear')
Gp_resamp = TableObjects.STObj(Gp_resamp_path)
Gp_resamp.plot(current_peel_directory + '/plot/cycle' +
str(usedcycles) + '/resamp',
phase_only=True)
Gp_inv_path = Gp_resamp.invert_table()
applycal(vis=updatedResidualMS.file_path,
gaintable=Gp_inv_path,
calwt=False,
flagbackup=False)
# Set the phase center again to the initial position
fixvis(vis=updatedResidualMS.file_path,
outputvis=updatedResidualMS.file_path,
phasecenter=orig_center_string)
# CHECK IF EVERYTHING WORKS
parms = {
'vis': updatedResidualMS.file_path,
'imagename': current_peel_directory + '/updated_res',
'gridmode': 'widefield',
'wprojplanes': 512,
'mode': 'mfs',
'nterms': 2,
'niter': 10000,
'gain': 0.1,
'psfmode': 'clark',
'imagermode': 'csclean',
'imsize': [5000],
'cell': cell,
'weighting': 'briggs',
'robust': rob,
'usescratch': True,
'mask': ''
}
utils.cleanmaskclean(parms, makemask=False)
exportfits(imagename=current_peel_directory + '/updated_res.image.tt0',
fitsimage=current_peel_directory +
'/updated_res.image.tt0.fits',
history=False,
overwrite=True)
residualrms = imstat(imagename=current_peel_directory +
'/updated_res.image.tt0')['rms'][0]
print '--'
print '--'
print '--'
print 'residual rms: ', residualrms
print '--'
print '--'
print '--'
return updatedResidualMS
def clipresidual(active_ms, f='', s=''):
"""
Create residuals in the CORRECTED_DATA (then unusable!)
and clip at 5 times the total flux of the model
NOTE: the ms CORRECTED_DATA will be corrupted!
"""
#default('uvsub')
uvsub(vis=active_ms)
# flag statistics before flagging
statsFlag(active_ms, note='Before BL flag')
logging.debug("Removing baselines with high residuals:")
import itertools
ms.open(active_ms, nomodify=False)
metadata = ms.metadata()
flag = {}
# datadesc ids are usually one per spw, but ms can also be splitted in corr
for datadescid in metadata.datadescids():
flag[datadescid] = {}
logging.debug("Working on datadesc: "+str(datadescid))
ms.msselect({'field':f, 'scan':s})
d = ms.getdata(['corrected_amplitude','flag','antenna1','antenna2','axis_info'], ifraxis=True)
# cycle on corr
for corr in xrange(len(d['corrected_amplitude'])):
flag[datadescid][corr] = {}
logging.debug("Working on corr: "+d['axis_info']['corr_axis'][corr])
# cycle on channels
for chan in xrange(len(d['corrected_amplitude'][corr])):
flag[datadescid][corr][chan] = {}
meds = []
# cycle on bl
for bl in xrange(len(d['corrected_amplitude'][corr][chan])):
amp = d['corrected_amplitude'][corr][chan][bl][~d['flag'][corr][chan][bl]] # get unflagged data
if amp != []: meds.append(np.median( amp[(amp == amp)] ))
if meds != []:
med = np.mean(meds)
rms = np.std(meds)
for bl in xrange(len(d['corrected_amplitude'][corr][chan])):
flag[datadescid][corr][chan][bl] = False
amp = d['corrected_amplitude'][corr][chan][bl][~d['flag'][corr][chan][bl]] # get unflagged data
if amp != []:
bl_med = np.median( amp[(amp == amp)] )
# if BL residuals are 3 times out of med rms, flag
if abs(bl_med - med) > 3*rms:
logging.debug("Flagging corr: "+d['axis_info']['corr_axis'][corr]+" - chan:"+str(chan)+" - BL: "+d['axis_info']['ifr_axis']['ifr_name'][bl])
flag[datadescid][corr][chan][bl] = True
ms.close()
# extend flags to all scans
ms.open(active_ms, nomodify=False)
metadata = ms.metadata()
# datadesc ids are usually one per spw, but ms can also be splitted in corr
for datadescid in metadata.datadescids():
ms.msselect()
w = ms.getdata(['flag'], ifraxis=True)
for corr in xrange(len(w['flag'])):
# TODO: extend flags on all chan for BLs which appear often
for chan in xrange(len(w['flag'][corr])):
for bl in xrange(len(w['flag'][corr][chan])):
#print '--'
#print len(d['corrected_amplitude'][corr][chan])
#print len(w['flag'][corr][chan])
#print len(flag[datadescid][corr][chan])
#print '--'
# So, sometimes len(w['flag'][corr][chan]) > len(flag[datadescid][corr][chan])
# CHECK IF THIS IS A GOOD SOLUTION
bl = min(bl, len(flag[datadescid][corr][chan])-1)
if flag[datadescid][corr][chan][bl] == True:
w['flag'][corr][chan][bl] = True
ms.putdata({'flag':w['flag']})
ms.close()
statsFlag(active_ms, note='After clipping')
