def flag_extreme_amplitudes(tablename, maxpctchange=50, pols=[0], channels=[0]): """ Flag out all gain amplitudes with >``maxpctchange``% change (e.g., for the default 50%, flag everything outside the range 0.5 < G < 1.5). This is a *very simple* cut, but it cannot be easily applied with existing tools since it is cutting on the value of the amplitude correction, not on any particular normal data selection type. It is still highly advisable to plot the amp vs snr or similar diagnostics after running this to make sure you understand what's going on. For example, after I ran this on a data set, I discovered that one antenna had high gain corrections even in the high SNR regime, which probably indicates a problem with that antenna. Parameters ---------- maxpctchange : float The maximum percent change permitted in an amplitude pols : list The list of polarizations to include in the heuristics channels : list The list of channels to include in the heuristics Returns ------- None """ tb.open(tablename) solns = tb.getcol('CPARAM') snr = tb.getcol('SNR') # true flag = flagged out, bad data flags = tb.getcol('FLAG') tb.close() amp = np.abs(solns) maxfrac = maxpctchange / 100. bad = ((amp[pols, channels] > (1+maxfrac)) | (amp[pols, channels] < (1-maxfrac))) bad_snr = snr[pols, channels, :][bad] print("Found {0} bad amplitudes with mean snr={1}".format(bad.sum(), bad_snr.mean())) print("Total flags in tb.flag: {0}".format(flags.sum())) flags[pols, channels, :] = bad | flags[pols, channels, :] assert all(flags[pols, channels, :][bad]), "Failed to modify array" tb.open(tablename, nomodify=False) tb.putcol(columnname='FLAG', value=flags) tb.flush() tb.close() tb.open(tablename, nomodify=True) flags = tb.getcol('FLAG') print("Total flags in tb.flag after: {0}".format(flags.sum())) assert all(flags[pols, channels, :][bad]), "Failed to modify table" tb.close()
def addPhasePerAntenna(self,phase,antennaId,dataType="DATA"): "Add a phase (degree) per antennaId. Careful with the order !" tb.open(self.visName,nomodify=False) ant1 = tb.getcol("ANTENNA1") ant2 = tb.getcol("ANTENNA2") data = tb.getcol(dataType) phase = DEGREE2RAD*phase print "Phase rotation: %f"%(phase) nDim = data.shape for i in range(nDim[0]): for j in range(nDim[1]): for k in range(nDim[2]): antenna1 = ant1[k] antenna2 = ant2[k] if antenna1 == antennaId: value = data[i][j][k] real = value.real image = value.imag cosRot = math.cos(phase) sinRot = math.sin(phase) xRot = cosRot*real-sinRot*image yRot = sinRot*real+sinRot*image data[i][j][k] = complex(xRot,yRot) # print "Amplitude:%f"%(math.sqrt(xRot*xRot+yRot*yRot)) if antenna2 == antennaId: value = data[i][j][k] real = value.real image = value.imag cosRot = math.cos(-phase) sinRot = math.sin(-phase) xRot = cosRot*real-sinRot*image yRot = sinRot*real+sinRot*image data[i][j][k] = complex(xRot,yRot) # print "Amplitude:%f"%(math.sqrt(xRot*xRot+yRot*yRot)) tb.putcol(dataType,data) tb.close()
def pixelmask2cleanmask(imagename='', maskname='mask0', maskimage='', usemasked=False): """ convert pixel(T/F) mask (in a CASA image) to a mask image (1/0) used for clean imagename - input imagename that contain a mask to be used maskname - mask name in the image (default: mask0) maskimage - output mask image name usemasked - if True use masked region as a valid region """ ia.open(imagename) masks = ia.maskhandler('get') ia.close() inmaskname = '' if type(masks) != list: masks = [masks] for msk in masks: if maskname == msk: inmaskname = msk break if inmaskname == '': raise Exception, "mask %s does not exist. Available masks are: %s" % ( maskname, masks) tb.open(imagename + '/' + maskname) dat0 = tb.getcol('PagedArray') tb.close() #os.system('cp -r %s %s' % (imagename, maskimage)) shutil.copytree(imagename, maskimage) ia.open(maskimage) # to unset mask ia.maskhandler('set', ['']) # make all valid if (usemasked): ia.set(1) else: ia.set(0) ia.close() # tb.open(maskimage, nomodify=False) imd = tb.getcol('map') # maybe shape check here #by default use True part of bool mask masked = 1 if (usemasked): masked = 0 imd[dat0] = masked tb.putcol('map', imd) tb.close()
def pixelmask2cleanmask(imagename='',maskname='mask0',maskimage='',usemasked=False): """ convert pixel(T/F) mask (in a CASA image) to a mask image (1/0) used for clean imagename - input imagename that contain a mask to be used maskname - mask name in the image (default: mask0) maskimage - output mask image name usemasked - if True use masked region as a valid region """ ia.open(imagename) masks=ia.maskhandler('get') ia.close() inmaskname='' if type(masks)!=list: masks=[masks] for msk in masks: if maskname == msk: inmaskname=msk break if inmaskname=='': raise Exception, "mask %s does not exist. Available masks are: %s" % (maskname,masks) tb.open(imagename+'/'+maskname) dat0=tb.getcol('PagedArray') tb.close() #os.system('cp -r %s %s' % (imagename, maskimage)) shutil.copytree(imagename,maskimage) ia.open(maskimage) # to unset mask ia.maskhandler('set',['']) # make all valid if (usemasked): ia.set(1) else: ia.set(0) ia.close() # tb.open(maskimage,nomodify=False) imd=tb.getcol('map') # maybe shape check here #by default use True part of bool mask masked=1 if (usemasked): masked=0 imd[dat0]=masked tb.putcol('map',imd) tb.close()
def bandpass_normalize(bandpass_table, bandpass_table_inv): shutil.copytree(bandpass_table, bandpass_table_inv) tb.open(bandpass_table_inv, nomodify=False) gain = tb.getcol('CPARAM') gain_norm = gain #for each antenna and for each channel of the bandpass I divide out by the modulo of the complex number for i in range(0, gain.shape[2]): for j in range(0, gain.shape[1]): a = gain[0, j, i] #if the real part of the antenna gain is set to 1 it means that that antenna a/o channel is flag, so don't bother looking at it if a.real != 1: gain_norm[0, j, i] = gain_norm[0, j, i] / abs(gain_norm[0, j, i]) gain_norm[1, j, i] = gain_norm[1, j, i] / abs(gain_norm[1, j, i]) # print abs(gain_norm[i,j,0]),abs(gain[i,j,0]),i,j # put back the normalized bandpass tb.putcol('CPARAM', gain_norm) tb.close() tb.done()
def fill_flagged_soln(caltable='', doplot=False): """ This is to replace the gaincal solution of flagged/failed solutions by the nearest valid one. If you do not do that and applycal blindly with the table your data gets flagged between calibration runs that have a bad/flagged solution at one edge. Can be pretty bad when you calibrate every hour or more (when you are betting on self-cal) of observation (e.g L-band of the EVLA)..one can lose the whole hour of good data without realizing ! """ tb.open(caltable, nomodify=False) flg=tb.getcol('FLAG') sol=tb.getcol('SOLUTION_OK') ant=tb.getcol('ANTENNA1') gain=tb.getcol('GAIN') t=tb.getcol('TIME') dd=tb.getcol('CAL_DESC_ID') maxant=np.max(ant) maxdd=np.max(dd) npol=len(gain[:,0,0]) nchan=len(gain[0,:,0]) k=1 if(doplot): pl.ion() pl.figure(1) pl.plot(t[(ant==k)], sol[0,0,(ant==k)], 'b+') pl.plot(t[(ant==k)], flg[0,0,(ant==k)], 'r+') pl.twinx() pl.plot(t[(ant==k)], abs(gain[0,0,(ant==k)]), 'go') print 'maxant', maxant numflag=0.0 for k in range(maxant+1): for j in range (maxdd+1): subflg=flg[:,:,(ant==k) & (dd==j)] subt=t[(ant==k) & (dd==j)] subsol=sol[:,:,(ant==k) & (dd==j)] subgain=gain[:,:,(ant==k) & (dd==j)] #print 'subgain', subgain.shape for kk in range(1, len(subt)): for chan in range(nchan): for pol in range(npol): if(subflg[pol,chan,kk] and not subflg[pol,chan,kk-1]): numflag += 1.0 subflg[pol,chan,kk]=False subsol[pol, chan, kk]=True subgain[pol,chan,kk]=subgain[pol,chan,kk-1] if(subflg[pol,chan,kk-1] and not subflg[pol,chan,kk]): numflag += 1.0 subflg[pol,chan,kk-1]=False subsol[pol, chan, kk-1]=True subgain[pol,chan,kk-1]=subgain[pol,chan,kk] flg[:,:,(ant==k) & (dd==j)]=subflg sol[:,:,(ant==k) & (dd==j)]=subsol gain[:,:,(ant==k) & (dd==j)]=subgain print 'numflag', numflag if(doplot): pl.figure(2) k=1 #pl.clf() pl.plot(t[(ant==k)], sol[0,0,(ant==k)], 'b+') pl.plot(t[(ant==k)], flg[0,0,(ant==k)], 'r+') pl.twinx() pl.plot(t[(ant==k)], abs(gain[0,0,(ant==k)]), 'go') pl.title('antenna='+str(k)) ### tb.putcol('FLAG', flg) tb.putcol('SOLUTION_OK', sol) tb.putcol('GAIN', gain) tb.done()
def flag_extreme_amplitudes(tablename, maxpctchange=50, pols=[0], channels=[0]): """ Flag out all gain amplitudes with >``maxpctchange``% change (e.g., for the default 50%, flag everything outside the range 0.5 < G < 1.5). This is a *very simple* cut, but it cannot be easily applied with existing tools since it is cutting on the value of the amplitude correction, not on any particular normal data selection type. It is still highly advisable to plot the amp vs snr or similar diagnostics after running this to make sure you understand what's going on. For example, after I ran this on a data set, I discovered that one antenna had high gain corrections even in the high SNR regime, which probably indicates a problem with that antenna. Parameters ---------- maxpctchange : float The maximum percent change permitted in an amplitude pols : list The list of polarizations to include in the heuristics channels : list The list of channels to include in the heuristics Returns ------- None """ tb.open(tablename) solns = tb.getcol('CPARAM') snr = tb.getcol('SNR') # true flag = flagged out, bad data flags = tb.getcol('FLAG') tb.close() amp = np.abs(solns) maxfrac = maxpctchange / 100. bad = ((amp[pols, channels] > (1 + maxfrac)) | (amp[pols, channels] < (1 - maxfrac))) bad_snr = snr[pols, channels, :][bad] print("Found {0} bad amplitudes with mean snr={1}".format( bad.sum(), bad_snr.mean())) print("Total flags in tb.flag: {0}".format(flags.sum())) flags[pols, channels, :] = bad | flags[pols, channels, :] assert all(flags[pols, channels, :][bad]), "Failed to modify array" tb.open(tablename, nomodify=False) tb.putcol(columnname='FLAG', value=flags) tb.flush() tb.close() tb.open(tablename, nomodify=True) flags = tb.getcol('FLAG') print("Total flags in tb.flag after: {0}".format(flags.sum())) assert all(flags[pols, channels, :][bad]), "Failed to modify table" tb.close()
def fill_flagged_soln(caltable='', doplot=False): """ This is to replace the gaincal solution of flagged/failed solutions by the nearest valid one. If you do not do that and applycal blindly with the table your data gets flagged between calibration runs that have a bad/flagged solution at one edge. Can be pretty bad when you calibrate every hour or more (when you are betting on self-cal) of observation (e.g L-band of the EVLA)..one can lose the whole hour of good data without realizing ! """ tb.open(caltable, nomodify=False) flg = tb.getcol('FLAG') sol = tb.getcol('SOLUTION_OK') ant = tb.getcol('ANTENNA1') gain = tb.getcol('GAIN') t = tb.getcol('TIME') dd = tb.getcol('CAL_DESC_ID') maxant = np.max(ant) maxdd = np.max(dd) npol = len(gain[:, 0, 0]) nchan = len(gain[0, :, 0]) k = 1 if (doplot): pl.ion() pl.figure(1) pl.plot(t[(ant == k)], sol[0, 0, (ant == k)], 'b+') pl.plot(t[(ant == k)], flg[0, 0, (ant == k)], 'r+') pl.twinx() pl.plot(t[(ant == k)], abs(gain[0, 0, (ant == k)]), 'go') print 'maxant', maxant numflag = 0.0 for k in range(maxant + 1): for j in range(maxdd + 1): subflg = flg[:, :, (ant == k) & (dd == j)] subt = t[(ant == k) & (dd == j)] subsol = sol[:, :, (ant == k) & (dd == j)] subgain = gain[:, :, (ant == k) & (dd == j)] #print 'subgain', subgain.shape for kk in range(1, len(subt)): for chan in range(nchan): for pol in range(npol): if (subflg[pol, chan, kk] and not subflg[pol, chan, kk - 1]): numflag += 1.0 subflg[pol, chan, kk] = False subsol[pol, chan, kk] = True subgain[pol, chan, kk] = subgain[pol, chan, kk - 1] if (subflg[pol, chan, kk - 1] and not subflg[pol, chan, kk]): numflag += 1.0 subflg[pol, chan, kk - 1] = False subsol[pol, chan, kk - 1] = True subgain[pol, chan, kk - 1] = subgain[pol, chan, kk] flg[:, :, (ant == k) & (dd == j)] = subflg sol[:, :, (ant == k) & (dd == j)] = subsol gain[:, :, (ant == k) & (dd == j)] = subgain print 'numflag', numflag if (doplot): pl.figure(2) k = 1 #pl.clf() pl.plot(t[(ant == k)], sol[0, 0, (ant == k)], 'b+') pl.plot(t[(ant == k)], flg[0, 0, (ant == k)], 'r+') pl.twinx() pl.plot(t[(ant == k)], abs(gain[0, 0, (ant == k)]), 'go') pl.title('antenna=' + str(k)) ### tb.putcol('FLAG', flg) tb.putcol('SOLUTION_OK', sol) tb.putcol('GAIN', gain) tb.done()
def make_primary_beams(image_name,lst,stokes_choice,beam_filename): # # define the frequencies of the simulated beams # freq_beams = np.zeros(11) freq_beams[0] = 100e6 for j in range(1,11): freq_beams[j] = freq_beams[j - 1] + 10e6 # # read the input cube # tb.open(image_name) q_cube = tb.getcol('map') tb.close() # ra = np.ndarray(shape=(image.shape[0],image.shape[1]),dtype=float) # dec = np.ndarray(shape=(image.shape[0],image.shape[1]),dtype=float) ia.open(image_name) summary = ia.summary() # read the image summary cube_shape = summary['shape'] # read the image shape: RA, DEC, Stokes, Freq ra = np.ndarray(shape=(cube_shape[0],cube_shape[1]),dtype=float) dec = np.ndarray(shape=(cube_shape[0],cube_shape[1]),dtype=float) nchan = cube_shape[3] # number of channels in the cube start_freq = summary['refval'][3] # start frequencies in Hz df = summary['incr'][3] # frequency increment in Hz # # ra and dec will contain the RA and DEC corresponding to the pixel values in the image # # for j in range(0,cube_shape[0]): for k in range(0,cube_shape[1]): a=ia.toworld([j,k,0,0]) # a dictionary is returned with the world coordinates of that pixel b=a['numeric'] # the array is extracted from the dictionary a ra[j,k] = b[0] # save the RA for pixel j,k dec[j,k] = b[1] # save the DEC for pixel j,k # print ra[j,k] * 12/np.pi,dec[j,k] * 180/np.pi,j,k # print 'RA and DEC calculated' ia.close() # # # # read the beams # fekoX=fmt.FEKO('/home/gianni/PAPER/beams/fitBeam/data/PAPER_FF_X.ffe') fekoY=fmt.FEKO('/home/gianni/PAPER/beams/fitBeam/data/PAPER_FF_Y.ffe') feko_xpol=fekoX.fields[0] feko_ypol=fekoY.fields[0] phi=feko_xpol.phi*np.pi/180. # phi is the azimuth theta=feko_xpol.theta*np.pi/180. # theta is the zenith angle theta = np.pi/2 - theta # pyephem wants the elevation rather than the zenith angle ra_beams = np.ndarray(shape=(phi.shape[0]),dtype=float) # array of RA corresponding to (phi,theta) dec_beams = np.ndarray(shape=(phi.shape[0]),dtype=float) # array of DEC corresponding to (phi,theta) # # compute complex beams # gxx=feko_xpol.etheta*np.conj(feko_xpol.etheta)+feko_xpol.ephi*np.conj(feko_xpol.ephi) # # define an array that will contain all the simulated beams # beams = np.ndarray(shape=(gxx.shape[0],4,freq_beams.shape[0]),dtype=float) # # read all the beam models and save them in the beams array # for j in range(0,freq_beams.shape[0]): feko_xpol = fekoX.fields[j] # these two lines give an error, I need to check with Griffin how to fix it feko_ypol = fekoY.fields[j] # gxx = feko_xpol.etheta*np.conj(feko_xpol.etheta)+feko_xpol.ephi*np.conj(feko_xpol.ephi) gyy = feko_ypol.etheta*np.conj(feko_ypol.etheta)+feko_ypol.ephi*np.conj(feko_ypol.ephi) gxy = feko_xpol.etheta*np.conj(feko_ypol.etheta)+feko_xpol.ephi*np.conj(feko_ypol.ephi) gyx = feko_ypol.etheta*np.conj(feko_xpol.etheta)+feko_ypol.ephi*np.conj(feko_xpol.ephi) # # make the stokes beams # beams[:,0,j] = (gxx+gyy).real # beams[:,0,j] = beams[:,0,j] / np.max(beams[:,0,j]) # normalize the beams to be 1 at zenith beams[:,1,j] = (gxx-gyy).real beams[:,2,j] = (gxy+gyx).real beams[:,3,j] = (gxy-gyx).imag # norm_beam = np.max(beams[:,0,5]) # beam peak at 150 MHz beams[:,0,:] = beams[:,0,:] / norm_beam # normalize the beams to be 1 at zenith at 150 MHz # print norm_beam,np.max(beams[:,0,:]) print 'Beams read' # # bring the beam to RA,DEC coordinates # # Create an observer # paper = Observer() # # Set the observer at the Karoo # paper.lat, paper.long, paper.elevation = '-30:43:17', '21:25:40.08', 0.0 # j0 = ephem.julian_date(0) # invoked this way if import ephem j0 = julian_date(0) # invoked this way if from ephem import * # paper.date = float(lst) paper.date = float(lst) - j0 + 5./60/24 # I think I need this. At http://stackoverflow.com/questions/8962426/convert-topocentric-coordinates-azimuth-elevation-to-equatorial-coordinates # they seem to suggest that this is needed in order to get the right date and I seem to get the right # RA if I include this. The factor 5./60/24 needs to be added as the lst reported in the filename refers to # the beginning of the integration which is ~10 min long for j in range(0,ra_beams.shape[0]): a = paper.radec_of(phi[j],theta[j]) ra_beams[j] = a[0] # RA is in radians dec_beams[j] = a[1] # DEC is in radians # # # now interpolate the beams in frequency # interp_beam = np.ndarray(shape=(beams.shape[0],beams.shape[1],nchan),dtype=float) cube_freq = start_freq for chan in range(0,nchan): a = np.max(q_cube[:,:,0,chan,0]) b = np.min(q_cube[:,:,0,chan,0]) if (a != 0 and b != 0): # if the image is not empty, then proceed freq_dist = np.abs(cube_freq - freq_beams) freq_dist_s = np.sort(freq_dist) w = np.where(freq_dist == freq_dist_s[0]) if freq_dist_s[0] == 0: # # if the beam is simulated at the exact frequency channel, then do not interpolate # for j in range(0,4): interp_beam[:,j,chan] = beams[:,j,w[0][0]] # # if they are not, perform a weighted average of the two closest beams in frequency. The weights are the inverse of the frequency distance squared # else: w1 = np.where(freq_dist == freq_dist_s[1]) for j in range(0,4): interp_beam[:,j,chan] = (beams[:,j,w[0][0]] * freq_dist_s[0]**(-2) + beams[:,j,w1[0][0]] * freq_dist_s[1]**(-2)) / (freq_dist_s[0]**(-2) + freq_dist_s[1]**(-2)) # cube_freq = cube_freq + df # print 'Beams interpolated in frequency' # # now interpolate the beam at the observed RA,DEC # interp_beam_maps_q = np.ndarray(shape=(ra.shape[0],ra.shape[1],1,nchan,1),dtype=float) # for j in range(0,ra.shape[0]): for k in range(0,ra.shape[1]): # # interpolating amongst the three closest points # # x = np.cos(ra[j,k])*np.cos(ra_beams)*np.cos(dec[j,k])*np.cos(dec_beams) # y = np.sin(ra[j,k])*np.sin(ra_beams)*np.cos(dec[j,k])*np.cos(dec_beams) # z = np.sin(dec[j,k])*np.sin(dec_beams) # dist = np.sqrt(x**2 + y**2 + z**2) # dist = np.sqrt((ra[j,k] - ra_beams)**2 + (dec[j,k] - dec_beams)**2) dist_s = np.sort(dist) w0 = np.where(dist == dist_s[0]) w1 = np.where(dist == dist_s[1]) w2 = np.where(dist == dist_s[2]) # interp_beam_maps_q[j,k,0,:,0] = interp_beam[w0[0][0],stokes_choice,:] / dist_s[0] + interp_beam[w1[0][0],stokes_choice,:] / dist_s[1] + interp_beam[w2[0][0],stokes_choice,:] / dist_s[2] interp_beam_maps_q[j,k,0,:,0] = interp_beam_maps_q[j,k,0,:,0] / (dist_s[0]**(-1) + dist_s[1]**(-1) + dist_s[2]**(-1)) # # nearest neighbour interpolation # # dist = np.sqrt((ra[j,k] - ra_beams)**2 + (dec[j,k] - dec_beams)**2) # dist_s = np.sort(dist) # w0 = np.where(dist == dist_s[0]) # interp_beam_maps_q[j,k,0,:,0] = interp_beam[w0[0][0],stokes_choice,:] # print 'Beams interpolated in angle' # # store the beams into an image # # beam_filename = image_name.strip('.image') cmd = 'rm -rf ' + beam_filename + '.beams' os.system(cmd) cmd = 'cp -r ' + image_name + ' ' + beam_filename + '.beams' os.system(cmd) # tb.open(beam_filename + '.beams',nomodify=False) tb.putcol('map',interp_beam_maps_q) tb.close() tb.done() ia.open(beam_filename + '.beams') ia.tofits(beam_filename + '_beams.fits',overwrite=true) ia.close()
# # specific PAPER options # #cmd = 'rm -rf '+filename #os.system(cmd) #cmd = 'python /home/gianni/Python_code/swap_column.py /home/gianni/PAPER/psa32/data/Original/'+filename+' '+filename #os.system(cmd) #ft(vis=filename,model='First_model/2455819.50285.model',usescratch=True) # cmd = 'rm -rf ' + filename os.system(cmd) cmd = 'cp -r /home/gianni/PAPER/psa32/data/Original/'+ filename + ' ' + filename os.system(cmd) tb.open(filename,nomodify=False) data = tb.getcol('DATA') tb.putcol('CORRECTED_DATA',data) # the CORRECTED_DATA column is initialized to DATA tb.close() tb.done() # tflagdata(vis=filename,mode='manual',spw='0:167~202',action='apply',datacolumn='DATA') flagdata(vis=filename,mode='clip',clipminmax=[0,300],correlation='ABS_XY,YX',action='apply',datacolumn='DATA') flagdata(vis=filename,mode='clip',clipminmax=[0,1000],correlation='ABS_XX,YY',action='apply',datacolumn='DATA') # out_fits = lst+'.fits' out_fits_res = lst+'_residual.fits' out_fits_model = lst+'_model.fits' # cmd = 'rm -rf '+ lst + '.image ' + lst + '.model ' + lst + '.residual ' + lst + '.flux' + lst + '.psf' os.system(cmd) # # make an image