def fieldflux(infile): invert = lib.miriad('invert') invert.vis = infile invert.map = 'fluxmap' invert.beam = 'fluxbeam' invert.imsize = 2049 invert.cell = 3 invert.stokes = 'ii' invert.options = 'mfs' invert.robust = 0 invert.slop = 1 invert.go() clean = lib.miriad('clean') clean.map = invert.map clean.beam = invert.beam clean.out = 'fluxmodel' clean.niters = 10000 clean.go() fits = lib.miriad('fits') fits.in_ = clean.out fits.op = 'xyout' fits.out = clean.out + '.fits' fits.go() pyfile = pyfits.open(fits.out) image = pyfile[0].data[0][0] pyfile.close() intflux = np.sum(image) os.system('rm -rf flux*') return intflux
def create_parametric_mask(self, dataset, radius, cutoff, cat, outputdir): ''' Creates a parametric mask using a model from an input catalogue. dataset (string): The dataset to get the coordiantes for the model from. radius (float): The radius around the pointing centre of the input dataset to consider sources in in deg. cutoff (float): The apparent flux percentage to consider sources from 0.0 accounts for no sources, 1.0 for all sources in the catalogue within the search radius of the target field. cat (string): The catalogue to search sources in. Possible options are 'NVSS', 'FIRST', and 'WENSS'. outputdir (string): The output directory to create the MIRIAD mask file in. The file is named mask. ''' lsm.write_mask(outputdir + '/mask.txt', lsm.lsm_mask(dataset, radius, cutoff, cat)) mskfile = open(outputdir + '/mask.txt', 'r') object = mskfile.readline().rstrip('\n') spar = mskfile.readline() mskfile.close() imgen = lib.miriad('imgen') imgen.imsize = self.selfcal_image_imsize imgen.cell = self.selfcal_image_cellsize imgen.object = object imgen.spar = spar imgen.out = outputdir + '/imgen' imgen.go() maths = lib.miriad('maths') maths.exp = '"<' + outputdir + '/imgen' + '>"' maths.mask = '"<' + outputdir + '/imgen>.gt.1e-6' + '"' maths.out = outputdir + '/mask' maths.go() subs.managefiles.director(self, 'rm', outputdir + '/imgen') self.single_mskcutoff = 1e-6 subs.managefiles.director(self, 'rm', outputdir + '/mask.txt')
def cal_mult_bp(scanlist, cfgfile, basedir, sourcename, gainint='60', gapint='60', bpint='60'): #now do the bandpass solutions - the point of all of this! #define intervals by default #numbers don't matter overly much - will be full solution of short scan #sourcename is important! Make sure it's a recognized calibrator format!!!! #read cfgfile, not sure this will actually matter here since I set basedir manually ccal = apercal.ccal(cfgfile) #set-up utilities for bandpass calibration mfcal = lib.miriad('mfcal') # Comment the next line out if you don't want to solve for delays mfcal.options = 'delay' mfcal.stokes = 'XX' mfcal.interval = gainint + ',' + gapint + ',' + bpint mfcal.tol = 0.1 # Set the tolerance a bit lower. Otherwise mfcal takes a long time to finsh #IMPORTANT! source names have beam number, need to make sure miriad canrecongize #these parts are global, and I do specific scans below puthd = lib.miriad('puthd') puthd.value = sourcename #Now iterate and # Execute the bandpass calibration #reading into different arrays #now iterate through scans and do calibration for i, scan in enumerate(scanlist): print i, scan ccal.basedir = basedir + '_' + str(i) + '/' ccal.crosscaldir = ccal.basedir + '00/crosscal/' #print ccal.basedir #print ccal.fluxcal #print ccal.crosscaldir puthdstring = ccal.crosscaldir + sourcename + '.mir/source' #print puthdstring puthd.in_ = puthdstring #extra underscore because python has a global in puthd.go() #convert.show() #print ccal.crosscaldir if os.path.exists(ccal.crosscaldir): print 'doing calibration' mfcal.vis = ccal.crosscaldir + ccal.fluxcal mfcal.go()
def getimagestats(self, image): ''' Subroutine to calculate the max,min and rms of an image image (string): The absolute path to the image file. returns (numpy array): The min, max and rms of the image ''' subs.setinit.setinitdirs(self) char_set = string.ascii_uppercase + string.digits if os.path.isdir(image) or os.path.isfile(image): if os.path.isdir(image): tempdir = subs.managetmp.manage_tempdir('images') temp_string = ''.join(random.sample(char_set * 8, 8)) fits = lib.miriad('fits') fits.op = 'xyout' fits.in_ = image fits.out = tempdir + '/' + temp_string + '.fits' fits.go() image_data = pyfits.open(tempdir + '/' + temp_string + '.fits') elif os.path.isfile(image): image_data = pyfits.open(image) else: print( 'Image format not supported. Only MIRIAD and FITS formats are supported!' ) data = image_data[0].data imagestats = np.full((3), np.nan) imagestats[0] = np.nanmin(data) # Get the maxmimum of the image imagestats[1] = np.nanmax(data) # Get the minimum of the image imagestats[2] = np.nanstd(data) # Get the standard deviation image_data.close() # Close the image subs.managetmp.clean_tempdir('images') else: print('Image does not seem to exist!') return imagestats
def calc_theoretical_noise(self, dataset): ''' Calculate the theoretical rms of a given dataset dataset (string): The input dataset to calculate the theoretical rms from returns (float): The theoretical rms of the input dataset as a float ''' uv = aipy.miriad.UV(dataset) obsrms = lib.miriad('obsrms') try: tsys = np.median(uv['systemp']) if np.isnan(tsys): obsrms.tsys = 30.0 else: obsrms.tsys = tsys except KeyError: obsrms.tsys = 30.0 obsrms.jyperk = uv['jyperk'] obsrms.antdiam = 25 obsrms.freq = uv['sfreq'] obsrms.theta = 15 obsrms.nants = uv['nants'] obsrms.bw = np.abs(uv['sdf'] * uv['nschan']) * 1000.0 obsrms.inttime = 12.0 * 60.0 obsrms.coreta = 0.88 theorms = float(obsrms.go()[-1].split()[3]) / 1000.0 return theorms
def get_bp(file): ''' Function to create a python array from a bandpass calibrated dataset to analyse file (str): u,v file with the bandpass calibration return(array, array): The bandpass array in the following order (antenna, frequencies, solution intervals) and a list of the frequencies ''' char_set = string.ascii_uppercase + string.digits # Create a charset for random gain log file generation tempdir = subs.managetmp.manage_tempdir('mirlog') # tempdir = os.path.expanduser('~') + '/apercal/temp/mirlog' bp_string = ''.join(random.sample(char_set * 8, 8)) gpplt = lib.miriad('gpplt') gpplt.vis = file gpplt.log = tempdir + '/' + bp_string gpplt.options = 'bandpass' gpplt.go() check_table(tempdir + '/' + bp_string, 'bp') t = Table.read(tempdir + '/' + bp_string, format='ascii') freqs = np.array(np.unique(t['col1'])) nfreq = len(freqs) nint = len(t['col1']) / nfreq nant = int(len(t[0]) - 1) bp_array = np.zeros((nant, nfreq, nint)) for ant in range(nant): bp_array[ant, :, :] = np.swapaxes( t['col' + str(ant + 2)].reshape(nint, nfreq), 0, 1) return bp_array, freqs
def theostats(infile): ''' theostats: Calculates the theoretical noise of an observation using the MIRIAD task obsrms infile: The input file to calculate the noise for return: The theoretical rms ''' obsrms = lib.miriad('obsrms') theorms = 0.00004 return theorms
def transfer_to_target(self): ''' Transfers the gains of the calibrators to the target field. Automatically checks if polarisation calibration has been done. ''' if self.crosscal_transfer_to_target: subs.setinit.setinitdirs(self) subs.setinit.setdatasetnamestomiriad(self) subs.managefiles.director(self, 'ch', self.crosscaldir) self.logger.info( '### Copying calibrator solutions to target dataset ###') gpcopy = lib.miriad('gpcopy') if os.path.isfile(self.crosscaldir + '/' + self.polcal + '/' + 'bandpass') and os.path.isfile( self.crosscaldir + '/' + self.polcal + '/' + 'gains') and os.path.isfile( self.crosscaldir + '/' + self.polcal + '/' + 'leakage'): gpcopy.vis = self.polcal self.logger.info( '# Copying calibrator solutions (bandpass, gains, leakage, angle) from polarised calibrator #' ) elif os.path.isfile(self.crosscaldir + '/' + self.fluxcal + '/' + 'bandpass') and os.path.isfile( self.crosscaldir + '/' + self.fluxcal + '/' + 'gains'): gpcopy.vis = self.fluxcal self.logger.info( '# Copying calibrator solutions (bandpass, gains) from flux calibrator #' ) self.logger.info( '# Polarisation calibration solutions (leakage, angle) not found #' ) else: self.logger.error( '# No calibrator solutions found! Exiting! #') sys.exit(1) datasets = glob.glob('../../*') self.logger.info('# Copying calibrator solutions to ' + str(len(datasets)) + ' beams! #') for n, ds in enumerate(datasets): if os.path.isfile(ds + '/' + self.crosscalsubdir + '/' + self.target + '/visdata'): gpcopy.out = ds + '/' + self.crosscalsubdir + '/' + self.target gpcopy.options = 'relax' gpcopy.go() self.logger.info('# Calibrator solutions copied to beam ' + str(n).zfill(2) + '! #') else: self.logger.warning('# Beam ' + str(n).zfill(2) + ' does not seem to contain data! #') self.logger.info( '### All solutions copied to target data set(s) ###') else: self.logger.info( '### No copying of calibrator solutions to target data done! ###' )
def resistats(infile): fits = lib.miriad('fits') fits.in_ = 'residual' fits.op = 'xyout' fits.out = 'rmsresidual' + '.fits' fits.go() pyfile = pyfits.open(fits.out) image = pyfile[0].data[0][0] pyfile.close() resimax = np.amax(image) resirms = np.std(image) os.system('rm -rf rms*') return resirms, resimax
def imagetofits(self, mirimage, fitsimage): ''' Converts a MIRIAD image to a FITS image mirimage: The MIRIAD image to convert fitsimage: The converted FITS image ''' subs.setinit.setinitdirs(self) fits = lib.miriad('fits') fits.op = 'xyout' fits.in_ = mirimage fits.out = fitsimage fits.go() if os.path.isfile(fitsimage): director(self, 'rm', mirimage)
def get_gains(self, chunk, yaxis): ''' Function to write and return the plot txt file chunk (int): The chunk to return the arrays for. yaxis (string): The axis you want to show the gains for. return (array, array): The array with the timestep information, the array with the gains for all antennas of a chunk. ''' subs.setinit.setinitdirs(self) char_set = string.ascii_uppercase + string.digits # Create a charset for random gain log file generation self.tempdir = os.path.expanduser('~') + '/apercal/temp/iaplot' gpplt = lib.miriad('gpplt') gpplt.vis = self.selfcaldir + '/' + str(chunk).zfill(2) + '/' + str( chunk).zfill(2) + '.mir' gpplt.log = self.tempdir + '/' + ''.join(random.sample( char_set * 8, 8)) + '.txt' gpplt.yaxis = yaxis gpplt.options = 'gains' gpplt.go() gainstring = '' with open(gpplt.log, 'r') as gaintxt: gainlines = gaintxt.readlines() nants = int(gainlines[3][-3:-1]) obsdate = gainlines[1][-19:-12] gainlines = gainlines[4:] joiner = range(0, len(gainlines), int(m.ceil(nants / 6) + 1)) for num, line in enumerate(gainlines): if num in joiner: joinedline = re.sub('\s+', ' ', ''.join( gainlines[num:num + 3])).strip() + '\n' joinedline = re.sub(':', ' ', joinedline) gainstring = gainstring + joinedline gainfile = self.tempdir + '/' + ''.join(random.sample(char_set * 8, 8)) + '.txt' with open(gainfile, 'w') as gainreformated: gainreformated.write(gainstring) with open(gainfile, 'r') as gainarrayfile: gainarray = np.loadtxt(gainarrayfile) starttime = datetime.datetime( int('20' + obsdate[0:2]), int(time.strptime(obsdate[2:5], '%b').tm_mon), int(obsdate[5:7])) timearray = [ starttime + datetime.timedelta(days=gainarray[gain, 0], hours=gainarray[gain, 1], minutes=gainarray[gain, 2], seconds=gainarray[gain, 3]) for gain in range(len(gainarray)) ] gainarray = gainarray[:, 4:] return timearray, gainarray
def imstats(infile, stokes): invert = lib.miriad('invert') invert.vis = infile invert.map = 'rmsmap' invert.beam = 'rmsbeam' invert.imsize = 2049 invert.cell = 3 invert.stokes = stokes invert.options = 'mfs' invert.robust = 0 invert.slop = 1 invert.go() fits = lib.miriad('fits') fits.in_ = invert.map fits.op = 'xyout' fits.out = invert.map + '.fits' fits.go() pyfile = pyfits.open(fits.out) image = pyfile[0].data[0][0] pyfile.close() inimax = np.amax(image) inirms = np.std(image) os.system('rm -rf rms*') return inimax, inirms
def calc_isum(self, image): ''' Function to calculate the sum of the values of the pixels in an image image (string): The name of the image file. Must be in MIRIAD-format returns (float): the sum of the pxiels in the image ''' fits = lib.miriad('fits') fits.op = 'xyout' fits.in_ = image fits.out = image + '.fits' fits.go() image_data = pyfits.open(image + '.fits') # Open the image data = image_data[0].data isum = np.nansum(data) # Get the maximum image_data.close() # Close the image subs.managefiles.director(self, 'rm', image + '.fits') return isum
def get_gains(file): ''' Function to create a complex python array of amplitude and phase gains from a dataset file (str): u,v file with the bandpass calibration return(array, array): an array with the amplitude and phase gains for each antenna and solution interval, a datetime array with the actual solution timesteps ''' char_set = string.ascii_uppercase + string.digits # Create a charset for random gain log file generation tempdir = subs.managetmp.manage_tempdir('mirlog') gains_string = ''.join(random.sample(char_set * 8, 8)) gpplt = lib.miriad('gpplt') gpplt.vis = file gpplt.log = tempdir + '/' + gains_string gpplt.options = 'gains' gpplt.yaxis = 'amp' cmd = gpplt.go() check_table(tempdir + '/' + gains_string, 'gains') obsdate = cmd[3].split(' ')[4][0:7] starttime = datetime.datetime( int('20' + obsdate[0:2]), int(time.strptime(obsdate[2:5], '%b').tm_mon), int(obsdate[5:7])) s = Table.read(tempdir + '/' + gains_string, format='ascii') gpplt.yaxis = 'phase' gpplt.go() check_table(tempdir + '/' + gains_string, 'gains') t = Table.read(tempdir + '/' + gains_string, format='ascii') days = np.array(t['col1']) times = np.array(t['col2']) nint = len(days) nant = int(len(t[0]) - 2) gain_array = np.zeros((nant, nint, 2)) for ant in range(nant): gain_array[ant, :, 0] = s['col' + str(ant + 3)] # gain_array[ant, :, 1] = t['col' + str(ant + 3)] gain_array[ant, :, 1] = np.unwrap(t['col' + str(ant + 3)], discont=90) time_array = [ starttime + datetime.timedelta(days=int(days[step]), hours=int(times[step][0:2]), minutes=int(times[step][3:5]), seconds=int(times[step][6:8])) for step in range(nint) ] return gain_array, time_array
def do_bp(cfgfile, inttime='30'): #function to do the bandpass solution, for given interval #inttime is the solution interval for bandpass, in minutes ccal = apercal.ccal(cfgfile) #change to working directory #workaround for long path names ccal.director('ch', ccal.crosscaldir) gainint = inttime gapint = inttime bpint = inttime mfcal = lib.miriad('mfcal') mfcal.vis = ccal.fluxcal print mfcal.vis #print mfcal.vis # Comment the next line out if you don't want to solve for delays mfcal.options = 'delay' mfcal.stokes = 'XX' mfcal.interval = gainint + ',' + gapint + ',' + bpint mfcal.tol = 0.1 # Set the tolerance a bit lower. Otherwise mfcal takes a long time to finsh mfcal.go() print 'calibration finished'
def bandpass(self): ''' Calibrates the bandpass for the flux calibrator using mfcal in MIRIAD. ''' if self.crosscal_bandpass: subs.setinit.setinitdirs(self) subs.setinit.setdatasetnamestomiriad(self) subs.managefiles.director(self, 'ch', self.crosscaldir) self.logger.info( '### Bandpass calibration on the flux calibrator data started ###' ) mfcal = lib.miriad('mfcal') mfcal.vis = self.fluxcal mfcal.stokes = 'ii' if self.crosscal_delay: mfcal.options = 'delay' else: pass mfcal.interval = 1000 mfcal.go() self.logger.info( '### Bandpass calibration on the flux calibrator data done ###' )
def get_delays(file): ''' Function to create a numpy array with the antenna delays for each solution interval file (str): u,v file with the bandpass calibration return(array, array): an array with the delays for each antenna and solution interval in nsec, a datetime array with the actual solution timesteps ''' char_set = string.ascii_uppercase + string.digits # Create a charset for random gain log file generation tempdir = subs.managetmp.manage_tempdir('mirlog') # tempdir = os.path.expanduser('~') + '/apercal/temp/mirlog' gains_string = ''.join(random.sample(char_set * 8, 8)) gpplt = lib.miriad('gpplt') gpplt.vis = file gpplt.log = tempdir + '/' + gains_string gpplt.options = 'delays' cmd = gpplt.go() check_table(tempdir + '/' + gains_string, 'delays') obsdate = cmd[3].split(' ')[4][0:7] starttime = datetime.datetime( int('20' + obsdate[0:2]), int(time.strptime(obsdate[2:5], '%b').tm_mon), int(obsdate[5:7])) t = Table.read(tempdir + '/' + gains_string, format='ascii') days = np.array(t['col1']) times = np.array(t['col2']) nint = len(days) nant = int(len(t[0]) - 2) delay_array = np.zeros((nant, nint)) for ant in range(nant): delay_array[ant, :] = t['col' + str(ant + 3)] time_array = [ starttime + datetime.timedelta(days=int(days[step]), hours=int(times[step][0:2]), minutes=int(times[step][3:5]), seconds=int(times[step][6:8])) for step in range(nint) ] return delay_array, time_array
def find_src_imsad(self): ''' Finds sources in continuum image above a threshold in flux specified by the user IN Continuum image: located in contdir IN cfga abs_ex_imsad_clip: sets flux threshold in Jy abs_ex_imsad_region: xmin,xmax,ymin,ymax defines regions where to search for sources OUT cont_src_imsad.txt: table with found sources stored in contdir ''' self.logger.info('### Find continuum sources ###') os.chdir(self.contdir) if os.path.exists(self.cont_im_mir) == False: fits = lib.miriad('fits') fits.op = 'xyin' fits.in_ = self.cont_im fits.out = self.cont_im_mir fits.go(rmfiles=True) if os.path.exists(self.cont_im) == False: fits = lib.miriad('fits') fits.op = 'xyout' fits.in_ = self.cont_im_mir fits.out = self.cont_im fits.go(rmfiles=True) imsad = lib.miriad('imsad') imsad.in_ = self.cont_im_mir imsad.out = self.src_imsad_out imsad.clip = self.abs_ex_imsad_clip #imsad.region = 'boxes\('+self.abs_ex_imsad_region+'\)' imsad.options = self.abs_ex_imsad_options imsad.go(rmfiles=True) #modify output of imsad for module load_src_csv src_list = open(self.src_imsad_out, 'r') lines = src_list.readlines() len_lines = len(lines) ra_tmp = [] dec_tmp = [] peak_tmp = [] for i in xrange(0, len_lines): lines[i] = lines[i].strip() tmp = lines[i].split(' ') ra_tmp.append(str(tmp[3])) dec_tmp.append(str(tmp[4])) peak_tmp.append(str(tmp[6])) ra_tmp = np.array(ra_tmp) dec_tmp = np.array(dec_tmp) peak_tmp = np.array(peak_tmp) #ID ids = np.array(np.arange(1, len_lines + 1, 1), dtype=str) #J2000 #convert ra ra_vec = [] for i in xrange(0, len_lines): line = ra_tmp[i].split(':') last_dig = int(round(float(line[2]), 0)) if last_dig < 10: last_dig = '0' + str(last_dig) ra_vec.append(line[0] + line[1] + str(last_dig)) #convert dec dec_vec = [] dec_coord = [] for i in xrange(0, len_lines): line = dec_tmp[i].split(':') last_dig = int(round(float(line[2]), 0)) first_dig = line[0].split('+') dec_vec.append(first_dig[2] + line[1] + str(last_dig)) dec_coord.append('+' + first_dig[2] + ':' + line[1] + ':' + str(last_dig)) J2000_tmp = np.array([a + '+' + b for a, b in zip(ra_vec, dec_vec)]) dec_coord = np.array(dec_coord) ### Find pixels of sources in continuum image #open continuum image hdulist = pyfits.open(self.cont_im) # read input # read data and header #what follows works for wcs, but can be written better prihdr = hdulist[0].header if prihdr['NAXIS'] == 4: del prihdr['CTYPE4'] del prihdr['CDELT4'] del prihdr['CRVAL4'] del prihdr['CRPIX4'] del prihdr['CTYPE3'] del prihdr['CDELT3'] del prihdr['CRVAL3'] del prihdr['CRPIX3'] del prihdr['NAXIS3'] del prihdr['NAXIS'] prihdr['NAXIS'] = 2 #load wcs system w = wcs.WCS(prihdr) self.pixels_cont = np.zeros([ra_tmp.shape[0], 2]) ra_list_tmp = np.zeros([ra_tmp.shape[0]]) for i in xrange(0, ra_tmp.shape[0]): ra_deg_tmp = self.ra2deg(ra_tmp[i]) dec_deg_tmp = self.dec2deg(dec_coord[i]) px, py = w.wcs_world2pix(ra_deg_tmp, dec_deg_tmp, 0) self.pixels_cont[i, 0] = str(round(px, 0)) self.pixels_cont[i, 1] = str(round(py, 0)) #make csv file with ID, J2000, Ra, Dec, Pix_y, Pix_y, Peak[Jy] tot = np.column_stack( (ids, J2000_tmp, ra_tmp, dec_coord, self.pixels_cont[:, 0], self.pixels_cont[:, 1], peak_tmp)) self.logger.info('# Continuum sources found. #') self.write_src_csv(tot) self.logger.info('### Continuum sources found ###') return 0
def selfcal_standard(self): ''' Executes the standard method of self-calibration with the given parameters ''' subs.setinit.setinitdirs(self) subs.setinit.setdatasetnamestomiriad(self) self.logger.info('### Starting standard self calibration routine ###') subs.managefiles.director(self, 'ch', self.selfcaldir) for chunk in self.list_chunks(): self.logger.info( '# Starting standard self-calibration routine on frequency chunk ' + chunk + ' #') subs.managefiles.director(self, 'ch', self.selfcaldir + '/' + chunk) if os.path.isfile(self.selfcaldir + '/' + chunk + '/' + chunk + '.mir/visdata'): theoretical_noise = self.calc_theoretical_noise( self.selfcaldir + '/' + chunk + '/' + chunk + '.mir') self.logger.info('# Theoretical noise for chunk ' + chunk + ' is ' + str(theoretical_noise) + ' Jy/beam #') theoretical_noise_threshold = self.calc_theoretical_noise_threshold( theoretical_noise) self.logger.info( '# Your theoretical noise threshold will be ' + str(self.selfcal_standard_nsigma) + ' times the theoretical noise corresponding to ' + str(theoretical_noise_threshold) + ' Jy/beam #') dr_list = self.calc_dr_maj( self.selfcal_standard_drinit, self.selfcal_standard_dr0, self.selfcal_standard_majorcycle, self.selfcal_standard_majorcycle_function) self.logger.info('# Your dynamic range limits are set to ' + str(dr_list) + ' for the major self-calibration cycles #') for majc in range(self.selfcal_standard_majorcycle): self.logger.info('# Major self-calibration cycle ' + str(majc) + ' for frequency chunk ' + chunk + ' started #') subs.managefiles.director( self, 'mk', self.selfcaldir + '/' + str(chunk) + '/' + str(majc).zfill(2)) dr_minlist = self.calc_dr_min( dr_list, majc, self.selfcal_standard_minorcycle, self.selfcal_standard_minorcycle_function ) # Calculate the dynamic ranges during minor cycles self.logger.info( '# The minor cycle dynamic range limits for major cycle ' + str(majc) + ' are ' + str(dr_minlist) + ' #') for minc in range(self.selfcal_standard_minorcycle): try: self.run_continuum_minoriteration( chunk, majc, minc, dr_minlist[minc], theoretical_noise_threshold) except: self.logger.warning( '# Chunk ' + chunk + ' does not seem to contain data to image #') break try: self.logger.info( '# Doing self-calibration with uvmin=' + str(self.selfcal_standard_uvmin[majc]) + ', uvmax=' + str(self.selfcal_standard_uvmax[majc]) + ', solution interval=' + str(self.selfcal_standard_solint[majc]) + ' minutes for major cycle ' + str(majc).zfill(2) + ' #') selfcal = lib.miriad('selfcal') selfcal.vis = chunk + '.mir' selfcal.select = '"' + 'uvrange(' + str( self.selfcal_standard_uvmin[majc]) + ',' + str( self.selfcal_standard_uvmax[majc]) + ')"' selfcal.model = str(majc).zfill(2) + '/model_' + str( minc).zfill(2) selfcal.interval = self.selfcal_standard_solint[majc] # Choose reference antenna if given if self.selfcal_refant == '': pass else: selfcal.refant = self.selfcal_refant # Enable amplitude calibration if triggered if self.selfcal_standard_amp == False: # See if we want to do amplitude calibration selfcal.options = 'mfs,phase' elif self.selfcal_standard_amp == True: selfcal.options = 'mfs,amp' elif self.selfcal_standard_amp == 'auto': modelflux = self.calc_isum( str(majc).zfill(2) + '/model_' + str(minc).zfill(2)) if modelflux >= self.selfcal_standard_amp_auto_limit: self.logger.info( '# Flux of clean model is ' + str(modelflux) + ' Jy. Doing amplitude calibration. #') selfcal.options = 'mfs,amp' else: selfcal.options = 'mfs,phase' if self.selfcal_standard_nfbin >= 1: selfcal.nfbin = self.selfcal_standard_nfbin selfcal.go() self.logger.info('# Major self-calibration cycle ' + str(majc) + ' for frequency chunk ' + chunk + ' finished #') except: self.logger.warning( '# Model for self-calibration not found. No further calibration on this chunk possible!' ) break self.logger.info( '# Standard self-calibration routine for chunk ' + chunk + ' finished #') else: self.logger.warning('# No data in chunk ' + chunk + '. Maybe all data is flagged? #') self.logger.info('### Standard self calibration routine finished ###')
def show_image(self, beam=None, step=None, chunk=None, major=None, minor=None, finaltype=None, type=None, imin=None, imax=None): ''' Function to show an image in the notebook beam (string): The beam number to show the image for. 'iterate' will iterate over beams. step (string): The step of the pipeline to show the image for. No default. selfcal and final are allowed. chunk (string): The frequency chunk to show the image for. 'iterate' will iterate over chunks. No default. major(string or int): Major iteration of clean to display. Default is last iteration. pm is also possible. Only for selfcal, not for final step. 'iterate' will iterate over images for major cycles. minor (string or int): Minor iteration of clean to display. Default is last iteration. 'iterate' will iterate over images for minor cycles. finaltype(string): Type of final image to show. mf and stack are possible. type(string): Type of image to show. mask, beam, image, residual, and in case of step=final final are possible. 'all' will show image, mask, residual, and model in one plot. imin(float): Minimum cutoff in the image. imax(float): Maximum cutoff in the image. return (string): String with a path and image name to use for displaying. ''' subs.setinit.setinitdirs(self) self.manage_tempdir() # Check if the temporary directory exists and if not create it self.clean_tempdir() # Remove any temorary files from the temporary directory char_set = string.ascii_uppercase + string.digits # Create a charset for random image name generation if any(it == 'iterate' for it in [beam, chunk, major, minor]): if beam == 'iterate': print('### Not implemented yet ###') elif chunk == 'iterate': imagelist = glob.glob(self.selfcaldir + '/*/' + str(major).zfill(2) + '/' + str(type) + '_' + str(minor).zfill(2)) plottext = 'Frequency \nchunk' elif major == 'iterate': imagelist = glob.glob(self.selfcaldir + '/' + str(chunk).zfill(2) + '/*/' + str(type) + '_' + str(minor).zfill(2)) plottext = 'Major \ncycle' elif minor == 'iterate': imagelist = glob.glob(self.selfcaldir + '/' + str(chunk).zfill(2) + '/' + str(major).zfill(2) + '/' + str(type) + '_*') plottext = 'Minor \ncycle' images = [] for image in imagelist: self.fitsimage = self.tempdir + '/' + ''.join(random.sample(char_set * 8, 8)) + '.fits' fits = lib.miriad('fits') fits.op = 'xyout' fits.in_ = image fits.out = self.fitsimage fits.go() images.append(maputils.FITSimage(self.fitsimage)) # Do the autoscaling of the image contrast if imin == None or imax == None: imagedata = pyfits.open(self.fitsimage)[0].data imagestd = np.nanstd(imagedata) if imin == None: imin = -1*imagestd else: pass if imax == None: imax = 3*imagestd else: pass else: pass print('Minimum/maximum colour range for image: ' + str(imin) + '/' + str(imax)) # Draw the plot fig = plt.figure(figsize=(12, 10)) frame = fig.add_axes([0.1, 0.15, 0.85, 0.8]) frame.set_title(str(type).capitalize()) # Add the slider for selecting the image slider_ax = fig.add_axes([0.103, 0.05, 0.672, 0.03]) slider = Slider(slider_ax, plottext, 0, len(images)-1, valinit = 0, valfmt='%1.0f', dragging=False, color='black') # Define the attributes for the plot image_object = images[int(round(slider.val))].Annotatedimage(frame, cmap='gist_gray', clipmin=imin, clipmax=imax) image_object.Image() image_object.Graticule() image_object.Colorbar() image_object.interact_toolbarinfo() image_object.interact_imagecolors() image_object.plot() fig.canvas.draw() # Update the plot if the slider is clicked def slider_on_changed(val): image_object = images[int(round(slider.val))].Annotatedimage(frame, cmap='gist_gray', clipmin=imin, clipmax=imax) image_object.Image() image_object.interact_toolbarinfo() image_object.interact_imagecolors() image_object.plot() fig.canvas.draw() slider.on_changed(slider_on_changed) plt.show() else: if type == 'all': # Make a 2x2 plot of image, residual, mask, and model of a cycle rawimage = self.get_image(beam, step, chunk, major, minor, finaltype, 'image') rawresidual = self.get_image(beam, step, chunk, major, minor, finaltype, 'residual') rawmask = self.get_image(beam, step, chunk, major, minor, finaltype, 'mask') rawmodel = self.get_image(beam, step, chunk, major, minor, finaltype, 'model') self.fitsimage = self.tempdir + '/' + ''.join(random.sample(char_set * 8, 8)) + '.fits' self.fitsresidual = self.tempdir + '/' + ''.join(random.sample(char_set * 8, 8)) + '.fits' self.fitsmask = self.tempdir + '/' + ''.join(random.sample(char_set * 8, 8)) + '.fits' self.fitsmodel = self.tempdir + '/' + ''.join(random.sample(char_set * 8, 8)) + '.fits' fits = lib.miriad('fits') fits.op = 'xyout' fits.in_ = rawimage fits.out = self.fitsimage fits.go() fits.in_ = rawresidual fits.out = self.fitsresidual fits.go() fits.in_ = rawmask fits.out = self.fitsmask fits.go() regrid = lib.miriad('regrid') regrid.in_ = rawmodel tempregrid = self.tempdir + '/' + ''.join(random.sample(char_set * 8, 8)) regrid.out = tempregrid regrid.axes = '1,2' regrid.tin = rawimage regrid.go() fits.in_ = tempregrid fits.out = self.fitsmodel fits.go() im = maputils.FITSimage(self.fitsimage) imdata = pyfits.open(self.fitsimage)[0].data imstd = np.nanstd(imdata) re = maputils.FITSimage(self.fitsresidual) redata = pyfits.open(self.fitsresidual)[0].data restd = np.nanstd(redata) ma = maputils.FITSimage(self.fitsmask) madata = pyfits.open(self.fitsmask)[0].data mastd = np.nanstd(madata) mo = maputils.FITSimage(self.fitsmodel) modata = pyfits.open(self.fitsmodel)[0].data mostd = np.nanstd(modata) fig = plt.figure(figsize=(12, 10)) frame1 = fig.add_axes([0.1,0.58,0.4,0.4]) frame1.set_title('Image') frame2 = fig.add_axes([0.59,0.58,0.4,0.4]) frame2.set_title('Residual') frame3 = fig.add_axes([0.1, 0.12, 0.4, 0.4]) frame3.set_title('Mask') frame4 = fig.add_axes([0.59, 0.12, 0.4, 0.4]) frame4.set_title('Model') annim1 = im.Annotatedimage(frame1, cmap='gist_gray') annim2 = re.Annotatedimage(frame2, cmap='gist_gray') annim3 = ma.Annotatedimage(frame3, cmap='gist_gray') annim4 = mo.Annotatedimage(frame4, cmap='gist_gray') annim1.Image(); annim2.Image(); annim3.Image(); annim4.Image() if imin == None: # Set the displayed colour range if not set immin, remin, mamin, momin = -1*imstd, -1*restd, -1*mastd, -1*mostd else: immin, remin, mamin, momin = imin, imin, imin, imin if imax == None: immax, remax, mamax, momax = 3*imstd, 3*restd, mastd, mostd else: immax, remax, mamax, momax = imax, imax, imax, imax print('Minimum colour range for image: ' + str(immin) + ' residual: ' + str(remin) + ' mask: ' + str(mamin) + ' model: ' + str(momin)) print('Maximum colour range for image: ' + str(immax) + ' residual: ' + str(remax) + ' mask: ' + str(mamax) + ' model: ' + str(momax)) annim1.set_norm(clipmin=immin, clipmax=immax); annim2.set_norm(clipmin=remin, clipmax=remax); annim3.set_norm(clipmin=0.0, clipmax=mamax); annim4.set_norm(clipmin=0.0, clipmax=momax) annim1.Colorbar(); annim2.Colorbar(); annim3.Colorbar(); annim4.Colorbar() annim1.Graticule(); annim2.Graticule(); annim3.Graticule(); annim4.Graticule() annim1.interact_toolbarinfo(); annim2.interact_toolbarinfo(); annim3.interact_toolbarinfo(); annim4.interact_toolbarinfo() annim1.interact_imagecolors(); annim2.interact_imagecolors(); annim3.interact_imagecolors(); annim4.interact_imagecolors() tdict = dict(color='g', fontsize=10, va='bottom', ha='left') fig.text(0.01, 0.01, annim1.get_colornavigation_info(), tdict) maputils.showall() else: # Make a simple plot of one specified image rawimage = self.get_image(beam, step, chunk, major, minor, finaltype, type) self.fitsimage = self.tempdir + '/' + ''.join(random.sample(char_set * 8, 8)) + '.fits' fits = lib.miriad('fits') fits.op = 'xyout' fits.in_ = rawimage fits.out = self.fitsimage fits.go() f = maputils.FITSimage(self.fitsimage) imdata = pyfits.open(self.fitsimage)[0].data imstd = np.nanstd(imdata) fig = plt.figure(figsize=(12,10)) frame = fig.add_axes([0.1, 0.15, 0.85, 0.8]) annim = f.Annotatedimage(frame, cmap='gist_gray') annim.Image() if imin == None: # Set the displayed colour range if not set imin = -1*imstd if imax == None: imax = 3*imstd print('Minimum/maximum colour range for image: ' + str(imin) + '/' + str(imax)) annim.set_norm(clipmin=imin,clipmax=imax) annim.Colorbar() annim.Graticule() annim.interact_toolbarinfo() annim.interact_imagecolors() units = 'unknown' if 'BUNIT' in f.hdr: units = f.hdr['BUNIT'] helptext = "File: [%s] Data units: [%s]\n" % (rawimage, units) helptext += annim.get_colornavigation_info() tdict = dict(color='g', fontsize=10, va='bottom', ha='left') fig.text(0.01, 0.01, helptext, tdict) maputils.showall()
def convert_lineuv2uvfits(self): ''' Looks for all calibrated datasets created by the line module, combines the chunks of individual beams and converts them to UVFITS format ''' subs.setinit.setinitdirs(self) subs.managefiles.director(self, 'ch', self.transferdir, verbose=False) beamlist = sorted(glob.glob(self.basedir + '[0-9][0-9]')) beamnames = [beam.split('/')[-1] for beam in beamlist] subs.param.add_param(self, 'transfer_input_beams', beamnames) # transferstatusarray = np.full((len(beamnames, 2), np.False)) for b, beam in enumerate(beamlist): uvgluestatusarray = np.full((len(beamnames)), False) uvfitsstatusarray = np.full((len(beamnames)), False) if os.path.isfile( self.target.rstrip('.mir') + '_B' + beam.split('/')[-1] + '.UVFITS'): self.logger.warning('# UVFITS file for beam ' + beam.split('/')[-1] + ' already exists! #') else: chunklist = sorted( glob.glob(beam + '/' + self.linesubdir + '/' + '[0-9][0-9]/[0-9][0-9]' + '.mir')) chunknames = [chunk.split('/')[-2] for chunk in chunklist] subs.param.add_param( self, 'transfer_input_beam_' + str(beamnames[b]) + '_chunks', chunknames) uvcatstatusarray = np.full((len(chunknames)), False) self.logger.debug( '# Starting combination of frequency chunks for beam ' + beam.split('/')[-1] + ' #') for c, chunk in enumerate(chunklist): uvcat = lib.miriad('uvcat') uvcat.vis = chunk uvcat.out = self.transferdir + '/' + 'B' + beam.split( '/')[-1] + '_' + str(c + 1) uvcat.go() if os.path.isdir( self.transferdir + '/' + 'B' + beam.split('/')[-1] + '_' + str(c + 1) ): # Check if file has been copied successfully self.logger.debug('# Chunk ' + str(chunk).zfill(2) + ' for beam ' + str(beam.split('/')[-1]) + ' copied successfully! #') uvcatstatusarray[c] = True else: self.logger.warning('# Chunk ' + str(chunk).zfill(2) + ' for beam ' + str(beam.split('/')[-1]) + ' NOT copied successfully! #') uvcatstatusarray[c] = False subs.param.add_param( self, 'transfer_input_beam_' + str(beamnames[b]) + '_copy_status', uvcatstatusarray) uvglue = lib.miriad('uvglue') uvglue.vis = 'B' + beam.split('/')[-1] uvglue.nfiles = len(chunklist) uvglue.out = self.target.rstrip('.mir') + '_B' + beam.split( '/')[-1] + '.mir' uvglue.go() if os.path.isdir( self.target.rstrip('.mir') + '_B' + beam.split('/')[-1] + '.mir'): self.logger.debug( '# Combination of frequency chunks for beam ' + beam.split('/')[-1] + ' successful! #') subs.managefiles.director(self, 'rm', 'B' + beam.split('/')[-1] + '*') uvgluestatusarray[b] = True else: self.logger.warning( '# Combination of frequency chunks for beam ' + beam.split('/')[-1] + ' not successful! #') uvgluestatusarray[b] = False fits = lib.miriad('fits') fits.op = 'uvout' fits.in_ = self.target.rstrip('.mir') + '_B' + beam.split( '/')[-1] + '.mir' fits.out = self.target.rstrip('.mir') + '_B' + beam.split( '/')[-1] + '.UVFITS' fits.go() if os.path.isfile( self.target.rstrip('.mir') + '_B' + beam.split('/')[-1] + '.UVFITS'): self.logger.debug( '# Conversion of MIRIAD file to UVFITS for beam ' + beam.split('/')[-1] + ' successful! #') subs.managefiles.director( self, 'rm', self.target.rstrip('.mir') + '_B' + beam.split('/')[-1] + '.mir') uvfitsstatusarray[b] = True else: self.logger.warning( '# Conversion of MIRIAD file to UVFITS for beam ' + beam.split('/')[-1] + ' NOT successful! #') uvfitsstatusarray[b] = False subs.param.add_param(self, 'transfer_input_beams_uvglue', uvgluestatusarray) subs.param.add_param(self, 'transfer_input_beams_uvfits', uvfitsstatusarray)
def spec_ex(self): ''' Extract spectrum at the coordinates of each source found by find_src_imsad IN Arrays stored in memory by load_src_csv IN cfga abs_ex_spec_format: .csv or .fits OUT Spectrum in .csv or .fits file format stored in abs/spec/ folder New line in abs_table.txt. Each line has the following information: Obs_ID, Beam, Source_ID, Ra, Dec, Peak Flux [Jy], r.m.s. spectrum ''' if self.abs_ex_spec_ex == True: self.logger.info( '### Extract spectra from the position of the peak of each continuum source ###' ) os.chdir(self.linedir) print os.getcwd() if os.path.exists(self.data_cube) == False: fits = lib.miriad('fits') fits.op = 'xyout' fits.in_ = self.data_cube_mir fits.out = self.data_cube fits.go(rmfiles=True) #open file hdulist = pyfits.open(self.data_cube) # read input # read data and header scidata = hdulist[0].data # reduce dimensions of the input cube if it his 4-dim sci = np.squeeze(scidata) prihdr = hdulist[0].header if (self.abs_ex_cube_zunit == 'frequency' or self.abs_ex_cube_zunit == 'freq' or self.abs_ex_cube_zunit == 'fr'): #convert frequencies in velocities freq = (np.linspace(1, sci.shape[0], sci.shape[0]) - prihdr['CRPIX3']) * prihdr['CDELT3'] + prihdr['CRVAL3'] v = C * ((HI - freq) / freq) freq0 = prihdr['CRVAL3'] freq_del = prihdr['CDELT3'] elif (self.abs_ex_cube_zunit == 'velocity' or self.abs_ex_cube_zunit == 'vel' or self.abs_ex_cube_zunit == 'v'): v = (np.linspace(1, sci.shape[0], sci.shape[0]) - prihdr['CRPIX3']) * prihdr['CDELT3'] + prihdr['CRVAL3'] v /= 1e3 freq = (C * HI) / (v + C) freq0 = (C * HI) / (prihdr['CRVAL3'] / 1e3 + C) freq_del = (freq0 - freq[-1]) / len(freq) #v_wrt_sys.append( C * ((HI - v_sys[j]) ) #find pixels where to extract the spectra self.coord_to_pix() # Load the list of coordinates in pixels self.abs_mean_rms = np.zeros(self.pixels.shape[0]) self.abs_los_rms = np.zeros(self.pixels.shape[0]) self.tau_los_rms = np.zeros(self.pixels.shape[0]) for i in xrange(0, self.pixels.shape[0]): # extract spectrum from each line of sight flux = np.zeros(freq.shape[0]) madfm = np.zeros(freq.shape[0]) pix_x_or = int(self.pixels[i][0]) pix_y_or = int(self.pixels[i][1]) if (0 < pix_x_or < prihdr['NAXIS1'] and 0 < pix_y_or < prihdr['NAXIS2']): for j in xrange(0, prihdr['NAXIS3']): #correct for chromatic aberration if self.abs_ex_chrom_aber == True: #depending if the cube is in velocity or frequency ? if freq_del <= 0.: scale = (freq0 + j * freq_del) / freq0 elif freq_del >= 0: scale = (freq0 - j * freq_del) / freq0 pix_x = (pix_x_or - prihdr['CRPIX1'] ) * scale + prihdr['CRPIX1'] pix_y = (pix_y_or - prihdr['CRPIX2'] ) * scale + prihdr['CRPIX2'] pix_x = int(round(pix_x, 0)) pix_y = int(round(pix_y, 0)) else: pix_x = pix_x_or pix_y = pix_y_or if (0 < pix_x < prihdr['NAXIS1'] and 0 < pix_y < prihdr['NAXIS2']): flux[j] = sci[j, pix_y, pix_x] else: flux[j] = 0.0 # determine the noise of the spectrum [Whiting 2012 et al.] in each channel # MADMF: median absolute deviation from the median # extract a region were to determine the noise: A BOX around the l.o.s. if (pix_x + 10 < prihdr['NAXIS1'] and pix_y + 5 < prihdr['NAXIS2'] and pix_y - 5 > 0): rms = np.nanmedian(sci[j, pix_x + 5:pix_x + 10, pix_y - 5:pix_y + 5]) med2 = abs(sci[j, pix_x, pix_y] - rms) madfm[j] = np.nanmedian(med2) / 0.6744888 else: madfm[j] = 0.0 self.abs_mean_rms[i] = np.nanmean(madfm) # measure noise in the spectrum outside of the line end_spec = float(sci.shape[0]) end_spec_th = int(end_spec / 3.) end_spec = int(end_spec) #print self.src_id[i], self.J2000_name[i], pix_x_or, pix_y_or, pix_x, pix_y tau = self.optical_depth(flux, self.peak_flux[i]) tau_noise = self.optical_depth(madfm, self.peak_flux[i]) #convert flux in optical depth self.abs_los_rms[i] = ( np.std(flux[0:end_spec_th]) + np.std(flux[end_spec - end_spec_th:end_spec])) / 2. self.tau_los_rms[i] = self.optical_depth( self.abs_los_rms[i], self.peak_flux[i]) self.logger.info('# Extracted spectrum of source ' + self.src_id[i] + ' ' + self.J2000_name[i] + ' #') #write spectrum tot = np.column_stack( (freq, v, flux, madfm, tau, tau_noise)) out_spec = str(self.specdir + self.src_id[i] + '_' + self.J2000_name[i]) #save spectrum if self.abs_ex_spec_format == 'csv': self.write_spec_csv(tot, out_spec) elif self.abs_ex_spec_format == 'fits': self.write_spec_fitstab(tot, out_spec) else: self.write_spec_csv(tot, out_spec) self.write_spec_fitstab(tot, out_spec) else: self.logger.warning( '# Source #' + self.src_id[i] + ' lies outside the fov of the data cube #') pass self.logger.info('### End of spectrum extraction ###') return 0
def flagline(self): ''' Creates an image cube of the different chunks and measures the rms in each channel. All channels with an rms outside of a given sigma interval are flagged in the continuum calibration, but are still used for line imaging. ''' if self.selfcal_flagline: subs.setinit.setinitdirs(self) subs.setinit.setdatasetnamestomiriad(self) self.logger.info( '### Automatic flagging of HI-line/RFI started ###') subs.managefiles.director(self, 'ch', self.selfcaldir) for chunk in self.list_chunks(): subs.managefiles.director(self, 'ch', self.selfcaldir + '/' + str(chunk)) self.logger.info('# Looking through data chunk ' + str(chunk) + ' #') invert = lib.miriad('invert') invert.vis = chunk + '.mir' invert.map = 'map' invert.beam = 'beam' invert.imsize = self.selfcal_image_imsize invert.cell = self.selfcal_image_cellsize invert.stokes = 'ii' invert.slop = 1 invert.go() if os.path.exists('map'): fits = lib.miriad('fits') fits.in_ = 'map' fits.op = 'xyout' fits.out = 'map.fits' fits.go() cube = pyfits.open('map.fits') data = cube[0].data std = np.nanstd(data, axis=(0, 2, 3)) median = np.median(std) stdall = np.nanstd(std) diff = std - median detections = np.where( np.abs(self.selfcal_flagline_sigma * diff) > stdall)[0] if len(detections) > 0: self.logger.info( '# Found high noise in channel(s) ' + str(detections).lstrip('[').rstrip(']') + ' #') for d in detections: uvflag = lib.miriad('uvflag') uvflag.vis = chunk + '.mir' uvflag.flagval = 'flag' uvflag.line = "'" + 'channel,1,' + str(d + 1) + "'" uvflag.go() self.logger.info( '# Flagged channel(s) ' + str(detections).lstrip('[').rstrip(']') + ' in data chunk ' + str(chunk) + ' #') else: self.logger.info( '# No high noise found in data chunk ' + str(chunk) + ' #') subs.managefiles.director( self, 'rm', self.selfcaldir + '/' + str(chunk) + '/' + 'map') subs.managefiles.director( self, 'rm', self.selfcaldir + '/' + str(chunk) + '/' + 'map.fits') subs.managefiles.director( self, 'rm', self.selfcaldir + '/' + str(chunk) + '/' + 'beam') else: self.logger.info('### No data in chunk ' + str(chunk) + '! ###') self.logger.info('### Automatic flagging of HI-line/RFI done ###')
def run_continuum_minoriteration(self, chunk, majc, minc, drmin, theoretical_noise_threshold): ''' Does a selfcal minor iteration for the standard mode chunk: The frequency chunk to image and calibrate maj: Current major iteration min: Current minor iteration drmin: maximum dynamic range for minor iteration theoretical_noise_threshold: calculated theoretical noise threshold ''' subs.setinit.setinitdirs(self) subs.setinit.setdatasetnamestomiriad(self) self.logger.info('# Minor self-calibration cycle ' + str(minc) + ' for frequency chunk ' + chunk + ' started #') if minc == 0: invert = lib.miriad('invert') # Create the dirty image invert.vis = chunk + '.mir' invert.map = str(majc).zfill(2) + '/map_' + str(minc).zfill(2) invert.beam = str(majc).zfill(2) + '/beam_' + str(minc).zfill(2) invert.imsize = self.selfcal_image_imsize invert.cell = self.selfcal_image_cellsize invert.stokes = 'ii' invert.options = 'mfs,double' invert.slop = 1 invert.robust = -2 invert.go() imax = self.calc_imax( str(majc).zfill(2) + '/map_' + str(minc).zfill(2)) noise_threshold = self.calc_noise_threshold(imax, minc, majc) dynamic_range_threshold = self.calc_dynamic_range_threshold( imax, drmin, self.selfcal_standard_minorcycle0_dr) mask_threshold, mask_threshold_type = self.calc_mask_threshold( theoretical_noise_threshold, noise_threshold, dynamic_range_threshold) self.logger.info('# Mask threshold for major/minor cycle ' + str(majc) + '/' + str(minc) + ' set to ' + str(mask_threshold) + ' Jy/beam #') self.logger.info('# Mask threshold set by ' + str(mask_threshold_type) + ' #') if majc == 0: maths = lib.miriad('maths') maths.out = str(majc).zfill(2) + '/mask_' + str(minc).zfill(2) maths.exp = '"<' + str(majc).zfill(2) + '/map_' + str( minc).zfill(2) + '>"' maths.mask = '"<' + str(majc).zfill(2) + '/map_' + str( minc).zfill(2) + '>.gt.' + str(mask_threshold) + '"' maths.go() self.logger.info('# Mask with threshold ' + str(mask_threshold) + ' Jy/beam created #') else: subs.managefiles.director( self, 'cp', str(majc).zfill(2) + '/mask_' + str(minc).zfill(2), file=str(majc - 1).zfill(2) + '/mask_' + str(self.selfcal_standard_minorcycle - 1).zfill(2)) self.logger.info( '# Mask from last minor iteration of last major cycle copied #' ) clean_cutoff = self.calc_clean_cutoff(mask_threshold) self.logger.info('# Clean threshold at major/minor cycle ' + str(majc) + '/' + str(minc) + ' was set to ' + str(clean_cutoff) + ' Jy/beam #') clean = lib.miriad( 'clean') # Clean the image down to the calculated threshold clean.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2) clean.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2) clean.out = str(majc).zfill(2) + '/model_' + str(minc).zfill(2) clean.cutoff = clean_cutoff clean.niters = 1000000 clean.region = '"' + 'mask(' + str(majc).zfill(2) + '/mask_' + str( minc).zfill(2) + ')' + '"' clean.go() self.logger.info('# Major/minor cycle ' + str(majc) + '/' + str(minc) + ' cleaning done #') restor = lib.miriad('restor') restor.model = str(majc).zfill(2) + '/model_' + str(minc).zfill(2) restor.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2) restor.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2) restor.out = str(majc).zfill(2) + '/image_' + str(minc).zfill(2) restor.mode = 'clean' restor.go() # Create the cleaned image self.logger.info('# Cleaned image for major/minor cycle ' + str(majc) + '/' + str(minc) + ' created #') restor.mode = 'residual' restor.out = str(majc).zfill(2) + '/residual_' + str(minc).zfill(2) restor.go() self.logger.info('# Residual image for major/minor cycle ' + str(majc) + '/' + str(minc) + ' created #') self.logger.info('# Peak of the residual image is ' + str( self.calc_imax( str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) + ' Jy/beam #') self.logger.info('# RMS of the residual image is ' + str( self.calc_irms( str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) + ' Jy/beam #') else: imax = self.calc_imax( str(majc).zfill(2) + '/map_' + str(0).zfill(2)) noise_threshold = self.calc_noise_threshold(imax, minc, majc) dynamic_range_threshold = self.calc_dynamic_range_threshold( imax, drmin, self.selfcal_standard_minorcycle0_dr) mask_threshold, mask_threshold_type = self.calc_mask_threshold( theoretical_noise_threshold, noise_threshold, dynamic_range_threshold) self.logger.info('# Mask threshold for major/minor cycle ' + str(majc) + '/' + str(minc) + ' set to ' + str(mask_threshold) + ' Jy/beam #') self.logger.info('# Mask threshold set by ' + str(mask_threshold_type) + ' #') maths = lib.miriad('maths') maths.out = str(majc).zfill(2) + '/mask_' + str(minc).zfill(2) maths.exp = '"<' + str(majc).zfill(2) + '/image_' + str( minc - 1).zfill(2) + '>"' maths.mask = '"<' + str(majc).zfill(2) + '/image_' + str( minc - 1).zfill(2) + '>.gt.' + str(mask_threshold) + '"' maths.go() self.logger.info('# Mask with threshold ' + str(mask_threshold) + ' Jy/beam created #') clean_cutoff = self.calc_clean_cutoff(mask_threshold) self.logger.info('# Clean threshold at major/minor cycle ' + str(majc) + '/' + str(minc) + ' was set to ' + str(clean_cutoff) + ' Jy/beam #') clean = lib.miriad( 'clean') # Clean the image down to the calculated threshold clean.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2) clean.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2) clean.model = str(majc).zfill(2) + '/model_' + str(minc - 1).zfill(2) clean.out = str(majc).zfill(2) + '/model_' + str(minc).zfill(2) clean.cutoff = clean_cutoff clean.niters = 1000000 clean.region = '"' + 'mask(' + str(majc).zfill(2) + '/mask_' + str( minc).zfill(2) + ')' + '"' clean.go() self.logger.info('# Major/minor cycle ' + str(majc) + '/' + str(minc) + ' cleaning done #') restor = lib.miriad('restor') restor.model = str(majc).zfill(2) + '/model_' + str(minc).zfill(2) restor.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2) restor.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2) restor.out = str(majc).zfill(2) + '/image_' + str(minc).zfill(2) restor.mode = 'clean' restor.go() # Create the cleaned image self.logger.info('# Cleaned image for major/minor cycle ' + str(majc) + '/' + str(minc) + ' created #') restor.mode = 'residual' restor.out = str(majc).zfill(2) + '/residual_' + str(minc).zfill(2) restor.go() self.logger.info('# Residual image for major/minor cycle ' + str(majc) + '/' + str(minc) + ' created #') self.logger.info('# Peak of the residual image is ' + str( self.calc_imax( str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) + ' Jy/beam #') self.logger.info('# RMS of the residual image is ' + str( self.calc_irms( str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) + ' Jy/beam #') self.logger.info('# Minor self-calibration cycle ' + str(minc) + ' for frequency chunk ' + chunk + ' finished #')
def ms2miriad(self): ''' Converts the data from MS to MIRIAD format via UVFITS using drivecasa. Does it for the flux calibrator, polarisation calibrator, and target field independently. ''' subs.setinit.setinitdirs(self) beams = 37 # Create the parameters for the parameter file for converting from MS to UVFITS format convertfluxcalmsavailable = get_param_def(self, 'convert_fluxcal_MSavailable', False ) # Flux calibrator MS dataset available? convertpolcalmsavailable = get_param_def(self, 'convert_polcal_MSavailable', False ) # Polarised calibrator MS dataset available? converttargetbeamsmsavailable = get_param_def(self, 'convert_targetbeams_MSavailable', np.full((beams), False) ) # Target beam MS dataset available? convertfluxcalms2uvfits = get_param_def(self, 'convert_fluxcal_MS2UVFITS', False ) # Flux calibrator MS dataset converted to UVFITS? convertpolcalms2uvfits = get_param_def(self, 'convert_polcal_MS2UVFITS', False ) # Polarised calibrator MS dataset converted to UVFITS? converttargetbeamsms2uvfits = get_param_def(self, 'convert_targetbeams_MS2UVFITS', np.full((beams), False) ) # Target beam MS dataset converted to UVFITS? convertfluxcaluvfitsavailable = get_param_def(self, 'convert_fluxcal_UVFITSavailable', False ) # Flux calibrator UVFITS dataset available? convertpolcaluvfitsavailable = get_param_def(self, 'convert_polcal_UVFITSavailable', False ) # Polarised calibrator UVFITS dataset available? converttargetbeamsuvfitsavailable = get_param_def(self, 'convert_targetbeams_UVFITSavailable', np.full((beams), False) ) # Target beam UVFITS dataset available? convertfluxcaluvfits2miriad = get_param_def(self, 'convert_fluxcal_UVFITS2MIRIAD', False ) # Flux calibrator UVFITS dataset converted to MIRIAD? convertpolcaluvfits2miriad = get_param_def(self, 'convert_polcal_UVFITS2MIRIAD', False ) # Polarised calibrator UVFITS dataset converted to MIRIAD? converttargetbeamsuvfits2miriad = get_param_def(self, 'convert_targetbeams_UVFITS2MIRIAD', np.full((beams), False) ) # Target beam UVFITS dataset converted to MIRIAD? ################################################### # Check which datasets are available in MS format # ################################################### if self.fluxcal != '': convertfluxcalmsavailable = os.path.isdir(self.basedir + '00' + '/' + self.rawsubdir + '/' + self.fluxcal) else: self.logger.warning('# Flux calibrator dataset not specified. Cannot convert flux calibrator! #') if self.polcal != '': convertpolcalmsavailable = os.path.isdir(self.basedir + '00' + '/' + self.rawsubdir + '/' + self.polcal) else: self.logger.warning('# Polarised calibrator dataset not specified. Cannot convert polarised calibrator! #') if self.target != '': for b in range(beams): converttargetbeamsmsavailable[b] = os.path.isdir(self.basedir + str(b).zfill(2) + '/' + self.rawsubdir + '/' + self.target) else: self.logger.warning('# Target beam dataset not specified. Cannot convert target beams! #') # Save the derived parameters for the availability to the parameter file subs.param.add_param(self, 'convert_fluxcal_MSavailable', convertfluxcalmsavailable) subs.param.add_param(self, 'convert_polcal_MSavailable', convertpolcalmsavailable) subs.param.add_param(self, 'convert_targetbeams_MSavailable', converttargetbeamsmsavailable) ############################################### # Convert the available MS-datasets to UVFITS # ############################################### # Convert the flux calibrator if self.convert_fluxcal: if self.fluxcal != '': if convertfluxcaluvfits2miriad == False: if convertfluxcalmsavailable: self.logger.debug('# Converting flux calibrator dataset from MS to UVFITS format. #') subs.managefiles.director(self, 'mk', self.basedir + '00' + '/' + self.crosscalsubdir, verbose=False) fc_convert = 'exportuvfits(vis="' + self.basedir + '00' + '/' + self.rawsubdir + '/' + self.fluxcal + '", fitsfile="' + self.basedir + '00' + '/' + self.crosscalsubdir + '/' + str(self.fluxcal).rstrip('MS') + 'UVFITS' + '", datacolumn="data", combinespw=True, padwithflags=True, multisource=True, writestation=True)' casacmd = [fc_convert] casa = drivecasa.Casapy() casa.run_script(casacmd, raise_on_severe=False, timeout=3600) if os.path.isfile( self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip('MS') + 'UVFITS'): convertfluxcalms2uvfits = True self.logger.info('# Converted flux calibrator dataset from MS to UVFITS format! #') else: convertfluxcalms2uvfits = False self.logger.warning('# Could not convert flux calibrator dataset from MS to UVFITS format! #') else: self.logger.warning('# Flux calibrator dataset not available! #') else: self.logger.info('# Flux calibrator dataset was already converted from MS to UVFITS format #') else: self.logger.warning('# Flux calibrator dataset not specified. Cannot convert flux calibrator! #') else: self.logger.warning('# Not converting flux calibrator dataset! #') # Convert the polarised calibrator if self.convert_polcal: if self.polcal != '': if convertpolcaluvfits2miriad == False: if convertpolcalmsavailable: self.logger.debug('# Converting polarised calibrator dataset from MS to UVFITS format. #') subs.managefiles.director(self, 'mk', self.basedir + '00' + '/' + self.crosscalsubdir, verbose=False) pc_convert = 'exportuvfits(vis="' + self.basedir + '00' + '/' + self.rawsubdir + '/' + self.polcal + '", fitsfile="' + self.basedir + '00' + '/' + self.crosscalsubdir + '/' + str(self.polcal).rstrip('MS') + 'UVFITS' + '", datacolumn="data", combinespw=True, padwithflags=True, multisource=True, writestation=True)' casacmd = [pc_convert] casa = drivecasa.Casapy() casa.run_script(casacmd, raise_on_severe=False, timeout=3600) if os.path.isfile( self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip('MS') + 'UVFITS'): convertpolcalms2uvfits = True self.logger.info('# Converted polarised calibrator dataset from MS to UVFITS format! #') else: convertpolcalms2uvfits = False self.logger.warning('# Could not convert polarised calibrator dataset from MS to UVFITS format! #') else: self.logger.warning('# Polarised calibrator dataset not available! #') else: self.logger.info('# Polarised calibrator dataset was already converted from MS to UVFITS format #') else: self.logger.warning('# Polarised calibrator dataset not specified. Cannot convert polarised calibrator! #') else: self.logger.warning('# Not converting polarised calibrator dataset! #') # Convert the target beams if self.convert_target: if self.target != '': self.logger.info('# Converting target beam datasets from MS to UVFITS format. #') if self.convert_targetbeams == 'all': datasets = glob.glob(self.basedir + '[0-9][0-9]' + '/' + self.rawsubdir + '/' + self.target) self.logger.debug('# Converting all available target beam datasets #') else: beams = self.convert_targetbeams.split(",") datasets = [self.basedir + str(b).zfill(2) + '/' + self.rawsubdir + '/' + self.target for b in beams] self.logger.debug('# Converting all selected target beam datasets #') for vis in datasets: if converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])] == False: if converttargetbeamsmsavailable[int(vis.split('/')[-3])]: subs.managefiles.director(self, 'mk', self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir, verbose=False) tg_convert = 'exportuvfits(vis="' + self.basedir + vis.split('/')[-3] + '/' + self.rawsubdir + '/' + self.target + '", fitsfile="' + self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS' + '", datacolumn="data", combinespw=True, padwithflags=True, multisource=True, writestation=True)' casacmd = [tg_convert] casa = drivecasa.Casapy() casa.run_script(casacmd, raise_on_severe=False, timeout=7200) if os.path.isfile( self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS'): converttargetbeamsms2uvfits[int(vis.split('/')[-3])] = True self.logger.debug('# Converted dataset of target beam ' + vis.split('/')[-3] + ' from MS to UVFITS format! #') else: converttargetbeamsms2uvfits[int(vis.split('/')[-3])] = False self.logger.warning('# Could not convert dataset for target beam ' + vis.split('/')[-3] + ' from MS to UVFITS format! #') else: self.logger.warning('# Dataset for target beam ' + vis.split('/')[-3] + ' not available! #') else: self.logger.info('# Dataset for target beam ' + vis.split('/')[-3] + ' was already converted from MS to UVFITS format #') else: self.logger.warning('# Target beam dataset(s) not specified. Cannot convert target beam datasets! #') else: self.logger.warning('# Not converting target beam dataset(s)! #') # Save the derived parameters for the MS to UVFITS conversion to the parameter file subs.param.add_param(self, 'convert_fluxcal_MS2UVFITS', convertfluxcalms2uvfits) subs.param.add_param(self, 'convert_polcal_MS2UVFITS', convertpolcalms2uvfits) subs.param.add_param(self, 'convert_targetbeams_MS2UVFITS', converttargetbeamsms2uvfits) ####################################################### # Check which datasets are available in UVFITS format # ####################################################### if self.fluxcal != '': convertfluxcaluvfitsavailable = os.path.isfile(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip('MS') + 'UVFITS') else: self.logger.warning('# Flux calibrator dataset not specified. Cannot convert flux calibrator! #') if self.polcal != '': convertpolcaluvfitsavailable = os.path.isfile(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip('MS') + 'UVFITS') else: self.logger.warning('# Polarised calibrator dataset not specified. Cannot convert polarised calibrator! #') if self.target != '': for b in range(beams): converttargetbeamsuvfitsavailable[b] = os.path.isfile(self.basedir + str(b).zfill(2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS') else: self.logger.warning('# Target beam dataset not specified. Cannot convert target beams! #') # Save the derived parameters for the availability to the parameter file subs.param.add_param(self, 'convert_fluxcal_UVFITSavailable', convertfluxcaluvfitsavailable) subs.param.add_param(self, 'convert_polcal_UVFITSavailable', convertpolcaluvfitsavailable) subs.param.add_param(self, 'convert_targetbeams_UVFITSavailable', converttargetbeamsuvfitsavailable) ########################################################## # Convert the available UVFITS-datasets to MIRIAD format # ########################################################## # Convert the flux calibrator if self.convert_fluxcal: if self.fluxcal != '': if convertfluxcaluvfits2miriad == False: if convertfluxcaluvfitsavailable: self.logger.debug('# Converting flux calibrator dataset from UVFITS to MIRIAD format. #') subs.managefiles.director(self, 'ch', self.basedir + '00' + '/' + self.crosscalsubdir, verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip('MS') + 'UVFITS' fits.out = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip('MS') + 'mir' fits.go() if os.path.isdir( self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip('MS') + 'mir'): convertfluxcaluvfits2miriad = True self.logger.info('# Converted flux calibrator dataset from UVFITS to MIRIAD format! #') else: convertfluxcaluvfits2miriad = False self.logger.warning('# Could not convert flux calibrator dataset from UVFITS to MIRIAD format! #') else: self.logger.warning('# Flux calibrator dataset not available! #') else: self.logger.info('# Flux calibrator dataset was already converted from UVFITS to MIRIAD format #') else: self.logger.warning('# Flux calibrator dataset not specified. Cannot convert flux calibrator! #') else: self.logger.warning('# Not converting flux calibrator dataset! #') # Convert the polarised calibrator if self.convert_polcal: if self.polcal != '': if convertpolcaluvfits2miriad == False: if convertpolcaluvfitsavailable: self.logger.debug('# Converting polarised calibrator dataset from UVFITS to MIRIAD format. #') subs.managefiles.director(self, 'ch',self.basedir + '00' + '/' + self.crosscalsubdir, verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip('MS') + 'UVFITS' fits.out = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip('MS') + 'mir' fits.go() if os.path.isdir(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip('MS') + 'mir'): convertpolcaluvfits2miriad = True self.logger.info('# Converted polarised calibrator dataset from UVFITS to MIRIAD format! #') else: convertpolcaluvfits2miriad = False self.logger.warning('# Could not convert polarised calibrator dataset from UVFITS to MIRIAD format! #') else: self.logger.warning('# Polarised calibrator dataset not available! #') else: self.logger.info('# Polarised calibrator dataset was already converted from UVFITS to MIRIAD format #') else: self.logger.warning('# Polarised calibrator dataset not specified. Cannot convert polarised calibrator! #') else: self.logger.warning('# Not converting polarised calibrator dataset! #') # Convert the target beams if self.convert_target: if self.target != '': self.logger.info('# Converting target beam datasets from UVFITS to MIRIAD format. #') if self.convert_targetbeams == 'all': datasets = glob.glob(self.basedir + '[0-9][0-9]' + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS') self.logger.debug('# Converting all available target beam datasets #') else: beams = self.convert_targetbeams.split(",") datasets = [self.basedir + str(b).zfill(2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS' for b in beams] self.logger.debug('# Converting all selected target beam datasets #') for vis in datasets: if converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])] == False: if converttargetbeamsuvfitsavailable[int(vis.split('/')[-3])]: subs.managefiles.director(self, 'ch', self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir, verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS' fits.out = self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'mir' fits.go() if os.path.isdir( self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'mir'): converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])] = True self.logger.debug('# Converted dataset of target beam ' + vis.split('/')[-3] + ' from UVFITS to MIRIAD format! #') else: converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])] = False self.logger.warning('# Could not convert dataset for target beam ' + vis.split('/')[-3] + ' from UVFITS to MIRIAD format! #') else: self.logger.warning('# Dataset for target beam ' + vis.split('/')[-3] + ' not available! #') else: self.logger.info('# Dataset for target beam ' + vis.split('/')[-3] + ' was already converted from MS to UVFITS format #') else: self.logger.warning('# Target beam dataset(s) not specified. Cannot convert target beam datasets! #') else: self.logger.warning('# Not converting target beam dataset(s)! #') # Save the derived parameters for the MS to UVFITS conversion to the parameter file subs.param.add_param(self, 'convert_fluxcal_UVFITS2MIRIAD', convertfluxcaluvfits2miriad) subs.param.add_param(self, 'convert_polcal_UVFITS2MIRIAD', convertpolcaluvfits2miriad) subs.param.add_param(self, 'convert_targetbeams_UVFITS2MIRIAD', converttargetbeamsuvfits2miriad) ##################################### # Remove the UVFITS files if wanted # ##################################### if self.convert_removeuvfits: self.logger.info('# Removing all UVFITS files #') subs.managefiles.director(self, 'rm', self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip('MS') + 'UVFITS') subs.managefiles.director(self, 'rm', self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip('MS') + 'UVFITS') for beam in range(beams): if os.path.isdir(self.basedir + str(beam).zfill(2) + '/' + self.crosscalsubdir): subs.managefiles.director(self, 'rm', self.basedir + str(beam).zfill(2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS') else: pass
def polarisation(self): ''' Derives the polarisation corrections (leakage, angle) from the polarised calibrator. Uses the bandpass from the bandpass calibrator. Does not account for freqeuncy dependent solutions at the moment. ''' if self.crosscal_polarisation: subs.setinit.setinitdirs(self) subs.setinit.setdatasetnamestomiriad(self) subs.managefiles.director(self, 'ch', self.crosscaldir) self.logger.info( '### Polarisation calibration on the polarised calibrator data started ###' ) if os.path.isfile(self.crosscaldir + '/' + self.fluxcal + '/' + 'bandpass'): self.logger.info( '# Bandpass solutions in flux calibrator data found. Using them! #' ) gpcopy = lib.miriad('gpcopy') gpcopy.vis = self.fluxcal gpcopy.out = self.polcal gpcopy.mode = 'copy' gpcopy.options = 'nopol,relax' gpcopy.go() self.logger.info( '# Bandpass from flux calibrator data copied to polarised calibrator data #' ) gpcal = lib.miriad('gpcal') gpcal.vis = self.polcal # uv = aipy.miriad.UV(self.polcal) # nchan = uv['nchan'] # gpcal.nfbin = round(nchan / self.crosscal_polarisation_nchan) gpcal.options = 'xyvary,linear' gpcal.go() self.logger.info( '# Solved for polarisation leakage and angle on polarised calibrator #' ) else: self.logger.info( '# Bandpass solutions from flux calibrator not found #') self.logger.info( '# Deriving bandpass from polarised calibrator using mfcal #' ) mfcal = lib.miriad('mfcal') mfcal.vis = self.polcal mfcal.go() self.logger.info( '# Bandpass solutions from polarised calibrator derived #') self.logger.info( '# Continuing with polarisation calibration (leakage, angle) from polarised calibrator data #' ) gpcal = lib.miriad('gpcal') gpcal.vis = self.polcal # uv = aipy.miriad.UV(self.polcal) # nchan = uv['nchan'] # gpcal.nfbin = round(nchan / self.crosscal_polarisation_nchan) gpcal.options = 'xyvary,linear' gpcal.go() self.logger.info( '# Solved for polarisation leakage and angle on polarised calibrator #' ) self.logger.info( '### Polarisation calibration on the polarised calibrator data done ###' ) else: self.logger.info('### No polarisation calibration done! ###')
#Will create almost empty configuration files for both cfg1 = '/home/adams/apertif/AAF/180216005.cfg' cfg2 = '/home/adams/apertif/AAF/180302009.cfg' #Now in miriad need to use uvimage to create image cube #Want to limit number of channels #Want to include near 9330, 20 subbands #Subbands start at multiples of 64 #take subbands 130-149 #channels 8320, 9599 #move to directory where data is ccal = apercal.ccal(cfg1) ccal.director('ch', ccal.crosscaldir) uvimage = lib.miriad('uvimage') uvimage.vis = ccal.target + 'mir' #uvaver.out uvimage.view = 'amplitude' targetname = ccal.target uvimage.out = targetname + '_amp_freq.im' #still too much data so also select a time range #parentheses in command so need double quote uvimage.select = "'-auto,time(11:00:00,12:00:00)'" uvimage.line = 'channel,1280,8320,1' uvimage.options = 'freq' uvimage.mode = 3 uvimage.go() #write to fits file fits = lib.miriad('fits') fits.in_ = uvimage.out
def splitdata(self): ''' Applies calibrator corrections to data, splits the data into chunks in frequency and bins it to the given frequency resolution for the self-calibration ''' if self.splitdata: subs.setinit.setinitdirs(self) subs.setinit.setdatasetnamestomiriad(self) subs.managefiles.director(self, 'ch', self.selfcaldir) self.logger.info( '### Splitting of target data into individual frequency chunks started ###' ) if os.path.isfile(self.selfcaldir + '/' + self.target): self.logger.info( '# Calibrator corrections already seem to have been applied #' ) else: self.logger.info( '# Applying calibrator solutions to target data before averaging #' ) uvaver = lib.miriad('uvaver') uvaver.vis = self.crosscaldir + '/' + self.target uvaver.out = self.selfcaldir + '/' + self.target uvaver.go() self.logger.info( '# Calibrator solutions to target data applied #') if self.selfcal_flagantenna != '': uvflag = lib.miriad('uvflag') uvflag.vis = self.selfcaldir + '/' + self.target uvflag.flagval = 'flag' uvflag.select = 'antenna(' + str( self.selfcal_flagantenna) + ')' uvflag.go() else: pass try: uv = aipy.miriad.UV(self.selfcaldir + '/' + self.target) except RuntimeError: self.logger.error( '### No data in your crosscal directory! Exiting pipeline! ###' ) sys.exit(1) try: nsubband = len(uv['nschan']) # Number of subbands in data except TypeError: nsubband = 1 # Only one subband in data since exception was triggered self.logger.info('# Found ' + str(nsubband) + ' subband(s) in target data #') counter = 0 # Counter for naming the chunks and directories for subband in range(nsubband): self.logger.info('# Started splitting of subband ' + str(subband) + ' #') if nsubband == 1: numchan = uv['nschan'] finc = np.fabs(uv['sdf']) else: numchan = uv['nschan'][ subband] # Number of channels per subband finc = np.fabs( uv['sdf'] [subband]) # Frequency increment for each channel subband_bw = numchan * finc # Bandwidth of one subband subband_chunks = round(subband_bw / self.selfcal_splitdata_chunkbandwidth) subband_chunks = int( np.power(2, np.ceil(np.log(subband_chunks) / np.log(2))) ) # Round to the closest power of 2 for frequency chunks with the same bandwidth over the frequency range of a subband if subband_chunks == 0: subband_chunks = 1 chunkbandwidth = (numchan / subband_chunks) * finc self.logger.info( '# Adjusting chunk size to ' + str(chunkbandwidth) + ' GHz for regular gridding of the data chunks over frequency #' ) for chunk in range(subband_chunks): self.logger.info('# Starting splitting of data chunk ' + str(chunk) + ' for subband ' + str(subband) + ' #') binchan = round( self.selfcal_splitdata_channelbandwidth / finc) # Number of channels per frequency bin chan_per_chunk = numchan / subband_chunks if chan_per_chunk % binchan == 0: # Check if the freqeuncy bin exactly fits self.logger.info( '# Using frequency binning of ' + str(self.selfcal_splitdata_channelbandwidth) + ' for all subbands #') else: while chan_per_chunk % binchan != 0: # Increase the frequency bin to keep a regular grid for the chunks binchan = binchan + 1 else: if chan_per_chunk >= binchan: # Check if the calculated bin is not larger than the subband channel number pass else: binchan = chan_per_chunk # Set the frequency bin to the number of channels in the chunk of the subband self.logger.info( '# Increasing frequency bin of data chunk ' + str(chunk) + ' to keep bandwidth of chunks equal over the whole bandwidth #' ) self.logger.info('# New frequency bin is ' + str(binchan * finc) + ' GHz #') nchan = int( chan_per_chunk / binchan) # Total number of output channels per chunk start = 1 + chunk * chan_per_chunk width = int(binchan) step = int(width) subs.managefiles.director( self, 'mk', self.selfcaldir + '/' + str(counter).zfill(2)) uvaver = lib.miriad('uvaver') uvaver.vis = self.selfcaldir + '/' + self.target uvaver.out = self.selfcaldir + '/' + str(counter).zfill( 2) + '/' + str(counter).zfill(2) + '.mir' uvaver.select = "'" + 'window(' + str(subband + 1) + ')' + "'" uvaver.line = "'" + 'channel,' + str(nchan) + ',' + str( start) + ',' + str(width) + ',' + str(step) + "'" uvaver.go() counter = counter + 1 self.logger.info('# Splitting of data chunk ' + str(chunk) + ' for subband ' + str(subband) + ' done #') self.logger.info('# Splitting of data for subband ' + str(subband) + ' done #') self.logger.info( '### Splitting of target data into individual frequency chunks done ###' )
def parametric(self): ''' Parametric self calibration using an NVSS/FIRST skymodel and calculating spectral indices by source matching with WENSS. ''' if self.selfcal_parametric: subs.setinit.setinitdirs(self) subs.setinit.setdatasetnamestomiriad(self) self.logger.info('### Doing parametric self calibration ###') subs.managefiles.director(self, 'ch', self.selfcaldir) for chunk in self.list_chunks(): self.logger.info( '# Starting parametric self calibration routine on chunk ' + chunk + ' #') subs.managefiles.director(self, 'ch', self.selfcaldir + '/' + chunk) subs.managefiles.director( self, 'mk', self.selfcaldir + '/' + chunk + '/' + 'pm') parametric_textfile = lsm.lsm_model( chunk + '.mir', self.selfcal_parametric_skymodel_radius, self.selfcal_parametric_skymodel_cutoff, self.selfcal_parametric_skymodel_distance) lsm.write_model( self.selfcaldir + '/' + chunk + '/' + 'pm' + '/model.txt', parametric_textfile) self.logger.info( '# Creating model from textfile model.txt for chunk ' + chunk + ' #') uv = aipy.miriad.UV(self.selfcaldir + '/' + chunk + '/' + chunk + '.mir') freq = uv['sfreq'] uvmodel = lib.miriad('uvmodel') uvmodel.vis = chunk + '.mir' parametric_modelfile = open( self.selfcaldir + '/' + str(chunk) + '/' + 'pm' + '/model.txt', 'r') for n, source in enumerate(parametric_modelfile.readlines()): if n == 0: uvmodel.options = 'replace,mfs' else: uvmodel.options = 'add,mfs' uvmodel.offset = source.split(',')[0] + ',' + source.split( ',')[1] uvmodel.flux = source.split(',')[2] + ',i,' + str( freq) + ',' + source.split(',')[4].rstrip( '\n') + ',0,0' uvmodel.out = 'pm/tmp' + str(n) uvmodel.go() uvmodel.vis = uvmodel.out subs.managefiles.director( self, 'rn', 'pm/model', uvmodel.out) # Rename the last modelfile to model subs.managefiles.director( self, 'rm', 'pm/tmp*') # Remove all the obsolete modelfiles self.logger.info( '# Doing parametric self-calibration on chunk ' + chunk + ' with solution interval ' + str(self.selfcal_parametric_solint) + ' min and uvrange limits of ' + str(self.selfcal_parametric_uvmin) + '~' + str(self.selfcal_parametric_uvmax) + ' klambda #') selfcal = lib.miriad('selfcal') selfcal.vis = chunk + '.mir' selfcal.model = 'pm/model' selfcal.interval = self.selfcal_parametric_solint selfcal.select = "'" + 'uvrange(' + str( self.selfcal_parametric_uvmin) + ',' + str( self.selfcal_parametric_uvmax) + ')' + "'" # Choose reference antenna if given if self.selfcal_refant == '': pass else: selfcal.refant = self.selfcal_refant # Do amplitude calibration if wanted if self.selfcal_parametric_amp: selfcal.options = 'mfs,amp' else: selfcal.options = 'mfs' selfcal.go() self.logger.info( '# Parametric self calibration routine on chunk ' + chunk + ' done! #') self.logger.info('### Parametric self calibration done ###') else: self.logger.info('### Parametric self calibration disabled ###')