def calc_isum(self, image): """ Function to calculate the sum of the values of the pixels in an image image (string): The name of the image file. Must be in MIRIAD-format returns (float): the sum of the pxiels in the image """ fits = lib.miriad('fits') fits.op = 'xyout' fits.in_ = image fits.out = image + '.fits' fits.go() image_data = pyfits.open(image + '.fits') # Open the image data = image_data[0].data isum = np.nansum(data) # Get the maximum image_data.close() # Close the image subs_managefiles.director(self, 'rm', image + '.fits') return isum
def getimagestats(self, image): """ Subroutine to calculate the min, max and rms of an image image (string): The absolute path to the image file. returns (numpy array): The min, max and rms of the image """ setinit.setinitdirs(self) char_set = string.ascii_uppercase + string.digits if os.path.isdir(image) or os.path.isfile(image): if os.path.isdir(image): temp_string = ''.join(random.sample(char_set * 8, 8)) fits = lib.miriad('fits') fits.op = 'xyout' fits.in_ = image with tempfile.TemporaryDirectory() as tempdir: fits.out = tempdir + '/' + temp_string + '.fits' fits.go() image_data = pyfits.open(tempdir + '/' + temp_string + '.fits') elif os.path.isfile(image): image_data = pyfits.open(image) else: error = 'Image format not supported. Only MIRIAD and FITS formats are supported!' logger.error(error) raise ApercalException(error) data = image_data[0].data imagestats = np.full(3, np.nan) if data.shape[-3] == 2: imagestats[0] = np.nanmin( data[0, 0, :, :]) # Get the maxmimum of the image imagestats[1] = np.nanmax( data[0, 0, :, :]) # Get the minimum of the image imagestats[2] = np.nanstd( data[0, 0, :, :]) # Get the standard deviation else: imagestats[0] = np.nanmin(data) # Get the maxmimum of the image imagestats[1] = np.nanmax(data) # Get the minimum of the image imagestats[2] = np.nanstd(data) # Get the standard deviation image_data.close() # Close the image else: error = 'Image does not seem to exist!' logger.error(error) raise ApercalException(error) return imagestats
def imagetofits(self, mirimage, fitsimage, remove=True): """ Converts a MIRIAD image to a FITS image mirimage: The MIRIAD image to convert fitsimage: The converted FITS image """ subs_setinit.setinitdirs(self) fits = lib.miriad('fits') fits.op = 'xyout' fits.in_ = mirimage fits.out = fitsimage fits.go() if os.path.isfile(fitsimage) and remove: director(self, 'rm', mirimage) elif os.path.isfile(fitsimage) and remove == False: pass else: pass
def get_gains(file_): """ Function to create a complex python array of amplitude and phase gains from a dataset file_ (str): u,v file with the bandpass calibration return(array, array): an array with the amplitude and phase gains for each antenna and solution interval, a datetime array with the actual solution timesteps """ char_set = string.ascii_uppercase + string.digits # Create a charset for random gain log file generation tempdir = subs.managetmp.manage_tempdir('mirlog') gains_string = ''.join(random.sample(char_set * 8, 8)) gpplt = lib.miriad('gpplt') gpplt.vis = file_ gpplt.log = tempdir + '/' + gains_string gpplt.options = 'gains' gpplt.yaxis = 'amp' cmd = gpplt.go() check_table(tempdir + '/' + gains_string, 'gains') obsdate = cmd[3].split(' ')[4][0:7] starttime = datetime.datetime( int('20' + obsdate[0:2]), int(time.strptime(obsdate[2:5], '%b').tm_mon), int(obsdate[5:7])) s = Table.read(tempdir + '/' + gains_string, format='ascii') gpplt.yaxis = 'phase' gpplt.go() check_table(tempdir + '/' + gains_string, 'gains') t = Table.read(tempdir + '/' + gains_string, format='ascii') days = np.array(t['col1']) times = np.array(t['col2']) nint = len(days) nant = int(len(t[0]) - 2) gain_array = np.zeros((nant, nint, 2)) for ant in range(nant): gain_array[ant, :, 0] = s['col' + str(ant + 3)] # gain_array[ant, :, 1] = t['col' + str(ant + 3)] gain_array[ant, :, 1] = np.unwrap(t['col' + str(ant + 3)], discont=90) time_array = [ starttime + datetime.timedelta(days=int(days[step]), hours=int(times[step][0:2]), minutes=int(times[step][3:5]), seconds=int(times[step][6:8])) for step in range(nint) ] return gain_array, time_array
def get_ndims(file_): """ Wrapper funtion to get the dimension of the selfcal gain file file_ (str): u,v file with the gain calibration returns (int, int, int): Number of antennas, number of frequency bins, number of time intervals """ char_set = string.ascii_uppercase + string.digits # Create a charset for random gain log file generation tempdir = subs.managetmp.manage_tempdir('mirlog') subs.managetmp.clean_tempdir('mirlog') gains_string = ''.join(random.sample(char_set * 8, 8)) gpplt = lib.miriad('gpplt') gpplt.vis = file_ gpplt.log = tempdir + '/' + gains_string gpplt.yaxis = 'amp' cmd = gpplt.go() nants = get_nants(tempdir + '/' + gains_string) nbins = get_nbins(tempdir + '/' + gains_string) nsols = get_nsols(tempdir + '/' + gains_string) return nants, nbins, nsols
def fixheader(beamname, beamdir): """ Generation of Gaussian beam maps results in the Gaussian parameters included as the "beam" This function removes those values Args: beamname (str): name of beam map beamdir (str): location of beam map """ # load function to delete header information delhd = lib.miriad('delhd') # delete bmaj delhd.in_ = '{0}/{1}/bmaj'.format(beamdir, beamname) delhd.go() # delete bmin delhd.in_ = '{0}/{1}/bmin'.format(beamdir, beamname) delhd.go() # delete bmpa delhd.in_ = '{0}/{1}/bpa'.format(beamdir, beamname) delhd.go()
def get_phases(file_): """ Function to create a python array of selfcal phase gains from a dataset file_ (str): u,v file with the gain calibration return(array, array): an array with the phase gains for each antenna, frequency bin and solution interval, a datetime array with the actual solution timesteps """ char_set = string.ascii_uppercase + string.digits # Create a charset for random gain log file generation tempdir = subs.managetmp.manage_tempdir('mirlog') subs.managetmp.clean_tempdir('mirlog') gains_string = ''.join(random.sample(char_set * 8, 8)) gpplt = lib.miriad('gpplt') gpplt.vis = file_ gpplt.log = tempdir + '/' + gains_string gpplt.yaxis = 'phase' cmd = gpplt.go() nants = get_nants(tempdir + '/' + gains_string) nbins = get_nbins(tempdir + '/' + gains_string) nsols = get_nsols(tempdir + '/' + gains_string) reformat_table(tempdir + '/' + gains_string) obsdate = cmd[3].split(' ')[4][0:7] starttime = datetime.datetime( int('20' + obsdate[0:2]), int(time.strptime(obsdate[2:5], '%b').tm_mon), int(obsdate[5:7])) s = Table.read(tempdir + '/' + gains_string, format='ascii') days = np.array(s['col1'][0:nsols]) times = np.array(s['col2'][0:nsols]) time_array = [ starttime + datetime.timedelta(days=int(days[step]), hours=int(times[step][0:2]), minutes=int(times[step][3:5]), seconds=int(times[step][6:8])) for step in range(nsols) ] gain_array = np.zeros([nants, nbins, nsols]) for ant in range(nants): gain_array[ant, :, :] = np.reshape(s['col' + str(ant + 3)][nsols:], (1, nbins, nsols)) return gain_array, time_array
def beam_cutoff(beamname, tmpbeamname, beamdir, cutoff): """ Function to apply a beam cutoff value Args: beamname (str): final name for beam map tmpbeamname (str): name of temporary map, to be deleted beamdir (str): output directory for beam map cutoff (float): relative level at which beam map should be cut off """ # load maths maths = lib.miriad('maths') # expression is just beam map to have cutoff value applied # Use brackets to guard against formatting issues maths.exp = "'<{0}/{1}>'".format(beamdir, tmpbeamname) # Apply cutoff value as mask, using miriad formatting, brackets around image name maths.mask = "'<{0}/{1}>.gt.{2}'".format(beamdir, tmpbeamname, cutoff) maths.out = '{0}/{1}'.format(beamdir, beamname) # run maths.go() # clean up temp file os.system('rm -rf {0}/{1}'.format(beamdir, tmpbeamname))
def get_delays(file_): """ Function to create a numpy array with the antenna delays for each solution interval file_ (str): u,v file with the bandpass calibration return(array, array): an array with the delays for each antenna and solution interval in nsec, a datetime array with the actual solution timesteps """ char_set = string.ascii_uppercase + string.digits # Create a charset for random gain log file_ generation tempdir = subs.managetmp.manage_tempdir('mirlog') # tempdir = os.path.expanduser('~') + '/apercal/temp/mirlog' gains_string = ''.join(random.sample(char_set * 8, 8)) gpplt = lib.miriad('gpplt') gpplt.vis = file_ gpplt.log = tempdir + '/' + gains_string gpplt.options = 'delays' cmd = gpplt.go() reformat_table(tempdir + '/' + gains_string) obsdate = cmd[3].split(' ')[4][0:7] starttime = datetime.datetime( int('20' + obsdate[0:2]), int(time.strptime(obsdate[2:5], '%b').tm_mon), int(obsdate[5:7])) t = Table.read(tempdir + '/' + gains_string, format='ascii') days = np.array(t['col1']) times = np.array(t['col2']) nint = len(days) nant = int(len(t[0]) - 2) delay_array = np.zeros((nant, nint)) for ant in range(nant): delay_array[ant, :] = t['col' + str(ant + 3)] time_array = [ starttime + datetime.timedelta(days=int(days[step]), hours=int(times[step][0:2]), minutes=int(times[step][3:5]), seconds=int(times[step][6:8])) for step in range(nint) ] return delay_array, time_array
def getmaskstats(self, image, size): """ Subroutine to calculate the number of pixels in a mask and its percentage of the full image image (string): The absolute path to the image file. size (int): Number of pixels along an axis of the original image. Assumes square images. returns (numpy array): The number of pixels and their percentage of the full image """ setinit.setinitdirs(self) char_set = string.ascii_uppercase + string.digits if os.path.isdir(image) or os.path.isfile(image): if os.path.isdir(image): temp_string = ''.join(random.sample(char_set * 8, 8)) fits = lib.miriad('fits') fits.op = 'xyout' fits.in_ = image with tempfile.TemporaryDirectory() as tempdir: fits.out = tempdir + '/' + temp_string + '.fits' fits.go() mask_data = pyfits.open(tempdir + '/' + temp_string + '.fits') elif os.path.isfile(image): mask_data = pyfits.open(image) else: error = 'Image format not supported. Only MIRIAD and FITS formats are supported!' logger.error(error) raise ApercalException(error) data = mask_data[0].data maskstats = np.full(2, np.nan) maskstats[0] = np.count_nonzero(~np.isnan(data)) maskstats[1] = maskstats[0] / (size**2) mask_data.close() else: error = 'Image does not seem to exist!' logger.error(error) raise ApercalException(error) return maskstats
def create_mask(self, image, mask, threshold, theoretical_noise, beampars=None, rms_map=None): """ Creates a mask from an image using pybdsf image (string): Input image to use in MIRIAD format mask (string): Output mask image in MIRIAD format threshold (float): Threshold in Jy to use theoretical_noise (float): Theoretical noise for calculating the adaptive threshold parameter inside pybdsf """ convim.mirtofits(image, image + '.fits') bdsf_threshold = threshold / theoretical_noise if beampars: # bdsf.process_image(image + '.fits', stop_at='isl', thresh_isl=bdsf_threshold, beam=beampars, adaptive_rms_box=True, rms_map=rms_map).export_image(outfile=mask + '.fits', img_format='fits', img_type='island_mask', pad_image=True) bdsf.process_image(image + '.fits', stop_at='isl', thresh_isl=bdsf_threshold, beam=beampars, adaptive_rms_box=True, rms_map=False, rms_value=theoretical_noise).export_image(outfile=mask + '.fits', img_format='fits', img_type='island_mask', pad_image=True) else: bdsf.process_image(image + '.fits', stop_at='isl', thresh_isl=bdsf_threshold, adaptive_rms_box=True, rms_map=False, rms_value=theoretical_noise).export_image(outfile=mask + '.fits', img_format='fits', img_type='island_mask', pad_image=True) if os.path.isfile(mask + '.fits'): # Add a random number to the masks to make it viewable in kvis fitsmask = pyfits.open(mask + '.fits') fitsmask_data = fitsmask[0].data fitsmask_hdr = fitsmask[0].header rand_array = np.random.rand(int(fitsmask_hdr['NAXIS1']), int(fitsmask_hdr['NAXIS2'])) fitsmask[0].data = np.multiply(rand_array, fitsmask_data) fitsmask.writeto(mask + '.fits', clobber=True) # Convert mask to MIRIAD and generate a usable one for MIRIAD convim.fitstomir(mask + '.fits', mask + '_pybdsf') maths = lib.miriad('maths') maths.out = mask maths.exp = '"<' + mask + '_pybdsf>"' maths.mask = '"<' + mask + '_pybdsf>.gt.0' + '"' maths.go() managefiles.director(self, 'rm', image + '.fits.pybdsf.log') managefiles.director(self, 'rm', image + '.fits') managefiles.director(self, 'rm', mask + '.fits') managefiles.director(self, 'rm', mask + '_pybdsf') else: pass
def make_gaussian_beam(beamdir, beamoutname, bm_size, cell, fwhm, cutoff): """ Function to create Gaussian beam Args: beamdir (str): destination directory for beam map beamoutname (str): name for beam map, based on beam number bm_size (integer): number of pixels for reference map cell (float): cell size of a pixel in arcsec fwhm (float): FWHM of Gaussian in arcsec cutoff (float): Relative level to cut beam off at """ # use a temp beam name because you have to apply a cutoff and then clean up tmpbeamname = beamoutname + '_tmp' # set peak level to 1 and PA = 0, plus arbitrary reference pixel pk = 1. bpa = 0. pos = [0., 60.] # set up imgen parameters imgen = lib.miriad('imgen') imgen.out = os.path.join(beamdir, tmpbeamname) imgen.imsize = bm_size imgen.cell = cell imgen.object = 'gaussian' imgen.spar = '{0},{1},{2},{3},{4},{5}'.format(str(pk), str(0), str(0), str(fwhm), str(fwhm), str(bpa)) imgen.radec = '{0},{1}'.format(str(pos[0]), str(pos[1])) # create image imgen.go() # apply beam cutoff beam_cutoff(beamoutname, tmpbeamname, beamdir, cutoff) # fix header fixheader(beamoutname, beamdir)
def getmodelstats(self, image): """ Subroutine to calculate the number of clean components and their flux image (string): The absolute path to the image file. returns (numpy array): The number of pixels with clean components and their summed flux in Jy """ setinit.setinitdirs(self) char_set = string.ascii_uppercase + string.digits if os.path.isdir(image) or os.path.isfile(image): if os.path.isdir(image): temp_string = ''.join(random.sample(char_set * 8, 8)) fits = lib.miriad('fits') fits.op = 'xyout' fits.in_ = image with tempfile.TemporaryDirectory() as tempdir: fits.out = tempdir + '/' + temp_string + '.fits' fits.go() model_data = pyfits.open(tempdir + '/' + temp_string + '.fits') elif os.path.isfile(image): model_data = pyfits.open(image) else: error = 'Image format not supported. Only MIRIAD and FITS formats are supported!' logger.error(error) raise ApercalException(error) data = model_data[0].data[:, 0, :, :] modelstats = np.full(2, np.nan) modelstats[0] = np.count_nonzero(data) modelstats[1] = np.sum(data) model_data.close() else: error = 'Image does not seem to exist!' logger.error(error) raise ApercalException(error) return modelstats
def splitdata(self): """ Applies calibrator corrections to data, splits the data into chunks in frequency and bins it to the given frequency resolution for the self-calibration """ if self.selfcal_splitdata: subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) subs_managefiles.director(self, 'ch', self.selfcaldir) logger.info(' Splitting of target data into individual frequency chunks started') if os.path.exists(self.selfcaldir + '/' + self.target): logger.info('Calibrator corrections already seem to have been applied #') else: logger.info('Applying calibrator solutions to target data before averaging #') uvaver = lib.miriad('uvaver') uvaver.vis = self.crosscaldir + '/' + self.target uvaver.out = self.selfcaldir + '/' + self.target uvaver.go() logger.info('Calibrator solutions to target data applied #') if self.selfcal_flagantenna != '': uvflag = lib.miriad('uvflag') uvflag.vis = self.selfcaldir + '/' + self.target uvflag.flagval = 'flag' uvflag.select = 'antenna(' + str(self.selfcal_flagantenna) + ')' uvflag.go() else: pass try: uv = aipy.miriad.UV(self.selfcaldir + '/' + self.target) except RuntimeError: raise ApercalException(' No data in your selfcal directory!') try: nsubband = len(uv['nschan']) # Number of subbands in data except TypeError: nsubband = 1 # Only one subband in data since exception was triggered logger.info('Found ' + str(nsubband) + ' subband(s) in target data #') counter = 0 # Counter for naming the chunks and directories for subband in range(nsubband): logger.info('Started splitting of subband ' + str(subband) + ' #') if nsubband == 1: numchan = uv['nschan'] finc = np.fabs(uv['sdf']) else: numchan = uv['nschan'][subband] # Number of channels per subband finc = np.fabs(uv['sdf'][subband]) # Frequency increment for each channel subband_bw = numchan * finc # Bandwidth of one subband subband_chunks = round(subband_bw / self.selfcal_splitdata_chunkbandwidth) # Round to the closest power of 2 for frequency chunks with the same bandwidth over the frequency # range of a subband subband_chunks = int(np.power(2, np.ceil(np.log(subband_chunks) / np.log(2)))) if subband_chunks == 0: subband_chunks = 1 chunkbandwidth = (numchan / subband_chunks) * finc logger.info('Adjusting chunk size to ' + str( chunkbandwidth) + ' GHz for regular gridding of the data chunks over frequency #') for chunk in range(subband_chunks): logger.info( 'Starting splitting of data chunk ' + str(chunk) + ' for subband ' + str(subband) + ' #') binchan = round( self.selfcal_splitdata_channelbandwidth / finc) # Number of channels per frequency bin chan_per_chunk = numchan / subband_chunks if chan_per_chunk % binchan == 0: # Check if the freqeuncy bin exactly fits logger.info('Using frequency binning of ' + str( self.selfcal_splitdata_channelbandwidth) + ' for all subbands #') else: # Increase the frequency bin to keep a regular grid for the chunks while chan_per_chunk % binchan != 0: binchan = binchan + 1 else: # Check if the calculated bin is not larger than the subband channel number if chan_per_chunk >= binchan: pass else: # Set the frequency bin to the number of channels in the chunk of the subband binchan = chan_per_chunk logger.info('Increasing frequency bin of data chunk ' + str( chunk) + ' to keep bandwidth of chunks equal over the whole bandwidth #') logger.info('New frequency bin is ' + str(binchan * finc) + ' GHz #') nchan = int(chan_per_chunk / binchan) # Total number of output channels per chunk start = 1 + chunk * chan_per_chunk width = int(binchan) step = int(width) subs_managefiles.director(self, 'mk', self.selfcaldir + '/' + str(counter).zfill(2)) uvaver = lib.miriad('uvaver') uvaver.vis = self.selfcaldir + '/' + self.target uvaver.out = self.selfcaldir + '/' + str(counter).zfill(2) + '/' + str(counter).zfill(2) + '.mir' uvaver.select = "'" + 'window(' + str(subband + 1) + ')' + "'" uvaver.line = "'" + 'channel,' + str(nchan) + ',' + str(start) + ',' + str(width) + ',' + str( step) + "'" uvaver.go() counter = counter + 1 logger.info('Splitting of data chunk ' + str(chunk) + ' for subband ' + str(subband) + ' done #') logger.info('Splitting of data for subband ' + str(subband) + ' done #') logger.info(' Splitting of target data into individual frequency chunks done')
def run_continuum_minoriteration(self, chunk, majc, minc, drmin, theoretical_noise_threshold): """ Does a selfcal minor iteration for the standard mode chunk: The frequency chunk to image and calibrate maj: Current major iteration min: Current minor iteration drmin: maximum dynamic range for minor iteration theoretical_noise_threshold: calculated theoretical noise threshold """ subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) logger.info('Minor self-calibration cycle ' + str(minc) + ' for frequency chunk ' + chunk + ' started #') if minc == 0: invert = lib.miriad('invert') # Create the dirty image invert.vis = chunk + '.mir' invert.map = str(majc).zfill(2) + '/map_' + str(minc).zfill(2) invert.beam = str(majc).zfill(2) + '/beam_' + str(minc).zfill(2) invert.imsize = self.selfcal_image_imsize invert.cell = self.selfcal_image_cellsize invert.stokes = 'ii' invert.options = 'mfs,double' invert.slop = 1 invert.robust = -2 invert.go() imax = self.calc_imax(str(majc).zfill(2) + '/map_' + str(minc).zfill(2)) noise_threshold = self.calc_noise_threshold(imax, minc, majc) dynamic_range_threshold = calc_dynamic_range_threshold(imax, drmin, self.selfcal_standard_minorcycle0_dr) mask_threshold, mask_threshold_type = calc_mask_threshold(theoretical_noise_threshold, noise_threshold, dynamic_range_threshold) logger.info('Mask threshold for major/minor cycle ' + str(majc) + '/' + str(minc) + ' set to ' + str( mask_threshold) + ' Jy/beam #') logger.info('Mask threshold set by ' + str(mask_threshold_type) + ' #') if majc == 0: maths = lib.miriad('maths') maths.out = str(majc).zfill(2) + '/mask_' + str(minc).zfill(2) maths.exp = '"<' + str(majc).zfill(2) + '/map_' + str(minc).zfill(2) + '>"' maths.mask = '"<' + str(majc).zfill(2) + '/map_' + str(minc).zfill(2) + '>.gt.' + str( mask_threshold) + '"' maths.go() logger.info('Mask with threshold ' + str(mask_threshold) + ' Jy/beam created #') else: subs_managefiles.director(self, 'cp', str(majc).zfill(2) + '/mask_' + str(minc).zfill(2), file_=str(majc - 1).zfill(2) + '/mask_' + str( self.selfcal_standard_minorcycle - 1).zfill(2)) logger.info('Mask from last minor iteration of last major cycle copied #') clean_cutoff = self.calc_clean_cutoff(mask_threshold) logger.info('Clean threshold at major/minor cycle ' + str(majc) + '/' + str(minc) + ' was set to ' + str( clean_cutoff) + ' Jy/beam #') clean = lib.miriad('clean') # Clean the image down to the calculated threshold clean.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2) clean.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2) clean.out = str(majc).zfill(2) + '/model_' + str(minc).zfill(2) clean.cutoff = clean_cutoff clean.niters = 1000000 clean.region = '"' + 'mask(' + str(majc).zfill(2) + '/mask_' + str(minc).zfill(2) + ')' + '"' clean.go() logger.info('Major/minor cycle ' + str(majc) + '/' + str(minc) + ' cleaning done #') restor = lib.miriad('restor') restor.model = str(majc).zfill(2) + '/model_' + str(minc).zfill(2) restor.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2) restor.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2) restor.out = str(majc).zfill(2) + '/image_' + str(minc).zfill(2) restor.mode = 'clean' restor.go() # Create the cleaned image logger.info('Cleaned image for major/minor cycle ' + str(majc) + '/' + str(minc) + ' created #') restor.mode = 'residual' restor.out = str(majc).zfill(2) + '/residual_' + str(minc).zfill(2) restor.go() logger.info('Residual image for major/minor cycle ' + str(majc) + '/' + str(minc) + ' created #') logger.info('Peak of the residual image is ' + str( self.calc_imax(str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) + ' Jy/beam #') logger.info('RMS of the residual image is ' + str( self.calc_irms(str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) + ' Jy/beam #') else: imax = self.calc_imax(str(majc).zfill(2) + '/map_' + str(0).zfill(2)) noise_threshold = self.calc_noise_threshold(imax, minc, majc) dynamic_range_threshold = calc_dynamic_range_threshold(imax, drmin, self.selfcal_standard_minorcycle0_dr) mask_threshold, mask_threshold_type = calc_mask_threshold(theoretical_noise_threshold, noise_threshold, dynamic_range_threshold) logger.info('Mask threshold for major/minor cycle ' + str(majc) + '/' + str(minc) + ' set to ' + str( mask_threshold) + ' Jy/beam #') logger.info('Mask threshold set by ' + str(mask_threshold_type) + ' #') maths = lib.miriad('maths') maths.out = str(majc).zfill(2) + '/mask_' + str(minc).zfill(2) maths.exp = '"<' + str(majc).zfill(2) + '/image_' + str(minc - 1).zfill(2) + '>"' maths.mask = '"<' + str(majc).zfill(2) + '/image_' + str(minc - 1).zfill(2) + '>.gt.' + str( mask_threshold) + '"' maths.go() logger.info('Mask with threshold ' + str(mask_threshold) + ' Jy/beam created #') clean_cutoff = self.calc_clean_cutoff(mask_threshold) logger.info('Clean threshold at major/minor cycle ' + str(majc) + '/' + str(minc) + ' was set to ' + str( clean_cutoff) + ' Jy/beam #') clean = lib.miriad('clean') # Clean the image down to the calculated threshold clean.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2) clean.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2) clean.model = str(majc).zfill(2) + '/model_' + str(minc - 1).zfill(2) clean.out = str(majc).zfill(2) + '/model_' + str(minc).zfill(2) clean.cutoff = clean_cutoff clean.niters = 1000000 clean.region = '"' + 'mask(' + str(majc).zfill(2) + '/mask_' + str(minc).zfill(2) + ')' + '"' clean.go() logger.info('Major/minor cycle ' + str(majc) + '/' + str(minc) + ' cleaning done #') restor = lib.miriad('restor') restor.model = str(majc).zfill(2) + '/model_' + str(minc).zfill(2) restor.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2) restor.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2) restor.out = str(majc).zfill(2) + '/image_' + str(minc).zfill(2) restor.mode = 'clean' restor.go() # Create the cleaned image logger.info('Cleaned image for major/minor cycle ' + str(majc) + '/' + str(minc) + ' created #') restor.mode = 'residual' restor.out = str(majc).zfill(2) + '/residual_' + str(minc).zfill(2) restor.go() logger.info('Residual image for major/minor cycle ' + str(majc) + '/' + str(minc) + ' created #') logger.info('Peak of the residual image is ' + str( self.calc_imax(str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) + ' Jy/beam #') logger.info('RMS of the residual image is ' + str( self.calc_irms(str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) + ' Jy/beam #') logger.info('Minor self-calibration cycle ' + str(minc) + ' for frequency chunk ' + chunk + ' finished #')
def selfcal_standard(self): """ Executes the standard method of self-calibration with the given parameters """ subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) logger.info(' Starting standard self calibration routine') subs_managefiles.director(self, 'ch', self.selfcaldir) for chunk in self.list_chunks(): logger.info('Starting standard self-calibration routine on frequency chunk ' + chunk + ' #') subs_managefiles.director(self, 'ch', self.selfcaldir + '/' + chunk) if os.path.isfile(self.selfcaldir + '/' + chunk + '/' + chunk + '.mir/visdata'): theoretical_noise = calc_theoretical_noise(self.selfcaldir + '/' + chunk + '/' + chunk + '.mir') logger.info('Theoretical noise for chunk ' + chunk + ' is ' + str(theoretical_noise) + ' Jy/beam #') theoretical_noise_threshold = self.calc_theoretical_noise_threshold(theoretical_noise) logger.info('Your theoretical noise threshold will be ' + str( self.selfcal_standard_nsigma) + ' times the theoretical noise corresponding to ' + str( theoretical_noise_threshold) + ' Jy/beam #') dr_list = calc_dr_maj(self.selfcal_standard_drinit, self.selfcal_standard_dr0, self.selfcal_standard_majorcycle, self.selfcal_standard_majorcycle_function) logger.info( 'Your dynamic range limits are set to ' + str(dr_list) + ' for the major self-calibration cycles #') for majc in range(self.selfcal_standard_majorcycle): logger.info( 'Major self-calibration cycle ' + str(majc) + ' for frequency chunk ' + chunk + ' started #') subs_managefiles.director(self, 'mk', self.selfcaldir + '/' + str(chunk) + '/' + str(majc).zfill(2)) # Calculate the dynamic ranges during minor cycles dr_minlist = self.calc_dr_min(dr_list, majc, self.selfcal_standard_minorcycle, self.selfcal_standard_minorcycle_function) logger.info('The minor cycle dynamic range limits for major cycle ' + str(majc) + ' are ' + str( dr_minlist) + ' #') for minc in range(self.selfcal_standard_minorcycle): try: self.run_continuum_minoriteration(chunk, majc, minc, dr_minlist[minc], theoretical_noise_threshold) except Exception: logger.warning('Chunk ' + chunk + ' does not seem to contain data to image #') break try: logger.info('Doing self-calibration with uvmin=' + str( self.selfcal_standard_uvmin[majc]) + ', uvmax=' + str( self.selfcal_standard_uvmax[majc]) + ', solution interval=' + str( self.selfcal_standard_solint[majc]) + ' minutes for major cycle ' + str(majc).zfill( 2) + ' #') selfcal = lib.miriad('selfcal') selfcal.vis = chunk + '.mir' selfcal.select = '"' + 'uvrange(' + str(self.selfcal_standard_uvmin[majc]) + ',' + str( self.selfcal_standard_uvmax[majc]) + ')"' selfcal.model = str(majc).zfill(2) + '/model_' + str(minc).zfill(2) selfcal.interval = self.selfcal_standard_solint[majc] # Choose reference antenna if given if self.selfcal_refant == '': pass else: selfcal.refant = self.selfcal_refant # Enable amplitude calibration if triggered if not self.selfcal_standard_amp: # See if we want to do amplitude calibration selfcal.options = 'mfs,phase' elif self.selfcal_standard_amp: selfcal.options = 'mfs,amp' elif self.selfcal_standard_amp == 'auto': modelflux = self.calc_isum(str(majc).zfill(2) + '/model_' + str(minc).zfill(2)) if modelflux >= self.selfcal_standard_amp_auto_limit: logger.info( 'Flux of clean model is ' + str(modelflux) + ' Jy. Doing amplitude calibration. #') selfcal.options = 'mfs,amp' else: selfcal.options = 'mfs,phase' if self.selfcal_standard_nfbin >= 1: selfcal.nfbin = self.selfcal_standard_nfbin selfcal.go() logger.info('Major self-calibration cycle ' + str( majc) + ' for frequency chunk ' + chunk + ' finished #') except Exception: logger.warning( 'Model for self-calibration not found. No further calibration on this chunk possible!') break logger.info('Standard self-calibration routine for chunk ' + chunk + ' finished #') else: logger.warning('No data in chunk ' + chunk + '. Maybe all data is flagged? #') logger.info(' Standard self calibration routine finished')
def parametric(self): """ Parametric self calibration using an NVSS/FIRST skymodel and calculating spectral indices by source matching with WENSS. """ if self.selfcal_parametric: subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) logger.info(' Doing parametric self calibration') subs_managefiles.director(self, 'ch', self.selfcaldir) for chunk in self.list_chunks(): logger.info('Starting parametric self calibration routine on chunk ' + chunk + ' #') subs_managefiles.director(self, 'ch', self.selfcaldir + '/' + chunk) subs_managefiles.director(self, 'mk', self.selfcaldir + '/' + chunk + '/' + 'pm') parametric_textfile = lsm.lsm_model(chunk + '.mir', self.selfcal_parametric_skymodel_radius, self.selfcal_parametric_skymodel_cutoff, self.selfcal_parametric_skymodel_distance) lsm.write_model(self.selfcaldir + '/' + chunk + '/' + 'pm' + '/model.txt', parametric_textfile) logger.info('Creating model from textfile model.txt for chunk ' + chunk + ' #') uv = aipy.miriad.UV(self.selfcaldir + '/' + chunk + '/' + chunk + '.mir') freq = uv['sfreq'] uvmodel = lib.miriad('uvmodel') uvmodel.vis = chunk + '.mir' parametric_modelfile = open(self.selfcaldir + '/' + str(chunk) + '/' + 'pm' + '/model.txt', 'r') for n, source in enumerate(parametric_modelfile.readlines()): if n == 0: uvmodel.options = 'replace,mfs' else: uvmodel.options = 'add,mfs' uvmodel.offset = source.split(',')[0] + ',' + source.split(',')[1] uvmodel.flux = source.split(',')[2] + ',i,' + str(freq) + ',' + source.split(',')[4].rstrip( '\n') + ',0,0' uvmodel.out = 'pm/tmp' + str(n) uvmodel.go() uvmodel.vis = uvmodel.out subs_managefiles.director(self, 'rn', 'pm/model', uvmodel.out) # Rename the last modelfile to model subs_managefiles.director(self, 'rm', 'pm/tmp*') # Remove all the obsolete modelfiles logger.info('Doing parametric self-calibration on chunk {} with solution interval {} min' 'and uvrange limits of {}~{} klambda #'.format(chunk, self.selfcal_parametric_solint, self.selfcal_parametric_uvmin, self.selfcal_parametric_uvmax)) selfcal = lib.miriad('selfcal') selfcal.vis = chunk + '.mir' selfcal.model = 'pm/model' selfcal.interval = self.selfcal_parametric_solint selfcal.select = "'" + 'uvrange(' + str(self.selfcal_parametric_uvmin) + ',' + str( self.selfcal_parametric_uvmax) + ')' + "'" # Choose reference antenna if given if self.selfcal_refant == '': pass else: selfcal.refant = self.selfcal_refant # Do amplitude calibration if wanted if self.selfcal_parametric_amp: selfcal.options = 'mfs,amp' else: selfcal.options = 'mfs' selfcal.go() logger.info('Parametric self calibration routine on chunk ' + chunk + ' done! #') logger.info(' Parametric self calibration done') else: logger.info(' Parametric self calibration disabled')
def convert_selfcaluv2uvfits(self): """ Looks for the last self-calibrated uv-fits file, copies its gains over to the original file, applies them and coverts to UVFITS format """ subs_setinit.setinitdirs(self) sbeam = 'selfcal_B' + str(self.beam).zfill(2) tbeam = 'transfer_B' + str(self.beam).zfill(2) # Create the parameters for the parameter file for the conversion of the UVFITS-files transfertargetbeamsselfcaluv2uvfitsstatus = get_param_def( self, tbeam + '_targetbeams_selfcaluv2uvfits_status', False) if self.transfer_convert_selfcaluv2uvfits: subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) subs_managefiles.director(self, 'ch', self.transferdir, verbose=True) if not transfertargetbeamsselfcaluv2uvfitsstatus: # Get the status of the selfcal for the specified beam selfcaltargetbeamsphasestatus = get_param_def( self, sbeam + '_targetbeams_phase_status', False) selfcaltargetbeamsampstatus = get_param_def( self, sbeam + '_targetbeams_amp_status', False) datasetname_amp = os.path.join( self.selfcaldir, self.target).rstrip('.mir') + '_amp.mir' datasetname_phase = os.path.join(self.selfcaldir, self.target) logger.debug("Setting amplitude selfcal file name: {}".format( datasetname_amp)) logger.debug("Setting phase selfcal file name: {}".format( datasetname_phase)) # datasetname_amp = self.get_target_path().rstrip('.mir') + '_amp.mir' # datasetname_phase = self.get_target_path() if os.path.isdir( datasetname_amp) and selfcaltargetbeamsampstatus: logger.info('Beam ' + self.beam + ': Using amplitude self-calibrated dataset!') dataset = datasetname_amp elif os.path.isdir( datasetname_phase) and selfcaltargetbeamsphasestatus: logger.info( 'Beam ' + self.beam + ': Using phase self-calibrated dataset. Amplitude calibration was not successful or not wanted!' ) dataset = datasetname_phase else: dataset = None if dataset is not None: # Copy the raw dataset to the transfer directory subs_managefiles.director( self, 'cp', self.transferdir + '/' + self.target, file_=self.crosscaldir + '/' + self.target) if selfcaltargetbeamsampstatus: gpcopy = lib.miriad('gpcopy') gpcopy.vis = datasetname_phase gpcopy.out = self.transferdir + '/' + self.target gpcopy.go() uvaver = lib.miriad('uvaver') uvaver.vis = self.transferdir + '/' + self.target uvaver.out = self.transferdir + '/' + \ self.target.rstrip('.mir') + '_phase.mir' uvaver.go() gpcopy = lib.miriad('gpcopy') gpcopy.vis = datasetname_amp gpcopy.out = self.transferdir + '/' + \ self.target.rstrip('.mir') + '_phase.mir' gpcopy.go() fits = lib.miriad('fits') fits.op = 'uvout' fits.in_ = self.transferdir + '/' + \ self.target.rstrip('.mir') + '_phase.mir' fits.out = self.transferdir + '/' + \ self.target.rstrip('.mir') + '.UVFITS' fits.go() if os.path.isfile(self.transferdir + '/' + self.target.rstrip('.mir') + '.UVFITS'): subs_managefiles.director( self, 'rm', self.transferdir + '/' + self.target) subs_managefiles.director( self, 'rm', self.transferdir + '/' + self.target.rstrip('.mir') + '_phase.mir') transfertargetbeamsselfcaluv2uvfitsstatus = True else: logger.error( 'Beam ' + self.beam + ': Conversion was not successful. No UVFITS-file generated!' ) transfertargetbeamsselfcaluv2uvfitsstatus = False elif selfcaltargetbeamsphasestatus: gpcopy = lib.miriad('gpcopy') gpcopy.vis = datasetname_phase gpcopy.out = self.transferdir + '/' + self.target gpcopy.go() fits = lib.miriad('fits') fits.op = 'uvout' fits.in_ = self.transferdir + '/' + self.target fits.out = self.transferdir + '/' + \ self.target.rstrip('.mir') + '.UVFITS' fits.go() if os.path.isfile(self.transferdir + '/' + self.target.rstrip('.mir') + '.UVFITS'): subs_managefiles.director( self, 'rm', self.transferdir + '/' + self.target) transfertargetbeamsselfcaluv2uvfitsstatus = True else: logger.error( 'Beam ' + self.beam + ': Conversion was not successful. No UVFITS-file generated!' ) transfertargetbeamsselfcaluv2uvfitsstatus = False else: logger.error( 'Beam ' + self.beam + ': Self-calibration was not successful. No conversion to UVFITS-format possible!' ) transfertargetbeamsselfcaluv2uvfitsstatus = False else: logger.info( 'Beam ' + self.beam + ': Conversion of final calibrated data to UVFITS-format already successfully executed!' ) else: logger.info( 'Beam ' + self.beam + ': Conversion of final calibrated data to UVFITS-format not selected!' ) # Save the derived parameters to the parameter file subs_param.add_param(self, tbeam + '_targetbeams_selfcaluv2uvfits_status', transfertargetbeamsselfcaluv2uvfitsstatus)
items = os.listdir('/data/apertif/'+str(data_dir)+'/mosaic/cont_tmp/') def convert_fits(): for i in range(len(items)): fits = lib.miriad('fits') fits.in_ = '/data/apertif/'+str(data_dir)+'/mosaic/cont_tmp/'+str(items[i]) fits.out= '/data/apertif/'+str(data_dir)+'/mosaic/cont_tmp/'+str(items[i][:-5])+'.mir' fits.op = 'xyin' fits.go() convert_fits() #----------------------------------- # create mosaic image with linmos linmos = lib.miriad('linmos') linmos.in_ = '/data/apertif/'+str(data_dir)+'/mosaic/cont_tmp/image_mf_*.mir' linmos.out = '/data/apertif/'+str(data_dir)+'/mosaic/'+str(data_dir)+'_mosaic_image' linmos.go() #----------------------------------- # convert mosaic miriad image into fits file fits = lib.miriad('fits') fits.in_ = '/data/apertif/'+str(data_dir)+'/mosaic/'+str(data_dir)+'_mosaic_image' fits.op = 'xyout' fits.out = '/data/apertif/'+str(data_dir)+'/mosaic/'+str(data_dir)+'_mosaic_image.fits' fits.go() #------------------------------------- #clen up temporary files
def regrid_pb(self): """ Convert PB image to miriad in working directory Regrid image """ #first get PB image and write to miriad in workingdir #naming convention might depend on type of PB image #drift, Alexander's method, etc #check if drift if len(self.pbname) == 6: #YYMMDD name used for driftscans #pbfits = os.path.join(self.pbdir,"{0}_{1}_I_model_reg.fits". # format(self.pbname,self.beam)) pbfits = os.path.join( self.pbdir, "{0}_{1}_I_model.fits".format(self.pbname, self.beam)) #may have to add a check about string length, but wait until it's a problem #check if gpall if self.pbname == 'gpall': pbfits = os.path.join(self.pbdir, "pb_{0}_gpall.fits".format(self.beam)) if self.pbname == 'norm': pbfits = os.path.join(self.pbdir, "{0}_norm.fits".format(self.beam)) if self.pbname == 'nonorm': pbfits = os.path.join(self.pbdir, "{0}_nonorm.fits".format(self.beam)) if self.pbname == 'gp_avg_norm': pbfits = os.path.join(self.pbdir, "{0}_gp_avg_norm.fits".format(self.beam)) if self.pbname == 'gp_avg_orig': pbfits = os.path.join(self.pbdir, "{0}_gp_avg_orig.fits".format(self.beam)) #having defined pbfits above, #check it exists, retrieve to miriad in workingdir #also check that output doesn't exist pbim = os.path.join(self.workingdir, "pb") if (os.path.exists(pbfits) and not os.path.isdir(pbim)): #get pbimage to miriad fits = lib.miriad('fits') fits.op = 'xyin' fits.in_ = pbfits fits.out = pbim try: fits.go() except Exception as e: self.status = False print(("Conversion to miriad image failed " "for primary beam {0}").format(self.beam)) print(e) #this seems to then have regrid do nothing #but it lets me run code and can update/fix this later #get projection center of continuum image for regridding #and update center of PB image manually #is this the right thing to do? #check w/ Thijs & Helga..... if (os.path.isdir(self.smimpath) and os.path.isdir(pbim)): #get center values gethd = lib.miriad('gethd') gethd.in_ = os.path.join(self.smimpath, 'crval1') #gethd.format = 'hms' ra_ref = gethd.go() gethd.in_ = os.path.join(self.smimpath, 'crval2') #gethd.format = 'dms' dec_ref = gethd.go() #update center puthd = lib.miriad('puthd') puthd.in_ = os.path.join(pbim, 'crval1') puthd.value = ra_ref[0] puthd.type = 'double' puthd.go() puthd.in_ = os.path.join(pbim, 'crval2') puthd.value = dec_ref[0] puthd.type = 'double' puthd.go() #keep ra & dec in object; will need later self.ra = ra_ref[0] * u.rad self.dec = dec_ref[0] * u.rad else: #getting projection center of (smoothed) image failed self.status = False print( ("Updating projection center from beam {0}, " "based on taskid {1}, failed").format(self.beam, self.taskid)) #regrid image #define out put, pbr - PB R egridded #check pbim exists for regridding #plus smoothed path for template #also that output doesn't exist if (os.path.isdir(pbim) and os.path.isdir(self.smimpath) and not os.path.isdir(self.pbpath)): regrid = lib.miriad('regrid') regrid.in_ = pbim regrid.out = self.pbpath regrid.tin = self.smimpath regrid.axes = '1,2' #what if I try options = offset #then maybe keep image values but on right grid #regrid.options = 'offset' #then my image is resize so things don't work later #what if I add desc value? #use helper function to get values ###ra_vals = self.get_hdr('ra',self.smimpath) ###dec_vals = self.get_hdr('dec',self.smimpath) ###desc = "{0},{1},{2},{3},{4},{5},{6},{7}".format( ### ra_vals[0],ra_vals[1],ra_vals[2],ra_vals[3], ### dec_vals[0],dec_vals[1],dec_vals[2],dec_vals[3]) try: regrid.go() except Exception as e: self.status = False print(("Regridding failed for primary beam {0}," "reference taskid {1}").format(self.beam, self.taskid)) print(e) else: self.status = False print(("Either primary beam or smoothed continuum " "image missing for beam {0}, taskid{1}").format( self.beam, self.taskid)) #write regridded pb to fits; need later if os.path.isdir(self.pbpath): fits = lib.miriad('fits') fits.in_ = self.pbpath fits.out = self.pbrfits fits.op = 'xyout' try: fits.go() print("Writing {0} to {1}".format(self.pbpath, self.pbrfits)) except Exception as e: print("Writing {0} failed".format(self.pbrfits)) print(e)
def ms2miriad(self): """ Converts the data from MS to MIRIAD format via UVFITS using drivecasa. Does it for the flux calibrator, polarisation calibrator, and target field independently. """ subs_setinit.setinitdirs(self) ccalbeam = 'ccal_B' + str(self.beam).zfill(2) cbeam = 'convert_B' + str(self.beam).zfill(2) # Read the parameters from crosscal # and check before doing anything # Status of the solution transfer for the target, flux calibrator and polarisation calibrator ccal_targetbeams_transfer = get_param_def( self, ccalbeam + '_targetbeams_transfer', False) ccal_calibration_calibrator_finished = get_param_def( self, ccalbeam + '_calibration_calibrator_finished', False) if not ccal_calibration_calibrator_finished: error = "Beam {}: Will not convert files to miriad format because cross-calibration failed.".format( str(self.beam).zfill(2)) logger.error(error) raise ApercalException(error) elif not ccal_targetbeams_transfer: error = "Beam {}: Will not convert files to miriad format because cross-calibration solutions were not successfully applied to target.".format( str(self.beam).zfill(2)) logger.error(error) raise ApercalException(error) # Create the parameters for the parameter file for converting from MS to UVFITS format # Flux calibrator MS dataset available? convertfluxcalmsavailable = get_param_def( self, cbeam + '_fluxcal_MSavailable', False) # Polarised calibrator MS dataset available? convertpolcalmsavailable = get_param_def(self, cbeam + '_polcal_MSavailable', False) # Target beam MS dataset available? converttargetbeamsmsavailable = get_param_def( self, cbeam + '_targetbeams_MSavailable', False) # Flux calibrator MS dataset converted to UVFITS? convertfluxcalms2uvfits = get_param_def(self, cbeam + '_fluxcal_MS2UVFITS', False) # Polarised calibrator MS dataset converted to UVFITS? convertpolcalms2uvfits = get_param_def(self, cbeam + '_polcal_MS2UVFITS', False) # Target beam MS dataset converted to UVFITS? converttargetbeamsms2uvfits = get_param_def( self, cbeam + '_targetbeams_MS2UVFITS', False) # Flux calibrator UVFITS dataset available? convertfluxcaluvfitsavailable = get_param_def( self, cbeam + '_fluxcal_UVFITSavailable', False) # Polarised calibrator UVFITS dataset available? convertpolcaluvfitsavailable = get_param_def( self, cbeam + '_polcal_UVFITSavailable', False) # Target beam UVFITS dataset available? converttargetbeamsuvfitsavailable = get_param_def( self, cbeam + '_targetbeams_UVFITSavailable', False) # Flux calibrator UVFITS dataset converted to MIRIAD? convertfluxcaluvfits2miriad = get_param_def( self, cbeam + '_fluxcal_UVFITS2MIRIAD', False) # Polarised calibrator UVFITS dataset converted to MIRIAD? convertpolcaluvfits2miriad = get_param_def( self, cbeam + '_polcal_UVFITS2MIRIAD', False) # Target beam UVFITS dataset converted to MIRIAD? converttargetbeamsuvfits2miriad = get_param_def( self, cbeam + '_targetbeams_UVFITS2MIRIAD', False) # Check which datasets are available in MS format # if self.fluxcal != '': convertfluxcalmsavailable = path.isdir(self.get_fluxcal_path()) else: logger.warning( 'Beam ' + self.beam + ': Flux calibrator dataset not specified. Cannot convert flux calibrator!' ) if self.polcal != '': convertpolcalmsavailable = path.isdir(self.get_polcal_path()) else: logger.warning( 'Beam ' + self.beam + ': Polarised calibrator dataset not specified. Cannot convert polarised calibrator!' ) if self.target != '': converttargetbeamsmsavailable = path.isdir(self.get_target_path()) else: logger.warning( 'Beam ' + self.beam + ': Target beam dataset not specified. Cannot convert target beams!' ) # Save the derived parameters for the availability to the parameter file subs_param.add_param(self, cbeam + '_fluxcal_MSavailable', convertfluxcalmsavailable) subs_param.add_param(self, cbeam + '_polcal_MSavailable', convertpolcalmsavailable) subs_param.add_param(self, cbeam + '_targetbeams_MSavailable', converttargetbeamsmsavailable) # Convert the flux calibrator if self.convert_fluxcal: if self.fluxcal != '': if not convertfluxcaluvfits2miriad: if convertfluxcalmsavailable: logger.debug( 'Beam ' + self.beam + ': Converting flux calibrator dataset from MS to UVFITS format.' ) subs_managefiles.director( self, 'mk', self.get_crosscalsubdir_path(), verbose=False) fluxcal_ms = self.get_fluxcal_path() # convert only if corrected data column exists if subs_msutils.has_correcteddata(fluxcal_ms): datacolumn = "corrected" fluxcal_fits = mspath_to_fitspath( self.get_crosscalsubdir_path(), fluxcal_ms) fc_convert = exportuvfits_cmd.format( vis=self.get_fluxcal_path(), fits=fluxcal_fits, datacolumn=datacolumn) lib.run_casa([fc_convert], timeout=3600) if path.isfile(fluxcal_fits): convertfluxcalms2uvfits = True logger.info( 'Beam ' + self.beam + ': Converted flux calibrator dataset from MS to UVFITS format!' ) else: convertfluxcalms2uvfits = False logger.warning( 'Beam ' + self.beam + ': Could not convert flux calibrator dataset {} ' 'from MS to UVFITS format!'.format( fluxcal_fits)) else: logger.warning( 'Beam ' + self.beam + ': Flux calibrator does not have a corrected_data column! Not ' 'converting flux calibrator dataset!') else: logger.warning( 'Beam ' + self.beam + ': Flux calibrator dataset {} not available!'. format(self.get_fluxcal_path())) else: logger.info( 'Beam ' + self.beam + ': Flux calibrator dataset was already converted from MS to UVFITS format' ) else: logger.warning( 'Beam ' + self.beam + ': Flux calibrator dataset not specified. Cannot convert flux calibrator!' ) else: logger.warning('Beam ' + self.beam + ': Not converting flux calibrator dataset!') # Convert the polarised calibrator if self.convert_polcal: if self.polcal != '': if not convertpolcaluvfits2miriad: if convertpolcalmsavailable: logger.debug( 'Beam ' + self.beam + ': Converting polarised calibrator dataset from MS to UVFITS format.' ) subs_managefiles.director( self, 'mk', self.get_crosscalsubdir_path(), verbose=False) polcal_ms = self.get_polcal_path() # convert only if corrected data column exists if subs_msutils.has_correcteddata(polcal_ms): datacolumn = "corrected" polcal_fits = mspath_to_fitspath( self.get_crosscalsubdir_path(), polcal_ms) pc_convert = exportuvfits_cmd.format( vis=polcal_ms, fits=polcal_fits, datacolumn=datacolumn) lib.run_casa([pc_convert], timeout=3600) if path.isfile(polcal_fits): convertpolcalms2uvfits = True logger.info( 'Beam ' + self.beam + ': Converted polarised calibrator dataset from MS to UVFITS format!' ) else: convertpolcalms2uvfits = False logger.warning( 'Beam ' + self.beam + ': Could not convert polarised calibrator dataset from MS to UVFITS format!' ) else: logger.warning( 'Beam ' + self.beam + ': Polarised calibrator does not have a corrected_data column! Not ' 'converting polarised calibrator dataset!') else: logger.warning( 'Beam ' + self.beam + ': Polarised calibrator dataset not available!') else: logger.info( 'Beam ' + self.beam + ': Polarised calibrator dataset was already converted from MS to UVFITS format' ) else: logger.warning( 'Beam ' + self.beam + ': Polarised calibrator dataset not specified. Cannot convert polarised calibrator!' ) else: logger.warning('Beam ' + self.beam + ': Not converting polarised calibrator dataset!') # Convert the target beams if self.convert_target: if self.target != '': logger.info( 'Beam ' + self.beam + ': Converting target beam dataset from MS to UVFITS format.' ) if not converttargetbeamsuvfits2miriad: if converttargetbeamsmsavailable: subs_managefiles.director( self, 'mk', self.get_crosscalsubdir_path(), verbose=False) target_ms = self.get_target_path() target_fits = mspath_to_fitspath( self.get_crosscalsubdir_path(), target_ms) # only convert if corrected data column exists if subs_msutils.has_correcteddata(target_ms): datacolumn = "corrected" tg_convert = exportuvfits_cmd.format( vis=target_ms, fits=target_fits, datacolumn=datacolumn) lib.run_casa([tg_convert], timeout=10000) if path.isfile(target_fits): converttargetbeamsms2uvfits = True logger.debug( 'Beam ' + self.beam + ': Converted dataset of target beam from MS to UVFITS format!' ) else: converttargetbeamsms2uvfits = False logger.warning( 'Beam ' + self.beam + ': Could not convert dataset for target beam from MS to UVFITS format!' ) else: logger.warning( 'Beam ' + self.beam + ': Target beam dataset does not have a corrected_data column! Not ' 'converting target beam dataset!') else: logger.warning('Beam ' + self.beam + ': Target beam dataset not available!') else: logger.info('Beam ' + self.beam + ': Target beam dataset was already ' 'converted from MS to UVFITS format') else: logger.warning( 'Beam ' + self.beam + ': Target beam dataset not specified. Cannot convert target beam dataset!' ) else: logger.warning('Beam ' + self.beam + ': Not converting target beam dataset!') # Save the derived parameters for the MS to UVFITS conversion to the parameter file subs_param.add_param(self, cbeam + '_fluxcal_MS2UVFITS', convertfluxcalms2uvfits) subs_param.add_param(self, cbeam + '_polcal_MS2UVFITS', convertpolcalms2uvfits) subs_param.add_param(self, cbeam + '_targetbeams_MS2UVFITS', converttargetbeamsms2uvfits) # Check which datasets are available in UVFITS format # if self.fluxcal != '': crosscal_fluxcal = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.fluxcal) convertfluxcaluvfitsavailable = path.isfile(crosscal_fluxcal) else: logger.warning( 'Beam ' + self.beam + ': Flux calibrator dataset not specified. Cannot convert flux calibrator!' ) if self.polcal != '': crosscal_polcal = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.polcal) convertpolcaluvfitsavailable = path.isfile(crosscal_polcal) else: logger.warning( 'Beam ' + self.beam + ': Polarised calibrator dataset not specified. Cannot convert polarised calibrator!' ) if self.target != '': crosscal_target = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.target) converttargetbeamsuvfitsavailable = path.isfile(crosscal_target) else: logger.warning( 'Beam ' + self.beam + ': Target beam dataset not specified. Cannot convert target beam!' ) # Save the derived parameters for the availability to the parameter file subs_param.add_param(self, cbeam + '_fluxcal_UVFITSavailable', convertfluxcaluvfitsavailable) subs_param.add_param(self, cbeam + '_polcal_UVFITSavailable', convertpolcaluvfitsavailable) subs_param.add_param(self, cbeam + '_targetbeams_UVFITSavailable', converttargetbeamsuvfitsavailable) # Convert the available UVFITS-datasets to MIRIAD format # # Convert the flux calibrator if self.convert_fluxcal: if self.fluxcal != '': if not convertfluxcaluvfits2miriad: if convertfluxcaluvfitsavailable: logger.debug( 'Beam ' + self.beam + ': Converting flux calibrator dataset from UVFITS to MIRIAD format.' ) subs_managefiles.director( self, 'ch', self.get_crosscalsubdir_path(), verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.fluxcal) fits.out = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.fluxcal, ext='mir') fits.go() if path.isdir(fits.out): convertfluxcaluvfits2miriad = True logger.info( 'Beam ' + self.beam + ': Converted flux calibrator dataset from UVFITS to MIRIAD format!' ) else: convertfluxcaluvfits2miriad = False logger.warning( 'Beam ' + self.beam + ': Could not convert flux calibrator dataset {} from UVFITS to ' 'MIRIAD format!'.format(fits.out)) else: logger.warning( 'Beam ' + self.beam + ': Flux calibrator dataset not available!') else: logger.info( 'Beam ' + self.beam + ': Flux calibrator dataset was already converted from UVFITS to MIRIAD format' ) else: logger.warning( 'Beam ' + self.beam + ': Flux calibrator dataset not specified. Cannot convert flux calibrator!' ) else: logger.warning('Beam ' + self.beam + ': Not converting flux calibrator dataset!') # Convert the polarised calibrator if self.convert_polcal: if self.polcal != '': if not convertpolcaluvfits2miriad: if convertpolcaluvfitsavailable: logger.debug( 'Beam ' + self.beam + ': Converting polarised calibrator dataset from UVFITS to MIRIAD format.' ) subs_managefiles.director( self, 'ch', self.get_crosscalsubdir_path(), verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.polcal) fits.out = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.polcal, ext='mir') fits.go() if path.isdir(fits.out): convertpolcaluvfits2miriad = True logger.info( 'Beam ' + self.beam + ': Converted polarised calibrator dataset from UVFITS to MIRIAD format!' ) else: convertpolcaluvfits2miriad = False logger.warning( 'Beam ' + self.beam + ': Could not convert polarised calibrator dataset from UVFITS to MIRIAD format!' ) else: logger.warning( 'Beam ' + self.beam + ': Polarised calibrator dataset not available!') else: logger.info( 'Beam ' + self.beam + ': Polarised calibrator dataset was already converted from UVFITS to MIRIAD format' ) else: logger.warning( 'Beam ' + self.beam + ': Polarised calibrator dataset not specified. Cannot convert polarised calibrator!' ) else: logger.warning('Beam ' + self.beam + ': Not converting polarised calibrator dataset!') # Convert the target beams if self.convert_target: if self.target != '': logger.info( 'Beam ' + self.beam + ': Converting target beam dataset from UVFITS to MIRIAD format.' ) if not converttargetbeamsuvfits2miriad: if converttargetbeamsuvfitsavailable: subs_managefiles.director( self, 'ch', self.get_crosscalsubdir_path(), verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.target) fits.out = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.target, ext='mir') fits.go() if path.isdir(fits.out): converttargetbeamsuvfits2miriad = True logger.debug( 'Beam ' + self.beam + ': Converted target beam dataset from ' 'UVFITS to MIRIAD format!') else: converttargetbeamsuvfits2miriad = False logger.warning( 'Beam ' + self.beam + ': Could not convert target beam dataset ' '{} from UVFITS to MIRIAD format!'.format( fits.out)) else: logger.warning('Beam ' + self.beam + ': Target beam dataset not available!') else: logger.info('Beam ' + self.beam + ': Target beam dataset was already converted ' 'from MS to UVFITS format') else: logger.warning( 'Beam ' + self.beam + ': Target beam dataset not specified. Cannot convert target beam datasets!' ) else: logger.warning('Beam ' + self.beam + ': Not converting target beam dataset!') # Save the derived parameters for the MS to UVFITS conversion to the parameter file subs_param.add_param(self, cbeam + '_fluxcal_UVFITS2MIRIAD', convertfluxcaluvfits2miriad) subs_param.add_param(self, cbeam + '_polcal_UVFITS2MIRIAD', convertpolcaluvfits2miriad) subs_param.add_param(self, cbeam + '_targetbeams_UVFITS2MIRIAD', converttargetbeamsuvfits2miriad) if self.convert_averagems and self.subdirification: logger.info('Beam ' + self.beam + ': Averaging down target measurement set') average_cmd = 'mstransform(vis="{vis}", outputvis="{outputvis}", chanaverage=True, chanbin=64)' vis = self.get_target_path() outputvis = vis.replace(".MS", "_avg.MS") lib.run_casa([average_cmd.format(vis=vis, outputvis=outputvis)], timeout=10000) # Remove measurement sets if wanted if self.convert_removems and self.subdirification: logger.info('Beam ' + self.beam + ': Removing measurement sets') vis = self.get_target_path() if path.exists(vis): subs_managefiles.director(self, 'rm', vis) # Remove the UVFITS files if wanted if self.convert_removeuvfits and self.subdirification: logger.info('Beam ' + self.beam + ': Removing all UVFITS files') if self.fluxcal != '' and path.exists( mspath_to_fitspath( self.get_crosscalsubdir_path(), self.fluxcal)) and convertfluxcalms2uvfits: subs_managefiles.director( self, 'rm', mspath_to_fitspath(self.get_crosscalsubdir_path(), self.fluxcal)) logger.info('Beam ' + self.beam + ': Removed fluxcal UVFITS files') else: logger.warning( 'Beam ' + self.beam + ': No fluxcal UVFITS file available for removing') if self.polcal != '' and path.exists( mspath_to_fitspath( self.get_crosscalsubdir_path(), self.polcal)) and convertpolcalms2uvfits: subs_managefiles.director( self, 'rm', mspath_to_fitspath(self.get_crosscalsubdir_path(), self.polcal)) logger.info('Beam ' + self.beam + ': Removed polcal UVFITS files') else: logger.warning( 'Beam ' + self.beam + ': No polcal UVFITS file available for removing') if self.target != '' and path.exists( mspath_to_fitspath( self.get_crosscalsubdir_path(), self.target)) and convertfluxcalms2uvfits: subs_managefiles.director( self, 'rm', mspath_to_fitspath(self.get_crosscalsubdir_path(), self.target)) logger.info('Beam ' + self.beam + ': Removed target UVFITS files') else: logger.warning( 'Beam ' + self.beam + ': No target UVFITS file available for removing')
def regrid_in_miriad(taskid, image_name, hdu_image, b, c): """ Find appropriate beam model and set center to center of image. Rescale the beam model to appropriate size for the center frequency of the cube. Regrid the beam model image in miriad to the HI image. Expand beam model imagine in 3D. """ # Change the reference pixel of beam model to reference pixel of image to correct cb_model = beam_lookup.model_lookup2(taskid, b) hdulist_cb = pyfits.open(cb_model) hdulist_cb[0].header['CRVAL1'] = hdu_image[0].header['CRVAL1'] hdulist_cb[0].header['CRVAL2'] = hdu_image[0].header['CRVAL2'] # Rescale to appropriate frequency. This should work for either drift scans or Gaussian regression (only tested on latter): avg_cube_freq = ( hdu_image[0].header['CRVAL3'] + hdu_image[0].header['CDELT3'] * hdu_image[0].data.shape[0]) * u.Hz hdulist_cb[0].header['CDELT1'] = (hdulist_cb[0].header['CDELT1'] * get_cb_model_freq().to(u.Hz) / avg_cube_freq).value hdulist_cb[0].header['CDELT2'] = (hdulist_cb[0].header['CDELT2'] * get_cb_model_freq().to(u.Hz) / avg_cube_freq).value cb2d_name = 'temp_b{}_c{}_cb-2d.fits'.format(b, c) hdulist_cb.writeto(cb2d_name) hdulist_cb.close() print('\tRegridding in miriad using model {}'.format(cb_model)) fits = lib.miriad('fits') regrid = lib.miriad('regrid') # Convert files to miriad: fits.in_ = image_name fits.out = '{}.mir'.format(image_name[:-5]) fits.op = 'xyin' fits.go() fits.in_ = cb2d_name fits.out = '{}.mir'.format(cb2d_name[:-5]) fits.op = 'xyin' fits.go() # Regrid beam image regrid.in_ = '{}.mir'.format(cb2d_name[:-5]) regrid.out = '{}_rgrid.mir'.format(cb2d_name[:-5]) regrid.tin = '{}.mir'.format(image_name[:-5]) regrid.axes = '1,2' regrid.go() # Convert regrided beam image to fits fits.in_ = '{}_rgrid.mir'.format(cb2d_name[:-5]) fits.out = '{}_rgrid.fits'.format(cb2d_name[:-5]) fits.op = 'xyout' fits.go() # Make cb 3D and save as FITS: hdu_cb = pyfits.open('{}_rgrid.fits'.format(cb2d_name[:-5])) d_new = np.ones((hdu_image[0].header['NAXIS3'], hdu_cb[0].header['NAXIS2'], hdu_cb[0].header['NAXIS2'])) d_beam_cube = d_new * hdu_cb[0].data hdu_cb[0].data = np.float32(d_beam_cube) print('\tWriting compound beam cube.') hdu_cb.writeto('{}_cb.fits'.format(image_name[:-5])) hdu_cb.close() # Clean up the extra Miriad & 2D cb files os.system('rm -rf {}*.mir'.format(image_name[:-5])) os.system('rm -rf {}*'.format(cb2d_name[:-5]))
def run2(i): global new_cleancube_data, new_modelcube_data, new_residualcube_data try: for name in [ 'map_{:02}'.format(minc), 'beam_{:02}'.format(minc), 'mask_{:02}'.format(minc) ]: imsub = lib.miriad('imsub') imsub.in_ = name imsub.out = name + "_" + str(chan[i]).zfill(4) imsub.region = '"images({})"'.format(chan[i] + 1) imsub.go() # print("[CLEAN2] Cleaning HI emission using SoFiA mask for Sources {}.".format(args.sources)) klean.map = 'map_{:02}_{:04}'.format(minc, chan[i]) klean.beam = 'beam_{:02}_{:04}'.format(minc, chan[i]) klean.out = 'model_{:02}_{:04}'.format(minc + 1, chan[i]) klean.cutoff = lineimagestats[2] * 0.5 klean.niters = 10000 klean.region = '"mask(mask_{:02}_{:04}/)"'.format(minc, chan[i]) klean.go() # print("[CLEAN2] Restoring line cube.") restor.model = 'model_{:02}_{:04}'.format(minc + 1, chan[i]) restor.beam = 'beam_{:02}_{:04}'.format(minc, chan[i]) restor.map = 'map_{:02}_{:04}'.format(minc, chan[i]) restor.out = 'image_{:02}_{:04}'.format(minc + 1, chan[i]) restor.mode = 'clean' restor.go() # print("[CLEAN2] Making residual cube.") out_array = ['image_{:02}_{:04}'.format(minc + 1, chan[i])] if args.all: restor.mode = 'residual' # Create the residual image restor.out = loc + 'residual_{:02}_{:04}'.format(minc + 1, chan[i]) restor.go() out_array = [ 'model_{:02}_{:04}'.format(minc + 1, chan[i]), 'image_{:02}_{:04}'.format(minc + 1, chan[i]), 'residual_{:02}_{:04}'.format(minc + 1, chan[i]) ] for name in out_array: fits.op = 'xyout' fits.in_ = name fits.out = name + '.fits' fits.go() new_cleancube_data[chan[i], :, :] = pyfits.getdata( 'image_{:02}_{:04}.fits'.format(minc + 1, chan[i])) if args.all: new_modelcube_data[chan[i], :, :] = pyfits.getdata( 'model_{:02}_{:04}.fits'.format(minc + 1, chan[i])) new_residualcube_data[chan[i], :, :] = pyfits.getdata( 'residual_{:02}_{:04}.fits'.format(minc + 1, chan[i])) except RuntimeError: print( "Channel {} probably all blank and klean failed. Continue to next." .format(chan[i])) return 'OK'
def flagline(self): """ Creates an image cube of the different chunks and measures the rms in each channel. All channels with an rms outside of a given sigma interval are flagged in the continuum calibration, but are still used for line imaging. """ if self.selfcal_flagline: subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) logger.info(' Automatic flagging of HI-line/RFI started') subs_managefiles.director(self, 'ch', self.selfcaldir) for chunk in self.list_chunks(): subs_managefiles.director(self, 'ch', self.selfcaldir + '/' + str(chunk)) logger.info('Looking through data chunk ' + str(chunk) + ' #') invert = lib.miriad('invert') invert.vis = chunk + '.mir' invert.map = 'map' invert.beam = 'beam' invert.imsize = self.selfcal_image_imsize invert.cell = self.selfcal_image_cellsize invert.stokes = 'ii' invert.slop = 1 invert.go() if os.path.exists('map'): fits = lib.miriad('fits') fits.in_ = 'map' fits.op = 'xyout' fits.out = 'map.fits' fits.go() cube = pyfits.open('map.fits') data = cube[0].data std = np.nanstd(data, axis=(0, 2, 3)) median = np.median(std) stdall = np.nanstd(std) diff = std - median detections = np.where( np.abs(self.selfcal_flagline_sigma * diff) > stdall)[0] if len(detections) > 0: logger.info('Found high noise in channel(s) ' + str(detections).lstrip('[').rstrip(']') + ' #') for d in detections: uvflag = lib.miriad('uvflag') uvflag.vis = chunk + '.mir' uvflag.flagval = 'flag' uvflag.line = "'" + 'channel,1,' + str(d + 1) + "'" uvflag.go() logger.info('Flagged channel(s) ' + str(detections).lstrip('[').rstrip(']') + ' in data chunk ' + str(chunk) + ' #') else: logger.info('No high noise found in data chunk ' + str(chunk) + ' #') subs_managefiles.director( self, 'rm', self.selfcaldir + '/' + str(chunk) + '/' + 'map') subs_managefiles.director( self, 'rm', self.selfcaldir + '/' + str(chunk) + '/' + 'map.fits') subs_managefiles.director( self, 'rm', self.selfcaldir + '/' + str(chunk) + '/' + 'beam') else: logger.info(' No data in chunk ' + str(chunk) + '!') logger.info(' Automatic flagging of HI-line/RFI done')
def get_measured_beam_maps(beam, beam_map_input_path, beam_map_output_path, beam_map_output_name, cutoff): """ Function to create beam map from drift scans Args: input: beam = beam number (int, e.g. 01) image_path = path to mir image beam_map_path = path to the directory with measured beam maps (/data5/apertif/driftscans/fits_files/191023/) output_name = output name of regridded beam map """ work_dir = os.getcwd() # file name of the beam model fits file which will be copied input_beam_model = os.path.join( beam_map_input_path, "{0}_{1}_I_model.fits".format(beam_map_input_path.split("/")[5], beam)) # file name of the fits file after copying temp_beam_model_name = os.path.join( beam_map_output_path, 'beam_model_{0}_temp.fits'.format(beam)) # file name of the miriad file before applying the cutoff temp_beam_map_output_name = beam_map_output_name.replace( ".map", "_temp.map") # copy beam model to work directory logger.debug("Copying beam model of beam {0} ({1} -> {2})".format( beam, input_beam_model, temp_beam_model_name)) # copy_cmd = 'cp -r {0} {1}'.format(input_beam_model, temp_beam_model_name) # copy_cmd = 'cp -r {0}191023_{1}_I_model.fits {2}'.format( # beam_map_input_path, beam, temp_beam_model_name) # logger.debug(copy_cmd) # switched to python command shutil.copy2(input_beam_model, temp_beam_model_name) # os.system(copy_cmd) # convert beam model to mir file # but only if it does not exists # if os.path.isdir('./beam_model_{0}_temp.mir'.format(beam)) == False: if os.path.isdir(os.path.join(beam_map_output_path, beam_map_output_name)) is False: logger.debug("Converting beam model of beam {}".format(beam)) fits = lib.miriad('fits') # fits.in_ = './beam_model_{0}_temp.fits'.format(beam) fits.in_ = temp_beam_model_name fits.op = 'xyin' # fits.out = './beam_model_{0}_temp.mir'.format(beam) fits.out = os.path.join(beam_map_output_path, temp_beam_map_output_name) fits.go() # now apply the cutoff logger.debug("Applying cutoff of {0} to beam model of beam {1}".format( cutoff, beam)) beam_cutoff(beam_map_output_name, temp_beam_map_output_name, beam_map_output_path, cutoff) # if os.path.isdir('{}.mir'.format(image_path[:-5])) == False: # fits = lib.miriad('fits') # fits.in_ = image_path # fits.op = 'xyin' # fits.out = '{}.mir'.format(image_path[:-5]) # fits.go() # replace centre of beam_model # gethd = lib.miriad('gethd') # gethd.in_ = '{0}.mir/crval1'.format(image_path[:-5]) # ra_ref=gethd.go() # gethd.in_ = '{0}.mir/crval2'.format(image_path[:-5]) # dec_ref=gethd.go() # print(ra_ref, dec_ref) # puthd = lib.miriad('puthd') # puthd.in_ = './beam_model_{0}_temp.mir/crval1'.format(beam) # puthd.value = ra_ref[0] # puthd.type = 'double' # puthd.go() # puthd.in_ = './beam_model_{0}_temp.mir/crval2'.format(beam) # puthd.value = dec_ref[0] # puthd.type = 'double' # puthd.go() # # regrid beam model # if os.path.isdir('./beam_model_{0}.mir'.format(beam)) == False: # regrid = lib.miriad('regrid') # regrid.in_ = './beam_model_{0}_temp.mir'.format(beam) # regrid.out = './{0}_{1}.mir'.format(output_name, beam) # regrid.axes = '1,2' # regrid.tin = '{0}.mir'.format(image_path[:-5]) # regrid.go() # print('DONE') # clean temp file logger.debug("Removing fits file of beam model of beam {}".format(beam)) os.remove(temp_beam_model_name)
def ms2miriad(self): """ Converts the data from MS to MIRIAD format via UVFITS using drivecasa. Does it for the flux calibrator, polarisation calibrator, and target field independently. """ subs_setinit.setinitdirs(self) nbeams = 37 # Create the parameters for the parameter file for converting from MS to UVFITS format # Flux calibrator MS dataset available? convertfluxcalmsavailable = get_param_def(self, 'convert_fluxcal_MSavailable', False) # Polarised calibrator MS dataset available? convertpolcalmsavailable = get_param_def(self, 'convert_polcal_MSavailable', False) # Target beam MS dataset available? converttargetbeamsmsavailable = get_param_def(self, 'convert_targetbeams_MSavailable', np.full(nbeams, False)) # Flux calibrator MS dataset converted to UVFITS? convertfluxcalms2uvfits = get_param_def(self, 'convert_fluxcal_MS2UVFITS', False) # Polarised calibrator MS dataset converted to UVFITS? convertpolcalms2uvfits = get_param_def(self, 'convert_polcal_MS2UVFITS', False) # Target beam MS dataset converted to UVFITS? converttargetbeamsms2uvfits = get_param_def(self, 'convert_targetbeams_MS2UVFITS', np.full(nbeams, False)) # Flux calibrator UVFITS dataset available? convertfluxcaluvfitsavailable = get_param_def(self, 'convert_fluxcal_UVFITSavailable', False) # Polarised calibrator UVFITS dataset available? convertpolcaluvfitsavailable = get_param_def(self, 'convert_polcal_UVFITSavailable', False) # Target beam UVFITS dataset available? converttargetbeamsuvfitsavailable = get_param_def(self, 'convert_targetbeams_UVFITSavailable', np.full(nbeams, False)) # Flux calibrator UVFITS dataset converted to MIRIAD? convertfluxcaluvfits2miriad = get_param_def(self, 'convert_fluxcal_UVFITS2MIRIAD', False) # Polarised calibrator UVFITS dataset converted to MIRIAD? convertpolcaluvfits2miriad = get_param_def(self, 'convert_polcal_UVFITS2MIRIAD', False) # Target beam UVFITS dataset converted to MIRIAD? converttargetbeamsuvfits2miriad = get_param_def(self, 'convert_targetbeams_UVFITS2MIRIAD', np.full(nbeams, False)) # Check which datasets are available in MS format # if self.fluxcal != '': convertfluxcalmsavailable = os.path.isdir(self.basedir + '00' + '/' + self.rawsubdir + '/' + self.fluxcal) else: logger.warning('Flux calibrator dataset not specified. Cannot convert flux calibrator!') if self.polcal != '': convertpolcalmsavailable = os.path.isdir(self.basedir + '00' + '/' + self.rawsubdir + '/' + self.polcal) else: logger.warning('Polarised calibrator dataset not specified. Cannot convert polarised calibrator!') if self.target != '': for b in range(nbeams): converttargetbeamsmsavailable[b] = os.path.isdir( self.basedir + str(b).zfill(2) + '/' + self.rawsubdir + '/' + self.target) else: logger.warning('Target beam dataset not specified. Cannot convert target beams!') # Save the derived parameters for the availability to the parameter file subs_param.add_param(self, 'convert_fluxcal_MSavailable', convertfluxcalmsavailable) subs_param.add_param(self, 'convert_polcal_MSavailable', convertpolcalmsavailable) subs_param.add_param(self, 'convert_targetbeams_MSavailable', converttargetbeamsmsavailable) # Convert the available MS-datasets to UVFITS # raw_convert_cmd = 'exportuvfits(vis="{basedir}00/{rawsubdir}/{cal}", ' \ 'fitsfile="{basedir}00/{crosscalsubdir}/{calbase}UVFITS", datacolumn="{datacolumn}", ' \ 'combinespw=True, padwithflags=True, multisource=True, writestation=True)' # Convert the flux calibrator if self.convert_fluxcal: if self.fluxcal != '': if not convertfluxcaluvfits2miriad: if convertfluxcalmsavailable: logger.debug('Converting flux calibrator dataset from MS to UVFITS format.') subs_managefiles.director(self, 'mk', self.basedir + '00' + '/' + self.crosscalsubdir, verbose=False) path = self.basedir + '00' + '/' + self.rawsubdir + '/' + self.fluxcal if subs_msutils.has_correcteddata(path): fc_convert = raw_convert_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir, cal=self.fluxcal, calbase=self.fluxcal[:-2], crosscalsubdir=self.crosscalsubdir, datacolumn="corrected") else: fc_convert = raw_convert_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir, cal=self.fluxcal, calbase=self.fluxcal[:-2], crosscalsubdir=self.crosscalsubdir, datacolumn="data") logger.warning('Flux calibrator does not have a corrected_data column! Using uncorrected' 'data for conversion!') lib.run_casa([fc_convert], timeout=3600) if os.path.isfile(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip( 'MS') + 'UVFITS'): convertfluxcalms2uvfits = True logger.info('Converted flux calibrator dataset from MS to UVFITS format!') else: convertfluxcalms2uvfits = False logger.warning('Could not convert flux calibrator dataset from MS to UVFITS format!') else: logger.warning('Flux calibrator dataset not available!') else: logger.info('Flux calibrator dataset was already converted from MS to UVFITS format') else: logger.warning('Flux calibrator dataset not specified. Cannot convert flux calibrator!') else: logger.warning('Not converting flux calibrator dataset!') # Convert the polarised calibrator if self.convert_polcal: if self.polcal != '': if not convertpolcaluvfits2miriad: if convertpolcalmsavailable: logger.debug('Converting polarised calibrator dataset from MS to UVFITS format.') subs_managefiles.director(self, 'mk', self.basedir + '00' + '/' + self.crosscalsubdir, verbose=False) path = self.basedir + '00' + '/' + self.rawsubdir + '/' + self.polcal if subs_msutils.has_correcteddata(path): pc_convert = raw_convert_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir, cal=self.fluxcal, calbase=self.polcal[:-2], crosscalsubdir=self.crosscalsubdir, datacolumn="corrected") else: pc_convert = raw_convert_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir, cal=self.fluxcal, calbase=self.polcal[:-2], crosscalsubdir=self.crosscalsubdir, datacolumn="data") logger.warning('Polarised calibrator does not have a corrected_data column! Using' 'uncorrected data for conversion!') lib.run_casa([pc_convert], timeout=3600) if os.path.isfile(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip( 'MS') + 'UVFITS'): convertpolcalms2uvfits = True logger.info('Converted polarised calibrator dataset from MS to UVFITS format!') else: convertpolcalms2uvfits = False logger.warning('Could not convert polarised calibrator dataset from MS to UVFITS format!') else: logger.warning('Polarised calibrator dataset not available!') else: logger.info('Polarised calibrator dataset was already converted from MS to UVFITS format') else: logger.warning('Polarised calibrator dataset not specified. Cannot convert polarised calibrator!') else: logger.warning('Not converting polarised calibrator dataset!') # Convert the target beams if self.convert_target: if self.target != '': logger.info('Converting target beam datasets from MS to UVFITS format.') if self.convert_targetbeams == 'all': datasets = glob.glob(self.basedir + '[0-9][0-9]' + '/' + self.rawsubdir + '/' + self.target) logger.debug('Converting all available target beam datasets') else: beams = self.convert_targetbeams.split(",") datasets = [self.basedir + str(b).zfill(2) + '/' + self.rawsubdir + '/' + self.target for b in beams] logger.debug('Converting all selected target beam datasets') for vis in datasets: if not converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])]: if converttargetbeamsmsavailable[int(vis.split('/')[-3])]: subs_managefiles.director(self, 'mk', self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir, verbose=False) beam_dataset = vis.split('/')[-3] raw_tg_cmd = 'exportuvfits(vis="{basedir}{beam_dataset}/{rawsubdir}/{target}", ' \ 'fitsfile="{basedir}{beam_dataset}/{crosscalsubdir}/{targetbase}UVFITS", ' \ 'datacolumn="{datacolumn}", combinespw=True, padwithflags=True, ' \ 'multisource=True, writestation=True)' path = self.basedir + beam_dataset + '/' + self.rawsubdir + '/' + self.target if subs_msutils.has_correcteddata(path): datacolumn = "corrected" else: datacolumn = "data" logger.warning('Target beam dataset {} does not have a corrected_data column! Using ' 'uncorrected data for conversion!'.format(beam_dataset)) tg_convert = raw_tg_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir, target=self.target, crosscalsubdir=self.crosscalsubdir, targetbase=self.target.rstrip('MS'), datacolumn=datacolumn, beam_dataset=beam_dataset) lib.run_casa([tg_convert], timeout=7200) if os.path.isfile(self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS'): converttargetbeamsms2uvfits[int(vis.split('/')[-3])] = True logger.debug('Converted dataset of target beam ' + vis.split('/')[ -3] + ' from MS to UVFITS format!') else: converttargetbeamsms2uvfits[int(vis.split('/')[-3])] = False logger.warning('Could not convert dataset for target beam ' + vis.split('/')[ -3] + ' from MS to UVFITS format!') else: logger.warning('Dataset for target beam ' + vis.split('/')[-3] + ' not available!') else: logger.info('Dataset for target beam ' + vis.split('/')[ -3] + ' was already converted from MS to UVFITS format') else: logger.warning('Target beam dataset(s) not specified. Cannot convert target beam datasets!') else: logger.warning('Not converting target beam dataset(s)!') # Save the derived parameters for the MS to UVFITS conversion to the parameter file subs_param.add_param(self, 'convert_fluxcal_MS2UVFITS', convertfluxcalms2uvfits) subs_param.add_param(self, 'convert_polcal_MS2UVFITS', convertpolcalms2uvfits) subs_param.add_param(self, 'convert_targetbeams_MS2UVFITS', converttargetbeamsms2uvfits) # Check which datasets are available in UVFITS format # if self.fluxcal != '': convertfluxcaluvfitsavailable = os.path.isfile( self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip('MS') + 'UVFITS') else: logger.warning('Flux calibrator dataset not specified. Cannot convert flux calibrator!') if self.polcal != '': convertpolcaluvfitsavailable = os.path.isfile( self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip('MS') + 'UVFITS') else: logger.warning('Polarised calibrator dataset not specified. Cannot convert polarised calibrator!') if self.target != '': for b in range(nbeams): converttargetbeamsuvfitsavailable[b] = os.path.isfile( self.basedir + str(b).zfill(2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip( 'MS') + 'UVFITS') else: logger.warning('Target beam dataset not specified. Cannot convert target beams!') # Save the derived parameters for the availability to the parameter file subs_param.add_param(self, 'convert_fluxcal_UVFITSavailable', convertfluxcaluvfitsavailable) subs_param.add_param(self, 'convert_polcal_UVFITSavailable', convertpolcaluvfitsavailable) subs_param.add_param(self, 'convert_targetbeams_UVFITSavailable', converttargetbeamsuvfitsavailable) # Convert the available UVFITS-datasets to MIRIAD format # # Convert the flux calibrator if self.convert_fluxcal: if self.fluxcal != '': if not convertfluxcaluvfits2miriad: if convertfluxcaluvfitsavailable: logger.debug('Converting flux calibrator dataset from UVFITS to MIRIAD format.') subs_managefiles.director(self, 'ch', self.basedir + '00' + '/' + self.crosscalsubdir, verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip( 'MS') + 'UVFITS' fits.out = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip( 'MS') + 'mir' fits.go() if os.path.isdir(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip( 'MS') + 'mir'): convertfluxcaluvfits2miriad = True logger.info('Converted flux calibrator dataset from UVFITS to MIRIAD format!') else: convertfluxcaluvfits2miriad = False logger.warning('Could not convert flux calibrator dataset from UVFITS to MIRIAD format!') else: logger.warning('Flux calibrator dataset not available!') else: logger.info('Flux calibrator dataset was already converted from UVFITS to MIRIAD format') else: logger.warning('Flux calibrator dataset not specified. Cannot convert flux calibrator!') else: logger.warning('Not converting flux calibrator dataset!') # Convert the polarised calibrator if self.convert_polcal: if self.polcal != '': if not convertpolcaluvfits2miriad: if convertpolcaluvfitsavailable: logger.debug('Converting polarised calibrator dataset from UVFITS to MIRIAD format.') subs_managefiles.director(self, 'ch', self.basedir + '00' + '/' + self.crosscalsubdir, verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip( 'MS') + 'UVFITS' fits.out = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip( 'MS') + 'mir' fits.go() if os.path.isdir(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip( 'MS') + 'mir'): convertpolcaluvfits2miriad = True logger.info('Converted polarised calibrator dataset from UVFITS to MIRIAD format!') else: convertpolcaluvfits2miriad = False logger.warning( 'Could not convert polarised calibrator dataset from UVFITS to MIRIAD format!') else: logger.warning('Polarised calibrator dataset not available!') else: logger.info('Polarised calibrator dataset was already converted from UVFITS to MIRIAD format') else: logger.warning('Polarised calibrator dataset not specified. Cannot convert polarised calibrator!') else: logger.warning('Not converting polarised calibrator dataset!') # Convert the target beams if self.convert_target: if self.target != '': logger.info('Converting target beam datasets from UVFITS to MIRIAD format.') if self.convert_targetbeams == 'all': datasets = glob.glob( self.basedir + '[0-9][0-9]' + '/' + self.crosscalsubdir + '/' + self.target.rstrip( 'MS') + 'UVFITS') logger.debug('Converting all available target beam datasets') else: beams = self.convert_targetbeams.split(",") datasets = [self.basedir + str(b).zfill(2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip( 'MS') + 'UVFITS' for b in beams] logger.debug('Converting all selected target beam datasets') for vis in datasets: if not converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])]: if converttargetbeamsuvfitsavailable[int(vis.split('/')[-3])]: subs_managefiles.director(self, 'ch', self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir, verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = self.basedir + vis.split('/')[ -3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS' fits.out = self.basedir + vis.split('/')[ -3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'mir' fits.go() if os.path.isdir(self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'mir'): converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])] = True logger.debug('Converted dataset of target beam ' + vis.split('/')[ -3] + ' from UVFITS to MIRIAD format!') else: converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])] = False logger.warning('Could not convert dataset for target beam ' + vis.split('/')[ -3] + ' from UVFITS to MIRIAD format!') else: logger.warning('Dataset for target beam ' + vis.split('/')[-3] + ' not available!') else: logger.info('Dataset for target beam ' + vis.split('/')[ -3] + ' was already converted from MS to UVFITS format') else: logger.warning('Target beam dataset(s) not specified. Cannot convert target beam datasets!') else: logger.warning('Not converting target beam dataset(s)!') # Save the derived parameters for the MS to UVFITS conversion to the parameter file subs_param.add_param(self, 'convert_fluxcal_UVFITS2MIRIAD', convertfluxcaluvfits2miriad) subs_param.add_param(self, 'convert_polcal_UVFITS2MIRIAD', convertpolcaluvfits2miriad) subs_param.add_param(self, 'convert_targetbeams_UVFITS2MIRIAD', converttargetbeamsuvfits2miriad) # Remove the UVFITS files if wanted # if self.convert_removeuvfits: logger.info('Removing all UVFITS files') subs_managefiles.director(self, 'rm', self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip( 'MS') + 'UVFITS') subs_managefiles.director(self, 'rm', self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip( 'MS') + 'UVFITS') for beam in range(nbeams): if os.path.isdir(self.basedir + str(beam).zfill(2) + '/' + self.crosscalsubdir): subs_managefiles.director(self, 'rm', self.basedir + str(beam).zfill( 2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS') else: pass
def ms2miriad(self): """ Converts the data from MS to MIRIAD format via UVFITS using drivecasa. Does it for the flux calibrator, polarisation calibrator, and target field independently. """ subs_setinit.setinitdirs(self) nbeams = 37 # Create the parameters for the parameter file for converting from MS to UVFITS format # Flux calibrator MS dataset available? convertfluxcalmsavailable = get_param_def( self, 'convert_fluxcal_MSavailable', False) # Polarised calibrator MS dataset available? convertpolcalmsavailable = get_param_def(self, 'convert_polcal_MSavailable', False) # Target beam MS dataset available? converttargetbeamsmsavailable = get_param_def( self, 'convert_targetbeams_MSavailable', np.full(nbeams, False)) # Flux calibrator MS dataset converted to UVFITS? convertfluxcalms2uvfits = get_param_def(self, 'convert_fluxcal_MS2UVFITS', False) # Polarised calibrator MS dataset converted to UVFITS? convertpolcalms2uvfits = get_param_def(self, 'convert_polcal_MS2UVFITS', False) # Target beam MS dataset converted to UVFITS? converttargetbeamsms2uvfits = get_param_def( self, 'convert_targetbeams_MS2UVFITS', np.full(nbeams, False)) # Flux calibrator UVFITS dataset available? convertfluxcaluvfitsavailable = get_param_def( self, 'convert_fluxcal_UVFITSavailable', False) # Polarised calibrator UVFITS dataset available? convertpolcaluvfitsavailable = get_param_def( self, 'convert_polcal_UVFITSavailable', False) # Target beam UVFITS dataset available? converttargetbeamsuvfitsavailable = get_param_def( self, 'convert_targetbeams_UVFITSavailable', np.full(nbeams, False)) # Flux calibrator UVFITS dataset converted to MIRIAD? convertfluxcaluvfits2miriad = get_param_def( self, 'convert_fluxcal_UVFITS2MIRIAD', False) # Polarised calibrator UVFITS dataset converted to MIRIAD? convertpolcaluvfits2miriad = get_param_def( self, 'convert_polcal_UVFITS2MIRIAD', False) # Target beam UVFITS dataset converted to MIRIAD? converttargetbeamsuvfits2miriad = get_param_def( self, 'convert_targetbeams_UVFITS2MIRIAD', np.full(nbeams, False)) # Check which datasets are available in MS format # if self.fluxcal != '': convertfluxcalmsavailable = path.isdir(self.get_fluxcal_path()) else: logger.warning( 'Flux calibrator dataset not specified. Cannot convert flux calibrator!' ) if self.polcal != '': convertpolcalmsavailable = path.isdir(self.get_polcal_path()) else: logger.warning( 'Polarised calibrator dataset not specified. Cannot convert polarised calibrator!' ) if self.target != '': for b in range(nbeams): converttargetbeamsmsavailable[b] = path.isdir( self.get_target_path(str(b).zfill(2))) else: logger.warning( 'Target beam dataset not specified. Cannot convert target beams!' ) # Save the derived parameters for the availability to the parameter file subs_param.add_param(self, 'convert_fluxcal_MSavailable', convertfluxcalmsavailable) subs_param.add_param(self, 'convert_polcal_MSavailable', convertpolcalmsavailable) subs_param.add_param(self, 'convert_targetbeams_MSavailable', converttargetbeamsmsavailable) # Convert the flux calibrator if self.convert_fluxcal: if self.fluxcal != '': if not convertfluxcaluvfits2miriad: if convertfluxcalmsavailable: logger.debug( 'Converting flux calibrator dataset from MS to UVFITS format.' ) subs_managefiles.director( self, 'mk', self.get_crosscalsubdir_path(), verbose=False) fluxcal_ms = self.get_fluxcal_path() if subs_msutils.has_correcteddata(fluxcal_ms): datacolumn = "corrected" else: datacolumn = "data" logger.warning( 'Flux calibrator does not have a corrected_data column! Using uncorrected' 'data for conversion!') fluxcal_fits = mspath_to_fitspath( self.get_crosscalsubdir_path(), fluxcal_ms) fc_convert = exportuvfits_cmd.format( vis=self.get_fluxcal_path(), fits=fluxcal_fits, datacolumn=datacolumn) lib.run_casa([fc_convert], timeout=3600) if path.isfile(fluxcal_fits): convertfluxcalms2uvfits = True logger.info( 'Converted flux calibrator dataset from MS to UVFITS format!' ) else: convertfluxcalms2uvfits = False logger.warning( 'Could not convert flux calibrator dataset from MS to UVFITS format!' ) else: logger.warning( 'Flux calibrator dataset not available!') else: logger.info( 'Flux calibrator dataset was already converted from MS to UVFITS format' ) else: logger.warning( 'Flux calibrator dataset not specified. Cannot convert flux calibrator!' ) else: logger.warning('Not converting flux calibrator dataset!') # Convert the polarised calibrator if self.convert_polcal: if self.polcal != '': if not convertpolcaluvfits2miriad: if convertpolcalmsavailable: logger.debug( 'Converting polarised calibrator dataset from MS to UVFITS format.' ) subs_managefiles.director( self, 'mk', self.get_crosscalsubdir_path(), verbose=False) polcal_ms = self.get_polcal_path() if subs_msutils.has_correcteddata(polcal_ms): datacolumn = "corrected" else: datacolumn = "data" logger.warning( 'Polarised calibrator does not have a corrected_data column! Using' 'uncorrected data for conversion!') polcal_fits = mspath_to_fitspath( self.get_crosscalsubdir_path(), polcal_ms) pc_convert = exportuvfits_cmd.format( vis=polcal_ms, fits=polcal_fits, datacolumn=datacolumn) lib.run_casa([pc_convert], timeout=3600) if path.isfile(polcal_fits): convertpolcalms2uvfits = True logger.info( 'Converted polarised calibrator dataset from MS to UVFITS format!' ) else: convertpolcalms2uvfits = False logger.warning( 'Could not convert polarised calibrator dataset from MS to UVFITS format!' ) else: logger.warning( 'Polarised calibrator dataset not available!') else: logger.info( 'Polarised calibrator dataset was already converted from MS to UVFITS format' ) else: logger.warning( 'Polarised calibrator dataset not specified. Cannot convert polarised calibrator!' ) else: logger.warning('Not converting polarised calibrator dataset!') # Convert the target beams if self.convert_target: if self.target != '': logger.info( 'Converting target beam datasets from MS to UVFITS format.' ) if self.convert_targetbeams == 'all': datasets = self.get_datasets() logger.debug( 'Converting all available target beam datasets') else: beams = self.convert_targetbeams.split(",") datasets = self.get_datasets(beams) logger.debug( 'Converting all selected target beam datasets') for vis, beam in datasets: if not converttargetbeamsuvfits2miriad[int(beam)]: if converttargetbeamsmsavailable[int(beam)]: subs_managefiles.director( self, 'mk', self.get_crosscalsubdir_path(beam), verbose=False) target_ms = self.get_target_path(beam) target_fits = mspath_to_fitspath( self.get_crosscalsubdir_path(beam), target_ms) if subs_msutils.has_correcteddata(target_ms): datacolumn = "corrected" else: datacolumn = "data" logger.warning( 'Target beam dataset {} does not have a corrected_data column! Using ' 'uncorrected data for conversion!'.format( beam)) cmd = exportuvfits_cmd.format( vis=target_ms, fits=target_fits, beam=beam, datacolumn=datacolumn) lib.run_casa([cmd], timeout=7200) if path.isfile(target_fits): converttargetbeamsms2uvfits[int(beam)] = True logger.debug( 'Converted dataset of target beam ' 'l{} from MS to UVFITS format!'.format( beam)) else: converttargetbeamsms2uvfits[int(beam)] = False logger.warning( 'Could not convert dataset for target beam ' '{} from MS to UVFITS format!'.format( beam)) else: logger.warning( 'Dataset for target beam {} not available!'. format(beam)) else: logger.info( 'Dataset for target beam {} was already ' 'converted from MS to UVFITS format'.format(beam)) else: logger.warning( 'Target beam dataset(s) not specified. Cannot convert target beam datasets!' ) else: logger.warning('Not converting target beam dataset(s)!') # Save the derived parameters for the MS to UVFITS conversion to the parameter file subs_param.add_param(self, 'convert_fluxcal_MS2UVFITS', convertfluxcalms2uvfits) subs_param.add_param(self, 'convert_polcal_MS2UVFITS', convertpolcalms2uvfits) subs_param.add_param(self, 'convert_targetbeams_MS2UVFITS', converttargetbeamsms2uvfits) # Check which datasets are available in UVFITS format # if self.fluxcal != '': crosscal_fluxcal = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.fluxcal) convertfluxcaluvfitsavailable = path.isfile(crosscal_fluxcal) else: logger.warning( 'Flux calibrator dataset not specified. Cannot convert flux calibrator!' ) if self.polcal != '': crosscal_polcal = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.polcal) convertpolcaluvfitsavailable = path.isfile(crosscal_polcal) else: logger.warning( 'Polarised calibrator dataset not specified. Cannot convert polarised calibrator!' ) if self.target != '': for b in range(nbeams): b_formatted = str(b).zfill(2) converttargetbeamsuvfitsavailable[b] = path.isfile( mspath_to_fitspath( self.get_crosscalsubdir_path(b_formatted), self.target)) else: logger.warning( 'Target beam dataset not specified. Cannot convert target beams!' ) # Save the derived parameters for the availability to the parameter file subs_param.add_param(self, 'convert_fluxcal_UVFITSavailable', convertfluxcaluvfitsavailable) subs_param.add_param(self, 'convert_polcal_UVFITSavailable', convertpolcaluvfitsavailable) subs_param.add_param(self, 'convert_targetbeams_UVFITSavailable', converttargetbeamsuvfitsavailable) # Convert the available UVFITS-datasets to MIRIAD format # # Convert the flux calibrator if self.convert_fluxcal: if self.fluxcal != '': if not convertfluxcaluvfits2miriad: if convertfluxcaluvfitsavailable: logger.debug( 'Converting flux calibrator dataset from UVFITS to MIRIAD format.' ) subs_managefiles.director( self, 'ch', self.get_crosscalsubdir_path(), verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.fluxcal) fits.out = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.fluxcal, ext='mir') fits.go() if path.isdir(fits.out): convertfluxcaluvfits2miriad = True logger.info( 'Converted flux calibrator dataset from UVFITS to MIRIAD format!' ) else: convertfluxcaluvfits2miriad = False logger.warning( 'Could not convert flux calibrator dataset from UVFITS to MIRIAD format!' ) else: logger.warning( 'Flux calibrator dataset not available!') else: logger.info( 'Flux calibrator dataset was already converted from UVFITS to MIRIAD format' ) else: logger.warning( 'Flux calibrator dataset not specified. Cannot convert flux calibrator!' ) else: logger.warning('Not converting flux calibrator dataset!') # Convert the polarised calibrator if self.convert_polcal: if self.polcal != '': if not convertpolcaluvfits2miriad: if convertpolcaluvfitsavailable: logger.debug( 'Converting polarised calibrator dataset from UVFITS to MIRIAD format.' ) subs_managefiles.director( self, 'ch', self.get_crosscalsubdir_path(), verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.polcal) fits.out = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.polcal, ext='mir') fits.go() if path.isdir(fits.out): convertpolcaluvfits2miriad = True logger.info( 'Converted polarised calibrator dataset from UVFITS to MIRIAD format!' ) else: convertpolcaluvfits2miriad = False logger.warning( 'Could not convert polarised calibrator dataset from UVFITS to MIRIAD format!' ) else: logger.warning( 'Polarised calibrator dataset not available!') else: logger.info( 'Polarised calibrator dataset was already converted from UVFITS to MIRIAD format' ) else: logger.warning( 'Polarised calibrator dataset not specified. Cannot convert polarised calibrator!' ) else: logger.warning('Not converting polarised calibrator dataset!') # Convert the target beams if self.convert_target: if self.target != '': logger.info( 'Converting target beam datasets from UVFITS to MIRIAD format.' ) if self.convert_targetbeams == 'all': datasets = glob.glob( mspath_to_fitspath( self.get_crosscalsubdir_path('[0-9][0-9]'), self.get_target_path())) logger.debug( 'Converting all available target beam datasets') else: beams = self.convert_targetbeams.split(",") datasets = [ mspath_to_fitspath( self.get_crosscalsubdir_path(str(b).zfill(2)), self.target) for b in beams ] logger.debug( 'Converting all selected target beam datasets') for vis in datasets: beam = vis.split('/')[-3] if not converttargetbeamsuvfits2miriad[int(beam)]: if converttargetbeamsuvfitsavailable[int(beam)]: subs_managefiles.director( self, 'ch', self.get_crosscalsubdir_path(beam), verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = mspath_to_fitspath( self.get_crosscalsubdir_path(beam), self.target) fits.out = mspath_to_fitspath( self.get_crosscalsubdir_path(beam), self.target, ext='mir') fits.go() if path.isdir(fits.out): converttargetbeamsuvfits2miriad[int( beam)] = True logger.debug( 'Converted dataset of target beam {} from ' 'UVFITS to MIRIAD format!'.format(beam)) else: converttargetbeamsuvfits2miriad[int( beam)] = False logger.warning( 'Could not convert dataset for target beam ' '{} from UVFITS to MIRIAD format!'.format( beam)) else: logger.warning( 'Dataset for target beam {} not available!'. format(beam)) else: logger.info( 'Dataset for target beam {} was already converted ' 'from MS to UVFITS format'.format(beam)) else: logger.warning( 'Target beam dataset(s) not specified. Cannot convert target beam datasets!' ) else: logger.warning('Not converting target beam dataset(s)!') # Save the derived parameters for the MS to UVFITS conversion to the parameter file subs_param.add_param(self, 'convert_fluxcal_UVFITS2MIRIAD', convertfluxcaluvfits2miriad) subs_param.add_param(self, 'convert_polcal_UVFITS2MIRIAD', convertpolcaluvfits2miriad) subs_param.add_param(self, 'convert_targetbeams_UVFITS2MIRIAD', converttargetbeamsuvfits2miriad) # Remove the UVFITS files if wanted # if self.convert_removeuvfits: logger.info('Removing all UVFITS files') subs_managefiles.director( self, 'rm', mspath_to_fitspath(self.get_crosscalsubdir_path(), self.fluxcal)) subs_managefiles.director( self, 'rm', mspath_to_fitspath(self.get_crosscalsubdir_path(), self.polcal)) for beam in range(nbeams): basedir = self.get_crosscalsubdir_path(str(beam).zfill(2)) if path.isdir(basedir): subs_managefiles.director(self, 'rm', basedir, self.target)
def mosaic_continuum_mf(self): """Looks for all available stacked continuum images and mosaics them into one large image.""" subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) ########################################################################################################## # Check if the parameter is already in the parameter file and load it otherwise create the needed arrays # ########################################################################################################## mosaiccontinuummfstatus = get_param_def( self, 'mosaic_continuum_mf_status', False) # Status of the continuum mf mosaic mosaiccontinuummfcontinuumstatus = get_param_def( self, 'mosaic_continuum_mf_continuumstatus', np.full(self.NBEAMS, False)) # Status of the continuum imaging mosaiccontinuummfcopystatus = get_param_def( self, 'mosaic_continuum_mf_copystatus', np.full(self.NBEAMS, False)) # Status of the copy of the images mosaiccontinuummfconvolstatus = get_param_def( self, 'mosaic_continuum_mf_convolstatus', np.full(self.NBEAMS, False)) # Status of the convolved images mosaiccontinuummfcontinuumbeamparams = get_param_def( self, 'mosaic_continuum_mf_continuumbeamparams', np.full((self.NBEAMS, 3), np.nan)) # Beam sizes of the input images mosaiccontinuummfcontinuumimagestats = get_param_def( self, 'mosaic_continuum_mf_continuumimagestats', np.full((self.NBEAMS, 3), np.nan)) # Image statistics of the input images # Start the mosaicking of the stacked continuum images if self.mosaic_continuum_mf: subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) subs_managefiles.director(self, 'ch', self.mosdir + '/continuum') if not mosaiccontinuummfstatus: logger.info('Mosaicking multi-frequency continuum images') # Acquire the results and statistics from continuum mf imaging for b in range(self.NBEAMS): mosaiccontinuummfcontinuumstatus[b] = get_param_def( self, 'continuum_B' + str(b).zfill(2) + '_targetbeams_mf_status', False) if mosaiccontinuummfcontinuumstatus[b]: finalminor = get_param_def( self, 'continuum_B' + str(b).zfill(2) + '_targetbeams_mf_final_minorcycle', np.nan) subs_managefiles.director( self, 'cp', str(b).zfill(2) + '.fits', file_=self.basedir + str(b).zfill(2) + '/' + self.contsubdir + '/' + 'image_mf_' + str(finalminor).zfill(2) + '.fits') if os.path.isfile(str(b).zfill(2) + '.fits'): mosaiccontinuummfcopystatus[b] = True subs_convim.fitstomir( str(b).zfill(2) + '.fits', str(b).zfill(2)) subs_managefiles.director( self, 'rm', str(b).zfill(2) + '.fits') else: mosaiccontinuummfcopystatus[b] = False logger.warning('Beam ' + str(b).zfill(2) + ' was not copied successfully!') # Copy the images over to the mosaic directory for b in range(self.NBEAMS): if mosaiccontinuummfcontinuumstatus[ b] and mosaiccontinuummfcopystatus[b]: # Get the image beam parameters and the image statistics mosaiccontinuummfcontinuumimagestats[ b, :] = subs_imstats.getimagestats( self, str(b).zfill(2)) mosaiccontinuummfcontinuumbeamparams[ b, :] = subs_readmirhead.getbeamimage( str(b).zfill(2)) else: logger.warning( 'Skipping Beam ' + str(b).zfill(2) + '! Continuum mf-imaging was not successful or continuum image not available!' ) # Calculate the synthesised beam and reject outliers (algorithm needs to be updated) rejbeams, beamparams = subs_combim.calc_synbeam( mosaiccontinuummfcontinuumbeamparams) # Convolve all the images to the calculated beam for b in range(self.NBEAMS): if mosaiccontinuummfcontinuumstatus[ b] and mosaiccontinuummfcopystatus[b]: try: convol = lib.miriad('convol') convol.map = str(b).zfill(2) convol.fwhm = str(beamparams[0]) + ',' + str( beamparams[1]) convol.pa = str(beamparams[2]) convol.options = 'final' convol.out = str(b).zfill(2) + '_cv' convol.go() if os.path.isdir(str(b).zfill(2) + '_cv'): mosaiccontinuummfconvolstatus[b] = True else: mosaiccontinuummfconvolstatus[b] = False logger.warning( 'Beam ' + str(b).zfill(2) + ' could not be convolved to the calculated beam size! File not there!' ) except: mosaiccontinuummfconvolstatus[b] = False logger.warning( 'Beam ' + str(b).zfill(2) + ' could not be convolved to the calculated beam size!' ) # Combine all the images using linmos (needs to be updated with proper primary beam model) linmosimages = '' linmosrms = '' for b in range(self.NBEAMS): if mosaiccontinuummfcontinuumstatus[ b] and mosaiccontinuummfcopystatus[ b] and mosaiccontinuummfconvolstatus[b]: linmosimages = linmosimages + str(b).zfill(2) + '_cv,' linmosrms = linmosrms + str( subs_imstats.getimagestats( self, str(b).zfill(2) + '_cv')[2]) + ',' linmos = lib.miriad('linmos') linmos.in_ = linmosimages.rstrip(',') linmos.rms = linmosrms.rstrip(',') linmos.out = self.target.rstrip('.MS') + '_mf' linmos.go() if os.path.isdir(self.target.rstrip('.MS') + '_mf'): mosaiccontinuummfstatus = True subs_convim.mirtofits( self.target.rstrip('.MS') + '_mf', self.target.rstrip('.MS') + '_mf.fits') logger.info( 'Mosaicking of multi-frequency image successful!') else: mosaiccontinuummfstatus = False logger.error( 'Multi-freqeuncy mosaic was not created successfully!') else: mosaiccontinuummfstatus = True logger.info( 'Multi-frequency continuum mosaic was already successfully created!' ) # Save the derived parameters to the parameter file subs_param.add_param(self, 'mosaic_continuum_mf_status', mosaiccontinuummfstatus) subs_param.add_param(self, 'mosaic_continuum_mf_continuumstatus', mosaiccontinuummfcontinuumstatus) subs_param.add_param(self, 'mosaic_continuum_mf_copystatus', mosaiccontinuummfcopystatus) subs_param.add_param(self, 'mosaic_continuum_mf_convolstatus', mosaiccontinuummfconvolstatus) subs_param.add_param(self, 'mosaic_continuum_mf_continuumbeamparams', mosaiccontinuummfcontinuumbeamparams) subs_param.add_param(self, 'mosaic_continuum_mf_continuumimagestats', mosaiccontinuummfcontinuumimagestats)
# Output what exactly is being used to clean the data print("\t{}".format(maskfits)) # Edit mask cube to trick Miriad into using the whole volume. m = pyfits.open(maskfits, mode='update') m[0].data[0, 0, 0] = -1 m[0].data[-1, -1, -1] = -1 m[0].scale('int16') m.flush() # Delete any pre-existing Miriad files. os.system('rm -rf model_* beam_* map_* image_* mask_* residual_*') print("[CLEAN] Reading in FITS files, making Miriad mask.") fits = lib.miriad('fits') fits.op = 'xyin' if args.nospline: fits.in_ = line_cube else: fits.in_ = new_splinefits fits.out = 'map_00' fits.go() if not os.path.isfile(beam_cube): print("[CLEAN] Retrieving synthesized beam cube from ALTA.") os.system('iget {}{}_AP_B0{:02}/HI_beam_cube{}.fits {}'.format( alta_dir, taskid, b, c, loc)) fits.in_ = beam_cube fits.out = 'beam_00' fits.go()
def get_cont_image(self): """ Get continuum image specified by taskid, beam Want to retrieve from ALTA Can use mosaic module in Apercal as a guide to how to do this Convert to miriad format """ #directory for beam/taskid altadir = ("/altaZone/archive/apertif_main/" "visibilities_default/{0}_AP_B{1}").format( self.taskid, self.beam.zfill(3)) #check if beam/taskid available in ALTA if self.check_alta_path(altadir) == 0: #found beam so can continue #have to find continuum image name since it can be different fits_name = '' alta_fits_path = '' for k in range(10): fits_name = "image_mf_{0:02d}.fits".format(k) alta_fits_path = os.path.join(altadir, fits_name) if self.check_alta_path(alta_fits_path) == 0: break else: # make empty again when no image was found fits_name = '' continue if fits_name == '': #failed to find image. Note this and print warning self.status = False ("No continuum image found on ALTA for " "beam {0} of taskid {1}").format(self.beam, self.taskid) else: #create working directory if not os.path.exists(self.workingdir): os.makedirs(self.workingdir) #check if file is already there self.fitspath = os.path.join(self.workingdir, fits_name) print(self.fitspath) if not os.path.exists(self.fitspath): #copy continuum image to this directory return_msg = self.get_data_from_alta( alta_fits_path, self.fitspath) if return_msg != 0.0: print(("Failed to retrieve continuum " "image for beam {0} of taskid {1}").format( self.beam, self.taskid)) self.status = False #now, as long as status is good, convert to miriad #change to check fitspath exist #setting status but not really using now if os.path.exists(self.fitspath): #check that miriad image doesn't already exist and do conversion if not os.path.isdir(self.impath): fits = lib.miriad('fits') fits.op = 'xyin' fits.in_ = self.fitspath fits.out = self.impath try: fits.go() except Exception as e: self.status = False print(("Conversion to miriad image failed " "for beam {0} of taskid {1}").format( self.beam, self.taskid)) print(e)