def create_parametric_mask(self, dataset, radius, cutoff, cat, outputdir): """ Creates a parametric mask using a model from an input catalogue. dataset (string): The dataset to get the coordiantes for the model from. radius (float): The radius around the pointing centre of the input dataset to consider sources in in deg. cutoff (float): The apparent flux percentage to consider sources from 0.0 accounts for no sources, 1.0 for all sources in the catalogue within the search radius of the target field. cat (string): The catalogue to search sources in. Possible options are 'NVSS', 'FIRST', and 'WENSS'. outputdir (string): The output directory to create the MIRIAD mask file in. The file is named mask. """ lsm.write_mask(outputdir + '/mask.txt', lsm.lsm_mask(dataset, radius, cutoff, cat)) mskfile = open(outputdir + '/mask.txt', 'r') object_ = mskfile.readline().rstrip('\n') spar = mskfile.readline() mskfile.close() imgen = lib.miriad('imgen') imgen.imsize = self.selfcal_image_imsize imgen.cell = self.selfcal_image_cellsize imgen.object = object_ imgen.spar = spar imgen.out = outputdir + '/imgen' imgen.go() maths = lib.miriad('maths') maths.exp = '"<' + outputdir + '/imgen' + '>"' maths.mask = '"<' + outputdir + '/imgen>.gt.1e-6' + '"' maths.out = outputdir + '/mask' maths.go() subs_managefiles.director(self, 'rm', outputdir + '/imgen') subs_managefiles.director(self, 'rm', outputdir + '/mask.txt')
def reset(self): """ Function to reset the current step and remove all generated data. Be careful! Deletes all data generated in this step! """ subs_setinit.setinitdirs(self) nbeams = 37 logger.warning(' Deleting all converted data.') for beam in range(nbeams): path = self.get_crosscalsubdir_path(str(beam).zfill(2)) if path.isdir(path): subs_managefiles.director(self, 'rm', path + '/*') logger.warning( ' Deleting all parameter file entries for CONVERT module') subs_param.del_param(self, 'convert_fluxcal_MSavailable') subs_param.del_param(self, 'convert_polcal_MSavailable') subs_param.del_param(self, 'convert_targetbeams_MSavailable') subs_param.del_param(self, 'convert_fluxcal_MS2UVFITS') subs_param.del_param(self, 'convert_polcal_MS2UVFITS') subs_param.del_param(self, 'convert_targetbeams_MS2UVFITS') subs_param.del_param(self, 'convert_fluxcal_UVFITSavailable') subs_param.del_param(self, 'convert_polcal_UVFITSavailable') subs_param.del_param(self, 'convert_targetbeams_UVFITSavailable') subs_param.del_param(self, 'convert_fluxcal_UVFITS2MIRIAD') subs_param.del_param(self, 'convert_polcal_UVFITS2MIRIAD') subs_param.del_param(self, 'convert_targetbeams_UVFITS2MIRIAD')
def reset(self): """ Function to reset the current step and remove all generated data. Be careful! Deletes all data generated in this step! """ subs_setinit.setinitdirs(self) logger.warning('Deleting all raw data products and their directories.') subs_managefiles.director(self, 'ch', self.basedir) deldirs = glob.glob(self.basedir + '[0-9][0-9]' + '/' + self.rawsubdir) for dir_ in deldirs: subs_managefiles.director(self, 'rm', dir_) logger.warning( 'Deleting all parameter file entries for PREPARE module') subs_param.del_param(self, 'prepare_fluxcal_requested') subs_param.del_param(self, 'prepare_fluxcal_diskstatus') subs_param.del_param(self, 'prepare_fluxcal_altastatus') subs_param.del_param(self, 'prepare_fluxcal_copystatus') subs_param.del_param(self, 'prepare_fluxcal_rejreason') subs_param.del_param(self, 'prepare_polcal_requested') subs_param.del_param(self, 'prepare_polcal_diskstatus') subs_param.del_param(self, 'prepare_polcal_altastatus') subs_param.del_param(self, 'prepare_polcal_copystatus') subs_param.del_param(self, 'prepare_polcal_rejreason') subs_param.del_param(self, 'prepare_targetbeams_requested') subs_param.del_param(self, 'prepare_targetbeams_diskstatus') subs_param.del_param(self, 'prepare_targetbeams_altastatus') subs_param.del_param(self, 'prepare_targetbeams_copystatus') subs_param.del_param(self, 'prepare_targetbeams_rejreason')
def get_theoretical_noise(self, dataset, gausslimit, startchan=None, endchan=None): """ Subroutine to create a Stokes V image from a dataset and measure the noise, which should be similar to the theoretical one image (string): The path to the dataset file. startchan(int): First channel to use for imaging, zero-based endchan(int): Last channel to use for imaging, zero-based returns (numpy array): The rms of the image """ invert = lib.miriad('invert') invert.vis = dataset invert.map = 'vrms' invert.beam = 'vbeam' invert.imsize = 1024 invert.cell = 5 invert.stokes = 'v' invert.slop = 1 invert.robust = -2 invert.options='mfs' if (startchan and endchan) != None: invert.line = 'channel,1,' + str(startchan + 1) + ',' + str(endchan - startchan + 1) + ',' + str(endchan - startchan + 1) else: pass invert.go() vmax, vmin, vstd = imstats.getimagestats(self, 'vrms') gaussianity = qa.checkimagegaussianity(self, 'vrms', gausslimit) if os.path.isdir('vrms') and os.path.isdir('vbeam'): managefiles.director(self, 'rm', 'vrms') managefiles.director(self, 'rm', 'vbeam') else: raise ApercalException('Stokes V image was not created successfully. Cannot calculate theoretical noise! No iterative selfcal possible!') return gaussianity, vstd
def reset_all(self): """ Function to reset the current step and remove all generated data for all beams. Be careful! Deletes all data generated in this step! """ subs_setinit.setinitdirs(self) for b in range(self.NBEAMS): cbeam = 'convert_B' + str(b).zfill(2) logger.warning('Beam ' + str(b).zfill(2) + ': Deleting all converted data.') path = self.get_crosscalsubdir_path(str(b).zfill(2)) if os.path.isdir(path): subs_managefiles.director(self, 'rm', path + '/*') logger.warning( 'Beam ' + str(b).zfill(2) + ': Deleting all parameter file entries for CONVERT module') subs_param.del_param(self, cbeam + '_fluxcal_MSavailable') subs_param.del_param(self, cbeam + '_polcal_MSavailable') subs_param.del_param(self, cbeam + '_targetbeams_MSavailable') subs_param.del_param(self, cbeam + '_fluxcal_MS2UVFITS') subs_param.del_param(self, cbeam + '_polcal_MS2UVFITS') subs_param.del_param(self, cbeam + '_targetbeams_MS2UVFITS') subs_param.del_param(self, cbeam + '_fluxcal_UVFITSavailable') subs_param.del_param(self, cbeam + '_polcal_UVFITSavailable') subs_param.del_param(self, cbeam + '_targetbeams_UVFITSavailable') subs_param.del_param(self, cbeam + '_fluxcal_UVFITS2MIRIAD') subs_param.del_param(self, cbeam + '_polcal_UVFITS2MIRIAD') subs_param.del_param(self, cbeam + '_targetbeams_UVFITS2MIRIAD')
def reset(self): """ Function to reset the current step and remove all generated data. Be careful! Deletes all data generated in this step! """ subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) logger.warning('Deleting all data products ready for transfer!') subs_managefiles.director(self, 'ch', self.basedir) subs_managefiles.director(self, 'rm', self.transferdir)
def reset(self): """ Function to reset the current step and remove all generated data. Be careful! Deletes all data generated in this step! """ subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) logger.warning(' Deleting all self-calibrated data.') subs_managefiles.director(self, 'ch', self.selfcaldir) subs_managefiles.director(self, 'rm', self.selfcaldir + '/*')
def flagline(self): """ Creates an image cube of the different chunks and measures the rms in each channel. All channels with an rms outside of a given sigma interval are flagged in the continuum calibration, but are still used for line imaging. """ if self.selfcal_flagline: subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) logger.info(' Automatic flagging of HI-line/RFI started') subs_managefiles.director(self, 'ch', self.selfcaldir) for chunk in self.list_chunks(): subs_managefiles.director(self, 'ch', self.selfcaldir + '/' + str(chunk)) logger.info('Looking through data chunk ' + str(chunk) + ' #') invert = lib.miriad('invert') invert.vis = chunk + '.mir' invert.map = 'map' invert.beam = 'beam' invert.imsize = self.selfcal_image_imsize invert.cell = self.selfcal_image_cellsize invert.stokes = 'ii' invert.slop = 1 invert.go() if os.path.exists('map'): fits = lib.miriad('fits') fits.in_ = 'map' fits.op = 'xyout' fits.out = 'map.fits' fits.go() cube = pyfits.open('map.fits') data = cube[0].data std = np.nanstd(data, axis=(0, 2, 3)) median = np.median(std) stdall = np.nanstd(std) diff = std - median detections = np.where(np.abs(self.selfcal_flagline_sigma * diff) > stdall)[0] if len(detections) > 0: logger.info('Found high noise in channel(s) ' + str(detections).lstrip('[').rstrip(']') + ' #') for d in detections: uvflag = lib.miriad('uvflag') uvflag.vis = chunk + '.mir' uvflag.flagval = 'flag' uvflag.line = "'" + 'channel,1,' + str(d + 1) + "'" uvflag.go() logger.info( 'Flagged channel(s) ' + str(detections).lstrip('[').rstrip(']') + ' in data chunk ' + str( chunk) + ' #') else: logger.info('No high noise found in data chunk ' + str(chunk) + ' #') subs_managefiles.director(self, 'rm', self.selfcaldir + '/' + str(chunk) + '/' + 'map') subs_managefiles.director(self, 'rm', self.selfcaldir + '/' + str(chunk) + '/' + 'map.fits') subs_managefiles.director(self, 'rm', self.selfcaldir + '/' + str(chunk) + '/' + 'beam') else: logger.info(' No data in chunk ' + str(chunk) + '!') logger.info(' Automatic flagging of HI-line/RFI done')
def reset_all(self): """ Function to reset the current step and remove all generated data. Be careful! Deletes all data generated in this step! """ subs_setinit.setinitdirs(self) logger.warning( 'Deleting all raw data products and their directories for all beams. You will need to ' 'start with the PREPARE step again!') subs_managefiles.director(self, 'ch', self.basedir) for b in range(self.NBEAMS): prebeam = 'prepare_B' + str(b).zfill(2) sbeam = 'split_B' + str(b).zfill(2) if os.path.isdir(self.basedir + str(b).zfill(2) + '/' + self.rawsubdir): try: logger.warning('Beam ' + str(b).zfill(2) + ': Deleting all raw data products.') subs_managefiles.director( self, 'rm', self.basedir + str(b).zfill(2) + '/' + self.rawsubdir) except: pass logger.warning( 'Beam ' + str(b).zfill(2) + ': Deleting all parameter file entries for PREPARE and SPLIT module.' ) subs_param.del_param(self, prebeam + '_fluxcal_requested') subs_param.del_param(self, prebeam + '_fluxcal_diskstatus') subs_param.del_param(self, prebeam + '_fluxcal_altastatus') subs_param.del_param(self, prebeam + '_fluxcal_copystatus') subs_param.del_param(self, prebeam + '_fluxcal_rejreason') subs_param.del_param(self, prebeam + '_polcal_requested') subs_param.del_param(self, prebeam + '_polcal_diskstatus') subs_param.del_param(self, prebeam + '_polcal_altastatus') subs_param.del_param(self, prebeam + '_polcal_copystatus') subs_param.del_param(self, prebeam + '_polcal_rejreason') subs_param.del_param(self, prebeam + '_targetbeams_requested') subs_param.del_param(self, prebeam + '_targetbeams_diskstatus') subs_param.del_param(self, prebeam + '_targetbeams_altastatus') subs_param.del_param(self, prebeam + '_targetbeams_copystatus') subs_param.del_param(self, prebeam + '_targetbeams_rejreason') subs_param.del_param(self, sbeam + '_fluxcal_status') subs_param.del_param(self, sbeam + '_polcal_status') subs_param.del_param(self, sbeam + '_targetbeams_status') else: logger.warning('Beam ' + str(b).zfill(2) + ': No raw data present.')
def calc_isum(self, image): """ Function to calculate the sum of the values of the pixels in an image image (string): The name of the image file. Must be in MIRIAD-format returns (float): the sum of the pxiels in the image """ fits = lib.miriad('fits') fits.op = 'xyout' fits.in_ = image fits.out = image + '.fits' fits.go() image_data = pyfits.open(image + '.fits') # Open the image data = image_data[0].data isum = np.nansum(data) # Get the maximum image_data.close() # Close the image subs_managefiles.director(self, 'rm', image + '.fits') return isum
def reset(self): """ Function to reset the current step and remove all generated data. Be careful! Deletes all data generated in this step! """ subs_setinit.setinitdirs(self) logger.warning('Beam ' + self.beam + ': Deleting all raw data and their directories.') subs_managefiles.director(self, 'ch', self.basedir) try: subs_managefiles.director( self, 'rm', self.basedir + self.beam + '/' + self.rawsubdir) except: pass logger.warning( 'Beam ' + self.beam + ': Deleting all parameter file entries for SPLIT and PREPARE module' ) prebeam = 'prepare_B' + str(self.beam).zfill(2) sbeam = 'split_B' + str(self.beam).zfill(2) subs_param.del_param(self, prebeam + '_fluxcal_requested') subs_param.del_param(self, prebeam + '_fluxcal_diskstatus') subs_param.del_param(self, prebeam + '_fluxcal_altastatus') subs_param.del_param(self, prebeam + '_fluxcal_copystatus') subs_param.del_param(self, prebeam + '_fluxcal_rejreason') subs_param.del_param(self, prebeam + '_polcal_requested') subs_param.del_param(self, prebeam + '_polcal_diskstatus') subs_param.del_param(self, prebeam + '_polcal_altastatus') subs_param.del_param(self, prebeam + '_polcal_copystatus') subs_param.del_param(self, prebeam + '_polcal_rejreason') subs_param.del_param(self, prebeam + '_targetbeams_requested') subs_param.del_param(self, prebeam + '_targetbeams_diskstatus') subs_param.del_param(self, prebeam + '_targetbeams_altastatus') subs_param.del_param(self, prebeam + '_targetbeams_copystatus') subs_param.del_param(self, prebeam + '_targetbeams_rejreason') subs_param.del_param(self, sbeam + '_fluxcal_status') subs_param.del_param(self, sbeam + '_polcal_status') subs_param.del_param(self, sbeam + '_targetbeams_status')
def get_beam(self, image, beam): """ Get the synthesised beam of an image with has not been cleaned image (string): Input image to use in MIRIAD format beam (string): Beam image for cleaning in MIRIAD format return (tuple): Synthesised beam parameters in the order bmaj, bmin, bpa """ clean = lib.miriad('clean') clean.map = image clean.beam = beam clean.out = 'tmp_beampars.cl' clean.niters = 1 clean.region = 'quarter' clean.go() restor = lib.miriad('restor') # Create the restored image restor.model = 'tmp_beampars.cl' restor.beam = beam restor.map = image restor.out = 'tmp_beampars.rstr' restor.mode = 'clean' restor.go() convim.mirtofits('tmp_beampars.rstr', 'tmp_beampars.fits') pyfile = pyfits.open('tmp_beampars.fits') header = pyfile[0].header bmaj = header['BMAJ'] bmin = header['BMIN'] bpa = header['BPA'] beampars = bmaj, bmin, bpa managefiles.director(self, 'rm', 'tmp_beampars.cl') managefiles.director(self, 'rm', 'tmp_beampars.rstr') managefiles.director(self, 'rm', 'tmp_beampars.fits') return beampars
def reset(self): """ Function to reset the current step and remove all generated data. Be careful! Deletes all data generated in this step! """ subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) if os.path.isdir(self.mosdir): logger.warning('Deleting all mosaicked data products.') subs_managefiles.director(self, 'ch', self.basedir) subs_managefiles.director(self, 'rm', self.mosdir) logger.warning( 'Deleting all parameter file entries for MOSAIC module') subs_param.del_param(self, 'mosaic_continuum_mf_status') subs_param.del_param(self, 'mosaic_continuum_mf_continuumstatus') subs_param.del_param(self, 'mosaic_continuum_mf_copystatus') subs_param.del_param(self, 'mosaic_continuum_mf_convolstatus') subs_param.del_param(self, 'mosaic_continuum_mf_continuumbeamparams') subs_param.del_param(self, 'mosaic_continuum_mf_continuumimagestats') else: logger.warning('Mosaicked data products are not present!')
def create_mask(self, image, mask, threshold, theoretical_noise, beampars=None, rms_map=None): """ Creates a mask from an image using pybdsf image (string): Input image to use in MIRIAD format mask (string): Output mask image in MIRIAD format threshold (float): Threshold in Jy to use theoretical_noise (float): Theoretical noise for calculating the adaptive threshold parameter inside pybdsf """ convim.mirtofits(image, image + '.fits') bdsf_threshold = threshold / theoretical_noise if beampars: # bdsf.process_image(image + '.fits', stop_at='isl', thresh_isl=bdsf_threshold, beam=beampars, adaptive_rms_box=True, rms_map=rms_map).export_image(outfile=mask + '.fits', img_format='fits', img_type='island_mask', pad_image=True) bdsf.process_image(image + '.fits', stop_at='isl', thresh_isl=bdsf_threshold, beam=beampars, adaptive_rms_box=True, rms_map=False, rms_value=theoretical_noise).export_image(outfile=mask + '.fits', img_format='fits', img_type='island_mask', pad_image=True) else: bdsf.process_image(image + '.fits', stop_at='isl', thresh_isl=bdsf_threshold, adaptive_rms_box=True, rms_map=False, rms_value=theoretical_noise).export_image(outfile=mask + '.fits', img_format='fits', img_type='island_mask', pad_image=True) if os.path.isfile(mask + '.fits'): # Add a random number to the masks to make it viewable in kvis fitsmask = pyfits.open(mask + '.fits') fitsmask_data = fitsmask[0].data fitsmask_hdr = fitsmask[0].header rand_array = np.random.rand(int(fitsmask_hdr['NAXIS1']), int(fitsmask_hdr['NAXIS2'])) fitsmask[0].data = np.multiply(rand_array, fitsmask_data) fitsmask.writeto(mask + '.fits', clobber=True) # Convert mask to MIRIAD and generate a usable one for MIRIAD convim.fitstomir(mask + '.fits', mask + '_pybdsf') maths = lib.miriad('maths') maths.out = mask maths.exp = '"<' + mask + '_pybdsf>"' maths.mask = '"<' + mask + '_pybdsf>.gt.0' + '"' maths.go() managefiles.director(self, 'rm', image + '.fits.pybdsf.log') managefiles.director(self, 'rm', image + '.fits') managefiles.director(self, 'rm', mask + '.fits') managefiles.director(self, 'rm', mask + '_pybdsf') else: pass
def blank_corners(self, mask, imsize): """ Blanks the outer edges of a mask to make it always work with mfclean mask (string): Input mask in MIRIAD format imsize (int): image size in pixels """ # Calculate the borders for the mask lowborder = 0.1 * imsize highborder = 0.9 * imsize # Do the masking and regrid maths = lib.miriad('maths') maths.out = mask + '_cut' maths.exp = '"<' + mask + '>"' maths.region = 'box"(' + str(lowborder) + ',' + str(lowborder) + ',' + str(highborder) + ',' + str(highborder) + ')"' maths.go() regrid = lib.miriad('regrid') regrid.in_ = mask + '_cut' regrid.out = mask + '_cut_regrid' regrid.axes = '1,2' regrid.tin = mask regrid.go() managefiles.director(self, 'rm', mask + '_cut') managefiles.director(self, 'rm', mask) managefiles.director(self, 'rn', mask, file_ = mask + '_cut_regrid')
def split(self): """ Splits out a certain frequency range from the datasets for the quicklook pipeline """ if self.prepare_split: logger.info('Splitting channel ' + str(self.prepare_split_startchannel) + ' until ' + str(self.prepare_split_endchannel)) # split the flux calibrator dataset logger.debug("self.fluxcal = {}".format(self.fluxcal)) logger.debug("os.path.isdir(self.get_fluxcal_path()) = {}".format( os.path.isdir(self.get_fluxcal_path()))) if self.fluxcal != '' and os.path.isdir(self.get_fluxcal_path()): fluxcal_split = 'split(vis = "' + self.get_fluxcal_path( ) + '", outputvis = "' + self.get_fluxcal_path().rstrip( '.MS') + '_split.MS"' + ', spw = "0:' + str( self.prepare_split_startchannel) + '~' + str( self.prepare_split_endchannel ) + '", datacolumn = "data")' lib.run_casa([fluxcal_split], log_output=True, timeout=3600) if os.path.isdir(self.get_fluxcal_path().rstrip('.MS') + '_split.MS'): subs_managefiles.director(self, 'rm', self.get_fluxcal_path()) subs_managefiles.director( self, 'rn', self.get_fluxcal_path(), file_=self.get_fluxcal_path().rstrip('.MS') + '_split.MS') else: logger.warning( 'Splitting of flux calibrator dataset not successful!') else: logger.warning( 'Fluxcal not set or dataset not available! Cannot split flux calibrator dataset!' ) # Split the polarised calibrator dataset logger.debug("self.polcal = {}".format(self.polcal)) logger.debug("os.path.isdir(self.get_polcal_path()) = {}".format( os.path.isdir(self.get_polcal_path()))) if self.polcal != '' and os.path.isdir(self.get_polcal_path()): polcal_split = 'split(vis = "' + self.get_polcal_path( ) + '", outputvis = "' + self.get_polcal_path().rstrip( '.MS') + '_split.MS"' + ', spw = "0:' + str( self.prepare_split_startchannel) + '~' + str( self.prepare_split_endchannel ) + '", datacolumn = "data")' lib.run_casa([polcal_split], log_output=True, timeout=3600) if os.path.isdir(self.get_polcal_path().rstrip('.MS') + '_split.MS'): subs_managefiles.director(self, 'rm', self.get_polcal_path()) subs_managefiles.director( self, 'rn', self.get_polcal_path(), file_=self.get_polcal_path().rstrip('.MS') + '_split.MS') else: logger.warning( 'Splitting of polarised calibrator dataset not successful!' ) else: logger.warning( 'Polcal not set or dataset not available! Cannot split polarised calibrator dataset!' ) # Split the target dataset logger.debug("self.target = {}".format(self.target)) logger.debug("os.path.isdir(self.get_target_path()) = {}".format( os.path.isdir(self.get_target_path()))) if self.target != '' and os.path.isdir(self.get_target_path()): target_split = 'split(vis = "' + self.get_target_path( ) + '", outputvis = "' + self.get_target_path().rstrip( '.MS') + '_split.MS"' + ', spw = "0:' + str( self.prepare_split_startchannel) + '~' + str( self.prepare_split_endchannel ) + '", datacolumn = "data")' lib.run_casa([target_split], log_output=True, timeout=3600) if os.path.isdir(self.get_target_path().rstrip('.MS') + '_split.MS'): subs_managefiles.director(self, 'rm', self.get_target_path()) subs_managefiles.director( self, 'rn', self.get_target_path(), file_=self.get_target_path().rstrip('.MS') + '_split.MS') else: logger.warning( 'Splitting of target dataset not successful!') else: logger.warning( 'Target not set or dataset not available! Cannot split target dataset!' ) else: pass
def splitdata(self): """ Applies calibrator corrections to data, splits the data into chunks in frequency and bins it to the given frequency resolution for the self-calibration """ if self.selfcal_splitdata: subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) subs_managefiles.director(self, 'ch', self.selfcaldir) logger.info(' Splitting of target data into individual frequency chunks started') if os.path.exists(self.selfcaldir + '/' + self.target): logger.info('Calibrator corrections already seem to have been applied #') else: logger.info('Applying calibrator solutions to target data before averaging #') uvaver = lib.miriad('uvaver') uvaver.vis = self.crosscaldir + '/' + self.target uvaver.out = self.selfcaldir + '/' + self.target uvaver.go() logger.info('Calibrator solutions to target data applied #') if self.selfcal_flagantenna != '': uvflag = lib.miriad('uvflag') uvflag.vis = self.selfcaldir + '/' + self.target uvflag.flagval = 'flag' uvflag.select = 'antenna(' + str(self.selfcal_flagantenna) + ')' uvflag.go() else: pass try: uv = aipy.miriad.UV(self.selfcaldir + '/' + self.target) except RuntimeError: raise ApercalException(' No data in your selfcal directory!') try: nsubband = len(uv['nschan']) # Number of subbands in data except TypeError: nsubband = 1 # Only one subband in data since exception was triggered logger.info('Found ' + str(nsubband) + ' subband(s) in target data #') counter = 0 # Counter for naming the chunks and directories for subband in range(nsubband): logger.info('Started splitting of subband ' + str(subband) + ' #') if nsubband == 1: numchan = uv['nschan'] finc = np.fabs(uv['sdf']) else: numchan = uv['nschan'][subband] # Number of channels per subband finc = np.fabs(uv['sdf'][subband]) # Frequency increment for each channel subband_bw = numchan * finc # Bandwidth of one subband subband_chunks = round(subband_bw / self.selfcal_splitdata_chunkbandwidth) # Round to the closest power of 2 for frequency chunks with the same bandwidth over the frequency # range of a subband subband_chunks = int(np.power(2, np.ceil(np.log(subband_chunks) / np.log(2)))) if subband_chunks == 0: subband_chunks = 1 chunkbandwidth = (numchan / subband_chunks) * finc logger.info('Adjusting chunk size to ' + str( chunkbandwidth) + ' GHz for regular gridding of the data chunks over frequency #') for chunk in range(subband_chunks): logger.info( 'Starting splitting of data chunk ' + str(chunk) + ' for subband ' + str(subband) + ' #') binchan = round( self.selfcal_splitdata_channelbandwidth / finc) # Number of channels per frequency bin chan_per_chunk = numchan / subband_chunks if chan_per_chunk % binchan == 0: # Check if the freqeuncy bin exactly fits logger.info('Using frequency binning of ' + str( self.selfcal_splitdata_channelbandwidth) + ' for all subbands #') else: # Increase the frequency bin to keep a regular grid for the chunks while chan_per_chunk % binchan != 0: binchan = binchan + 1 else: # Check if the calculated bin is not larger than the subband channel number if chan_per_chunk >= binchan: pass else: # Set the frequency bin to the number of channels in the chunk of the subband binchan = chan_per_chunk logger.info('Increasing frequency bin of data chunk ' + str( chunk) + ' to keep bandwidth of chunks equal over the whole bandwidth #') logger.info('New frequency bin is ' + str(binchan * finc) + ' GHz #') nchan = int(chan_per_chunk / binchan) # Total number of output channels per chunk start = 1 + chunk * chan_per_chunk width = int(binchan) step = int(width) subs_managefiles.director(self, 'mk', self.selfcaldir + '/' + str(counter).zfill(2)) uvaver = lib.miriad('uvaver') uvaver.vis = self.selfcaldir + '/' + self.target uvaver.out = self.selfcaldir + '/' + str(counter).zfill(2) + '/' + str(counter).zfill(2) + '.mir' uvaver.select = "'" + 'window(' + str(subband + 1) + ')' + "'" uvaver.line = "'" + 'channel,' + str(nchan) + ',' + str(start) + ',' + str(width) + ',' + str( step) + "'" uvaver.go() counter = counter + 1 logger.info('Splitting of data chunk ' + str(chunk) + ' for subband ' + str(subband) + ' done #') logger.info('Splitting of data for subband ' + str(subband) + ' done #') logger.info(' Splitting of target data into individual frequency chunks done')
def mosaic_continuum_mf(self): """Looks for all available stacked continuum images and mosaics them into one large image.""" subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) ########################################################################################################## # Check if the parameter is already in the parameter file and load it otherwise create the needed arrays # ########################################################################################################## mosaiccontinuummfstatus = get_param_def( self, 'mosaic_continuum_mf_status', False) # Status of the continuum mf mosaic mosaiccontinuummfcontinuumstatus = get_param_def( self, 'mosaic_continuum_mf_continuumstatus', np.full(self.NBEAMS, False)) # Status of the continuum imaging mosaiccontinuummfcopystatus = get_param_def( self, 'mosaic_continuum_mf_copystatus', np.full(self.NBEAMS, False)) # Status of the copy of the images mosaiccontinuummfconvolstatus = get_param_def( self, 'mosaic_continuum_mf_convolstatus', np.full(self.NBEAMS, False)) # Status of the convolved images mosaiccontinuummfcontinuumbeamparams = get_param_def( self, 'mosaic_continuum_mf_continuumbeamparams', np.full((self.NBEAMS, 3), np.nan)) # Beam sizes of the input images mosaiccontinuummfcontinuumimagestats = get_param_def( self, 'mosaic_continuum_mf_continuumimagestats', np.full((self.NBEAMS, 3), np.nan)) # Image statistics of the input images # Start the mosaicking of the stacked continuum images if self.mosaic_continuum_mf: subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) subs_managefiles.director(self, 'ch', self.mosdir + '/continuum') if not mosaiccontinuummfstatus: logger.info('Mosaicking multi-frequency continuum images') # Acquire the results and statistics from continuum mf imaging for b in range(self.NBEAMS): mosaiccontinuummfcontinuumstatus[b] = get_param_def( self, 'continuum_B' + str(b).zfill(2) + '_targetbeams_mf_status', False) if mosaiccontinuummfcontinuumstatus[b]: finalminor = get_param_def( self, 'continuum_B' + str(b).zfill(2) + '_targetbeams_mf_final_minorcycle', np.nan) subs_managefiles.director( self, 'cp', str(b).zfill(2) + '.fits', file_=self.basedir + str(b).zfill(2) + '/' + self.contsubdir + '/' + 'image_mf_' + str(finalminor).zfill(2) + '.fits') if os.path.isfile(str(b).zfill(2) + '.fits'): mosaiccontinuummfcopystatus[b] = True subs_convim.fitstomir( str(b).zfill(2) + '.fits', str(b).zfill(2)) subs_managefiles.director( self, 'rm', str(b).zfill(2) + '.fits') else: mosaiccontinuummfcopystatus[b] = False logger.warning('Beam ' + str(b).zfill(2) + ' was not copied successfully!') # Copy the images over to the mosaic directory for b in range(self.NBEAMS): if mosaiccontinuummfcontinuumstatus[ b] and mosaiccontinuummfcopystatus[b]: # Get the image beam parameters and the image statistics mosaiccontinuummfcontinuumimagestats[ b, :] = subs_imstats.getimagestats( self, str(b).zfill(2)) mosaiccontinuummfcontinuumbeamparams[ b, :] = subs_readmirhead.getbeamimage( str(b).zfill(2)) else: logger.warning( 'Skipping Beam ' + str(b).zfill(2) + '! Continuum mf-imaging was not successful or continuum image not available!' ) # Calculate the synthesised beam and reject outliers (algorithm needs to be updated) rejbeams, beamparams = subs_combim.calc_synbeam( mosaiccontinuummfcontinuumbeamparams) # Convolve all the images to the calculated beam for b in range(self.NBEAMS): if mosaiccontinuummfcontinuumstatus[ b] and mosaiccontinuummfcopystatus[b]: try: convol = lib.miriad('convol') convol.map = str(b).zfill(2) convol.fwhm = str(beamparams[0]) + ',' + str( beamparams[1]) convol.pa = str(beamparams[2]) convol.options = 'final' convol.out = str(b).zfill(2) + '_cv' convol.go() if os.path.isdir(str(b).zfill(2) + '_cv'): mosaiccontinuummfconvolstatus[b] = True else: mosaiccontinuummfconvolstatus[b] = False logger.warning( 'Beam ' + str(b).zfill(2) + ' could not be convolved to the calculated beam size! File not there!' ) except: mosaiccontinuummfconvolstatus[b] = False logger.warning( 'Beam ' + str(b).zfill(2) + ' could not be convolved to the calculated beam size!' ) # Combine all the images using linmos (needs to be updated with proper primary beam model) linmosimages = '' linmosrms = '' for b in range(self.NBEAMS): if mosaiccontinuummfcontinuumstatus[ b] and mosaiccontinuummfcopystatus[ b] and mosaiccontinuummfconvolstatus[b]: linmosimages = linmosimages + str(b).zfill(2) + '_cv,' linmosrms = linmosrms + str( subs_imstats.getimagestats( self, str(b).zfill(2) + '_cv')[2]) + ',' linmos = lib.miriad('linmos') linmos.in_ = linmosimages.rstrip(',') linmos.rms = linmosrms.rstrip(',') linmos.out = self.target.rstrip('.MS') + '_mf' linmos.go() if os.path.isdir(self.target.rstrip('.MS') + '_mf'): mosaiccontinuummfstatus = True subs_convim.mirtofits( self.target.rstrip('.MS') + '_mf', self.target.rstrip('.MS') + '_mf.fits') logger.info( 'Mosaicking of multi-frequency image successful!') else: mosaiccontinuummfstatus = False logger.error( 'Multi-freqeuncy mosaic was not created successfully!') else: mosaiccontinuummfstatus = True logger.info( 'Multi-frequency continuum mosaic was already successfully created!' ) # Save the derived parameters to the parameter file subs_param.add_param(self, 'mosaic_continuum_mf_status', mosaiccontinuummfstatus) subs_param.add_param(self, 'mosaic_continuum_mf_continuumstatus', mosaiccontinuummfcontinuumstatus) subs_param.add_param(self, 'mosaic_continuum_mf_copystatus', mosaiccontinuummfcopystatus) subs_param.add_param(self, 'mosaic_continuum_mf_convolstatus', mosaiccontinuummfconvolstatus) subs_param.add_param(self, 'mosaic_continuum_mf_continuumbeamparams', mosaiccontinuummfcontinuumbeamparams) subs_param.add_param(self, 'mosaic_continuum_mf_continuumimagestats', mosaiccontinuummfcontinuumimagestats)
def ms2miriad(self): """ Converts the data from MS to MIRIAD format via UVFITS using drivecasa. Does it for the flux calibrator, polarisation calibrator, and target field independently. """ subs_setinit.setinitdirs(self) ccalbeam = 'ccal_B' + str(self.beam).zfill(2) cbeam = 'convert_B' + str(self.beam).zfill(2) # Read the parameters from crosscal # and check before doing anything # Status of the solution transfer for the target, flux calibrator and polarisation calibrator ccal_targetbeams_transfer = get_param_def( self, ccalbeam + '_targetbeams_transfer', False) ccal_calibration_calibrator_finished = get_param_def( self, ccalbeam + '_calibration_calibrator_finished', False) if not ccal_calibration_calibrator_finished: error = "Beam {}: Will not convert files to miriad format because cross-calibration failed.".format( str(self.beam).zfill(2)) logger.error(error) raise ApercalException(error) elif not ccal_targetbeams_transfer: error = "Beam {}: Will not convert files to miriad format because cross-calibration solutions were not successfully applied to target.".format( str(self.beam).zfill(2)) logger.error(error) raise ApercalException(error) # Create the parameters for the parameter file for converting from MS to UVFITS format # Flux calibrator MS dataset available? convertfluxcalmsavailable = get_param_def( self, cbeam + '_fluxcal_MSavailable', False) # Polarised calibrator MS dataset available? convertpolcalmsavailable = get_param_def(self, cbeam + '_polcal_MSavailable', False) # Target beam MS dataset available? converttargetbeamsmsavailable = get_param_def( self, cbeam + '_targetbeams_MSavailable', False) # Flux calibrator MS dataset converted to UVFITS? convertfluxcalms2uvfits = get_param_def(self, cbeam + '_fluxcal_MS2UVFITS', False) # Polarised calibrator MS dataset converted to UVFITS? convertpolcalms2uvfits = get_param_def(self, cbeam + '_polcal_MS2UVFITS', False) # Target beam MS dataset converted to UVFITS? converttargetbeamsms2uvfits = get_param_def( self, cbeam + '_targetbeams_MS2UVFITS', False) # Flux calibrator UVFITS dataset available? convertfluxcaluvfitsavailable = get_param_def( self, cbeam + '_fluxcal_UVFITSavailable', False) # Polarised calibrator UVFITS dataset available? convertpolcaluvfitsavailable = get_param_def( self, cbeam + '_polcal_UVFITSavailable', False) # Target beam UVFITS dataset available? converttargetbeamsuvfitsavailable = get_param_def( self, cbeam + '_targetbeams_UVFITSavailable', False) # Flux calibrator UVFITS dataset converted to MIRIAD? convertfluxcaluvfits2miriad = get_param_def( self, cbeam + '_fluxcal_UVFITS2MIRIAD', False) # Polarised calibrator UVFITS dataset converted to MIRIAD? convertpolcaluvfits2miriad = get_param_def( self, cbeam + '_polcal_UVFITS2MIRIAD', False) # Target beam UVFITS dataset converted to MIRIAD? converttargetbeamsuvfits2miriad = get_param_def( self, cbeam + '_targetbeams_UVFITS2MIRIAD', False) # Check which datasets are available in MS format # if self.fluxcal != '': convertfluxcalmsavailable = path.isdir(self.get_fluxcal_path()) else: logger.warning( 'Beam ' + self.beam + ': Flux calibrator dataset not specified. Cannot convert flux calibrator!' ) if self.polcal != '': convertpolcalmsavailable = path.isdir(self.get_polcal_path()) else: logger.warning( 'Beam ' + self.beam + ': Polarised calibrator dataset not specified. Cannot convert polarised calibrator!' ) if self.target != '': converttargetbeamsmsavailable = path.isdir(self.get_target_path()) else: logger.warning( 'Beam ' + self.beam + ': Target beam dataset not specified. Cannot convert target beams!' ) # Save the derived parameters for the availability to the parameter file subs_param.add_param(self, cbeam + '_fluxcal_MSavailable', convertfluxcalmsavailable) subs_param.add_param(self, cbeam + '_polcal_MSavailable', convertpolcalmsavailable) subs_param.add_param(self, cbeam + '_targetbeams_MSavailable', converttargetbeamsmsavailable) # Convert the flux calibrator if self.convert_fluxcal: if self.fluxcal != '': if not convertfluxcaluvfits2miriad: if convertfluxcalmsavailable: logger.debug( 'Beam ' + self.beam + ': Converting flux calibrator dataset from MS to UVFITS format.' ) subs_managefiles.director( self, 'mk', self.get_crosscalsubdir_path(), verbose=False) fluxcal_ms = self.get_fluxcal_path() # convert only if corrected data column exists if subs_msutils.has_correcteddata(fluxcal_ms): datacolumn = "corrected" fluxcal_fits = mspath_to_fitspath( self.get_crosscalsubdir_path(), fluxcal_ms) fc_convert = exportuvfits_cmd.format( vis=self.get_fluxcal_path(), fits=fluxcal_fits, datacolumn=datacolumn) lib.run_casa([fc_convert], timeout=3600) if path.isfile(fluxcal_fits): convertfluxcalms2uvfits = True logger.info( 'Beam ' + self.beam + ': Converted flux calibrator dataset from MS to UVFITS format!' ) else: convertfluxcalms2uvfits = False logger.warning( 'Beam ' + self.beam + ': Could not convert flux calibrator dataset {} ' 'from MS to UVFITS format!'.format( fluxcal_fits)) else: logger.warning( 'Beam ' + self.beam + ': Flux calibrator does not have a corrected_data column! Not ' 'converting flux calibrator dataset!') else: logger.warning( 'Beam ' + self.beam + ': Flux calibrator dataset {} not available!'. format(self.get_fluxcal_path())) else: logger.info( 'Beam ' + self.beam + ': Flux calibrator dataset was already converted from MS to UVFITS format' ) else: logger.warning( 'Beam ' + self.beam + ': Flux calibrator dataset not specified. Cannot convert flux calibrator!' ) else: logger.warning('Beam ' + self.beam + ': Not converting flux calibrator dataset!') # Convert the polarised calibrator if self.convert_polcal: if self.polcal != '': if not convertpolcaluvfits2miriad: if convertpolcalmsavailable: logger.debug( 'Beam ' + self.beam + ': Converting polarised calibrator dataset from MS to UVFITS format.' ) subs_managefiles.director( self, 'mk', self.get_crosscalsubdir_path(), verbose=False) polcal_ms = self.get_polcal_path() # convert only if corrected data column exists if subs_msutils.has_correcteddata(polcal_ms): datacolumn = "corrected" polcal_fits = mspath_to_fitspath( self.get_crosscalsubdir_path(), polcal_ms) pc_convert = exportuvfits_cmd.format( vis=polcal_ms, fits=polcal_fits, datacolumn=datacolumn) lib.run_casa([pc_convert], timeout=3600) if path.isfile(polcal_fits): convertpolcalms2uvfits = True logger.info( 'Beam ' + self.beam + ': Converted polarised calibrator dataset from MS to UVFITS format!' ) else: convertpolcalms2uvfits = False logger.warning( 'Beam ' + self.beam + ': Could not convert polarised calibrator dataset from MS to UVFITS format!' ) else: logger.warning( 'Beam ' + self.beam + ': Polarised calibrator does not have a corrected_data column! Not ' 'converting polarised calibrator dataset!') else: logger.warning( 'Beam ' + self.beam + ': Polarised calibrator dataset not available!') else: logger.info( 'Beam ' + self.beam + ': Polarised calibrator dataset was already converted from MS to UVFITS format' ) else: logger.warning( 'Beam ' + self.beam + ': Polarised calibrator dataset not specified. Cannot convert polarised calibrator!' ) else: logger.warning('Beam ' + self.beam + ': Not converting polarised calibrator dataset!') # Convert the target beams if self.convert_target: if self.target != '': logger.info( 'Beam ' + self.beam + ': Converting target beam dataset from MS to UVFITS format.' ) if not converttargetbeamsuvfits2miriad: if converttargetbeamsmsavailable: subs_managefiles.director( self, 'mk', self.get_crosscalsubdir_path(), verbose=False) target_ms = self.get_target_path() target_fits = mspath_to_fitspath( self.get_crosscalsubdir_path(), target_ms) # only convert if corrected data column exists if subs_msutils.has_correcteddata(target_ms): datacolumn = "corrected" tg_convert = exportuvfits_cmd.format( vis=target_ms, fits=target_fits, datacolumn=datacolumn) lib.run_casa([tg_convert], timeout=10000) if path.isfile(target_fits): converttargetbeamsms2uvfits = True logger.debug( 'Beam ' + self.beam + ': Converted dataset of target beam from MS to UVFITS format!' ) else: converttargetbeamsms2uvfits = False logger.warning( 'Beam ' + self.beam + ': Could not convert dataset for target beam from MS to UVFITS format!' ) else: logger.warning( 'Beam ' + self.beam + ': Target beam dataset does not have a corrected_data column! Not ' 'converting target beam dataset!') else: logger.warning('Beam ' + self.beam + ': Target beam dataset not available!') else: logger.info('Beam ' + self.beam + ': Target beam dataset was already ' 'converted from MS to UVFITS format') else: logger.warning( 'Beam ' + self.beam + ': Target beam dataset not specified. Cannot convert target beam dataset!' ) else: logger.warning('Beam ' + self.beam + ': Not converting target beam dataset!') # Save the derived parameters for the MS to UVFITS conversion to the parameter file subs_param.add_param(self, cbeam + '_fluxcal_MS2UVFITS', convertfluxcalms2uvfits) subs_param.add_param(self, cbeam + '_polcal_MS2UVFITS', convertpolcalms2uvfits) subs_param.add_param(self, cbeam + '_targetbeams_MS2UVFITS', converttargetbeamsms2uvfits) # Check which datasets are available in UVFITS format # if self.fluxcal != '': crosscal_fluxcal = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.fluxcal) convertfluxcaluvfitsavailable = path.isfile(crosscal_fluxcal) else: logger.warning( 'Beam ' + self.beam + ': Flux calibrator dataset not specified. Cannot convert flux calibrator!' ) if self.polcal != '': crosscal_polcal = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.polcal) convertpolcaluvfitsavailable = path.isfile(crosscal_polcal) else: logger.warning( 'Beam ' + self.beam + ': Polarised calibrator dataset not specified. Cannot convert polarised calibrator!' ) if self.target != '': crosscal_target = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.target) converttargetbeamsuvfitsavailable = path.isfile(crosscal_target) else: logger.warning( 'Beam ' + self.beam + ': Target beam dataset not specified. Cannot convert target beam!' ) # Save the derived parameters for the availability to the parameter file subs_param.add_param(self, cbeam + '_fluxcal_UVFITSavailable', convertfluxcaluvfitsavailable) subs_param.add_param(self, cbeam + '_polcal_UVFITSavailable', convertpolcaluvfitsavailable) subs_param.add_param(self, cbeam + '_targetbeams_UVFITSavailable', converttargetbeamsuvfitsavailable) # Convert the available UVFITS-datasets to MIRIAD format # # Convert the flux calibrator if self.convert_fluxcal: if self.fluxcal != '': if not convertfluxcaluvfits2miriad: if convertfluxcaluvfitsavailable: logger.debug( 'Beam ' + self.beam + ': Converting flux calibrator dataset from UVFITS to MIRIAD format.' ) subs_managefiles.director( self, 'ch', self.get_crosscalsubdir_path(), verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.fluxcal) fits.out = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.fluxcal, ext='mir') fits.go() if path.isdir(fits.out): convertfluxcaluvfits2miriad = True logger.info( 'Beam ' + self.beam + ': Converted flux calibrator dataset from UVFITS to MIRIAD format!' ) else: convertfluxcaluvfits2miriad = False logger.warning( 'Beam ' + self.beam + ': Could not convert flux calibrator dataset {} from UVFITS to ' 'MIRIAD format!'.format(fits.out)) else: logger.warning( 'Beam ' + self.beam + ': Flux calibrator dataset not available!') else: logger.info( 'Beam ' + self.beam + ': Flux calibrator dataset was already converted from UVFITS to MIRIAD format' ) else: logger.warning( 'Beam ' + self.beam + ': Flux calibrator dataset not specified. Cannot convert flux calibrator!' ) else: logger.warning('Beam ' + self.beam + ': Not converting flux calibrator dataset!') # Convert the polarised calibrator if self.convert_polcal: if self.polcal != '': if not convertpolcaluvfits2miriad: if convertpolcaluvfitsavailable: logger.debug( 'Beam ' + self.beam + ': Converting polarised calibrator dataset from UVFITS to MIRIAD format.' ) subs_managefiles.director( self, 'ch', self.get_crosscalsubdir_path(), verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.polcal) fits.out = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.polcal, ext='mir') fits.go() if path.isdir(fits.out): convertpolcaluvfits2miriad = True logger.info( 'Beam ' + self.beam + ': Converted polarised calibrator dataset from UVFITS to MIRIAD format!' ) else: convertpolcaluvfits2miriad = False logger.warning( 'Beam ' + self.beam + ': Could not convert polarised calibrator dataset from UVFITS to MIRIAD format!' ) else: logger.warning( 'Beam ' + self.beam + ': Polarised calibrator dataset not available!') else: logger.info( 'Beam ' + self.beam + ': Polarised calibrator dataset was already converted from UVFITS to MIRIAD format' ) else: logger.warning( 'Beam ' + self.beam + ': Polarised calibrator dataset not specified. Cannot convert polarised calibrator!' ) else: logger.warning('Beam ' + self.beam + ': Not converting polarised calibrator dataset!') # Convert the target beams if self.convert_target: if self.target != '': logger.info( 'Beam ' + self.beam + ': Converting target beam dataset from UVFITS to MIRIAD format.' ) if not converttargetbeamsuvfits2miriad: if converttargetbeamsuvfitsavailable: subs_managefiles.director( self, 'ch', self.get_crosscalsubdir_path(), verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.target) fits.out = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.target, ext='mir') fits.go() if path.isdir(fits.out): converttargetbeamsuvfits2miriad = True logger.debug( 'Beam ' + self.beam + ': Converted target beam dataset from ' 'UVFITS to MIRIAD format!') else: converttargetbeamsuvfits2miriad = False logger.warning( 'Beam ' + self.beam + ': Could not convert target beam dataset ' '{} from UVFITS to MIRIAD format!'.format( fits.out)) else: logger.warning('Beam ' + self.beam + ': Target beam dataset not available!') else: logger.info('Beam ' + self.beam + ': Target beam dataset was already converted ' 'from MS to UVFITS format') else: logger.warning( 'Beam ' + self.beam + ': Target beam dataset not specified. Cannot convert target beam datasets!' ) else: logger.warning('Beam ' + self.beam + ': Not converting target beam dataset!') # Save the derived parameters for the MS to UVFITS conversion to the parameter file subs_param.add_param(self, cbeam + '_fluxcal_UVFITS2MIRIAD', convertfluxcaluvfits2miriad) subs_param.add_param(self, cbeam + '_polcal_UVFITS2MIRIAD', convertpolcaluvfits2miriad) subs_param.add_param(self, cbeam + '_targetbeams_UVFITS2MIRIAD', converttargetbeamsuvfits2miriad) if self.convert_averagems and self.subdirification: logger.info('Beam ' + self.beam + ': Averaging down target measurement set') average_cmd = 'mstransform(vis="{vis}", outputvis="{outputvis}", chanaverage=True, chanbin=64)' vis = self.get_target_path() outputvis = vis.replace(".MS", "_avg.MS") lib.run_casa([average_cmd.format(vis=vis, outputvis=outputvis)], timeout=10000) # Remove measurement sets if wanted if self.convert_removems and self.subdirification: logger.info('Beam ' + self.beam + ': Removing measurement sets') vis = self.get_target_path() if path.exists(vis): subs_managefiles.director(self, 'rm', vis) # Remove the UVFITS files if wanted if self.convert_removeuvfits and self.subdirification: logger.info('Beam ' + self.beam + ': Removing all UVFITS files') if self.fluxcal != '' and path.exists( mspath_to_fitspath( self.get_crosscalsubdir_path(), self.fluxcal)) and convertfluxcalms2uvfits: subs_managefiles.director( self, 'rm', mspath_to_fitspath(self.get_crosscalsubdir_path(), self.fluxcal)) logger.info('Beam ' + self.beam + ': Removed fluxcal UVFITS files') else: logger.warning( 'Beam ' + self.beam + ': No fluxcal UVFITS file available for removing') if self.polcal != '' and path.exists( mspath_to_fitspath( self.get_crosscalsubdir_path(), self.polcal)) and convertpolcalms2uvfits: subs_managefiles.director( self, 'rm', mspath_to_fitspath(self.get_crosscalsubdir_path(), self.polcal)) logger.info('Beam ' + self.beam + ': Removed polcal UVFITS files') else: logger.warning( 'Beam ' + self.beam + ': No polcal UVFITS file available for removing') if self.target != '' and path.exists( mspath_to_fitspath( self.get_crosscalsubdir_path(), self.target)) and convertfluxcalms2uvfits: subs_managefiles.director( self, 'rm', mspath_to_fitspath(self.get_crosscalsubdir_path(), self.target)) logger.info('Beam ' + self.beam + ': Removed target UVFITS files') else: logger.warning( 'Beam ' + self.beam + ': No target UVFITS file available for removing')
def parametric(self): """ Parametric self calibration using an NVSS/FIRST skymodel and calculating spectral indices by source matching with WENSS. """ if self.selfcal_parametric: subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) logger.info(' Doing parametric self calibration') subs_managefiles.director(self, 'ch', self.selfcaldir) for chunk in self.list_chunks(): logger.info('Starting parametric self calibration routine on chunk ' + chunk + ' #') subs_managefiles.director(self, 'ch', self.selfcaldir + '/' + chunk) subs_managefiles.director(self, 'mk', self.selfcaldir + '/' + chunk + '/' + 'pm') parametric_textfile = lsm.lsm_model(chunk + '.mir', self.selfcal_parametric_skymodel_radius, self.selfcal_parametric_skymodel_cutoff, self.selfcal_parametric_skymodel_distance) lsm.write_model(self.selfcaldir + '/' + chunk + '/' + 'pm' + '/model.txt', parametric_textfile) logger.info('Creating model from textfile model.txt for chunk ' + chunk + ' #') uv = aipy.miriad.UV(self.selfcaldir + '/' + chunk + '/' + chunk + '.mir') freq = uv['sfreq'] uvmodel = lib.miriad('uvmodel') uvmodel.vis = chunk + '.mir' parametric_modelfile = open(self.selfcaldir + '/' + str(chunk) + '/' + 'pm' + '/model.txt', 'r') for n, source in enumerate(parametric_modelfile.readlines()): if n == 0: uvmodel.options = 'replace,mfs' else: uvmodel.options = 'add,mfs' uvmodel.offset = source.split(',')[0] + ',' + source.split(',')[1] uvmodel.flux = source.split(',')[2] + ',i,' + str(freq) + ',' + source.split(',')[4].rstrip( '\n') + ',0,0' uvmodel.out = 'pm/tmp' + str(n) uvmodel.go() uvmodel.vis = uvmodel.out subs_managefiles.director(self, 'rn', 'pm/model', uvmodel.out) # Rename the last modelfile to model subs_managefiles.director(self, 'rm', 'pm/tmp*') # Remove all the obsolete modelfiles logger.info('Doing parametric self-calibration on chunk {} with solution interval {} min' 'and uvrange limits of {}~{} klambda #'.format(chunk, self.selfcal_parametric_solint, self.selfcal_parametric_uvmin, self.selfcal_parametric_uvmax)) selfcal = lib.miriad('selfcal') selfcal.vis = chunk + '.mir' selfcal.model = 'pm/model' selfcal.interval = self.selfcal_parametric_solint selfcal.select = "'" + 'uvrange(' + str(self.selfcal_parametric_uvmin) + ',' + str( self.selfcal_parametric_uvmax) + ')' + "'" # Choose reference antenna if given if self.selfcal_refant == '': pass else: selfcal.refant = self.selfcal_refant # Do amplitude calibration if wanted if self.selfcal_parametric_amp: selfcal.options = 'mfs,amp' else: selfcal.options = 'mfs' selfcal.go() logger.info('Parametric self calibration routine on chunk ' + chunk + ' done! #') logger.info(' Parametric self calibration done') else: logger.info(' Parametric self calibration disabled')
def ms2miriad(self): """ Converts the data from MS to MIRIAD format via UVFITS using drivecasa. Does it for the flux calibrator, polarisation calibrator, and target field independently. """ subs_setinit.setinitdirs(self) nbeams = 37 # Create the parameters for the parameter file for converting from MS to UVFITS format # Flux calibrator MS dataset available? convertfluxcalmsavailable = get_param_def(self, 'convert_fluxcal_MSavailable', False) # Polarised calibrator MS dataset available? convertpolcalmsavailable = get_param_def(self, 'convert_polcal_MSavailable', False) # Target beam MS dataset available? converttargetbeamsmsavailable = get_param_def(self, 'convert_targetbeams_MSavailable', np.full(nbeams, False)) # Flux calibrator MS dataset converted to UVFITS? convertfluxcalms2uvfits = get_param_def(self, 'convert_fluxcal_MS2UVFITS', False) # Polarised calibrator MS dataset converted to UVFITS? convertpolcalms2uvfits = get_param_def(self, 'convert_polcal_MS2UVFITS', False) # Target beam MS dataset converted to UVFITS? converttargetbeamsms2uvfits = get_param_def(self, 'convert_targetbeams_MS2UVFITS', np.full(nbeams, False)) # Flux calibrator UVFITS dataset available? convertfluxcaluvfitsavailable = get_param_def(self, 'convert_fluxcal_UVFITSavailable', False) # Polarised calibrator UVFITS dataset available? convertpolcaluvfitsavailable = get_param_def(self, 'convert_polcal_UVFITSavailable', False) # Target beam UVFITS dataset available? converttargetbeamsuvfitsavailable = get_param_def(self, 'convert_targetbeams_UVFITSavailable', np.full(nbeams, False)) # Flux calibrator UVFITS dataset converted to MIRIAD? convertfluxcaluvfits2miriad = get_param_def(self, 'convert_fluxcal_UVFITS2MIRIAD', False) # Polarised calibrator UVFITS dataset converted to MIRIAD? convertpolcaluvfits2miriad = get_param_def(self, 'convert_polcal_UVFITS2MIRIAD', False) # Target beam UVFITS dataset converted to MIRIAD? converttargetbeamsuvfits2miriad = get_param_def(self, 'convert_targetbeams_UVFITS2MIRIAD', np.full(nbeams, False)) # Check which datasets are available in MS format # if self.fluxcal != '': convertfluxcalmsavailable = os.path.isdir(self.basedir + '00' + '/' + self.rawsubdir + '/' + self.fluxcal) else: logger.warning('Flux calibrator dataset not specified. Cannot convert flux calibrator!') if self.polcal != '': convertpolcalmsavailable = os.path.isdir(self.basedir + '00' + '/' + self.rawsubdir + '/' + self.polcal) else: logger.warning('Polarised calibrator dataset not specified. Cannot convert polarised calibrator!') if self.target != '': for b in range(nbeams): converttargetbeamsmsavailable[b] = os.path.isdir( self.basedir + str(b).zfill(2) + '/' + self.rawsubdir + '/' + self.target) else: logger.warning('Target beam dataset not specified. Cannot convert target beams!') # Save the derived parameters for the availability to the parameter file subs_param.add_param(self, 'convert_fluxcal_MSavailable', convertfluxcalmsavailable) subs_param.add_param(self, 'convert_polcal_MSavailable', convertpolcalmsavailable) subs_param.add_param(self, 'convert_targetbeams_MSavailable', converttargetbeamsmsavailable) # Convert the available MS-datasets to UVFITS # raw_convert_cmd = 'exportuvfits(vis="{basedir}00/{rawsubdir}/{cal}", ' \ 'fitsfile="{basedir}00/{crosscalsubdir}/{calbase}UVFITS", datacolumn="{datacolumn}", ' \ 'combinespw=True, padwithflags=True, multisource=True, writestation=True)' # Convert the flux calibrator if self.convert_fluxcal: if self.fluxcal != '': if not convertfluxcaluvfits2miriad: if convertfluxcalmsavailable: logger.debug('Converting flux calibrator dataset from MS to UVFITS format.') subs_managefiles.director(self, 'mk', self.basedir + '00' + '/' + self.crosscalsubdir, verbose=False) path = self.basedir + '00' + '/' + self.rawsubdir + '/' + self.fluxcal if subs_msutils.has_correcteddata(path): fc_convert = raw_convert_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir, cal=self.fluxcal, calbase=self.fluxcal[:-2], crosscalsubdir=self.crosscalsubdir, datacolumn="corrected") else: fc_convert = raw_convert_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir, cal=self.fluxcal, calbase=self.fluxcal[:-2], crosscalsubdir=self.crosscalsubdir, datacolumn="data") logger.warning('Flux calibrator does not have a corrected_data column! Using uncorrected' 'data for conversion!') lib.run_casa([fc_convert], timeout=3600) if os.path.isfile(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip( 'MS') + 'UVFITS'): convertfluxcalms2uvfits = True logger.info('Converted flux calibrator dataset from MS to UVFITS format!') else: convertfluxcalms2uvfits = False logger.warning('Could not convert flux calibrator dataset from MS to UVFITS format!') else: logger.warning('Flux calibrator dataset not available!') else: logger.info('Flux calibrator dataset was already converted from MS to UVFITS format') else: logger.warning('Flux calibrator dataset not specified. Cannot convert flux calibrator!') else: logger.warning('Not converting flux calibrator dataset!') # Convert the polarised calibrator if self.convert_polcal: if self.polcal != '': if not convertpolcaluvfits2miriad: if convertpolcalmsavailable: logger.debug('Converting polarised calibrator dataset from MS to UVFITS format.') subs_managefiles.director(self, 'mk', self.basedir + '00' + '/' + self.crosscalsubdir, verbose=False) path = self.basedir + '00' + '/' + self.rawsubdir + '/' + self.polcal if subs_msutils.has_correcteddata(path): pc_convert = raw_convert_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir, cal=self.fluxcal, calbase=self.polcal[:-2], crosscalsubdir=self.crosscalsubdir, datacolumn="corrected") else: pc_convert = raw_convert_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir, cal=self.fluxcal, calbase=self.polcal[:-2], crosscalsubdir=self.crosscalsubdir, datacolumn="data") logger.warning('Polarised calibrator does not have a corrected_data column! Using' 'uncorrected data for conversion!') lib.run_casa([pc_convert], timeout=3600) if os.path.isfile(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip( 'MS') + 'UVFITS'): convertpolcalms2uvfits = True logger.info('Converted polarised calibrator dataset from MS to UVFITS format!') else: convertpolcalms2uvfits = False logger.warning('Could not convert polarised calibrator dataset from MS to UVFITS format!') else: logger.warning('Polarised calibrator dataset not available!') else: logger.info('Polarised calibrator dataset was already converted from MS to UVFITS format') else: logger.warning('Polarised calibrator dataset not specified. Cannot convert polarised calibrator!') else: logger.warning('Not converting polarised calibrator dataset!') # Convert the target beams if self.convert_target: if self.target != '': logger.info('Converting target beam datasets from MS to UVFITS format.') if self.convert_targetbeams == 'all': datasets = glob.glob(self.basedir + '[0-9][0-9]' + '/' + self.rawsubdir + '/' + self.target) logger.debug('Converting all available target beam datasets') else: beams = self.convert_targetbeams.split(",") datasets = [self.basedir + str(b).zfill(2) + '/' + self.rawsubdir + '/' + self.target for b in beams] logger.debug('Converting all selected target beam datasets') for vis in datasets: if not converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])]: if converttargetbeamsmsavailable[int(vis.split('/')[-3])]: subs_managefiles.director(self, 'mk', self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir, verbose=False) beam_dataset = vis.split('/')[-3] raw_tg_cmd = 'exportuvfits(vis="{basedir}{beam_dataset}/{rawsubdir}/{target}", ' \ 'fitsfile="{basedir}{beam_dataset}/{crosscalsubdir}/{targetbase}UVFITS", ' \ 'datacolumn="{datacolumn}", combinespw=True, padwithflags=True, ' \ 'multisource=True, writestation=True)' path = self.basedir + beam_dataset + '/' + self.rawsubdir + '/' + self.target if subs_msutils.has_correcteddata(path): datacolumn = "corrected" else: datacolumn = "data" logger.warning('Target beam dataset {} does not have a corrected_data column! Using ' 'uncorrected data for conversion!'.format(beam_dataset)) tg_convert = raw_tg_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir, target=self.target, crosscalsubdir=self.crosscalsubdir, targetbase=self.target.rstrip('MS'), datacolumn=datacolumn, beam_dataset=beam_dataset) lib.run_casa([tg_convert], timeout=7200) if os.path.isfile(self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS'): converttargetbeamsms2uvfits[int(vis.split('/')[-3])] = True logger.debug('Converted dataset of target beam ' + vis.split('/')[ -3] + ' from MS to UVFITS format!') else: converttargetbeamsms2uvfits[int(vis.split('/')[-3])] = False logger.warning('Could not convert dataset for target beam ' + vis.split('/')[ -3] + ' from MS to UVFITS format!') else: logger.warning('Dataset for target beam ' + vis.split('/')[-3] + ' not available!') else: logger.info('Dataset for target beam ' + vis.split('/')[ -3] + ' was already converted from MS to UVFITS format') else: logger.warning('Target beam dataset(s) not specified. Cannot convert target beam datasets!') else: logger.warning('Not converting target beam dataset(s)!') # Save the derived parameters for the MS to UVFITS conversion to the parameter file subs_param.add_param(self, 'convert_fluxcal_MS2UVFITS', convertfluxcalms2uvfits) subs_param.add_param(self, 'convert_polcal_MS2UVFITS', convertpolcalms2uvfits) subs_param.add_param(self, 'convert_targetbeams_MS2UVFITS', converttargetbeamsms2uvfits) # Check which datasets are available in UVFITS format # if self.fluxcal != '': convertfluxcaluvfitsavailable = os.path.isfile( self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip('MS') + 'UVFITS') else: logger.warning('Flux calibrator dataset not specified. Cannot convert flux calibrator!') if self.polcal != '': convertpolcaluvfitsavailable = os.path.isfile( self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip('MS') + 'UVFITS') else: logger.warning('Polarised calibrator dataset not specified. Cannot convert polarised calibrator!') if self.target != '': for b in range(nbeams): converttargetbeamsuvfitsavailable[b] = os.path.isfile( self.basedir + str(b).zfill(2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip( 'MS') + 'UVFITS') else: logger.warning('Target beam dataset not specified. Cannot convert target beams!') # Save the derived parameters for the availability to the parameter file subs_param.add_param(self, 'convert_fluxcal_UVFITSavailable', convertfluxcaluvfitsavailable) subs_param.add_param(self, 'convert_polcal_UVFITSavailable', convertpolcaluvfitsavailable) subs_param.add_param(self, 'convert_targetbeams_UVFITSavailable', converttargetbeamsuvfitsavailable) # Convert the available UVFITS-datasets to MIRIAD format # # Convert the flux calibrator if self.convert_fluxcal: if self.fluxcal != '': if not convertfluxcaluvfits2miriad: if convertfluxcaluvfitsavailable: logger.debug('Converting flux calibrator dataset from UVFITS to MIRIAD format.') subs_managefiles.director(self, 'ch', self.basedir + '00' + '/' + self.crosscalsubdir, verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip( 'MS') + 'UVFITS' fits.out = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip( 'MS') + 'mir' fits.go() if os.path.isdir(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip( 'MS') + 'mir'): convertfluxcaluvfits2miriad = True logger.info('Converted flux calibrator dataset from UVFITS to MIRIAD format!') else: convertfluxcaluvfits2miriad = False logger.warning('Could not convert flux calibrator dataset from UVFITS to MIRIAD format!') else: logger.warning('Flux calibrator dataset not available!') else: logger.info('Flux calibrator dataset was already converted from UVFITS to MIRIAD format') else: logger.warning('Flux calibrator dataset not specified. Cannot convert flux calibrator!') else: logger.warning('Not converting flux calibrator dataset!') # Convert the polarised calibrator if self.convert_polcal: if self.polcal != '': if not convertpolcaluvfits2miriad: if convertpolcaluvfitsavailable: logger.debug('Converting polarised calibrator dataset from UVFITS to MIRIAD format.') subs_managefiles.director(self, 'ch', self.basedir + '00' + '/' + self.crosscalsubdir, verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip( 'MS') + 'UVFITS' fits.out = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip( 'MS') + 'mir' fits.go() if os.path.isdir(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip( 'MS') + 'mir'): convertpolcaluvfits2miriad = True logger.info('Converted polarised calibrator dataset from UVFITS to MIRIAD format!') else: convertpolcaluvfits2miriad = False logger.warning( 'Could not convert polarised calibrator dataset from UVFITS to MIRIAD format!') else: logger.warning('Polarised calibrator dataset not available!') else: logger.info('Polarised calibrator dataset was already converted from UVFITS to MIRIAD format') else: logger.warning('Polarised calibrator dataset not specified. Cannot convert polarised calibrator!') else: logger.warning('Not converting polarised calibrator dataset!') # Convert the target beams if self.convert_target: if self.target != '': logger.info('Converting target beam datasets from UVFITS to MIRIAD format.') if self.convert_targetbeams == 'all': datasets = glob.glob( self.basedir + '[0-9][0-9]' + '/' + self.crosscalsubdir + '/' + self.target.rstrip( 'MS') + 'UVFITS') logger.debug('Converting all available target beam datasets') else: beams = self.convert_targetbeams.split(",") datasets = [self.basedir + str(b).zfill(2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip( 'MS') + 'UVFITS' for b in beams] logger.debug('Converting all selected target beam datasets') for vis in datasets: if not converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])]: if converttargetbeamsuvfitsavailable[int(vis.split('/')[-3])]: subs_managefiles.director(self, 'ch', self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir, verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = self.basedir + vis.split('/')[ -3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS' fits.out = self.basedir + vis.split('/')[ -3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'mir' fits.go() if os.path.isdir(self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'mir'): converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])] = True logger.debug('Converted dataset of target beam ' + vis.split('/')[ -3] + ' from UVFITS to MIRIAD format!') else: converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])] = False logger.warning('Could not convert dataset for target beam ' + vis.split('/')[ -3] + ' from UVFITS to MIRIAD format!') else: logger.warning('Dataset for target beam ' + vis.split('/')[-3] + ' not available!') else: logger.info('Dataset for target beam ' + vis.split('/')[ -3] + ' was already converted from MS to UVFITS format') else: logger.warning('Target beam dataset(s) not specified. Cannot convert target beam datasets!') else: logger.warning('Not converting target beam dataset(s)!') # Save the derived parameters for the MS to UVFITS conversion to the parameter file subs_param.add_param(self, 'convert_fluxcal_UVFITS2MIRIAD', convertfluxcaluvfits2miriad) subs_param.add_param(self, 'convert_polcal_UVFITS2MIRIAD', convertpolcaluvfits2miriad) subs_param.add_param(self, 'convert_targetbeams_UVFITS2MIRIAD', converttargetbeamsuvfits2miriad) # Remove the UVFITS files if wanted # if self.convert_removeuvfits: logger.info('Removing all UVFITS files') subs_managefiles.director(self, 'rm', self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip( 'MS') + 'UVFITS') subs_managefiles.director(self, 'rm', self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip( 'MS') + 'UVFITS') for beam in range(nbeams): if os.path.isdir(self.basedir + str(beam).zfill(2) + '/' + self.crosscalsubdir): subs_managefiles.director(self, 'rm', self.basedir + str(beam).zfill( 2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS') else: pass
"chan", "chan", "-", "Jy/beam", "Jy/beam", "Jy/beam", "-", "-", "Jy/beam", "chan", "chan", "pix", "pix", "pix", "pix", "pix", "deg", "deg", "pix", "pix", "pix", "Jy/beam", "-", "-", "-") catParFormt = ("%12s", "%7i", "%10.3f", "%10.3f", "%10.3f", "%7i", "%7i", "%7i", "%7i", "%7i", "%7i", "%8i", "%10.7f", "%10.7f", "%12.6f", "%8.6f", "%7i", "%12.6f", "%10.3f", "%10.3f", "%10.3f", "%10.3f", "%10.3f", "%10.3f", "%10.3f", "%10.3f", "%10.3f", "%10.3f", "%10.3f", "%10.3f", "%12.6f", "%10i", "%7i", "%7i") prepare = apercal.prepare() for b in beams: loc = '/tank/hess/apertif/' + taskid + '/B0' + str(b).zfill(2) + '/' print("\t{}".format(loc)) clean_catalog = loc + 'clean_cat.txt' managefiles.director(prepare, 'ch', loc) for c in cubes: line_cube = cube_name + '{0}.fits'.format(c) beam_cube = beam_name + '{0}.fits'.format(c) maskfits = cube_name + '{0}_4sig_mask.fits'.format(c) mask2dfits = cube_name + '{0}_4sig_mask-2d.fits'.format(c) filteredfits = cube_name + '{0}_filtered.fits'.format(c) splinefits = cube_name + '{0}_filtered_spline.fits'.format(c) new_splinefits = cube_name + '{0}_all_spline.fits'.format(c) catalog_file = cube_name + '{0}_4sig_cat.txt'.format(c) if os.path.isfile(maskfits): catalog = ascii.read(catalog_file, header_start=10) if args.sources == 'all': mask_expr = '"(<mask_sofia>.eq.-1).or.(<mask_sofia>.ge.1)"'
def split_data(self): """ Splits out a certain frequency range from the datasets """ subs_setinit.setinitdirs(self) sbeam = 'split_B' + str(self.beam).zfill(2) splitfluxcalstatus = get_param_def(self, sbeam + '_fluxcal_status', False) splitpolcalstatus = get_param_def(self, sbeam + '_polcal_status', False) splittargetbeamsstatus = get_param_def(self, sbeam + '_targetbeams_status', False) logger.info('Beam ' + self.beam + ': Splitting channel ' + str(self.split_startchannel) + ' until ' + str(self.split_endchannel)) # split the flux calibrator dataset logger.debug("self.fluxcal = {}".format(self.fluxcal)) logger.debug("os.path.isdir(self.get_fluxcal_path()) = {}".format( os.path.isdir(self.get_fluxcal_path()))) if self.fluxcal != '' and os.path.isdir(self.get_fluxcal_path()): fluxcal_split = 'split(vis = "' + self.get_fluxcal_path() + '", outputvis = "' + self.get_fluxcal_path().rstrip('.MS') + '_split.MS"' + \ ', spw = "0:' + str(self.split_startchannel) + '~' + str(self.split_endchannel) + '", datacolumn = "data")' lib.run_casa([fluxcal_split], log_output=True, timeout=30000) if os.path.isdir(self.get_fluxcal_path().rstrip('.MS') + '_split.MS'): subs_managefiles.director(self, 'rm', self.get_fluxcal_path()) subs_managefiles.director( self, 'rn', self.get_fluxcal_path(), file_=self.get_fluxcal_path().rstrip('.MS') + '_split.MS') splitfluxcalstatus = True else: splitfluxcalstatus = False logger.warning( 'Beam ' + self.beam + ': Splitting of flux calibrator dataset not successful!') else: splitfluxcalstatus = False logger.warning( 'Beam ' + self.beam + ': Fluxcal not set or dataset not available! Cannot split flux calibrator dataset!' ) subs_param.add_param(self, sbeam + '_fluxcal_status', splitfluxcalstatus) # Split the polarised calibrator dataset logger.debug("self.polcal = {}".format(self.polcal)) logger.debug("os.path.isdir(self.get_polcal_path()) = {}".format( os.path.isdir(self.get_polcal_path()))) if self.polcal != '' and os.path.isdir(self.get_polcal_path()): polcal_split = 'split(vis = "' + self.get_polcal_path() + '", outputvis = "' + self.get_polcal_path().rstrip('.MS') + '_split.MS"' + \ ', spw = "0:' + str(self.split_startchannel) + '~' + str(self.split_endchannel) + '", datacolumn = "data")' lib.run_casa([polcal_split], log_output=True, timeout=30000) if os.path.isdir(self.get_polcal_path().rstrip('.MS') + '_split.MS'): subs_managefiles.director(self, 'rm', self.get_polcal_path()) subs_managefiles.director( self, 'rn', self.get_polcal_path(), file_=self.get_polcal_path().rstrip('.MS') + '_split.MS') splitpolcalstatus = True else: splitpolcalstatus = False logger.warning( 'Beam ' + self.beam + ': Splitting of polarised calibrator dataset not successful!' ) else: splitpolcalstatus = False logger.warning( 'Beam ' + self.beam + ': Polcal not set or dataset not available! Cannot split polarised calibrator dataset!' ) subs_param.add_param(self, sbeam + '_polcal_status', splitpolcalstatus) # Split the target dataset logger.debug("self.target = {}".format(self.target)) logger.debug("os.path.isdir(self.get_target_path()) = {}".format( os.path.isdir(self.get_target_path()))) if self.target != '' and os.path.isdir(self.get_target_path()): target_split = 'split(vis = "' + self.get_target_path() + '", outputvis = "' + self.get_target_path().rstrip('.MS') + '_split.MS"' + \ ', spw = "0:' + str(self.split_startchannel) + '~' + str(self.split_endchannel) + '", datacolumn = "data")' lib.run_casa([target_split], log_output=True, timeout=30000) if os.path.isdir(self.get_target_path().rstrip('.MS') + '_split.MS'): subs_managefiles.director(self, 'rm', self.get_target_path()) subs_managefiles.director( self, 'rn', self.get_target_path(), file_=self.get_target_path().rstrip('.MS') + '_split.MS') splittargetbeamsstatus = True else: splittargetbeamsstatus = False logger.warning('Beam ' + self.beam + ': Splitting of target dataset not successful!') else: splittargetbeamsstatus = False logger.warning( 'Beam ' + self.beam + ': Target not set or dataset not available! Cannot split target beam dataset!' ) subs_param.add_param(self, sbeam + '_targetbeams_status', splittargetbeamsstatus)
def copyobs(self): """ Prepares the directory structure and copies over the needed data from ALTA. Checks for data in the current working directories and copies only missing data. """ subs_setinit.setinitdirs(self) # Check if the parameter is already in the parameter file and load it otherwise create the needed arrays # if not os.path.isdir(self.basedir): os.mkdir(self.basedir) # Is the fluxcal data requested? preparefluxcalrequested = get_param_def(self, 'prepare_fluxcal_requested', False) # Is the polcal data requested? preparepolcalrequested = get_param_def(self, 'prepare_polcal_requested', False) # Is the target data requested? One entry per beam preparetargetbeamsrequested = get_param_def( self, 'prepare_targetbeams_requested', np.full(self.NBEAMS, False)) # Is the fluxcal data already on disk? preparefluxcaldiskstatus = get_param_def(self, 'prepare_fluxcal_diskstatus', False) # Is the polcal data already on disk? preparepolcaldiskstatus = get_param_def(self, 'prepare_polcal_diskstatus', False) # Is the target data already on disk? One entry per beam preparetargetbeamsdiskstatus = get_param_def( self, 'prepare_targetbeams_diskstatus', np.full(self.NBEAMS, False)) # Is the fluxcal data on ALTA? preparefluxcalaltastatus = get_param_def(self, 'prepare_fluxcal_altastatus', False) # Is the polcal data on ALTA? preparepolcalaltastatus = get_param_def(self, 'prepare_polcal_altastatus', False) # Is the target data on disk? One entry per beam preparetargetbeamsaltastatus = get_param_def( self, 'prepare_targetbeams_altastatus', np.full(self.NBEAMS, False)) # Is the fluxcal data copied? preparefluxcalcopystatus = get_param_def(self, 'prepare_fluxcal_copystatus', False) # Is the polcal data on copied? preparepolcalcopystatus = get_param_def(self, 'prepare_polcal_copystatus', False) # Is the target data copied? One entry per beam preparetargetbeamscopystatus = get_param_def( self, 'prepare_targetbeams_copystatus', np.full(self.NBEAMS, False)) # Reason for flux calibrator dataset not being there preparefluxcalrejreason = get_param_def(self, 'prepare_fluxcal_rejreason', np.full(1, '', dtype='U50')) # Reason for polarisation calibrator dataset not being there preparepolcalrejreason = get_param_def(self, 'prepare_polcal_rejreason', np.full(1, '', dtype='U50')) # Reason for a beam dataset not being there preparetargetbeamsrejreason = get_param_def( self, 'prepare_targetbeams_rejreason', np.full(self.NBEAMS, '', dtype='U50')) ################################################ # Start the preparation of the flux calibrator # ################################################ if self.fluxcal != '': # If the flux calibrator is requested preparefluxcalrejreason[0] = '' # Empty the comment string preparefluxcalrequested = True fluxcal = self.get_fluxcal_path() preparefluxcaldiskstatus = os.path.isdir(fluxcal) if preparefluxcaldiskstatus: logger.debug( 'Flux calibrator dataset found on disk ({})'.format( fluxcal)) else: logger.debug( 'Flux calibrator dataset not on disk ({})'.format(fluxcal)) if hasattr(self, 'prepare_bypass_alta') and self.prepare_bypass_alta: logger.debug("Skipping fetching dataset from ALTA") else: # Check if the flux calibrator dataset is available on ALTA preparefluxcalaltastatus = getstatus_alta( self.prepare_date, self.prepare_obsnum_fluxcal, self.beam) if preparefluxcalaltastatus: logger.debug('Flux calibrator dataset available on ALTA') else: logger.warning( 'Flux calibrator dataset not available on ALTA') # Copy the flux calibrator data from ALTA if needed if preparefluxcaldiskstatus and preparefluxcalaltastatus: preparefluxcalcopystatus = True elif preparefluxcaldiskstatus and not preparefluxcalaltastatus: preparefluxcalcopystatus = True logger.warning( 'Flux calibrator data available on disk, but not in ALTA!' ) elif not preparefluxcaldiskstatus and preparefluxcalaltastatus: subs_managefiles.director(self, 'mk', self.basedir + self.beam + '/' + self.rawsubdir, verbose=False) getdata_alta(int(self.prepare_date), int(self.prepare_obsnum_fluxcal), 0, targetdir=self.rawdir + '/' + self.fluxcal) if os.path.isdir(self.get_fluxcal_path()): preparefluxcalcopystatus = True logger.debug( 'Flux calibrator dataset successfully copied from ALTA' ) else: preparefluxcalcopystatus = False preparefluxcalrejreason[ 0] = 'Copy from ALTA not successful' logger.error( 'Flux calibrator dataset available on ALTA, but NOT successfully copied!' ) if self.prepare_flip_ra: flip_ra(self.rawdir + '/' + self.fluxcal, logger=logger) elif not preparefluxcaldiskstatus and not preparefluxcalaltastatus: preparefluxcalcopystatus = False preparefluxcalrejreason[0] = 'Dataset not on ALTA or disk' logger.error( 'Flux calibrator dataset not available on disk nor in ALTA! The next steps will not work!' ) else: # In case the flux calibrator is not specified meaning the parameter is empty. preparefluxcalrequested = False preparefluxcaldiskstatus = False preparefluxcalaltastatus = False preparefluxcalcopystatus = False preparefluxcalrejreason[0] = 'Dataset not specified' logger.error( 'No flux calibrator dataset specified. The next steps will not work!' ) # Save the derived parameters for the fluxcal to the parameter file subs_param.add_param(self, 'prepare_fluxcal_requested', preparefluxcalrequested) subs_param.add_param(self, 'prepare_fluxcal_diskstatus', preparefluxcaldiskstatus) subs_param.add_param(self, 'prepare_fluxcal_altastatus', preparefluxcalaltastatus) subs_param.add_param(self, 'prepare_fluxcal_copystatus', preparefluxcalcopystatus) subs_param.add_param(self, 'prepare_fluxcal_rejreason', preparefluxcalrejreason) ######################################################## # Start the preparation of the polarisation calibrator # ######################################################## if self.polcal != '': # If the polarised calibrator is requested preparepolcalrejreason[0] = '' # Empty the comment string preparepolcalrequested = True preparepolcaldiskstatus = os.path.isdir(self.get_polcal_path()) if preparepolcaldiskstatus: logger.debug('Polarisation calibrator dataset found on disk') else: logger.debug('Polarisation calibrator dataset not on disk') if hasattr(self, 'prepare_bypass_alta') and self.prepare_bypass_alta: logger.debug("Skipping fetching dataset from ALTA") else: # Check if the polarisation calibrator dataset is available on ALTA preparepolcalaltastatus = getstatus_alta( self.prepare_date, self.prepare_obsnum_polcal, self.beam) if preparepolcalaltastatus: logger.debug( 'Polarisation calibrator dataset available on ALTA') else: logger.warning( 'Polarisation calibrator dataset not available on ALTA' ) # Copy the polarisation calibrator data from ALTA if needed if preparepolcaldiskstatus and preparepolcalaltastatus: preparepolcalcopystatus = True elif preparepolcaldiskstatus and not preparepolcalaltastatus: preparepolcalcopystatus = True logger.warning( 'Polarisation calibrator data available on disk, but not in ALTA!' ) elif not preparepolcaldiskstatus and preparepolcalaltastatus: subs_managefiles.director(self, 'mk', self.basedir + self.beam + '/' + self.rawsubdir, verbose=False) getdata_alta(int(self.prepare_date), int(self.prepare_obsnum_polcal), 0, targetdir=self.rawdir + '/' + self.polcal) if os.path.isdir(self.get_polcal_path()): preparepolcalcopystatus = True logger.debug( 'Polarisation calibrator dataset successfully copied from ALTA' ) else: preparepolcalcopystatus = False preparepolcalrejreason[ 0] = 'Copy from ALTA not successful' logger.error( 'Polarisation calibrator dataset available on ALTA, but NOT successfully copied!' ) if self.prepare_flip_ra: flip_ra(self.rawdir + '/' + self.polcal, logger=logger) elif not preparepolcaldiskstatus and not preparepolcalaltastatus: preparepolcalcopystatus = False preparepolcalrejreason[0] = 'Dataset not on ALTA or disk' logger.warning( 'Polarisation calibrator dataset not available on disk nor in ALTA! Polarisation calibration will not work!' ) else: # In case the polarisation calibrator is not specified meaning the parameter is empty. preparepolcalrequested = False preparepolcaldiskstatus = False preparepolcalaltastatus = False preparepolcalcopystatus = False preparepolcalrejreason[0] = 'Dataset not specified' logger.warning( 'No polarisation calibrator dataset specified. Polarisation calibration will not work!' ) # Save the derived parameters for the polcal to the parameter file subs_param.add_param(self, 'prepare_polcal_requested', preparepolcalrequested) subs_param.add_param(self, 'prepare_polcal_diskstatus', preparepolcaldiskstatus) subs_param.add_param(self, 'prepare_polcal_altastatus', preparepolcalaltastatus) subs_param.add_param(self, 'prepare_polcal_copystatus', preparepolcalcopystatus) subs_param.add_param(self, 'prepare_polcal_rejreason', preparepolcalrejreason) ################################################ # Start the preparation of the target datasets # ################################################ if self.prepare_obsnum_target and self.prepare_obsnum_target != '': if self.prepare_target_beams == 'all': # if all beams are requested reqbeams_int = range( self.NBEAMS) # create a list of numbers for the beams reqbeams = [str(b).zfill(2) for b in reqbeams_int] # Add the leading zeros else: # if only certain beams are requested reqbeams = self.prepare_target_beams.split(",") reqbeams_int = [int(b) for b in reqbeams] reqbeams = [str(b).zfill(2) for b in reqbeams_int] # Add leading zeros for beam in reqbeams: preparetargetbeamsrequested[int(beam)] = True for b in reqbeams_int: # Check which target beams are already on disk preparetargetbeamsrejreason[int( b)] = '' # Empty the comment string preparetargetbeamsdiskstatus[b] = os.path.isdir( self.basedir + str(b).zfill(2) + '/' + self.rawsubdir + '/' + self.target) if preparetargetbeamsdiskstatus[b]: logger.debug('Target dataset for beam ' + str(b).zfill(2) + ' found on disk') else: logger.debug('Target dataset for beam ' + str(b).zfill(2) + ' NOT found on disk') if hasattr(self, 'prepare_bypass_alta') and self.prepare_bypass_alta: logger.debug("Skipping fetching dataset from ALTA") else: # Check which target datasets are available on ALTA preparetargetbeamsaltastatus[b] = getstatus_alta( self.prepare_date, self.prepare_obsnum_target, str(b).zfill(2)) if preparetargetbeamsaltastatus[b]: logger.debug('Target dataset for beam ' + str(b).zfill(2) + ' available on ALTA') else: logger.debug('Target dataset for beam ' + str(b).zfill(2) + ' NOT available on ALTA') if hasattr(self, 'prepare_bypass_alta') and self.prepare_bypass_alta: logger.debug("Skipping fetching dataset from ALTA") else: # Set the copystatus of the beams and copy beams which are requested but not on disk for c in reqbeams_int: if preparetargetbeamsdiskstatus[ c] and preparetargetbeamsaltastatus[c]: preparetargetbeamscopystatus[c] = True elif preparetargetbeamsdiskstatus[ c] and not preparetargetbeamsaltastatus[c]: preparetargetbeamscopystatus[c] = True logger.warning('Target dataset for beam ' + str(c).zfill(2) + ' available on disk, but not in ALTA!') elif not preparetargetbeamsdiskstatus[ c] and preparetargetbeamsaltastatus[c] and str( c ).zfill( 2 ) in reqbeams: # if target dataset is requested, but not on disk subs_managefiles.director(self, 'mk', self.basedir + str(c).zfill(2) + '/' + self.rawsubdir, verbose=False) getdata_alta(int(self.prepare_date), int(self.prepare_obsnum_target), int(str(c).zfill(2)), targetdir=self.basedir + str(c).zfill(2) + '/' + self.rawsubdir + '/' + self.target) # Check if copy was successful if os.path.isdir(self.basedir + str(c).zfill(2) + '/' + self.rawsubdir + '/' + self.target): preparetargetbeamscopystatus[c] = True else: preparetargetbeamscopystatus[c] = False preparetargetbeamsrejreason[int( c)] = 'Copy from ALTA not successful' logger.error( 'Target beam dataset available on ALTA, but NOT successfully copied!' ) if self.prepare_flip_ra: flip_ra(self.basedir + str(c).zfill(2) + '/' + self.rawsubdir + '/' + self.target, logger=logger) elif not preparetargetbeamsdiskstatus[ c] and not preparetargetbeamsaltastatus[c] and str( c).zfill(2) in reqbeams: preparetargetbeamscopystatus[c] = False preparetargetbeamsrejreason[int( c)] = 'Dataset not on ALTA or disk' logger.error( 'Target beam dataset not available on disk nor in ALTA! Requested beam cannot be processed!' ) else: # If no target dataset is requested meaning the parameter is empty logger.warning('No target datasets specified!') for b in range(self.NBEAMS): preparetargetbeamsrequested[b] = False preparetargetbeamsdiskstatus[b] = False preparetargetbeamsaltastatus[b] = False preparetargetbeamscopystatus[b] = False preparetargetbeamsrejreason[int(b)] = 'Dataset not specified' # Save the derived parameters for the target beams to the parameter file subs_param.add_param(self, 'prepare_targetbeams_requested', preparetargetbeamsrequested) subs_param.add_param(self, 'prepare_targetbeams_diskstatus', preparetargetbeamsdiskstatus) subs_param.add_param(self, 'prepare_targetbeams_altastatus', preparetargetbeamsaltastatus) subs_param.add_param(self, 'prepare_targetbeams_copystatus', preparetargetbeamscopystatus) subs_param.add_param(self, 'prepare_targetbeams_rejreason', preparetargetbeamsrejreason)
def selfcal_standard(self): """ Executes the standard method of self-calibration with the given parameters """ subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) logger.info(' Starting standard self calibration routine') subs_managefiles.director(self, 'ch', self.selfcaldir) for chunk in self.list_chunks(): logger.info('Starting standard self-calibration routine on frequency chunk ' + chunk + ' #') subs_managefiles.director(self, 'ch', self.selfcaldir + '/' + chunk) if os.path.isfile(self.selfcaldir + '/' + chunk + '/' + chunk + '.mir/visdata'): theoretical_noise = calc_theoretical_noise(self.selfcaldir + '/' + chunk + '/' + chunk + '.mir') logger.info('Theoretical noise for chunk ' + chunk + ' is ' + str(theoretical_noise) + ' Jy/beam #') theoretical_noise_threshold = self.calc_theoretical_noise_threshold(theoretical_noise) logger.info('Your theoretical noise threshold will be ' + str( self.selfcal_standard_nsigma) + ' times the theoretical noise corresponding to ' + str( theoretical_noise_threshold) + ' Jy/beam #') dr_list = calc_dr_maj(self.selfcal_standard_drinit, self.selfcal_standard_dr0, self.selfcal_standard_majorcycle, self.selfcal_standard_majorcycle_function) logger.info( 'Your dynamic range limits are set to ' + str(dr_list) + ' for the major self-calibration cycles #') for majc in range(self.selfcal_standard_majorcycle): logger.info( 'Major self-calibration cycle ' + str(majc) + ' for frequency chunk ' + chunk + ' started #') subs_managefiles.director(self, 'mk', self.selfcaldir + '/' + str(chunk) + '/' + str(majc).zfill(2)) # Calculate the dynamic ranges during minor cycles dr_minlist = self.calc_dr_min(dr_list, majc, self.selfcal_standard_minorcycle, self.selfcal_standard_minorcycle_function) logger.info('The minor cycle dynamic range limits for major cycle ' + str(majc) + ' are ' + str( dr_minlist) + ' #') for minc in range(self.selfcal_standard_minorcycle): try: self.run_continuum_minoriteration(chunk, majc, minc, dr_minlist[minc], theoretical_noise_threshold) except Exception: logger.warning('Chunk ' + chunk + ' does not seem to contain data to image #') break try: logger.info('Doing self-calibration with uvmin=' + str( self.selfcal_standard_uvmin[majc]) + ', uvmax=' + str( self.selfcal_standard_uvmax[majc]) + ', solution interval=' + str( self.selfcal_standard_solint[majc]) + ' minutes for major cycle ' + str(majc).zfill( 2) + ' #') selfcal = lib.miriad('selfcal') selfcal.vis = chunk + '.mir' selfcal.select = '"' + 'uvrange(' + str(self.selfcal_standard_uvmin[majc]) + ',' + str( self.selfcal_standard_uvmax[majc]) + ')"' selfcal.model = str(majc).zfill(2) + '/model_' + str(minc).zfill(2) selfcal.interval = self.selfcal_standard_solint[majc] # Choose reference antenna if given if self.selfcal_refant == '': pass else: selfcal.refant = self.selfcal_refant # Enable amplitude calibration if triggered if not self.selfcal_standard_amp: # See if we want to do amplitude calibration selfcal.options = 'mfs,phase' elif self.selfcal_standard_amp: selfcal.options = 'mfs,amp' elif self.selfcal_standard_amp == 'auto': modelflux = self.calc_isum(str(majc).zfill(2) + '/model_' + str(minc).zfill(2)) if modelflux >= self.selfcal_standard_amp_auto_limit: logger.info( 'Flux of clean model is ' + str(modelflux) + ' Jy. Doing amplitude calibration. #') selfcal.options = 'mfs,amp' else: selfcal.options = 'mfs,phase' if self.selfcal_standard_nfbin >= 1: selfcal.nfbin = self.selfcal_standard_nfbin selfcal.go() logger.info('Major self-calibration cycle ' + str( majc) + ' for frequency chunk ' + chunk + ' finished #') except Exception: logger.warning( 'Model for self-calibration not found. No further calibration on this chunk possible!') break logger.info('Standard self-calibration routine for chunk ' + chunk + ' finished #') else: logger.warning('No data in chunk ' + chunk + '. Maybe all data is flagged? #') logger.info(' Standard self calibration routine finished')
def run_continuum_minoriteration(self, chunk, majc, minc, drmin, theoretical_noise_threshold): """ Does a selfcal minor iteration for the standard mode chunk: The frequency chunk to image and calibrate maj: Current major iteration min: Current minor iteration drmin: maximum dynamic range for minor iteration theoretical_noise_threshold: calculated theoretical noise threshold """ subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) logger.info('Minor self-calibration cycle ' + str(minc) + ' for frequency chunk ' + chunk + ' started #') if minc == 0: invert = lib.miriad('invert') # Create the dirty image invert.vis = chunk + '.mir' invert.map = str(majc).zfill(2) + '/map_' + str(minc).zfill(2) invert.beam = str(majc).zfill(2) + '/beam_' + str(minc).zfill(2) invert.imsize = self.selfcal_image_imsize invert.cell = self.selfcal_image_cellsize invert.stokes = 'ii' invert.options = 'mfs,double' invert.slop = 1 invert.robust = -2 invert.go() imax = self.calc_imax(str(majc).zfill(2) + '/map_' + str(minc).zfill(2)) noise_threshold = self.calc_noise_threshold(imax, minc, majc) dynamic_range_threshold = calc_dynamic_range_threshold(imax, drmin, self.selfcal_standard_minorcycle0_dr) mask_threshold, mask_threshold_type = calc_mask_threshold(theoretical_noise_threshold, noise_threshold, dynamic_range_threshold) logger.info('Mask threshold for major/minor cycle ' + str(majc) + '/' + str(minc) + ' set to ' + str( mask_threshold) + ' Jy/beam #') logger.info('Mask threshold set by ' + str(mask_threshold_type) + ' #') if majc == 0: maths = lib.miriad('maths') maths.out = str(majc).zfill(2) + '/mask_' + str(minc).zfill(2) maths.exp = '"<' + str(majc).zfill(2) + '/map_' + str(minc).zfill(2) + '>"' maths.mask = '"<' + str(majc).zfill(2) + '/map_' + str(minc).zfill(2) + '>.gt.' + str( mask_threshold) + '"' maths.go() logger.info('Mask with threshold ' + str(mask_threshold) + ' Jy/beam created #') else: subs_managefiles.director(self, 'cp', str(majc).zfill(2) + '/mask_' + str(minc).zfill(2), file_=str(majc - 1).zfill(2) + '/mask_' + str( self.selfcal_standard_minorcycle - 1).zfill(2)) logger.info('Mask from last minor iteration of last major cycle copied #') clean_cutoff = self.calc_clean_cutoff(mask_threshold) logger.info('Clean threshold at major/minor cycle ' + str(majc) + '/' + str(minc) + ' was set to ' + str( clean_cutoff) + ' Jy/beam #') clean = lib.miriad('clean') # Clean the image down to the calculated threshold clean.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2) clean.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2) clean.out = str(majc).zfill(2) + '/model_' + str(minc).zfill(2) clean.cutoff = clean_cutoff clean.niters = 1000000 clean.region = '"' + 'mask(' + str(majc).zfill(2) + '/mask_' + str(minc).zfill(2) + ')' + '"' clean.go() logger.info('Major/minor cycle ' + str(majc) + '/' + str(minc) + ' cleaning done #') restor = lib.miriad('restor') restor.model = str(majc).zfill(2) + '/model_' + str(minc).zfill(2) restor.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2) restor.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2) restor.out = str(majc).zfill(2) + '/image_' + str(minc).zfill(2) restor.mode = 'clean' restor.go() # Create the cleaned image logger.info('Cleaned image for major/minor cycle ' + str(majc) + '/' + str(minc) + ' created #') restor.mode = 'residual' restor.out = str(majc).zfill(2) + '/residual_' + str(minc).zfill(2) restor.go() logger.info('Residual image for major/minor cycle ' + str(majc) + '/' + str(minc) + ' created #') logger.info('Peak of the residual image is ' + str( self.calc_imax(str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) + ' Jy/beam #') logger.info('RMS of the residual image is ' + str( self.calc_irms(str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) + ' Jy/beam #') else: imax = self.calc_imax(str(majc).zfill(2) + '/map_' + str(0).zfill(2)) noise_threshold = self.calc_noise_threshold(imax, minc, majc) dynamic_range_threshold = calc_dynamic_range_threshold(imax, drmin, self.selfcal_standard_minorcycle0_dr) mask_threshold, mask_threshold_type = calc_mask_threshold(theoretical_noise_threshold, noise_threshold, dynamic_range_threshold) logger.info('Mask threshold for major/minor cycle ' + str(majc) + '/' + str(minc) + ' set to ' + str( mask_threshold) + ' Jy/beam #') logger.info('Mask threshold set by ' + str(mask_threshold_type) + ' #') maths = lib.miriad('maths') maths.out = str(majc).zfill(2) + '/mask_' + str(minc).zfill(2) maths.exp = '"<' + str(majc).zfill(2) + '/image_' + str(minc - 1).zfill(2) + '>"' maths.mask = '"<' + str(majc).zfill(2) + '/image_' + str(minc - 1).zfill(2) + '>.gt.' + str( mask_threshold) + '"' maths.go() logger.info('Mask with threshold ' + str(mask_threshold) + ' Jy/beam created #') clean_cutoff = self.calc_clean_cutoff(mask_threshold) logger.info('Clean threshold at major/minor cycle ' + str(majc) + '/' + str(minc) + ' was set to ' + str( clean_cutoff) + ' Jy/beam #') clean = lib.miriad('clean') # Clean the image down to the calculated threshold clean.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2) clean.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2) clean.model = str(majc).zfill(2) + '/model_' + str(minc - 1).zfill(2) clean.out = str(majc).zfill(2) + '/model_' + str(minc).zfill(2) clean.cutoff = clean_cutoff clean.niters = 1000000 clean.region = '"' + 'mask(' + str(majc).zfill(2) + '/mask_' + str(minc).zfill(2) + ')' + '"' clean.go() logger.info('Major/minor cycle ' + str(majc) + '/' + str(minc) + ' cleaning done #') restor = lib.miriad('restor') restor.model = str(majc).zfill(2) + '/model_' + str(minc).zfill(2) restor.beam = str(majc).zfill(2) + '/beam_' + str(0).zfill(2) restor.map = str(majc).zfill(2) + '/map_' + str(0).zfill(2) restor.out = str(majc).zfill(2) + '/image_' + str(minc).zfill(2) restor.mode = 'clean' restor.go() # Create the cleaned image logger.info('Cleaned image for major/minor cycle ' + str(majc) + '/' + str(minc) + ' created #') restor.mode = 'residual' restor.out = str(majc).zfill(2) + '/residual_' + str(minc).zfill(2) restor.go() logger.info('Residual image for major/minor cycle ' + str(majc) + '/' + str(minc) + ' created #') logger.info('Peak of the residual image is ' + str( self.calc_imax(str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) + ' Jy/beam #') logger.info('RMS of the residual image is ' + str( self.calc_irms(str(majc).zfill(2) + '/residual_' + str(minc).zfill(2))) + ' Jy/beam #') logger.info('Minor self-calibration cycle ' + str(minc) + ' for frequency chunk ' + chunk + ' finished #')
def convert_selfcaluv2uvfits(self): """ Looks for the last self-calibrated uv-fits file, copies its gains over to the original file, applies them and coverts to UVFITS format """ subs_setinit.setinitdirs(self) sbeam = 'selfcal_B' + str(self.beam).zfill(2) tbeam = 'transfer_B' + str(self.beam).zfill(2) # Create the parameters for the parameter file for the conversion of the UVFITS-files transfertargetbeamsselfcaluv2uvfitsstatus = get_param_def( self, tbeam + '_targetbeams_selfcaluv2uvfits_status', False) if self.transfer_convert_selfcaluv2uvfits: subs_setinit.setinitdirs(self) subs_setinit.setdatasetnamestomiriad(self) subs_managefiles.director(self, 'ch', self.transferdir, verbose=True) if not transfertargetbeamsselfcaluv2uvfitsstatus: # Get the status of the selfcal for the specified beam selfcaltargetbeamsphasestatus = get_param_def( self, sbeam + '_targetbeams_phase_status', False) selfcaltargetbeamsampstatus = get_param_def( self, sbeam + '_targetbeams_amp_status', False) datasetname_amp = os.path.join( self.selfcaldir, self.target).rstrip('.mir') + '_amp.mir' datasetname_phase = os.path.join(self.selfcaldir, self.target) logger.debug("Setting amplitude selfcal file name: {}".format( datasetname_amp)) logger.debug("Setting phase selfcal file name: {}".format( datasetname_phase)) # datasetname_amp = self.get_target_path().rstrip('.mir') + '_amp.mir' # datasetname_phase = self.get_target_path() if os.path.isdir( datasetname_amp) and selfcaltargetbeamsampstatus: logger.info('Beam ' + self.beam + ': Using amplitude self-calibrated dataset!') dataset = datasetname_amp elif os.path.isdir( datasetname_phase) and selfcaltargetbeamsphasestatus: logger.info( 'Beam ' + self.beam + ': Using phase self-calibrated dataset. Amplitude calibration was not successful or not wanted!' ) dataset = datasetname_phase else: dataset = None if dataset is not None: # Copy the raw dataset to the transfer directory subs_managefiles.director( self, 'cp', self.transferdir + '/' + self.target, file_=self.crosscaldir + '/' + self.target) if selfcaltargetbeamsampstatus: gpcopy = lib.miriad('gpcopy') gpcopy.vis = datasetname_phase gpcopy.out = self.transferdir + '/' + self.target gpcopy.go() uvaver = lib.miriad('uvaver') uvaver.vis = self.transferdir + '/' + self.target uvaver.out = self.transferdir + '/' + \ self.target.rstrip('.mir') + '_phase.mir' uvaver.go() gpcopy = lib.miriad('gpcopy') gpcopy.vis = datasetname_amp gpcopy.out = self.transferdir + '/' + \ self.target.rstrip('.mir') + '_phase.mir' gpcopy.go() fits = lib.miriad('fits') fits.op = 'uvout' fits.in_ = self.transferdir + '/' + \ self.target.rstrip('.mir') + '_phase.mir' fits.out = self.transferdir + '/' + \ self.target.rstrip('.mir') + '.UVFITS' fits.go() if os.path.isfile(self.transferdir + '/' + self.target.rstrip('.mir') + '.UVFITS'): subs_managefiles.director( self, 'rm', self.transferdir + '/' + self.target) subs_managefiles.director( self, 'rm', self.transferdir + '/' + self.target.rstrip('.mir') + '_phase.mir') transfertargetbeamsselfcaluv2uvfitsstatus = True else: logger.error( 'Beam ' + self.beam + ': Conversion was not successful. No UVFITS-file generated!' ) transfertargetbeamsselfcaluv2uvfitsstatus = False elif selfcaltargetbeamsphasestatus: gpcopy = lib.miriad('gpcopy') gpcopy.vis = datasetname_phase gpcopy.out = self.transferdir + '/' + self.target gpcopy.go() fits = lib.miriad('fits') fits.op = 'uvout' fits.in_ = self.transferdir + '/' + self.target fits.out = self.transferdir + '/' + \ self.target.rstrip('.mir') + '.UVFITS' fits.go() if os.path.isfile(self.transferdir + '/' + self.target.rstrip('.mir') + '.UVFITS'): subs_managefiles.director( self, 'rm', self.transferdir + '/' + self.target) transfertargetbeamsselfcaluv2uvfitsstatus = True else: logger.error( 'Beam ' + self.beam + ': Conversion was not successful. No UVFITS-file generated!' ) transfertargetbeamsselfcaluv2uvfitsstatus = False else: logger.error( 'Beam ' + self.beam + ': Self-calibration was not successful. No conversion to UVFITS-format possible!' ) transfertargetbeamsselfcaluv2uvfitsstatus = False else: logger.info( 'Beam ' + self.beam + ': Conversion of final calibrated data to UVFITS-format already successfully executed!' ) else: logger.info( 'Beam ' + self.beam + ': Conversion of final calibrated data to UVFITS-format not selected!' ) # Save the derived parameters to the parameter file subs_param.add_param(self, tbeam + '_targetbeams_selfcaluv2uvfits_status', transfertargetbeamsselfcaluv2uvfitsstatus)
def ms2miriad(self): """ Converts the data from MS to MIRIAD format via UVFITS using drivecasa. Does it for the flux calibrator, polarisation calibrator, and target field independently. """ subs_setinit.setinitdirs(self) nbeams = 37 # Create the parameters for the parameter file for converting from MS to UVFITS format # Flux calibrator MS dataset available? convertfluxcalmsavailable = get_param_def( self, 'convert_fluxcal_MSavailable', False) # Polarised calibrator MS dataset available? convertpolcalmsavailable = get_param_def(self, 'convert_polcal_MSavailable', False) # Target beam MS dataset available? converttargetbeamsmsavailable = get_param_def( self, 'convert_targetbeams_MSavailable', np.full(nbeams, False)) # Flux calibrator MS dataset converted to UVFITS? convertfluxcalms2uvfits = get_param_def(self, 'convert_fluxcal_MS2UVFITS', False) # Polarised calibrator MS dataset converted to UVFITS? convertpolcalms2uvfits = get_param_def(self, 'convert_polcal_MS2UVFITS', False) # Target beam MS dataset converted to UVFITS? converttargetbeamsms2uvfits = get_param_def( self, 'convert_targetbeams_MS2UVFITS', np.full(nbeams, False)) # Flux calibrator UVFITS dataset available? convertfluxcaluvfitsavailable = get_param_def( self, 'convert_fluxcal_UVFITSavailable', False) # Polarised calibrator UVFITS dataset available? convertpolcaluvfitsavailable = get_param_def( self, 'convert_polcal_UVFITSavailable', False) # Target beam UVFITS dataset available? converttargetbeamsuvfitsavailable = get_param_def( self, 'convert_targetbeams_UVFITSavailable', np.full(nbeams, False)) # Flux calibrator UVFITS dataset converted to MIRIAD? convertfluxcaluvfits2miriad = get_param_def( self, 'convert_fluxcal_UVFITS2MIRIAD', False) # Polarised calibrator UVFITS dataset converted to MIRIAD? convertpolcaluvfits2miriad = get_param_def( self, 'convert_polcal_UVFITS2MIRIAD', False) # Target beam UVFITS dataset converted to MIRIAD? converttargetbeamsuvfits2miriad = get_param_def( self, 'convert_targetbeams_UVFITS2MIRIAD', np.full(nbeams, False)) # Check which datasets are available in MS format # if self.fluxcal != '': convertfluxcalmsavailable = path.isdir(self.get_fluxcal_path()) else: logger.warning( 'Flux calibrator dataset not specified. Cannot convert flux calibrator!' ) if self.polcal != '': convertpolcalmsavailable = path.isdir(self.get_polcal_path()) else: logger.warning( 'Polarised calibrator dataset not specified. Cannot convert polarised calibrator!' ) if self.target != '': for b in range(nbeams): converttargetbeamsmsavailable[b] = path.isdir( self.get_target_path(str(b).zfill(2))) else: logger.warning( 'Target beam dataset not specified. Cannot convert target beams!' ) # Save the derived parameters for the availability to the parameter file subs_param.add_param(self, 'convert_fluxcal_MSavailable', convertfluxcalmsavailable) subs_param.add_param(self, 'convert_polcal_MSavailable', convertpolcalmsavailable) subs_param.add_param(self, 'convert_targetbeams_MSavailable', converttargetbeamsmsavailable) # Convert the flux calibrator if self.convert_fluxcal: if self.fluxcal != '': if not convertfluxcaluvfits2miriad: if convertfluxcalmsavailable: logger.debug( 'Converting flux calibrator dataset from MS to UVFITS format.' ) subs_managefiles.director( self, 'mk', self.get_crosscalsubdir_path(), verbose=False) fluxcal_ms = self.get_fluxcal_path() if subs_msutils.has_correcteddata(fluxcal_ms): datacolumn = "corrected" else: datacolumn = "data" logger.warning( 'Flux calibrator does not have a corrected_data column! Using uncorrected' 'data for conversion!') fluxcal_fits = mspath_to_fitspath( self.get_crosscalsubdir_path(), fluxcal_ms) fc_convert = exportuvfits_cmd.format( vis=self.get_fluxcal_path(), fits=fluxcal_fits, datacolumn=datacolumn) lib.run_casa([fc_convert], timeout=3600) if path.isfile(fluxcal_fits): convertfluxcalms2uvfits = True logger.info( 'Converted flux calibrator dataset from MS to UVFITS format!' ) else: convertfluxcalms2uvfits = False logger.warning( 'Could not convert flux calibrator dataset from MS to UVFITS format!' ) else: logger.warning( 'Flux calibrator dataset not available!') else: logger.info( 'Flux calibrator dataset was already converted from MS to UVFITS format' ) else: logger.warning( 'Flux calibrator dataset not specified. Cannot convert flux calibrator!' ) else: logger.warning('Not converting flux calibrator dataset!') # Convert the polarised calibrator if self.convert_polcal: if self.polcal != '': if not convertpolcaluvfits2miriad: if convertpolcalmsavailable: logger.debug( 'Converting polarised calibrator dataset from MS to UVFITS format.' ) subs_managefiles.director( self, 'mk', self.get_crosscalsubdir_path(), verbose=False) polcal_ms = self.get_polcal_path() if subs_msutils.has_correcteddata(polcal_ms): datacolumn = "corrected" else: datacolumn = "data" logger.warning( 'Polarised calibrator does not have a corrected_data column! Using' 'uncorrected data for conversion!') polcal_fits = mspath_to_fitspath( self.get_crosscalsubdir_path(), polcal_ms) pc_convert = exportuvfits_cmd.format( vis=polcal_ms, fits=polcal_fits, datacolumn=datacolumn) lib.run_casa([pc_convert], timeout=3600) if path.isfile(polcal_fits): convertpolcalms2uvfits = True logger.info( 'Converted polarised calibrator dataset from MS to UVFITS format!' ) else: convertpolcalms2uvfits = False logger.warning( 'Could not convert polarised calibrator dataset from MS to UVFITS format!' ) else: logger.warning( 'Polarised calibrator dataset not available!') else: logger.info( 'Polarised calibrator dataset was already converted from MS to UVFITS format' ) else: logger.warning( 'Polarised calibrator dataset not specified. Cannot convert polarised calibrator!' ) else: logger.warning('Not converting polarised calibrator dataset!') # Convert the target beams if self.convert_target: if self.target != '': logger.info( 'Converting target beam datasets from MS to UVFITS format.' ) if self.convert_targetbeams == 'all': datasets = self.get_datasets() logger.debug( 'Converting all available target beam datasets') else: beams = self.convert_targetbeams.split(",") datasets = self.get_datasets(beams) logger.debug( 'Converting all selected target beam datasets') for vis, beam in datasets: if not converttargetbeamsuvfits2miriad[int(beam)]: if converttargetbeamsmsavailable[int(beam)]: subs_managefiles.director( self, 'mk', self.get_crosscalsubdir_path(beam), verbose=False) target_ms = self.get_target_path(beam) target_fits = mspath_to_fitspath( self.get_crosscalsubdir_path(beam), target_ms) if subs_msutils.has_correcteddata(target_ms): datacolumn = "corrected" else: datacolumn = "data" logger.warning( 'Target beam dataset {} does not have a corrected_data column! Using ' 'uncorrected data for conversion!'.format( beam)) cmd = exportuvfits_cmd.format( vis=target_ms, fits=target_fits, beam=beam, datacolumn=datacolumn) lib.run_casa([cmd], timeout=7200) if path.isfile(target_fits): converttargetbeamsms2uvfits[int(beam)] = True logger.debug( 'Converted dataset of target beam ' 'l{} from MS to UVFITS format!'.format( beam)) else: converttargetbeamsms2uvfits[int(beam)] = False logger.warning( 'Could not convert dataset for target beam ' '{} from MS to UVFITS format!'.format( beam)) else: logger.warning( 'Dataset for target beam {} not available!'. format(beam)) else: logger.info( 'Dataset for target beam {} was already ' 'converted from MS to UVFITS format'.format(beam)) else: logger.warning( 'Target beam dataset(s) not specified. Cannot convert target beam datasets!' ) else: logger.warning('Not converting target beam dataset(s)!') # Save the derived parameters for the MS to UVFITS conversion to the parameter file subs_param.add_param(self, 'convert_fluxcal_MS2UVFITS', convertfluxcalms2uvfits) subs_param.add_param(self, 'convert_polcal_MS2UVFITS', convertpolcalms2uvfits) subs_param.add_param(self, 'convert_targetbeams_MS2UVFITS', converttargetbeamsms2uvfits) # Check which datasets are available in UVFITS format # if self.fluxcal != '': crosscal_fluxcal = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.fluxcal) convertfluxcaluvfitsavailable = path.isfile(crosscal_fluxcal) else: logger.warning( 'Flux calibrator dataset not specified. Cannot convert flux calibrator!' ) if self.polcal != '': crosscal_polcal = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.polcal) convertpolcaluvfitsavailable = path.isfile(crosscal_polcal) else: logger.warning( 'Polarised calibrator dataset not specified. Cannot convert polarised calibrator!' ) if self.target != '': for b in range(nbeams): b_formatted = str(b).zfill(2) converttargetbeamsuvfitsavailable[b] = path.isfile( mspath_to_fitspath( self.get_crosscalsubdir_path(b_formatted), self.target)) else: logger.warning( 'Target beam dataset not specified. Cannot convert target beams!' ) # Save the derived parameters for the availability to the parameter file subs_param.add_param(self, 'convert_fluxcal_UVFITSavailable', convertfluxcaluvfitsavailable) subs_param.add_param(self, 'convert_polcal_UVFITSavailable', convertpolcaluvfitsavailable) subs_param.add_param(self, 'convert_targetbeams_UVFITSavailable', converttargetbeamsuvfitsavailable) # Convert the available UVFITS-datasets to MIRIAD format # # Convert the flux calibrator if self.convert_fluxcal: if self.fluxcal != '': if not convertfluxcaluvfits2miriad: if convertfluxcaluvfitsavailable: logger.debug( 'Converting flux calibrator dataset from UVFITS to MIRIAD format.' ) subs_managefiles.director( self, 'ch', self.get_crosscalsubdir_path(), verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.fluxcal) fits.out = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.fluxcal, ext='mir') fits.go() if path.isdir(fits.out): convertfluxcaluvfits2miriad = True logger.info( 'Converted flux calibrator dataset from UVFITS to MIRIAD format!' ) else: convertfluxcaluvfits2miriad = False logger.warning( 'Could not convert flux calibrator dataset from UVFITS to MIRIAD format!' ) else: logger.warning( 'Flux calibrator dataset not available!') else: logger.info( 'Flux calibrator dataset was already converted from UVFITS to MIRIAD format' ) else: logger.warning( 'Flux calibrator dataset not specified. Cannot convert flux calibrator!' ) else: logger.warning('Not converting flux calibrator dataset!') # Convert the polarised calibrator if self.convert_polcal: if self.polcal != '': if not convertpolcaluvfits2miriad: if convertpolcaluvfitsavailable: logger.debug( 'Converting polarised calibrator dataset from UVFITS to MIRIAD format.' ) subs_managefiles.director( self, 'ch', self.get_crosscalsubdir_path(), verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.polcal) fits.out = mspath_to_fitspath( self.get_crosscalsubdir_path(), self.polcal, ext='mir') fits.go() if path.isdir(fits.out): convertpolcaluvfits2miriad = True logger.info( 'Converted polarised calibrator dataset from UVFITS to MIRIAD format!' ) else: convertpolcaluvfits2miriad = False logger.warning( 'Could not convert polarised calibrator dataset from UVFITS to MIRIAD format!' ) else: logger.warning( 'Polarised calibrator dataset not available!') else: logger.info( 'Polarised calibrator dataset was already converted from UVFITS to MIRIAD format' ) else: logger.warning( 'Polarised calibrator dataset not specified. Cannot convert polarised calibrator!' ) else: logger.warning('Not converting polarised calibrator dataset!') # Convert the target beams if self.convert_target: if self.target != '': logger.info( 'Converting target beam datasets from UVFITS to MIRIAD format.' ) if self.convert_targetbeams == 'all': datasets = glob.glob( mspath_to_fitspath( self.get_crosscalsubdir_path('[0-9][0-9]'), self.get_target_path())) logger.debug( 'Converting all available target beam datasets') else: beams = self.convert_targetbeams.split(",") datasets = [ mspath_to_fitspath( self.get_crosscalsubdir_path(str(b).zfill(2)), self.target) for b in beams ] logger.debug( 'Converting all selected target beam datasets') for vis in datasets: beam = vis.split('/')[-3] if not converttargetbeamsuvfits2miriad[int(beam)]: if converttargetbeamsuvfitsavailable[int(beam)]: subs_managefiles.director( self, 'ch', self.get_crosscalsubdir_path(beam), verbose=False) fits = lib.miriad('fits') fits.op = 'uvin' fits.in_ = mspath_to_fitspath( self.get_crosscalsubdir_path(beam), self.target) fits.out = mspath_to_fitspath( self.get_crosscalsubdir_path(beam), self.target, ext='mir') fits.go() if path.isdir(fits.out): converttargetbeamsuvfits2miriad[int( beam)] = True logger.debug( 'Converted dataset of target beam {} from ' 'UVFITS to MIRIAD format!'.format(beam)) else: converttargetbeamsuvfits2miriad[int( beam)] = False logger.warning( 'Could not convert dataset for target beam ' '{} from UVFITS to MIRIAD format!'.format( beam)) else: logger.warning( 'Dataset for target beam {} not available!'. format(beam)) else: logger.info( 'Dataset for target beam {} was already converted ' 'from MS to UVFITS format'.format(beam)) else: logger.warning( 'Target beam dataset(s) not specified. Cannot convert target beam datasets!' ) else: logger.warning('Not converting target beam dataset(s)!') # Save the derived parameters for the MS to UVFITS conversion to the parameter file subs_param.add_param(self, 'convert_fluxcal_UVFITS2MIRIAD', convertfluxcaluvfits2miriad) subs_param.add_param(self, 'convert_polcal_UVFITS2MIRIAD', convertpolcaluvfits2miriad) subs_param.add_param(self, 'convert_targetbeams_UVFITS2MIRIAD', converttargetbeamsuvfits2miriad) # Remove the UVFITS files if wanted # if self.convert_removeuvfits: logger.info('Removing all UVFITS files') subs_managefiles.director( self, 'rm', mspath_to_fitspath(self.get_crosscalsubdir_path(), self.fluxcal)) subs_managefiles.director( self, 'rm', mspath_to_fitspath(self.get_crosscalsubdir_path(), self.polcal)) for beam in range(nbeams): basedir = self.get_crosscalsubdir_path(str(beam).zfill(2)) if path.isdir(basedir): subs_managefiles.director(self, 'rm', basedir, self.target)
def start_apercal_pipeline(targets, fluxcals, polcals, dry_run=False, basedir=None, flip_ra=False, steps=None, configfilename=None): """ Trigger the start of a fluxcal pipeline. Returns when pipeline is done. Example for taskid, name, beamnr: (190108926, '3C147_36', 36) Fluxcals and polcals can be specified in the wrong order, if the polcal is not polarised they will be flipped. If both polcals and fluxcals are set, they should both be the same length. A list of config files can be provided, i.e., one for each beam. If a single config file is given, copies of it will be created so that there is one config per beam. If no config file is given, the default one is used and copies for each beam are made. Args: targets (Tuple[int, str, List[int]]): taskid, name, list of beamnrs fluxcals (List[Tuple[int, str, int]]): fluxcals: taskid, name, beamnr polcals (List[Tuple[int, str, int]]): polcals: taskid, name, beamnr (can be None) dry_run (bool): interpret arguments, do not actually run pipeline basedir (str): base directory; if not specified will be /data/apertif/{target_taskid} flip_ra (bool): flip RA (for old measurement sets where beamweights were flipped) steps (List[str]): list of steps to perform configfilename (List[str]): Custom configfile (should be full path for now) Returns: Tuple[Dict[int, List[str]], str], str: Tuple of a dict, the formatted runtime, and possibly an exception. The dict contains beam numbers (ints) as keys, a list of failed steps as values. Failed is defined here as 'threw an exception', only for target steps. Please also read logs. """ if steps is None: steps = [ "prepare", "split", "preflag", "ccal", "convert", "scal", "continuum", "polarisation", "line", "transfer" ] (taskid_target, name_target, beamlist_target) = targets # set the base directory if none was provided if not basedir: basedir = '/data/apertif/{}/'.format(taskid_target) elif len(basedir) > 0 and basedir[-1] != '/': basedir = basedir + '/' if not os.path.exists(basedir): os.mkdir(basedir) logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) gitinfo = subprocess.check_output('cd ' + os.path.dirname(apercal.__file__) + '&& git describe --tag; cd', shell=True).strip() logger.info("Apercal version: " + gitinfo) logger.info( "start_apercal called with arguments targets={}; fluxcals={}; polcals={}" .format(targets, fluxcals, polcals)) logger.info("steps = {}".format(steps)) # number of beams to process n_beams = len(beamlist_target) # check the input config file # get the default configfile if none was provided if not configfilename: logger.info("No config file provided, getting default config") # create a list of config file name configfilename_list = [ os.path.join( basedir, "{0}_B{1}_Apercal_settings.cfg".format(taskid_target, str(beam).zfill(2))) for beam in beamlist_target ] # get the default config settings config = lib.get_default_config() # go through the config files and create them for beam_index in range(n_beams): with open(configfilename_list[beam_index], "w") as fp: config.write(fp) logger.info("Beam {} config file saved to {}".format( beamlist_target[beam_index], configfilename_list[beam_index])) # if configfile(s) are given as a list elif type(configfilename) is list: # if it is just one, create copies for each beam in the base directory if len(configfilename) == 1: logger.info( "A single config file was provided. Creating copies of {}". format(configfilename[0])) configfilename_list = [ os.path.join( basedir, "{0}_B{1}_Apercal_settings.cfg".format( taskid_target, str(beam).zfill(2))) for beam in beamlist_target ] # make the copies for config in configfilename_list: lib.basher("cp " + str(configfilename[0]) + " " + str(config)) elif len(configfilename) == n_beams: logger.info("Number of config files and target beams match.") configfilename_list = configfilename else: error = "Number of config files and target beams did not match. Abort" logger.error(error) raise RuntimeError(error) # if configfilename is just a string elif type(configfilename) is str: logger.info( "A single config file was provided. Creating copies of {}".format( configfilename)) configfilename_list = [ os.path.join( basedir, "{0}_B{1}_Apercal_settings.cfg".format(taskid_target, str(beam).zfill(2))) for beam in beamlist_target ] # make the copies for config in configfilename_list: lib.basher("cp " + str(configfilename) + " " + str(config)) else: error = "Unknown input for configfilename. Abort" logger.error(error) raise RuntimeError(error) status = pymp.shared.dict({beamnr: [] for beamnr in beamlist_target}) if fluxcals: name_fluxcal = str(fluxcals[0][1]).strip().split('_')[0].upper() else: name_fluxcal = '' if polcals: name_polcal = str(polcals[0][1]).strip().split('_')[0].upper() else: name_polcal = '' name_target = str(name_target).strip() # .upper() # If both fluxcal and polcal polarized, remove polcal if subs_calmodels.is_polarised( name_polcal) and subs_calmodels.is_polarised(name_fluxcal): name_polcal = "" if (fluxcals and fluxcals != '') and (polcals and polcals != ''): assert (len(fluxcals) == len(polcals)) # avoid symmetry bias, if there is only a polcal but no fluxcal, switch them if fluxcals is None and polcals is not None: logger.info( "Only polcal was provided. Setting polcal {} to fluxcal".format( name_polcal)) fluxcals, polcals = polcals, fluxcals name_polcal = "" # Exchange polcal and fluxcal if specified in the wrong order elif not subs_calmodels.is_polarised(name_polcal) and name_polcal != '': if subs_calmodels.is_polarised(name_fluxcal): logger.info("Switching polcal and fluxcal because " + name_polcal + " is not polarised") fluxcals, polcals = polcals, fluxcals name_polcal = str(polcals[0][1]).strip() else: logger.info("Setting polcal to '' since " + name_polcal + " is not polarised") name_polcal = "" elif name_polcal != '': logger.info("Polcal " + name_polcal + " is polarised, all good") def name_to_ms(name): if not name: return '' elif '3C' in name: return name.upper().strip().split('_')[0] + '.MS' else: return name + '.MS' def name_to_mir(name): if not name: return '' elif '3C' in name: return name.upper().strip().split('_')[0] + '.mir' else: return name + '.mir' def set_files(p): """ Set the basedir, fluxcal, polcal, target properties Args: p (BaseModule): apercal step object (e.g. prepare) Returns: None """ p.basedir = basedir p.fluxcal = name_to_ms(name_fluxcal) p.polcal = name_to_ms(name_polcal) p.target = name_to_ms(name_target) # debug_msg = """ # p.basedir = basedir = {0}; # p.fluxcal = name_to_ms(name_fluxcal) = {1}; # p.polcal = name_to_ms(name_polcal) = {2}; # p.target = name_to_ms(name_target) = {3}; # """.format(basedir, name_to_ms(name_fluxcal), name_to_ms(name_polcal), name_to_ms(name_target)) # logger.debug(debug_msg) beamnrs_fluxcal = [f[2] for f in fluxcals] if len(fluxcals) > 1: # Check every target beam has a fluxcal beam for beamnr_target in beamlist_target: assert (beamnr_target in beamnrs_fluxcal) # creating a copy of the target beamlist as a normal array # to avoid using np.where() for such a small thing if type(beamlist_target) == np.ndarray: beamlist_target_for_config = beamlist_target.tolist() else: beamlist_target_for_config = beamlist_target time_start = time() try: # ======= # Prepare # ======= # keep a start-finish record of step in the main log file if "prepare" in steps: logger.info("Running prepare") start_time_prepare = time() else: logger.info("Skipping prepare") # Prepare fluxcals for (taskid_fluxcal, name_fluxcal, beamnr_fluxcal) in fluxcals: p0 = prepare(file_=configfilename_list[ beamlist_target_for_config.index(beamnr_fluxcal)]) p0.basedir = basedir #set_files(p0) p0.prepare_flip_ra = flip_ra # the following two need to be empty strings for prepare p0.fluxcal = '' p0.polcal = '' p0.target = name_to_ms(name_fluxcal) p0.prepare_target_beams = str(beamnr_fluxcal) p0.prepare_date = str(taskid_fluxcal)[:6] p0.prepare_obsnum_target = validate_taskid(taskid_fluxcal) if "prepare" in steps and not dry_run: try: p0.go() except Exception as e: logger.warning("Prepare failed for fluxcal " + str(taskid_fluxcal) + " beam " + str(beamnr_fluxcal)) logger.exception(e) if 'prepare' in steps: # copy the param file generated here param_file = os.path.join(basedir, 'param.npy') director(p0, 'rn', param_file.replace( ".npy", "_prepare_{}.npy".format(name_fluxcal.split('_')[0])), file_=param_file, ignore_nonexistent=True) # Prepare polcals if name_polcal != '': for (taskid_polcal, name_polcal, beamnr_polcal) in polcals: p0 = prepare(file_=configfilename_list[ beamlist_target_for_config.index(beamnr_polcal)]) p0.basedir = basedir #set_files(p0) p0.prepare_flip_ra = flip_ra # the following two need to be empty strings for prepare p0.fluxcal = '' p0.polcal = '' p0.target = name_to_ms(name_polcal) p0.prepare_target_beams = str(beamnr_polcal) p0.prepare_date = str(taskid_polcal)[:6] p0.prepare_obsnum_target = validate_taskid(taskid_polcal) if "prepare" in steps and not dry_run: try: p0.go() except Exception as e: logger.warning("Prepare failed for polcal " + str(taskid_polcal) + " beam " + str(beamnr_polcal)) logger.exception(e) if 'prepare' in steps: # copy the param file generated here param_file = os.path.join(basedir, 'param.npy') director(p0, 'rn', param_file.replace( ".npy", "_prepare_{}.npy".format( name_polcal.split('_')[0])), file_=param_file, ignore_nonexistent=True) # Prepare target for beamnr in beamlist_target: p0 = prepare(file_=configfilename_list[ beamlist_target_for_config.index(beamnr)]) p0.basedir = basedir # set_files(p0) p0.prepare_flip_ra = flip_ra # the following two need to be empty strings for prepare p0.fluxcal = '' p0.polcal = '' p0.target = name_to_ms(name_target) p0.prepare_date = str(taskid_target)[:6] p0.prepare_obsnum_target = validate_taskid(taskid_target) p0.prepare_target_beams = ','.join( ['{:02d}'.format(beamnr) for beamnr in beamlist_target]) if "prepare" in steps and not dry_run: try: p0.go() except Exception as e: logger.warning("Prepare failed for target " + str(taskid_target) + " beam " + str(beamnr)) logger.exception(e) status[beamnr] += ['prepare'] # keep a start-finish record of step in the main log file if "prepare" in steps: logger.info("Running prepare ... Done ({0:.0f}s)".format( time() - start_time_prepare)) # copy the param file generated here param_file = os.path.join(basedir, 'param.npy') director(p0, 'rn', param_file.replace(".npy", "_prepare_{}.npy".format(name_target)), file_=param_file, ignore_nonexistent=True) # ===== # Split # ===== # keep a start-finish record of step in the main log file if 'split' in steps: logger.info("Running split") start_time_split = time() else: logger.info("Skipping split") # Splitting a small chunk of data for quicklook pipeline # at the moment it all relies on the target beams # what if there are more calibrator than target beams-> realistic? with pymp.Parallel(5) as p: for beam_index in p.range(n_beams): beamnr = beamlist_target[beam_index] # individual logfiles for each process logfilepath = os.path.join(basedir, 'apercal{:02d}.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.debug("Starting logfile for beam " + str(beamnr)) try: s0 = split(file_=configfilename_list[beam_index]) set_files(s0) s0.beam = "{:02d}".format(beamnr) if "split" in steps and not dry_run: s0.go() except Exception as e: logger.warning("Split failed for {0} beam {1}".format( str(taskid_target), str(beamnr))) logger.exception(e) # not sure if following line is necessary status[beamnr] += ['split'] # keep a start-finish record of step in the main log file if "split" in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info( "Running split ... Done ({0:.0f}s)".format(time() - start_time_split)) # copy the param file generated here # param_file = os.path.join(basedir, 'param.npy') # director( # p0, 'rn', param_file.replace(".npy", "_split.npy"), file_=param_file, ignore_nonexistent=True) # ======= # Preflag # ======= # keep a record of the parallalised step in the main log file if "preflag" in steps: logger.info("Running preflag") start_time_preflag = time() else: logger.info("Skipping preflag") # In order to run in parallel, the bandpass table needs to exists # doing it here is not elegant but requires the least amount of changes # to preflage # with pymp.Parallel(10) as p: # for beam_index in p.range(n_beams): # beamnr = beamlist_target[beam_index] # # individual logfiles for each process # logfilepath = os.path.join( # basedir, 'apercal{:02d}.log'.format(beamnr)) # lib.setup_logger('debug', logfile=logfilepath) # logger = logging.getLogger(__name__) # logger.debug("Starting logfile for beam " + str(beamnr)) # p1 = preflag(filename=configfilename) # p1.paramfilename = 'param_{:02d}.npy'.format(beamnr) # p1.basedir = basedir # p1.fluxcal = '' # p1.polcal = '' # p1.target = name_to_ms(name_fluxcal) # p1.beam = "{:02d}".format(beamnr) # p1.preflag_targetbeams = "{:02d}".format(beamnr) # if "preflag" in steps and not dry_run: # try: # bandpass_start_time = time() # logger.info("Running aoflagger bandpass for flux calibrator {0} in beam {1}".format( # p1.target, p1.beam)) # # director( # # p1, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) # p1.go() # # director(p1, 'rm', basedir + '/param.npy', # # ignore_nonexistent=True) # # it is necessary to move the param files in order to keep them # param_file = basedir + \ # '/param_{:02d}.npy'.format(beamnr) # director( # p1, 'mv', param_file, file_=param_file.replace(".npy", "_preflag_{0}.npy".format(name_fluxcal)), ignore_nonexistent=True) # p1.aoflagger_bandpass() # except Exception as e: # logger.warning("Running aoflagger bandpass for flux calibrator {0} in beam {1} ... Failed ({2:.0f}s)".format( # p1.target, p1.beam, time() - bandpass_start_time)) # logger.exception(e) # status[beamnr] += ['preflag_bandpass'] # else: # logger.info("Running aoflagger bandpass for flux calibrator {0} in beam {1} ... Done ({2:.0f}s)".format( # p1.target, p1.beam, time() - bandpass_start_time)) # Flag fluxcal (pretending it's a target, parallelised version) # 5 in parallel with pymp.Parallel(5) as p: for beam_index in p.range(n_beams): beamnr = beamlist_target[beam_index] # individual logfiles for each process logfilepath = os.path.join(basedir, 'apercal{:02d}.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.debug("Starting logfile for beam " + str(beamnr)) try: p1 = preflag(filename=configfilename_list[beam_index]) p1.paramfilename = 'param_{:02d}.npy'.format(beamnr) p1.basedir = basedir p1.fluxcal = '' p1.polcal = '' p1.target = name_to_ms(name_fluxcal) p1.beam = "{:02d}".format(beamnr) p1.preflag_targetbeams = "{:02d}".format(beamnr) if beam_index < 2: p1.preflag_aoflagger_threads = 9 else: p1.preflag_aoflagger_threads = 10 if "preflag" in steps and not dry_run: logger.info( "Running preflag for flux calibrator {0} in beam {1}" .format(p1.target, p1.beam)) preflag_flux_cal_start_time = time() # director( # p1, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) p1.go() # it is necessary to move the param files in order to keep them param_file = os.path.join( basedir, 'param_{:02d}.npy'.format(beamnr)) director(p1, 'rn', param_file.replace( ".npy", "_preflag_{0}.npy".format( name_fluxcal.split('_')[0])), file_=param_file, ignore_nonexistent=True) logger.info( "Running preflag for flux calibrator {0} in beam {1} ... Done ({2:.0f}s)" .format(p1.target, p1.beam, time() - preflag_flux_cal_start_time)) except Exception as e: logger.warning( "Running preflag for flux calibrator {0} in beam {1} ... Failed ({2:.0f}s)" .format(p1.target, p1.beam, time() - preflag_flux_cal_start_time)) logger.exception(e) status[beamnr] += ['preflag'] # Flag polcal (pretending it's a target, parallel version) # 5 in parallel with pymp.Parallel(5) as p: for beam_index in p.range(n_beams): beamnr = beamlist_target[beam_index] # individual logfiles for each process logfilepath = os.path.join(basedir, 'apercal{:02d}.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.debug("Starting logfile for beam " + str(beamnr)) try: p1 = preflag(filename=configfilename_list[beam_index]) # remove next line in final version p1.preflag_aoflagger_version = 'local' p1.basedir = basedir p1.paramfilename = 'param_{:02d}.npy'.format(beamnr) p1.basedir = basedir if name_polcal != '': p1.fluxcal = '' p1.polcal = '' p1.target = name_to_ms(name_polcal) p1.beam = "{:02d}".format(beamnr) p1.preflag_targetbeams = "{:02d}".format(beamnr) if beam_index < 2: p1.preflag_aoflagger_threads = 9 else: p1.preflag_aoflagger_threads = 10 if "preflag" in steps and not dry_run: logger.info( "Running preflag for pol calibrator {0} in beam {1}" .format(p1.target, p1.beam)) preflag_pol_cal_start_time = time() # director( # p1, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) p1.go() # it is necessary to move the param files in order to keep them param_file = os.path.join( basedir, 'param_{:02d}.npy'.format(beamnr)) director(p1, 'rn', param_file.replace( ".npy", "_preflag_{0}.npy".format( name_polcal.split('_')[0])), file_=param_file, ignore_nonexistent=True) logger.info( "Running preflag for pol calibrator {0} in beam {1} ... Done ({2:.0f}s)" .format(p1.target, p1.beam, time() - preflag_pol_cal_start_time)) except Exception as e: logger.warning( "Running preflag for pol calibrator {0} in beam {1} ... Failed ({2:.0f}s)" .format(p1.target, p1.beam, time() - preflag_pol_cal_start_time)) logger.exception(e) status[beamnr] += ['preflag'] # Flag target # 5 in parallel with pymp.Parallel(5) as p: for beam_index in p.range(n_beams): beamnr = beamlist_target[beam_index] # individual logfiles for each process logfilepath = os.path.join(basedir, 'apercal{:02d}.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.debug("Starting logfile for beam " + str(beamnr)) try: p1 = preflag(filename=configfilename_list[beam_index]) # remove next line in final version p1.preflag_aoflagger_version = 'local' p1.paramfilename = 'param_{:02d}.npy'.format(beamnr) p1.basedir = basedir p1.fluxcal = '' p1.polcal = '' p1.target = name_to_ms(name_target) p1.beam = "{:02d}".format(beamnr) p1.preflag_targetbeams = "{:02d}".format(beamnr) if beam_index < 2: p1.preflag_aoflagger_threads = 9 else: p1.preflag_aoflagger_threads = 10 if "preflag" in steps and not dry_run: logger.info( "Running preflag for target {0} in beam {1}". format(p1.target, p1.beam)) preflag_target_start_time = time() # director( # p1, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) p1.go() # it is necessary to move the param files in order to keep them param_file = os.path.join( basedir, 'param_{:02d}.npy'.format(beamnr)) director(p1, 'rn', param_file.replace( ".npy", "_preflag_{0}.npy".format(name_target)), file_=param_file, ignore_nonexistent=True) logger.info( "Running preflag for target {0} in beam {1} ... Done ({2:.0f}s)" .format(p1.target, p1.beam, time() - preflag_target_start_time)) except Exception as e: logger.info( "Running preflag for target {0} in beam {1} ... Failed ({2:.0f}s)" .format(p1.target, p1.beam, time() - preflag_target_start_time)) logger.exception(e) status[beamnr] += ['preflag'] # keep a record of the parallalised step in the main log file if "preflag" in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Running preflag ... Done ({0:.0f}s)".format( time() - start_time_preflag)) # =============== # Crosscal # =============== # keep a record of the parallalised step in the main log file if 'ccal' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Running crosscal") start_time_crosscal = time() else: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Skipping crosscal") if len(fluxcals) == 1 and fluxcals[0][-1] == 0 and n_beams > 1: raise ApercalException( "Sorry, one fluxcal is not supported anymore at the moment") with pymp.Parallel(10) as p: for beam_index in p.range(n_beams): beamnr = beamlist_target[beam_index] logfilepath = os.path.join(basedir, 'apercal{:02d}.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.debug("Starting logfile for beam " + str(beamnr)) try: p2 = ccal(file_=configfilename_list[beam_index]) p2.paramfilename = 'param_{:02d}.npy'.format(beamnr) set_files(p2) p2.beam = "{:02d}".format(beamnr) p2.crosscal_transfer_to_target_targetbeams = "{:02d}".format( beamnr) if "ccal" in steps and not dry_run: # director( # p2, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) p2.go() # it is necessary to move the param files in order to keep them param_file = os.path.join( basedir, 'param_{:02d}.npy'.format(beamnr)) director(p2, 'rn', param_file.replace(".npy", "_crosscal.npy"), file_=param_file, ignore_nonexistent=True) except Exception as e: # Exception was already logged just before logger.warning( "Failed beam {}, skipping that from crosscal".format( beamnr)) logger.exception(e) status[beamnr] += ['crosscal'] # keep a record of the parallalised step in the main log file if 'ccal' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Running crosscal ... Done ({0:.0f}s)".format( time() - start_time_crosscal)) # ======= # Convert # ======= # keep a record of the parallalised step in the main log file if 'convert' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Running convert") start_time_convert = time() else: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Skipping convert") # 5 threads to not hammer the disks too much, convert is only IO with pymp.Parallel(5) as p: for beam_index in p.range(n_beams): beamnr = beamlist_target[beam_index] logfilepath = os.path.join(basedir, 'apercal{:02d}.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) try: p3 = convert(file_=configfilename_list[beam_index]) p3.paramfilename = 'param_{:02d}.npy'.format(beamnr) set_files(p3) p3.beam = "{:02d}".format(beamnr) p3.convert_targetbeams = "{:02d}".format(beamnr) if "convert" in steps and not dry_run: # director( # p3, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) p3.go() # it is necessary to move the param files in order to keep them param_file = os.path.join( basedir, 'param_{:02d}.npy'.format(beamnr)) director(p3, 'rn', param_file.replace(".npy", "_convert.npy"), file_=param_file, ignore_nonexistent=True) # director( # p3, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) except Exception as e: logger.warning( "Failed beam {}, skipping that from convert".format( beamnr)) logger.exception(e) status[beamnr] += ['convert'] if 'convert' in steps: # keep a record of the parallalised step in the main log file logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Running convert ... Done ({0:.0f}s)".format( time() - start_time_convert)) # ================================== # Selfcal + Continuum + Polarisation # ================================== # keep a record of the parallalised step in the main log file if 'scal' in steps or 'continuum' in steps or 'polarisation' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Running selfcal and/or continuum and/or polarisation") start_time_selfcal_continuum_polarisation = time() else: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Skipping selfcal and continuum and polarisation") with pymp.Parallel(10) as p: for beam_index in p.range(n_beams): beamnr = beamlist_target[beam_index] logfilepath = os.path.join(basedir, 'apercal{:02d}.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) try: p4 = scal(file_=configfilename_list[beam_index]) p4.paramfilename = 'param_{:02d}.npy'.format(beamnr) p4.basedir = basedir p4.beam = "{:02d}".format(beamnr) p4.target = name_target + '.mir' if "scal" in steps and not dry_run: p4.go() except Exception as e: # Exception was already logged just before logger.warning( "Failed beam {}, skipping that from scal".format( beamnr)) logger.exception(e) status[beamnr] += ['scal'] try: p5 = continuum(file_=configfilename_list[beam_index]) p5.paramfilename = 'param_{:02d}.npy'.format(beamnr) p5.basedir = basedir p5.beam = "{:02d}".format(beamnr) p5.target = name_target + '.mir' if "continuum" in steps and not dry_run: p5.go() except Exception as e: # Exception was already logged just before logger.warning( "Failed beam {}, skipping that from continuum".format( beamnr)) logger.exception(e) status[beamnr] += ['continuum'] try: p6 = polarisation(file_=configfilename_list[beam_index]) p6.paramfilename = 'param_{:02d}.npy'.format(beamnr) p6.basedir = basedir p6.beam = "{:02d}".format(beamnr) p6.polcal = name_to_mir(name_polcal) p6.target = name_to_mir(name_target) if "polarisation" in steps and not dry_run: p6.go() except Exception as e: # Exception was already logged just before logger.warning( "Failed beam {}, skipping that from polarisation". format(beamnr)) logger.exception(e) status[beamnr] += ['polarisation'] # keep a record of the parallalised step in the main log file if 'scal' in steps or 'continuum' in steps or 'polarisation' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info( "Running selfcal and/or continuum and/or polarisation ... Done ({0:.0f}s)" .format(time() - start_time_selfcal_continuum_polarisation)) # ==== # Line # ==== # keep a record of the parallalised step in the main log file if 'line' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger.info("Running line") start_time_line = time() else: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger.info("Skipping line") for beamnr in beamlist_target: # Because of the amount of information coming from line # this module gets its own logfile logfilepath = os.path.join(basedir, 'apercal{:02d}_line.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) try: p7 = line(file_=configfilename_list[ beamlist_target_for_config.index(beamnr)]) if beamnr not in p7.line_beams: logger.debug( "Skipping line imaging for beam {}".format(beamnr)) continue p7.basedir = basedir p7.beam = "{:02d}".format(beamnr) p7.target = name_target + '.mir' if "line" in steps and not dry_run: p7.go() except Exception as e: # Exception was already logged just before logger.warning( "Failed beam {}, skipping that from line".format(beamnr)) logger.exception(e) status[beamnr] += ['line'] # with pymp.Parallel(5) as p: # for beam_index in p.range(n_beams): # beamnr = beamlist_target[beam_index] # logfilepath = os.path.join( # basedir, 'apercal{:02d}.log'.format(beamnr)) # lib.setup_logger('debug', logfile=logfilepath) # logger = logging.getLogger(__name__) # try: # p7 = line(file_=configfilename) # if beamnr not in p7.line_beams: # logger.debug( # "Skipping line imaging for beam {}".format(beamnr)) # continue # p7.basedir = basedir # p7.beam = "{:02d}".format(beamnr) # p7.target = name_target + '.mir' # if "line" in steps and not dry_run: # p7.go() # except Exception as e: # # Exception was already logged just before # logger.warning( # "Failed beam {}, skipping that from line".format(beamnr)) # logger.exception(e) # status[beamnr] += ['line'] # keep a record of the parallalised step in the main log file if 'line' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info( "Running line ... Done ({0:.0f}s)".format(time() - start_time_line)) # ======== # Transfer # ======== # keep a record of the parallalised step in the main log file if 'transfer' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger.info("Running transfer") start_time_transfer = time() else: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger.info("Skipping transfer") # 5 threads to not hammer the disks too much during copying with pymp.Parallel(5) as p: for beam_index in p.range(n_beams): beamnr = beamlist_target[beam_index] logfilepath = os.path.join(basedir, 'apercal{:02d}.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) try: p8 = transfer(file_=configfilename_list[beam_index]) p8.paramfilename = 'param_{:02d}.npy'.format(beamnr) p8.basedir = basedir p8.target = name_target + '.mir' p8.beam = "{:02d}".format(beamnr) if "transfer" in steps and not dry_run: # director( # p8, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) p8.go() except Exception as e: logger.warning( "Failed beam {}, skipping that from transfer".format( beamnr)) logger.exception(e) status[beamnr] += ['transfer'] # keep a record of the parallalised step in the main log file if 'transfer' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Running transfer ... Done ({0:.0f}s)".format( time() - start_time_transfer)) # Polarisation # ============ # keep a record of the parallalised step in the main log file # if 'polarisation' in steps: # logfilepath = os.path.join(basedir, 'apercal.log') # lib.setup_logger('debug', logfile=logfilepath) # logger = logging.getLogger(__name__) # logger.info("Running polarisation") # start_time_polarisation = time() # else: # logfilepath = os.path.join(basedir, 'apercal.log') # lib.setup_logger('debug', logfile=logfilepath) # logger = logging.getLogger(__name__) # logger.info("Skipping polarisation") # with pymp.Parallel(5) as p: # for beam_index in p.range(n_beams): # beamnr = beamlist_target[beam_index] # logfilepath = os.path.join( # basedir, 'apercal{:02d}.log'.format(beamnr)) # lib.setup_logger('debug', logfile=logfilepath) # logger = logging.getLogger(__name__) # try: # p7 = polarisation(file_=configfilename) # p7.paramfilename = 'param_{:02d}.npy'.format(beamnr) # p7.basedir = basedir # p7.beam = "{:02d}".format(beamnr) # p7.target = name_to_mir(name_target) # if "polarisation" in steps and not dry_run: # p7.go() # except Exception as e: # # Exception was already logged just before # logger.warning( # "Failed beam {}, skipping that from polarisation".format(beamnr)) # logger.exception(e) # status[beamnr] += ['polarisation'] # # keep a record of the parallalised step in the main log file # if 'polarisation' in steps: # logfilepath = os.path.join(basedir, 'apercal.log') # lib.setup_logger('debug', logfile=logfilepath) # logger = logging.getLogger(__name__) # logger.info("Running polarisation ... Done ({0:.0f}s)".format( # time() - start_time_polarisation)) # if "ccalqa" in steps and not dry_run: # logger.info("Starting crosscal QA plots") # try: # make_all_ccal_plots( # taskid_target, name_fluxcal.upper().strip().split('_')[0]) # except Exception as e: # logger.warning("Failed crosscal QA plots") # logger.exception(e) # logger.info("Done with crosscal QA plots") status = status.copy() # Convert pymp shared dict to a normal one msg = "Apercal finished after " + \ str(timedelta(seconds=time() - time_start)) logger.info(msg) return status, str(timedelta(seconds=time() - time_start)), None except Exception as e: msg = "Apercal threw an error after " + \ str(timedelta(seconds=time() - time_start)) logger.exception(msg) return status, str(timedelta(seconds=time() - time_start)), str(e)
def apercc(cal_list=None, task_id=None, cal_name=None, base_dir=None, search_all_nodes=False, steps=None): """ Main function to run the cross-calibration stability evaluation. For a list of calibrator scans or a given task id and calibrator name, this functions runs the cross-calibration evaluation. The function can get the data from ALTA and flags them using the Apercal modules prepare and preflag. It compares the bandpass solutions and gain factors between beams and between observations of the same calibrators The different steps can be selected individually. Example: scanid, source name, beam: [190108926, '3C147_36', 36] steps: ['prepare', 'preflag', 'crosscal', bpass_compare', 'gain_comare', 'bpass_compare_obs', 'gain_compare_obs'] function cal: apercc(cal_list=[[190108926, '3C147_36', 36], [190108927, '3C147_37', 37]) or apercc(task_id = 190409056, cal_name='3C196') Args: cal_list (List(List(int, str, int)): scan id, source name, beam, optional base_dir (str): Name of directory to store the data, if not specified it will be /data/apertif/crosscal/<scanid> when new data is fetched or /data/apertif/<scanid> when existing data is used task_id (int): ID of scan to be used as main ID and for the directory, if not specified it will be the first scan id cal_name (str): Name of the calibrator, if not specified the first name in the calibrator list will be used search_all_nodes (bool): steps (List(str)): List of steps in this task To Do: Use existing data using the task_id option and the name of the calibrator? """ # General setup # ============= # start time of this function start_time = time() # check input # if no list of a calibrators is given cal_list_mode = True if cal_list is None: # then it needs the task id and the calibrator name to look for existing data if task_id is not None and cal_name is not None: print( "Using task id and calibrator name. Assuming to use existing data. Will not run preflag, prepare and crosscal" ) # check that if steps were provided, they don't contain preflag, prepare and crosscal if steps is not None: if 'prepare' in steps: steps.remove('prepare') if 'preflag' in steps: steps.remove('preflag') if 'crosscal' in steps: steps.remove('crosscal') else: steps = [ 'bpass_compare', 'gain_comare', 'bpass_compare_obs', 'gain_compare_obs' ] # using existing data cal_list_mode = False # otherwise it won't do anything else: print( "Input parameters incorrect. Please specify either cal_list or task_id and cal_name. Abort" ) return -1 else: print("Using list of calibrators") if not steps: steps = [ 'prepare', 'preflag', 'crosscal', 'bpass_compare', 'gain_comare', 'bpass_compare_obs', 'gain_compare_obs' ] # # check that preflag is in it if prepare is run # else: # if 'prepare' in steps and not 'preflag' in steps: # steps.insert(1, 'preflag') # get the scan id to be used as the task id if not task_id: task_id = cal_list[0][0] else: task_id = task_id # create data directory unless specified using the first id unless otherwise specified # if no directory is specified if not base_dir: # and existing data is used, assume the base dir is an apercal processed data directory if task_id is not None and cal_name is not None: base_dir = '/data/apertif/{}/'.format(task_id) # if new data is fetched from the archive use a different default directory else: base_dir = '/data/apertif/crosscal/{}/'.format(task_id) elif len(base_dir) > 0 and base_dir[-1] != '/': base_dir = base_dir + '/' if not os.path.exists(base_dir) and cal_list_mode: try: os.mkdir(base_dir) except Exception as e: print("Creating the base directory failed. Abort") return -1 elif not os.path.exists(base_dir): print("Directory was not found. Abort") return -1 logfilepath = os.path.join(base_dir, 'apercc.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) # gitinfo = subprocess.check_output('cd ' + os.path.dirname(apercal.__file__) + # '&& git describe --tag; cd', shell=True).strip() # logger.info("Apercal version: " + gitinfo) logger.info("Apertif cross-calibration stability evaluation") if cal_list_mode: logger.info("Using list of calibrators as input !!!!") else: logger.info("Using task id and calibrator name as input !!!!") logger.debug("apercc called with arguments ...") logger.debug("cal_list={}".format(cal_list)) logger.debug("task_id = {}".format(task_id)) logger.debug("cal_name = {}".format(cal_name)) logger.debug("base_dir = {}".format(base_dir)) logger.debug("search_all_nodes = {}".format(search_all_nodes)) logger.debug("steps = {}".format(steps)) # number of calibrators if cal_list is not None: n_cals = len(cal_list) # get a list of beams beam_list = np.array([cal_list[k][2] for k in range(n_cals)]) else: n_cals = 1 # get the name of the flux calibrator if cal_name is None: name_cal = str(cal_list[0][1]).strip().split('_')[0] else: name_cal = cal_name # Getting the data using prepare # ============================== if "prepare" in steps: start_time_prepare = time() logger.info("Getting data for calibrators") # go through the list of calibrators and run prepare for (task_id_cal, name_cal, beamnr_cal) in cal_list: logger.info("Running prepare for {0} of beam {1}".format( name_cal, beamnr_cal)) # create prepare object without config file prep = prepare(filename=None) # where to store the data prep.basedir = base_dir # give the calibrator as a target to prepare prep.fluxcal = '' prep.polcal = '' prep.target = name_cal.upper().strip().split('_')[0] + '.MS' prep.prepare_target_beams = str(beamnr_cal) prep.prepare_date = str(task_id_cal)[:6] prep.prepare_obsnum_target = str(task_id_cal)[-3:] try: prep.go() except Exception as e: logger.warning( "Prepare failed for calibrator {0} ({1}) beam {2}".format( str(task_id), name_cal, beamnr_cal)) logger.exception(e) else: logger.info("Prepare successful for {0} of beam {1}".format( name_cal, beamnr_cal)) logger.info("Getting data for calibrators ... Done ({0:.0f}s)".format( time() - start_time_prepare)) else: logger.info("Skipping getting data for calibrators") # Running preflag for calibrators # =============================== if 'preflag' in steps: start_time_flag = time() logger.info("Flagging data of calibrators") # Flag fluxcal (pretending it's a target) # needs to be changed for parallel preflag and make it a loop flag = preflag(filename=None) flag.basedir = base_dir flag.fluxcal = '' flag.polcal = '' flag.target = name_cal.upper().strip().split('_')[0] + '.MS' flag.beam = "{:02d}".format(beam_list[0]) flag.preflag_targetbeams = "{:02d}".format(beamnr) try: director(flag, 'rm', base_dir + '/param.npy', ignore_nonexistent=True) flag.go() except Exception as e: logger.warning("Preflag failed") logger.exception(e) else: logger.info( "Flagging data of calibrators ... Done ({0:.0f}s)".format( time() - start_time_flag)) else: logger.info("Skipping running preflag for calibrators") # Running crosscal for calibrators # =============================== if 'crosscal' in steps: start_time_crosscal = time() logger.info("Running crosscal for calibrators") for beam_nr in beam_list: logger.info("Running crosscal for beam {0}".format(beam_nr)) crosscal = ccal(file_=None) crosscal.basedir = base_dir crosscal.fluxcal = name_cal.upper().strip().split('_')[0] + '.MS' # p.polcal = name_to_ms(name_polcal) # p.target = name_to_ms(name_target) # p2.paramfilename = 'param_{:02d}.npy'.format(beamnr) crosscal.beam = "{:02d}".format(beam_nr) crosscal.crosscal_transfer_to_target = False # p2.crosscal_transfer_to_target_targetbeams = "{:02d}".format( # beamnr) try: director(crosscal, 'rm', base_dir + '/param.npy', ignore_nonexistent=True) # director( # p2, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) crosscal.go() except Exception as e: # Exception was already logged just before logger.warning("Failed crosscal for beam {}".format(beam_nr)) logger.exception(e) else: logger.info( "Running crosscal for beam {0} ... Done".format(beam_nr)) logger.info( "Running crosscal for calibrators ... Done ({0:.0f}s)".format( time() - start_time_crosscal)) else: logger.info("Skipping running crosscal for calibrators") # Running Bandbpass comparison # ============================ if 'bpass_compare' in steps: start_time_prepare = time() logger.info("Comparing bandpass") logger.info("#### Doing nothing here yet ####") logger.info( "Comparing bandpass ... Done ({0:.0f})".format(time() - start_time_prepare)) else: logger.info("Skipping comparing bandpass") # Running Bandbpass comparison # ============================ if 'gain_compare' in steps: start_time_gain = time() logger.info("Comparing gain solutions") logger.info("#### Doing nothing here yet ####") logger.info("Comparing gain solutions ... Done ({0:.0f})".format( time() - start_time_gain)) else: logger.info("Skipping comparing gain solutions") # Running Bandbpass comparison between observations # ================================================= if 'bpass_compare_obs' in steps: start_time_bandpass = time() logger.info("Comparing banpdass solutions across observations") logger.info("#### Doing nothing here yet ####") logger.info( "Comparing banpdass solutions across observations ... Done ({0:.0f})" .format(time() - start_time_bandpass)) else: logger.info( "Skipping comparing banpdass solutions across observations") # Running Bandbpass comparison between observations # ================================================= if 'bpass_compare_obs' in steps: start_time_gain = time() logger.info("Comparing banpdass solutions across observations") logger.info("#### Doing nothing here yet ####") logger.info( "Comparing banpdass solutions across observations ... Done ({0:.0f})" .format(time() - start_time_gain)) else: logger.info( "Skipping comparing banpdass solutions across observations") logger.info( "Apertif cross-calibration stability evaluation ... Done ({0:.0f}s)". format(time() - start_time))