def calibration_plots(primary: str, secondary: str, freq: str): """Common function to create calibration plots of the primary and secondary visibility files Arguments: primary {str} -- Filename of primary calibrator secondary {str} -- Filename of the secondary calibrator freq {str} -- Frequency of the IF """ plt = [ m(f'uvplt vis={primary} axis=time,amp options=nob,nof stokes=i device=primary_timeamp_{freq}.png/PNG' ), m(f'uvplt vis={primary} axis=re,im options=nob,nof,eq stokes=i,q,u,v device=primary_reim_{freq}.png/PNG' ), m(f'uvplt vis={primary} axis=uc,vc options=nob,nof stokes=i device=primary_ucvc_{freq}.png/PNG' ), m(f'uvplt vis={primary} axis=FREQ,amp options=nob,nof stokes=i device=primary_freqamp_{freq}.png/PNG' ), m(f'uvplt vis={secondary} axis=time,amp options=nob,nof stokes=i device=secondary_timeamp_{freq}.png/PNG' ), m(f'uvplt vis={secondary} axis=re,im options=nob,nof,eq stokes=i,q,u,v device=secondary_reim_{freq}.png/PNG' ), m(f'uvplt vis={secondary} axis=uc,vc options=nob,nof stokes=i device=secondary_ucvc_{freq}.png/PNG' ), m(f'uvplt vis={secondary} axis=FREQ,amp options=nob,nof stokes=i device=secondary_freqamp_{freq}.png/PNG' ), m(f'uvfmeas vis={secondary} stokes=i log=secondary_uvfmeas_{freq}_log.txt device=secondary_uvfmeas_{freq}.png/PNG' ) ] pool = Pool(7) result = pool.map(run, plt) pool.close() pool.join()
def mosaic_pgflag(src): """The name of the C3171 mosaic file to flag Arguments: src {str} -- Name of the file to flag """ pgflag = m(f"pgflag vis={src} command='<b' stokes=i,v,q,u flagpar=8,2,2,3,6,3 "\ f"options=nodisp").run() logger.log(logging.INFO, pgflag) pgflag = m(f"pgflag vis={src} command='<b' stokes=i,v,u,q flagpar=8,2,2,3,6,3 "\ f"options=nodisp").run() logger.log(logging.INFO, pgflag)
def pgflag_flag(s): """Flag uvdata based on the smonrms statistic s {str} -- Path to the filename to flag """ # Automated flagging pgflag = m(f"pgflag vis={s} command='<b' stokes=i,q,u,v flagpar=8,5,0,3,6,3 options=nodisp").run() print(pgflag) pgflag = m(f"pgflag vis={s} command='<b' stokes=i,v,q,u flagpar=8,2,0,3,6,3 options=nodisp").run() print(pgflag) pgflag = m(f"pgflag vis={s} command='<b' stokes=i,v,u,q flagpar=8,2,0,3,6,3 options=nodisp").run() print(pgflag)
def resolution(self): """Fit to the restor image to return the image resolutions """ if 'restor' not in self.img_tasks.keys(): return None invert = self.img_tasks['invert'] imfit = m(f"imfit in={invert.beam} object=beam region='perc(1)(1)'").run() print(imfit) src_data = {} obj1 = False for line in imfit.p.stdout.split('\n'): if 'Object type: beam' in line: obj1 = True continue elif not obj1: continue if 'Major axis' in line: items = line.split() src_data['bmaj'] = float(items[-3]) src_data['bmaj_e'] = float(items[-1]) elif 'Minor axis' in line: items = line.split() src_data['bmin'] = float(items[-3]) src_data['bmin_e'] = float(items[-1]) elif 'Position angle' in line: items = line.split() src_data['pa'] = float(items[-3]) src_data['pa_e'] = float(items[-1]) return src_data
def mosaic_src_calibration(src: str): """Apply any common calibration steps for each source file Arguments: src {str} -- uv source file of item to process """ gpaver = m(f"gpaver vis={src} interval=5 options=scalar").run() logger.log(logging.INFO, gpaver)
def calibrator_pgflag(src): """A series of pgflag steps common to most (if not all) of the primary and secondary miriad uv files. This is following the flagging outlined in Do_Flag.csh` on the ATCAGAMA wiki Arguments: src {str} -- The filename of the data to flag """ primary = True if '1934-638' in src else False if primary: pgflag = m( f"pgflag vis={src} stokes=v flagpar=15,3,3,3,5,3 command='<' options=nodisp" ).run() logger.log(logging.INFO, pgflag) pgflag = m( f"pgflag vis={src} stokes=q flagpar=15,3,3,3,5,3 command='<' options=nodisp" ).run() logger.log(logging.INFO, pgflag) pgflag = m( f"pgflag vis={src} stokes=u flagpar=15,3,3,3,5,3 command='<' options=nodisp" ).run() logger.log(logging.INFO, pgflag) pgflag = m( f"pgflag vis={src} stokes=i flagpar=15,3,3,3,5,3 command='<' options=nodisp" ).run() logger.log(logging.INFO, pgflag) else: pgflag = m( f"pgflag vis={src} stokes=i,q,u,v flagpar=10,1,1,3,5,3 command='<' options=nodisp" ).run() logger.log(logging.INFO, pgflag) pgflag = m( f"pgflag vis={src} stokes=i,u flagpar=10,1,1,3,5,3 command='<' options=nodisp" ).run() logger.log(logging.INFO, pgflag) pgflag = m( f"pgflag vis={src} stokes=i,q flagpar=10,1,1,3,5,3 command='<' options=nodisp" ).run() logger.log(logging.INFO, pgflag) pgflag = m( f"pgflag vis={src} stokes=i flagpar=10,1,1,3,5,3 command='<' options=nodisp" ).run() logger.log(logging.INFO, pgflag)
def calibrator_pgflag(src): """A series of pgflag steps common to most (if not all) of the primary and secondary miriad uv files Arguments: src {str} -- The filename of the data to flag """ # Automated flagging pgflag = m( f"pgflag vis={src} command='<b' stokes=i,q,u,v flagpar=8,5,5,3,6,3 options=nodisp" ).run() logger.log(logging.INFO, pgflag) pgflag = m( f"pgflag vis={src} command='<b' stokes=i,v,q,u flagpar=8,2,2,3,6,3 options=nodisp" ).run() logger.log(logging.INFO, pgflag) pgflag = m( f"pgflag vis={src} command='<b' stokes=i,v,u,q flagpar=8,2,2,3,6,3 options=nodisp" ).run() logger.log(logging.INFO, pgflag)
def selfcal(self, *args, round: int=0, self_kwargs: dict=None, preprocess=None, postprocess=None, **kwargs): """Apply a round of selfcalibration to the uv-file if appropriate. Keyword Arguments: round {int} -- The selfcalibration round the process is up to (default: {0}) self_kwargs {dict} -- Options for selfcalibration. Can be overwritten by attempt_selfcal returns (default: {None}) preprocess {callable} -- callable to accept an instance of uv-file before selfcalibration (default: {None}) postprocess {callable} -- callable to accept an instance of uv-file after selfcalibration (default: {None}) """ run_self = self.attempt_selfcal(**kwargs) if isinstance(run_self, tuple): run_self, self_kwargs = run_self # Conditions are not met. Return instance of self. if not run_self: return self # Apply calibration tables in existing uv-file uvaver = m(f"uvaver vis={self.uv} out={self.uv}.{round}").run() print(uvaver) # Action any described todos if preprocess is not None: preprocess(uvaver.out) # Derive selfcalibrated solutions selfcal = m(f"selfcal vis={uvaver.out} model={self.img_tasks['mfclean'].out} "\ f"options=mfs,phase interval=0.1 nfbin=4", over=self_kwargs).run() print(selfcal) # Action any described todos if postprocess is not None: postprocess(uvaver.out) # Return new instance to selfcaled file return uv(uvaver.out)
def flag_inttime(uv: str, threshold: int = 9, logfile: str = 'dump.txt', selection: str = 'flagging.txt', delete: bool = False): """Flag out short integration cycles from a uv-file Arguments: uv {str} -- uv-file to flag Keyword Arguments: threshold {int} -- Minimum length of time for each cycle (default: {9}) logfile {str} -- File to `uvdump` to (default: {'dump.txt'}) selection {str} -- File to place select statements in (default: {'flagging.txt'}) delete {bool} -- Clean up log files (default: {False}) """ uvdump = m( f"uvdump vis={uv} vars=inttime,ant1,ant2,time,source log={logfile}" ).run() print(uvdump) var = pd.read_csv(logfile, names=('inttime', 'ant1', 'ant2', 'time', 'source')) var = var[(var['ant1'].values == 1) & (var['ant2'].values == 2)] # Smaller list of selects var[var['inttime'] < threshold].apply(flag_cycle, axis=1, selection=selection) uvflag = m(f"uvflag vis={uv} select=@{selection} flagval=flag").run() print(uvflag) if delete: os.remove(logfile) os.remove(selection)
def mosaic_uvsplit(mosaic: str): """uvsplit a mosaic source file up based on source names. Will return a list of new source files to operate upon. Arguments: mosaic {str} -- Name of the mosaic file to split """ uvsplit = m(f"uvsplit vis={mosaic}").run() logger.log(logging.INFO, uvsplit) srcs = [] for line in str(uvsplit).splitlines(): if 'Creating' not in line: continue srcs.append(line.split()[1]) return srcs
def sc_round_4(s): """Logic for the four round of selfcalibration Arguments: s {uv} -- An instance of the uv-class. """ from astropy.io import fits as pyfits restor = s.img_tasks['restor'] fits = m(f"fits in={restor.out} out={restor.out}.4.fits op=xyout " \ "region='images(1,1)'").run() data = pyfits.open(fits.out)[0].data.squeeze() delete_miriad(fits.out) # Threshold selected by experimentation (does it look bad?) restor_max = data.max() if restor_max > 400*s.img_tasks['stokes_v_rms']: return True, {'options':'mfs,amp', 'interval':'1'} else: return False
def sc_round_2(s): """Logic for the second round of selfcalibration Arguments: s {uv} -- An instance of the uv-class. """ from astropy.io import fits as pyfits restor = s.img_tasks['restor'] fits = m(f"fits in={restor.out} out={restor.out}.2.fits op=xyout " \ "region='images(1,1)'").run() data = pyfits.open(fits.out)[0].data.squeeze() delete_miriad(fits.out) # Threshold selected by dumb luck and subsequet experimentation restor_max = data.max() if restor_max > 50*s.img_tasks['stokes_v_rms']: return True, {'interval':'1'} else: return False
def mosaic_src_pgflag(src): """Thw flagging procedure applied to the source data. THis follows Minh's script called Do_Flag.csh on the ATCAGAMA wiki. For ease it is applied before uvsplit. Time convolution is turned off for the moment. Arguments: src {[type]} -- [description] """ pgflag = m(f"pgflag vis={src} command='<' stokes=i,q,u,v flagpar=10,1,0,3,5,3 "\ f"options=nodisp").run() logger.log(logging.INFO, pgflag) pgflag = m(f"pgflag vis={src} command='<' stokes=i,q,u,v flagpar=10,1,0,3,5,3 "\ f"options=nodisp").run() logger.log(logging.INFO, pgflag) pgflag = m(f"pgflag vis={src} command='<' stokes=i,q flagpar=10,1,0,3,5,3 "\ f"options=nodisp").run() logger.log(logging.INFO, pgflag) pgflag = m(f"pgflag vis={src} command='<' stokes=i,q flagpar=10,1,0,3,5,3 "\ f"options=nodisp").run() logger.log(logging.INFO, pgflag) pgflag = m(f"pgflag vis={src} command='<' stokes=i,u flagpar=10,1,0,3,5,3 "\ f"options=nodisp").run() logger.log(logging.INFO, pgflag) pgflag = m(f"pgflag vis={src} command='<' stokes=i,u flagpar=10,1,0,3,5,3 "\ f"options=nodisp").run() logger.log(logging.INFO, pgflag) pgflag = m(f"pgflag vis={src} command='<' stokes=i flagpar=10,1,0,3,5,3 "\ f"options=nodisp").run() logger.log(logging.INFO, pgflag) pgflag = m(f"pgflag vis={src} command='<' stokes=i flagpar=10,1,0,3,5,3 "\ f"options=nodisp").run() logger.log(logging.INFO, pgflag)
def run_linmos(mos: list, round: int=0, mode: str='restor'): """The the miriad task linmos agaisnt list of uv files Arguments: mos {list} -- A list of uv-instances round {int} -- A int for self calibration round mode {str} -- The mode to linmos together """ convol_files = f'temp_file_{round}.{mode}.dat' with open(f"{convol_files}", 'w') as f: for i in mos: print(f"{i.img_tasks[f'convol_{mode}'].out}", file=f) out = f"all_days.{round}.{mode}.linmos" delete_miriad(out) linmos = m(f"linmos in=@{convol_files} bw=4.096 out={out}").run() print(linmos) os.remove(convol_files) return linmos
def convol(self, bmaj: float, bmin: float, pa: float, mode: str='restor'): """Convol a restored image to a different resolution Arguments: bmaj {float} -- Beam major in arcseconds bmin {float} -- Beam minor in arcseconds pa {float} -- Beam position angle in degrees mode {str} -- The mode to convol (default: {restor}) """ if mode not in self.img_tasks.keys() or \ f'convol_{mode}' in self.img_tasks.keys(): return None restor = self.img_tasks[mode] print(restor.out) convol = m(f"convol map={restor.out} out={self.uv}.{mode}.convol fwhm={bmaj},{bmin} "\ f"pa={pa} options=final").run() print(convol) self.img_tasks[f'convol_{mode}'] = convol
def image(self, invert_kwargs: dict=None, cut_thres_1: {float}=5, cut_thres_2: {float}=None): """A simple MFCLEAN imaging pipeline Keyword Arguments: invert_kwargs {dict} -- Options to pass to the invert map/beam stage (default: {None}) cut_thres_1 {float} -- First cutoff option provided to mfclean, depth to clean to (default: {5}) invert_kwargs {dict} -- Second option provided to mfclean if not None, depth to clean existing components down to (default: {None}) """ # No work to do, as data has been imaged if 'restor' in self.img_tasks.keys(): return invert = m(f"invert vis={self.uv} options=mfs,sdb,double,mosaic,systemp " \ f"offset=3:32:22.0,-27:48:37 stokes=i imsize=5,5,beam " \ f"map={self.uv}.map beam={self.uv}.beam robust=1 cell=0.25", over=invert_kwargs).run() print(invert) stokes_v = m(f"invert vis={self.uv} imsize=3,3,beam options=mfs,sdb,double,mosaic,systemp " \ f"offset=3:32:22.0,-27:48:37 stokes=v " \ f"map={self.uv}.v.map cell=0.35").run() print(stokes_v) sigest = m(f"sigest in={stokes_v.map}").run() print(sigest) for l in sigest.p.stdout.split('\n'): if 'Estimated rms' in l: v_rms = float(l.split()[-1]) cutoff = f"{cut_thres_1*v_rms},{cut_thres_2*v_rms}" if cut_thres_2 is not None else f"{cut_thres_1*v_rms}" mfclean = m(f"mfclean map={invert.map} beam={invert.beam} out={self.uv}.clean "\ f"region='perc(66)' niters=1500 cutoff={cutoff}").run() print(mfclean) restor = m(f"restor map={invert.map} beam={invert.beam} model={mfclean.out} "\ f"options=mfs out={self.uv}.restor").run() print(restor) residual = m(f"restor map={invert.map} beam={invert.beam} model={mfclean.out} "\ f"mode=residual out={self.uv}.residual").run() print(residual) self.img_tasks = {'invert':invert, 'invert_v':stokes_v, 'mfclean':mfclean, 'restor':restor, 'residual':residual, 'stokes_v_rms':v_rms}
def run(a): a.run() logger.log(logging.INFO, a) # Initial pass of the data loads in without the Tsys correction to produce a # robust spectrum of the secondary calibrator for index, freq in enumerate(FREQS): ifsel = index + 1 primary = f"{mu.primary}.{freq}" secondary = f"{mu.secondary}.{freq}" mosaic = f"{mu.science}.{freq}" # Load in the data atlod = m(f"atlod in=raw/*C3171 options=notsys,rfiflag,birdie,xycorr,noauto "\ f"ifsel={ifsel} out=data{ifsel}.uv").run() logger.log(logging.INFO, atlod) # Flag the known bad channels out if ifsel == 1: mu.uvflag(atlod.out, mu.flags_7) else: mu.uvflag(atlod.out, mu.flags_9) # Split the data up uvsplit = m(f"uvsplit vis={atlod.out} options=mosaic "\ f"select=source({mu.primary}),source({mu.secondary})").run() logger.log(logging.INFO, uvsplit) mfcal = m(f"mfcal vis={primary} interval=0.1,0.1,5").run() gpcal = m(f"gpcal vis={primary} interval=0.1 nfbin={NFBIN} "\
NFBIN = 4 FREQ = 9500 # Load in files assuming the setup file/s have been renamed or deleted # Can lead to problems with 0 and 9s. files = glob('raw/*C3132') # Example loading in files assuming first is setup # files = glob('raw/*C3132').pop(0) # Glob order is not the same as sort order files = sorted(files) # Load in data. Remember to set ifsel appropriately atlod = m( f"atlod in={','.join(files)} out=data9.uv ifsel=2 options=birdie,rfiflag,noauto,xycorr" ).run() logger.log(logging.INFO, atlod) # Flag out known bad channels mu.uvflag(atlod.out, mu.flags_9) uvsplit = m(f"uvsplit vis={atlod.out} options=mosaic").run() logger.log(logging.INFO, uvsplit) # Deduce the objects in the observing run srcs = mu.derive_obs_sources(uvsplit, FREQ) primary, secondary, mosaic_targets = srcs mu.calibrator_pgflag(primary)
def seeing_flag(s): """Flag uvdata based on the smonrms statistic """ uvflag = m(f"uvflag vis={s} select=-seeing(500) flagval=flag").run() print(uvflag)
def attempt_selfcal(self, mode: str=None): """Logic to decide whether selcalibration should be attempted. Mode can be a function that is passed into the method. It should except as a single argument a reference to uv-instance. Keyword Arguments: mode {str} -- The mode used to evaluate decision to self (default: {None}) Returns: bool -- Whether selfcal should be used dict -- optional return with selfcal key/values to use """ # Nothing set if mode is None: return False # If it quacks like a duck try: return mode(self) except Exception as e: print(e) pass if mode == 'test': # Example of conditional. Used to test subequent imaging if '100' in self.uv: return False elif mode == 'restor_max': from astropy.io import fits as pyfits restor = self.img_tasks['restor'] fits = m(f"fits in={restor.out} out={restor.out}.fits op=xyout " \ "region='images(1,1)'").run() data = pyfits.open(fits.out)[0].data.squeeze() delete_miriad(fits.out) # Threshold selected by dumb luck and subsequet experimentation restor_max = data.max() if restor_max > 150*self.img_tasks['stokes_v_rms']: return True, {'options':'mfs,amp'} elif restor_max > 50*self.img_tasks['stokes_v_rms']: return True else: return False elif mode == 'clean_sum': from astropy.io import fits as pyfits clean = self.img_tasks['mfclean'] fits = m(f"fits in={clean.out} out={clean.out}.fits op=xyout " \ "region='images(1,1)'").run() data = pyfits.open(fits.out)[0].data.squeeze() delete_miriad(fits.out) clean_sum = data.sum() if clean_sum > 500*self.img_tasks['stokes_v_rms']: return True, {'options':'mfs,amp'} elif clean_sum > 100*self.img_tasks['stokes_v_rms']: return True else: return False return True