def main(): # Get the name of the config file args = config_parser.parse_args() # Parse config file taskvals, config = config_parser.parse_config(args['config']) visname = va(taskvals, 'data', 'vis', str) fields = bookkeeping.get_field_ids(taskvals['fields']) calcrefant = va(taskvals, 'crosscal', 'calcrefant', bool, default=False) # Calculate reference antenna if calcrefant: if len(fields.fluxfield.split(',')) > 1: field = fields.fluxfield.split(',')[0] else: field = fields.fluxfield refant, badants = get_ref_ant(visname, field) # Overwrite config file with new refant config_parser.overwrite_config(args['config'], conf_sec='crosscal', conf_dict={'refant': refant}) config_parser.overwrite_config(args['config'], conf_sec='crosscal', conf_dict={'badants': badants})
def validateinput(): """ Parse the input config file (command line argument) and validate that the parameters look okay """ logger.info('This is version {0} of the pipeline'.format(processMeerKAT.__version__)) # Get the name of the config file args = config_parser.parse_args() # Parse config file taskvals, config = config_parser.parse_config(args['config']) visname = va(taskvals, 'data', 'vis', str) calcrefant = va(taskvals, 'crosscal', 'calcrefant', bool) refant = va(taskvals, 'crosscal', 'refant', str) fields = bookkeeping.get_field_ids(taskvals['fields']) # Check if the reference antenna exists, and complain and quit if it doesn't if not calcrefant: refant = va(taskvals, 'crosscal', 'refant', str) msmd.open(visname) get_fields.check_refant(MS=visname, refant=refant, warn=False) msmd.close() msmd.done() if not os.path.exists(visname): raise IOError("Path to MS %s not found" % (visname))
def get_imaging_params(): # Get the name of the config file args = config_parser.parse_args() # Parse config file taskvals, config = config_parser.parse_config(args['config']) params = taskvals['image'] params['vis'] = taskvals['data']['vis'] return args, params
def run_script(func, logfile=''): # Get the name of the config file args = config_parser.parse_args() # Parse config file taskvals, config = config_parser.parse_config(args['config']) continue_run = config_parser.validate_args(taskvals, 'run', 'continue', bool, default=True) spw = config_parser.validate_args(taskvals, 'crosscal', 'spw', str) nspw = config_parser.validate_args(taskvals, 'crosscal', 'nspw', int) if continue_run: try: func(args, taskvals) #rename_logs(logfile) except Exception as err: logger.error( 'Exception found in the pipeline of type {0}: {1}'.format( type(err), err)) logger.error(traceback.format_exc()) config_parser.overwrite_config( args['config'], conf_dict={'continue': False}, conf_sec='run', sec_comment='# Internal variables for pipeline execution') if nspw > 1: for SPW in spw.split(','): spw_config = '{0}/{1}'.format(SPW.replace('0:', ''), args['config']) config_parser.overwrite_config( spw_config, conf_dict={'continue': False}, conf_sec='run', sec_comment= '# Internal variables for pipeline execution') rename_logs(logfile) sys.exit(1) else: logger.error( 'Exception found in previous pipeline job, which set "continue=False" in [run] section of "{0}". Skipping "{1}".' .format(args['config'], os.path.split(sys.argv[2])[1])) #os.system('./killJobs.sh') # and cancelling remaining jobs (scancel not found since /opt overwritten) rename_logs(logfile) sys.exit(1)
def main(): config = config_parser.parse_args()['config'] # Parse config file taskvals, config = config_parser.parse_config(config) visname = va(taskvals, 'data', 'vis', str) keepmms = va(taskvals, 'crosscal', 'keepmms', bool) calfiles, caldir = bookkeeping.bookkeeping(visname) fields = bookkeeping.get_field_ids(taskvals['fields']) msmd.open(visname) if not os.path.exists(PLOT_DIR): os.mkdir(PLOT_DIR) #Plot solutions for bandpass calibrator plotms(vis=calfiles.bpassfile, xaxis='Real', yaxis='Imag', coloraxis='corr', plotfile='{0}/bpass_real_imag.png'.format(PLOT_DIR),showgui=False) plotms(vis=calfiles.bpassfile, xaxis='freq', yaxis='Amp', coloraxis='antenna1', plotfile='{0}/bpass_freq_amp.png'.format(PLOT_DIR),showgui=False) plotms(vis=calfiles.bpassfile, xaxis='freq', yaxis='Phase', coloraxis='antenna1', plotfile='{0}/bpass_freq_phase.png'.format(PLOT_DIR),showgui=False) #Plot solutions for phase calibrator plotms(vis=calfiles.gainfile, xaxis='Real', yaxis='Imag', coloraxis='corr', plotfile='{0}/phasecal_real_imag.png'.format(PLOT_DIR),showgui=False) plotms(vis=calfiles.gainfile, xaxis='Time', yaxis='Amp', coloraxis='antenna1', plotfile='{0}/phasecal_time_amp.png'.format(PLOT_DIR),showgui=False) plotms(vis=calfiles.gainfile, xaxis='Time', yaxis='Phase', coloraxis='antenna1', plotfile='{0}/phasecal_time_phase.png'.format(PLOT_DIR),showgui=False) #Plot solutions for individual antennas of bandpass and phase calibrator in 3x2 panels plot_antennas('bpass',fields,calfiles,xaxis='freq',yaxis='amp') plot_antennas('bpass',fields,calfiles,xaxis='freq',yaxis='phase') plot_antennas('phasecal',fields,calfiles,xaxis='time',yaxis='amp') plot_antennas('phasecal',fields,calfiles,xaxis='time',yaxis='phase') extn = 'mms' if keepmms else 'ms' for field in fields: for subf in field.split(','): fname = msmd.namesforfields(int(subf))[0] inname = '%s.%s.%s' % (os.path.splitext(visname)[0], fname, extn) if not os.path.exists('{0}/{1}_freq_amp.png'.format(PLOT_DIR,fname)): plotms(vis=inname, xaxis='freq', yaxis='Amp', coloraxis='corr', plotfile='{0}/{1}_freq_amp.png'.format(PLOT_DIR,fname),showgui=False) plotms(vis=inname, xaxis='Real', yaxis='Imag', coloraxis='corr', plotfile='{0}/{1}_real_imag.png'.format(PLOT_DIR,fname),showgui=False) msmd.close() msmd.done()
def main(): config = config_parser.parse_args()['config'] # Parse config file taskvals, config = config_parser.parse_config(config) visname = va(taskvals, 'data', 'vis', str) calfiles, caldir = bookkeeping.bookkeeping(visname) fields = bookkeeping.get_field_ids(taskvals['fields']) #Plot solutions for bandpass calibrator fastplot(calfiles.bpassfile, col='CPARAM', xaxis='Real', yaxis='Imag', fname='bpass_real_imag.png') fastplot(calfiles.bpassfile, col='CPARAM', xaxis='chan', yaxis='Amp', logy=True, fname='bpass_chan_amp.png') fastplot(calfiles.bpassfile, col='CPARAM', xaxis='chan', yaxis='Phase', fname='bpass_chan_phase.png') #Plot solutions for phase calibrator fastplot(calfiles.gainfile, col='CPARAM', xaxis='Amp', yaxis='Time', fname='phasecal_time_amp.png', markersize=2, extent=1e-8) fastplot(calfiles.gainfile, col='CPARAM', xaxis='Phase', yaxis='Time', fname='phasecal_time_phase.png', markersize=2, extent=1e-8)
def main(): # Get the name of the config file args = config_parser.parse_args() # Parse config file taskvals, config = config_parser.parse_config(args['config']) visname = va(taskvals, 'data', 'vis', str) calcrefant = va(taskvals, 'crosscal', 'calcrefant', bool, default=False) refant = va(taskvals, 'crosscal', 'refant', str, default='m005') spw = va(taskvals, 'crosscal', 'spw', str, default='') mvis = do_partition(visname, spw) mvis = "'{0}'".format(mvis) vis = "'{0}'".format(visname) config_parser.overwrite_config(args['config'], conf_sec='data', conf_dict={'vis': mvis}) config_parser.overwrite_config(args['config'], conf_sec='data', conf_dict={'orig_vis': vis})
overwrite=True, writeflags=True) # now flag using 'rflag' option flagdata(vis=visname, mode="rflag", datacolumn="corrected", field=fields.targetfield, timecutoff=5.0, freqcutoff=5.0, timefit="poly", freqfit="poly", flagdimension="freqtime", extendflags=False, timedevscale=5.0, freqdevscale=5.0, spectralmax=500.0, extendpols=False, growaround=False, flagneartime=False, flagnearfreq=False, action="apply", flagbackup=True, overwrite=True, writeflags=True) # Now summary flagdata(vis=visname, mode="summary", datacolumn="corrected", extendflags=True, name=visname + 'summary.split', action="apply", flagbackup=True, overwrite=True, writeflags=True) if __name__ == '__main__': # Get the name of the config file args = config_parser.parse_args() # Parse config file taskvals, config = config_parser.parse_config(args['config']) visname = va(taskvals, 'data', 'vis', str) calfiles, caldir = bookkeeping.bookkeeping(visname) fields = bookkeeping.get_field_ids(taskvals['fields']) do_pre_flag_2(visname, fields)
def get_selfcal_params(): #Flag for input errors exit = False # Get the name of the config file args = config_parser.parse_args() # Parse config file taskvals, config = config_parser.parse_config(args['config']) params = taskvals['selfcal'] check_params = params.keys() check_params.pop(check_params.index('nloops')) check_params.pop(check_params.index('restart_no')) params['vis'] = taskvals['data']['vis'] params['refant'] = taskvals['crosscal']['refant'] if 'loop' not in params: params['loop'] = 0 else: check_params.pop(check_params.index('loop')) for arg in check_params: # Multiscale needs to be a list of lists (if specifying multiple scales) # or a simple list (if specifying a single scale). So make sure these two # cases are covered. Likewise for imsize. if arg in ['multiscale', 'imsize']: # Not a list of lists, so turn it into one of right length if type(params[arg]) is list and (len(params[arg]) == 0 or type( params[arg][0]) is not list): params[arg] = [ params[arg], ] * (params['nloops'] + 1) # Not a list at all, so put it into a list elif type(params[arg]) is not list: params[arg] = [ [ params[arg], ], ] * (params['nloops'] + 1) # A list of lists of length 1, so put into list of lists of right length elif type(params[arg]) is list and type( params[arg][0]) is list and len(params[arg]) == 1: params[arg] = [ params[arg][0], ] * (params['nloops'] + 1) if len(params[arg]) != params['nloops'] + 1: logger.error( "Parameter '{0}' in '{1}' is the wrong length. It is {2} but must be a single value or equal to 'nloops' + 1 ({3})." .format(arg, args['config'], len(params[arg]), params['nloops'] + 1)) exit = True else: if type(params[arg]) is not list: if arg in ['solint', 'calmode']: params[arg] = [params[arg]] * (params['nloops']) else: params[arg] = [params[arg]] * (params['nloops'] + 1) if arg in ['solint', 'calmode' ] and len(params[arg]) != params['nloops']: logger.error( "Parameter '{0}' in '{1}' is the wrong length. It is {2} long but must be 'nloops' ({3}) long or a single value (not a list)." .format(arg, args['config'], len(params[arg]), params['nloops'])) exit = True elif arg not in ['solint', 'calmode' ] and len(params[arg]) != params['nloops'] + 1: logger.error( "Parameter '{0}' in '{1}' is the wrong length. It is {2} long but must 'nloops' + 1 ({3}) long or a single value (not a list)." .format(arg, args['config'], len(params[arg]), params['nloops'] + 1)) exit = True if exit: sys.exit(1) return args, params
""" Main module that controls the complete graph QA pipeline """ import math from config_parser import parse_args from trainer import Trainer from evaluator import Evaluator from data.data_loader import getMetaData, getDataLoader if __name__ == "__main__": args = parse_args() args.gen_config() # Calculate Meta Parameters from dataset train_max_vertices, train_feature_size, train_data_size, train_max_total_vertices = getMetaData( args, args.train_dir) val_max_vertices, val_feature_size, val_data_size, val_max_total_vertices = getMetaData( args, args.val_dir) test_max_vertices, test_feature_size, test_data_size, test_max_total_vertices = getMetaData( args, args.test_dir) args.max_vertices = max(train_max_vertices, val_max_vertices, test_max_vertices) args.feature_size = max(train_feature_size, val_feature_size, test_feature_size) args.data_size = max(train_data_size, val_data_size, test_data_size) args.max_total_vertices = max(train_max_total_vertices, val_max_total_vertices, test_max_total_vertices) args.train_data_size = train_data_size
def get_selfcal_params(): #Flag for input errors exit = False # Get the name of the config file args = config_parser.parse_args() # Parse config file taskvals, config = config_parser.parse_config(args['config']) params = taskvals['selfcal'] other_params = list(params.keys()) params['vis'] = taskvals['data']['vis'] params['refant'] = taskvals['crosscal']['refant'] params['dopol'] = taskvals['run']['dopol'] if params['dopol'] and 'G' in params['gaintype']: logger.warning( "dopol is True, but gaintype includes 'G'. Use gaintype='T' for polarisation on linear feeds (e.g. MeerKAT)." ) single_args = ['nloops', 'loop', 'discard_nloops', 'outlier_threshold'] #need to be 1 long (i.e. not a list) gaincal_args = ['solint', 'calmode', 'gaintype', 'flag'] #need to be nloops long list_args = ['imsize'] #allowed to be lists of lists for arg in single_args: if arg in other_params: other_params.pop(other_params.index(arg)) for arg in single_args: if type(params[arg]) is list or type( params[arg]) is str and ',' in params[arg]: logger.error( "Parameter '{0}' in '{1}' cannot be a list. It must be a single value." .format(arg, args['config'])) exit = True for arg in other_params: if type(params[arg]) is str and ',' in params[arg]: logger.error( "Parameter '{0}' in '{1}' cannot use comma-seprated values. It must be a list or values, or a single value." .format(arg, args['config'])) exit = True # These can be a list of lists or a simple list (if specifying a single value). # So make sure these two cases are covered. if arg in list_args: # Not a list of lists, so turn it into one of right length if type(params[arg]) is list and (len(params[arg]) == 0 or type( params[arg][0]) is not list): params[arg] = [ params[arg], ] * (params['nloops'] + 1) # Not a list at all, so put it into a list elif type(params[arg]) is not list: params[arg] = [ [ params[arg], ], ] * (params['nloops'] + 1) # A list of lists of length 1, so put into list of lists of right length elif type(params[arg]) is list and type( params[arg][0]) is list and len(params[arg]) == 1: params[arg] = [ params[arg][0], ] * (params['nloops'] + 1) elif type(params[arg]) is not list: if arg in gaincal_args: params[arg] = [params[arg]] * (params['nloops']) else: params[arg] = [params[arg]] * (params['nloops'] + 1) for arg in other_params: #By this point params[arg] will be a list if arg in gaincal_args and len(params[arg]) != params['nloops']: logger.error( "Parameter '{0}' in '{1}' is the wrong length. It is {2} long but must be 'nloops' ({3}) long or a single value (not a list)." .format(arg, args['config'], len(params[arg]), params['nloops'] - 1)) exit = True elif arg not in gaincal_args and len( params[arg]) != params['nloops'] + 1: logger.error( "Parameter '{0}' in '{1}' is the wrong length. It is {2} long but must be 'nloops' + 1 ({3}) long or a single value (not a list)." .format(arg, args['config'], len(params[arg]), params['nloops'] + 2)) exit = True if exit: sys.exit(1) return args, params