if do_gaincal:
predict_model(vis, imagename, imsize, cell, gridder, wprojplanes,
deconvolver, robust, niter, multiscale, threshold,
nterms, pixmask, loop)
solnorm = 'a' in calmode[loop]
gaincal(vis=vis,
caltable=caltable,
selectdata=False,
refant=refant,
solint=solint[loop],
solnorm=solnorm,
normtype='median',
gaintable=prev_caltables,
calmode=calmode[loop],
append=False,
parang=False)
loop += 1
return loop
if __name__ == '__main__':
args, params = bookkeeping.get_selfcal_params()
loop = selfcal_part2(**params)
config_parser.overwrite_config(args['config'],
conf_dict={'loop': loop},
conf_sec='selfcal')
def format_args_iris(config, submit, quiet, dependencies):
"""Format (and validate) arguments from config file, to be passed into write_jobs() function.
Arguments:
----------
config : str
Path to config file.
submit : bool
Allow user to force submitting to queue immediately.
quiet : bool
Activate quiet mode, with suppressed output?
dependencies : str
Comma-separated list of SLURM job dependencies.
selfcal : bool
Is selfcal being performed?
Returns:
--------
kwargs : dict
Keyword arguments extracted from [iris] section of config file, to be passed into write_jobs() function."""
#Ensure all keys exist in these sections
kwargs = get_config_kwargs(config, 'iris', globals.IRIS_CONFIG_KEYS)
data_kwargs = get_config_kwargs(config, 'data', ['vis'])
get_config_kwargs(config, 'fields', globals.FIELDS_CONFIG_KEYS)
crosscal_kwargs = get_config_kwargs(config, 'crosscal',
globals.CROSSCAL_CONFIG_KEYS)
#Check selfcal params
if config_parser.has_section(config, 'selfcal'):
selfcal_kwargs = get_config_kwargs(config, 'selfcal',
globals.SELFCAL_CONFIG_KEYS)
bookkeeping.get_selfcal_params()
#Force submit=True if user has requested it during [-R --run]
if submit:
kwargs['submit'] = True
#Ensure nspw is integer
if type(crosscal_kwargs['nspw']) is not int:
logger.logger.warn(
"Argument 'nspw'={0} in '{1}' is not an integer. Will set to integer ({2})."
.format(crosscal_kwargs['nspw']), config,
int(crosscal_kwargs['nspw']))
crosscal_kwargs['nspw'] = int(crosscal_kwargs['nspw'])
spw = crosscal_kwargs['spw']
nspw = crosscal_kwargs['nspw']
#mem = int(kwargs['mem'])
if nspw > 1 and len(kwargs['scripts']) == 0:
logger.logger.warn(
'Setting nspw=1, since no "scripts" parameter in "{0}" is empty, so there\'s nothing run inside SPW directories.'
.format(config))
config_parser.overwrite_config(config,
conf_dict={'nspw': 1},
conf_sec='crosscal')
nspw = 1
#If nspw = 1 and precal or postcal scripts present, overwrite config and reload
if nspw == 1:
if len(kwargs['precal_scripts']) > 0 or len(
kwargs['postcal_scripts']) > 0:
logger.logger.warn(
'Appending "precal_scripts" to beginning of "scripts", and "postcal_script" to end of "scripts", since nspw=1. Overwritting this in "{0}".'
.format(config))
#Drop first instance of calc_refant.py from precal scripts in preference for one in scripts (after flag_round_1.py)
if 'calc_refant.py' in [
i[0] for i in kwargs['precal_scripts']
] and 'calc_refant.py' in [i[0] for i in kwargs['scripts']]:
kwargs['precal_scripts'].pop([
i[0] for i in kwargs['precal_scripts']
].index('calc_refant.py'))
scripts = kwargs['precal_scripts'] + kwargs['scripts'] + kwargs[
'postcal_scripts']
config_parser.overwrite_config(config,
conf_dict={'scripts': scripts},
conf_sec='iris')
config_parser.overwrite_config(config,
conf_dict={'precal_scripts': []},
conf_sec='iris')
config_parser.overwrite_config(config,
conf_dict={'postcal_scripts': []},
conf_sec='iris')
kwargs = get_config_kwargs(config, 'iris',
globals.IRIS_CONFIG_KEYS)
else:
scripts = kwargs['scripts']
else:
scripts = kwargs['precal_scripts'] + kwargs['postcal_scripts']
kwargs['num_precal_scripts'] = len(kwargs['precal_scripts'])
# Validate kwargs along with MS
kwargs['MS'] = data_kwargs['vis']
validate_args_iris(kwargs, config)
#Reformat scripts tuple/list, to extract scripts, threadsafe, and containers as parallel lists
#Check that path to each script and container exists or is ''
kwargs['scripts'] = [check_path(i[0]) for i in scripts]
kwargs['threadsafe'] = [i[1] for i in scripts]
kwargs['containers'] = [check_path(i[2]) for i in scripts]
if not crosscal_kwargs['createmms']:
logger.logger.info(
"You've set 'createmms = False' in '{0}', so forcing 'keepmms = False'. Will use single CPU for every job other than 'quick_tclean.py', if present."
.format(config))
config_parser.overwrite_config(config,
conf_dict={'keepmms': False},
conf_sec='crosscal')
kwargs['threadsafe'] = [False] * len(scripts)
elif not crosscal_kwargs['keepmms']:
#Set threadsafe=False for split and postcal scripts (since working with MS not MMS).
if 'split.py' in kwargs['scripts']:
kwargs['threadsafe'][kwargs['scripts'].index('split.py')] = False
if nspw != 1:
kwargs['threadsafe'][kwargs['num_precal_scripts']:] = [
False
] * len(kwargs['postcal_scripts'])
#Set threadsafe=True for quick-tclean, as this uses MPI even for an MS
if 'quick_tclean.py' in kwargs['scripts']:
kwargs['threadsafe'][kwargs['scripts'].index('quick_tclean.py')] = True
#Only reduce the memory footprint if we're not using all CPUs on each node
# if kwargs['ntasks_per_node'] < NTASKS_PER_NODE_LIMIT:
# mem = mem // nspw
dopol = config_parser.get_key(config, 'run', 'dopol')
if not dopol and ('xy_yx_solve.py' in kwargs['scripts']
or 'xy_yx_apply.py' in kwargs['scripts']):
logger.logger.warn(
"Cross-hand calibration scripts 'xy_yx_*' found in scripts. Forcing dopol=True in '[run]' section of '{0}'."
.format(config))
config_parser.overwrite_config(
config,
conf_dict={'dopol': True},
conf_sec='run',
sec_comment='# Internal variables for pipeline execution')
includes_partition = any('partition' in script
for script in kwargs['scripts'])
#If single correctly formatted spw, split into nspw directories, and process each spw independently
if nspw > 1:
#Write timestamp to this pipeline run
kwargs['timestamp'] = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
config_parser.overwrite_config(
config,
conf_dict={'timestamp': "'{0}'".format(kwargs['timestamp'])},
conf_sec='run',
sec_comment='# Internal variables for pipeline execution')
nspw = spw_split_iris(spw,
nspw,
config,
crosscal_kwargs['badfreqranges'],
kwargs['MS'],
includes_partition,
createmms=crosscal_kwargs['createmms'])
config_parser.overwrite_config(config,
conf_dict={'nspw': "{0}".format(nspw)},
conf_sec='crosscal')
#Pop script to calculate reference antenna if calcrefant=False. Assume it won't be in postcal scripts
if not crosscal_kwargs['calcrefant']:
if pop_script(kwargs, 'calc_refant.py'):
kwargs['num_precal_scripts'] -= 1
#Replace empty containers with default container and remove unwanted kwargs
for i in range(len(kwargs['containers'])):
if kwargs['containers'][i] == '':
kwargs['containers'][i] = kwargs['container']
kwargs.pop('container')
kwargs.pop('MS')
kwargs.pop('precal_scripts')
kwargs.pop('postcal_scripts')
kwargs['quiet'] = quiet
#Force overwrite of dependencies
if dependencies != '':
kwargs['dependencies'] = dependencies
if len(kwargs['scripts']) == 0:
logger.logger.error(
'Nothing to do. Please insert scripts into "scripts" parameter in "{0}".'
.format(config))
sys.exit(1)
#If everything up until here has passed, we can copy config file to TMP_CONFIG (in case user runs sbatch manually) and inform user
logger.logger.debug(
"Copying '{0}' to '{1}', and using this to run pipeline.".format(
config, globals.TMP_CONFIG))
copyfile(config, globals.TMP_CONFIG)
if not quiet:
logger.logger.warn(
"Changing [iris] section in your config will have no effect until you submit your jdl"
)
return kwargs