def start_apercal_pipeline(targets,
fluxcals,
polcals,
dry_run=False,
basedir=None,
flip_ra=False,
steps=None,
configfilename=None):
"""
Trigger the start of a fluxcal pipeline. Returns when pipeline is done.
Example for taskid, name, beamnr: (190108926, '3C147_36', 36)
Fluxcals and polcals can be specified in the wrong order, if the polcal is not polarised
they will be flipped.
If both polcals and fluxcals are set, they should both be the same length.
A list of config files can be provided, i.e., one for each beam. If a single config file
is given, copies of it will be created so that there is one config per beam. If no
config file is given, the default one is used and copies for each beam are made.
Args:
targets (Tuple[int, str, List[int]]): taskid, name, list of beamnrs
fluxcals (List[Tuple[int, str, int]]): fluxcals: taskid, name, beamnr
polcals (List[Tuple[int, str, int]]): polcals: taskid, name, beamnr (can be None)
dry_run (bool): interpret arguments, do not actually run pipeline
basedir (str): base directory; if not specified will be /data/apertif/{target_taskid}
flip_ra (bool): flip RA (for old measurement sets where beamweights were flipped)
steps (List[str]): list of steps to perform
configfilename (List[str]): Custom configfile (should be full path for now)
Returns:
Tuple[Dict[int, List[str]], str], str: Tuple of a dict, the formatted runtime, and possibly
an exception. The dict
contains beam numbers (ints) as keys, a list of failed
steps as values. Failed is defined here as 'threw an
exception', only for target steps. Please also read logs.
"""
if steps is None:
steps = [
"prepare", "split", "preflag", "ccal", "convert", "scal",
"continuum", "polarisation", "line", "transfer"
]
(taskid_target, name_target, beamlist_target) = targets
# set the base directory if none was provided
if not basedir:
basedir = '/data/apertif/{}/'.format(taskid_target)
elif len(basedir) > 0 and basedir[-1] != '/':
basedir = basedir + '/'
if not os.path.exists(basedir):
os.mkdir(basedir)
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
gitinfo = subprocess.check_output('cd ' +
os.path.dirname(apercal.__file__) +
'&& git describe --tag; cd',
shell=True).strip()
logger.info("Apercal version: " + gitinfo)
logger.info(
"start_apercal called with arguments targets={}; fluxcals={}; polcals={}"
.format(targets, fluxcals, polcals))
logger.info("steps = {}".format(steps))
# number of beams to process
n_beams = len(beamlist_target)
# check the input config file
# get the default configfile if none was provided
if not configfilename:
logger.info("No config file provided, getting default config")
# create a list of config file name
configfilename_list = [
os.path.join(
basedir,
"{0}_B{1}_Apercal_settings.cfg".format(taskid_target,
str(beam).zfill(2)))
for beam in beamlist_target
]
# get the default config settings
config = lib.get_default_config()
# go through the config files and create them
for beam_index in range(n_beams):
with open(configfilename_list[beam_index], "w") as fp:
config.write(fp)
logger.info("Beam {} config file saved to {}".format(
beamlist_target[beam_index], configfilename_list[beam_index]))
# if configfile(s) are given as a list
elif type(configfilename) is list:
# if it is just one, create copies for each beam in the base directory
if len(configfilename) == 1:
logger.info(
"A single config file was provided. Creating copies of {}".
format(configfilename[0]))
configfilename_list = [
os.path.join(
basedir, "{0}_B{1}_Apercal_settings.cfg".format(
taskid_target,
str(beam).zfill(2))) for beam in beamlist_target
]
# make the copies
for config in configfilename_list:
lib.basher("cp " + str(configfilename[0]) + " " + str(config))
elif len(configfilename) == n_beams:
logger.info("Number of config files and target beams match.")
configfilename_list = configfilename
else:
error = "Number of config files and target beams did not match. Abort"
logger.error(error)
raise RuntimeError(error)
# if configfilename is just a string
elif type(configfilename) is str:
logger.info(
"A single config file was provided. Creating copies of {}".format(
configfilename))
configfilename_list = [
os.path.join(
basedir,
"{0}_B{1}_Apercal_settings.cfg".format(taskid_target,
str(beam).zfill(2)))
for beam in beamlist_target
]
# make the copies
for config in configfilename_list:
lib.basher("cp " + str(configfilename) + " " + str(config))
else:
error = "Unknown input for configfilename. Abort"
logger.error(error)
raise RuntimeError(error)
status = pymp.shared.dict({beamnr: [] for beamnr in beamlist_target})
if fluxcals:
name_fluxcal = str(fluxcals[0][1]).strip().split('_')[0].upper()
else:
name_fluxcal = ''
if polcals:
name_polcal = str(polcals[0][1]).strip().split('_')[0].upper()
else:
name_polcal = ''
name_target = str(name_target).strip() # .upper()
# If both fluxcal and polcal polarized, remove polcal
if subs_calmodels.is_polarised(
name_polcal) and subs_calmodels.is_polarised(name_fluxcal):
name_polcal = ""
if (fluxcals and fluxcals != '') and (polcals and polcals != ''):
assert (len(fluxcals) == len(polcals))
# avoid symmetry bias, if there is only a polcal but no fluxcal, switch them
if fluxcals is None and polcals is not None:
logger.info(
"Only polcal was provided. Setting polcal {} to fluxcal".format(
name_polcal))
fluxcals, polcals = polcals, fluxcals
name_polcal = ""
# Exchange polcal and fluxcal if specified in the wrong order
elif not subs_calmodels.is_polarised(name_polcal) and name_polcal != '':
if subs_calmodels.is_polarised(name_fluxcal):
logger.info("Switching polcal and fluxcal because " + name_polcal +
" is not polarised")
fluxcals, polcals = polcals, fluxcals
name_polcal = str(polcals[0][1]).strip()
else:
logger.info("Setting polcal to '' since " + name_polcal +
" is not polarised")
name_polcal = ""
elif name_polcal != '':
logger.info("Polcal " + name_polcal + " is polarised, all good")
def name_to_ms(name):
if not name:
return ''
elif '3C' in name:
return name.upper().strip().split('_')[0] + '.MS'
else:
return name + '.MS'
def name_to_mir(name):
if not name:
return ''
elif '3C' in name:
return name.upper().strip().split('_')[0] + '.mir'
else:
return name + '.mir'
def set_files(p):
"""
Set the basedir, fluxcal, polcal, target properties
Args:
p (BaseModule): apercal step object (e.g. prepare)
Returns:
None
"""
p.basedir = basedir
p.fluxcal = name_to_ms(name_fluxcal)
p.polcal = name_to_ms(name_polcal)
p.target = name_to_ms(name_target)
# debug_msg = """
# p.basedir = basedir = {0};
# p.fluxcal = name_to_ms(name_fluxcal) = {1};
# p.polcal = name_to_ms(name_polcal) = {2};
# p.target = name_to_ms(name_target) = {3};
# """.format(basedir, name_to_ms(name_fluxcal), name_to_ms(name_polcal), name_to_ms(name_target))
# logger.debug(debug_msg)
beamnrs_fluxcal = [f[2] for f in fluxcals]
if len(fluxcals) > 1:
# Check every target beam has a fluxcal beam
for beamnr_target in beamlist_target:
assert (beamnr_target in beamnrs_fluxcal)
# creating a copy of the target beamlist as a normal array
# to avoid using np.where() for such a small thing
if type(beamlist_target) == np.ndarray:
beamlist_target_for_config = beamlist_target.tolist()
else:
beamlist_target_for_config = beamlist_target
time_start = time()
try:
# =======
# Prepare
# =======
# keep a start-finish record of step in the main log file
if "prepare" in steps:
logger.info("Running prepare")
start_time_prepare = time()
else:
logger.info("Skipping prepare")
# Prepare fluxcals
for (taskid_fluxcal, name_fluxcal, beamnr_fluxcal) in fluxcals:
p0 = prepare(file_=configfilename_list[
beamlist_target_for_config.index(beamnr_fluxcal)])
p0.basedir = basedir
#set_files(p0)
p0.prepare_flip_ra = flip_ra
# the following two need to be empty strings for prepare
p0.fluxcal = ''
p0.polcal = ''
p0.target = name_to_ms(name_fluxcal)
p0.prepare_target_beams = str(beamnr_fluxcal)
p0.prepare_date = str(taskid_fluxcal)[:6]
p0.prepare_obsnum_target = validate_taskid(taskid_fluxcal)
if "prepare" in steps and not dry_run:
try:
p0.go()
except Exception as e:
logger.warning("Prepare failed for fluxcal " +
str(taskid_fluxcal) + " beam " +
str(beamnr_fluxcal))
logger.exception(e)
if 'prepare' in steps:
# copy the param file generated here
param_file = os.path.join(basedir, 'param.npy')
director(p0,
'rn',
param_file.replace(
".npy",
"_prepare_{}.npy".format(name_fluxcal.split('_')[0])),
file_=param_file,
ignore_nonexistent=True)
# Prepare polcals
if name_polcal != '':
for (taskid_polcal, name_polcal, beamnr_polcal) in polcals:
p0 = prepare(file_=configfilename_list[
beamlist_target_for_config.index(beamnr_polcal)])
p0.basedir = basedir
#set_files(p0)
p0.prepare_flip_ra = flip_ra
# the following two need to be empty strings for prepare
p0.fluxcal = ''
p0.polcal = ''
p0.target = name_to_ms(name_polcal)
p0.prepare_target_beams = str(beamnr_polcal)
p0.prepare_date = str(taskid_polcal)[:6]
p0.prepare_obsnum_target = validate_taskid(taskid_polcal)
if "prepare" in steps and not dry_run:
try:
p0.go()
except Exception as e:
logger.warning("Prepare failed for polcal " +
str(taskid_polcal) + " beam " +
str(beamnr_polcal))
logger.exception(e)
if 'prepare' in steps:
# copy the param file generated here
param_file = os.path.join(basedir, 'param.npy')
director(p0,
'rn',
param_file.replace(
".npy", "_prepare_{}.npy".format(
name_polcal.split('_')[0])),
file_=param_file,
ignore_nonexistent=True)
# Prepare target
for beamnr in beamlist_target:
p0 = prepare(file_=configfilename_list[
beamlist_target_for_config.index(beamnr)])
p0.basedir = basedir
# set_files(p0)
p0.prepare_flip_ra = flip_ra
# the following two need to be empty strings for prepare
p0.fluxcal = ''
p0.polcal = ''
p0.target = name_to_ms(name_target)
p0.prepare_date = str(taskid_target)[:6]
p0.prepare_obsnum_target = validate_taskid(taskid_target)
p0.prepare_target_beams = ','.join(
['{:02d}'.format(beamnr) for beamnr in beamlist_target])
if "prepare" in steps and not dry_run:
try:
p0.go()
except Exception as e:
logger.warning("Prepare failed for target " +
str(taskid_target) + " beam " + str(beamnr))
logger.exception(e)
status[beamnr] += ['prepare']
# keep a start-finish record of step in the main log file
if "prepare" in steps:
logger.info("Running prepare ... Done ({0:.0f}s)".format(
time() - start_time_prepare))
# copy the param file generated here
param_file = os.path.join(basedir, 'param.npy')
director(p0,
'rn',
param_file.replace(".npy",
"_prepare_{}.npy".format(name_target)),
file_=param_file,
ignore_nonexistent=True)
# =====
# Split
# =====
# keep a start-finish record of step in the main log file
if 'split' in steps:
logger.info("Running split")
start_time_split = time()
else:
logger.info("Skipping split")
# Splitting a small chunk of data for quicklook pipeline
# at the moment it all relies on the target beams
# what if there are more calibrator than target beams-> realistic?
with pymp.Parallel(5) as p:
for beam_index in p.range(n_beams):
beamnr = beamlist_target[beam_index]
# individual logfiles for each process
logfilepath = os.path.join(basedir,
'apercal{:02d}.log'.format(beamnr))
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.debug("Starting logfile for beam " + str(beamnr))
try:
s0 = split(file_=configfilename_list[beam_index])
set_files(s0)
s0.beam = "{:02d}".format(beamnr)
if "split" in steps and not dry_run:
s0.go()
except Exception as e:
logger.warning("Split failed for {0} beam {1}".format(
str(taskid_target), str(beamnr)))
logger.exception(e)
# not sure if following line is necessary
status[beamnr] += ['split']
# keep a start-finish record of step in the main log file
if "split" in steps:
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.info(
"Running split ... Done ({0:.0f}s)".format(time() -
start_time_split))
# copy the param file generated here
# param_file = os.path.join(basedir, 'param.npy')
# director(
# p0, 'rn', param_file.replace(".npy", "_split.npy"), file_=param_file, ignore_nonexistent=True)
# =======
# Preflag
# =======
# keep a record of the parallalised step in the main log file
if "preflag" in steps:
logger.info("Running preflag")
start_time_preflag = time()
else:
logger.info("Skipping preflag")
# In order to run in parallel, the bandpass table needs to exists
# doing it here is not elegant but requires the least amount of changes
# to preflage
# with pymp.Parallel(10) as p:
# for beam_index in p.range(n_beams):
# beamnr = beamlist_target[beam_index]
# # individual logfiles for each process
# logfilepath = os.path.join(
# basedir, 'apercal{:02d}.log'.format(beamnr))
# lib.setup_logger('debug', logfile=logfilepath)
# logger = logging.getLogger(__name__)
# logger.debug("Starting logfile for beam " + str(beamnr))
# p1 = preflag(filename=configfilename)
# p1.paramfilename = 'param_{:02d}.npy'.format(beamnr)
# p1.basedir = basedir
# p1.fluxcal = ''
# p1.polcal = ''
# p1.target = name_to_ms(name_fluxcal)
# p1.beam = "{:02d}".format(beamnr)
# p1.preflag_targetbeams = "{:02d}".format(beamnr)
# if "preflag" in steps and not dry_run:
# try:
# bandpass_start_time = time()
# logger.info("Running aoflagger bandpass for flux calibrator {0} in beam {1}".format(
# p1.target, p1.beam))
# # director(
# # p1, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True)
# p1.go()
# # director(p1, 'rm', basedir + '/param.npy',
# # ignore_nonexistent=True)
# # it is necessary to move the param files in order to keep them
# param_file = basedir + \
# '/param_{:02d}.npy'.format(beamnr)
# director(
# p1, 'mv', param_file, file_=param_file.replace(".npy", "_preflag_{0}.npy".format(name_fluxcal)), ignore_nonexistent=True)
# p1.aoflagger_bandpass()
# except Exception as e:
# logger.warning("Running aoflagger bandpass for flux calibrator {0} in beam {1} ... Failed ({2:.0f}s)".format(
# p1.target, p1.beam, time() - bandpass_start_time))
# logger.exception(e)
# status[beamnr] += ['preflag_bandpass']
# else:
# logger.info("Running aoflagger bandpass for flux calibrator {0} in beam {1} ... Done ({2:.0f}s)".format(
# p1.target, p1.beam, time() - bandpass_start_time))
# Flag fluxcal (pretending it's a target, parallelised version)
# 5 in parallel
with pymp.Parallel(5) as p:
for beam_index in p.range(n_beams):
beamnr = beamlist_target[beam_index]
# individual logfiles for each process
logfilepath = os.path.join(basedir,
'apercal{:02d}.log'.format(beamnr))
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.debug("Starting logfile for beam " + str(beamnr))
try:
p1 = preflag(filename=configfilename_list[beam_index])
p1.paramfilename = 'param_{:02d}.npy'.format(beamnr)
p1.basedir = basedir
p1.fluxcal = ''
p1.polcal = ''
p1.target = name_to_ms(name_fluxcal)
p1.beam = "{:02d}".format(beamnr)
p1.preflag_targetbeams = "{:02d}".format(beamnr)
if beam_index < 2:
p1.preflag_aoflagger_threads = 9
else:
p1.preflag_aoflagger_threads = 10
if "preflag" in steps and not dry_run:
logger.info(
"Running preflag for flux calibrator {0} in beam {1}"
.format(p1.target, p1.beam))
preflag_flux_cal_start_time = time()
# director(
# p1, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True)
p1.go()
# it is necessary to move the param files in order to keep them
param_file = os.path.join(
basedir, 'param_{:02d}.npy'.format(beamnr))
director(p1,
'rn',
param_file.replace(
".npy", "_preflag_{0}.npy".format(
name_fluxcal.split('_')[0])),
file_=param_file,
ignore_nonexistent=True)
logger.info(
"Running preflag for flux calibrator {0} in beam {1} ... Done ({2:.0f}s)"
.format(p1.target, p1.beam,
time() - preflag_flux_cal_start_time))
except Exception as e:
logger.warning(
"Running preflag for flux calibrator {0} in beam {1} ... Failed ({2:.0f}s)"
.format(p1.target, p1.beam,
time() - preflag_flux_cal_start_time))
logger.exception(e)
status[beamnr] += ['preflag']
# Flag polcal (pretending it's a target, parallel version)
# 5 in parallel
with pymp.Parallel(5) as p:
for beam_index in p.range(n_beams):
beamnr = beamlist_target[beam_index]
# individual logfiles for each process
logfilepath = os.path.join(basedir,
'apercal{:02d}.log'.format(beamnr))
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.debug("Starting logfile for beam " + str(beamnr))
try:
p1 = preflag(filename=configfilename_list[beam_index])
# remove next line in final version
p1.preflag_aoflagger_version = 'local'
p1.basedir = basedir
p1.paramfilename = 'param_{:02d}.npy'.format(beamnr)
p1.basedir = basedir
if name_polcal != '':
p1.fluxcal = ''
p1.polcal = ''
p1.target = name_to_ms(name_polcal)
p1.beam = "{:02d}".format(beamnr)
p1.preflag_targetbeams = "{:02d}".format(beamnr)
if beam_index < 2:
p1.preflag_aoflagger_threads = 9
else:
p1.preflag_aoflagger_threads = 10
if "preflag" in steps and not dry_run:
logger.info(
"Running preflag for pol calibrator {0} in beam {1}"
.format(p1.target, p1.beam))
preflag_pol_cal_start_time = time()
# director(
# p1, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True)
p1.go()
# it is necessary to move the param files in order to keep them
param_file = os.path.join(
basedir, 'param_{:02d}.npy'.format(beamnr))
director(p1,
'rn',
param_file.replace(
".npy", "_preflag_{0}.npy".format(
name_polcal.split('_')[0])),
file_=param_file,
ignore_nonexistent=True)
logger.info(
"Running preflag for pol calibrator {0} in beam {1} ... Done ({2:.0f}s)"
.format(p1.target, p1.beam,
time() - preflag_pol_cal_start_time))
except Exception as e:
logger.warning(
"Running preflag for pol calibrator {0} in beam {1} ... Failed ({2:.0f}s)"
.format(p1.target, p1.beam,
time() - preflag_pol_cal_start_time))
logger.exception(e)
status[beamnr] += ['preflag']
# Flag target
# 5 in parallel
with pymp.Parallel(5) as p:
for beam_index in p.range(n_beams):
beamnr = beamlist_target[beam_index]
# individual logfiles for each process
logfilepath = os.path.join(basedir,
'apercal{:02d}.log'.format(beamnr))
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.debug("Starting logfile for beam " + str(beamnr))
try:
p1 = preflag(filename=configfilename_list[beam_index])
# remove next line in final version
p1.preflag_aoflagger_version = 'local'
p1.paramfilename = 'param_{:02d}.npy'.format(beamnr)
p1.basedir = basedir
p1.fluxcal = ''
p1.polcal = ''
p1.target = name_to_ms(name_target)
p1.beam = "{:02d}".format(beamnr)
p1.preflag_targetbeams = "{:02d}".format(beamnr)
if beam_index < 2:
p1.preflag_aoflagger_threads = 9
else:
p1.preflag_aoflagger_threads = 10
if "preflag" in steps and not dry_run:
logger.info(
"Running preflag for target {0} in beam {1}".
format(p1.target, p1.beam))
preflag_target_start_time = time()
# director(
# p1, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True)
p1.go()
# it is necessary to move the param files in order to keep them
param_file = os.path.join(
basedir, 'param_{:02d}.npy'.format(beamnr))
director(p1,
'rn',
param_file.replace(
".npy",
"_preflag_{0}.npy".format(name_target)),
file_=param_file,
ignore_nonexistent=True)
logger.info(
"Running preflag for target {0} in beam {1} ... Done ({2:.0f}s)"
.format(p1.target, p1.beam,
time() - preflag_target_start_time))
except Exception as e:
logger.info(
"Running preflag for target {0} in beam {1} ... Failed ({2:.0f}s)"
.format(p1.target, p1.beam,
time() - preflag_target_start_time))
logger.exception(e)
status[beamnr] += ['preflag']
# keep a record of the parallalised step in the main log file
if "preflag" in steps:
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.info("Running preflag ... Done ({0:.0f}s)".format(
time() - start_time_preflag))
# ===============
# Crosscal
# ===============
# keep a record of the parallalised step in the main log file
if 'ccal' in steps:
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.info("Running crosscal")
start_time_crosscal = time()
else:
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.info("Skipping crosscal")
if len(fluxcals) == 1 and fluxcals[0][-1] == 0 and n_beams > 1:
raise ApercalException(
"Sorry, one fluxcal is not supported anymore at the moment")
with pymp.Parallel(10) as p:
for beam_index in p.range(n_beams):
beamnr = beamlist_target[beam_index]
logfilepath = os.path.join(basedir,
'apercal{:02d}.log'.format(beamnr))
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.debug("Starting logfile for beam " + str(beamnr))
try:
p2 = ccal(file_=configfilename_list[beam_index])
p2.paramfilename = 'param_{:02d}.npy'.format(beamnr)
set_files(p2)
p2.beam = "{:02d}".format(beamnr)
p2.crosscal_transfer_to_target_targetbeams = "{:02d}".format(
beamnr)
if "ccal" in steps and not dry_run:
# director(
# p2, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True)
p2.go()
# it is necessary to move the param files in order to keep them
param_file = os.path.join(
basedir, 'param_{:02d}.npy'.format(beamnr))
director(p2,
'rn',
param_file.replace(".npy", "_crosscal.npy"),
file_=param_file,
ignore_nonexistent=True)
except Exception as e:
# Exception was already logged just before
logger.warning(
"Failed beam {}, skipping that from crosscal".format(
beamnr))
logger.exception(e)
status[beamnr] += ['crosscal']
# keep a record of the parallalised step in the main log file
if 'ccal' in steps:
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.info("Running crosscal ... Done ({0:.0f}s)".format(
time() - start_time_crosscal))
# =======
# Convert
# =======
# keep a record of the parallalised step in the main log file
if 'convert' in steps:
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.info("Running convert")
start_time_convert = time()
else:
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.info("Skipping convert")
# 5 threads to not hammer the disks too much, convert is only IO
with pymp.Parallel(5) as p:
for beam_index in p.range(n_beams):
beamnr = beamlist_target[beam_index]
logfilepath = os.path.join(basedir,
'apercal{:02d}.log'.format(beamnr))
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
try:
p3 = convert(file_=configfilename_list[beam_index])
p3.paramfilename = 'param_{:02d}.npy'.format(beamnr)
set_files(p3)
p3.beam = "{:02d}".format(beamnr)
p3.convert_targetbeams = "{:02d}".format(beamnr)
if "convert" in steps and not dry_run:
# director(
# p3, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True)
p3.go()
# it is necessary to move the param files in order to keep them
param_file = os.path.join(
basedir, 'param_{:02d}.npy'.format(beamnr))
director(p3,
'rn',
param_file.replace(".npy", "_convert.npy"),
file_=param_file,
ignore_nonexistent=True)
# director(
# p3, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True)
except Exception as e:
logger.warning(
"Failed beam {}, skipping that from convert".format(
beamnr))
logger.exception(e)
status[beamnr] += ['convert']
if 'convert' in steps:
# keep a record of the parallalised step in the main log file
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.info("Running convert ... Done ({0:.0f}s)".format(
time() - start_time_convert))
# ==================================
# Selfcal + Continuum + Polarisation
# ==================================
# keep a record of the parallalised step in the main log file
if 'scal' in steps or 'continuum' in steps or 'polarisation' in steps:
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.info("Running selfcal and/or continuum and/or polarisation")
start_time_selfcal_continuum_polarisation = time()
else:
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.info("Skipping selfcal and continuum and polarisation")
with pymp.Parallel(10) as p:
for beam_index in p.range(n_beams):
beamnr = beamlist_target[beam_index]
logfilepath = os.path.join(basedir,
'apercal{:02d}.log'.format(beamnr))
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
try:
p4 = scal(file_=configfilename_list[beam_index])
p4.paramfilename = 'param_{:02d}.npy'.format(beamnr)
p4.basedir = basedir
p4.beam = "{:02d}".format(beamnr)
p4.target = name_target + '.mir'
if "scal" in steps and not dry_run:
p4.go()
except Exception as e:
# Exception was already logged just before
logger.warning(
"Failed beam {}, skipping that from scal".format(
beamnr))
logger.exception(e)
status[beamnr] += ['scal']
try:
p5 = continuum(file_=configfilename_list[beam_index])
p5.paramfilename = 'param_{:02d}.npy'.format(beamnr)
p5.basedir = basedir
p5.beam = "{:02d}".format(beamnr)
p5.target = name_target + '.mir'
if "continuum" in steps and not dry_run:
p5.go()
except Exception as e:
# Exception was already logged just before
logger.warning(
"Failed beam {}, skipping that from continuum".format(
beamnr))
logger.exception(e)
status[beamnr] += ['continuum']
try:
p6 = polarisation(file_=configfilename_list[beam_index])
p6.paramfilename = 'param_{:02d}.npy'.format(beamnr)
p6.basedir = basedir
p6.beam = "{:02d}".format(beamnr)
p6.polcal = name_to_mir(name_polcal)
p6.target = name_to_mir(name_target)
if "polarisation" in steps and not dry_run:
p6.go()
except Exception as e:
# Exception was already logged just before
logger.warning(
"Failed beam {}, skipping that from polarisation".
format(beamnr))
logger.exception(e)
status[beamnr] += ['polarisation']
# keep a record of the parallalised step in the main log file
if 'scal' in steps or 'continuum' in steps or 'polarisation' in steps:
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.info(
"Running selfcal and/or continuum and/or polarisation ... Done ({0:.0f}s)"
.format(time() - start_time_selfcal_continuum_polarisation))
# ====
# Line
# ====
# keep a record of the parallalised step in the main log file
if 'line' in steps:
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger.info("Running line")
start_time_line = time()
else:
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger.info("Skipping line")
for beamnr in beamlist_target:
# Because of the amount of information coming from line
# this module gets its own logfile
logfilepath = os.path.join(basedir,
'apercal{:02d}_line.log'.format(beamnr))
lib.setup_logger('debug', logfile=logfilepath)
try:
p7 = line(file_=configfilename_list[
beamlist_target_for_config.index(beamnr)])
if beamnr not in p7.line_beams:
logger.debug(
"Skipping line imaging for beam {}".format(beamnr))
continue
p7.basedir = basedir
p7.beam = "{:02d}".format(beamnr)
p7.target = name_target + '.mir'
if "line" in steps and not dry_run:
p7.go()
except Exception as e:
# Exception was already logged just before
logger.warning(
"Failed beam {}, skipping that from line".format(beamnr))
logger.exception(e)
status[beamnr] += ['line']
# with pymp.Parallel(5) as p:
# for beam_index in p.range(n_beams):
# beamnr = beamlist_target[beam_index]
# logfilepath = os.path.join(
# basedir, 'apercal{:02d}.log'.format(beamnr))
# lib.setup_logger('debug', logfile=logfilepath)
# logger = logging.getLogger(__name__)
# try:
# p7 = line(file_=configfilename)
# if beamnr not in p7.line_beams:
# logger.debug(
# "Skipping line imaging for beam {}".format(beamnr))
# continue
# p7.basedir = basedir
# p7.beam = "{:02d}".format(beamnr)
# p7.target = name_target + '.mir'
# if "line" in steps and not dry_run:
# p7.go()
# except Exception as e:
# # Exception was already logged just before
# logger.warning(
# "Failed beam {}, skipping that from line".format(beamnr))
# logger.exception(e)
# status[beamnr] += ['line']
# keep a record of the parallalised step in the main log file
if 'line' in steps:
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.info(
"Running line ... Done ({0:.0f}s)".format(time() -
start_time_line))
# ========
# Transfer
# ========
# keep a record of the parallalised step in the main log file
if 'transfer' in steps:
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger.info("Running transfer")
start_time_transfer = time()
else:
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger.info("Skipping transfer")
# 5 threads to not hammer the disks too much during copying
with pymp.Parallel(5) as p:
for beam_index in p.range(n_beams):
beamnr = beamlist_target[beam_index]
logfilepath = os.path.join(basedir,
'apercal{:02d}.log'.format(beamnr))
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
try:
p8 = transfer(file_=configfilename_list[beam_index])
p8.paramfilename = 'param_{:02d}.npy'.format(beamnr)
p8.basedir = basedir
p8.target = name_target + '.mir'
p8.beam = "{:02d}".format(beamnr)
if "transfer" in steps and not dry_run:
# director(
# p8, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True)
p8.go()
except Exception as e:
logger.warning(
"Failed beam {}, skipping that from transfer".format(
beamnr))
logger.exception(e)
status[beamnr] += ['transfer']
# keep a record of the parallalised step in the main log file
if 'transfer' in steps:
logfilepath = os.path.join(basedir, 'apercal.log')
lib.setup_logger('debug', logfile=logfilepath)
logger = logging.getLogger(__name__)
logger.info("Running transfer ... Done ({0:.0f}s)".format(
time() - start_time_transfer))
# Polarisation
# ============
# keep a record of the parallalised step in the main log file
# if 'polarisation' in steps:
# logfilepath = os.path.join(basedir, 'apercal.log')
# lib.setup_logger('debug', logfile=logfilepath)
# logger = logging.getLogger(__name__)
# logger.info("Running polarisation")
# start_time_polarisation = time()
# else:
# logfilepath = os.path.join(basedir, 'apercal.log')
# lib.setup_logger('debug', logfile=logfilepath)
# logger = logging.getLogger(__name__)
# logger.info("Skipping polarisation")
# with pymp.Parallel(5) as p:
# for beam_index in p.range(n_beams):
# beamnr = beamlist_target[beam_index]
# logfilepath = os.path.join(
# basedir, 'apercal{:02d}.log'.format(beamnr))
# lib.setup_logger('debug', logfile=logfilepath)
# logger = logging.getLogger(__name__)
# try:
# p7 = polarisation(file_=configfilename)
# p7.paramfilename = 'param_{:02d}.npy'.format(beamnr)
# p7.basedir = basedir
# p7.beam = "{:02d}".format(beamnr)
# p7.target = name_to_mir(name_target)
# if "polarisation" in steps and not dry_run:
# p7.go()
# except Exception as e:
# # Exception was already logged just before
# logger.warning(
# "Failed beam {}, skipping that from polarisation".format(beamnr))
# logger.exception(e)
# status[beamnr] += ['polarisation']
# # keep a record of the parallalised step in the main log file
# if 'polarisation' in steps:
# logfilepath = os.path.join(basedir, 'apercal.log')
# lib.setup_logger('debug', logfile=logfilepath)
# logger = logging.getLogger(__name__)
# logger.info("Running polarisation ... Done ({0:.0f}s)".format(
# time() - start_time_polarisation))
# if "ccalqa" in steps and not dry_run:
# logger.info("Starting crosscal QA plots")
# try:
# make_all_ccal_plots(
# taskid_target, name_fluxcal.upper().strip().split('_')[0])
# except Exception as e:
# logger.warning("Failed crosscal QA plots")
# logger.exception(e)
# logger.info("Done with crosscal QA plots")
status = status.copy() # Convert pymp shared dict to a normal one
msg = "Apercal finished after " + \
str(timedelta(seconds=time() - time_start))
logger.info(msg)
return status, str(timedelta(seconds=time() - time_start)), None
except Exception as e:
msg = "Apercal threw an error after " + \
str(timedelta(seconds=time() - time_start))
logger.exception(msg)
return status, str(timedelta(seconds=time() - time_start)), str(e)
def run_triggered_qa(targets,
fluxcals,
polcals,
steps=None,
basedir=None,
osa=''):
"""Function to run all QA steps.
| Args:
| target (list(int, str, list(str))): the target field to be reduced
| fluxcals (list(int, str, int)): the flux calibrators for the beams to be reduced
| polcals (list(int, str, int)): the polarisation calibrator for the beams to be reduced
| steps (list(str)): The QA steps that should be exectuted
| basedir (str): The directory where the taskid is located
| osa (str): The OSA of the given taskid
| Function is called from autocal as
| ``run_triggered_qa(tdict['target'], tdict['cal1'], tdict['cal2'])``
| With the first three variables defined (the same way as autocal) as
| targets = (190505048, 'LH_WSRT', array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))
fluxcals = [(190505017, '3C147_9', 9), (190505016, '3C147_8', 8), (190505015, '3C147_7', 7), (190505014, '3C147_6', 6), (190505013, '3C147_5', 5), (190505012, '3C147_4', 4), (190505011, '3C147_3', 3), (190505010, '3C147_2', 2), (190505009, '3C147_1', 1), (190505008, '3C147_0', 0)]
polcals = [(190506001, '3C286_0', 0), (190506002, '3C286_1', 1), (190506003, '3C286_2', 2), (190506004, '3C286_3', 3), (190506005,'3C286_4', 4), (190506006, '3C286_5', 5), (190506007, '3C286_6', 6), (190506008, '3C286_7', 7), (190506009, '3C286_8', 8), (190506010, '3C286_9', 9)]
| If steps is not provided then all steps except mosaic will be performed:
| steps = ['inspection_plots', 'preflag', 'crosscal', 'selfcal', 'continuum', 'line', 'report']
| It is possible to select a certain step:
| steps = ['inspection_plots']
| test call can look like this:
| ``from dataqa.run_qa import run_triggered_qa``
| ``run_triggered_qa((190505048, 'LH_WSRT', [0]), [(190505048, '3C147_10', 10)], [(190505048, '3C286_10', 10)], steps=['report'])``
"""
# for time measurement
start_time = time.time()
# Process input parameters
# (same as in start_apercal_pipeline)
# ========================
(taskid_target, name_target, beamlist_target) = targets
if fluxcals:
name_fluxcal = str(fluxcals[0][1]).strip().split('_')[0].upper()
else:
name_fluxcal = ''
if polcals:
name_polcal = str(polcals[0][1]).strip().split('_')[0].upper()
else:
name_polcal = ''
if steps is None:
# steps = ['preflag', 'crosscal', 'selfcal',
# 'continuum', 'line', 'mosaic', 'report']
# steps = ['inspection_plots', 'beamweights', 'preflag', 'crosscal', 'selfcal',
# 'continuum', 'line', 'report']
# due to isssue with beamweights script, this step is currently not performed
steps = [
'inspection_plots', 'preflag', 'crosscal', 'selfcal', 'continuum',
'line', 'report'
]
# Set up
# ======
# Get host name
host_name = socket.gethostname()
# QA directory
if basedir is not None:
qa_dir = get_default_imagepath(taskid_target, basedir=basedir)
else:
qa_dir = get_default_imagepath(taskid_target)
basedir = "/data/apertif"
# check that path exists
if not os.path.exists(qa_dir):
print("Directory {0:s} does not exist and will be created".format(
qa_dir))
try:
os.mkdir(qa_dir)
except Exception as e:
print(e)
# start log file
# logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s',
# filename='{0:s}{1:s}_triggered_qa.log'.format(qa_dir, host_name), level=logging.DEBUG)
lib.setup_logger('debug',
logfile='{0:s}{1:s}_triggered_qa.log'.format(
qa_dir, host_name))
logger = logging.getLogger(__name__)
logger.info("#######################")
logger.info("Input parameters:")
logger.info("target={0:s}".format(str(targets)))
logger.info("fluxcals={0:s}".format(str(fluxcals)))
logger.info("polcals={0:s}".format(str(polcals)))
logger.info("#######################")
logger.info('#######################')
logger.info('#### Running all QA steps on {0:s}'.format(host_name))
logger.info('#######################')
# If both fluxcal and polcal polarized, remove polcal
# (taken from start_pipeline)
if subs_calmodels.is_polarised(
name_polcal) and subs_calmodels.is_polarised(name_fluxcal):
name_polcal = ""
if (fluxcals and fluxcals != '') and (polcals and polcals != ''):
assert (len(fluxcals) == len(polcals))
# avoid symmetry bias, if there is only a polcal but no fluxcal, switch them
if fluxcals is None and polcals is not None:
logger.info(
"Only polcal was provided. Setting polcal {} to fluxcal".format(
name_polcal))
fluxcals, polcals = polcals, fluxcals
name_fluxcal, name_polcal = name_polcal, name_fluxcal
name_polcal = ""
# Exchange polcal and fluxcal if specified in the wrong order
# (taken from start_pipeline)
# (except for how the names are switched)
elif not subs_calmodels.is_polarised(name_polcal) and name_polcal != '':
if subs_calmodels.is_polarised(name_fluxcal):
logger.debug("Switching polcal and fluxcal because " +
name_polcal + " is not polarised")
fluxcals, polcals = polcals, fluxcals
name_fluxcal, name_polcal = name_polcal, name_fluxcal
#name_polcal = str(polcals[0][1]).strip()
else:
logger.debug("Setting polcal to '' since " + name_polcal +
" is not polarised")
name_polcal = ""
elif name_polcal != '':
logger.debug("Polcal " + name_polcal + " is polarised, all good")
logger.info(
"## Observation of target: {0:s}, flux calibrator: {1:s}, polarisation calibrator: {2:s}"
.format(name_target, name_fluxcal, name_polcal))
# Write information about the observation into a txt file for later
# This information is important for the OSA report
# =================================================================
# flux calibrator tid list
flux_cal_tid_list = [cal[0] for cal in fluxcals]
# pol calibrator tid list
if name_polcal != '':
pol_cal_tid_list = [cal[0] for cal in polcals]
else:
pol_cal_tid_list = []
summary_table = Table(
[[taskid_target], [name_target], [name_fluxcal],
[str(flux_cal_tid_list).replace("]", "").replace("[", "")],
[name_polcal],
[str(pol_cal_tid_list).replace("]", "").replace("[", "")], [osa]],
names=('Obs_ID', 'Target', 'Flux_Calibrator',
'Flux_Calibrator_Obs_IDs', 'Pol_Calibrator',
'Pol_Calibrator_Obs_IDs', 'OSA'))
table_name = "{0}_obs.ecsv".format(taskid_target)
table_name_with_path = os.path.join(qa_dir, table_name)
try:
summary_table.write(table_name_with_path,
format='ascii.ecsv',
overwrite=True)
except Exception as e:
logger.warning("Saving observation information in {0} failed.".format(
table_name_with_path))
logger.exception(e)
else:
logger.info(("Saving observation information in {0} ... Done.".format(
table_name_with_path)))
# Inspection Plots
# ================
if 'inspection_plots' in steps:
start_time_inspection_plot = time.time()
# for the target it is enough to do it only for happili-01
# as they do not depend on the beam
# for the flux and pol calibrator, they have to be run on every node
# get inspection plots for target
if host_name == "happili-01":
logger.info(
"#### Inspection plot QA for {}...".format(name_target))
try:
inspection_plot_msg = os.system(
'python /home/apercal/dataqa/run_inspection_plot.py {0:d} {1:s} --basedir={2}'
.format(taskid_target, name_target, basedir))
logger.info(
"Getting inspection plots finished with msg {0}".format(
inspection_plot_msg))
logger.info(
"#### Inspection plot QA {0}... Done ".format(name_target))
except Exception as e:
logger.warning(
"Inspection plot QA for {} failed.".format(name_target))
logger.exception(e)
# get inspection plot for flux calibrator
logger.info("#### Inspection plot QA for {}...".format(name_fluxcal))
for (taskid_cal, name_cal, beamnr_cal) in fluxcals:
try:
inspection_plot_msg = os.system(
'python /home/apercal/dataqa/run_inspection_plot.py {0:d} {1:s} -c --beam={2:d} --cal_id={3:d} --basedir={4}'
.format(taskid_target, name_fluxcal, beamnr_cal,
taskid_cal, basedir))
logger.info(
"Getting inspection plots finished with msg {0}".format(
inspection_plot_msg))
logger.info(
"#### Inspection plot QA for {0} beam {1} ... Done".format(
name_fluxcal, beamnr_cal))
except Exception as e:
logger.warning(
"Inspection plot QA for {} beam {1} failed.".format(
name_fluxcal, beamnr_cal))
logger.exception(e)
# get inspection plot for pol calibrator if it exists
if name_polcal != '':
logger.info(
"#### Inspection plot QA for {}...".format(name_polcal))
for (taskid_cal, name_cal, beamnr_cal) in polcals:
try:
inspection_plot_msg = os.system(
'python /home/apercal/dataqa/run_inspection_plot.py {0:d} {1:s} -c --beam={2:d} --cal_id={3:d} --basedir={4}'
.format(taskid_target, name_polcal, beamnr_cal,
taskid_cal, basedir))
logger.info(
"Getting inspection plots finished with msg {0}".
format(inspection_plot_msg))
logger.info(
"#### Inspection plot QA for {0} beam {1} ... Done".
format(name_polcal, beamnr_cal))
except Exception as e:
logger.warning(
"Inspection plot QA for {} beam {1} failed.".format(
name_polcal, beamnr_cal))
logger.exception(e)
logger.info("#### Inspection plot QA ... Done (time {0:.1f}s)".format(
time.time() - start_time_inspection_plot))
else:
logger.warning("#### Did not perform inspection plot QA")
# Beamweights Plots
# =================
if 'beamweights' in steps:
start_time_beamweights = time.time()
# this needs to run on every node
logger.info("#### Beamweights QA for {}...".format(name_fluxcal))
try:
beamweights_msg = os.system(
'python /home/apercal/dataqa/run_beamweights_plots.py {0:d} {1:s} -t 20'
.format(taskid_target, name_fluxcal))
logger.info("Getting Beamweightss finished with msg {0}".format(
beamweights_msg))
except Exception as e:
logger.warning(
"Beamweights QA for {} failed.".format(name_fluxcal))
logger.exception(e)
else:
logger.info("#### Beamweights QA ... Done (time {0:.1f}s)".format(
time.time() - start_time_beamweights))
else:
logger.warning("#### Did not perform Beamweights QA")
# Preflag QA
# ==========
if 'preflag' in steps and host_name == "happili-01":
logger.info("#### Running preflag QA ...")
start_time_preflag = time.time()
try:
preflag_msg = os.system(
'python /home/apercal/dataqa/run_preflag_qa.py {0:d} --basedir={1}'
.format(taskid_target, basedir))
logger.info("Preflag QA finished with msg {0}".format(preflag_msg))
logger.info(
"#### Running preflag QA ... Done (time {0:.1f}s)".format(
time.time() - start_time_preflag))
except Exception as e:
logger.warning("Preflag QA failed. Continue with next QA")
logger.exception(e)
# Disabled rfinder
# try:
# preflag_msg = os.system(
# 'python /home/apercal/dataqa/run_rfinder.py {0:d} {1:s} --trigger_mode'.format(taskid_target, name_fluxcal))
# logger.info(
# "Preflag QA finished with msg {0}".format(preflag_msg))
# logger.info("#### Running preflag QA ... Done (time {0:.1f}s)".format(
# time.time()-start_time_preflag))
# except Exception as e:
# logger.warning("Preflag QA failed. Continue with next QA")
# logger.exception(e)
else:
logger.warning("#### Did not perform preflag QA")
# Crosscal QA
# ===========
if 'crosscal' in steps and name_fluxcal != '':
logger.info('#### Running crosscal QA ...')
start_time_crosscal = time.time()
try:
crosscal_msg = os.system(
'python /home/apercal/dataqa/run_ccal_plots.py {0:d} "{1:s}" "{2:s}" --basedir={3} --trigger_mode'
.format(taskid_target, name_fluxcal, name_polcal, basedir))
logger.info(
"Crosscal QA finished with msg {0}".format(crosscal_msg))
logger.info(
"#### Running crosscal QA ... Done (time {0:.1f}s)".format(
time.time() - start_time_crosscal))
except Exception as e:
logger.warning("Crosscal QA failed. Continue with next QA")
logger.exception(e)
else:
logger.warning("#### Did not perform crosscal QA")
# Selfcal QA
# ==========
if 'selfcal' in steps:
logger.info('#### Running selfcal QA ...')
start_time_selfcal = time.time()
try:
selfcal_msg = os.system(
'python /home/apercal/dataqa/run_scal_plots.py {0:d} {1:s} --basedir={2} --trigger_mode'
.format(taskid_target, name_target, basedir))
logger.info("Selfcal QA finished with msg {0}".format(selfcal_msg))
logger.info(
"#### Running selfcal QA ... Done (time {0:.1f}s)".format(
time.time() - start_time_selfcal))
except Exception as e:
logger.warning("Selfcal QA failed. Continue with next QA")
logger.exception(e)
else:
logger.warning("#### Did not perform selfcal QA")
# Mosaic QA
# ==========
if 'mosaic' in steps and host_name == 'happili-01':
logger.info('#### Mosaic QA is currently not available ...')
# logger.info('#### Running mosaic QA ...')
# start_time_mosaic = time.time()
# try:
# # Create the mosaic
# logger.info('## Making the mosaic ...')
# start_time_make_mosaic = time.time()
# make_mosaic_msg = os.system(
# 'python /home/apercal/dataqa/make_mosaic_image.py {0:d}'.format(taskid_target))
# logger.info(
# "Making mosaic finished with msg {0}".format(make_mosaic_msg))
# logger.info("## Making the mosaic ... Done (time {0:.1f}s)".format(
# time.time()-start_time_make_mosaic))
# # Run the validation tool
# logger.info('## Run validation ...')
# start_time_mosaic_validation = time.time()
# mosaic_validation_msg = os.system(
# 'python /home/apercal/dataqa/run_continuum_validation.py {0:d} --for_mosaic'.format(taskid_target))
# logger.info(
# "Mosaic validation finished with msg {0}".format(mosaic_validation_msg))
# logger.info("## Run validation ... Done (time {0:.1f}s)".format(
# time.time()-start_time_mosaic_validation))
# logger.info("#### Running mosaic QA ... Done (time {0:.1f}s)".format(
# time.time()-start_time_mosaic))
# except Exception as e:
# logger.warning("Mosaic QA failed. Continue with next QA")
# logger.exception(e)
else:
logger.warning("#### Did not perform mosaic QA")
# Line QA
# =======
if 'line' in steps:
logger.info('#### Running line QA ...')
start_time_line = time.time()
try:
# Get cube statistic without continuum subtraction
logger.info('## Get cube statistic ...')
start_time_get_cube_stat = time.time()
cube_stat_msg = os.system(
'python /home/apercal/dataqa/run_cube_stats.py {0:d} --basedir={1} --trigger_mode'
.format(taskid_target, basedir))
logger.info(
"Cube stat finished with msg {0}".format(cube_stat_msg))
logger.info(
"## Get cube statistic ... Done (time {0:.1f}s)".format(
time.time() - start_time_get_cube_stat))
# Subtract continuum
# logger.info('## Subtract continuum ...')
# start_time_subtract_continuum = time.time()
# subtract_cont_msg = os.system(
# 'python /home/apercal/dataqa/subtract_continuum.py {0:d} --trigger_mode'.format(taskid_target))
# logger.info(
# "Continuum subtraction finished with msg {0}".format(subtract_cont_msg))
# logger.info("## Subtract continuum ... Done (time {0:.1f}s)".format(
# time.time()-start_time_subtract_continuum))
# # Get cube statistic after continuum subtraction
# logger.info(
# '## Get cube statistic after continuum subtraction ...')
# start_time_get_cube_stat_cont = time.time()
# get_cube_stat_cont_msg = os.system(
# 'python /home/apercal/dataqa/run_cube_stats_cont.py {0:d} --trigger_mode'.format(taskid_target))
# logger.info(
# "Cube stat cont finished with msg {0}".format(get_cube_stat_cont_msg))
# logger.info("## Get cube statistic after continuum subtraction ... Done (time {0:.1f}s)".format(
# time.time()-start_time_get_cube_stat_cont))
logger.info("#### Running line QA ... Done (time {0:.1f}s)".format(
time.time() - start_time_line))
except Exception as e:
logger.warning("Line QA failed. Continue with next QA")
logger.exception(e)
else:
logger.warning("#### Did not perform line QA")
# Continuum QA
# ============
if 'continuum' in steps:
logger.info('#### Running continuum QA ...')
start_time_continuum = time.time()
try:
continuum_msg = os.system(
'python /home/apercal/dataqa/run_continuum_validation.py {0:d} --basedir={1} --trigger_mode'
.format(taskid_target, basedir))
logger.info(
"Continuum QA finished with msg {0}".format(continuum_msg))
logger.info(
"#### Running continuum QA ... Done (time {0:.1f}s)".format(
time.time() - start_time_continuum))
except Exception as e:
logger.warning("Continuum QA failed. Continue with next QA")
logger.exception(e)
else:
logger.warning("#### Did not perform continuum QA")
# Create report
# =============
if 'report' in steps:
# merge the crosscal and selfcal plots for the report
if host_name == 'happili-01':
logger.info('#### Merge crosscal and selfcal plots...')
start_time_merge = time.time()
try:
report_msg = os.system(
'python /home/apercal/dataqa/run_merge_plots.py {0:d} --basedir={1} --do_ccal --do_scal --run_parallel'
.format(taskid_target, basedir))
logger.info("Merging finished with msg {0}".format(report_msg))
logger.info(
"#### Merge crosscal and selfcal plots ... Done (time {0:.1f}s)"
.format(time.time() - start_time_merge))
except Exception as e:
logger.warning("Merge crosscal and selfcal plots failed.")
logger.exception(e)
# now create the report
logger.info('#### Create report ...')
start_time_report = time.time()
try:
report_msg = os.system(
'python /home/apercal/dataqa/create_report.py {0:d} --basedir={1} --trigger_mode'
.format(taskid_target, basedir))
logger.info("Report finished with msg {0}".format(report_msg))
logger.info("#### Create report ... Done (time {0:.1f}s)".format(
time.time() - start_time_report))
except Exception as e:
logger.warning("Creating report failed.")
logger.exception(e)
else:
logger.warning("#### Did not create a report")
# Finish
# ======
logger.info('#######################')
logger.info(
'#### Running all QA steps on {0:s} ... Done (time {1:.1f}s)'.format(
host_name,
time.time() - start_time))
logger.info('#######################')