def go(self):
super(imager_create_dbs, self).go()
# get assoc_theta, convert from empty string if needed
assoc_theta = self.inputs["assoc_theta"]
if assoc_theta == "":
assoc_theta = None
# Load mapfile data from files
self.logger.info(self.inputs["slice_paths_mapfile"])
slice_paths_map = MultiDataMap.load(self.inputs["slice_paths_mapfile"])
input_map = DataMap.load(self.inputs['args'][0])
source_list_map = DataMap.load(self.inputs['source_list_map_path'])
if self._validate_input_data(input_map, slice_paths_map):
return 1
# Run the nodes with now collected inputs
jobs, output_map = self._run_create_dbs_node(
input_map, slice_paths_map, assoc_theta,
source_list_map)
# Collect the output of the node scripts write to (map) files
return self._collect_and_assign_outputs(jobs, output_map,
slice_paths_map)
def go(self):
super(imager_create_dbs, self).go()
# get assoc_theta, convert from empty string if needed
assoc_theta = self.inputs["assoc_theta"]
if assoc_theta == "":
assoc_theta = None
# Load mapfile data from files
self.logger.info(self.inputs["slice_paths_mapfile"])
slice_paths_map = MultiDataMap.load(self.inputs["slice_paths_mapfile"])
input_map = DataMap.load(self.inputs['args'][0])
source_list_map = DataMap.load(self.inputs['source_list_map_path'])
if self._validate_input_data(input_map, slice_paths_map):
return 1
# Run the nodes with now collected inputs
jobs, output_map = self._run_create_dbs_node(
input_map, slice_paths_map, assoc_theta,
source_list_map)
# Collect the output of the node scripts write to (map) files
return self._collect_and_assign_outputs(jobs, output_map,
slice_paths_map)
def run(self, bbs_executable, parset, ms_list_path, parmdb_list_path,
sky_list_path):
"""
imager_bbs functionality. Called by framework performing all the work
"""
self.logger.debug("Starting imager_bbs Node")
# *********************************************************************
# 1. Load mapfiles
# read in the mapfiles to data maps: The master recipe added the single
# path to a mapfilem which allows usage of default data methods
# (load_data_map)
# TODO: Datamap
ms_map = MultiDataMap.load(ms_list_path)
parmdb_map = MultiDataMap.load(parmdb_list_path)
sky_list = MultiDataMap.load(sky_list_path)
source_db = sky_list[0].file[0] # the sourcedb is the first file entry
try:
bbs_process_group = SubProcessGroup(self.logger,
self.resourceMonitor)
# *****************************************************************
# 2. start the bbs executable with data
for (measurement_set, parmdm) in zip(ms_map[0].file,
parmdb_map[0].file):
command = [
bbs_executable,
"--sourcedb={0}".format(source_db),
"--parmdb={0}".format(parmdm) ,
measurement_set,
parset]
self.logger.info("Executing bbs command: {0}".format(" ".join(
command)))
bbs_process_group.run(command)
# *****************************************************************
# 3. check status of the processes
if bbs_process_group.wait_for_finish() != None:
self.logger.error(
"Failed bbs run detected Aborting")
return 1 # If bbs failed we need to abort: the concat
# is now corrupt
except OSError, exception:
self.logger.error("Failed to execute bbs: {0}".format(str(
exception)))
return 1
def run(self, bbs_executable, parset, ms_list_path, parmdb_list_path,
sky_list_path):
"""
imager_bbs functionality. Called by framework performing all the work
"""
self.logger.debug("Starting imager_bbs Node")
# *********************************************************************
# 1. Load mapfiles
# read in the mapfiles to data maps: The master recipe added the single
# path to a mapfilem which allows usage of default data methods
# (load_data_map)
# TODO: Datamap
ms_map = MultiDataMap.load(ms_list_path)
parmdb_map = MultiDataMap.load(parmdb_list_path)
sky_list = MultiDataMap.load(sky_list_path)
source_db = sky_list[0].file[0] # the sourcedb is the first file entry
try:
bbs_process_group = SubProcessGroup(self.logger,
self.resourceMonitor)
# *****************************************************************
# 2. start the bbs executable with data
for (measurement_set, parmdm) in zip(ms_map[0].file,
parmdb_map[0].file):
command = [
bbs_executable, "--sourcedb={0}".format(source_db),
"--parmdb={0}".format(parmdm), measurement_set, parset
]
self.logger.info("Executing bbs command: {0}".format(
" ".join(command)))
bbs_process_group.run(command)
# *****************************************************************
# 3. check status of the processes
if bbs_process_group.wait_for_finish() != None:
self.logger.error("Failed bbs run detected Aborting")
return 1 # If bbs failed we need to abort: the concat
# is now corrupt
except OSError as exception:
self.logger.error("Failed to execute bbs: {0}".format(
str(exception)))
return 1
return 0
def _bbs(self,
timeslice_map_path,
parmdbs_map_path,
sourcedb_map_path,
skip=False):
"""
Perform a calibration step. First with a set of sources from the
gsm and in later iterations also on the found sources
"""
# create parset for bbs run
parset = self.parset.makeSubset("BBS.")
parset_path = self._write_parset_to_file(
parset, "bbs", "Parset for calibration with a local sky model")
# create the output file path
output_mapfile = self._write_datamap_to_file(
None, "bbs_output", "Mapfile with calibrated measurement sets.")
converted_sourcedb_map_path = self._write_datamap_to_file(
None, "source_db", "correctly shaped mapfile for input sourcedbs")
if skip:
return output_mapfile
# The create db step produces a mapfile with a single sourcelist for
# the different timeslices. Generate a mapfile with copies of the
# sourcelist location: This allows validation of maps in combination
# get the original map data
sourcedb_map = DataMap.load(sourcedb_map_path)
parmdbs_map = MultiDataMap.load(parmdbs_map_path)
converted_sourcedb_map = []
# sanity check for correcy output from previous recipes
if not validate_data_maps(sourcedb_map, parmdbs_map):
self.logger.error("The input files for bbs do not contain "
"matching host names for each entry content:")
self.logger.error(repr(sourcedb_map))
self.logger.error(repr(parmdbs_map))
raise PipelineException("Invalid input data for imager_bbs recipe")
self.run_task("imager_bbs",
timeslice_map_path,
parset=parset_path,
instrument_mapfile=parmdbs_map_path,
sourcedb_mapfile=sourcedb_map_path,
mapfile=output_mapfile,
working_directory=self.scratch_directory)
return output_mapfile
def _bbs(self, timeslice_map_path, parmdbs_map_path, sourcedb_map_path,
skip = False):
"""
Perform a calibration step. First with a set of sources from the
gsm and in later iterations also on the found sources
"""
# create parset for bbs run
parset = self.parset.makeSubset("BBS.")
parset_path = self._write_parset_to_file(parset, "bbs",
"Parset for calibration with a local sky model")
# create the output file path
output_mapfile = self._write_datamap_to_file(None, "bbs_output",
"Mapfile with calibrated measurement sets.")
converted_sourcedb_map_path = self._write_datamap_to_file(None,
"source_db", "correctly shaped mapfile for input sourcedbs")
if skip:
return output_mapfile
# The create db step produces a mapfile with a single sourcelist for
# the different timeslices. Generate a mapfile with copies of the
# sourcelist location: This allows validation of maps in combination
# get the original map data
sourcedb_map = DataMap.load(sourcedb_map_path)
parmdbs_map = MultiDataMap.load(parmdbs_map_path)
converted_sourcedb_map = []
# sanity check for correcy output from previous recipes
if not validate_data_maps(sourcedb_map, parmdbs_map):
self.logger.error("The input files for bbs do not contain "
"matching host names for each entry content:")
self.logger.error(repr(sourcedb_map))
self.logger.error(repr(parmdbs_map))
raise PipelineException("Invalid input data for imager_bbs recipe")
self.run_task("imager_bbs",
timeslice_map_path,
parset = parset_path,
instrument_mapfile = parmdbs_map_path,
sourcedb_mapfile = sourcedb_map_path,
mapfile = output_mapfile,
working_directory = self.scratch_directory)
return output_mapfile
def plugin_main(args, **kwargs):
#print 'PLUGIN KWARG: ', kwargs
result = {}
datamap = None
fileid = kwargs['mapfile_in']
fileid2 = kwargs['mapfile_ref']
datamap = MultiDataMap.load(fileid)
datamap2 = DataMap.load(fileid2)
newmap = []
for item in datamap2:
entry = {}
entry['host'] = item.host
entry['file'] = datamap.data[0].file
entry['skip'] = item.skip
newmap.append(entry)
outfileid = os.path.join(kwargs['mapfile_dir'], kwargs['filename'])
outmap = open(outfileid, 'w')
outmap.write(repr(newmap))
outmap.close()
result['mapfile'] = outfileid
return result
def go(self):
"""
imager_bbs functionality. Called by framework performing all the work
"""
super(imager_bbs, self).go()
self.logger.info("Starting imager_bbs run")
# ********************************************************************
# 1. Load the and validate the data
ms_map = MultiDataMap.load(self.inputs['args'][0])
parmdb_map = MultiDataMap.load(self.inputs['instrument_mapfile'])
sourcedb_map = DataMap.load(self.inputs['sourcedb_mapfile'])
# TODO: DataMap extention
# #Check if the input has equal length and on the same nodes
# if not validate_data_maps(ms_map, parmdb_map):
# self.logger.error("The combination of mapfiles failed validation:")
# self.logger.error("ms_map: \n{0}".format(ms_map))
# self.logger.error("parmdb_map: \n{0}".format(parmdb_map))
# return 1
# *********************************************************************
# 2. Start the node scripts
jobs = []
node_command = " python %s" % (self.__file__.replace("master", "nodes"))
map_dir = os.path.join(
self.config.get("layout", "job_directory"), "mapfiles")
run_id = str(self.inputs.get("id"))
# Update the skip fields of the four maps. If 'skip' is True in any of
# these maps, then 'skip' must be set to True in all maps.
for w, x, y in zip(ms_map, parmdb_map, sourcedb_map):
w.skip = x.skip = y.skip = (
w.skip or x.skip or y.skip
)
ms_map.iterator = parmdb_map.iterator = sourcedb_map.iterator = \
DataMap.SkipIterator
for (ms, parmdb, sourcedb) in zip(ms_map, parmdb_map, sourcedb_map):
#host is same for each entry (validate_data_maps)
host, ms_list = ms.host, ms.file
# Write data maps to MultaDataMaps
ms_list_path = os.path.join(
map_dir, host + "_ms_" + run_id + ".map")
MultiDataMap([tuple([host, ms_list, False])]).save(ms_list_path)
parmdb_list_path = os.path.join(
map_dir, host + "_parmdb_" + run_id + ".map")
MultiDataMap(
[tuple([host, parmdb.file, False])]).save(parmdb_list_path)
sourcedb_list_path = os.path.join(
map_dir, host + "_sky_" + run_id + ".map")
MultiDataMap(
[tuple([host, [sourcedb.file], False])]).save(sourcedb_list_path)
arguments = [self.inputs['bbs_executable'],
self.inputs['parset'],
ms_list_path, parmdb_list_path, sourcedb_list_path]
jobs.append(ComputeJob(host, node_command, arguments))
# start and wait till all are finished
self._schedule_jobs(jobs)
# **********************************************************************
# 3. validate the node output and construct the output mapfile.
if self.error.isSet(): #if one of the nodes failed
self.logger.error("One of the nodes failed while performing"
"a BBS run. Aborting: concat.ms corruption")
return 1
# return the output: The measurement set that are calibrated:
# calibrated data is placed in the ms sets
MultiDataMap(ms_map).save(self.inputs['mapfile'])
self.logger.info("Wrote file with calibrated data")
self.outputs['mapfile'] = self.inputs['mapfile']
return 0
def test_new_style_load_store(self):
tmp_file = self.new_style_map_file + '.tmp'
data_map = MultiDataMap(self.new_style_map)
data_map.save(tmp_file)
reloaded_data_map = MultiDataMap.load(tmp_file)
self.assertEqual(data_map, reloaded_data_map)
def go(self):
"""
imager_bbs functionality. Called by framework performing all the work
"""
super(imager_bbs, self).go()
self.logger.info("Starting imager_bbs run")
# ********************************************************************
# 1. Load the and validate the data
ms_map = MultiDataMap.load(self.inputs['args'][0])
parmdb_map = MultiDataMap.load(self.inputs['instrument_mapfile'])
sourcedb_map = DataMap.load(self.inputs['sourcedb_mapfile'])
# TODO: DataMap extention
# #Check if the input has equal length and on the same nodes
# if not validate_data_maps(ms_map, parmdb_map):
# self.logger.error("The combination of mapfiles failed validation:")
# self.logger.error("ms_map: \n{0}".format(ms_map))
# self.logger.error("parmdb_map: \n{0}".format(parmdb_map))
# return 1
# *********************************************************************
# 2. Start the node scripts
jobs = []
node_command = " python3 %s" % (self.__file__.replace("master", "nodes"))
map_dir = os.path.join(
self.config.get("layout", "job_directory"), "mapfiles")
run_id = str(self.inputs.get("id"))
# Update the skip fields of the four maps. If 'skip' is True in any of
# these maps, then 'skip' must be set to True in all maps.
for w, x, y in zip(ms_map, parmdb_map, sourcedb_map):
w.skip = x.skip = y.skip = (
w.skip or x.skip or y.skip
)
ms_map.iterator = parmdb_map.iterator = sourcedb_map.iterator = \
DataMap.SkipIterator
for (idx, (ms, parmdb, sourcedb)) in enumerate(zip(ms_map, parmdb_map, sourcedb_map)):
# host is same for each entry (validate_data_maps)
host, ms_list = ms.host, ms.file
# Write data maps to MultaDataMaps
ms_list_path = os.path.join(
map_dir, "%s-%s_map_%s.map" % (host, idx, run_id))
MultiDataMap([tuple([host, ms_list, False])]).save(ms_list_path)
parmdb_list_path = os.path.join(
map_dir, "%s-%s_parmdb_%s.map" % (host, idx, run_id))
MultiDataMap(
[tuple([host, parmdb.file, False])]).save(parmdb_list_path)
sourcedb_list_path = os.path.join(
map_dir, "%s-%s_sky_%s.map" % (host, idx, run_id))
MultiDataMap(
[tuple([host, [sourcedb.file], False])]).save(sourcedb_list_path)
arguments = [self.inputs['bbs_executable'],
self.inputs['parset'],
ms_list_path, parmdb_list_path, sourcedb_list_path]
jobs.append(ComputeJob(host, node_command, arguments,
resources = {
"cores": self.inputs['nthreads']
}))
# start and wait till all are finished
self._schedule_jobs(jobs)
# **********************************************************************
# 3. validate the node output and construct the output mapfile.
if self.error.isSet(): # if one of the nodes failed
self.logger.error("One of the nodes failed while performing"
"a BBS run. Aborting: concat.ms corruption")
return 1
# return the output: The measurement set that are calibrated:
# calibrated data is placed in the ms sets
MultiDataMap(ms_map).save(self.inputs['mapfile'])
self.logger.info("Wrote file with calibrated data")
self.outputs['mapfile'] = self.inputs['mapfile']
return 0
def go(self):
"""
imager_bbs functionality. Called by framework performing all the work
"""
super(selfcal_bbs, self).go()
self.logger.info("Starting imager_bbs run")
# ********************************************************************
# 1. Load the and validate the data
ms_map = MultiDataMap.load(self.inputs['args'][0])
parmdb_map = MultiDataMap.load(self.inputs['instrument_mapfile'])
sourcedb_map = DataMap.load(self.inputs['sourcedb_mapfile'])
concat_ms_map = DataMap.load(self.inputs['concat_ms_map_path'])
# *********************************************************************
# 2. Start the node scripts
jobs = []
node_command = " python %s" % (self.__file__.replace(
"master", "nodes"))
map_dir = os.path.join(self.config.get("layout", "job_directory"),
"mapfiles")
run_id = str(self.inputs.get("id"))
# Update the skip fields of the four maps. If 'skip' is True in any of
# these maps, then 'skip' must be set to True in all maps.
align_data_maps(ms_map, parmdb_map, sourcedb_map, concat_ms_map)
ms_map.iterator = parmdb_map.iterator = sourcedb_map.iterator = \
concat_ms_map.iterator = DataMap.SkipIterator
# *********************************************************************
for (ms, parmdb, sourcedb, concat_ms) in zip(ms_map, parmdb_map,
sourcedb_map,
concat_ms_map):
#host is same for each entry (validate_data_maps)
host, ms_list = ms.host, ms.file
# Write data maps to MultaDataMaps
ms_list_path = os.path.join(map_dir,
host + "_ms_" + run_id + ".map")
MultiDataMap([tuple([host, ms_list, False])]).save(ms_list_path)
parmdb_list_path = os.path.join(
map_dir, host + "_parmdb_" + run_id + ".map")
MultiDataMap([tuple([host, parmdb.file,
False])]).save(parmdb_list_path)
sourcedb_list_path = os.path.join(map_dir,
host + "_sky_" + run_id + ".map")
MultiDataMap([tuple([host, [sourcedb.file],
False])]).save(sourcedb_list_path)
# THe concat ms does not have to be written: It already is a
# singular item (it is the output of the reduce step)
# redmine issue #6021
arguments = [
self.inputs['bbs_executable'], self.inputs['parset'],
ms_list_path, parmdb_list_path, sourcedb_list_path,
concat_ms.file, self.inputs['major_cycle']
]
jobs.append(ComputeJob(host, node_command, arguments))
# start and wait till all are finished
self._schedule_jobs(jobs)
# **********************************************************************
# 3. validate the node output and construct the output mapfile.
if self.error.isSet(): #if one of the nodes failed
self.logger.warn("Failed bbs node run detected, skipping work"
"on this work item for further computations")
# find failed job and set the skip field
for (ms_item, concat_item, job) in zip(ms_map, concat_ms_map, jobs):
if job.results["returncode"] == 0:
continue
else:
ms_item.skip = True
concat_item.skip = True
self.logger.warn("bbs failed on item: {0}".format(
ms_item.file))
# return the output: The measurement set that are calibrated:
# calibrated data is placed in the ms sets
MultiDataMap(ms_map).save(self.inputs['mapfile'])
# also save the concat_ms map with possible skips
DataMap(concat_ms_map).save(self.inputs['concat_ms_map_path'])
self.logger.info("Wrote file with calibrated data")
self.outputs['mapfile'] = self.inputs['mapfile']
return 0
def test_new_style_load_store(self):
tmp_file = self.new_style_map_file + '.tmp'
data_map = MultiDataMap(self.new_style_map)
data_map.save(tmp_file)
reloaded_data_map = MultiDataMap.load(tmp_file)
self.assertEqual(data_map, reloaded_data_map)
def run(self, bbs_executable, parset, ms_list_path, parmdb_list_path,
sky_list_path, concat_ms_path, major_cycle):
"""
selfcal_bbs functionality. Called by framework performing all the work
"""
self.logger.debug("Starting selfcal_bbs Node")
# *********************************************************************
# 1. Load mapfiles
# read in the mapfiles to data maps: The master recipe added the single
# path to a mapfilem which allows usage of default data methods
# (load_data_map)
# TODO: Datamap
ms_map = MultiDataMap.load(ms_list_path)
parmdb_map = MultiDataMap.load(parmdb_list_path)
sky_list = MultiDataMap.load(sky_list_path)
source_db = sky_list[0].file[0] # the sourcedb is the first file entry
try:
bbs_process_group = SubProcessGroup(self.logger,
self.resourceMonitor)
# *****************************************************************
# 2. start the bbs executable with data
# The data is located in multimaps. We need the first entry
# TODO: THis is not 'nice' usage of the multimap
for (measurement_set, parmdm) in zip(ms_map[0].file,
parmdb_map[0].file):
command = [
bbs_executable, "--sourcedb={0}".format(source_db),
"--parmdb={0}".format(parmdm), measurement_set, parset
]
self.logger.info("Executing bbs command: {0}".format(
" ".join(command)))
bbs_process_group.run(command)
# *****************************************************************
# 3. check status of the processes
if bbs_process_group.wait_for_finish() != None:
self.logger.error("Failed bbs run detected Aborting")
return 1
except OSError as exception:
self.logger.error("Failed to execute bbs: {0}".format(
str(exception)))
return 1
# *********************************************************************
# 4. Concat in time after bbs calibration your MSs using
# msconcat (pyrap.tables module) (added by N.Vilchez)
# this step has te be performed on this location. because the bbs run
# might add additional columns not present in the original ms
# and therefore not produced in the concat done in the prepare phase
# redmine issue #6021
pt.msconcat(ms_map[0].file, concat_ms_path, concatTime=True)
# *********************************************************************
# 5. copy time slives directory to a new one
# This is done for debugging purpose: The copy is not used for anything
# The actual selfcal steps are done in place
# (added by N.Vilchez)
# THe save location is created relative to the concat.ms
# we could also use the self.scratch_directory from the toplevel recipe
# this would need an aditional ingredient
# This is a 'debugging' step and should never ever cause a failure of \
# the pipeline
try:
working_dir = os.path.dirname(concat_ms_path)
time_slice_dir = os.path.join(working_dir, 'time_slices')
time_slice_copy_dir = os.path.join(
working_dir, 'time_slices_cycle_{0}'.format(major_cycle))
cmd = "cp -r {0} {1}".format(time_slice_dir, time_slice_copy_dir)
os.system(cmd)
except:
self.logger.warn(
"Debug copy of temporary files failed: continue operations")
pass # Do nothing
return 0
