def go(self): super(imager_create_dbs, self).go() # get assoc_theta, convert from empty string if needed assoc_theta = self.inputs["assoc_theta"] if assoc_theta == "": assoc_theta = None # Load mapfile data from files self.logger.info(self.inputs["slice_paths_mapfile"]) slice_paths_map = MultiDataMap.load(self.inputs["slice_paths_mapfile"]) input_map = DataMap.load(self.inputs['args'][0]) source_list_map = DataMap.load(self.inputs['source_list_map_path']) if self._validate_input_data(input_map, slice_paths_map): return 1 # Run the nodes with now collected inputs jobs, output_map = self._run_create_dbs_node( input_map, slice_paths_map, assoc_theta, source_list_map) # Collect the output of the node scripts write to (map) files return self._collect_and_assign_outputs(jobs, output_map, slice_paths_map)
def run(self, bbs_executable, parset, ms_list_path, parmdb_list_path, sky_list_path): """ imager_bbs functionality. Called by framework performing all the work """ self.logger.debug("Starting imager_bbs Node") # ********************************************************************* # 1. Load mapfiles # read in the mapfiles to data maps: The master recipe added the single # path to a mapfilem which allows usage of default data methods # (load_data_map) # TODO: Datamap ms_map = MultiDataMap.load(ms_list_path) parmdb_map = MultiDataMap.load(parmdb_list_path) sky_list = MultiDataMap.load(sky_list_path) source_db = sky_list[0].file[0] # the sourcedb is the first file entry try: bbs_process_group = SubProcessGroup(self.logger, self.resourceMonitor) # ***************************************************************** # 2. start the bbs executable with data for (measurement_set, parmdm) in zip(ms_map[0].file, parmdb_map[0].file): command = [ bbs_executable, "--sourcedb={0}".format(source_db), "--parmdb={0}".format(parmdm) , measurement_set, parset] self.logger.info("Executing bbs command: {0}".format(" ".join( command))) bbs_process_group.run(command) # ***************************************************************** # 3. check status of the processes if bbs_process_group.wait_for_finish() != None: self.logger.error( "Failed bbs run detected Aborting") return 1 # If bbs failed we need to abort: the concat # is now corrupt except OSError, exception: self.logger.error("Failed to execute bbs: {0}".format(str( exception))) return 1
def run(self, bbs_executable, parset, ms_list_path, parmdb_list_path, sky_list_path): """ imager_bbs functionality. Called by framework performing all the work """ self.logger.debug("Starting imager_bbs Node") # ********************************************************************* # 1. Load mapfiles # read in the mapfiles to data maps: The master recipe added the single # path to a mapfilem which allows usage of default data methods # (load_data_map) # TODO: Datamap ms_map = MultiDataMap.load(ms_list_path) parmdb_map = MultiDataMap.load(parmdb_list_path) sky_list = MultiDataMap.load(sky_list_path) source_db = sky_list[0].file[0] # the sourcedb is the first file entry try: bbs_process_group = SubProcessGroup(self.logger, self.resourceMonitor) # ***************************************************************** # 2. start the bbs executable with data for (measurement_set, parmdm) in zip(ms_map[0].file, parmdb_map[0].file): command = [ bbs_executable, "--sourcedb={0}".format(source_db), "--parmdb={0}".format(parmdm), measurement_set, parset ] self.logger.info("Executing bbs command: {0}".format( " ".join(command))) bbs_process_group.run(command) # ***************************************************************** # 3. check status of the processes if bbs_process_group.wait_for_finish() != None: self.logger.error("Failed bbs run detected Aborting") return 1 # If bbs failed we need to abort: the concat # is now corrupt except OSError as exception: self.logger.error("Failed to execute bbs: {0}".format( str(exception))) return 1 return 0
def _bbs(self, timeslice_map_path, parmdbs_map_path, sourcedb_map_path, skip=False): """ Perform a calibration step. First with a set of sources from the gsm and in later iterations also on the found sources """ # create parset for bbs run parset = self.parset.makeSubset("BBS.") parset_path = self._write_parset_to_file( parset, "bbs", "Parset for calibration with a local sky model") # create the output file path output_mapfile = self._write_datamap_to_file( None, "bbs_output", "Mapfile with calibrated measurement sets.") converted_sourcedb_map_path = self._write_datamap_to_file( None, "source_db", "correctly shaped mapfile for input sourcedbs") if skip: return output_mapfile # The create db step produces a mapfile with a single sourcelist for # the different timeslices. Generate a mapfile with copies of the # sourcelist location: This allows validation of maps in combination # get the original map data sourcedb_map = DataMap.load(sourcedb_map_path) parmdbs_map = MultiDataMap.load(parmdbs_map_path) converted_sourcedb_map = [] # sanity check for correcy output from previous recipes if not validate_data_maps(sourcedb_map, parmdbs_map): self.logger.error("The input files for bbs do not contain " "matching host names for each entry content:") self.logger.error(repr(sourcedb_map)) self.logger.error(repr(parmdbs_map)) raise PipelineException("Invalid input data for imager_bbs recipe") self.run_task("imager_bbs", timeslice_map_path, parset=parset_path, instrument_mapfile=parmdbs_map_path, sourcedb_mapfile=sourcedb_map_path, mapfile=output_mapfile, working_directory=self.scratch_directory) return output_mapfile
def _bbs(self, timeslice_map_path, parmdbs_map_path, sourcedb_map_path, skip = False): """ Perform a calibration step. First with a set of sources from the gsm and in later iterations also on the found sources """ # create parset for bbs run parset = self.parset.makeSubset("BBS.") parset_path = self._write_parset_to_file(parset, "bbs", "Parset for calibration with a local sky model") # create the output file path output_mapfile = self._write_datamap_to_file(None, "bbs_output", "Mapfile with calibrated measurement sets.") converted_sourcedb_map_path = self._write_datamap_to_file(None, "source_db", "correctly shaped mapfile for input sourcedbs") if skip: return output_mapfile # The create db step produces a mapfile with a single sourcelist for # the different timeslices. Generate a mapfile with copies of the # sourcelist location: This allows validation of maps in combination # get the original map data sourcedb_map = DataMap.load(sourcedb_map_path) parmdbs_map = MultiDataMap.load(parmdbs_map_path) converted_sourcedb_map = [] # sanity check for correcy output from previous recipes if not validate_data_maps(sourcedb_map, parmdbs_map): self.logger.error("The input files for bbs do not contain " "matching host names for each entry content:") self.logger.error(repr(sourcedb_map)) self.logger.error(repr(parmdbs_map)) raise PipelineException("Invalid input data for imager_bbs recipe") self.run_task("imager_bbs", timeslice_map_path, parset = parset_path, instrument_mapfile = parmdbs_map_path, sourcedb_mapfile = sourcedb_map_path, mapfile = output_mapfile, working_directory = self.scratch_directory) return output_mapfile
def plugin_main(args, **kwargs): #print 'PLUGIN KWARG: ', kwargs result = {} datamap = None fileid = kwargs['mapfile_in'] fileid2 = kwargs['mapfile_ref'] datamap = MultiDataMap.load(fileid) datamap2 = DataMap.load(fileid2) newmap = [] for item in datamap2: entry = {} entry['host'] = item.host entry['file'] = datamap.data[0].file entry['skip'] = item.skip newmap.append(entry) outfileid = os.path.join(kwargs['mapfile_dir'], kwargs['filename']) outmap = open(outfileid, 'w') outmap.write(repr(newmap)) outmap.close() result['mapfile'] = outfileid return result
def go(self): """ imager_bbs functionality. Called by framework performing all the work """ super(imager_bbs, self).go() self.logger.info("Starting imager_bbs run") # ******************************************************************** # 1. Load the and validate the data ms_map = MultiDataMap.load(self.inputs['args'][0]) parmdb_map = MultiDataMap.load(self.inputs['instrument_mapfile']) sourcedb_map = DataMap.load(self.inputs['sourcedb_mapfile']) # TODO: DataMap extention # #Check if the input has equal length and on the same nodes # if not validate_data_maps(ms_map, parmdb_map): # self.logger.error("The combination of mapfiles failed validation:") # self.logger.error("ms_map: \n{0}".format(ms_map)) # self.logger.error("parmdb_map: \n{0}".format(parmdb_map)) # return 1 # ********************************************************************* # 2. Start the node scripts jobs = [] node_command = " python %s" % (self.__file__.replace("master", "nodes")) map_dir = os.path.join( self.config.get("layout", "job_directory"), "mapfiles") run_id = str(self.inputs.get("id")) # Update the skip fields of the four maps. If 'skip' is True in any of # these maps, then 'skip' must be set to True in all maps. for w, x, y in zip(ms_map, parmdb_map, sourcedb_map): w.skip = x.skip = y.skip = ( w.skip or x.skip or y.skip ) ms_map.iterator = parmdb_map.iterator = sourcedb_map.iterator = \ DataMap.SkipIterator for (ms, parmdb, sourcedb) in zip(ms_map, parmdb_map, sourcedb_map): #host is same for each entry (validate_data_maps) host, ms_list = ms.host, ms.file # Write data maps to MultaDataMaps ms_list_path = os.path.join( map_dir, host + "_ms_" + run_id + ".map") MultiDataMap([tuple([host, ms_list, False])]).save(ms_list_path) parmdb_list_path = os.path.join( map_dir, host + "_parmdb_" + run_id + ".map") MultiDataMap( [tuple([host, parmdb.file, False])]).save(parmdb_list_path) sourcedb_list_path = os.path.join( map_dir, host + "_sky_" + run_id + ".map") MultiDataMap( [tuple([host, [sourcedb.file], False])]).save(sourcedb_list_path) arguments = [self.inputs['bbs_executable'], self.inputs['parset'], ms_list_path, parmdb_list_path, sourcedb_list_path] jobs.append(ComputeJob(host, node_command, arguments)) # start and wait till all are finished self._schedule_jobs(jobs) # ********************************************************************** # 3. validate the node output and construct the output mapfile. if self.error.isSet(): #if one of the nodes failed self.logger.error("One of the nodes failed while performing" "a BBS run. Aborting: concat.ms corruption") return 1 # return the output: The measurement set that are calibrated: # calibrated data is placed in the ms sets MultiDataMap(ms_map).save(self.inputs['mapfile']) self.logger.info("Wrote file with calibrated data") self.outputs['mapfile'] = self.inputs['mapfile'] return 0
def test_new_style_load_store(self): tmp_file = self.new_style_map_file + '.tmp' data_map = MultiDataMap(self.new_style_map) data_map.save(tmp_file) reloaded_data_map = MultiDataMap.load(tmp_file) self.assertEqual(data_map, reloaded_data_map)
def go(self): """ imager_bbs functionality. Called by framework performing all the work """ super(imager_bbs, self).go() self.logger.info("Starting imager_bbs run") # ******************************************************************** # 1. Load the and validate the data ms_map = MultiDataMap.load(self.inputs['args'][0]) parmdb_map = MultiDataMap.load(self.inputs['instrument_mapfile']) sourcedb_map = DataMap.load(self.inputs['sourcedb_mapfile']) # TODO: DataMap extention # #Check if the input has equal length and on the same nodes # if not validate_data_maps(ms_map, parmdb_map): # self.logger.error("The combination of mapfiles failed validation:") # self.logger.error("ms_map: \n{0}".format(ms_map)) # self.logger.error("parmdb_map: \n{0}".format(parmdb_map)) # return 1 # ********************************************************************* # 2. Start the node scripts jobs = [] node_command = " python3 %s" % (self.__file__.replace("master", "nodes")) map_dir = os.path.join( self.config.get("layout", "job_directory"), "mapfiles") run_id = str(self.inputs.get("id")) # Update the skip fields of the four maps. If 'skip' is True in any of # these maps, then 'skip' must be set to True in all maps. for w, x, y in zip(ms_map, parmdb_map, sourcedb_map): w.skip = x.skip = y.skip = ( w.skip or x.skip or y.skip ) ms_map.iterator = parmdb_map.iterator = sourcedb_map.iterator = \ DataMap.SkipIterator for (idx, (ms, parmdb, sourcedb)) in enumerate(zip(ms_map, parmdb_map, sourcedb_map)): # host is same for each entry (validate_data_maps) host, ms_list = ms.host, ms.file # Write data maps to MultaDataMaps ms_list_path = os.path.join( map_dir, "%s-%s_map_%s.map" % (host, idx, run_id)) MultiDataMap([tuple([host, ms_list, False])]).save(ms_list_path) parmdb_list_path = os.path.join( map_dir, "%s-%s_parmdb_%s.map" % (host, idx, run_id)) MultiDataMap( [tuple([host, parmdb.file, False])]).save(parmdb_list_path) sourcedb_list_path = os.path.join( map_dir, "%s-%s_sky_%s.map" % (host, idx, run_id)) MultiDataMap( [tuple([host, [sourcedb.file], False])]).save(sourcedb_list_path) arguments = [self.inputs['bbs_executable'], self.inputs['parset'], ms_list_path, parmdb_list_path, sourcedb_list_path] jobs.append(ComputeJob(host, node_command, arguments, resources = { "cores": self.inputs['nthreads'] })) # start and wait till all are finished self._schedule_jobs(jobs) # ********************************************************************** # 3. validate the node output and construct the output mapfile. if self.error.isSet(): # if one of the nodes failed self.logger.error("One of the nodes failed while performing" "a BBS run. Aborting: concat.ms corruption") return 1 # return the output: The measurement set that are calibrated: # calibrated data is placed in the ms sets MultiDataMap(ms_map).save(self.inputs['mapfile']) self.logger.info("Wrote file with calibrated data") self.outputs['mapfile'] = self.inputs['mapfile'] return 0
def go(self): """ imager_bbs functionality. Called by framework performing all the work """ super(selfcal_bbs, self).go() self.logger.info("Starting imager_bbs run") # ******************************************************************** # 1. Load the and validate the data ms_map = MultiDataMap.load(self.inputs['args'][0]) parmdb_map = MultiDataMap.load(self.inputs['instrument_mapfile']) sourcedb_map = DataMap.load(self.inputs['sourcedb_mapfile']) concat_ms_map = DataMap.load(self.inputs['concat_ms_map_path']) # ********************************************************************* # 2. Start the node scripts jobs = [] node_command = " python %s" % (self.__file__.replace( "master", "nodes")) map_dir = os.path.join(self.config.get("layout", "job_directory"), "mapfiles") run_id = str(self.inputs.get("id")) # Update the skip fields of the four maps. If 'skip' is True in any of # these maps, then 'skip' must be set to True in all maps. align_data_maps(ms_map, parmdb_map, sourcedb_map, concat_ms_map) ms_map.iterator = parmdb_map.iterator = sourcedb_map.iterator = \ concat_ms_map.iterator = DataMap.SkipIterator # ********************************************************************* for (ms, parmdb, sourcedb, concat_ms) in zip(ms_map, parmdb_map, sourcedb_map, concat_ms_map): #host is same for each entry (validate_data_maps) host, ms_list = ms.host, ms.file # Write data maps to MultaDataMaps ms_list_path = os.path.join(map_dir, host + "_ms_" + run_id + ".map") MultiDataMap([tuple([host, ms_list, False])]).save(ms_list_path) parmdb_list_path = os.path.join( map_dir, host + "_parmdb_" + run_id + ".map") MultiDataMap([tuple([host, parmdb.file, False])]).save(parmdb_list_path) sourcedb_list_path = os.path.join(map_dir, host + "_sky_" + run_id + ".map") MultiDataMap([tuple([host, [sourcedb.file], False])]).save(sourcedb_list_path) # THe concat ms does not have to be written: It already is a # singular item (it is the output of the reduce step) # redmine issue #6021 arguments = [ self.inputs['bbs_executable'], self.inputs['parset'], ms_list_path, parmdb_list_path, sourcedb_list_path, concat_ms.file, self.inputs['major_cycle'] ] jobs.append(ComputeJob(host, node_command, arguments)) # start and wait till all are finished self._schedule_jobs(jobs) # ********************************************************************** # 3. validate the node output and construct the output mapfile. if self.error.isSet(): #if one of the nodes failed self.logger.warn("Failed bbs node run detected, skipping work" "on this work item for further computations") # find failed job and set the skip field for (ms_item, concat_item, job) in zip(ms_map, concat_ms_map, jobs): if job.results["returncode"] == 0: continue else: ms_item.skip = True concat_item.skip = True self.logger.warn("bbs failed on item: {0}".format( ms_item.file)) # return the output: The measurement set that are calibrated: # calibrated data is placed in the ms sets MultiDataMap(ms_map).save(self.inputs['mapfile']) # also save the concat_ms map with possible skips DataMap(concat_ms_map).save(self.inputs['concat_ms_map_path']) self.logger.info("Wrote file with calibrated data") self.outputs['mapfile'] = self.inputs['mapfile'] return 0
def run(self, bbs_executable, parset, ms_list_path, parmdb_list_path, sky_list_path, concat_ms_path, major_cycle): """ selfcal_bbs functionality. Called by framework performing all the work """ self.logger.debug("Starting selfcal_bbs Node") # ********************************************************************* # 1. Load mapfiles # read in the mapfiles to data maps: The master recipe added the single # path to a mapfilem which allows usage of default data methods # (load_data_map) # TODO: Datamap ms_map = MultiDataMap.load(ms_list_path) parmdb_map = MultiDataMap.load(parmdb_list_path) sky_list = MultiDataMap.load(sky_list_path) source_db = sky_list[0].file[0] # the sourcedb is the first file entry try: bbs_process_group = SubProcessGroup(self.logger, self.resourceMonitor) # ***************************************************************** # 2. start the bbs executable with data # The data is located in multimaps. We need the first entry # TODO: THis is not 'nice' usage of the multimap for (measurement_set, parmdm) in zip(ms_map[0].file, parmdb_map[0].file): command = [ bbs_executable, "--sourcedb={0}".format(source_db), "--parmdb={0}".format(parmdm), measurement_set, parset ] self.logger.info("Executing bbs command: {0}".format( " ".join(command))) bbs_process_group.run(command) # ***************************************************************** # 3. check status of the processes if bbs_process_group.wait_for_finish() != None: self.logger.error("Failed bbs run detected Aborting") return 1 except OSError as exception: self.logger.error("Failed to execute bbs: {0}".format( str(exception))) return 1 # ********************************************************************* # 4. Concat in time after bbs calibration your MSs using # msconcat (pyrap.tables module) (added by N.Vilchez) # this step has te be performed on this location. because the bbs run # might add additional columns not present in the original ms # and therefore not produced in the concat done in the prepare phase # redmine issue #6021 pt.msconcat(ms_map[0].file, concat_ms_path, concatTime=True) # ********************************************************************* # 5. copy time slives directory to a new one # This is done for debugging purpose: The copy is not used for anything # The actual selfcal steps are done in place # (added by N.Vilchez) # THe save location is created relative to the concat.ms # we could also use the self.scratch_directory from the toplevel recipe # this would need an aditional ingredient # This is a 'debugging' step and should never ever cause a failure of \ # the pipeline try: working_dir = os.path.dirname(concat_ms_path) time_slice_dir = os.path.join(working_dir, 'time_slices') time_slice_copy_dir = os.path.join( working_dir, 'time_slices_cycle_{0}'.format(major_cycle)) cmd = "cp -r {0} {1}".format(time_slice_dir, time_slice_copy_dir) os.system(cmd) except: self.logger.warn( "Debug copy of temporary files failed: continue operations") pass # Do nothing return 0