def _collect_and_assign_outputs(self, jobs, output_map, slice_paths_map): """ Collect and combine the outputs of the individual create_dbs node recipes. Combine into output mapfiles and save these at the supplied path locations """ # Create a container for the output parmdbs: same host and output_map.iterator = DataMap.TupleIterator parmdbs_list = [] # loop over the raw data including the skip file (use the data member) for output_entry in output_map.data: parms_tuple = tuple([output_entry.host, [], output_entry.skip]) parmdbs_list.append(parms_tuple) parmdbs_map = MultiDataMap(parmdbs_list) output_map.iterator = parmdbs_map.iterator = DataMap.SkipIterator # The maps are synced succesfull_run = False for (output_item, parmdbs_item, job) in zip( output_map, parmdbs_map, jobs): node_succeeded = job.results.has_key("parmdbs") and \ job.results.has_key("sourcedb") host = output_item.host # The current job has to be skipped (due to skip field) # Or if the node failed: if not node_succeeded: self.logger.warn("Warning failed ImagerCreateDBs run " "detected: No sourcedb file created, {0} continue".format( host)) output_item.file = "failed" output_item.skip = True parmdbs_item.file = ["failed"] parmdbs_item.skip = True # Else it succeeded and we can write te results else: succesfull_run = True output_item.file = job.results["sourcedb"] parmdbs_item.file = job.results["parmdbs"] # we also need to manually set the skip for this new # file list parmdbs_item.file_skip = [False] * len(job.results["parmdbs"]) # Fail if none of the nodes returned all data if not succesfull_run: self.logger.error("The creation of dbs on the nodes failed:") self.logger.error("Not a single node produces all needed data") self.logger.error( "products. sourcedb_files: {0}".format(output_map)) self.logger.error("parameter dbs: {0}".format(parmdbs_map)) return 1 # write the mapfiles output_map.save(self.inputs["sourcedb_map_path"]) parmdbs_map.save(self.inputs["parmdbs_map_path"]) self.logger.debug("Wrote sourcedb dataproducts: {0} \n {1}".format( self.inputs["sourcedb_map_path"], self.inputs["parmdbs_map_path"])) # Set the outputs self.outputs['sourcedb_map_path'] = self.inputs["sourcedb_map_path"] self.outputs['parmdbs_map_path'] = self.inputs["parmdbs_map_path"] return 0
def _collect_and_assign_outputs(self, jobs, output_map, slice_paths_map): """ Collect and combine the outputs of the individual create_dbs node recipes. Combine into output mapfiles and save these at the supplied path locations """ # Create a container for the output parmdbs: same host and output_map.iterator = DataMap.TupleIterator parmdbs_list = [] # loop over the raw data including the skip file (use the data member) for output_entry in output_map.data: parms_tuple = tuple([output_entry.host, [], output_entry.skip]) parmdbs_list.append(parms_tuple) parmdbs_map = MultiDataMap(parmdbs_list) output_map.iterator = parmdbs_map.iterator = DataMap.SkipIterator # The maps are synced succesfull_run = False for (output_item, parmdbs_item, job) in zip( output_map, parmdbs_map, jobs): node_succeeded = "parmdbs" in job.results and \ "sourcedb" in job.results host = output_item.host # The current job has to be skipped (due to skip field) # Or if the node failed: if not node_succeeded: self.logger.warn("Warning failed selfcalCreateDBs run " "detected: No sourcedb file created, {0} continue".format( host)) output_item.file = "failed" output_item.skip = True parmdbs_item.file = [] parmdbs_item.skip = True # Else it succeeded and we can write te results else: succesfull_run = True output_item.file = job.results["sourcedb"] parmdbs_item.file = job.results["parmdbs"] # we also need to manually set the skip for this new # file list parmdbs_item.file_skip = [False] * len(job.results["parmdbs"]) # Fail if none of the nodes returned all data if not succesfull_run: self.logger.error("The creation of dbs on the nodes failed:") self.logger.error("Not a single node produces all needed data") self.logger.error( "products. sourcedb_files: {0}".format(output_map)) self.logger.error("parameter dbs: {0}".format(parmdbs_map)) return 1 # write the mapfiles output_map.save(self.inputs["sourcedb_map_path"]) parmdbs_map.save(self.inputs["parmdbs_map_path"]) self.logger.debug("Wrote sourcedb dataproducts: {0} \n {1}".format( self.inputs["sourcedb_map_path"], self.inputs["parmdbs_map_path"])) # Set the outputs self.outputs['sourcedb_map_path'] = self.inputs["sourcedb_map_path"] self.outputs['parmdbs_map_path'] = self.inputs["parmdbs_map_path"] return 0
def test_new_style_load_store(self): tmp_file = self.new_style_map_file + '.tmp' data_map = MultiDataMap(self.new_style_map) data_map.save(tmp_file) reloaded_data_map = MultiDataMap.load(tmp_file) self.assertEqual(data_map, reloaded_data_map)
def test_new_style_load_store(self): tmp_file = self.new_style_map_file + '.tmp' data_map = MultiDataMap(self.new_style_map) data_map.save(tmp_file) reloaded_data_map = MultiDataMap.load(tmp_file) self.assertEqual(data_map, reloaded_data_map)