def test_fusion_parameters(self): """ TEST : Fusion.dempster_shafer_fusion_parameters """ from iota2.Classification import Fusion from iota2.Common import IOTA2Directory from iota2.Common import ServiceConfigFile as SCF # define inputs cfg = SCF.serviceConfigFile(self.config_test) iota2_dir = os.path.join(self.test_working_directory, "fusionTest") cfg.setParam('chain', 'outputPath', iota2_dir) IOTA2Directory.generate_directories(iota2_dir, check_inputs=False) iota2_ds_confusions_dir = os.path.join(iota2_dir, "dataAppVal", "bymodels") fut.ensure_dir(iota2_ds_confusions_dir) # generate some fake data nb_seed = 2 for i in range(nb_seed): fake_classif_opt = os.path.join( iota2_dir, "classif", "Classif_T31TCJ_model_1_seed_{}.tif".format(i)) fake_classif_sar = os.path.join( iota2_dir, "classif", "Classif_T31TCJ_model_1_seed_{}_SAR.tif".format(i)) fake_confidence_opt = os.path.join( iota2_dir, "classif", "T31TCJ_model_1_confidence_seed_{}.tif".format(i)) fake_confidence_sar = os.path.join( iota2_dir, "classif", "T31TCJ_model_1_confidence_seed_{}_SAR.tif".format(i)) fake_model_confusion_sar = os.path.join( iota2_ds_confusions_dir, "model_1_seed_{}_SAR.csv".format(i)) fake_model_confusion_opt = os.path.join( iota2_ds_confusions_dir, "model_1_seed_{}.csv".format(i)) with open(fake_classif_opt, "w") as new_file: new_file.write("TEST") with open(fake_classif_sar, "w") as new_file: new_file.write("TEST") with open(fake_confidence_opt, "w") as new_file: new_file.write("TEST") with open(fake_confidence_sar, "w") as new_file: new_file.write("TEST") with open(fake_model_confusion_sar, "w") as new_file: new_file.write("TEST") with open(fake_model_confusion_opt, "w") as new_file: new_file.write("TEST") parameters_test = Fusion.dempster_shafer_fusion_parameters(iota2_dir) # parameters_test depend of execution environement, remove local path is necessary for param_group in parameters_test: for key, value in list(param_group.items()): param_group[key] = value.replace(iota2_dir, "") #reverse parameters_test if necessary if "seed_1" in parameters_test[0]["sar_model"]: parameters_test = parameters_test[::-1] # assert self.assertTrue(all(param_group_test == param_group_ref for param_group_test, param_group_ref in zip( parameters_test, self.parameter_ref)), msg="input parameters generation failed")
def mpi_schedule(iota2_step, param_array_origin, mpi_service=MPIService(), logPath=None, logger_lvl="INFO", enable_console=False): """ A simple MPI scheduler to execute jobs in parallel. """ if mpi_service.rank != 0: return None job = iota2_step.step_execute() returned_data_list = [] parameters_success = [] if not param_array_origin: raise Exception( "JobArray must contain a list of parameter as argument") sys.exit(1) try: if os.path.exists(logPath): os.remove(logPath) if callable(param_array_origin): param_array = param_array_origin() else: #shallowCopy param_array = [param for param in param_array_origin] if mpi_service.size > 1: # master nb_completed_tasks = 0 nb_tasks = len(param_array) for i in range(1, mpi_service.size): if len(param_array) > 0: task_param = param_array.pop(0) mpi_service.comm.send( [job, task_param, logger_lvl, enable_console], dest=i, tag=0) while nb_completed_tasks < nb_tasks: [ worker_rank, [start, end, worker_complete_log, returned_data, success] ] = mpi_service.comm.recv(source=MPI.ANY_SOURCE, tag=0) returned_data_list.append(returned_data) parameters_success.append(success) #Write worker log fut.ensure_dir(os.path.split(logPath)[0]) with open(logPath, "a+") as log_f: log_f.write(worker_complete_log) nb_completed_tasks += 1 if len(param_array) > 0: task_param = param_array.pop(0) mpi_service.comm.send( [job, task_param, logger_lvl, enable_console], dest=worker_rank, tag=0) else: #if not lanch thanks to mpirun, launch each parameters one by one for param in param_array: worker_log = sLog.Log_task(logger_lvl, enable_console) worker_complete_log, start_date, end_date, returned_data, success = launchTask( job, param, worker_log) fut.ensure_dir(os.path.split(logPath)[0]) with open(logPath, "a+") as log_f: log_f.write(worker_complete_log) returned_data_list.append(returned_data) parameters_success.append(success) except KeyboardInterrupt: raise except Exception as e: print(e) parameters_success.append(False) if mpi_service.rank == 0 and mpi_service.size > 1: print("Something went wrong, we should log errors.") traceback.print_exc() stop_workers(mpi_service) sys.exit(1) step_completed = all(parameters_success) if step_completed: iota2_step.step_clean() return returned_data_list, step_completed