def train(self): X_train, y_train, _ = self.load_results_from_result_paths(self.train_paths) X_val, y_val, _ = self.load_results_from_result_paths(self.val_paths) self.model.fit(X_train, y_train) train_pred, var_train = self.model.predict(X_train), None val_pred, var_val = self.model.predict(X_val), None #self.save() fig_train = utils.scatter_plot(np.array(train_pred), np.array(y_train), xlabel='Predicted', ylabel='True', title='') fig_train.savefig(os.path.join(self.log_dir, 'pred_vs_true_train.jpg')) plt.close() fig_val = utils.scatter_plot(np.array(val_pred), np.array(y_val), xlabel='Predicted', ylabel='True', title='') fig_val.savefig(os.path.join(self.log_dir, 'pred_vs_true_val.jpg')) plt.close() train_metrics = utils.evaluate_metrics(y_train, train_pred, prediction_is_first_arg=False) valid_metrics = utils.evaluate_metrics(y_val, val_pred, prediction_is_first_arg=False) logging.info('train metrics: %s', train_metrics) logging.info('valid metrics: %s', valid_metrics) return valid_metrics
def train(self): X_train, y_train, _ = self.load_results_from_result_paths( self.train_paths) X_val, y_val, _ = self.load_results_from_result_paths(self.val_paths) base_learner_config = self.parse_config("base:") param_config = self.parse_config("param:") # train base_learner = DecisionTreeRegressor(criterion='friedman_mse', random_state=None, splitter='best', **base_learner_config) self.model = NGBRegressor(Dist=Normal, Base=base_learner, Score=LogScore, verbose=True, **param_config) self.model = self.model.fit( X_train, y_train, X_val=X_val, Y_val=y_val, early_stopping_rounds=self.model_config["early_stopping_rounds"]) train_pred, var_train = self.model.predict(X_train), None val_pred, var_val = self.model.predict(X_val), None # self.save() fig_train = utils.scatter_plot(np.array(train_pred), np.array(y_train), xlabel='Predicted', ylabel='True', title='') fig_train.savefig(os.path.join(self.log_dir, 'pred_vs_true_train.jpg')) plt.close() fig_val = utils.scatter_plot(np.array(val_pred), np.array(y_val), xlabel='Predicted', ylabel='True', title='') fig_val.savefig(os.path.join(self.log_dir, 'pred_vs_true_val.jpg')) plt.close() train_metrics = utils.evaluate_metrics(y_train, train_pred, prediction_is_first_arg=False) valid_metrics = utils.evaluate_metrics(y_val, val_pred, prediction_is_first_arg=False) logging.info('train metrics: %s', train_metrics) logging.info('valid metrics: %s', valid_metrics) return valid_metrics
def train(self): X_train, y_train, _ = self.load_results_from_result_paths( self.train_paths) X_val, y_val, _ = self.load_results_from_result_paths(self.val_paths) logging.info( "LGBOOST TRAIN: Careful categoricals not specified in dataset conversion" ) dtrain = lgb.Dataset(X_train, label=y_train) dval = lgb.Dataset(X_val, label=y_val) param_config = self.parse_param_config() param_config["seed"] = self.seed self.model = lgb.train( param_config, dtrain, early_stopping_rounds=self.model_config["early_stopping_rounds"], verbose_eval=1, valid_sets=[dval]) train_pred, var_train = self.model.predict(X_train), None val_pred, var_val = self.model.predict(X_val), None # self.save() fig_train = utils.scatter_plot(np.array(train_pred), np.array(y_train), xlabel='Predicted', ylabel='True', title='') fig_train.savefig(os.path.join(self.log_dir, 'pred_vs_true_train.jpg')) plt.close() fig_val = utils.scatter_plot(np.array(val_pred), np.array(y_val), xlabel='Predicted', ylabel='True', title='') fig_val.savefig(os.path.join(self.log_dir, 'pred_vs_true_val.jpg')) plt.close() train_metrics = utils.evaluate_metrics(y_train, train_pred, prediction_is_first_arg=False) valid_metrics = utils.evaluate_metrics(y_val, val_pred, prediction_is_first_arg=False) logging.info('train metrics: %s', train_metrics) logging.info('valid metrics: %s', valid_metrics) return valid_metrics
def evaluate(self, result_paths): X_test, y_test, _ = self.load_results_from_result_paths(result_paths) test_pred, var_test = self.model.predict(X_test), None test_metrics = utils.evaluate_metrics(y_test, test_pred, prediction_is_first_arg=False) return test_metrics, test_pred, y_test
def validate(self): preds = [] targets = [] self.model.eval() valid_queue = self.load_results_from_result_paths(self.val_paths) for step, (arch_path_enc, y_true) in enumerate(valid_queue): arch_path_enc = arch_path_enc.to(self.device).float() y_true = y_true.to(self.device).float() pred = self.model(arch_path_enc) preds.extend(pred.detach().cpu().numpy() * 100) targets.extend(y_true.detach().cpu().numpy()) fig = utils.scatter_plot(np.array(preds), np.array(targets), xlabel='Predicted', ylabel='True', title='') fig.savefig(os.path.join(self.log_dir, 'pred_vs_true_valid.jpg')) plt.close() val_results = utils.evaluate_metrics(np.array(targets), np.array(preds), prediction_is_first_arg=False) logging.info('validation metrics %s', val_results) return val_results
def test(self): preds = [] targets = [] self.model.eval() test_queue = self.load_results_from_result_paths(self.test_paths) for step, graph_batch in enumerate(test_queue): graph_batch = graph_batch.to(self.device) if self.model_config['model'] == 'gnn_vs_gae_classifier': pred_bins, pred = self.model(graph_batch=graph_batch) else: pred = self.model(graph_batch=graph_batch) preds.extend(pred.detach().cpu().numpy() * 100) targets.extend(graph_batch.y.detach().cpu().numpy()) fig = utils.scatter_plot(np.array(preds), np.array(targets), xlabel='Predicted', ylabel='True', title='') fig.savefig(os.path.join(self.log_dir, 'pred_vs_true_test.jpg')) plt.close() test_results = utils.evaluate_metrics(np.array(targets), np.array(preds), prediction_is_first_arg=False) logging.info('test metrics %s', test_results) return test_results
def leave_one_out_analysis(analysis_directory, nasbench_data): result_dict = {} data_min, data_max = 100, 0 optimizers = ['darts', 'bananas', 'combo', 'de', 're', 'tpe', 'pc_darts', 'gdas', 'drnas'] result_loader = utils.ResultLoader('', '', '', 0) for optimizer in optimizers: try: model_log_dir = glob.glob(os.path.join(analysis_directory, optimizer, '*', '*'))[0] data_config = json.load(open(os.path.join(model_log_dir, 'data_config.json'), 'r')) model_config = json.load(open(os.path.join(model_log_dir, 'model_config.json'), 'r')) # Instantiate model surrogate_model = utils.model_dict[model_config['model']](data_root=nasbench_data, log_dir=None, seed=data_config['seed'], data_config=data_config, model_config=model_config) surrogate_model.load(os.path.join(model_log_dir, 'surrogate_model.model')) left_out_optimizer_paths = result_loader.filter_duplicate_dirs( glob.glob(os.path.join(nasbench_data, optimizer, '*'))) _, val_preds, val_true = surrogate_model.evaluate(left_out_optimizer_paths) data_min = min(data_min, *val_preds, *val_true) data_max = max(data_max, *val_preds, *val_true) if type(val_preds) is not list: val_preds, val_true = val_preds.tolist(), val_true.tolist() result_dict[optimizer] = {'val_preds': val_preds, 'val_true': val_true} except FileNotFoundError as e: pass fig = plt.figure(figsize=(3, 20)) grid = ImageGrid(fig, 111, # as in plt.subplot(111) nrows_ncols=(len(optimizers), 1), axes_pad=0.5, share_all=True) counter = 0 statistics = {} for ax, (optimizer, results) in zip(grid, result_dict.items()): if counter == 0: ax.set_xlabel('Predicted') ax.set_ylabel('True') counter += 1 ax.scatter(results['val_preds'], results['val_true'], s=1, alpha=0.15) kendall = stats.kendalltau(np.round(np.array(results['val_preds']), decimals=1), results['val_true']) statistics[optimizer] = { **evaluate_metrics(results['val_true'], results['val_preds'], prediction_is_first_arg=False) } ax.set_xlim(data_min, data_max) ax.set_ylim(data_min, data_max) ax.plot([data_min, data_max], [data_min, data_max], c='r', alpha=0.3) ax.grid(True, which="both", ls="-", alpha=0.1) ax.set_title('{} - K:{}'.format(optimizer.upper(), "{:.3f}".format(kendall.correlation))) plt.savefig(os.path.join(analysis_directory, 'loo_analysis.png'), dpi=600) # Dump statistics json.dump(statistics, open(os.path.join(analysis_directory, 'statistics.json'), 'w')) # Dump the results from the analysis json.dump(result_dict, open(os.path.join(analysis_directory, 'analysis_results.json'), 'w'))
def validate(self): X_val, y_val, _ = self.load_results_from_result_paths(self.val_paths) val_pred, var_val = self.model.predict(X_val), None valid_metrics = utils.evaluate_metrics(y_val, val_pred, prediction_is_first_arg=False) logging.info('validation metrics %s', valid_metrics) return valid_metrics
def test(self): X_test, y_test, _ = self.load_results_from_result_paths(self.test_paths) test_pred, var_test = self.model.predict(X_test), None fig = utils.scatter_plot(np.array(test_pred), np.array(y_test), xlabel='Predicted', ylabel='True', title='') fig.savefig(os.path.join(self.log_dir, 'pred_vs_true_test.jpg')) plt.close() test_metrics = utils.evaluate_metrics(y_test, test_pred, prediction_is_first_arg=False) logging.info('test metrics %s', test_metrics) return test_metrics
def validate_and_test_surrogate_model(model_log_dir): # Instantiate surrogate model surrogate_model = load_surrogate_model(model_log_dir) flatten = lambda l: [item for sublist in l for item in sublist] aposteriori_analysis = {} for split in ['val', 'test']: paths = flatten([ json.load(open(val_opt)) for val_opt in glob.glob( os.path.join(model_log_dir, '*_{}_paths.json'.format(split))) ]) _, preds, true = surrogate_model.evaluate(paths) aposteriori_analysis[split] = utils.evaluate_metrics( true, preds, prediction_is_first_arg=False) print(aposteriori_analysis)
def train(self): X_train, y_train, _ = self.load_results_from_result_paths(self.train_paths) X_val, y_val, _ = self.load_results_from_result_paths(self.val_paths) dtrain = xgb.DMatrix(X_train, label=y_train) dval = xgb.DMatrix(X_val, label=y_val) param_config = self.parse_param_config() param_config["seed"] = self.seed self.model = xgb.train(param_config, dtrain, num_boost_round=self.model_config["param:num_rounds"], early_stopping_rounds=self.model_config["early_stopping_rounds"], verbose_eval=1, evals=[(dval, 'val')]) train_pred, var_train = self.model.predict(dtrain), None val_pred, var_val = self.model.predict(dval), None # self.save() fig_train = utils.scatter_plot(np.array(train_pred), np.array(y_train), xlabel='Predicted', ylabel='True', title='') fig_train.savefig(os.path.join(self.log_dir, 'pred_vs_true_train.jpg')) plt.close() fig_val = utils.scatter_plot(np.array(val_pred), np.array(y_val), xlabel='Predicted', ylabel='True', title='') fig_val.savefig(os.path.join(self.log_dir, 'pred_vs_true_val.jpg')) plt.close() train_metrics = utils.evaluate_metrics(y_train, train_pred, prediction_is_first_arg=False) valid_metrics = utils.evaluate_metrics(y_val, val_pred, prediction_is_first_arg=False) logging.info('train metrics: %s', train_metrics) logging.info('valid metrics: %s', valid_metrics) return valid_metrics
def evaluate(self, result_paths): # Get evaluation data eval_queue = self.load_results_from_result_paths(result_paths) preds = [] targets = [] self.model.eval() for step, (arch_path_enc, y_true) in enumerate(eval_queue): arch_path_enc = arch_path_enc.to(self.device).float() y_true = y_true.to(self.device).float() pred = self.model(arch_path_enc) preds.extend(pred.detach().cpu().numpy() * 100) targets.extend(y_true.detach().cpu().numpy()) test_metrics = utils.evaluate_metrics(np.array(targets), np.array(preds), prediction_is_first_arg=False) return test_metrics, preds, targets
def evaluate(self, result_paths): # Get evaluation data eval_queue = self.load_results_from_result_paths(result_paths) preds = [] targets = [] self.model.eval() for step, graph_batch in enumerate(eval_queue): graph_batch = graph_batch.to(self.device) pred = self.model(graph_batch=graph_batch) preds.extend(pred.detach().cpu().numpy() * 100) targets.extend(graph_batch.y.detach().cpu().numpy()) test_metrics = utils.evaluate_metrics(np.array(targets), np.array(preds), prediction_is_first_arg=False) return test_metrics, np.array(preds), np.array(targets)
def infer(self, train_queue, valid_queue, model, criterion, optimizer, lr, epoch): objs = utils.AvgrageMeter() # VALIDATION preds = [] targets = [] for step, (arch_path_enc, y_true) in enumerate(valid_queue): arch_path_enc = arch_path_enc.to(self.device).float() y_true = y_true.to(self.device).float() pred = self.model(arch_path_enc) loss = torch.mean( torch.abs((self.normalize_data(pred) / self.normalize_data(y_true / 100)) - 1)) preds.extend(pred.detach().cpu().numpy() * 100) targets.extend(y_true.detach().cpu().numpy()) objs.update(loss.data.item(), len(arch_path_enc)) if step % self.data_config['report_freq'] == 0: logging.info('valid %03d %e ', step, objs.avg) fig = utils.scatter_plot(np.array(preds), np.array(targets), xlabel='Predicted', ylabel='True', title='') fig.savefig( os.path.join(self.log_dir, 'pred_vs_true_valid_{}.jpg'.format(epoch))) plt.close() val_results = utils.evaluate_metrics(np.array(targets), np.array(preds), prediction_is_first_arg=False) return objs.avg, val_results
def evaluate_ensemble(self, result_paths, apply_noise): """Evaluates the metrics on the result paths using ensemble predicitons""" preds, targets = [], [] # Collect individuals predictions for member_model in self.ensemble_members: member_metrics, member_preds, member_targets = member_model.evaluate( result_paths) logging.info("==> Eval member metrics: %s", member_metrics) if len(targets) == 0: preds.append(member_preds) targets = member_targets continue if np.any((targets - member_targets) > 1e-5): raise ValueError("Ensemble members have different targets!") preds.append(member_preds) means = np.mean(preds, axis=0) stddevs = np.std(preds, axis=0) # Apply noise if apply_noise: noisy_predictions = [ np.random.normal(loc=mean, scale=stddev, size=1)[0] for mean, stddev in zip(means, stddevs) ] ensemble_predictions = noisy_predictions else: ensemble_predictions = means # Evaluate metrics metrics = utils.evaluate_metrics(targets, ensemble_predictions, prediction_is_first_arg=False) return metrics, ensemble_predictions, stddevs, targets
def infer(self, train_queue, valid_queue, model, criterion, optimizer, lr, epoch): objs = utils.AvgrageMeter() # VALIDATION preds = [] targets = [] model.eval() for step, graph_batch in enumerate(valid_queue): graph_batch = graph_batch.to(self.device) if self.model_config['model'] == 'gnn_vs_gae_classifier': pred_bins, pred = self.model(graph_batch=graph_batch) criterion = torch.nn.BCELoss() criterion_2 = torch.nn.MSELoss() bins = self.create_bins(lower_bound=0, width=10, quantity=9) binned_weights = [] for value in graph_batch.y.cpu().numpy(): bin_index = self.find_bin(value, bins) binned_weights.append(bin_index) bins = torch.FloatTensor(binned_weights) make_one_hot = lambda index: torch.eye(self.model_config[ 'no_bins'])[index.view(-1).long()] binns_one_hot = make_one_hot(bins).to(self.device) loss_1 = criterion(pred_bins, binns_one_hot) loss_2 = criterion_2(pred, self.normalize_data(graph_batch.y)) alpha = self.model_config['classification_loss'] beta = self.model_config['regression_loss'] loss = alpha * loss_1 + beta * loss_2 else: pred = self.model(graph_batch=graph_batch) loss = criterion(self.normalize_data(pred), self.normalize_data(graph_batch.y / 100)) preds.extend(pred.detach().cpu().numpy() * 100) targets.extend(graph_batch.y.detach().cpu().numpy()) n = graph_batch.num_graphs objs.update(loss.data.item(), n) if step % self.data_config['report_freq'] == 0: logging.info('valid %03d %e ', step, objs.avg) fig = utils.scatter_plot(np.array(preds), np.array(targets), xlabel='Predicted', ylabel='True', title='') fig.savefig( os.path.join(self.log_dir, 'pred_vs_true_valid_{}.jpg'.format(epoch))) plt.close() val_results = utils.evaluate_metrics(np.array(targets), np.array(preds), prediction_is_first_arg=False) return objs.avg, val_results
def train_epoch(self, train_queue, valid_queue, model, criterion, optimizer, lr, epoch): objs = utils.AvgrageMeter() # TRAINING preds = [] targets = [] model.train() for step, graph_batch in enumerate(train_queue): graph_batch = graph_batch.to(self.device) # print(step) if self.model_config['model'] == 'gnn_vs_gae_classifier': pred_bins, pred = self.model(graph_batch=graph_batch) criterion = torch.nn.BCELoss() criterion_2 = torch.nn.MSELoss() bins = self.create_bins(lower_bound=0, width=10, quantity=9) binned_weights = [] for value in graph_batch.y.cpu().numpy(): bin_index = self.find_bin(value, bins) binned_weights.append(bin_index) bins = torch.FloatTensor(binned_weights) make_one_hot = lambda index: torch.eye(self.model_config[ 'no_bins'])[index.view(-1).long()] binns_one_hot = make_one_hot(bins).to(self.device) loss_1 = criterion(pred_bins, binns_one_hot) loss_2 = criterion_2(pred, self.normalize_data(graph_batch.y)) alpha = self.model_config['classification_loss'] beta = self.model_config['regression_loss'] loss = alpha * loss_1 + beta * loss_2 else: pred = self.model(graph_batch=graph_batch) if self.model_config['loss:loss_log_transform']: loss = criterion(self.normalize_data(pred), self.normalize_data(graph_batch.y / 100)) else: loss = criterion(pred, graph_batch.y / 100) if self.model_config['loss:pairwise_ranking_loss']: m = 0.1 ''' y = list(map(lambda y_i: 1 if y_i == True else -1, graph_batch.y[0: -1] > graph_batch.y[1:])) pairwise_ranking_loss = torch.nn.HingeEmbeddingLoss(margin=m)(pred[0:-1] - pred[1:], target=torch.from_numpy(np.array(y))) ''' pairwise_ranking_loss = [] sort_idx = torch.argsort(graph_batch.y, descending=True) for idx, idx_y_i in enumerate(sort_idx): for idx_y_i_p1 in sort_idx[idx + 1:]: pairwise_ranking_loss.append( torch.max( torch.tensor(0.0, dtype=torch.float), m - (pred[idx_y_i] - pred[idx_y_i_p1]))) pairwise_ranking_loss = torch.mean( torch.stack(pairwise_ranking_loss)) loss += pairwise_ranking_loss if step % self.data_config['report_freq'] == 0: logging.info('Pairwise ranking loss {}'.format( pairwise_ranking_loss)) preds.extend(pred.detach().cpu().numpy() * 100) targets.extend(graph_batch.y.detach().cpu().numpy()) optimizer.zero_grad() loss.backward() optimizer.step() n = graph_batch.num_graphs objs.update(loss.data.item(), n) if step % self.data_config['report_freq'] == 0: logging.info('train %03d %e', step, objs.avg) fig = utils.scatter_plot(np.array(preds), np.array(targets), xlabel='Predicted', ylabel='True', title='') fig.savefig( os.path.join(self.log_dir, 'pred_vs_true_train_{}.jpg'.format(epoch))) plt.close() train_results = utils.evaluate_metrics(np.array(targets), np.array(preds), prediction_is_first_arg=False) return objs.avg, train_results
def train_epoch(self, train_queue, valid_queue, model, criterion, optimizer, lr, epoch): objs = utils.AvgrageMeter() # TRAINING preds = [] targets = [] model.train() for step, (arch_path_enc, y_true) in enumerate(train_queue): arch_path_enc = arch_path_enc.to(self.device).float() y_true = y_true.to(self.device).float() pred = self.model(arch_path_enc) if self.model_config['loss:loss_log_transform']: loss = torch.mean( torch.abs((self.normalize_data(pred) / self.normalize_data(y_true / 100)) - 1)) else: loss = criterion(1 - pred, 1 - y_true / 100) if self.model_config['loss:pairwise_ranking_loss']: m = 0.1 pairwise_ranking_loss = [] sort_idx = torch.argsort(y_true, descending=True) for idx, idx_y_i in enumerate(sort_idx): for idx_y_i_p1 in sort_idx[idx + 1:]: pairwise_ranking_loss.append( torch.max(torch.tensor(0.0, dtype=torch.float), m - (pred[idx_y_i] - pred[idx_y_i_p1]))) pairwise_ranking_loss = torch.mean( torch.stack(pairwise_ranking_loss)) loss += pairwise_ranking_loss if step % self.data_config['report_freq'] == 0: logging.info('Pairwise ranking loss {}'.format( pairwise_ranking_loss)) preds.extend(pred.detach().cpu().numpy() * 100) targets.extend(y_true.detach().cpu().numpy()) optimizer.zero_grad() loss.backward() optimizer.step() objs.update(loss.data.item(), len(arch_path_enc)) if step % self.data_config['report_freq'] == 0: logging.info('train %03d %e', step, objs.avg) fig = utils.scatter_plot(np.array(preds), np.array(targets), xlabel='Predicted', ylabel='True', title='') fig.savefig( os.path.join(self.log_dir, 'pred_vs_true_train_{}.jpg'.format(epoch))) plt.close() train_results = utils.evaluate_metrics(np.array(targets), np.array(preds), prediction_is_first_arg=False) return objs.avg, train_results
def analyze_cell_topology(): config_loader = ConfigLoader('configspace.json') # Load groundtruth data cell_depths = [ compute_depth_of_cell_simple_path(config_loader[gt][0])[0] for gt in glob.glob( '/home/user/projects/nasbench_201_2/analysis/nb_301_cell_topology/cell_topology_analysis/results_*.json' ) ] gt_data_paths = [ gt for gt in glob.glob( '/home/user/projects/nasbench_201_2/analysis/nb_301_cell_topology/cell_topology_analysis/results_*.json' ) ] surrogate_models = { 'GIN': load_surrogate_model( '/home/user/projects/nasbench_201_2/experiments/surrogate_models/gnn_gin/20200919-135631-6' ), 'LGB': load_surrogate_model( '/home/user/projects/nasbench_201_2/experiments/surrogate_models/lgb/20200919-135720-6' ), 'XGB': load_surrogate_model( '/home/user/projects/nasbench_201_2/experiments/surrogate_models/xgb/20200919-135720-6' ) } surrogate_model_results = {'cell_depth': cell_depths} for surrogate_model_name, surrogate_model in surrogate_models.items(): test_metrics, preds, targets = surrogate_model.evaluate(gt_data_paths) surrogate_model_results[surrogate_model_name + '_preds'] = preds surrogate_model_results[surrogate_model_name + '_targets'] = targets fig, ax_left = plt.subplots(figsize=(4, 3)) ax_left.set_ylabel('sKendall Tau') for surrogate_model in surrogate_models: idx = 0 for cell_depth, group in pd.DataFrame(surrogate_model_results).groupby( 'cell_depth'): preds, targets = group['{}_preds'.format(surrogate_model)], group[ '{}_targets'.format(surrogate_model)] metrics = evaluate_metrics(targets, preds, prediction_is_first_arg=False) print(surrogate_model, cell_depth, metrics) ax_left.scatter( cell_depth, metrics['kendall_tau_1_dec'], marker=surrogate_model_markers[surrogate_model.upper()], label=surrogate_model.upper() if idx == 0 else None, c=surrogate_model_colors[surrogate_model.upper()]) idx += 1 plt.legend() plt.grid(True, which="both", ls="-", alpha=0.5) ''' ax_right = ax_left.twinx() # instantiate a second axes that shares the same x-axis ax_right.set_ylabel('R² (x)') for surrogate_model in surrogate_models: idx = 0 for cell_depth, group in pd.DataFrame(surrogate_model_results).groupby('cell_depth'): preds, targets = group['{}_preds'.format(surrogate_model)], group['{}_targets'.format(surrogate_model)] metrics = evaluate_metrics(targets, preds, prediction_is_first_arg=False) ax_right.scatter(cell_depth, metrics['r2'], marker='x', c=surrogate_model_colors[surrogate_model.upper()]) idx += 1 ''' ax_left.set_xlabel('Cell Depth') plt.tight_layout() plt.savefig('surrogate_models/analysis/cell_topology_analysis.pdf')