eng = matlab.engine.start_matlab() eng.addpath(CAUSALITY_ROOT + 'comparison_methods/Mooij16/cep') eng.startup(nargout=0) eng.local_config(nargout=0) methodpars = eng.struct() methodpars['FITC'] = 0 methodpars['minimize'] = 'minimize_lbfgsb' methodpars['evaluation'] = 'pHSIC' accuracy = 0 correct_decisions = 0 undecided = 0 for i in range(FIRST_ID - 1, LAST_ID): (x, y), true_direction = get_pair(i, BENCHMARK) if true_direction == -1: continue x_ = matlab.double(x.tolist()) x_T = eng.transpose(x_) y_ = matlab.double(y.tolist()) y_T = eng.transpose(y_) result = eng.cep_anm(x_T, y_T, methodpars) predicted_direction = result['decision'] if predicted_direction == true_direction: correct_decisions += 1 if predicted_direction == 0: undecided += 1
# prediction_file = './benchmark_predictions/{}_{}.txt'.format(BENCHMARK, NAME) # if os.path.isfile(prediction_file): # c = 0 # while os.path.isfile(prediction_file): # c += 1 # prediction_file = './benchmark_predictions/{}_{}_{}.txt'.format( # BENCHMARK, NAME, c) accuracy = 0 sum_of_weights = 0 weighted_correct = 0 for i in range(FIRST_ID-1, LAST_ID): (x, y), true_direction, weight = get_pair(i, BENCHMARK) if true_direction == 0: continue df = pd.DataFrame(index=[0], columns=['SampleID', 'A', 'B'], dtype='object') df.SampleID.iloc[0] = 'pair1' df.A.iloc[0] = x df.B.iloc[0] = y model = cgnn.GNN(backend="TensorFlow") predictions = model.predict_dataset(df, printout='_printout.csv') predictions = pd.DataFrame(predictions, columns=["Predictions"]) if predictions.iloc[0, 0] > 0: predicted_direction = 1 else: predicted_direction = -1 if predicted_direction == true_direction:
prediction_file = './benchmark_predictions/{}_{}.txt'.format(BENCHMARK, NAME) if os.path.isfile(prediction_file): c = 0 while os.path.isfile(prediction_file): c += 1 prediction_file = './benchmark_predictions/{}_{}_{}.txt'.format( BENCHMARK, NAME, c) np.random.seed(1) accuracy = 0 sum_of_weights = 0 weighted_correct = 0 for i in range(FIRST_ID-1, LAST_ID): (x, y), true_direction, weight = get_pair( i, BENCHMARK, subsample_size=SUBSAMPLE) if true_direction == 0: continue scaler = MinMaxScaler(scale) x, y = scaler.fit_transform(np.array((x, y)).T).T minimizer = nifty5.RelaxedNewton(controller=nifty5.GradientNormController( tol_rel_gradnorm=TOL_REL_GRADNORM, iteration_limit=ITERATION_LIMIT, convergence_level=5, )) bcm = bayesian_causal_model_nifty.cause_model_shallow.CausalModelShallow( N_bins=N_BINS, noise_var=NOISE_VAR,
"bcs_10samples", "tcep", "tcep_20samples", ]: fig = plt.figure(figsize=(16, 16)) if benchmark.startswith("bcs"): subplots = fig.subplots(10, 10) num_plots = 100 elif benchmark.startswith("tcep"): subplots = fig.subplots(11, 10) num_plots = 108 idx = -1 for i in range(num_plots): (x, y), true_direction, w = get_pair(i, benchmark) if true_direction == -1: c = color2 elif true_direction == +1: c = color1 else: continue idx += 1 ax = subplots[idx // 10, idx % 10] ax.scatter(x, y, s=2, c=c) ax.tick_params( axis="both", which="both", bottom=False, top=False, left=False,
def main(): parser = BCMParser() args = get_args(parser) verbosity = args.verbosity benchmark = args.benchmark print( f"performing {args.benchmark} benchmark for ids {args.first_id} to {args.last_id},\n" f"with N_bins: {args.n_bins},\n" f"noise variance: {args.noise_variance}\n" f"power spectrum beta: {args.power_spectrum_beta_str}\n" f"power spectrum f: {args.power_spectrum_f_str}\n" f"rho: {args.rho}\n" f"scale_max: {args.scale_max}\n" f"storing results with suffix {args.name}" ) power_spectrum_beta = lambda q: eval(args.power_spectrum_beta_str) power_spectrum_f = lambda q: eval(args.power_spectrum_f_str) scale = (0, args.scale_max) prediction_file = "./benchmark_predictions/{}_{}.txt".format(benchmark, args.name) if os.path.isfile(prediction_file): c = 0 while os.path.isfile(prediction_file): c += 1 prediction_file = "./benchmark_predictions/{}_{}_{}.txt".format( benchmark, args.name, c ) sum_of_weights = 0 weighted_correct = 0 for rep in range(args.repetitions): np.random.seed(rep) for i in range(args.first_id - 1, args.last_id): (x, y), true_direction, weight = get_pair( i, benchmark, subsample_size=args.subsample ) if true_direction == 0: continue scaler = MinMaxScaler(scale) x, y = scaler.fit_transform(np.array((x, y)).T).T bcm = bayesian_causal_model.cause_model_shallow.CausalModelShallow( N_bins=args.n_bins, noise_var=args.noise_variance, rho=args.rho, power_spectrum_beta=power_spectrum_beta, power_spectrum_f=power_spectrum_f, ) bcm.set_data(x, y) H1 = bcm.get_evidence(direction=1, verbosity=verbosity - 1) H2 = bcm.get_evidence(direction=-1, verbosity=verbosity - 1) predicted_direction = 1 if int(H1 < H2) else -1 if predicted_direction == true_direction: fore = colorama.Fore.GREEN weighted_correct += weight else: fore = colorama.Fore.RED sum_of_weights += weight accuracy = weighted_correct / sum_of_weights if verbosity > 0: print( "dataset {}, {} true direction: {}, predicted direction {}\n" "H1: {:.2e},\n H2: {:.2e},\n{}" "accuracy so far: {:.2f}".format( i, fore, true_direction, predicted_direction, H1, H2, colorama.Style.RESET_ALL, accuracy, ) ) with open(prediction_file, "a") as f: f.write("{} {} {} {}\n".format(i + 1, predicted_direction, H1, H2)) print("accuracy: {:.2f}".format(accuracy)) benchmark_information = { "benchmark": benchmark, "n_bins": args.n_bins, "noise_var": args.noise_var, "rho": args.rho, "power_spectrum_beta": args.power_spectrum_beta_str, "power_spectrum_f": args.power_spectrum_f_str, "accuracy": accuracy, "prediction_file": prediction_file, } with open("benchmark_predictions/benchmarks_meta.txt", "a") as f: f.write(str(benchmark_information) + "\n")