def main(argv): del argv # Unused. df = io_utils.read_dataframe_from_hdf5(FLAGS.original_df) ## Generate the percentile-normalized embeddings emb_df_normalized = normalize_ljosa(df) ## Generate the post-factor analysis embeddings np.random.seed(0) emb_df_fa = transform.factor_analysis(emb_df_normalized, 0.15, 50) io_utils.write_dataframe_to_hdf5(emb_df_fa, FLAGS.post_fa_path) io_utils.write_dataframe_to_hdf5(emb_df_normalized, FLAGS.post_normliazation_path)
def main(argv): del argv emb_df_clean = io_utils.read_dataframe_from_hdf5(FLAGS.input_df) if "treatment_group" not in emb_df_clean.index.names: raise ValueError("Must have treatment_group in embeddings index names.") contents = load_contents(FLAGS.transformation_file) ## dictionary to save things save_dict = {} ## Get steps over training steps = list(contents.keys()) steps.remove("params") steps = np.sort(steps) ## Truncate list of steps steps = steps[:FLAGS.num_steps] ## embeddings without unevaluated compound emb_df_valid = transform.drop_unevaluated_comp(emb_df_clean) if "treatment_group" not in emb_df_valid.index.names: raise ValueError("Must have treatment_group in embeddings index names.") ## list of compounds and number of compounds comp_list = emb_df_valid.index.get_level_values( level=metadata.COMPOUND).unique() n_comp = len(comp_list) ## Set up data structure for leave-one-out cross validation list_of_comp_set = [] for i in range(n_comp): comp_set = {} comp_set["b"] = comp_list[i] comp_set["a"] = list(set(comp_list).difference([comp_list[i]])) list_of_comp_set.append(comp_set) ## Cross validation training with leave-one-out and variable stopping time. (steps_max, cross_validated_scores) = cross_val_train( emb_df_clean, contents, steps, list_of_comp_set, n_comp, percent_norm=FLAGS.percentile_normalize, factor_analys=FLAGS.factor_analysis) ## Find first and last timesteps used, to use for bootstraps boot_steps = [steps[i] for i, v in enumerate(steps) if np.max(steps_max) >= v >= np.min(steps_max)] metrics_dict = evaluate_metrics(contents, emb_df_clean, steps_max, boot_steps, list_of_comp_set, num_bootstrap=FLAGS.num_bootstrap, percent_norm=FLAGS.percentile_normalize, factor_analys=FLAGS.factor_analysis) save_dict["metrics_dict"] = metrics_dict ## time steps where max cross validation results were found save_dict["list_of_time_step_max"] = steps_max ## accuracy for not same compound or batch, obtained at time_step_max ## for each individual compound. save_dict["metrics_dict"]["wdn"]["cross_val_scores"] = cross_validated_scores with gfile.GFile(FLAGS.output_file, mode="w") as f: f.write(pickle.dumps(save_dict))
def testReadHDF5(self): df = pandas.DataFrame(FOLD_DATA) actual = io_utils.read_dataframe_from_hdf5(self.base_path) self.assertTrue(actual.equals(df))