def init_level(name, level): """Initialize a given level of the search by saving all of the structures which need to be evaluated.""" if not storage.exists(experiment_dir(name)): raise RuntimeError('Experiment %s not yet initialized.' % name) params = storage.load(params_file(name)) splits = storage.load(splits_file(name)) if level == 1: init_structures = ['g'] else: init_structures = storage.load(winning_structure_file(name, level - 1)) structure_pairs = list_structure_pairs(init_structures, params.rules, params.expand_noise) data_matrix = storage.load(data_file(name)) X_train = data_matrix lab = None node_mat = np.zeros([params.num_splits * params.num_samples, 200, 200, 2]) pruned_pairs = [] os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' config = tf.ConfigProto() config.gpu_options.allow_growth = True # ; config.gpu_options.per_process_gpu_memory_fraction = 0.4 sess = tf.Session(config=config) c = Classifier() real = tf.placeholder(shape=[None, 200, 200, 2], dtype=tf.float32) c_out = tf.reduce_mean(tf.nn.softmax(c(real), axis=-1), axis=0, keepdims=True) c_params = c.vars saver = tf.train.Saver(c_params) sess.run(tf.global_variables_initializer()) saver.restore(sess, "saved_model/d") for (init_structure, structure) in structure_pairs: for split_id in range(params.num_splits): for sample_id in range(params.num_samples): train_rows, train_cols, test_rows, test_cols = splits[split_id] X_train = data_matrix[train_rows[:, nax], train_cols[nax, :]] if level == 1: init = X_train.sample_latent_values(np.zeros((X_train.m, X_train.n)), 1.) prev_model = recursive.GaussianNode(init, 'scalar', 1.) else: try: prev_model = storage.load(samples_file(name, level - 1, init_structure, split_id, sample_id)) except: print("structure", grammar.pretty_print(init_structure), "never exists") continue if isinstance(prev_model, recursive.Decomp): prev_model = prev_model.root node, old_dist, rule = recursive.find_changed_node(prev_model, init_structure, structure) lab = labelize(rule) node_mat[split_id * params.num_samples + sample_id] = pad(random_shrink(node.value())) if_continue = sess.run(tf.nn.top_k(c_out, 3), feed_dict={real: node_mat}) if lab in if_continue.indices: print("transformation structure ", grammar.pretty_print(init_structure), "->", grammar.pretty_print(structure), "i.e. lab ", lab, " included with top_k", if_continue) pruned_pairs.append((init_structure, structure)) else: print("transformation structure ", grammar.pretty_print(init_structure), "->", grammar.pretty_print(structure), "i.e. lab ", lab, " emitted, with top_k", if_continue) structure_pairs = pruned_pairs storage.dump(structure_pairs, structures_file(name, level))
def collect_scores(name, level, structure): """Collect the held-out predictive log-likelihood scores for all CV splits and order them according to the indices of the original data matrix.""" params = storage.load(params_file(name)) splits = storage.load(splits_file(name)) row_loglik_all = [] col_loglik_all = [] failed = False for split_id, (train_rows, train_cols, test_rows, test_cols) in enumerate(splits): row_loglik_curr, col_loglik_curr = [], [] num_samples = params.num_samples for sample_id in range(num_samples): try: row_loglik_single, col_loglik_single = storage.load(scores_file(name, level, structure, split_id, sample_id)) except: row_loglik_single = np.nan * np.ones(len(test_rows)) col_loglik_single = np.nan * np.ones(len(test_cols)) failed = True row_loglik_curr.append(row_loglik_single) col_loglik_curr.append(col_loglik_single) row_loglik_all.append(np.array(row_loglik_curr)) col_loglik_all.append(np.array(col_loglik_curr)) if failed: print termcolor.colored(' failed: %s' % grammar.pretty_print(structure), 'red') storage.dump((row_loglik_all, col_loglik_all), collected_scores_file(name, level, structure))
def collect_scores(name, level, structure): """Collect the held-out predictive log-likelihood scores for all CV splits and order them according to the indices of the original data matrix.""" params = storage.load(params_file(name)) splits = storage.load(splits_file(name)) row_loglik_all = [] col_loglik_all = [] failed = False for split_id, (train_rows, train_cols, test_rows, test_cols) in enumerate(splits): row_loglik_curr, col_loglik_curr = [], [] num_samples = params.num_samples for sample_id in range(num_samples): try: row_loglik_single, col_loglik_single = storage.load(scores_file(name, level, structure, split_id, sample_id)) except Exception as e: if isinstance(e, FileNotFoundError): return row_loglik_single = np.nan * np.ones(len(test_rows)) col_loglik_single = np.nan * np.ones(len(test_cols)) failed = True row_loglik_curr.append(row_loglik_single) col_loglik_curr.append(col_loglik_single) row_loglik_all.append(np.array(row_loglik_curr)) col_loglik_all.append(np.array(col_loglik_curr)) if failed: print(termcolor.colored(' failed: %s' % grammar.pretty_print(structure), 'red')) storage.dump((row_loglik_all, col_loglik_all), collected_scores_file(name, level, structure))
def fit_winning_sequence(name, sample_id): """After the sequence of models is identified, sample factorizations from each of the models on the full data matrix.""" data_matrix = storage.load(data_file(name)) sequence = sequence_of_structures(name) params = storage.load(params_file(name)) decomps = recursive.fit_sequence(sequence, data_matrix, gibbs_steps=params.gibbs_steps) storage.dump(decomps, winning_samples_file(name, sample_id))
def run(params): if isinstance(params, str): params = get_params(params) v = gnp.garray(datasets.SubsampledMNISTInfo.load().as_matrix()) v = 0.999 * v + 0.001 * 0.5 tparams = rbm_training.TrainingParams.defaults('pcd') rbm, _ = rbm_training.train_rbm(v, 20, tparams, show_progress=True) storage.dump(rbm, rbm_file(params['name']))
def init_level(name, level): """Initialize a given level of the search by saving all of the structures which need to be evaluated.""" if not storage.exists(experiment_dir(name)): raise RuntimeError('Experiment %s not yet initialized.' % name) params = storage.load(params_file(name)) if level == 1: init_structures = ['g'] else: init_structures = storage.load(winning_structure_file(name, level - 1)) structure_pairs = list_structure_pairs(init_structures, params.rules, params.expand_noise) storage.dump(structure_pairs, structures_file(name, level))
def after_step(self, rbm, trainer, i): it = i + 1 save = it in self.expt.save_after display = it in self.expt.show_after if save: if self.expt.save_particles: storage.dump(trainer.fantasy_particles, self.expt.pcd_particles_file(it)) storage.dump(rbm, self.expt.rbm_file(it)) if hasattr(trainer, 'avg_rbm'): storage.dump(trainer.avg_rbm, self.expt.avg_rbm_file(it)) storage.dump(time.time() - self.t0, self.expt.time_file(it)) if 'particles' in self.subset and (save or display): fig = rbm_vis.show_particles(rbm, trainer.fantasy_particles, self.expt.dataset, display=display, figtitle='PCD particles ({} updates)'.format(it)) if display: pylab.gcf().canvas.draw() if save: misc.save_image(fig, self.expt.pcd_particles_figure_file(it)) if 'gibbs_chains' in self.subset and (save or display): fig = diagnostics.show_chains(rbm, trainer.fantasy_particles, self.expt.dataset, display=display, figtitle='Gibbs chains (iteration {})'.format(it)) if save: misc.save_image(fig, self.expt.gibbs_chains_figure_file(it)) if 'objective' in self.subset: self.log_prob_tracker.update(rbm, trainer.fantasy_particles) if display: pylab.gcf().canvas.draw()
def compute_init_samples(name, level, structure, split_id, sample_id): """For one of the high-performing structures in the previous level, sample from the posterior so that it can be used to initialize the current level. This is only needed if params.save_samples == False. The log-likelihood scores are saved as well for purposes of determining statistical significance of the improvement over the previous level.""" if level == 1: return init_structure = init_structure_for(name, level-1, structure) root = sample_from_model(name, level-1, init_structure, structure, split_id, sample_id) storage.dump(root, init_samples_file(name, level, structure, split_id, sample_id)) row_loglik, col_loglik = evaluate_decomp(name, level-1, init_structure, split_id, sample_id, root) storage.dump((row_loglik, col_loglik), init_scores_file(name, level, structure, split_id, sample_id))
def save_exact_log_Z(expt): """Compute the exact partition functions for small RBMs.""" if isinstance(expt, str): expt = get_experiment(expt) tr_expt = get_training_expt(expt) for it in tr_expt.save_after: for avg in AVG_VALS: print 'iteration', it, avg try: rbm = load_rbm(expt, it, avg) except: continue log_Z = tractable.exact_partition_function(rbm) storage.dump(log_Z, expt.log_Z_file(it, avg))
def save_exact_samples(expt): """Save exact samples from the RBM distribution.""" if isinstance(expt, str): expt = get_experiment(expt) tr_expt = get_training_expt(expt) for it in tr_expt.save_after: for avg in AVG_VALS: print 'Iteration', it, avg try: rbm = load_rbm(expt, it, avg) except: continue states = tractable.exact_samples(rbm, expt.annealing.num_samples) storage.dump(states, expt.gibbs_states_file(it, avg))
def compute_init_samples(name, level, structure, split_id, sample_id): """For one of the high-performing structures in the previous level, sample from the posterior so that it can be used to initialize the current level. This is only needed if params.save_samples == False. The log-likelihood scores are saved as well for purposes of determining statistical significance of the improvement over the previous level.""" if level == 1: return init_structure = init_structure_for(name, level-1, structure) try: root = sample_from_model(name, level-1, init_structure, structure, split_id, sample_id) except: return storage.dump(root, init_samples_file(name, level, structure, split_id, sample_id)) row_loglik, col_loglik = evaluate_decomp(name, level-1, init_structure, split_id, sample_id, root) storage.dump((row_loglik, col_loglik), init_scores_file(name, level, structure, split_id, sample_id))
def init_experiment(name, data_matrix, params, components=None, clean_data_matrix=None): """Initialize the structure search by saving the matrix, and possibly auxiliary information, to files, and generating cross-validation splits.""" check_required_directories() if not storage.exists(experiment_dir(name)): storage.mkdir(experiment_dir(name)) storage.dump(params, params_file(name)) splits = nfold_cv(data_matrix.m, data_matrix.n, params.num_splits) storage.dump(splits, splits_file(name)) if clean_data_matrix is not None: storage.dump(clean_data_matrix, clean_data_file(name)) storage.dump(data_matrix, data_file(name)) if components is not None: storage.dump(components, components_file(name))
def run_ais(expt, save=True, show_progress=False): """Run AIS for all the RBMs, and save the estimated log partition functions and the final particles.""" if isinstance(expt, str): expt = get_experiment(expt) mkl.set_num_threads(1) tr_expt = get_training_expt(expt) for it in tr_expt.save_after: for avg in AVG_VALS: print 'iteration', it, avg t0 = time.time() try: rbm = load_rbm(expt, it, avg) except: continue moments = compute_moments(tr_expt, rbm) brm = moments.full_base_rate_moments() init_rbm = binary_rbms.RBM.from_moments(brm) path = ais.GeometricRBMPath(init_rbm, rbm) schedule = np.linspace(0., 1., expt.annealing.num_steps) state, log_Z, _ = ais.ais(path, schedule, expt.annealing.num_particles, show_progress=show_progress) if save: storage.dump(log_Z, expt.log_Z_file(it, avg)) storage.dump(state, expt.final_states_file(it, avg)) storage.dump(time.time() - t0, expt.time_file(it, avg))
def run_gibbs(expt, save=True, show_progress=False): """Run Gibbs chains starting from the AIS particles (sampled proportionally to their weights), and save the final particles.""" if isinstance(expt, str): expt = get_experiment(expt) tr_expt = get_training_expt(expt) for it in tr_expt.save_after: for avg in AVG_VALS: print 'Iteration', it, avg try: rbm = load_rbm(expt, it, avg) except: continue log_Z = storage.load(expt.log_Z_file(it, avg)).as_numpy_array() final_states = storage.load(expt.final_states_file(it, avg)) # sample the states proportionally to the Z estimates p = log_Z - np.logaddexp.reduce(log_Z) p /= p.sum( ) # not needed in theory, but numpy complains if it doesn't sum exactly to 1 idxs = np.random.multinomial( 1, p, size=expt.annealing.num_particles).argmax(1) states = binary_rbms.RBMState(final_states.v[idxs, :], final_states.h[idxs, :]) if show_progress: pbar = misc.pbar(expt.gibbs_steps) for st in range(expt.gibbs_steps): states = rbm.step(states) if show_progress: pbar.update(st) if show_progress: pbar.finish() if save: storage.dump(states, expt.gibbs_states_file(it, avg))
def run_model(name, level, init_structure, structure, split_id, sample_id, save=True, save_sample=False): """Sample from the posterior given the training data, and evaluate on heldout rows/columns.""" params = storage.load(params_file(name)) t0 = time.time() root = sample_from_model(name, level, init_structure, structure, split_id, sample_id) if save and (save_sample or params.save_samples): storage.dump(root, samples_file(name, level, structure, split_id, sample_id)) print 'Saved.' row_loglik, col_loglik = evaluate_decomp(name, level, init_structure, split_id, sample_id, root) print 'Row:', row_loglik.mean() print 'Col:', col_loglik.mean() if save: storage.dump((row_loglik, col_loglik), scores_file(name, level, structure, split_id, sample_id)) storage.dump( time.time() - t0, running_time_file(name, level, structure, split_id, sample_id))
def after_step(self, rbm, trainer, i): it = i + 1 save = it in self.expt.save_after display = it in self.expt.show_after if save: if self.expt.save_particles: storage.dump(trainer.fantasy_particles, self.expt.pcd_particles_file(it)) storage.dump(rbm, self.expt.rbm_file(it)) if hasattr(trainer, 'avg_rbm'): storage.dump(trainer.avg_rbm, self.expt.avg_rbm_file(it)) storage.dump(time.time() - self.t0, self.expt.time_file(it)) if 'particles' in self.subset and (save or display): fig = rbm_vis.show_particles( rbm, trainer.fantasy_particles, self.expt.dataset, display=display, figtitle='PCD particles ({} updates)'.format(it)) if display: pylab.gcf().canvas.draw() if save: misc.save_image(fig, self.expt.pcd_particles_figure_file(it)) if 'gibbs_chains' in self.subset and (save or display): fig = diagnostics.show_chains( rbm, trainer.fantasy_particles, self.expt.dataset, display=display, figtitle='Gibbs chains (iteration {})'.format(it)) if save: misc.save_image(fig, self.expt.gibbs_chains_figure_file(it)) if 'objective' in self.subset: self.log_prob_tracker.update(rbm, trainer.fantasy_particles) if display: pylab.gcf().canvas.draw()
def run_model(name, level, init_structure, structure, split_id, sample_id, save=True, save_sample=False): """Sample from the posterior given the training data, and evaluate on heldout rows/columns.""" params = storage.load(params_file(name)) t0 = time.time() root = sample_from_model(name, level, init_structure, structure, split_id, sample_id) if save and (save_sample or params.save_samples): storage.dump(root, samples_file(name, level, structure, split_id, sample_id)) print 'Saved.' row_loglik, col_loglik = evaluate_decomp(name, level, init_structure, split_id, sample_id, root) print 'Row:', row_loglik.mean() print 'Col:', col_loglik.mean() if save: storage.dump((row_loglik, col_loglik), scores_file(name, level, structure, split_id, sample_id)) storage.dump(time.time() - t0, running_time_file(name, level, structure, split_id, sample_id))
def save_winning_structures(name, level): storage.dump(winning_structures(name, level), winning_structure_file(name, level))
def sample_matrix(name, sample_id, level, size, output_file): decomps = storage.load(winning_samples_file(name, sample_id)) data = decomps[level].sample_matrix(size) storage.dump(data, output_file) print('Samples saved at: "{}".'.format(output_file))