def loadSyntheticData(self): cache_file = os.path.join(self.CACHE_DIR, 'lsh_programs.pkl') if not self.evict_cache and os.path.isfile(cache_file): data = load_json(cache_file) prog_items = data['raw_programs'] anon_progs = data['anon_programs'] else: standard_path = self.sampledDataPath + '/standard/train' + SYNTH_NAME uniform_path = self.sampledDataPath + '/uniform/train' + SYNTH_NAME tempered_path = self.sampledDataPath + '/tempered/train' + SYNTH_NAME standardDict = pickle.load(open(standard_path, "rb" )) uniformDict = pickle.load(open(uniform_path, "rb" )) temperedDict = pickle.load(open(tempered_path, "rb" )) all_dicts = [standardDict, uniformDict, temperedDict] # this step is not stable across different runs if caching forest # so this needs to be cached too prog_items = list(standardDict.keys() | uniformDict.keys() | temperedDict.keys()) anon_progs = [self.multi_dict_get(prog, all_dicts) for prog in prog_items] data = dict(raw_programs=prog_items, anon_programs=anon_progs) os.makedirs(self.CACHE_DIR, exist_ok=True) save_json(data, cache_file) # if we dont load cache here, we should regenerate forest too self.evict_cache = True return prog_items, anon_progs
def finalise(self): self.logger.info("Saving final versions of model...") self.save_checkpoint(filename='final.pth.tar') save_json(self.accuracies, os.path.join(self.config.out_dir, 'accuracies.json')) self.summary_writer.export_scalars_to_json( os.path.join(self.config.summary_dir, "all_scalars.json".format())) self.summary_writer.close()
def backup(self): save_json(self.accuracies, os.path.join(self.config.out_dir, 'accuracies.json')) self.summary_writer.export_scalars_to_json( os.path.join(self.config.summary_dir, "all_scalars.json".format())) self.summary_writer.close() self.logger.info("Backing up current version of model...") self.save_checkpoint(filename='backup.pth.tar')
def run(self, results_dir, problem, exp_dir, error_rate=0): grammar_dir = paths.grammar_path(problem) inf_e = EngineGuided(grammar_dir, exp_dir) N = len(inf_e.dataset) n_lim = 100 data, raw_prgs = self.create_data_loader(inf_e.dataset, N) data_loader = DataLoader(data, batch_size=1) tqdm_batch = tqdm(data_loader, total=N) time_data = [] failed = [] for i, data_list in enumerate(tqdm_batch): program_args, _, _, _, _ = data_list[:-4], data_list[ -4], data_list[-3], data_list[-2], data_list[-1] program = raw_prgs[i] if error_rate > 0: program_args = self.dropout_words(program_args, error_rate) num_steps = self.infer_matches(inf_e, program, program_args, n_lim=n_lim) if num_steps < 0: failed.append(program) time_data.append(num_steps) error_rate_str = str(error_rate).replace('.', '_') # TODO: make sure failed programs are safe to save out_json = dict(failed_progs=failed, time_data=time_data, error_rate=error_rate) out_file = os.path.join(results_dir, f'data_err_{error_rate_str}.json') save_json(out_json, out_file) plt.hist(time_data, bins=50, color='purple') plt.title( f'Num samples to recover validation program (N={N}, max_steps={n_lim}, error_rate={error_rate})' ) plt.xlabel('Number of steps (neg. = not recovered)') plt.ylabel('Frequency') plt.savefig( os.path.join(results_dir, f'validation_recovery_err_{error_rate_str}.png'))
def make_vocabs(problem, domain='education'): vocab_paths = paths.vocab_paths(problem, domain) os.makedirs(vocab_paths['data_path'], exist_ok=True) all_programs, all_anon_programs = [], [] for sampling_strategy in ['standard', 'uniform', 'tempered']: (counts_paths, labels_paths,rv_order_paths, tiers_paths, anon_mapping_paths, all_rvs_path) = \ paths.raw_data_paths(problem, 'train', domain, sampling_strategy) n_shards = len(counts_paths) for i in range(n_shards): programs_i, anon_programs_i, _, _, _, _ = load_raw_rubric_data( counts_paths[i], labels_paths[i], rv_order_paths[i], tiers_paths[i], anon_mapping_paths[i]) all_programs.extend(programs_i) all_anon_programs.extend(anon_programs_i) vocab = build_vocab_rnn(all_programs, character_level=False) char_vocab = build_vocab_rnn(all_programs, character_level=True) anon_vocab = build_vocab_rnn(all_anon_programs, character_level=False) anon_char_vocab = build_vocab_rnn(all_anon_programs, character_level=True) io.save_json(vocab, vocab_paths['vocab_path']) io.save_json(char_vocab, vocab_paths['char_vocab_path']) io.save_json(anon_vocab, vocab_paths['anon_vocab_path']) io.save_json(anon_char_vocab, vocab_paths['anon_char_vocab_path'])
def run(self, n, out_dir, grammar_dir, problem): std_data = self._run_base(n, grammar_dir, SAMPLE_STANDARD) unif_data = self._run_base(n, grammar_dir, SAMPLE_UNIFORM) data = dict(standard=std_data, uniform=unif_data, tempered=dict()) for r in [0.01, 0.1, 1.]: for d in [0.3, 0.6, 0.9]: data['tempered'][f'{r}_{d}'] = self._run_base(n, grammar_dir, SAMPLE_TEMPERED, reward=r, discount=d) save_json(data, os.path.join(out_dir, f'{problem}.json'))
def evaluateAgent(self, agent_name, inf_nn): tqdm_batch = tqdm(self.dataloader, total=len(self.student_data)) distance_data = dict() nn_map = dict() for i, data_list in enumerate(tqdm_batch): program_args, program, anon_program = data_list[:2], data_list[ -2], data_list[-1] # for non-anon, non-char model # program_args, program, anon_program = data_list[:4], data_list[-2], data_list[-1] # for anon xor char model # TODO: hack to make it work right now. Investigate why this is happening if program_args[1].item() == 0: print('SKIPPING STUDENT PROG OF LENGTH 0') continue # access 0 index to unbatch the batch of size 1 program = program[0] anon_program = anon_program[0] clean_prog = self.transformRawProgram(program) if agent_name == 'lsh': top_k_progs = self.lsh_nn.findNearestNeighbours(anon_program) elif agent_name == 'random': top_k_progs = self.random_nn.findNearestNeighbours( anon_program) else: top_k_progs = inf_nn.findNearestNeighbours( program, program_args=program_args) if len(top_k_progs) == 0: print(f'SKIPPING programs with 0 top_k for agent={agent_name}') print(clean_prog) continue if len(top_k_progs) != self.top_k: print( f'[WARNING] Found programs with not enough top_k. Expected {self.top_k}, received [{agent_name}_len={len(top_k_progs)}]' ) best_prog, _ = self.getBestProgram(clean_prog, top_k_progs) nn_map[program] = best_prog self.computeDistances(distance_data, clean_prog, top_k_progs) all_data = dict(distances=distance_data, nns=nn_map) save_json(all_data, os.path.join(self.results_dir, f'{agent_name}.json'))
def compareNNs(self): tqdm_batch = tqdm(self.dataloader, total=len(self.student_data)) metadata = dict(top_k=self.top_k) distance_data = dict(metadata=metadata) for i, data_list in enumerate(tqdm_batch): # TODO: this indexing is super hardcoded program_args, program, anon_program = data_list[:2], data_list[ -2], data_list[-1] # for non-anon, non-char model #program_args, program, anon_program = data_list[:4], data_list[-2], data_list[-1] # for anon, not_char model # TODO: hack to make it work right now. Investigate why this is happening if program_args[1].item() == 0: print('SKIPPING STUDENT PROG OF LENGTH 0') continue # access 0 index to unbatch the batch of size 1 program = program[0] anon_program = anon_program[0] clean_prog = self.transformRawProgram(program) top_k_lsh, top_k_inf = self.getNNs(program, anon_program, program_args) if len(top_k_lsh) == 0: print(f'SKIPPING programs with 0 LSH') print(clean_prog) continue if len(top_k_lsh) != self.top_k or len(top_k_inf) != self.top_k: print( f'[WARNING] Found programs with not enough top_k. Expected {self.top_k}, received (lsh={len(top_k_lsh)}, inf={len(top_k_inf)})' ) self.computeDistances(distance_data, clean_prog, top_k_lsh, top_k_inf) save_json( distance_data, os.path.join(self.results_dir, f'recovery_similarity_k_{self.top_k}.json'))
def make_rnn_data(problem, split, domain='education', sampling_strategy='standard'): rnn_paths = paths.rnn_data_paths(problem, split, domain, sampling_strategy) vocab_paths = paths.vocab_paths(problem, domain) os.makedirs(rnn_paths['data_path'], exist_ok=True) (counts_paths, labels_paths, rv_order_paths, tiers_paths, anon_mapping_paths, all_rvs_path) = \ paths.raw_data_paths(problem, split, domain, sampling_strategy) n_shards = len(counts_paths) # get info that has to be collected across all shards max_lens = get_merged_info(counts_paths, labels_paths, rv_order_paths, tiers_paths, anon_mapping_paths) vocab, char_vocab, anon_vocab, anon_char_vocab = load_vocabs(vocab_paths) max_len, char_max_len, anon_max_len, anon_char_max_len = max_lens all_rvs = io.load_json(all_rvs_path) rv_info = create_rv_info(all_rvs) # save all_rvs into rv_info rv_info['values'] = all_rvs data_len = 0 shard_size = 0 for i in range(n_shards): programs_i, anon_programs_i, labels_i, rv_order_i, tiers_i, _ = load_raw_rubric_data( counts_paths[i], labels_paths[i], rv_order_paths[i], tiers_paths[i], anon_mapping_paths[i]) # assumes equally sized shards (except smaller remaining last one) shard_size = max(shard_size, len(programs_i)) data_len += len(programs_i) feat_labels_i = featurise_labels(labels_i, rv_info, all_rvs) feat_rv_order_i, rv_order_lengths_i = featurise_rv_order( rv_order_i, rv_info) feat_programs_i, program_lengths_i = featurise_programs_rnn( programs_i, vocab, max_len) anon_feat_programs_i, anon_program_lengths_i = \ featurise_programs_rnn(anon_programs_i, anon_vocab, anon_max_len) char_feat_programs_i, char_program_lengths_i = \ featurise_programs_rnn(programs_i, char_vocab, char_max_len, character_level=True) anon_char_feat_programs_i, anon_char_program_lengths_i = \ featurise_programs_rnn(anon_programs_i, anon_char_vocab, anon_char_max_len, character_level=True) program_mats_i = dict(programs=feat_programs_i, lengths=program_lengths_i, tiers=tiers_i) char_program_mats_i = dict(programs=char_feat_programs_i, lengths=char_program_lengths_i, tiers=tiers_i) anon_program_mats_i = dict(programs=anon_feat_programs_i, lengths=anon_program_lengths_i, tiers=tiers_i) anon_char_program_mats_i = dict(programs=anon_char_feat_programs_i, lengths=anon_char_program_lengths_i, tiers=tiers_i) rv_order_mats_i = dict(rv_orders=feat_rv_order_i, lengths=rv_order_lengths_i) io.save_pickle(programs_i, rnn_paths['raw_programs_path'].format(i)) io.savemat(program_mats_i, rnn_paths['feat_programs_path'].format(i)) io.savemat(char_program_mats_i, rnn_paths['char_feat_programs_path'].format(i)) # TODO: save raw labels in raw_labels_path io.save_np(feat_labels_i, rnn_paths['feat_labels_path'].format(i)) io.save_pickle(anon_programs_i, rnn_paths['anon_raw_programs_path'].format(i)) io.savemat(anon_program_mats_i, rnn_paths['anon_feat_programs_path'].format(i)) io.savemat(anon_char_program_mats_i, rnn_paths['anon_char_feat_programs_path'].format(i)) io.save_pickle(rv_order_i, rnn_paths['raw_rvOrder_path'].format(i)) io.savemat(rv_order_mats_i, rnn_paths['feat_rvOrder_path'].format(i)) io.save_json(rv_info, rnn_paths['rv_info_path']) metadata = dict(max_len=max_len, char_max_len=char_max_len, anon_max_len=anon_max_len, anon_char_max_len=anon_char_max_len, data_len=data_len, num_shards=n_shards, shard_size=shard_size) io.save_json(metadata, rnn_paths['metadata_path'])
offset_idx = render.index(to_find) end_idx = offset_idx + len(to_find) render = render[:offset_idx] + format_data[key] + render[end_idx:] for rv_set, (idx, n) in rv_data[key]: ret_idxs.append((rv_set, (idx + offset_idx, n))) if len(rvs) > 0: ret_idxs.append((rvs, (0, len(render)))) return nonterminal, render, ret_idxs def tagged_data(data): ret = [] for d in tqdm(data): p = d[0] ret.append(conv_sample(d[1])) return ret if __name__ == "__main__": import argparse arg_parser = argparse.ArgumentParser() arg_parser.add_argument('data_file', type=str, help='which results to process') args = arg_parser.parse_args() data = load_json(args.data_file) conv_data = tagged_data(data) save_json(conv_data, os.path.join('output.json'))
from src.utils.io_utils import save_json if __name__ == "__main__": import argparse arg_parser = argparse.ArgumentParser() arg_parser.add_argument('results_dir', type=str, help='where to save results') args = arg_parser.parse_args() MODEL_DIR = 'experiments/liftoff_final_100k_tempered/2019-04-20--13_57_14' GRAMMAR_DIR = 'src/rubricsampling/grammars/liftoff_hacks' student_data = StudentPrograms('liftoff', include_anonymized=False) dataloader = DataLoader(student_data, batch_size=1, shuffle=False) tqdm_batch = tqdm(dataloader, total=len(student_data)) inf_nn = InferenceNNHighlight(GRAMMAR_DIR, MODEL_DIR) results = [] for i, (seq_tokens, seq_lengths, program) in enumerate(tqdm_batch): program_args = (seq_tokens, seq_lengths) try: program, highlights, decisions = inf_nn.guided_sample(program_args) except: continue results.append((program, highlights)) save_json(results, os.path.join(args.results_dir, 'student_highlights.json'))
def make_scene_graph_data(device, problem, split, sampling_strategy='standard', use_resnet=False): data_paths = paths.scene_graph_data_paths(problem, split, sampling_strategy) os.makedirs(data_paths['data_path'], exist_ok=True) (counts_paths, labels_paths, images_paths, rv_order_paths, tiers_paths, all_rvs_path) = \ paths.raw_scene_graph_data_paths(problem, split, sampling_strategy) n_shards = len(counts_paths) all_rvs = io.load_json(all_rvs_path) rv_info = create_rv_info(all_rvs) # save all_rvs into rv_info rv_info['values'] = all_rvs data_len = 0 shard_size = 0 if use_resnet: # load huge model :( print('loading deep net for feature extraction...') net, expected_input_dim = load_classification_model() net = net.to(device) image_transforms = load_data_transforms(expected_input_dim) for i in range(n_shards): scene_graphs_i, images_i, labels_i, rv_order_i, tiers_i, _ = load_raw_scene_graph_data( counts_paths[i], labels_paths[i], rv_order_paths[i], images_paths[i], tiers_paths[i]) n_items_i = len(scene_graphs_i) if use_resnet: feat_images_i = featurise_images(images_i, net, device, image_transforms) else: feat_images_i = images_i # assumes equally sized shards (except smaller remaining last one) shard_size = max(shard_size, n_items_i) data_len += n_items_i feat_labels_i = featurise_labels(labels_i, rv_info, all_rvs) feat_rv_order_i, rv_order_lengths_i = featurise_rv_order( rv_order_i, rv_info) image_mats_i = dict(images=feat_images_i, tiers=tiers_i) rv_order_mats_i = dict(rv_orders=feat_rv_order_i, lengths=rv_order_lengths_i) io.savemat(image_mats_i, data_paths['feat_images_path'].format(i)) io.save_np(feat_labels_i, data_paths['feat_labels_path'].format(i)) io.save_pickle(rv_order_i, data_paths['raw_rvOrder_path'].format(i)) io.savemat(rv_order_mats_i, data_paths['feat_rvOrder_path'].format(i)) io.save_json(rv_info, data_paths['rv_info_path']) metadata = dict(data_len=data_len, num_shards=n_shards, shard_size=shard_size) io.save_json(metadata, data_paths['metadata_path'])
def getModelConfig(self): return self.inf_e.config def guided_sample(self, program_args): # something that will crash if accessed without setting initAssignments = 1000000 * torch.ones(1, self.inf_e.model.num_nodes) program, highlights, labels, decisions, rvOrder, rvAssignments_pred = \ self.inf_e.renderProgram(program_args, initAssignments) return program, highlights, decisions if __name__ == "__main__": import argparse parser = argparse.ArgumentParser() parser.add_argument('--N', type=int, default=1) args = parser.parse_args() data = [] e = EngineHighlight('src/rubricsampling/grammars/liftoff_hacks') for i in range(args.N): program, highlights = e.renderProgram() print(program) print(highlights) print('----') print() data.append((program, highlights)) save_json(data, os.path.join('in_grammar_highlights.json'))