def run(): if len(sys.argv) < 3: print("** Usage: python3 " + sys.argv[0] + " <<Model Directory>> <<Test Set>>") sys.exit(1) np.random.seed(42) model_dir = sys.argv[1] config = Config.load( ['./default.conf', os.path.join(model_dir, 'model.conf')]) model = create_model(config) test_data = load_data(sys.argv[2], config.dictionary, config.grammar, config.max_length) print("unknown", unknown_tokens) with tf.Graph().as_default(): tf.set_random_seed(1234) with tf.device('/cpu:0'): model.build() test_eval = Seq2SeqEvaluator(model, config.grammar, test_data, 'test', config.reverse_dictionary, beam_size=config.beam_size, batch_size=config.batch_size) loader = tf.train.Saver() with tf.Session() as sess: loader.restore(sess, os.path.join(model_dir, 'best')) test_eval.eval(sess, save_to_file=True)
def config(self): if not hasattr(self, '_config'): path = config_path() if os.path.exists(path): self._config = Config.load(path) else: self._config = Config() self._config.path = path return self._config
def run(): if len(sys.argv) < 4: print("** Usage: python3 " + sys.argv[0] + " <<Model Directory>> <<Everything Set>> <<Test Set>>") sys.exit(1) np.random.seed(42) model_dir = sys.argv[1] config = Config.load( ['./default.conf', os.path.join(model_dir, 'model.conf')]) model = create_model(config) everything_labels, everything_label_lengths = load_programs( config, sys.argv[2]) test_labels, test_label_lengths = load_programs(config, sys.argv[3]) #test_labels, test_label_lengths = sample(config.grammar, test_labels, test_label_lengths) print("unknown", unknown_tokens) with tf.Graph().as_default(): tf.set_random_seed(1234) model.build() loader = tf.train.Saver() train_bag_of_tokens = bag_of_tokens(config, everything_labels, everything_label_lengths) V, mean = pca_fit(train_bag_of_tokens, n_components=2) eval_bag_of_tokens = bag_of_tokens(config, test_labels, test_label_lengths) transformed = pca_transform(eval_bag_of_tokens, V, mean) with tf.Session() as sess: loader.restore(sess, os.path.join(model_dir, 'best')) transformed = transformed.eval(session=sess) programs = reconstruct_programs(test_labels, test_label_lengths, config.grammar.tokens) show_pca(transformed, programs)
def load_language(app, tokenizer_service, tag, model_dir): config = Config.load([ './default.conf', './default.' + tag + '.conf', os.path.join(model_dir, 'model.conf') ]) model = create_model(config) graph = tf.Graph() session = tf.Session(graph=graph) with graph.as_default(): # Force everything to run on CPU, we run on single inputs so there is not much point # on going through the GPU with tf.device('/cpu:0'): model.build() loader = tf.train.Saver() with session.as_default(): loader.restore(session, os.path.join(model_dir, 'best')) tokenizer = Tokenizer(tokenizer_service, tag) app.add_language(tag, LanguageContext(tag, tokenizer, session, config, model)) print('Loaded language ' + tag)
def run(): if len(sys.argv) < 3: print("** Usage: python3 " + sys.argv[0] + " <<Model Directory>> <<Test Set>>") sys.exit(1) np.random.seed(42) model_dir = sys.argv[1] config = Config.load( ['./default.conf', os.path.join(model_dir, 'model.conf')]) model = create_model(config) test_data = load_data(sys.argv[2], config.dictionary, config.grammar, config.max_length) with tf.Graph().as_default(): tf.set_random_seed(1234) model.build() loader = tf.train.Saver() inputs, input_lengths, parses, labels, label_lengths = test_data final_encoder_state = tf.concat(nest.flatten( model.final_encoder_state), axis=1) final_encoder_size = final_encoder_state.get_shape()[1] final_states = OrderedDict() with tf.Session() as sess: loader.restore(sess, os.path.join(model_dir, 'best')) # capture all the final encoder states for input_batch, input_length_batch, parse_batch, label_batch, label_length_batch in get_minibatches( [inputs, input_lengths, parses, labels, label_lengths], config.batch_size): feed_dict = model.create_feed_dict(input_batch, input_length_batch, parse_batch) state_array = sess.run(final_encoder_state, feed_dict=feed_dict) #print state_array.shape for state, input, input_length, label, length in zip( state_array, input_batch, input_length_batch, label_batch, label_length_batch): label = label[:length] program = ' '.join( config.grammar.tokens[x] for x in label) # if is_function(config.grammar.tokens[x])) if not program in final_states: final_states[program] = [(state, input[:input_length])] else: final_states[program].append( (state, input[:input_length])) prog_array = [prog for prog in final_states ] #if len(final_states[prog]) > 1] prog_index = dict() num_programs = len(prog_array) print('num programs', num_programs) centers = np.zeros((num_programs, final_encoder_size), dtype=np.float32) for i, program in enumerate(prog_array): prog_index[program] = i centers[i] = np.mean([x[0] for x in final_states[program]], axis=0) eval_data = [] with open(sys.argv[3]) as fp: for line in fp: sentence, gold, predicted, _ = line.strip().split('\t') if gold == predicted: continue gold += ' <<EOS>>' predicted += ' <<EOS>>' if gold in prog_index and predicted in prog_index: sentence_vector, sentence_length = vectorize( sentence, config.dictionary, config.max_length) gold_index = prog_index[gold] gold_center = centers[gold_index] predicted_index = prog_index[predicted] predicted_center = centers[predicted_index] eval_data.append( (gold, predicted, gold_center, predicted_center, sentence_vector, sentence_length)) #print(np.linalg.norm(gold_center-predicted_center), gold, predicted, sentence, sep='\t') elif gold not in prog_index: #print('no gold', gold, file=sys.stderr) pass elif predicted not in prog_index: #print('no predicted', file=sys.stderr) pass with tf.Session() as sess: loader.restore(sess, os.path.join(model_dir, 'best')) def flip(list_of_tuples): inner_length = len(list_of_tuples[0]) tuple_of_lists = [[x[i] for x in list_of_tuples] for i in range(inner_length)] return tuple_of_lists with open('./eval.tsv', 'w') as out: for gold_batch, predicted_batch, gold_center_batch, predicted_center_batch, input_batch, input_length_batch in get_minibatches( flip(eval_data), config.batch_size): parse_batch = np.zeros( (len(input_batch), 2 * config.max_length - 1), dtype=np.bool) feed_dict = model.create_feed_dict(input_batch, input_length_batch, parse_batch) state_array = sess.run(final_encoder_state, feed_dict=feed_dict) assert len(state_array) == len(gold_batch) for state, input, input_length, gold, predicted, gold_center, predicted_center in zip( state_array, input_batch, input_length_batch, gold_batch, predicted_batch, gold_center_batch, predicted_center_batch): gold_predicted_dist = np.linalg.norm(gold_center - predicted_center) sentence_gold_dist = np.linalg.norm(state - gold_center) sentence_predicted_dist = np.linalg.norm( state - predicted_center) sentence = ' '.join(config.reverse_dictionary[x] for x in input[:input_length]) print(gold_predicted_dist, sentence_gold_dist, sentence_predicted_dist, gold, predicted, sentence, sep='\t', file=out) print('written eval.tsv') num_good_sentences = np.zeros((num_programs, ), dtype=np.int32) sum_good_distance = np.zeros((num_programs, ), dtype=np.float32) num_bad_sentences = np.zeros((num_programs, ), dtype=np.int32) sum_bad_distance = np.zeros((num_programs, ), dtype=np.float32) for i, program in enumerate(prog_array): num_good_sentences[i] = len(final_states[program]) for encoding, sentence in final_states[program]: dist = np.linalg.norm(encoding - centers[i]) sum_good_distance[i] += dist # negative examples for negative in np.random.choice(prog_array, size=(10, ), replace=False): if negative == program: continue num_bad_sentences[i] += len(final_states[negative]) for negative_enc, negative_sentence in final_states[negative]: dist = np.linalg.norm(negative_enc - centers[i]) sum_bad_distance[i] += dist avg_good_distance = sum_good_distance / num_good_sentences avg_bad_distance = sum_bad_distance / num_bad_sentences with open('./encoded.csv', 'w') as fp: writer = csv.writer(fp) writer.writerows( zip(num_good_sentences, num_bad_sentences, avg_good_distance, avg_bad_distance, sum_good_distance, sum_bad_distance))
def run(): if len(sys.argv) < 4: print("** Usage: python3 " + sys.argv[0] + " <<Model Directory>> <<Train Set>> <<Test Set>>") sys.exit(1) np.random.seed(42) model_dir = sys.argv[1] config = Config.load( ['./default.conf', os.path.join(model_dir, 'model.conf')]) model = create_model(config) train_data = load_data(sys.argv[2], config.dictionary, config.grammar, config.max_length) pca_data = load_data(sys.argv[3], config.dictionary, config.grammar, config.max_length) print("unknown", unknown_tokens) with tf.Graph().as_default(): model.build() loader = tf.train.Saver() with tf.Session() as sess: loader.restore(sess, os.path.join(model_dir, 'best')) inputs, input_lengths, parses, _, _ = train_data final_encoder_state = tf.concat(nest.flatten( model.final_encoder_state), axis=1) final_encoder_size = final_encoder_state.get_shape()[1] final_states_arrays = [] # capture all the final encoder states for input_batch, input_length_batch, parse_batch in get_minibatches( [inputs, input_lengths, parses], config.batch_size): feed_dict = model.create_feed_dict(input_batch, input_length_batch, parse_batch) state_array = sess.run(final_encoder_state, feed_dict=feed_dict) #print state_array.shape final_states_arrays.append(state_array) X = np.concatenate(final_states_arrays, axis=0) assert X.shape == (len(inputs), final_encoder_size) X = tf.constant(X) mean = tf.reduce_mean(X, axis=0) centered_X = X - mean S, U, V = tf.svd(centered_X) # take only the top 2 components V = V[:2] V_array, mean_array = sess.run([V, mean]) inputs, input_lengths, parses, labels, label_lengths = pca_data X = final_encoder_state centered_X = X - tf.constant(mean_array) transformed_X = tf.matmul(centered_X, tf.constant(V_array.T)) feed_dict = model.create_feed_dict(inputs, input_lengths, parses) X_pca = sess.run(transformed_X, feed_dict=feed_dict) if False: sentences = reconstruct_sentences(inputs, input_lengths, config.reverse_dictionary) else: sentences = reconstruct_sentences(labels, label_lengths, config.grammar.tokens) show_pca(X_pca, sentences)
def run(): if len(sys.argv) < 3: print("** Usage: python3 " + sys.argv[0] + " <<Model Directory>> <<Train Set>> [<<Dev Set>>]") sys.exit(1) np.random.seed(42) model_dir = sys.argv[1] model_conf = os.path.join(model_dir, 'model.conf') config = Config.load(['./default.conf', model_conf]) model = create_model(config) train_data = load_data(sys.argv[2], config.dictionary, config.grammar, config.max_length) if len(sys.argv) > 3: dev_data = load_data(sys.argv[3], config.dictionary, config.grammar, config.max_length) else: dev_data = None print("unknown", unknown_tokens) try: os.mkdir(model_dir) except OSError: pass if not os.path.exists(model_conf): config.save(model_conf) with tf.Graph().as_default(): tf.set_random_seed(1234) model.build() init = tf.global_variables_initializer() saver = tf.train.Saver(max_to_keep=config.n_epochs) train_eval = Seq2SeqEvaluator(model, config.grammar, train_data, 'train', config.reverse_dictionary, beam_size=config.beam_size, batch_size=config.batch_size) dev_eval = Seq2SeqEvaluator(model, config.grammar, dev_data, 'dev', config.reverse_dictionary, beam_size=config.beam_size, batch_size=config.batch_size) trainer = Trainer(model, train_data, train_eval, dev_eval, saver, model_dir=model_dir, max_length=config.max_length, batch_size=config.batch_size, n_epochs=config.n_epochs, dropout=config.dropout) with tf.Session() as sess: # Run the Op to initialize the variables. sess.run(init) #sess = tf_debug.LocalCLIDebugWrapperSession(sess) #sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan) # Fit the model best_dev, best_train = trainer.fit(sess) print("best train", best_train) print("best dev", best_dev)
def app_config(): schema = Config() config_data = json.load(open('league_config.json')) return schema.load(config_data)