def test_qaranker_local_integration(self): relations = Relations.read(self.qa_path + "/relations.txt") assert len(relations) == 4 text_set = TextSet.read_csv(self.qa_path + "/question_corpus.csv") assert text_set.get_uris() == ["Q1", "Q2"] transformed = text_set.tokenize().normalize().word2idx( ).shape_sequence(5) relation_pairs = TextSet.from_relation_pairs(relations, transformed, transformed) pair_samples = relation_pairs.get_samples() assert len(pair_samples) == 2 for sample in pair_samples: assert list(sample.feature.shape) == [2, 10] assert np.allclose(sample.label.to_ndarray(), np.array([[1.0], [0.0]])) relation_lists = TextSet.from_relation_lists(relations, transformed, transformed) relation_samples = relation_lists.get_samples() assert len(relation_samples) == 2 for sample in relation_samples: assert list(sample.feature.shape) == [2, 10] assert list(sample.label.shape) == [2, 1] knrm = KNRM(5, 5, self.glove_path, word_index=transformed.get_word_index()) model = Sequential().add(TimeDistributed(knrm, input_shape=(2, 10))) model.compile("sgd", "rank_hinge") model.fit(relation_pairs, batch_size=2, nb_epoch=2) print(knrm.evaluate_ndcg(relation_lists, 3)) print(knrm.evaluate_map(relation_lists))
parser.add_option("-l", "--learning_rate", dest="learning_rate", default="0.001") parser.add_option("-m", "--model", dest="model") parser.add_option("--output_path", dest="output_path") (options, args) = parser.parse_args(sys.argv) sc = init_nncontext("QARanker Example") q_set = TextSet.read_csv(options.data_path + "/question_corpus.csv", sc, int(options.partition_num)).tokenize().normalize()\ .word2idx(min_freq=2).shape_sequence(int(options.question_length)) a_set = TextSet.read_csv(options.data_path+"/answer_corpus.csv", sc, int(options.partition_num)).tokenize().normalize()\ .word2idx(min_freq=2, existing_map=q_set.get_word_index())\ .shape_sequence(int(options.answer_length)) train_relations = Relations.read(options.data_path + "/relation_train.csv", sc, int(options.partition_num)) train_set = TextSet.from_relation_pairs(train_relations, q_set, a_set) validate_relations = Relations.read(options.data_path + "/relation_valid.csv", sc, int(options.partition_num)) validate_set = TextSet.from_relation_lists(validate_relations, q_set, a_set) if options.model: knrm = KNRM.load_model(options.model) else: word_index = a_set.get_word_index() knrm = KNRM(int(options.question_length), int(options.answer_length), options.embedding_file, word_index) model = Sequential().add( TimeDistributed( knrm, input_shape=(2, int(options.question_length) + int(options.answer_length))))