def demo(): # Start Example TCP socket client for message in samplelist: # see samplelist in /data_streams/samples.py SocketClient(message=message).start() # Start HTTP example client HTTPClient().start() # Start Example Random Number Stream DataStream(random_nr_config, random_nr).start()
def test(): with open('config_judge.json', encoding='utf-8') as infile: config = json.load(infile) int2bool = {1: True, 0: False} sentiment_words_path = config["train_config"]["SENTIMENT_WORDS_PATH"] batch_size = config["train_config"]["BATCH_SIZE"] is_loop = int2bool[config["train_config"]["Is_LOOP"]] is_sort = int2bool[config["train_config"]["IS_SORT"]] dropout_rate = config["train_config"]["DROPOUT_RATE"] nb_classes = config["train_config"]["NB_CLASSES"] attention_dim = config["train_config"]["ATTENTION_DIM"] nb_hops = config["train_config"]["NB_HOPS"] drop_template_path = config["train_config"]["DROP_JUDGE_TEMPLATE_PATH"] use_bert = int2bool[config["train_config"]["USE_BERT"]] optimizer = config["train_config"]["OPTIMIZER"] learning_rate = config["train_config"]["LEARNING_RATE"] grad_clipper = config["train_config"]["GRAD_CLIPPER"] drop_judge_dev_path = config["train_config"]["DROP_JUDGE_DEV_PATH"] best_path = config["train_config"]["BEST_PATH"] question2targets_path = config["train_config"]["QUESTION2TARGETS_PATH"] use_extra_feature = config["train_config"]["USE_EXTRA_FEATURE"] ner_dict_size = config["train_config"]["NER_DICT_SIZE"] pos_dict_size = config["train_config"]["POS_DICT_SIZE"] extra_feature_dim = config["train_config"]["EXTRA_FEATURE_DIM"] ner_dict_path = config["train_config"]["NER_DICT_PATH"] pos_dict_path = config["train_config"]["POS_DICT_PATH"] rnn_dim = config["train_config"]["RNN_DIM"] lambda_l2 = config["train_config"]["LAMBDA_L2"] ans_max_len = config["train_config"]["ANS_MAX_LEN"] que_max_len = config["train_config"]["QUE_MAX_LEN"] sentiment_polarity_multiple = config["train_config"]["POLARITY_MULTIPLE"] use_w2v = True if use_bert: use_w2v = False char_voc_path = config["w2v_config"]["CHAR_VOC_PATH"] char_embedding_matrix_path = config["w2v_config"][ "CHAR_EMBEDDING_MATRIX_PATH"] word_voc_path = config["w2v_config"]["WORD_VOC_PATH"] word_embedding_matrix_path = config["w2v_config"][ "WORD_EMBEDDING_MATRIX_PATH"] bert_model_path = config["bert_config"]["BERT_MODEL_PATH"] bert_config_file = config["bert_config"]["CONFIG_FILE"] bert_checkpoint_path = config["bert_config"]["INIT_CHECKPOINT"] bert_voc_path = config["bert_config"]["VOC_FILE"] sen2id_path = config["bert_config"]["SEN2ID_PATH"] judge_samples, _, _ = read_file(drop_judge_dev_path) judge_template, _, _ = read_file(drop_template_path) max_sequence_len = max( max([len(sample['question']) for sample in judge_samples]), max([len(sample['answer']) for sample in judge_samples])) with open(char_voc_path, 'rb') as infile: char_voc = pickle.load(infile) with open(word_voc_path, 'rb') as infile: word_voc = pickle.load(infile) bert_encoder = BertEncoder(model_root=bert_model_path, bert_config_file=bert_config_file, init_checkpoint=bert_checkpoint_path, vocab_file=bert_voc_path, max_sequence_len=max_sequence_len, embedding_batch=3, embedding_matrix_path=None, sen2id_path=sen2id_path, vec_dim=768) instances_judge_dev = make_instances(judge_samples, char_voc, word_voc, sentiment_words_path, ner_dict_path=ner_dict_path, pos_dict_path=pos_dict_path, use_extra_feature=use_extra_feature, question2targets=question2targets, is_training=False, need_augment=False) instances_judge_dev_with_match_result(instances_judge_dev) data_stream_judge_dev = DataStream(instances=instances_judge_dev, is_shuffle=False, is_loop=is_loop, batch_size=batch_size, ans_max_len=ans_max_len, que_max_len=que_max_len, use_bert=use_bert, bert_encoder=bert_encoder, is_sort=is_sort) with tf.Graph().as_default(): with tf.variable_scope("Model", reuse=False, initializer=tf.glorot_uniform_initializer()): answer_understander_dev = AnswerUnderstander( use_bert=use_bert, use_w2v=use_w2v, rnn_unit='lstm', dropout_rate=dropout_rate, optimizer=optimizer, learning_rate=learning_rate, grad_clipper=grad_clipper, global_step=None, attention_dim=attention_dim, nb_hops=nb_hops, rnn_dim=rnn_dim, lambda_l2=lambda_l2, is_training=False, sentiment_polarity_multiple=sentiment_polarity_multiple, nb_classes=nb_classes, use_extra_feature=use_extra_feature, ner_dict_size=ner_dict_size, pos_dict_size=pos_dict_size, extra_feature_dim=extra_feature_dim, ans_max_len=ans_max_len, que_max_len=que_max_len, char_w2v_embedding_matrix_path=char_embedding_matrix_path, word_w2v_embedding_matrix_path=word_embedding_matrix_path) saver = tf.train.Saver() sess = tf.Session() initializer = tf.global_variables_initializer() sess.run(initializer) saver.restore(sess, best_path) judge_acc = evaluation(sess, answer_understander_dev, data_stream_judge_dev, 'result_{}.txt'.format(loop_index)) print("the final judge accuracy:{}".format(judge_acc)) return judge_acc
# need_augment=False) # data_stream_judge_template = DataStream(instances=instances_judge_template, # is_shuffle=False, # is_loop=is_loop, # batch_size=batch_size, # ans_max_len=ans_max_len, # que_max_len=que_max_len, # use_bert=use_bert, # bert_encoder=bert_encoder, # is_sort=False) data_stream_judge_dev = DataStream(instances=instances_judge_dev, is_shuffle=False, is_loop=is_loop, batch_size=batch_size, ans_max_len=ans_max_len, que_max_len=que_max_len, use_bert=use_bert, bert_encoder=bert_encoder, is_sort=False) # get match answer # question2id, question2answer, question2label = \ # gather_question_template(judge_template) with tf.Graph().as_default(): with tf.variable_scope("Model", reuse=None, initializer=tf.glorot_uniform_initializer()): answer_understander_dev = AnswerUnderstander( use_bert=use_bert,
parser.add_argument('--seed_data', default=1) parser.add_argument('--batch_size', default=16, type=int) parser.add_argument('--image_size', default=512, type=int) parser.add_argument('--learning_rate', type=float, default=0.0001) parser.add_argument('--data_dir', type=str, default='/scratch/users/aruch/nerdd') args = parser.parse_args() print(args) # fixed random seeds rng_data = np.random.RandomState(args.seed_data) rng = np.random.RandomState(args.seed) theano_rng = MRG_RandomStreams(rng.randint(2 ** 15)) lasagne.random.set_rng(np.random.RandomState(rng.randint(2 ** 15))) # load CIFAR-10 ds_train = DataStream(d=args.data_dir, img_size=args.image_size, batch_size=args.batch_size, h5_name="train_sub.hdf5") ds_test = DataStream(d=args.data_dir, img_size=args.image_size, batch_size=args.batch_size, h5_name="val_sub.hdf5") print("DATA LOADERS CREATED") # specify discriminative model disc_layers = [ll.InputLayer(shape=(None, 1, args.image_size, args.image_size))] disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5)) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 32, (3,3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5)) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 64, (3,3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5)) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 96, (3,3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5)) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 192, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5))
use_extra_feature=use_extra_feature, question2targets=question2targets, is_training=True, need_augment=True) instances_train, instances_dev = split_train_dev(instances, dev_size=0.2) drop_instances_to_excel(instances_train, drop_train_path) drop_instances_to_excel(instances_dev, drop_path) # drop_instances_to_excel(ins_template, drop_template_path) instances_train, instances_valid = split_train_dev(instances_train, dev_size=0.1) data_stream_train = DataStream(instances=instances_train, is_shuffle=is_shuffle, is_loop=is_loop, batch_size=batch_size, ans_max_len=ans_max_len, que_max_len=que_max_len, use_bert=use_bert, bert_encoder=bert_encoder, is_sort=is_sort) data_stream_valid = DataStream(instances=instances_valid, is_shuffle=False, is_loop=is_loop, batch_size=batch_size, ans_max_len=ans_max_len, que_max_len=que_max_len, use_bert=use_bert, bert_encoder=bert_encoder, is_sort=is_sort) with tf.Graph().as_default():