Python DataProcessor.get_batches示例

示例#1

        model.l2_penalty: l2,
        model.drop_penalty: regularize_drop_penalty
    }
    lstm_feed.update(char_embedding_feed)

    if batch_dataset == 'A':
        run_list = [train_op_A, model.loss_A]
    elif batch_dataset == 'B':
        run_list = [train_op_B, model.loss_B]
    elif batch_dataset == 'C':
        run_list = [train_op_C, model.loss_C]

    _, loss = sess.run(run_list, feed_dict=lstm_feed)

    return loss


test_batches_B = dp.get_batches('cdr_test', batch_size, random=False)
test_batches_C = dp.get_batches('bc_test', batch_size, random=False)

model_path = '/mnt/nfs/work1/mccallum/tbansal/nathan/disjoint-dataset/models/'
save_path = model_path + 'exp1_%d_%.1f.ckpt' % (hidden_size, args.dropout)

saver.restore(sess, save_path)
print('\nModel loaded from %s' % save_path)

print('Best model results:')

run_evaluation(test_batches_B, 'B', 'cdr_test')
run_evaluation(test_batches_C, 'C', 'bc_test')

示例#2

    if batch_dataset == 'A':
        run_list = [train_op_A, model.loss_A]
    elif batch_dataset == 'B':
        run_list = [train_op_B, model.loss_B]
    elif batch_dataset == 'C':
        run_list = [train_op_C, model.loss_C]

    _, loss = sess.run(run_list, feed_dict=lstm_feed)

    return loss


epochs = 6000

train_batches_A = dp.get_batches('cdr_train_weak', batch_size, random=False)
train_batches_B = dp.get_batches('cdr_train_gold', batch_size, random=False)
train_batches_C = dp.get_batches('bc_train', batch_size, random=False)

dev_batches_B = dp.get_batches('cdr_dev', batch_size, random=False)
dev_batches_C = dp.get_batches('bc_dev', batch_size, random=False)

test_batches_B = dp.get_batches('cdr_test', batch_size, random=False)
test_batches_C = dp.get_batches('bc_test', batch_size, random=False)

max_f1 = 0
epoch_last_saved = 0
last_saved = 'N/A'
for i in range(epochs):
    batches_A = dp.get_batches('cdr_train_weak', batch_size)
    batches_B = dp.get_batches('cdr_train_gold', batch_size)

示例#3

        model.input_dropout_keep_prob: 1 - input_dropout,
        model.l2_penalty: l2,
        model.drop_penalty: regularize_drop_penalty
    }
    lstm_feed.update(char_embedding_feed)

    if A:
        _, loss = sess.run([train_op_A, model.loss_A], feed_dict=lstm_feed)
    else:
        _, loss = sess.run([train_op_B, model.loss_B], feed_dict=lstm_feed)

    return loss


epochs = 6000
dev_batches_A = dp.get_batches('cdr_dev', batch_size, random=False)
dev_batches_B = dp.get_batches('bc_dev', batch_size, random=False)
max_f1 = 0
last_saved = 'N/A'
for i in range(epochs):
    batches_A = dp.get_batches('cdr_train_gold', batch_size) + dp.get_batches(
        'cdr_train_weak', batch_size)
    np.random.shuffle(batches_A)
    batches_B = dp.get_batches('bc_train', batch_size)

    if dataset == 'joint':
        l = min(len(batches_A), len(batches_B))
    elif dataset == 'A':
        l = len(batches_A)
    elif dataset == 'B':
        l = len(batches_B)

示例#4

文件： processB.py 项目： ngreenberg/em-crf

        model.input_dropout_keep_prob: 1 - input_dropout,
        model.l2_penalty: l2,
        model.drop_penalty: regularize_drop_penalty
    }
    lstm_feed.update(char_embedding_feed)

    if A:
        _, loss = sess.run([train_op_A, model.loss_A], feed_dict=lstm_feed)
    else:
        _, loss = sess.run([train_op_B, model.loss_B], feed_dict=lstm_feed)

    return loss


epochs = 600
dev_batches_A = dp.get_batches('cdr_dev', batch_size, random=False)
dev_batches_B = dp.get_batches('bc_dev', batch_size, random=False)
max_f1 = 0
last_saved = 0
for i in range(epochs):
    f1_A, _ = run_evaluation(dev_batches_A,
                             'cdr_dev (iteration %d)' % i,
                             A=True)
    f1_B, _ = run_evaluation(dev_batches_B,
                             'bc_dev (iteration %d)' % i,
                             A=False)

    # if np.mean((f1_A, f1_B)) > max_f1:
    if f1_B > max_f1:
        model_path = 'modelpath/bc/'
        save_path = saver.save(

示例#5

    if batch_dataset == 'A':
        run_list = [train_op_A, model.loss_A]
    elif batch_dataset == 'B':
        run_list = [train_op_B, model.loss_B]
    elif batch_dataset == 'C':
        run_list = [train_op_C, model.loss_C]

    _, loss = sess.run(run_list, feed_dict=lstm_feed)

    return loss


epochs = 6000

train_batches_A = dp.get_batches('A_train', batch_size, random=False)
train_batches_B = dp.get_batches('B_train', batch_size, random=False)

dev_batches = dp.get_batches('dev', batch_size, random=False)

test_batches = dp.get_batches('test', batch_size, random=False)

max_f1 = 0
epoch_last_saved = 0
last_saved = 'N/A'
for i in range(epochs):
    batches_A = dp.get_batches('A_train', batch_size)
    batches_B = dp.get_batches('B_train', batch_size)

    l = min(len(batches_A), len(batches_B))

示例#6

文件： loadexp_split.py 项目： ngreenberg/em-crf

        model.middle_dropout_keep_prob: 1 - middle_dropout,
        model.input_dropout_keep_prob: 1 - input_dropout,
        model.l2_penalty: l2,
        model.drop_penalty: regularize_drop_penalty
    }
    lstm_feed.update(char_embedding_feed)

    if batch_dataset == 'A':
        run_list = [train_op_A, model.loss_A]
    elif batch_dataset == 'B':
        run_list = [train_op_B, model.loss_B]
    elif batch_dataset == 'C':
        run_list = [train_op_C, model.loss_C]

    _, loss = sess.run(run_list, feed_dict=lstm_feed)

    return loss


test_batches = dp.get_batches('test', batch_size, random=False)

model_path = 'modelpath/3_sets/'
save_path = model_path + 'exp_split_%d_%.1f.ckpt' % (hidden_size, args.dropout)

saver.restore(sess, save_path)
print('\nModel loaded from %s' % save_path)

print('Best model results:')

run_evaluation(test_batches, 'full', 'test')