def train_golden(RUN_TYPE, opts):
print(opts.CLASSNAME, RUN_TYPE)
# loading testing data
# (remember, here we are testing on ALL golden... so it doesn't matter what the test fold is
golden_1, golden_2 = data_io.load_splits(test_fold=0)
train_files = golden_1 + golden_2
test_files = golden_2
if small:
test_files = test_files[:3]
train_files = train_files[:3]
train_X, train_y = data_io.load_data(
train_files, opts.SPEC_TYPE, opts.LEARN_LOG, opts.CLASSNAME, opts.A, opts.B)
test_X, test_y = data_io.load_data(
test_files, opts.SPEC_TYPE, opts.LEARN_LOG, opts.CLASSNAME, opts.A, opts.B)
for idx in range(opts.ENSEMBLE_MEMBERS):
logging_dir = data_io.base + 'predictions/%s/%d/%s/' % (RUN_TYPE, idx, opts.CLASSNAME)
force_make_dir(logging_dir)
# sys.stdout = ui.Logger(logging_dir + 'log.txt')
opts.height = train_X[0].shape[0]
with open(logging_dir + 'network_opts.yaml', 'w') as f:
yaml.dump(opts, f, default_flow_style=False)
train_and_test(train_X, test_X, train_y, test_y, test_files, logging_dir, opts)
def exp(dataset, data_seed, init_seed):
'''
dataset - name of dataset
data_seed - data_seed corresponds to train/dev/test split
init_seed - seed for initializing NN weights
'''
print('running {} on {}'.format(FLAGS.model, dataset))
tf.reset_default_graph()
adj, subgraphs, features, labels, train_mask, val_mask, test_mask = load_data(
dataset, data_seed)
features = preprocess_features(features)
# if early_stopping is not used, then put validation data into the testing data
if FLAGS.early_stop == 0:
mask = np.logical_or(val_mask, test_mask)
test_mask = mask
val_mask = mask
config = tf.ConfigProto()
config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1
#config.log_device_placement = True
config.gpu_options.allow_growth = True
train_loss = []
train_acc = []
valid_loss = []
valid_acc = []
with tf.Graph().as_default():
random.seed(init_seed)
np.random.seed(init_seed)
tf.set_random_seed(init_seed)
with tf.Session(config=config) as sess:
model, support, placeholders = build_model(adj, features,
labels.shape[1],
subgraphs)
sess.run(tf.global_variables_initializer())
def evaluate(labels_mask, noise=0., dropout=0.):
feed_dict_val = construct_feed_dict(features, support, labels,
labels_mask, placeholders,
noise, dropout)
outs_val = sess.run([model.loss, model.accuracy],
feed_dict=feed_dict_val)
return outs_val[0], outs_val[1]
start_t = time.time()
for epoch in range(FLAGS.epochs):
feed_dict = construct_feed_dict(features, support, labels,
train_mask, placeholders,
FLAGS.fisher_noise,
FLAGS.dropout)
feed_dict.update({tf.keras.backend.learning_phase(): 1})
outs = sess.run([model.opt_op, model.loss, model.accuracy],
feed_dict=feed_dict)
train_loss.append(outs[1])
train_acc.append(outs[2])
# Validation
outs = evaluate(val_mask)
valid_loss.append(outs[0])
valid_acc.append(outs[1])
if (epoch + 1) % 10 == 0:
print("Epoch:", '%04d' % (epoch + 1), "train_loss=",
"{:.5f}".format(train_loss[-1]), "train_acc=",
"{:.5f}".format(train_acc[-1]), "val_loss=",
"{:.5f}".format(valid_loss[-1]), "val_acc=",
"{:.5f}".format(valid_acc[-1]))
#print( 'perterbation radius:', sess.run( pradius ) )
if FLAGS.early_stop == 0:
if epoch > 10 and (train_loss[-1] > 1.5 * train_loss[0]
or np.isnan(train_loss[-1])):
print("Early stopping at epoch {}...".format(epoch))
break
elif FLAGS.early_stop == 1: # simple early stopping
if epoch > 20 and valid_loss[-1] > np.mean( valid_loss[-10:] ) \
and valid_acc[-1] < np.mean( valid_acc[-10:] ):
print("Early stopping at epoch {}...".format(epoch))
break
elif FLAGS.early_stop == 2: # more strict conditions
if epoch > 100 \
and np.mean( valid_loss[-10:] ) > np.mean( valid_loss[-100:] ) \
and np.mean( valid_acc[-10:] ) < np.mean( valid_acc[-100:] ):
print("Early stopping at epoch {}...".format(epoch))
break
else:
print('unknown early stopping strategy:', FLAGS.early_stop)
sys.exit(0)
test_loss, test_acc = evaluate(test_mask)
sec_per_epoch = (time.time() - start_t) / epoch
print("Test set results:", "loss=", "{:.5f}".format(test_loss),
"accuracy=", "{:.5f}".format(test_acc), "epoch_secs=",
"{:.2f}".format(sec_per_epoch))
tf.reset_default_graph()
return {
'train_loss': train_loss,
'train_acc': train_acc,
'valid_loss': valid_loss,
'valid_acc': valid_acc,
'test_loss': test_loss,
'test_acc': test_acc,
}
def predict(A, B, ENSEMBLE_MEMBERS, SPEC_TYPE, CLASSNAME, HWW_X, HWW_Y,
DO_AUGMENTATION, LEARN_LOG, NUM_FILTERS, WIGGLE_ROOM,
CONV_FILTER_WIDTH, NUM_DENSE_UNITS, DO_BATCH_NORM, MAX_EPOCHS,
LEARNING_RATE):
# Loading data
golden_1, golden_2 = data_io.load_splits(test_fold=0)
test_files = golden_1 + golden_2
test_X, test_y = data_io.load_data(test_files, SPEC_TYPE, LEARN_LOG,
CLASSNAME, A, B)
# # creaging samplers and batch iterators
test_sampler = SpecSampler(64,
HWW_X,
HWW_Y,
False,
LEARN_LOG,
randomise=False,
seed=10,
balanced=True)
height = test_X[0].shape[0]
net = train_helpers.create_net(height, HWW_X, LEARN_LOG, NUM_FILTERS,
WIGGLE_ROOM, CONV_FILTER_WIDTH,
NUM_DENSE_UNITS, DO_BATCH_NORM)
x_in = net['input'].input_var
test_output = lasagne.layers.get_output(net['prob'], deterministic=True)
pred_fn = theano.function([x_in], test_output)
test_sampler = SpecSampler(64,
HWW_X,
HWW_Y,
False,
LEARN_LOG,
randomise=False,
seed=10,
balanced=False)
y_preds_proba = collections.defaultdict(list)
y_gts = {}
def bal_acc(y_true, y_pred_class):
total = {}
total['tm'] = y_true.shape[0]
total['tp'] = np.logical_and(y_true == y_pred_class, y_true == 1).sum()
total['tn'] = np.logical_and(y_true == y_pred_class, y_true == 0).sum()
total['fp'] = np.logical_and(y_true != y_pred_class, y_true == 0).sum()
total['fn'] = np.logical_and(y_true != y_pred_class, y_true == 1).sum()
A = float(total['tp']) / sum(total[key] for key in ['tp', 'fn'])
B = float(total['tn']) / sum(total[key] for key in ['fp', 'tn'])
return (A + B) / 2.0
# Load network weights
for idx in range(ENSEMBLE_MEMBERS):
print "Predicting with ensemble member: %d" % idx
weights = pickle.load(open(load_path % (train_name, idx, classname)))
lasagne.layers.set_all_param_values(net['prob'], weights)
#######################
# TESTING
for fname, spec, y in zip(test_files, test_X, test_y):
probas = []
y_true = []
for Xb, yb in test_sampler([spec], [y]):
probas.append(pred_fn(Xb))
y_true.append(yb)
y_preds_proba[fname].append(np.vstack(probas))
y_gts[fname] = np.hstack(y_true)
all_pred_prob = np.vstack(y_preds_proba[fname][-1]
for fname in test_files)
all_pred = np.argmax(all_pred_prob, axis=1)
all_gt = np.hstack(y_gts[fname] for fname in test_files)
print bal_acc(all_gt, all_pred)
# aggregating ensemble members
all_probs = []
all_gt = []
for fname in test_files:
combined_preds_prob = np.stack(y_preds_proba[fname]).mean(0)
combined_preds = np.argmax(combined_preds_prob, axis=1)
y_gt = y_gts[fname]
with open(predictions_savedir + fname, 'w') as f:
pickle.dump([y_gt, combined_preds_prob], f, -1)
all_probs.append(combined_preds)
all_gt.append(y_gt)
y_pred_class = np.hstack(all_probs)
y_true = np.hstack(all_gt)
print "Combined:", bal_acc(y_true, y_pred_class)
def train(data_path, output_model, transfer_learning_flag):
num_labels = 6
print('[INFO] Load data ... ')
x_train, y_train = dio.load_data(
os.path.join(data_path, 'vaihingen_train.hdf5'))
print('[INFO] Training samples: {}'.format(x_train.shape))
x_val, y_val = dio.load_data(os.path.join(data_path, 'vaihingen_val.hdf5'))
print('[INFO] Validation samples: {}'.format(x_val.shape))
print('[INFO] preprocess the input data (normalization, centering) ... ')
x_train = preprocess_input(x_train, mode='tf')
x_val = preprocess_input(x_val, mode='tf')
print('[INFO] preprocess the labels ... ')
y_train -= 1
y_val -= 1
y_train = to_categorical(y_train, num_labels)
y_val = to_categorical(y_val, num_labels)
print('[INFO] load model ... ')
if args.n_gpus > 1:
with tf.device('/device:CPU:0'):
resnet50_fcn_model_tmpl = model_generator.generate_resnet50_fcn(
use_pretraining=transfer_learning_flag)
resnet50_fcn_model_tmpl.summary()
print("[INFO] using {} GPUs".format(args.n_gpus))
resnet50_fcn_model = multi_gpu_model(resnet50_fcn_model_tmpl,
gpus=args.n_gpus)
else:
resnet50_fcn_model = model_generator.generate_resnet50_fcn(
use_pretraining=transfer_learning_flag)
resnet50_fcn_model.summary()
print('[INFO] compile model ... ')
resnet50_fcn_model.compile(optimizer=keras.optimizers.Adam(),
loss=keras.losses.categorical_crossentropy,
metrics=['accuracy'])
batch_size = BATCH_SIZE #*args.n_gpus ### compare with fixed batch size, strong scaling (?)
time_callback = TimeHistory()
resnet50_fcn_model.fit(
x_train,
y_train,
batch_size=batch_size,
epochs=args.n_epochs,
validation_data=(x_val, y_val),
#steps_per_epoch=len(x_train)//batch_size,
callbacks=[time_callback])
print(time_callback.total_duration)
resnet50_fcn_model.save_weights(output_model)
_mn = time_callback.total_duration / args.n_epochs
mn = np.mean(time_callback.durations)
sd = np.std(time_callback.durations, ddof=1)
print("[INFO] Mean: ", _mn, mn)
return ParamEntry(datetime.now(), os.path.basename(__file__),
time_callback.val_acc, time_callback.val_loss,
time_callback.total_duration, mn, sd, args.augmentation,
args.transfer_learning, batch_size, args.n_epochs,
args.n_gpus)
def test():
data = load_data("test", input_type.CSV)
def main(args):
input_table, out_table, trans_cols, method, paras = parse_cmd(args)
input_data = load_data(input_table, input_type.CSV)
def doCV():
SEED = 42
rnd = np.random.RandomState(SEED)
model_lr = linear_model.LogisticRegression(C=3)
model_rf = ensemble.RandomForestClassifier(
n_estimators=10, min_samples_split=10, compute_importances=False, n_jobs=2, random_state=rnd, verbose=2
)
print "loading data for random forest..."
y, X = data_io.load_data_pd("train_orig.csv", use_labels=True)
_, X_test = data_io.load_data_pd("test_orig.csv", use_labels=False)
xtrain = getRFX(X)
xtest = getRFX_test(X_test)
xtrain = xtrain[:, 1:]
xtest = xtest[:, 1:]
xtrain.dump("num_train.dat")
xtest.dump("num_test.dat")
print "dumped..!"
print "loading data for logistic regression..."
ysp, Xsp = data_io.load_data("train_orig.csv")
y_testsp, X_testsp = data_io.load_data("test_orig.csv", use_labels=False)
# === one-hot encoding === #
# we want to encode the category IDs encountered both in
# the training and the test set, so we fit the encoder on both
encoder = preprocessing.OneHotEncoder()
# print Xsp.shape, X_testsp.shape
encoder.fit(np.vstack((Xsp, X_testsp)))
Xsp = encoder.transform(Xsp) # Returns a sparse matrix (see numpy.sparse)
X_testsp = encoder.transform(X_testsp)
print "starting cross validation..."
nFeatures = X.shape[0]
niter = 10
cv = cross_validation.ShuffleSplit(nFeatures, n_iter=niter, test_size=0.2, random_state=rnd)
mean_auc = 0.0
i = 0
for train, test in cv:
xtrain = X.ix[train]
ytrain = y[train]
xtest = X.ix[test]
ytest = y[test]
xtrain_sp = Xsp[train]
xtest_sp = X_testsp[test]
ytrainsp = ysp[train]
xtrain = getRFX(xtrain)
xtest = getRFX_test(xtest)
xtrain = xtrain[:, 1:]
xtest = xtest[:, 1:]
print "fitting random forest...."
model_rf.fit(xtrain, ytrain)
preds_rf = model_rf.predict_proba(xtest)[:, 1]
print "fitting logistic regression..."
model_lr.fit(xtrain_sp, ytrainsp)
preds_lr = model_lr.predict_proba(xtest_sp)[:, 1]
preds = [np.mean(x) for x in zip(preds_rf, preds_lr)]
fpr, tpr, _ = metrics.roc_curve(ytest, preds)
roc_auc = metrics.auc(fpr, tpr)
print "AUC (fold %d/%d): %f" % (i + 1, niter, roc_auc)
mean_auc += roc_auc
i += 1
print "Mean AUC: ", mean_auc / niter
def main():
np.random.seed(0)
# Quick run
# vocab_size = 1000
# max_len = 600
# n_samples = 128
vocab_size = 2000
max_len = 1000
n_samples = 128
noise_size = 100
data, labels = load_data(max_len=max_len,
vocab_size=vocab_size,
n_samples=n_samples)
generative_optimizer = Adam(lr=1e-4)
discriminative_optimizer = Adam(lr=1e-3)
# In: (None, 100) <-- rnd noise
# Out: (None, 1000, 2000) <-- sentence of (max_len) words encoded in (vocab_size)
generative_model = build_generative(noise_size, max_len, vocab_size)
generative_model.compile(loss='binary_crossentropy',
optimizer=generative_optimizer)
# print(generative_model.summary())
# In: (None, 1000, 2000) <-- sentence of (max_len) words encoded in (vocab_size)
# Out: (None, 1) <-- probability of the sentence being real
discriminative_model = build_discriminative(max_len, vocab_size)
discriminative_model.compile(loss='binary_crossentropy',
optimizer=discriminative_optimizer)
# print(discriminative_model.summary())
# Stacked GAN
# In: (None, 100) <-- rnd noise
# Out: (None, 1) <-- probability of the sentence being real
gan = build_gan(noise_size, discriminative_model, generative_model)
gan.compile(loss='binary_crossentropy', optimizer=generative_optimizer)
# print(gan.summary())
# -- Training the discriminator alone
print('=' * 10 + 'Training discriminative model' + '=' * 10)
print('-' * 10 + 'Building training data' + '-' * 10)
training_samples, training_outputs = generate_mixed_data(
data.train,
generative_model,
noise_size=noise_size,
vocab_size=vocab_size,
real_samples_size=100,
generated_samples_size=100)
print('Training samples shape: ', training_samples.shape)
print('Training outputs shape: ', training_outputs.shape)
print('-' * 10 + 'Building testing data' + '-' * 10)
testing_samples, testing_outputs = generate_mixed_data(
data.test,
generative_model,
noise_size=noise_size,
vocab_size=vocab_size,
real_samples_size=100,
generated_samples_size=100)
print('Testing samples shape: ', testing_samples.shape)
print('Testing outputs shape: ', testing_outputs.shape)
print('-' * 10 + 'Running the training process' + '-' * 10)
make_trainable(discriminative_model, True)
discriminative_model.fit(training_samples,
training_outputs,
batch_size=128,
epochs=2)
print('-' * 10 + 'Evaluating the discriminative model' + '-' * 10)
scores = discriminative_model.evaluate(testing_samples, testing_outputs)
print('Loss on testing samples: {:.2%}'.format(scores))
losses = {"d": [], "g": []}
try:
change_lr(gan, 1e-4)
change_lr(discriminative_model, 1e-3)
losses = train(nb_epochs=6000,
batch_size=32,
training_data=data.train,
discriminative_model=discriminative_model,
generative_model=generative_model,
gan_model=gan,
noise_size=noise_size,
vocab_size=vocab_size,
losses=losses)
export('1', losses, discriminative_model, generative_model, gan)
change_lr(gan, 1e-5)
change_lr(discriminative_model, 1e-4)
losses = train(nb_epochs=2000,
batch_size=32,
training_data=data.train,
discriminative_model=discriminative_model,
generative_model=generative_model,
gan_model=gan,
noise_size=noise_size,
vocab_size=vocab_size,
losses=losses)
export('2', losses, discriminative_model, generative_model, gan)
change_lr(gan, 1e-6)
change_lr(discriminative_model, 1e-5)
losses = train(nb_epochs=2000,
batch_size=32,
training_data=data.train,
discriminative_model=discriminative_model,
generative_model=generative_model,
gan_model=gan,
noise_size=noise_size,
vocab_size=vocab_size,
losses=losses)
export('3', losses, discriminative_model, generative_model, gan)
except KeyboardInterrupt as _:
export('quitedInBetween', losses, discriminative_model,
generative_model, gan)