def submission():
"""
Generate submission file for the trained models.
"""
print('Loading and compiling models...')
model_systole = get_model()
model_diastole = get_model()
print('Loading models weights...')
model_systole.load_weights('weights_systole23.hdf5')
model_diastole.load_weights('weights_diastole50.hdf5')
# load val losses to use as sigmas for CDF
with open('val_loss.txt', mode='r') as f:
val_loss_systole = float(f.readline())
val_loss_diastole = float(f.readline())
print('Loading validation data...')
X, ids = load_validation_data()
#print('Pre-processing images...')
#X = preprocess(X)
batch_size = 32
print('Predicting on validation data...')
pred_systole = model_systole.predict(X, batch_size=batch_size, verbose=1)
pred_diastole = model_diastole.predict(X, batch_size=batch_size, verbose=1)
# real predictions to CDF
cdf_pred_systole = real_to_cdf(pred_systole, val_loss_systole)
cdf_pred_diastole = real_to_cdf(pred_diastole, val_loss_diastole)
print('Accumulating results...')
sub_systole = accumulate_study_results(ids, cdf_pred_systole)
sub_diastole = accumulate_study_results(ids, cdf_pred_diastole)
# write to submission file
print('Writing submission to file...')
fi = csv.reader(open('data/sample_submission_validate.csv'))
f = open('submission.csv', 'w')
fo = csv.writer(f, lineterminator='\n')
fo.writerow(fi.next())
for line in fi:
idx = line[0]
key, target = idx.split('_')
key = int(key)
out = [idx]
if key in sub_systole:
if target == 'Diastole':
out.extend(list(sub_diastole[key][0]))
else:
out.extend(list(sub_systole[key][0]))
else:
print('Miss {0}'.format(idx))
fo.writerow(out)
f.close()
print('Done.')
def submission():
"""
Generate submission file for the trained models.
"""
logging.info('Loading and compiling models...')
model_systole = get_model()
model_diastole = get_model()
logging.info('Loading models weights...')
model_systole.load_weights('../models/weights/weights_systole_best.hdf5')
model_diastole.load_weights('../models/weights/weights_diastole_best.hdf5')
logging.info('Loading validation data...')
X, ids = load_validation_data()
logging.info('Pre-processing images...')
X = preprocess(X)
batch_size = 32
logging.info('Predicting on validation data...')
pred_systole = model_systole.predict(X, batch_size=batch_size, verbose=1)
pred_diastole = model_diastole.predict(X, batch_size=batch_size, verbose=1)
# real predictions to CDF
cdf_pred_systole = correct_cdf(pred_systole)
cdf_pred_diastole = correct_cdf(pred_diastole)
logging.info('Accumulating results...')
sub_systole = accumulate_study_results(ids, cdf_pred_systole)
sub_diastole = accumulate_study_results(ids, cdf_pred_diastole)
# write to submission file
logging.info('Writing submission to file...')
fi = csv.reader(open('../input/sample_submission_validate.csv'))
f = open('../submissions/submission_13.csv', 'w')
fo = csv.writer(f, lineterminator='\n')
fo.writerow(next(fi))
for line in fi:
idx = line[0]
key, target = idx.split('_')
key = int(key)
out = [idx]
if key in sub_systole:
if target == 'Diastole':
out.extend(list(sub_diastole[key][0]))
else:
out.extend(list(sub_systole[key][0]))
else:
logging.info('Miss {0}'.format(idx))
fo.writerow(out)
f.close()
logging.info('Done.')
def train(args):
device = args.device
load_path = args.load_path
# load data
train_data = load_data('train')
val_data = load_data('validation')
# load model
with tf.device('/gpu:%d' % device):
model = get_model('policy')
# trainer init
optimizer = Config.optimizer
train_step = optimizer.minimize(model.loss)
# init session and server
sess = tf.InteractiveSession()
saver = tf.train.Saver()
if load_path==None:
sess.run(tf.initialize_all_variables())
else:
saver.restore(sess, load_path)
print("Model restored from %s" % load_path)
# accuracy
pred = tf.reshape(model.pred, [-1, 9*10*16])
label = tf.reshape(model.label, [-1, 9*10*16])
correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(label,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
logging.basicConfig(filename='log.txt', level=logging.DEBUG)
# train steps
for i in range(Config.n_epoch):
# training step
batch_data, batch_label = train_data.next_batch(Config.minibatch_size)
input_dict = {model.label:batch_label}
for var, data in zip(model.inputs, batch_data):
input_dict[var]=data
#from IPython import embed;embed()
sess.run(train_step, feed_dict=input_dict)
# evalue step
if (i+1)%Config.evalue_point == 0:
batch_data, batch_label = val_data.next_batch(Config.minibatch_size)
val_dict = {model.label:batch_label}
for var, data in zip(model.inputs, batch_data):
val_dict[var]=data
score = accuracy.eval(feed_dict=val_dict)
print("epoch %d, accuracy is %.2f" % (i,score))
logging.info("epoch %d, accuracy is %.2f" % (i,score))
# save step
if (i+1)%Config.check_point == 0:
save_path = saver.save(sess, "%s/epoch-%d" %(Config.save_path, i))
print("Model saved in file: %s" % save_path)
logging.info("Model saved in file: %s" % save_path)
def export_input_graph(model_folder):
sys.path.append(model_folder)
from model import get_model
with tf.Session() as sess:
model = get_model('policy')
saver = tf.train.Saver()
tf.train.write_graph(sess.graph_def, model_folder, 'input_graph.pb', as_text=True)
def hard_train(data_prefix, prefix, seed, col):
what = ['systole', 'diastole'][col % 2]
print('We are going to train hard {} {}'.format(what, col))
print('Loading training data...')
X, y = load_train_data(data_prefix, seed)
X_train, y_train, X_test, y_test = split_data(X, y, split_ratio=0.2)
model = get_model()
nb_iter = 200
epochs_per_iter = 1
batch_size = 32
min_val = sys.float_info.max
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=15, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=True) # randomly flip images
print('-'*50)
print('Training...')
print('-'*50)
datagen.fit(X_train)
checkpointer_best = ModelCheckpoint(filepath=prefix + "weights_{}_best.hdf5".format(what), verbose=1, save_best_only=True)
checkpointer = ModelCheckpoint(filepath=prefix + "weights_{}.hdf5".format(what), verbose=1, save_best_only=False)
hist = model.fit_generator(datagen.flow(X_train, y_train[:, col], batch_size=batch_size),
samples_per_epoch=X_train.shape[0],
nb_epoch=nb_iter, show_accuracy=False,
validation_data=(X_test, y_test[:, col]),
callbacks=[checkpointer, checkpointer_best],
nb_worker=4)
loss = hist.history['loss'][-1]
val_loss = hist.history['val_loss'][-1]
with open(prefix + 'val_loss.txt', mode='w+') as f:
f.write(str(min(hist.history['val_loss'])))
f.write('\n')
def _main(config, config_idx, train):
base_filename = config.name + '_cfg' + str(config_idx)
logger = set_up_logger('logs/' + base_filename + '.log')
title = '{}: {} ({}) config index {}'.format(__file__, config.name, config.desc, config_idx)
logger.info('START ' + title + '\n\n{}\n'.format(config))
data = get_data(config, train)
if config.device != 'cpu':
assert 'theano' not in sys.modules
import theano.sandbox.cuda
theano.sandbox.cuda.use(config.device)
from model import get_model
model = get_model(config, data)
if not train:
assert config.tst_load_model_path
if not model.load(config.tst_load_model_path):
raise AssertionError('Failed loading model weights from {}'.format(config.tst_load_model_path))
ans_hats = _tst_epoch(config, model, data)
write_test_predictions(ans_hats, config.pred_json_path)
logger.info('END ' + title)
return
# Training loop
epoch_results = []
max_em = -np.inf
max_f1 = -np.inf
np_rng = np.random.RandomState(config.seed // 2)
for epoch in range(1, config.max_num_epochs+1):
trn_loss, trn_acc, trn_samples_per_sec = _trn_epoch(config, model, data, epoch, np_rng)
dev_loss, dev_acc, dev_em, dev_f1 = _dev_epoch(config, model, data)
if dev_em > max_em:
model.save('models/' + base_filename + '_best_em.pkl')
max_em = dev_em
if dev_f1 > max_f1:
model.save('models/' + base_filename + '_best_f1.pkl')
max_f1 = dev_f1
if config.save_freq and epoch % config.save_freq == 0:
model.save('models/' + base_filename + '_e{:03d}.pkl'.format(epoch))
epoch_results.append(
EpochResult(trn_loss, trn_acc, dev_loss, dev_acc, dev_em, dev_f1))
if config.plot:
plot_epoch_results(epoch_results, 'logs/' + base_filename + '.png')
logger.info('\n\nFinished epoch {} for: (config index {}) (samples/sec: {:<.1f})\n{}\n\nResults:\n{}\n\n'.format(
epoch, config_idx, trn_samples_per_sec, config.format_compared(), format_epoch_results(epoch_results)))
logger.info('END ' + title)
def train(args):
normalLogger.debug('loading data...')
data = pd.read_csv(args.data_dir)
normalLogger.debug('split data into train and test...')
target, features = data[args.y_col], data.drop([args.y_col], axis=1)
X_train, X_test, y_train, y_test = train_test_split(features,
target,
test_size=0.25,
random_state=33)
##### output test_set.csv #####
# X_test[args.y_col] = y_test
# X_test.to_csv('test_set.csv',index=False)
# X_test.drop([args.y_col], axis=1, inplace=True)
#####
drop_useful_col(X_train)
drop_useful_col(X_test)
#####
normalLogger.debug('X_train size:' + str(X_train.shape))
normalLogger.debug('X_test size:' + str(X_test.shape))
normalLogger.debug('y_train size:' + str(y_train.shape))
normalLogger.debug('y_test size:' + str(y_test.shape))
normalLogger.debug('create preprocess from training data...')
preprocessor = preprocess(encoder='label',
normalize=(args.algorithm == 'nn'))
if args.algorithm == 'nn':
# note: target is for target encoder and nn to get output class count.
# if you don't use target encoder and algorithm is not nn, then target is not matter
X_train_encoder = preprocessor.fit_transform(X_train,
auto_fill=True,
target=y_train)
else:
X_train_encoder = preprocessor.fit_transform(X_train,
auto_fill=False,
target=y_train)
# save preprocessor to pickle
with open('./model_data/preprocessor.pkl', 'wb') as output:
pickle.dump(preprocessor, output, pickle.HIGHEST_PROTOCOL)
scaler = sum(y_train != 1) / sum(y_train == 1)
#restrict the max scale time
if sum(y_train != 1) / sum(y_train == 1) > 100:
scaler = 100 #np.floor(np.sqrt(sum(y_train!=1)/sum(y_train==1)))
normalLogger.debug('initialize %s model...' % args.algorithm)
model, param_grid = get_model(args.algorithm,
scaler=scaler,
in_features=len(X_train.columns),
num_classes=len(set(y_train)),
mid_features=256)
normalLogger.debug('getting model: ')
normalLogger.debug(model)
if args.algorithm == 'nn':
from nn_factory import nn_factory
import torch
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
grid = nn_factory(model,
device,
X_train_encoder,
y_train,
batch_size=32)
grid.fit(epoch=30)
else:
normalLogger.debug('Hyperparameter tuning...')
grid_start = time.time()
scroer = make_scorer(fbeta_score, beta=3)
if len(param_grid) > 0:
grid = GridSearchCV(estimator=model,
cv=5,
n_jobs=-1,
param_grid=param_grid,
scoring=scroer)
#grid = RandomizedSearchCV(estimator=model,cv=5, n_jobs=-1 , param_distributions=param_grid, scoring='f1_micro', n_iter=100)
else:
grid = model
grid.fit(X_train_encoder, y_train)
grid_end = time.time() - grid_start
normalLogger.debug('finish grid search, it took %.5f min' %
(grid_end / 60))
#print(grid.cv_results_)
# save model for future inference
normalLogger.debug('saving model to ./model_data')
normalLogger.debug(grid.best_estimator_)
joblib.dump(grid.best_estimator_,
os.path.join(cwd, 'model_data', 'grid.pkl'))
normalLogger.debug('saving feature importance')
feature_importance(X_train_encoder, grid.best_estimator_)
#see training performance
normalLogger.debug('prediction on training set...')
train_preds = grid.predict(X_train_encoder)
train_auc = roc_auc_score(y_train, train_preds)
train_recall = recall_score(y_train, train_preds, average=None)
normalLogger.debug('compute and save the confusion matrix...')
train_conf = confusion_matrix(y_train, train_preds)
# graph confusion table and save
colormap = sns.diverging_palette(220, 10, as_cmap=True)
sns.set(font_scale=1.4)
plt.figure()
train_plot = sns.heatmap(train_conf,
cmap=colormap,
annot=True,
cbar=False,
fmt='d')
train_fig = train_plot.get_figure()
plt.title('train auc: %.3f, recall:%s' % (train_auc, str(train_recall)))
train_fig.savefig("train_confusion.png")
##### see testset performance #####
normalLogger.debug('prediction on testing set...')
normalLogger.debug('preprocess for testing set...')
del preprocessor
with open('./model_data/preprocessor.pkl', 'rb') as input:
preprocessor_test = pickle.load(input)
X_test_encoder = preprocessor_test.transform(X_test)
test_preds = grid.predict(X_test_encoder)
test_auc = roc_auc_score(y_test, test_preds)
test_recall = recall_score(y_test, test_preds, average=None)
test_conf = confusion_matrix(y_test, test_preds)
# graph confusion table and save
plt.figure()
test_plot = sns.heatmap(test_conf,
cmap=colormap,
annot=True,
cbar=False,
fmt='d')
test_fig = test_plot.get_figure()
plt.title('test auc: %.3f, recall:%s' % (test_auc, str(test_recall)))
test_fig.savefig("test_confusion.png")
def main(_):
start_time = time.time()
MAX_STEPS = 10000 # Maximum steps to train
logging.info("TensorFlow version: %s", tf.__version__)
logging.info("TensorFlow git version: %s", tf.__git_version__)
if KUBERNETES:
tf_config_json = os.environ.get("TF_CONFIG", "{}")
tf_config = json.loads(tf_config_json)
logging.info("tf_config: {}".format(tf_config))
task = tf_config.get("task", {})
task_index = task["index"]
job_name = task["type"]
logging.info("task: {}".format(task))
cluster_spec = tf_config.get("cluster", {})
logging.info("cluster_spec: {}".format(cluster_spec))
else: # Local testing
task_index = FLAGS.task_index
job_name = FLAGS.job_name
cluster_spec = {
"ps": ["localhost:2222"],
"worker": ["localhost:2223", "localhost:2224"]
}
worker_list = cluster_spec.get("worker", "{}")
ps_list = cluster_spec.get("ps", "{}")
logging.info("job_name: {}".format(job_name))
logging.info("task_index: {}".format(task_index))
config = tf.ConfigProto(inter_op_parallelism_threads=num_inter_op_threads,
intra_op_parallelism_threads=num_intra_op_threads)
cluster = tf.train.ClusterSpec(cluster_spec)
server = tf.train.Server(cluster, job_name=job_name, task_index=task_index)
is_sync = (FLAGS.is_sync == 1) # Synchronous or asynchronous updates
is_chief = (task_index == 0) # Am I the chief node (always task 0)
if job_name == "ps":
logging.info(
"I am parameter server #{}. "
"I will join the server and will "
"need to be explicitly terminated when all jobs end. "
"Kubernetes should do this automatically."
"Otherwise, use CTRL-\\ for manual termination".format(task_index))
server.join()
elif job_name == "worker":
if is_chief:
logging.info("I am the chief worker {} with task #{}".format(
worker_list[task_index], task_index))
else:
logging.info("I am worker {} with task #{}".format(
worker_list[task_index], task_index))
# Graph
worker_device = "/job:{}/task:{}".format(job_name, task_index)
setter = tf.train. \
replica_device_setter(ps_tasks=len(ps_list),
worker_device=worker_device)
with tf.device(setter):
"""
BEGIN: MODEL DEFINE
"""
input_tensor = tf.placeholder(tf.float32)
label_tensor = tf.placeholder(tf.float32)
model = get_model(input_tensor, label_tensor, FLAGS, is_chief,
MAX_STEPS, len(worker_list))
"""
END: MODEL DEFINE
"""
# Monitored Training Session
checkpoint_dir = None
# if is_chief:
# checkpoint_dir = FLAGS.CHECKPOINTS_DIRECTORY
# else:
# checkpoint_dir = None
params = dict(master=server.target,
is_chief=is_chief,
config=config,
hooks=model["hooks"],
checkpoint_dir=checkpoint_dir,
stop_grace_period_secs=10)
sess = tf.train.MonitoredTrainingSession(**params)
logging.info("Starting training on worker {}".format(task_index))
if is_chief:
time.sleep(5)
"""
Just predict a simple line.
"""
slope = 8.16
intercept = -19.71
while not sess.should_stop():
train_x = np.random.randn(1) * 10
train_y = slope * train_x + \
intercept + np.random.randn(1) * 0.33
_, result, loss, m, b, step = sess.run([
model["optimizer"], model["prediction"], model["loss"],
model["m"], model["b"], model["global_step"]
],
feed_dict={
input_tensor: train_x,
label_tensor: train_y
})
logging.info("worker {}, step {}: loss = {:.4}, "
"target: [{}, {}], prediction: [{:.4}, {:.4}]".format(
task_index, step, loss, slope, intercept, m, b))
logging.info("Finished on task {}".format(task_index))
logging.info(
"Session from worker {} closed cleanly".format(task_index))
# training config
settings = {}
settings['model_selection'] = 'cnn_version_1'
#settings['model_selection'] = 'resnet_version_1'
settings['time_len'] = 1024
# tf computing graph
noised_inp_ = tf.placeholder(dtype=tf.float32,
shape=[None, settings['time_len']],
name='model_input')
clear_ground_truth_ = tf.placeholder(dtype=tf.float32,
shape=[None, settings['time_len']],
name='ground_truth')
denoised_out_ = model.get_model(noised_inp_, settings['model_selection'])
# check_point
saver = tf.compat.v1.train.Saver(tf.compat.v1.global_variables(),
max_to_keep=100)
#################################################
# load check point (load weights)
if 1:
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
data_str = 'd2020-08-11_t1624'
checkpoint_str = 'E:\\dnn_denoising\\traing_log\\' + settings[
'model_selection'] + '\\' + data_str + '\\chech_point-19'
#checkpoint_str = r'E:\dnn_denoising\traing_log'+'/'+settings['model_selection']+'/'+ data_str+'r\chech_point-19'
print('### checkpoint_str: {0}'.format(checkpoint_str))
saver.restore(sess, checkpoint_str)
def train(epoch=None, show=None):
"""
训练模型
"""
model = get_model()
model.train(epoch=epoch, show=show)
def run_main(ARGS):
print("#######################################")
print("Current execution paramenters:")
for arg, value in sorted(vars(ARGS).items()):
print("{}: {}".format(arg, value))
print("#######################################")
bandwidth = ARGS.conv_depth / (16 * 3)
print("\nBandwidth: ", bandwidth)
dirname = "mode{mode}/alpha{alpha:.2f}B{b}E{e}band{band:.2f}".format(
mode=ARGS.loss_type,
alpha=ARGS.alpha,
b=int(ARGS.snr_legit_train),
e=int(ARGS.snr_adv_train),
band=bandwidth)
train_dir = ARGS.train_dir + "/" + dirname + "/train"
save_dir = ARGS.train_dir + "/" + dirname + "/saved"
test_dir = ARGS.test_dir + "/" + dirname + "/test"
for dir in [train_dir, save_dir, test_dir]:
if not os.path.exists(dir):
os.makedirs(dir)
img_height = DATASETS[ARGS.dataset]._HEIGHT
img_width = DATASETS[ARGS.dataset]._WIDTH
num_channels = DATASETS[ARGS.dataset]._NUM_CHANNELS
num_classes = DATASETS[ARGS.dataset]._NUM_CLASSES
u = tf.compat.v1.placeholder(
tf.float32,
shape=[None, img_height, img_width, num_channels],
name='u')
p = tf.compat.v1.placeholder(tf.float32,
shape=[None, num_classes],
name='p')
snr_legit = ARGS.snr_legit_train
snr_adv = ARGS.snr_adv_train
model_vars, model_metrics, model_losses, model_collections = get_model(
u,
p,
params={
'ARGS': ARGS,
'snr_legit': snr_legit,
'snr_adv': snr_adv
})
DATASETS[ARGS.dataset].data_path = ARGS.data_dir_test
DATASETS[ARGS.dataset].maybe_download_and_extract()
u_test, s_test, p_test = DATASETS[ARGS.dataset].load_test_data()
def test(session):
feed_dict_test = {u: u_test, p: p_test}
results = session.run(model_metrics, feed_dict=feed_dict_test)
return results
session_config = tf.ConfigProto(inter_op_parallelism_threads=0,
intra_op_parallelism_threads=0,
allow_soft_placement=True)
session_config.gpu_options.allow_growth = True
if ARGS.mode in ("train", "Train"):
DATASETS[ARGS.dataset].data_path = ARGS.data_dir_train
DATASETS[ARGS.dataset].maybe_download_and_extract()
u_train, s_train, p_train = DATASETS[ARGS.dataset].load_training_data()
batch_size = ARGS.batch_size
train_iters_legit_1 = ARGS.train_iters_legit_1
train_iters_legit_3 = ARGS.train_iters_legit_3
train_iters_adv_2 = ARGS.train_iters_adv_2
train_iters_adv_3 = ARGS.train_iters_adv_3
global_step = tf.Variable(initial_value=0,
name='global_step',
trainable=False)
legit_vars = model_collections['legitimate_vars']
adv_vars = model_collections['adversary_vars']
optimizers = [
tf.train.AdamOptimizer(learning_rate=ARGS.learn_rate).minimize(
model_losses['loss_legit_prelim'], var_list=legit_vars),
tf.train.AdamOptimizer(learning_rate=ARGS.learn_rate).minimize(
model_losses['loss_adv'], var_list=adv_vars),
tf.train.AdamOptimizer(learning_rate=ARGS.learn_rate).minimize(
model_losses['loss_legit'], var_list=legit_vars)
]
def get_random_batch(batch_size):
idx = np.random.choice(len(u_train), size=batch_size)
u_batch = u_train[idx, :, :, :]
p_batch = p_train[idx, :]
return u_batch, p_batch
def train(session, optimizer, iters):
for i in range(iters):
u_batch, p_batch = get_random_batch(batch_size)
feed_dict_train = {u: u_batch, p: p_batch}
i_global, _ = session.run([global_step, optimizer],
feed_dict=feed_dict_train)
for n in range(ARGS.num_simulations):
print("Beginning simulation number: ", n)
sim_slug = "/sim{}".format(n)
for dir in [train_dir, save_dir, test_dir]:
if not os.path.exists(dir + sim_slug):
os.makedirs(dir + sim_slug)
session = tf.Session(config=session_config)
init = tf.compat.v1.global_variables_initializer()
session.run(init)
for phase in range(3):
measures = {
'legit_iters': [],
'adv_iters': [],
'mse': [],
'psnr': [],
'cross_entropy': [],
'accuracy': [],
'avg_power_y': [],
'avg_power_z': []
}
def write_results():
filename = train_dir + sim_slug + "/results_" + str(
phase + 1) + ".csv"
df = pd.DataFrame(measures)
df.to_csv(filename)
print("\tResults saved in ", filename)
t_epoch = 0
tot_iters_legit = 0
tot_iters_adv = 0
test_results = test(session)
def append_results(results):
print("\t -Total main network iterations: ",
tot_iters_legit)
print("\t -Total adversary iterations: ", tot_iters_adv)
measures['legit_iters'].append(tot_iters_legit)
measures['adv_iters'].append(tot_iters_adv)
for key in results:
if key in measures:
if t_epoch % 1 == 0:
print("\t -", key,
": {0}".format(results[key]))
measures[key].append(results[key])
if phase == 0:
# Phase 1: preliminary training of main network
print("\tPhase 1: Preliminary training of main network.")
saver = tf.compat.v1.train.Saver(var_list=legit_vars)
save_path = save_dir + sim_slug + "/model_phase_1.ckpt"
if os.path.exists(save_path + ".index"
) and not ARGS.delete_prev_model_1:
saver.restore(session, save_path)
print("\tModel restored from: ", save_path)
if ARGS.skip_phase_1:
print("\tSkipping phase 1")
continue
start_time = time.time()
test_results = test(session)
append_results(test_results)
mse_new = test_results['mse']
while True:
t_epoch = t_epoch + 1
print("\tEpoch number: ", t_epoch)
mse_old = mse_new
train(session, optimizers[0], train_iters_legit_1)
test_results = test(session)
tot_iters_legit = tot_iters_legit + train_iters_legit_1
append_results(test_results)
write_results()
mse_new = test_results['mse']
# save model every 10 epochs
if t_epoch % 10 == 0:
saver.save(session, save_path)
print("\tModel saved in path: ", save_path)
# if a specific condition is satisfied, exit from the loop
condition = t_epoch >= ARGS.train_epochs_1
if ARGS.reach_convergence_1:
condition = abs(mse_old -
mse_new) < ARGS.mse_epsilon_1
if condition:
break
elif phase == 1:
# Phase 2: preliminary training of the adversary
print("\tPhase 2: Preliminary training of the adversary.")
saver = tf.compat.v1.train.Saver(var_list=adv_vars)
save_path = save_dir + sim_slug + "/model_phase_2.ckpt"
if os.path.exists(save_path + ".index"
) and not ARGS.delete_prev_model_2:
saver.restore(session, save_path)
print("\tModel restored from: ", save_path)
if ARGS.skip_phase_2:
print("\tSkipping phase 2")
continue
start_time = time.time()
test_results = test(session)
append_results(test_results)
mse_new = test_results['accuracy']
while True:
t_epoch = t_epoch + 1
print("\tEpoch number: ", t_epoch)
mse_old = mse_new
train(session, optimizers[1], train_iters_adv_2)
test_results = test(session)
tot_iters_adv = tot_iters_adv + train_iters_adv_2
append_results(test_results)
write_results()
mse_new = test_results['accuracy']
# save model every 10 epochs
if t_epoch % 10 == 0:
saver.save(session, save_path)
print("\tModel saved in path: ", save_path)
# if a specific condition is satisfied, exit from the loop
condition = t_epoch >= ARGS.train_epochs_2
if ARGS.reach_convergence_2:
condition = abs(acc_old -
acc_new) < ARGS.acc_epsilon_2
if condition:
break
else:
# Phase 3: adversarial training of the network (minimax)
print(
"\tPhase 3: Adversarial training of the network (minimax)."
)
saver = tf.compat.v1.train.Saver()
save_path = save_dir + sim_slug + "/model.ckpt"
start_time = time.time()
test_results = test(session)
append_results(test_results)
mse_new = test_results['mse']
acc_new = test_results['accuracy']
while True:
t_epoch = t_epoch + 1
print("\tEpoch number: ", t_epoch)
mse_old = mse_new
acc_old = acc_new
train(session, optimizers[2], train_iters_legit_3)
test_results = test(session)
tot_iters_legit = tot_iters_legit + train_iters_legit_3
append_results(test_results)
write_results()
train(session, optimizers[1], train_iters_adv_3)
test_results = test(session)
tot_iters_adv = tot_iters_adv + train_iters_adv_3
append_results(test_results)
write_results()
acc_new = test_results['accuracy']
# save model every 10 epochs
if t_epoch % 10 == 0:
saver.save(session, save_path)
print("\tModel saved in path: ", save_path)
# if a specific condition is satisfied, exit from the loop
condition = t_epoch >= ARGS.train_epochs_3
if ARGS.reach_convergence_3:
condition = abs(mse_old -
mse_new) < ARGS.mse_epsilon_3
condition = condition and abs(
acc_old - acc_new) < ARGS.acc_epsilon_3
if condition:
break
elif ARGS.mode in ("test", "Test"):
DATASETS[ARGS.dataset].data_path = ARGS.data_dir_test
DATASETS[ARGS.dataset].maybe_download_and_extract()
# count the number of simulation folders
num_simulations = 0
folders = os.walk(save_dir)[1]
for name in folders:
if "sim" in name:
num_simulations = num_simulations + 1
for n in num_simulations:
print("Beginning simulation number: ", n)
sim_slug = "/sim{}".format(n)
measures = {
'snr_legit': [],
'snr_adv': [],
'mse': [],
'psnr': [],
'cross_entropy': [],
'accuracy': [],
'avg_power_y': [],
'avg_power_z': []
}
def write_results():
filename = test_dir + sim_slug + "/results.csv"
df = pd.DataFrame(measures)
df.to_csv(filename)
print("\tResults saved in ", filename)
def append_results(results):
measures['snr_legit'].append(snr_legit)
measures['snr_adv'].append(snr_adv)
for key in results:
if key in measures:
if (t_epoch % 1 == 0):
print("\t -", key, ": {0}".format(results[key]))
measures[key].append(results[key])
snr_range = [5 * i
for i in range(-5, 6)] # [-25, -20, ..., 20, 25]
snr_legit = ARGS.snr_legit_train
for snr_adv in snr_range:
model_vars, model_metrics, model_losses, model_collections = get_model(
u,
p,
params={
'ARGS': ARGS,
'snr_legit': snr_legit,
'snr_adv': snr_adv
})
session = tf.Session(config=session_config)
saver = tf.compat.v1.train.Saver()
save_path = save_dir + sim_slug + "/model.ckpt"
if os.path.exists(save_path + ".index"):
saver.restore(session, save_path)
print("Model restored from: ", save_path)
test_results = test(session)
append_results(test_results)
write_results()
else:
print("ERROR: Model not found.")
else:
print("Error: the only available options are 'train' or 'test'.")
default="dataset/predict/input",
help="input video folder")
parser.add_argument('--output',
type=str,
default="dataset/predict/output",
help="output video folder")
parser.add_argument(
"--scale",
type=int,
default=4,
help='Increase the frames by Nx. Example scale=2 ==> 2x frames')
args = parser.parse_args()
# Flow compute
flow_compute = get_model("FC")
model_load(flow_compute, "FC", args.checkpoint)
device = model_device()
flow_compute.to(device)
flow_compute.eval()
# Flow interpolate
flow_interpolate = get_model("AT")
model_load(flow_interpolate, "AT", args.checkpoint)
flow_interpolate.to(device)
flow_interpolate.eval()
# if os.environ["ENABLE_APEX"] == "YES":
# from apex import amp
# model = amp.initialize(model, opt_level="O1")
def train():
"""
Training systole and diastole models.
"""
print('Loading and compiling models...')
model_systole = get_model()
model_diastole = get_model()
#import best model if it exists
if os.path.isfile('/data/run2/weights_systole_best.hdf5'):
print('loading weights')
model_systole.load_weights('/data/run2/weights_systole_best.hdf5')
if os.path.isfile('/data/run2/weights_diastole_best.hdf5'):
model_diastole.load_weights('/data/run2/weights_diastole_best.hdf5')
print('Loading training data...')
X, y, metadata = load_train_data()
#print('Pre-processing images...')
#X = preprocess(X)
#np.save('/data/pre/pre/X_train.npy', X)
# split to training and test
X_train, y_train, X_test, y_test, metadata_train, metadata_test = split_data(X, y, metadata, split_ratio=0.2)
nb_iter = 200
epochs_per_iter = 1
batch_size = 8
calc_crps = 5 # calculate CRPS every n-th iteration (set to 0 if CRPS estimation is not needed)
# remember min val. losses (best iterations), used as sigmas for submission
min_val_loss_systole = sys.float_info.max
min_val_loss_diastole = sys.float_info.max
print('-'*50)
print('Training...')
print('-'*50)
for i in range(0,nb_iter):
print('-'*50)
print('Iteration {0}/{1}'.format(i + 1, nb_iter))
print('-'*50)
# print('Augmenting images - rotations')
# X_train_aug = rotation_augmentation(X_train, 15)
# print('Augmenting images - shifts')
# X_train_aug = shift_augmentation(X_train_aug, 0.1, 0.1)
# print('Augmenting images - shifts')
# X_train_aug = shift_augmentation(X_train, 0.1, 0.1)
X_train_aug = X_train
print('Fitting systole model...')
hist_systole = model_systole.fit({'input1':X_train_aug, 'input2':metadata_train, 'output':y_train[:, 0]}, shuffle=True, nb_epoch=epochs_per_iter,
batch_size=batch_size, validation_data={'input1':X_test,'input2':metadata_test, 'output':y_test[:, 0]})
print('Fitting diastole model...')
hist_diastole = model_diastole.fit({'input1':X_train_aug, 'input2':metadata_train, 'output':y_train[:, 1]}, shuffle=True, nb_epoch=epochs_per_iter,
batch_size=batch_size, validation_data={'input1':X_test, 'input2':metadata_test, 'output':y_test[:, 1]})
# sigmas for predicted data, actually loss function values (RMSE)
loss_systole = hist_systole.history['loss'][-1]
loss_diastole = hist_diastole.history['loss'][-1]
val_loss_systole = hist_systole.history['val_loss'][-1]
val_loss_diastole = hist_diastole.history['val_loss'][-1]
if calc_crps > 0 and i % calc_crps == 0:
print('Evaluating CRPS...')
pred_systole = model_systole.predict({'input1':X_train, 'input2':metadata_train, 'output':y_train[:, 0]}, batch_size=batch_size, verbose=1)['output']
pred_diastole = model_diastole.predict({'input1':X_train, 'input2':metadata_train, 'output':y_train[:, 1]}, batch_size=batch_size, verbose=1)['output']
val_pred_systole = model_systole.predict({'input1':X_test, 'input2':metadata_test, 'output':y_test[:, 0]}, batch_size=batch_size, verbose=1)['output']
val_pred_diastole = model_diastole.predict({'input1':X_test, 'input2':metadata_test, 'output':y_test[:, 1]}, batch_size=batch_size, verbose=1)['output']
# Get sigmas
# sigma_systole = as_tensor_variable(root_mean_squared_error(y_train[:, 0], pred_systole))
# sigma_diastole = as_tensor_variable(root_mean_squared_error(y_train[:, 1], pred_systole))
# val_sigma_systole = as_tensor_variable(root_mean_squared_error(y_test[:, 0], val_pred_systole))
# val_sigma_diastole = as_tensor_variable(root_mean_squared_error(y_test[:, 1], val_pred_diastole))
# CDF for train and test data (actually a step function)
cdf_train = real_to_cdf(np.concatenate((y_train[:, 0], y_train[:, 1])))
cdf_test = real_to_cdf(np.concatenate((y_test[:, 0], y_test[:, 1])))
# CDF for predicted data
cdf_pred_systole = real_to_cdf(pred_systole, loss_systole)
cdf_pred_diastole = real_to_cdf(pred_diastole, loss_diastole)
cdf_val_pred_systole = real_to_cdf(val_pred_systole, val_loss_systole)
cdf_val_pred_diastole = real_to_cdf(val_pred_diastole, val_loss_diastole)
# evaluate CRPS on training data
crps_train = crps(cdf_train, np.concatenate((cdf_pred_systole, cdf_pred_diastole)))
print('CRPS(train) = {0}'.format(crps_train))
# evaluate CRPS on test data
crps_test = crps(cdf_test, np.concatenate((cdf_val_pred_systole, cdf_val_pred_diastole)))
print('CRPS(test) = {0}'.format(crps_test))
# for best (lowest) val losses, save weights
if val_loss_systole < min_val_loss_systole:
min_val_loss_systole = val_loss_systole
model_systole.save_weights('/data/run2/weights_systole_best.hdf5', overwrite=True)
if val_loss_diastole < min_val_loss_diastole:
min_val_loss_diastole = val_loss_diastole
model_diastole.save_weights('/data/run2/weights_diastole_best.hdf5', overwrite=True)
# save best (lowest) val losses in file (to be later used for generating submission)
with open('/data/run2/val_loss.txt', mode='w+') as f:
f.write(str(min_val_loss_systole))
f.write('\n')
f.write(str(min_val_loss_diastole))
with open("/data/run2/loss.txt", "a+") as myfile:
myfile.write('\t'.join((str(i+1), str(loss_systole),str(loss_diastole),str(val_loss_systole),str(val_loss_diastole), str(crps_train), str(crps_test))))
myfile.write('\n')
def test():
"""
模型测试
"""
model = get_model()
return model.test()
def cut(sentence):
"""
分词
"""
model = get_model()
return model.cut(sentence)
params["n_sents"],
params["n_targets"],
params["batch_size"],
encoding)
# add post-processing experiment params
embedding_input = batches.char_indexer.vocab_len()
n_classes = batches.word_indexer.vocab_len()
params.update({"embedding_input": embedding_input, "n_classes": n_classes})
X_test, y_test = batches.train_test_split()
y_test = np_utils.to_categorical(y_test, n_classes)
# model
print("compiling model...")
m = get_model(**params)
print("learning...")
if params["batch_training"]:
for e in range(params["n_epoch"]):
print("epoch number: %d" % e)
for X_batch, y_batch in batches:
y_batch = np_utils.to_categorical(y_batch, n_classes)
m.fit(X_batch, y_batch, batch_size=params["mini_batch_size"],
nb_epoch=1, verbose=1, validation_data=(X_test, y_test))
# loss = m.train_on_batch(X_batch, y_batch)
# print("loss: [%d]" % loss[0])
else:
# canister callback
def refresh(self):
fname, fkey = self.reference.split(".")
fmod = get_model(fname)
self.count = fmod.query(getattr(fmod, fkey) == self.target).count()
from theano.tensor.nnet import categorical_crossentropy
from transformers import OneHotEncode, RandomHorizontalFlip
batch_size = 32
test_dataset = CIFAR10(('test',), subset=slice(0, 9984))
test_stream = DataStream.default_stream(
test_dataset,
iteration_scheme=SequentialScheme(test_dataset.num_examples, batch_size)
)
test_stream = OneHotEncode(test_stream, which_sources=('targets',))
X = tensor.ftensor4('features')
targets = tensor.fmatrix('targets')
output, output_test, all_parameters, acc_parameters = get_model(X, batch_size, (32, 32))
saved_parameters = numpy.load('./best_weights.npz')['arr_0']
for param_w, param in zip(saved_parameters, all_parameters):
param.set_value(param_w)
saved_parameters_bn = numpy.load('./best_weights_bn.npz')['arr_0']
for param_w, param in zip(saved_parameters_bn, acc_parameters):
param[1].set_value(param_w)
error_test = tensor.neq(tensor.argmax(output_test[:,:,0,0], axis=1), tensor.argmax(targets, axis=1)).mean()
error_test.name = 'error_test'
f_valid = theano.function(
inputs=[X, targets],
outputs=[error_test],
def train(continue_training=False):
'''
Run training
'''
print('Loading training data...')
train = np.load(train_file)
train,val = split_data(train, split_ratio = 0.2)
test = np.load(test_file)
print "done."
print('Prepare data...')
input_train = model_data(train)
input_val = model_data(val)
input_test = model_data(test)
nFeatures = input_train["X_input"].shape[1]
nTin = input_train["RNN_input"].shape[1]
nRNNFeatures = input_train["RNN_input"].shape[2]
nTin2 = input_train["RNN2_input"].shape[1]
nRNNFeatures2 = input_train["RNN2_input"].shape[2]
print "using %d/%d/%d samples with %d features RNN(t,features,hidden)=(%d,%d,%d) and (%d,%d,%d)" % \
(input_train["X_input"].shape[0],input_val["X_input"].shape[0],input_test["X_input"].shape[0],\
nFeatures,nTin,nRNNFeatures,nRNNHidden,nTin2,nRNNFeatures2,nRNNHidden2)
print('Loading and compiling models...')
model = get_model(nTin=nTin, nRNNFeatures=nRNNFeatures,nRNNHidden=nRNNHidden, \
nTin2=nTin2, nRNNFeatures2=nRNNFeatures2,nRNNHidden2=nRNNHidden, \
nFeatures=nFeatures, nOutput=N_ZONES)
if continue_training:
print('Loading models weights...')
model.load_weights(output_weights_best_file)
print "done."
print('-'*50)
print('Training model...')
print('-'*50)
nIterations = 10000 # 300
epochs_per_iter = 1
batch_size = 64 # 1024 # 64
loss_val_min = sys.float_info.max
losses_train = []
losses_val = []
losses_test = []
errors_train = []
errors_val = []
errors_test = []
for iIteration in range(nIterations):
print('-'*50)
print('Iteration {0}/{1}'.format(iIteration + 1,nIterations))
print('-'*50)
print('Fitting model...')
hist = model.fit(input_train, validation_data=(input_val), \
shuffle=True, nb_epoch=epochs_per_iter, verbose=1,batch_size=batch_size)
loss_train = hist.history['loss'][-1]
loss_val = hist.history['val_loss'][-1]
losses_train.append(loss_train)
losses_val.append(loss_val)
print("Loss train/test = %f / %f" % (loss_train, loss_val))
print('Calculate predictions...')
Ypred_train = model.predict(input_train, batch_size=batch_size, verbose=1)["out"]
Ypred_val = model.predict(input_val, batch_size=batch_size, verbose=1)["out"]
Y_train = input_train["out"]
Y_val = input_val["out"]
error_train = np.sqrt(np.mean((Y_train - Ypred_train) * (Y_train - Ypred_train))) / np.mean(Y_train)
error_val = np.sqrt(np.mean((Y_val - Ypred_val) * (Y_val - Ypred_val))) / np.mean(Y_val)
errors_train.append(error_train)
errors_val.append(error_val)
print("Error train/test = %f / %f" % (error_train, error_val))
print "done."
print('Save Losses...')
csv_file = open(output_losses_file, "w")
csv_file.write("iter,train_loss,test_loss\n")
for i in range(len(losses_train)):
csv_file.write("%d,%f,%f,%f,%f\n" % (i, losses_train[i], losses_val[i],errors_train[i],errors_val[i]))
csv_file.close()
print "done."
print('Saving weights...')
model.save_weights(output_weights_file, overwrite=True)
if loss_val < loss_val_min:
# csv_file = open("output/kerasRNN/accuracy", "w")
# csv_file.write("id,actual,predicted\n")
# for id in id_to_pred_test.keys():
# csv_file.write("%s,%f,%f\n" % (id, id_to_actual_test[id], id_to_pred_test[id]))
# csv_file.close()
loss_val_min = loss_val
model.save_weights(output_weights_best_file, overwrite=True)
print "done."
# force deletion
del hist
model.training_data = None
model.validation_data = None
gc.collect()
print "Plot example..."
plot(Y_val,Ypred_val,i=0,iZone=0)
print "done."
print "END."
def tagger(sentence):
"""
词性标注
"""
model = get_model()
return model.predict(sentence)
def main():
""" main function
"""
### header
parser = argparse.ArgumentParser()
# path
parser.add_argument('--root-path', default=CFG.root_path, help="root path")
parser.add_argument('--log-path', default=CFG.log_path, help="log path")
parser.add_argument('--model-path',
default=CFG.model_path,
help="model path")
parser.add_argument('--pretrained-path', help='pretrained path')
# image
parser.add_argument('--transform-version',
default=0,
type=int,
help="image transform version ex) 0, 1, 2 ...")
parser.add_argument('--image-size',
default=256,
type=int,
help="image size(256)")
# model
parser.add_argument('--model-name',
default=CFG.model_name,
help=f"model name({CFG.model_name})")
parser.add_argument('--backbone-name',
default=CFG.backbone_name,
help=f"backbone name({CFG.backbone_name})")
# learning
parser.add_argument('--batch-size',
default=CFG.batch_size,
type=int,
help=f"batch size({CFG.batch_size})")
parser.add_argument('--learning-rate',
default=CFG.learning_rate,
type=float,
help=f"learning rate({CFG.learning_rate})")
parser.add_argument('--num-epochs',
default=CFG.num_epochs,
type=int,
help=f"number of epochs({CFG.num_epochs})")
# etc
parser.add_argument("--seed",
default=CFG.seed,
type=int,
help=f"seed({CFG.seed})")
parser.add_argument("--workers",
default=CFG.workers,
type=int,
help=f"number of workers({CFG.workers})")
parser.add_argument("--debug", action="store_true", help="debug mode")
args = parser.parse_args()
# path
CFG.root_path = args.root_path
CFG.model_path = args.model_path
CFG.log_path = args.log_path
CFG.pretrained_path = args.pretrained_path
# image
CFG.transform_version = args.transform_version
CFG.image_size = args.image_size
# model
CFG.model_name = args.model_name
CFG.backbone_name = args.backbone_name
# learning
CFG.batch_size = args.batch_size
CFG.learning_rate = args.learning_rate
CFG.num_epochs = args.num_epochs
# etc
CFG.seed = args.seed
CFG.workers = args.workers
CFG.debug = args.debug
# get device
CFG.device = get_device()
# get version
_, version, _ = sys.argv[0].split('/')
CFG.version = version
# update log path
if not CFG.debug:
CFG.log_path = os.path.join(CFG.log_path, CFG.version)
os.makedirs(CFG.log_path, exist_ok=True)
CFG.log_path = os.path.join(
CFG.log_path, f'exp_{get_exp_id(CFG.log_path, prefix="exp_")}')
os.makedirs(CFG.log_path, exist_ok=True)
else:
CFG.log_path = os.path.join(CFG.log_path, "debug")
os.makedirs(CFG.log_path, exist_ok=True)
CFG.log_path = os.path.join(CFG.log_path, "debug")
os.makedirs(CFG.log_path, exist_ok=True)
# update model path
if not CFG.debug:
CFG.model_path = os.path.join(CFG.model_path, version)
os.makedirs(CFG.model_path, exist_ok=True)
CFG.model_path = os.path.join(
CFG.model_path, f'exp_{get_exp_id(CFG.model_path, prefix="exp_")}')
os.makedirs(CFG.model_path, exist_ok=True)
else:
CFG.model_path = os.path.join(CFG.model_path, "debug")
os.makedirs(CFG.model_path, exist_ok=True)
CFG.model_path = os.path.join(CFG.model_path, "debug")
os.makedirs(CFG.model_path, exist_ok=True)
pprint({k: v for k, v in dict(CFG.__dict__).items() if '__' not in k})
json.dump({k: v
for k, v in dict(CFG.__dict__).items() if '__' not in k},
open(os.path.join(CFG.log_path, 'CFG.json'), "w"))
print()
### seed all
seed_everything(CFG.seed)
### data related
# load data
print("Load Raw Data")
data_df, test_df = load_data(CFG)
# preprocess data
print("Preprocess Data")
data_df = preprocess_data(CFG, data_df)
# split data into train with valid
print("Split Data")
data_df = split_data(CFG, data_df)
# get transform
print("Get Transform")
train_transforms, test_transforms = get_transform(CFG)
# train test split
for fold in range(CFG.n_folds):
print(f"\nValidation Fold: {fold}")
train_df = data_df[data_df['fold'] != fold].reset_index(drop=True)
valid_df = data_df[data_df['fold'] == fold].reset_index(drop=True)
print(
f"... Train Shape: {train_df.shape}, Valid Shape: {valid_df.shape}"
)
# dataset
trn_data = MelanomaDataset(CFG, train_df, train_transforms)
val_data = MelanomaDataset(CFG, valid_df, test_transforms)
### model related
# get learner
learner = Learner(CFG)
learner.name = f"model.fold_{fold}"
if CFG.pretrained_path:
print("Load Pretrained Model")
print(f"... Pretrained Info - {CFG.pretrained_path}")
learner.load(CFG.pretrained_path, f"model_state_dict")
# get model
if CFG.pretrained_path:
print(f"Get Model")
model = learner.best_model.to(CFG.deivce)
else:
print(f"Get Model")
model = get_model(CFG)
model = model.to(CFG.device)
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
# get optimizer
optimizer = optim.Adam(model.parameters(), lr=CFG.learning_rate)
optimizer = torchcontrib.optim.SWA(optimizer)
# get scheduler
# scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
# optimizer, mode='min', patience=1, verbose=False, factor=0.2)
# scheduler = CosineAnnealingLRWarmup(optimizer, T_min=int(CFG.num_epochs / 5), T_max=CFG.num_epochs)
scheduler = CosineAnnealingLRWarmup(optimizer,
T_min=0,
T_max=CFG.num_epochs // 4)
### train related
# train model
learner.train(trn_data, val_data, model, optimizer, scheduler)
print()
def train():
"""
训练模型
"""
model = get_model()
model.train()
from tqdm import tqdm
from util import load_data, get_tokens, get_keep_tokens
from config import BaseConfig
from model import get_model
from dataset import data_generator
# 加载数据集
data = load_data(BaseConfig.train_path)
train_data = [d for i, d in enumerate(data) if i % 10 != 0]
valid_data = [d for i, d in enumerate(data) if i % 10 == 0]
test_data = load_data(BaseConfig.test_path)
test_generator = data_generator(test_data, BaseConfig.batch_size)
# 数据集词频
tokens = get_tokens(data + test_data)
# BERT词频
keep_tokens = get_keep_tokens()
model = get_model(tokens, keep_tokens)
model.load_weights('best_model.h5')
F = open("result.csv", mode="w")
for x_true, _ in tqdm(test_generator):
y_pred = model.predict(x_true)[:, 0, 5:7]
y_pred = y_pred[:, 1] / (y_pred.sum(axis=1) + 1e-8)
for p in y_pred:
F.write('%f\n' % p)
F.close()
def recognize(sentence):
"""
命名实体识别
"""
model = get_model()
return model.predict(sentence)
def train():
"""
Training model.
"""
# Compile training and testing functions
[model, train_fn, val_fn, predict_fn] = get_model()
# Load training data
print('Loading training data...')
X, y = load_train_data()
#print('Pre-processing images...')
#X = preprocess(X)
# split to training and test
X_train, y_train, X_test, y_test = split_data(X, y, split_ratio=0.2)
nb_epoch = 200
batch_size = 32
calc_crps = 0 # calculate CRPS every n-th iteration (set to 0 if CRPS estimation is not needed) NOT IMPLEMENTED YET
print('-'*50)
print('Training...')
print('-'*50)
min_val_err = sys.float_info.max
patience = 0
for i in range(nb_epoch):
print('-'*50)
print('Iteration {0}/{1}'.format(i + 1, nb_epoch))
print('-'*50)
print('Augmenting images - rotations')
X_train_aug = rotation_augmentation(X_train, 15)
print('Augmenting images - shifts')
X_train_aug = shift_augmentation(X_train_aug, 0.1, 0.1)
# In each epoch, we do a full pass over the training data:
print('Fitting model...')
train_err = 0
train_batches = 0
for batch in iterate_minibatches(X_train_aug, y_train, batch_size, shuffle=True):
inputs, targets = batch
train_err += train_fn(inputs, targets)
train_batches += 1
# And a full pass over the validation data:
val_err = 0
val_batches = 0
for batch in iterate_minibatches(X_test, y_test, batch_size, shuffle=False):
inputs, targets = batch
val_err += val_fn(inputs, targets)
val_batches += 1
print('Saving weights...')
# save weights so they can be loaded later
# np.savez('weights.npz', *get_all_param_values(model))
# for best (lowest) val losses, save weights
if val_err < min_val_err:
patience = 0
min_val_err = val_err
np.savez('weights_best.npz', *get_all_param_values(model))
else:
patience += 1
print('error on validation set: ' + str(val_err))
print('patience variable is: ' + str(patience))
print('\n')
# save best (lowest) val losses in file (to be later used for generating submission)
with open('val_loss.txt', mode='a') as f:
f.write(str(val_err))
f.write('\n')
if (patience == 8):
break
def main():
opt = parse_args()
logging.info(json.dumps(vars(opt), indent=2))
rootpath = opt.rootpath
testCollection = opt.testCollection
assert collectionStrt == "multiple"
resume = os.path.join(opt.logger_name, opt.checkpoint_name)
if not os.path.exists(resume):
logging.info(resume + ' not exists.')
sys.exit(0)
checkpoint = torch.load(resume)
start_epoch = checkpoint['epoch']
best_rsum = checkpoint['best_rsum']
logging.info("=> loaded checkpoint '{}' (epoch {}, best_rsum {})".format(
resume, start_epoch, best_rsum))
options = checkpoint['opt']
trainCollection = options.trainCollection
valCollection = options.valCollection
visual_feat_file = BigFile(
os.path.join(rootpath, testCollection, 'FeatureData',
options.visual_feature))
assert options.visual_feat_dim == visual_feat_file.ndims
video2frame = read_dict(
os.path.join(rootpath, testCollection, 'FeatureData',
options.visual_feature, 'video2frames.txt'))
vid_data_loader = data.get_vis_data_loader(visual_feat_file,
opt.batch_size, opt.workers,
video2frame)
vis_embs = None
# set bow vocabulary and encoding
bow_vocab_file = os.path.join(rootpath, options.trainCollection,
'TextData', 'vocabulary', 'bow',
options.vocab + '.pkl')
bow_vocab = pickle.load(open(bow_vocab_file, 'rb'))
bow2vec = get_text_encoder('bow')(bow_vocab)
options.bow_vocab_size = len(bow_vocab)
# set rnn vocabulary
rnn_vocab_file = os.path.join(rootpath, options.trainCollection,
'TextData', 'vocabulary', 'rnn',
options.vocab + '.pkl')
rnn_vocab = pickle.load(open(rnn_vocab_file, 'rb'))
options.vocab_size = len(rnn_vocab)
model = get_model(options.model)(options)
model.load_state_dict(checkpoint['model'])
model.val_start()
output_dir = resume.replace(trainCollection, testCollection)
for query_set in opt.query_sets.strip().split(','):
output_dir_tmp = output_dir.replace(
valCollection,
'%s/%s/%s' % (query_set, trainCollection, valCollection))
output_dir_tmp = output_dir_tmp.replace('/%s/' % options.cv_name,
'/results/')
pred_result_file = os.path.join(output_dir_tmp, 'id.sent.score.txt')
logging.info(pred_result_file)
if checkToSkip(pred_result_file, opt.overwrite):
sys.exit(0)
makedirsforfile(pred_result_file)
# query data loader
query_file = os.path.join(rootpath, testCollection, 'TextData',
query_set)
query_loader = data.get_txt_data_loader(query_file, rnn_vocab, bow2vec,
opt.batch_size, opt.workers)
# encode videos
if vis_embs is None:
start = time.time()
if options.space == 'hybrid':
video_embs, video_tag_probs, video_ids = evaluation.encode_text_or_vid_tag_hist_prob(
model.embed_vis, vid_data_loader)
else:
video_embs, video_ids = evaluation.encode_text_or_vid(
model.embed_vis, vid_data_loader)
logging.info("encode video time: %.3f s" % (time.time() - start))
# encode text
start = time.time()
if options.space == 'hybrid':
query_embs, query_tag_probs, query_ids = evaluation.encode_text_or_vid_tag_hist_prob(
model.embed_txt, query_loader)
else:
query_embs, query_ids = evaluation.encode_text_or_vid(
model.embed_txt, query_loader)
logging.info("encode text time: %.3f s" % (time.time() - start))
if options.space == 'hybrid':
t2v_matrix_1 = evaluation.cal_simi(query_embs, video_embs)
# eval_avs(t2v_matrix_1, query_ids, video_ids, pred_result_file, rootpath, testCollection, query_set)
t2v_matrix_2 = evaluation.cal_simi(query_tag_probs,
video_tag_probs)
# pred_result_file = os.path.join(output_dir_tmp, 'id.sent.score_2.txt')
# eval_avs(t2v_matrix_2, query_ids, video_ids, pred_result_file, rootpath, testCollection, query_set)
t2v_matrix_1 = norm_score(t2v_matrix_1)
t2v_matrix_2 = norm_score(t2v_matrix_2)
for w in [0.8]:
print("\n")
t2v_matrix = w * t2v_matrix_1 + (1 - w) * t2v_matrix_2
pred_result_file = os.path.join(output_dir_tmp,
'id.sent.score_%.1f.txt' % w)
eval_avs(t2v_matrix, query_ids, video_ids, pred_result_file,
rootpath, testCollection, query_set)
else:
t2v_matrix_1 = evaluation.cal_simi(query_embs, video_embs)
eval_avs(t2v_matrix_1, query_ids, video_ids, pred_result_file,
rootpath, testCollection, query_set)
def train(data_prefix, prefix, seed, run):
"""
Training systole and diastole models.
"""
print('Loading training data...')
X, y = load_train_data(data_prefix, seed)
print('Loading and compiling models...')
model_systole = get_model()
model_diastole = get_model()
# split to training and test
X_train, y_train, X_test, y_test = split_data(X, y, split_ratio=0.2)
nb_iter = 200
epochs_per_iter = 1
batch_size = 32
calc_crps = 1 # calculate CRPS every n-th iteration (set to 0 if CRPS estimation is not needed)
# remember min val. losses (best iterations), used as sigmas for submission
min_val_loss_systole = sys.float_info.max
min_val_loss_diastole = sys.float_info.max
print('-'*50)
print('Training...')
print('-'*50)
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=15, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=True) # randomly flip images
# compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied)
datagen.fit(X_train)
systole_checkpointer_best = ModelCheckpoint(filepath=prefix + "weights_systole_best.hdf5", verbose=1, save_best_only=True)
diastole_checkpointer_best = ModelCheckpoint(filepath=prefix + "weights_diastole_best.hdf5", verbose=1, save_best_only=True)
systole_checkpointer = ModelCheckpoint(filepath=prefix + "weights_systole.hdf5", verbose=1, save_best_only=False)
diastole_checkpointer = ModelCheckpoint(filepath=prefix + "weights_diastole.hdf5", verbose=1, save_best_only=False)
if run == 0 or run == 1:
print('Fitting Systole Shapes')
hist_systole = model_systole.fit_generator(datagen.flow(X_train, y_train[:, 2], batch_size=batch_size),
samples_per_epoch=X_train.shape[0],
nb_epoch=nb_iter, show_accuracy=False,
validation_data=(X_test, y_test[:, 2]),
callbacks=[systole_checkpointer, systole_checkpointer_best],
nb_worker=4)
if run == 0 or run == 2:
print('Fitting Diastole Shapes')
hist_diastole = model_diastole.fit_generator(datagen.flow(X_train, y_train[:, 2], batch_size=batch_size),
samples_per_epoch=X_train.shape[0],
nb_epoch=nb_iter, show_accuracy=False,
validation_data=(X_test, y_test[:, 2]),
callbacks=[diastole_checkpointer, diastole_checkpointer_best],
nb_worker=4)
if run == 0 or run == 1:
loss_systole = hist_systole.history['loss'][-1]
val_loss_systole = hist_systole.history['val_loss'][-1]
if run == 0 or run == 2:
loss_diastole = hist_diastole.history['loss'][-1]
val_loss_diastole = hist_diastole.history['val_loss'][-1]
if calc_crps > 0 and run == 0:
print('Evaluating CRPS...')
pred_systole = model_systole.predict(X_train, batch_size=batch_size, verbose=1)
val_pred_systole = model_systole.predict(X_test, batch_size=batch_size, verbose=1)
pred_diastole = model_diastole.predict(X_train, batch_size=batch_size, verbose=1)
val_pred_diastole = model_diastole.predict(X_test, batch_size=batch_size, verbose=1)
# CDF for train and test data (actually a step function)
cdf_train = real_to_cdf(np.concatenate((y_train[:, 0], y_train[:, 1])))
cdf_test = real_to_cdf(np.concatenate((y_test[:, 0], y_test[:, 1])))
# CDF for predicted data
cdf_pred_systole = real_to_cdf(pred_systole, loss_systole)
cdf_val_pred_systole = real_to_cdf(val_pred_systole, val_loss_systole)
cdf_pred_diastole = real_to_cdf(pred_diastole, loss_diastole)
cdf_val_pred_diastole = real_to_cdf(val_pred_diastole, val_loss_diastole)
# evaluate CRPS on training data
crps_train = crps(cdf_train, np.concatenate((cdf_pred_systole, cdf_pred_diastole)))
print('CRPS(train) = {0}'.format(crps_train))
# evaluate CRPS on test data
crps_test = crps(cdf_test, np.concatenate((cdf_val_pred_systole, cdf_val_pred_diastole)))
print('CRPS(test) = {0}'.format(crps_test))
# save best (lowest) val losses in file (to be later used for generating submission)
with open(prefix + 'val_loss.txt', mode='w+') as f:
if run == 0 or run == 1:
f.write(str(min(hist_systole.history['val_loss'])))
f.write('\n')
if run == 0 or run == 2:
f.write(str(min(hist_diastole.history['val_loss'])))
def submission():
"""
Generate submission file for the trained models.
"""
print('Loading and compiling models...')
[model, train_fn, val_fn, predict_fn] = get_model()
print('Loading models weights...')
with np.load('weights_best.hdf5.npz') as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
set_all_param_values(model, param_values)
# load val losses to use as sigmas for CDF
# with open('val_loss.txt', mode='r') as f:
# val_loss_systole = float(f.readline())
# val_loss_diastole = float(f.readline())
print('Loading validation data...')
X, ids = load_validation_data()
print('Pre-processing images...')
#X = preprocess(X)
#batch_size = 32
pred_systole = np.zeros([X.shape[0], 600])
pred_diastole = np.zeros([X.shape[0], 600])
print('Predicting on validation data...')
nb_of_batches = 10
list_indexes = np.linspace(0,X.shape[0],nb_of_batches + 1,dtype=np.int)
for i in range(nb_of_batches):
pred = predict_fn(X[list_indexes[i]:list_indexes[i+1]])
pred_systole[list_indexes[i]:list_indexes[i+1]] = pred[:,:600]
pred_diastole[list_indexes[i]:list_indexes[i+1]] = pred[:,600:]
# CDF for train and test prediction
cdf_pred_systole = np.cumsum(pred_systole, axis=1)
cdf_pred_diastole = np.cumsum(pred_diastole, axis=1)
print('Accumulating results...')
sub_systole = accumulate_study_results(ids, cdf_pred_systole)
sub_diastole = accumulate_study_results(ids, cdf_pred_diastole)
# write to submission file
print('Writing submission to file...')
fi = csv.reader(open('data/sample_submission_validate.csv'))
f = open('submission.csv', 'w')
fo = csv.writer(f, lineterminator='\n')
fo.writerow(fi.next())
for line in fi:
idx = line[0]
key, target = idx.split('_')
key = int(key)
out = [idx]
if key in sub_systole:
if target == 'Diastole':
out.extend(list(sub_diastole[key][0]))
else:
out.extend(list(sub_systole[key][0]))
else:
print('Miss {0}'.format(idx))
fo.writerow(out)
f.close()
def load_model(input_shape, n_outputs=100):
'''Loads and compiles pre-trained model to be used for real-time predictions'''
model = m.get_model(input_size=input_shape, n_outputs=n_outputs)
model.load_weights("pre_trained_weights/latest_model_weights.hdf5")
model.compile(loss='categorical_crossentropy', optimizer='adam')
return model
def build_submission(config):
model_systole = get_model()
model_diastole = get_model()
print('Loading models weights...')
model_systole.load_weights(config.systole_weights)
model_diastole.load_weights(config.diastole_weights)
# load val losses to use as sigmas for CDF
with open(config.val_loss_systole, 'r') as f:
val_loss_systole = float(f.readline())
with open(config.val_loss_diastole, 'r') as f:
val_loss_diastole = float(f.readline())
print('Loading validation data...')
X, ids, mult = load_validation_data()
batch_size = 32
print('Predicting on validation data...')
pred_normed_systole = model_systole.predict(X, batch_size=batch_size, verbose=1)
pred_normed_diastole = model_diastole.predict(X, batch_size=batch_size, verbose=1)
print('Normed_systole:', pred_normed_systole.shape)
print('Normed_diastole:', pred_normed_diastole.shape)
print('mult:', mult.shape)
pred_systole = pred_normed_systole[:,0] * mult
pred_diastole = pred_normed_diastole[:,0] * mult
print('systole:', pred_systole.shape)
print('diastole:', pred_diastole.shape)
# real predictions to CDF
cdf_pred_systole = real_to_cdf(pred_systole, val_loss_systole)
cdf_pred_diastole = real_to_cdf(pred_diastole, val_loss_diastole)
print('Accumulating results...')
sub_systole = accumulate_study_results(ids, cdf_pred_systole)
sub_diastole = accumulate_study_results(ids, cdf_pred_diastole)
# write to submission file
print('Writing submission to file...')
fi = csv.reader(open('/data/sample_submission_validate.csv'))
f = open(config.submission, 'w')
fo = csv.writer(f, lineterminator='\n')
fo.writerow(next(fi))
for line in fi:
idx = line[0]
key, target = idx.split('_')
key = int(key)
out = [idx]
if key in sub_systole:
if target == 'Diastole':
out.extend(list(sub_diastole[key][0]))
else:
out.extend(list(sub_systole[key][0]))
else:
print('Miss {0}'.format(idx))
fo.writerow(out)
f.close()
print('Done.')
from flask import Flask, render_template
import preprocessor
from preprocessor import preprocess_text
import model
from model import get_model
import tokenizer
from tokenizer import get_tokenizer
import pandas as pd
import numpy as np
from keras.preprocessing.sequence import pad_sequences
from mappings import map_emoji
tokenizer = get_tokenizer()
model = get_model()
app = Flask(__name__)
@app.route('/<string:text>')
def index(text):
text_processed = preprocess_text(text)
df_test = pd.DataFrame({'Text': [text_processed]})
X_predict = tokenizer.texts_to_sequences(df_test['Text'].values)
print("Here")
X_predict = pad_sequences(X_predict)
pred = model.predict(X_predict)
pred_labels = np.argmax(pred, axis=1)
pred_emojis = map_emoji(pred_labels)
return pred_emojis[0]
def train(config):
self_summary_strs = [] # summary string to print out for later
# first, read both data and filter stuff, to get the word2vec idx,
train_data = read_data(config, 'train', config.load)
val_data = read_data(
config, 'val', True
) # dev should always load model shared data(word2idx etc.) from train
# now that the dataset is loaded , we get the max_word_size from the dataset
# then adjust the max based on the threshold as well
# also get the vocab size
config_vars = vars(config)
str_ = "threshold setting--\n" + "\t" + " ,".join(
["%s:%s" % (key, config_vars[key]) for key in config.thresmeta])
print str_
self_summary_strs.append(str_)
# cap the numbers
# max sentence word count etc.
update_config(config, [train_data, val_data],
showMeta=True) # all word num is <= max_thres
str_ = "renewed ----\n" + "\t" + " ,".join(
["%s:%s" % (key, config_vars[key]) for key in config.maxmeta])
print str_
self_summary_strs.append(str_)
# now we initialize the matrix for word embedding for word not in glove
word2vec_dict = train_data.shared['word2vec']
word2idx_dict = train_data.shared[
'word2idx'] # this is the word not in word2vec
# we are not fine tuning , so this should be empty
idx2vec_dict = {
word2idx_dict[word]: vec
for word, vec in word2vec_dict.items() if word in word2idx_dict
}
#print len(idx2vec_dict) # 0
# config.word_vocab_size = len(train_data.shared['word2idx']) # the word not in word2vec
# so the emb_mat should all be a random vector
# np.random.multivariate_normal gets mean of zero and co of 1 for each dim, like
#>>> np.random.multivariate_normal(np.zeros(5),np.eye(5))
#array([-0.73663652, -1.16417783, -0.74083293, -0.80016731, 0.060182 ])
# random initial embedding matrix for new words
config.emb_mat = np.array([
idx2vec_dict[idx]
if idx2vec_dict.has_key(idx) else np.random.multivariate_normal(
np.zeros(config.word_emb_size), np.eye(config.word_emb_size))
for idx in xrange(config.word_vocab_size)
],
dtype="float32")
model = get_model(config) # construct model under gpu0
trainer = Trainer(model, config)
tester = Tester(model, config)
saver = tf.train.Saver(max_to_keep=5) # how many model to keep
bestsaver = tf.train.Saver(max_to_keep=5) # just for saving the best model
save_period = config.save_period # also the eval period
# start training!
# allow_soft_placement : tf will auto select other device if the tf.device(*) not available
tfconfig = tf.ConfigProto(allow_soft_placement=True)
tfconfig.gpu_options.allow_growth = True # this way it will only allocate nessasary gpu, not take all
# or you can set hard limit
#tfconfig.gpu_options.per_process_gpu_memory_fraction = 0.4
with tf.Session(config=tfconfig) as sess:
# calculate total parameters
totalParam = cal_total_param()
str_ = "total parameters: %s" % (totalParam)
print str_
self_summary_strs.append(str_)
initialize(load=config.load,
load_best=config.load_best,
model=model,
config=config,
sess=sess)
# the total step (iteration) the model will run
last_time = time.time()
# total / batchSize * epoch
num_steps = int(
math.ceil(train_data.num_examples /
float(config.batch_size))) * config.num_epochs
# get_batches is a generator, run on the fly
# there will be num_steps batch
str_ = " batch_size:%s, epoch:%s,total step:%s,eval/save every %s steps" % (
config.batch_size, config.num_epochs, num_steps,
config.save_period)
print str_
self_summary_strs.append(str_)
best = {
"acc": 0.0,
"step": -1
} # remember the best eval acc during training
finalAcc = None
isStart = True
for batch in tqdm(train_data.get_batches(config.batch_size,
num_steps=num_steps),
total=num_steps):
# each batch has (batch_idxs,Dataset(batch_data, full_shared))
# batch_data has {"q":,"y":..."pidx2feat",.."photo_idxs"..}
global_step = sess.run(model.global_step) + 1 # start from 0
# if load from existing model, save if first
if config.load and isStart:
tqdm.write("saving original model...")
tqdm.write("\tsaving model...")
saver.save(sess,
config.save_dir_model,
global_step=global_step)
tqdm.write("\tdone")
isStart = False
id2predanswers = {}
id2realanswers = {}
for evalbatch in val_data.get_batches(
config.batch_size,
num_steps=config.val_num_batches,
shuffle=False,
cap=True):
yp = tester.step(
sess, evalbatch
) # [N,4] # id2realanswersprob for each answer
pred, gt = getAnswers(
yp, evalbatch) # from here we get the qid:yindx,
id2predanswers.update(pred)
id2realanswers.update(gt)
evalAcc = getEvalScore(id2predanswers, id2realanswers)
tqdm.write(
"\teval on validation %s batches Acc:%s, (best:%s at step %s) "
% (config.val_num_batches, evalAcc, best['acc'],
best['step']))
# remember the best acc
if (evalAcc > best['acc']):
best['acc'] = evalAcc
best['step'] = global_step
# save the best model
tqdm.write("\t saving best model...")
bestsaver.save(sess,
config.save_dir_best_model,
global_step=global_step)
tqdm.write("\t done.")
finalAcc = evalAcc
loss, train_op = trainer.step(sess, batch)
if global_step % save_period == 0: # time to save model
duration = time.time() - last_time # in seconds
sec_per_step = duration / float(save_period)
last_time = time.time()
#use tqdm to print
tqdm.write(
"step:%s/%s (epoch:%.3f), took %s, loss:%s, estimate remaining:%s"
% (global_step, num_steps,
(config.num_epochs * global_step / float(num_steps)),
sec2time(duration), loss,
sec2time((num_steps - global_step) * sec_per_step)))
tqdm.write("\tsaving model...")
saver.save(sess,
config.save_dir_model,
global_step=global_step)
tqdm.write("\tdone")
id2predanswers = {}
id2realanswers = {}
for evalbatch in val_data.get_batches(
config.batch_size,
num_steps=config.val_num_batches,
shuffle=False,
cap=True):
yp = tester.step(
sess, evalbatch
) # [N,4] # id2realanswersprob for each answer
pred, gt = getAnswers(
yp, evalbatch) # from here we get the qid:yindx,
id2predanswers.update(pred)
id2realanswers.update(gt)
evalAcc = getEvalScore(id2predanswers, id2realanswers)
tqdm.write(
"\teval on validation %s batches Acc:%s, (best:%s at step %s) "
% (config.val_num_batches, evalAcc, best['acc'],
best['step']))
# remember the best acc
if (evalAcc > best['acc']):
best['acc'] = evalAcc
best['step'] = global_step
# save the best model
tqdm.write("\t saving best model...")
bestsaver.save(sess,
config.save_dir_best_model,
global_step=global_step)
tqdm.write("\t done.")
finalAcc = evalAcc
if global_step % save_period != 0: # time to save model
saver.save(sess, config.save_dir_model, global_step=global_step)
str_ = "best eval on val Accurucy: %s at %s step, final step %s Acc is %s" % (
best['acc'], best['step'], global_step, finalAcc)
print str_
self_summary_strs.append(str_)
if config.write_self_sum:
f = open(config.self_summary_path, "w")
f.writelines("%s" % ("\n".join(self_summary_strs)))
f.close()
def main(_):
start_time = time.time()
logging.info("TensorFlow version: %s", tf.__version__)
logging.info("TensorFlow git version: %s", tf.__git_version__)
if KUBERNETES:
tf_config_json = os.environ.get("TF_CONFIG", "{}")
tf_config = json.loads(tf_config_json)
logging.info("tf_config: {}".format(tf_config))
task = tf_config.get("task", {})
task_index = task["index"]
job_name = task["type"]
logging.info("task: {}".format(task))
cluster_spec = tf_config.get("cluster", {})
logging.info("cluster_spec: {}".format(cluster_spec))
else: # Local testing
task_index = FLAGS.task_index
job_name = FLAGS.job_name
cluster_spec = {"ps": ["localhost:2222"],
"worker": ["localhost:2223",
"localhost:2224"]}
worker_list = cluster_spec.get("worker", "{}")
ps_list = cluster_spec.get("ps", "{}")
logging.info("job_name: {}".format(job_name))
logging.info("task_index: {}".format(task_index))
config = tf.ConfigProto(
inter_op_parallelism_threads=num_inter_op_threads,
intra_op_parallelism_threads=num_intra_op_threads)
cluster = tf.train.ClusterSpec(cluster_spec)
server = tf.train.Server(cluster, job_name=job_name, task_index=task_index)
is_chief = (task_index == 0) # Am I the chief node (always task 0)
if job_name == "ps":
logging.info("I am parameter server #{}. "
"I will join the server and will "
"need to be explicitly terminated when all jobs end. "
"Kubernetes should do this automatically."
"Otherwise, use CTRL-\\ for manual termination".
format(task_index))
server.join()
elif job_name == "worker":
if is_chief:
logging.info("I am the chief worker {} with task #{}".format(
worker_list[task_index], task_index))
else:
logging.info("I am worker {} with task #{}".format(
worker_list[task_index], task_index))
# Graph
worker_device = "/job:{}/task:{}".format(job_name, task_index)
setter = tf.train. \
replica_device_setter(ps_tasks=len(ps_list),
worker_device=worker_device)
with tf.device(setter):
"""
BEGIN: Data loader
"""
data = load_data()
"""
END: Data loader
"""
"""
BEGIN: MODEL DEFINE
"""
model = get_model(data,
FLAGS,
is_chief,
len(worker_list))
"""
END: MODEL DEFINE
"""
"""
Monitored Training Session
"""
checkpoint_dir = None
# if is_chief:
# checkpoint_dir = FLAGS.CHECKPOINTS_DIRECTORY
# else:
# checkpoint_dir = None
params = dict(master=server.target,
is_chief=is_chief,
config=config,
hooks=model["hooks"],
checkpoint_dir=checkpoint_dir,
stop_grace_period_secs=10)
with tf.train.MonitoredTrainingSession(**params) as sess:
logging.info("Starting training on worker {}".format(
task_index))
batch_start = 0
batch_stop = batch_start + FLAGS.batch_size
epoch = 1
while not sess.should_stop():
x_train = data["x_train"][batch_start:batch_stop]
y_train = data["y_train"][batch_start:batch_stop]
_, loss, step = sess.run([model["optimizer"],
model["loss"],
model["global_step"]],
feed_dict={
model["input_tensor"]: x_train,
model["label_tensor"]: y_train})
batch_start += FLAGS.batch_size
if batch_start >= data["num_train"]:
batch_start = 0
epoch += 1
logging.info("Epoch {}, Batch Size = {}".format(
epoch, FLAGS.batch_size))
batch_stop = batch_start + FLAGS.batch_size
logging.info("worker {}, step {} of {}: loss = {:.4}".
format(task_index,
step,
FLAGS.steps_to_train,
loss))
if ((step % FLAGS.steps_to_validate) == 0) and is_chief:
loss, accuracy = sess.run([model["loss"],
model["accuracy"]],
feed_dict={
model["input_tensor"]:
data["x_test"],
model["label_tensor"]:
data["y_test"]})
logging.info("{} test validation: loss = {:.4f}"
", accuracy of the model is {:.3f}".format(
data["name"], loss, accuracy))
logging.info("Finished on task {} in {} seconds".format(
task_index, time.time() - start_time))
logging.info(
"Session from worker {} closed cleanly".format(task_index))
def test(config):
if config.is_test_on_val:
test_data = read_data(config, 'val', True)
print "total val samples:%s" % test_data.num_examples
else:
test_data = read_data(
config, 'test', True
) # here will load shared.p from config.outpath (outbase/modelname/runId/)
print "total test samples:%s" % test_data.num_examples
# get the max_sent_size and other stuff
print "threshold setting--"
config_vars = vars(config)
print "\t" + " ,".join(
["%s:%s" % (key, config_vars[key]) for key in config.thresmeta])
# cap the numbers
update_config(config, [test_data], showMeta=True)
print "renewed ----"
print "\t" + " ,".join(
["%s:%s" % (key, config_vars[key]) for key in config.maxmeta])
model = get_model(config)
# update each batch forward into this dict
id2predanswers = {}
id2realanswers = {}
id2yp = {}
tfconfig = tf.ConfigProto(allow_soft_placement=True)
tfconfig.gpu_options.allow_growth = True # this way it will only allocate nessasary gpu, not take all
# or you can set hard limit
#tfconfig.gpu_options.per_process_gpu_memory_fraction = 0.4
with tf.Session(config=tfconfig) as sess:
initialize(load=True,
load_best=config.load_best,
model=model,
config=config,
sess=sess)
if config.is_save_weights:
weight_dict = {}
weight_sum = open(os.path.join(config.weights_path, "all.txt"),
"w")
for var in tf.trainable_variables():
shape = var.get_shape()
weight_sum.writelines("%s %s\n" % (var.name, shape))
var_val = sess.run(var)
weight_dict[var.name] = var_val
np.savez(os.path.join(config.weights_path, "weights.npz"),
**weight_dict)
weight_sum.close()
last_time = time.time()
# num_epoch should be 1
num_steps = int(
math.ceil(test_data.num_examples /
float(config.batch_size))) * config.num_epochs
# load the graph and variables
tester = Tester(model, config, sess)
count = 0
print "total step:%s" % num_steps
for batch in tqdm(test_data.get_batches(config.batch_size,
num_steps=num_steps,
shuffle=False),
total=num_steps):
count += 1
yp = tester.step(
sess, batch) # [N,4] # id2realanswersprob for each answer
if config.get_yp:
pred, gt, yp = getAnswers_yp(yp, batch)
id2yp.update(yp)
else:
pred, gt = getAnswers(yp,
batch) # from here we get the qid:yindx,
id2predanswers.update(pred)
id2realanswers.update(gt)
acc = getEvalScore(id2predanswers, id2realanswers)
print "done, got %s answers, accuracy:%s" % (len(id2predanswers), acc)
json.dump(id2predanswers, open("%s/answers.json" % config.val_path, "w"))
if config.get_yp:
json.dump({id_: "%s" % (id2yp[id_])
for id_ in id2yp}, open("%s/yps.json" % config.val_path,
"w"))
NUM_LAYERS = args['num_layers']
MODEL_WEIGHTS = args['model_weights']
if TAR_PATH is not None:
untar(TAR_PATH)
if TEST_TAR_PATH is not None:
untar(TEST_TAR_PATH)
X, Y, input_vocab, output_vocab, input_dict, output_dict = prep_data(
DATASET_PATH, INPUT_LENGTH, OUTPUT_LENGTH)
X_test, Y_test, test_input_vocab, test_output_vocab, _, _ = prep_data(
TEST_PATH, INPUT_LENGTH, OUTPUT_LENGTH)
auto_encoder = get_model(len(input_vocab), INPUT_LENGTH, len(output_vocab),
OUTPUT_LENGTH, HIDDEN_SIZE, NUM_LAYERS)
auto_encoder.load_weights(MODEL_WEIGHTS)
auto_encoder.summary()
expected_codes = vec_to_words(Y_test, test_output_vocab)
tasks = input_vec_to_words(X_test, test_input_vocab)
codes = predict_codes(auto_encoder, X_test, input_dict, INPUT_LENGTH,
output_vocab)
print('*************************************************')
print('MODEL EVALUATION')
print(average_bleu(expected_codes, codes))
print(average_code_compilance(codes))
pp_results(tasks, codes)
print('*************************************************')
def train():
"""
model training
"""
model = get_model()
model.train()
X_train.append(training_set_scaled[i - 30:i, 0])
y_train.append(training_set_scaled[i, 0])
X_train, y_train = np.array(X_train), np.array(y_train)
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
X_test = []
y_test = []
for i in range(30, test_set.shape[0]):
X_test.append(test_set_scaled[i - 30:i, 0])
y_test.append(test_set_scaled[i, 0])
X_test, y_test = np.array(X_test), np.array(y_test)
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
model = get_model(X_train)
model.compile(loss='mse', optimizer='Adam')
model.fit(X_train[:X_train.shape[0] * 80 // 100],
y_train[:X_train.shape[0] * 80 // 100],
epochs=100,
batch_size=512,
validation_data=(X_train[X_train.shape[0] * 80 // 100:],
y_train[X_train.shape[0] * 80 // 100:]),
verbose=2,
shuffle=False)
model.save(os.path.join(experiment_name, "model.h5"))
def denoise(self):
# DEBUG, INFO, WARN, ERROR, or FATAL
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.FATAL) # silence deprecate warning message
config = tf.ConfigProto() # for ubuntu nvidia driver
config.gpu_options.allow_growth = True # for ubuntu nvidia driver
tf.keras.backend.set_session(tf.Session(config=config)) # for ubuntu nvidia driver
image_dir = self.input_dir
weight_file = self.weight_path
is_sharpening = False
val_noise_model = get_noise_model('gaussian,20,20')
model = get_model(self.model_category)
model.load_weights(weight_file)
# if args.output_dir:
# output_dir = Path(args.output_dir)
# output_dir.mkdir(parents=True, exist_ok=True)
#
# save_arg_filename = Path(output_dir) / 'args.txt'
# with open(str(save_arg_filename), 'w') as f:
# json.dump(args.__dict__, f, indent=2)
# --kw: glob -> rglob--
image_paths = list(Path(image_dir).expanduser().rglob("*.png"))
image_paths += list(Path(image_dir).expanduser().rglob("*.tif"))
image_paths += list(Path(image_dir).expanduser().rglob("*.jpg"))
image_paths += list(Path(image_dir).expanduser().rglob("*.bmp"))
for image_path in tqdm(image_paths):
image_gray = cv2.imread(str(image_path), cv2.IMREAD_GRAYSCALE)
image = cv2.merge([image_gray, image_gray, image_gray])
h, w, _ = image.shape
image = image[:(h // 16) * 16, :(w // 16) * 16] # for stride (maximum 16)
h, w, _ = image.shape
out_image = np.zeros((h, w * 3, 3), dtype=np.uint8)
noise_image = val_noise_model(image)
prediction = model.predict(np.expand_dims(noise_image, 0))
denoised_image = self.clip_image(prediction[0])
# if is_sharpening == 'True':
# print('sharpening')
# dst = cv2.Laplacian(denoised_image, cv2.CV_64F, ksize=1)
# denoised_image_float = denoised_image.astype(np.float64)
# denoised_image = denoised_image_float - dst
# denoised_image = np.clip(denoised_image, 0, 255).astype(np.uint8)
out_image[:, :w] = image
out_image[:, w:w * 2] = noise_image
out_image[:, w * 2:] = denoised_image
out_image = cv2.cvtColor(out_image, cv2.COLOR_BGR2GRAY)
# diff = cv2.absdiff(out_image[:, w * 2:], out_image[:, :w])
# fig_diff, ax = plt.subplots(figsize=(10, 5))
# fig_diff.subplots_adjust(hspace=0.2, wspace=0.2) # 設定子圖的間隔
# plt.subplot(1, 2, 1)
# plt.imshow(diff, vmin=0, vmax=255)
# plt.subplot(1, 2, 2)
# plt.hist(diff.ravel(), bins=256, range=[0, 256])
# plt.show()
if not self.debugMode:
output_dir = Path(self.output_dir).expanduser().resolve()
# print('output_dir', output_dir)
rawImgNameWithoutExtension = str(Path(image_path).stem)
save_file_name = output_dir / str('deN-'+rawImgNameWithoutExtension+'.png')
# save_histogram_name = str(save_img_name)[:-4] + '_histogram.png'
# print('save_img_name', save_img_name)
# print('save_histogram_name', save_histogram_name)
if not Path(output_dir).exists():
os.makedirs(output_dir)
cv2.imwrite(str(save_file_name), out_image)
# plt.savefig(save_histogram_name, dpi=300)
# plt.close()
else:
cv2.imshow("result", out_image)
key = cv2.waitKey(-1)
# "q": quit
if key == 113:
return 0
model = joblib.load(f)
elif os.path.basename(args.model).endswith('.pt'): # pytorch model
align_data = preprocessor.align_data
train_target = preprocessor.target
import torch
import torch.nn.functional as F
device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
normalLogger.debug('load model .pt file...')
model, _ = get_model(
args.algorithm, #scaler=scaler,
in_features=len(align_data.columns),
num_classes=len(set(train_target)),
mid_features=256)
state = torch.load(args.model)
#state = torch.load('./checkpoint/epoch-5-val_loss0.037-val_acc0.972.pt')
model.load_state_dict(state['state_dict'])
print(model)
model.eval()
normalLogger.debug('successfuly load model')
except:
normalLogger.debug(
'fail to load model...check model path or do training model')
assert False, 'fail to load model'
type=str,
default="config.yaml",
help='path of config file')
args = parser.parse_args()
with open(args.config, 'r') as file:
config = yaml.load(file)
paths = config["paths"]
data = config["data"]
dataloader = DataLoader(config)
dataloader.load()
input_shape = (data["imsize"], data["imsize"], data["imchannel"])
model = get_model(input_shape, config, top=False)
model.load_weights(paths["load"], by_name=True)
X_batch, y_batch = dataloader.get_random_batch(k=-1)
#embeddings = X_batch.reshape(-1, 784)
embeddings = model.predict(X_batch,
batch_size=config["train"]["batch-size"],
verbose=1)
tsne = TSNE(n_components=2,
perplexity=config["tsne"]["perplexity"],
verbose=1,
n_iter=config["tsne"]["n_iter"])
tsne_embeds = tsne.fit_transform(embeddings)
def main():
opt = parse_args()
print(json.dumps(vars(opt), indent=2))
rootpath = opt.rootpath
collectionStrt = opt.collectionStrt
resume = os.path.join(opt.logger_name, opt.checkpoint_name)
if not os.path.exists(resume):
logging.info(resume + ' not exists.')
sys.exit(0)
checkpoint = torch.load(resume)
start_epoch = checkpoint['epoch']
best_rsum = checkpoint['best_rsum']
print("=> loaded checkpoint '{}' (epoch {}, best_rsum {})".format(
resume, start_epoch, best_rsum))
options = checkpoint['opt']
# collection setting
testCollection = opt.testCollection
collections_pathname = options.collections_pathname
collections_pathname['test'] = testCollection
trainCollection = options.trainCollection
output_dir = resume.replace(trainCollection, testCollection)
if 'checkpoints' in output_dir:
output_dir = output_dir.replace('/checkpoints/', '/results/')
else:
output_dir = output_dir.replace(
'/%s/' % options.cv_name,
'/results/%s/%s/' % (options.cv_name, trainCollection))
result_pred_sents = os.path.join(output_dir, 'id.sent.score.txt')
pred_error_matrix_file = os.path.join(output_dir,
'pred_errors_matrix.pth.tar')
if checkToSkip(pred_error_matrix_file, opt.overwrite):
sys.exit(0)
makedirsforfile(pred_error_matrix_file)
log_config(output_dir)
logging.info(json.dumps(vars(opt), indent=2))
# data loader prepare
test_cap = os.path.join(rootpath, collections_pathname['test'], 'TextData',
'%s.caption.txt' % testCollection)
if collectionStrt == 'single':
test_cap = os.path.join(
rootpath, collections_pathname['test'], 'TextData',
'%s%s.caption.txt' % (testCollection, opt.split))
elif collectionStrt == 'multiple':
test_cap = os.path.join(rootpath, collections_pathname['test'],
'TextData', '%s.caption.txt' % testCollection)
else:
raise NotImplementedError('collection structure %s not implemented' %
collectionStrt)
caption_files = {'test': test_cap}
img_feat_path = os.path.join(rootpath, collections_pathname['test'],
'FeatureData', options.visual_feature)
visual_feats = {'test': BigFile(img_feat_path)}
assert options.visual_feat_dim == visual_feats['test'].ndims
video2frames = {
'test':
read_dict(
os.path.join(rootpath, collections_pathname['test'], 'FeatureData',
options.visual_feature, 'video2frames.txt'))
}
# set bow vocabulary and encoding
bow_vocab_file = os.path.join(rootpath, collections_pathname['train'],
'TextData', 'vocabulary', 'bow',
options.vocab + '.pkl')
bow_vocab = pickle.load(open(bow_vocab_file, 'rb'))
bow2vec = get_text_encoder('bow')(bow_vocab)
options.bow_vocab_size = len(bow_vocab)
# set rnn vocabulary
rnn_vocab_file = os.path.join(rootpath, collections_pathname['train'],
'TextData', 'vocabulary', 'rnn',
options.vocab + '.pkl')
rnn_vocab = pickle.load(open(rnn_vocab_file, 'rb'))
options.vocab_size = len(rnn_vocab)
# Construct the model
model = get_model(options.model)(options)
model.load_state_dict(checkpoint['model'])
model.Eiters = checkpoint['Eiters']
model.val_start()
# set data loader
video_ids_list = data.read_video_ids(caption_files['test'])
vid_data_loader = data.get_vis_data_loader(visual_feats['test'],
opt.batch_size,
opt.workers,
video2frames['test'],
video_ids=video_ids_list)
text_data_loader = data.get_txt_data_loader(caption_files['test'],
rnn_vocab, bow2vec,
opt.batch_size, opt.workers)
# mapping
if options.space == 'hybrid':
video_embs, video_tag_probs, video_ids = evaluation.encode_text_or_vid_tag_hist_prob(
model.embed_vis, vid_data_loader)
cap_embs, cap_tag_probs, caption_ids = evaluation.encode_text_or_vid_tag_hist_prob(
model.embed_txt, text_data_loader)
else:
video_embs, video_ids = evaluation.encode_text_or_vid(
model.embed_vis, vid_data_loader)
cap_embs, caption_ids = evaluation.encode_text_or_vid(
model.embed_txt, text_data_loader)
v2t_gt, t2v_gt = metrics.get_gt(video_ids, caption_ids)
logging.info("write into: %s" % output_dir)
if options.space != 'latent':
tag_vocab_path = os.path.join(
rootpath, collections_pathname['train'], 'TextData', 'tags',
'video_label_th_1', 'tag_vocab_%d.json' % options.tag_vocab_size)
evaluation.pred_tag(video_tag_probs, video_ids, tag_vocab_path,
os.path.join(output_dir, 'video'))
evaluation.pred_tag(cap_tag_probs, caption_ids, tag_vocab_path,
os.path.join(output_dir, 'text'))
if options.space in ['latent', 'hybrid']:
# logging.info("=======Latent Space=======")
t2v_all_errors_1 = evaluation.cal_error(video_embs, cap_embs,
options.measure)
if options.space in ['concept', 'hybrid']:
# logging.info("=======Concept Space=======")
t2v_all_errors_2 = evaluation.cal_error_batch(video_tag_probs,
cap_tag_probs,
options.measure_2)
if options.space in ['hybrid']:
w = 0.6
t2v_all_errors_1 = norm_score(t2v_all_errors_1)
t2v_all_errors_2 = norm_score(t2v_all_errors_2)
t2v_tag_all_errors = w * t2v_all_errors_1 + (1 - w) * t2v_all_errors_2
cal_perf(t2v_tag_all_errors, v2t_gt, t2v_gt)
torch.save(
{
'errors': t2v_tag_all_errors,
'videos': video_ids,
'captions': caption_ids
}, pred_error_matrix_file)
logging.info("write into: %s" % pred_error_matrix_file)
elif options.space in ['latent']:
cal_perf(t2v_all_errors_1, v2t_gt, t2v_gt)
torch.save(
{
'errors': t2v_all_errors_1,
'videos': video_ids,
'captions': caption_ids
}, pred_error_matrix_file)
logging.info("write into: %s" % pred_error_matrix_file)
def predict(*args, **kwargs):
def remove_sents(sentences_t, labels_t, max_length):
remove_idxs = []
for i, sentence in enumerate(sentences_t):
if len(sentence) > max_length:
remove_idxs.append(i)
for i in remove_idxs:
sentences_t.pop(i)
labels_t.pop(i)
word_index = {}
label_index = {}
max_length = 0
with open(configs.DICT_FILE, 'r', encoding='utf-8') as file:
dicts = file.read()
dicts = dicts.split("\n")
word_index = eval(dicts[0])
label_index = eval(dicts[1])
max_length = eval(dicts[2])
with open(configs.EMBEDDINGS_FILE, 'rb') as file:
word_embeddings = np.load(file)
#Loading test sequences
sentences_t, labels_t, max_length_t = prd.read_data(configs.TEST_FILE)
remove_sents(sentences_t, labels_t, max_length)
print('Total no of test sequences: ', len(sentences_t))
char_index = prd.get_vocabulory(word_index)
char_idxs = prd.get_chars(sentences_t, max_length, char_index)
label_idxs = prd.get_sequence_indices(labels_t, label_index,
max_length)
seq_idxs = prd.get_sequence_indices(sentences_t, word_index,
max_length)
assert (len(seq_idxs) == len(label_idxs)
), "length of I/O sequences doesn't match"
index_labels = {}
for item, i in label_index.items():
index_labels[i] = item
model_t = mdl.get_model(word_embeddings, max_length, len(char_index),
len(index_labels), True)
# Predict labels for test data
pred_label = np.asarray(model_t.predict([seq_idxs, char_idxs]))
pred_label = np.argmax(pred_label, axis=-1)
#Skipping padded sequences
pred_label = prd.get_orig_labels(pred_label, index_labels, labels_t)
print("Predicted Labels--->\n", pred_label[0])
outputfile = configs.OUTPUT_FILE
with open(outputfile, 'w', encoding='utf-8') as f:
f.write('WORD' + ' ' + 'TRUE_LABEL' + ' ' + 'PRED_LABEL')
f.write('\n')
f.write('\n')
for i in range(len(pred_label)):
cur_sentences = sentences_t[i]
cur_labels = labels_t[i]
cur_pred = pred_label[i]
for j in range(len(cur_sentences)):
f.write(cur_sentences[j] + ' ' + cur_labels[j] + ' ' +
cur_pred[j])
f.write('\n')
f.write('\n')
f.write('\n')
with open(outputfile, 'r', encoding='utf-8') as f:
for line in f.readlines():
print(line)
return ([labels_t, pred_label])
if not args_dict.model_file: # run all training
model, model_val = init_models(args_dict)
model, model_name = trainloop(args_dict,model,model_val = model_val)
epoch_start = args_dict.nepochs
del model
K.clear_session()
else:
epoch_start = 0
### Fine Tune ConvNet ###
# get fine tuning params in place
args_dict.lr = args_dict.ftlr
args_dict.nepochs = args_dict.ftnepochs
model = get_model(args_dict)
opt = get_opt(args_dict)
if args_dict.model_file:
print "Loading snapshot: ", args_dict.model_file
model.load_weights(args_dict.model_file)
else:
model.load_weights(model_name)
for i,layer in enumerate(model.layers[1].layers):
if i > args_dict.finetune_start_layer:
layer.trainable = True
model.compile(optimizer=opt,loss='categorical_crossentropy',
sample_weight_mode="temporal")
args_dict.cnn_train = True
return
if __name__ == "__main__":
print("Loading Config")
config_file = "./default-config,yaml"
config = load_config_file(config_file)
# dataloaders
print("Getting Dataloader")
test_loader, total_classes = load_dataset(config)
# creating model
print("Loading Model")
model = get_model(num_classes=total_classes)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model.to(device)
# loading model
if not os.path.isfile(config["MODEL_TEST_PATH"]):
print("Model checkpoint weights does no exist, exiting....")
import sys
sys.exit()
else:
ckpt = torch.load(config["MODEL_TEST_PATH"])
if "model_state_dict" in ckpt:
model.load_state_dict(ckpt["model_state_dict"])
if "accuracy" in ckpt:
print(
"The loaded model has Validation accuracy of: {:.2f} %\n".
import dataset
import engine
from model import get_model
if __name__ == '__main__':
data_path = 'C:/Users/Arpan/Downloads/Education/ML/Approaching-almost-any-Machine-Learning-Problem/Computer Vision/Data'
device = 'cuda'
df = pd.read_csv(os.path.join(data_path, 'train.csv'))
epochs = 10
images = df['ImageId'].values.tolist()
images = [os.path.join(data_path, 'train_png', i + '.png') for i in images]
targets = df['target'].values
model = get_model(pretrained=True, model_name='resnet34')
model.to(device)
# imagenet stats
mean = (0.485, 0.456, 0.406)
std = (0.229, 0.224, 0.225)
aug = albumentations.Compose([
albumentations.Normalize(mean,
std,
max_pixel_value=255,
always_apply=True)
])
train_images, val_images, train_targets, val_targets = train_test_split(
def run(epochs=500, training_percentage=0.4, validation_percentage=0.1, extract=True, cont=True, size=256, top_k=5):
'''Does the routine required to get the data, put them in needed format and start training the model
saves weights whenever the model produces a better test result and keeps track of the best loss'''
if extract:
print("Extracting data..")
X, y = data.extract_data(size=size)
print("Preprocessing data..")
X, y, nb_samples, num_categories = data.preprocess_data(X, y, save=True, subtract_mean=True)
else:
print("Loading data..")
h5f = h5py.File('data.hdf5', 'r')
nb_samples = h5f['nb_samples'].value
num_categories = h5f['n_categories'].value
h5f.close()
print("Number of categories: {}".format(num_categories))
print("Number of samples {}".format(nb_samples))
data_ids = np.arange(start=0, stop=nb_samples)
val_ids = data.produce_validation_indices(data_ids, nb_samples * validation_percentage)
train_ids = data.produce_train_indices(dataset_indx=data_ids, number_of_samples=nb_samples * training_percentage,
val_indx=val_ids)
# X_train, y_train, X_test, y_test = data.split_data(X, y, split_ratio=split)
X_train, y_train, X_val, y_val = data.load_dataset_bit_from_hdf5(train_ids, val_ids, only_train=False)
X_val = X_val / 255
print("Building and Compiling model..")
model = m.get_model(n_outputs=num_categories, input_size=size)
if cont:
# model.load_weights_until_layer("pre_trained_weights/latest_model_weights.hdf5", 26)
model.load_weights("pre_trained_weights/latest_model_weights.hdf5")
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=["accuracy"])
print("Training..")
best_performance = np.inf
for i in range(epochs):
train_ids = data.produce_train_indices(dataset_indx=data_ids, number_of_samples=15000, val_indx=val_ids)
X_train, y_train = data.load_dataset_bit_from_hdf5(train_ids, val_ids, only_train=True)
X_train = X_train / 255
X_train = data.augment_data(X_train)
# fit the model on the batches generated by datagen.flow()
metadata = model.fit(X_train, y_train, validation_data=[X_val, y_val], batch_size=64,
nb_epoch=1, verbose=1, shuffle=True, class_weight=None,
sample_weight=None)
current_loss = metadata.history['loss'][-1]
current_val_loss = metadata.history['val_loss'][-1]
preds = model.predict_proba(X_val, batch_size=64)
print("Loss: {}".format(current_loss))
print("Val_loss: {}".format(current_val_loss))
top_3_error = get_top_n_error(preds, y_val, top_k)
print("Top 3 error: {}".format(top_3_error))
if current_val_loss < best_performance:
model.save_weights("pre_trained_weights/model_weights.hdf5", overwrite=True)
best_performance = current_val_loss
print("Saving weights..")
model.save_weights("pre_trained_weights/latest_model_weights.hdf5", overwrite=True)
import os
import glob
import sys
sys.path.append('/home/ubuntu/vdp/clevr_inference/scene_parse/attr_net')
import json
from options import get_options
from datasets import get_dataloader
from model import get_model
import utils
COMP_CAT_DICT_PATH = '/home/ubuntu/vdp/clevr_inference/scene_parse/attr_net/tools/clevr_comp_cat_dict.json'
opt = get_options('test')
test_loader = get_dataloader(opt, 'test')
model = get_model(opt)
if opt.use_cat_label:
with open(COMP_CAT_DICT_PATH) as f:
cat_dict = utils.invert_dict(json.load(f))
if opt.dataset == 'clevr':
num_images = len(glob.glob(opt.clevr_val_img_dir + '/*.png'))
scenes = [{
'image_index': i,
'image_filename': 'CLEVR_val_%06d.png' % i,
'objects': []
} for i in range(num_images)]
# print("run_test.py", scenes)
count = 0
def training(model_name):
feature, label, _ = data.get_matrix('train', True)
classifier = model.get_model(model_name)
classifier.fit(feature, label)
return classifier
vocab = pickle.load(open(os.path.join(data_dir, 'instr_vocab.pkl'),'rb'))
ingr_vocab_size = len(ingrs_vocab)
instrs_vocab_size = len(vocab)
output_dim = instrs_vocab_size
print(ingr_vocab_size)
print(instrs_vocab_size)
kk = 'cuda'
device = torch.device(kk)
t= time.time()
import sys
sys.argv = ['']
del sys
args = get_parser()
args.maxseqlen = 15
args.ingr_only = False
model = get_model(args,ingr_vocab_size,instrs_vocab_size)
model.load_state_dict(torch.load('../checkpoints/inversecooking/im2ingr/checkpoints/modelbest.ckpt', map_location = None))
model.to(device)
model.eval()
model.ingrs_only = False
model.recipe_only = False
transf_list_batch = []
transf_list_batch.append(transforms.ToTensor())
transf_list_batch.append(transforms.Normalize((0.485, 0.456, 0.406),(0.229, 0.224, 0.225)))
to_input_transf = transforms.Compose(transf_list_batch)
greedy = [True, False, False, False]
beam = [-1, -1, -1, -1]
temperature = 1.0
def train():
print('Loading and compiling models...')
model_systole = get_model()
model_diastole = get_model()
print('Loading training data...')
X, y = load_train_data()
print('Pre-processing images...')
X = preprocess(X)
X_train, y_train, X_test, y_test = split_data(X, y, split_ratio=0.2)
nb_iter = 200
epochs_per_iter = 1
batch_size = 32
calc_crps = 1
min_val_loss_systole = sys.float_info.max
min_val_loss_diastole = sys.float_info.max
print('-'*50)
print('Training...')
print('-'*50)
for i in range(nb_iter):
print('-'*50)
print('Iteration {0}/{1}'.format(i + 1, nb_iter))
print('-'*50)
print('Augmenting images - rotations')
X_train_aug = rotation_augmentation(X_train, 15)
print('Augmenting images - shifts')
X_train_aug = shift_augmentation(X_train_aug, 0.1, 0.1)
print('Fitting systole model...')
hist_systole = model_systole.fit(X_train_aug, y_train[:, 0], shuffle=True, nb_epoch=epochs_per_iter,
batch_size=batch_size, validation_data=(X_test, y_test[:, 0]))
print('Fitting diastole model...')
hist_diastole = model_diastole.fit(X_train_aug, y_train[:, 1], shuffle=True, nb_epoch=epochs_per_iter,
batch_size=batch_size, validation_data=(X_test, y_test[:, 1]))
loss_systole = hist_systole.history['loss'][-1]
loss_diastole = hist_diastole.history['loss'][-1]
val_loss_systole = hist_systole.history['val_loss'][-1]
val_loss_diastole = hist_diastole.history['val_loss'][-1]
if calc_crps > 0 and i % calc_crps == 0:
print('Evaluating CRPS...')
pred_systole = model_systole.predict(X_train, batch_size=batch_size, verbose=1)
pred_diastole = model_diastole.predict(X_train, batch_size=batch_size, verbose=1)
val_pred_systole = model_systole.predict(X_test, batch_size=batch_size, verbose=1)
val_pred_diastole = model_diastole.predict(X_test, batch_size=batch_size, verbose=1)
cdf_train = real_to_cdf(np.concatenate((y_train[:, 0], y_train[:, 1])))
cdf_test = real_to_cdf(np.concatenate((y_test[:, 0], y_test[:, 1])))
cdf_pred_systole = real_to_cdf(pred_systole, loss_systole)
cdf_pred_diastole = real_to_cdf(pred_diastole, loss_diastole)
cdf_val_pred_systole = real_to_cdf(val_pred_systole, val_loss_systole)
cdf_val_pred_diastole = real_to_cdf(val_pred_diastole, val_loss_diastole)
crps_train = crps(cdf_train, np.concatenate((cdf_pred_systole, cdf_pred_diastole)))
print('CRPS(train) = {0}'.format(crps_train))
crps_test = crps(cdf_test, np.concatenate((cdf_val_pred_systole, cdf_val_pred_diastole)))
print('CRPS(test) = {0}'.format(crps_test))
print('Saving weights...')
model_systole.save_weights('weights_systole.hdf5', overwrite=True)
model_diastole.save_weights('weights_diastole.hdf5', overwrite=True)
if val_loss_systole < min_val_loss_systole:
min_val_loss_systole = val_loss_systole
model_systole.save_weights('weights_systole_best.hdf5', overwrite=True)
if val_loss_diastole < min_val_loss_diastole:
min_val_loss_diastole = val_loss_diastole
model_diastole.save_weights('weights_diastole_best.hdf5', overwrite=True)
with open('val_loss.txt', mode='w+') as f:
f.write(str(min_val_loss_systole))
f.write('\n')
f.write(str(min_val_loss_diastole))
# plt.ylabel('RMSE')
# plt.title('RMSE Loss Function')
# plt.legend('')
# plt.show()
# #crps
# plt.plot(x,y)
# plt.xlabel('Iteration')
# plt.ylabel('CRPS')
# plt.title('')
# plt.legend('')
# plt.show()
print('Loading and compiling models...')
model_systole = get_model()
model_diastole = get_model()
#import best model if it exists
if os.path.isfile('/data/run2/weights_systole_best.hdf5'):
print('loading weights')
model_systole.load_weights('/data/run2/weights_systole_best.hdf5')
if os.path.isfile('/data/run2/weights_diastole_best.hdf5'):
model_diastole.load_weights('/data/run2/weights_diastole_best.hdf5')
print('Loading training data...')
X, y, metadata = load_train_data()
print(metadata[0, :].shape, metadata[0].shape, metadata[0:1, :].shape)
pred_systole = model_systole.predict({'input1':X[40:50, :, : , :], 'input2':metadata[40:50, :], 'output':y[40:50, 0]})['output']
def main():
height = 512
width = 512
noise = 'gauss'
mode = 'clean'
args = get_args()
image_dir = args.image_dir
weight_file = 'weights_{}_{}.hdf5'.format(noise, mode) #args.weight_file
if mode != 'clean':
val_noise_model = get_noise_model(args.test_noise_model)
else:
model = get_model(height, width, args.model)
model.load_weights(weight_file)
model.summary()
# saved_model
tf.saved_model.save(
model, 'saved_model_{}_{}_{}_{}x{}'.format(args.model, noise, mode,
height, width))
# pb
full_model = tf.function(lambda inputs: model(inputs))
full_model = full_model.get_concrete_function(
inputs=[tf.TensorSpec(model.inputs[0].shape, model.inputs[0].dtype)])
frozen_func = convert_variables_to_constants_v2(full_model,
lower_control_flow=False)
frozen_func.graph.as_graph_def()
tf.io.write_graph(graph_or_graph_def=frozen_func.graph,
logdir=".",
name="noise2noise_{}_{}_{}_{}x{}_float32.pb".format(
args.model, noise, mode, height, width),
as_text=False)
# No Quantization - Input/Output=float32
converter = tf.lite.TFLiteConverter.from_keras_model(model)
tflite_model = converter.convert()
with open(
'noise2noise_{}_{}_{}_{}x{}_float32.tflite'.format(
args.model, noise, mode, height, width), 'wb') as w:
w.write(tflite_model)
print(
"tflite convert complete! - noise2noise_{}_{}_{}_{}x{}_float32.tflite".
format(args.model, noise, mode, height, width))
# Weight Quantization - Input/Output=float32
converter = tf.lite.TFLiteConverter.from_keras_model(model)
converter.optimizations = [tf.lite.Optimize.OPTIMIZE_FOR_SIZE]
tflite_model = converter.convert()
with open(
'noise2noise_{}_{}_{}_{}x{}_weight_quant.tflite'.format(
args.model, noise, mode, height, width), 'wb') as w:
w.write(tflite_model)
print(
'Weight Quantization complete! - noise2noise_{}_{}_{}_{}x{}_weight_quant.tflite'
.format(args.model, noise, mode, height, width))
# Float16 Quantization - Input/Output=float32
converter = tf.lite.TFLiteConverter.from_keras_model(model)
converter.optimizations = [tf.lite.Optimize.DEFAULT]
converter.target_spec.supported_types = [tf.float16]
tflite_quant_model = converter.convert()
with open(
'noise2noise_{}_{}_{}_{}x{}_float16_quant.tflite'.format(
args.model, noise, mode, height, width), 'wb') as w:
w.write(tflite_quant_model)
print(
'Float16 Quantization complete! - noise2noise_{}_{}_{}_{}x{}_float16_quant.tflite'
.format(args.model, noise, mode, height, width))
def representative_dataset_gen():
for data in raw_test_data.take(10):
image = data['image'].numpy()
image = tf.image.resize(image, (height, width))
image = image[np.newaxis, :, :, :]
# image = image / 127.5 - 1.0
yield [image]
raw_test_data, info = tfds.load(name="coco/2017",
with_info=True,
split="test",
data_dir="~/TFDS",
download=False)
# Integer Quantization - Input/Output=float32
converter = tf.lite.TFLiteConverter.from_keras_model(model)
converter.optimizations = [tf.lite.Optimize.DEFAULT]
converter.representative_dataset = representative_dataset_gen
tflite_quant_model = converter.convert()
with open(
'noise2noise_{}_{}_{}_{}x{}_integer_quant.tflite'.format(
args.model, noise, mode, height, width), 'wb') as w:
w.write(tflite_quant_model)
print(
'Integer Quantization complete! - noise2noise_{}_{}_{}_{}x{}_integer_quant.tflite'
.format(args.model, noise, mode, height, width))
# Full Integer Quantization - Input/Output=int8
converter = tf.lite.TFLiteConverter.from_keras_model(model)
converter.optimizations = [tf.lite.Optimize.DEFAULT]
converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]
converter.inference_input_type = tf.uint8
converter.inference_output_type = tf.uint8
converter.representative_dataset = representative_dataset_gen
tflite_quant_model = converter.convert()
with open(
'noise2noise_{}_{}_{}_{}x{}_full_integer_quant.tflite'.format(
args.model, noise, mode, height, width), 'wb') as w:
w.write(tflite_quant_model)
print(
'Integer Quantization complete! - noise2noise_{}_{}_{}_{}x{}_full_integer_quant.tflite'
.format(args.model, noise, mode, height, width))
# # EdgeTPU
# import subprocess
# result = subprocess.check_output(["edgetpu_compiler", "-s", "noise2noise_{}_{}_{}_{}x{}_full_integer_quant.tflite".format(args.model, noise, mode, height, width)])
# print(result)
sys.exit(0)
if args.output_dir:
output_dir = Path(args.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
image_paths = list(Path(image_dir).glob("*.*"))
for image_path in image_paths:
image = cv2.imread(str(image_path))
h, w, _ = image.shape
image = image[:(h // 16) * 16, :(w // 16) *
16] # for stride (maximum 16)
h, w, _ = image.shape
out_image = np.zeros((h, w * 3, 3), dtype=np.uint8)
noise_image = val_noise_model(image)
pred = model.predict(np.expand_dims(noise_image, 0))
denoised_image = get_image(pred[0])
out_image[:, :w] = image
out_image[:, w:w * 2] = noise_image
out_image[:, w * 2:] = denoised_image
if args.output_dir:
cv2.imwrite(
str(output_dir.joinpath(image_path.name))[:-4] + ".png",
out_image)
else:
cv2.imshow("result", out_image)
key = cv2.waitKey(-1)
# "q": quit
if key == 113:
return 0
from __future__ import print_function from keras.callbacks import LambdaCallback from model import get_model from data import data_load from config import Config from sample import print_sample (chars, char_indices, indices_char, maxlen, X, y, text) = data_load(Config) model = get_model(Config, maxlen, chars) model_name = Config['model_name'] print_sample(Config, text, maxlen, chars, char_indices, model, indices_char)
def main():
nb_epochs = config.MAXIMUM_EPOCHS
batch_size = config.batch_size
lr = 0.1
momentum = 0.9
model_name = 'ResNet50'
image_size = config.IMAGE_SIZE
output_dir = 'checkpoints'
experiment_name = 'data_augmentation'
early_stop_patience = config.EARLY_STOP_EPOCHS
train_path = os.path.join(image_directory, 'train')
validation_path = os.path.join(image_directory, 'validation')
test_path = os.path.join(image_directory, 'test')
PARAMS = {
'epoch_nr': nb_epochs,
'batch_size': batch_size,
'learning_rate': lr,
'momentum': momentum,
# 'input_shape': (512, 32, 3),
'early_stop': early_stop_patience,
'image_size': image_size,
'network': model_name
}
if log_experiment:
neptune.init(project_qualified_name='4ND4/sandbox')
neptune_tb.integrate_with_keras()
result = neptune.create_experiment(name=experiment_name, params=PARAMS)
name = result.id
print(name)
else:
name = 'debug'
train_gen, val_gen, test_gen = getdata(train_path, validation_path,
test_path)
model = get_model(model_name=model_name,
image_size=image_size,
number_classes=nb_classes)
sgd = SGD(lr=lr, momentum=momentum, nesterov=True)
model.compile(optimizer=sgd,
loss="categorical_crossentropy",
metrics=[age_mae])
model.summary()
output_dir = Path(__file__).resolve().parent.joinpath(output_dir)
if not output_dir.exists():
output_dir.mkdir(parents=True)
if not os.path.exists('checkpoints/{}'.format(name)):
os.mkdir('checkpoints/{}'.format(name))
callbacks = [
EarlyStopping(monitor='val_age_mae',
mode='min',
verbose=1,
patience=early_stop_patience),
LearningRateScheduler(schedule=Schedule(nb_epochs, initial_lr=lr)),
ModelCheckpoint(os.path.join(output_dir, name) +
"/weights.{epoch:03d}-{val_loss:.3f}-{"
"val_age_mae:.3f}.hdf5",
monitor="val_age_mae",
verbose=1,
save_best_only=True,
mode="min")
]
hist = model.fit_generator(generator=train_gen,
steps_per_epoch=train_gen.samples // batch_size,
validation_data=val_gen,
validation_steps=val_gen.samples // batch_size,
epochs=nb_epochs,
verbose=1,
callbacks=callbacks)
np.savez(str(output_dir.joinpath("history_{}.npz".format(name))),
history=hist.history)
def train():
"""
Training systole and diastole models.
"""
print('Loading and compiling models...')
model_systole = get_model()
model_diastole = get_model()
print('Loading training data...')
X, y = load_train_data()
print('Pre-processing images...')
X = preprocess(X)
# split to training and test
X_train, y_train, X_test, y_test = split_data(X, y, split_ratio=0.2)
nb_iter = 200
epochs_per_iter = 1
batch_size = 32
calc_crps = 1 # calculate CRPS every n-th iteration (set to 0 if CRPS estimation is not needed)
# remember min val. losses (best iterations), used as sigmas for submission
min_val_loss_systole = sys.float_info.max
min_val_loss_diastole = sys.float_info.max
print('-'*50)
print('Training...')
print('-'*50)
for i in range(nb_iter):
print('-'*50)
print('Iteration {0}/{1}'.format(i + 1, nb_iter))
print('-'*50)
print('Augmenting images - rotations')
X_train_aug = rotation_augmentation(X_train, 15)
print('Augmenting images - shifts')
X_train_aug = shift_augmentation(X_train_aug, 0.1, 0.1)
print('Fitting systole model...')
hist_systole = model_systole.fit(X_train_aug, y_train[:, 0], shuffle=True, nb_epoch=epochs_per_iter,
batch_size=batch_size, validation_data=(X_test, y_test[:, 0]))
print('Fitting diastole model...')
hist_diastole = model_diastole.fit(X_train_aug, y_train[:, 1], shuffle=True, nb_epoch=epochs_per_iter,
batch_size=batch_size, validation_data=(X_test, y_test[:, 1]))
# sigmas for predicted data, actually loss function values (RMSE)
loss_systole = hist_systole.history['loss'][-1]
loss_diastole = hist_diastole.history['loss'][-1]
val_loss_systole = hist_systole.history['val_loss'][-1]
val_loss_diastole = hist_diastole.history['val_loss'][-1]
if calc_crps > 0 and i % calc_crps == 0:
print('Evaluating CRPS...')
pred_systole = model_systole.predict(X_train, batch_size=batch_size, verbose=1)
pred_diastole = model_diastole.predict(X_train, batch_size=batch_size, verbose=1)
val_pred_systole = model_systole.predict(X_test, batch_size=batch_size, verbose=1)
val_pred_diastole = model_diastole.predict(X_test, batch_size=batch_size, verbose=1)
# CDF for train and test data (actually a step function)
cdf_train = real_to_cdf(np.concatenate((y_train[:, 0], y_train[:, 1])))
cdf_test = real_to_cdf(np.concatenate((y_test[:, 0], y_test[:, 1])))
# CDF for predicted data
cdf_pred_systole = real_to_cdf(pred_systole, loss_systole)
cdf_pred_diastole = real_to_cdf(pred_diastole, loss_diastole)
cdf_val_pred_systole = real_to_cdf(val_pred_systole, val_loss_systole)
cdf_val_pred_diastole = real_to_cdf(val_pred_diastole, val_loss_diastole)
# evaluate CRPS on training data
crps_train = crps(cdf_train, np.concatenate((cdf_pred_systole, cdf_pred_diastole)))
print('CRPS(train) = {0}'.format(crps_train))
# evaluate CRPS on test data
crps_test = crps(cdf_test, np.concatenate((cdf_val_pred_systole, cdf_val_pred_diastole)))
print('CRPS(test) = {0}'.format(crps_test))
print('Saving weights...')
# save weights so they can be loaded later
model_systole.save_weights('weights_systole.hdf5', overwrite=True)
model_diastole.save_weights('weights_diastole.hdf5', overwrite=True)
# for best (lowest) val losses, save weights
if val_loss_systole < min_val_loss_systole:
min_val_loss_systole = val_loss_systole
model_systole.save_weights('weights_systole_best.hdf5', overwrite=True)
if val_loss_diastole < min_val_loss_diastole:
min_val_loss_diastole = val_loss_diastole
model_diastole.save_weights('weights_diastole_best.hdf5', overwrite=True)
# save best (lowest) val losses in file (to be later used for generating submission)
with open('val_loss.txt', mode='w+') as f:
f.write(str(min_val_loss_systole))
f.write('\n')
f.write(str(min_val_loss_diastole))
def train():
"""
训练模型
"""
model = get_model()
model.train()
def parser(sentence):
"""
依存句法分析
"""
model = get_model()
return model.predict(sentence)
def train():
"""
Training systole and diastole models.
"""
print('Loading and compiling models...')
model_systole = get_model(img_size)
model_diastole = get_model(img_size)
print('Loading training data...')
X, y = load_train_data()
print('Pre-processing images...')
X = preprocess(X)
# split to training and test
X_train, y_train, X_test, y_test = split_data(X, y, split_ratio=0.2)
# define image generator for random rotations
datagen = ImageDataGenerator(featurewise_center=False,
featurewise_std_normalization=False,
rotation_range=15)
nb_iter = 300
epochs_per_iter = 1
batch_size = 64
calc_crps = 1 # calculate CRPS every n-th iteration (set to 0 if CRPS estimation is not needed)
# remember min val. losses (best iterations), used as sigmas for submission
min_val_loss_systole = sys.float_info.max
min_val_loss_diastole = sys.float_info.max
if not os.path.exists(STATS):
os.makedirs(STATS)
with open(STATS + 'RMSE_CRPS.txt', 'w') as f:
names = ['train_RMSE_d', 'train_RMSE_s', 'test_RMSE_d', 'test_RMSE_s', 'train_crps', 'test_crps']
f.write('\t'.join([str(name) for name in names]) + '\n')
print('-'*50)
print('Training...')
print('-'*50)
for i in range(nb_iter):
print('-'*50)
print('Iteration {0}/{1}'.format(i + 1, nb_iter))
print('-'*50)
print('Augmenting images - rotations')
X_train_aug = rotation_augmentation(X_train, 15)
print('Augmenting images - shifts')
X_train_aug = shift_augmentation(X_train_aug, 0.1, 0.1)
print('Fitting systole model...')
hist_systole = model_systole.fit(X_train_aug, y_train[:, 0], shuffle=True, nb_epoch=epochs_per_iter, batch_size=batch_size, validation_data=(X_test, y_test[:, 0]))
print('Fitting diastole model...')
hist_diastole = model_diastole.fit(X_train_aug, y_train[:, 1], shuffle=True, nb_epoch=epochs_per_iter, batch_size=batch_size, validation_data=(X_test, y_test[:, 1]))
# sigmas for predicted data, actually loss function values (RMSE)
loss_systole = hist_systole.history['loss'][-1]
loss_diastole = hist_diastole.history['loss'][-1]
val_loss_systole = hist_systole.history['val_loss'][-1]
val_loss_diastole = hist_diastole.history['val_loss'][-1]
if calc_crps > 0 and i % calc_crps == 0:
print('Evaluating CRPS...')
pred_systole = model_systole.predict(X_train, batch_size=batch_size, verbose=1)
pred_diastole = model_diastole.predict(X_train, batch_size=batch_size, verbose=1)
val_pred_systole = model_systole.predict(X_test, batch_size=batch_size, verbose=1)
val_pred_diastole = model_diastole.predict(X_test, batch_size=batch_size, verbose=1)
# CDF for train and test data (actually a step function)
cdf_train = real_to_cdf(np.concatenate((y_train[:, 0], y_train[:, 1])))
cdf_test = real_to_cdf(np.concatenate((y_test[:, 0], y_test[:, 1])))
# CDF for predicted data
cdf_pred_systole = real_to_cdf(pred_systole, loss_systole)
cdf_pred_diastole = real_to_cdf(pred_diastole, loss_diastole)
cdf_val_pred_systole = real_to_cdf(val_pred_systole, val_loss_systole)
cdf_val_pred_diastole = real_to_cdf(val_pred_diastole, val_loss_diastole)
# evaluate CRPS on training data
crps_train = crps(cdf_train, np.concatenate((cdf_pred_systole, cdf_pred_diastole)))
print('CRPS(train) = {0}'.format(crps_train))
# evaluate CRPS on test data
crps_test = crps(cdf_test, np.concatenate((cdf_val_pred_systole, cdf_val_pred_diastole)))
print('CRPS(test) = {0}'.format(crps_test))
print('Saving weights...')
# save weights so they can be loaded later
model_systole.save_weights(MODELS + 'weights_systole.hdf5', overwrite=True)
model_diastole.save_weights(MODELS + 'weights_diastole.hdf5', overwrite=True)
# for best (lowest) val losses, save weights
if val_loss_systole < min_val_loss_systole:
min_val_loss_systole = val_loss_systole
model_systole.save_weights(MODELS + 'weights_systole_best.hdf5', overwrite=True)
if val_loss_diastole < min_val_loss_diastole:
min_val_loss_diastole = val_loss_diastole
model_diastole.save_weights(MODELS + 'weights_diastole_best.hdf5', overwrite=True)
# save best (lowest) val losses in file (to be later used for generating submission)
with open(MODELS + 'val_loss.txt', mode='w+') as f:
f.write(str(min_val_loss_systole))
f.write('\n')
f.write(str(min_val_loss_diastole))
with open(STATS + 'RMSE_CRPS.txt', 'a') as f:
# train_RMSE_d train_RMSE_s test_RMSE_d test_RMSE_s train_crps test_crps
rmse_values = [loss_diastole, loss_systole, val_loss_diastole, val_loss_systole]
crps_values = [crps_train, crps_test]
f.write('\t'.join([str(val) for val in rmse_values + crps_values]) + '\n')
print('Saving stats images...')
write_images(STATS)
if (i != 0) & ((i + 1) % 100 == 0):
print('Submitting learned model....')
SUBMISSION_FOLDER = SUBMISSION + preproc_type + "/" + model_name + "/" + get_name() + "_ITERS" + str(i + 1) + "/"
if not os.path.exists(SUBMISSION_FOLDER):
os.makedirs(SUBMISSION_FOLDER)
copyfile(MODELS + 'weights_systole_best.hdf5', SUBMISSION_FOLDER + 'weights_systole_best.hdf5')
copyfile(MODELS + 'weights_diastole_best.hdf5', SUBMISSION_FOLDER + 'weights_diastole_best.hdf5')
copyfile(MODELS + 'val_loss.txt', SUBMISSION_FOLDER + 'val_loss.txt')
os.system('python submission.py %s %s %s' % (preproc_type, model_name, SUBMISSION_FOLDER))
