def main():
parser = argparse.ArgumentParser()
parser.add_argument('--patch_size', type=int, default=128)
parser.add_argument('--keep_range', action='store_true')
parser.add_argument('--batch_size', type=int, default=16)
parser.add_argument('--depth', type=int, default=4)
parser.add_argument('--epochs', type=int, default=20)
parser.add_argument('--lr', type=float, default=1e-4)
parser.add_argument('--lr_gamma', type=float, default=0.5)
parser.add_argument('--milestones', nargs='+', default=[10, 15])
parser.add_argument('--exp_name', type=str, default='baseline')
parser.add_argument('--save_as', type=str, default='models/deblur.hdf5')
cfg = parser.parse_args()
# For checking the GPU usage
#tf.debugging.set_log_device_placement(True)
# For limiting the GPU usage
gpus = tf.config.experimental.list_physical_devices('GPU')
if gpus:
try:
tf.config.experimental.set_memory_growth(gpus[0], True)
except RuntimeError as e:
print(e)
dataset_train = data.REDS(
cfg.batch_size,
patch_size=cfg.patch_size,
train=True,
keep_range=cfg.keep_range,
)
# We have 3,000 validation frames.
# Note that each frame will be center-cropped for the validation.
dataset_val = data.REDS(
20,
patch_size=cfg.patch_size,
train=False,
keep_range=cfg.keep_range,
)
if cfg.depth == 4:
net = model.Baseline(cfg.patch_size, cfg.patch_size)
else:
net_class = getattr(model, 'Small{}'.format(cfg.depth))
net = net_class(cfg.patch_size, cfg.patch_size)
net.build(input_shape=(None, cfg.patch_size, cfg.patch_size, 3))
kwargs = {'optimizer': 'adam', 'loss': 'mse'}
if cfg.keep_range:
net.compile(**kwargs, metrics=[metric.psnr_full])
else:
net.compile(**kwargs, metrics=[metric.psnr])
net.summary()
# Callback functions
# For TensorBoard logging
logging = callbacks.TensorBoard(
log_dir=path.join('logs', cfg.exp_name),
update_freq=100,
)
# For checkpointing
os.makedirs(path.dirname(cfg.save_as), exist_ok=True)
checkpointing = callbacks.ModelCheckpoint(
cfg.save_as,
verbose=1,
save_weights_only=True,
)
def scheduler(epoch):
idx = bisect.bisect_right(cfg.milestones, epoch)
lr = cfg.lr * (cfg.lr_gamma**idx)
return lr
# For learning rate scheduling
scheduling = callbacks.LearningRateScheduler(scheduler, verbose=1)
net.fit_generator(
dataset_train,
epochs=cfg.epochs,
callbacks=[logging, checkpointing, scheduling],
validation_data=dataset_val,
validation_freq=1,
max_queue_size=16,
workers=8,
use_multiprocessing=True,
)
# Date time string.
datetime_string = utils.get_datetime_string() + "_{}-{}".format(
len(qrcodes_train), len(qrcodes_validate)) + "_".join(
dataset_parameters["output_targets"])
# Output path. Ensure its existence.
output_path = os.path.join("/whhdata/models", datetime_string)
if os.path.exists(output_path) == False:
os.makedirs(output_path)
print("Using output path:", output_path)
# Important things.
pp = pprint.PrettyPrinter(indent=4)
log_dir = os.path.join("/whhdata/models", "logs", datetime_string)
tensorboard_callback = callbacks.TensorBoard(log_dir=log_dir)
histories = {}
# Training network.
def train_rgbmaps():
sequence_length = dataset_parameters["sequence_length"]
model = models.Sequential()
if sequence_length == 0:
model.add(
layers.Conv2D(64, (3, 3),
activation="relu",
input_shape=(image_size, image_size, 3)))
else:
model.add(
optimizer=optimizers.RMSprop(lr=2e-5),
metrics=['acc'])
#สร้าง folder TensorBoard
root_logdir = '/media/tohn/SSD/FP_All_Nor_Abnor_B3NA_25/my_logs'
def get_run_logdir():
import time
run_id = time.strftime("run_%Y_%m_%d_%H_%M_%S")
return os.path.join(root_logdir, run_id)
run_logdir = get_run_logdir()
tensorboard_cb = callbacks.TensorBoard(run_logdir)
#ส้รางไฟล์เก็บโมเดล
os.makedirs("./models", exist_ok=True)
history = model2.fit_generator(train_generator,
steps_per_epoch=NUM_TRAIN // batch_size,
epochs=epochs,
validation_data=validation_generator,
validation_steps=NUM_TEST // batch_size,
verbose=1,
use_multiprocessing=True,
workers=1,
callbacks=[
tensorboard_cb,
callbacks.ModelCheckpoint(
def _build_callback_hooks(self,
models_dir: str,
logs_dir: str,
is_training=True,
logging_frequency=25):
"""
Build callback hooks for the training loop
Returns:
callbacks_list: list of callbacks
"""
callbacks_list: list = list()
if is_training:
# Model checkpoint
if models_dir:
checkpoints_path = os.path.join(models_dir, CHECKPOINT_FNAME)
cp_callback = callbacks.ModelCheckpoint(
filepath=checkpoints_path,
save_weights_only=False,
verbose=1,
save_best_only=True,
mode="max",
monitor="val_new_MRR",
)
callbacks_list.append(cp_callback)
# Early Stopping
early_stopping_callback = callbacks.EarlyStopping(
monitor="val_new_MRR",
mode="max",
patience=2,
verbose=1,
restore_best_weights=True)
callbacks_list.append(early_stopping_callback)
# TensorBoard
if logs_dir:
tensorboard_callback = callbacks.TensorBoard(log_dir=logs_dir,
histogram_freq=1,
update_freq=5)
callbacks_list.append(tensorboard_callback)
# Debugging/Logging
logger = self.logger
class DebuggingCallback(callbacks.Callback):
def __init__(self, patience=0):
super(DebuggingCallback, self).__init__()
self.epoch = 0
def on_train_batch_end(self, batch, logs=None):
if batch % logging_frequency == 0:
logger.info("[epoch: {} | batch: {}] {}".format(
self.epoch, batch, logs))
def on_epoch_end(self, epoch, logs=None):
logger.info("End of Epoch {}".format(self.epoch))
logger.info(logs)
def on_epoch_begin(self, epoch, logs=None):
self.epoch = epoch + 1
logger.info("Starting Epoch : {}".format(self.epoch))
logger.info(logs)
def on_train_begin(self, logs):
logger.info("Training Model")
def on_test_begin(self, logs):
logger.info("Evaluating Model")
def on_predict_begin(self, logs):
logger.info("Predicting scores using model")
def on_train_end(self, logs):
logger.info("Completed training model")
logger.info(logs)
def on_test_end(self, logs):
logger.info("Completed evaluating model")
logger.info(logs)
def on_predict_end(self, logs):
logger.info("Completed Predicting scores using model")
logger.info(logs)
callbacks_list.append(DebuggingCallback())
# Add more here
return callbacks_list
def train_model(training_data,
output_model,
loss_function='categorical_crossentropy',
validation_data=None,
fine_tune=None,
tensorboard=False):
"""
Train message segmentation model.
:param training_data: JSON file with training data
:param output_model: path prefix for model checkpoints
:param loss_function: optimization loss function
:param validation_data: JSON file with validation data
:param fine_tune: fine-tune model from given file instead of training from scratch
:param tensorboard: Tensorboard log data directory
"""
tb_callback = callbacks.TensorBoard(log_dir='./data/graph/' +
str(datetime.now()),
update_freq='batch',
write_graph=False,
write_images=False)
es_callback = callbacks.EarlyStopping(monitor='val_loss',
verbose=1,
patience=5)
cp_callback = callbacks.ModelCheckpoint(
output_model + '.epoch-{epoch:02d}.val_loss-{val_loss:.3f}.h5')
cp_callback_no_val = callbacks.ModelCheckpoint(
output_model + '.epoch-{epoch:02d}.loss-{loss:.3f}.h5')
def get_line_model():
line_input = layers.Input(shape=(LINE_LEN, INPUT_DIM))
masking = layers.Masking(0)(line_input)
bi_seq = layers.Bidirectional(layers.GRU(128),
merge_mode='sum')(masking)
bi_seq = layers.BatchNormalization()(bi_seq)
bi_seq = layers.Activation('relu')(bi_seq)
return line_input, bi_seq
def get_context_model():
context_input = layers.Input(shape=CONTEXT_SHAPE)
conv2d = layers.Conv2D(128, (4, 4))(context_input)
conv2d = layers.BatchNormalization()(conv2d)
conv2d = layers.Activation('relu')(conv2d)
conv2d = layers.Conv2D(128, (3, 3))(conv2d)
conv2d = layers.Activation('relu')(conv2d)
conv2d = layers.MaxPooling2D(2)(conv2d)
flatten = layers.Flatten()(conv2d)
dense = layers.Dense(128)(flatten)
dense = layers.Activation('relu')(dense)
return context_input, dense
def get_base_model():
line_input_cur, line_model_cur = get_line_model()
line_input_prev, line_model_prev = get_line_model()
context_input, context_model = get_context_model()
concat = layers.concatenate(
[line_model_cur, line_model_prev, context_model])
dropout = layers.Dropout(0.25)(concat)
dense = layers.Dense(len(MailLinesSequence.LABEL_MAP))(dropout)
output = layers.Activation('softmax')(dense)
return models.Model(
inputs=[line_input_cur, line_input_prev, context_input],
outputs=output)
if fine_tune is None:
segmenter = get_base_model()
else:
segmenter = models.load_model(fine_tune)
logger.info('Freezing layers')
for layer in segmenter.layers[:-2]:
layer.trainable = False
compile_args = {
'optimizer': 'adam',
'loss': loss_function,
'metrics': ['categorical_accuracy']
}
effective_callbacks = [
es_callback, cp_callback
] if validation_data is not None else [cp_callback_no_val]
if tensorboard:
effective_callbacks.append(tb_callback)
segmenter.compile(**compile_args)
segmenter.summary()
train_seq = MailLinesSequence(training_data,
CONTEXT_SHAPE,
labeled=True,
batch_size=TRAIN_BATCH_SIZE)
val_seq = MailLinesSequence(
validation_data,
CONTEXT_SHAPE,
labeled=True,
batch_size=INF_BATCH_SIZE) if validation_data else None
epochs = 20 if fine_tune is None else 10
segmenter.fit_generator(train_seq,
epochs=epochs,
validation_data=val_seq,
shuffle=True,
use_multiprocessing=False,
workers=train_seq.num_workers,
max_queue_size=train_seq.max_queue_size,
callbacks=effective_callbacks)
return tf.matmul(inputs, self.kernel)
def compute_output_shape(self, input_shape):
shape = tf.TensorShape(input_shape).as_list()
shape[-1] = self.output_dim
return tf.TensorShape(shape)
def get_config(self):
base_config = super(MyLayer, self).get_config()
base_config['output_dim'] = self.output_dim
return base_config
@classmethod
def from_config(cls, config):
return cls(**config)
callbacks = [
# Interrupt training if `val_loss` stops improving for over 2 epochs
callbacks.EarlyStopping(patience=2, monitor='val_loss'),
# Write TensorBoard logs to `./logs` directory
callbacks.TensorBoard(log_dir='./logs')
]
# model.fit(data, labels, batch_size=32, epochs=5, callbacks=callbacks,
# validation_data=(val_data, val_labels))
# show_network(model)
if __name__ == '__main__':
pass
def get_callbacks(CONF, use_lr_decay=True):
"""
Get a callback list to feed fit_generator.
#TODO Use_remote callback needs proper configuration
#TODO Add ReduceLROnPlateau callback?
Parameters
----------
CONF: dict
Returns
-------
List of callbacks
"""
calls = []
# Add mandatory callbacks
calls.append(callbacks.TerminateOnNaN())
calls.append(LRHistory())
# Add optional callbacks
if use_lr_decay:
milestones = np.array(
CONF['training']['lr_step_schedule']) * CONF['training']['epochs']
milestones = milestones.astype(np.int)
calls.append(
LR_scheduler(lr_decay=CONF['training']['lr_step_decay'],
epoch_milestones=milestones.tolist()))
if CONF['monitor']['use_tensorboard']:
calls.append(
callbacks.TensorBoard(log_dir=paths.get_logs_dir(),
write_graph=False))
# # Let the user launch Tensorboard
# print('Monitor your training in Tensorboard by executing the following comand on your console:')
# print(' tensorboard --logdir={}'.format(paths.get_logs_dir()))
# Run Tensorboard on a separate Thread/Process on behalf of the user
port = os.getenv('monitorPORT', 6006)
port = int(port) if len(str(port)) >= 4 else 6006
subprocess.run(['fuser', '-k', '{}/tcp'.format(port)
]) # kill any previous process in that port
p = Process(target=launch_tensorboard, args=(port, ), daemon=True)
p.start()
if CONF['monitor']['use_remote']:
calls.append(callbacks.RemoteMonitor())
if CONF['training']['use_validation'] and CONF['training'][
'use_early_stopping']:
calls.append(
callbacks.EarlyStopping(patience=int(0.1 *
CONF['training']['epochs'])))
if CONF['training']['ckpt_freq'] is not None:
calls.append(
callbacks.ModelCheckpoint(os.path.join(paths.get_checkpoints_dir(),
'epoch-{epoch:02d}.hdf5'),
verbose=1,
period=max(
1,
int(CONF['training']['ckpt_freq'] *
CONF['training']['epochs']))))
if not calls:
calls = None
return calls
try:
learning_rates = np.logspace(*learning_rate, opts['epochs'])
except ValueError:
raise ValueError(
'`learning_rate` has to either a single float or a tuple of two floats [rate_beginning, rate_end]'
)
def lr_callback_func(epoch):
learning_rate = learning_rates[epoch - initial_epoch]
tf.summary.scalar('learning rate', data=learning_rate, step=epoch)
return learning_rate
lr_callback = callbacks.LearningRateScheduler(lr_callback_func)
checkpoint_path = output_dir / 'logs' / 'models' / 'cp-{epoch:04d}.ckpt'
model_checkpoint_callback = callbacks.ModelCheckpoint(str(checkpoint_path))
writer = tf.summary.create_file_writer(logdir=str(output_dir / 'logs' /
'test'))
tensorboard_callback = callbacks.TensorBoard(log_dir=output_dir / 'logs')
images = np.array(*test_dataset_batched.take(1).as_numpy_iterator())[0]
model.fit(train_dataset_batched,
initial_epoch=initial_epoch,
epochs=initial_epoch + opts.get('epochs', 10),
validation_data=test_dataset_batched,
callbacks=[
lr_callback, tensorboard_callback,
TestImagesTensorboard(images, writer),
model_checkpoint_callback
])
model.add(layers.MaxPooling2D(pool_size=(2, 2)))
for i in range(conv_layer - 1):
model.add(layers.Conv2D(layer_size, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D(pool_size=(2, 2)))
model.add(layers.Flatten())
model.add(layers.Dropout(dropout_set))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer=optimizers.Adam(learning_rate=0.1),
metrics=['acc'])
NAME = "{}-conv-{}-nodes-{}-dropout-{}".format(
conv_layer, layer_size, dropout_set,
datetime.now().strftime("%H%M%S"))
tensorboard = callbacks.TensorBoard(log_dir='catdog\\logs\\%s' %
NAME)
model.fit(x=X,
y=Y,
batch_size=32,
epochs=12,
validation_split=0.3,
callbacks=[tensorboard])
# %%
model.save('catdog\\models\\catdog-64x4')
# %%
def train(self,
download_dir: str = '/tmp/seq2seq/downloads/',
cache_dir: Optional[str] = '/tmp/seq2seq/cache/',
log_dir: Optional[str] = '/tmp/seq2seq/logs/',
save_dir: Optional[str] = '/tmp/seq2seq/models',
epochs: int = 1,
optimizer: str = 'adam',
glove_n_components: int = 10,
gaussian_noise: float = 0,
capacity: int = 1,
val_split: float = 0.1,
test_split: float = 0.2,
l2: float = 0.01,
progress: Optional[Callable] = None):
self.settings['seq2seq'] = { # type: ignore
'epochs': epochs,
'optimizer': optimizer,
'n_components': glove_n_components,
'gaussian_noise': gaussian_noise,
'capacity': capacity,
'val_split': val_split,
'test_split': test_split,
'l2': l2,
}
embedding_input = layers.Input((None, glove_n_components),
name='tokens',
dtype=tf.float32)
tfidf_input = layers.Input((None, 1), name='tfidf', dtype=tf.float32)
keyphraseness_input = layers.Input((None, 1),
name='keyphraseness',
dtype=tf.float32)
pos_input = layers.Input((None, 35), name='pos', dtype=tf.float32)
net_inputs = (embedding_input, tfidf_input, keyphraseness_input,
pos_input)
if 0 < gaussian_noise < 1:
embedding_input = layers.GaussianNoise(gaussian_noise)(
embedding_input)
net = layers.Concatenate()(
[embedding_input, tfidf_input, keyphraseness_input, pos_input])
net = layers.Bidirectional(
layers.GRU(16 * capacity, return_sequences=True))(net)
net = layers.Bidirectional(
layers.GRU(16 * capacity, return_sequences=True))(net)
net = layers.Bidirectional(
layers.GRU(16 * capacity, return_sequences=True))(net)
net = layers.Dense(32 * capacity,
activation='relu',
kernel_regularizer=regularizers.l2(l2))(net)
net = layers.BatchNormalization()(net)
net = layers.Dense(32 * capacity,
activation='relu',
kernel_regularizer=regularizers.l2(l2))(net)
net = layers.BatchNormalization()(net)
prediction_layer = layers.Dense(1, activation='sigmoid')(net)
self.model = models.Model(inputs=net_inputs, outputs=prediction_layer)
self.model.compile(optimizer=optimizer,
loss='binary_crossentropy',
metrics=['acc'],
sample_weight_mode='temporal')
self.model.summary()
(x_train,
y_train), validation_data, (x_test, y_test) = self._create_data(
download_dir=download_dir,
glove_n_components=glove_n_components,
val_split=val_split,
test_split=test_split,
cache_dir=cache_dir or '.',
progress=progress)
class_count_counter = Counter(int(a) for b in y_train for a in b)
print("Training set class counts", class_count_counter)
num_classes = max(class_count_counter.keys()) + 1
print('Num classes', num_classes)
class_counts = np.zeros(num_classes, dtype=int)
for cl, count in class_count_counter.items():
class_counts[cl] = count
self.class_weights = -np.log(class_counts / np.sum(class_counts))
print("Training class weights", self.class_weights)
sample_weights = np.array([[self.class_weights[y2] for y2 in y1]
for y1 in y_train.squeeze()],
dtype=float)
print('Training sample weights:', sample_weights.shape)
cb = []
if log_dir:
tb = callbacks.TensorBoard(log_dir,
write_graph=True,
histogram_freq=5)
cb.append(tb)
try:
self.fit_result = self.model.fit(epochs=epochs,
x=x_train,
y=y_train,
sample_weight=sample_weights,
validation_data=validation_data,
callbacks=cb)
except KeyboardInterrupt:
print('Model fit() interrupted by user input.')
test_sample_weights = np.array([[self.class_weights[y2] for y2 in y1]
for y1 in y_test.squeeze()],
dtype=float)
print('Evaluation of test samples')
self.evaluation = self.model.evaluate(
x_test, y_test, sample_weight=test_sample_weights)
if save_dir:
os.makedirs(save_dir, exist_ok=True)
now = datetime.datetime.now()
filename = now.strftime('%y-%m-%d.%H:%M:%S.model')
filepath = os.path.join(save_dir, filename)
print('Saving model to', filepath)
self.save(filepath)
def train_deepdrug(batch_size,
lr,
epoch,
input,
log_dir,
k_fold=10,
augmentation=False):
# optimize gpu memory usage
physical_devices = tf.config.experimental.list_physical_devices('GPU')
assert len(physical_devices) > 0
tf.config.experimental.set_memory_growth(physical_devices[0], True)
# load the data
with open(input, "rb") as f:
grids_dict = pk.load(f)
grids = list(grids_dict.values())
def get_label(key):
if "active" in key: return 1
else: return 0
labels = list(map(get_label, grids_dict.keys()))
# shuffle input
idx_list = list(range(len(labels)))
rd.shuffle(idx_list)
grids = list(np.array(grids)[idx_list])
labels = list(np.array(labels)[idx_list])
# print("grids type:", type(grids))
# print("grid type:", type(grids[0]), "shape:", grids[0].shape)
# print("labels type:", type(labels))
# print("label type:", type(labels[0]), "shape:", labels[0].shape)
histories = list()
timestamp = time.strftime("%Y%m%d_%H%M%S", time.localtime())
for k in range(k_fold):
tf.keras.backend.clear_session()
# get training data with respect to the kth fold
x, y, val_x, val_y = split_dataset_with_kfold(grids, labels, k, k_fold)
# to balance different classes
sample_weights = compute_sample_weight('balanced', y)
val_data = (val_x, val_y)
# build & compile model
mdl = DeepDrug3DBuilder.build(classes=1)
adam = Adam(
lr=lr, beta_1=0.9, beta_2=0.999, epsilon=None,
decay=0.0, amsgrad=False)
loss = "binary_crossentropy"
metric = "binary_accuracy"
auc_metric = tf.keras.metrics.AUC()
precision_metric = tf.keras.metrics.Precision()
recall_metric = tf.keras.metrics.Recall()
mdl.compile(
optimizer=adam, loss=loss,
metrics=[metric, auc_metric, precision_metric, recall_metric])
# callback function for model checking
log_d = os.path.join(log_dir, timestamp, "fold_{}".format(k))
os.makedirs(log_d, exist_ok=True)
tfCallBack = callbacks.TensorBoard(log_dir=log_d)
history = mdl.fit(
x, y, epochs = epoch, batch_size = batch_size,
sample_weight=sample_weights, validation_data=val_data,
shuffle = True, callbacks = [tfCallBack])
histories.append(history)
val_accs = list()
val_aucs = list()
val_precisions = list()
val_recalls = list()
for his in histories:
try:
val_accs.append(his.history["val_binary_accuracy"])
except KeyError:
val_accs.append(his.history["val_categorical_accuracy"])
val_aucs.append(his.history["val_auc"])
val_precisions.append(his.history["val_precision"])
val_recalls.append(his.history["val_recall"])
# find best epoch
val_accs = np.array(val_accs)
avgs = np.mean(val_accs, axis=0)
best_epoch = np.argmax(avgs)
# get the accuracy and standard deviation of the best epoch
max_accs = val_accs[:, best_epoch]
acc_avg = np.mean(max_accs)
acc_std = np.std(max_accs)
# get the auc score of the best epoch
max_aucs = np.array(val_aucs)[:, best_epoch]
auc_avg = np.mean(max_aucs)
auc_std = np.std(max_accs)
# get the precision, racall, f1 score of the best epoch
max_precisions = np.array(val_precisions)[:, best_epoch]
precision_avg = np.mean(max_precisions)
precision_std = np.std(max_precisions)
max_recalls = np.array(val_recalls)[:, best_epoch]
recall_avg = np.mean(max_recalls)
recall_std = np.std(max_recalls)
max_f1s = 2 * max_precisions * max_recalls / (max_precisions + max_recalls)
f1_avg = np.mean(max_f1s)
f1_std = np.std(max_f1s)
# print and save the training results
print(
"{}-fold cross validation performs the best "
"at epoch {}".format(k_fold, best_epoch))
print("Accuracy is {} +- {}".format(acc_avg, acc_std))
print("AUC ROC is {} +- {}".format(auc_avg, auc_std))
print("Precision is {} +- {}".format(precision_avg, precision_std))
print("Recall is {} +- {}".format(recall_avg, recall_std))
print("F1 score is {} +- {}".format(f1_avg, f1_std))
print()
with open(os.path.join(log_dir, timestamp, "readme"), "w") as f:
print("dataset: {}".format(os.path.basename(input)), file=f)
print("batch size: {}".format(batch_size), file=f)
print("learning rate: {}".format(lr), file=f)
print("epochs: {}".format(epoch), file=f)
print("validation folds: {}".format(k_fold), file=f)
print(
"{}-fold cross validation performs the best "
"at epoch {}".format(k_fold, best_epoch),
file=f)
print("Accuracy is {} +- {}".format(acc_avg, acc_std), file=f)
print("AUC ROC is {} +- {}".format(auc_avg, auc_std), file=f)
print("Precision is {} +- {}".format(
precision_avg, precision_std), file=f)
print("Recall is {} +- {}".format(recall_avg, recall_std), file=f)
print("F1 score is {} +- {}".format(f1_avg, f1_std), file=f)
def _internal_fit(self,
x_train,
y_train,
validation_x=None,
validation_y=None):
"""Fit internal model with given data."""
# As a new model will be trained, remove old stateful model, if any
self._inference_model = None
self._inference_batch_size = None
# x_train data shape is (samples, steps, lags, series)
if self._options.sequential_mini_step > 0:
# In this case, we always perform one step prediction, so ignore
# other steps of y_train for training, and use them only for eval.
# Note we duplicate y_train in memory and don't overwrite it
y_train = y_train[:, :self._options.sequential_mini_step, :]
if validation_y is not None:
validation_y = validation_y[:, :self._options.
sequential_mini_step, :]
if self._options.nn_use_variable_sigma:
# Note we add a dummy output that is ignored by metrics. It is
# because metrics need same input size in prediction and y_train.
# Sigma predictions are ignored (except for the loss).
# This duplicates y_train in memory, but it does not overwrite it
y_train = np.stack([y_train, np.zeros(y_train.shape)], axis=1)
if validation_y is not None:
validation_y = np.stack(
[validation_y, np.zeros(validation_y.shape)], axis=1)
metrics = [
util.sigma_mean_squared_error, util.sigma_mean_absolute_error,
util.sigma_mean_absolute_percentage_error
]
else:
metrics = [
losses.mean_squared_error, losses.mean_absolute_error,
losses.mean_absolute_percentage_error
]
# We create model here
input_layer = layers.Input(shape=x_train.shape[1:])
last_layer = self._create_net(input_layer)
self._model = models.Model(inputs=input_layer, outputs=last_layer)
optimizer = getattr(
optimizers,
self._options.nn_optimizer)(lr=self._options.nn_learning_rate)
self._model.compile(loss=self._loss(),
optimizer=optimizer,
metrics=metrics)
logging.info(self._model.summary())
validation_data = None
calls = None
if validation_x is not None:
validation_data = (validation_x, validation_y)
if self._options.nn_patience >= 0:
if self._options.flow_use_temperature:
calls = [
CustomStopper(
monitor='val_loss',
patience=self._options.nn_patience,
restore_best_weights=True,
start_epoch=self._options.flow_temperature_steps)
]
else:
calls = [
callbacks.EarlyStopping(
monitor='val_loss',
patience=self._options.nn_patience,
restore_best_weights=True)
]
additional_calls = self._callbacks()
if self._options.nn_tensorboard:
tb_call = callbacks.TensorBoard(log_dir="./logs")
if additional_calls is None:
additional_calls = [tb_call]
else:
additional_calls.append(tb_call)
if calls is None:
calls = additional_calls
elif additional_calls is not None:
calls += additional_calls
self._model.fit(x=x_train,
y=y_train,
validation_data=validation_data,
epochs=self._options.nn_epochs,
callbacks=calls,
batch_size=self._options.nn_batch_size)
# Store real number of epochs where it stopped
self._stopped_epoch = None
if self._options.nn_patience and calls:
for one_call in calls:
if hasattr(one_call, 'stopped_epoch'):
self._stopped_epoch = (one_call.stopped_epoch -
self._options.nn_patience)
if self._stopped_epoch is None:
self._stopped_epoch = self._options.nn_epochs
if not self._options.lstm_stateful:
# If not stateful prediction, then use same model for inference
self._inference_model = self._model
self._inference_batch_size = self._options.nn_batch_size
train_generator = DataGenerator(img_root=imgs_path, list_IDs=train_img_names, labels=train_labels,
batch_size=batch_size, label_max_length=max_labels)
test_generator = DataGenerator(img_root=imgs_path, list_IDs=test_img_names, labels=test_labels,
batch_size=batch_size, label_max_length=max_labels)
# In[ ]:
checkpoint = callbacks.ModelCheckpoint("./models/efb7-val_loss.{val_loss:.2f}.h5", monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=False, mode='auto', period=1)
# checkpoint = callbacks.ModelCheckpoint("./models/loss.{loss:.2f}-acc.{acc:.2f}-val_loss.{val_loss:.2f}-val_acc.{val_acc:.2f}.h5", monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=False, mode='auto', period=1)
# early = EarlyStopping(monitor='val_acc', min_delta=0, patience=3, verbose=1, mode='auto')
# reduce_lr = callbacks.ReduceLROnPlateau(monitor='acc', factor=0.1, patience=2, min_lr=0.000001)
learningRateScheduler = callbacks.LearningRateScheduler(lr_decay)
tensorboard = callbacks.TensorBoard(log_dir='./logs_b7')
# In[ ]:
history = model.fit_generator(generator=train_generator,
steps_per_epoch=ceil(len(train_labels) / batch_size),
validation_data=test_generator,
validation_steps=ceil(len(test_labels) / batch_size),
epochs=nb_epochs,
callbacks = [checkpoint, tensorboard, learningRateScheduler],
use_multiprocessing=True,
workers=6, verbose=1)
X_test = np.load("../audio_data/audio_test.npy")
Y_train = pd.read_csv("../audio_data/labels_train.csv")
# Simple preprocessing
X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], 1)
X_test = X_test.reshape(X_test.shape[0], X_test.shape[1], 1)
Y_train = np.array(Y_train)[:, -1] # set(Y_train) = {0, 1, 3, 4, 5, 6, 7, 8, 9}
# Y_train = utils.to_categorical(Y_train) for "categorical_crossentropy" loss.
print("The shape of X_train\X_test\Y_train: ", X_train.shape, X_test.shape, Y_train.shape)
# Build 1-D CNN
input_shape = (X_train.shape[1], 1)
n_classes = int(Y_train.max()+1) # 10
model = CNN_1d(input_shape, n_classes, opt='adam')
model.summary()
# Train the model
tbCallBack = callbacks.TensorBoard(log_dir="./runs/TB_cnn1d")
model.fit(X_train, Y_train, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS,
validation_split=0.2, callbacks=[tbCallBack])
#model.save("./runs/checkpoint_cnn1d")
# Generate predicted label
#models.load_model("./runs/checkpoint_cnn1d")
prob_table = model.predict(X_test)
Y_pred = backend.argmax(prob_table)
print("The shape of Y_pred: ", Y_pred.shape)
# Make submission file
make_submission(Y_pred, "submission")
image_shape=image_shape,
batch_size=batch_size)
validation_generator = get_validation_generator(directory=validation_dir,
image_shape=image_shape,
batch_size=batch_size)
model = get_model(image_shape=image_shape, Net=Net)
print(model.summary())
if gpu_count > 1:
model = multi_gpu_model(model, gpus=gpu_count)
tensorboard_cb = callbacks.TensorBoard(
log_dir=log_dir,
histogram_freq=1,
)
early_stopping_cb = callbacks.EarlyStopping(monitor='val_loss',
patience=10,
verbose=0,
mode='min')
checkpoint_path = 'model.h5'
checkpoint_cb = callbacks.ModelCheckpoint(
checkpoint_path,
save_best_only=True,
monitor=
'val_accuracy' # for tensorflow 2 change to val_accuracy, for tensorflow 1 change to val_acc
)
def start_pong():
UP_ACTION = 2
DOWN_ACTION = 3
gamma = 0.99
x_train = []
y_train = []
rewards = []
reward_sum = 0
episode_num = 0
POSITIVE = 1
NEGATIVE = 0
running_reward = None
env = gym.make('Pong-v0')
observation = env.reset()
prev_input = None
model = get_model()
tbCallBack = callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=0,
write_graph=True,
write_images=True)
while True:
# preproces observation and set input x_train input as difference between two images(frames)
current_input = prepro(observation)
x_diff = current_input - prev_input if prev_input is not None else np.zeros(
80 * 80)
prev_input = current_input
# use policy network to action to get what model thinks about UP action
prediction = model.predict(np.expand_dims(x_diff, axis=1).T)
action = UP_ACTION if np.random.uniform() < prediction else DOWN_ACTION
# print('chosen action:', action)
y_diff = POSITIVE if action == UP_ACTION else NEGATIVE
# print('x frame', x_diff)
# save results to training set
x_train.append(x_diff)
y_train.append(y_diff)
# make next action step
observation, reward, done, info = env.step(action)
# print('observation', observation)
# print('reward', reward)
# print('done', done)
# print('info', info)
# log rewards data
rewards.append(reward)
# log rewards sum
reward_sum += reward
env.render()
if done:
print('End of episode:', episode_num, 'Total reward: ', reward_sum)
# increment nomber of episode
episode_num += 1
# make a model training with episode results - stack training data and labels. Set training data weights
# regarding rewards received per episode
model.fit(
x=np.vstack(x_train),
y=np.vstack(y_train),
verbose=0,
# callbacks=[tbCallBack],
sample_weight=discount_rewards(rewards, gamma))
running_reward = reward_sum if running_reward is None else running_reward * 0.99 + reward_sum * 0.01
# tflog('running_reward', running_reward)
save_model(model)
# reinitialisation learning data
x_train = []
y_train = []
rewards = []
reward_sum = 0
prev_input = None
# reset game* (restart env)
observation = env.reset()
# In[ ]:
checkpoint = callbacks.ModelCheckpoint(
"./models_transform/epoch.{epoch:03d}-loss.{loss:.2f}-val_loss.{val_loss:.2f}-b7.h5",
monitor='val_loss',
verbose=0,
save_best_only=True,
save_weights_only=True,
mode='auto',
period=1)
# checkpoint = callbacks.ModelCheckpoint("./models/loss.{loss:.2f}-acc.{acc:.2f}-val_loss.{val_loss:.2f}-val_acc.{val_acc:.2f}.h5", monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=False, mode='auto', period=1)
# early = EarlyStopping(monitor='val_acc', min_delta=0, patience=3, verbose=1, mode='auto')
# reduce_lr = callbacks.ReduceLROnPlateau(monitor='acc', factor=0.1, patience=2, min_lr=0.000001)
learningRateScheduler = callbacks.LearningRateScheduler(poly_decay)
tensorboard = callbacks.TensorBoard(log_dir='./logs_transform')
# In[ ]:
model.fit_generator(generator=train_generator,
steps_per_epoch=ceil(len(train_labels) / batch_size),
validation_data=test_generator,
validation_steps=ceil(len(test_labels) / batch_size),
epochs=nb_epochs,
callbacks=[checkpoint, tensorboard, learningRateScheduler],
use_multiprocessing=True,
workers=6,
verbose=1)
# In[ ]:
def _build_callback_hooks(
self,
models_dir: str,
logs_dir: Optional[str] = None,
is_training=True,
logging_frequency=25,
monitor_metric: str = "",
monitor_mode: str = "",
patience=2,
):
"""
Build callback hooks for the training and evaluation loop
Parameters
----------
models_dir : str
Path to directory to save model checkpoints
logs_dir : str
Path to directory to save tensorboard logs
is_training : bool, optional
Whether we are building callbacks for training or evaluation
logging_frequency : int, optional
How often, in number of epochs, to log training and evaluation progress
monitor_metric : str, optional
Name of metric to be used for ModelCheckpoint and EarlyStopping callbacks
monitor_mode : {"max", "min"}, optional
Mode for maximizing or minimizing the ModelCheckpoint and EarlyStopping
patience : int, optional
Number of epochs to wait before early stopping if metric change is below tolerance
Returns
-------
callbacks_list : list
List of callbacks to be used with the RelevanceModel training and evaluation
"""
callbacks_list: list = list()
if is_training:
# Model checkpoint
if models_dir and monitor_metric:
checkpoints_path = os.path.join(
models_dir, RelevanceModelConstants.CHECKPOINT_FNAME
)
cp_callback = callbacks.ModelCheckpoint(
filepath=checkpoints_path,
save_weights_only=False,
verbose=1,
save_best_only=True,
mode=monitor_mode,
monitor=monitor_metric,
)
callbacks_list.append(cp_callback)
# Early Stopping
if monitor_metric:
early_stopping_callback = callbacks.EarlyStopping(
monitor=monitor_metric,
mode=monitor_mode,
patience=patience,
verbose=1,
restore_best_weights=True,
)
callbacks_list.append(early_stopping_callback)
# TensorBoard
if logs_dir:
tensorboard_callback = callbacks.TensorBoard(
log_dir=logs_dir, histogram_freq=1, update_freq=5
)
callbacks_list.append(tensorboard_callback)
# Debugging/Logging
callbacks_list.append(DebuggingCallback(self.logger, logging_frequency))
# Add more here
return callbacks_list
def main(batch_size,
training_dir,
checkpoint_dir,
epochs,
n_fixed_layers,
logger_filename,
weight_file,
class_weight):
config = tf.ConfigProto()
config.gpu_options.allow_growth = True # dynamically grow the memory used on the GPU
sess = tf.Session(config=config)
tf.keras.backend.set_session(sess)
df_train = pd.read_csv("dataset/training.csv")
df_train = df_train.sample(frac=1,random_state=42)
df_val = pd.read_csv("dataset/validation.csv")
#3 using subset of training data
df_train['Filename'] = training_dir+"/"+df_train['Filename'].astype(str)
df_val['Filename'] = training_dir+"/"+df_val['Filename'].astype(str)
#df_train = df_train[:100]
#df_val = df_val[:100]
generator = preprocess.tfdata_generator(df_train['Filename'].values,
df_train['Drscore'].values,
is_training=True,
buffer_size=50,
batch_size=batch_size)
validation_generator = preprocess.tfdata_generator(df_val['Filename'].values,
df_val['Drscore'].values,
is_training=False,
buffer_size=50,
batch_size=batch_size)
## various callbacks
tensorboard_cbk = callbacks.TensorBoard(log_dir=checkpoint_dir,
update_freq='epoch',
write_grads=False,
histogram_freq=0)
checkpoint_cbk = callbacks.ModelCheckpoint(
filepath=os.path.join(checkpoint_dir,'weights-{epoch:03d}.hdf5'),
save_best_only=True,
monitor='val_loss',
verbose=1,
save_weights_only=False)
earlystop_ckb = callbacks.EarlyStopping(monitor='val_loss',
patience=5,
restore_best_weights=False)
csv_callback = callbacks.CSVLogger(os.path.join(checkpoint_dir,logger_filename),append=True)
reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5,
patience=3, min_lr=1e-6)
lr_scheduler = callbacks.LearningRateScheduler(step_decay)
model,base_model = create_model()
## freeze upper layers
files = sorted(glob(os.path.join(checkpoint_dir, 'weights-*.hdf5')))
if weight_file:
model_file = weight_file
initial_epoch = int(model_file[-8:-5])
print('Resuming using saved model %s.' % model_file)
model = tf.keras.models.load_model(model_file)
elif files:
model_file = files[-1]
initial_epoch = int(model_file[-8:-5])
print('Resuming using saved model %s.' % model_file)
model = tf.keras.models.load_model(model_file)
else:
#model,base_model = create_model()
initial_epoch = 0
if n_fixed_layers:
for layer in base_model.layers[:n_fixed_layers]:
layer.trainable = False
for layer in base_model.layers[n_fixed_layers:]:
layer.trainable = True
print("training layer {}".format(layer.name))
if class_weight:
class_weights=compute_class_weight('balanced',
np.unique(df_train["Drscore"].values),
df_train["Drscore"].values)
weight_dict = dict([(i,class_weights[i]) for i in range(len(class_weights))])
else:
weight_dict=None
model.fit(
generator,
epochs=epochs,
initial_epoch=initial_epoch,
steps_per_epoch=df_train.shape[0]//batch_size,
verbose=1,
validation_data=validation_generator,
validation_steps=df_val.shape[0]//batch_size,
class_weight=np.array(class_weights),
callbacks=[tensorboard_cbk,
checkpoint_cbk,
csv_callback,
reduce_lr])
self.batch_size,
triplet_selector).make_one_shot_iterator().get_next())
self.summary_writer = tf.summary.FileWriter("tb/%s" % opts.run)
def on_epoch_end(self, epoch, logs):
# TODO: how do we just use the models iterator here? don't care
# that it's "wastes" examples doing this eval, it's all generator
# based anyways...
next_egs = self.sess.run(self.examples)
sess = tf.keras.backend.get_session()
summaries = sess.run(loss_fn.summaries, feed_dict={inputs: next_egs})
# percentage_non_zero = np.count_nonzero(per_elem_loss) / self.batch_size
# log stats
for summary in summaries:
self.summary_writer.add_summary(summary, global_step=epoch)
self.summary_writer.flush()
callbacks = [
callbacks.ModelCheckpoint(filepath="runs/%s/model.{epoch}.hdf5" %
opts.run),
callbacks.TensorBoard(log_dir="tb/%s" % opts.run),
callbacks.TerminateOnNaN(),
NumZeroLossCB()
]
model.fit(examples,
epochs=opts.epochs,
verbose=1,
steps_per_epoch=opts.steps_per_epoch,
callbacks=callbacks)
def train_test_model(logdir, hparams):
filter_kernel_2 = json.loads(hparams['filter_kernel_2'])
units = json.loads(hparams['units'])
# C3 Convolution.
c3_input = keras.Input(shape=(x_train_c3[0].shape[0],
x_train_c3[0].shape[1], 1),
name='c3_input')
c3_model = layers.Conv2D(filters=int(hparams['filter_1']),
kernel_size=int(hparams['kernel_1']),
activation='relu')(c3_input)
c3_model = layers.MaxPooling2D(pool_size=2)(c3_model)
c3_model = layers.Dropout(hparams['dropout'])(c3_model)
if int(filter_kernel_2[0]) > 0:
c3_model = layers.Conv2D(filters=int(filter_kernel_2[0]),
kernel_size=int(filter_kernel_2[1]),
activation='relu')(c3_model)
c3_model = layers.MaxPooling2D(pool_size=2)(c3_model)
c3_model = layers.Dropout(hparams['dropout'])(c3_model)
c3_model = layers.Flatten()(c3_model)
# C4 Convolution.
c4_input = keras.Input(shape=(x_train_c3[0].shape[0],
x_train_c3[0].shape[1], 1),
name='c4_input')
c4_model = layers.Conv2D(filters=int(hparams['filter_1']),
kernel_size=int(hparams['kernel_1']),
activation='relu')(c4_input)
c4_model = layers.MaxPooling2D(pool_size=2)(c4_model)
c4_model = layers.Dropout(hparams['dropout'])(c4_model)
if int(filter_kernel_2[0]) > 0:
c4_model = layers.Conv2D(filters=int(filter_kernel_2[0]),
kernel_size=int(filter_kernel_2[1]),
activation='relu')(c4_model)
c4_model = layers.MaxPooling2D(pool_size=2)(c4_model)
c4_model = layers.Dropout(hparams['dropout'])(c4_model)
c4_model = layers.Flatten()(c4_model)
# Dense concatenation.
model = layers.Concatenate()([c3_model, c4_model])
for unit in units:
model = layers.Dense(unit, activation='relu')(model)
model = layers.Dropout(hparams['dropout'])(model)
model = layers.Dense(1, activation='sigmoid')(model)
model = tf.keras.models.Model(inputs=[c3_input, c4_input], outputs=model)
model.summary()
if not os.path.exists(logdir):
os.makedirs(logdir)
plot_model(model,
logdir + '/model.png',
show_shapes=True,
show_layer_names=False)
model.compile(optimizer=optimizers.Adam(learning_rate=hparams['lr'],
decay=0.001),
loss='binary_crossentropy',
metrics=['accuracy'])
cb = [
callbacks.TensorBoard(log_dir=logdir),
hp.KerasCallback(logdir, hparams)
]
history = model.fit({
'c3_input': x_train_c3,
'c4_input': x_train_c4
},
y_train,
validation_data=({
'c3_input': x_test_c3,
'c4_input': x_test_c4
}, y_test),
batch_size=64,
epochs=500,
callbacks=cb,
verbose=0)
return model, history
from tensorflow.keras import callbacks
cs = [
callbacks.TerminateOnNaN(),
callbacks.ModelCheckpoint(Config.log.tensorboard + '/weights.h5',
save_best_only=True,
save_weights_only=True,
verbose=1),
callbacks.ReduceLROnPlateau(
patience=Config.training.reduce_lr_on_plateau_pacience,
factor=Config.training.reduce_lr_on_plateau_factor,
verbose=1),
callbacks.EarlyStopping(patience=Config.training.early_stopping_patience,
verbose=1),
callbacks.TensorBoard(Config.log.tensorboard, write_graph=False)
]
try:
disc.fit(train_ds,
validation_data=val_ds,
epochs=Config.training.epochs,
initial_epoch=0,
callbacks=cs)
except KeyboardInterrupt:
print('stopped')
# In[ ]:
disc.load_weights(Config.log.tensorboard + '/weights.h5')
return model
model = model_final(preproc_source_sentences.shape,
preproc_target_sentences.shape[1],
len(source_tokenizer.word_index) + 1,
len(target_tokenizer.word_index) + 1)
model.summary()
#CallBacks
mfile = 'models/Glove_training_bach32.model.h5'
model_checkpoint = callbacks.ModelCheckpoint(mfile,
monitor='accuracy',
save_best_only=True,
save_weights_only=True)
logger = callbacks.CSVLogger('results/training_bach_32.log')
tensorboard = callbacks.TensorBoard(log_dir='results/training_bach_32')
callbacks = [logger, tensorboard]
#Training model and save callbacks:
#model.fit(X_train, Y_train, batch_size=1024, epochs=25, validation_split=0.1, callbacks=callbacks)
#Training model and save callbacks:
model.fit(X_train,
Y_train,
batch_size=32,
epochs=10,
validation_split=0.01)
Predicted_by_Glove = model.predict(X_test, len(X_test))
#Save Model
def scheduler(epoch):
if epoch < 10:
return lr
else:
return lr * tf.math.pow(0.9, epoch-10) # tf.math.pow(0.9, 10)=0.35, tf.math.pow(0.9, 20)=0.122, tf.math.pow(0.9, 50)=0.005
learningRateScheduler_poly = callbacks.LearningRateScheduler(poly_decay)
learningRateScheduler_scheduler = callbacks.LearningRateScheduler(scheduler)
ReduceLROnPlateau = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=3, min_lr=0.000001)
logs_dir = f'logs/{datetime.date.today()}'
if not os.path.exists(logs_dir):
os.makedirs(logs_dir)
tensorboard = callbacks.TensorBoard(log_dir=logs_dir)
checkpoint = callbacks.ModelCheckpoint(
os.path.join(checkpoints_dir, 'tf_yolov3_{epoch:02d}_{loss:.4f}_{val_loss:.4f}_weights.h5'),
verbose=1, save_weights_only=True, save_best_only=True, monitor='val_loss', mode='auto'
)
model.fit_generator(
generator=train_generator,
steps_per_epoch=ceil(len(train_list_IDs) // batch_size),
initial_epoch=0,
epochs=Epochs,
# callbacks=[learningRateScheduler, checkpoint, tensorboard],
callbacks=[ReduceLROnPlateau, checkpoint, tensorboard],
validation_data=val_generator,
validation_steps=ceil(len(test_list_IDs) // batch_size),
def train_deepdrug(batch_size,
lr,
epoch,
output,
k_fold=10,
subset="all",
suffix="resigrids"):
# optimize gpu memory usage
physical_devices = tf.config.experimental.list_physical_devices('GPU')
assert len(physical_devices) > 0
tf.config.experimental.set_memory_growth(physical_devices[0], True)
# load the data
with open("../data/tough_c1/control-pocket." + suffix, "rb") as f:
grids = list(pk.load(f).values())
labels = [np.array([0])] * len(grids)
if subset == "nucleotide":
with open("../data/tough_c1/nucleotide-pocket." + suffix, "rb") as f:
grids += list(pk.load(f).values())
labels += [np.array([1])] * (len(grids) - len(labels))
classes = 1
elif subset == "heme":
with open("../data/tough_c1/heme-pocket." + suffix, "rb") as f:
grids += list(pk.load(f).values())
labels += [np.array([1])] * (len(grids) - len(labels))
classes = 1
elif subset == "all":
with open("../data/tough_c1/nucleotide-pocket." + suffix, "rb") as f:
grids += list(pk.load(f).values())
labels += [np.array([1])] * (len(grids) - len(labels))
with open("../data/tough_c1/heme-pocket." + suffix, "rb") as f:
grids += list(pk.load(f).values())
labels += [np.array([2])] * (len(grids) - len(labels))
with open("../data/tough_c1/steroid-pocket." + suffix, "rb") as f:
grids += list(pk.load(f).values())
labels += [np.array([3])] * (len(grids) - len(labels))
classes = 4
else:
print("Invalid subset name. Please choose from all/nucleotide/heme.")
return
# shuffle input
idx_list = list(range(len(labels)))
rd.shuffle(idx_list)
grids = list(np.array(grids)[idx_list])
labels = list(np.array(labels)[idx_list])
# print("grids type:", type(grids))
# print("grid type:", type(grids[0]), "shape:", grids[0].shape)
# print("labels type:", type(labels))
# print("label type:", type(labels[0]), "shape:", labels[0].shape)
histories = list()
timestamp = time.strftime("%Y%m%d_%H%M%S", time.localtime())
for k in range(k_fold):
tf.keras.backend.clear_session()
# get training data with respect to the kth fold
x, y, val_x, val_y = split_dataset_with_kfold(grids, labels, k, k_fold)
if classes > 1:
y = to_categorical(y, num_classes=classes)
val_y = to_categorical(val_y, num_classes=classes)
# to balance different classes
sample_weights = compute_sample_weight('balanced', y)
val_data = (val_x, val_y)
# build & compile model
mdl = DeepDrug3DBuilder.build(classes=classes)
adam = Adam(lr=lr,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
loss = "binary_crossentropy" if classes==1 \
else "categorical_crossentropy"
metric = "binary_accuracy" if classes == 1 else "categorical_accuracy"
auc_metric = tf.keras.metrics.AUC()
precision_metric = tf.keras.metrics.Precision()
recall_metric = tf.keras.metrics.Recall()
mdl.compile(
optimizer=adam,
loss=loss,
metrics=[metric, auc_metric, precision_metric, recall_metric])
# callback function for model checking
log_dir = os.path.join('training_logs', timestamp, "fold_{}".format(k))
os.makedirs(log_dir, exist_ok=True)
tfCallBack = callbacks.TensorBoard(log_dir=log_dir)
history = mdl.fit(x,
y,
epochs=epoch,
batch_size=batch_size,
sample_weight=sample_weights,
validation_data=val_data,
shuffle=True,
callbacks=[tfCallBack])
histories.append(history)
# log the metrics after each cross-validation training
val_accs = list()
val_aucs = list()
val_precisions = list()
val_recalls = list()
for his in histories:
try:
val_accs.append(his.history["val_binary_accuracy"])
except KeyError:
val_accs.append(his.history["val_categorical_accuracy"])
val_aucs.append(his.history["val_auc"])
val_precisions.append(his.history["val_precision"])
val_recalls.append(his.history["val_recall"])
# find best epoch
val_accs = np.array(val_accs)
avgs = np.mean(val_accs, axis=0)
best_epoch = np.argmax(avgs)
# get the accuracy and standard deviation of the best epoch
max_accs = val_accs[:, best_epoch]
acc_avg = np.mean(max_accs)
acc_std = np.std(max_accs)
# get the auc score of the best epoch
max_aucs = np.array(val_aucs)[:, best_epoch]
auc_avg = np.mean(max_aucs)
auc_std = np.std(max_accs)
# get the precision, racall, f1 score of the best epoch
max_precisions = np.array(val_precisions)[:, best_epoch]
precision_avg = np.mean(max_precisions)
precision_std = np.std(max_precisions)
max_recalls = np.array(val_recalls)[:, best_epoch]
recall_avg = np.mean(max_recalls)
recall_std = np.std(max_recalls)
max_f1s = 2 * max_precisions * max_recalls / (max_precisions + max_recalls)
f1_avg = np.mean(max_f1s)
f1_std = np.std(max_f1s)
# print and save the training results
print("{}-fold cross validation performs the best "
"at epoch {}".format(k_fold, best_epoch + 1))
print("Accuracy is {} +- {}".format(acc_avg, acc_std))
print("AUC ROC is {} +- {}".format(auc_avg, auc_std))
print("Precision is {} +- {}".format(precision_avg, precision_std))
print("Recall is {} +- {}".format(recall_avg, recall_std))
print("F1 score is {} +- {}".format(f1_avg, f1_std))
print()
with open(os.path.join("training_logs", timestamp, "readme"), "w") as f:
print("dataset: {}".format(subset), file=f)
print("grid type: {}".format(suffix), file=f)
print("batch size: {}".format(batch_size), file=f)
print("learning rate: {}".format(lr), file=f)
print("epochs: {}".format(epoch), file=f)
print("validation folds: {}".format(k_fold), file=f)
print("{}-fold cross validation performs the best "
"at epoch {}".format(k_fold, best_epoch + 1),
file=f)
print("Accuracy is {} +- {}".format(acc_avg, acc_std), file=f)
print("AUC ROC is {} +- {}".format(auc_avg, auc_std), file=f)
print("Precision is {} +- {}".format(precision_avg, precision_std),
file=f)
print("Recall is {} +- {}".format(recall_avg, recall_std), file=f)
print("F1 score is {} +- {}".format(f1_avg, f1_std), file=f)
def train(self,
x_train_lstm,
x_train_svm,
y_train,
x_val_lstm=None,
x_val_svm=None,
y_val=None,
x_train_u=None,
x_val_u=None,
n_epochs=20,
class_weight=None):
logdir = os.path.join("tensorboard_log", self.detection,
self.save_model_name, self.num_set)
cmd = "mkdir -p " + logdir
os.system(cmd)
cmd = "rm -r " + logdir + "/"
os.system(cmd)
tensorboard_callback = callbacks.TensorBoard(log_dir=logdir)
h5_save_path = os.path.join(
"Models", self.detection,
self.model_name + "_" + self.num_set + ".h5")
acc = []
loss = []
val_acc = []
val_loss = []
x_train_dict = {
'lstm_features': x_train_lstm,
'svm_features': x_train_svm,
'attention_params': x_train_u
}
x_val_dict = {
'lstm_features': x_val_lstm,
'svm_features': x_val_svm,
'attention_params': x_val_u
}
train_set = tensorflow.data.Dataset.from_tensor_slices(
(x_train_dict, y_train)).shuffle(
buffer_size=Config.BUFFER_SIZE).batch(32).prefetch(
tensorflow.data.experimental.AUTOTUNE)
val_set = tensorflow.data.Dataset.from_tensor_slices(
(x_val_dict, y_val)).shuffle(
buffer_size=Config.BUFFER_SIZE).batch(32).prefetch(
tensorflow.data.experimental.AUTOTUNE)
model_cp = callbacks.ModelCheckpoint(filepath=h5_save_path,
monitor="val_loss",
verbose=0,
save_best_only=True,
save_weights_only=False,
mode="min",
period=10)
"""
history = self.model.fit(x = x_train_dict,
y = y_train,
shuffle = 1,
validation_data = (x_val_dict, y_val),
batch_size = Config.BATCH_SIZE,
epochs = Config.EPOCHS,
class_weight = class_weight,
callbacks = [model_cp, tensorboard_callback],
use_multiprocessing = False)
"""
history = self.model.fit(x=train_set,
validation_data=val_set,
epochs=Config.EPOCHS,
verbose=1,
class_weight=class_weight,
callbacks=[model_cp, tensorboard_callback],
use_multiprocessing=False)
# 训练集上的损失值和准确率
loss = history.history['loss']
val_loss = history.history["val_loss"]
figfile = os.path.join("Fig", self.detection,
self.model_name + "_" + self.num_set + ".png")
pickle.dump(loss, open(figfile.split(".p")[0] + "_loss.cpickle", "wb"))
pickle.dump(val_loss,
open(figfile.split(".p")[0] + "_val_loss.cpickle", "wb"))
plotCurve(loss, val_loss, 'Model Loss', 'loss', figfile)
with open(args.json_file) as f:
params = json.load(f)
print("Params loaded: ", params)
train_data, test_data = load_imdb(params)
model = cnn_zhang(params)
model.summary()
model.compile(loss='binary_crossentropy',
optimizer=tf.optimizers.Adam(learning_rate=params["lr"]),
metrics=['accuracy'])
cb = []
cb.append(callbacks.TensorBoard(params["logs_dir"]))
checkp = callbacks.ModelCheckpoint(params["weights_dir"] + '/weights.h5',
save_best_only=True,
save_weights_only=True)
cb.append(checkp)
if os.path.exists(params["weights_dir"] + '/weights.h5'):
print("Loading weights")
model.load_weights(params["weights_dir"] + '/weights.h5')
if not os.path.exists(params["weights_dir"]):
os.mkdir(params["weights_dir"])
model.fit(train_data,
epochs=params["epochs"],
validation_data=test_data,
map_feature = Lambda(reshape_query)(map_feature)
pred = Lambda(lambda x:prior_dist(x))([out_feature, map_feature]) #negative distance
combine = Model([sample, inp], pred)
optimizer = Adam(0.001)
combine.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['categorical_accuracy'])
print(combine.summary())
train_loader = DataGenerator(way=train_way, query=train_query, shot=shot, num_batch=1000)
(x,y),z = train_loader[0]
print(x.shape, y.shape, z.shape)
val_loader = DataGenerator(data_type='val',way=val_way, shot=shot)
(x,y),z = val_loader[0]
print(x.shape, y.shape, z.shape)
save_conv = SaveConv()
reduce_lr = cb.ReduceLROnPlateau(monitor='val_loss', factor=0.4,patience=2, min_lr=1e-8)
lr_sched = cb.LearningRateScheduler(scheduler)
tensorboard = cb.TensorBoard()
combine.fit_generator(train_loader,epochs=1000,validation_data=val_loader, use_multiprocessing=False, workers=4, shuffle=False, callbacks=[save_conv, lr_sched, tensorboard])
save_model(conv, "model/omniglot_conv")
# images, labels = zip(*list(loader('python/images_background')))
# images = np.expand_dims(images, axis=-1)
# images = np.repeat(images, repeats=3, axis=-1)
# print(images.shape)
# main_labels, sub_labels= [x[0] for x in labels], [x[1] for x in labels]
# encoder = LabelBinarizer()
# enc_main_labels = encoder.fit_transform(main_labels)
# output_num = len(np.unique(main_labels))
# bottleneck_model = conv_model()
# bottleneck_model.trainable = False
x_train, y_train = read_dataset(args.train_dataset_filename)
validation_data = None
if args.val_dataset_filename is not None:
validation_data = read_dataset(args.val_dataset_filename)
model = build_model()
optimizer = build_optimizer()
loss=build_loss()
model.compile(loss=loss, optimizer=optimizer, metrics = args.metrics)
model.summary()
tf_callbacks = []
if args.logsout_path:
tf_callbacks.append(tfcb.TensorBoard(log_dir=args.logsout_path, histogram_freq=0, write_graph=True, write_images=True))
if args.model_snapshots_path:
tf_callbacks.append(EpochLogger())
if args.best_model_monitor:
tf_callbacks.append(tfcb.ModelCheckpoint(
filepath = args.model_path,
save_best_only = True,
monitor = args.best_model_monitor,
mode = 'auto',
verbose=1))
start_time = time.time()
history = model.fit(
x_train, y_train,
epochs=args.epochs,
batch_size=args.batch_size,
layers.Dense(2, activation='softmax')
])
k_model.compile(optimizer='Adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
filepath = os.path.join(dir, 'models/defaults_keras_model.hdf5')
log_dir = os.path.join(dir, 'logs/1')
callbacks = [
callbacks.ModelCheckpoint(filepath=filepath,
verbose=1,
monitor='val_loss',
save_best_only=True),
#Check out the train history later with Tensorboard
callbacks.TensorBoard(log_dir=log_dir),
callbacks.EarlyStopping(patience=1)
]
k_model.fit(x_train,
y_train,
epochs=1000,
validation_split=0.1,
verbose=2,
callbacks=callbacks)
del k_model
k_model = models.load_model(filepath=filepath)
test_result = k_model.evaluate(x=x_test, y=y_test)
print(
f'Test result:\nTest loss = {test_result[0]}, Test accuracy = {test_result[1]}'
)