def fit(
model: tf.keras.models.Model,
train_iterator: tk.data.Iterator,
val_iterator: tk.data.Iterator = None,
val_freq: int | typing.Sequence[int] | str | None = "auto",
class_weight: dict[int, float] = None,
epochs: int = 1800,
callbacks: list[tf.keras.callbacks.Callback] = None,
verbose: int = 1,
initial_epoch: int = 0,
):
"""学習。
Args:
model: モデル
train_iterator: 訓練データ
val_iterator: 検証データ。Noneなら省略。
val_freq: 検証を行うエポック数の間隔、またはエポック数のリスト。0ならvalidationしない(独自仕様)。"auto"なら適当に決める(独自仕様)。
class_weight: クラスごとの重みのdict
epochs: エポック数
callbacks: コールバック。EpochLoggerとErrorOnNaNとhorovod関連は自動追加。
verbose: 1ならプログレスバー表示、2ならepoch毎の結果だけ表示。
initial_epoch: 学習を開始するエポック数 - 1
"""
# Horovodはそれぞれのワーカーが勝手にvalidateするのでshuffleする必要がある。
# shuffleするならデータ数分だけでは全体をカバーできないため3倍回す。
# horovodのexamplesの真似:
# <https://github.com/horovod/horovod/blob/9bdd70d/examples/keras_mnist_advanced.py#L112,L115>
use_horovod = tk.hvd.is_active()
if val_freq == 0 or val_iterator is None:
# val_freq == 0ならvalidationしない(独自仕様)
val_freq = None
val_iterator = None
elif val_freq == "auto":
# "auto"なら適当に決める(独自仕様)
val_freq = make_val_freq(
val_freq,
epochs,
len(train_iterator.dataset),
len(val_iterator.dataset) * (3 if use_horovod else 1),
)
train_ds, train_steps = train_iterator.data_loader.get_ds(
train_iterator.dataset, shuffle=True)
val_ds, val_steps = (val_iterator.data_loader.get_ds(val_iterator.dataset,
shuffle=use_horovod)
if val_iterator is not None else (None, 0))
logger.info(f"fit(train): {train_ds.element_spec} {train_steps=}")
if val_ds is not None:
logger.info(f"fit(val): {val_ds.element_spec} {val_steps=}")
callbacks = make_callbacks(callbacks, training=True)
fit_kwargs = {}
if val_freq is not None:
fit_kwargs["validation_freq"] = val_freq
with tk.log.trace("fit"):
model.fit(
train_ds,
steps_per_epoch=train_steps // tk.hvd.size(),
validation_data=val_ds,
validation_steps=(val_steps * 3 //
tk.hvd.size() if use_horovod else val_steps)
if val_iterator is not None else None,
class_weight=class_weight,
epochs=epochs,
callbacks=callbacks,
verbose=verbose if tk.hvd.is_master() else 0,
initial_epoch=initial_epoch,
**fit_kwargs,
)
def model_compile_fit(
hparams: Dict,
model: tf.keras.models.Model,
dataset: DatasetDF,
model_file: AnyStr = None,
log_dir: AnyStr = None,
best_only=True,
verbose=settings['verbose']['fit'],
):
hparams = {**settings['hparam_defaults'], **hparams}
optimiser = getattr(tf.keras.optimizers, hparams['optimizer'])
schedule = scheduler(hparams, dataset, verbose=verbose)
callbacks = [
EarlyStopping(monitor='val_loss',
mode='min',
verbose=verbose,
patience=hparams.get('patience', hparams['patience']),
restore_best_weights=best_only),
schedule,
KaggleTimeoutCallback(hparams["timeout"], verbose=False),
# ProgbarLogger(count_mode='samples', stateful_metrics=None)
]
if model_file:
callbacks += [
ModelCheckpoint(
model_file,
monitor='val_loss',
verbose=False,
save_best_only=best_only,
save_weights_only=False,
mode='auto',
)
]
if log_dir and settings['verbose']['tensorboard']:
callbacks += [
tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1), # log metrics
KerasCallback(log_dir, hparams) # log train_hparams
]
timer_start = time.time()
model.compile(loss=tf.keras.losses.categorical_crossentropy,
optimizer=optimiser(
learning_rate=hparams.get('learning_rate', 0.0001)),
metrics=['accuracy'])
history = model.fit(dataset.X["train"],
dataset.Y["train"],
batch_size=hparams.get("batch_size", 128),
epochs=999,
verbose=verbose,
validation_data=(dataset.X["valid"],
dataset.Y["valid"]),
callbacks=callbacks)
timer_seconds = int(time.time() - timer_start)
if 'val_loss' in history.history:
best_epoch = history.history['val_loss'].index(
min(history.history['val_loss'])) if best_only else -1
model_stats = {
key: value[best_epoch]
for key, value in history.history.items()
}
model_stats['time'] = timer_seconds
model_stats['epochs'] = len(history.history['loss'])
else:
model_stats = None
return model_stats
def summary(model: tf.keras.models.Model):
"""summaryを実行するだけ。"""
model.summary(print_fn=logger.info
if tk.hvd.is_master() else lambda x: None) # type: ignore
def _predict_on_batch(model: tf.keras.models.Model, X):
return model.predict_on_batch(X)
def stochastic_from_keras(
model: tf.keras.models.Model,
input_tensors=None,
clone_function=None,
expect_determinism=False,
temp_weights_path="tmp/weights",
):
"""
Creates a stochastic instance from a given `tf.keras.models.Sequential` model:
The new model will have the same structure (layers) and weights as the passed model.
All stochastic layers (e.g. tf.keras.layers.Dropout) will be used for randomization during randomized predictions.
If no stochastic layers are present, a ValueError is thrown.
The raising of the error can be suppressed by setting `expect_determinism` to true.
:param model: The model to copy. Remains unchanged.
:param input_tensors: Optional tensors to use as input_tensors for new model. See the corresponding parameter in `tf.keras.models.clone_model` for details.
:param _clone_function: Optional function to use to clone layers. Will be applied to all layers except input layers and stochastic layers. See the corresponding parameter in `tf.keras.models.clone_model` for more details.
:param expect_determinism: If True, deterministic models (e.g. models without stochastic layers) are accepted and no ValueError is thrown.
:param temp_weights_path: The model weights are temporarily saved to the disk at this path. Folder is deleted after successful completion.
:return: A newly created stochastic model
"""
# _clone_function is some layer cloning behavior that can be specified by the user
# If none is specified, we use keras default (see `tf.keras.models.clone_model`)
if clone_function is None:
def _clone_function(layer):
return layer.__class__.from_config(layer.get_config())
# We wrap the users (or default) clone function in a clone function
# that replaces stochastic layers with uncertainty wizard stochastic layers
stochastic_mode = StochasticMode()
is_stochastic_layer = []
def _uncertainty_wizard_aware_clone_function(layer):
new_layer = Stochastic._replace_layer_if_possible(
layer, stochastic_mode=stochastic_mode)
if new_layer == layer:
# Layer was not mapped to an uncertainty wizard layer, thus the default clone function is applied
new_layer = _clone_function(layer)
is_stochastic_layer.append(False)
else:
is_stochastic_layer.append(True)
return new_layer
# Clone the keras model to become the new inner model
new_inner = tf.keras.models.clone_model(
model=model,
input_tensors=input_tensors,
clone_function=_uncertainty_wizard_aware_clone_function,
)
new_inner.stochastic_mode_tensor = stochastic_mode.as_tensor()
if not expect_determinism and not any(is_stochastic_layer):
raise ValueError(
"The passed model had no stochastic layers."
"If that is intended (and you do not plan to use any sampling based quantifiers)"
"you can set the flag `expect_determinism = True`, i.e., "
"calling `SequentialStochastic.clone_from_keras(keras_model,expect_determinism = True)`"
)
# Restore the Weights
model.save_weights(temp_weights_path)
new_inner.load_weights(temp_weights_path)
# Remove temporarily stored weights
shutil.rmtree(temp_weights_path, ignore_errors=True)
# Put the wrapper around the new model
if isinstance(model, tf.keras.models.Sequential):
target_class = StochasticSequential
else:
target_class = StochasticFunctional
# Consenting Adults: The _wrap method is intended to be used here
# but declared private as it is not intended to be used by the uwiz user
return target_class._wrap(
inner=new_inner, stochastic_mode_tensor=stochastic_mode.as_tensor())
def check(
train_model: tf.keras.models.Model,
pred_model: tf.keras.models.Model,
models_dir: tk.typing.PathLike,
dataset: tk.data.Dataset = None,
train_data_loader: tk.data.DataLoader = None,
pred_data_loader: tk.data.DataLoader = None,
save_mode: str = "hdf5",
):
"""モデルの簡易動作確認用コード。
Args:
train_model: 学習用モデル
pred_model: 推論用モデル
models_dir: 情報の保存先ディレクトリ
dataset: チェック用データ (少数にしておくこと)
train_data_loader: 学習用DataLoader
pred_data_loader: 推論用DataLoader
save_mode: 保存形式 ("hdf5", "saved_model", "onnx", "tflite"のいずれか)
"""
models_dir = pathlib.Path(models_dir)
logger = tk.log.get(__name__)
# summary表示
tk.models.summary(train_model)
# グラフを出力
tk.models.plot(train_model, models_dir / "model.png")
# save/loadの動作確認 (とりあえず落ちなければOKとする)
with tempfile.TemporaryDirectory() as tmpdir:
save_path = pathlib.Path(tmpdir) / f"model.{save_mode}"
tk.models.save(pred_model, save_path)
pred_model = tk.models.load(save_path)
# train_model.evaluate
if dataset is not None and train_data_loader is not None:
ds, steps = train_data_loader.get_ds(dataset, shuffle=True)
logger.info(f"train_model.evaluate: {ds.element_spec} {steps=}")
values = train_model.evaluate(ds, steps=steps, verbose=1)
if len(train_model.metrics_names) == 1:
evals = {train_model.metrics_names[0]: values}
else:
evals = dict(zip(train_model.metrics_names, values))
logger.info(f"check.evaluate: {tk.evaluations.to_str(evals)}")
# pred_model.predict
if dataset is not None and pred_data_loader is not None:
ds, steps = pred_data_loader.get_ds(dataset)
logger.info(f"pred_model.evaluate: {ds.element_spec} {steps=}")
pred = pred_model.predict(ds, steps=steps, verbose=1)
if isinstance(pred, (list, tuple)):
logger.info(f"check.predict: shape={[p.shape for p in pred]}")
else:
logger.info(f"check.predict: shape={pred.shape}")
# train_model.fit
if dataset is not None and train_data_loader is not None:
ds, steps = train_data_loader.get_ds(dataset, shuffle=True)
train_model.fit(ds, steps_per_epoch=steps, epochs=1, verbose=1)
def summary(model: tf.keras.models.Model):
"""summaryを実行するだけ。"""
model.summary(print_fn=tk.log.get(__name__).info if tk.hvd.is_master(
) else lambda x: None)
def dump_features(self, dataset_partition: str, name: str,
model: tf.keras.models.Model, input_shape: Tuple,
feature_size: int, preprocess_fn: Callable) -> None:
"""Dumps the features of a partition of database: Train, val or test. The features are extracted
by a model and then dumped into a .npy file. Also the filenames for which the features are dumped will be
stored in another .npy file. The name of these .npy files are features.npy and files_names.npy and they are
stored in a directory under database_address. The name of directory is the join of dataset_partition and
name argument.
Inputs:
dataset_partition: Should be train, val or test.
name: The name of features to dump. For example VGG19_layer_4
model: The network or model which is used to dump features. The output of this model is the feature which
will be stored.
input_shape: The input shape of each image file.
feature_size: The length or depth of features.
preprocess_fn: The preprocessing function which is applied to the raw image before passing it through the model.
Use None to ignore it.
Returns: None
This function does not return anything but stores two numpy files. In order to load them use
load_dumped_features function. """
if dataset_partition == 'train':
files_dir = self.train_folders
elif dataset_partition == 'val':
files_dir = self.val_folders
elif dataset_partition == 'test':
files_dir = self.test_folders
else:
raise Exception('Pratition is not train val or test!')
assert (dataset_partition in ('train', 'val', 'test'))
dir_path = os.path.join(self.database_address,
f'{name}_{dataset_partition}')
if not os.path.exists(dir_path):
os.mkdir(dir_path)
all_files = list()
for class_name in files_dir:
all_files.extend([
os.path.join(class_name, file_name)
for file_name in os.listdir(class_name)
])
files_names_address = os.path.join(dir_path, 'files_names.npy')
np.save(files_names_address, all_files)
features_address = os.path.join(dir_path, 'features.npy')
n = len(all_files)
m = feature_size
features = np.zeros(shape=(n, m))
begin_time = datetime.now()
for index, sampled_file in enumerate(all_files):
if index % 1000 == 0:
print(f'{index}/{len(all_files)} images dumped')
img = tf.keras.preprocessing.image.load_img(
sampled_file, target_size=(input_shape[:2]))
img = tf.keras.preprocessing.image.img_to_array(img)
img = np.expand_dims(img, axis=0)
if preprocess_fn is not None:
img = preprocess_fn(img)
features[index, :] = model.predict(img).reshape(-1)
np.save(features_address, features)
end_time = datetime.now()
print('Features dumped')
print(f'Time to dump features: {str(end_time - begin_time)}')
def create_epoch_tuple(self, encoder, embedding_model: tf.keras.models.Model):
logger.info(">> Creating tuples for an epoch -----")
self.qidxs = np.random.choice(self.indexes, self.qsize)
self.mask = np.ones(len(self.qclusters), dtype=np.uint8)
self.indexes = np.arange(len(self.qidxs))
self.pidxs = []
self.nidxs = []
for idx in self.qidxs:
self.mask[idx] = 0
cluster_idx = self.idx2cluster[idx]
self.cluster_bitmap[cluster_idx][idx] = 0
all_pos_idxs = np.where(self.cluster_bitmap[cluster_idx])[0]
pos_idxs = np.random.choice(all_pos_idxs, size=self.pos_size)
self.cluster_bitmap[cluster_idx][idx] = 1
for t in pos_idxs:
self.pidxs.append(t)
for t in all_pos_idxs:
self.mask[t] = 0
pool_idxs = np.where(self.mask)[0]
llen_pool_idxs = len(pool_idxs)
logger.info("Negative masks: %s", llen_pool_idxs)
if llen_pool_idxs >= self.pool_size:
llen_pool_idxs = self.pool_size
else:
logger.info("Pool size are limited to %s", llen_pool_idxs)
self.idx2pool_neg = np.random.choice(np.where(self.mask)[0], size=llen_pool_idxs)
X_pos = embedding_model.predict(encoder(self.X[self.pidxs]), batch_size=128, verbose=0)
X_neg = embedding_model.predict(encoder(self.X[self.idx2pool_neg]), batch_size=64, verbose=0)
logger.info(">> %s %s" % (X_pos.shape, X_neg.shape))
logger.info(">> Searching for hard negatives...")
scores = tf.matmul(X_pos, X_neg, transpose_b=True)
top_val, top_indices = tf.math.top_k(scores, k=self.neg_size, )
avg_ndist = tf.Variable(0.0, trainable=False, dtype=tf.float32)
n_ndist = tf.Variable(0.0, trainable=False, dtype=tf.float32)
for q in range(len(self.pidxs)):
qcluster = self.idx2cluster[q]
clusters = [qcluster]
r = 0
nidxs = []
while len(nidxs) < self.neg_size and r < len(top_indices[q]):
potential = self.idx2pool_neg[top_indices[q, r]]
if not self.idx2cluster[potential] in clusters:
nidxs.append(potential)
clusters.append(self.idx2cluster[potential])
avg_ndist.assign_add(pairwise_dist(X_pos[q], X_neg[top_indices[q, r]]))
n_ndist.assign_add(1)
r += 1
self.nidxs.append(nidxs)
avg_dist = tf.divide(avg_ndist, n_ndist).numpy()
logger.info("Average negative l2-distance: {:.6f}".format(avg_dist))
return avg_dist
def create_epoch_tuple(self, encoder, embedding_model: tf.keras.models.Model):
logger.info(">> Creating tuples for an epoch -----")
self.mask = np.ones(len(self.qclusters), dtype=np.uint8)
self.qidxs = np.zeros(self.qsize, dtype=np.uint8)
self.indexes = np.arange(len(self.qidxs))
# Select positive item and mask all related ones
for i in range(self.qsize):
idx = np.random.choice(np.where(self.mask)[0], size=1)[0]
self.qidxs[i] = idx
_, y_pos_idxs = self.item2item[idx].nonzero()
for t in y_pos_idxs:
self.mask[t] = 0
X_emb = embedding_model.predict(encoder(self.X), batch_size=1024, verbose=1)
X_dist = np.dot(X_emb, X_emb.T)
# del X_emb
mask_pos = ~self.item2item[self.qidxs].toarray().astype(np.bool)
mask_neg = ~mask_pos
X_emd_query = X_dist[self.qidxs]
X_dist_pos = np.ma.masked_array(X_emd_query, mask=mask_pos)
X_dist_neg = np.ma.masked_array(X_emd_query, mask=mask_neg)
logger.info(">> Searching for hard negatives...")
# # Select hardest positive, if it's similar with model => skip
min_pos_dist = X_dist_pos.min(axis=1)
max_neg_dist = X_dist_neg.max(axis=1)
self.pos_pair = []
self.neg_pair = []
cum_ndist_pos, cum_ndist_neg = tf.Variable(0.0, trainable=False, dtype=tf.float32), tf.Variable(0.0,
trainable=False,
dtype=tf.float32)
n_ndist, p_ndist = 0, 0
for i in np.where(min_pos_dist >= self.threshold)[0]:
pos_idx_by_random = np.random.choice(np.where(X_dist_pos[i] == min_pos_dist[i])[0], size=1)[0]
left_idx, right_idx = self.qidxs[i], pos_idx_by_random
cum_ndist_pos.assign_add(pairwise_dist(X_emb[left_idx], X_emb[right_idx]))
p_ndist += 1
self.pos_pair.append([left_idx, right_idx, 1])
for i in np.where(max_neg_dist < self.threshold)[0]:
neg_idxs = np.random.choice(np.where(X_dist_neg[i] < max_neg_dist[i] + 0.1)[0], size=self.neg_size)
for neg_idx in neg_idxs:
left_idx, right_idx = self.qidxs[i], neg_idx
cum_ndist_neg.assign_add(pairwise_dist(X_emb[left_idx], X_emb[right_idx]))
n_ndist += 1
self.neg_pair.append([left_idx, right_idx, 0])
logger.info("Average negative l2-distance: {:.6f}".format(cum_ndist_pos.value() / p_ndist))
logger.info("Average positive l2-distance: {:.6f}".format(cum_ndist_neg.value() / n_ndist))
self.pair = np.append(self.pos_pair, self.neg_pair, axis=0)
del X_emb, X_dist, X_emd_query
gc.collect()