def __init__(self,
name=None,
units=1,
tensor=None,
dtype=None):
if not dtype:
dtype = K.floatx()
elif not dtype == K.floatx():
K.set_floatx(dtype)
if units < 1:
raise ValueError(
'Expected at least one unit size - was provided `units`={:d}'.format(units)
)
layer = InputLayer(
batch_input_shape=(None, units),
input_tensor=tensor,
name=name,
dtype=dtype
)
super(RadialBasisBase, self).__init__(
layers=to_list(layer),
inputs=to_list(layer.input),
outputs=to_list(layer.output),
)
def train(model_func, params):
image_width = params['image_width']
image_height = params['image_height']
image_format = params['image_format']
distort_color = params['distort_color']
momentum = params['momentum']
loss_scale = params['loss_scale']
data_dir = params['data_dir']
data_idx_dir = params['data_idx_dir']
batch_size = params['batch_size']
num_iter = params['num_iter']
iter_unit = params['iter_unit']
log_dir = params['log_dir']
export_dir = params['export_dir']
tensorboard_dir = params['tensorboard_dir']
display_every = params['display_every']
precision = params['precision']
dali_mode = params['dali_mode']
use_xla = params['use_xla']
if data_dir is not None:
file_format = os.path.join(data_dir, '%s-*')
train_files = sorted(tf.io.gfile.glob(file_format % 'train'))
valid_files = sorted(tf.io.gfile.glob(file_format % 'validation'))
num_train_samples = common.get_num_records(train_files)
num_valid_samples = common.get_num_records(valid_files)
else:
num_train_samples = 1281982
num_valid_samples = 5000
train_idx_files = None
valid_idx_files = None
if data_idx_dir is not None:
file_format = os.path.join(data_idx_dir, '%s-*')
train_idx_files = sorted(tf.io.gfile.glob(file_format % 'train'))
valid_idx_files = sorted(tf.io.gfile.glob(file_format % 'validation'))
if iter_unit.lower() == 'epoch':
num_epochs = num_iter
nstep_per_epoch = num_train_samples // (batch_size * hvd.size())
nstep_per_valid = num_valid_samples // (batch_size * hvd.size())
else:
assert iter_unit.lower() == 'batch'
num_epochs = 1
nstep_per_epoch = min(num_iter,
num_train_samples // (batch_size * hvd.size()))
nstep_per_valid = min(10,
num_valid_samples // (batch_size * hvd.size()))
initial_epoch = 0
if log_dir:
# We save check points only when using the real data.
assert data_dir, "--data_dir cannot be empty when using --log_dir"
assert os.path.exists(log_dir)
ckpt_format = log_dir + "/model-{epoch:02d}-{val_top1:.2f}.hdf5"
# Looks for the most recent checkpoint and sets the initial epoch from it.
for filename in os.listdir(log_dir):
if filename.startswith('model-'):
initial_epoch = max(int(re.findall(r'\d+', filename)[0]),
initial_epoch)
if tensorboard_dir:
assert os.path.exists(tensorboard_dir)
if export_dir:
assert os.path.exists(export_dir)
save_format = export_dir + "/saved_model_rn50.h5"
if use_xla:
tf.config.optimizer.set_jit(True)
# Horovod: pin GPU to be used to process local rank (one GPU per process)
gpus = tf.config.experimental.list_physical_devices('GPU')
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
if gpus:
tf.config.experimental.set_visible_devices(gpus[hvd.local_rank()],
'GPU')
if precision == 'fp16':
policy = keras.mixed_precision.experimental.Policy(
'mixed_float16', loss_scale)
keras.mixed_precision.experimental.set_policy(policy)
lr_schedule = common.create_piecewise_constant_decay_with_warmup(
batch_size=batch_size * hvd.size(),
epoch_size=num_train_samples,
warmup_epochs=common.LR_SCHEDULE[0][1],
boundaries=list(p[1] for p in common.LR_SCHEDULE[1:]),
multipliers=list(p[0] for p in common.LR_SCHEDULE),
compute_lr_on_cpu=True)
opt = keras.optimizers.SGD(learning_rate=lr_schedule, momentum=momentum)
# Horovod: add Horovod DistributedOptimizer. We use a modified version to
# support the custom learning rate schedule.
opt = hvd_patch.DistributedOptimizer(opt)
backend.set_image_data_format(image_format)
dtype = 'float16' if precision == 'fp16' else 'float32'
backend.set_floatx(dtype)
model = model_func(num_classes=image_processing.NUM_CLASSES)
loss_func = 'sparse_categorical_crossentropy',
top5 = tf.keras.metrics.SparseTopKCategoricalAccuracy(k=5, name='top5')
top1 = tf.keras.metrics.SparseTopKCategoricalAccuracy(k=1, name='top1')
# Horovod: Specify `experimental_run_tf_function=False` to ensure TensorFlow
# uses hvd.DistributedOptimizer() to compute gradients. However, this option
# will disable the overlapping of the data loading and compute and hurt the
# performace if the model is not under the scope of distribution strategy
# scope.
model.compile(optimizer=opt,
loss=loss_func,
metrics=[top1, top5],
experimental_run_tf_function=False)
training_hooks = []
training_hooks.append(hvd.callbacks.BroadcastGlobalVariablesCallback(0))
training_hooks.append(_ProfileKerasFitCallback(batch_size, display_every))
if log_dir and hvd.rank() == 0:
ckpt_callback = keras.callbacks.ModelCheckpoint(
ckpt_format,
monitor='val_top1',
verbose=1,
save_best_only=False,
save_weights_only=False,
save_frequency=1)
training_hooks.append(ckpt_callback)
if tensorboard_dir and hvd.rank() == 0:
tensorboard_callback = tf.keras.callbacks.TensorBoard(
log_dir=tensorboard_dir)
training_hooks.append(tensorboard_callback)
if data_dir is not None:
num_preproc_threads = params['dali_threads'] if dali_mode else 10
train_input = image_processing.image_set(
train_files,
batch_size,
image_height,
image_width,
training=True,
distort_color=distort_color,
deterministic=False,
num_threads=num_preproc_threads,
use_dali=dali_mode,
idx_filenames=train_idx_files)
valid_input = image_processing.image_set(
valid_files,
batch_size,
image_height,
image_width,
training=False,
distort_color=False,
deterministic=False,
num_threads=num_preproc_threads,
use_dali=dali_mode,
idx_filenames=valid_idx_files)
if dali_mode:
train_input = train_input.get_device_dataset()
valid_input = valid_input.get_device_dataset()
valid_params = {
'validation_data': valid_input,
'validation_steps': nstep_per_valid,
'validation_freq': 1
}
else:
train_input = image_processing.fake_image_set(batch_size, image_height,
image_width)
valid_params = {}
try:
verbose = 2 if hvd.rank() == 0 else 0
model.fit(train_input,
epochs=num_epochs,
callbacks=training_hooks,
steps_per_epoch=nstep_per_epoch,
verbose=verbose,
initial_epoch=initial_epoch,
**valid_params)
except KeyboardInterrupt:
print("Keyboard interrupt")
if export_dir and hvd.rank() == 0:
model.save(save_format)
print(f"The model is saved to {save_format}")
def retinanet_mask_3D(inputs,
num_classes,
retinanet_model=None,
anchor_params=None,
nms=True,
class_specific_filter=True,
crop_size=(14, 14),
mask_size=(28, 28, 28),
name='retinanet-mask-3D',
roi_submodels=None,
mask_dtype=K.floatx(),
**kwargs):
"""Construct a RetinaNet mask model on top of a retinanet bbox model.
Uses the retinanet bbox model and appends layers to compute masks.
Args:
inputs: List of tensorflow.keras.layers.Input.
The first input is the image, the second input the blob of masks.
num_classes: Integer, number of classes to classify.
retinanet_model: deepcell.model_zoo.retinanet.retinanet model,
returning regression and classification values.
anchor_params: Struct containing anchor parameters.
nms: Boolean, whether to use NMS.
class_specific_filter: Boolean, use class specific filtering.
roi_submodels: Submodels for processing ROIs.
mask_dtype: Data type of the masks, can be different from the main one.
name: Name of the model.
**kwargs: Additional kwargs to pass to the retinanet bbox model.
Returns:
Model with inputs as input and as output the output of each submodel
for each pyramid level and the detections. The order is as defined in
submodels.
```
[
regression, classification, other[0], other[1], ...,
boxes_masks, boxes, scores, labels, masks, other[0], other[1], ...
]
```
"""
if anchor_params is None:
anchor_params = AnchorParameters.default
if roi_submodels is None:
retinanet_dtype = K.floatx()
K.set_floatx(mask_dtype)
roi_submodels = default_roi_submodels(num_classes, crop_size,
mask_size, mask_dtype,
retinanet_dtype)
K.set_floatx(retinanet_dtype)
image = inputs
image_shape = Shape()(image)
if retinanet_model is None:
retinanet_model = retinanet(inputs=image,
num_classes=num_classes,
num_anchors=anchor_params.num_anchors(),
**kwargs)
# parse outputs
regression = retinanet_model.outputs[0]
classification = retinanet_model.outputs[1]
other = retinanet_model.outputs[2:]
features = [
retinanet_model.get_layer(name).output
for name in ['P3', 'P4', 'P5', 'P6', 'P7']
]
# build boxes
anchors = __build_anchors(anchor_params, features)
boxes = RegressBoxes(name='boxes')([anchors, regression])
boxes = ClipBoxes(name='clipped_boxes')([image, boxes])
# filter detections (apply NMS / score threshold / select top-k)
detections = FilterDetections(
nms=nms,
class_specific_filter=class_specific_filter,
max_detections=100,
name='filtered_detections')([boxes, classification] + other)
# split up in known outputs and "other"
boxes = detections[0]
scores = detections[1]
# get the region of interest features
roi_input = [image_shape, boxes, classification] + features
rois = RoiAlign(crop_size=crop_size)(roi_input)
# execute maskrcnn submodels
maskrcnn_outputs = [submodel(rois) for _, submodel in roi_submodels]
# concatenate boxes for loss computation
trainable_outputs = [
ConcatenateBoxes(name=name)([boxes, output])
for (name, _), output in zip(roi_submodels, maskrcnn_outputs)
]
# reconstruct the new output
outputs = [regression, classification] + other + trainable_outputs + \
detections + maskrcnn_outputs
return Model(inputs=inputs, outputs=outputs, name=name)
def retinanet_mask(inputs,
backbone_dict,
num_classes,
frames_per_batch=1,
backbone_levels=['C3', 'C4', 'C5'],
pyramid_levels=['P3', 'P4', 'P5', 'P6', 'P7'],
retinanet_model=None,
anchor_params=None,
nms=True,
panoptic=False,
class_specific_filter=True,
crop_size=(14, 14),
mask_size=(28, 28),
name='retinanet-mask',
roi_submodels=None,
max_detections=100,
score_threshold=0.05,
nms_threshold=0.5,
mask_dtype=K.floatx(),
**kwargs):
"""Construct a RetinaNet mask model on top of a retinanet bbox model.
Uses the retinanet bbox model and appends layers to compute masks.
Args:
inputs (tensor): List of tensorflow.keras.layers.Input.
The first input is the image, the second input the blob of masks.
backbone_dict (dict): A dictionary with the backbone layers.
num_classes (int): Integer, number of classes to classify.
frames_per_batch (int): Size of z axis in generated batches.
If equal to 1, assumes 2D data.
backbone_levels (list): The backbone levels to be used.
to create the feature pyramid. Defaults to ['C3', 'C4', 'C5'].
pyramid_levels (list): The pyramid levels to attach regression and
classification heads to. Defaults to ['P3', 'P4', 'P5', 'P6', 'P7'].
retinanet_model (tensorflow.keras.Model): RetinaNet model that predicts
regression and classification values.
anchor_params (AnchorParameters): Struct containing anchor parameters.
nms (bool): Whether to use non-maximum suppression
for the filtering step.
panoptic (bool): Flag for adding the semantic head for panoptic
segmentation tasks. Defaults to false.
class_specific_filter (bool): Use class specific filtering.
crop_size (tuple): 2-length tuple for the x-y size of the crops.
Used to create default roi_submodels.
mask_size (tuple): 2-length tuple for the x-y size of the masks.
Used to create default roi_submodels.
name (str): Name of the model.
roi_submodels (list): Submodels for processing ROIs.
max_detections (int): The maximum number of detections allowed.
score_threshold (float): Minimum score for the FilterDetections layer.
nms_threshold (float): Minimimum NMS for the FilterDetections layer.
mask_dtype (str): Dtype to use for mask tensors.
kwargs (dict): Additional kwargs to pass to the retinanet bbox model.
Returns:
tensorflow.keras.Model: Model with inputs as input and as output
the output of each submodel for each pyramid level and the
detections. The order is as defined in submodels.
```
[
regression, classification, other[0], ...,
boxes_masks, boxes, scores, labels, masks, other[0], ...
]
```
"""
if anchor_params is None:
anchor_params = AnchorParameters.default
if roi_submodels is None:
retinanet_dtype = K.floatx()
K.set_floatx(mask_dtype)
roi_submodels = default_roi_submodels(num_classes, crop_size,
mask_size, frames_per_batch,
mask_dtype, retinanet_dtype)
K.set_floatx(retinanet_dtype)
image = inputs
image_shape = Shape()(image)
if retinanet_model is None:
retinanet_model = retinanet(inputs=image,
backbone_dict=backbone_dict,
num_classes=num_classes,
backbone_levels=backbone_levels,
pyramid_levels=pyramid_levels,
panoptic=panoptic,
num_anchors=anchor_params.num_anchors(),
frames_per_batch=frames_per_batch,
**kwargs)
# parse outputs
regression = retinanet_model.outputs[0]
classification = retinanet_model.outputs[1]
if panoptic:
# Determine the number of semantic heads
n_semantic_heads = len([
1 for layer in retinanet_model.layers if 'semantic' in layer.name
])
# The panoptic output should not be sent to filter detections
other = retinanet_model.outputs[2:-n_semantic_heads]
semantic = retinanet_model.outputs[-n_semantic_heads:]
else:
other = retinanet_model.outputs[2:]
features = [
retinanet_model.get_layer(name).output for name in pyramid_levels
]
# build boxes
anchors = __build_anchors(anchor_params,
features,
frames_per_batch=frames_per_batch)
boxes = RegressBoxes(name='boxes')([anchors, regression])
boxes = ClipBoxes(name='clipped_boxes')([image, boxes])
# filter detections (apply NMS / score threshold / select top-k)
detections = FilterDetections(
nms=nms,
nms_threshold=nms_threshold,
score_threshold=score_threshold,
class_specific_filter=class_specific_filter,
max_detections=max_detections,
name='filtered_detections')([boxes, classification] + other)
# split up in known outputs and "other"
boxes = detections[0]
scores = detections[1]
# get the region of interest features
#
# roi_input = [image_shape, boxes, classification] + features
# rois = _RoiAlign(crop_size=crop_size)(roi_input)
fpn = features[0]
fpn = UpsampleLike()([fpn, image])
rois = RoiAlign(crop_size=crop_size)([boxes, fpn])
# execute maskrcnn submodels
maskrcnn_outputs = [submodel(rois) for _, submodel in roi_submodels]
# concatenate boxes for loss computation
trainable_outputs = [
ConcatenateBoxes(name=name)([boxes, output])
for (name, _), output in zip(roi_submodels, maskrcnn_outputs)
]
# reconstruct the new output
outputs = [regression, classification] + other + trainable_outputs + \
detections + maskrcnn_outputs
if panoptic:
outputs += list(semantic)
model = Model(inputs=inputs, outputs=outputs, name=name)
model.backbone_levels = backbone_levels
model.pyramid_levels = pyramid_levels
return model
def __init__(self,
fields=None,
variables=None,
hidden_layers=None,
activation="tanh",
output_activation="linear",
rnn_type="SimpleRNN",
recurrent_activation="tanh",
kernel_initializer=None,
bias_initializer=None,
kernel_regularizer=None,
bias_regularizer=None,
dtype=None,
trainable=True,
**kwargs):
# check data-type.
if dtype is None:
dtype = K.floatx()
elif not K.floatx() == dtype:
K.set_floatx(dtype)
# prepare hidden layers.
if hidden_layers is None:
hidden_layers = []
else:
hidden_layers = to_list(hidden_layers)
# check for copy constructor.
if all([x in kwargs for x in ('inputs', 'outputs', 'layers')]):
self._inputs = kwargs['inputs'].copy()
self._outputs = kwargs['outputs'].copy()
self._layers = kwargs['layers'].copy()
self._set_model()
return
# prepare kernel initializers.
activations, def_biasinit, def_kerinit = \
prepare_default_activations_and_initializers(
len(hidden_layers) * [activation] + [output_activation]
)
if kernel_initializer is None:
kernel_initializer = def_kerinit
elif isinstance(kernel_initializer, (float, int)):
kernel_initializer = default_weight_initializer(
len(hidden_layers) * [activation] + [output_activation],
'constant',
scale=kernel_initializer)
else:
kernel_initializer = [
kernel_initializer for l in len(hidden_layers) * [activation] +
[output_activation]
]
# prepare bias initializers.
if bias_initializer is None:
bias_initializer = def_biasinit
elif isinstance(bias_initializer, (float, int)):
bias_initializer = default_weight_initializer(
len(hidden_layers) * [activation] + [output_activation],
'constant',
scale=bias_initializer)
else:
bias_initializer = [
bias_initializer for l in len(hidden_layers) * [activation] +
[output_activation]
]
# prepare regularizers.
kernel_regularizer = default_regularizer(kernel_regularizer)
bias_regularizer = default_regularizer(bias_regularizer)
# prepares fields.
fields = to_list(fields)
if all([isinstance(fld, str) for fld in fields]):
output_fields = [
RNNField(
name=fld,
dtype=dtype,
kernel_initializer=kernel_initializer[-1],
bias_initializer=bias_initializer[-1],
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
trainable=trainable,
) for fld in fields
]
elif all([validations.is_field(fld) for fld in fields]):
output_fields = fields
else:
raise TypeError('Please provide a "list" of field names of' +
' type "String" or "Field" objects.')
# prepare inputs/outputs/layers.
inputs = []
layers = []
variables = to_list(variables)
if all([isinstance(var, RNNFunctional) for var in variables]):
for var in variables:
inputs += var.outputs
# for var in variables:
# for lay in var.layers:
# layers.append(lay)
else:
raise TypeError(
"Input error: Please provide a `list` of `Functional`s. \n"
"Provided - {}".format(variables))
# prepare hidden layers.
if hidden_layers is None:
hidden_layers = []
else:
hidden_layers = to_list(hidden_layers)
# Check and convert activation functions to proper format.
assert not isinstance(activation, list), \
'Expected an activation function name not a "list". '
afunc = get_activation(activation)
# Input layers.
if len(inputs) == 1:
net_input = inputs[0]
else:
layer = Concatenate(name=graph_unique_name('conct'))
net_input = layer(inputs)
# Define the networks.
net = [net_input]
assert len(
hidden_layers) > 0, 'Minimum of 1 RNN hidden layer is needed.'
# Adding hidden layers
for nLay, nNeuron in enumerate(hidden_layers):
if nLay < 1000:
# First layer starts with RNN.
if rnn_type == 'LSTM':
layer = LSTM(nNeuron,
return_sequences=True,
recurrent_activation=recurrent_activation,
kernel_initializer=kernel_initializer[nLay],
bias_initializer=bias_initializer[nLay],
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
trainable=trainable,
dtype=dtype,
unroll=True,
name=graph_unique_name(
"LSTM{:d}b_".format(nNeuron)))
elif rnn_type == 'SimpleRNN':
layer = SimpleRNN(
nNeuron,
return_sequences=True,
kernel_initializer=kernel_initializer[nLay],
bias_initializer=bias_initializer[nLay],
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
trainable=trainable,
dtype=dtype,
unroll=True,
name=graph_unique_name("SRNN{:d}b_".format(nNeuron)))
else:
raise ValueError('Invalid entry for `rnn_type` -- '
'accepts from (`SimpleRNN`, `LSTM`).')
else:
# Add the dense layer.
layer = Dense(nNeuron,
kernel_initializer=kernel_initializer[nLay],
bias_initializer=bias_initializer[nLay],
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
trainable=trainable,
dtype=dtype,
name=graph_unique_name("D{:d}b".format(nNeuron)))
layers.append(layer)
net[-1] = layer(net[-1])
# Apply the activation.
if afunc.__name__ != 'linear':
layer = activations[nLay]
layers.append(layer)
net[-1] = layer(net[-1])
# store output layers.
for out in output_fields:
layers.append(out)
# Assign to the output variable
if len(net) == 1:
net_output = net[0]
else:
raise ValueError("Legacy for Enrichment: Must be updated. ")
layer = Concatenate(name=graph_unique_name("{}_".format("conct")))
net_output = layer(net)
# check output activation functions.
output_func = get_activation(output_activation)
# Define the final outputs of each network
outputs = []
for out in output_fields:
# add the activation on the output.
if output_func.__name__ != 'linear':
layer = activations[-1]
layers.append(layer)
outputs.append(layer(out(net_output)))
else:
outputs.append(out(net_output))
self._inputs = inputs
self._outputs = outputs
self._layers = layers
self._set_model()
# Copyright 2020 Graphcore Ltd.
"""Helper utility to download imagenet weights for Densenet model."""
from pathlib import Path
from typing import List
import numpy as np
import tensorflow.compat.v1 as tf
from tensorflow.contrib.framework.python.framework.checkpoint_utils import \
list_variables, load_variable
from tensorflow.python.keras import backend as keras_backend
from tensorflow.python.keras.applications.densenet import DenseNet121
keras_backend.set_floatx('float16')
def get_densenet_weights(save_dir: Path = Path('densenet_121')) -> Path:
"""Download pre-trained imagenet weights for densenet model.
Args:
save_dir: Path to where checkpoint must be downloaded.
Returns: Path to checkpoint file.
"""
g = tf.Graph()
with tf.Session(graph=g) as sess:
keras_backend.set_session(sess)
save_dir.mkdir(parents=True, exist_ok=True)
_ = DenseNet121(weights='imagenet')
saver = tf.train.Saver()
def predict(hparams,
model_dir, checkpoint_path, output_dir,
test_source_files, test_target_files):
if hparams.half_precision:
backend.set_floatx(tf.float16.name)
backend.set_epsilon(1e-4)
audio = Audio(hparams)
def predict_input_fn():
source = tf.data.TFRecordDataset(list(test_source_files))
target = tf.data.TFRecordDataset(list(test_target_files))
dataset = dataset_factory(source, target, hparams)
batched = dataset.prepare_and_zip().group_by_batch(
batch_size=1).move_mel_to_source()
return batched.dataset
estimator = model_factory(hparams, model_dir, None)
predictions = map(
lambda p: PredictedMel(p["id"], p["key"], p["mel"], p.get("mel_postnet"), p["mel"].shape[1], p["mel"].shape[0],
p["ground_truth_mel"],
p["alignment"], p.get("alignment2"), p.get("alignment3"), p.get("alignment4"),
p.get("alignment5"), p.get("alignment6"), p.get("attention2_gate_activation"),
p["source"], p["text"], p.get("accent_type"), p),
estimator.predict(predict_input_fn, checkpoint_path=checkpoint_path))
for v in predictions:
key = v.key.decode('utf-8')
mel_filename = f"{key}.{hparams.predicted_mel_extension}"
mel_filepath = os.path.join(output_dir, mel_filename)
ground_truth_mel = v.ground_truth_mel.astype(np.float32)
predicted_mel = v.predicted_mel.astype(np.float32)
mel_denormalized = audio.denormalize_mel(predicted_mel)
linear_spec = audio.logmelspc_to_linearspc(mel_denormalized)
wav = audio.griffin_lim(linear_spec)
audio.save_wav(wav, os.path.join(output_dir, f"{key}.wav"))
assert mel_denormalized.shape[1] == hparams.num_mels
mel_denormalized.tofile(mel_filepath, format='<f4')
text = v.text.decode("utf-8")
plot_filename = f"{key}.png"
plot_filepath = os.path.join(output_dir, plot_filename)
alignments = [x.astype(np.float32) for x in [
v.alignment, v.alignment2, v.alignment3, v.alignment4, v.alignment5, v.alignment6] if x is not None]
if hparams.model == "SSNTModel":
ssnt_metrics.save_alignment_and_log_probs([v.alignment],
[v.all_fields["log_emit_and_shift_probs"]],
[v.all_fields["log_output_prob"]],
[None],
text, v.key, 0,
os.path.join(output_dir, f"{key}_probs.png"))
ssnt_metrics.write_prediction_result(v.id, key, text,
v.all_fields["log_emit_and_shift_probs"],
v.all_fields["log_output_prob"],
os.path.join(output_dir, f"{key}_probs.tfrecord"))
plot_predictions(alignments, ground_truth_mel, predicted_mel, text, v.key, plot_filepath)
prediction_filename = f"{key}.tfrecord"
prediction_filepath = os.path.join(output_dir, prediction_filename)
write_prediction_result(v.id, key, alignments, mel_denormalized, audio.denormalize_mel(ground_truth_mel),
text, v.source, prediction_filepath)
# %%
num_meta_tasks = 20
num_test_tasks = 5
num_datapoints = 5
meta, test = generate_meta_and_test_tasks(num_datapoints, num_meta_tasks, num_test_tasks)
# %% [markdown]
# ## Create the mean function
# We will use a Keras model Deep Neural Network as mean function.
# %%
from tensorflow.python.keras import backend as K
from gpflow.config import default_float
K.set_floatx("float64")
assert default_float() == np.float64
def build_mean_function():
inputs = tf.keras.layers.Input(shape=(1,))
x = tf.keras.layers.Dense(64, activation="relu")(inputs)
x = tf.keras.layers.Dense(64, activation="relu")(x)
outputs = tf.keras.layers.Dense(1)(x)
return tf.keras.Model(inputs=inputs, outputs=outputs)
# %% [markdown]
# ## Build the GP metamodel
# Metalearning boils down to learning a good prior that can generalize to new tasks with a small number of data points. This framework is prevalent in GP modeling, where we usually maximize the marginal likelihood to learn a good set of hyperparameters that specify the GP prior.
#
import tensorflow.compat.v1 as tf
import numpy as np
import tensorflow.python.keras.backend as K
from keras.models import load_model
from werkzeug.utils import secure_filename, redirect
import cv2
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
tf.disable_v2_behavior()
app = Flask(__name__)
config = tf.ConfigProto(device_count={'GPU': 0})
sess = tf.Session(config=config)
graph = tf.get_default_graph()
set_session(sess)
K.set_floatx('float16')
#model = load_model('/home/code_broom/PycharmProjects/DistractedDrivers/Models/class_7/class_7_flask_trial.h5')
@app.route('/', methods=['GET', 'POST'])
def main_page():
if request.method == 'POST':
file = request.files['file']
filename = secure_filename(file.filename)
file.save(os.path.join('uploads', filename))
return redirect(url_for('prediction', filename=filename))
return render_template('index.html')
@app.route('/prediction/<filename>')
def prediction(filename):
def train_ctl(model_func, params):
image_width = params['image_width']
image_height = params['image_height']
image_format = params['image_format']
distort_color = params['distort_color']
momentum = params['momentum']
loss_scale = params['loss_scale']
data_dir = params['data_dir']
data_idx_dir = params['data_idx_dir']
batch_size = params['batch_size']
num_iter = params['num_iter']
iter_unit = params['iter_unit']
log_dir = params['log_dir']
export_dir = params['export_dir']
tensorboard_dir = params['tensorboard_dir']
display_every = params['display_every']
precision = params['precision']
dali_mode = params['dali_mode']
use_xla = params['use_xla']
if data_dir is not None:
file_format = os.path.join(data_dir, '%s-*')
train_files = sorted(tf.io.gfile.glob(file_format % 'train'))
valid_files = sorted(tf.io.gfile.glob(file_format % 'validation'))
num_train_samples = common.get_num_records(train_files)
num_valid_samples = common.get_num_records(valid_files)
else:
num_train_samples = 1281982
num_valid_samples = 5000
train_idx_files = None
valid_idx_files = None
if data_idx_dir is not None:
file_format = os.path.join(data_idx_dir, '%s-*')
train_idx_files = sorted(tf.io.gfile.glob(file_format % 'train'))
valid_idx_files = sorted(tf.io.gfile.glob(file_format % 'validation'))
if iter_unit.lower() == 'epoch':
num_epochs = num_iter
nstep_per_epoch = num_train_samples // (batch_size * hvd.size())
nstep_per_valid = num_valid_samples // (batch_size * hvd.size())
else:
assert iter_unit.lower() == 'batch'
num_epochs = 1
nstep_per_epoch = min(num_iter,
num_train_samples // (batch_size * hvd.size()))
nstep_per_valid = min(10,
num_valid_samples // (batch_size * hvd.size()))
if export_dir:
assert os.path.exists(export_dir)
save_format = export_dir + "/saved_model_rn50.h5"
if use_xla:
tf.config.optimizer.set_jit(True)
# Horovod: pin GPU to be used to process local rank (one GPU per process)
gpus = tf.config.experimental.list_physical_devices('GPU')
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
if gpus:
tf.config.experimental.set_visible_devices(gpus[hvd.local_rank()],
'GPU')
if tensorboard_dir and hvd.rank() == 0:
assert os.path.exists(tensorboard_dir)
summary_writer = tf.summary.create_file_writer(tensorboard_dir)
else:
summary_writer = None
if precision == 'fp16':
if tf.__version__ >= "2.4.0":
policy = keras.mixed_precision.Policy('mixed_float16')
keras.mixed_precision.set_global_policy(policy)
else:
policy = keras.mixed_precision.experimental.Policy(
'mixed_float16', loss_scale)
keras.mixed_precision.experimental.set_policy(policy)
lr_schedule = common.create_piecewise_constant_decay_with_warmup(
batch_size=batch_size * hvd.size(),
epoch_size=num_train_samples,
warmup_epochs=common.LR_SCHEDULE[0][1],
boundaries=list(p[1] for p in common.LR_SCHEDULE[1:]),
multipliers=list(p[0] for p in common.LR_SCHEDULE),
compute_lr_on_cpu=True)
opt = keras.optimizers.SGD(learning_rate=lr_schedule, momentum=momentum)
backend.set_image_data_format(image_format)
dtype = 'float16' if precision == 'fp16' else 'float32'
backend.set_floatx(dtype)
model = model_func(num_classes=image_processing.NUM_CLASSES,
batch_size=batch_size)
loss_func = keras.losses.SparseCategoricalCrossentropy()
train_top1 = tf.keras.metrics.SparseTopKCategoricalAccuracy(
k=1, name='train_top1')
train_top5 = tf.keras.metrics.SparseTopKCategoricalAccuracy(
k=5, name='train_top5')
val_loss = tf.keras.metrics.Mean(name='val_loss', dtype=tf.float32)
val_top1 = tf.keras.metrics.SparseTopKCategoricalAccuracy(k=1,
name='val_top1')
val_top5 = tf.keras.metrics.SparseTopKCategoricalAccuracy(k=5,
name='val_top5')
if log_dir:
# We save check points only when using the real data.
assert data_dir, "--data_dir cannot be empty when using --log_dir"
assert os.path.exists(log_dir)
ckpt = tf.train.Checkpoint(epoch=tf.Variable(0),
optimizer=opt,
net=model)
manager = tf.train.CheckpointManager(ckpt,
log_dir,
max_to_keep=3,
checkpoint_name="model-ckpt")
@tf.function
def train_step(inputs, first_batch):
images, labels = inputs
with tf.GradientTape() as tape:
predictions = model(images, training=True)
loss = loss_func(labels, predictions)
loss += tf.reduce_sum(model.losses)
loss_copy = loss
# Scale the losses
if precision == 'fp16':
loss = loss * tf.cast(loss_scale, loss.dtype)
tape = hvd.DistributedGradientTape(tape)
old_grads = tape.gradient(loss, model.trainable_variables)
# Unscale the grads
if precision == 'fp16':
loss_scale_reciprocal = 1. / loss_scale
grads = [
g * tf.cast(loss_scale_reciprocal, g.dtype)
if g is not None else None for g in old_grads
]
else:
grads = old_grads
opt.apply_gradients(zip(grads, model.trainable_variables))
train_top1.update_state(labels, predictions)
train_top5.update_state(labels, predictions)
if hvd.size() > 1 and first_batch:
hvd.broadcast_variables(model.variables, root_rank=0)
hvd.broadcast_variables(opt.variables(), root_rank=0)
return loss_copy
@tf.function
def valid_step(inputs):
images, labels = inputs
predictions = model(images, training=False)
loss = loss_func(labels, predictions)
val_loss.update_state(loss)
val_top1.update_state(labels, predictions)
val_top5.update_state(labels, predictions)
if data_dir is not None:
num_preproc_threads = 4 if dali_mode else 10
train_input = image_processing.image_set(
train_files,
batch_size,
image_height,
image_width,
training=True,
distort_color=distort_color,
deterministic=False,
num_threads=num_preproc_threads,
use_dali=dali_mode,
idx_filenames=train_idx_files)
valid_input = image_processing.image_set(
valid_files,
batch_size,
image_height,
image_width,
training=False,
distort_color=False,
deterministic=False,
num_threads=num_preproc_threads,
use_dali=dali_mode,
idx_filenames=valid_idx_files)
else:
if dali_mode:
raise ValueError("Must provide --data_dir if Dali is enabled")
else:
train_input = image_processing.fake_image_set(
batch_size, image_height, image_width)
global_steps = 0
log_steps = display_every
try:
initial_epoch = 0
if log_dir:
ckpt.restore(manager.latest_checkpoint)
if manager.latest_checkpoint:
if hvd.rank() == 0:
print("Restored from {}".format(manager.latest_checkpoint))
initial_epoch = max(
int(re.findall(r'\d+', manager.latest_checkpoint)[0]),
initial_epoch)
else:
if hvd.rank() == 0:
print("Initializing from scratch.")
# Training Loop
for epoch in range(num_epochs):
if epoch < initial_epoch:
continue
# on_epoch_begin
epoch_start = time.time()
total_loss = 0.0
num_batches = 0
train_top1.reset_states()
train_top5.reset_states()
if not dali_mode:
train_iter = iter(train_input)
for _ in range(nstep_per_epoch):
# on_batch_begin
global_steps += 1
if global_steps == 1:
start_time = time.time()
if global_steps == 1 and hvd.rank() == 0 and summary_writer:
tf.summary.trace_on(graph=True, profiler=True)
if not dali_mode:
x = next(train_iter)
else:
x = train_input.get_device_minibatches()
total_loss += train_step(x, global_steps == 1)
if global_steps == 1 and hvd.rank() == 0 and summary_writer:
with summary_writer.as_default():
tf.summary.trace_export(
name="train_step",
step=0,
profiler_outdir=tensorboard_dir)
# on_batch_end
if global_steps % log_steps == 0:
timestamp = time.time()
elapsed_time = timestamp - start_time
examples_per_second = \
(batch_size * hvd.size() * log_steps) / elapsed_time
if hvd.rank() == 0:
print("global_step: %d images_per_sec: %.1f" %
(global_steps, examples_per_second))
start_time = timestamp
num_batches += 1
train_loss = total_loss / num_batches
# on_epoch_end
epoch_run_time = time.time() - epoch_start
if hvd.rank() == 0:
print("epoch: %d time_taken: %.1f" % (epoch, epoch_run_time))
if data_dir is not None:
val_loss.reset_states()
val_top1.reset_states()
val_top5.reset_states()
if not dali_mode:
test_iter = iter(valid_input)
for _ in range(nstep_per_valid):
if not dali_mode:
x = next(test_iter)
else:
x = valid_input.get_device_minibatches()
valid_step(x)
if log_dir:
ckpt.epoch.assign_add(1)
if hvd.rank() == 0:
save_path = manager.save()
print("Saved checkpoint for epoch {}: {}".format(
int(ckpt.epoch), save_path))
if hvd.rank() == 0:
output_str = (
"loss: {} - top1: {} - top5: {} - val_loss: {} - "
"val_top1: {} - val_top5: {}")
print(
output_str.format(train_loss, train_top1.result(),
train_top5.result(), val_loss.result(),
val_top1.result(), val_top5.result()))
if hvd.rank() == 0 and summary_writer:
with summary_writer.as_default():
tf.summary.scalar('train_loss', train_loss, global_steps)
tf.summary.scalar('train_top1', train_top1.result(),
global_steps)
tf.summary.scalar('train_top5', train_top5.result(),
global_steps)
tf.summary.scalar('val_loss', val_loss.result(),
global_steps)
tf.summary.scalar('val_top1', val_top1.result(),
global_steps)
tf.summary.scalar('val_top5', val_top5.result(),
global_steps)
if hvd.rank() == 0 and summary_writer:
summary_writer.close()
except KeyboardInterrupt:
print("Keyboard interrupt")
if export_dir and hvd.rank() == 0:
model.save(save_format)
print(f"The model is saved to {save_format}")
from tensorflow.python.keras.utils import layer_utils
from tensorflow.python.keras.utils.data_utils import get_file
from tensorflow.python.keras.applications.imagenet_utils import preprocess_input
from tensorflow.python.keras.datasets import cifar10
# import pydot
# from IPython.display import SVG
from tensorflow.python.keras.utils.vis_utils import model_to_dot
from tensorflow.python.keras.utils import plot_model
from tensorflow.python.keras.initializers import glorot_uniform
# import scipy.misc
# from matplotlib.pyplot import imshow
# # %matplotlib inline
from tensorflow.python.keras import backend as K
K.set_image_data_format('channels_last')
K.set_learning_phase(1)
K.set_floatx('float32')
import os
import json
def identity_block(X, f, filters, stage, block):
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
F1, F2, F3 = filters
X_shortcut = X
X = Conv2D(filters=F1,
kernel_size=(1, 1),
strides=(1, 1),
# %%
num_meta_tasks = 20
num_test_tasks = 5
num_datapoints = 5
meta, test = generate_meta_and_test_tasks(num_datapoints, num_meta_tasks,
num_test_tasks)
# %% [markdown]
# ## Create the mean function
# We will use a Keras model Deep Neural Network as mean function.
# %%
from tensorflow.python.keras import backend as K
from gpflow.config import default_float
K.set_floatx('float64')
assert default_float() == np.float64
def build_mean_function():
inputs = tf.keras.layers.Input(shape=(1, ))
x = tf.keras.layers.Dense(64, activation="relu")(inputs)
x = tf.keras.layers.Dense(64, activation="relu")(x)
outputs = tf.keras.layers.Dense(1)(x)
return tf.keras.Model(inputs=inputs, outputs=outputs)
# %% [markdown]
# ## Build the GP metamodel
# Metalearning boils down to learning a good prior that can generalize to new tasks with a small number of data points. This framework is prevalent in GP modeling, where we usually maximize the marginal likelihood to learn a good set of hyperparameters that specify the GP prior.
#
import tensorflow as tf
tf.compat.v1.enable_eager_execution()
import tensorflow.python.keras.backend as K
dtype = 'float16'
K.set_floatx(dtype)
K.set_epsilon(1e-4)
# import numpy as np
import os, glob
# os.environ['TF_ENABLE_AUTO_MIXED_PRECISION'] = '1'
from shutil import copy
from utils.logger import set_logger
from dataset_generator import DataGenerator
from Dataset.digest_train_csv import Digestive
from tensorflow.python.keras.callbacks import ModelCheckpoint, CSVLogger, TensorBoard, EarlyStopping, ReduceLROnPlateau
from segcaps import CapsNet, Container
from competition_losses_metrics import *
from config import conf
class SteelSeg:
def __init__(self, conf):
self.conf = conf
self.data_reader = DataGenerator(conf,
Digestive(conf).masks_at_least(1))
self.model = self.build()
# if reload_step is not zero it means we are reading data from already existing folders
self.log_dir, self.model_dir, self.save_dir = self.set_dirs()
def __init__(self,
fields=None,
variables=None,
hidden_layers=None,
activation="tanh",
output_activation="linear",
res_net=False,
kernel_initializer=None,
bias_initializer=None,
kernel_regularizer=None,
bias_regularizer=None,
dtype=None,
trainable=True,
**kwargs):
# check data-type.
if dtype is None:
dtype = K.floatx()
elif not K.floatx() == dtype:
K.set_floatx(dtype)
# prepare hidden layers.
if hidden_layers is None:
hidden_layers = []
else:
hidden_layers = to_list(hidden_layers)
# check for copy constructor.
if all([x in kwargs for x in ('inputs', 'outputs', 'layers')]):
self._inputs = kwargs['inputs'].copy()
self._outputs = kwargs['outputs'].copy()
self._layers = kwargs['layers'].copy()
self._set_model()
return
# prepare kernel initializers.
activations, def_biasinit, def_kerinit = \
prepare_default_activations_and_initializers(
len(hidden_layers) * [activation] + [output_activation]
)
if kernel_initializer is None:
kernel_initializer = def_kerinit
elif isinstance(kernel_initializer, (float, int)):
kernel_initializer = default_weight_initializer(
len(hidden_layers) * [activation] + [output_activation],
'constant',
scale=kernel_initializer)
else:
kernel_initializer = [
kernel_initializer for l in len(hidden_layers) * [activation] +
[output_activation]
]
# prepare bias initializers.
if bias_initializer is None:
bias_initializer = def_biasinit
elif isinstance(bias_initializer, (float, int)):
bias_initializer = default_weight_initializer(
len(hidden_layers) * [activation] + [output_activation],
'constant',
scale=bias_initializer)
else:
bias_initializer = [
bias_initializer for l in len(hidden_layers) * [activation] +
[output_activation]
]
# prepare regularizers.
kernel_regularizer = default_regularizer(kernel_regularizer)
bias_regularizer = default_regularizer(bias_regularizer)
# prepares fields.
fields = to_list(fields)
if all([isinstance(fld, str) for fld in fields]):
output_fields = [
Field(
name=fld,
dtype=dtype,
kernel_initializer=kernel_initializer[-1],
bias_initializer=bias_initializer[-1],
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
trainable=trainable,
) for fld in fields
]
elif all([validations.is_field(fld) for fld in fields]):
output_fields = fields
else:
raise TypeError('Please provide a "list" of field names of' +
' type "String" or "Field" objects.')
# prepare inputs/outputs/layers.
inputs = []
layers = []
variables = to_list(variables)
if all([isinstance(var, Functional) for var in variables]):
for var in variables:
inputs += var.outputs
# for var in variables:
# for lay in var.layers:
# layers.append(lay)
else:
raise TypeError(
"Input error: Please provide a `list` of `Functional`s. \n"
"Provided - {}".format(variables))
# Input layers.
if len(inputs) == 1:
net_input = inputs[0]
else:
layer = Concatenate(name=graph_unique_name('conct'))
net_input = layer(inputs)
# Define the output network.
net = [net_input]
# define the ResNet networks.
if res_net is True:
res_layers = []
res_outputs = []
for rl in ["U", "V", "H"]:
layers.append(
Dense(hidden_layers[0],
kernel_initializer=kernel_initializer[0],
bias_initializer=bias_initializer[0],
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
trainable=trainable,
dtype=dtype,
name=graph_unique_name(
"DRes" + rl + "{:d}b".format(hidden_layers[0]))))
res_output = layers[-1](net_input)
# Apply the activation.
if activations[0].activation.__name__ != 'linear':
layers.append(activations[0])
res_outputs.append(layers[-1](res_output))
net[-1] = res_outputs[-1]
for nLay, nNeuron in enumerate(hidden_layers):
# Add the layer.
layer = Dense(nNeuron,
kernel_initializer=kernel_initializer[nLay],
bias_initializer=bias_initializer[nLay],
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
trainable=trainable,
dtype=dtype,
name=graph_unique_name("D{:d}b".format(nNeuron)))
layers.append(layer)
net[-1] = layer(net[-1])
# Apply the activation.
if activations[
nLay].activation.__name__ != 'linear': #nLay<len(hidden_layers)-1 and
layer = activations[nLay]
layers.append(layer)
net[-1] = layer(net[-1])
# Add the resnet layer
if res_net is True:
layer = Lambda(lambda xs: (1 - xs[0]) * xs[1] + xs[0] * xs[2],
name=graph_unique_name("ResLayer"))
net[-1] = layer([net[-1]] + res_outputs[:2])
# store output layers.
for out in output_fields:
layers.append(out)
# Assign to the output variable
if len(net) == 1:
net_output = net[0]
else:
raise ValueError("Legacy for Enrichment: Must be updated. ")
layer = Concatenate(name=graph_unique_name('conct'))
net_output = layer(net)
# Define the final outputs of each network
outputs = []
for out in output_fields:
# add the activation on the output.
if activations[-1].activation.__name__ != 'linear':
layer = activations[-1]
layers.append(layer)
outputs.append(layer(out(net_output)))
else:
outputs.append(out(net_output))
self._inputs = inputs
self._outputs = outputs
self._layers = layers
self._set_model()
