def _build_models(self, batch_size, embedding_size, rnn_size, num_layers):
model = Sequential()
model.add(
Embedding(self.vectorizer.vocab_size,
embedding_size,
batch_input_shape=(batch_size, None)))
for layer in range(num_layers):
model.add(LSTM(rnn_size, stateful=True, return_sequences=True))
model.add(Dropout(0.2))
model.add(
TimeDistributed(
Dense(self.vectorizer.vocab_size, activation='softmax')))
# With sparse_categorical_crossentropy we can leave as labels as
# integers instead of one-hot vectors
model.compile(loss='sparse_categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
model.summary()
# Keep a separate model with batch_size 1 for sampling
self.train_model = model
config = model.get_config()
config['layers'][0]['config']['batch_input_shape'] = (1, None)
self.sample_model = Sequential.from_config(config)
self.sample_model.trainable = False
def create_dropout_predict_function(model, dropout):
"""
Hard-codes a dropout rate to the Dropout layers of a trained model.
When initially training model, Dropout layers must have training=True,
otherwise dropout will not be applied to predictions.
Parameters:
model : trained keras model
dropout : dropout rate to apply to all Dropout layers
Returns:
predict_with_dropout : keras model that will apply dropout when making predictions
"""
# Load the config of the original model
conf = model.get_config()
# Add the specified dropout to all layers
for layer in conf['layers']:
# Dropout layers
if layer["class_name"]=="Dropout":
layer["config"]["rate"] = dropout
# Recurrent layers with dropout
elif "dropout" in layer["config"].keys():
layer["config"]["dropout"] = dropout
# Create a new model with specified dropout
if type(model)==Sequential:
# Sequential
model_dropout = Sequential.from_config(conf)
else:
# Functional
model_dropout = Model.from_config(conf)
model_dropout.set_weights(model.get_weights())
return model_dropout
def export_model_to_tensorflow(path_to_trained_keras_model: str):
print("Loading model for exporting to Protocol Buffer format...")
model = tensorflow.keras.models.load_model(path_to_trained_keras_model)
sess = tensorflow.keras.backend.get_session()
# serialize the model and get its weights, for quick re-building
config = model.get_config()
weights = model.get_weights()
# re-build a model where the learning phase is now hard-coded to 0
new_model = Sequential.from_config(config)
new_model.set_weights(weights)
export_path = os.path.abspath(os.path.join("export", "simple")) # where to save the exported graph
os.makedirs(export_path)
checkpoint_state_name = "checkpoint_state"
export_version = 1 # version number (integer)
saver = tensorflow.train.Saver(sharded=True, name=checkpoint_state_name)
model_exporter = exporter.Exporter(saver)
signature = exporter.classification_signature(input_tensor=model.input, scores_tensor=model.output)
# # Version 1 of exporter
# model_exporter.init(sess.graph.as_graph_def(), default_graph_signature=signature)
# model_exporter.export(export_path, tensorflow.constant(export_version), sess)
#
# # Version 2 of exporter
# tensorflow.train.write_graph(sess.graph.as_graph_def(), logdir=".", name="simple.pbtxt", as_text=True)
# Version 3 with Freezer from https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/tools/freeze_graph_test.py
input_graph_name = "input_graph.pb"
output_graph_name = "output_graph.pb"
saver_write_version = saver_pb2.SaverDef.V2
# We'll create an input graph that has a single variable containing 1.0,
# and that then multiplies it by 2.
saver = tensorflow.train.Saver(write_version=saver_write_version)
checkpoint_path = saver.save(sess, export_path, global_step=0, latest_filename=checkpoint_state_name)
graph_io.write_graph(sess.graph, export_path, input_graph_name)
# We save out the graph to disk, and then call the const conversion
# routine.
input_graph_path = os.path.join(export_path, input_graph_name)
input_saver_def_path = ""
input_binary = False
output_node_names = "output_node/Softmax"
restore_op_name = "save/restore_all"
filename_tensor_name = "save/Const:0"
output_graph_path = os.path.join(export_path, output_graph_name)
clear_devices = False
freeze_graph.freeze_graph(input_graph_path, input_saver_def_path,
input_binary, checkpoint_path, output_node_names,
restore_op_name, filename_tensor_name,
output_graph_path, clear_devices, "")
shutil.copy(os.path.join("export", "simple", "output_graph.pb"), output_graph_name)
shutil.rmtree("export")
print("Exported model: {0}".format(os.path.abspath(output_graph_name)))
def train_keras_tensorflow(model_param, train_param, data, verbose=0):
#sess = tf.Session()
backend.clear_session()
numpy.random.seed(train_param.np_random_seed)
tf.set_random_seed(train_param.tf_random_seed)
data_x = data.x_data
data_y = data.y_data
with tf.Session() as sess:
cb = []
if train_param.early_stopping != {}:
cb.append(
tf.keras.callbacks.EarlyStopping(**train_param.early_stopping))
if train_param.tensorboard['log_dir'] != None:
tb_param = copy.deepcopy(train_param.tensorboard)
tb_param['log_dir'] = tb_param['log_dir'] + '/' + dt.datetime.now(
).strftime("%Y%m%dT%H%M%S")
cb.append(
tf.keras.callbacks.TensorBoard(profile_batch=0, **tb_param))
#reduce_lr = ReduceLROnPlateau(monitor = 'loss', factor=0.8, patience=100, min_lr=0.00001)
# fix random seed for reproducibility
model = Sequential.from_config(model_param.param)
logger.info("Compiling model.")
model.compile(loss=train_param.loss,
optimizer=train_param.get_optimizer())
logger.info("Start training, epochs: " + str(train_param.epochs) +
", batch_size: " + str(train_param.batch_size))
history = model.fit(data_x,
data_y,
epochs=train_param.epochs,
batch_size=train_param.batch_size,
callbacks=cb,
verbose=verbose,
validation_split=train_param.validation_split)
logger.info("Finished training")
h = None
# sess.close()
if 'loss' in history.history:
loss = numpy.asarray(history.history['loss'])
return TensorflowKerasModel(model.get_config(),
model.get_weights(),
repo_info={}), TensorflowKerasHistory(
loss, repo_info={})
else:
return TensorflowKerasModel(model.get_config(),
model.get_weights(),
repo_info={})
def clone_model(model):
clone = Sequential.from_config(model.get_config())
clone.set_weights(model.get_weights())
return clone
def target_train_tpu_convert_main(input_file, output_weight_file,
output_json_file):
with CustomObjectScope({'RandomLayer': RandomLayer}):
input_model = load_model(input_file)
input_model.save_weights(output_weight_file)
config = input_model.get_config()
config['layers'][0]['config']['batch_input_shape'] = (None, None, 139)
config['layers'][0]['config']['layer']['config'][
'kernel_initializer'] = 'zeros'
config['layers'][1]['config']['kernel_initializer'] = 'zeros'
config['layers'][4]['config']['layer']['config'][
'kernel_initializer'] = 'zeros'
config['layers'][5]['config']['kernel_initializer'] = 'zeros'
config['layers'][6]['config']['target_shape'] = (-1, 64)
config['layers'][9]['config']['layer']['config'][
'kernel_initializer'] = 'zeros'
config['layers'][10]['config']['kernel_initializer'] = 'zeros'
config['layers'][11]['config']['target_shape'] = (-1, 32)
config['layers'][14]['config']['layer']['config'][
'kernel_initializer'] = 'zeros'
config['layers'][15]['config']['kernel_initializer'] = 'zeros'
config['layers'][16]['config']['target_shape'] = (-1, 16)
config['layers'][19]['config']['layer']['config'][
'kernel_initializer'] = 'zeros'
config['layers'][20]['config']['kernel_initializer'] = 'zeros'
model = Sequential.from_config(config)
json_dic = json.loads(model.to_json())
del json_dic['config']['layers'][0]['config']['layer']['config'][
'time_major']
del json_dic['config']['layers'][0]['config']['layer']['config'][
'zero_output_for_mask']
json_dic['config']['layers'][2]['class_name'] = 'BatchNormalization'
json_dic['config']['layers'][2]['config']['axis'] = 2
del json_dic['config']['layers'][4]['config']['layer']['config'][
'time_major']
del json_dic['config']['layers'][4]['config']['layer']['config'][
'zero_output_for_mask']
json_dic['config']['layers'][7]['class_name'] = 'BatchNormalization'
json_dic['config']['layers'][7]['config']['axis'] = 2
del json_dic['config']['layers'][9]['config']['layer']['config'][
'time_major']
del json_dic['config']['layers'][9]['config']['layer']['config'][
'zero_output_for_mask']
json_dic['config']['layers'][12]['class_name'] = 'BatchNormalization'
json_dic['config']['layers'][12]['config']['axis'] = 2
del json_dic['config']['layers'][14]['config']['layer']['config'][
'time_major']
del json_dic['config']['layers'][14]['config']['layer']['config'][
'zero_output_for_mask']
json_dic['config']['layers'][17]['class_name'] = 'BatchNormalization'
json_dic['config']['layers'][17]['config']['axis'] = 2
del json_dic['config']['layers'][19]['config']['layer']['config'][
'time_major']
del json_dic['config']['layers'][19]['config']['layer']['config'][
'zero_output_for_mask']
with open(output_json_file, 'w') as f:
json.dump(json_dic, f)
def convert(prevmodel, export_path, export_name):
graph1 = tf.Graph()
with graph1.as_default():
sess1 = tf.Session()
with sess1.as_default():
previous_model = load_model(prevmodel)
previous_model.summary()
config = previous_model.get_config()
weights = previous_model.get_weights()
graph2 = tf.Graph()
with graph2.as_default():
sess2 = tf.Session()
with sess2.as_default():
K.set_learning_phase(0)
try:
model = Sequential.from_config(config)
except:
model = Model.from_config(config)
model.set_weights(weights)
with open(
os.path.join(export_path,
export_name + '.input_output.txt'),
'w') as fid:
model_input_name = model.input.name
#model_input_node_name
model_output_name = model.output.name
print(f"Input name: {model_input_name}")
print(f"Output name: {model_output_name}")
fid.write(f"{model_input_name}\n")
fid.write(f"{model_output_name}\n")
model_output_node_name = model.output.name.split(':')[0]
graph_file = os.path.join(export_path,
export_name + ".graph.pbtxt")
ckpt_file = os.path.join(export_path, export_name + ".ckpt")
saver = tf.train.Saver()
tf.train.write_graph(sess2.graph_def, '', graph_file)
save_path = saver.save(sess2, ckpt_file)
# freeze the graph
frozen_graph_def = tf.graph_util.convert_variables_to_constants(
sess2, # The session is used to retrieve the weights
graph2.as_graph_def(
), # The graph_def is used to retrieve the nodes
model_output_node_name.split(
","
) # The output node names are used to select the usefull nodes
)
# Finally we serialize and dump the output graph to the filesystem
frozen_graph_path = os.path.join(export_path,
export_name + ".frozen.pb")
with tf.gfile.GFile(frozen_graph_path, "wb") as f:
f.write(frozen_graph_def.SerializeToString())
#tf.reset_default_graph()
#frozen_graph_def = freeze_graph(export_path, model_output_node_name)
tf.reset_default_graph()
train_writer = tf.summary.FileWriter(export_path)
train_writer.add_graph(frozen_graph_def)
train_writer.flush()
# optimize for inference
optimized_graph_def = optimize_for_inference_lib.optimize_for_inference(
frozen_graph_def, [model_input_name.split(':')[0]],
[model_output_name.split(':')[0]], tf.float32.as_datatype_enum)
#tf.int32.as_datatype_enum
optimized_graph_path = os.path.join(
export_path, export_name + ".optimized_for_inference.pb")
with tf.gfile.GFile(optimized_graph_path, "wb") as f:
f.write(optimized_graph_def.SerializeToString())
# tflite
# ## from frozen graph
converter = tflite.TocoConverter.from_frozen_graph(
optimized_graph_path, [model_input_name.split(':')[0]],
[model_output_name.split(':')[0]],
{model_input_name.split(':')[0]: [1, 32, 32, 3]})
#
## from keras model file
# only available in tensorflow >=1.11
#converter = tflite.TocoConverter.from_keras_model_file(prevmodel)
converter.post_training_quantize = True
#converter.inference_type = tf.quint8
#converter.inference_input_type = tf.float32
tflite_quantized_model = converter.convert()
optimized_graph_path = os.path.join(export_path, export_name + ".tflite")
open(optimized_graph_path, "wb").write(tflite_quantized_model)
def build_pruned_model(model, new_model_param, layer_types, num_new_neurons,
num_new_filters, comp):
"""
The new number of neurons and filters are changed in the model config.
Load the new weight matrices into the model.
Args:
model: Model which should be pruned
new_model_param: Stores the new weights of the model
layer_types: The types of all layers of the model
num_new_neurons: Number of neurons of the dense layers
num_new_filters: Number of filters of the conv layers
Return:
pruned_model: New model after pruning all dense and conv layers
"""
model_config = model.get_config()
'''
For functional model first layer is the input layer.
For sequential model the first layer is the layer after the input layer
'''
a = 1
if layer_types[0] == 'InputLayer':
a = 0
for i in range(0, len(model_config['layers']) - 3):
if model_config['layers'][i + a][
'class_name'] == "Dense": #i+1 because first layer of model is the inputlayer
print("Dense")
model_config['layers'][i +
a]['config']['units'] = num_new_neurons[i]
elif model_config['layers'][i + a]['class_name'] == "Conv2D":
print("Conv2D")
model_config['layers'][i +
a]['config']['filters'] = num_new_filters[i]
elif model_config['layers'][i + a]['class_name'] == "Reshape":
temp_list = list(
model_config['layers'][i + a]['config']['target_shape'])
cur_layer = i
cur_filters = num_new_filters[cur_layer]
#Get number of filters of last Conv layer
if cur_filters == 0:
while cur_filters == 0:
cur_layer -= 1
cur_filters = num_new_filters[cur_layer]
temp_list[2] = cur_filters
temp_tuple = tuple(temp_list)
model_config['layers'][i +
a]['config']['target_shape'] = temp_tuple
else:
print("No Dense or Conv2D")
print("Before pruning:")
model.summary()
if "Sequential" in str(model):
pruned_model = Sequential.from_config(model_config)
elif "Functional" in str(model):
pruned_model = Model.from_config(model_config)
print("After pruning:")
pruned_model.summary()
for i in range(0, len(pruned_model.layers)):
if len(new_model_param[i]) != 0:
pruned_model.layers[i].set_weights(new_model_param[i])
else:
None
pruned_model.compile(**comp)
return pruned_model
def target_train_tpu_main(gen_targets_dir,
model_file_path,
early_stopping_patience=None,
length=None,
batch_size=1,
period=1,
retrain_file=None,
retrain_do_compile=False,
base_model_file_path='target_common.h5',
optimizer=Adam(),
optimizer_lr=0.001,
epochs=100000):
gc.collect()
with CustomObjectScope({'RandomLayer': RandomLayer}):
if retrain_file is None:
gen = VoiceGeneratorTargetTpu(gen_targets_dir,
0.1,
batch_size,
length,
train=True)
shape0 = gen[0][0].shape[1]
val_gen = VoiceGeneratorTargetTpu(gen_targets_dir,
0.1,
train=False,
max_size=shape0)
model = load_model(base_model_file_path)
config = model.get_config()
config['layers'][0]['config']['batch_input_shape'] = (None, shape0,
139)
config['layers'][3]['config']['rate'] = 0.1
config['layers'][6]['config']['target_shape'] = (shape0 * 2, 64)
config['layers'][8]['config']['rate'] = 0.1
config['layers'][11]['config']['target_shape'] = (shape0 * 4, 32)
config['layers'][13]['config']['rate'] = 0.1
config['layers'][16]['config']['target_shape'] = (shape0 * 8, 16)
config['layers'][18]['config']['rate'] = 0.1
model = Sequential.from_config(config)
model.load_weights(base_model_file_path, by_name=True)
model.summary()
model.compile(loss='mse', optimizer=optimizer)
baseline = None
else:
model = load_model(retrain_file)
if retrain_do_compile:
model.compile(loss='mse', optimizer=optimizer)
config = model.get_config()
shape0 = config['layers'][0]['config']['batch_input_shape'][1]
gen = VoiceGeneratorTargetTpu(gen_targets_dir,
0.1,
batch_size,
length,
train=True,
max_size=shape0)
val_gen = VoiceGeneratorTargetTpu(gen_targets_dir,
0.1,
train=False,
max_size=shape0)
baseline = model.test_on_batch(val_gen[0][0], val_gen[0][1])
tpu_grpc_url = 'grpc://' + os.environ['COLAB_TPU_ADDR']
tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
tpu_grpc_url)
strategy = keras_support.TPUDistributionStrategy(tpu_cluster_resolver)
model = tf.contrib.tpu.keras_to_tpu_model(model, strategy=strategy)
cp = ModelCheckpoint(filepath=model_file_path,
monitor='val_loss',
save_best_only=True,
period=period)
if baseline is not None:
cp.best = baseline
def lr_scheduler(epoch):
return optimizer_lr
scheduler = LearningRateScheduler(lr_scheduler)
if early_stopping_patience is not None:
es = EarlyStopping(monitor='val_loss',
patience=early_stopping_patience,
verbose=0,
mode='auto',
baseline=baseline)
callbacks = [es, cp, scheduler]
else:
callbacks = [cp, scheduler]
model.fit_generator(gen,
shuffle=True,
epochs=epochs,
verbose=1,
callbacks=callbacks,
validation_data=val_gen)
K.clear_session()
def get_model(self):
model = Sequential.from_config(self.keras_model_config)
model.set_weights(self.model_weights)
return model
def f_convert_model(model, optimizer, L_W, L_A, custom_obj, verbose=False):
cfg = model.get_config()
cfg_reg = {
'class_name': 'discretize_reg',
'config': {
"FL": L_W,
"lg": 1e-4,
"l2": 0
}
}
weights_new = []
idx1, idx2 = 0, 0
while cfg['layers'][idx1]['class_name'] != 'GlobalAveragePooling2D':
if cfg['layers'][idx1]['class_name'] == 'Conv2dNorm':
cfg_conv2d = deepcopy(cfg['layers'][idx1])
cfg_conv2d['config']['kernel_regularizer'] = cfg_reg
cfg_conv2d['config']['bias_regularizer'] = cfg_reg
cfg_conv2d['class_name'] = 'Conv2D'
cfg_conv2d['config']['name'] = 'conv2d_new_' + str(idx2)
del cfg_conv2d['config']['momentum'], cfg_conv2d['config'][
'max_scale'], cfg_conv2d['config']['L_W'], cfg_conv2d[
'config']['L_A']
cfg_aktiv = {
'class_name': 'ActivationLayer_relu',
'config': {
"L_A": L_A
}
}
del cfg['layers'][idx1]
cfg['layers'].insert(idx1, cfg_aktiv)
cfg['layers'].insert(idx1, cfg_conv2d)
weights_tmp = model.layers[idx2].get_weights()
weights_new.append(weights_tmp[0] * weights_tmp[2])
weights_new.append(weights_tmp[1] * weights_tmp[2])
idx1 += 1
elif cfg['layers'][idx1]['class_name'] == 'Conv2D':
cfg['layers'][idx1]['config']['kernel_regularizer'] = cfg_reg
cfg['layers'][idx1]['config']['bias_regularizer'] = cfg_reg
weights_tmp = model.layers[idx2].get_weights()
for wei in weights_tmp:
weights_new.append(wei)
elif cfg['layers'][idx1]['class_name'] == 'SepConv2DNorm':
cfg_sepcon = deepcopy(cfg['layers'][idx1]['config'])
cfg_depthw = {
'class_name': 'DepthwiseConv2D',
'config': {
"depthwise_initializer":
cfg_sepcon['depthwise_initializer'],
'depthwise_regularizer': cfg_reg,
'depthwise_constraint': cfg_sepcon['depthwise_constraint'],
'name': 'sepconv2d_new_' + str(idx2),
'strides': cfg_sepcon['strides'],
'trainable': True,
'kernel_size': (3, 3),
'use_bias': False,
'padding': 'same',
}
}
cfg_aktiv = {
'class_name': 'ActivationLayer_relu',
'config': {
"L_A": L_A
}
}
cfg_conv2d = {
'class_name': 'Conv2D',
'config': {
"kernel_initializer": cfg_sepcon['kernel_initializer'],
"bias_initializer": cfg_sepcon['bias_initializer'],
'kernel_constraint': cfg_sepcon['kernel_constraint'],
'bias_constraint': cfg_sepcon['bias_constraint'],
'kernel_regularizer': cfg_reg,
'bias_regularizer': cfg_reg,
'name': 'conv2d_new_' + str(idx2),
'filters': cfg_sepcon['filters'],
'kernel_size': (1, 1),
'padding': 'same',
'strides': cfg_sepcon['strides'],
'trainable': True,
'use_bias': True
}
}
del cfg['layers'][idx1]
cfg['layers'].insert(idx1, cfg_aktiv)
cfg['layers'].insert(idx1, cfg_conv2d)
cfg['layers'].insert(idx1, cfg_aktiv)
cfg['layers'].insert(idx1, cfg_depthw)
weights_tmp = model.layers[idx2].get_weights()
weights_new.append(weights_tmp[0])
weights_new.append(weights_tmp[1] * weights_tmp[3])
weights_new.append(weights_tmp[2] * weights_tmp[3])
idx1 += 3
else:
weights_tmp = model.layers[idx2].get_weights()
for wei in weights_tmp:
weights_new.append(wei)
idx1 += 1
idx2 += 1
model_conv = Sequential.from_config(cfg, custom_objects=custom_obj)
model_conv.set_weights(weights_new)
model_conv.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['acc', cat_crossentropy_from_logits])
if verbose == True:
model_conv.summary()
return model_conv
