def CRNN(n_classes,
batch_size,
optimizer_type="sgd",
training=True,
lr=.001,
reg=1e-6,
dropout_chance=0.2,
cnn_weights_path=None,
compile_model=True):
images = Input(shape=(32, 128, 1), name='images')
#x = Dropout(0.2)(images)
# extract patches of 32hx16w with stride 2
x = Lambda(identity_conv, arguments={
'patch_size': (32, 16),
'stride': 2
})(images)
num_windows = x.shape[1]
x = Reshape((num_windows, 32, 16, 1))(x)
# expand width-wise 32x16 to 32x32
x = TimeDistributed(Lambda(expander),
name="expander")(x) # (None, 32, 32, 1)
# =============================================================================
# # pad width-wise with 0's to expand 32x16 to 32x32
# x = TimeDistributed(Lambda(padder), name="padder")(x) # (None, 32, 32, 1)
# =============================================================================
# =============================================================================
# # expand 32x16 to 32x32 deconvolution
# x = TimeDistributed(Conv2DTranspose(filters=1,
# kernel_size=3,
# strides=(1,2),
# padding='same'), name='expand_patch')(x) # (None, 32, 32, 1)
# =============================================================================
cnn = CNN(n_classes - 1, reg=reg, compile_model=False)
if cnn_weights_path:
cnn.load_weights(cnn_weights_path)
cnn = Model(inputs=cnn.inputs, outputs=cnn.layers[-2].output)
#cnn.trainable = False
x = TimeDistributed(cnn, name='convnet')(x) # (None, num_windows, 512)
# =============================================================================
# # CNN to RNN
# x = Conv1D(filters=256,
# kernel_size=3,
# kernel_initializer="he_normal",
# padding="same")(x) # (None, num_windows, 256)
# =============================================================================
# RNN
x = Bidirectional(
GRU(units=256, return_sequences=True,
kernel_initializer="he_normal"))(x) # (None, num_windows, 512)
x = BatchNormalization()(x)
# =============================================================================
# x = Bidirectional(
# GRU(units=256,
# return_sequences=True,
# kernel_initializer="he_normal"))(x) # (None, num_windows, 512)
# x = BatchNormalization()(x)
# =============================================================================
y_pred = Dense(n_classes, activation="softmax",
name="logits_layer")(x) # (None, num_windows, 63)
model = Model(inputs=images, outputs=y_pred)
logit_length = [[model.layers[-1].output_shape[1]]] * batch_size
# Optimizer
if optimizer_type == "sgd":
optimizer = SGD(lr=lr, momentum=0.9, decay=0.01)
elif optimizer_type == "nesterov_sgd":
optimizer = SGD(lr=lr, momentum=0.9, decay=0.01, nesterov=True)
elif optimizer_type == "adam":
optimizer = Adam(lr=lr)
if compile_model:
model.compile(loss=CTCLoss(logit_length=logit_length),
optimizer=optimizer,
metrics=[LevenshteinMetric(batch_size=batch_size)])
print(model.summary())
return model
class TensorflowClient(object):
def __init__(self, iden, X, y):
self.iden = iden
self.X = X
self.y = y
# TODO: Should be randomized.
cut_off = int(X.shape[0] * 0.8)
self.X_train = X[:cut_off]
self.y_train = y[:cut_off]
self.X_test = X[cut_off:]
self.y_test = y[cut_off:]
def setup_model(self, model_type):
self.model_type = model_type
if model_type == "perceptron":
self.model = Perceptron()
elif model_type == "cnn-mnist":
self.model = CNN()
elif model_type == "cnn-cifar10":
self.model = CNN()
else:
raise ValueError("Model {0} not supported.".format(model_type))
def train(self, weights, config):
logging.info('Training just started.')
assert weights != None, 'weights must not be None.'
assert config["averaging_type"] in ["data_size", "val_acc"]
if config["averaging_type"] == "data_size":
X, y = self.X, self.y
elif config["averaging_type"] == "val_acc":
X, y = self.X_train, self.y_train
batch_size = X.shape[0] \
if config["batch_size"] == -1 else config["batch_size"]
epochs = config["epochs"]
learning_rate = config["learning_rate"]
params = {'new_weights': weights, 'learning_rate': learning_rate}
classifier = tf.estimator.Estimator(
model_fn=self.model.get_model,
model_dir=self.get_checkpoints_folder(),
params=params)
train_input_fn = tf.estimator.inputs.numpy_input_fn(
x={"x": X},
y=y,
batch_size=batch_size,
num_epochs=epochs,
shuffle=True)
classifier.train(input_fn=train_input_fn)
logging.info('Training complete.')
new_weights = self.model.get_weights(self.get_latest_checkpoint())
if config["averaging_type"] == "data_size":
omega = X.shape[0]
elif config["averaging_type"] == "val_acc":
eval_classifier = tf.estimator.Estimator(
model_fn=self.model.get_model,
model_dir=self.get_checkpoints_folder(),
params={'new_weights': new_weights})
eval_input_fn = tf.estimator.inputs.numpy_input_fn(
x={"x": self.X_test},
y=self.y_test,
num_epochs=1,
shuffle=False)
eval_results = eval_classifier.evaluate(input_fn=eval_input_fn)
omega = eval_results["accuracy"]
shutil.rmtree("./checkpoints-{0}/".format(self.iden))
return new_weights, omega
def validate(self, t, weights, config):
# check if this is needed
model_type = config["model_type"]
self.setup_model(model_type)
classifier = tf.estimator.Estimator(
model_fn=self.model.get_model,
model_dir=self.get_checkpoints_folder(),
params={
'new_weights': weights,
'learning_rate': 0.0
})
train_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x": self.X},
y=self.y,
batch_size=1,
num_epochs=None,
shuffle=False)
classifier.train(input_fn=train_input_fn, steps=1)
metagraph_file = self.get_checkpoints_folder() + '.meta'
self.model.load_weights(weights, self.get_latest_checkpoint(),
self.get_checkpoints_folder())
logging.info('Main model updated.')
self.setup_model(model_type)
classifier = tf.estimator.Estimator(
model_fn=self.model.get_model,
model_dir=self.get_checkpoints_folder(),
params={'new_weights': weights})
eval_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x": self.X},
y=self.y,
num_epochs=1,
shuffle=False)
eval_results = classifier.evaluate(input_fn=eval_input_fn)
logging.info("[Round {0}] Validation results: {1}".format(
t, eval_results))
return eval_results
def get_initial_weights(self, model_type):
tf.reset_default_graph()
if model_type == "perceptron":
m = Perceptron()
inputs = tf.placeholder(tf.float32, shape=(None, 28 * 28))
_ = m.get_model(features={"x": inputs},
labels=None,
mode='predict',
params=None)
elif model_type == 'cnn-mnist':
m = CNN()
inputs = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))
_ = m.get_model(features={"x": inputs},
labels=None,
mode='predict',
params=None)
elif model_type == 'cnn-cifar10':
m = CNN()
inputs = tf.placeholder(tf.float32, shape=(None, 32, 32, 3))
_ = m.get_model(features={"x": inputs},
labels=None,
mode='predict',
params=None)
else:
raise ValueError(
"Model {model_type} not supported.".format(model_type))
with tf.Session().as_default() as sess:
sess.run(tf.global_variables_initializer())
collection = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
weights = {tensor.name: sess.run(tensor) for tensor in collection}
tf.reset_default_graph()
return weights
def get_checkpoints_folder(self):
return "./checkpoints-{0}/{1}/".format(self.iden, self.model_type)
def get_latest_checkpoint(self):
return tf.train.latest_checkpoint(self.get_checkpoints_folder())
patches = tf.reshape(patches, (patches.shape[1], 32, 16, 1))
print('patches reshape', patches.shape)
# =============================================================================
# for i in range(patches.shape[0]):
# show_img(patches[i])
# =============================================================================
cnn = CNN(62, reg=3e-4, compile_model=True)
# =============================================================================
# cnn_weights_path = '../checkpoints/cnn_best_weights/' + \
# 'epoch.158_val_loss.0.628467.h5'
# =============================================================================
cnn_weights_path = '../checkpoints/cnn_best_weights/' + \
'epoch.152_val_loss.0.600990.h5'
cnn.load_weights(cnn_weights_path)
#cnn = Model(inputs=cnn.inputs, outputs=cnn.layers[-2].output)
# =============================================================================
# lenet = LeNet(62, reg=3e-4, compile_model=True)
# lenet_weights_path = '../checkpoints/cnn_best_weights/' + \
# 'lenet_epoch.276_val_loss.1.301563.h5'
# lenet.load_weights(lenet_weights_path)
# =============================================================================
probs = []
word = []
# =============================================================================
# # padding test
# for i in range(patches.shape[0]):
# t = patches[i]