def train_classifier_depricated(tpu=False):
scope = strategy.scope()
print("Number of accelerators: ", strategy.num_replicas_in_sync)
data = mnist_data()
backbone = ResNet()
discriminator = Discriminator(backbone)
classifier = Classifier(backbone, 10)
preds = classifier.predict(data.get_test()[0])
classifier.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
classifier.summary()
if (tpu):
classifier = convert_model_for_tpu(classifier)
checkpoint = keras.callbacks.ModelCheckpoint(
'./checkpoints/classifier/classifier_{epoch:.2f}.h5',
monitor='val_loss',
verbose=0,
save_best_only=False,
save_weights_only=True)
# classifier.fit(x=x_train,y=y_train,batch_size=6000,epochs=1, validation_data=(x_vali,y_vali),callbacks=[checkpoint])
classifier.fit(x=data.get_n_samples(35)[0],
y=data.get_n_samples(35)[1],
batch_size=6000,
epochs=20,
validation_data=data.get_vali(),
callbacks=[checkpoint])
# import pdb; pdb.set_trace() # breakpoint 396fe169 //
backbone = classifier.get_backbone()
backbone.save_weights('backbone_weights.h5')
return (classifier, x_test)
class MnistTrainer(object):
def __init__(
self,
input_shape,
output_dim,
patience=4,
structure='wide_res_net',
):
self.model = None
if structure == 'wide_res_net':
self.model = WideResNet(input_shape=input_shape,
output_dim=output_dim)
elif structure == 'res_net':
self.model = ResNet(input_shape=input_shape, output_dim=output_dim)
else:
raise Exception('no structure')
self.criterion = tf.keras.losses.CategoricalCrossentropy()
self.optimizer = tf.keras.optimizers.SGD(learning_rate=0.1)
self.train_loss = tf.keras.metrics.Mean()
self.train_acc = tf.keras.metrics.CategoricalAccuracy()
self.val_loss = tf.keras.metrics.Mean()
self.val_acc = tf.keras.metrics.CategoricalAccuracy()
self.history = {
'train_loss': [],
'val_loss': [],
'train_acc': [],
'val_acc': []
}
self.es = {'loss': float('inf'), 'patience': patience, 'step': 0}
self.save_dir = './logs'
if not os.path.exists(self.save_dir):
os.mkdir('logs')
def train(self, data_loader, epochs, batch_size, early_stopping=False):
n_batches_val = data_loader.val_size // batch_size
x_val = data_loader.x_val
t_val = data_loader.t_val
for epoch in range(epochs):
x_, t_ = data_loader.get_train_data()
self.train_loss.reset_states()
self.val_loss.reset_states()
self.train_acc.reset_states()
self.val_acc.reset_states()
for batch in tqdm(range(n_batches_train)):
start = batch * batch_size
end = start + batch_size
x_batch = x_[start:end]
t_batch = t_[start:end]
self.train_step(x_batch, t_batch)
for batch in tqdm(range(n_batches_val)):
start = batch * batch_size
end = start + batch_size
x_batch = x_val[start:end]
t_batch = t_val[start:end]
self.val_step(x_batch, t_batch)
self.history['train_loss'].append(self.train_loss.result())
self.history['val_loss'].append(self.val_loss.result())
self.history['train_acc'].append(self.train_acc.result())
self.history['val_acc'].append(self.val_acc.result())
print(
'epoch {} => train_loss: {}, train_acc: {}, val_loss: {}, val_acc: {}'
.format(epoch + 1, self.train_loss.result(),
self.train_acc.result(), self.val_loss.result(),
self.val_acc.result()))
if early_stopping:
if self.early_stopping(self.val_loss.result()):
break
fig = plt.figure(figsize=(20, 20))
ax1 = fig.add_subplot(2, 2, 1)
ax2 = fig.add_subplot(2, 2, 2)
ax3 = fig.add_subplot(2, 2, 3)
ax4 = fig.add_subplot(2, 2, 4)
ax1.plot(self.history['train_loss'])
ax2.plot(self.history['train_acc'])
ax3.plot(self.history['val_loss'])
ax4.plot(self.history['val_acc'])
ax1.set_title('train_loss')
ax2.set_title('train_acc')
ax3.set_title('val_loss')
ax4.set_title('val_acc')
plt.show()
def train_step(self, x, t):
with tf.GradientTape() as tape:
preds = self.model(x)
loss = self.criterion(t, preds)
grads = tape.gradient(loss, self.model.trainable_variables)
self.optimizer.apply_gradients(
zip(grads, self.model.trainable_variables))
self.train_loss(loss)
self.train_acc(t, preds)
def val_step(self, x, t):
preds = self.model(x)
loss = self.criterion(t, preds)
self.val_loss(loss)
self.val_acc(t, preds)
def evaluate(self, x_test, t_test):
accuracy = tf.metrics.CategoricalAccuracy()
preds = self.model(x_test)
loss = self.criterion(preds, t_test)
accuracy(t_test, preds)
print('accuracy: {}, loss: {}'.format(accuracy.result(), loss))
return (accuracy.result().numpy(), loss.numpy())
def save(self, name):
path = self.save_dir + '/' + name
self.model.save_weights(path)
def load(self, name):
path = self.save_dir + '/' + name
self.model.load_weights(path)
def early_stopping(self, loss):
if loss > self.es['loss']:
self.es['step'] += 1
if self.es['step'] > self.es['patience']:
print('early stopping')
self.load('early_stopping_saving')
return True
else:
self.es['loss'] = loss
self.es['step'] = 0
self.save('early_stopping_saving')
return False
class Trainer(object):
def __init__(self,
input_shape=(128, 128, 3),
output_dim=10,
patience=5,
structure='wide_res_net',
loss='categorical_crossentropy',
name='latest'):
self.model = None
if structure == 'wide_res_net':
self.model = WideResNet(input_shape=input_shape,
output_dim=output_dim)
elif structure == 'res_net':
self.model = ResNet(input_shape=input_shape, output_dim=output_dim)
else:
raise Exception('no structure')
if loss == 'categorical_crossentropy':
self.criterion = tf.keras.losses.CategoricalCrossentropy()
elif loss == 'emd':
self.criterion = EMD
self.optimizer = tf.keras.optimizers.SGD(learning_rate=0.1,
momentum=0.1)
self.train_acc = tf.keras.metrics.CategoricalAccuracy()
self.train_loss = tf.keras.metrics.Mean()
self.val_acc = tf.keras.metrics.CategoricalAccuracy()
self.val_loss = tf.keras.metrics.Mean()
self.history = {
'train_acc': [],
'train_loss': [],
'val_acc': [],
'val_loss': [],
'start': 0,
}
self.es = {'loss': float('inf'), 'patience': patience, 'step': 0}
self.save_dir = './logs'
if not os.path.exists(self.save_dir):
os.mkdir('logs')
self.name = name
def train(self,
data_loader,
epochs,
batch_size,
image_path,
early_stopping=False):
n_batches_val = data_loader.val_size // batch_size
x_val = data_loader.x_val
t_val = data_loader.t_val
start = self.history['start']
for epoch in range(start, epochs):
self.history['start'] = epoch
x_, t_ = data_loader.get_train_data()
n_batches_train = x_.shape[0] // batch_size
self.train_acc.reset_states()
self.train_loss.reset_states()
self.val_acc.reset_states()
self.val_loss.reset_states()
for batch in tqdm(range(n_batches_train)):
start = batch * batch_size
end = start + batch_size
x_batch = x_[start:end]
t_batch = t_[start:end]
img_batch = self.get_image(image_path, x_batch)
self.train_step(img_batch, t_batch)
for batch in tqdm(range(n_batches_val + 1)):
start = batch * batch_size
end = start + batch_size
end = min(end, x_val.shape[0])
x_batch = x_val[start:end]
t_batch = t_val[start:end]
img_batch = self.get_image(image_path, x_batch)
self.val_step(img_batch, t_batch)
self.history['train_loss'].append(self.train_loss.result().numpy())
self.history['val_loss'].append(self.val_loss.result().numpy())
self.history['train_acc'].append(self.train_acc.result().numpy())
self.history['val_acc'].append(self.val_acc.result().numpy())
print(
'epoch {} => train_loss: {}, train_acc: {}, val_loss: {}, val_acc: {}'
.format(epoch + 1, self.train_loss.result(),
self.train_acc.result(), self.val_loss.result(),
self.val_acc.result()))
if self.early_stopping(self.val_loss.result()):
if early_stopping:
break
self.save(self.name)
self.plot()
def get_image(self, image_path, x_batch, standard=255):
img_batch = list()
for id in x_batch:
image = np.load(image_path + '/' + str(id) + '.npy')
img_batch.append(image / standard)
return np.array(img_batch)
def train_step(self, x, t):
with tf.GradientTape() as tape:
preds = self.model(x)
loss = self.criterion(t, preds)
grads = tape.gradient(loss, self.model.trainable_variables)
self.optimizer.apply_gradients(
zip(grads, self.model.trainable_variables))
self.train_loss(loss)
self.train_acc(t, preds)
def val_step(self, x, t):
preds = self.model(x)
loss = self.criterion(t, preds)
self.val_loss(loss)
self.val_acc(t, preds)
def evaluate(self, x_test, t_test, batch_size=1000):
loss = tf.keras.metrics.Mean()
accuracy = tf.keras.metrics.CategoricalAccuracy()
n_batches = x_test.shape[0] // batch_size
for batch in tqdm(range(n_batches)):
start = batch_size * batch
end = start + batch_size
x_batch = x_test[start:end]
t_batch = t_test[start:end]
img_batch = self.get_image(x_batch)
preds = self.model(img_batch)
loss(self.criterion(t_batch, preds))
accuracy(t_batch, preds)
start = batch_size * n_batches
end = x_test.shape[0]
x_batch = x_test[start:end]
t_batch = t_test[start:end]
img_batch = self.get_image(x_batch)
preds = self.model(img_batch)
loss(self.criterion(t_batch, preds))
accuracy(t_batch, preds)
print('loss: {}, accuracy: {}'.format(loss.result(),
accuracy.result()))
return (loss.result().numpy(), accuracy.result().numpy())
def early_stopping(self, loss):
name = self.name + '_es'
if loss > self.es['loss']:
self.es['step'] += 1
if self.es['step'] > self.es['patience']:
print('early stopping')
return True
else:
self.es['loss'] = loss
self.es['step'] = 0
self.save(name)
return False
def save(self, name):
path = self.save_dir + '/' + name
self.model.save_weights(path)
dic = {'history': self.history, 'es': self.es}
with open(self.save_dir + '/' + name + '.json', 'w') as f:
json.dump(dic, f)
def load(self, name):
path = self.save_dir + '/' + name
self.model.load_weights(path)
with open(self.save_dir + '/' + name + '.json', 'r') as f:
data = f.read()
dic = json.load(data)
self.hisotory = dic['history']
self.es = dic['es']
def plot(self):
fig = plt.figure(figsize=(10, 10))
ax1 = fig.add_subplot(2, 2, 1)
ax2 = fig.add_subplot(2, 2, 2)
ax3 = fig.add_subplot(2, 2, 3)
ax4 = fig.add_subplot(2, 2, 4)
ax1.plot(self.history['train_loss'])
ax2.plot(self.history['train_acc'])
ax3.plot(self.history['val_loss'])
ax4.plot(self.history['val_acc'])
ax1.set_title('train_loss')
ax2.set_title('train_acc')
ax3.set_title('val_loss')
ax4.set_title('val_acc')
plt.show()
