def test_confusion_matrix(self):
from sklearn.metrics import confusion_matrix
y1 = np.random.randint(0, 8, size=100)
y2 = np.random.randint(0, 8, size=100)
y_pred = K.variable(y1)
y_true = K.variable(y2)
confusion = K.confusion_matrix(y_pred, y_true)
r1 = K.eval(confusion)
r2 = confusion_matrix(y1, y2)
self.assertEqual(np.sum(r1 - r2), 0.)
N.Flatten(outdim=2),
N.Dense(256, activation=K.relu),
N.Dense(10, activation=K.softmax)
],
debug=True)
ops = cPickle.loads(cPickle.dumps(ops)) # test if the ops is pickle-able
K.set_training(True)
y_pred_train = ops(X)
K.set_training(False)
y_pred_score = ops(X)
cost_train = K.mean(K.categorical_crossentropy(y_pred_train, y))
cost_test_1 = K.mean(K.categorical_crossentropy(y_pred_score, y))
cost_test_2 = K.mean(K.categorical_accuracy(y_pred_score, y))
cost_test_3 = K.confusion_matrix(y_pred_score, y, labels=range(10))
parameters = ops.parameters
optimizer = K.optimizers.SGD(lr=arg['lr'])
updates = optimizer(cost_train, parameters)
print('Building training functions ...')
f_train = K.function([X, y], [cost_train, optimizer.norm], updates=updates)
print('Building testing functions ...')
f_test = K.function([X, y], [cost_test_1, cost_test_2, cost_test_3])
print('Building predicting functions ...')
f_pred = K.function(X, y_pred_score)
# ===========================================================================
# Build trainer
# ===========================================================================
print('Start training ...')
def standard_trainer(train_data,
valid_data,
X,
y_train,
y_score,
y_target,
parameters,
test_data=None,
cost_train=None,
cost_score=None,
optimizer=None,
confusion_matrix=False,
gradient_norm=True,
save_path=None,
save_obj=None,
batch_size=64,
nb_epoch=3,
valid_freq=0.6,
seed=1208,
shuffle_level=2,
patience=3,
earlystop=5,
report_path=None):
"""
Parameters
----------
cost_train: list of callable
each function will be apply to a pair y_train and y_target
Return
------
MainLoop, and History
Note
----
"""
from odin import backend as K
# ====== prepare variables and cost ====== #
# check optimizer
if optimizer is None:
optimizer = K.optimizers.SGD(lr=0.0001, momentum=0.9, nesterov=True)
elif not isinstance(optimizer, K.optimizers.Optimizer) and \
not hasattr(optimizer, "get_updates"):
raise ValueError(
"Invalid optimizer, the optimizer must be instance of "
"backend.optimizers.Optimizer or having function "
"get_updates(self, loss_or_grads, params).")
# check the cost functions
if cost_train is None:
cost_train = K.categorical_crossentropy
if cost_score is None:
cost_score = K.categorical_crossentropy
cost_train = as_tuple(cost_train)
cost_score = as_tuple(cost_score)
# check input X, y, parameters
X = as_tuple(X)
y_train = as_tuple(y_train)
y_score = as_tuple(y_score)
y_target = as_tuple(y_target)
parameters = as_tuple(parameters)
if len(X) == 0 or len(y_train) == 0 or len(y_score) == 0 or \
len(y_target) == 0 or len(parameters) == 0:
raise ValueError(
"X(len=%d), y_train(len=%d), y_score(len=%d), y_target(len=%d),"
"and parameters(len=%d) must be list or tuple with length > 0." %
(len(X), len(y_train), len(y_score), len(y_target),
len(parameters)))
# get all cost
if len(y_train) == 1:
y_train = y_train * len(cost_train)
if len(y_score) == 1:
y_score = y_score * len(cost_score)
cost_train = [
K.mean(f_cost(y_, y), axis=0) for f_cost, y_, y in zip(
cost_train, y_train,
y_target * len(cost_train) if len(y_target) == 1 else y_target)
]
cost_score = [
K.mean(f_cost(y_, y), axis=0) for f_cost, y_, y in zip(
cost_score, y_score,
y_target * len(cost_score) if len(y_target) == 1 else y_target)
]
# add confusion matrix
if confusion_matrix:
if not is_number(confusion_matrix) and \
not isinstance(confusion_matrix, (tuple, list, np.ndarray)):
raise ValueError(
"confusion_matrix must be an integer, or list, tuple"
" specifies number of classes, or list of all classes.")
if is_number(confusion_matrix):
confusion_matrix = list(range(int(confusion_matrix)))
for y_, y in zip(y_score, y_target):
cost_score.append(
K.confusion_matrix(y_pred=y_,
y_true=y,
labels=confusion_matrix))
# get the update
updates = optimizer.get_updates(cost_train[0], parameters)
# ====== create function ====== #
grad_norm = [] if not gradient_norm or not hasattr(optimizer, 'norm') else \
[optimizer.norm]
cost_train = cost_train + grad_norm
print('Building training functions ...')
f_train = K.function(inputs=X + y_target,
outputs=cost_train,
updates=updates)
print('Building scoring functions ...')
f_score = K.function(inputs=X + y_target, outputs=cost_score)
# ====== Create trainer ====== #
task = MainLoop(batch_size=batch_size,
seed=seed,
shuffle_level=shuffle_level)
if save_path is not None and save_obj is not None:
task.set_save(save_path, save_obj, save_hist=True)
# set task
task.set_task(f_train, train_data, epoch=nb_epoch, name='train')
task.set_subtask(f_score, valid_data, freq=valid_freq, name='valid')
if test_data is not None:
task.set_subtask(f_score, test_data, when=-1, epoch=1, name='test')
# format for score
score_format = 'Results:' + __format_string(
len(cost_score) - (1 if confusion_matrix else 0))
score_tracking = {
(len(cost_score) - 1): lambda x: sum(x)
} if confusion_matrix else []
# set the callback
history = History()
task.set_callback([
ProgressMonitor(name='train',
format='Results:' + __format_string(len(cost_train))),
ProgressMonitor(name='valid',
format=score_format,
tracking=score_tracking),
(ProgressMonitor(
name='test', format=score_format, tracking=score_tracking)
if test_data is not None else None), history,
EarlyStopGeneralizationLoss(
'valid',
threshold=earlystop,
patience=patience,
get_value=lambda x: np.mean([i[0] for i in x]
if isinstance(x[0],
(tuple, list)) else x)),
NaNDetector(('train', 'valid'), patience=patience, rollback=True)
])
return task, history
# Build network
# ===========================================================================
ops = N.Sequence([
N.Flatten(outdim=2),
N.Dense(512, activation=K.relu),
N.Dense(256, activation=K.relu),
N.Dense(10, activation=K.softmax)
])
ops = cPickle.loads(cPickle.dumps(ops)) # test if the ops is pickle-able
y_pred_train = ops(X_train)
y_pred_score = ops(X_score)
cost_train = K.mean(K.categorical_crossentropy(y_pred_train, y))
cost_test_1 = K.mean(K.categorical_crossentropy(y_pred_score, y))
cost_test_2 = K.mean(K.categorical_accuracy(y_pred_score, y))
cost_test_3 = K.confusion_matrix(y_pred_score, y, labels=range(10))
parameters = ops.parameters
optimizer = K.optimizers.RMSProp(lr= 0.0001, clipnorm=100.)
updates = optimizer(cost_train, parameters)
print('Building training functions ...')
f_train = K.function([X_train, y], [cost_train, optimizer.norm],
updates=updates)
print('Building testing functions ...')
f_test = K.function([X_score, y], [cost_test_1, cost_test_2, cost_test_3])
# ====== normalize 0-1 ====== #
if False:
print('Normalized data in range [0-1]')
X_train = ds['X_train'][:]
X_train = (X_train - np.min(X_train, 0)) / (np.max(X_train) - np.min(X_train))
