def test_confusion_matrix(self): from sklearn.metrics import confusion_matrix y1 = np.random.randint(0, 8, size=100) y2 = np.random.randint(0, 8, size=100) y_pred = K.variable(y1) y_true = K.variable(y2) confusion = K.confusion_matrix(y_pred, y_true) r1 = K.eval(confusion) r2 = confusion_matrix(y1, y2) self.assertEqual(np.sum(r1 - r2), 0.)
N.Flatten(outdim=2), N.Dense(256, activation=K.relu), N.Dense(10, activation=K.softmax) ], debug=True) ops = cPickle.loads(cPickle.dumps(ops)) # test if the ops is pickle-able K.set_training(True) y_pred_train = ops(X) K.set_training(False) y_pred_score = ops(X) cost_train = K.mean(K.categorical_crossentropy(y_pred_train, y)) cost_test_1 = K.mean(K.categorical_crossentropy(y_pred_score, y)) cost_test_2 = K.mean(K.categorical_accuracy(y_pred_score, y)) cost_test_3 = K.confusion_matrix(y_pred_score, y, labels=range(10)) parameters = ops.parameters optimizer = K.optimizers.SGD(lr=arg['lr']) updates = optimizer(cost_train, parameters) print('Building training functions ...') f_train = K.function([X, y], [cost_train, optimizer.norm], updates=updates) print('Building testing functions ...') f_test = K.function([X, y], [cost_test_1, cost_test_2, cost_test_3]) print('Building predicting functions ...') f_pred = K.function(X, y_pred_score) # =========================================================================== # Build trainer # =========================================================================== print('Start training ...')
def standard_trainer(train_data, valid_data, X, y_train, y_score, y_target, parameters, test_data=None, cost_train=None, cost_score=None, optimizer=None, confusion_matrix=False, gradient_norm=True, save_path=None, save_obj=None, batch_size=64, nb_epoch=3, valid_freq=0.6, seed=1208, shuffle_level=2, patience=3, earlystop=5, report_path=None): """ Parameters ---------- cost_train: list of callable each function will be apply to a pair y_train and y_target Return ------ MainLoop, and History Note ---- """ from odin import backend as K # ====== prepare variables and cost ====== # # check optimizer if optimizer is None: optimizer = K.optimizers.SGD(lr=0.0001, momentum=0.9, nesterov=True) elif not isinstance(optimizer, K.optimizers.Optimizer) and \ not hasattr(optimizer, "get_updates"): raise ValueError( "Invalid optimizer, the optimizer must be instance of " "backend.optimizers.Optimizer or having function " "get_updates(self, loss_or_grads, params).") # check the cost functions if cost_train is None: cost_train = K.categorical_crossentropy if cost_score is None: cost_score = K.categorical_crossentropy cost_train = as_tuple(cost_train) cost_score = as_tuple(cost_score) # check input X, y, parameters X = as_tuple(X) y_train = as_tuple(y_train) y_score = as_tuple(y_score) y_target = as_tuple(y_target) parameters = as_tuple(parameters) if len(X) == 0 or len(y_train) == 0 or len(y_score) == 0 or \ len(y_target) == 0 or len(parameters) == 0: raise ValueError( "X(len=%d), y_train(len=%d), y_score(len=%d), y_target(len=%d)," "and parameters(len=%d) must be list or tuple with length > 0." % (len(X), len(y_train), len(y_score), len(y_target), len(parameters))) # get all cost if len(y_train) == 1: y_train = y_train * len(cost_train) if len(y_score) == 1: y_score = y_score * len(cost_score) cost_train = [ K.mean(f_cost(y_, y), axis=0) for f_cost, y_, y in zip( cost_train, y_train, y_target * len(cost_train) if len(y_target) == 1 else y_target) ] cost_score = [ K.mean(f_cost(y_, y), axis=0) for f_cost, y_, y in zip( cost_score, y_score, y_target * len(cost_score) if len(y_target) == 1 else y_target) ] # add confusion matrix if confusion_matrix: if not is_number(confusion_matrix) and \ not isinstance(confusion_matrix, (tuple, list, np.ndarray)): raise ValueError( "confusion_matrix must be an integer, or list, tuple" " specifies number of classes, or list of all classes.") if is_number(confusion_matrix): confusion_matrix = list(range(int(confusion_matrix))) for y_, y in zip(y_score, y_target): cost_score.append( K.confusion_matrix(y_pred=y_, y_true=y, labels=confusion_matrix)) # get the update updates = optimizer.get_updates(cost_train[0], parameters) # ====== create function ====== # grad_norm = [] if not gradient_norm or not hasattr(optimizer, 'norm') else \ [optimizer.norm] cost_train = cost_train + grad_norm print('Building training functions ...') f_train = K.function(inputs=X + y_target, outputs=cost_train, updates=updates) print('Building scoring functions ...') f_score = K.function(inputs=X + y_target, outputs=cost_score) # ====== Create trainer ====== # task = MainLoop(batch_size=batch_size, seed=seed, shuffle_level=shuffle_level) if save_path is not None and save_obj is not None: task.set_save(save_path, save_obj, save_hist=True) # set task task.set_task(f_train, train_data, epoch=nb_epoch, name='train') task.set_subtask(f_score, valid_data, freq=valid_freq, name='valid') if test_data is not None: task.set_subtask(f_score, test_data, when=-1, epoch=1, name='test') # format for score score_format = 'Results:' + __format_string( len(cost_score) - (1 if confusion_matrix else 0)) score_tracking = { (len(cost_score) - 1): lambda x: sum(x) } if confusion_matrix else [] # set the callback history = History() task.set_callback([ ProgressMonitor(name='train', format='Results:' + __format_string(len(cost_train))), ProgressMonitor(name='valid', format=score_format, tracking=score_tracking), (ProgressMonitor( name='test', format=score_format, tracking=score_tracking) if test_data is not None else None), history, EarlyStopGeneralizationLoss( 'valid', threshold=earlystop, patience=patience, get_value=lambda x: np.mean([i[0] for i in x] if isinstance(x[0], (tuple, list)) else x)), NaNDetector(('train', 'valid'), patience=patience, rollback=True) ]) return task, history
# Build network # =========================================================================== ops = N.Sequence([ N.Flatten(outdim=2), N.Dense(512, activation=K.relu), N.Dense(256, activation=K.relu), N.Dense(10, activation=K.softmax) ]) ops = cPickle.loads(cPickle.dumps(ops)) # test if the ops is pickle-able y_pred_train = ops(X_train) y_pred_score = ops(X_score) cost_train = K.mean(K.categorical_crossentropy(y_pred_train, y)) cost_test_1 = K.mean(K.categorical_crossentropy(y_pred_score, y)) cost_test_2 = K.mean(K.categorical_accuracy(y_pred_score, y)) cost_test_3 = K.confusion_matrix(y_pred_score, y, labels=range(10)) parameters = ops.parameters optimizer = K.optimizers.RMSProp(lr= 0.0001, clipnorm=100.) updates = optimizer(cost_train, parameters) print('Building training functions ...') f_train = K.function([X_train, y], [cost_train, optimizer.norm], updates=updates) print('Building testing functions ...') f_test = K.function([X_score, y], [cost_test_1, cost_test_2, cost_test_3]) # ====== normalize 0-1 ====== # if False: print('Normalized data in range [0-1]') X_train = ds['X_train'][:] X_train = (X_train - np.min(X_train, 0)) / (np.max(X_train) - np.min(X_train))