normalize=(mean, std), process_func=parallel_augment, testing=True) # Transform batchiterator to a threaded iterator train_iter = threaded_iterator(train_iter) valid_iter = threaded_iterator(valid_iter) # Count the number of trainable parameters in the model num_params = nn.layers.count_params(output_layer, trainable=True) print('Number of trainable parameters: {}'.format(num_params)) # Construct loss function & accuracy predictions = nn.layers.get_output(output_layer) train_log_loss = categorical_crossentropy(predictions, y) train_log_loss = train_log_loss.mean() train_kappa_loss = quad_kappa_loss(predictions, y, y_pow=y_pow) params = nn.layers.get_all_params(output_layer, regularizable=True) regularization = sum(T.sum(p**2) for p in params) train_hybrid_loss = train_kappa_loss + log_scale * T.clip( train_log_loss, log_cutoff, 10**3) + l2_reg * regularization train_accuracy = accuracy(predictions, y) valid_predictions = nn.layers.get_output(output_layer, deterministic=True) valid_log_loss = categorical_crossentropy(valid_predictions, y) valid_log_loss = valid_log_loss.mean() valid_kappa_loss = quad_kappa_loss(valid_predictions, y) valid_loss = valid_kappa_loss valid_accuracy = accuracy(valid_predictions, y) # Scale grads all_params = nn.layers.get_all_params(output_layer, trainable=True)
def loss(y, t): return quad_kappa_loss(y, t, y_pow=y_pow)
predictions = nn.layers.get_output(output_layer, deterministic=True) train_log_loss, train_reg_loss, train_multi_loss = multi_task_loss(predictions, y) # train_loss1 = categorical_crossentropy(predictions[:, :num_class], y).mean() # train_loss2 = categorical_crossentropy(predictions[:, num_class:], y).mean() # train_hybrid_loss = hybrid_loss(predictions, y) params = nn.layers.get_all_params(output_layer, regularizable=True) regularization = sum(T.sum(p ** 2) for p in params) # train_loss = train_loss1 + train_loss2 + l2_reg * regularization train_loss = train_multi_loss + l2_reg * regularization # train_loss = train_hybrid_loss + l2_reg * regularization # train_accuracy = accuracy(predictions[:, num_class:], y) # train_kappa = quad_kappa_loss(predictions[:, num_class:], y) train_accuracy = accuracy(predictions[:, :num_class], y) train_kappa = quad_kappa_loss(predictions[:, :num_class], y) # train_accuracy = accuracy(predictions, y) # train_kappa = quad_kappa_loss(predictions, y) valid_predictions = nn.layers.get_output(output_layer, deterministic=True) valid_log_loss, valid_reg_loss, valid_multi_loss = multi_task_loss(valid_predictions, y) # valid_loss = categorical_crossentropy(valid_predictions[:, num_class:], y).mean() # valid_loss = hybrid_loss(valid_predictions, y) valid_accuracy = accuracy(valid_predictions[:, :num_class], y) # valid_accuracy = accuracy(valid_predictions, y) valid_kappa = quad_kappa_loss(valid_predictions[:, :num_class], y) # valid_kappa = quad_kappa_loss(valid_predictions, y) # Scale grads all_params = nn.layers.get_all_params(output_layer, trainable=True)
def hybrid_loss(y, t): log_loss = categorical_crossentropy(y, t).mean() kappa_loss = quad_kappa_loss(y, t, y_pow=2) return kappa_loss + 0.5 * T.clip(log_loss, 0.6, 10 ** 3)
def hybrid_loss(y, t): log_loss = categorical_crossentropy(y, t).mean() kappa_loss = quad_kappa_loss(y, t, y_pow=2) return kappa_loss + 0.5 * T.clip(log_loss, 0.6, 10**3)
predictions = nn.layers.get_output(output_layer, deterministic=True) train_log_loss, train_reg_loss, train_multi_loss = multi_task_loss( predictions, y) #train_loss1 = categorical_crossentropy(predictions[:, :num_class], y).mean() #train_loss2 = categorical_crossentropy(predictions[:, num_class:], y).mean() #train_hybrid_loss = hybrid_loss(predictions, y) params = nn.layers.get_all_params(output_layer, regularizable=True) regularization = sum(T.sum(p**2) for p in params) #train_loss = train_loss1 + train_loss2 + l2_reg * regularization train_loss = train_multi_loss + l2_reg * regularization #train_loss = train_hybrid_loss + l2_reg * regularization #train_accuracy = accuracy(predictions[:, num_class:], y) #train_kappa = quad_kappa_loss(predictions[:, num_class:], y) train_accuracy = accuracy(predictions[:, :num_class], y) train_kappa = quad_kappa_loss(predictions[:, :num_class], y) #train_accuracy = accuracy(predictions, y) #train_kappa = quad_kappa_loss(predictions, y) valid_predictions = nn.layers.get_output(output_layer, deterministic=True) valid_log_loss, valid_reg_loss, valid_multi_loss = multi_task_loss( valid_predictions, y) #valid_loss = categorical_crossentropy(valid_predictions[:, num_class:], y).mean() #valid_loss = hybrid_loss(valid_predictions, y) valid_accuracy = accuracy(valid_predictions[:, :num_class], y) #valid_accuracy = accuracy(valid_predictions, y) valid_kappa = quad_kappa_loss(valid_predictions[:, :num_class], y) #valid_kappa = quad_kappa_loss(valid_predictions, y) # Scale grads all_params = nn.layers.get_all_params(output_layer, trainable=True)
return log_loss, reg_loss, 0.75 * log_loss + 0.25 * reg_loss def hybrid_loss(y, t): log_loss = categorical_crossentropy(y, t).mean() kappa_loss = quad_kappa_loss(y, t, y_pow=2) return kappa_loss + 0.5 * T.clip(log_loss, 0.6, 10**3) predictions = nn.layers.get_output(output_layer, deterministic=False) log_loss = categorical_crossentropy(predictions, y).mean() params = nn.layers.get_all_params(output_layer, regularizable=True) regularization = sum(T.sum(p**2) for p in params) train_loss = log_loss + l2_reg * regularization train_accuracy = accuracy(predictions, y) train_kappa = quad_kappa_loss(predictions, y) valid_predictions = nn.layers.get_output(output_layer, deterministic=True) valid_loss = categorical_crossentropy(valid_predictions, y).mean() valid_accuracy = accuracy(valid_predictions, y) valid_kappa = quad_kappa_loss(valid_predictions, y) # Scale grads all_params = nn.layers.get_all_params(output_layer, trainable=True) all_grads = T.grad(train_loss, all_params) #scaled_grads = nn.updates.total_norm_constraint(all_grads, max_norm=5, return_norm=False) # Construct update updates = nn.updates.nesterov_momentum(all_grads, all_params, learning_rate=lr,
normalize=(mean, std), process_func=parallel_augment, testing=True) # Transform batchiterator to a threaded iterator train_iter = threaded_iterator(train_iter) valid_iter = threaded_iterator(valid_iter) # Count the number of trainable parameters in the model num_params = nn.layers.count_params(output_layer, trainable=True) print('Number of trainable parameters: {}'.format(num_params)) # Construct loss function & accuracy predictions = nn.layers.get_output(output_layer) train_log_loss = categorical_crossentropy(predictions, y) train_log_loss = train_log_loss.mean() train_kappa_loss = quad_kappa_loss(predictions, y, y_pow=y_pow) params = nn.layers.get_all_params(output_layer, regularizable=True) regularization = sum(T.sum(p ** 2) for p in params) train_hybrid_loss = train_kappa_loss + log_scale * T.clip(train_log_loss, log_cutoff, 10 ** 3) + l2_reg * regularization train_accuracy = accuracy(predictions, y) valid_predictions = nn.layers.get_output(output_layer, deterministic=True) valid_log_loss = categorical_crossentropy(valid_predictions, y) valid_log_loss = valid_log_loss.mean() valid_kappa_loss = quad_kappa_loss(valid_predictions, y) valid_loss = valid_kappa_loss valid_accuracy = accuracy(valid_predictions, y) # Scale grads all_params = nn.layers.get_all_params(output_layer, trainable=True)
return log_loss, reg_loss, 0.75 * log_loss + 0.25 * reg_loss def hybrid_loss(y, t): log_loss = categorical_crossentropy(y, t).mean() kappa_loss = quad_kappa_loss(y, t, y_pow=2) return kappa_loss + 0.5 * T.clip(log_loss, 0.6, 10 ** 3) predictions = nn.layers.get_output(output_layer, deterministic=False) log_loss = categorical_crossentropy(predictions, y).mean() params = nn.layers.get_all_params(output_layer, regularizable=True) regularization = sum(T.sum(p ** 2) for p in params) train_loss = log_loss + l2_reg * regularization train_accuracy = accuracy(predictions, y) train_kappa = quad_kappa_loss(predictions, y) valid_predictions = nn.layers.get_output(output_layer, deterministic=True) valid_loss = categorical_crossentropy(valid_predictions, y).mean() valid_accuracy = accuracy(valid_predictions, y) valid_kappa = quad_kappa_loss(valid_predictions, y) # Scale grads all_params = nn.layers.get_all_params(output_layer, trainable=True) all_grads = T.grad(train_loss, all_params) # scaled_grads = nn.updates.total_norm_constraint(all_grads, max_norm=5, return_norm=False) # Construct update updates = nn.updates.nesterov_momentum(all_grads, all_params, learning_rate=lr, momentum=momentum) # updates = nn.updates.adam(all_grads, all_params, learning_rate=0.0001)