def compute_gradients(self, trainable_params, feats, labels):
predictions = self.wrapper([self.repeated_params, trainable_params, feats], training=False)
kld_gradients = self.compute_kld_gradients(trainable_params, predictions)
if self.use_second_order_derivatives:
loss = K.sum(losses.get('categorical_crossentropy')(K.squeeze(K.one_hot(labels, 4208), 3), predictions), axis=[1,2]) / 1000.
return K.squeeze(K.gradients(loss, [trainable_params]), 0) + kld_gradients
else:
loss = K.sum(losses.get(self.wrapper.loss)(labels, predictions), axis=[1,2]) / 1000.
return K.stop_gradient(K.squeeze(K.gradients(loss, [trainable_params]), 0)) + kld_gradients
def loss_functions(model, loss):
# Prepare loss functions.
print(model)
print(loss)
if isinstance(loss, dict):
for name in loss:
print(name)
if name not in model.output_names:
raise ValueError('Unknown entry in loss '
'dictionary: "' + name + '". '
'Only expected the following keys: ' +
str(model.output_names))
loss_functions = []
for name in model.output_names:
if name not in loss:
warnings.warn('Output "' + name +
'" missing from loss dictionary. '
'We assume this was done on purpose, '
'and we will not be expecting '
'any data to be passed to "' + name +
'" during training.', stacklevel=2)
loss_functions.append(losses.get(loss.get(name)))
elif isinstance(loss, list):
if len(loss) != len(model.outputs):
print('Warning : When passing a list as loss, '
'it should have one entry per model outputs. '
'The model has ' + str(len(model.outputs)) +
' outputs, but you passed loss=' +
str(loss))
loss_functions = []
for l in loss:
i = losses.get(l)
print('loss : ' , l, ' losses.get(l): ',i)
loss_functions.append(i)
# loss_functions = [losses.get(l) for l in loss]
else:
loss_function = losses.get(loss)
loss_functions = [loss_function for _ in range(len(model.outputs))]
model.loss_functions = loss_functions
weighted_losses = [_weighted_masked_objective(fn) for fn in loss_functions]
skip_target_indices = []
skip_target_weighing_indices = []
model._feed_outputs = []
model._feed_output_names = []
model._feed_output_shapes = []
model._feed_loss_fns = []
for i in range(len(weighted_losses)):
if weighted_losses[i] is None:
skip_target_indices.append(i)
skip_target_weighing_indices.append(i)
def compute_gradients(self, trainable_params, feats, labels):
predictions = self.wrapper(
[self.repeated_params,
K.transpose(trainable_params), feats])
loss = K.mean(losses.get(self.wrapper.loss)(labels, predictions))
kld_loss = K.mean(
losses.get('kld')(self.original_predictions, predictions))
return (K.stop_gradient(
K.squeeze(K.gradients(loss, [trainable_params]), 0)),
K.stop_gradient(
K.squeeze(K.gradients(kld_loss, [trainable_params]), 0)))
def compute_gradients(self, trainable_params, feats, labels):
predictions = self.wrapper([self.params, trainable_params, feats],
training=False)
loss = K.sum(losses.get(self.wrapper.loss)(labels, predictions),
axis=[1, 2]) / 1000.
return K.stop_gradient(
K.squeeze(K.gradients(loss, [trainable_params]), 0))
def _extract_losses(self):
# Can be either a string, function, list or dict according to the Keras API. In the end, we want a dictionary
# that maps an output tensor id to a loss.
if isinstance(self._model.loss, str) or callable(self._model.loss):
# normalize to list
losses = [self._model.loss] * len(self._output_specs)
else:
losses = self._model.loss
if not isinstance(losses, dict):
# must be a list, normalize to dict
losses = {
self._output_specs[i].identifier: losses[i]
for i in range(len(self._output_specs))
}
else:
# Keras stores dicts that map an output layer name to a loss. We want an output tensor id.
temp = {}
for layer_name, loss in losses.items():
for tensor_id in self._output_layer_tensor_ids[layer_name]:
temp[tensor_id] = loss
losses = temp
import keras.losses as kl
for tensor, loss in losses.items():
losses[tensor] = kl.get(loss).__name__
return losses
def compute_kld_gradients(self, trainable_params, predictions):
if not self.use_kld_regularization:
return 0
kld = K.mean(losses.get('kld')(K.stop_gradient(self.original_predictions), predictions), axis=[1,2])
if self.use_second_order_derivatives:
return self.kld_weight * K.squeeze(K.gradients(kld, [trainable_params]), 0)
else:
return self.kld_weight * K.stop_gradient(K.squeeze(K.gradients(kld, [trainable_params]), 0))
def Dave_recompile(model, params):
optimizer = params["optimizer"]
loss = KL.get(params["loss"])
a_loss = KL.get(params["a_loss"])
loss_weights = params["weight_ratio"]
weight_main = loss_weights[0]
weight_assi = loss_weights[1]
def losses(y_true, y_pred):
true_main = y_true[:,:params["output_categories"][0]]
true_assi = y_true[:,params["output_categories"][0]:]
pred_main = y_pred[:,:params["output_categories"][0]]
pred_assi = y_pred[:,params["output_categories"][0]:]
loss_main = loss(true_main, pred_main)
loss_assi = a_loss(true_assi, pred_assi)
return weight_main*loss_main + weight_assi*loss_assi
# return KL.mean_absolute_error(y_pred, y_pred)
model.compile(optimizer=optimizer, loss=losses, metrics=["accuracy"])
return model
def test2(self):
A = np.array([[0, 0, 1, 0, 0]])
B = np.array([[2]])
C = np.array([[3]])
A = K.variable(value=A)
B = K.variable(value=B)
C = K.variable(value=C)
cc = l.get("categorical_crossentropy")
print K.eval(cc(A, B))
print K.eval(cc(A, C))
print K.eval(cc(A, A))
def test_loss_masking():
weighted_loss = weighted_masked_objective(losses.get('mae'))
shape = (3, 4, 2)
x = np.arange(24).reshape(shape)
y = 2 * x
# Normally the trailing 1 is added by standardize_weights
weights = np.ones((3, ))
mask = np.ones((3, 4))
mask[1, 0] = 0
out = K.eval(
weighted_loss(K.variable(x), K.variable(y), K.variable(weights),
K.variable(mask)))
def test_loss_masking():
weighted_loss = _weighted_masked_objective(losses.get('mae'))
shape = (3, 4, 2)
x = np.arange(24).reshape(shape)
y = 2 * x
# Normally the trailing 1 is added by standardize_weights
weights = np.ones((3,))
mask = np.ones((3, 4))
mask[1, 0] = 0
out = K.eval(weighted_loss(K.variable(x),
K.variable(y),
K.variable(weights),
K.variable(mask)))
def test_loss_wrapper(self):
loss_fn = losses.get('mse')
mse_obj = losses.LossFunctionWrapper(loss_fn, name=loss_fn.__name__)
assert mse_obj.name == 'mean_squared_error'
assert (
mse_obj.reduction == losses_utils.Reduction.SUM_OVER_BATCH_SIZE)
y_true = K.constant([[1., 9.], [2., 5.]])
y_pred = K.constant([[4., 8.], [12., 3.]])
sample_weight = K.constant([1.2, 0.5])
loss = mse_obj(y_true, y_pred, sample_weight=sample_weight)
# mse = [((4 - 1)^2 + (8 - 9)^2) / 2, ((12 - 2)^2 + (3 - 5)^2) / 2]
# mse = [5, 52]
# weighted_mse = [5 * 1.2, 52 * 0.5] = [6, 26]
# reduced_weighted_mse = (6 + 26) / 2 =
np.allclose(K.eval(loss), 16, atol=1e-2)
def prepare_for_training_fgbg_weigthed(model,
optimizer='adam',
loss='binary_crossentropy'):
# Add the weigth input
weight_inp = layers.Input(model.inputs[0].get_shape().dims[1:-1],
name='weight_map')
m = models.Model(inputs=[model.input, weight_inp], outputs=model.outputs)
# Get the pixel loss
pixel_loss = losses.get(loss)
# Define the weighted loss
def _weighted_loss(y_true, y_pred):
return weight_inp * pixel_loss(y_true, y_pred)
# Compile the model
m.compile(optimizer, loss=_weighted_loss)
return m
def contractive_loss(y_pred, y_true):
# Get the base_loss.
if isinstance(self.loss, str):
base_loss = losses.get(self.loss)(y_pred, y_true)
else:
base_loss = self.loss(y_pred, y_true)
# Get the contractive loss.
encoder_output = self.encoder.layers[-1]
weigths = K.variable(value=encoder_output.get_weights()[0])
weigths = K.transpose(weigths) # N_hidden x N
h = encoder_output.output
dh = h * (1 - h)
contractive = lam * K.sum(dh**2 * K.sum(weigths**2, axis=1),
axis=1)
return base_loss + contractive
def vae_loss(loss_inputs, loss_outputs):
# Flatten all to accept different dimensions.
loss_inputs = K.flatten(loss_inputs)
loss_outputs = K.flatten(loss_outputs)
# Reconstruction loss.
if isinstance(self.loss, str):
r_loss = losses.get(self.loss)
else:
r_loss = self.loss
r_loss *= inputs_dim
# kl loss.
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
# VAE loss.
vae_loss = K.mean(r_loss + kl_loss)
vae_loss /= inputs_dim
return vae_loss
def compute_inputs(self, trainable_params, feats, labels):
predictions = self.wrapper(
[self.params, K.transpose(trainable_params), feats])
loss = K.mean(losses.get(self.wrapper.loss)(labels, predictions))
gradients = K.stop_gradient(
K.squeeze(K.gradients(loss, [trainable_params]), 0))
loss = loss * K.ones_like(trainable_params)
preprocessed_gradients = self.preprocess(gradients)
preprocessed_loss = self.preprocess(loss)
inputs = K.stop_gradient(
K.concatenate([
self.param_coordinates,
preprocessed_gradients[0],
preprocessed_gradients[1],
preprocessed_loss[0],
preprocessed_loss[1],
],
axis=1))
return inputs, gradients
def _get_loss_object(self, loss):
"""Returns a `Loss` object.
Converts the user-supplied loss to a `Loss` object. Also allows
`SUM_OVER_BATCH_SIZE` reduction to be used for this loss.
Args:
loss: A string, function, or `Loss` object.
Returns:
A `Loss` object.
"""
if loss is None:
return None # Ok to have no loss for an output.
loss = losses_mod.get(loss)
if not isinstance(loss, losses_mod.Loss):
loss_name = get_custom_object_name(loss)
if loss_name is None:
raise ValueError('Loss should be a callable, found: {}'.format(loss))
loss = losses_mod.LossFunctionWrapper(loss, name=loss_name)
loss._allow_sum_over_batch_size = True # pylint: disable=protected-access
return loss
def compile_tfrecord(self, optimizer, loss, y, metrics=None, y_val=None):
"""Configures the model for training.
# Arguments
optimizer: str (name of optimizer) or optimizer object.
See [optimizers](/optimizers).
loss: str (name of objective function) or objective function.
See [losses](/losses).
If the model has multiple outputs, you can use a different loss
on each output by passing a dictionary or a list of losses.
The loss value that will be minimized by the model
will then be the sum of all individual losses.
metrics: list of metrics to be evaluated by the model
during training and testing.
Typically you will use `metrics=['accuracy']`.
To specify different metrics for different outputs of a
multi-output model, you could also pass a dictionary,
such as `metrics={'output_a': 'accuracy'}`.
# Raises
ValueError: In case of invalid arguments for
`optimizer`, `loss`, `metrics` or `sample_weight_mode`.
"""
loss = loss or {}
self.optimizer = optimizers.get(optimizer)
self.loss = loss
self.sample_weight_mode = None
self.loss_weights = None
self.y_val = y_val
do_validation = bool(len(self.val_inputs) > 0)
if do_validation and y_val is None:
raise ValueError('When you use validation inputs, '
'you should provide y_val.')
# Prepare loss functions.
if isinstance(loss, dict):
for name in loss:
if name not in self.output_names:
raise ValueError('Unknown entry in loss '
'dictionary: "' + name + '". '
'Only expected the following keys: ' +
str(self.output_names))
loss_functions = []
for name in self.output_names:
if name not in loss:
warnings.warn('Output "' + name +
'" missing from loss dictionary. '
'We assume this was done on purpose, '
'and we will not be expecting '
'any data to be passed to "' + name +
'" during training.',
stacklevel=2)
loss_functions.append(losses.get(loss.get(name)))
elif isinstance(loss, list):
if len(loss) != len(self.outputs):
raise ValueError('When passing a list as loss, '
'it should have one entry per model outputs. '
'The model has ' + str(len(self.outputs)) +
' outputs, but you passed loss=' + str(loss))
loss_functions = [losses.get(l) for l in loss]
else:
loss_function = losses.get(loss)
loss_functions = [loss_function for _ in range(len(self.outputs))]
self.loss_functions = loss_functions
# Prepare training targets of model.
if isinstance(y, (list, tuple)):
y = list(y) # Tensor or list of tensors.
else:
y = [y]
self.targets = []
for i in range(len(self.outputs)):
target = y[i]
self.targets.append(target)
# Prepare validation targets of model.
if isinstance(y_val, (list, tuple)):
y_val = list(y_val) # Tensor or list of tensors.
else:
y_val = [y_val]
self.y_val = y_val
self.val_targets = []
for i in range(len(self.val_outputs)):
val_target = y_val[i]
self.val_targets.append(val_target)
# Prepare metrics.
self.metrics = metrics
self.metrics_names = ['loss']
self.metrics_tensors = []
self.val_metrics_names = ['loss']
self.val_metrics_tensors = []
# Compute total training loss.
total_loss = None
for i in range(len(self.outputs)):
y_true = self.targets[i]
y_pred = self.outputs[i]
loss_function = loss_functions[i]
val_output_loss = K.mean(loss_function(y_true, y_pred))
if len(self.outputs) > 1:
self.metrics_tensors.append(val_output_loss)
self.metrics_names.append(self.output_names[i] + '_loss')
if total_loss is None:
total_loss = val_output_loss
else:
total_loss += val_output_loss
if total_loss is None:
if not self.losses:
raise RuntimeError('The model cannot be compiled '
'because it has no loss to optimize.')
else:
total_loss = 0.
# Compute total validation loss.
val_total_loss = None
for i in range(len(self.val_outputs)):
y_true = self.val_targets[i]
y_pred = self.val_outputs[i]
loss_function = loss_functions[i]
val_output_loss = K.mean(loss_function(y_true, y_pred))
if len(self.outputs) > 1:
self.val_metrics_tensors.append(val_output_loss)
self.val_metrics_names.append(self.output_names[i] +
'_val_loss')
if val_total_loss is None:
val_total_loss = val_output_loss
else:
val_total_loss += val_output_loss
if val_total_loss is None:
if not self.losses and do_validation:
raise RuntimeError('The model cannot be compiled '
'because it has no loss to optimize.')
else:
val_total_loss = 0.
# Add regularization penalties
# and other layer-specific losses.
for loss_tensor in self.losses:
total_loss += loss_tensor
val_total_loss += loss_tensor
# List of same size as output_names.
# contains tuples (metrics for output, names of metrics).
nested_metrics = _collect_metrics(metrics, self.output_names)
def append_metric(layer_num, metric_name, metric_tensor):
"""Helper function used in loop below."""
if len(self.output_names) > 1:
metric_name = self.output_layers[
layer_num].name + '_' + metric_name
self.metrics_names.append(metric_name)
self.metrics_tensors.append(metric_tensor)
for i in range(len(self.outputs)):
y_true = self.targets[i]
y_pred = self.outputs[i]
output_metrics = nested_metrics[i]
for metric in output_metrics:
if metric == 'accuracy' or metric == 'acc':
# custom handling of accuracy
# (because of class mode duality)
output_shape = self.internal_output_shapes[i]
acc_fn = None
if (output_shape[-1] == 1 or self.loss_functions[i]
== losses.binary_crossentropy):
# case: binary accuracy
acc_fn = metrics_module.binary_accuracy
elif self.loss_functions[
i] == losses.sparse_categorical_crossentropy:
# case: categorical accuracy with sparse targets
acc_fn = metrics_module.sparse_categorical_accuracy
else:
acc_fn = metrics_module.categorical_accuracy
append_metric(i, 'acc', K.mean(acc_fn(y_true, y_pred)))
else:
metric_fn = metrics_module.get(metric)
metric_result = metric_fn(y_true, y_pred)
metric_result = {metric_fn.__name__: metric_result}
for name, tensor in six.iteritems(metric_result):
append_metric(i, name, tensor)
def append_val_metric(layer_num, metric_name, metric_tensor):
"""Helper function used in loop below."""
if len(self.output_names) > 1:
metric_name = self.output_layers[
layer_num].name + '_val_' + metric_name
self.val_metrics_names.append(metric_name)
self.val_metrics_tensors.append(metric_tensor)
for i in range(len(self.val_outputs)):
y_true = self.val_targets[i]
y_pred = self.val_outputs[i]
output_metrics = nested_metrics[i]
for metric in output_metrics:
if metric == 'accuracy' or metric == 'acc':
# custom handling of accuracy
# (because of class mode duality)
output_shape = self.internal_output_shapes[i]
acc_fn = None
if (output_shape[-1] == 1 or self.loss_functions[i]
== losses.binary_crossentropy):
# case: binary accuracy
acc_fn = metrics_module.binary_accuracy
elif self.loss_functions[
i] == losses.sparse_categorical_crossentropy:
# case: categorical accuracy with sparse targets
acc_fn = metrics_module.sparse_categorical_accuracy
else:
acc_fn = metrics_module.categorical_accuracy
append_val_metric(i, 'acc', K.mean(acc_fn(y_true, y_pred)))
else:
metric_fn = metrics_module.get(metric)
metric_result = metric_fn(y_true, y_pred)
metric_result = {metric_fn.__name__: metric_result}
for name, tensor in six.iteritems(metric_result):
append_val_metric(i, name, tensor)
# Prepare gradient updates and state updates.
self.total_loss = total_loss
self.val_total_loss = val_total_loss
# Functions for train, test and predict will
# be compiled lazily when required.
# This saves time when the user is not using all functions.
self.train_function = None
self.val_function = None
self.test_function = None
self.predict_function = None
# Collected trainable weights and sort them deterministically.
trainable_weights = self.trainable_weights
# Sort weights by name.
if trainable_weights:
trainable_weights.sort(key=lambda x: x.name)
self._collected_trainable_weights = trainable_weights
def Dave(params=None):
if params is None:
return "Dave"
else:
inputs = []
for n in range(params["input_points"][1]):
i = Input(shape=(params["input_points"][0],), name="input_{}".format(n+1))
inputs.append(i)
drop_rate = params["dropout_rate"]
max_norm = params["max_norm"]
activation = params["layer_activation"]
layers = []
mod = 1
layer_tracker = 0
merge_layer = 0
for i,node_count in enumerate(params["node_per_layer"]):
if node_count == 0:
if merge_layer == 0:
merge_layer = layer_tracker
mod = 0
continue
else:
layers.append(Flatten())
layer_tracker += 1
continue
if type(node_count) is tuple:
layers.append(Conv1D(node_count[0], node_count[1], activation=activation, name="conv1D_{}".format(i+mod), kernel_constraint=maxnorm(max_norm), padding="same"))
layers.append(MaxPool1D(node_count[2], name="max_pooling1D_{}".format(i+mod)))
layer_tracker += 2
else:
layers.append(Dense(node_count, activation=activation, name="dense_{}".format(i+mod), kernel_constraint=maxnorm(max_norm)))
layer_tracker += 1
layers.append(Dropout(drop_rate))
layers.append(BatchNormalization())
layer_tracker += 2
r = Reshape((params["input_points"][0], 1))
outs = []
for x in inputs:
x = r(x)
for layer in layers[:merge_layer]:
x = layer(x)
outs.append(x)
try:
activation = params["output_activation"]
except KeyError:
pass
if len(outs) > 1:
x = concatenate(outs, axis=-1)
else:
x = outs[0]
for layer in layers[merge_layer:]:
x = layer(x)
output1 = Dense(params["output_categories"][0], activation=activation, name="output_main")(x)
output2 = Dense(params["output_categories"][1], activation=activation, name="output_assister")(x)
output = concatenate([output1, output2])
outputs = [output]
model = Model(inputs=inputs, outputs=outputs)
optimizer = params["optimizer"]
loss = KL.get(params["loss"])
a_loss = KL.get(params["a_loss"])
loss_weights = params["weight_ratio"]
weight_main = loss_weights[0]
weight_assi = loss_weights[1]
def losses(y_true, y_pred):
true_main = y_true[:,:params["output_categories"][0]]
true_assi = y_true[:,params["output_categories"][0]:]
pred_main = y_pred[:,:params["output_categories"][0]]
pred_assi = y_pred[:,params["output_categories"][0]:]
loss_main = loss(true_main, pred_main)
loss_assi = a_loss(true_assi, pred_assi)
return weight_main*loss_main + weight_assi*loss_assi
# return KL.mean_absolute_error(y_pred, y_pred)
model.compile(optimizer=optimizer, loss=losses, metrics=["accuracy"])
return model
def test_sequential(in_tmpdir):
(x_train, y_train), (x_test, y_test) = _get_test_data()
# TODO: factor out
def data_generator(x, y, batch_size=50):
index_array = np.arange(len(x))
while 1:
batches = make_batches(len(x_test), batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
x_batch = x[batch_ids]
y_batch = y[batch_ids]
yield (x_batch, y_batch)
model = Sequential()
model.add(Dense(num_hidden, input_shape=(input_dim,)))
model.add(Activation('relu'))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1,
validation_data=(x_test, y_test))
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=2,
validation_split=0.1)
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=0)
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1,
shuffle=False)
model.train_on_batch(x_train[:32], y_train[:32])
loss = model.evaluate(x_test, y_test)
prediction = model.predict_generator(data_generator(x_test, y_test), 1,
max_queue_size=2, verbose=1)
gen_loss = model.evaluate_generator(data_generator(x_test, y_test, 50), 1,
max_queue_size=2)
pred_loss = K.eval(K.mean(losses.get(model.loss)(K.variable(y_test),
K.variable(prediction))))
assert(np.isclose(pred_loss, loss))
assert(np.isclose(gen_loss, loss))
model.predict(x_test, verbose=0)
model.predict_classes(x_test, verbose=0)
model.predict_proba(x_test, verbose=0)
fname = 'test_sequential_temp.h5'
model.save_weights(fname, overwrite=True)
model = Sequential()
model.add(Dense(num_hidden, input_shape=(input_dim,)))
model.add(Activation('relu'))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.load_weights(fname)
os.remove(fname)
nloss = model.evaluate(x_test, y_test, verbose=0)
assert(loss == nloss)
# Test serialization
config = model.get_config()
assert 'name' in config
new_model = Sequential.from_config(config)
assert new_model.weights # Model should be built.
model.summary()
json_str = model.to_json()
model_from_json(json_str)
yaml_str = model.to_yaml()
model_from_yaml(yaml_str)
def loss(self, y_true, y_pred):
return get(self.loss_identifier)(y_true, y_pred)
def get(name):
try:
return kloss.get(name)
except ValueError:
return get_from_module(name, globals())
def compute_gradients(self, trainable_params, feats, labels):
predictions = self.wrapper(
[self.params, K.transpose(trainable_params), feats])
loss = K.mean(losses.get(self.wrapper.loss)(labels, predictions))
return K.stop_gradient(
K.squeeze(K.gradients(loss, [trainable_params]), 0))
def __str__(self) -> str:
if isinstance(self.loss_identifier, str):
return self.loss_identifier
return str(get(self.loss_identifier))
def subset_sampling(train_states, hyperparameters, new_block_result):
random.seed(hyperparameters['start_seed'] * 1000 +
train_states['block_number'])
y_train = np.load(hyperparameters['y_train_file'])
nb_sample = y_train.shape[0]
nb_select = int(np.floor(nb_sample * hyperparameters['subset_percentage']))
if nb_select == nb_sample:
return list(range(nb_sample))
if hyperparameters['sampling_strategy'] == 'random-with-class':
subset_indices = list(range(nb_sample))
random.shuffle(subset_indices)
subset_indices = subset_indices[:nb_select]
subset_indices = []
y_train_lb = np.argmax(y_train, axis=-1)
for lb in np.unique(y_train_lb):
indices = np.where(y_train_lb == lb)[0].tolist()
random.shuffle(indices)
nb_subset_select = int(
np.floor(len(indices) * hyperparameters['subset_percentage']))
subset_indices += indices[:nb_subset_select]
return subset_indices
if hyperparameters['sampling_strategy'] == 'random-without-class':
subset_indices = list(range(nb_sample))
random.shuffle(subset_indices)
subset_indices = subset_indices[:nb_select]
return subset_indices
topology = copy.copy(train_states['topology'])
layer_iter = train_states['layer_iter']
block_iter = train_states['block_iter']
weights = train_states['weights']
# add new block to topology
if layer_iter == 0 and block_iter == 0:
topology[1].append(hyperparameters['block_size'])
elif layer_iter > 0 and block_iter == 0:
output_layer = topology.pop(-1)
topology.append([hyperparameters['block_size']])
topology.append(output_layer)
else:
topology[-2].append(hyperparameters['block_size'])
# build the model
model = network_builder(topology, None, None, None)
model.compile('adam', hyperparameters['loss'], hyperparameters['metrics'])
# set the weights of old blocks
for layer in model.layers:
if layer.name in weights.keys():
layer.set_weights(weights[layer.name])
# set the weights of new block
model.get_layer('dense%d_%d' % (layer_iter, block_iter)).set_weights(
new_block_result['dense_weights'])
model.get_layer('bn%d_%d' % (layer_iter, block_iter)).set_weights(
new_block_result['bn_weights'])
model.get_layer('output%d_%d' % (layer_iter, block_iter)).set_weights(
new_block_result['output_weights'])
# load full train data
x_train = np.load(hyperparameters['x_train_file'])
feature_model = Model(inputs=model.input,
outputs=model.get_layer('pre_predictions').output)
features = feature_model.predict(x_train,
batch_size=hyperparameters['batch_size'],
verbose=0)
if topology[-1][-1] is not None:
y_pred = softmax(features)
else:
y_pred = features
# compute the loss induced by each sample
losses = K.eval(
keras_losses.get(hyperparameters['loss'])(K.variable(y_train),
K.variable(y_pred)))
#
if hyperparameters['sampling_strategy'] == 'top_loss':
subset_indices = np.argsort(losses)[-nb_select:]
return subset_indices
else:
nb_cluster = min(hyperparameters['cluster_scale'] * topology[-1][0],
nb_select)
if hyperparameters['cluster_algo'] == 'kmean':
clusterer = Cluster.KMeans(n_clusters=nb_cluster,
n_init=1,
n_jobs=-1,
max_iter=100,
algorithm='elkan')
else:
clusterer = Cluster.AgglomerativeClustering(n_clusters=nb_cluster)
# perform clustering on the prediction
cluster_labels = clusterer.fit_predict(features)
subset_indices = cluster_sampling(cluster_labels, nb_cluster,
hyperparameters['subset_percentage'],
losses,
hyperparameters['sampling_routine'])
return subset_indices