def test_siamese_data_generator_invalid_data(self):
generator = image_generators.SiameseDataGenerator(
featurewise_center=True,
samplewise_center=True,
featurewise_std_normalization=True,
samplewise_std_normalization=True,
zca_whitening=True,
data_format='channels_last')
feats = ['appearance', 'distance', 'neighborhood', 'regionprop']
# Test fit with invalid data
with self.assertRaises(ValueError):
x = np.random.random((3, 10, 10))
generator.fit(x)
# Test flow with invalid dimensions
with self.assertRaises(ValueError):
train_dict = {
'X': np.random.random((8, 10, 10)),
'y': np.random.random((8, 10, 10)),
'daughters': {}
}
generator.flow(train_dict, features=feats)
# Test flow with non-matching batches
with self.assertRaises(Exception):
train_dict = {
'X': np.random.random((8, 11, 10, 10, 1)),
'y': np.random.random((7, 11, 10, 10, 1)),
'daughters': {}
}
generator.flow(train_dict, features=feats)
# Test flow without daughters
with self.assertRaises(ValueError):
train_dict = {
'X': np.random.random((8, 11, 10, 10, 1)),
'y': np.random.random((7, 11, 10, 10, 1)),
}
generator.flow(train_dict, features=feats)
# Invalid number of channels: will work but raise a warning
generator.fit(np.random.random((8, 10, 10, 5)))
with self.assertRaises(ValueError):
generator = image_generators.SiameseDataGenerator(
data_format='unknown')
generator = image_generators.SiameseDataGenerator(zoom_range=(2, 2))
with self.assertRaises(ValueError):
generator = image_generators.SiameseDataGenerator(zoom_range=(2, 2,
2))
def test_siamese_data_generator(self):
frames = 5
# TODO: image generator should handle RGB as well as grayscale
for test_images in _generate_test_images()[1:]:
img_list = []
for im in test_images:
frame_list = []
for _ in range(frames):
frame_list.append(img_to_array(im)[None, ...])
img_stack = np.vstack(frame_list)
img_list.append(img_stack)
images = np.vstack(img_list)
batches = images.shape[0] // frames
images = np.reshape(images, (batches, frames, *images.shape[1:]))
generator = image_generators.SiameseDataGenerator(
featurewise_center=True,
samplewise_center=True,
featurewise_std_normalization=True,
samplewise_std_normalization=True,
zca_whitening=True,
rotation_range=90.,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.5,
zoom_range=0.2,
channel_shift_range=1.,
brightness_range=(1, 5),
fill_mode='nearest',
cval=0.5,
horizontal_flip=True,
vertical_flip=True)
feats = ['appearance', 'distance', 'neighborhood', 'regionprop']
# Basic test before fit
train_dict = {
# TODO: image generator should handle RGB as well as grayscale
'X':
np.random.random((8, 5, 10, 10, 1)),
'y':
np.random.randint(low=0, high=4, size=(8, 5, 10, 10, 1)),
'daughters': [{j: [{
1: [2, 3]
}]
for j in range(1, 4)} for k in range(8)]
}
generator.flow(train_dict, features=feats)
# Temp dir to save generated images
temp_dir = self.get_temp_dir()
y_shape = tuple(list(images.shape)[:-1] + [1])
train_dict['X'] = images
train_dict['y'] = np.random.randint(low=0, high=4, size=y_shape)
train_dict['daughters'] = [{j: [{
1: [2, 3]
}]
for j in range(1, 4)}
for k in range(y_shape[0])]
def train_model_siamese_daughter(model,
dataset,
expt='',
test_size=.1,
n_epoch=100,
batch_size=1,
num_gpus=None,
crop_dim=32,
min_track_length=1,
neighborhood_scale_size=10,
features=None,
optimizer=SGD(lr=0.01,
decay=1e-6,
momentum=0.9,
nesterov=True),
log_dir='/data/tensorboard_logs',
model_dir='/data/models',
model_name=None,
focal=False,
gamma=0.5,
lr_sched=rate_scheduler(lr=0.01, decay=0.95),
rotation_range=0,
flip=True,
shear=0,
zoom_range=0,
seed=None,
**kwargs):
is_channels_first = K.image_data_format() == 'channels_first'
if model_name is None:
todays_date = datetime.datetime.now().strftime('%Y-%m-%d')
data_name = os.path.splitext(os.path.basename(dataset))[0]
model_name = '{}_{}_[{}]_neighs={}_epochs={}_seed={}_{}'.format(
todays_date, data_name, ','.join(f[0] for f in sorted(features)),
neighborhood_scale_size, n_epoch, seed, expt)
model_path = os.path.join(model_dir, '{}.h5'.format(model_name))
loss_path = os.path.join(model_dir, '{}.npz'.format(model_name))
print('training on dataset:', dataset)
print('saving model at:', model_path)
print('saving loss at:', loss_path)
train_dict, val_dict = get_data(dataset,
mode='siamese_daughters',
seed=seed,
test_size=test_size)
# the data, shuffled and split between train and test sets
print('X_train shape:', train_dict['X'].shape)
print('y_train shape:', train_dict['y'].shape)
print('X_test shape:', val_dict['X'].shape)
print('y_test shape:', val_dict['y'].shape)
print('Output Shape:', model.layers[-1].output_shape)
n_classes = model.layers[-1].output_shape[1 if is_channels_first else -1]
def loss_function(y_true, y_pred):
if focal:
return losses.weighted_focal_loss(y_true,
y_pred,
gamma=gamma,
n_classes=n_classes,
from_logits=False)
return losses.weighted_categorical_crossentropy(y_true,
y_pred,
n_classes=n_classes,
from_logits=False)
if num_gpus is None:
num_gpus = train_utils.count_gpus()
if num_gpus >= 2:
batch_size = batch_size * num_gpus
model = train_utils.MultiGpuModel(model, num_gpus)
print('Training on {} GPUs'.format(num_gpus))
model.compile(loss=loss_function,
optimizer=optimizer,
metrics=['accuracy'])
print('Using real-time data augmentation.')
# this will do preprocessing and realtime data augmentation
datagen = image_generators.SiameseDataGenerator(
rotation_range=rotation_range,
shear_range=shear,
zoom_range=zoom_range,
horizontal_flip=flip,
vertical_flip=flip)
datagen_val = image_generators.SiameseDataGenerator(rotation_range=0,
zoom_range=0,
shear_range=0,
horizontal_flip=0,
vertical_flip=0)
total_train_pairs = tracking_utils.count_pairs(train_dict['y'],
same_probability=5.0)
total_test_pairs = tracking_utils.count_pairs(val_dict['y'],
same_probability=5.0)
train_data = datagen.flow(train_dict,
crop_dim=crop_dim,
batch_size=batch_size,
min_track_length=min_track_length,
neighborhood_scale_size=neighborhood_scale_size,
features=features)
val_data = datagen_val.flow(
val_dict,
crop_dim=crop_dim,
batch_size=batch_size,
min_track_length=min_track_length,
neighborhood_scale_size=neighborhood_scale_size,
features=features)
print('total_train_pairs:', total_train_pairs)
print('total_test_pairs:', total_test_pairs)
print('batch size:', batch_size)
print('validation_steps: ', total_test_pairs // batch_size)
# fit the model on the batches generated by datagen.flow()
loss_history = model.fit_generator(
train_data,
steps_per_epoch=total_train_pairs // batch_size,
epochs=n_epoch,
validation_data=val_data,
validation_steps=total_test_pairs // batch_size,
callbacks=[
callbacks.LearningRateScheduler(lr_sched),
callbacks.ModelCheckpoint(model_path,
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=num_gpus >= 2),
callbacks.TensorBoard(log_dir=os.path.join(log_dir, model_name))
])
model.save_weights(model_path)
np.savez(loss_path, loss_history=loss_history.history)
return model
def train_model_siamese_daughter(model,
dataset,
expt='',
test_size=.2,
n_epoch=100,
batch_size=1,
num_gpus=None,
crop_dim=32,
min_track_length=1,
neighborhood_scale_size=10,
features=None,
optimizer=SGD(lr=0.01,
decay=1e-6,
momentum=0.9,
nesterov=True),
log_dir='/data/tensorboard_logs',
model_dir='/data/models',
model_name=None,
focal=False,
gamma=0.5,
lr_sched=rate_scheduler(lr=0.01, decay=0.95),
rotation_range=0,
flip=True,
shear=0,
zoom_range=0,
seed=0,
**kwargs):
is_channels_first = K.image_data_format() == 'channels_first'
if model_name is None:
todays_date = datetime.datetime.now().strftime('%Y-%m-%d')
data_name = os.path.splitext(os.path.basename(dataset))[0]
model_name = '{}_{}_[{}]_neighs={}_epochs={}_seed={}_{}'.format(
todays_date, data_name, ','.join(f[0] for f in sorted(features)),
neighborhood_scale_size, n_epoch, seed, expt)
model_path = os.path.join(model_dir, '{}.h5'.format(model_name))
loss_path = os.path.join(model_dir, '{}.npz'.format(model_name))
print('training on dataset:', dataset)
print('saving model at:', model_path)
print('saving loss at:', loss_path)
train_dict, val_dict = get_data(dataset,
mode='siamese_daughters',
seed=seed,
test_size=test_size)
# the data, shuffled and split between train and test sets
print('X_train shape:', train_dict['X'].shape)
print('y_train shape:', train_dict['y'].shape)
print('X_test shape:', val_dict['X'].shape)
print('y_test shape:', val_dict['y'].shape)
print('Output Shape:', model.layers[-1].output_shape)
n_classes = model.layers[-1].output_shape[1 if is_channels_first else -1]
def loss_function(y_true, y_pred):
if focal:
return losses.weighted_focal_loss(y_true,
y_pred,
gamma=gamma,
n_classes=n_classes,
from_logits=False)
return losses.weighted_categorical_crossentropy(y_true,
y_pred,
n_classes=n_classes,
from_logits=False)
if num_gpus is None:
num_gpus = train_utils.count_gpus()
print('Training on {} GPUs'.format(num_gpus))
model.compile(loss=loss_function,
optimizer=optimizer,
metrics=['accuracy'])
print('Using real-time data augmentation.')
# this will do preprocessing and realtime data augmentation
datagen = image_generators.SiameseDataGenerator(
rotation_range=rotation_range,
shear_range=shear,
zoom_range=zoom_range,
horizontal_flip=flip,
vertical_flip=flip)
datagen_val = image_generators.SiameseDataGenerator(rotation_range=0,
zoom_range=0,
shear_range=0,
horizontal_flip=0,
vertical_flip=0)
# same_probability values have varied from 0.5 to 5.0
total_train_pairs = tracking_utils.count_pairs(train_dict['y'],
same_probability=5.0)
total_test_pairs = tracking_utils.count_pairs(val_dict['y'],
same_probability=5.0)
train_data = datagen.flow(train_dict,
seed=seed,
crop_dim=crop_dim,
batch_size=batch_size,
min_track_length=min_track_length,
neighborhood_scale_size=neighborhood_scale_size,
features=features)
val_data = datagen_val.flow(
val_dict,
seed=seed,
crop_dim=crop_dim,
batch_size=batch_size,
min_track_length=min_track_length,
neighborhood_scale_size=neighborhood_scale_size,
features=features)
print('total_train_pairs:', total_train_pairs)
print('total_test_pairs:', total_test_pairs)
print('batch size:', batch_size)
print('validation_steps: ', total_test_pairs // batch_size)
# Make dicts to map the two generator outputs to the Dataset and model
# input here is model input and output is model output
features = sorted(features)
input_type_dict = {}
input_shape_dict = {}
for feature in features:
feature_name1 = '{}_input1'.format(feature)
feature_name2 = '{}_input2'.format(feature)
input_type_dict[feature_name1] = tf.float32
input_type_dict[feature_name2] = tf.float32
if feature == 'appearance':
app1 = tuple([
None, train_data.min_track_length, train_data.crop_dim,
train_data.crop_dim, 1
])
app2 = tuple(
[None, 1, train_data.crop_dim, train_data.crop_dim, 1])
input_shape_dict[feature_name1] = app1
input_shape_dict[feature_name2] = app2
elif feature == 'distance':
dist1 = tuple([None, train_data.min_track_length, 2])
dist2 = tuple([None, 1, 2])
input_shape_dict[feature_name1] = dist1
input_shape_dict[feature_name2] = dist2
elif feature == 'neighborhood':
neighborhood_size = 2 * train_data.neighborhood_scale_size + 1
neigh1 = tuple([
None, train_data.min_track_length, neighborhood_size,
neighborhood_size, 1
])
neigh2 = tuple([None, 1, neighborhood_size, neighborhood_size, 1])
input_shape_dict[feature_name1] = neigh1
input_shape_dict[feature_name2] = neigh2
elif feature == 'regionprop':
rprop1 = tuple([None, train_data.min_track_length, 3])
rprop2 = tuple([None, 1, 3])
input_shape_dict[feature_name1] = rprop1
input_shape_dict[feature_name2] = rprop2
output_type_dict = {'classification': tf.int32}
# Ouput_shape has to be None because we dont know how many cells
output_shape_dict = {'classification': (None, 3)}
train_dataset = Dataset.from_generator(lambda: train_data,
(input_type_dict, output_type_dict),
output_shapes=(input_shape_dict,
output_shape_dict))
val_dataset = Dataset.from_generator(lambda: val_data,
(input_type_dict, output_type_dict),
output_shapes=(input_shape_dict,
output_shape_dict))
train_callbacks = get_callbacks(model_path,
lr_sched=lr_sched,
tensorboard_log_dir=log_dir,
save_weights_only=num_gpus >= 2,
monitor='val_loss',
verbose=1)
# fit the model on the batches generated by datagen.flow()
loss_history = model.fit(train_dataset,
steps_per_epoch=total_train_pairs // batch_size,
epochs=n_epoch,
validation_data=val_dataset,
validation_steps=total_test_pairs // batch_size,
callbacks=train_callbacks)
np.savez(loss_path, loss_history=loss_history.history)
return model
def train_model_siamese(model=None,
dataset=None,
optimizer=None,
expt='',
it=0,
batch_size=1,
n_epoch=100,
direc_save='/data/models',
direc_data='/data/npz_data',
focal=False,
gamma=0.5,
lr_sched=rate_scheduler(lr=0.01, decay=0.95),
rotation_range=0,
flip=True,
shear=0,
class_weight=None):
is_channels_first = K.image_data_format() == 'channels_first'
training_data_file_name = os.path.join(direc_data, dataset + '.npz')
todays_date = datetime.datetime.now().strftime('%Y-%m-%d')
file_name_save = os.path.join(
direc_save, '{}_{}_{}_{}.h5'.format(todays_date, dataset, expt, it))
file_name_save_loss = os.path.join(
direc_save, '{}_{}_{}_{}.npz'.format(todays_date, dataset, expt, it))
train_dict, test_dict = get_data(training_data_file_name, mode='siamese')
class_weights = train_dict['class_weights']
# the data, shuffled and split between train and test sets
print('X_train shape:', train_dict['X'].shape)
print('y_train shape:', train_dict['y'].shape)
print('X_test shape:', test_dict['X'].shape)
print('y_test shape:', test_dict['y'].shape)
print('Output Shape:', model.layers[-1].output_shape)
n_classes = model.layers[-1].output_shape[1 if is_channels_first else -1]
def loss_function(y_true, y_pred):
if focal:
return losses.weighted_focal_loss(y_true,
y_pred,
gamma=gamma,
n_classes=n_classes,
from_logits=False)
else:
return losses.weighted_categorical_crossentropy(
y_true, y_pred, n_classes=n_classes, from_logits=False)
model.compile(loss=loss_function,
optimizer=optimizer,
metrics=['accuracy'])
print('Using real-time data augmentation.')
# this will do preprocessing and realtime data augmentation
datagen = generators.SiameseDataGenerator(
rotation_range=
rotation_range, # randomly rotate images by 0 to rotation_range degrees
shear_range=
shear, # randomly shear images in the range (radians , -shear_range to shear_range)
horizontal_flip=flip, # randomly flip images
vertical_flip=flip) # randomly flip images
datagen_val = generators.SiameseDataGenerator(
rotation_range=
0, # randomly rotate images by 0 to rotation_range degrees
shear_range=
0, # randomly shear images in the range (radians , -shear_range to shear_range)
horizontal_flip=0, # randomly flip images
vertical_flip=0) # randomly flip images
def count_pairs(y):
"""
Compute number of training samples needed to (stastically speaking)
observe all cell pairs.
Assume that the number of images is encoded in the second dimension.
Assume that y values are a cell-uniquely-labeled mask.
Assume that a cell is paired with one of its other frames 50% of the time
and a frame from another cell 50% of the time.
"""
# TODO: channels_first axes
total_pairs = 0
for image_set in range(y.shape[0]):
set_cells = 0
cells_per_image = []
for image in range(y.shape[1]):
image_cells = int(y[image_set, image, :, :, :].max())
set_cells = set_cells + image_cells
cells_per_image.append(image_cells)
# Since there are many more possible non-self pairings than there are self pairings,
# we want to estimate the number of possible non-self pairings and then multiply
# that number by two, since the odds of getting a non-self pairing are 50%, to
# find out how many pairs we would need to sample to (statistically speaking)
# observe all possible cell-frame pairs.
# We're going to assume that the average cell is present in every frame. This will
# lead to an underestimate of the number of possible non-self pairings, but it's
# unclear how significant the underestimate is.
average_cells_per_frame = int(
sum(cells_per_image) / len(cells_per_image))
non_self_cellframes = (average_cells_per_frame -
1) * len(cells_per_image)
non_self_pairings = non_self_cellframes * max(cells_per_image)
cell_pairings = non_self_pairings * 2
total_pairs = total_pairs + cell_pairings
return total_pairs
# This shouldn't remain long term.
magic_number = 2048 # A power of 2 chosen just to reduce training time.
total_train_pairs = count_pairs(train_dict['y'])
total_train_pairs = int(total_train_pairs // magic_number)
total_test_pairs = count_pairs(test_dict['y'])
total_test_pairs = int(total_test_pairs // magic_number)
# fit the model on the batches generated by datagen.flow()
loss_history = model.fit_generator(
datagen.flow(train_dict, batch_size=batch_size),
steps_per_epoch=total_train_pairs // batch_size,
epochs=n_epoch,
validation_data=datagen_val.flow(test_dict, batch_size=batch_size),
validation_steps=total_test_pairs // batch_size,
callbacks=[
callbacks.LearningRateScheduler(lr_sched),
callbacks.ModelCheckpoint(file_name_save,
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=num_gpus >= 2),
])
model.save_weights(file_name_save)
np.savez(file_name_save_loss, loss_history=loss_history.history)
return model
