def run(image_shape, batch_size, device, data_dir, ckpt_file, dataset_seed,
train_n_patches, test_n_patches, patches_used_in_eval, arch, n_jobs):
tf_config = tf.ConfigProto(allow_soft_placement=True)
sess = tf.Session(config=tf_config)
K.set_session(sess)
vangogh = VanGogh(base_dir=data_dir,
image_shape=image_shape,
random_state=dataset_seed,
train_n_patches=train_n_patches,
valid_n_patches=train_n_patches,
test_n_patches=test_n_patches,
n_jobs=n_jobs)
phases = ('train', 'valid', 'test')
print('loading data...')
start = time()
data = dict(zip(phases, vangogh.load_patches(*phases)))
print('done (%.1f sec).' % (time() - start))
data['train'] = list(map(np.concatenate, zip(data['train'],
data['valid'])))
del data['valid']
data['train'][0] /= 127.5
data['train'][0] -= 1.
data['test'][0] /= 127.5
data['test'][0] -= 1.
with tf.device(device):
model = build_siamese_model(x_shape=image_shape,
arch=arch,
weights=None)
model.load_weights(ckpt_file)
scores = evaluate(model=model,
data=data,
batch_size=batch_size,
patches_used=patches_used_in_eval)
print('max test score: %.2f%%' % (100 * scores['test']))
def run(dataset_seed, image_shape, batch_size, device, data_dir, output_dir,
phases, architecture,
o_meta, limb_weights, joint_weights, weights, pooling,
dense_layers, use_gram_matrix, last_base_layer, override,
embedded_files_max_size, selected_layers):
os.makedirs(output_dir, exist_ok=True)
with tf.device(device):
print('building model...')
model = build_siamese_model(image_shape, architecture, 0.0, weights,
last_base_layer=last_base_layer,
use_gram_matrix=use_gram_matrix,
dense_layers=dense_layers, pooling=pooling,
include_base_top=False, include_top=True,
trainable_limbs=False,
limb_weights=limb_weights,
predictions_activation=[o['a'] for o in o_meta],
predictions_name=[o['n'] for o in o_meta],
classes=[o['u'] for o in o_meta],
embedding_units=[o['e'] for o in o_meta],
joints=[o['j'] for o in o_meta])
# Restore best parameters.
print('loading weights from:', joint_weights)
model.load_weights(joint_weights)
model = model.get_layer('model_2')
available_layers = [l.name for l in model.layers]
if set(selected_layers) - set(available_layers):
print('available layers:', available_layers)
raise ValueError('selection contains unknown layers: %s' % selected_layers)
style_features = [model.get_layer(l).output for l in selected_layers]
if use_gram_matrix:
gram_layer = layers.Lambda(gram_matrix, arguments=dict(norm_by_channels=False))
style_features = [gram_layer(f) for f in style_features]
model = Model(inputs=model.inputs, outputs=style_features)
g = ImageDataGenerator(preprocessing_function=get_preprocess_fn(architecture))
for phase in phases:
phase_data_dir = os.path.join(data_dir, phase)
output_file_name = os.path.join(output_dir, phase + '.%i.pickle')
already_embedded = os.path.exists(output_file_name % 0)
phase_exists = os.path.exists(phase_data_dir)
if already_embedded and not override or not phase_exists:
print('%s transformation skipped' % phase)
continue
# Shuffle must always be off in order to keep names consistent.
data = g.flow_from_directory(phase_data_dir,
target_size=image_shape[:2],
class_mode='sparse',
batch_size=batch_size, shuffle=False,
seed=dataset_seed)
print('transforming %i %s samples from %s' % (data.n, phase, phase_data_dir))
part_id = 0
samples_seen = 0
displayed_once = False
while samples_seen < data.n:
z, y = {n: [] for n in selected_layers}, []
chunk_size = 0
chunk_start = samples_seen
while chunk_size < embedded_files_max_size and samples_seen < data.n:
_x, _y = next(data)
outputs = model.predict_on_batch(_x)
chunk_size += sum(o.nbytes for o in outputs)
for l, o in zip(selected_layers, outputs):
z[l].append(o)
y.append(_y)
samples_seen += _x.shape[0]
chunk_p = int(100 * (samples_seen / data.n))
if chunk_p % 10 == 0:
if not displayed_once:
print('\n%i%% (%.2f MB)'
% (chunk_p, chunk_size / 1024 ** 2),
flush=True, end='')
displayed_once = True
else:
displayed_once = False
print('.', end='')
for layer in selected_layers:
z[layer] = np.concatenate(z[layer])
with open(output_file_name % part_id, 'wb') as f:
pickle.dump({'data': z,
'target': np.concatenate(y),
'names': np.asarray(data.filenames[chunk_start: samples_seen])},
f, pickle.HIGHEST_PROTOCOL)
part_id += 1
print('done.')
def run(_run, image_shape, data_dir, patches, estimator_type, submission_info,
solution, architecture, weights, batch_size, last_base_layer,
use_gram_matrix, pooling, dense_layers, device, num_classes,
limb_weights, predictions_activation, joint, embedding_units,
dropout_rate, ckpt, results_file, submission_file, use_multiprocessing,
workers):
report_dir = _run.observers[0].dir
with tf.device(device):
print('building...')
model = build_siamese_model(
image_shape,
architecture,
dropout_rate,
weights,
num_classes,
last_base_layer,
use_gram_matrix,
dense_layers,
pooling,
include_base_top=False,
include_top=True,
predictions_activation=predictions_activation,
limb_weights=limb_weights,
trainable_limbs=False,
embedding_units=embedding_units,
joints=joint)
model.summary()
print('loading weights from', ckpt)
model.load_weights(ckpt)
limb, rest = model.get_layer('model_2'), model.layers[3:]
x = i = Input(shape=(num_classes, ))
for l in limb.layers[-5:]:
x = l(x)
limb = Model(inputs=i, outputs=x)
ia, ib = Input(shape=(num_classes, )), Input(shape=(num_classes, ))
ya = limb(ia)
yb = limb(ib)
x = [ya, yb]
for l in rest:
x = l(x)
model = Model(inputs=[ia, ib], outputs=x)
print('loading submission and solution...')
pairs = pd.read_csv(submission_info, quotechar='"',
delimiter=',').values[:, 1:]
labels = pd.read_csv(solution, quotechar='"',
delimiter=',').values[:, 1:].flatten()
print('loading sequential predictions...')
d = load_pickle_data(data_dir, phases=['test'], keys=['data', 'names'])
d, names = d['test']
samples = d['predictions']
del d
samples, names = group_by_paintings(samples, names=names)
names = np.asarray([n.split('/')[1] + '.jpg' for n in names])
print('test data shape:', samples.shape)
print('\n# test evaluation')
test_data = ArrayPairsSequence(samples, names, pairs, labels,
batch_size)
probabilities = model.predict_generator(
test_data,
use_multiprocessing=use_multiprocessing,
workers=workers,
verbose=1).reshape(-1, patches)
del model
K.clear_session()
layer_results = evaluate(labels, probabilities, estimator_type)
layer_results['phase'] = 'test'
evaluation_results = [layer_results]
# generate results file.
with open(os.path.join(report_dir, results_file), 'w') as file:
json.dump(evaluation_results, file)
# generate submission file to Kaggle.
for v in layer_results['evaluations']:
predictions_field = 'binary_probabilities' if 'binary_probabilities' in v else 'p'
p = v[predictions_field]
with open(submission_file.format(strategy=v['strategy']), 'w') as f:
f.write('index,sameArtist\n')
f.writelines(['%i,%f\n' % (i, _p) for i, _p in enumerate(p)])
def run(_run, image_shape, train_info, data_dir, train_pairs, valid_pairs,
train_shuffle, valid_shuffle, subdirectories, architecture, weights,
batch_size, last_base_layer, pooling, device, opt_params, dropout_rate,
resuming_ckpt, ckpt, steps_per_epoch, epochs, validation_steps,
workers, use_multiprocessing, initial_epoch, early_stop_patience,
use_gram_matrix, dense_layers, limb_weights, trainable_limbs,
outputs_meta):
report_dir = _run.observers[0].dir
print('reading train-info...')
outputs, name_map = load_multiple_outputs(train_info,
outputs_meta,
encode='sparse')
g = ImageDataGenerator(
horizontal_flip=True,
vertical_flip=True,
zoom_range=.2,
rotation_range=.2,
height_shift_range=.2,
width_shift_range=.2,
fill_mode='reflect',
preprocessing_function=get_preprocess_fn(architecture))
print('loading train meta-data...')
train_data = BalancedDirectoryPairsMultipleOutputsSequence(
os.path.join(data_dir, 'train'),
outputs,
name_map,
g,
batch_size=batch_size,
target_size=image_shape[:2],
subdirectories=subdirectories,
shuffle=train_shuffle,
pairs=train_pairs)
print('loading valid meta-data...')
valid_data = BalancedDirectoryPairsMultipleOutputsSequence(
os.path.join(data_dir, 'valid'),
outputs,
name_map,
g,
batch_size=batch_size,
target_size=image_shape[:2],
subdirectories=subdirectories,
shuffle=valid_shuffle,
pairs=valid_pairs)
with tf.device(device):
print('building...')
model = build_siamese_model(
image_shape,
architecture,
dropout_rate,
weights,
last_base_layer=last_base_layer,
use_gram_matrix=use_gram_matrix,
dense_layers=dense_layers,
pooling=pooling,
include_base_top=False,
include_top=True,
trainable_limbs=trainable_limbs,
limb_weights=limb_weights,
predictions_activation=[o['a'] for o in outputs_meta],
predictions_name=[o['n'] for o in outputs_meta],
classes=[o['u'] for o in outputs_meta],
embedding_units=[o['e'] for o in outputs_meta],
joints=[o['j'] for o in outputs_meta])
print('siamese model summary:')
model.summary()
if resuming_ckpt:
print('loading weights...')
model.load_weights(resuming_ckpt)
model.compile(optimizer=optimizers.Adam(**opt_params),
loss=dict((o['n'] + '_binary_predictions', o['l'])
for o in outputs_meta),
metrics=dict((o['n'] + '_binary_predictions', o['m'])
for o in outputs_meta))
print('training from epoch %i...' % initial_epoch)
try:
model.fit_generator(
train_data,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=valid_data,
validation_steps=validation_steps,
initial_epoch=initial_epoch,
use_multiprocessing=use_multiprocessing,
workers=workers,
verbose=2,
callbacks=[
callbacks.TerminateOnNaN(),
callbacks.EarlyStopping(patience=early_stop_patience),
callbacks.ReduceLROnPlateau(min_lr=1e-10,
patience=early_stop_patience //
3),
callbacks.TensorBoard(report_dir, batch_size=batch_size),
callbacks.ModelCheckpoint(os.path.join(report_dir, ckpt),
save_best_only=True,
verbose=1),
])
except KeyboardInterrupt:
print('interrupted by user')
else:
print('done')
def run(image_shape, data_dir, valid_pairs, classes, num_classes, architecture,
weights, batch_size, last_base_layer, pooling, device,
predictions_activation, dropout_rate, ckpt, validation_steps,
use_multiprocessing, use_gram_matrix, dense_layers, embedding_units,
limb_weights, trainable_limbs):
if isinstance(classes, int):
classes = sorted(os.listdir(os.path.join(data_dir, 'train')))[:classes]
g = ImageDataGenerator(
preprocessing_function=utils.get_preprocess_fn(architecture))
valid_data = BalancedDirectoryPairsSequence(os.path.join(
data_dir, 'valid'),
g,
target_size=image_shape[:2],
pairs=valid_pairs,
classes=classes,
batch_size=batch_size)
if validation_steps is None:
validation_steps = len(valid_data)
with tf.device(device):
print('building...')
model = build_siamese_model(
image_shape,
architecture,
dropout_rate,
weights,
num_classes,
last_base_layer,
use_gram_matrix,
dense_layers,
pooling,
include_base_top=False,
include_top=True,
predictions_activation=predictions_activation,
limb_weights=limb_weights,
trainable_limbs=trainable_limbs,
embedding_units=embedding_units,
joints='multiply')
print('siamese model summary:')
model.summary()
if ckpt:
print('loading weights...')
model.load_weights(ckpt)
enqueuer = None
try:
enqueuer = OrderedEnqueuer(valid_data,
use_multiprocessing=use_multiprocessing)
enqueuer.start()
output_generator = enqueuer.get()
y, p = [], []
for step in range(validation_steps):
x, _y = next(output_generator)
_p = model.predict(x, batch_size=batch_size)
y.append(_y)
p.append(_p)
y, p = (np.concatenate(e).flatten() for e in (y, p))
print('actual:', y[:80])
print('expected:', p[:80])
print('accuracy:', metrics.accuracy_score(y, p >= 0.5))
print(metrics.classification_report(y, p >= 0.5))
print(metrics.confusion_matrix(y, p >= 0.5))
finally:
if enqueuer is not None:
enqueuer.stop()
def run(batch_size, data_dir, train_dir, valid_dir, balanced_classes,
image_shape, train_augmentations, train_shuffle, train_dataset_seed,
valid_augmentations, valid_shuffle, valid_dataset_seed, device,
opt_params, ckpt_file, pre_trained_weights, arch, network_weights,
train_samples_per_epoch, nb_epoch, nb_val_samples, nb_worker,
early_stop_patience, reduce_lr_on_plateau_patience, tensorboard_file,
initial_epoch):
os.makedirs(os.path.dirname(ckpt_file), exist_ok=True)
tf_config = tf.ConfigProto(allow_soft_placement=True)
tf_config.gpu_options.allow_growth = True
sess = tf.Session(config=tf_config)
K.set_session(sess)
g = ImageDataGenerator(featurewise_center=True,
horizontal_flip=True,
vertical_flip=True,
rotation_range=30,
height_shift_range=.2,
width_shift_range=.2,
rescale=2. / 255.,
fill_mode='reflect')
g.mean = 1.
train_data = BalancedDirectoryPairsSequence(
train_dir,
image_data_generator=g,
target_size=image_shape[:2],
augmentations=train_augmentations,
batch_size=batch_size,
balanced_classes=balanced_classes,
shuffle=train_shuffle,
seed=train_dataset_seed)
valid_data = BalancedDirectoryPairsSequence(
valid_dir,
image_data_generator=g,
target_size=image_shape[:2],
augmentations=valid_augmentations,
batch_size=batch_size,
balanced_classes=balanced_classes,
shuffle=valid_shuffle,
seed=valid_dataset_seed)
with tf.device(device):
model = build_siamese_model(x_shape=image_shape,
arch=arch,
weights=network_weights)
opt = optimizers.Adam(**opt_params)
model.compile(optimizer=opt, loss=contrastive_loss)
if pre_trained_weights:
model.load_weights(pre_trained_weights)
elif initial_epoch > 0:
print('re-loading weights...')
model.load_weights(ckpt_file)
print('training network...')
try:
model.fit_generator(
train_data,
samples_per_epoch=train_samples_per_epoch,
nb_epoch=nb_epoch,
validation_data=valid_data,
nb_val_samples=nb_val_samples,
nb_worker=nb_worker,
callbacks=[
callbacks.ReduceLROnPlateau(
min_lr=1e-10, patience=reduce_lr_on_plateau_patience),
callbacks.EarlyStopping(patience=early_stop_patience),
callbacks.TensorBoard(tensorboard_file, histogram_freq=5),
callbacks.ModelCheckpoint(ckpt_file,
verbose=1,
save_best_only=True),
],
initial_epoch=initial_epoch)
except KeyboardInterrupt:
print('training interrupted by user.')
else:
print('done.')
def run(_run, image_shape, data_dir, train_pairs, valid_pairs, classes,
num_classes, architecture, weights, batch_size, last_base_layer,
pooling, device, predictions_activation, opt_params, dropout_rate,
resuming_ckpt, ckpt, steps_per_epoch, epochs, validation_steps, joints,
workers, use_multiprocessing, initial_epoch, early_stop_patience,
use_gram_matrix, dense_layers, embedding_units, limb_weights,
trainable_limbs, tensorboard_tag):
report_dir = _run.observers[0].dir
if isinstance(classes, int):
classes = sorted(os.listdir(os.path.join(data_dir, 'train')))[:classes]
g = ImageDataGenerator(
horizontal_flip=True,
vertical_flip=True,
zoom_range=.2,
rotation_range=.2,
height_shift_range=.2,
width_shift_range=.2,
fill_mode='reflect',
preprocessing_function=utils.get_preprocess_fn(architecture))
train_data = BalancedDirectoryPairsSequence(os.path.join(
data_dir, 'train'),
g,
target_size=image_shape[:2],
pairs=train_pairs,
classes=classes,
batch_size=batch_size)
valid_data = BalancedDirectoryPairsSequence(os.path.join(
data_dir, 'valid'),
g,
target_size=image_shape[:2],
pairs=valid_pairs,
classes=classes,
batch_size=batch_size)
if steps_per_epoch is None:
steps_per_epoch = len(train_data)
if validation_steps is None:
validation_steps = len(valid_data)
with tf.device(device):
print('building...')
model = build_siamese_model(
image_shape,
architecture,
dropout_rate,
weights,
num_classes,
last_base_layer,
use_gram_matrix,
dense_layers,
pooling,
include_base_top=False,
include_top=True,
predictions_activation=predictions_activation,
limb_weights=limb_weights,
trainable_limbs=trainable_limbs,
embedding_units=embedding_units,
joints=joints)
print('siamese model summary:')
model.summary()
if resuming_ckpt:
print('loading weights...')
model.load_weights(resuming_ckpt)
model.compile(loss='binary_crossentropy',
metrics=['accuracy'],
optimizer=optimizers.Adam(**opt_params))
print('training from epoch %i...' % initial_epoch)
try:
model.fit_generator(
train_data,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=valid_data,
validation_steps=validation_steps,
initial_epoch=initial_epoch,
use_multiprocessing=use_multiprocessing,
workers=workers,
verbose=2,
callbacks=[
callbacks.TerminateOnNaN(),
callbacks.EarlyStopping(patience=early_stop_patience),
callbacks.ReduceLROnPlateau(min_lr=1e-10,
patience=int(
early_stop_patience // 3)),
callbacks.TensorBoard(os.path.join(report_dir,
tensorboard_tag),
batch_size=batch_size),
callbacks.ModelCheckpoint(os.path.join(report_dir, ckpt),
save_best_only=True,
save_weights_only=True,
verbose=1),
])
except KeyboardInterrupt:
print('interrupted by user')
else:
print('done')