def setup_clbks(conf: Config,
setup_tb=False,
setup_mc=True,
setup_ts=False,
setup_es=False):
logm('Setting up callbacks', cur_frame=currentframe(), mtype='I')
callbacks = []
if setup_tb:
logm('Setting up tensorboard', cur_frame=currentframe(), mtype='I')
if conf.use_tb_embeddings:
paths, labels = parse_csv(conf.test_data_csv, conf.data_path)
test_set = TestDataset(paths, labels)
with open(os.path.join(conf.log_dir, 'metadata.tsv'), 'w') as f:
np.savetxt(f, labels, delimiter=",", fmt='%s')
logm(
f'Loading test data ({len(paths)} samples) '
'for tensorboard callback...',
cur_frame=currentframe(),
mtype='I')
y_test = test_set()[1]
x_test = np.asarray([conf.data_loader(x) for x in test_set()[0]])
logm(
f'Loading test data ({len(paths)} samples) for '
'tensorboard callback... Done',
cur_frame=currentframe(),
mtype='I')
print('x_test shape:', x_test.shape)
tb = TensorBoard(log_dir=os.path.join(conf.log_dir, 'tensorboard'),
histogram_freq=1,
batch_size=conf.batch_size,
write_graph=True,
write_grads=True,
write_images=True,
embeddings_freq=5,
embeddings_layer_names=['features'],
embeddings_metadata='metadata.tsv',
embeddings_data=x_test)
else:
tb = TensorBoard(log_dir=conf.log_dir,
histogram_freq=1,
batch_size=conf.batch_size,
write_graph=True)
callbacks.append(tb)
if setup_ts:
logm('Setting up TimedStopping', cur_frame=currentframe(), mtype='I')
if conf.max_seconds_per_run:
callbacks.append(TimedStopping(conf.max_seconds_per_run))
else:
logm(
'Could not set up TimedStopping: '
'conf.max_seconds_per_run is set as None',
cur_frame=currentframe(),
mtype='W')
if setup_es:
logm('Setting up EarlyStopping', cur_frame=currentframe(), mtype='I')
callbacks.append(EarlyStopping(patience=5))
if setup_mc:
logm('Setting up ModelCheckpoint', cur_frame=currentframe(), mtype='I')
callbacks.append(
ModelCheckpoint(f'{conf.model_checkpoint_location}.h5',
period=1,
save_best_only=True))
logm('Setting up callbacks... Done', cur_frame=currentframe(), mtype='I')
return callbacks
def train(model: keras.Model, conf: Config, developing=False):
logm(f'Running train for {conf}', cur_frame=currentframe(), mtype='I')
paths, labels = parse_csv(conf.train_data_csv, conf.data_path)
if developing:
logm('Developing is set as true: limiting size of dataset',
cur_frame=currentframe(),
mtype='I')
paths_labels = list(zip(paths, labels))
random.shuffle(paths_labels)
paths, labels = zip(*paths_labels)
paths = paths[:100]
labels = labels[:100]
epochs = 2
conf.steps_per_epoch = 10
dataset = Dataset(paths,
labels,
shuffle=True,
val_split=0.005,
name=conf.dataset_name)
logm(f'Loading validation data...', cur_frame=currentframe(), mtype='I')
X_val = np.asarray([conf.data_loader(x) for x in dataset.validation[0]])
y_val = dataset.validation[1]
logm(f'Loading validation data... Done',
cur_frame=currentframe(),
mtype='I')
logm(f'Validation data shape is {X_val.shape}',
cur_frame=currentframe(),
mtype='I')
if conf.use_generator:
logm('Using generator', cur_frame=currentframe(), mtype='I')
batch_size = (conf.batch_size
if conf.batch_size < len(dataset.train[0]) else len(
dataset.train[0]))
train_gen = Generator(paths=dataset.train[0],
labels=dataset.train[1],
loader_fn=conf.data_loader,
batch_size=batch_size,
heap=setup_heap(conf),
expected_shape=(SPEC_SHAPE_HEIGTH,
SPEC_SHAPE_WIDTH, 3))
history = model.fit_generator(
generator=train_gen,
validation_data=(X_val, y_val),
use_multiprocessing=True,
max_queue_size=96,
workers=12,
steps_per_epoch=conf.steps_per_epoch,
epochs=conf.epochs if not developing else 2,
callbacks=setup_clbks(conf, conf.log_dir),
verbose=1)
else:
X_train = np.asarray([conf.data_loader(x) for x in dataset.train[0]])
y_train = dataset.train[1]
history = model.fit(x=X_train,
y=y_train,
batch_size=conf.batch_size,
validation_data=(X_val, y_val),
epochs=conf.epochs if not developing else 2,
callbacks=setup_clbks(conf, conf.log_dir),
verbose=1)
pickle.dump(history,
open(os.path.join(conf.log_dir, f'history_{conf}.pkl'), 'wb'),
protocol=3)
return history
def main(conf: Config):
model = build_model(conf)
conf.model_name = 'mit'
logm(f'Current configuration is:\n{repr(conf)}',
cur_frame=currentframe(),
mtype='I')
model.summary()
setup_dirs(conf)
if TRAIN:
logm(f'Start train: {str(conf)}', cur_frame=currentframe(), mtype='I')
with Timer() as t:
train(model, conf, developing=DEVELOPING)
logm(f'End train: total time taken: {str(t.interval)}',
cur_frame=currentframe(),
mtype='I')
if SAVE_MODEL:
logm(f'Saving model: {str(conf.model_name)} at {conf.model_location}',
cur_frame=currentframe(),
mtype='I')
model.save(conf.model_location + '.h5')
if EVALUATE:
logm(f'Start evaluation: {str(conf)}',
cur_frame=currentframe(),
mtype='I')
test_paths, test_labels = parse_csv(conf.test_data_csv, conf.data_path)
if DEVELOPING:
logm('Developing is set as true: limiting size of dataset',
cur_frame=currentframe(),
mtype='I')
paths_labels = list(zip(test_paths, test_labels))
random.shuffle(paths_labels)
test_paths, test_labels = zip(*paths_labels)
test_paths = test_paths[:10]
test_labels = test_labels[:10]
logm(f'Get test data: total of {len(test_labels)}',
cur_frame=currentframe(),
mtype='I')
if not TRAIN:
logm(f'Loading model from {conf.model_location}',
cur_frame=currentframe(),
mtype='I')
if conf.model_location is None:
logm(
'Model location is None: '
'Load the configuration file of a valid trained '
'model',
cur_frame=currentframe(),
mtype='E')
raise ValueError('Not valid configuration file for evaluation')
model = load_model(conf.model_location + '.h5')
logm(f'Start evaluation of {str(conf)}',
cur_frame=currentframe(),
mtype='I')
test_dataset = TestDataset(test_paths,
test_labels,
name=conf.dataset_name,
num_classes=conf.num_classes)
with Timer() as t:
evaluate(model, conf, test_dataset, 'final')
logm(f'End evaluation: total time taken: {str(t.interval)}',
cur_frame=currentframe(),
mtype='I')
if EVALUATE_BEST_MODEL:
logm('Evaluating best model', cur_frame=currentframe(), mtype='I')
logm(f'Loading model from {conf.model_checkpoint_location}',
cur_frame=currentframe(),
mtype='I')
model = load_model(conf.model_checkpoint_location + '.h5')
logm(f'Start evaluation of {str(conf)} (best model)',
cur_frame=currentframe(),
mtype='I')
with Timer() as t:
evaluate(model, conf, test_dataset, 'best')
logm(
'End evaluation (best model): total time taken: '
f'{str(t.interval)}',
cur_frame=currentframe(),
mtype='I')
def train(model: Model, conf: Config, batch_size, developing=False):
logm(f'Running train for {conf}', cur_frame=currentframe(), mtype='I')
train_paths, train_labels = parse_csv(conf.train_data_csv, conf.data_path)
val_paths, val_labels = parse_csv(conf.eval_data_csv, conf.data_path)
if developing:
logm('Developing is set as true: limiting size of dataset',
cur_frame=currentframe(),
mtype='I')
train_paths_labels = list(zip(train_paths, train_labels))
random.shuffle(train_paths_labels)
train_paths, train_labels = zip(*train_paths_labels)
train_paths = train_paths[:100]
train_labels = train_labels[:100]
val_paths = val_paths[:10]
val_labels = val_labels[:10]
epochs = 2
conf.steps_per_epoch = 10
train_dataset = Dataset(train_paths,
train_labels,
name=conf.dataset_name + '[TRAIN]',
num_classes=NUM_CLASSES)
val_dataset = Dataset(val_paths,
val_labels,
name=conf.dataset_name + '[VALIDATION]',
num_classes=NUM_CLASSES)
epochs = int(len(train_dataset) // (batch_size * conf.steps_per_epoch)) + 1
logm(
f'Calculated number of epochs to process all data at least 1 time: {epochs}',
cur_frame=currentframe())
logm(f'Loading validation data...', cur_frame=currentframe(), mtype='I')
X_val = np.asarray([conf.data_loader(x) for x in val_dataset()[0]])
y_val = val_dataset()[1]
logm(f'Loading validation data... Done',
cur_frame=currentframe(),
mtype='I')
logm(f'Validation data shape is {X_val.shape}',
cur_frame=currentframe(),
mtype='I')
if conf.use_generator:
logm('Using generator', cur_frame=currentframe(), mtype='I')
train_gen = Generator(paths=train_dataset()[0],
labels=train_dataset()[1],
loader_fn=conf.data_loader,
batch_size=int(batch_size),
heap=setup_heap(conf) if conf.use_heap else None,
expected_shape=(SPEC_SHAPE_HEIGTH,
SPEC_SHAPE_WIDTH, CHANNELS))
history = model.fit_generator(generator=train_gen,
validation_data=(X_val, y_val),
use_multiprocessing=True,
max_queue_size=96,
workers=12,
steps_per_epoch=conf.steps_per_epoch,
epochs=epochs,
callbacks=setup_clbks(conf),
verbose=1)
else:
X_train = np.asarray([conf.data_loader(x) for x in train_dataset()[0]])
y_train = train_dataset()[1]
history = model.fit(x=X_train,
y=y_train,
batch_size=int(batch_size),
validation_data=(X_val, y_val),
epochs=epochs,
callbacks=setup_clbks(conf),
verbose=1)
return history, model