def train():
# train_val_df, test_df, xray14_labels = load_from_text(cfg.data_root)
train_val_df, test_df, xray14_labels = load_from_npy()
train_df, valid_df = split_train_val(train_val_df, ratio=0.25)
print('*'*40, 'tain data', '*'*40)
describe_data(train_df, xray14_labels)
print('*' * 40, 'val data', '*' * 40)
describe_data(valid_df, xray14_labels)
train_gen, val_gen = creat_tain_val_generator(train_df, valid_df, cfg.input_shape[:-1],
batch_size=cfg.batch_size)
model = Xception(cfg.input_shape, include_top=True, n_class=len(xray14_labels), pretrain_weights='imagenet')
model.compile(optimizer=Adam(), loss=[focal_loss()], metrics=['binary_accuracy', 'mae'])
log_path = os.path.join(cfg.log_dir, 'xray14_focal')
makedir(log_path)
weights_path = os.path.join(log_path, cfg.weights_name)
checkpoint = ModelCheckpoint(weights_path, monitor='val_loss',
verbose=1, save_best_only=True,
mode='min', save_weights_only=True)
callbacks = [checkpoint, LearningRateScheduler(lr_schedule)]
train_steps = get_number_of_steps(len(train_df), cfg.batch_size)*5
val_steps = get_number_of_steps(len(valid_df), cfg.batch_size)*2
model.fit_generator(train_gen, epochs=cfg.epochs,
steps_per_epoch=train_steps,
callbacks=callbacks,
validation_data=val_gen,
workers=cfg.n_works,
max_queue_size=cfg.n_queue,
use_multiprocessing=True,
validation_steps=val_steps,
initial_epoch=0)
def train():
train_val_df, test_df, xray14_labels = load_from_text(cfg.data_root)
train_df, val_df = split_patients_by_patient_ID(train_val_df, 4)
print('*' * 40, 'tain data', '*' * 40)
describe_data(train_df, xray14_labels)
print('*' * 40, 'val data', '*' * 40)
describe_data(val_df, xray14_labels)
train_transformer = ImageTransformer(samplewise_normalization=True,
rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.1,
zoom_range=[0.7, 1.5],
horizontal_flip=True)
val_transformer = ImageTransformer(samplewise_normalization=True)
train_gen = random_image_generator(train_transformer,
train_val_df,
cfg.input_shape[:-1],
xray14_labels,
batch_size=cfg.batch_size,
color_mode='grayscale')
val_gen = ImageGeneratorFromPath(val_transformer,
test_df['path'],
test_df['xray14_vec'],
shuffle=False,
target_size=cfg.input_shape[:-1],
batch_size=cfg.batch_size)
model = Xception(cfg.input_shape,
include_top=True,
n_class=len(xray14_labels),
pretrain_weights='imagenet')
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['binary_accuracy', 'mae'])
log_path = os.path.join(cfg.log_dir, 'random_disease')
makedir(log_path)
weights_path = os.path.join(log_path, cfg.weights_name)
checkpoint = ModelCheckpoint(weights_path,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min',
save_weights_only=True)
callbacks = [checkpoint]
train_steps = get_number_of_steps(len(train_df), cfg.batch_size)
val_steps = get_number_of_steps(len(val_df), cfg.batch_size)
model.fit_generator(train_gen,
epochs=cfg.epochs,
steps_per_epoch=train_steps,
callbacks=callbacks,
validation_data=val_gen,
workers=cfg.n_works,
max_queue_size=cfg.n_queue,
use_multiprocessing=True,
validation_steps=val_steps,
initial_epoch=0)
def train():
train_val_df = load_train_csv(cfg)
train_df, val_df = split_train_val(train_val_df, 0.25)
train_gen = BaseGenerator(train_df, cfg.train_dir, batch_size=cfg.batch_size,
aug_args=cfg.aug_args,
target_shape=cfg.input_shape[:2],
use_yellow=False)
val_gen = BaseGenerator(val_df, cfg.train_dir, batch_size=cfg.batch_size,
aug_args=cfg.aug_args,
target_shape=(512, 512),
use_yellow=False)
if n_gpus > 0:
with tf.device('/cpu:0'):
cpu_model = Xception(cfg.input_shape, include_top=True, n_class=len(cfg.label_names))
model = multi_gpu_model(cpu_model, gpus=n_gpus)
else:
model = Xception(cfg.input_shape, include_top=True, n_class=len(cfg.label_names))
model.compile(optimizer=Adam(1e-3), loss=roc_auc_loss,
metrics=['binary_accuracy', 'mae'])
log_dir = os.path.join(cfg.log_dir, 'base_xception')
makedir(log_dir)
weights_path = os.path.join(log_dir, cfg.weights_file)
checkpoint = ModelCheckpoint(weights_path, monitor='val_loss',
verbose=1, save_best_only=True,
mode='min', save_weights_only=True)
if n_gpus > 0:
del checkpoint
checkpoint = MultiGPUCheckpoint(weights_path, cpu_model, monitor='val_loss')
callbacks = [checkpoint]
callbacks += [ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, verbose=1, mode='min')]
train_steps = get_number_of_steps(len(train_df), cfg.batch_size)
val_steps = get_number_of_steps(len(val_df), cfg.batch_size)
model.fit_generator(train_gen,
epochs=cfg.epochs,
steps_per_epoch=train_steps,
callbacks=callbacks,
validation_data=val_gen,
workers=cfg.n_works,
max_queue_size=cfg.n_queue,
use_multiprocessing=True,
validation_steps=val_steps,
initial_epoch=0)
K.clear_session()
def pretrain(task_name='base_resnet',
epochs=10,
lr=1e-1,
target_shape=(512, 512),
trainable=True,
pretrain_weights='imagenet',
init_epoch=0):
cfg.input_shape = list(target_shape) + [3]
train_val_df = load_train_csv(cfg)
train_df, val_df = split_train_val(train_val_df, 0.25, seed=42)
train_gen = BaseGenerator(train_df,
cfg.train_dir,
batch_size=cfg.batch_size,
aug_args=cfg.aug_args.copy(),
target_shape=target_shape,
use_yellow=False,
preprocessing_function=preprocess_input)
val_gen = BaseGenerator(val_df,
cfg.train_dir,
batch_size=cfg.batch_size,
aug_args=cfg.aug_args.copy(),
target_shape=target_shape,
use_yellow=False,
preprocessing_function=preprocess_input)
if n_gpus > 1:
print('use multi gpu')
with tf.device('/cpu:0'):
cpu_model = ResNet50(input_shape=cfg.input_shape,
classes=len(cfg.label_names),
trainable=trainable,
weights=pretrain_weights)
model = multi_gpu_model(cpu_model, gpus=n_gpus)
else:
print('use single gpu')
model = ResNet50(input_shape=cfg.input_shape,
classes=len(cfg.label_names),
trainable=trainable,
weights=pretrain_weights)
model.compile(optimizer=KO.Adam(lr=lr, amsgrad=True),
loss='binary_crossentropy',
metrics=[f1_score, 'mae'])
log_dir = os.path.join(cfg.log_dir, task_name)
makedir(log_dir)
weights_path = os.path.join(log_dir, cfg.weights_file)
checkpoint = ModelCheckpoint(weights_path,
monitor='f1_score',
verbose=1,
save_best_only=True,
mode='max',
save_weights_only=True)
if n_gpus > 1:
del checkpoint
checkpoint = MultiGPUCheckpoint(weights_path,
cpu_model,
verbose=1,
monitor='f1_score',
mode='max',
save_weights_only=True,
save_best_only=True)
callbacks = [checkpoint]
callbacks += [
ReduceLROnPlateau(monitor='f1_score',
factor=0.5,
patience=3,
verbose=1,
mode='max')
]
# callbacks += [LearningRateScheduler(lr_schedule)]
train_steps = get_number_of_steps(len(train_df), cfg.batch_size)
val_steps = get_number_of_steps(len(val_df), cfg.batch_size)
model.fit_generator(train_gen,
epochs=epochs,
steps_per_epoch=train_steps,
callbacks=callbacks,
validation_data=val_gen,
workers=cfg.n_works,
max_queue_size=cfg.n_queue,
use_multiprocessing=True,
validation_steps=val_steps,
initial_epoch=init_epoch)
K.clear_session()
def train(exp_name='xray14_wce_06'):
# train_val_df, test_df, xray14_labels = load_from_text(cfg.data_root)
train_val_test_df_name = 'train_val_test_df.npy'
# np.save(train_val_test_df_name, [train_val_df, test_df, xray14_labels])
train_val_df, test_df, xray14_labels = np.load(train_val_test_df_name)
train_df, val_df = split_patients_by_patient_ID(train_val_df, 4,
cfg.random_seed)
print('*' * 40, 'tain data', '*' * 40)
describe_data(train_df, xray14_labels)
print('*' * 40, 'val data', '*' * 40)
describe_data(val_df, xray14_labels)
train_transformer = ImageTransformer(samplewise_normalization=True,
rotation_range=10,
width_shift_range=0.05,
height_shift_range=0.1,
shear_range=0.1,
zoom_range=[0.8, 1.2],
horizontal_flip=True)
val_transformer = ImageTransformer(samplewise_normalization=True)
train_gen = train_gennerator(train_transformer,
train_val_df,
cfg.input_shape[:-1],
xray14_labels,
batch_size=cfg.batch_size,
color_mode='rgb')
val_gen = ImageGeneratorFromPath(val_transformer,
test_df['path'],
test_df['xray14_vec'],
shuffle=False,
target_size=cfg.input_shape[:-1],
batch_size=cfg.batch_size,
color_mode='rgb')
cb_gen = ImageGeneratorFromPath(val_transformer,
test_df['path'],
test_df['xray14_vec'],
shuffle=False,
target_size=cfg.input_shape[:-1],
batch_size=cfg.batch_size,
color_mode='rgb')
log_path = os.path.join(cfg.log_dir, exp_name)
makedir(log_path)
trainable = True
if cfg.n_gpus > 1:
with tf.device('/cpu:0'):
base_model = Xception_CBAM(cfg.input_shape,
include_top=True,
classes=len(xray14_labels),
pretrain_weights='imagenet',
layer_trainable=trainable)
parallel_model = multi_gpu_model(base_model, gpus=cfg.n_gpus)
callbacks = get_callbacks(cb_gen,
xray14_labels,
log_path,
base_model=base_model)
model = parallel_model
else:
model = Xception_CBAM(cfg.input_shape,
include_top=True,
classes=len(xray14_labels),
pretrain_weights='imagenet',
layer_trainable=trainable)
callbacks = get_callbacks(cb_gen, xray14_labels, log_path)
base_model = model
class_weights = get_class_weight(train_val_df, xray14_labels)
print('class weights ---', class_weights)
model.compile(optimizer=Adam(1e-3),
loss=weight_binary_ce(class_weights),
metrics=['binary_accuracy', 'mae'])
train_steps = get_number_of_steps(len(train_val_df), cfg.batch_size)
val_steps = get_number_of_steps(len(test_df), cfg.batch_size)
model.fit_generator(train_gen,
epochs=cfg.epochs,
steps_per_epoch=train_steps,
callbacks=callbacks,
validation_data=val_gen,
workers=cfg.n_works,
max_queue_size=cfg.n_queue,
use_multiprocessing=True,
validation_steps=val_steps,
initial_epoch=0)
base_model.save_weights(__file__.split('.'[0]) + '.hdf5')
def train(task_name='base_inception',
epochs=6,
target_shape=(512, 512),
lr_schedule=None,
weights='imagenet',
trainable=True,
seed=42,
save_best_only=True,
initial_epoch=0,
use_extra_data=False):
np.random.seed(seed + 111)
random.seed(seed + 111)
tf.set_random_seed(seed + 111)
if not use_extra_data:
train_df, val_df, train_gen, val_gen = load_local_gen(target_shape)
else:
train_df, val_df, train_gen, val_gen = load_extra_data_gen(
target_shape)
if n_gpus > 1:
print('use multi gpu')
with tf.device('/cpu:0'):
cpu_model = InceptionResNetV2(input_shape=list(target_shape) + [3],
classes=len(cfg.label_names),
trainable=trainable)
if weights == "imagenet":
cpu_model.load_weights(cfg.inception_imagenet_weights,
by_name=True,
skip_mismatch=True)
else:
cpu_model.load_weights(weights)
model = multi_gpu_model(cpu_model, gpus=n_gpus)
else:
print('use single gpu')
model = InceptionResNetV2(input_shape=cfg.input_shape,
classes=len(cfg.label_names),
trainable=False)
if weights == "imagenet":
cpu_model.load_weights(cfg.inception_imagenet_weights,
by_name=True,
skip_mismatch=True)
else:
cpu_model.load_weights(weights)
print('load weights from ', weights)
model.compile(optimizer=KO.Adam(lr=lr_schedule[0][0], amsgrad=True),
loss='binary_crossentropy',
metrics=['binary_accuracy', 'mae'])
log_dir = os.path.join(cfg.log_dir, task_name)
makedir(log_dir)
weights_path = os.path.join(log_dir, cfg.weights_file)
checkpoint = ModelCheckpoint(weights_path,
monitor='f1_score',
verbose=1,
mode='max',
save_best_only=save_best_only,
save_weights_only=True)
if n_gpus > 1:
del checkpoint
checkpoint = MultiGPUCheckpoint(weights_path,
cpu_model,
monitor='f1_score',
mode='max',
save_best_only=save_best_only)
callbacks = [checkpoint]
callbacks += [
LearningRateScheduler(lambda epoch: schedule_steps(epoch, lr_schedule))
]
train_steps = get_number_of_steps(len(train_df), cfg.batch_size) * 4
val_steps = get_number_of_steps(len(val_df), cfg.batch_size) * 4
model.fit_generator(train_gen,
epochs=epochs,
steps_per_epoch=train_steps,
callbacks=callbacks,
validation_data=val_gen,
workers=cfg.n_works,
max_queue_size=cfg.n_queue,
use_multiprocessing=True,
validation_steps=val_steps,
initial_epoch=initial_epoch)
del model
del checkpoint
K.clear_session()
def train(task_name='base_xception',
epochs=6,
target_shape=(512, 512),
lr_schedule=None,
weights='imagenet',
trainable=True,
seed=42,
save_best_only=True,
initial_epoch=0,
drop_rate=0):
train_val_df = load_train_csv(cfg)
train_df, val_df = split_train_val(train_val_df, 0.25, seed=42)
np.random.seed(seed + 111)
random.seed(seed + 111)
tf.set_random_seed(seed + 111)
train_gen = BaseGenerator(train_df,
cfg.train_dir,
batch_size=cfg.batch_size,
aug_args=cfg.aug_args.copy(),
target_shape=target_shape,
use_yellow=False,
preprocessing_function=preprocess_input)
val_gen = BaseGenerator(val_df,
cfg.train_dir,
batch_size=cfg.batch_size,
aug_args=cfg.aug_args.copy(),
target_shape=target_shape,
use_yellow=False,
preprocessing_function=preprocess_input)
if n_gpus > 1:
print('use multi gpu')
with tf.device('/cpu:0'):
cpu_model = Xception(cfg.input_shape,
include_top=True,
n_class=len(cfg.label_names),
weights=weights,
trainable=trainable,
drop_rate=drop_rate)
model = multi_gpu_model(cpu_model, gpus=n_gpus)
else:
print('use single gpu')
model = Xception(cfg.input_shape,
include_top=True,
n_class=len(cfg.label_names),
weights=weights,
trainable=trainable,
drop_rate=drop_rate)
model.compile(optimizer=KO.Adam(lr=lr_schedule[0][0], amsgrad=True),
loss='binary_crossentropy',
metrics=[f1_score, 'mae'])
log_dir = os.path.join(cfg.log_dir, task_name)
makedir(log_dir)
weights_path = os.path.join(log_dir, cfg.weights_file)
checkpoint = ModelCheckpoint(weights_path,
monitor='f1_score',
mode='max',
verbose=1,
save_best_only=save_best_only,
save_weights_only=True)
if n_gpus > 1:
del checkpoint
checkpoint = MultiGPUCheckpoint(weights_path,
cpu_model,
monitor='f1_score',
mode='max',
save_best_only=save_best_only)
callbacks = []
callbacks += [checkpoint]
callbacks += [
LearningRateScheduler(lambda epoch: schedule_steps(epoch, lr_schedule))
]
train_steps = get_number_of_steps(len(train_df), cfg.batch_size) * 4
val_steps = get_number_of_steps(len(val_df), cfg.batch_size)
model.fit_generator(train_gen,
epochs=epochs,
steps_per_epoch=train_steps,
callbacks=callbacks,
validation_data=val_gen,
workers=cfg.n_works,
max_queue_size=cfg.n_queue,
use_multiprocessing=True,
validation_steps=val_steps,
initial_epoch=initial_epoch)
del model
del checkpoint
K.clear_session()