def train(model):
dataGen = DataGenerator("train", 169728, BATCH_SIZE, True)
dataGenval = DataGenerator("val", 42432, BATCH_SIZE, False)
dataGentest = DataGenerator("test", 48832, 64, False)
filepath = "BioElmoTextModel-{epoch:02d}-{val_loss:.2f}.hdf5"
modelckp = ModelCheckpoint(filepath,
monitor='val_loss',
verbose=1,
save_best_only=False,
mode='min')
clr = CyclicLR(base_lr=0.0001, max_lr=0.0006, step_size=2000.)
es = EarlyStopping(monitor="val_loss", mode=min, verbose=1)
callbacks_list = [
modelckp, checkpoint, clr, es,
roc_callback(dataGentest, np.array(dataGentest.labels))
]
model.fit_generator(dataGen,
callbacks_list=callbacks_list,
validation_data=dataGenval,
use_multiprocessing=False,
verbose=1,
epochs=EPOCHS,
callbacks=call_list,
workers=4)
model.save_weights("BioElmoTextModel.h5")
def train(model):
train_df = pd.read_csv(open("./train.csv", 'rU'),
usecols=range(16),
encoding='utf-8',
engine='c',
dtype={'Report': str})
val_df = pd.read_csv(open("./validate.csv", 'rU'),
usecols=range(16),
encoding='utf-8',
engine='c',
dtype={'Report': str})
test_df = pd.read_csv(open("./test.csv", 'rU'),
usecols=range(16),
encoding='utf-8',
engine='c',
dtype={'Report': str})
x1_train, x2_train, y_train, x1_val, x2_val, y_val, x1_test, x2_test, y_test = getData(
train_df, val_df, test_df)
print("Prepare Generators")
trainGen = ChestXRaySequence(x1_train,
x2_train,
y_train,
batch_size=BATCH_SIZE)
valGen = ChestXRaySequence(x1_val, x2_val, y_val, batch_size=BATCH_SIZE)
testGen = ChestXRaySequence(x1_test,
x2_test,
y_test,
batch_size=BATCH_SIZE)
print("Start training")
filepath = "weights-improvement-MERGED-{epoch:02d}-{val_loss:.2f}.hdf5"
checkpoint = ModelCheckpoint(filepath,
monitor='val_loss',
verbose=1,
save_best_only=False,
mode='min')
clr = CyclicLR(base_lr=0.0001, max_lr=0.0006, step_size=2000.)
callbacks_list = [
checkpoint, clr,
roc_callback(testGen, np.array(testGen.labels))
]
model.fit_generator(generator=trainGen,
validation_data=valGen,
use_multiprocessing=False,
verbose=1,
epochs=EPOCHS,
callbacks=callbacks_list,
workers=6)
model.save("merged.h5")
def train(model,x_train, y_train, x_val, y_val,x_test, y_test):
filepath = "BioFastTextModel-{epoch:02d}-{val_loss:.2f}.hdf5"
modelckp = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=False,
mode='min')
metrics=roc_callback(x_test,y_test)
clr = CyclicLR(base_lr=0.0001, max_lr=0.0006, step_size=2000.)
es = EarlyStopping(monitor="val_loss", mode=min, verbose=1)
callbacks_list = [modelckp, checkpoint, clr, es, metrics]
print("Begining training")
model.fit(x_train, y_train, epochs=EPOCHS, batch_size=BATCH_SIZE,
shuffle=True, validation_data=(x_val, y_val), verbose=1, callbacks=callbacks_list)
def trainGrayscale(model):
labels = ['No Finding','Enlarged Cardiomediastinum',
'Cardiomegaly','Lung Opacity','Lung Lesion',
'Edema','Consolidation','Pneumonia','Atelectasis',
'Pneumothorax','Pleural Effusion','Pleural Other','Fracture','Support Devices']
datagen = image.ImageDataGenerator(rescale=1. / 255)
traindf = pd.read_csv(TRAIN)
validatedf = pd.read_csv(VAL)
testdf = pd.read_csv(TEST)
traingenerator = datagen.flow_from_dataframe(traindf,
directory=None,
color_mode='grayscale',
target_size=(256, 256),
x_col='Path',
y_col=labels,
class_mode="other",
shuffle=True,
batch_size=BATCH,
drop_duplicates=False)
validategenerator = datagen.flow_from_dataframe(validatedf,
directory=None,
color_mode='grayscale',
target_size=(256, 256),
x_col='Path',
y_col=labels,
class_mode="other",
shuffle=False,
batch_size=BATCH,
drop_duplicates=False)
testgenerator = datagen.flow_from_dataframe(testdf,
directory=None,
color_mode='grayscale',
target_size=(256, 256),
x_col='Path',
y_col=labels,
class_mode="other",
shuffle=False,
batch_size=BATCH,
drop_duplicates=False)
print(traingenerator.n)
print(validategenerator.n)
print(testgenerator.n)
filepath = "DenseNet-grayscale-{epoch:02d}-{val_loss:.2f}.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=False,
mode='min')
clr = CyclicLR(base_lr=0.0001, max_lr=0.0006, step_size=2000.)
es = EarlyStopping(monitor="val_loss",mode=min, verbose=1)
callbacks_list = [checkpoint,clr,es,roc_callback(testgenerator,np.array(testgenerator.labels))]
model.fit_generator(generator=traingenerator,
validation_data=validategenerator,
epochs=EPOCHS,
steps_per_epoch=traingenerator.n / BATCH,
validation_steps=validategenerator.n / BATCH,
callbacks=callbacks_list,
workers=THREAD,
verbose=1)
model.save_weights("DenseNet.h5")
model.save('DenseNet.h5')