def predict_test_tta(input_size, mode='grayscale', epochs=5):
df_test = load_to_dataframe("./mldata/test_data_order.csv",
y_label=0,
split=0,
shuffle=0)
test_generator = test_gen_tta.flow_from_dataframe(
df_test,
directory='./mldata/test_set/',
x_col='image_filename',
target_size=input_size,
color_mode=mode,
shuffle=False,
classes=[0, 1, 2, 3],
batch_size=1,
class_mode=None)
predictions = []
for i in range(epochs):
preds = model.predict_generator(test_generator,
steps=test_generator.samples)
predictions.append(preds)
pred = np.mean(predictions, axis=0)
print(pred)
df_test['class_number'] = np.argmax(pred, 1)
df_test.to_csv('/home/seldon/mauna_kea/mldata/results.csv',
index=None,
header=True)
def predict_val(val_model):
print("Cross val " + str(val_model.name) + " ...")
mode = 'grayscale' if (val_model.input_size[-1] == 1) else 'rgb'
_, df_val_tmp = load_to_dataframe("./mldata/train_y.csv", shuffle = True)
val_generator = test_gen.flow_from_dataframe(
df_val_tmp,
directory = './mldata/train_set/',
x_col = 'image_filename',
target_size = val_model.input_size[:2],
color_mode = mode,
shuffle = False,
classes = [0, 1, 2, 3],
batch_size = 1,
class_mode = None)
results = val_model.model.predict_generator(val_generator, steps = val_generator.samples)
df_val_tmp['class_number'] = np.argmax(results, 1)
df_val_tmp.to_csv('/home/seldon/mauna_kea/mldata/cv/results_val_' + str(val_model.name) + '.csv', index = None, header=True)
def pseudo_labelling(results):
preds = np.mean(results, axis = 0)
count = 0
copy = {}
for idx, pred in enumerate(preds):
max_val = max(pred)
if max_val > 0.975:
copy[idx] = np.argmax(pred, 0)
count += 1
print(count)
cop = {2:3, 4:1, 6:2, 12:0}
keys = list(copy.keys())
vals = list(copy.values())
df_conv = pd.DataFrame()
df_test = load_to_dataframe("./mldata/test_data_order.csv", y_label= 0, split = 0, shuffle = 0)
df_conv['image_filename'] = df_test.iloc[keys]['image_filename']
df_conv['class_number'] = vals
df_conv.to_csv('/home/seldon/mauna_kea/mldata/train_y_add.csv', index = None, header=True)
def predict_test(testing_model):
print("Testing " + str(testing_model.name) + " ...")
mode = 'grayscale' if (testing_model.input_size[-1] == 1) else 'rgb'
df_test = load_to_dataframe("./mldata/test_data_order.csv", y_label= 0, split = 0, shuffle = 0)
test_generator = test_gen.flow_from_dataframe(
df_test,
directory = './mldata/test_set/',
x_col = 'image_filename',
target_size = testing_model.input_size[:2],
color_mode = mode,
shuffle = False,
classes = [0, 1, 2, 3],
batch_size = 1,
class_mode = None)
test_model = testing_model.model
results = test_model.predict_generator(test_generator, steps = test_generator.samples)
df_test['class_number'] = np.argmax(results, 1)
df_test.to_csv ('/home/seldon/mauna_kea/mldata/results_' + str(testing_model.name) + '.csv', index = None, header=True)
def predict_test(input_size, mode='grayscale'):
df_test = load_to_dataframe("./mldata/test_data_order.csv",
y_label=0,
split=0,
shuffle=0)
test_generator = test_gen.flow_from_dataframe(
df_test,
directory='./mldata/test_set/',
x_col='image_filename',
target_size=input_size,
color_mode=mode,
shuffle=False,
classes=[0, 1, 2, 3],
batch_size=1,
class_mode=None)
results = model.predict_generator(test_generator,
steps=test_generator.samples)
df_test['class_number'] = np.argmax(results, 1)
df_test.to_csv('/home/seldon/mauna_kea/mldata/results.csv',
index=None,
header=True)
epochs=nb_epochs,
steps_per_epoch=training_generator.samples //
training_generator.batch_size + 1,
# callbacks = callbacks),
validation_data=valid_generator,
validation_steps=valid_generator.samples // valid_generator.batch_size
+ 1)
# MAIN
if __name__ == '__main__':
#loading data
from load_data import load_to_dataframe
df_train, df_val = load_to_dataframe("./mldata/train_y.csv", shuffle=True)
# df_train = load_to_dataframe("./mldata/train_y.csv", shuffle = True, split = False)
# Chosoe good input according to model
cnn_input_size = (512, 512)
define_generators(cnn_input_size, 'grayscale')
# Choose one Model
# model = custom_model()
# model = vgg16_scratch(0.00005)
autoencoder_model = Autoencoder()
autoencoder_model.load_weights()
model = autoencoder_model.model
# model = transfer_learning(0.0001)
#TRAINING/LOADING
batch_size = 1,
class_mode = None)
predictions = []
test_model = testing_model.model
for i in range(epochs):
print(i, end = '')
preds = test_model.predict_generator(test_generator, steps = test_generator.samples)
predictions.append(preds)
pred = np.mean(predictions, axis = 0)
return pred
if __name__ == '__main__':
#LOADING DATA
from load_data import load_to_dataframe
# df_train, df_val, class_weights = load_to_dataframe("./mldata/train_y.csv", shuffle = True, weights = 1)
df_train, class_weights = load_to_dataframe("./mldata/train_y.csv", split = 0, shuffle = True, weights = 1)
df_val = None
# Models
models = []
# models.append(custom_model(name = 'autoencoder', lr = 0.001))
models.append(resnext_scratch(name = 'resnext', lr = 0.0008))
models.append(pretrained_resnet(name = 'pretrained_resnetv2', lr = 0.001)) #start at 0.01
models.append(pretrained_vgg16(name = 'pretrained_vgg', lr = 0.0005))
models.append(pretrained_xception(name = 'xception', lr = 0.005)) # start at 0.045
models.append(pretrained_mobilenet(name = 'mobilenet', lr = 0.01))
models.append(pretrained_nasnet(name = 'nasnet', lr = 0.002)) # start at 0.01
models.append(pretrained_densenet(name = 'densenet', lr = 0.0001)) # start at 0.001
list_epochs = [10]#, 80, 80]