def main():
parser = argparse.ArgumentParser()
parser.add_argument("-p",
"--project-path",
default=r"/media/megatron/Projects/DeepSolaris",
help="The project path to use")
parser.add_argument("-d",
"--dataset",
required=True,
help="The dataset to train/validate on")
parser.add_argument(
"-t",
"--test-size",
default=0.25,
help="The fraction of the complete data to use as test set")
parser.add_argument(
"-v",
"--validation-size",
default=0.10,
help="The fraction of the training set to use for validation")
args = vars(parser.parse_args())
with Project(project_path=args["project_path"]) as project:
dataset = project.dataset(args["dataset"])
train_val_dataset, test_dataset = dataset.split(
test_size=args["test_size"],
first_name="{}_train_val".format(args["dataset"]),
second_name="{}_test".format(args["dataset"]))
train_dataset, validation_dataset = dataset.split(
test_size=args["validation_size"],
first_name="{}_train".format(args["dataset"]),
second_name="{}_val".format(args["dataset"]))
train_dataset.save_dataset()
test_dataset.save_dataset()
validation_dataset.save_dataset()
def main():
project_path = r"/media/megatron/Projects/DeepSolaris"
project_path = r"/media/tim/Data/Work/CBS/DeepSolaris"
with Project(project_path=project_path) as project:
dataset = project.dataset("Heerlen-HR")
#dataset.import_numpy_dataset(os.path.join(project_path, r"Images/hr_2018_18m_all.npy"),
# os.path.join(project_path, r"Images/hr_2018_18m_all_labels.npy"))
train_dataset, test_dataset = dataset.split(test_size=0.2)
image_shape = dataset.data[0].shape
for name, cnn_model in create_models(input_shape=image_shape):
model = project.model(name)
model.create_model(cnn_model)
cnn_model.summary()
model.random_search(train_dataset,
test_dataset,
param_distributions=dict(
epochs=[3],
batch_size=[32],
loss_function=["binary_crossentropy"],
lr=uniform(0, 0.1),
momentum=[0.7, 0.8, 0.9],
nesterov=[True, False],
decay=[0]))
def main():
project_path = r"/media/megatron/Projects/DeepSolaris"
#project_path = r"/media/tim/Data/Work/CBS/DeepSolaris"
with Project(project_path=project_path) as project:
dataset = project.dataset("Heerlen-HR")
train_dataset, test_dataset = dataset.split(test_size=0.2)
image_shape = dataset.data[0].shape
name, cnn_model = create_vgg16_model(input_shape=image_shape)
model = project.model(name)
model.create_model(cnn_model)
model.plot()
cnn_model.summary()
model.random_search(
train_dataset,
test_dataset,
param_distributions=dict(
epochs=[40],
batch_size=[32],
loss_function=["binary_crossentropy"],
lr=[1e-5], #[1e-4],
momentum=[0.8, 0.9],
nesterov=[True, False],
calculate_decay=[True, False],
callbacks=[[EarlyStopping(patience=5)]],
),
n_iter=10)
def main():
#project_path = r"/media/megatron/Projects/DeepSolaris"
project_path = r"/media/tim/Data/Work/CBS/DeepSolaris"
with Project(project_path=project_path) as project:
dataset = project.dataset("Heerlen-HR")
dataset.data = dataset.data[:, :, :, ::-1]
train_dataset, test_dataset = dataset.split(test_size=0.2)
train_generator = ImageGenerator(train_dataset)\
.with_rescale(1/255.)\
.with_seed(42)\
.with_rotation_range(30)\
.with_width_shift_range(0.1)\
.with_height_shift_range(0.1)\
.with_zoom_range(0.2)\
.with_shear_range(0.2)\
.with_horizontal_flip(True)\
.with_fill_mode("reflect")
test_generator = ImageGenerator(test_dataset)\
.with_rescale(1/255.)\
.with_seed(84)
image_shape = dataset.data[0].shape
name, cnn_model = create_vgg16_model(input_shape=image_shape)
model = project.model(name)
model.create_model(cnn_model)
model.plot()
cnn_model.summary()
reduce_lr = ReduceLROnPlateau(monitor='val_acc',
factor=0.1,
patience=5,
min_lr=1e-5)
model.random_search(
train_generator,
test_generator,
param_distributions=dict(
epochs=[40],
batch_size=[32],
loss_function=["binary_crossentropy"],
lr=[1e-3],
momentum=[0.9],
nesterov=[False],
decay=[0],
callbacks=[[EarlyStopping(patience=5), reduce_lr]],
),
n_iter=10)
def main():
#project_path = r"/media/megatron/Projects/DeepSolaris"
project_path = r"/media/tim/Data/Work/CBS/DeepSolaris"
with Project(project_path=project_path) as project:
dataset = project.dataset("Heerlen-HR")
dataset.data = dataset.data[:, :, :, ::-1]
train_dataset, test_dataset = dataset.split(test_size=0.2)
train_generator = ImageGenerator(train_dataset)\
.with_rescale(1/255.)\
.with_rotation_range(30)\
.with_width_shift_range(0.1)\
.with_height_shift_range(0.1)\
.with_zoom_range(0.2)\
.with_shear_range(0.2)\
.with_horizontal_flip(True)\
.with_fill_mode("reflect")
test_generator = ImageGenerator(test_dataset)\
.with_rescale(1/255.)
image_shape = dataset.data[0].shape
name = "vgg16_rmsprop_fine_tune_sgd"
model = project.model(name)
model.import_model(
r"/media/tim/Data/Work/CBS/DeepSolaris/models/vgg16_full_fc512_fc512_fc1_aug_frozen_rms_prop/runs/2019/5/25/11:56:19/model.hdf5"
)
model.plot()
name, cnn_model = train_model_until(model.model)
cnn_model.summary()
epochs = 15
lr = 1e-2
decay = lr / epochs
# .with_callbacks([EarlyStopping(patience=5)])\
with model.run().with_epochs(epochs)\
.with_batch_size(64)\
.with_loss_function("binary_crossentropy")\
.with_optimizer(SGDSettings(lr=lr, momentum=0.9, decay=decay, nesterov=False))\
.with_metric_callbacks([ClassificationReportCallback(), ConfusionMatrixCallback(), PlotRocCallback()])\
.with_class_weights(train_dataset.class_weights)\
.with_train_dataset(train_generator)\
.with_test_dataset(test_generator)\
.with_evaluation_dataset(test_generator) as run:
run.train()
run.evaluate()
def main():
#project_path = r"/media/megatron/Projects/DeepSolaris"
project_path = r"/media/tim/Data/Work/CBS/DeepSolaris"
with Project(project_path=project_path) as project:
dataset = project.dataset("Heerlen-HR")
dataset.data = preprocess_input(dataset.data[:, :, :, ::-1])
train_dataset, test_dataset = dataset.split(test_size=0.25)
class_weights = train_dataset.class_weights
train_dataset.labels = to_categorical(train_dataset.labels)
test_dataset.labels = to_categorical(test_dataset.labels)
train_generator = ImageGenerator(train_dataset)\
.with_seed(42)\
.with_rotation_range(30)\
.with_width_shift_range(0.1)\
.with_height_shift_range(0.1)\
.with_zoom_range(0.2)\
.with_shear_range(0.2)\
.with_horizontal_flip(True)\
.with_fill_mode("reflect")
test_generator = ImageGenerator(test_dataset)\
.with_seed(84)
image_shape = dataset.data[0].shape
name, cnn_model = create_vgg16_model(input_shape=image_shape,
layer_index=7)
model = project.model(name)
model.create_model(cnn_model)
model.plot()
cnn_model.summary()
# .with_callbacks([EarlyStopping(patience=5)])\
with model.run().with_epochs(10)\
.with_batch_size(64)\
.with_loss_function("binary_crossentropy")\
.with_optimizer(RMSPropSettings(lr=1e-5))\
.with_metric_callbacks([ClassificationReportCallback(), ConfusionMatrixCallback(), PlotRocCallback()])\
.with_class_weights(class_weights)\
.with_train_dataset(train_generator)\
.with_test_dataset(test_generator)\
.with_evaluation_dataset(test_dataset) as run:
run.train()
run.evaluate()
include_top=False,
weights="imagenet",
)
for layer in base_model.layers:
layer.trainable = True
last_conv_layer = base_model.get_layer("block4_conv3")
x = GlobalAveragePooling2D()(last_conv_layer.output)
x = Dense(512, activation="relu")(x)
x = BatchNormalization(axis=-1)(x)
x = Dropout(0.25)(x)
predictions = Dense(1, activation="sigmoid")(x)
return Model(base_model.input, predictions)
with Project(project_path=r"/media/megatron/Projects/DeepSolaris") as project:
dataset = project.dataset("Heerlen-HR")
dataset.import_numpy_dataset(
r"/media/megatron/9827-B092/hr_2018_18m_all.npy",
r"/media/megatron/9827-B092/hr_2018_18m_all_labels.npy")
train_dataset, test_dataset = dataset.split(test_size=0.2)
train_generator = ImageGenerator(train_dataset)\
.with_shuffle_data(True)\
.with_rotation_range(45)\
.with_width_shift_range(0.1)\
.with_height_shift_range(0.1)\
.with_shear_range(0.1)\
.with_zoom_range(0.1)\
.with_channel_shift_range(0.1)\
.with_horizontal_flip(0.1)\
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-p",
"--project-path",
default=r"/media/megatron/Projects/DeepSolaris",
help="The project path to use")
parser.add_argument("-d",
"--training-set",
required=True,
help="The dataset to train on")
parser.add_argument("-t",
"--test-set",
required=True,
help="The dataset to test on")
parser.add_argument("-v",
"--validation-set",
required=True,
help="The dataset to validate on")
parser.add_argument("-s",
"--sample-size",
required=True,
type=float,
help="The sample fraction of the training set")
parser.add_argument("-e",
"--epochs",
default=10,
type=int,
help="The number of epochs to train the model")
parser.add_argument("-b",
"--batch_size",
default=32,
type=int,
help="The batch_size to train with")
parser.add_argument("-l",
"--learning_rate",
default=1e-4,
type=float,
help="The learning rate to train with")
#parser.add_argument("-m", "--model-filename", required=True, help="The model to evaluate")
#parser.add_argument("-n", "--model-name", required=True, help="The name of the model to evaluate")
args = vars(parser.parse_args())
with Project(project_path=args["project_path"]) as project:
dataset = project.dataset(args["training_set"])
dataset.data = dataset.data[:, :, :, ::-1]
train_dataset = dataset.sample(sample_size=args["sample_size"],
random_state=42)
test_dataset = project.dataset(args["test_set"])
validation_dataset = project.dataset(args["validation_set"])
train_generator = ImageGenerator(train_dataset)\
.with_rescale(1/255.)\
.with_seed(42)\
.with_rotation_range(30)\
.with_width_shift_range(0.1)\
.with_height_shift_range(0.1)\
.with_zoom_range(0.2)\
.with_shear_range(0.2)\
.with_horizontal_flip(True)\
.with_fill_mode("reflect")
test_generator = ImageGenerator(test_dataset)\
.with_rescale(1/255.)\
.with_seed(84)
validation_generator = ImageGenerator(validation_dataset)\
.with_rescale(1/255.)\
.with_seed(84)
#model_name, cnn_model = load_model(args["model_filename"])
model_name, cnn_model = create_vgg16_model(input_shape=(187, 187, 3))
model = project.model("{}_{}".format(model_name,
str(args["sample_size"])))
model.create_model(cnn_model)
model.plot()
cnn_model.summary()
with model.run().with_epochs(args["epochs"])\
.with_batch_size(args["batch_size"])\
.with_loss_function("binary_crossentropy")\
.with_optimizer(RMSPropSettings(lr=args["learning_rate"]))\
.with_metric_callbacks([ClassificationReportCallback(), ConfusionMatrixCallback(), PlotRocCallback()])\
.with_class_weights(train_dataset.class_weights)\
.with_train_dataset(train_generator)\
.with_test_dataset(test_generator)\
.with_evaluation_dataset(test_generator)\
.with_evaluation_dataset(validation_generator) as run:
run.train()
run.evaluate()