#tf.set_random_seed(seed=seed) # Summary Information SUMMARY_PATH = "/mnt/data/results" # SUMMARY_PATH="c:/temp/results" # SUMMARY_PATH="/tmp/results" NETWORK_FORMAT = "Unimodal" IMAGE_FORMAT = "2D" SUMMARY_BASEPATH = create_results_dir(SUMMARY_PATH, NETWORK_FORMAT, IMAGE_FORMAT) # how many times to execute the training/validation/test cycle CYCLES = 20 # # Execution Attributes attr = ExecutionAttribute() # dimensions of our images. attr.img_width, attr.img_height = 96, 96 # network parameters # attr.path='C:/Users/hp/Downloads/cars_train' # attr.path='/home/amenegotto/dataset/2d/sem_pre_proc_mini/ attr.path = '/mnt/data/image/2d/com_pre_proc/' attr.summ_basename = get_base_name(SUMMARY_BASEPATH) attr.s3_path = NETWORK_FORMAT + '/' + IMAGE_FORMAT attr.epochs = 100 attr.batch_size = 128 attr.set_dir_names() if K.image_data_format() == 'channels_first':
from keras import backend as K
# fix seed for reproducible results (only works on CPU, not GPU)
# seed = 9
# np.random.seed(seed=seed)
# tf.set_random_seed(seed=seed)
# Summary Information
SUMMARY_PATH = "/mnt/data/results"
NETWORK_FORMAT = "Unimodal"
IMAGE_FORMAT = "2D"
SUMMARY_BASEPATH = create_results_dir(SUMMARY_PATH, NETWORK_FORMAT,
IMAGE_FORMAT)
# Execution Attributes
attr = ExecutionAttribute()
attr.architecture = 'InceptionV3'
results_path = create_results_dir(SUMMARY_BASEPATH, 'fine-tuning',
attr.architecture)
attr.summ_basename = get_base_name(results_path)
attr.s3_path = NETWORK_FORMAT + '/' + IMAGE_FORMAT
attr.path = '/mnt/data/image/2d/sem_pre_proc'
attr.set_dir_names()
attr.batch_size = 128 # try 4, 8, 16, 32, 64, 128, 256 dependent on CPU/GPU memory capacity (powers of 2 values).
attr.epochs = 500
# how many times to execute the training/validation/test cycle
CYCLES = 1
for i in range(0, CYCLES):
# Summary Information IMG_TYPE = "sem_pre_proc/" SUMMARY_PATH = "/mnt/data/results" # SUMMARY_PATH="c:/temp/results" # SUMMARY_PATH="/tmp/results" NETWORK_FORMAT = "Multimodal" IMAGE_FORMAT = "2D" SUMMARY_BASEPATH = create_results_dir(SUMMARY_PATH, NETWORK_FORMAT, IMAGE_FORMAT) INTERMEDIATE_FUSION = True LATE_FUSION = False # how many times to execute the training/validation/test cycle CYCLES = 1 # Execution Attributes attr = ExecutionAttribute() # dimensions of our images. attr.img_width, attr.img_height = 96, 96 # network parameters attr.csv_path = 'csv/clinical_data.csv' attr.path = '/mnt/data/image/2d/' + IMG_TYPE # attr.path = '/home/amenegotto/dataset/2d/' + IMG_TYPE attr.numpy_path = '/mnt/data/image/2d/numpy/' + IMG_TYPE # attr.numpy_path = '/home/amenegotto/dataset/2d/numpy/' + IMG_TYPE attr.summ_basename = get_base_name(SUMMARY_BASEPATH) attr.epochs = 2 attr.batch_size = 32 attr.set_dir_names()
# Summary Information
SUMMARY_PATH = "/mnt/data/results"
# SUMMARY_PATH="c:/temp/results"
# SUMMARY_PATH="/tmp/results"
NETWORK_FORMAT = "Multimodal"
IMAGE_FORMAT = "2D"
SUMMARY_BASEPATH = create_results_dir(SUMMARY_PATH, NETWORK_FORMAT,
IMAGE_FORMAT)
# how many times to execute the training/validation/test cycle
CYCLES = 1
#
# Execution Attributes
attr = ExecutionAttribute()
# dimensions of our images.
attr.img_width, attr.img_height = 96, 96
# network parameters
# attr.path='C:/Users/hp/Downloads/cars_train'
# attr.path='/home/amenegotto/dataset/2d/sem_pre_proc_mini/
numpy_path = '/mnt/data/image/2d/numpy/sem_pre_proc/'
# numpy_path = '/home/amenegotto/dataset/2d/numpy/sem_pre_proc_mini/'
attr.csv_path = 'csv/clinical_data.csv'
# attr.path = '/mnt/data/image/2d/com_pre_proc/'
attr.summ_basename = get_base_name(SUMMARY_BASEPATH)
attr.epochs = 1
attr.batch_size = 32
attr.set_dir_names()
# Summary Information SUMMARY_PATH = "/mnt/data/results" # SUMMARY_PATH="c:/temp/results" # SUMMARY_PATH="/tmp/results" NETWORK_FORMAT = "Multimodal" IMAGE_FORMAT = "2D" SUMMARY_BASEPATH = create_results_dir(SUMMARY_PATH, NETWORK_FORMAT, IMAGE_FORMAT) INTERMEDIATE_FUSION = False LATE_FUSION = True # how many times to execute the training/validation/test cycle CYCLES = 1 # # Execution Attributes attr = ExecutionAttribute() # numpy_path = '/home/amenegotto/dataset/2d/numpy/sem_pre_proc_mini/' attr.numpy_path = '/mnt/data/image/2d/numpy/sem_pre_proc/' # dimensions of our images. attr.img_width, attr.img_height = 96, 96 # network parameters # attr.path='C:/Users/hp/Downloads/cars_train' # attr.path='/home/amenegotto/dataset/2d/sem_pre_proc_mini/ attr.csv_path = 'csv/clinical_data.csv' # attr.path = '/mnt/data/image/2d/com_pre_proc/' attr.summ_basename = get_base_name(SUMMARY_BASEPATH) attr.epochs = 5 attr.batch_size = 32 attr.set_dir_names()
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
# Summary Information
IMG_TYPE = "com_pre_proc/"
SUMMARY_PATH = "/mnt/data/results"
# SUMMARY_PATH="c:/temp/results"
# SUMMARY_PATH="/tmp/results"
NETWORK_FORMAT = "Multimodal"
IMAGE_FORMAT = "2D"
SUMMARY_BASEPATH = create_results_dir(SUMMARY_PATH, NETWORK_FORMAT,
IMAGE_FORMAT)
INTERMEDIATE_FUSION = True
LATE_FUSION = False
# Execution Attributes
attr = ExecutionAttribute()
attr.architecture = 'vgg19'
attr.csv_path = 'csv/clinical_data.csv'
attr.s3_path = NETWORK_FORMAT + '/' + IMAGE_FORMAT
attr.numpy_path = '/mnt/data/image/2d/numpy/' + IMG_TYPE
# attr.numpy_path = '/home/amenegotto/dataset/2d/numpy/' + IMG_TYPE
attr.path = '/mnt/data/image/2d/' + IMG_TYPE
results_path = create_results_dir(SUMMARY_BASEPATH, 'fine-tuning',
attr.architecture)
attr.summ_basename = get_base_name(results_path)
attr.set_dir_names()
attr.batch_size = 128
attr.epochs = 500
attr.img_width = 224
# tf.set_random_seed(seed=seed)
# Summary Information
IMG_TYPE = "sem_pre_proc/"
SUMMARY_PATH = "/mnt/data/results"
# SUMMARY_PATH="c:/temp/results"
# SUMMARY_PATH="/tmp/results"
NETWORK_FORMAT = "Multimodal"
IMAGE_FORMAT = "2D"
SUMMARY_BASEPATH = create_results_dir(SUMMARY_PATH, NETWORK_FORMAT,
IMAGE_FORMAT)
INTERMEDIATE_FUSION = False
LATE_FUSION = True
# Execution Attributes
attr = ExecutionAttribute()
attr.architecture = 'InceptionV3'
attr.numpy_path = '/mnt/data/image/2d/numpy/' + IMG_TYPE
attr.path = '/mnt/data/image/2d/' + IMG_TYPE
results_path = create_results_dir(SUMMARY_BASEPATH, 'fine-tuning',
attr.architecture)
attr.summ_basename = get_base_name(results_path)
attr.s3_path = NETWORK_FORMAT + '/' + IMAGE_FORMAT
attr.set_dir_names()
attr.batch_size = 128 # try 4, 8, 16, 32, 64, 128, 256 dependent on CPU/GPU memory capacity (powers of 2 values).
attr.epochs = 500
# how many times to execute the training/validation/test cycle
CYCLES = 1
from keras.layers import Input, Conv2D, MaxPooling2D, concatenate
from keras.layers import Activation, Dropout, Flatten, Dense, BatchNormalization
from keras import backend as K
from keras.optimizers import RMSprop, Adam
from ExecutionAttributes import ExecutionAttribute
import cv2
import os
import pandas as pd
import numpy as np
from sklearn.metrics import classification_report, confusion_matrix, cohen_kappa_score, roc_auc_score, roc_curve
from keras.preprocessing.image import load_img, img_to_array
os.environ["PATH"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin'
# Execution Attributes
attr = ExecutionAttribute()
# dimensions of our images.
attr.img_width, attr.img_height = 32, 32
# network parameters
attr.path = 'C:/Users/hp/Downloads/cars_train'
attr.epochs = 200
attr.batch_size = 8
attr.set_dir_names()
if K.image_data_format() == 'channels_first':
input_s = (3, attr.img_width, attr.img_height)
else:
input_s = (attr.img_width, attr.img_height, 3)
# Summary Information
SUMMARY_PATH="/mnt/data/fine-tuning/Xception"
#SUMMARY_PATH="c:/temp/results"
#SUMMARY_PATH="/tmp/results"
NETWORK_FORMAT="Unimodal"
IMAGE_FORMAT="2D"
SUMMARY_BASEPATH=create_results_dir(SUMMARY_PATH, NETWORK_FORMAT, IMAGE_FORMAT)
# how many times to execute the training/validation/test
CYCLES = 1
#
# Execution Attributes
INITIAL_EPOCH=1
attr = ExecutionAttribute()
attr.architecture = 'Xception'
results_path = create_results_dir('/mnt/data', 'fine-tuning', attr.architecture)
attr.summ_basename = get_base_name(results_path)
attr.path='/mnt/data/image/2d/com_pre_proc'
attr.set_dir_names()
attr.batch_size = 64 # try 4, 8, 16, 32, 64, 128, 256 dependent on CPU/GPU memory capacity (powers of 2 values).
attr.epochs = 15
# hyper parameters for model
nb_classes = 2 # number of classes
based_model_last_block_layer_number = 126 # value is based on based model selected.
attr.img_width, attr.img_height = 299, 299 # change based on the shape/structure of your images
learn_rate = 1e-4 # sgd learning rate
momentum = .9 # sgd momentum to avoid local minimum
# PURPOSE: # create numpy arrays to store all data needed on multimodal networks and save them to disk # for later reuse on training/validation/test cycle for faster initialization import numpy as np from Datasets import create_data_as_numpy, create_data_as_list from ExecutionAttributes import ExecutionAttribute attr = ExecutionAttribute() # dimensions of our images. attr.img_width, attr.img_height = 96, 96 # network parameters attr.csv_path = 'csv/clinical_data.csv' is_categorical = True image_type = 'com_pre_proc/' npy_path = '/mnt/data/image/2d/numpy/' + image_type # npy_path = '/home/amenegotto/dataset/2d/numpy/' + image_type attr.path = '/mnt/data/image/2d/' + image_type # attr.path = '/home/amenegotto/dataset/2d/' + image_type attr.set_dir_names() # First multimodal approach using all dataset as numpy array aka Fat Memory Approach... #create_data_as_numpy(attr.path, attr.csv_path, attr.img_width, attr.img_height, True, npy_path) # Third multimodal approach for a thread-safe custom generator create_data_as_list(attr.path, attr.csv_path, True, npy_path, is_categorical)
def read_attributes(path):
print("Reading execution attributes from ", path)
attr = ExecutionAttribute()
lines = [line.rstrip('\n') for line in open(path)]
attr.seq = int(lines[0].split("=", 1)[1])
attr.img_width = int(lines[1].split("=", 1)[1])
attr.img_height = int(lines[2].split("=", 1)[1])
attr.path = lines[3].split("=", 1)[1]
attr.summ_basename = lines[4].split("=", 1)[1]
attr.epochs = int(lines[5].split("=", 1)[1])
attr.batch_size = int(lines[6].split("=", 1)[1])
attr.train_data_dir = lines[7].split("=", 1)[1]
attr.validation_data_dir = lines[8].split("=", 1)[1]
attr.test_data_dir = lines[9].split("=", 1)[1]
attr.steps_train = int(lines[10].split("=", 1)[1])
attr.steps_valid = int(lines[11].split("=", 1)[1])
attr.steps_test = int(lines[12].split("=", 1)[1])
attr.architecture = lines[13].split("=", 1)[1]
attr.curr_basename = lines[14].split("=", 1)[1]
return attr
# fix seed for reproducible results (only works on CPU, not GPU) #seed = 9 #np.random.seed(seed=seed) #tf.set_random_seed(seed=seed) # Summary Information SUMMARY_PATH = "/mnt/data/results" NETWORK_FORMAT = "Unimodal" IMAGE_FORMAT = "2D" SUMMARY_BASEPATH = create_results_dir(SUMMARY_PATH, NETWORK_FORMAT, IMAGE_FORMAT) # how many times to execute the training/validation/test cycle CYCLES = 1 # Execution Attributes attr = ExecutionAttribute() attr.architecture = 'vgg19' results_path = create_results_dir(SUMMARY_BASEPATH, 'fine-tuning', attr.architecture) attr.summ_basename = get_base_name(results_path) attr.s3_path = NETWORK_FORMAT + '/' + IMAGE_FORMAT attr.path = '/mnt/data/image/2d/com_pre_proc' attr.set_dir_names() attr.batch_size = 128 attr.epochs = 500 attr.img_width = 224 attr.img_height = 224 for i in range(0, CYCLES):