def Main(inputParamsConfig):
#input_shape,learning_rate, momentum, num_epochs, batchsize, data_path, train_set_size, test_set_size,
# positive_set_ratio, dropout,nonlinearityToUse,augmentation
experiment_id = str(time.strftime("%Y%m%d%H%M%S"))
input_shape = inputParamsConfig['input_shape']
learning_rate = inputParamsConfig['learning_rate']
momentum = inputParamsConfig['momentum']
num_epochs = inputParamsConfig['num_epochs']
batch_size = inputParamsConfig['batch_size']
data_path = inputParamsConfig['data_path']
train_set_size = inputParamsConfig['train_set_size']
test_set_size = inputParamsConfig['test_set_size']
positive_set_ratio = inputParamsConfig['positive_set_ratio']
dropout = inputParamsConfig['dropout']
nonlinearityToUse = inputParamsConfig['nonlinearityToUse']
numberOfLayers = inputParamsConfig['numberOfLayers']
augmentationFlag = inputParamsConfig['augmentationFlag']
print(
" Learning rate: '%s' , momentum: '%s', num_epochs: '%s' ,batch size: '%s' ,data_path: '%s' ,Train Set Size: '%s' ,Test set Size: '%s' ,Positive set Ratio '%s' , dropout: '%s', nonlinearityToUse: '%s',augmentationFlag: '%s',number of layers: '%s'"
% (str(learning_rate), str(momentum), str(num_epochs), str(batch_size),
data_path, str(train_set_size), str(test_set_size),
str(positive_set_ratio), str(dropout), str(nonlinearityToUse),
str(augmentationFlag), str(numberOfLayers)))
num_epochs = int(num_epochs)
batch_size = int(batch_size)
# We save the created train and test set of size X and posetive ration r to reduce the overhead in running the pipline
ps = []
ng = []
patient_id = []
with h5py.File(os.path.join('/diskStation/temp/test_500_0.3_28288 .hdf5'),
'r') as hf:
print('List of arrays in this file: \n', hf.keys())
tmp_test_paths = hf.get(
'Test_set') # Reading list of patients and test file paths
ps = np.array(tmp_test_paths) #full paths to all positive test patches
tmp_test_paths = hf.get('neg_test_set')
ng = np.array(tmp_test_paths)
# path_to_pos_tmp='/home/shamidian/Summer2016/DeepMed/28288/pos_28288'
# path_to_neg_tmp='/home/shamidian/Summer2016/DeepMed/28288/neg_smp_0_28288'
# for item in os.listdir(path_to_pos_tmp):
# ps.append(os.path.join(path_to_pos_tmp,item))
# for items in os.listdir(path_to_neg_tmp):
# ng.append(os.path.join(path_to_neg_tmp,items))
test_set, test_label = SupportFuncs.mat_generate_from_path(ps, ng)
#
# train_set, train_label, test_set, test_label, val_set, val_label = SupportFuncs.load_data(data_path, int(train_set_size),
# int(test_set_size),
# int(augmentationFlag),float(positive_set_ratio))
if nonlinearityToUse == 'relu':
nonLinearity = lasagne.nonlinearities.rectify
elif nonlinearityToUse == 'tanh':
nonLinearity = lasagne.nonlinearities.tanh
elif nonlinearityToUse == 'sigmoid':
nonLinearity = lasagne.nonlinearities.sigmoid
else:
raise Exception(
'nonlinearityToUse: Unsupported nonlinearity type has been selected for the network, retry with a supported one!'
)
dtensor5 = T.TensorType('float32', (False, ) * 5)
input_var = dtensor5('inputs')
target_var = T.ivector('targets')
inputParamsNetwork = dict(n_layer=numberOfLayers,
shape=input_shape,
dropout=float(dropout),
nonLinearity=nonLinearity)
network = Build_3dcnn(weight_init, inputParamsNetwork, input_var)
lasagne.layers.set_all_param_values(network, param_values)
# Create a loss expression for training, i.e., a scalar objective we want
# to minimize (for our multi-class problem, it is the cross-entropy loss):
prediction = lasagne.layers.get_output(network)
loss = lasagne.objectives.categorical_crossentropy(prediction, target_var)
loss = loss.mean()
# loss=np.mean(loss)
# We could add some weight decay as well here, see lasagne.regularization.
# Create update expressions for training, i.e., how to modify the
# parameters at each training step. Here, we'll use Stochastic Gradient
# Descent (SGD) with Nesterov momentum, but Lasagne offers plenty more.
params = lasagne.layers.get_all_params(network, trainable=True)
updates = lasagne.updates.nesterov_momentum(
loss,
params,
learning_rate=float(learning_rate),
momentum=float(momentum))
# Create a loss expression for validation/testing. The crucial difference
# here is that we do a deterministic forward pass through the network,
# disabling dropout layers.
test_prediction = lasagne.layers.get_output(network, deterministic=True)
test_loss = lasagne.objectives.categorical_crossentropy(
test_prediction, target_var)
# test_loss = test_loss.mean()
test_loss = test_loss.mean()
# As a bonus, also create an expression for the classification accuracy:
test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), target_var),
dtype=theano.config.floatX)
# Compile a function performing a training step on a mini-batch (by giving
# the updates dictionary) and returning the corresponding training loss:
train_fn = theano.function([input_var, target_var], loss,
updates=updates) # mode='DebugMode'
# Compile a second function computing the validation loss and accuracy:
val_fn = theano.function(
[input_var, target_var],
[test_loss, test_acc, test_prediction]) # ,mode='DebugMode')
# Finally, launch the training loop.
print("Starting training...")
# We iterate over epochs:
# After training, we compute and print the test error:
test_err = 0
test_acc = 0
test_batches = 0
all_test_pred = np.empty(
(0, 2), dtype=float
) # initialize; array n_samplesx2 for the 2 class predictions for all test samples
all_test_labels = np.empty(
(0, 1), dtype=float
) # initialize; array n_samplesx1 for labels of all test samples
for batch in Iterate_minibatches(test_set,
test_label,
int(batch_size),
shuffle=False):
inputs, targets = batch
inputs = np.float32(inputs)
err, acc, test_pred = val_fn(inputs, targets)
test_err += err
test_acc += acc
test_batches += 1
all_test_pred = np.vstack((all_test_pred, test_pred))
all_test_labels = np.append(all_test_labels, targets)
test_AUC, test_varAUC = SupportFuncs.Pred2AUC(all_test_pred,
all_test_labels)
savedSamplesFile = np.load(pathSavedSamples)
inputParamsConfigLocal = savedSamplesFile['inputParamsConfig'].item(
) #saved dict becomes an object, so need to turn back into dict!
screeningModelForPatches = inputParamsConfigLocal['fp_model_to_use']
masterPatchFolder = 'diskStation/LIDC/36368/screenPatches'
test_pos_paths = os.listdir(os.path.join(masterPatchFolder, 'pos'))
test_neg_paths = os.listdir(os.path.join(masterPatchFolder, 'neg'))
test_pos_paths = [
os.path.join(masterPatchFolder, 'pos', x) for x in test_pos_paths
]
test_neg_paths = [
os.path.join(masterPatchFolder, 'neg', x) for x in test_neg_paths
]
test_samples, test_labels, test_len_pos, test_len_neg = SupportFuncs.mat_generate_from_path(
test_pos_paths, test_neg_paths)
#test_samples = savedSamplesFile['inputs'] #the test samples/labels were saved in a particular order, so they can be loaded in same order
#test_labels = savedSamplesFile['targets']
#test_pred_orig = savedSamplesFile['test_pred']
########################
######Input Params######
#inputParamsConfigLocal = {}
#inputParamsConfigLocal['input_shape'] = '36, 36, 8'
#inputParamsConfigLocal['learning_rate'] = '0.04'
#inputParamsConfigLocal['momentum'] = '0.9'
#inputParamsConfigLocal['num_epochs'] = '35'
#inputParamsConfigLocal['batch_size'] = '100'
#inputParamsConfigLocal['noduleCaseFilterParams'] = 'NumberOfObservers,>=,2;IURatio,>=,0.2;SliceThicknessDicom,>=,1.5;SliceThicknessDicom,<=,3'
#inputParamsConfigLocal['train_set_size'] = '150000'
#inputParamsConfigLocal['test_set_size'] = '500'
#inputParamsConfigLocal['positive_set_ratio'] = '0.5'