def SVM_test_function(vals):
s = GpyOptOption.convert_to_objects(*vals[0])
dbd = GpyOptOption.convert_to_db_description(*vals[0])
dbr = " ".join(GpyOptOption.tagit(vals[0]))
print('Started: {} {}'.format(
datetime.datetime.now().strftime("%D %H:%M:%S"), dbr))
# algorithm
version = (1, 2, 0)
norma_data, norma_feature = s['norma_data'], s['norma_feature']
norma_feature = norma_feature()
ds = DatasetGenerator(150000, train_iterations=1, train=True)
alg = LinearSVC(penalty=s['penalty_type'],
C=s['penalty'],
dual=s['dual'],
loss=s['loss'])
acc_final, tested = 0., 0.
for (Xqueue, Yqueue), (Xweek, Yweek) in ds.queue_iterator():
Xqueue = norma_data(norma_feature.fit_transform(Xqueue))
Xweek = norma_data(norma_feature.transform(Xweek))
alg.fit(Xqueue, Yqueue)
acc = alg.score(Xweek, Yweek)
acc_final += acc
tested += 1.
print("Act: {}".format(acc))
acc_final /= tested
write_kwargs_to_db(
TABLE='SVM',
VERSION=version,
RESULT=acc_final,
**dbd,
) # dataset info
print('Ended with {}: {}'.format(acc_final, dbr))
return -acc_final
print("Dice coefficient = {:.4f}, ".format(dice_coef), end="")
plt.title("Prediction\nDice = {:.4f}".format(dice_coef), fontsize=20)
save_name = os.path.join(png_directory,
"prediction_tf_{}.png".format(idx))
print("Saved as: {}".format(save_name))
plt.savefig(save_name)
if __name__ == "__main__":
model_filename = os.path.join(args.output_path, args.inference_filename)
ds_testing = DatasetGenerator(os.path.join(args.data_path,
"testing/*.npz"),
crop_dim=args.crop_dim,
batch_size=128,
augment=False,
seed=args.seed)
# Load model
if args.use_pconv:
from model_pconv import unet
unet_model = unet(use_pconv=True)
else:
from model import unet
unet_model = unet()
model = unet_model.load_model(model_filename)
# Create output directory for images
png_directory = args.output_pngs
if not os.path.exists(png_directory):
def NN_test_function(vals):
s = GpyOptOption.convert_to_objects(*vals[0])
dbd = GpyOptOption.convert_to_db_description(*vals[0])
dbr = " ".join(GpyOptOption.tagit(vals[0]))
print('Started: {} {}'.format(
datetime.datetime.now().strftime("%D %H:%M:%S"), dbr))
start_time = datetime.datetime.now()
# algorithm
version = (1, 1, 0)
layers = layersoo.get_layers_from_object_dict(s)
batch_size = s['batch_size']
l1 = s['l1_reg']
l2 = s['l2_reg']
dropout = s['dropout'] if s['use_dropout'] else 0.
gaussian_noise_stddev = s['gaussian_noise'] if s[
'use_gaussian_noise'] else 0.
batchnormalization = s['use_batchnorm']
norma_feature = s['norma_feature']()
ds = DatasetGenerator(150000, train_iterations=1, train=True)
network = NeuralNetworkClassifier(
input_dim=540,
output_dim=2,
layers=layers,
activation=s['activation'],
batch_size=batch_size,
learning_rate=s['learning_rate'],
learning_rate_decay=s['learning_rate_decay'],
l1=l1,
l2=l2,
dropout=dropout,
gaussian_noise_stddev=gaussian_noise_stddev,
batchnormalization=batchnormalization,
use_amsgrad=s['use_amsgrad'],
alpha=s['alpha'],
verbose=1)
acc_final, tested = 0., 0.
for (Xqueue, Yqueue), (Xweek, Yweek) in ds.queue_iterator():
Xqueue = norma_feature.fit_transform(Xqueue)
Xweek = norma_feature.transform(Xweek)
network.fit(Xqueue, Yqueue)
acc = network.score(Xweek, Yweek)[1]
acc_final += acc
tested += 1.
print("Act: {}".format(acc))
acc_final /= tested
end_time = (datetime.datetime.now() - start_time).seconds
write_kwargs_to_db(TABLE='NeuralNetwork',
VERSION=version,
RESULT=acc_final,
DURATION=end_time,
**dbd)
print('Ended with {} ({:.5}min): {}'.format(acc_final, end_time / 60, dbr))
return -acc_final
def train_and_predict(data_path, crop_dim, batch_size, n_epoch):
"""
Create a model, load the data, and train it.
"""
"""
Step 1: Load the data
"""
print("-" * 30)
print("Loading the data from the NumPy files to tf.dataset ...")
print("-" * 30)
ds_train = DatasetGenerator(os.path.join(data_path, "train/*.npz"),
crop_dim=crop_dim,
batch_size=batch_size,
augment=True,
seed=args.seed)
ds_validation = DatasetGenerator(os.path.join(data_path,
"validation/*.npz"),
crop_dim=crop_dim,
batch_size=batch_size,
augment=False,
seed=args.seed)
ds_testing = DatasetGenerator(os.path.join(data_path, "testing/*.npz"),
crop_dim=crop_dim,
batch_size=batch_size,
augment=False,
seed=args.seed)
print("-" * 30)
print("Creating and compiling model ...")
print("-" * 30)
"""
Step 2: Define the model
"""
if args.use_pconv:
from model_pconv import unet
else:
from model import unet
unet_model = unet()
model = unet_model.create_model(ds_train.get_input_shape(),
ds_train.get_output_shape())
model_filename, model_callbacks = unet_model.get_callbacks()
# If there is a current saved file, then load weights and start from
# there.
saved_model = os.path.join(args.output_path, args.inference_filename)
if os.path.isfile(saved_model):
model.load_weights(saved_model)
"""
Step 3: Train the model on the data
"""
print("-" * 30)
print("Fitting model with training data ...")
print("-" * 30)
model.fit(ds_train.get_dataset(),
epochs=n_epoch,
validation_data=ds_validation.get_dataset(),
verbose=1,
callbacks=model_callbacks)
"""
Step 4: Evaluate the best model
"""
print("-" * 30)
print("Loading the best trained model ...")
print("-" * 30)
unet_model.evaluate_model(model_filename, ds_testing.get_dataset())
"""
Step 5: Print the command to convert TensorFlow model into OpenVINO format with model optimizer.
"""
print("-" * 30)
print("-" * 30)
unet_model.print_openvino_mo_command(model_filename,
ds_testing.get_input_shape())
save_name = os.path.join(png_directory,
"prediction_tf_{}_{}.png".format(batch_num, idx))
print("Saved as: {}".format(save_name))
plt.savefig(save_name)
if __name__ == "__main__":
model_filename = os.path.join(args.output_path, args.inference_filename)
trainFiles, validateFiles, testFiles = get_decathlon_filelist(
data_path=args.data_path, seed=args.seed, split=args.split)
ds_test = DatasetGenerator(testFiles,
batch_size=128,
crop_dim=[args.crop_dim, args.crop_dim],
augment=False,
seed=args.seed)
# Load model
if args.use_pconv:
from model_pconv import unet
unet_model = unet(use_pconv=True)
else:
from model import unet
unet_model = unet()
model = unet_model.load_model(model_filename)
# Create output directory for images
png_directory = args.output_pngs
tf.pywrap_tensorflow.IsMklEnabled())
test_intel_tensorflow() # Prints if Intel-optimized TensorFlow is used.
"""
crop_dim = Dimensions to crop the input tensor
"""
crop_dim = (args.tile_height, args.tile_width, args.tile_depth,
args.number_input_channels)
"""
1. Load the dataset
"""
brats_data = DatasetGenerator(crop_dim,
data_path=args.data_path,
batch_size=args.batch_size,
train_test_split=args.train_test_split,
validate_test_split=args.validate_test_split,
number_output_classes=args.number_output_classes,
random_seed=args.random_seed,
shard=hvd.rank())
if (hvd.rank() == 0):
print("{} workers".format(hvd.size()))
brats_data.print_info() # Print dataset information
"""
2. Create the TensorFlow model
"""
model = unet_3d(input_dim=crop_dim,
filters=args.filters,
number_output_classes=args.number_output_classes,
use_upsampling=args.use_upsampling,
concat_axis=-1,
"""
"""
Step 1: Define a data loader
"""
print("-" * 30)
print(
"Loading the data from the Medical Decathlon directory to a TensorFlow data loader ..."
)
print("-" * 30)
trainFiles, validateFiles, testFiles = get_decathlon_filelist(
data_path=args.data_path, seed=args.seed, split=args.split)
ds_train = DatasetGenerator(trainFiles,
batch_size=args.batch_size,
crop_dim=[args.crop_dim, args.crop_dim],
augment=True,
seed=args.seed)
ds_validation = DatasetGenerator(validateFiles,
batch_size=args.batch_size,
crop_dim=[args.crop_dim, args.crop_dim],
augment=False,
seed=args.seed)
ds_test = DatasetGenerator(testFiles,
batch_size=args.batch_size,
crop_dim=[args.crop_dim, args.crop_dim],
augment=False,
seed=args.seed)
print("-" * 30)
print("Creating and compiling model ...")
print(args)
test_intel_tensorflow() # Prints if Intel-optimized TensorFlow is used.
"""
crop_dim = Dimensions to crop the input tensor
"""
crop_dim = (args.tile_height, args.tile_width, args.tile_depth,
args.number_input_channels)
"""
1. Load the dataset
"""
brats_data = DatasetGenerator(crop_dim,
data_path=args.data_path,
batch_size=args.batch_size,
train_test_split=args.train_test_split,
validate_test_split=args.validate_test_split,
number_output_classes=args.number_output_classes,
random_seed=args.random_seed)
brats_data.print_info() # Print dataset information
"""
2. Create the TensorFlow model
"""
model = unet_3d(input_dim=crop_dim,
filters=args.filters,
number_output_classes=args.number_output_classes,
use_upsampling=args.use_upsampling,
concat_axis=-1,
model_name=args.saved_model_name)
def VAE_train_test(idexp):
last_sample, last_week_nn, last_week_vae, scaler = saver.load_result()
ds = DatasetGenerator(150000,
train_iterations=1,
train=False,
skip_weeks=idexp)
debug = True
if debug:
maximum_epochs = 1
generate_number = 150
else:
maximum_epochs = 1000
generate_number = 150000
i = 1
for (Xqueue, Yqueue), (Xweek, Yweek) in ds.queue_iterator():
print('Started {}: {}'.format(
i,
datetime.datetime.now().strftime("%D %H:%M:%S")))
if i <= last_sample:
print("Skipping iteration {} (already done)".format(i))
i += 1
continue
else:
print("Started iteration no. {}".format(i))
if i == 1:
scaler = MinMaxScaler()
#scaler = PowerTransformer(standardize=False)
print('Transform')
Xqueue = scaler.fit_transform(Xqueue)
Xweek = scaler.transform(Xweek)
c = MyChecker.CheckManager(3, MyChecker.NanChecker.NanError)
while c.not_done():
with c:
print('\tFit NN')
last_week_nn = create_new_NN_V2()
last_week_nn.fit(Xqueue,
Yqueue,
maximum_epochs=maximum_epochs)
if c.not_done():
last_week_nn.free()
last_week_nn = None
print('\tScore NN')
labels, probabilities = last_week_nn.get_label_and_probability(
Xweek)
print('\tSave result')
saver.save_result(i, np.mean(np.equal(labels, Yweek)),
probabilities)
c = MyChecker.CheckManager(3, MyChecker.NanChecker.NanError)
while c.not_done():
with c:
print('\tFit VAE')
last_week_vae = create_new_VAE_V2()
last_week_vae.fit(Xqueue, maximum_epochs=maximum_epochs)
if c.not_done():
last_week_vae.free()
last_week_vae = None
print('\tSave models')
saver.save_models(i, last_week_nn, last_week_vae, scaler)
else:
print('\tScore with previous NN')
labels, probabilities = last_week_nn.get_label_and_probability(
scaler.transpose(Xweek))
print('\tSave result')
res = np.mean(np.equal(labels, Yweek))
saver.save_result(i, res, probabilities)
print('\t\tresult: {}'.format(res))
c = MyChecker.CheckManager(3, MyChecker.NanChecker.NanError)
while c.not_done():
with c:
print('\tGenerate')
generated_data = last_week_vae.generate(generate_number)
print('\tPredict')
generated_prediction = last_week_nn.get_label(generated_data)
print('\tInverse transform')
generated_data = scaler.inverse_transform(generated_data)
print('\tConcatenate old with new')
Xqueue = np.concatenate([generated_data, Xqueue], axis=0)
Yqueue = np.concatenate([generated_prediction, Yqueue], axis=0)
print('\tTransform')
pred = Xqueue
Xqueue = scaler.fit_transform(Xqueue)
np.savez('spatne.npz', tet=Xqueue, predtim=pred)
c = MyChecker.CheckManager(3, MyChecker.NanChecker.NanError)
while c.not_done():
with c:
print('\tFit new NN')
new_week_nn = create_new_NN_V2()
new_week_nn.fit(Xqueue,
Yqueue,
maximum_epochs=maximum_epochs)
if c.not_done():
new_week_nn.free()
new_week_nn = None
c = MyChecker.CheckManager(3, MyChecker.NanChecker.NanError)
while c.not_done():
with c:
print('\tFit VAE')
new_week_vae = create_new_VAE_V2()
new_week_vae.fit(Xqueue, maximum_epochs=maximum_epochs)
if c.not_done():
new_week_vae.free()
new_week_vae = None
print('\tSave models')
saver.save_models(i, new_week_nn, new_week_vae, scaler)
last_week_nn.free()
last_week_vae.free()
last_week_nn = new_week_nn
last_week_vae = new_week_vae
print('\tClear old models')
i += 1