def get_initialised_driver(starting_iter=0):
system_param = {
'SYSTEM':
ParserNamespace(action='train',
num_threads=2,
num_gpus=4,
cuda_devices='6',
model_dir='./testing_data'),
'NETWORK':
ParserNamespace(batch_size=20, name='tests.toy_application.TinyNet'),
'TRAINING':
ParserNamespace(
starting_iter=starting_iter,
max_iter=500,
save_every_n=20,
tensorboard_every_n=1,
max_checkpoints=20,
optimiser='niftynet.engine.application_optimiser.Adagrad',
lr=0.01),
'CUSTOM':
ParserNamespace(vector_size=100,
mean=10.0,
stddev=2.0,
name='tests.toy_application.ToyApplication')
}
app_driver = ApplicationDriver()
app_driver.initialise_application(system_param, {})
return app_driver
def get_initialised_driver(starting_iter=0,
model_dir_rand=True,
vars_to_restore='',
application='tests.toy_application.ToyApplication'):
if model_dir_rand:
model_dir = os.path.join('.', 'testing_data', 'tmp', str(uuid.uuid4()))
os.makedirs(model_dir)
else:
model_dir = os.path.join('.', 'testing_data')
system_param = {
'SYSTEM':
ParserNamespace(action='train',
num_threads=2,
num_gpus=4,
cuda_devices='6',
model_dir=model_dir,
dataset_split_file=os.path.join(
'.', 'testing_data', 'testtoyapp.csv'),
event_handler=[
'niftynet.engine.handler_model.ModelRestorer',
'niftynet.engine.handler_sampler.SamplerThreading',
'niftynet.engine.handler_gradient.ApplyGradients'
],
iteration_generator=None),
'NETWORK':
ParserNamespace(batch_size=20, name='tests.toy_application.TinyNet'),
'TRAINING':
ParserNamespace(
starting_iter=starting_iter,
max_iter=500,
save_every_n=20,
tensorboard_every_n=1,
max_checkpoints=20,
optimiser='niftynet.engine.application_optimiser.Adagrad',
validation_every_n=-1,
exclude_fraction_for_validation=0.1,
exclude_fraction_for_inference=0.1,
vars_to_restore=vars_to_restore,
patience=100,
lr=0.01),
'CUSTOM':
ParserNamespace(vector_size=100,
mean=10.0,
stddev=2.0,
name=application)
}
app_driver = ApplicationDriver()
app_driver.initialise_application(system_param, {})
# set parameters without __init__
app_driver.app.action_param = system_param['TRAINING']
app_driver.app.net_param = system_param['NETWORK']
app_driver.app.action = 'train'
return app_driver
def test_no_data_param_infer(self):
clear_target()
user_param = generate_input_params(SYSTEM={
'action': 'inference',
'dataset_split_file': TARGET_FILE
})
app_driver = ApplicationDriver()
app_driver.initialise_application(user_param, {})
self.assertTrue(app_driver._data_partitioner is not None)
self.assertFalse(os.path.isfile(TARGET_FILE))
partitioner = app_driver._data_partitioner
self.assertFalse(partitioner.all_files)
def test_no_data_param_infer(self):
clear_target()
user_param = generate_input_params(
SYSTEM={'action': 'inference',
'dataset_split_file': TARGET_FILE
}
)
app_driver = ApplicationDriver()
app_driver.initialise_application(user_param, {})
self.assertTrue(app_driver._data_partitioner is not None)
self.assertFalse(os.path.isfile(TARGET_FILE))
partitioner = app_driver._data_partitioner
self.assertFalse(partitioner.all_files)
def write_target():
clear_target()
user_param = generate_input_params(
SYSTEM={'action': 'train',
'dataset_split_file': TARGET_FILE
},
TRAINING={'validation_every_n': 2,
'validation_max_iter': 1,
'exclude_fraction_for_validation': 0.1,
'exclude_fraction_for_inference': 0.1,
}
)
data_param = _generate_data_param()
app_driver = ApplicationDriver()
app_driver.initialise_application(user_param, data_param)
assert os.path.isfile(TARGET_FILE)
return
def write_target():
clear_target()
user_param = generate_input_params(
SYSTEM={'action': 'train',
'dataset_split_file': TARGET_FILE
},
TRAINING={'validation_every_n': 2,
'validation_max_iter': 1,
'exclude_fraction_for_validation': 0.1,
'exclude_fraction_for_inference': 0.1,
}
)
data_param = _generate_data_param()
app_driver = ApplicationDriver()
app_driver.initialise_application(user_param, data_param)
assert os.path.isfile(TARGET_FILE)
return
def test_no_data_param_train(self):
clear_target()
user_param = generate_input_params(
SYSTEM={'action': 'train',
'dataset_split_file': TARGET_FILE
},
TRAINING={'validation_every_n': -1,
'exclude_fraction_for_validation': 0.1,
'exclude_fraction_for_inference': 0.1,
}
)
app_driver = ApplicationDriver()
app_driver.initialise_application(user_param, {})
self.assertTrue(app_driver._data_partitioner is not None)
self.assertFalse(os.path.isfile(TARGET_FILE))
partitioner = app_driver._data_partitioner
self.assertFalse(partitioner.all_files)
def test_no_data_param_train(self):
clear_target()
user_param = generate_input_params(
SYSTEM={'action': 'train',
'dataset_split_file': TARGET_FILE
},
TRAINING={'validation_every_n': -1,
'exclude_fraction_for_validation': 0.1,
'exclude_fraction_for_inference': 0.1,
}
)
app_driver = ApplicationDriver()
app_driver.initialise_application(user_param, {})
self.assertTrue(app_driver._data_partitioner is not None)
self.assertFalse(os.path.isfile(TARGET_FILE))
partitioner = app_driver._data_partitioner
self.assertFalse(partitioner.all_files)
def test_training(self):
write_target()
user_param = generate_input_params(
SYSTEM={'action': 'train',
'dataset_split_file': TARGET_FILE
},
TRAINING={'validation_every_n': 2,
'validation_max_iter': 1,
'exclude_fraction_for_validation': 0.1,
'exclude_fraction_for_inference': 0.1,
}
)
data_param = _generate_data_param()
app_driver = ApplicationDriver()
app_driver.initialise_application(user_param, data_param)
partitioner = app_driver._data_partitioner
self.assertTrue(partitioner.has_training)
self.assertTrue(partitioner.has_inference)
self.assertTrue(partitioner.has_validation)
def test_training(self):
write_target()
user_param = generate_input_params(
SYSTEM={'action': 'train',
'dataset_split_file': TARGET_FILE
},
TRAINING={'validation_every_n': 2,
'validation_max_iter': 1,
'exclude_fraction_for_validation': 0.1,
'exclude_fraction_for_inference': 0.1,
}
)
data_param = _generate_data_param()
app_driver = ApplicationDriver()
app_driver.initialise_application(user_param, data_param)
partitioner = app_driver._data_partitioner
self.assertTrue(partitioner.has_training)
self.assertTrue(partitioner.has_inference)
self.assertTrue(partitioner.has_validation)
def test_inference(self):
clear_target()
user_param = generate_input_params(
SYSTEM={'action': 'inference',
'dataset_split_file': TARGET_FILE
},
TRAINING={'validation_every_n': 1,
'validation_max_iter': 1,
'exclude_fraction_for_validation': 0.1,
'exclude_fraction_for_inference': 0.0,
}
)
data_param = _generate_data_param()
app_driver = ApplicationDriver()
app_driver.initialise_application(user_param, data_param)
self.assertTrue(app_driver._data_partitioner is not None)
self.assertFalse(os.path.isfile(TARGET_FILE))
partitioner = app_driver._data_partitioner
self.assertTrue(partitioner._partition_ids is None)
def test_inference(self):
clear_target()
user_param = generate_input_params(
SYSTEM={'action': 'inference',
'dataset_split_file': TARGET_FILE
},
TRAINING={'validation_every_n': 1,
'validation_max_iter': 1,
'exclude_fraction_for_validation': 0.1,
'exclude_fraction_for_inference': 0.0,
}
)
data_param = _generate_data_param()
app_driver = ApplicationDriver()
app_driver.initialise_application(user_param, data_param)
self.assertTrue(app_driver._data_partitioner is not None)
self.assertFalse(os.path.isfile(TARGET_FILE))
partitioner = app_driver._data_partitioner
self.assertTrue(partitioner._partition_ids is None)
def main():
system_param, input_data_param = user_parameters_parser.run()
if util.has_bad_inputs(system_param):
return -1
# print all parameters to txt file for future reference
all_param = {}
all_param.update(system_param)
all_param.update(input_data_param)
txt_file = 'settings_{}.txt'.format(system_param['SYSTEM'].action)
model_folder = touch_folder(system_param['SYSTEM'].model_dir)
txt_file = os.path.join(model_folder, txt_file)
util.print_save_input_parameters(all_param, txt_file)
# keep all commandline outputs
log_file_name = os.path.join(
model_folder, '{}_{}'.format(all_param['SYSTEM'].action,
'niftynet_log'))
set_logger(file_name=log_file_name)
# start application
app_driver = ApplicationDriver()
app_driver.initialise_application(system_param, input_data_param)
app_driver.run_application()
return 0
def get_initialised_driver(starting_iter=0):
system_param = {
'SYSTEM':
ParserNamespace(action='train',
num_threads=2,
num_gpus=4,
cuda_devices='6',
model_dir=os.path.join('.', 'testing_data'),
dataset_split_file=os.path.join(
'.', 'testing_data', 'testtoyapp.csv')),
'NETWORK':
ParserNamespace(batch_size=20, name='tests.toy_application.TinyNet'),
'TRAINING':
ParserNamespace(
starting_iter=starting_iter,
max_iter=500,
save_every_n=20,
tensorboard_every_n=1,
max_checkpoints=20,
optimiser='niftynet.engine.application_optimiser.Adagrad',
validation_every_n=-1,
exclude_fraction_for_validation=0.1,
exclude_fraction_for_inference=0.1,
lr=0.01),
'CUSTOM':
ParserNamespace(vector_size=100,
mean=10.0,
stddev=2.0,
name='tests.toy_application.ToyApplication')
}
app_driver = ApplicationDriver()
app_driver.initialise_application(system_param, {})
# set parameters without __init__
app_driver.app.action_param = system_param['TRAINING']
app_driver.app.net_param = system_param['NETWORK']
app_driver.app.is_training = True
return app_driver
def get_initialised_driver(starting_iter=0):
system_param = {
'SYSTEM': ParserNamespace(
action='train',
num_threads=2,
num_gpus=4,
cuda_devices='6',
model_dir=os.path.join('.', 'testing_data'),
dataset_split_file=os.path.join(
'.', 'testing_data', 'testtoyapp.csv')),
'NETWORK': ParserNamespace(
batch_size=20,
name='tests.toy_application.TinyNet'),
'TRAINING': ParserNamespace(
starting_iter=starting_iter,
max_iter=500,
save_every_n=20,
tensorboard_every_n=1,
max_checkpoints=20,
optimiser='niftynet.engine.application_optimiser.Adagrad',
validation_every_n=-1,
exclude_fraction_for_validation=0.1,
exclude_fraction_for_inference=0.1,
lr=0.01),
'CUSTOM': ParserNamespace(
vector_size=100,
mean=10.0,
stddev=2.0,
name='tests.toy_application.ToyApplication')
}
app_driver = ApplicationDriver()
app_driver.initialise_application(system_param, {})
# set parameters without __init__
app_driver.app.action_param = system_param['TRAINING']
app_driver.app.net_param = system_param['NETWORK']
app_driver.app.is_training = True
return app_driver
def main():
system_param, input_data_param = user_parameters_parser.run()
if util.has_bad_inputs(system_param):
return -1
# print all parameters to txt file for future reference
all_param = {}
all_param.update(system_param)
all_param.update(input_data_param)
# Set up path for niftynet model_root
# (rewriting user input with an absolute path)
system_param['SYSTEM'].model_dir = resolve_module_dir(
system_param['SYSTEM'].model_dir,
create_new=system_param['SYSTEM'].action == "train")
# writing all params for future reference
txt_file = 'settings_{}.txt'.format(system_param['SYSTEM'].action)
txt_file = os.path.join(system_param['SYSTEM'].model_dir, txt_file)
util.print_save_input_parameters(all_param, txt_file)
# keep all commandline outputs to model_root
log_file_name = os.path.join(
system_param['SYSTEM'].model_dir,
'{}_{}'.format(all_param['SYSTEM'].action, 'niftynet_log'))
set_logger(file_name=log_file_name)
# set up all model folder related parameters here
# see https://cmiclab.cs.ucl.ac.uk/CMIC/NiftyNet/issues/168
# 1. resolve mapping file:
try:
if system_param['NETWORK'].histogram_ref_file:
system_param['NETWORK'].histogram_ref_file = to_absolute_path(
input_path=system_param['NETWORK'].histogram_ref_file,
model_root=system_param['SYSTEM'].model_dir)
except (AttributeError, KeyError):
pass
# 2. resolve output file:
try:
if system_param['INFERENCE'].save_seg_dir:
system_param['INFERENCE'].save_seg_dir = to_absolute_path(
input_path=system_param['INFERENCE'].save_seg_dir,
model_root=system_param['SYSTEM'].model_dir)
except (AttributeError, KeyError):
pass
# start application
app_driver = ApplicationDriver()
app_driver.initialise_application(system_param, input_data_param)
app_driver.run_application()
return 0
def test_wrong_init(self):
app_driver = ApplicationDriver()
with self.assertRaisesRegexp(AttributeError, ''):
app_driver.initialise_application([], [])
def main():
set_logger()
system_param, input_data_param = user_parameters_parser.run()
if util.has_bad_inputs(system_param):
return -1
# print all parameters to txt file for future reference
all_param = {}
all_param.update(system_param)
all_param.update(input_data_param)
# Set up path for niftynet model_root
# (rewriting user input with an absolute path)
system_param['SYSTEM'].model_dir = resolve_module_dir(
system_param['SYSTEM'].model_dir,
create_new=system_param['SYSTEM'].action == "train")
# writing all params for future reference
txt_file = 'settings_{}.txt'.format(system_param['SYSTEM'].action)
txt_file = os.path.join(system_param['SYSTEM'].model_dir, txt_file)
try:
util.print_save_input_parameters(all_param, txt_file)
except IOError:
tf.logging.fatal(
'Unable to write %s,\nplease check '
'model_dir parameter, current value: %s',
txt_file, system_param['SYSTEM'].model_dir)
raise
# keep all commandline outputs to model_root
log_file_name = os.path.join(
system_param['SYSTEM'].model_dir,
'{}_{}'.format(all_param['SYSTEM'].action, 'niftynet_log'))
set_logger(file_name=log_file_name)
# set up all model folder related parameters here
# see https://cmiclab.cs.ucl.ac.uk/CMIC/NiftyNet/issues/168
# 1. resolve mapping file:
try:
if system_param['NETWORK'].histogram_ref_file:
system_param['NETWORK'].histogram_ref_file = to_absolute_path(
input_path=system_param['NETWORK'].histogram_ref_file,
model_root=system_param['SYSTEM'].model_dir)
except (AttributeError, KeyError):
pass
# 2. resolve output file:
try:
if system_param['INFERENCE'].save_seg_dir:
system_param['INFERENCE'].save_seg_dir = to_absolute_path(
input_path=system_param['INFERENCE'].save_seg_dir,
model_root=system_param['SYSTEM'].model_dir)
except (AttributeError, KeyError):
pass
# 3. resolve dataset splitting file:
try:
if system_param['SYSTEM'].dataset_split_file:
system_param['SYSTEM'].dataset_split_file = to_absolute_path(
input_path=system_param['SYSTEM'].dataset_split_file,
model_root=system_param['SYSTEM'].model_dir)
except (AttributeError, KeyError):
pass
# start application
app_driver = ApplicationDriver()
app_driver.initialise_application(system_param, input_data_param)
app_driver.run_application()
return 0
def main():
set_logger()
print("Hola como vai estoy en el init de niftynet ################")
system_param, input_data_param = user_parameters_parser.run()
if util.has_bad_inputs(system_param):
return -1
# print all parameters to txt file for future reference
all_param = {}
all_param.update(system_param)
all_param.update(input_data_param)
# Set up path for niftynet model_root
# (rewriting user input with an absolute path)
system_param['SYSTEM'].model_dir = resolve_module_dir(
system_param['SYSTEM'].model_dir,
create_new=system_param['SYSTEM'].action == "train")
# writing all params for future reference
txt_file = 'settings_{}.txt'.format(system_param['SYSTEM'].action)
txt_file = os.path.join(system_param['SYSTEM'].model_dir, txt_file)
try:
util.print_save_input_parameters(all_param, txt_file)
except IOError:
tf.logging.fatal(
'Unable to write %s,\nplease check '
'model_dir parameter, current value: %s',
txt_file, system_param['SYSTEM'].model_dir)
raise
# keep all commandline outputs to model_root
log_file_name = os.path.join(
system_param['SYSTEM'].model_dir,
'{}_{}'.format(all_param['SYSTEM'].action, 'niftynet_log'))
set_logger(file_name=log_file_name)
# set up all model folder related parameters here
# see https://cmiclab.cs.ucl.ac.uk/CMIC/NiftyNet/issues/168
# 1. resolve mapping file:
try:
if system_param['NETWORK'].histogram_ref_file:
system_param['NETWORK'].histogram_ref_file = to_absolute_path(
input_path=system_param['NETWORK'].histogram_ref_file,
model_root=system_param['SYSTEM'].model_dir)
except (AttributeError, KeyError):
pass
# 2. resolve output file:
try:
if system_param['INFERENCE'].save_seg_dir:
system_param['INFERENCE'].save_seg_dir = to_absolute_path(
input_path=system_param['INFERENCE'].save_seg_dir,
model_root=system_param['SYSTEM'].model_dir)
except (AttributeError, KeyError):
pass
# 3. resolve dataset splitting file:
try:
if system_param['SYSTEM'].dataset_split_file:
system_param['SYSTEM'].dataset_split_file = to_absolute_path(
input_path=system_param['SYSTEM'].dataset_split_file,
model_root=system_param['SYSTEM'].model_dir)
except (AttributeError, KeyError):
pass
# start application
app_driver = ApplicationDriver()
print("Initializing application$$$$$$$$$$$$$$$$$$$$$BEGIN")
app_driver.initialise_application(system_param, input_data_param)
print("Initializing application$$$$$$$$$$$$$$$$$$$$$END")
print("Initializing RUN application$$$$$$$$$$$$$$$$$$$$$BEGIN")
app_driver.run_application()
print("Initializing RUN application$$$$$$$$$$$$$$$$$$$$$END")
return 0
def test_wrong_init(self):
app_driver = ApplicationDriver()
with self.assertRaisesRegexp(AttributeError, ''):
app_driver.initialise_application([], [])