def make_logfile_name(params=None, net_save_filename=None, timestamp = None, train=True):
'''
Returns the name of the logfile for the current training/forward pass run
'''
#Need to specify either a params object, or the net save prefix
utils.assert_arglist(params,
[net_save_filename])
if params is not None:
if train:
_net_save_filename = params['train_save_net']
else:
_net_save_filename = params['output_prefix']
if net_save_filename is not None:
_net_save_filename = net_save_filename
save_prefix = os.path.splitext( _net_save_filename )[0]
save_prefix_valid = len(save_prefix) > 0
assert(save_prefix_valid)
if timestamp is None:
timestamp = utils.timestamp()
mode = "train" if train else "forward"
directory_name = os.path.dirname( save_prefix )
if not os.path.exists(directory_name):
os.mkdir(directory_name)
save_filename = "{}_{}_{}.log".format(save_prefix, mode, timestamp)
return save_filename
def record_config_file(params=None,
config_filename=None,
net_save_filename=None,
timestamp=None,
train=True):
'''
Copies the config file used for the current run of ZNN under the same
prefix as the network save prefix
Format: {net_save_prefix}_{train/forward}_{timestamp}.cfg
e.g. net_train_20151007_17:48:55.cfg
'''
import shutil
#Need to specify either a params object, or all of the other optional args
#"ALL" optional args excludes train
utils.assert_arglist(params, [config_filename, net_save_filename])
#Args default to params values, override by options
if params is not None:
_config_filename = params['fconfig']
if train:
_net_save_filename = params['train_save_net']
else:
_net_save_filename = params['']
#Option override
if config_filename is not None:
_config_filename = config_filename
if net_save_filename is not None:
_net_save_filename = net_save_filename
#More error checking
save_prefix = os.path.splitext(_net_save_filename)[0]
config_file_exists = os.path.isfile(_config_filename)
assert (config_file_exists)
save_prefix_valid = len(save_prefix) > 0
assert (save_prefix_valid)
#Deriving destination filename information
if timestamp is None:
timestamp = utils.timestamp()
mode = "train" if train else "forward"
#Actually saving
# get directory name from filename
directory_name = os.path.dirname(save_prefix)
if not os.path.exists(directory_name):
os.mkdir(directory_name)
save_filename = "{}_{}_{}.cfg".format(save_prefix, mode, timestamp)
shutil.copy(_config_filename, save_filename)
def record_config_file(params=None, config_filename=None, net_save_filename=None,
timestamp=None, train=True):
'''
Copies the config file used for the current run of ZNN under the same
prefix as the network save prefix
Format: {net_save_prefix}_{train/forward}_{timestamp}.cfg
e.g. net_train_20151007_17:48:55.cfg
'''
import shutil
#Need to specify either a params object, or all of the other optional args
#"ALL" optional args excludes train
utils.assert_arglist(params,
[config_filename, net_save_filename]
)
#Args default to params values, override by options
if params is not None:
_config_filename = params['fconfig']
if train:
_net_save_filename = params['train_save_net']
else:
_net_save_filename = params['']
#Option override
if config_filename is not None:
_config_filename = config_filename
if net_save_filename is not None:
_net_save_filename = net_save_filename
#More error checking
save_prefix = os.path.splitext( _net_save_filename )[0]
config_file_exists = os.path.isfile( _config_filename )
assert(config_file_exists)
save_prefix_valid = len(save_prefix) > 0
assert(save_prefix_valid)
#Deriving destination filename information
if timestamp is None:
timestamp = utils.timestamp()
mode = "train" if train else "forward"
#Actually saving
# get directory name from filename
directory_name = os.path.dirname( save_prefix )
if not os.path.exists(directory_name):
os.mkdir(directory_name)
save_filename = "{}_{}_{}.cfg".format(save_prefix, mode, timestamp)
shutil.copy( _config_filename, save_filename)
def init_network( params=None, train=True, network_specfile=None,
output_patch_shape=None, num_threads=None, optimize=None,
force_fft=None ):
'''
Initializes a random network using the Boost Python interface and configuration
file options.
The function will define this network by a parameter object
(as generated by the front_end.parse function), or by the specified options.
If both a parameter object and any optional arguments are specified,
the parameter object will form the default options, and those will be
overwritten by the other optional arguments
'''
#Need to specify either a params object, or all of the other optional args
#"ALL" optional args excludes train
assert_arglist(params,
[network_specfile, output_patch_shape,
num_threads, optimize]
)
#Defining phase argument by train argument
phase = int(not train)
#If a params object exists, then those options are the default
if params is not None:
if train:
_output_patch_shape = params['train_outsz']
_optimize = params['is_train_optimize']
else:
_output_patch_shape = params['forward_outsz']
_optimize = params['is_forward_optimize']
_force_fft = params['force_fft']
_network_specfile = params['fnet_spec']
_num_threads = params['num_threads']
#Overwriting defaults with any other optional args
if network_specfile is not None:
_network_specfile = network_specfile
if output_patch_shape is not None:
_output_patch_shape = output_patch_shape
if num_threads is not None:
_num_threads = num_threads
if optimize is not None:
_optimize = optimize
if force_fft is not None:
_force_fft = force_fft
return pyznn.CNet(_network_specfile, _output_patch_shape,
_num_threads, _optimize, phase, _force_fft)
def init_network( params=None, train=True, network_specfile=None,
output_patch_shape=None, num_threads=None, optimize=None,
force_fft=None ):
'''
Initializes a random network using the Boost Python interface and configuration
file options.
The function will define this network by a parameter object
(as generated by the front_end.parse function), or by the specified options.
If both a parameter object and any optional arguments are specified,
the parameter object will form the default options, and those will be
overwritten by the other optional arguments
'''
#Need to specify either a params object, or all of the other optional args
#"ALL" optional args excludes train
assert_arglist(params,
[network_specfile, output_patch_shape,
num_threads, optimize]
)
#Defining phase argument by train argument
phase = int(not train)
#If a params object exists, then those options are the default
if params is not None:
if train:
_output_patch_shape = params['train_outsz']
_optimize = params['is_train_optimize']
else:
_output_patch_shape = params['forward_outsz']
_optimize = params['is_forward_optimize']
_force_fft = params['force_fft']
_network_specfile = params['fnet_spec']
_num_threads = params['num_threads']
#Overwriting defaults with any other optional args
if network_specfile is not None:
_network_specfile = network_specfile
if output_patch_shape is not None:
_output_patch_shape = output_patch_shape
if num_threads is not None:
_num_threads = num_threads
if optimize is not None:
_optimize = optimize
if force_fft is not None:
_force_fft = force_fft
return pyznn.CNet(_network_specfile, _output_patch_shape,
_num_threads, _optimize, phase, _force_fft)
def load_network(params=None,
train=True,
hdf5_filename=None,
network_specfile=None,
output_patch_shape=None,
num_threads=None,
optimize=None):
'''
Loads a network from an hdf5 file.
The function will define the loading process by a parameter object
(as generated by the front_end.parse function), or by the specified options.
If both a parameter object and any optional arguments are specified,
the parameter object will form the default options, and those will be
overwritten by the other optional arguments
'''
#Need to specify either a params object, or all of the other optional args
params_defined = params is not None
#"ALL" optional args excludes train (it has a default)
assert_arglist(params, [
hdf5_filename, network_specfile, output_patch_shape, num_threads,
optimize
])
#Defining phase argument by train argument
phase = int(not train)
#If a params object exists, then those options are the default
if params_defined:
if train:
_hdf5_filename = params['train_load_net']
_output_patch_shape = params['train_outsz']
_optimize = params['is_train_optimize']
else:
_hdf5_filename = params['forward_net']
_output_patch_shape = params['forward_outsz']
_optimize = params['is_forward_optimize']
_network_specfile = params['fnet_spec']
_num_threads = params['num_threads']
#Overwriting defaults with any other optional args
if hdf5_filename is not None:
_hdf5_filename = hdf5_filename
if network_specfile is not None:
_network_specfile = network_specfile
if output_patch_shape is not None:
_output_patch_shape = output_patch_shape
if num_threads is not None:
_num_threads = num_threads
#ACTUAL LOADING FUNCTIONALITY
#This is a little strange to allow for "seeding" larger
# nets with other training runs
# 1) Initialize template net for network_specfile
# 2) Load options from hdf5_filename (possibly containing the seed net)
# 3) Consolidate options from the template and seed (see consolidate_opts above)
# NOTE: the seed network could simply be the network we want to load,
# in which case this will overwrite all of the relevant template opts
# 4) Return consolidated CNet object
template = init_network(params, train, _network_specfile,
_output_patch_shape, _num_threads, False)
#If the file doesn't exist, init a new network
if os.path.isfile(_hdf5_filename):
load_options = load_opts(_hdf5_filename)
template_options = template.get_opts()
del template
print "consolidating options..."
final_options = consolidate_opts(load_options, template_options,
params)
else:
final_options = template.get_opts()
del template
return pyznn.CNet(final_options, _network_specfile, _output_patch_shape,
_num_threads, _optimize, phase)
def load_network( params=None, is_seed=False, train=True, hdf5_filename=None,
network_specfile=None, output_patch_shape=None, num_threads=None,
optimize=None, force_fft=None ):
'''
Loads a network from an hdf5 file.
The function will define the loading process by a parameter object
(as generated by the front_end.parse function), or by the specified options.
If both a parameter object and any optional arguments are specified,
the parameter object will form the default options, and those will be
overwritten by the other optional arguments
'''
#Need to specify either a params object, or all of the other optional args
params_defined = params is not None
#"ALL" optional args excludes train (it has a default)
assert_arglist(params,
[is_seed, hdf5_filename, network_specfile, output_patch_shape,
num_threads, optimize, force_fft])
#Defining phase argument by train argument
phase = int(not train)
#If a params object exists, then those options are the default
if params_defined:
if train:
_hdf5_filename = params['train_load_net']
_output_patch_shape = params['train_outsz']
_optimize = params['is_train_optimize']
else:
_hdf5_filename = params['forward_net']
_output_patch_shape = params['forward_outsz']
_optimize = params['is_forward_optimize']
_force_fft = params['force_fft']
_network_specfile = params['fnet_spec']
_num_threads = params['num_threads']
#Overwriting defaults with any other optional args
if hdf5_filename is not None:
_hdf5_filename = hdf5_filename
if network_specfile is not None:
_network_specfile = network_specfile
if output_patch_shape is not None:
_output_patch_shape = output_patch_shape
if num_threads is not None:
_num_threads = num_threads
#ACTUAL LOADING FUNCTIONALITY
#This is a little strange to allow for "seeding" larger
# nets with other training runs
# 1) Initialize template net for network_specfile
# 2) Load options from hdf5_filename (possibly containing the seed net)
# 3) Consolidate options from the template and seed (see consolidate_opts above)
# NOTE: the seed network could simply be the network we want to load,
# in which case this will overwrite all of the relevant template opts
# 4) Return consolidated CNet object
template = init_network( params, train, _network_specfile, _output_patch_shape,
_num_threads, False, False )
#If the file doesn't exist, init a new network
if os.path.isfile( _hdf5_filename ):
load_options = load_opts(_hdf5_filename)
template_options = template.get_opts()
del template
print "consolidating options..."
final_options = consolidate_opts(load_options, template_options, params, is_seed)
else:
final_options = template.get_opts()
del template
return pyznn.CNet(final_options, _network_specfile, _output_patch_shape,
_num_threads, _optimize, phase, _force_fft)