def save_args(self, args_file="config.pkl"):
"""
This saves the model's initial configuration parameters (`self.args`) in a pickle file.
Parameters
----------
args_file : str, optional
Filename of pickled configuration parameters. Defaults to 'config.pkl'.
Returns
-------
bool
Whether or not successfully saved the file.
"""
# make sure outdir is not set to False (no outputs/saving)
if getattr(self, 'outdir', None):
args_path = os.path.join(self.outdir, args_file)
args_file = os.path.realpath(args_path)
ftype = file_ops.get_extension_type(args_file)
args = self.args.copy()
args[class_key] = self._classname
# force extension to be .pkl if it isn't a pickle file
if ftype != file_ops.PKL:
args_file = ''.join([args_file, '.pkl'])
log.debug('Saving %s configuration to %s', self._classname,
str(args_file))
# try to dump the args values
with open(args_file, 'wb') as f:
try:
pickle.dump(args, f, protocol=pickle.HIGHEST_PROTOCOL)
except Exception as e:
log.exception(
"Some issue saving model %s configuration to %s! Exception: %s",
self._classname, str(args_file), str(e))
return False
finally:
f.close()
return True
else:
return False
def save_args(self, args_file="config.pkl"):
"""
This saves the model's initial configuration parameters (`self.args`) in a pickle file.
Parameters
----------
args_file : str, optional
Filename of pickled configuration parameters. Defaults to 'config.pkl'.
Returns
-------
bool
Whether or not successfully saved the file.
"""
# make sure outdir is not set to False (no outputs/saving)
if getattr(self, 'outdir', None):
args_path = os.path.join(self.outdir, args_file)
args_file = os.path.realpath(args_path)
ftype = file_ops.get_extension_type(args_file)
args = self.args.copy()
args[class_key] = self._classname
# force extension to be .pkl if it isn't a pickle file
if ftype != file_ops.PKL:
args_file = ''.join([args_file, '.pkl'])
log.debug('Saving %s configuration to %s',
self._classname, str(args_file))
# try to dump the args values
with open(args_file, 'wb') as f:
try:
pickle.dump(args, f, protocol=pickle.HIGHEST_PROTOCOL)
except Exception as e:
log.exception("Some issue saving model %s configuration to %s! Exception: %s",
self._classname, str(args_file), str(e))
return False
finally:
f.close()
return True
else:
return False
def save_run(self, filename):
"""
Saves (pickle) the compiled theano function for running the model.
Parameters
----------
filename : str
Filepath to save the compiled run function
Returns
-------
tuple(bool, str)
Tuple of [whether or not successful] and [complete filepath to saved file].
"""
# make sure outdir is not set to False (no outputs/saving)
if getattr(self, 'outdir', None):
filepath = os.path.join(self.outdir, filename)
save_file = os.path.realpath(filepath)
ftype = file_ops.get_extension_type(save_file)
# force extension to be .pkl if it isn't a pickle file
if ftype != file_ops.PKL:
save_file = ''.join([save_file, '.pkl'])
log.debug('Saving %s compiled run function to %s',
self._classname, str(save_file))
# try to dump the param values
with open(save_file, 'wb') as f:
try:
run_fn = self.compile_run_fn()
pickle.dump(run_fn, f, protocol=pickle.HIGHEST_PROTOCOL)
except Exception as e:
if "maximum recursion depth exceeded" in str(e):
recursion_limit = 50000
import sys
while "maximum recursion depth exceeded" in str(e):
log.debug("found recursion depth bug when pickling function...bumping limit to %d"
% recursion_limit)
sys.setrecursionlimit(recursion_limit)
try:
run_fn = self.compile_run_fn()
pickle.dump(run_fn, f, protocol=pickle.HIGHEST_PROTOCOL)
return (True, save_file)
except Exception as e:
if "maximum recursion depth exceeded" not in str(e):
log.exception("Some issue saving model %s run function to %s! Exception: %s",
self._classname, str(save_file), str(e))
return (False, save_file)
recursion_limit += 10000
else:
log.exception("Some issue saving model %s run function to %s! Exception: %s",
self._classname, str(save_file), str(e))
return (False, save_file)
finally:
f.close()
# all done
return (True, save_file)
else:
return (False, None)
def save_params(self, param_file, use_hdf5=False):
"""
This saves the model's parameters (HDF5 file or pickles them) to the `param_file`.
Parameters
----------
param_file : str
Filename of HDF5 or pickled params file to save to.
use_hdf5 : bool
Whether to use an HDF5 file for the saved parameters (if h5py is installed).
Otherwise, it will use pickle.
Returns
-------
bool
Whether or not successfully saved the file.
"""
# make sure outdir was not set to false (no saving or outputs)
if getattr(self, 'outdir', None):
param_path = os.path.join(self.outdir, param_file)
param_file = os.path.realpath(param_path)
ftype = file_ops.get_extension_type(param_file)
params_dict = self.get_param_values(borrow=False)
if HAS_H5PY and use_hdf5:
# force extension to be .hdf5
if ftype != file_ops.HDF5:
param_file = ''.join([param_file, '.hdf5'])
log.debug('Saving %s parameters to %s',
self._classname, str(param_file))
# try to dump the param values
f = h5py.File(param_file, 'w')
try:
if hdf5_param_key not in f:
param_group = f.create_group(hdf5_param_key)
else:
param_group = f[hdf5_param_key]
for name, param in params_dict.items():
if name in param_group:
dset = param_group[name]
dset[...] = param
else:
dset = param_group.create_dataset(name, data=param)
f.flush()
except Exception as e:
log.exception("Some issue saving model %s parameters to %s! Exception: %s",
self._classname, str(param_file), str(e))
return False
finally:
f.close()
else:
# force extension to be .pkl if it isn't a pickle file
if ftype != file_ops.PKL:
param_file = ''.join([param_file, '.pkl'])
log.debug('Saving %s parameters to %s',
self._classname, str(param_file))
# try to dump the param values
with open(param_file, 'wb') as f:
try:
pickle.dump(params_dict, f, protocol=pickle.HIGHEST_PROTOCOL)
except Exception as e:
log.exception("Some issue saving model %s parameters to %s! Exception: %s",
self._classname, str(param_file), str(e))
return False
finally:
f.close()
# all done
return True
else:
return False
def save_run(self, filename):
"""
Saves (pickle) the compiled theano function for running the model.
Parameters
----------
filename : str
Filepath to save the compiled run function
Returns
-------
tuple(bool, str)
Tuple of [whether or not successful] and [complete filepath to saved file].
"""
# make sure outdir is not set to False (no outputs/saving)
if getattr(self, 'outdir', None):
filepath = os.path.join(self.outdir, filename)
save_file = os.path.realpath(filepath)
ftype = file_ops.get_extension_type(save_file)
# force extension to be .pkl if it isn't a pickle file
if ftype != file_ops.PKL:
save_file = ''.join([save_file, '.pkl'])
log.debug('Saving %s compiled run function to %s', self._classname,
str(save_file))
# try to dump the param values
with open(save_file, 'wb') as f:
try:
run_fn = self.compile_run_fn()
pickle.dump(run_fn, f, protocol=pickle.HIGHEST_PROTOCOL)
except Exception as e:
if "maximum recursion depth exceeded" in str(e):
recursion_limit = 50000
import sys
while "maximum recursion depth exceeded" in str(e):
log.debug(
"found recursion depth bug when pickling function...bumping limit to %d"
% recursion_limit)
sys.setrecursionlimit(recursion_limit)
try:
run_fn = self.compile_run_fn()
pickle.dump(run_fn,
f,
protocol=pickle.HIGHEST_PROTOCOL)
return (True, save_file)
except Exception as e:
if "maximum recursion depth exceeded" not in str(
e):
log.exception(
"Some issue saving model %s run function to %s! Exception: %s",
self._classname, str(save_file),
str(e))
return (False, save_file)
recursion_limit += 10000
else:
log.exception(
"Some issue saving model %s run function to %s! Exception: %s",
self._classname, str(save_file), str(e))
return (False, save_file)
finally:
f.close()
# all done
return (True, save_file)
else:
return (False, None)
def save_params(self, param_file, use_hdf5=False):
"""
This saves the model's parameters (HDF5 file or pickles them) to the `param_file`.
Parameters
----------
param_file : str
Filename of HDF5 or pickled params file to save to.
use_hdf5 : bool
Whether to use an HDF5 file for the saved parameters (if h5py is installed).
Otherwise, it will use pickle.
Returns
-------
bool
Whether or not successfully saved the file.
"""
# make sure outdir was not set to false (no saving or outputs)
if getattr(self, 'outdir', None):
param_path = os.path.join(self.outdir, param_file)
param_file = os.path.realpath(param_path)
ftype = file_ops.get_extension_type(param_file)
params_dict = self.get_param_values(borrow=False)
if HAS_H5PY and use_hdf5:
# force extension to be .hdf5
if ftype != file_ops.HDF5:
param_file = ''.join([param_file, '.hdf5'])
log.debug('Saving %s parameters to %s', self._classname,
str(param_file))
# try to dump the param values
f = h5py.File(param_file, 'w')
try:
if hdf5_param_key not in f:
param_group = f.create_group(hdf5_param_key)
else:
param_group = f[hdf5_param_key]
for name, param in params_dict.items():
if name in param_group:
dset = param_group[name]
dset[...] = param
else:
dset = param_group.create_dataset(name, data=param)
f.flush()
except Exception as e:
log.exception(
"Some issue saving model %s parameters to %s! Exception: %s",
self._classname, str(param_file), str(e))
return False
finally:
f.close()
else:
# force extension to be .pkl if it isn't a pickle file
if ftype != file_ops.PKL:
param_file = ''.join([param_file, '.pkl'])
log.debug('Saving %s parameters to %s', self._classname,
str(param_file))
# try to dump the param values
with open(param_file, 'wb') as f:
try:
pickle.dump(params_dict,
f,
protocol=pickle.HIGHEST_PROTOCOL)
except Exception as e:
log.exception(
"Some issue saving model %s parameters to %s! Exception: %s",
self._classname, str(param_file), str(e))
return False
finally:
f.close()
# all done
return True
else:
return False
