def save_args(self, args_file="config.pkl"): """ This saves the model's initial configuration parameters (`self.args`) in a pickle file. Parameters ---------- args_file : str, optional Filename of pickled configuration parameters. Defaults to 'config.pkl'. Returns ------- bool Whether or not successfully saved the file. """ # make sure outdir is not set to False (no outputs/saving) if getattr(self, 'outdir', None): args_path = os.path.join(self.outdir, args_file) args_file = os.path.realpath(args_path) ftype = file_ops.get_extension_type(args_file) args = self.args.copy() args[class_key] = self._classname # force extension to be .pkl if it isn't a pickle file if ftype != file_ops.PKL: args_file = ''.join([args_file, '.pkl']) log.debug('Saving %s configuration to %s', self._classname, str(args_file)) # try to dump the args values with open(args_file, 'wb') as f: try: pickle.dump(args, f, protocol=pickle.HIGHEST_PROTOCOL) except Exception as e: log.exception( "Some issue saving model %s configuration to %s! Exception: %s", self._classname, str(args_file), str(e)) return False finally: f.close() return True else: return False
def save_args(self, args_file="config.pkl"): """ This saves the model's initial configuration parameters (`self.args`) in a pickle file. Parameters ---------- args_file : str, optional Filename of pickled configuration parameters. Defaults to 'config.pkl'. Returns ------- bool Whether or not successfully saved the file. """ # make sure outdir is not set to False (no outputs/saving) if getattr(self, 'outdir', None): args_path = os.path.join(self.outdir, args_file) args_file = os.path.realpath(args_path) ftype = file_ops.get_extension_type(args_file) args = self.args.copy() args[class_key] = self._classname # force extension to be .pkl if it isn't a pickle file if ftype != file_ops.PKL: args_file = ''.join([args_file, '.pkl']) log.debug('Saving %s configuration to %s', self._classname, str(args_file)) # try to dump the args values with open(args_file, 'wb') as f: try: pickle.dump(args, f, protocol=pickle.HIGHEST_PROTOCOL) except Exception as e: log.exception("Some issue saving model %s configuration to %s! Exception: %s", self._classname, str(args_file), str(e)) return False finally: f.close() return True else: return False
def save_run(self, filename): """ Saves (pickle) the compiled theano function for running the model. Parameters ---------- filename : str Filepath to save the compiled run function Returns ------- tuple(bool, str) Tuple of [whether or not successful] and [complete filepath to saved file]. """ # make sure outdir is not set to False (no outputs/saving) if getattr(self, 'outdir', None): filepath = os.path.join(self.outdir, filename) save_file = os.path.realpath(filepath) ftype = file_ops.get_extension_type(save_file) # force extension to be .pkl if it isn't a pickle file if ftype != file_ops.PKL: save_file = ''.join([save_file, '.pkl']) log.debug('Saving %s compiled run function to %s', self._classname, str(save_file)) # try to dump the param values with open(save_file, 'wb') as f: try: run_fn = self.compile_run_fn() pickle.dump(run_fn, f, protocol=pickle.HIGHEST_PROTOCOL) except Exception as e: if "maximum recursion depth exceeded" in str(e): recursion_limit = 50000 import sys while "maximum recursion depth exceeded" in str(e): log.debug("found recursion depth bug when pickling function...bumping limit to %d" % recursion_limit) sys.setrecursionlimit(recursion_limit) try: run_fn = self.compile_run_fn() pickle.dump(run_fn, f, protocol=pickle.HIGHEST_PROTOCOL) return (True, save_file) except Exception as e: if "maximum recursion depth exceeded" not in str(e): log.exception("Some issue saving model %s run function to %s! Exception: %s", self._classname, str(save_file), str(e)) return (False, save_file) recursion_limit += 10000 else: log.exception("Some issue saving model %s run function to %s! Exception: %s", self._classname, str(save_file), str(e)) return (False, save_file) finally: f.close() # all done return (True, save_file) else: return (False, None)
def save_params(self, param_file, use_hdf5=False): """ This saves the model's parameters (HDF5 file or pickles them) to the `param_file`. Parameters ---------- param_file : str Filename of HDF5 or pickled params file to save to. use_hdf5 : bool Whether to use an HDF5 file for the saved parameters (if h5py is installed). Otherwise, it will use pickle. Returns ------- bool Whether or not successfully saved the file. """ # make sure outdir was not set to false (no saving or outputs) if getattr(self, 'outdir', None): param_path = os.path.join(self.outdir, param_file) param_file = os.path.realpath(param_path) ftype = file_ops.get_extension_type(param_file) params_dict = self.get_param_values(borrow=False) if HAS_H5PY and use_hdf5: # force extension to be .hdf5 if ftype != file_ops.HDF5: param_file = ''.join([param_file, '.hdf5']) log.debug('Saving %s parameters to %s', self._classname, str(param_file)) # try to dump the param values f = h5py.File(param_file, 'w') try: if hdf5_param_key not in f: param_group = f.create_group(hdf5_param_key) else: param_group = f[hdf5_param_key] for name, param in params_dict.items(): if name in param_group: dset = param_group[name] dset[...] = param else: dset = param_group.create_dataset(name, data=param) f.flush() except Exception as e: log.exception("Some issue saving model %s parameters to %s! Exception: %s", self._classname, str(param_file), str(e)) return False finally: f.close() else: # force extension to be .pkl if it isn't a pickle file if ftype != file_ops.PKL: param_file = ''.join([param_file, '.pkl']) log.debug('Saving %s parameters to %s', self._classname, str(param_file)) # try to dump the param values with open(param_file, 'wb') as f: try: pickle.dump(params_dict, f, protocol=pickle.HIGHEST_PROTOCOL) except Exception as e: log.exception("Some issue saving model %s parameters to %s! Exception: %s", self._classname, str(param_file), str(e)) return False finally: f.close() # all done return True else: return False
def save_run(self, filename): """ Saves (pickle) the compiled theano function for running the model. Parameters ---------- filename : str Filepath to save the compiled run function Returns ------- tuple(bool, str) Tuple of [whether or not successful] and [complete filepath to saved file]. """ # make sure outdir is not set to False (no outputs/saving) if getattr(self, 'outdir', None): filepath = os.path.join(self.outdir, filename) save_file = os.path.realpath(filepath) ftype = file_ops.get_extension_type(save_file) # force extension to be .pkl if it isn't a pickle file if ftype != file_ops.PKL: save_file = ''.join([save_file, '.pkl']) log.debug('Saving %s compiled run function to %s', self._classname, str(save_file)) # try to dump the param values with open(save_file, 'wb') as f: try: run_fn = self.compile_run_fn() pickle.dump(run_fn, f, protocol=pickle.HIGHEST_PROTOCOL) except Exception as e: if "maximum recursion depth exceeded" in str(e): recursion_limit = 50000 import sys while "maximum recursion depth exceeded" in str(e): log.debug( "found recursion depth bug when pickling function...bumping limit to %d" % recursion_limit) sys.setrecursionlimit(recursion_limit) try: run_fn = self.compile_run_fn() pickle.dump(run_fn, f, protocol=pickle.HIGHEST_PROTOCOL) return (True, save_file) except Exception as e: if "maximum recursion depth exceeded" not in str( e): log.exception( "Some issue saving model %s run function to %s! Exception: %s", self._classname, str(save_file), str(e)) return (False, save_file) recursion_limit += 10000 else: log.exception( "Some issue saving model %s run function to %s! Exception: %s", self._classname, str(save_file), str(e)) return (False, save_file) finally: f.close() # all done return (True, save_file) else: return (False, None)
def save_params(self, param_file, use_hdf5=False): """ This saves the model's parameters (HDF5 file or pickles them) to the `param_file`. Parameters ---------- param_file : str Filename of HDF5 or pickled params file to save to. use_hdf5 : bool Whether to use an HDF5 file for the saved parameters (if h5py is installed). Otherwise, it will use pickle. Returns ------- bool Whether or not successfully saved the file. """ # make sure outdir was not set to false (no saving or outputs) if getattr(self, 'outdir', None): param_path = os.path.join(self.outdir, param_file) param_file = os.path.realpath(param_path) ftype = file_ops.get_extension_type(param_file) params_dict = self.get_param_values(borrow=False) if HAS_H5PY and use_hdf5: # force extension to be .hdf5 if ftype != file_ops.HDF5: param_file = ''.join([param_file, '.hdf5']) log.debug('Saving %s parameters to %s', self._classname, str(param_file)) # try to dump the param values f = h5py.File(param_file, 'w') try: if hdf5_param_key not in f: param_group = f.create_group(hdf5_param_key) else: param_group = f[hdf5_param_key] for name, param in params_dict.items(): if name in param_group: dset = param_group[name] dset[...] = param else: dset = param_group.create_dataset(name, data=param) f.flush() except Exception as e: log.exception( "Some issue saving model %s parameters to %s! Exception: %s", self._classname, str(param_file), str(e)) return False finally: f.close() else: # force extension to be .pkl if it isn't a pickle file if ftype != file_ops.PKL: param_file = ''.join([param_file, '.pkl']) log.debug('Saving %s parameters to %s', self._classname, str(param_file)) # try to dump the param values with open(param_file, 'wb') as f: try: pickle.dump(params_dict, f, protocol=pickle.HIGHEST_PROTOCOL) except Exception as e: log.exception( "Some issue saving model %s parameters to %s! Exception: %s", self._classname, str(param_file), str(e)) return False finally: f.close() # all done return True else: return False