def load_rbm_params(self, param_file): """ Loads the parameters for the RBM only from param_file - used if the RBM was pre-trained :param param_file: location of rbm parameters :type param_file: string :return: whether successful :rtype: boolean """ param_file = os.path.realpath(param_file) # make sure it is a pickle file ftype = file_ops.get_file_type(param_file) if ftype == file_ops.PKL: log.debug("loading model %s parameters from %s", str(type(self)), str(param_file)) # try to grab the pickled params from the specified param_file path with open(param_file, 'r') as f: loaded_params = pickle.load(f) ############################################################################# # set the W, bv, and bh values (make sure same order as saved in RBM class) # ############################################################################# self.W.set_value(loaded_params[0]) self.bv.set_value(loaded_params[1]) self.bh.set_value(loaded_params[2]) return True # if get_file_type didn't return pkl or none, it wasn't a pickle file elif ftype: log.error("Param file %s doesn't have a supported pickle extension!", str(param_file)) return False # if get_file_type returned none, it couldn't find the file else: log.error("Param file %s couldn't be found!", str(param_file)) return False
def load_params(self, param_file): """ This loads the model's parameters from the param_file (pickle file) ------------------ :param param_file: filename of pickled params file :type param_file: String :return: whether or not successful :rtype: Boolean """ param_file = os.path.realpath(param_file) # make sure it is a pickle file ftype = file_ops.get_file_type(param_file) if ftype == file_ops.PKL: log.debug("loading model %s parameters from %s", str(type(self)), str(param_file)) # try to grab the pickled params from the specified param_file path with open(param_file, 'r') as f: loaded_params = pickle.load(f) self.set_param_values(loaded_params) return True # if get_file_type didn't return pkl or none, it wasn't a pickle file elif ftype: log.error("Param file %s doesn't have a supported pickle extension!", str(param_file)) return False # if get_file_type returned none, it couldn't find the file else: log.error("Param file %s couldn't be found!", str(param_file)) return False
def load_params(self, param_file): """ This loads the model's parameters from the param_file (pickle file) ------------------ :param param_file: filename of pickled params file :type param_file: String :return: whether or not successful :rtype: Boolean """ param_file = os.path.realpath(param_file) # make sure it is a pickle file ftype = file_ops.get_file_type(param_file) if ftype == file_ops.PKL: log.debug("loading model %s parameters from %s...", str(type(self)), str(param_file)) # try to grab the pickled params from the specified param_file path with open(param_file, 'r') as f: loaded_params = cPickle.load(f) self.set_param_values(loaded_params) return True # if get_file_type didn't return pkl or none, it wasn't a pickle file elif ftype: log.error( "Param file %s doesn't have a supported pickle extension!", str(param_file)) return False # if get_file_type returned none, it couldn't find the file else: log.error("Param file %s couldn't be found!", str(param_file)) return False
def load_rbm_params(self, param_file): """ Loads the parameters for the RBM only from param_file - used if the RBM was pre-trained :param param_file: location of rbm parameters :type param_file: string :return: whether successful :rtype: boolean """ param_file = os.path.realpath(param_file) # make sure it is a pickle file ftype = file_ops.get_file_type(param_file) if ftype == file_ops.PKL: log.debug("loading model %s parameters from %s", str(type(self)), str(param_file)) # try to grab the pickled params from the specified param_file path with open(param_file, 'r') as f: loaded_params = pickle.load(f) ############################################################################# # set the W, bv, and bh values (make sure same order as saved in RBM class) # ############################################################################# self.W.set_value(loaded_params[0]) self.bv.set_value(loaded_params[1]) self.bh.set_value(loaded_params[2]) return True # if get_file_type didn't return pkl or none, it wasn't a pickle file elif ftype: log.error("Param file %s doesn't have a supported pickle extension!", str(param_file)) return False # if get_file_type returned none, it couldn't find the file else: log.error("Param file %s couldn't be found!", str(param_file)) return False
def load_params(self, param_file): """ This loads the model's parameters from the param_file (hdf5 or pickle file) Parameters ---------- param_file : str Filename of hdf5 or pickled params file (the file holding the model parameters). Returns ------- bool Whether or not successfully loaded parameters. """ param_file = os.path.realpath(param_file) # make sure it is a pickle file ftype = file_ops.get_file_type(param_file) log.debug("loading %s model parameters from %s", self._classname, str(param_file)) if ftype == file_ops.PKL: # try to grab the pickled params from the specified param_file path with open(param_file, 'rb') as f: loaded_params = pickle.load(f) self.set_param_values(loaded_params, borrow=False) return True elif ftype == file_ops.HDF5: if HAS_H5PY: f = h5py.File(param_file) try: params = f[hdf5_param_key] self.set_param_values(params) except Exception as e: log.exception( "Some issue loading model %s parameters from %s! Exception: %s", self._classname, str(param_file), str(e)) return False finally: f.close() else: log.error( "Please install the h5py package to read HDF5 files!") return False # if get_file_type didn't return pkl, hdf5, or none elif ftype: log.error( "Param file %s doesn't have a supported pickle or HDF5 extension!", str(param_file)) return False # if get_file_type returned none, it couldn't find the file else: log.error("Param file %s couldn't be found!", str(param_file)) return False
def load_params(self, param_file): """ This loads the model's parameters from the param_file (hdf5 or pickle file) Parameters ---------- param_file : str Filename of hdf5 or pickled params file (the file holding the model parameters). Returns ------- bool Whether or not successfully loaded parameters. """ param_file = os.path.realpath(param_file) # make sure it is a pickle file ftype = file_ops.get_file_type(param_file) log.debug("loading %s model parameters from %s", self._classname, str(param_file)) if ftype == file_ops.PKL: # try to grab the pickled params from the specified param_file path with open(param_file, 'rb') as f: loaded_params = pickle.load(f) self.set_param_values(loaded_params, borrow=False) return True elif ftype == file_ops.HDF5: if HAS_H5PY: f = h5py.File(param_file) try: params = f[hdf5_param_key] self.set_param_values(params) except Exception as e: log.exception("Some issue loading model %s parameters from %s! Exception: %s", self._classname, str(param_file), str(e)) return False finally: f.close() else: log.error("Please install the h5py package to read HDF5 files!") return False # if get_file_type didn't return pkl, hdf5, or none elif ftype: log.error("Param file %s doesn't have a supported pickle or HDF5 extension!", str(param_file)) return False # if get_file_type returned none, it couldn't find the file else: log.error("Param file %s couldn't be found!", str(param_file)) return False
def load(config_file, param_file=None): """ Returns a new Model from a configuration file. Parameters ---------- config_file : str Filename of pickled configuration file. param_file : str, optional Filename of hdf5 or pickle file holding the model parameters (in a separate file from `config_file` if you want to load some starting parameters). Returns ------- :class:`Model` A `Model` instance from the configuration and optionally loaded parameters. """ config_file = os.path.realpath(config_file) ftype = file_ops.get_file_type(config_file) # deal with pickle if ftype == file_ops.PKL: log.debug("loading model from %s", str(config_file)) with open(config_file, 'rb') as f: loaded_config = pickle.load(f) # if get_file_type didn't return pkl, or none elif ftype: log.exception( "Config file %s doesn't have a supported pickle extension!", str(config_file)) raise AssertionError( "Config file %s doesn't have a supported pickle extension!", str(config_file)) # if get_file_type returned none, it couldn't find the file else: log.exception("Config file %s couldn't be found!", str(config_file)) raise AssertionError("Config file %s couldn't be found!", str(config_file)) classname = loaded_config.pop(class_key) class_ = getattr(opendeep.models, classname) model = class_(**loaded_config) if param_file is not None: model.load_params(param_file=param_file) return model
def load(config_file, param_file=None): """ Returns a new Model from a configuration file. Parameters ---------- config_file : str Filename of pickled configuration file. param_file : str, optional Filename of hdf5 or pickle file holding the model parameters (in a separate file from `config_file` if you want to load some starting parameters). Returns ------- :class:`Model` A `Model` instance from the configuration and optionally loaded parameters. """ config_file = os.path.realpath(config_file) ftype = file_ops.get_file_type(config_file) # deal with pickle if ftype == file_ops.PKL: log.debug("loading model from %s", str(config_file)) with open(config_file, 'rb') as f: loaded_config = pickle.load(f) # if get_file_type didn't return pkl, or none elif ftype: log.exception("Config file %s doesn't have a supported pickle extension!", str(config_file)) raise AssertionError("Config file %s doesn't have a supported pickle extension!", str(config_file)) # if get_file_type returned none, it couldn't find the file else: log.exception("Config file %s couldn't be found!", str(config_file)) raise AssertionError("Config file %s couldn't be found!", str(config_file)) classname = loaded_config.pop(class_key) class_ = getattr(opendeep.models, classname) model = class_(**loaded_config) if param_file is not None: model.load_params(param_file=param_file) return model
def load_gsn_params(self, param_file): """ Loads the parameters for the GSN only from param_file - used if the GSN was pre-trained Parameters ---------- param_file : str Relative location of GSN parameters. Returns ------- bool Whether or not successful. """ param_file = os.path.realpath(param_file) # make sure it is a pickle file ftype = file_ops.get_file_type(param_file) if ftype == file_ops.PKL: log.debug("loading model %s parameters from %s", str(type(self)), str(param_file)) # try to grab the pickled params from the specified param_file path with open(param_file, 'r') as f: loaded_params = pickle.load(f) # set the GSN parameters for i, weight in enumerate(self.weights_list): weight.set_value(loaded_params[i]) for i, bias in enumerate(self.bias_list): bias.set_value(loaded_params[i+len(self.weights_list)]) return True # if get_file_type didn't return pkl or none, it wasn't a pickle file elif ftype: log.error("Param file %s doesn't have a supported pickle extension!", str(param_file)) return False # if get_file_type returned none, it couldn't find the file else: log.error("Param file %s couldn't be found!", str(param_file)) return False
def load_gsn_params(self, param_file): """ Loads the parameters for the GSN only from param_file - used if the GSN was pre-trained Parameters ---------- param_file : str Relative location of GSN parameters. Returns ------- bool Whether or not successful. """ param_file = os.path.realpath(param_file) # make sure it is a pickle file ftype = file_ops.get_file_type(param_file) if ftype == file_ops.PKL: log.debug("loading model %s parameters from %s", str(type(self)), str(param_file)) # try to grab the pickled params from the specified param_file path with open(param_file, 'r') as f: loaded_params = pickle.load(f) # set the GSN parameters for i, weight in enumerate(self.weights_list): weight.set_value(loaded_params[i]) for i, bias in enumerate(self.bias_list): bias.set_value(loaded_params[i+len(self.weights_list)]) return True # if get_file_type didn't return pkl or none, it wasn't a pickle file elif ftype: log.error("Param file %s doesn't have a supported pickle extension!", str(param_file)) return False # if get_file_type returned none, it couldn't find the file else: log.error("Param file %s couldn't be found!", str(param_file)) return False
def install(self): ''' Method to both download and extract the dataset from the internet (if there) or verify connection settings ''' file_type = None if self.filename is not None: log.info('Installing dataset %s', str(self.filename)) # construct the actual path to the dataset prevdir = os.getcwd() os.chdir(os.path.split(os.path.realpath(__file__))[0]) dataset_dir = os.path.realpath(self.dataset_dir) try: mkdir_p(dataset_dir) dataset_location = os.path.join(dataset_dir, self.filename) except Exception as e: log.error("Couldn't make the dataset path with directory %s and filename %s", dataset_dir, str(self.filename)) log.exception("%s", str(e)) dataset_location = None finally: os.chdir(prevdir) # check if the dataset is already in the source, otherwise download it. # first check if the base filename exists - without all the extensions. # then, add each extension on and keep checking until the upper level, when you download from http. if dataset_location is not None: (dirs, fname) = os.path.split(dataset_location) split_fname = fname.split('.') accumulated_name = split_fname[0] found = False # first check if the filename was a directory (like for the midi datasets) if os.path.exists(os.path.join(dirs, accumulated_name)): found = True file_type = get_file_type(os.path.join(dirs, accumulated_name)) dataset_location = os.path.join(dirs, accumulated_name) log.debug('Found file %s', dataset_location) # now go through the file extensions starting with the lowest level and check if the file exists if not found and len(split_fname) > 1: for chunk in split_fname[1:]: accumulated_name = '.'.join((accumulated_name, chunk)) file_type = get_file_type(os.path.join(dirs, accumulated_name)) if file_type is not None: dataset_location = os.path.join(dirs, accumulated_name) log.debug('Found file %s', dataset_location) break # if the file wasn't found, download it if a source was provided. Otherwise, raise error. download_success = True if self.source is not None: if file_type is None: download_success = download_file(self.source, dataset_location) file_type = get_file_type(dataset_location) else: log.error("Filename %s couldn't be found, and no URL source to download was provided.", str(self.filename)) raise RuntimeError("Filename %s couldn't be found, and no URL source to download was provided." % str(self.filename)) # if the file type is a zip, unzip it. unzip_success = True if file_type is files.ZIP: (dirs, fname) = os.path.split(dataset_location) post_unzip = os.path.join(dirs, '.'.join(fname.split('.')[0:-1])) unzip_success = files.unzip(dataset_location, post_unzip) # if the unzip was successful if unzip_success: # remove the zipfile and update the dataset location and file type log.debug('Removing file %s', dataset_location) os.remove(dataset_location) dataset_location = post_unzip file_type = get_file_type(dataset_location) if download_success and unzip_success: log.info('Installation complete. Yay!') else: log.warning('Something went wrong installing dataset. Boo :(') return dataset_location, file_type
def install(self): ''' Method to both download and extract the dataset from the internet (if applicable) or verify that the file exists in the dataset_dir. Returns ------- str The absolute path to the dataset location on disk. int The integer representing the file type for the dataset, as defined in the opendeep.utils.file_ops module. ''' file_type = None if self.filename is not None: log.info('Installing dataset %s', str(self.filename)) # construct the actual path to the dataset prevdir = os.getcwd() os.chdir(os.path.split(os.path.realpath(__file__))[0]) dataset_dir = os.path.realpath(self.dataset_dir) try: mkdir_p(dataset_dir) dataset_location = os.path.join(dataset_dir, self.filename) except Exception as e: log.error("Couldn't make the dataset path with directory %s and filename %s", dataset_dir, str(self.filename)) log.exception("%s", str(e)) dataset_location = None finally: os.chdir(prevdir) # check if the dataset is already in the source, otherwise download it. # first check if the base filename exists - without all the extensions. # then, add each extension on and keep checking until the upper level, when you download from http. if dataset_location is not None: (dirs, fname) = os.path.split(dataset_location) split_fname = fname.split('.') accumulated_name = split_fname[0] found = False # first check if the filename was a directory (like for the midi datasets) if os.path.exists(os.path.join(dirs, accumulated_name)): found = True file_type = get_file_type(os.path.join(dirs, accumulated_name)) dataset_location = os.path.join(dirs, accumulated_name) log.debug('Found file %s', dataset_location) # now go through the file extensions starting with the lowest level and check if the file exists if not found and len(split_fname) > 1: for chunk in split_fname[1:]: accumulated_name = '.'.join((accumulated_name, chunk)) file_type = get_file_type(os.path.join(dirs, accumulated_name)) if file_type is not None: dataset_location = os.path.join(dirs, accumulated_name) log.debug('Found file %s', dataset_location) break # if the file wasn't found, download it if a source was provided. Otherwise, raise error. download_success = True if self.source is not None: if file_type is None: download_success = download_file(self.source, dataset_location) file_type = get_file_type(dataset_location) else: log.error("Filename %s couldn't be found, and no URL source to download was provided.", str(self.filename)) raise RuntimeError("Filename %s couldn't be found, and no URL source to download was provided." % str(self.filename)) # if the file type is a zip, unzip it. unzip_success = True if file_type is files.ZIP: (dirs, fname) = os.path.split(dataset_location) post_unzip = os.path.join(dirs, '.'.join(fname.split('.')[0:-1])) unzip_success = files.unzip(dataset_location, post_unzip) # if the unzip was successful if unzip_success: # remove the zipfile and update the dataset location and file type log.debug('Removing file %s', dataset_location) os.remove(dataset_location) dataset_location = post_unzip file_type = get_file_type(dataset_location) if download_success and unzip_success: log.info('Installation complete. Yay!') else: log.warning('Something went wrong installing dataset. Boo :(') return dataset_location, file_type