def cached_solver(activities, targets, rng=None, E=None): try: args, _, _, defaults = inspect.getargspec(solver) except TypeError: args, _, _, defaults = inspect.getargspec(solver.__call__) args = args[-len(defaults):] if rng is None and 'rng' in args: rng = defaults[args.index('rng')] if E is None and 'E' in args: E = defaults[args.index('E')] key = self._get_cache_key(solver, activities, targets, rng, E) decoder_path = self._get_decoder_path(key) solver_info_path = self._get_solver_info_path(key) try: decoders = np.load(decoder_path) with open(solver_info_path, 'rb') as f: solver_info = pickle.load(f) except: logger.info("Cache miss [{0}].".format(key)) decoders, solver_info = solver( activities, targets, rng=rng, E=E) if not self.read_only: np.save(decoder_path, decoders) with open(solver_info_path, 'wb') as f: pickle.dump(solver_info, f) else: logger.info( "Cache hit [{0}]: Loaded stored decoders.".format(key)) return decoders, solver_info
def _write_index(self): assert self._lock.acquired with open(self.index_path + '.part', 'wb') as f: # Use protocol 2 for version information to ensure that # all Python versions supported by Nengo will be able to # read it in the future. pickle.dump((self.VERSION, pickle.HIGHEST_PROTOCOL), f, 2) # Use highest available protocol for index data for maximum # performance. pickle.dump(self._index, f, pickle.HIGHEST_PROTOCOL) try: replace(self.index_path + '.part', self.index_path) except (CalledProcessError, PermissionError): # It may fail when # another program like a virus scanner is accessing the file to be # moved. There is not a lot we could do about this. See # <https://github.com/nengo/nengo/issues/1200> for more info. warnings.warn( "The cache index could not be updated because another program " "blocked access to it. This is commonly caused by anti-virus " "software. It is safe to ignore this warning. But if you see " "it a lot, you might want to consider doing one of the " "following for the best Nengo performance:\n" "1. Configure your anti-virus to ignore the Nengo cache " "folder ('{cache_dir}').\n" "2. Disable the cache.\n".format(cache_dir=self.cache_dir), category=CacheIOWarning) if os.path.exists(self.legacy_path): os.remove(self.legacy_path)
def plain_pickle(loadfile, savefile): load_dic = IGPUModel.load_checkpoint(loadfile) options = {} for o in load_dic['op'].get_options_list(): options[o.name] = o.value load_dic['op'] = options with open(savefile, 'wb') as f: pickle.dump(load_dic, f, protocol=-1) print("Wrote %r" % savefile)
def _write_index(self): assert self._lock.acquired with open(self.index_path + '.part', 'wb') as f: # Use protocol 2 for version information to ensure that # all Python versions supported by Nengo will be able to # read it in the future. pickle.dump((self.VERSION, pickle.HIGHEST_PROTOCOL), f, 2) # Use highest available protocol for index data for maximum # performance. pickle.dump(self._index, f, pickle.HIGHEST_PROTOCOL) replace(self.index_path + '.part', self.index_path) if os.path.exists(self.legacy_path): os.remove(self.legacy_path)
def _write_index(self): assert self._lock.acquired with open(self.index_path + '.part', 'wb') as f: # Use protocol 2 for version information to ensure that # all Python versions supported by Nengo will be able to # read it in the future. pickle.dump( (self.VERSION, pickle.HIGHEST_PROTOCOL), f, 2) # Use highest available protocol for index data for maximum # performance. pickle.dump(self._index, f, pickle.HIGHEST_PROTOCOL) replace(self.index_path + '.part', self.index_path) if os.path.exists(self.legacy_path): os.remove(self.legacy_path)
def sync(self): try: with self._lock: self._index = self._load_index() self._index.update(self._updates) for key in self._deletes: del self._index[key] with open(self.filename, 'wb') as f: pickle.dump( {k: v for k, v in self._index.items() if v[0] not in self._removed_files}, f, pickle.HIGHEST_PROTOCOL) except TimeoutError: warnings.warn( "Decoder cache index could not acquire lock. " "Cache index was not synced.") self._updates.clear() self._deletes.clear()
def sync(self): try: with self._lock: self._index = self._load_index() self._index.update(self._updates) for key in self._deletes: del self._index[key] with open(self.filename, 'wb') as f: pickle.dump( { k: v for k, v in self._index.items() if v[0] not in self._removed_files }, f, pickle.HIGHEST_PROTOCOL) except TimeoutError: warnings.warn("Decoder cache index could not acquire lock. " "Cache index was not synced.") self._updates.clear() self._deletes.clear()