def save_dictionary(data, filename): """Save dictionary in a single file .spydata file""" filename = osp.abspath(filename) old_cwd = getcwd() os.chdir(osp.dirname(filename)) error_message = None try: saved_arrays = {} if load_array is not None: # Saving numpy arrays with np.save arr_fname = osp.splitext(filename)[0] for name in list(data.keys()): if isinstance(data[name], np.ndarray) and data[name].size > 0: # Saving arrays at data root fname = __save_array(data[name], arr_fname, len(saved_arrays)) saved_arrays[(name, None)] = osp.basename(fname) data.pop(name) elif isinstance(data[name], (list, dict)): # Saving arrays nested in lists or dictionaries if isinstance(data[name], list): iterator = enumerate(data[name]) else: iterator = iter(list(data[name].items())) to_remove = [] for index, value in iterator: if isinstance(value, np.ndarray) and value.size > 0: fname = __save_array(value, arr_fname, len(saved_arrays)) saved_arrays[(name, index)] = osp.basename(fname) to_remove.append(index) for index in sorted(to_remove, reverse=True): data[name].pop(index) if saved_arrays: data['__saved_arrays__'] = saved_arrays pickle_filename = osp.splitext(filename)[0] + '.pickle' with open(pickle_filename, 'wb') as fdesc: pickle.dump(data, fdesc, 2) tar = tarfile.open(filename, "w") for fname in [pickle_filename ] + [fn for fn in list(saved_arrays.values())]: tar.add(osp.basename(fname)) os.remove(fname) tar.close() if saved_arrays: data.pop('__saved_arrays__') except (RuntimeError, pickle.PicklingError, TypeError) as error: error_message = to_text_string(error) os.chdir(old_cwd) return error_message
def save_dictionary(data, filename): """Save dictionary in a single file .spydata file""" filename = osp.abspath(filename) old_cwd = getcwd() os.chdir(osp.dirname(filename)) error_message = None try: saved_arrays = {} if load_array is not None: # Saving numpy arrays with np.save arr_fname = osp.splitext(filename)[0] for name in list(data.keys()): if isinstance(data[name], np.ndarray) and data[name].size > 0: # Saving arrays at data root fname = __save_array(data[name], arr_fname, len(saved_arrays)) saved_arrays[(name, None)] = osp.basename(fname) data.pop(name) elif isinstance(data[name], (list, dict)): # Saving arrays nested in lists or dictionaries if isinstance(data[name], list): iterator = enumerate(data[name]) else: iterator = iter(list(data[name].items())) to_remove = [] for index, value in iterator: if isinstance(value, np.ndarray) and value.size > 0: fname = __save_array(value, arr_fname, len(saved_arrays)) saved_arrays[(name, index)] = osp.basename(fname) to_remove.append(index) for index in sorted(to_remove, reverse=True): data[name].pop(index) if saved_arrays: data['__saved_arrays__'] = saved_arrays pickle_filename = osp.splitext(filename)[0]+'.pickle' with open(pickle_filename, 'wb') as fdesc: pickle.dump(data, fdesc, 2) tar = tarfile.open(filename, "w") for fname in [pickle_filename]+[fn for fn in list(saved_arrays.values())]: tar.add(osp.basename(fname)) os.remove(fname) tar.close() if saved_arrays: data.pop('__saved_arrays__') except (RuntimeError, pickle.PicklingError, TypeError) as error: error_message = to_text_string(error) os.chdir(old_cwd) return error_message
def __setitem__(self, key, value): """ db['key'] = 5 """ fil = self.root / key parent = fil.parent if parent and not parent.isdir(): parent.makedirs() # We specify protocol 2, so that we can mostly go between Python 2 # and Python 3. We can upgrade to protocol 3 when Python 2 is obsolete. pickled = pickle.dump(value, fil.open('wb'), protocol=2) try: self.cache[fil] = (value, fil.mtime) except OSError as e: if e.errno != errno.ENOENT: raise
def __setitem__(self, key, value): """ db['key'] = 5 """ fil = self.root / key parent = fil.parent if parent and not parent.isdir(): parent.makedirs() # We specify protocol 2, so that we can mostly go between Python 2 # and Python 3. We can upgrade to protocol 3 when Python 2 is obsolete. pickled = pickle.dump(value, fil.open('wb'), protocol=2) try: self.cache[fil] = (value, fil.mtime) except OSError as e: if e.errno != errno.ENOENT: raise
def save(self): while len(self.rdata) > self.max_entries: self.rdata.pop(-1) pickle.dump([self.VERSION]+self.rdata, open(self.DATAPATH, 'wb'), 2)
def save(self): while len(self.rdata) > self.max_entries: self.rdata.pop(-1) pickle.dump([self.VERSION]+self.rdata, open(self.DATAPATH, 'wb'), 2)