def save_dictionary(data, filename): """Save dictionary in a single file .spydata file""" filename = osp.abspath(filename) old_cwd = getcwd_or_home() os.chdir(osp.dirname(filename)) error_message = None # Copy dictionary before modifying to fix #6689 try: data = copy.deepcopy(data) except NotImplementedError: try: data = copy.copy(data) except Exception: data = data try: saved_arrays = {} if load_array is not None: # Saving numpy arrays with np.save arr_fname = osp.splitext(filename)[0] for name in list(data.keys()): if isinstance(data[name], np.ndarray) and data[name].size > 0: # Saving arrays at data root fname = __save_array(data[name], arr_fname, len(saved_arrays)) saved_arrays[(name, None)] = osp.basename(fname) data.pop(name) elif isinstance(data[name], (list, dict)): # Saving arrays nested in lists or dictionaries if isinstance(data[name], list): iterator = enumerate(data[name]) else: iterator = iter(list(data[name].items())) to_remove = [] for index, value in iterator: if isinstance(value, np.ndarray) and value.size > 0: fname = __save_array(value, arr_fname, len(saved_arrays)) saved_arrays[(name, index)] = osp.basename(fname) to_remove.append(index) for index in sorted(to_remove, reverse=True): data[name].pop(index) if saved_arrays: data['__saved_arrays__'] = saved_arrays pickle_filename = osp.splitext(filename)[0] + '.pickle' with open(pickle_filename, 'w+b') as fdesc: pickle.dump(data, fdesc, 2) tar = tarfile.open(filename, "w") for fname in [pickle_filename ] + [fn for fn in list(saved_arrays.values())]: tar.add(osp.basename(fname)) os.remove(fname) tar.close() if saved_arrays: data.pop('__saved_arrays__') except (RuntimeError, pickle.PicklingError, TypeError) as error: error_message = to_text_string(error) os.chdir(old_cwd) return error_message
def save_dictionary(data, filename): """Save dictionary in a single file .spydata file""" filename = osp.abspath(filename) old_cwd = getcwd_or_home() os.chdir(osp.dirname(filename)) error_message = None try: saved_arrays = {} if load_array is not None: # Saving numpy arrays with np.save arr_fname = osp.splitext(filename)[0] for name in list(data.keys()): if isinstance(data[name], np.ndarray) and data[name].size > 0: # Saving arrays at data root fname = __save_array(data[name], arr_fname, len(saved_arrays)) saved_arrays[(name, None)] = osp.basename(fname) data.pop(name) elif isinstance(data[name], (list, dict)): # Saving arrays nested in lists or dictionaries if isinstance(data[name], list): iterator = enumerate(data[name]) else: iterator = iter(list(data[name].items())) to_remove = [] for index, value in iterator: if isinstance(value, np.ndarray) and value.size > 0: fname = __save_array(value, arr_fname, len(saved_arrays)) saved_arrays[(name, index)] = osp.basename(fname) to_remove.append(index) for index in sorted(to_remove, reverse=True): data[name].pop(index) if saved_arrays: data['__saved_arrays__'] = saved_arrays pickle_filename = osp.splitext(filename)[0]+'.pickle' with open(pickle_filename, 'wb') as fdesc: pickle.dump(data, fdesc, 2) tar = tarfile.open(filename, "w") for fname in [pickle_filename]+[fn for fn in list(saved_arrays.values())]: tar.add(osp.basename(fname)) os.remove(fname) tar.close() if saved_arrays: data.pop('__saved_arrays__') except (RuntimeError, pickle.PicklingError, TypeError) as error: error_message = to_text_string(error) os.chdir(old_cwd) return error_message
def save(self): while len(self.rdata) > self.max_entries: self.rdata.pop(-1) pickle.dump([self.VERSION] + self.rdata, open(self.DATAPATH, 'wb'), 2)
def save(self): while len(self.rdata) > self.max_entries: self.rdata.pop(-1) pickle.dump([self.VERSION]+self.rdata, open(self.DATAPATH, 'wb'), 2)