def read_data(self, node_name, sl): sl = to_slice(sl) # TODO: allow arbitrary slices to be taken, now they have to match output = [ d for key, d in self._persisted.items() if get_key_slice(key) == sl ] if len(output) == 0: raise IndexError("No matching slice found.") return get_named_item(output[0], 'data')
def set(self, name, sl, data): if type(sl) == int: sl = to_slice(sl) if name not in self.store.keys(): self.store[name] = [] l = self.store[name] while len(l) < sl.stop: l.extend([None] * self.batch_size) l[sl] = data
def set(self, name, sl, data): if type(sl) == int: sl = to_slice(sl) i = sl.start j = 0 rows = [] for i in range(sl.start, sl.stop): try: ser = self.serialize(data[j]) except Exception as e: logger.critical("Could not serialize data!") logger.critical("Error: {}".format(e)) raise rows.append({"idx": i, "data": ser}) j += 1 self.db.add_rows(name, rows)
def _read_data(self, name, sl): """Operation for reading from storage object. Parameters ---------- name : string sl : slice Returns ------- Values matching slice as np.ndarray """ sl = to_slice(sl) if name not in self.store.keys(): return [] return np.array(self.store[name][sl])
def __getitem__(self, sl): """ Returns the delayed data in slice `sl` """ sl = to_slice(sl) outputs = self._get_output_datalist(sl) # Return the data_slice if len(outputs) == 0: empty = np.zeros(shape=(0, 0)) output = delayed(empty) elif len(outputs) == 1: output = outputs[0] else: key = reset_key_slice(outputs[0].key, sl) output = delayed(np.vstack)(tuple(outputs), dask_key_name=key) return output
def _read_data(self, name, sl): """Operation for reading from storage object. Parameters ---------- name : string sl : slice Returns ------- Values matching slice as np.ndarray """ sl = to_slice(sl) filt = lambda row: sl.start <= row["idx"] < sl.stop rows = self.db.filter_rows(name, filt) try: ret = np.array([self.deserialize(row["data"]) for row in rows]) except Exception as e: logger.critical("Could not deserialize data!") logger.critical("Error: {}".format(e)) raise return ret
def __getitem__(self, sl): sl = to_slice(sl) return self._delayed_outputs[sl]