def read_data(self, node_name, sl):
sl = to_slice(sl)
# TODO: allow arbitrary slices to be taken, now they have to match
output = [
d for key, d in self._persisted.items() if get_key_slice(key) == sl
]
if len(output) == 0:
raise IndexError("No matching slice found.")
return get_named_item(output[0], 'data')
def set(self, name, sl, data):
if type(sl) == int:
sl = to_slice(sl)
if name not in self.store.keys():
self.store[name] = []
l = self.store[name]
while len(l) < sl.stop:
l.extend([None] * self.batch_size)
l[sl] = data
def set(self, name, sl, data):
if type(sl) == int:
sl = to_slice(sl)
i = sl.start
j = 0
rows = []
for i in range(sl.start, sl.stop):
try:
ser = self.serialize(data[j])
except Exception as e:
logger.critical("Could not serialize data!")
logger.critical("Error: {}".format(e))
raise
rows.append({"idx": i, "data": ser})
j += 1
self.db.add_rows(name, rows)
def _read_data(self, name, sl):
"""Operation for reading from storage object.
Parameters
----------
name : string
sl : slice
Returns
-------
Values matching slice as np.ndarray
"""
sl = to_slice(sl)
if name not in self.store.keys():
return []
return np.array(self.store[name][sl])
def __getitem__(self, sl):
"""
Returns the delayed data in slice `sl`
"""
sl = to_slice(sl)
outputs = self._get_output_datalist(sl)
# Return the data_slice
if len(outputs) == 0:
empty = np.zeros(shape=(0, 0))
output = delayed(empty)
elif len(outputs) == 1:
output = outputs[0]
else:
key = reset_key_slice(outputs[0].key, sl)
output = delayed(np.vstack)(tuple(outputs), dask_key_name=key)
return output
def _read_data(self, name, sl):
"""Operation for reading from storage object.
Parameters
----------
name : string
sl : slice
Returns
-------
Values matching slice as np.ndarray
"""
sl = to_slice(sl)
filt = lambda row: sl.start <= row["idx"] < sl.stop
rows = self.db.filter_rows(name, filt)
try:
ret = np.array([self.deserialize(row["data"]) for row in rows])
except Exception as e:
logger.critical("Could not deserialize data!")
logger.critical("Error: {}".format(e))
raise
return ret
def __getitem__(self, sl):
sl = to_slice(sl)
return self._delayed_outputs[sl]