def _receive(self):
# Continuous data
if self._dimensions == 2:
if self.i.ready():
if not self._X_columns:
self._X_columns = list(self.i.data.columns)
if self._shape and (self.i.data.shape[1] != self._shape):
self.logger.warning("Invalid shape")
else:
self._X = self.i.data.values
self._X_indices = self.i.data.index.values
self._X_meta = self.i.meta
# Epochs
if self._dimensions == 3:
for name, _, port in self.iterate("i_*"):
if port.ready(
) and "training" not in name and "events" not in name:
data = port.data.values
meta = port.meta
indices = port.data.index.values
label = get_meta(port, self.meta_label)
if not self._X_columns:
self._X_columns = list(port.data.columns)
if self._shape and (data.shape != self._shape):
self.logger.warning("Invalid shape")
continue
if not self.fit and self.meta_label is not None and label is None:
self.logger.warning("Invalid label")
continue
if self._X is None:
self._X = []
if self._y is None and label is not None:
self._y = []
if self._X_meta is None:
self._X_meta = []
self._X.append(data)
self._X_indices.append(indices)
self._X_meta.append(meta)
if label is not None:
self._y.append(label)
def _accumulate(self, start, stop):
# Do nothing if no fitting required
if not self.fit:
return
# Set defaults
indices = np.array([], dtype=np.datetime64)
# Accumulate continuous data
if self._dimensions == 2:
if self.i_training.ready():
data = self.i_training.data
mask = (data.index >= start) & (data.index < stop)
data = data[mask]
if not data.empty:
if self._X_train is None:
self._X_train = data.values
self._shape = self._X_train.shape[1]
indices = data.index.values
else:
if data.shape[1] == self._shape:
self._X_train = np.vstack((self._X_train, data.values))
indices = data.index.values
else:
self.logger.warning("Invalid shape")
# Accumulate epoched data
if self._dimensions == 3:
for _, _, port in self.iterate("i_training_*"):
if port.ready():
index = port.data.index.values[0]
if index >= start and index < stop:
data = port.data.values
label = get_meta(port, self.meta_label)
if self._shape and (data.shape != self._shape):
self.logger.warning("Invalid shape")
continue
if self.meta_label is not None and label is None:
self.logger.warning("Invalid label")
continue
if self._X_train is None:
self._X_train = np.array([data])
self._shape = self._X_train.shape[1:]
else:
self._X_train = np.vstack((self._X_train, [data]))
indices = np.append(indices, index)
if label is not None:
if self._y_train is None:
self._y_train = np.array([label])
else:
self._y_train = np.append(self._y_train, [label])
# Store indices
if indices.size != 0:
self._X_train_indices = np.append(self._X_train_indices, indices)
# Trim
if self._X_train is not None:
mask = (self._X_train_indices >= start) & (self._X_train_indices < stop)
self._X_train = self._X_train[mask]
self._X_train_indices = self._X_train_indices[mask]
if self._y_train is not None:
self._y_train = self._y_train[mask]
def test_get_meta_no_key():
port = Port()
assert get_meta(port, None) == None
assert get_meta(port, None, False) == False
def test_get_meta_not_found():
port = Port()
assert get_meta(port, ('foo')) == None
assert get_meta(port, ('foo'), False) == False
def test_get_meta_deep():
port = Port()
port.meta = {'foo': {'bar': {'baz': 42}}}
assert get_meta(port, ('foo', 'bar', 'baz')) == 42
def test_get_meta():
port = Port()
port.meta = {'target': True}
assert get_meta(port, 'target') == True