def _receive(self): # Continuous data if self._dimensions == 2: if self.i.ready(): if not self._X_columns: self._X_columns = list(self.i.data.columns) if self._shape and (self.i.data.shape[1] != self._shape): self.logger.warning("Invalid shape") else: self._X = self.i.data.values self._X_indices = self.i.data.index.values self._X_meta = self.i.meta # Epochs if self._dimensions == 3: for name, _, port in self.iterate("i_*"): if port.ready( ) and "training" not in name and "events" not in name: data = port.data.values meta = port.meta indices = port.data.index.values label = get_meta(port, self.meta_label) if not self._X_columns: self._X_columns = list(port.data.columns) if self._shape and (data.shape != self._shape): self.logger.warning("Invalid shape") continue if not self.fit and self.meta_label is not None and label is None: self.logger.warning("Invalid label") continue if self._X is None: self._X = [] if self._y is None and label is not None: self._y = [] if self._X_meta is None: self._X_meta = [] self._X.append(data) self._X_indices.append(indices) self._X_meta.append(meta) if label is not None: self._y.append(label)
def _accumulate(self, start, stop): # Do nothing if no fitting required if not self.fit: return # Set defaults indices = np.array([], dtype=np.datetime64) # Accumulate continuous data if self._dimensions == 2: if self.i_training.ready(): data = self.i_training.data mask = (data.index >= start) & (data.index < stop) data = data[mask] if not data.empty: if self._X_train is None: self._X_train = data.values self._shape = self._X_train.shape[1] indices = data.index.values else: if data.shape[1] == self._shape: self._X_train = np.vstack((self._X_train, data.values)) indices = data.index.values else: self.logger.warning("Invalid shape") # Accumulate epoched data if self._dimensions == 3: for _, _, port in self.iterate("i_training_*"): if port.ready(): index = port.data.index.values[0] if index >= start and index < stop: data = port.data.values label = get_meta(port, self.meta_label) if self._shape and (data.shape != self._shape): self.logger.warning("Invalid shape") continue if self.meta_label is not None and label is None: self.logger.warning("Invalid label") continue if self._X_train is None: self._X_train = np.array([data]) self._shape = self._X_train.shape[1:] else: self._X_train = np.vstack((self._X_train, [data])) indices = np.append(indices, index) if label is not None: if self._y_train is None: self._y_train = np.array([label]) else: self._y_train = np.append(self._y_train, [label]) # Store indices if indices.size != 0: self._X_train_indices = np.append(self._X_train_indices, indices) # Trim if self._X_train is not None: mask = (self._X_train_indices >= start) & (self._X_train_indices < stop) self._X_train = self._X_train[mask] self._X_train_indices = self._X_train_indices[mask] if self._y_train is not None: self._y_train = self._y_train[mask]
def test_get_meta_no_key(): port = Port() assert get_meta(port, None) == None assert get_meta(port, None, False) == False
def test_get_meta_not_found(): port = Port() assert get_meta(port, ('foo')) == None assert get_meta(port, ('foo'), False) == False
def test_get_meta_deep(): port = Port() port.meta = {'foo': {'bar': {'baz': 42}}} assert get_meta(port, ('foo', 'bar', 'baz')) == 42
def test_get_meta(): port = Port() port.meta = {'target': True} assert get_meta(port, 'target') == True