def compose(self, left: topology.Composable) -> pipeline.Segment: """Dummy composition.""" track = left.expand() trainer = node.Worker(spec, 1, 1) applier = trainer.fork() extractor = node.Worker(spec, 1, 1) trainer.train(track.train.publisher, extractor[0]) return track.use( label=track.train.extend(extractor)).extend(applier)
def path(request, head: grnode.Worker, spec: task.Spec) -> view.Path: """Path fixture.""" grnode1 = grnode.Worker(spec, 1, 2) grnode2 = grnode.Worker(spec, 2, 1) grnode1[0].subscribe(head[0]) grnode2[0].subscribe(grnode1[0]) grnode2[1].subscribe(grnode1[1]) if request.param: # stateful grnode3 = grnode.Worker(spec, 1, 1) grnode2[0].publish(grnode3, port.Train()) return view.Path(head)
def builder(self, head: pipeline.Segment, inner: pipeline.Segment) -> 'FullStacker.Builder': """Create a builder (folding context). Args: head: Head of the crossvalidation segment. inner: Exclusive instance of the inner composition. Returns: Builder instance. """ trained: node.Worker = node.Worker(ndframe.Concat.spec(axis='columns'), len(self.bases), 1) applied: node.Worker = trained.fork() stack_forks: typing.Iterable[node.Worker] = node.Worker.fgen( ndframe.Concat.spec(axis='index'), self.nsplits, 1) merge_forks: typing.Iterable[node.Worker] = node.Worker.fgen( ndframe.Apply.spec(function=self._merge), self.nsplits, 1) stackers: typing.Dict[topology.Composable, node.Worker] = dict() mergers: typing.Dict[topology.Composable, node.Worker] = dict() for index, (base, stack, merge) in enumerate( zip(self.bases, stack_forks, merge_forks)): stackers[base] = stack mergers[base] = merge trained[index].subscribe(stackers[base][0]) applied[index].subscribe(merge[0]) return self.Builder(head, stackers, mergers, trained, applied)
def compose(self, left: topology.Composable) -> pipeline.Segment: """Composition implementation. Args: left: Left side. Returns: Composed track. """ left: pipeline.Segment = left.expand() train_dumper: node.Worker = node.Worker( TrainDumper.spec(path=self._path('train'), label=self.label), 1, 1) apply_dumper: node.Worker = node.Worker( ApplyDumper.spec(path=self._path('apply')), 1, 1) train_dumper.train(left.train.publisher, left.label.publisher) self._instances += 1 return left.extend(apply=view.Path(apply_dumper), train=view.Path(train_dumper.fork()))
def compose(self, left: topology.Composable) -> pipeline.Segment: """Compose the publisher segment track. Returns: Sink segment track. """ apply: node.Worker = node.Worker(self._writer, 1, 0) train: node.Worker = apply.fork() return left.expand().extend(apply, train)
def compose(self, left: topology.Composable) -> pipeline.Segment: """Abstract composition implementation. Args: left: Left side track builder. Returns: Composed track. """ return self.apply(node.Worker(self.spec, self.SZIN, self.SZOUT), left.expand())
def compose(self, left: topology.Composable) -> pipeline.Segment: """Compose the source segment track. Returns: Source segment track. """ if not isinstance(left, topology.Origin): raise error.Unexpected('Source not origin') apply: view.Path = view.Path(node.Worker(self._apply, 0, 1)) train: view.Path = view.Path(node.Worker(self._train, 0, 1)) label: typing.Optional[view.Path] = None if self._label: train_tail = node.Future() label_tail = node.Future() extract = node.Worker(self._label, 1, 2) extract[0].subscribe(train.publisher) train_tail[0].subscribe(extract[0]) label_tail[0].subscribe(extract[1]) train = train.extend(tail=train_tail) label = train.extend(tail=label_tail) return pipeline.Segment(apply, train, label)
def compose(self, left: topology.Composable) -> pipeline.Segment: """Composition implementation. Args: left: Left side. Returns: Composed track. """ left: pipeline.Segment = left.expand() inserter: node.Worker = node.Worker(self.inserter, 1, 1) inserter.train(left.train.publisher, left.label.publisher) return left.extend(train=view.Path(inserter.fork()))
def score(self, ytrue: port.Publishable, ypred: port.Publishable) -> node.Atomic: """Metric scoring routing. Args: ytrue: Publisher of the tru labels. ypred: Publisher of the predicted values. Returns: Scoring worker node. """ scorer = node.Worker(self.metric, 2, 1) scorer[0].subscribe(ytrue) scorer[1].subscribe(ypred) return scorer
def worker( mode: typing.Optional[task.Spec] ) -> typing.Optional[node.Worker]: """Create a worker for given spec if not None. Args: mode: Task spec for given mode. Returns: Worker instance or None. """ if mode: mode = node.Worker(mode, 1, 1) return mode
def builder(self, head: pipeline.Segment, inner: pipeline.Segment) -> 'MergingScorer.Builder': """Create a builder (folding context). Args: head: Head of the crossvalidation segment. inner: Exclusive instance of the inner composition. Returns: Builder instance. """ # TO-DO: apply path based on trainset is confusing compiler # inner.apply.subscribe(head.train.publisher) # scorer = self.score(head.label.publisher, inner.apply.publisher) merger = node.Worker(self.merger, self.nsplits, 1) # return self.Builder(head.use(apply=head.train.extend(tail=scorer)), merger) return self.Builder(head, merger)
def compose(self, left: topology.Composable) -> pipeline.Segment: """Ensemble composition. Args: left: left segment. Returns: Composed segment track. """ head: pipeline.Segment = pipeline.Segment() splitter = node.Worker(self.splitter, 1, 2 * self.nsplits) splitter.train(head.train.publisher, head.label.publisher) features: node.Worker = splitter.fork() features[0].subscribe(head.train.publisher) labels: node.Worker = splitter.fork() labels[0].subscribe(head.label.publisher) builder = self.builder(head, left.expand()) for fold in range(self.nsplits): inner: pipeline.Segment = left.expand() self.fold(fold, builder, inner, features, labels) return builder.build()
def node(spec: task.Spec) -> grnode.Worker: """Node fixture.""" return grnode.Worker(spec, 1, 1)
def simple(spec: task.Spec) -> grnode.Worker: """Simple node fixture with 1 input and 1 output apply port.""" return grnode.Worker(spec, 1, 1)
def head(spec: task.Spec) -> grnode.Worker: """Path head fixture.""" return grnode.Worker(spec, 1, 1)
def multi(spec: task.Spec) -> grnode.Worker: """Multi port node fixture (2 input and 2 output apply port).""" return grnode.Worker(spec, 2, 2)
def node3(spec: task.Spec) -> node.Worker: """Node fixture.""" return node.Worker(spec, 1, 1)
def compose(self, left: topology.Composable) -> pipeline.Segment: """Dummy composition.""" trainer = node.Worker(spec, 1, 1) applier = trainer.fork() return pipeline.Segment(applier, trainer)