def compose(self, left: topology.Composable) -> pipeline.Segment:
"""Dummy composition."""
track = left.expand()
trainer = node.Worker(spec, 1, 1)
applier = trainer.fork()
extractor = node.Worker(spec, 1, 1)
trainer.train(track.train.publisher, extractor[0])
return track.use(
label=track.train.extend(extractor)).extend(applier)
def path(request, head: grnode.Worker, spec: task.Spec) -> view.Path:
"""Path fixture."""
grnode1 = grnode.Worker(spec, 1, 2)
grnode2 = grnode.Worker(spec, 2, 1)
grnode1[0].subscribe(head[0])
grnode2[0].subscribe(grnode1[0])
grnode2[1].subscribe(grnode1[1])
if request.param: # stateful
grnode3 = grnode.Worker(spec, 1, 1)
grnode2[0].publish(grnode3, port.Train())
return view.Path(head)
def builder(self, head: pipeline.Segment,
inner: pipeline.Segment) -> 'FullStacker.Builder':
"""Create a builder (folding context).
Args:
head: Head of the crossvalidation segment.
inner: Exclusive instance of the inner composition.
Returns:
Builder instance.
"""
trained: node.Worker = node.Worker(ndframe.Concat.spec(axis='columns'),
len(self.bases), 1)
applied: node.Worker = trained.fork()
stack_forks: typing.Iterable[node.Worker] = node.Worker.fgen(
ndframe.Concat.spec(axis='index'), self.nsplits, 1)
merge_forks: typing.Iterable[node.Worker] = node.Worker.fgen(
ndframe.Apply.spec(function=self._merge), self.nsplits, 1)
stackers: typing.Dict[topology.Composable, node.Worker] = dict()
mergers: typing.Dict[topology.Composable, node.Worker] = dict()
for index, (base, stack, merge) in enumerate(
zip(self.bases, stack_forks, merge_forks)):
stackers[base] = stack
mergers[base] = merge
trained[index].subscribe(stackers[base][0])
applied[index].subscribe(merge[0])
return self.Builder(head, stackers, mergers, trained, applied)
def compose(self, left: topology.Composable) -> pipeline.Segment:
"""Composition implementation.
Args:
left: Left side.
Returns:
Composed track.
"""
left: pipeline.Segment = left.expand()
train_dumper: node.Worker = node.Worker(
TrainDumper.spec(path=self._path('train'), label=self.label), 1, 1)
apply_dumper: node.Worker = node.Worker(
ApplyDumper.spec(path=self._path('apply')), 1, 1)
train_dumper.train(left.train.publisher, left.label.publisher)
self._instances += 1
return left.extend(apply=view.Path(apply_dumper),
train=view.Path(train_dumper.fork()))
def compose(self, left: topology.Composable) -> pipeline.Segment:
"""Compose the publisher segment track.
Returns:
Sink segment track.
"""
apply: node.Worker = node.Worker(self._writer, 1, 0)
train: node.Worker = apply.fork()
return left.expand().extend(apply, train)
def compose(self, left: topology.Composable) -> pipeline.Segment:
"""Abstract composition implementation.
Args:
left: Left side track builder.
Returns:
Composed track.
"""
return self.apply(node.Worker(self.spec, self.SZIN, self.SZOUT), left.expand())
def compose(self, left: topology.Composable) -> pipeline.Segment:
"""Compose the source segment track.
Returns:
Source segment track.
"""
if not isinstance(left, topology.Origin):
raise error.Unexpected('Source not origin')
apply: view.Path = view.Path(node.Worker(self._apply, 0, 1))
train: view.Path = view.Path(node.Worker(self._train, 0, 1))
label: typing.Optional[view.Path] = None
if self._label:
train_tail = node.Future()
label_tail = node.Future()
extract = node.Worker(self._label, 1, 2)
extract[0].subscribe(train.publisher)
train_tail[0].subscribe(extract[0])
label_tail[0].subscribe(extract[1])
train = train.extend(tail=train_tail)
label = train.extend(tail=label_tail)
return pipeline.Segment(apply, train, label)
def compose(self, left: topology.Composable) -> pipeline.Segment:
"""Composition implementation.
Args:
left: Left side.
Returns:
Composed track.
"""
left: pipeline.Segment = left.expand()
inserter: node.Worker = node.Worker(self.inserter, 1, 1)
inserter.train(left.train.publisher, left.label.publisher)
return left.extend(train=view.Path(inserter.fork()))
def score(self, ytrue: port.Publishable, ypred: port.Publishable) -> node.Atomic:
"""Metric scoring routing.
Args:
ytrue: Publisher of the tru labels.
ypred: Publisher of the predicted values.
Returns:
Scoring worker node.
"""
scorer = node.Worker(self.metric, 2, 1)
scorer[0].subscribe(ytrue)
scorer[1].subscribe(ypred)
return scorer
def worker(
mode: typing.Optional[task.Spec]
) -> typing.Optional[node.Worker]:
"""Create a worker for given spec if not None.
Args:
mode: Task spec for given mode.
Returns:
Worker instance or None.
"""
if mode:
mode = node.Worker(mode, 1, 1)
return mode
def builder(self, head: pipeline.Segment, inner: pipeline.Segment) -> 'MergingScorer.Builder':
"""Create a builder (folding context).
Args:
head: Head of the crossvalidation segment.
inner: Exclusive instance of the inner composition.
Returns:
Builder instance.
"""
# TO-DO: apply path based on trainset is confusing compiler
# inner.apply.subscribe(head.train.publisher)
# scorer = self.score(head.label.publisher, inner.apply.publisher)
merger = node.Worker(self.merger, self.nsplits, 1)
# return self.Builder(head.use(apply=head.train.extend(tail=scorer)), merger)
return self.Builder(head, merger)
def compose(self, left: topology.Composable) -> pipeline.Segment:
"""Ensemble composition.
Args:
left: left segment.
Returns:
Composed segment track.
"""
head: pipeline.Segment = pipeline.Segment()
splitter = node.Worker(self.splitter, 1, 2 * self.nsplits)
splitter.train(head.train.publisher, head.label.publisher)
features: node.Worker = splitter.fork()
features[0].subscribe(head.train.publisher)
labels: node.Worker = splitter.fork()
labels[0].subscribe(head.label.publisher)
builder = self.builder(head, left.expand())
for fold in range(self.nsplits):
inner: pipeline.Segment = left.expand()
self.fold(fold, builder, inner, features, labels)
return builder.build()
def node(spec: task.Spec) -> grnode.Worker:
"""Node fixture."""
return grnode.Worker(spec, 1, 1)
def simple(spec: task.Spec) -> grnode.Worker:
"""Simple node fixture with 1 input and 1 output apply port."""
return grnode.Worker(spec, 1, 1)
def head(spec: task.Spec) -> grnode.Worker:
"""Path head fixture."""
return grnode.Worker(spec, 1, 1)
def multi(spec: task.Spec) -> grnode.Worker:
"""Multi port node fixture (2 input and 2 output apply port)."""
return grnode.Worker(spec, 2, 2)
def node3(spec: task.Spec) -> node.Worker:
"""Node fixture."""
return node.Worker(spec, 1, 1)
def compose(self, left: topology.Composable) -> pipeline.Segment:
"""Dummy composition."""
trainer = node.Worker(spec, 1, 1)
applier = trainer.fork()
return pipeline.Segment(applier, trainer)
