Python TopologyBuilder.add_spout Examples

Example #1

File: window_size_topology.py Project: yangxu1362850636/heron

'''Example WindowSizeTopology'''
import sys

import heronpy.api.api_constants as constants
from heronpy.api.topology import TopologyBuilder
from heronpy.api.stream import Grouping
from heronpy.api.bolt.window_bolt import SlidingWindowBolt
from heron.examples.src.python.spout import WordSpout
from examples.src.python.bolt import WindowSizeBolt

# Topology is defined using a topology builder
# Refer to multi_stream_topology for defining a topology by subclassing Topology
if __name__ == '__main__':
  if len(sys.argv) != 2:
    print "Topology's name is not specified"
    sys.exit(1)

  builder = TopologyBuilder(name=sys.argv[1])

  word_spout = builder.add_spout("word_spout", WordSpout, par=2)
  count_bolt = builder.add_bolt("count_bolt", WindowSizeBolt, par=2,
                                inputs={word_spout: Grouping.fields('word')},
                                config={SlidingWindowBolt.WINDOW_DURATION_SECS: 10,
                                        SlidingWindowBolt.WINDOW_SLIDEINTERVAL_SECS: 2})

  topology_config = {constants.TOPOLOGY_RELIABILITY_MODE:
                         constants.TopologyReliabilityMode.ATLEAST_ONCE}
  builder.set_config(topology_config)

  builder.build_and_submit()

Example #2

File: window_size_topology.py Project: lucperkins/heron

import sys

import heronpy.api.api_constants as constants
from heronpy.api.topology import TopologyBuilder
from heronpy.api.stream import Grouping
from heronpy.api.bolt.window_bolt import SlidingWindowBolt
from heron.examples.src.python.spout import WordSpout
from examples.src.python.bolt import WindowSizeBolt

# Topology is defined using a topology builder
# Refer to multi_stream_topology for defining a topology by subclassing Topology
# pylint: disable=superfluous-parens
if __name__ == '__main__':
  if len(sys.argv) != 2:
    print("Topology's name is not specified")
    sys.exit(1)

  builder = TopologyBuilder(name=sys.argv[1])

  word_spout = builder.add_spout("word_spout", WordSpout, par=2)
  count_bolt = builder.add_bolt("count_bolt", WindowSizeBolt, par=2,
                                inputs={word_spout: Grouping.fields('word')},
                                config={SlidingWindowBolt.WINDOW_DURATION_SECS: 10,
                                        SlidingWindowBolt.WINDOW_SLIDEINTERVAL_SECS: 2})

  topology_config = {constants.TOPOLOGY_RELIABILITY_MODE:
                         constants.TopologyReliabilityMode.ATLEAST_ONCE}
  builder.set_config(topology_config)

  builder.build_and_submit()

Example #3

File: topology.py Project: tpawlowski/mri_pipeline

    p: float fraction of data which should be taken into consideration. Maybe it would be better to take a threshold. 
    stable_threshold: Number of iteration for which set of voxels were not changed.
    """

    config = {
        "benchmark_config": dict(data_path=op.join(op.expanduser('~'), 'data', 'rtfmri', 'func_0000.nii.gz'),
                                 classes_path=op.join(op.expanduser('~'), 'data', 'rtfmri', 'timingFiles', 'fullRunLabels.txt'),
                                 pure_training_size=10,
                                 shuffle_seed=1,  # 1 for no shuffle, 1000000007 for a nice random
                                 )
    }

    builder = TopologyBuilder("heron")

    # Load demo data (without loading whole file at once) (nibabel - mmap) (with classification results)
    image_spout = builder.add_spout("image_spout", ImageSpout, par=1, config=config)

    # Optional: low/er down the resolution

    # Affine registration - fixing possible moves or rotations (multicore)
    mask_bolt = builder.add_bolt("mask_bolt", MaskBolt, par=1, config=config,
                                 inputs={image_spout: Grouping.ALL})

    affine_registration_bolt = builder.add_bolt("affine_registration_bolt", AffineRegistrationBolt, par=6, config=config,
                                                inputs={mask_bolt: Grouping.ALL,
                                                        image_spout: Grouping.SHUFFLE})

    classifiers = []

    model = 'SGD'
    for loss in ['hinge', 'log', 'modified_huber']:

Example #4

from heronpy.api.topology import TopologyBuilder
from heronpy.api.stream import Grouping
from examples.src.python.spout import StatefulWordSpout
from examples.src.python.bolt import StatefulCountBolt

# Topology is defined using a topology builder
# Refer to multi_stream_topology for defining a topology by subclassing Topology
# pylint: disable=superfluous-parens
if __name__ == '__main__':
    if len(sys.argv) != 2:
        print("Topology's name is not specified")
        sys.exit(1)

    builder = TopologyBuilder(name=sys.argv[1])

    word_spout = builder.add_spout("word_spout", StatefulWordSpout, par=2)
    count_bolt = builder.add_bolt(
        "count_bolt",
        StatefulCountBolt,
        par=2,
        inputs={word_spout: Grouping.fields('word')},
        config={constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS: 10})

    topology_config = {
        constants.TOPOLOGY_RELIABILITY_MODE:
        constants.TopologyReliabilityMode.EFFECTIVELY_ONCE,
        constants.TOPOLOGY_STATEFUL_CHECKPOINT_INTERVAL_SECONDS: 30
    }
    builder.set_config(topology_config)

    builder.build_and_submit()

Example #5

from heronpy.api.stream import Grouping
from heronpy.api.topology import TopologyBuilder

from spout import TestWordSpout
from bolt import ExclamationBolt

if __name__ == '__main__':
    builder = TopologyBuilder('my-python-topology')
    word = builder.add_spout('word', TestWordSpout, par=2)
    exclaim1 = builder.add_bolt('exclaim1', \
      ExclamationBolt, par=2, \
      inputs={word['stream1']: Grouping.SHUFFLE, \
              word['stream2']: Grouping.SHUFFLE})
    exclaim2 = builder.add_bolt('exclaim2', \
      ExclamationBolt, par=2, \
      inputs={exclaim1: Grouping.SHUFFLE})
    builder.build_and_submit()