from examples.src.python.bolt import HalfAckBolt # Topology is defined using a topology builder # Refer to multi_stream_topology for defining a topology by subclassing Topology # pylint: disable=superfluous-parens if __name__ == '__main__': if len(sys.argv) != 2: print("Topology's name is not specified") sys.exit(1) builder = TopologyBuilder(name=sys.argv[1]) word_spout = builder.add_spout("word_spout", WordSpout, par=2) half_ack_bolt = builder.add_bolt( "half_ack_bolt", HalfAckBolt, par=2, inputs={word_spout: Grouping.fields('word')}, config={constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS: 10}) topology_config = { constants.TOPOLOGY_RELIABILITY_MODE: constants.TopologyReliabilityMode.ATLEAST_ONCE, constants.TOPOLOGY_MAX_SPOUT_PENDING: 100000000, constants.TOPOLOGY_MESSAGE_TIMEOUT_SECS: 300 } builder.set_config(topology_config) builder.build_and_submit()
'''Example WindowSizeTopology''' import sys import heronpy.api.api_constants as constants from heronpy.api.topology import TopologyBuilder from heronpy.api.stream import Grouping from heronpy.api.bolt.window_bolt import SlidingWindowBolt from heron.examples.src.python.spout import WordSpout from examples.src.python.bolt import WindowSizeBolt # Topology is defined using a topology builder # Refer to multi_stream_topology for defining a topology by subclassing Topology if __name__ == '__main__': if len(sys.argv) != 2: print "Topology's name is not specified" sys.exit(1) builder = TopologyBuilder(name=sys.argv[1]) word_spout = builder.add_spout("word_spout", WordSpout, par=2) count_bolt = builder.add_bolt("count_bolt", WindowSizeBolt, par=2, inputs={word_spout: Grouping.fields('word')}, config={SlidingWindowBolt.WINDOW_DURATION_SECS: 10, SlidingWindowBolt.WINDOW_SLIDEINTERVAL_SECS: 2}) topology_config = {constants.TOPOLOGY_RELIABILITY_MODE: constants.TopologyReliabilityMode.ATLEAST_ONCE} builder.set_config(topology_config) builder.build_and_submit()
import sys import heronpy.api.api_constants as constants from heronpy.api.topology import TopologyBuilder from heronpy.api.stream import Grouping from heronpy.api.bolt.window_bolt import SlidingWindowBolt from heron.examples.src.python.spout import WordSpout from examples.src.python.bolt import WindowSizeBolt # Topology is defined using a topology builder # Refer to multi_stream_topology for defining a topology by subclassing Topology # pylint: disable=superfluous-parens if __name__ == '__main__': if len(sys.argv) != 2: print("Topology's name is not specified") sys.exit(1) builder = TopologyBuilder(name=sys.argv[1]) word_spout = builder.add_spout("word_spout", WordSpout, par=2) count_bolt = builder.add_bolt("count_bolt", WindowSizeBolt, par=2, inputs={word_spout: Grouping.fields('word')}, config={SlidingWindowBolt.WINDOW_DURATION_SECS: 10, SlidingWindowBolt.WINDOW_SLIDEINTERVAL_SECS: 2}) topology_config = {constants.TOPOLOGY_RELIABILITY_MODE: constants.TopologyReliabilityMode.ATLEAST_ONCE} builder.set_config(topology_config) builder.build_and_submit()
from heronpy.api.stream import Grouping from examples.src.python.spout import StatefulWordSpout from examples.src.python.bolt import StatefulCountBolt # Topology is defined using a topology builder # Refer to multi_stream_topology for defining a topology by subclassing Topology # pylint: disable=superfluous-parens if __name__ == '__main__': if len(sys.argv) != 2: print("Topology's name is not specified") sys.exit(1) builder = TopologyBuilder(name=sys.argv[1]) word_spout = builder.add_spout("word_spout", StatefulWordSpout, par=2) count_bolt = builder.add_bolt( "count_bolt", StatefulCountBolt, par=2, inputs={word_spout: Grouping.fields('word')}, config={constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS: 10}) topology_config = { constants.TOPOLOGY_RELIABILITY_MODE: constants.TopologyReliabilityMode.EFFECTIVELY_ONCE, constants.TOPOLOGY_STATEFUL_CHECKPOINT_INTERVAL_SECONDS: 30 } builder.set_config(topology_config) builder.build_and_submit()
import heronpy.api.api_constants as constants from heronpy.api.topology import TopologyBuilder from heronpy.api.stream import Grouping from examples.src.python.spout import WordSpout from examples.src.python.bolt import HalfAckBolt # Topology is defined using a topology builder # Refer to multi_stream_topology for defining a topology by subclassing Topology # pylint: disable=superfluous-parens if __name__ == '__main__': if len(sys.argv) != 2: print("Topology's name is not specified") sys.exit(1) builder = TopologyBuilder(name=sys.argv[1]) word_spout = builder.add_spout("word_spout", WordSpout, par=2) half_ack_bolt = builder.add_bolt("half_ack_bolt", HalfAckBolt, par=2, inputs={word_spout: Grouping.fields('word')}, config={constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS: 10}) topology_config = {constants.TOPOLOGY_RELIABILITY_MODE: constants.TopologyReliabilityMode.ATLEAST_ONCE, constants.TOPOLOGY_MAX_SPOUT_PENDING: 100000000, constants.TOPOLOGY_MESSAGE_TIMEOUT_SECS: 300} builder.set_config(topology_config) builder.build_and_submit()
"benchmark_config": dict(data_path=op.join(op.expanduser('~'), 'data', 'rtfmri', 'func_0000.nii.gz'), classes_path=op.join(op.expanduser('~'), 'data', 'rtfmri', 'timingFiles', 'fullRunLabels.txt'), pure_training_size=10, shuffle_seed=1, # 1 for no shuffle, 1000000007 for a nice random ) } builder = TopologyBuilder("heron") # Load demo data (without loading whole file at once) (nibabel - mmap) (with classification results) image_spout = builder.add_spout("image_spout", ImageSpout, par=1, config=config) # Optional: low/er down the resolution # Affine registration - fixing possible moves or rotations (multicore) mask_bolt = builder.add_bolt("mask_bolt", MaskBolt, par=1, config=config, inputs={image_spout: Grouping.ALL}) affine_registration_bolt = builder.add_bolt("affine_registration_bolt", AffineRegistrationBolt, par=6, config=config, inputs={mask_bolt: Grouping.ALL, image_spout: Grouping.SHUFFLE}) classifiers = [] model = 'SGD' for loss in ['hinge', 'log', 'modified_huber']: for penalty in ['l1', 'elasticnet']: config["sgd_config"] = {'model': model, 'loss': loss, 'penalty': penalty} classifiers.append(builder.add_bolt("classifier_{}_{}_{}".format(model, loss, penalty), ClassifierBolt, par=1, config=config.copy(), inputs={affine_registration_bolt: Grouping.SHUFFLE})) builder.add_bolt("validation_{}_{}_{}".format(model, loss, penalty), CrossValidationBolt, par=1, config=config.copy(),
from heronpy.api.stream import Grouping from heronpy.api.topology import TopologyBuilder from spout import TestWordSpout from bolt import ExclamationBolt if __name__ == '__main__': builder = TopologyBuilder('my-python-topology') word = builder.add_spout('word', TestWordSpout, par=2) exclaim1 = builder.add_bolt('exclaim1', \ ExclamationBolt, par=2, \ inputs={word['stream1']: Grouping.SHUFFLE, \ word['stream2']: Grouping.SHUFFLE}) exclaim2 = builder.add_bolt('exclaim2', \ ExclamationBolt, par=2, \ inputs={exclaim1: Grouping.SHUFFLE}) builder.build_and_submit()