def run(self, name, config, builder):
"""Builds the topology and submits it"""
if not isinstance(name, str):
raise RuntimeError("Name has to be a string type")
if not isinstance(config, Config):
raise RuntimeError("config has to be a Config type")
if not isinstance(builder, Builder):
raise RuntimeError("builder has to be a Builder type")
bldr = TopologyBuilder(name=name)
builder.build(bldr)
bldr.set_config(config._api_config)
bldr.build_and_submit()
def test_constructor(self):
builder = TopologyBuilder("WordCount")
self.assertEqual(builder.topology_name, "WordCount")
with self.assertRaises(AssertionError):
TopologyBuilder("Topology")
with self.assertRaises(AssertionError):
TopologyBuilder(123)
with self.assertRaises(AssertionError):
TopologyBuilder(None)
def test_add_spec(self):
builder = TopologyBuilder("Test")
with self.assertRaises(ValueError):
builder.add_spec(HeronComponentSpec(None, "path", True, 1))
with self.assertRaises(TypeError):
builder.add_spec(None)
self.assertEqual(len(builder._specs), 0)
# add 10 specs
specs = []
for i in range(10):
specs.append(HeronComponentSpec(str(i), "path", True, 1))
builder.add_spec(*specs)
self.assertEqual(len(builder._specs), 10)
def run(self, name, config, builder):
"""Builds the topology and submits it"""
if not isinstance(name, str):
raise RuntimeError("Name has to be a string type")
if not isinstance(config, Config):
raise RuntimeError("config has to be a Config type")
if not isinstance(builder, Builder):
raise RuntimeError("builder has to be a Builder type")
bldr = TopologyBuilder(name=name)
builder.build(bldr)
bldr.set_config(config._api_config)
bldr.build_and_submit()
def test_add_spec(self):
builder = TopologyBuilder("Test")
with self.assertRaises(ValueError):
builder.add_spec(HeronComponentSpec(None, "path", True, 1))
with self.assertRaises(TypeError):
builder.add_spec(None)
self.assertEqual(len(builder._specs), 0)
# add 10 specs
specs = []
for i in range(10):
specs.append(HeronComponentSpec(str(i), "path", True, 1))
builder.add_spec(*specs)
self.assertEqual(len(builder._specs), 10)
def run(self, name, config=None):
"""Runs the Streamlet. This is run as a Heron python topology under the name
'name'. The config attached is passed on to this Heron topology
Once submitted, run returns immediately
"""
if name is None or not isinstance(name, str):
raise RuntimeError("Job Name has to be a string")
bldr = TopologyBuilder(name=name)
stage_names = {}
bldr = self._build(bldr, stage_names)
if config is not None:
if not isinstance(config, dict):
raise RuntimeError("config has to be a dict")
bldr.set_config(config)
bldr.build_and_submit()
'''Example WindowSizeTopology'''
import sys
import heronpy.api.api_constants as constants
from heronpy.api.topology import TopologyBuilder
from heronpy.api.stream import Grouping
from heronpy.api.bolt.window_bolt import SlidingWindowBolt
from heron.examples.src.python.spout import WordSpout
from examples.src.python.bolt import WindowSizeBolt
# Topology is defined using a topology builder
# Refer to multi_stream_topology for defining a topology by subclassing Topology
if __name__ == '__main__':
if len(sys.argv) != 2:
print "Topology's name is not specified"
sys.exit(1)
builder = TopologyBuilder(name=sys.argv[1])
word_spout = builder.add_spout("word_spout", WordSpout, par=2)
count_bolt = builder.add_bolt("count_bolt", WindowSizeBolt, par=2,
inputs={word_spout: Grouping.fields('word')},
config={SlidingWindowBolt.WINDOW_DURATION_SECS: 10,
SlidingWindowBolt.WINDOW_SLIDEINTERVAL_SECS: 2})
topology_config = {constants.TOPOLOGY_RELIABILITY_MODE:
constants.TopologyReliabilityMode.ATLEAST_ONCE}
builder.set_config(topology_config)
builder.build_and_submit()
import heronpy.api.api_constants as constants
from heronpy.api.topology import TopologyBuilder
from heronpy.api.stream import Grouping
from examples.src.python.spout import WordSpout
from examples.src.python.bolt import HalfAckBolt
# Topology is defined using a topology builder
# Refer to multi_stream_topology for defining a topology by subclassing Topology
# pylint: disable=superfluous-parens
if __name__ == '__main__':
if len(sys.argv) != 2:
print("Topology's name is not specified")
sys.exit(1)
builder = TopologyBuilder(name=sys.argv[1])
word_spout = builder.add_spout("word_spout", WordSpout, par=2)
half_ack_bolt = builder.add_bolt(
"half_ack_bolt",
HalfAckBolt,
par=2,
inputs={word_spout: Grouping.fields('word')},
config={constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS: 10})
topology_config = {
constants.TOPOLOGY_RELIABILITY_MODE:
constants.TopologyReliabilityMode.ATLEAST_ONCE,
constants.TOPOLOGY_MAX_SPOUT_PENDING: 100000000,
constants.TOPOLOGY_MESSAGE_TIMEOUT_SECS: 300
}
import sys
import heronpy.api.api_constants as constants
from heronpy.api.topology import TopologyBuilder
from heronpy.api.stream import Grouping
from heronpy.api.bolt.window_bolt import SlidingWindowBolt
from heron.examples.src.python.spout import WordSpout
from examples.src.python.bolt import WindowSizeBolt
# Topology is defined using a topology builder
# Refer to multi_stream_topology for defining a topology by subclassing Topology
# pylint: disable=superfluous-parens
if __name__ == '__main__':
if len(sys.argv) != 2:
print("Topology's name is not specified")
sys.exit(1)
builder = TopologyBuilder(name=sys.argv[1])
word_spout = builder.add_spout("word_spout", WordSpout, par=2)
count_bolt = builder.add_bolt("count_bolt", WindowSizeBolt, par=2,
inputs={word_spout: Grouping.fields('word')},
config={SlidingWindowBolt.WINDOW_DURATION_SECS: 10,
SlidingWindowBolt.WINDOW_SLIDEINTERVAL_SECS: 2})
topology_config = {constants.TOPOLOGY_RELIABILITY_MODE:
constants.TopologyReliabilityMode.ATLEAST_ONCE}
builder.set_config(topology_config)
builder.build_and_submit()
import heronpy.api.api_constants as constants
from heronpy.api.topology import TopologyBuilder
from heronpy.api.stream import Grouping
from examples.src.python.spout import WordSpout
from examples.src.python.bolt import HalfAckBolt
# Topology is defined using a topology builder
# Refer to multi_stream_topology for defining a topology by subclassing Topology
# pylint: disable=superfluous-parens
if __name__ == '__main__':
if len(sys.argv) != 2:
print("Topology's name is not specified")
sys.exit(1)
builder = TopologyBuilder(name=sys.argv[1])
word_spout = builder.add_spout("word_spout", WordSpout, par=2)
half_ack_bolt = builder.add_bolt("half_ack_bolt", HalfAckBolt, par=2,
inputs={word_spout: Grouping.fields('word')},
config={constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS: 10})
topology_config = {constants.TOPOLOGY_RELIABILITY_MODE:
constants.TopologyReliabilityMode.ATLEAST_ONCE,
constants.TOPOLOGY_MAX_SPOUT_PENDING: 100000000,
constants.TOPOLOGY_MESSAGE_TIMEOUT_SECS: 300}
builder.set_config(topology_config)
builder.build_and_submit()
Aim: Select part of the brain image which activity is correlated with the classification.
Parameters:
p: float fraction of data which should be taken into consideration. Maybe it would be better to take a threshold.
stable_threshold: Number of iteration for which set of voxels were not changed.
"""
config = {
"benchmark_config": dict(data_path=op.join(op.expanduser('~'), 'data', 'rtfmri', 'func_0000.nii.gz'),
classes_path=op.join(op.expanduser('~'), 'data', 'rtfmri', 'timingFiles', 'fullRunLabels.txt'),
pure_training_size=10,
shuffle_seed=1, # 1 for no shuffle, 1000000007 for a nice random
)
}
builder = TopologyBuilder("heron")
# Load demo data (without loading whole file at once) (nibabel - mmap) (with classification results)
image_spout = builder.add_spout("image_spout", ImageSpout, par=1, config=config)
# Optional: low/er down the resolution
# Affine registration - fixing possible moves or rotations (multicore)
mask_bolt = builder.add_bolt("mask_bolt", MaskBolt, par=1, config=config,
inputs={image_spout: Grouping.ALL})
affine_registration_bolt = builder.add_bolt("affine_registration_bolt", AffineRegistrationBolt, par=6, config=config,
inputs={mask_bolt: Grouping.ALL,
image_spout: Grouping.SHUFFLE})
classifiers = []
from heronpy.api.stream import Grouping
from heronpy.api.topology import TopologyBuilder
from spout import TestWordSpout
from bolt import ExclamationBolt
if __name__ == '__main__':
builder = TopologyBuilder('my-python-topology')
word = builder.add_spout('word', TestWordSpout, par=2)
exclaim1 = builder.add_bolt('exclaim1', \
ExclamationBolt, par=2, \
inputs={word['stream1']: Grouping.SHUFFLE, \
word['stream2']: Grouping.SHUFFLE})
exclaim2 = builder.add_bolt('exclaim2', \
ExclamationBolt, par=2, \
inputs={exclaim1: Grouping.SHUFFLE})
builder.build_and_submit()
"""
Example empty topology. See Heron docs on building python topologies.
"""
from heronpy.api.topology import TopologyBuilder
builder = TopologyBuilder("MyTopology")
# TODO: add spouts and bolts
# this is the what produces output about the topology
builder.build_and_submit()
