def launch_topologies(cl_args, topology_file, tmp_dir):
'''
Launch topologies
:param cl_args:
:param topology_file:
:param tmp_dir:
:return: list(Responses)
'''
# the submitter would have written the .defn file to the tmp_dir
defn_files = glob.glob(tmp_dir + '/*.defn')
if len(defn_files) == 0:
return SimpleResult(Status.HeronError,
"No topologies found under %s" % tmp_dir)
results = []
for defn_file in defn_files:
# load the topology definition from the file
topology_defn = topology_pb2.Topology()
try:
handle = open(defn_file, "rb")
topology_defn.ParseFromString(handle.read())
handle.close()
except Exception as e:
err_context = "Cannot load topology definition '%s': %s" % (
defn_file, e)
return SimpleResult(Status.HeronError, err_context)
# launch the topology
mode = " in dry-run mode" if cl_args['dry_run'] else ''
Log.info("Launching topology: \'%s\'%s", topology_defn.name, mode)
res = launch_a_topology(cl_args, tmp_dir, topology_file, defn_file,
topology_defn.name)
results.append(res)
return results
def create_mock_simple_topology(self,
spout_parallelism=1,
bolt_parallelism=1):
"""
Simple topology contains one spout and one bolt.
"""
topology = protoTopology.Topology()
topology.id = MockProto.topology_id
topology.name = MockProto.topology_name
# Stream1
stream1 = protoTopology.StreamId()
stream1.id = "mock_stream1"
stream1.component_name = "mock_spout"
# Spout1
spout = self.create_mock_spout("mock_spout", [stream1],
spout_parallelism)
topology.spouts.extend([spout])
# Bolt1
bolt = self.create_mock_bolt("mock_bolt", [stream1], [],
bolt_parallelism)
topology.bolts.extend([bolt])
return topology
def launch_topologies(cl_args, topology_file, tmp_dir):
# the submitter would have written the .defn file to the tmp_dir
defn_files = glob.glob(tmp_dir + '/*.defn')
if len(defn_files) == 0:
raise Exception("No topologies found")
try:
for defn_file in defn_files:
# load the topology definition from the file
topology_defn = topology_pb2.Topology()
try:
f = open(defn_file, "rb")
topology_defn.ParseFromString(f.read())
f.close()
except:
raise Exception("Could not open and parse topology defn file %s" % defn_file)
# launch the topology
try:
Log.info("Launching topology \'%s\'" % topology_defn.name)
launch_a_topology(cl_args, tmp_dir, topology_file, defn_file)
Log.info("Topology \'%s\' launched successfully" % topology_defn.name)
except Exception as ex:
Log.error('Failed to launch topology \'%s\' because %s' % (topology_defn.name, str(ex)))
raise
except:
raise
def launch_topologies(cl_args, topology_file, tmp_dir):
'''
Launch topologies
:param cl_args:
:param topology_file:
:param tmp_dir:
:return: list(Responses)
'''
# the submitter would have written the .defn file to the tmp_dir
defn_files = glob.glob(tmp_dir + '/*.defn')
if len(defn_files) == 0:
return Response(Status.HeronError,
"No topologies found under %s" % tmp_dir)
responses = []
for defn_file in defn_files:
# load the topology definition from the file
topology_defn = topology_pb2.Topology()
try:
handle = open(defn_file, "rb")
topology_defn.ParseFromString(handle.read())
handle.close()
except Exception as e:
msg = "Cannot load topology definition '%s'" % defn_file
return Response(Status.HeronError, msg, str(e))
# launch the topology
Log.info("Launching topology: \'%s\'", topology_defn.name)
resp = launch_a_topology(cl_args, tmp_dir, topology_file, defn_file,
topology_defn.name)
responses.append(resp)
return responses
def init_topology(mcs, classname, class_dict):
"""Initializes a topology protobuf"""
if classname == 'Topology':
# Base class can't initialize protobuf
return
heron_options = TopologyType.get_heron_options_from_env()
initial_state = heron_options.get("cmdline.topology.initial.state",
"RUNNING")
tmp_directory = heron_options.get("cmdline.topologydefn.tmpdirectory",
None)
if tmp_directory is None:
raise RuntimeError(
"Topology definition temp directory not specified")
topology_name = heron_options.get("cmdline.topology.name", classname)
topology_id = topology_name + str(uuid.uuid4())
# create protobuf
topology = topology_pb2.Topology()
topology.id = topology_id
topology.name = topology_name
topology.state = topology_pb2.TopologyState.Value(initial_state)
topology.topology_config.CopyFrom(
TopologyType.get_topology_config_protobuf(class_dict))
TopologyType.add_bolts_and_spouts(topology, class_dict)
class_dict['topology_name'] = topology_name
class_dict['topology_id'] = topology_id
class_dict['protobuf_topology'] = topology
class_dict['topologydefn_tmpdir'] = tmp_directory
class_dict['heron_runtime_options'] = heron_options
def create_mock_medium_topology(
self,
spout_parallelism=1,
bolt1_parallelism=1,
bolt2_parallelism=1,
bolt3_parallelism=1):
"""
Medium topology is a three stage topology
with one spout, two mid stage bolts, and one
last stage bolt.
S -str1-> B1 -str3-> B3
S -str2-> B2 -str4-> B3
"""
topology = protoTopology.Topology()
topology.id = "mock_topology_id"
topology.name = "mock_topology_name"
# Streams
stream1 = protoTopology.StreamId()
stream1.id = "mock_stream1"
stream1.component_name = "mock_spout1"
stream2 = protoTopology.StreamId()
stream2.id = "mock_stream2"
stream2.component_name = "mock_spout1"
stream3 = protoTopology.StreamId()
stream3.id = "mock_stream3"
stream3.component_name = "mock_bolt1"
stream4 = protoTopology.StreamId()
stream4.id = "mock_stream4"
stream4.component_name = "mock_bolt2"
# Spouts
spout1 = self.create_mock_spout("mock_spout1",
[stream1, stream2],
spout_parallelism)
topology.spouts.extend([spout1])
# Bolts
bolt1 = self.create_mock_bolt("mock_bolt1",
[stream1],
[stream3],
bolt1_parallelism)
bolt2 = self.create_mock_bolt("mock_bolt2",
[stream2],
[stream4],
bolt2_parallelism)
bolt3 = self.create_mock_bolt("mock_bolt3",
[stream3, stream4],
[],
bolt3_parallelism)
topology.bolts.extend([bolt1, bolt2, bolt3])
return topology
def launch_topologies(cl_args, topology_file, tmp_dir):
'''
Launch topologies
:param cl_args:
:param topology_file:
:param tmp_dir:
:return: list(Responses)
'''
# the submitter would have written the .defn file to the tmp_dir
defn_files = glob.glob(tmp_dir + '/*.defn')
if len(defn_files) == 0:
return SimpleResult(Status.HeronError,
"No topologies found under %s" % tmp_dir)
results = []
for defn_file in defn_files:
# load the topology definition from the file
topology_defn = topology_pb2.Topology()
try:
handle = open(defn_file, "rb")
topology_defn.ParseFromString(handle.read())
handle.close()
except Exception as e:
err_context = "Cannot load topology definition '%s': %s" % (
defn_file, e)
return SimpleResult(Status.HeronError, err_context)
# log topology and components configurations
Log.debug("Topology config: %s", topology_defn.topology_config)
Log.debug("Component config:")
for spout in topology_defn.spouts:
Log.debug("%s => %s", spout.comp.name, spout.comp.config)
for bolt in topology_defn.bolts:
Log.debug("%s => %s", bolt.comp.name, bolt.comp.config)
# launch the topology
Log.info("Launching topology: \'%s\'%s", topology_defn.name,
launch_mode_msg(cl_args))
# check if we have to do server or direct based deployment
if cl_args['deploy_mode'] == config.SERVER_MODE:
res = launch_topology_server(cl_args, topology_file, defn_file,
topology_defn.name)
else:
res = launch_a_topology(cl_args, tmp_dir, topology_file, defn_file,
topology_defn.name)
results.append(res)
return results
def strip_java_objects(topology):
"""
The argument is the proto object for topology.
The java objects are huge objects and topology
is stripped off these objects so that it can be stored
in zookeeper.
"""
stripped_topology = topology_pb2.Topology()
stripped_topology.CopyFrom(topology)
components = list(stripped_topology.spouts) + list(stripped_topology.bolts)
for component in components:
if component.comp.HasField("serialized_object"):
component.comp.ClearField("serialized_object")
return stripped_topology
def get_mock_topology(id="topology_id", name="topology_name", state=1, spouts=[], bolts=[]):
"""Returns a mock protobuf Topology object from topology_pb2"""
# topology
topology = topology_pb2.Topology()
topology.id = id
topology.name = name
topology.state = state
for sp in spouts:
added = topology.spouts.add()
added.CopyFrom(sp)
for bl in bolts:
added = topology.bolts.add()
added.CopyFrom(bl)
return topology