def heron_pex(topology_pex, topology_class_name, args=None):
Log.debug("Importing %s from %s", topology_class_name, topology_pex)
if topology_class_name == '-':
# loading topology by running its main method (if __name__ == "__main__")
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
cmd = [topology_pex]
if args is not None:
cmd.extend(args)
Log.debug("Invoking class using command: ``%s''", ' '.join(cmd))
Log.debug('Heron options: {%s}', str(heron_env['HERON_OPTIONS']))
# invoke the command with subprocess and print error message, if any
proc = subprocess.Popen(cmd, env=heron_env, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, bufsize=1)
# todo(rli): improve python topology submission workflow
return ProcessResult(proc)
else:
try:
# loading topology from Topology's subclass (no main method)
# to support specifying the name of topology
Log.debug("args: %s", args)
if args is not None and isinstance(args, (list, tuple)) and len(args) > 0:
opts.set_config('cmdline.topology.name', args[0])
os.environ["HERON_OPTIONS"] = opts.get_heron_config()
Log.debug("Heron options: {%s}", os.environ["HERON_OPTIONS"])
pex_loader.load_pex(topology_pex)
topology_class = pex_loader.import_and_get_class(topology_pex, topology_class_name)
topology_class.write()
return SimpleResult(Status.Ok)
except Exception as ex:
Log.debug(traceback.format_exc())
err_context = "Topology %s failed to be loaded from the given pex: %s" %\
(topology_class_name, ex)
return SimpleResult(Status.HeronError, err_context)
def heron_pex(topology_pex, topology_class_name):
Log.debug("Importing %s from %s" % (topology_class_name, topology_pex))
if topology_class_name == '-':
# loading topology by running its main method (if __name__ == "__main__")
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
cmd = [topology_pex]
Log.debug('$> %s' % cmd[0])
Log.debug('Heron options: %s' % str(heron_env['HERON_OPTIONS']))
# invoke the command with subprocess and print error message, if any
status = subprocess.call(cmd, env=heron_env)
if status != 0:
err_str = "Topology failed to be loaded from the given pex, with status: %d. Bailing out..." \
% status
raise RuntimeError(err_str)
else:
try:
# loading topology from Topology's subclass (no main method)
os.environ["HERON_OPTIONS"] = opts.get_heron_config()
pex_loader.load_pex(topology_pex)
topology_class = pex_loader.import_and_get_class(topology_pex, topology_class_name)
topology_class.write()
except Exception:
Log.debug(traceback.format_exc())
err_str = "Topology failed to be loaded from the given pex. Bailing out..."
raise RuntimeError(err_str)
def _setup_custom_grouping(self, topology):
"""Checks whether there are any bolts that consume any of my streams using custom grouping"""
for i in range(len(topology.bolts)):
for in_stream in topology.bolts[i].inputs:
if in_stream.stream.component_name == self.my_component_name and \
in_stream.gtype == topology_pb2.Grouping.Value("CUSTOM"):
# this bolt takes my output in custom grouping manner
if in_stream.type == topology_pb2.CustomGroupingObjectType.Value(
"PYTHON_OBJECT"):
custom_grouping_obj = default_serializer.deserialize(
in_stream.custom_grouping_object)
if isinstance(custom_grouping_obj, str):
pex_loader.load_pex(self.topology_pex_abs_path)
grouping_cls = \
pex_loader.import_and_get_class(self.topology_pex_abs_path, custom_grouping_obj)
custom_grouping_obj = grouping_cls()
assert isinstance(custom_grouping_obj, ICustomGrouping)
self.custom_grouper.add(
in_stream.stream.id,
self._get_taskids_for_component(
topology.bolts[i].comp.name),
custom_grouping_obj, self.my_component_name)
elif in_stream.type == topology_pb2.CustomGroupingObjectType.Value(
"JAVA_OBJECT"):
raise NotImplementedError(
"Java-serialized custom grouping is not yet supported "
"for python topology")
else:
raise ValueError(
"Unrecognized custom grouping type found: %s" %
str(in_stream.type))
def heron_pex(topology_pex, topology_class_name, args=None):
Log.debug("Importing %s from %s", topology_class_name, topology_pex)
if topology_class_name == '-':
# loading topology by running its main method (if __name__ == "__main__")
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
cmd = [topology_pex]
if args is not None:
cmd.extend(args)
Log.debug("Invoking class using command: ``%s''", ' '.join(cmd))
Log.debug('Heron options: {%s}', str(heron_env['HERON_OPTIONS']))
# invoke the command with subprocess and print error message, if any
process = subprocess.Popen(cmd, env=heron_env, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, bufsize=1)
# todo(rli): improve python topology submission workflow
return ProcessResult(process)
else:
try:
# loading topology from Topology's subclass (no main method)
# to support specifying the name of topology
Log.debug("args: %s", args)
if args is not None and isinstance(args, (list, tuple)) and len(args) > 0:
opts.set_config('cmdline.topology.name', args[0])
os.environ["HERON_OPTIONS"] = opts.get_heron_config()
Log.debug("Heron options: {%s}", os.environ["HERON_OPTIONS"])
pex_loader.load_pex(topology_pex)
topology_class = pex_loader.import_and_get_class(topology_pex, topology_class_name)
topology_class.write()
return SimpleResult(Status.Ok)
except Exception as ex:
Log.debug(traceback.format_exc())
err_context = "Topology %s failed to be loaded from the given pex: %s" %\
(topology_class_name, ex)
return SimpleResult(Status.HeronError, err_context)
def _setup_custom_grouping(self, topology):
"""Checks whether there are any bolts that consume any of my streams using custom grouping"""
for i in range(len(topology.bolts)):
for in_stream in topology.bolts[i].inputs:
if in_stream.stream.component_name == self.my_component_name and \
in_stream.gtype == topology_pb2.Grouping.Value("CUSTOM"):
# this bolt takes my output in custom grouping manner
if in_stream.type == topology_pb2.CustomGroupingObjectType.Value("PYTHON_OBJECT"):
custom_grouping_obj = default_serializer.deserialize(in_stream.custom_grouping_object)
if isinstance(custom_grouping_obj, str):
pex_loader.load_pex(self.topology_pex_abs_path)
grouping_cls = \
pex_loader.import_and_get_class(self.topology_pex_abs_path, custom_grouping_obj)
custom_grouping_obj = grouping_cls()
assert isinstance(custom_grouping_obj, ICustomGrouping)
self.custom_grouper.add(in_stream.stream.id,
self._get_taskids_for_component(topology.bolts[i].comp.name),
custom_grouping_obj,
self.my_component_name)
elif in_stream.type == topology_pb2.CustomGroupingObjectType.Value("JAVA_OBJECT"):
raise NotImplementedError("Java-serialized custom grouping is not yet supported "
"for python topology")
else:
raise ValueError("Unrecognized custom grouping type found: %s" % str(in_stream.type))
def test_load_pex(self):
# Testing load_pex without including deps (including deps requires an actual zip file)
test_path = ['sample.pex', 'sample_123.pex', '/tmp/path.pex']
for path in test_path:
pex_loader.load_pex(path, include_deps=False)
abs_path = os.path.abspath(path)
self.assertIn(abs_path, sys.path)
def test_sample(self): path = self.get_path_of_sample(constants.SAMPLE_PEX) print path pex_loader.load_pex(path) cls = pex_loader.import_and_get_class(path, constants.SAMPLE_PEX_CLASSPATH) self.assertIsNotNone(cls) self.assertEqual(cls.name, "sample class") self.assertEqual(cls.age, 100)
def heron_pex(topology_pex, topology_class_name, tmp_dir):
Log.debug("Importing %s from %s" % (topology_class_name, topology_pex))
try:
pex_loader.load_pex(topology_pex)
topology_class = pex_loader.import_and_get_class(topology_pex, topology_class_name)
topology_class.write(tmp_dir)
except Exception:
traceback.print_exc()
err_str = "Topology pex failed to be loaded. Bailing out..."
raise RuntimeError(err_str)
def get_serializer(context):
"""Returns a serializer for a given context"""
cluster_config = context.get_cluster_config()
serializer_clsname = cluster_config.get(constants.TOPOLOGY_SERIALIZER_CLASSNAME, None)
if serializer_clsname is None:
return PythonSerializer()
else:
try:
topo_pex_path = context.get_topology_pex_path()
pex_loader.load_pex(topo_pex_path)
serializer_cls = pex_loader.import_and_get_class(topo_pex_path, serializer_clsname)
serializer = serializer_cls()
return serializer
except Exception as e:
raise RuntimeError("Error with loading custom serializer class: %s, with error message: %s"
% (serializer_clsname, e.message))
def get_serializer(context):
"""Returns a serializer for a given context"""
cluster_config = context.get_cluster_config()
serializer_clsname = cluster_config.get(constants.TOPOLOGY_SERIALIZER_CLASSNAME, None)
if serializer_clsname is None:
return PythonSerializer()
else:
try:
topo_pex_path = context.get_topology_pex_path()
pex_loader.load_pex(topo_pex_path)
serializer_cls = pex_loader.import_and_get_class(topo_pex_path, serializer_clsname)
serializer = serializer_cls()
return serializer
except Exception as e:
raise RuntimeError("Error with loading custom serializer class: %s, with error message: %s"
% (serializer_clsname, str(e)))
def _init_task_hooks(self):
task_hooks_cls_list = self.get_cluster_config().get(constants.TOPOLOGY_AUTO_TASK_HOOKS, None)
if task_hooks_cls_list is None:
return
# load pex first
topo_pex_path = self.get_topology_pex_path()
pex_loader.load_pex(topo_pex_path)
for class_name in task_hooks_cls_list:
try:
task_hook_cls = pex_loader.import_and_get_class(topo_pex_path, class_name)
task_hook_instance = task_hook_cls()
assert isinstance(task_hook_instance, ITaskHook)
self[self.TASK_HOOKS].append(task_hook_instance)
except AssertionError:
raise RuntimeError("Auto-registered task hook not instance of ITaskHook")
except Exception as e:
raise RuntimeError("Error with loading task hook class: %s, with error message: %s"
% (class_name, e.message))
def load_py_instance(self, is_spout):
"""Loads user defined component (spout/bolt)"""
try:
if is_spout:
spout_proto = self.pplan_helper.get_my_spout()
py_classpath = spout_proto.comp.class_name
self.logger.info("Loading Spout from: %s", py_classpath)
else:
bolt_proto = self.pplan_helper.get_my_bolt()
py_classpath = bolt_proto.comp.class_name
self.logger.info("Loading Bolt from: %s", py_classpath)
pex_loader.load_pex(self.pplan_helper.topology_pex_abs_path)
spbl_class = pex_loader.import_and_get_class(self.pplan_helper.topology_pex_abs_path,
py_classpath)
except Exception as e:
spbl = "spout" if is_spout else "bolt"
self.logger.error(traceback.format_exc())
raise RuntimeError("Error when loading a %s from pex: %s" % (spbl, str(e)))
return spbl_class
def load_py_instance(self, is_spout):
"""Loads user defined component (spout/bolt)"""
try:
if is_spout:
spout_proto = self.pplan_helper.get_my_spout()
py_classpath = spout_proto.comp.class_name
self.logger.info("Loading Spout from: %s", py_classpath)
else:
bolt_proto = self.pplan_helper.get_my_bolt()
py_classpath = bolt_proto.comp.class_name
self.logger.info("Loading Bolt from: %s", py_classpath)
pex_loader.load_pex(self.pplan_helper.topology_pex_abs_path)
spbl_class = pex_loader.import_and_get_class(self.pplan_helper.topology_pex_abs_path,
py_classpath)
except Exception as e:
spbl = "spout" if is_spout else "bolt"
self.logger.error(traceback.format_exc())
raise RuntimeError("Error when loading a %s from pex: %s" % (spbl, str(e)))
return spbl_class
def _init_task_hooks(self):
task_hooks_cls_list = self.get_cluster_config().get(api_constants.TOPOLOGY_AUTO_TASK_HOOKS,
None)
if task_hooks_cls_list is None:
return
# load pex first
topo_pex_path = self.get_topology_pex_path()
pex_loader.load_pex(topo_pex_path)
for class_name in task_hooks_cls_list:
try:
task_hook_cls = pex_loader.import_and_get_class(topo_pex_path, class_name)
task_hook_instance = task_hook_cls()
assert isinstance(task_hook_instance, ITaskHook)
self.task_hooks.append(task_hook_instance)
except AssertionError:
raise RuntimeError("Auto-registered task hook not instance of ITaskHook")
except Exception as e:
raise RuntimeError("Error with loading task hook class: %s, with error message: %s"
% (class_name, e.message))
def heron_pex(topology_pex, topology_class_name, args=None):
Log.debug("Importing %s from %s" % (topology_class_name, topology_pex))
if topology_class_name == '-':
# loading topology by running its main method (if __name__ == "__main__")
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
cmd = [topology_pex]
if args is not None:
cmd.extend(args)
Log.debug('$> %s' % ' '.join(cmd))
Log.debug('Heron options: %s' % str(heron_env['HERON_OPTIONS']))
# invoke the command with subprocess and print error message, if any
status = subprocess.call(cmd, env=heron_env)
if status != 0:
err_str = "Topology failed to be loaded from the given pex, with status: %d. Bailing out..." \
% status
raise RuntimeError(err_str)
else:
try:
# loading topology from Topology's subclass (no main method)
# to support specifying the name of topology
Log.debug("args: %s" % args)
if args is not None and isinstance(
args, (list, tuple)) and len(args) > 0:
opts.set_config('cmdline.topology.name', args[0])
os.environ["HERON_OPTIONS"] = opts.get_heron_config()
Log.debug("Heron options: %s" % os.environ["HERON_OPTIONS"])
pex_loader.load_pex(topology_pex)
topology_class = pex_loader.import_and_get_class(
topology_pex, topology_class_name)
topology_class.write()
except Exception:
Log.debug(traceback.format_exc())
err_str = "Topology failed to be loaded from the given pex. Bailing out..."
raise RuntimeError(err_str)
def heron_pex(topology_pex, topology_class_name, args=None):
Log.debug("Importing %s from %s" % (topology_class_name, topology_pex))
if topology_class_name == '-':
# loading topology by running its main method (if __name__ == "__main__")
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
cmd = [topology_pex]
if args is not None:
cmd.extend(args)
Log.debug('$> %s' % ' '.join(cmd))
Log.debug('Heron options: %s' % str(heron_env['HERON_OPTIONS']))
# invoke the command with subprocess and print error message, if any
status = subprocess.call(cmd, env=heron_env)
if status != 0:
err_str = "Topology failed to be loaded from the given pex, with status: %d. Bailing out..." \
% status
raise RuntimeError(err_str)
else:
try:
# loading topology from Topology's subclass (no main method)
# to support specifying the name of topology
Log.debug("args: %s" % args)
if args is not None and isinstance(args, (list, tuple)) and len(args) > 0:
opts.set_config('cmdline.topology.name', args[0])
os.environ["HERON_OPTIONS"] = opts.get_heron_config()
Log.debug("Heron options: %s" % os.environ["HERON_OPTIONS"])
pex_loader.load_pex(topology_pex)
topology_class = pex_loader.import_and_get_class(topology_pex, topology_class_name)
topology_class.write()
except Exception:
Log.debug(traceback.format_exc())
err_str = "Topology failed to be loaded from the given pex. Bailing out..."
raise RuntimeError(err_str)
def _load_user_spout(pex_file, classpath):
pex_loader.load_pex(pex_file)
cls = pex_loader.import_and_get_class(pex_file, classpath)
return cls
def _load_user_bolt(pex_file, classpath): pex_loader.load_pex(pex_file) cls = pex_loader.import_and_get_class(pex_file, classpath) return cls