def heron_pex(topology_pex, topology_class_name, args=None):
Log.debug("Importing %s from %s", topology_class_name, topology_pex)
if topology_class_name == '-':
# loading topology by running its main method (if __name__ == "__main__")
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
cmd = [topology_pex]
if args is not None:
cmd.extend(args)
Log.debug("Invoking class using command: ``%s''", ' '.join(cmd))
Log.debug('Heron options: {%s}', str(heron_env['HERON_OPTIONS']))
# invoke the command with subprocess and print error message, if any
proc = subprocess.Popen(cmd, env=heron_env, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, bufsize=1)
# todo(rli): improve python topology submission workflow
return ProcessResult(proc)
else:
try:
# loading topology from Topology's subclass (no main method)
# to support specifying the name of topology
Log.debug("args: %s", args)
if args is not None and isinstance(args, (list, tuple)) and len(args) > 0:
opts.set_config('cmdline.topology.name', args[0])
os.environ["HERON_OPTIONS"] = opts.get_heron_config()
Log.debug("Heron options: {%s}", os.environ["HERON_OPTIONS"])
pex_loader.load_pex(topology_pex)
topology_class = pex_loader.import_and_get_class(topology_pex, topology_class_name)
topology_class.write()
return SimpleResult(Status.Ok)
except Exception as ex:
Log.debug(traceback.format_exc())
err_context = "Topology %s failed to be loaded from the given pex: %s" %\
(topology_class_name, ex)
return SimpleResult(Status.HeronError, err_context)
def heron_pex(topology_pex, topology_class_name, args=None):
Log.debug("Importing %s from %s", topology_class_name, topology_pex)
if topology_class_name == '-':
# loading topology by running its main method (if __name__ == "__main__")
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
cmd = [topology_pex]
if args is not None:
cmd.extend(args)
Log.debug("Invoking class using command: ``%s''", ' '.join(cmd))
Log.debug('Heron options: {%s}', str(heron_env['HERON_OPTIONS']))
# invoke the command with subprocess and print error message, if any
process = subprocess.Popen(cmd, env=heron_env, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, bufsize=1)
# todo(rli): improve python topology submission workflow
return ProcessResult(process)
else:
try:
# loading topology from Topology's subclass (no main method)
# to support specifying the name of topology
Log.debug("args: %s", args)
if args is not None and isinstance(args, (list, tuple)) and len(args) > 0:
opts.set_config('cmdline.topology.name', args[0])
os.environ["HERON_OPTIONS"] = opts.get_heron_config()
Log.debug("Heron options: {%s}", os.environ["HERON_OPTIONS"])
pex_loader.load_pex(topology_pex)
topology_class = pex_loader.import_and_get_class(topology_pex, topology_class_name)
topology_class.write()
return SimpleResult(Status.Ok)
except Exception as ex:
Log.debug(traceback.format_exc())
err_context = "Topology %s failed to be loaded from the given pex: %s" %\
(topology_class_name, ex)
return SimpleResult(Status.HeronError, err_context)
def heron_class(class_name,
lib_jars,
extra_jars=None,
args=None,
java_defines=None):
'''
Execute a heron class given the args and the jars needed for class path
:param class_name:
:param lib_jars:
:param extra_jars:
:param args:
:param java_defines:
:return:
'''
# default optional params to empty list if not provided
if extra_jars is None:
extra_jars = []
if args is None:
args = []
if java_defines is None:
java_defines = []
# Format all java -D options that need to be passed while running
# the class locally.
java_opts = ['-D' + opt for opt in java_defines]
java_path = config.get_java_path()
if java_path is None:
err_context = "Unable to find java command"
return SimpleResult(Status.InvocationError, err_context)
# Construct the command line for the sub process to run
# Because of the way Python execute works,
# the java opts must be passed as part of the list
all_args = [java_path, "-client", "-Xmx1g"] + \
java_opts + \
["-cp", config.get_classpath(extra_jars + lib_jars)]
all_args += [class_name] + list(args)
# set heron_config environment variable
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
# print the verbose message
Log.debug("Invoking class using command: `%s`",
' '.join(shlex.quote(a) for a in all_args))
Log.debug("Heron options: {%s}", str(heron_env["HERON_OPTIONS"]))
# invoke the command with subprocess and print error message, if any
# pylint: disable=consider-using-with
process = subprocess.Popen(all_args,
env=heron_env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
bufsize=1)
# stdout message has the information Java program sends back
# stderr message has extra information, such as debugging message
return ProcessResult(process)
def heron_class(class_name,
lib_jars,
extra_jars=None,
args=None,
java_defines=None):
'''
Execute a heron class given the args and the jars needed for class path
:param class_name:
:param lib_jars:
:param extra_jars:
:param args:
:param java_defines:
:return:
'''
# default optional params to empty list if not provided
if extra_jars is None:
extra_jars = []
if args is None:
args = []
if java_defines is None:
java_defines = []
# Format all java -D options that need to be passed while running
# the class locally.
java_opts = ['-D' + opt for opt in java_defines]
# Construct the command line for the sub process to run
# Because of the way Python execute works,
# the java opts must be passed as part of the list
all_args = [config.get_java_path(), "-client", "-Xmx1g"] + \
java_opts + \
["-cp", config.get_classpath(extra_jars + lib_jars)]
all_args += [class_name] + list(args)
# set heron_config environment variable
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
# print the verbose message
Log.debug("Invoking class using command: ``%s''", ' '.join(all_args))
Log.debug("Heron options: {%s}", str(heron_env["HERON_OPTIONS"]))
# invoke the command with subprocess and print error message, if any
proc = subprocess.Popen(all_args,
env=heron_env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# stdout message has the information Java program sends back
# stderr message has extra information, such as debugging message
msg, detailed_msg = proc.communicate()
# remove trailing newlines
if msg and msg[-1] == '\n':
msg = msg[:-1]
if detailed_msg and detailed_msg[-1] == '\n':
detailed_msg = detailed_msg[:-1]
Log.debug("shelled-out program's return code: %d", proc.returncode)
return Response(proc.returncode, msg, detailed_msg)
def heron_pex(topology_pex, topology_class_name, args=None):
Log.debug("Importing %s from %s" % (topology_class_name, topology_pex))
if topology_class_name == '-':
# loading topology by running its main method (if __name__ == "__main__")
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
cmd = [topology_pex]
if args is not None:
cmd.extend(args)
Log.debug('$> %s' % ' '.join(cmd))
Log.debug('Heron options: %s' % str(heron_env['HERON_OPTIONS']))
# invoke the command with subprocess and print error message, if any
status = subprocess.call(cmd, env=heron_env)
if status != 0:
err_str = "Topology failed to be loaded from the given pex, with status: %d. Bailing out..." \
% status
raise RuntimeError(err_str)
else:
try:
# loading topology from Topology's subclass (no main method)
# to support specifying the name of topology
Log.debug("args: %s" % args)
if args is not None and isinstance(
args, (list, tuple)) and len(args) > 0:
opts.set_config('cmdline.topology.name', args[0])
os.environ["HERON_OPTIONS"] = opts.get_heron_config()
Log.debug("Heron options: %s" % os.environ["HERON_OPTIONS"])
pex_loader.load_pex(topology_pex)
topology_class = pex_loader.import_and_get_class(
topology_pex, topology_class_name)
topology_class.write()
except Exception:
Log.debug(traceback.format_exc())
err_str = "Topology failed to be loaded from the given pex. Bailing out..."
raise RuntimeError(err_str)
def heron_class(class_name, lib_jars, extra_jars=None, args=None, java_defines=None):
'''
Execute a heron class given the args and the jars needed for class path
:param class_name:
:param lib_jars:
:param extra_jars:
:param args:
:param java_defines:
:return:
'''
# default optional params to empty list if not provided
if extra_jars is None:
extra_jars = []
if args is None:
args = []
if java_defines is None:
java_defines = []
# Format all java -D options that need to be passed while running
# the class locally.
java_opts = ['-D' + opt for opt in java_defines]
# Construct the command line for the sub process to run
# Because of the way Python execute works,
# the java opts must be passed as part of the list
all_args = [config.get_java_path(), "-client", "-Xmx1g"] + \
java_opts + \
["-cp", config.get_classpath(extra_jars + lib_jars)]
all_args += [class_name] + list(args)
# set heron_config environment variable
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
# print the verbose message
Log.debug("Invoking class using command: ``%s''", ' '.join(all_args))
Log.debug("Heron options: {%s}", str(heron_env["HERON_OPTIONS"]))
# invoke the command with subprocess and print error message, if any
proc = subprocess.Popen(all_args, env=heron_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# stdout message has the information Java program sends back
# stderr message has extra information, such as debugging message
msg, detailed_msg = proc.communicate()
# remove trailing newlines
if msg and msg[-1] == '\n':
msg = msg[:-1]
if detailed_msg and detailed_msg[-1] == '\n':
detailed_msg = detailed_msg[:-1]
Log.debug("shelled-out program's return code: %d", proc.returncode)
return Response(proc.returncode, msg, detailed_msg)
def heron_pex(topology_pex, topology_class_name, args=None):
Log.debug("Importing %s from %s" % (topology_class_name, topology_pex))
if topology_class_name == '-':
# loading topology by running its main method (if __name__ == "__main__")
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
cmd = [topology_pex]
if args is not None:
cmd.extend(args)
Log.debug('$> %s' % ' '.join(cmd))
Log.debug('Heron options: %s' % str(heron_env['HERON_OPTIONS']))
# invoke the command with subprocess and print error message, if any
status = subprocess.call(cmd, env=heron_env)
if status != 0:
err_str = "Topology failed to be loaded from the given pex, with status: %d. Bailing out..." \
% status
raise RuntimeError(err_str)
else:
try:
# loading topology from Topology's subclass (no main method)
# to support specifying the name of topology
Log.debug("args: %s" % args)
if args is not None and isinstance(args, (list, tuple)) and len(args) > 0:
opts.set_config('cmdline.topology.name', args[0])
os.environ["HERON_OPTIONS"] = opts.get_heron_config()
Log.debug("Heron options: %s" % os.environ["HERON_OPTIONS"])
pex_loader.load_pex(topology_pex)
topology_class = pex_loader.import_and_get_class(topology_pex, topology_class_name)
topology_class.write()
except Exception:
Log.debug(traceback.format_exc())
err_str = "Topology failed to be loaded from the given pex. Bailing out..."
raise RuntimeError(err_str)
def heron_cpp(topology_binary, args=None):
Log.debug("Executing %s", topology_binary)
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
cmd = [topology_binary]
if args is not None:
cmd.extend(args)
Log.debug("Invoking binary using command: ``%s''", ' '.join(cmd))
Log.debug('Heron options: {%s}', str(heron_env['HERON_OPTIONS']))
print("Invoking class using command: ``%s''" % ' '.join(cmd))
print('Heron options: {%s}' % str(heron_env['HERON_OPTIONS']))
# invoke the command with subprocess and print error message, if any
proc = subprocess.Popen(cmd, env=heron_env, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, bufsize=1)
return ProcessResult(proc)
def heron_class(class_name,
lib_jars,
extra_jars=None,
args=None,
java_defines=None):
'''
Execute a heron class given the args and the jars needed for class path
:param class_name:
:param lib_jars:
:param extra_jars:
:param args:
:param java_defines:
:return:
'''
# default optional params to empty list if not provided
if extra_jars is None:
extra_jars = []
if args is None:
args = []
if java_defines is None:
java_defines = []
# Format all java -D options that need to be passed while running
# the class locally.
java_opts = ['-D' + opt for opt in java_defines]
# Construct the command line for the sub process to run
# Because of the way Python execute works,
# the java opts must be passed as part of the list
all_args = [config.get_java_path(), "-client", "-Xmx1g"] + \
java_opts + \
["-cp", config.get_classpath(lib_jars + extra_jars)]
all_args += [class_name] + list(args)
# set heron_config environment variable
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
# print the verbose message
Log.debug('$> %s' % ' '.join(all_args))
Log.debug('Heron options: %s' % str(heron_env["HERON_OPTIONS"]))
# invoke the command with subprocess and print error message, if any
status = subprocess.call(all_args, env=heron_env)
if status != 0:
err_str = "User main failed with status %d. Bailing out..." % status
raise RuntimeError(err_str)
def heron_class(class_name, lib_jars, extra_jars=None, args=None, java_defines=None):
'''
Execute a heron class given the args and the jars needed for class path
:param class_name:
:param lib_jars:
:param extra_jars:
:param args:
:param java_defines:
:return:
'''
# default optional params to empty list if not provided
if extra_jars is None:
extra_jars = []
if args is None:
args = []
if java_defines is None:
java_defines = []
# Format all java -D options that need to be passed while running
# the class locally.
java_opts = ['-D' + opt for opt in java_defines]
# Construct the command line for the sub process to run
# Because of the way Python execute works,
# the java opts must be passed as part of the list
all_args = [config.get_java_path(), "-client", "-Xmx1g"] + \
java_opts + \
["-cp", config.get_classpath(lib_jars + extra_jars)]
all_args += [class_name] + list(args)
# set heron_config environment variable
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
# print the verbose message
Log.debug('$> %s' % ' '.join(all_args))
Log.debug('Heron options: %s' % str(heron_env["HERON_OPTIONS"]))
# invoke the command with subprocess and print error message, if any
status = subprocess.call(all_args, env=heron_env)
if status != 0:
err_str = "User main failed with status %d. Bailing out..." % status
raise RuntimeError(err_str)
def heron_cpp(topology_binary, args=None):
Log.debug("Executing %s", topology_binary)
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
cmd = [topology_binary]
if args is not None:
cmd.extend(args)
Log.debug("Invoking binary using command: ``%s''", ' '.join(cmd))
Log.debug('Heron options: {%s}', str(heron_env['HERON_OPTIONS']))
print(f"""Invoking class using command: ``{' '.join(cmd)}''""")
print(f"Heron options: {str(heron_env['HERON_OPTIONS'])}")
# invoke the command with subprocess and print error message, if any
# pylint: disable=consider-using-with
proc = subprocess.Popen(cmd,
env=heron_env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
bufsize=1)
return ProcessResult(proc)
