def launch_topology_server(cl_args, topology_file, topology_defn_file,
topology_name):
'''
Launch a topology given topology jar, its definition file and configurations
:param cl_args:
:param topology_file:
:param topology_defn_file:
:param topology_name:
:return:
'''
service_apiurl = cl_args['service_url'] + rest.ROUTE_SIGNATURES['submit'][1]
service_method = rest.ROUTE_SIGNATURES['submit'][0]
data = dict(
name=topology_name,
cluster=cl_args['cluster'],
role=cl_args['role'],
environment=cl_args['environ'],
user=cl_args['submit_user'],
)
Log.info("" + str(cl_args))
overrides = dict()
if 'config_property' in cl_args:
overrides = config.parse_override_config(cl_args['config_property'])
if overrides:
data.update(overrides)
if cl_args['dry_run']:
data["dry_run"] = True
if cl_args['verbose_gc']:
data['verbose_gc'] = True
files = dict(
definition=open(topology_defn_file, 'rb'),
topology=open(topology_file, 'rb'),
)
err_ctxt = "Failed to launch topology '%s' %s" % (topology_name,
launch_mode_msg(cl_args))
succ_ctxt = "Successfully launched topology '%s' %s" % (
topology_name, launch_mode_msg(cl_args))
try:
r = service_method(service_apiurl, data=data, files=files)
ok = r.status_code is requests.codes.ok
created = r.status_code is requests.codes.created
s = Status.Ok if created or ok else Status.HeronError
if s is Status.HeronError:
Log.error(r.json().get(
'message', "Unknown error from API server %d" % r.status_code))
elif ok:
# this case happens when we request a dry_run
print(r.json().get("response"))
except (requests.exceptions.ConnectionError,
requests.exceptions.HTTPError) as err:
Log.error(err)
return SimpleResult(Status.HeronError, err_ctxt, succ_ctxt)
return SimpleResult(s, err_ctxt, succ_ctxt)
def run_server(command, cl_args, action, extra_args=dict()):
'''
helper function to take action on topologies using REST API
:param command:
:param cl_args:
:param action: description of action taken
:return:
'''
topology_name = cl_args['topology-name']
service_endpoint = cl_args['service_url']
apiroute = rest.ROUTE_SIGNATURES[command][1] % (
cl_args['cluster'], cl_args['role'], cl_args['environ'], topology_name)
service_apiurl = service_endpoint + apiroute
service_method = rest.ROUTE_SIGNATURES[command][0]
# convert the dictionary to a list of tuples
data = flatten_args(extra_args)
err_msg = "Failed to %s: %s" % (action, topology_name)
succ_msg = "Successfully %s: %s" % (action, topology_name)
try:
r = service_method(service_apiurl, data=data)
s = Status.Ok if r.status_code == requests.codes.ok else Status.HeronError
if r.status_code != requests.codes.ok:
Log.error(r.json().get(
'message', "Unknown error from API server %d" % r.status_code))
except (requests.exceptions.ConnectionError,
requests.exceptions.HTTPError) as err:
Log.error(err)
return SimpleResult(Status.HeronError, err_msg, succ_msg)
return SimpleResult(s, err_msg, succ_msg)
def launch_topologies(cl_args, topology_file, tmp_dir):
'''
Launch topologies
:param cl_args:
:param topology_file:
:param tmp_dir:
:return: list(Responses)
'''
# the submitter would have written the .defn file to the tmp_dir
defn_files = glob.glob(tmp_dir + '/*.defn')
if len(defn_files) == 0:
return SimpleResult(Status.HeronError,
"No topologies found under %s" % tmp_dir)
results = []
for defn_file in defn_files:
# load the topology definition from the file
topology_defn = topology_pb2.Topology()
try:
handle = open(defn_file, "rb")
topology_defn.ParseFromString(handle.read())
handle.close()
except Exception as e:
err_context = "Cannot load topology definition '%s': %s" % (
defn_file, e)
return SimpleResult(Status.HeronError, err_context)
# launch the topology
mode = " in dry-run mode" if cl_args['dry_run'] else ''
Log.info("Launching topology: \'%s\'%s", topology_defn.name, mode)
res = launch_a_topology(cl_args, tmp_dir, topology_file, defn_file,
topology_defn.name)
results.append(res)
return results
def heron_pex(topology_pex, topology_class_name, args=None):
Log.debug("Importing %s from %s", topology_class_name, topology_pex)
if topology_class_name == '-':
# loading topology by running its main method (if __name__ == "__main__")
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
cmd = [topology_pex]
if args is not None:
cmd.extend(args)
Log.debug("Invoking class using command: ``%s''", ' '.join(cmd))
Log.debug('Heron options: {%s}', str(heron_env['HERON_OPTIONS']))
# invoke the command with subprocess and print error message, if any
process = subprocess.Popen(cmd, env=heron_env, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, bufsize=1)
# todo(rli): improve python topology submission workflow
return ProcessResult(process)
else:
try:
# loading topology from Topology's subclass (no main method)
# to support specifying the name of topology
Log.debug("args: %s", args)
if args is not None and isinstance(args, (list, tuple)) and len(args) > 0:
opts.set_config('cmdline.topology.name', args[0])
os.environ["HERON_OPTIONS"] = opts.get_heron_config()
Log.debug("Heron options: {%s}", os.environ["HERON_OPTIONS"])
pex_loader.load_pex(topology_pex)
topology_class = pex_loader.import_and_get_class(topology_pex, topology_class_name)
topology_class.write()
return SimpleResult(Status.Ok)
except Exception as ex:
Log.debug(traceback.format_exc())
err_context = "Topology %s failed to be loaded from the given pex: %s" %\
(topology_class_name, ex)
return SimpleResult(Status.HeronError, err_context)
def run(command, parser, cl_args, unknown_args):
'''
:param command:
:param parser:
:param args:
:param unknown_args:
:return:
'''
cluster = cl_args['cluster']
# server mode
if cluster:
config_file = config.heron_rc_file()
client_confs = {}
# Read the cluster definition, if not found
client_confs = cdefs.read_server_mode_cluster_definition(
cluster, cl_args, config_file)
if not client_confs[cluster]:
Log.error(
'Neither service url nor %s cluster definition in %s file',
cluster, config_file)
return SimpleResult(Status.HeronError)
# if cluster definition exists, but service_url is not set, it is an error
if not 'service_url' in client_confs[cluster]:
Log.error('No service url for %s cluster in %s', cluster,
config_file)
sys.exit(1)
service_endpoint = cl_args['service_url']
service_apiurl = service_endpoint + rest.ROUTE_SIGNATURES[command][1]
service_method = rest.ROUTE_SIGNATURES[command][0]
try:
r = service_method(service_apiurl)
if r.status_code != requests.codes.ok:
Log.error(r.json().get(
'message',
f"Unknown error from API server {r.status_code}"))
sorted_items = sorted(list(r.json().items()),
key=lambda tup: tup[0])
for key, value in sorted_items:
print(f"{key} : {value}")
except (requests.exceptions.ConnectionError,
requests.exceptions.HTTPError) as err:
Log.error(err)
return SimpleResult(Status.HeronError)
else:
config.print_build_info()
return SimpleResult(Status.Ok)
def launch_topologies(cl_args, topology_file, tmp_dir):
'''
Launch topologies
:param cl_args:
:param topology_file:
:param tmp_dir:
:return: list(Responses)
'''
# the submitter would have written the .defn file to the tmp_dir
defn_files = glob.glob(tmp_dir + '/*.defn')
if len(defn_files) == 0:
return SimpleResult(Status.HeronError,
"No topologies found under %s" % tmp_dir)
results = []
for defn_file in defn_files:
# load the topology definition from the file
topology_defn = topology_pb2.Topology()
try:
handle = open(defn_file, "rb")
topology_defn.ParseFromString(handle.read())
handle.close()
except Exception as e:
err_context = "Cannot load topology definition '%s': %s" % (
defn_file, e)
return SimpleResult(Status.HeronError, err_context)
# log topology and components configurations
Log.debug("Topology config: %s", topology_defn.topology_config)
Log.debug("Component config:")
for spout in topology_defn.spouts:
Log.debug("%s => %s", spout.comp.name, spout.comp.config)
for bolt in topology_defn.bolts:
Log.debug("%s => %s", bolt.comp.name, bolt.comp.config)
# launch the topology
Log.info("Launching topology: \'%s\'%s", topology_defn.name,
launch_mode_msg(cl_args))
# check if we have to do server or direct based deployment
if cl_args['deploy_mode'] == config.SERVER_MODE:
res = launch_topology_server(cl_args, topology_file, defn_file,
topology_defn.name)
else:
res = launch_a_topology(cl_args, tmp_dir, topology_file, defn_file,
topology_defn.name)
results.append(res)
return results
def run(command, parser, cl_args, unknown_args):
""" run the update command """
Log.debug("Update Args: %s", cl_args)
# Build jar list
extra_lib_jars = jars.packing_jars()
action = "update topology%s" % (' in dry-run mode'
if cl_args["dry_run"] else '')
# Build extra args
dict_extra_args = {}
try:
dict_extra_args = build_extra_args_dict(cl_args)
except Exception as err:
return SimpleResult(Status.InvocationError, err.message)
# Execute
if cl_args['deploy_mode'] == config.SERVER_MODE:
return cli_helper.run_server(command, cl_args, action, dict_extra_args)
else:
# Convert extra argument to commandline format and then execute
list_extra_args = convert_args_dict_to_list(dict_extra_args)
return cli_helper.run_direct(command, cl_args, action, list_extra_args,
extra_lib_jars)
def heron_class(class_name,
lib_jars,
extra_jars=None,
args=None,
java_defines=None):
'''
Execute a heron class given the args and the jars needed for class path
:param class_name:
:param lib_jars:
:param extra_jars:
:param args:
:param java_defines:
:return:
'''
# default optional params to empty list if not provided
if extra_jars is None:
extra_jars = []
if args is None:
args = []
if java_defines is None:
java_defines = []
# Format all java -D options that need to be passed while running
# the class locally.
java_opts = ['-D' + opt for opt in java_defines]
java_path = config.get_java_path()
if java_path is None:
err_context = "Unable to find java command"
return SimpleResult(Status.InvocationError, err_context)
# Construct the command line for the sub process to run
# Because of the way Python execute works,
# the java opts must be passed as part of the list
all_args = [java_path, "-client", "-Xmx1g"] + \
java_opts + \
["-cp", config.get_classpath(extra_jars + lib_jars)]
all_args += [class_name] + list(args)
# set heron_config environment variable
heron_env = os.environ.copy()
heron_env['HERON_OPTIONS'] = opts.get_heron_config()
# print the verbose message
Log.debug("Invoking class using command: `%s`",
' '.join(shlex.quote(a) for a in all_args))
Log.debug("Heron options: {%s}", str(heron_env["HERON_OPTIONS"]))
# invoke the command with subprocess and print error message, if any
# pylint: disable=consider-using-with
process = subprocess.Popen(all_args,
env=heron_env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
bufsize=1)
# stdout message has the information Java program sends back
# stderr message has extra information, such as debugging message
return ProcessResult(process)
def _unset(cl_args):
cluster, prop = cl_args['cluster'], cl_args['property']
if cliconfig.is_valid_property(prop):
cliconfig.unset_property(cluster, prop)
print("Cleared property [%s] for cluster %s" % (prop, cluster))
else:
print("Error: Unknown property [%s] for cluster %s" % (prop, cluster))
return SimpleResult(Status.Ok)
def _set(cl_args):
cluster, prop, value = cl_args['cluster'], cl_args['property'], cl_args['value']
if cliconfig.is_valid_property(prop):
cliconfig.set_property(cluster, prop, value)
print("Updated property [%s] for cluster %s" % (prop, cluster))
else:
print("Error: Unknown property [%s] for cluster %s" % (prop, cluster))
return SimpleResult(Status.Ok)
def run(command, parser, args, unknown_args): ''' :param command: :param parser: :param args: :param unknown_args: :return: ''' config.print_build_info() return SimpleResult(Status.Ok)
def _list(cl_args):
cluster = cl_args['cluster']
config = cliconfig.cluster_config(cluster)
if config:
for k, v in config.items():
print("%s = %s" % (str(k), str(v)))
else:
print("No config for cluster %s" % cluster)
return SimpleResult(Status.Ok)
def _list(cl_args):
cluster = cl_args['cluster']
config = cliconfig.cluster_config(cluster)
if config:
for k, v in list(config.items()):
print(f"{str(k)} = {str(v)}")
else:
print(f"No config for cluster {cluster}")
return SimpleResult(Status.Ok)
def run(command, parser, args, unknown_args):
'''
:param command:
:param parser:
:param args:
:param unknown_args:
:return:
'''
# get the command for detailed help
command_help = args['help-command']
# if no command is provided, just print main help
if command_help == 'help':
parser.print_help()
return SimpleResult(Status.Ok)
# get the subparser for the specific command
subparser = config.get_subparser(parser, command_help)
if subparser:
print(subparser.format_help())
return SimpleResult(Status.Ok)
Log.error("Unknown subcommand \'%s\'", command_help)
return SimpleResult(Status.InvocationError)
def run(command, parser, cl_args, unknown_args):
'''
runs parser
'''
action = cl_args["action"]
action_type = cl_args["type"]
if action == ACTION.SET:
if action_type == SET.ZOOKEEPERS:
call_editor(get_role_definition_file(SET.ZOOKEEPERS, cl_args))
update_zookeeper_config_files(cl_args)
elif action_type == SET.SLAVES:
call_editor(get_role_definition_file(SET.SLAVES, cl_args))
update_slave_config_files(cl_args)
elif action_type == SET.MASTERS:
call_editor(get_role_definition_file(SET.MASTERS, cl_args))
update_master_config_files(cl_args)
else:
raise ValueError("Invalid set type %s" % action_type)
elif action == ACTION.CLUSTER:
if action_type == CLUSTER.START:
start_cluster(cl_args)
elif action_type == CLUSTER.STOP:
if check_sure(
cl_args, "Are you sure you want to stop the cluster?"
" This will terminate everything running in "
"the cluster and remove any scheduler state."):
stop_cluster(cl_args)
else:
raise ValueError("Invalid cluster action %s" % action_type)
elif action == ACTION.TEMPLATE:
update_zookeeper_config_files(cl_args)
update_master_config_files(cl_args)
update_slave_config_files(cl_args)
else:
raise ValueError("Invalid action %s" % action)
return SimpleResult(Status.Ok)
def run(command, parser, cl_args, unknown_args):
'''
runs parser
'''
action = cl_args["action"]
if action == Action.SET:
call_editor(get_inventory_file(cl_args))
update_config_files(cl_args)
elif action == Action.CLUSTER:
action_type = cl_args["type"]
if action_type == Cluster.START:
start_cluster(cl_args)
elif action_type == Cluster.STOP:
if check_sure(
cl_args, "Are you sure you want to stop the cluster?"
" This will terminate everything running in "
"the cluster and remove any scheduler state."):
stop_cluster(cl_args)
else:
raise ValueError("Invalid cluster action %s" % action_type)
elif action == Action.TEMPLATE:
update_config_files(cl_args)
elif action == Action.GET:
action_type = cl_args["type"]
if action_type == Get.SERVICE_URL:
print get_service_url(cl_args)
elif action_type == Get.HERON_UI_URL:
print get_heron_ui_url(cl_args)
elif action_type == Get.HERON_TRACKER_URL:
print get_heron_tracker_url(cl_args)
else:
raise ValueError("Invalid get action %s" % action_type)
elif action == Action.INFO:
print_cluster_info(cl_args)
else:
raise ValueError("Invalid action %s" % action)
return SimpleResult(Status.Ok)
def run(command, parser, cl_args, unknown_args):
'''
Submits the topology to the scheduler
* Depending on the topology file name extension, we treat the file as a
fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz).
* We upload the topology file to the packer, update zookeeper and launch
scheduler jobs representing that topology
* You can see your topology in Heron UI
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:return:
'''
# get the topology file name
topology_file = cl_args['topology-file-name']
# check to see if the topology file exists
if not os.path.isfile(topology_file):
err_context = "Topology file '%s' does not exist" % topology_file
return SimpleResult(Status.InvocationError, err_context)
# check if it is a valid file type
jar_type = topology_file.endswith(".jar")
tar_type = topology_file.endswith(".tar") or topology_file.endswith(
".tar.gz")
pex_type = topology_file.endswith(".pex")
if not jar_type and not tar_type and not pex_type:
ext_name = os.path.splitext(topology_file)
err_context = "Unknown file type '%s'. Please use .tar or .tar.gz or .jar or .pex file"\
% ext_name
return SimpleResult(Status.InvocationError, err_context)
# check if extra launch classpath is provided and if it is validate
if cl_args['extra_launch_classpath']:
valid_classpath = classpath.valid_java_classpath(
cl_args['extra_launch_classpath'])
if not valid_classpath:
err_context = "One of jar or directory in extra launch classpath does not exist: %s" % \
cl_args['extra_launch_classpath']
return SimpleResult(Status.InvocationError, err_context)
# create a temporary directory for topology definition file
tmp_dir = tempfile.mkdtemp()
opts.cleaned_up_files.append(tmp_dir)
# if topology needs to be launched in deactivated state, do it so
if cl_args['deploy_deactivated']:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED)
else:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING)
# set the tmp dir and deactivated state in global options
opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir)
opts.set_config('cmdline.topology.initial.state', initial_state)
# check the extension of the file name to see if it is tar/jar file.
if jar_type:
return submit_fatjar(cl_args, unknown_args, tmp_dir)
elif tar_type:
return submit_tar(cl_args, unknown_args, tmp_dir)
else:
return submit_pex(cl_args, unknown_args, tmp_dir)
def run(command, parser, cl_args, unknown_args):
'''
Submits the topology to the scheduler
* Depending on the topology file name extension, we treat the file as a
fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz).
* We upload the topology file to the packer, update zookeeper and launch
scheduler jobs representing that topology
* You can see your topology in Heron UI
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:return:
'''
Log.debug("Submit Args %s", cl_args)
# get the topology file name
topology_file = cl_args['topology-file-name']
if urlparse.urlparse(topology_file).scheme:
cl_args['topology-file-name'] = download(topology_file, cl_args['cluster'])
topology_file = cl_args['topology-file-name']
Log.debug("download uri to local file: %s", topology_file)
# check to see if the topology file exists
if not os.path.isfile(topology_file):
err_context = "Topology file '%s' does not exist" % topology_file
return SimpleResult(Status.InvocationError, err_context)
# check if it is a valid file type
jar_type = topology_file.endswith(".jar")
tar_type = topology_file.endswith(".tar") or topology_file.endswith(".tar.gz")
pex_type = topology_file.endswith(".pex")
cpp_type = topology_file.endswith(".dylib") or topology_file.endswith(".so")
if not (jar_type or tar_type or pex_type or cpp_type):
_, ext_name = os.path.splitext(topology_file)
err_context = "Unknown file type '%s'. Please use .tar "\
"or .tar.gz or .jar or .pex or .dylib or .so file"\
% ext_name
return SimpleResult(Status.InvocationError, err_context)
# check if extra launch classpath is provided and if it is validate
if cl_args['extra_launch_classpath']:
valid_classpath = classpath.valid_java_classpath(cl_args['extra_launch_classpath'])
if not valid_classpath:
err_context = "One of jar or directory in extra launch classpath does not exist: %s" % \
cl_args['extra_launch_classpath']
return SimpleResult(Status.InvocationError, err_context)
# create a temporary directory for topology definition file
tmp_dir = tempfile.mkdtemp()
opts.cleaned_up_files.append(tmp_dir)
# if topology needs to be launched in deactivated state, do it so
if cl_args['deploy_deactivated']:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED)
else:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING)
# set the tmp dir and deactivated state in global options
opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir)
opts.set_config('cmdline.topology.initial.state', initial_state)
opts.set_config('cmdline.topology.role', cl_args['role'])
opts.set_config('cmdline.topology.environment', cl_args['environ'])
# Use CLI release yaml file if the release_yaml_file config is empty
if not cl_args['release_yaml_file']:
cl_args['release_yaml_file'] = config.get_heron_release_file()
# check the extension of the file name to see if it is tar/jar file.
if jar_type:
return submit_fatjar(cl_args, unknown_args, tmp_dir)
elif tar_type:
return submit_tar(cl_args, unknown_args, tmp_dir)
elif cpp_type:
return submit_cpp(cl_args, unknown_args, tmp_dir)
else:
return submit_pex(cl_args, unknown_args, tmp_dir)
