def run_bolts(command, parser, cl_args, unknown_args):
cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args['environ']
topology = cl_args['topology-name']
try:
result = utils.get_topology_info(cluster, env, topology, role)
bolts = result['physical_plan']['bolts'].keys()
bolt_name = cl_args['bolt']
if bolt_name:
if bolt_name in bolts:
bolts = [bolt_name]
else:
Log.error('Unknown bolt: \'%s\'' % bolt_name)
raise
except Exception:
return False
bolts_result = []
for bolt in bolts:
try:
metrics = utils.get_component_metrics(bolt, cluster, env, topology, role)
stat, header = to_table(metrics)
bolts_result.append((bolt, stat, header))
except Exception:
return False
for i, (bolt, stat, header) in enumerate(bolts_result):
if i != 0:
print('')
print('\'%s\' metrics:' % bolt)
print(tabulate(stat, headers=header))
return True
def extract_common_args(command, parser, cl_args):
try:
cluster_role_env = cl_args.pop('cluster/[role]/[env]')
config_path = cl_args['config_path']
override_config_file = utils.parse_override_config(cl_args['config_property'])
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = utils.get_subparser(parser, command)
print(subparser.format_help())
return dict()
cluster = utils.get_heron_cluster(cluster_role_env)
config_path = utils.get_heron_cluster_conf_dir(cluster, config_path)
if not os.path.isdir(config_path):
Log.error("Config path cluster directory does not exist: %s" % config_path)
return dict()
new_cl_args = dict()
try:
cluster_tuple = utils.parse_cluster_role_env(cluster_role_env, config_path)
new_cl_args['cluster'] = cluster_tuple[0]
new_cl_args['role'] = cluster_tuple[1]
new_cl_args['environ'] = cluster_tuple[2]
new_cl_args['config_path'] = config_path
new_cl_args['override_config_file'] = override_config_file
except Exception as e:
Log.error("Argument cluster/[role]/[env] is not correct: %s" % str(e))
return dict()
cl_args.update(new_cl_args)
return cl_args
def main():
'''
Run the command
:return:
'''
# verify if the environment variables are correctly set
check_environment()
# create the argument parser
parser = create_parser()
# if no argument is provided, print help and exit
if len(sys.argv[1:]) == 0:
parser.print_help()
return 0
# insert the boolean values for some of the options
sys.argv = parse.insert_bool_values(sys.argv)
# parse the args
args, unknown_args = parser.parse_known_args()
command_line_args = vars(args)
try:
if command_line_args['verbose']:
opts.set_verbose()
if command_line_args['trace_execution']:
opts.set_trace_execution()
except:
pass
# command to be execute
command = command_line_args['subcommand']
# file resources to be cleaned when exit
files = []
if command != 'help' and command != 'version':
command_line_args = extract_common_args(command, parser, command_line_args)
# bail out if args are empty
if not command_line_args:
return 1
# register dirs cleanup function during exit
files.append(command_line_args['override_config_file'])
atexit.register(cleanup, files)
# print the input parameters, if verbose is enabled
if opts.verbose():
print command_line_args
start = time.time()
retcode = run(command, parser, command_line_args, unknown_args)
end = time.time()
if command != 'help':
sys.stdout.flush()
Log.info('Elapsed time: %.3fs.' % (end - start))
return 0 if retcode else 1
def run_bolts(command, parser, cl_args, unknown_args):
""" run bolts subcommand """
cluster, role, env = cl_args["cluster"], cl_args["role"], cl_args["environ"]
topology = cl_args["topology-name"]
try:
result = utils.get_topology_info(cluster, env, topology, role)
bolts = result["physical_plan"]["bolts"].keys()
bolt_name = cl_args["bolt"]
if bolt_name:
if bolt_name in bolts:
bolts = [bolt_name]
else:
Log.error("Unknown bolt: '%s'" % bolt_name)
raise
except Exception:
return False
bolts_result = []
for bolt in bolts:
try:
metrics = utils.get_component_metrics(bolt, cluster, env, topology, role)
stat, header = to_table(metrics)
bolts_result.append((bolt, stat, header))
except Exception:
return False
for i, (bolt, stat, header) in enumerate(bolts_result):
if i != 0:
print ""
print "'%s' metrics:" % bolt
print tabulate(stat, headers=header)
return True
def extract_common_args(command, parser, cl_args):
try:
cluster_role_env = cl_args.pop('cluster/[role]/[env]')
config_path = cl_args['config_path']
override_config_file = utils.parse_override_config(cl_args['config_property'])
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = utils.get_subparser(parser, command)
print(subparser.format_help())
return dict()
cluster = utils.get_heron_cluster(cluster_role_env)
config_path = utils.get_heron_cluster_conf_dir(cluster, config_path)
if not os.path.isdir(config_path):
Log.error("Config path cluster directory does not exist: %s" % config_path)
return dict()
new_cl_args = dict()
try:
cluster_tuple = utils.parse_cluster_role_env(cluster_role_env, config_path)
new_cl_args['cluster'] = cluster_tuple[0]
new_cl_args['role'] = cluster_tuple[1]
new_cl_args['environ'] = cluster_tuple[2]
new_cl_args['config_path'] = config_path
new_cl_args['override_config_file'] = override_config_file
except Exception as e:
Log.error("Argument cluster/[role]/[env] is not correct: %s" % str(e))
return dict()
cl_args.update(new_cl_args)
return cl_args
def submit_fatjar(cl_args, unknown_args, tmp_dir):
# execute main of the topology to create the topology definition
topology_file = cl_args['topology-file-name']
try:
execute.heron_class(
cl_args['topology-class-name'],
utils.get_heron_libs(jars.topology_jars()),
extra_jars = [topology_file],
args = tuple(unknown_args),
javaDefines = cl_args['javaDefines'])
except Exception as ex:
Log.error("Unable to execute topology main class")
return False
try:
launch_topologies(cl_args, topology_file, tmp_dir)
except Exception as ex:
return False
finally:
shutil.rmtree(tmp_dir)
return True
def run_metrics(command, parser, cl_args, unknown_args):
""" run metrics subcommand """
cluster, role, env = cl_args["cluster"], cl_args["role"], cl_args["environ"]
topology = cl_args["topology-name"]
try:
result = utils.get_topology_info(cluster, env, topology, role)
spouts = result["physical_plan"]["spouts"].keys()
bolts = result["physical_plan"]["bolts"].keys()
components = spouts + bolts
cname = cl_args["component"]
if cname:
if cname in components:
components = [cname]
else:
Log.error("Unknown component: '%s'" % cname)
raise
except Exception:
return False
cresult = []
for comp in components:
try:
metrics = utils.get_component_metrics(comp, cluster, env, topology, role)
stat, header = to_table(metrics)
cresult.append((comp, stat, header))
except:
return False
for i, (comp, stat, header) in enumerate(cresult):
if i != 0:
print ""
print "'%s' metrics:" % comp
print tabulate(stat, headers=header)
return True
def get_topology_info(*args):
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_topology_info(*args))
except Exception as ex:
Log.error(str(ex))
raise
def extract_common_args(command, parser, cl_args):
try:
# do not pop like cli because ``topologies`` subcommand still needs it
cluster_role_env = cl_args['cluster/[role]/[env]']
config_path = cl_args['config_path']
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = utils.get_subparser(parser, command)
print(subparser.format_help())
return dict()
cluster = utils.get_heron_cluster(cluster_role_env)
config_path = utils.get_heron_cluster_conf_dir(cluster, config_path)
new_cl_args = dict()
try:
cluster_tuple = utils.parse_cluster_role_env(cluster_role_env, config_path)
new_cl_args['cluster'] = cluster_tuple[0]
new_cl_args['role'] = cluster_tuple[1]
new_cl_args['environ'] = cluster_tuple[2]
new_cl_args['config_path'] = config_path
except Exception as e:
Log.error("Unable to get valid topology location: %s" % str(e))
return dict()
cl_args.update(new_cl_args)
return cl_args
def get(self):
""" get method """
try:
cluster = self.get_argument_cluster()
role = self.get_argument_role()
environ = self.get_argument_environ()
topology_name = self.get_argument_topology()
container = self.get_argument(constants.PARAM_CONTAINER)
path = self.get_argument(constants.PARAM_PATH)
offset = self.get_argument_offset()
length = self.get_argument_length()
topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)
stmgr_id = "stmgr-" + container
stmgr = topology_info["physical_plan"]["stmgrs"][stmgr_id]
host = stmgr["host"]
shell_port = stmgr["shell_port"]
file_data_url = "http://%s:%d/filedata/%s?offset=%s&length=%s" % \
(host, shell_port, path, offset, length)
http_client = tornado.httpclient.AsyncHTTPClient()
response = yield http_client.fetch(file_data_url)
self.write_success_response(json.loads(response.body))
self.finish()
except Exception as e:
Log.debug(traceback.format_exc())
self.write_error_response(e)
def run_metrics(command, parser, cl_args, unknown_args):
cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args['environ']
topology = cl_args['topology-name']
try:
result = utils.get_topology_info(cluster, env, topology, role)
spouts = result['physical_plan']['spouts'].keys()
bolts = result['physical_plan']['bolts'].keys()
components = spouts + bolts
cname = cl_args['component']
if cname:
if cname in components:
components = [cname]
else:
Log.error('Unknown component: \'%s\'' % cname)
raise
except Exception:
return False
cresult = []
for comp in components:
try:
metrics = utils.get_component_metrics(comp, cluster, env, topology, role)
stat, header = to_table(metrics)
cresult.append((comp, stat, header))
except:
return False
for i, (comp, stat, header) in enumerate(cresult):
if i != 0:
print('')
print('\'%s\' metrics:' % comp)
print(tabulate(stat, headers=header))
return True
def get_cluster_topologies(cluster):
"""Synced API call to get topologies under a cluster"""
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_cluster_topologies(cluster))
except Exception:
Log.info(traceback.format_exc())
raise
def get_topology_metrics(*args):
"""Synced API call to get topology metrics"""
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_comp_metrics(*args))
except Exception:
Log.info(traceback.format_exc())
raise
def get_topology_metrics(*args):
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_comp_metrics(*args))
except Exception as ex:
Log.error(str(ex))
Log.error("Failed to retrive metrics of component \'%s\'" % args[3])
raise
def unregister_watch(self, uid):
"""
Unregister the watch with the given UUID.
"""
# Do not raise an error if UUID is
# not present in the watches.
Log.info("Unregister a watch with uid: " + str(uid))
self.watches.pop(uid, None)
def get_clusters():
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_clusters())
except Exception as ex:
Log.error('Error: %s' % str(ex))
Log.error('Failed to retrive clusters')
raise
def get_topology_metrics(*args):
"""Synced API call to get topology metrics"""
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_comp_metrics(*args))
except Exception:
Log.info(traceback.format_exc())
raise
def get_logical_plan(cluster, env, topology, role):
"""Synced API call to get logical plans"""
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_logical_plan(cluster, env, topology, role))
except Exception:
Log.info(traceback.format_exc())
raise
def get_cluster_topologies(cluster):
"""Synced API call to get topologies under a cluster"""
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_cluster_topologies(cluster))
except Exception:
Log.info(traceback.format_exc())
raise
def get_cluster_role_env_topologies(cluster, role, env):
"""Synced API call to get topologies under a cluster submitted by a role under env"""
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_cluster_role_env_topologies(cluster, role, env))
except Exception:
Log.info(traceback.format_exc())
raise
def get_cluster_topologies(cluster):
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_cluster_topologies(cluster))
except Exception as ex:
Log.error(str(ex))
Log.error('Failed to retrive topologies running in cluster \'%s\'' % cluster)
raise
def main():
# verify if the environment variables are correctly set
check_environment()
# create the argument parser
parser = create_parser()
# if no argument is provided, print help and exit
if len(sys.argv[1:]) == 0:
parser.print_help()
return 0
# insert the boolean values for some of the options
sys.argv = parse.insert_bool_values(sys.argv)
# parse the args
args, unknown_args = parser.parse_known_args()
command_line_args = vars(args)
try:
if command_line_args['verbose']:
opts.set_verbose()
if command_line_args['trace_execution']:
opts.set_trace_execution()
except:
pass
# command to be execute
command = command_line_args['subcommand']
# file resources to be cleaned when exit
files = []
if command != 'help' and command != 'version':
command_line_args = extract_common_args(command, parser,
command_line_args)
# bail out if args are empty
if not command_line_args:
return 1
# register dirs cleanup function during exit
files.append(command_line_args['override_config_file'])
atexit.register(cleanup, files)
# print the input parameters, if verbose is enabled
if opts.verbose():
print command_line_args
start = time.time()
retcode = run(command, parser, command_line_args, unknown_args)
end = time.time()
if command != 'help':
sys.stdout.flush()
Log.info('Elapsed time: %.3fs.' % (end - start))
return 0 if retcode == True else 1
def get_cluster_role_env_topologies(cluster, role, env):
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_cluster_role_env_topologies(cluster, role, env))
except Exception as ex:
Log.error(str(ex))
Log.error('Failed to retrive topologies running in cluster'
'\'%s\' submitted by %s under environment %s' % (cluster, role, env))
raise
def parse_cluster_role_env(cluster_role_env, config_path):
"""Parse cluster/[role]/[environ], supply default, if not provided, not required"""
parts = cluster_role_env.split('/')[:3]
Log.info("Using config file under %s" % config_path)
if not os.path.isdir(config_path):
Log.error("Config path cluster directory does not exist: %s" %
config_path)
raise Exception("Invalid config path")
# if cluster/role/env is not completely provided, check further
if len(parts) < 3:
cli_conf_file = os.path.join(config_path, CLIENT_YAML)
# if client conf doesn't exist, use default value
if not os.path.isfile(cli_conf_file):
if len(parts) == 1:
parts.append(getpass.getuser())
if len(parts) == 2:
parts.append(ENVIRON)
else:
cli_confs = {}
with open(cli_conf_file, 'r') as conf_file:
tmp_confs = yaml.load(conf_file)
# the return value of yaml.load can be None if conf_file is an empty file
if tmp_confs is not None:
cli_confs = tmp_confs
else:
print "Failed to read: %s due to it is empty" % (
CLIENT_YAML)
# if role is required but not provided, raise exception
if len(parts) == 1:
if (IS_ROLE_REQUIRED in cli_confs) and (
cli_confs[IS_ROLE_REQUIRED] is True):
raise Exception(
"role required but not provided (cluster/role/env = %s). See %s in %s"
% (cluster_role_env, IS_ROLE_REQUIRED, CLIENT_YAML))
else:
parts.append(getpass.getuser())
# if environ is required but not provided, raise exception
if len(parts) == 2:
if (IS_ENV_REQUIRED
in cli_confs) and (cli_confs[IS_ENV_REQUIRED] is True):
raise Exception(
"environ required but not provided (cluster/role/env = %s). See %s in %s"
% (cluster_role_env, IS_ENV_REQUIRED, CLIENT_YAML))
else:
parts.append(ENVIRON)
# if cluster or role or environ is empty, print
if len(parts[0]) == 0 or len(parts[1]) == 0 or len(parts[2]) == 0:
print "Failed to parse"
sys.exit(1)
return (parts[0], parts[1], parts[2])
def get_logical_plan(cluster, env, topology, role):
"""Synced API call to get logical plans"""
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(
lambda: API.get_logical_plan(cluster, env, topology, role))
except Exception:
Log.info(traceback.format_exc())
raise
def get_clusters():
"""Synced API call to get all cluster names"""
instance = tornado.ioloop.IOLoop.instance()
# pylint: disable=unnecessary-lambda
try:
return instance.run_sync(lambda: API.get_clusters())
except Exception:
Log.info(traceback.format_exc())
raise
def get_cluster_role_env_topologies(cluster, role, env):
"""Synced API call to get topologies under a cluster submitted by a role under env"""
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(
lambda: API.get_cluster_role_env_topologies(cluster, role, env))
except Exception:
Log.info(traceback.format_exc())
raise
def check_release_file_exists():
release_file = get_heron_release_file()
# if the file does not exist and is not a file
if not os.path.isfile(release_file):
Log.error("%s file not found: %s" % release_file)
return False
return True
def check_release_file_exists():
release_file = get_heron_release_file()
# if the file does not exist and is not a file
if not os.path.isfile(release_file):
Log.error("Required file not found: %s" % release_file)
return False
return True
def get_logical_plan(cluster, env, topology, role):
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_logical_plan(cluster, env, topology, role))
except Exception as ex:
Log.error('Error: %s' % str(ex))
Log.error('Failed to retrive logical plan info of topology \'%s\''
% ('/'.join([cluster, role, env, topology])))
raise
def get_clusters():
"""Synced API call to get all cluster names"""
instance = tornado.ioloop.IOLoop.instance()
# pylint: disable=unnecessary-lambda
try:
return instance.run_sync(lambda: API.get_clusters())
except Exception:
Log.info(traceback.format_exc())
raise
def check_release_file_exists():
"""Check if the release.yaml file exists"""
release_file = get_heron_release_file()
# if the file does not exist and is not a file
if not os.path.isfile(release_file):
Log.error("Required file not found: %s" % release_file)
return False
return True
def get_clusters():
"""Synced API call to get all cluster names"""
instance = tornado.ioloop.IOLoop.instance()
# pylint: disable=unnecessary-lambda
try:
return instance.run_sync(lambda: API.get_clusters())
except Exception as ex:
Log.error('Error: %s' % str(ex))
Log.error('Failed to retrive clusters')
raise
def get_component_metrics(component, cluster, env, topology, role):
"""Synced API call to get component metrics"""
all_queries = metric_queries()
try:
result = get_topology_metrics(cluster, env, topology, component, [],
all_queries, [0, -1], role)
return result["metrics"]
except Exception:
Log.info(traceback.format_exc())
raise
def get_component_metrics(component, cluster, env, topology, role):
"""Synced API call to get component metrics"""
all_queries = metric_queries()
try:
result = get_topology_metrics(cluster, env, topology, component, [],
all_queries, [0, -1], role)
return result["metrics"]
except Exception:
Log.info(traceback.format_exc())
raise
def parse_topo_loc(cl_args):
try:
topo_loc = cl_args['cluster/[role]/[env]'].split('/')
topo_loc.append(cl_args['topology-name'])
if len(topo_loc) != 4:
raise
return topo_loc
except Exception:
Log.error('Error: invalid topology location')
raise
def get_cluster_role_topologies(cluster, role):
"""Synced API call to get topologies under a cluster submitted by a role"""
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_cluster_role_topologies(cluster, role))
except Exception as ex:
Log.error(str(ex))
Log.error('Failed to retrive topologies running in cluster'
'\'%s\' submitted by %s' % (cluster, role))
raise
def run(command, parser, cl_args, unknown_args):
'''
Submits the topology to the scheduler
* Depending on the topology file name extension, we treat the file as a
fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz).
* We upload the topology file to the packer, update zookeeper and launch
scheduler jobs representing that topology
* You can see your topology in Heron UI
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:return:
'''
# get the topology file name
topology_file = cl_args['topology-file-name']
# check to see if the topology file exists
if not os.path.isfile(topology_file):
Log.error("Topology jar|tar|pex file %s does not exist" % topology_file)
return False
# check if it is a valid file type
jar_type = topology_file.endswith(".jar")
tar_type = topology_file.endswith(".tar") or topology_file.endswith(".tar.gz")
pex_type = topology_file.endswith(".pex")
if not jar_type and not tar_type and not pex_type:
Log.error("Unknown file type. Please use .tar or .tar.gz or .jar or .pex file")
return False
# create a temporary directory for topology definition file
tmp_dir = tempfile.mkdtemp()
# if topology needs to be launched in deactivated state, do it so
if cl_args['deploy_deactivated']:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED)
else:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING)
# set the tmp dir and deactivated state in global options
opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir)
opts.set_config('cmdline.topology.initial.state', initial_state)
# check the extension of the file name to see if it is tar/jar file.
if jar_type:
return submit_fatjar(cl_args, unknown_args, tmp_dir)
elif tar_type:
return submit_tar(cl_args, unknown_args, tmp_dir)
elif pex_type:
return submit_pex(cl_args, unknown_args, tmp_dir)
return False
def run(command, parser, cl_args, unknown_args):
""" run command """
try:
clusters = utils.get_clusters()
except:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
print 'Available clusters:'
for cluster in clusters:
print ' %s' % cluster
return True
def run(command, parser, cl_args, unknown_args):
""" run command """
try:
clusters = utils.get_clusters()
except:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
print 'Available clusters:'
for cluster in clusters:
print ' %s' % cluster
return True
def parse_topo_loc(cl_args):
""" parse topology location """
try:
topo_loc = cl_args['cluster/[role]/[env]'].split('/')
topo_loc.append(cl_args['topology-name'])
if len(topo_loc) != 4:
raise
return topo_loc
except Exception:
Log.error('Error: invalid topology location')
raise
def run(command, parser, cl_args, unknown_args):
location = cl_args['cluster/[role]/[env]'].split('/')
if len(location) == 1:
return show_cluster(*location)
elif len(location) == 2:
return show_cluster_role(*location)
elif len(location) == 3:
return show_cluster_role_env(*location)
else:
Log.error('Invalid topologies selection')
return False
def run(command, parser, cl_args, unknown_args):
""" run command """
location = cl_args['cluster/[role]/[env]'].split('/')
if len(location) == 1:
return show_cluster(cl_args, *location)
elif len(location) == 2:
return show_cluster_role(cl_args, *location)
elif len(location) == 3:
return show_cluster_role_env(cl_args, *location)
else:
Log.error('Invalid topologies selection')
return False
def get(self):
""" get method """
try:
cluster = self.get_argument_cluster()
role = self.get_argument_role()
environ = self.get_argument_environ()
topology_name = self.get_argument_topology()
topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)
self.write_success_response(topology_info)
except Exception as e:
Log.debug(traceback.format_exc())
self.write_error_response(e)
def run(command, parser, cl_args, unknown_args, action):
'''
helper function to take action on topologies
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:param action: description of action taken
:return:
'''
try:
topology_name = cl_args['topology-name']
new_args = [
"--cluster",
cl_args['cluster'],
"--role",
cl_args['role'],
"--environment",
cl_args['environ'],
"--heron_home",
utils.get_heron_dir(),
"--config_path",
cl_args['config_path'],
"--override_config_file",
cl_args['override_config_file'],
"--release_file",
utils.get_heron_release_file(),
"--topology_name",
topology_name,
"--command",
command,
]
if opts.verbose():
new_args.append("--verbose")
lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args)
except Exception:
Log.error('Failed to %s \'%s\'' % (action, topology_name))
return False
Log.info('Successfully %s \'%s\'' % (action, topology_name))
return True
def run(command, parser, cl_args, unknown_args):
'''
Submits the topology to the scheduler
* Depending on the topology file name extension, we treat the file as a
fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz).
* We upload the topology file to the packer, update zookeeper and launch
scheduler jobs representing that topology
* You can see your topology in Heron UI
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:return:
'''
# get the topology file name
topology_file = cl_args['topology-file-name']
# check to see if the topology file exists
if not os.path.isfile(topology_file):
Log.error("Topology jar|tar file %s does not exist" % topology_file)
return False
# check if it is a valid file type
jar_type = topology_file.endswith(".jar")
tar_type = topology_file.endswith(".tar") or topology_file.endswith(
".tar.gz")
if not jar_type and not tar_type:
Log.error("Unknown file type. Please use .tar or .tar.gz or .jar file")
return False
# create a temporary directory for topology definition file
tmp_dir = tempfile.mkdtemp()
# if topology needs to be launched in deactivated state, do it so
if cl_args['deploy_deactivated']:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED)
else:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING)
# set the tmp dir and deactivated state in global options
opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir)
opts.set_config('cmdline.topology.initial.state', initial_state)
# check the extension of the file name to see if it is tar/jar file.
if jar_type:
return submit_fatjar(cl_args, unknown_args, tmp_dir)
elif tar_type:
return submit_tar(cl_args, unknown_args, tmp_dir)
return False
def check_java_home_set():
"""Check if the java home set"""
# check if environ variable is set
if "JAVA_HOME" not in os.environ:
Log.error("JAVA_HOME not set")
return False
# check if the value set is correct
java_path = get_java_path()
if os.path.isfile(java_path) and os.access(java_path, os.X_OK):
return True
Log.error("JAVA_HOME/bin/java either does not exist or not an executable")
return False
def run_containers(command, parser, cl_args, unknown_args):
""" run containers subcommand """
cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args[
'environ']
topology = cl_args['topology-name']
container_id = cl_args['id']
try:
result = utils.get_topology_info(cluster, env, topology, role)
except:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
containers = result['physical_plan']['stmgrs']
all_bolts, all_spouts = set(), set()
for _, bolts in result['physical_plan']['bolts'].items():
all_bolts = all_bolts | set(bolts)
for _, spouts in result['physical_plan']['spouts'].items():
all_spouts = all_spouts | set(spouts)
stmgrs = containers.keys()
stmgrs.sort()
if container_id is not None:
try:
normalized_cid = container_id - 1
if normalized_cid < 0:
raise
stmgrs = [stmgrs[normalized_cid]]
except:
Log.error('Invalid container id: %d' % container_id)
return False
table = []
for sid, name in enumerate(stmgrs):
cid = sid + 1
host = containers[name]["host"]
port = containers[name]["port"]
pid = containers[name]["pid"]
instances = containers[name]["instance_ids"]
bolt_nums = len(
[instance for instance in instances if instance in all_bolts])
spout_nums = len(
[instance for instance in instances if instance in all_spouts])
table.append(
[cid, host, port, pid, bolt_nums, spout_nums,
len(instances)])
headers = [
"container", "host", "port", "pid", "#bolt", "#spout", "#instance"
]
sys.stdout.flush()
print tabulate(table, headers=headers)
return True
def show_cluster(cl_args, cluster):
''' print topologies information to stdout '''
try:
result = utils.get_cluster_topologies(cluster)
if not result:
Log.error('Unknown cluster \'%s\'' % cluster)
return False
result = result[cluster]
except Exception:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
table, header, rest_count = to_table(result)
print 'Topologies running in cluster \'%s\'' % cluster
if rest_count:
print ' with %d more...' % rest_count
print tabulate(table, headers=header)
return True
def run(command, parser, cl_args, unknown_args):
try:
topology_name = cl_args['topology-name']
new_args = [
"--cluster",
cl_args['cluster'],
"--role",
cl_args['role'],
"--environment",
cl_args['environ'],
"--heron_home",
utils.get_heron_dir(),
"--config_path",
cl_args['config_path'],
"--override_config_file",
cl_args['override_config_file'],
"--release_file",
utils.get_heron_release_file(),
"--topology_name",
topology_name,
"--command",
command,
]
if opts.verbose():
new_args.append("--verbose")
lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args)
except Exception as ex:
print 'Error: %s' % str(ex)
Log.error('Failed to activate topology \'%s\'' % topology_name)
return False
Log.info('Successfully activated topology \'%s\'' % topology_name)
return True
def run(command, parser, args, unknown_args):
""" run command """
# get the command for detailed help
command_help = args['help-command']
# if no command is provided, just print main help
if command_help == 'help':
parser.print_help()
return True
# get the subparser for the specific command
subparser = utils.get_subparser(parser, command_help)
if subparser:
print subparser.format_help()
return True
else:
Log.error("Unknown subcommand \'%s\'" % command_help)
return False
def show_cluster_role_env(cl_args, cluster, role, env):
''' print topologies information to stdout '''
try:
result = utils.get_cluster_role_env_topologies(cluster, role, env)
if not result:
Log.error('Unknown cluster/role/env \'%s\'' %
'/'.join([cluster, role, env]))
return False
result = result[cluster]
except Exception:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
table, header, rest_count = to_table(result)
print 'Topologies running in cluster \'%s\', submitted by \'%s\', and\
under environment \'%s\':' % (cluster, role, env)
if rest_count:
print ' with %d more...' % rest_count
print tabulate(table, headers=header)
return True
def launch_topologies(cl_args, topology_file, tmp_dir):
# the submitter would have written the .defn file to the tmp_dir
defn_files = glob.glob(tmp_dir + '/*.defn')
if len(defn_files) == 0:
raise Exception("No topologies found")
try:
for defn_file in defn_files:
# load the topology definition from the file
topology_defn = topology_pb2.Topology()
try:
f = open(defn_file, "rb")
topology_defn.ParseFromString(f.read())
f.close()
except:
raise Exception("Could not open and parse topology defn file %s" % defn_file)
# launch the topology
try:
Log.info("Launching topology \'%s\'" % topology_defn.name)
launch_a_topology(cl_args, tmp_dir, topology_file, defn_file)
Log.info("Topology \'%s\' launched successfully" % topology_defn.name)
except Exception as ex:
Log.error('Failed to launch topology \'%s\' because %s' % (topology_defn.name, str(ex)))
raise
except:
raise
def run_bolts(command, parser, cl_args, unknown_args):
""" run bolts subcommand """
cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args[
'environ']
topology = cl_args['topology-name']
try:
result = utils.get_topology_info(cluster, env, topology, role)
bolts = result['physical_plan']['bolts'].keys()
bolt_name = cl_args['bolt']
if bolt_name:
if bolt_name in bolts:
bolts = [bolt_name]
else:
Log.error('Unknown bolt: \'%s\'' % bolt_name)
raise
except Exception:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
bolts_result = []
for bolt in bolts:
try:
metrics = utils.get_component_metrics(bolt, cluster, env, topology,
role)
stat, header = to_table(metrics)
bolts_result.append((bolt, stat, header))
except Exception:
Log.error("Fail to connect to tracker: \'%s\'",
cl_args["tracker_url"])
return False
for i, (bolt, stat, header) in enumerate(bolts_result):
if i != 0:
print ''
print '\'%s\' metrics:' % bolt
print tabulate(stat, headers=header)
return True
def run(cl_args, compo_type):
""" run command """
cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args[
'environ']
topology = cl_args['topology-name']
spouts_only, bolts_only = cl_args['spout'], cl_args['bolt']
try:
components = utils.get_logical_plan(cluster, env, topology, role)
topo_info = utils.get_topology_info(cluster, env, topology, role)
table, header = to_table(components, topo_info)
if spouts_only == bolts_only:
print tabulate(table, headers=header)
elif spouts_only:
table, header = filter_spouts(table, header)
print tabulate(table, headers=header)
else:
table, header = filter_bolts(table, header)
print tabulate(table, headers=header)
return True
except:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
def submit_fatjar(cl_args, unknown_args, tmp_dir):
'''
We use the packer to make a package for the jar and dump it
to a well-known location. We then run the main method of class
with the specified arguments. We pass arguments as heron.options.
This will run the jar file with the topology_class_name. The submitter
inside will write out the topology defn file to a location that
we specify. Then we write the topology defn file to a well known
location. We then write to appropriate places in zookeeper
and launch the scheduler jobs
:param cl_args:
:param unknown_args:
:param tmp_dir:
:return:
'''
# execute main of the topology to create the topology definition
topology_file = cl_args['topology-file-name']
try:
execute.heron_class(cl_args['topology-class-name'],
utils.get_heron_libs(jars.topology_jars()),
extra_jars=[topology_file],
args=tuple(unknown_args),
java_defines=cl_args['topology_main_jvm_property'])
except Exception:
Log.error("Unable to execute topology main class")
return False
try:
launch_topologies(cl_args, topology_file, tmp_dir)
except Exception:
return False
finally:
shutil.rmtree(tmp_dir)
return True
def run(command, parser, cl_args, unknown_args):
'''
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:return:
'''
try:
topology_name = cl_args['topology-name']
container_id = cl_args['container-id']
new_args = [
"--cluster", cl_args['cluster'], "--role", cl_args['role'],
"--environment", cl_args['environ'], "--heron_home",
utils.get_heron_dir(), "--config_path", cl_args['config_path'],
"--override_config_file", cl_args['override_config_file'],
"--release_file",
utils.get_heron_release_file(), "--topology_name", topology_name,
"--command", command, "--container_id",
str(container_id)
]
lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args)
except Exception as ex:
print 'Error: %s' % str(ex)
Log.error('Failed to restart topology \'%s\'' % topology_name)
return False
Log.info('Successfully restarted topology \'%s\'' % topology_name)
return True
def run(command, parser, cl_args, unknown_args):
# get the topology file name
topology_file = cl_args['topology-file-name']
# check to see if the topology file exists
if not os.path.isfile(topology_file):
Log.error("Topology jar|tar file %s does not exist" % topology_file)
return False
# check if it is a valid file type
jar_type = topology_file.endswith(".jar")
tar_type = topology_file.endswith(".tar") or topology_file.endswith(".tar.gz")
if not jar_type and not tar_type:
Log.error("Unknown file type. Please use .tar or .tar.gz or .jar file")
return False
# create a temporary directory for topology definition file
tmp_dir = tempfile.mkdtemp()
# if topology needs to be launched in deactivated state, do it so
if cl_args['deploy_deactivated']:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED)
else:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING)
# set the tmp dir and deactivated state in global options
opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir)
opts.set_config('cmdline.topology.initial.state', initial_state)
# check the extension of the file name to see if it is tar/jar file.
if jar_type:
return submit_fatjar(cl_args, unknown_args, tmp_dir)
elif tar_type:
return submit_tar(cl_args, unknown_args, tmp_dir)
return False
def run_metrics(command, parser, cl_args, unknown_args):
""" run metrics subcommand """
cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args[
'environ']
topology = cl_args['topology-name']
try:
result = utils.get_topology_info(cluster, env, topology, role)
spouts = result['physical_plan']['spouts'].keys()
bolts = result['physical_plan']['bolts'].keys()
components = spouts + bolts
cname = cl_args['component']
if cname:
if cname in components:
components = [cname]
else:
Log.error('Unknown component: \'%s\'' % cname)
raise
except Exception:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
cresult = []
for comp in components:
try:
metrics = utils.get_component_metrics(comp, cluster, env, topology,
role)
except:
Log.error("Fail to connect to tracker: \'%s\'",
cl_args["tracker_url"])
return False
stat, header = to_table(metrics)
cresult.append((comp, stat, header))
for i, (comp, stat, header) in enumerate(cresult):
if i != 0:
print ''
print '\'%s\' metrics:' % comp
print tabulate(stat, headers=header)
return True
