def extract_common_args(command, parser, cl_args):
try:
cluster_role_env = cl_args.pop('cluster/[role]/[env]')
config_path = cl_args['config_path']
override_config_file = utils.parse_override_config(cl_args['config_property'])
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = utils.get_subparser(parser, command)
print(subparser.format_help())
return dict()
cluster = utils.get_heron_cluster(cluster_role_env)
config_path = utils.get_heron_cluster_conf_dir(cluster, config_path)
if not os.path.isdir(config_path):
Log.error("Config path cluster directory does not exist: %s" % config_path)
return dict()
new_cl_args = dict()
try:
cluster_tuple = utils.parse_cluster_role_env(cluster_role_env, config_path)
new_cl_args['cluster'] = cluster_tuple[0]
new_cl_args['role'] = cluster_tuple[1]
new_cl_args['environ'] = cluster_tuple[2]
new_cl_args['config_path'] = config_path
new_cl_args['override_config_file'] = override_config_file
except Exception as e:
Log.error("Argument cluster/[role]/[env] is not correct: %s" % str(e))
return dict()
cl_args.update(new_cl_args)
return cl_args
def run_metrics(command, parser, cl_args, unknown_args):
cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args['environ']
topology = cl_args['topology-name']
try:
result = utils.get_topology_info(cluster, env, topology, role)
spouts = result['physical_plan']['spouts'].keys()
bolts = result['physical_plan']['bolts'].keys()
components = spouts + bolts
cname = cl_args['component']
if cname:
if cname in components:
components = [cname]
else:
Log.error('Unknown component: \'%s\'' % cname)
raise
except Exception:
return False
cresult = []
for comp in components:
try:
metrics = utils.get_component_metrics(comp, cluster, env, topology, role)
stat, header = to_table(metrics)
cresult.append((comp, stat, header))
except:
return False
for i, (comp, stat, header) in enumerate(cresult):
if i != 0:
print('')
print('\'%s\' metrics:' % comp)
print(tabulate(stat, headers=header))
return True
def run_bolts(command, parser, cl_args, unknown_args):
""" run bolts subcommand """
cluster, role, env = cl_args["cluster"], cl_args["role"], cl_args["environ"]
topology = cl_args["topology-name"]
try:
result = utils.get_topology_info(cluster, env, topology, role)
bolts = result["physical_plan"]["bolts"].keys()
bolt_name = cl_args["bolt"]
if bolt_name:
if bolt_name in bolts:
bolts = [bolt_name]
else:
Log.error("Unknown bolt: '%s'" % bolt_name)
raise
except Exception:
return False
bolts_result = []
for bolt in bolts:
try:
metrics = utils.get_component_metrics(bolt, cluster, env, topology, role)
stat, header = to_table(metrics)
bolts_result.append((bolt, stat, header))
except Exception:
return False
for i, (bolt, stat, header) in enumerate(bolts_result):
if i != 0:
print ""
print "'%s' metrics:" % bolt
print tabulate(stat, headers=header)
return True
def launch_topologies(cl_args, topology_file, tmp_dir):
# the submitter would have written the .defn file to the tmp_dir
defn_files = glob.glob(tmp_dir + '/*.defn')
if len(defn_files) == 0:
raise Exception("No topologies found")
try:
for defn_file in defn_files:
# load the topology definition from the file
topology_defn = topology_pb2.Topology()
try:
f = open(defn_file, "rb")
topology_defn.ParseFromString(f.read())
f.close()
except:
raise Exception("Could not open and parse topology defn file %s" % defn_file)
# launch the topology
try:
Log.info("Launching topology \'%s\'" % topology_defn.name)
launch_a_topology(cl_args, tmp_dir, topology_file, defn_file)
Log.info("Topology \'%s\' launched successfully" % topology_defn.name)
except Exception as ex:
Log.error('Failed to launch topology \'%s\' because %s' % (topology_defn.name, str(ex)))
raise
except:
raise
def run_bolts(command, parser, cl_args, unknown_args):
cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args['environ']
topology = cl_args['topology-name']
try:
result = utils.get_topology_info(cluster, env, topology, role)
bolts = result['physical_plan']['bolts'].keys()
bolt_name = cl_args['bolt']
if bolt_name:
if bolt_name in bolts:
bolts = [bolt_name]
else:
Log.error('Unknown bolt: \'%s\'' % bolt_name)
raise
except Exception:
return False
bolts_result = []
for bolt in bolts:
try:
metrics = utils.get_component_metrics(bolt, cluster, env, topology, role)
stat, header = to_table(metrics)
bolts_result.append((bolt, stat, header))
except Exception:
return False
for i, (bolt, stat, header) in enumerate(bolts_result):
if i != 0:
print('')
print('\'%s\' metrics:' % bolt)
print(tabulate(stat, headers=header))
return True
def extract_common_args(command, parser, cl_args):
try:
# do not pop like cli because ``topologies`` subcommand still needs it
cluster_role_env = cl_args['cluster/[role]/[env]']
config_path = cl_args['config_path']
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = utils.get_subparser(parser, command)
print(subparser.format_help())
return dict()
cluster = utils.get_heron_cluster(cluster_role_env)
config_path = utils.get_heron_cluster_conf_dir(cluster, config_path)
new_cl_args = dict()
try:
cluster_tuple = utils.parse_cluster_role_env(cluster_role_env, config_path)
new_cl_args['cluster'] = cluster_tuple[0]
new_cl_args['role'] = cluster_tuple[1]
new_cl_args['environ'] = cluster_tuple[2]
new_cl_args['config_path'] = config_path
except Exception as e:
Log.error("Unable to get valid topology location: %s" % str(e))
return dict()
cl_args.update(new_cl_args)
return cl_args
def submit_fatjar(cl_args, unknown_args, tmp_dir):
# execute main of the topology to create the topology definition
topology_file = cl_args['topology-file-name']
try:
execute.heron_class(
cl_args['topology-class-name'],
utils.get_heron_libs(jars.topology_jars()),
extra_jars = [topology_file],
args = tuple(unknown_args),
javaDefines = cl_args['javaDefines'])
except Exception as ex:
Log.error("Unable to execute topology main class")
return False
try:
launch_topologies(cl_args, topology_file, tmp_dir)
except Exception as ex:
return False
finally:
shutil.rmtree(tmp_dir)
return True
def get_topology_info(*args):
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_topology_info(*args))
except Exception as ex:
Log.error(str(ex))
raise
def register_watch(self, callback):
"""
Returns the UUID with which the watch is
registered. This UUID can be used to unregister
the watch.
Returns None if watch could not be registered.
The argument 'callback' must be a function that takes
exactly one argument, the topology on which
the watch was triggered.
Note that the watch will be unregistered in case
it raises any Exception the first time.
This callback is also called at the time
of registration.
"""
RETRY_COUNT = 5
# Retry in case UID is previously
# generated, just in case...
for _ in range(RETRY_COUNT):
# Generate a random UUID.
uid = uuid.uuid4()
if uid not in self.watches:
Log.info("Registering a watch with uid: " + str(uid))
try:
callback(self)
except Exception as e:
Log.error("Caught exception while triggering callback: " + str(e))
Log.debug(traceback.format_exc())
return None
self.watches[uid] = callback
return uid
return None
def extract_common_args(command, parser, cl_args):
try:
cluster_role_env = cl_args.pop('cluster/[role]/[env]')
config_path = cl_args['config_path']
override_config_file = utils.parse_override_config(cl_args['config_property'])
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = utils.get_subparser(parser, command)
print(subparser.format_help())
return dict()
cluster = utils.get_heron_cluster(cluster_role_env)
config_path = utils.get_heron_cluster_conf_dir(cluster, config_path)
if not os.path.isdir(config_path):
Log.error("Config path cluster directory does not exist: %s" % config_path)
return dict()
new_cl_args = dict()
try:
cluster_tuple = utils.parse_cluster_role_env(cluster_role_env, config_path)
new_cl_args['cluster'] = cluster_tuple[0]
new_cl_args['role'] = cluster_tuple[1]
new_cl_args['environ'] = cluster_tuple[2]
new_cl_args['config_path'] = config_path
new_cl_args['override_config_file'] = override_config_file
except Exception as e:
Log.error("Argument cluster/[role]/[env] is not correct: %s" % str(e))
return dict()
cl_args.update(new_cl_args)
return cl_args
def run_metrics(command, parser, cl_args, unknown_args):
""" run metrics subcommand """
cluster, role, env = cl_args["cluster"], cl_args["role"], cl_args["environ"]
topology = cl_args["topology-name"]
try:
result = utils.get_topology_info(cluster, env, topology, role)
spouts = result["physical_plan"]["spouts"].keys()
bolts = result["physical_plan"]["bolts"].keys()
components = spouts + bolts
cname = cl_args["component"]
if cname:
if cname in components:
components = [cname]
else:
Log.error("Unknown component: '%s'" % cname)
raise
except Exception:
return False
cresult = []
for comp in components:
try:
metrics = utils.get_component_metrics(comp, cluster, env, topology, role)
stat, header = to_table(metrics)
cresult.append((comp, stat, header))
except:
return False
for i, (comp, stat, header) in enumerate(cresult):
if i != 0:
print ""
print "'%s' metrics:" % comp
print tabulate(stat, headers=header)
return True
def get_clusters():
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_clusters())
except Exception as ex:
Log.error('Error: %s' % str(ex))
Log.error('Failed to retrive clusters')
raise
def get_topology_metrics(*args):
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_comp_metrics(*args))
except Exception as ex:
Log.error(str(ex))
Log.error("Failed to retrive metrics of component \'%s\'" % args[3])
raise
def get_cluster_topologies(cluster):
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_cluster_topologies(cluster))
except Exception as ex:
Log.error(str(ex))
Log.error('Failed to retrive topologies running in cluster \'%s\'' % cluster)
raise
def get_cluster_role_env_topologies(cluster, role, env):
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_cluster_role_env_topologies(cluster, role, env))
except Exception as ex:
Log.error(str(ex))
Log.error('Failed to retrive topologies running in cluster'
'\'%s\' submitted by %s under environment %s' % (cluster, role, env))
raise
def get_logical_plan(cluster, env, topology, role):
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_logical_plan(cluster, env, topology, role))
except Exception as ex:
Log.error('Error: %s' % str(ex))
Log.error('Failed to retrive logical plan info of topology \'%s\''
% ('/'.join([cluster, role, env, topology])))
raise
def check_release_file_exists():
release_file = get_heron_release_file()
# if the file does not exist and is not a file
if not os.path.isfile(release_file):
Log.error("%s file not found: %s" % release_file)
return False
return True
def check_release_file_exists():
release_file = get_heron_release_file()
# if the file does not exist and is not a file
if not os.path.isfile(release_file):
Log.error("Required file not found: %s" % release_file)
return False
return True
def parse_cluster_role_env(cluster_role_env, config_path):
"""Parse cluster/[role]/[environ], supply default, if not provided, not required"""
parts = cluster_role_env.split('/')[:3]
Log.info("Using config file under %s" % config_path)
if not os.path.isdir(config_path):
Log.error("Config path cluster directory does not exist: %s" %
config_path)
raise Exception("Invalid config path")
# if cluster/role/env is not completely provided, check further
if len(parts) < 3:
cli_conf_file = os.path.join(config_path, CLIENT_YAML)
# if client conf doesn't exist, use default value
if not os.path.isfile(cli_conf_file):
if len(parts) == 1:
parts.append(getpass.getuser())
if len(parts) == 2:
parts.append(ENVIRON)
else:
cli_confs = {}
with open(cli_conf_file, 'r') as conf_file:
tmp_confs = yaml.load(conf_file)
# the return value of yaml.load can be None if conf_file is an empty file
if tmp_confs is not None:
cli_confs = tmp_confs
else:
print "Failed to read: %s due to it is empty" % (
CLIENT_YAML)
# if role is required but not provided, raise exception
if len(parts) == 1:
if (IS_ROLE_REQUIRED in cli_confs) and (
cli_confs[IS_ROLE_REQUIRED] is True):
raise Exception(
"role required but not provided (cluster/role/env = %s). See %s in %s"
% (cluster_role_env, IS_ROLE_REQUIRED, CLIENT_YAML))
else:
parts.append(getpass.getuser())
# if environ is required but not provided, raise exception
if len(parts) == 2:
if (IS_ENV_REQUIRED
in cli_confs) and (cli_confs[IS_ENV_REQUIRED] is True):
raise Exception(
"environ required but not provided (cluster/role/env = %s). See %s in %s"
% (cluster_role_env, IS_ENV_REQUIRED, CLIENT_YAML))
else:
parts.append(ENVIRON)
# if cluster or role or environ is empty, print
if len(parts[0]) == 0 or len(parts[1]) == 0 or len(parts[2]) == 0:
print "Failed to parse"
sys.exit(1)
return (parts[0], parts[1], parts[2])
def check_release_file_exists():
"""Check if the release.yaml file exists"""
release_file = get_heron_release_file()
# if the file does not exist and is not a file
if not os.path.isfile(release_file):
Log.error("Required file not found: %s" % release_file)
return False
return True
def get_cluster_role_topologies(cluster, role):
"""Synced API call to get topologies under a cluster submitted by a role"""
instance = tornado.ioloop.IOLoop.instance()
try:
return instance.run_sync(lambda: API.get_cluster_role_topologies(cluster, role))
except Exception as ex:
Log.error(str(ex))
Log.error('Failed to retrive topologies running in cluster'
'\'%s\' submitted by %s' % (cluster, role))
raise
def get_clusters():
"""Synced API call to get all cluster names"""
instance = tornado.ioloop.IOLoop.instance()
# pylint: disable=unnecessary-lambda
try:
return instance.run_sync(lambda: API.get_clusters())
except Exception as ex:
Log.error('Error: %s' % str(ex))
Log.error('Failed to retrive clusters')
raise
def parse_topo_loc(cl_args):
try:
topo_loc = cl_args['cluster/[role]/[env]'].split('/')
topo_loc.append(cl_args['topology-name'])
if len(topo_loc) != 4:
raise
return topo_loc
except Exception:
Log.error('Error: invalid topology location')
raise
def parse_topo_loc(cl_args):
""" parse topology location """
try:
topo_loc = cl_args['cluster/[role]/[env]'].split('/')
topo_loc.append(cl_args['topology-name'])
if len(topo_loc) != 4:
raise
return topo_loc
except Exception:
Log.error('Error: invalid topology location')
raise
def run(command, parser, cl_args, unknown_args):
""" run command """
try:
clusters = utils.get_clusters()
except:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
print 'Available clusters:'
for cluster in clusters:
print ' %s' % cluster
return True
def run(command, parser, cl_args, unknown_args):
'''
Submits the topology to the scheduler
* Depending on the topology file name extension, we treat the file as a
fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz).
* We upload the topology file to the packer, update zookeeper and launch
scheduler jobs representing that topology
* You can see your topology in Heron UI
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:return:
'''
# get the topology file name
topology_file = cl_args['topology-file-name']
# check to see if the topology file exists
if not os.path.isfile(topology_file):
Log.error("Topology jar|tar|pex file %s does not exist" % topology_file)
return False
# check if it is a valid file type
jar_type = topology_file.endswith(".jar")
tar_type = topology_file.endswith(".tar") or topology_file.endswith(".tar.gz")
pex_type = topology_file.endswith(".pex")
if not jar_type and not tar_type and not pex_type:
Log.error("Unknown file type. Please use .tar or .tar.gz or .jar or .pex file")
return False
# create a temporary directory for topology definition file
tmp_dir = tempfile.mkdtemp()
# if topology needs to be launched in deactivated state, do it so
if cl_args['deploy_deactivated']:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED)
else:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING)
# set the tmp dir and deactivated state in global options
opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir)
opts.set_config('cmdline.topology.initial.state', initial_state)
# check the extension of the file name to see if it is tar/jar file.
if jar_type:
return submit_fatjar(cl_args, unknown_args, tmp_dir)
elif tar_type:
return submit_tar(cl_args, unknown_args, tmp_dir)
elif pex_type:
return submit_pex(cl_args, unknown_args, tmp_dir)
return False
def run(command, parser, cl_args, unknown_args):
""" run command """
try:
clusters = utils.get_clusters()
except:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
print 'Available clusters:'
for cluster in clusters:
print ' %s' % cluster
return True
def run(command, parser, cl_args, unknown_args):
location = cl_args['cluster/[role]/[env]'].split('/')
if len(location) == 1:
return show_cluster(*location)
elif len(location) == 2:
return show_cluster_role(*location)
elif len(location) == 3:
return show_cluster_role_env(*location)
else:
Log.error('Invalid topologies selection')
return False
def parse_topo_loc(cl_args):
""" parse topology location """
try:
topo_loc = cl_args["cluster/[role]/[env]"].split("/")
topo_name = cl_args["topology-name"]
topo_loc.append(topo_name)
if len(topo_loc) != 4:
raise
return topo_loc
except Exception:
Log.error("Invalid topology location")
raise
def run(command, parser, cl_args, unknown_args):
""" run command """
location = cl_args['cluster/[role]/[env]'].split('/')
if len(location) == 1:
return show_cluster(cl_args, *location)
elif len(location) == 2:
return show_cluster_role(cl_args, *location)
elif len(location) == 3:
return show_cluster_role_env(cl_args, *location)
else:
Log.error('Invalid topologies selection')
return False
def parse_cluster_role_env(cluster_role_env, config_path):
"""Parse cluster/[role]/[environ], supply default, if not provided, not required"""
parts = cluster_role_env.split('/')[:3]
Log.info("Using config file under %s" % config_path)
if not os.path.isdir(config_path):
Log.error("Config path cluster directory does not exist: %s" % config_path)
raise Exception("Invalid config path")
# if cluster/role/env is not completely provided, check further
if len(parts) < 3:
cli_conf_file = os.path.join(config_path, CLIENT_YAML)
# if client conf doesn't exist, use default value
if not os.path.isfile(cli_conf_file):
if len(parts) == 1:
parts.append(getpass.getuser())
if len(parts) == 2:
parts.append(ENVIRON)
else:
cli_confs = {}
with open(cli_conf_file, 'r') as conf_file:
tmp_confs = yaml.load(conf_file)
# the return value of yaml.load can be None if conf_file is an empty file
if tmp_confs is not None:
cli_confs = tmp_confs
else:
print "Failed to read: %s due to it is empty" % (CLIENT_YAML)
# if role is required but not provided, raise exception
if len(parts) == 1:
if (IS_ROLE_REQUIRED in cli_confs) and (cli_confs[IS_ROLE_REQUIRED] is True):
raise Exception(
"role required but not provided (cluster/role/env = %s). See %s in %s" %
(cluster_role_env, IS_ROLE_REQUIRED, CLIENT_YAML))
else:
parts.append(getpass.getuser())
# if environ is required but not provided, raise exception
if len(parts) == 2:
if (IS_ENV_REQUIRED in cli_confs) and (cli_confs[IS_ENV_REQUIRED] is True):
raise Exception(
"environ required but not provided (cluster/role/env = %s). See %s in %s" %
(cluster_role_env, IS_ENV_REQUIRED, CLIENT_YAML))
else:
parts.append(ENVIRON)
# if cluster or role or environ is empty, print
if len(parts[0]) == 0 or len(parts[1]) == 0 or len(parts[2]) == 0:
print "Failed to parse"
sys.exit(1)
return (parts[0], parts[1], parts[2])
def run(command, parser, cl_args, unknown_args, action):
'''
helper function to take action on topologies
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:param action: description of action taken
:return:
'''
try:
topology_name = cl_args['topology-name']
new_args = [
"--cluster",
cl_args['cluster'],
"--role",
cl_args['role'],
"--environment",
cl_args['environ'],
"--heron_home",
utils.get_heron_dir(),
"--config_path",
cl_args['config_path'],
"--override_config_file",
cl_args['override_config_file'],
"--release_file",
utils.get_heron_release_file(),
"--topology_name",
topology_name,
"--command",
command,
]
if opts.verbose():
new_args.append("--verbose")
lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args)
except Exception:
Log.error('Failed to %s \'%s\'' % (action, topology_name))
return False
Log.info('Successfully %s \'%s\'' % (action, topology_name))
return True
def run(command, parser, cl_args, unknown_args):
'''
Submits the topology to the scheduler
* Depending on the topology file name extension, we treat the file as a
fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz).
* We upload the topology file to the packer, update zookeeper and launch
scheduler jobs representing that topology
* You can see your topology in Heron UI
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:return:
'''
# get the topology file name
topology_file = cl_args['topology-file-name']
# check to see if the topology file exists
if not os.path.isfile(topology_file):
Log.error("Topology jar|tar file %s does not exist" % topology_file)
return False
# check if it is a valid file type
jar_type = topology_file.endswith(".jar")
tar_type = topology_file.endswith(".tar") or topology_file.endswith(
".tar.gz")
if not jar_type and not tar_type:
Log.error("Unknown file type. Please use .tar or .tar.gz or .jar file")
return False
# create a temporary directory for topology definition file
tmp_dir = tempfile.mkdtemp()
# if topology needs to be launched in deactivated state, do it so
if cl_args['deploy_deactivated']:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED)
else:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING)
# set the tmp dir and deactivated state in global options
opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir)
opts.set_config('cmdline.topology.initial.state', initial_state)
# check the extension of the file name to see if it is tar/jar file.
if jar_type:
return submit_fatjar(cl_args, unknown_args, tmp_dir)
elif tar_type:
return submit_tar(cl_args, unknown_args, tmp_dir)
return False
def check_java_home_set():
"""Check if the java home set"""
# check if environ variable is set
if "JAVA_HOME" not in os.environ:
Log.error("JAVA_HOME not set")
return False
# check if the value set is correct
java_path = get_java_path()
if os.path.isfile(java_path) and os.access(java_path, os.X_OK):
return True
Log.error("JAVA_HOME/bin/java either does not exist or not an executable")
return False
def check_java_home_set():
# check if environ variable is set
if "JAVA_HOME" not in os.environ:
Log.error("JAVA_HOME not set")
return False
# check if the value set is correct
java_path = get_java_path()
if os.path.isfile(java_path) and os.access(java_path, os.X_OK):
return True
Log.error("JAVA_HOME/bin/java either does not exist or not an executable")
return False
def show_cluster(cluster):
try:
result = utils.get_cluster_topologies(cluster)
if not result:
Log.error('Unknown cluster \'%s\'' % cluster)
return False
result = result[cluster]
except Exception:
return False
table, header, rest_count = to_table(result)
print('Topologies running in cluster \'%s\'' % cluster)
if rest_count:
print(' with %d more...' % rest_count)
print(tabulate(table, headers=header))
return True
def show_cluster_role(cluster, role):
try:
result = utils.get_cluster_role_topologies(cluster, role)
if not result:
Log.error('Unknown cluster/role \'%s\'' % '/'.join([cluster, role]))
return False
result = result[cluster]
except Exception:
return False
table, header, rest_count = to_table(result)
print('Topologies running in cluster \'%s\' submitted by \'%s\':' % (cluster, role))
if rest_count:
print(' with %d more...' % rest_count)
print(tabulate(table, headers=header))
return True
def run_containers(command, parser, cl_args, unknown_args):
""" run containers subcommand """
cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args[
'environ']
topology = cl_args['topology-name']
container_id = cl_args['id']
try:
result = utils.get_topology_info(cluster, env, topology, role)
except:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
containers = result['physical_plan']['stmgrs']
all_bolts, all_spouts = set(), set()
for _, bolts in result['physical_plan']['bolts'].items():
all_bolts = all_bolts | set(bolts)
for _, spouts in result['physical_plan']['spouts'].items():
all_spouts = all_spouts | set(spouts)
stmgrs = containers.keys()
stmgrs.sort()
if container_id is not None:
try:
normalized_cid = container_id - 1
if normalized_cid < 0:
raise
stmgrs = [stmgrs[normalized_cid]]
except:
Log.error('Invalid container id: %d' % container_id)
return False
table = []
for sid, name in enumerate(stmgrs):
cid = sid + 1
host = containers[name]["host"]
port = containers[name]["port"]
pid = containers[name]["pid"]
instances = containers[name]["instance_ids"]
bolt_nums = len(
[instance for instance in instances if instance in all_bolts])
spout_nums = len(
[instance for instance in instances if instance in all_spouts])
table.append(
[cid, host, port, pid, bolt_nums, spout_nums,
len(instances)])
headers = [
"container", "host", "port", "pid", "#bolt", "#spout", "#instance"
]
sys.stdout.flush()
print tabulate(table, headers=headers)
return True
def show_cluster(cl_args, cluster):
''' print topologies information to stdout '''
try:
result = utils.get_cluster_topologies(cluster)
if not result:
Log.error('Unknown cluster \'%s\'' % cluster)
return False
result = result[cluster]
except Exception:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
table, header, rest_count = to_table(result)
print 'Topologies running in cluster \'%s\'' % cluster
if rest_count:
print ' with %d more...' % rest_count
print tabulate(table, headers=header)
return True
def show_cluster(cl_args, cluster):
''' print topologies information to stdout '''
try:
result = tracker_access.get_cluster_topologies(cluster)
if not result:
Log.error('No topologies in cluster \'%s\'' % cluster)
return False
result = result[cluster]
except Exception:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
table, header, rest_count = to_table(result)
print 'Topologies running in cluster \'%s\'' % cluster
if rest_count:
print ' with %d more...' % rest_count
print tabulate(table, headers=header)
return True
def run(command, parser, args, unknown_args):
# get the command for detailed help
command_help = args['help-command']
# if no command is provided, just print main help
if command_help == 'help':
parser.print_help()
return True
# get the subparser for the specific command
subparser = utils.get_subparser(parser, command_help)
if subparser:
print(subparser.format_help())
return True
else:
Log.error("Unknown subcommand \'%s\'" % command_help)
return False
def show_cluster_role(cl_args, cluster, role):
''' print topologies information to stdout '''
try:
result = tracker_access.get_cluster_role_topologies(cluster, role)
if not result:
Log.error('Unknown cluster/role \'%s\'' % '/'.join([cluster, role]))
return False
result = result[cluster]
except Exception:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
table, header, rest_count = to_table(result)
print 'Topologies running in cluster \'%s\' submitted by \'%s\':' % (cluster, role)
if rest_count:
print ' with %d more...' % rest_count
print tabulate(table, headers=header)
return True
def run(command, parser, cl_args, unknown_args):
try:
topology_name = cl_args['topology-name']
new_args = [
"--cluster",
cl_args['cluster'],
"--role",
cl_args['role'],
"--environment",
cl_args['environ'],
"--heron_home",
utils.get_heron_dir(),
"--config_path",
cl_args['config_path'],
"--override_config_file",
cl_args['override_config_file'],
"--release_file",
utils.get_heron_release_file(),
"--topology_name",
topology_name,
"--command",
command,
]
if opts.verbose():
new_args.append("--verbose")
lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args)
except Exception as ex:
print 'Error: %s' % str(ex)
Log.error('Failed to activate topology \'%s\'' % topology_name)
return False
Log.info('Successfully activated topology \'%s\'' % topology_name)
return True
def run(command, parser, args, unknown_args):
""" run command """
# get the command for detailed help
command_help = args['help-command']
# if no command is provided, just print main help
if command_help == 'help':
parser.print_help()
return True
# get the subparser for the specific command
subparser = utils.get_subparser(parser, command_help)
if subparser:
print subparser.format_help()
return True
else:
Log.error("Unknown subcommand \'%s\'" % command_help)
return False
def show_cluster_role_env(cl_args, cluster, role, env):
''' print topologies information to stdout '''
try:
result = utils.get_cluster_role_env_topologies(cluster, role, env)
if not result:
Log.error('Unknown cluster/role/env \'%s\'' %
'/'.join([cluster, role, env]))
return False
result = result[cluster]
except Exception:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
table, header, rest_count = to_table(result)
print 'Topologies running in cluster \'%s\', submitted by \'%s\', and\
under environment \'%s\':' % (cluster, role, env)
if rest_count:
print ' with %d more...' % rest_count
print tabulate(table, headers=header)
return True
def run_bolts(command, parser, cl_args, unknown_args):
""" run bolts subcommand """
cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args[
'environ']
topology = cl_args['topology-name']
try:
result = utils.get_topology_info(cluster, env, topology, role)
bolts = result['physical_plan']['bolts'].keys()
bolt_name = cl_args['bolt']
if bolt_name:
if bolt_name in bolts:
bolts = [bolt_name]
else:
Log.error('Unknown bolt: \'%s\'' % bolt_name)
raise
except Exception:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
bolts_result = []
for bolt in bolts:
try:
metrics = utils.get_component_metrics(bolt, cluster, env, topology,
role)
stat, header = to_table(metrics)
bolts_result.append((bolt, stat, header))
except Exception:
Log.error("Fail to connect to tracker: \'%s\'",
cl_args["tracker_url"])
return False
for i, (bolt, stat, header) in enumerate(bolts_result):
if i != 0:
print ''
print '\'%s\' metrics:' % bolt
print tabulate(stat, headers=header)
return True
def submit_fatjar(cl_args, unknown_args, tmp_dir):
'''
We use the packer to make a package for the jar and dump it
to a well-known location. We then run the main method of class
with the specified arguments. We pass arguments as heron.options.
This will run the jar file with the topology_class_name. The submitter
inside will write out the topology defn file to a location that
we specify. Then we write the topology defn file to a well known
location. We then write to appropriate places in zookeeper
and launch the scheduler jobs
:param cl_args:
:param unknown_args:
:param tmp_dir:
:return:
'''
# execute main of the topology to create the topology definition
topology_file = cl_args['topology-file-name']
try:
execute.heron_class(cl_args['topology-class-name'],
utils.get_heron_libs(jars.topology_jars()),
extra_jars=[topology_file],
args=tuple(unknown_args),
java_defines=cl_args['topology_main_jvm_property'])
except Exception:
Log.error("Unable to execute topology main class")
return False
try:
launch_topologies(cl_args, topology_file, tmp_dir)
except Exception:
return False
finally:
shutil.rmtree(tmp_dir)
return True
def run(cl_args, compo_type):
""" run command """
cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args[
'environ']
topology = cl_args['topology-name']
spouts_only, bolts_only = cl_args['spout'], cl_args['bolt']
try:
components = utils.get_logical_plan(cluster, env, topology, role)
topo_info = utils.get_topology_info(cluster, env, topology, role)
table, header = to_table(components, topo_info)
if spouts_only == bolts_only:
print tabulate(table, headers=header)
elif spouts_only:
table, header = filter_spouts(table, header)
print tabulate(table, headers=header)
else:
table, header = filter_bolts(table, header)
print tabulate(table, headers=header)
return True
except:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
def run(command, parser, cl_args, unknown_args):
'''
:param command:
:param parser:
:param cl_args:
:param unknown_args:
:return:
'''
try:
topology_name = cl_args['topology-name']
container_id = cl_args['container-id']
new_args = [
"--cluster", cl_args['cluster'], "--role", cl_args['role'],
"--environment", cl_args['environ'], "--heron_home",
utils.get_heron_dir(), "--config_path", cl_args['config_path'],
"--override_config_file", cl_args['override_config_file'],
"--release_file",
utils.get_heron_release_file(), "--topology_name", topology_name,
"--command", command, "--container_id",
str(container_id)
]
lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
jars.statemgr_jars())
# invoke the runtime manager to kill the topology
execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
lib_jars,
extra_jars=[],
args=new_args)
except Exception as ex:
print 'Error: %s' % str(ex)
Log.error('Failed to restart topology \'%s\'' % topology_name)
return False
Log.info('Successfully restarted topology \'%s\'' % topology_name)
return True
def run(command, parser, cl_args, unknown_args):
# get the topology file name
topology_file = cl_args['topology-file-name']
# check to see if the topology file exists
if not os.path.isfile(topology_file):
Log.error("Topology jar|tar file %s does not exist" % topology_file)
return False
# check if it is a valid file type
jar_type = topology_file.endswith(".jar")
tar_type = topology_file.endswith(".tar") or topology_file.endswith(".tar.gz")
if not jar_type and not tar_type:
Log.error("Unknown file type. Please use .tar or .tar.gz or .jar file")
return False
# create a temporary directory for topology definition file
tmp_dir = tempfile.mkdtemp()
# if topology needs to be launched in deactivated state, do it so
if cl_args['deploy_deactivated']:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED)
else:
initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING)
# set the tmp dir and deactivated state in global options
opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir)
opts.set_config('cmdline.topology.initial.state', initial_state)
# check the extension of the file name to see if it is tar/jar file.
if jar_type:
return submit_fatjar(cl_args, unknown_args, tmp_dir)
elif tar_type:
return submit_tar(cl_args, unknown_args, tmp_dir)
return False
def run_metrics(command, parser, cl_args, unknown_args):
""" run metrics subcommand """
cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args[
'environ']
topology = cl_args['topology-name']
try:
result = utils.get_topology_info(cluster, env, topology, role)
spouts = result['physical_plan']['spouts'].keys()
bolts = result['physical_plan']['bolts'].keys()
components = spouts + bolts
cname = cl_args['component']
if cname:
if cname in components:
components = [cname]
else:
Log.error('Unknown component: \'%s\'' % cname)
raise
except Exception:
Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
return False
cresult = []
for comp in components:
try:
metrics = utils.get_component_metrics(comp, cluster, env, topology,
role)
except:
Log.error("Fail to connect to tracker: \'%s\'",
cl_args["tracker_url"])
return False
stat, header = to_table(metrics)
cresult.append((comp, stat, header))
for i, (comp, stat, header) in enumerate(cresult):
if i != 0:
print ''
print '\'%s\' metrics:' % comp
print tabulate(stat, headers=header)
return True
