def run_bolts(command, parser, cl_args, unknown_args): cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args['environ'] topology = cl_args['topology-name'] try: result = utils.get_topology_info(cluster, env, topology, role) bolts = result['physical_plan']['bolts'].keys() bolt_name = cl_args['bolt'] if bolt_name: if bolt_name in bolts: bolts = [bolt_name] else: Log.error('Unknown bolt: \'%s\'' % bolt_name) raise except Exception: return False bolts_result = [] for bolt in bolts: try: metrics = utils.get_component_metrics(bolt, cluster, env, topology, role) stat, header = to_table(metrics) bolts_result.append((bolt, stat, header)) except Exception: return False for i, (bolt, stat, header) in enumerate(bolts_result): if i != 0: print('') print('\'%s\' metrics:' % bolt) print(tabulate(stat, headers=header)) return True
def extract_common_args(command, parser, cl_args): try: cluster_role_env = cl_args.pop('cluster/[role]/[env]') config_path = cl_args['config_path'] override_config_file = utils.parse_override_config(cl_args['config_property']) except KeyError: # if some of the arguments are not found, print error and exit subparser = utils.get_subparser(parser, command) print(subparser.format_help()) return dict() cluster = utils.get_heron_cluster(cluster_role_env) config_path = utils.get_heron_cluster_conf_dir(cluster, config_path) if not os.path.isdir(config_path): Log.error("Config path cluster directory does not exist: %s" % config_path) return dict() new_cl_args = dict() try: cluster_tuple = utils.parse_cluster_role_env(cluster_role_env, config_path) new_cl_args['cluster'] = cluster_tuple[0] new_cl_args['role'] = cluster_tuple[1] new_cl_args['environ'] = cluster_tuple[2] new_cl_args['config_path'] = config_path new_cl_args['override_config_file'] = override_config_file except Exception as e: Log.error("Argument cluster/[role]/[env] is not correct: %s" % str(e)) return dict() cl_args.update(new_cl_args) return cl_args
def main(): ''' Run the command :return: ''' # verify if the environment variables are correctly set check_environment() # create the argument parser parser = create_parser() # if no argument is provided, print help and exit if len(sys.argv[1:]) == 0: parser.print_help() return 0 # insert the boolean values for some of the options sys.argv = parse.insert_bool_values(sys.argv) # parse the args args, unknown_args = parser.parse_known_args() command_line_args = vars(args) try: if command_line_args['verbose']: opts.set_verbose() if command_line_args['trace_execution']: opts.set_trace_execution() except: pass # command to be execute command = command_line_args['subcommand'] # file resources to be cleaned when exit files = [] if command != 'help' and command != 'version': command_line_args = extract_common_args(command, parser, command_line_args) # bail out if args are empty if not command_line_args: return 1 # register dirs cleanup function during exit files.append(command_line_args['override_config_file']) atexit.register(cleanup, files) # print the input parameters, if verbose is enabled if opts.verbose(): print command_line_args start = time.time() retcode = run(command, parser, command_line_args, unknown_args) end = time.time() if command != 'help': sys.stdout.flush() Log.info('Elapsed time: %.3fs.' % (end - start)) return 0 if retcode else 1
def run_bolts(command, parser, cl_args, unknown_args): """ run bolts subcommand """ cluster, role, env = cl_args["cluster"], cl_args["role"], cl_args["environ"] topology = cl_args["topology-name"] try: result = utils.get_topology_info(cluster, env, topology, role) bolts = result["physical_plan"]["bolts"].keys() bolt_name = cl_args["bolt"] if bolt_name: if bolt_name in bolts: bolts = [bolt_name] else: Log.error("Unknown bolt: '%s'" % bolt_name) raise except Exception: return False bolts_result = [] for bolt in bolts: try: metrics = utils.get_component_metrics(bolt, cluster, env, topology, role) stat, header = to_table(metrics) bolts_result.append((bolt, stat, header)) except Exception: return False for i, (bolt, stat, header) in enumerate(bolts_result): if i != 0: print "" print "'%s' metrics:" % bolt print tabulate(stat, headers=header) return True
def submit_fatjar(cl_args, unknown_args, tmp_dir): # execute main of the topology to create the topology definition topology_file = cl_args['topology-file-name'] try: execute.heron_class( cl_args['topology-class-name'], utils.get_heron_libs(jars.topology_jars()), extra_jars = [topology_file], args = tuple(unknown_args), javaDefines = cl_args['javaDefines']) except Exception as ex: Log.error("Unable to execute topology main class") return False try: launch_topologies(cl_args, topology_file, tmp_dir) except Exception as ex: return False finally: shutil.rmtree(tmp_dir) return True
def run_metrics(command, parser, cl_args, unknown_args): """ run metrics subcommand """ cluster, role, env = cl_args["cluster"], cl_args["role"], cl_args["environ"] topology = cl_args["topology-name"] try: result = utils.get_topology_info(cluster, env, topology, role) spouts = result["physical_plan"]["spouts"].keys() bolts = result["physical_plan"]["bolts"].keys() components = spouts + bolts cname = cl_args["component"] if cname: if cname in components: components = [cname] else: Log.error("Unknown component: '%s'" % cname) raise except Exception: return False cresult = [] for comp in components: try: metrics = utils.get_component_metrics(comp, cluster, env, topology, role) stat, header = to_table(metrics) cresult.append((comp, stat, header)) except: return False for i, (comp, stat, header) in enumerate(cresult): if i != 0: print "" print "'%s' metrics:" % comp print tabulate(stat, headers=header) return True
def get_topology_info(*args): instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_topology_info(*args)) except Exception as ex: Log.error(str(ex)) raise
def extract_common_args(command, parser, cl_args): try: # do not pop like cli because ``topologies`` subcommand still needs it cluster_role_env = cl_args['cluster/[role]/[env]'] config_path = cl_args['config_path'] except KeyError: # if some of the arguments are not found, print error and exit subparser = utils.get_subparser(parser, command) print(subparser.format_help()) return dict() cluster = utils.get_heron_cluster(cluster_role_env) config_path = utils.get_heron_cluster_conf_dir(cluster, config_path) new_cl_args = dict() try: cluster_tuple = utils.parse_cluster_role_env(cluster_role_env, config_path) new_cl_args['cluster'] = cluster_tuple[0] new_cl_args['role'] = cluster_tuple[1] new_cl_args['environ'] = cluster_tuple[2] new_cl_args['config_path'] = config_path except Exception as e: Log.error("Unable to get valid topology location: %s" % str(e)) return dict() cl_args.update(new_cl_args) return cl_args
def get(self): """ get method """ try: cluster = self.get_argument_cluster() role = self.get_argument_role() environ = self.get_argument_environ() topology_name = self.get_argument_topology() container = self.get_argument(constants.PARAM_CONTAINER) path = self.get_argument(constants.PARAM_PATH) offset = self.get_argument_offset() length = self.get_argument_length() topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ) stmgr_id = "stmgr-" + container stmgr = topology_info["physical_plan"]["stmgrs"][stmgr_id] host = stmgr["host"] shell_port = stmgr["shell_port"] file_data_url = "http://%s:%d/filedata/%s?offset=%s&length=%s" % \ (host, shell_port, path, offset, length) http_client = tornado.httpclient.AsyncHTTPClient() response = yield http_client.fetch(file_data_url) self.write_success_response(json.loads(response.body)) self.finish() except Exception as e: Log.debug(traceback.format_exc()) self.write_error_response(e)
def run_metrics(command, parser, cl_args, unknown_args): cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args['environ'] topology = cl_args['topology-name'] try: result = utils.get_topology_info(cluster, env, topology, role) spouts = result['physical_plan']['spouts'].keys() bolts = result['physical_plan']['bolts'].keys() components = spouts + bolts cname = cl_args['component'] if cname: if cname in components: components = [cname] else: Log.error('Unknown component: \'%s\'' % cname) raise except Exception: return False cresult = [] for comp in components: try: metrics = utils.get_component_metrics(comp, cluster, env, topology, role) stat, header = to_table(metrics) cresult.append((comp, stat, header)) except: return False for i, (comp, stat, header) in enumerate(cresult): if i != 0: print('') print('\'%s\' metrics:' % comp) print(tabulate(stat, headers=header)) return True
def get_cluster_topologies(cluster): """Synced API call to get topologies under a cluster""" instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_cluster_topologies(cluster)) except Exception: Log.info(traceback.format_exc()) raise
def get_topology_metrics(*args): """Synced API call to get topology metrics""" instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_comp_metrics(*args)) except Exception: Log.info(traceback.format_exc()) raise
def get_topology_metrics(*args): instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_comp_metrics(*args)) except Exception as ex: Log.error(str(ex)) Log.error("Failed to retrive metrics of component \'%s\'" % args[3]) raise
def unregister_watch(self, uid): """ Unregister the watch with the given UUID. """ # Do not raise an error if UUID is # not present in the watches. Log.info("Unregister a watch with uid: " + str(uid)) self.watches.pop(uid, None)
def get_clusters(): instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_clusters()) except Exception as ex: Log.error('Error: %s' % str(ex)) Log.error('Failed to retrive clusters') raise
def get_logical_plan(cluster, env, topology, role): """Synced API call to get logical plans""" instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_logical_plan(cluster, env, topology, role)) except Exception: Log.info(traceback.format_exc()) raise
def get_cluster_role_env_topologies(cluster, role, env): """Synced API call to get topologies under a cluster submitted by a role under env""" instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_cluster_role_env_topologies(cluster, role, env)) except Exception: Log.info(traceback.format_exc()) raise
def get_cluster_topologies(cluster): instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_cluster_topologies(cluster)) except Exception as ex: Log.error(str(ex)) Log.error('Failed to retrive topologies running in cluster \'%s\'' % cluster) raise
def main(): # verify if the environment variables are correctly set check_environment() # create the argument parser parser = create_parser() # if no argument is provided, print help and exit if len(sys.argv[1:]) == 0: parser.print_help() return 0 # insert the boolean values for some of the options sys.argv = parse.insert_bool_values(sys.argv) # parse the args args, unknown_args = parser.parse_known_args() command_line_args = vars(args) try: if command_line_args['verbose']: opts.set_verbose() if command_line_args['trace_execution']: opts.set_trace_execution() except: pass # command to be execute command = command_line_args['subcommand'] # file resources to be cleaned when exit files = [] if command != 'help' and command != 'version': command_line_args = extract_common_args(command, parser, command_line_args) # bail out if args are empty if not command_line_args: return 1 # register dirs cleanup function during exit files.append(command_line_args['override_config_file']) atexit.register(cleanup, files) # print the input parameters, if verbose is enabled if opts.verbose(): print command_line_args start = time.time() retcode = run(command, parser, command_line_args, unknown_args) end = time.time() if command != 'help': sys.stdout.flush() Log.info('Elapsed time: %.3fs.' % (end - start)) return 0 if retcode == True else 1
def get_cluster_role_env_topologies(cluster, role, env): instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_cluster_role_env_topologies(cluster, role, env)) except Exception as ex: Log.error(str(ex)) Log.error('Failed to retrive topologies running in cluster' '\'%s\' submitted by %s under environment %s' % (cluster, role, env)) raise
def parse_cluster_role_env(cluster_role_env, config_path): """Parse cluster/[role]/[environ], supply default, if not provided, not required""" parts = cluster_role_env.split('/')[:3] Log.info("Using config file under %s" % config_path) if not os.path.isdir(config_path): Log.error("Config path cluster directory does not exist: %s" % config_path) raise Exception("Invalid config path") # if cluster/role/env is not completely provided, check further if len(parts) < 3: cli_conf_file = os.path.join(config_path, CLIENT_YAML) # if client conf doesn't exist, use default value if not os.path.isfile(cli_conf_file): if len(parts) == 1: parts.append(getpass.getuser()) if len(parts) == 2: parts.append(ENVIRON) else: cli_confs = {} with open(cli_conf_file, 'r') as conf_file: tmp_confs = yaml.load(conf_file) # the return value of yaml.load can be None if conf_file is an empty file if tmp_confs is not None: cli_confs = tmp_confs else: print "Failed to read: %s due to it is empty" % ( CLIENT_YAML) # if role is required but not provided, raise exception if len(parts) == 1: if (IS_ROLE_REQUIRED in cli_confs) and ( cli_confs[IS_ROLE_REQUIRED] is True): raise Exception( "role required but not provided (cluster/role/env = %s). See %s in %s" % (cluster_role_env, IS_ROLE_REQUIRED, CLIENT_YAML)) else: parts.append(getpass.getuser()) # if environ is required but not provided, raise exception if len(parts) == 2: if (IS_ENV_REQUIRED in cli_confs) and (cli_confs[IS_ENV_REQUIRED] is True): raise Exception( "environ required but not provided (cluster/role/env = %s). See %s in %s" % (cluster_role_env, IS_ENV_REQUIRED, CLIENT_YAML)) else: parts.append(ENVIRON) # if cluster or role or environ is empty, print if len(parts[0]) == 0 or len(parts[1]) == 0 or len(parts[2]) == 0: print "Failed to parse" sys.exit(1) return (parts[0], parts[1], parts[2])
def get_logical_plan(cluster, env, topology, role): """Synced API call to get logical plans""" instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync( lambda: API.get_logical_plan(cluster, env, topology, role)) except Exception: Log.info(traceback.format_exc()) raise
def get_clusters(): """Synced API call to get all cluster names""" instance = tornado.ioloop.IOLoop.instance() # pylint: disable=unnecessary-lambda try: return instance.run_sync(lambda: API.get_clusters()) except Exception: Log.info(traceback.format_exc()) raise
def get_cluster_role_env_topologies(cluster, role, env): """Synced API call to get topologies under a cluster submitted by a role under env""" instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync( lambda: API.get_cluster_role_env_topologies(cluster, role, env)) except Exception: Log.info(traceback.format_exc()) raise
def check_release_file_exists(): release_file = get_heron_release_file() # if the file does not exist and is not a file if not os.path.isfile(release_file): Log.error("%s file not found: %s" % release_file) return False return True
def check_release_file_exists(): release_file = get_heron_release_file() # if the file does not exist and is not a file if not os.path.isfile(release_file): Log.error("Required file not found: %s" % release_file) return False return True
def get_logical_plan(cluster, env, topology, role): instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_logical_plan(cluster, env, topology, role)) except Exception as ex: Log.error('Error: %s' % str(ex)) Log.error('Failed to retrive logical plan info of topology \'%s\'' % ('/'.join([cluster, role, env, topology]))) raise
def check_release_file_exists(): """Check if the release.yaml file exists""" release_file = get_heron_release_file() # if the file does not exist and is not a file if not os.path.isfile(release_file): Log.error("Required file not found: %s" % release_file) return False return True
def get_clusters(): """Synced API call to get all cluster names""" instance = tornado.ioloop.IOLoop.instance() # pylint: disable=unnecessary-lambda try: return instance.run_sync(lambda: API.get_clusters()) except Exception as ex: Log.error('Error: %s' % str(ex)) Log.error('Failed to retrive clusters') raise
def get_component_metrics(component, cluster, env, topology, role): """Synced API call to get component metrics""" all_queries = metric_queries() try: result = get_topology_metrics(cluster, env, topology, component, [], all_queries, [0, -1], role) return result["metrics"] except Exception: Log.info(traceback.format_exc()) raise
def parse_topo_loc(cl_args): try: topo_loc = cl_args['cluster/[role]/[env]'].split('/') topo_loc.append(cl_args['topology-name']) if len(topo_loc) != 4: raise return topo_loc except Exception: Log.error('Error: invalid topology location') raise
def get_cluster_role_topologies(cluster, role): """Synced API call to get topologies under a cluster submitted by a role""" instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_cluster_role_topologies(cluster, role)) except Exception as ex: Log.error(str(ex)) Log.error('Failed to retrive topologies running in cluster' '\'%s\' submitted by %s' % (cluster, role)) raise
def run(command, parser, cl_args, unknown_args): ''' Submits the topology to the scheduler * Depending on the topology file name extension, we treat the file as a fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz). * We upload the topology file to the packer, update zookeeper and launch scheduler jobs representing that topology * You can see your topology in Heron UI :param command: :param parser: :param cl_args: :param unknown_args: :return: ''' # get the topology file name topology_file = cl_args['topology-file-name'] # check to see if the topology file exists if not os.path.isfile(topology_file): Log.error("Topology jar|tar|pex file %s does not exist" % topology_file) return False # check if it is a valid file type jar_type = topology_file.endswith(".jar") tar_type = topology_file.endswith(".tar") or topology_file.endswith(".tar.gz") pex_type = topology_file.endswith(".pex") if not jar_type and not tar_type and not pex_type: Log.error("Unknown file type. Please use .tar or .tar.gz or .jar or .pex file") return False # create a temporary directory for topology definition file tmp_dir = tempfile.mkdtemp() # if topology needs to be launched in deactivated state, do it so if cl_args['deploy_deactivated']: initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED) else: initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING) # set the tmp dir and deactivated state in global options opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir) opts.set_config('cmdline.topology.initial.state', initial_state) # check the extension of the file name to see if it is tar/jar file. if jar_type: return submit_fatjar(cl_args, unknown_args, tmp_dir) elif tar_type: return submit_tar(cl_args, unknown_args, tmp_dir) elif pex_type: return submit_pex(cl_args, unknown_args, tmp_dir) return False
def run(command, parser, cl_args, unknown_args): """ run command """ try: clusters = utils.get_clusters() except: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False print 'Available clusters:' for cluster in clusters: print ' %s' % cluster return True
def parse_topo_loc(cl_args): """ parse topology location """ try: topo_loc = cl_args['cluster/[role]/[env]'].split('/') topo_loc.append(cl_args['topology-name']) if len(topo_loc) != 4: raise return topo_loc except Exception: Log.error('Error: invalid topology location') raise
def run(command, parser, cl_args, unknown_args): location = cl_args['cluster/[role]/[env]'].split('/') if len(location) == 1: return show_cluster(*location) elif len(location) == 2: return show_cluster_role(*location) elif len(location) == 3: return show_cluster_role_env(*location) else: Log.error('Invalid topologies selection') return False
def run(command, parser, cl_args, unknown_args): """ run command """ location = cl_args['cluster/[role]/[env]'].split('/') if len(location) == 1: return show_cluster(cl_args, *location) elif len(location) == 2: return show_cluster_role(cl_args, *location) elif len(location) == 3: return show_cluster_role_env(cl_args, *location) else: Log.error('Invalid topologies selection') return False
def get(self): """ get method """ try: cluster = self.get_argument_cluster() role = self.get_argument_role() environ = self.get_argument_environ() topology_name = self.get_argument_topology() topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ) self.write_success_response(topology_info) except Exception as e: Log.debug(traceback.format_exc()) self.write_error_response(e)
def run(command, parser, cl_args, unknown_args, action): ''' helper function to take action on topologies :param command: :param parser: :param cl_args: :param unknown_args: :param action: description of action taken :return: ''' try: topology_name = cl_args['topology-name'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", utils.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", utils.get_heron_release_file(), "--topology_name", topology_name, "--command", command, ] if opts.verbose(): new_args.append("--verbose") lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) except Exception: Log.error('Failed to %s \'%s\'' % (action, topology_name)) return False Log.info('Successfully %s \'%s\'' % (action, topology_name)) return True
def run(command, parser, cl_args, unknown_args): ''' Submits the topology to the scheduler * Depending on the topology file name extension, we treat the file as a fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz). * We upload the topology file to the packer, update zookeeper and launch scheduler jobs representing that topology * You can see your topology in Heron UI :param command: :param parser: :param cl_args: :param unknown_args: :return: ''' # get the topology file name topology_file = cl_args['topology-file-name'] # check to see if the topology file exists if not os.path.isfile(topology_file): Log.error("Topology jar|tar file %s does not exist" % topology_file) return False # check if it is a valid file type jar_type = topology_file.endswith(".jar") tar_type = topology_file.endswith(".tar") or topology_file.endswith( ".tar.gz") if not jar_type and not tar_type: Log.error("Unknown file type. Please use .tar or .tar.gz or .jar file") return False # create a temporary directory for topology definition file tmp_dir = tempfile.mkdtemp() # if topology needs to be launched in deactivated state, do it so if cl_args['deploy_deactivated']: initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED) else: initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING) # set the tmp dir and deactivated state in global options opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir) opts.set_config('cmdline.topology.initial.state', initial_state) # check the extension of the file name to see if it is tar/jar file. if jar_type: return submit_fatjar(cl_args, unknown_args, tmp_dir) elif tar_type: return submit_tar(cl_args, unknown_args, tmp_dir) return False
def check_java_home_set(): """Check if the java home set""" # check if environ variable is set if "JAVA_HOME" not in os.environ: Log.error("JAVA_HOME not set") return False # check if the value set is correct java_path = get_java_path() if os.path.isfile(java_path) and os.access(java_path, os.X_OK): return True Log.error("JAVA_HOME/bin/java either does not exist or not an executable") return False
def run_containers(command, parser, cl_args, unknown_args): """ run containers subcommand """ cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args[ 'environ'] topology = cl_args['topology-name'] container_id = cl_args['id'] try: result = utils.get_topology_info(cluster, env, topology, role) except: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False containers = result['physical_plan']['stmgrs'] all_bolts, all_spouts = set(), set() for _, bolts in result['physical_plan']['bolts'].items(): all_bolts = all_bolts | set(bolts) for _, spouts in result['physical_plan']['spouts'].items(): all_spouts = all_spouts | set(spouts) stmgrs = containers.keys() stmgrs.sort() if container_id is not None: try: normalized_cid = container_id - 1 if normalized_cid < 0: raise stmgrs = [stmgrs[normalized_cid]] except: Log.error('Invalid container id: %d' % container_id) return False table = [] for sid, name in enumerate(stmgrs): cid = sid + 1 host = containers[name]["host"] port = containers[name]["port"] pid = containers[name]["pid"] instances = containers[name]["instance_ids"] bolt_nums = len( [instance for instance in instances if instance in all_bolts]) spout_nums = len( [instance for instance in instances if instance in all_spouts]) table.append( [cid, host, port, pid, bolt_nums, spout_nums, len(instances)]) headers = [ "container", "host", "port", "pid", "#bolt", "#spout", "#instance" ] sys.stdout.flush() print tabulate(table, headers=headers) return True
def show_cluster(cl_args, cluster): ''' print topologies information to stdout ''' try: result = utils.get_cluster_topologies(cluster) if not result: Log.error('Unknown cluster \'%s\'' % cluster) return False result = result[cluster] except Exception: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False table, header, rest_count = to_table(result) print 'Topologies running in cluster \'%s\'' % cluster if rest_count: print ' with %d more...' % rest_count print tabulate(table, headers=header) return True
def run(command, parser, cl_args, unknown_args): try: topology_name = cl_args['topology-name'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", utils.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", utils.get_heron_release_file(), "--topology_name", topology_name, "--command", command, ] if opts.verbose(): new_args.append("--verbose") lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) except Exception as ex: print 'Error: %s' % str(ex) Log.error('Failed to activate topology \'%s\'' % topology_name) return False Log.info('Successfully activated topology \'%s\'' % topology_name) return True
def run(command, parser, args, unknown_args): """ run command """ # get the command for detailed help command_help = args['help-command'] # if no command is provided, just print main help if command_help == 'help': parser.print_help() return True # get the subparser for the specific command subparser = utils.get_subparser(parser, command_help) if subparser: print subparser.format_help() return True else: Log.error("Unknown subcommand \'%s\'" % command_help) return False
def show_cluster_role_env(cl_args, cluster, role, env): ''' print topologies information to stdout ''' try: result = utils.get_cluster_role_env_topologies(cluster, role, env) if not result: Log.error('Unknown cluster/role/env \'%s\'' % '/'.join([cluster, role, env])) return False result = result[cluster] except Exception: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False table, header, rest_count = to_table(result) print 'Topologies running in cluster \'%s\', submitted by \'%s\', and\ under environment \'%s\':' % (cluster, role, env) if rest_count: print ' with %d more...' % rest_count print tabulate(table, headers=header) return True
def launch_topologies(cl_args, topology_file, tmp_dir): # the submitter would have written the .defn file to the tmp_dir defn_files = glob.glob(tmp_dir + '/*.defn') if len(defn_files) == 0: raise Exception("No topologies found") try: for defn_file in defn_files: # load the topology definition from the file topology_defn = topology_pb2.Topology() try: f = open(defn_file, "rb") topology_defn.ParseFromString(f.read()) f.close() except: raise Exception("Could not open and parse topology defn file %s" % defn_file) # launch the topology try: Log.info("Launching topology \'%s\'" % topology_defn.name) launch_a_topology(cl_args, tmp_dir, topology_file, defn_file) Log.info("Topology \'%s\' launched successfully" % topology_defn.name) except Exception as ex: Log.error('Failed to launch topology \'%s\' because %s' % (topology_defn.name, str(ex))) raise except: raise
def run_bolts(command, parser, cl_args, unknown_args): """ run bolts subcommand """ cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args[ 'environ'] topology = cl_args['topology-name'] try: result = utils.get_topology_info(cluster, env, topology, role) bolts = result['physical_plan']['bolts'].keys() bolt_name = cl_args['bolt'] if bolt_name: if bolt_name in bolts: bolts = [bolt_name] else: Log.error('Unknown bolt: \'%s\'' % bolt_name) raise except Exception: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False bolts_result = [] for bolt in bolts: try: metrics = utils.get_component_metrics(bolt, cluster, env, topology, role) stat, header = to_table(metrics) bolts_result.append((bolt, stat, header)) except Exception: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False for i, (bolt, stat, header) in enumerate(bolts_result): if i != 0: print '' print '\'%s\' metrics:' % bolt print tabulate(stat, headers=header) return True
def run(cl_args, compo_type): """ run command """ cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args[ 'environ'] topology = cl_args['topology-name'] spouts_only, bolts_only = cl_args['spout'], cl_args['bolt'] try: components = utils.get_logical_plan(cluster, env, topology, role) topo_info = utils.get_topology_info(cluster, env, topology, role) table, header = to_table(components, topo_info) if spouts_only == bolts_only: print tabulate(table, headers=header) elif spouts_only: table, header = filter_spouts(table, header) print tabulate(table, headers=header) else: table, header = filter_bolts(table, header) print tabulate(table, headers=header) return True except: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False
def submit_fatjar(cl_args, unknown_args, tmp_dir): ''' We use the packer to make a package for the jar and dump it to a well-known location. We then run the main method of class with the specified arguments. We pass arguments as heron.options. This will run the jar file with the topology_class_name. The submitter inside will write out the topology defn file to a location that we specify. Then we write the topology defn file to a well known location. We then write to appropriate places in zookeeper and launch the scheduler jobs :param cl_args: :param unknown_args: :param tmp_dir: :return: ''' # execute main of the topology to create the topology definition topology_file = cl_args['topology-file-name'] try: execute.heron_class(cl_args['topology-class-name'], utils.get_heron_libs(jars.topology_jars()), extra_jars=[topology_file], args=tuple(unknown_args), java_defines=cl_args['topology_main_jvm_property']) except Exception: Log.error("Unable to execute topology main class") return False try: launch_topologies(cl_args, topology_file, tmp_dir) except Exception: return False finally: shutil.rmtree(tmp_dir) return True
def run(command, parser, cl_args, unknown_args): ''' :param command: :param parser: :param cl_args: :param unknown_args: :return: ''' try: topology_name = cl_args['topology-name'] container_id = cl_args['container-id'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", utils.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", utils.get_heron_release_file(), "--topology_name", topology_name, "--command", command, "--container_id", str(container_id) ] lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) except Exception as ex: print 'Error: %s' % str(ex) Log.error('Failed to restart topology \'%s\'' % topology_name) return False Log.info('Successfully restarted topology \'%s\'' % topology_name) return True
def run(command, parser, cl_args, unknown_args): # get the topology file name topology_file = cl_args['topology-file-name'] # check to see if the topology file exists if not os.path.isfile(topology_file): Log.error("Topology jar|tar file %s does not exist" % topology_file) return False # check if it is a valid file type jar_type = topology_file.endswith(".jar") tar_type = topology_file.endswith(".tar") or topology_file.endswith(".tar.gz") if not jar_type and not tar_type: Log.error("Unknown file type. Please use .tar or .tar.gz or .jar file") return False # create a temporary directory for topology definition file tmp_dir = tempfile.mkdtemp() # if topology needs to be launched in deactivated state, do it so if cl_args['deploy_deactivated']: initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED) else: initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING) # set the tmp dir and deactivated state in global options opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir) opts.set_config('cmdline.topology.initial.state', initial_state) # check the extension of the file name to see if it is tar/jar file. if jar_type: return submit_fatjar(cl_args, unknown_args, tmp_dir) elif tar_type: return submit_tar(cl_args, unknown_args, tmp_dir) return False
def run_metrics(command, parser, cl_args, unknown_args): """ run metrics subcommand """ cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args[ 'environ'] topology = cl_args['topology-name'] try: result = utils.get_topology_info(cluster, env, topology, role) spouts = result['physical_plan']['spouts'].keys() bolts = result['physical_plan']['bolts'].keys() components = spouts + bolts cname = cl_args['component'] if cname: if cname in components: components = [cname] else: Log.error('Unknown component: \'%s\'' % cname) raise except Exception: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False cresult = [] for comp in components: try: metrics = utils.get_component_metrics(comp, cluster, env, topology, role) except: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False stat, header = to_table(metrics) cresult.append((comp, stat, header)) for i, (comp, stat, header) in enumerate(cresult): if i != 0: print '' print '\'%s\' metrics:' % comp print tabulate(stat, headers=header) return True