def extract_common_args(command, parser, cl_args): try: cluster_role_env = cl_args.pop('cluster/[role]/[env]') config_path = cl_args['config_path'] override_config_file = utils.parse_override_config(cl_args['config_property']) except KeyError: # if some of the arguments are not found, print error and exit subparser = utils.get_subparser(parser, command) print(subparser.format_help()) return dict() cluster = utils.get_heron_cluster(cluster_role_env) config_path = utils.get_heron_cluster_conf_dir(cluster, config_path) if not os.path.isdir(config_path): Log.error("Config path cluster directory does not exist: %s" % config_path) return dict() new_cl_args = dict() try: cluster_tuple = utils.parse_cluster_role_env(cluster_role_env, config_path) new_cl_args['cluster'] = cluster_tuple[0] new_cl_args['role'] = cluster_tuple[1] new_cl_args['environ'] = cluster_tuple[2] new_cl_args['config_path'] = config_path new_cl_args['override_config_file'] = override_config_file except Exception as e: Log.error("Argument cluster/[role]/[env] is not correct: %s" % str(e)) return dict() cl_args.update(new_cl_args) return cl_args
def run_metrics(command, parser, cl_args, unknown_args): cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args['environ'] topology = cl_args['topology-name'] try: result = utils.get_topology_info(cluster, env, topology, role) spouts = result['physical_plan']['spouts'].keys() bolts = result['physical_plan']['bolts'].keys() components = spouts + bolts cname = cl_args['component'] if cname: if cname in components: components = [cname] else: Log.error('Unknown component: \'%s\'' % cname) raise except Exception: return False cresult = [] for comp in components: try: metrics = utils.get_component_metrics(comp, cluster, env, topology, role) stat, header = to_table(metrics) cresult.append((comp, stat, header)) except: return False for i, (comp, stat, header) in enumerate(cresult): if i != 0: print('') print('\'%s\' metrics:' % comp) print(tabulate(stat, headers=header)) return True
def run_bolts(command, parser, cl_args, unknown_args): """ run bolts subcommand """ cluster, role, env = cl_args["cluster"], cl_args["role"], cl_args["environ"] topology = cl_args["topology-name"] try: result = utils.get_topology_info(cluster, env, topology, role) bolts = result["physical_plan"]["bolts"].keys() bolt_name = cl_args["bolt"] if bolt_name: if bolt_name in bolts: bolts = [bolt_name] else: Log.error("Unknown bolt: '%s'" % bolt_name) raise except Exception: return False bolts_result = [] for bolt in bolts: try: metrics = utils.get_component_metrics(bolt, cluster, env, topology, role) stat, header = to_table(metrics) bolts_result.append((bolt, stat, header)) except Exception: return False for i, (bolt, stat, header) in enumerate(bolts_result): if i != 0: print "" print "'%s' metrics:" % bolt print tabulate(stat, headers=header) return True
def launch_topologies(cl_args, topology_file, tmp_dir): # the submitter would have written the .defn file to the tmp_dir defn_files = glob.glob(tmp_dir + '/*.defn') if len(defn_files) == 0: raise Exception("No topologies found") try: for defn_file in defn_files: # load the topology definition from the file topology_defn = topology_pb2.Topology() try: f = open(defn_file, "rb") topology_defn.ParseFromString(f.read()) f.close() except: raise Exception("Could not open and parse topology defn file %s" % defn_file) # launch the topology try: Log.info("Launching topology \'%s\'" % topology_defn.name) launch_a_topology(cl_args, tmp_dir, topology_file, defn_file) Log.info("Topology \'%s\' launched successfully" % topology_defn.name) except Exception as ex: Log.error('Failed to launch topology \'%s\' because %s' % (topology_defn.name, str(ex))) raise except: raise
def run_bolts(command, parser, cl_args, unknown_args): cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args['environ'] topology = cl_args['topology-name'] try: result = utils.get_topology_info(cluster, env, topology, role) bolts = result['physical_plan']['bolts'].keys() bolt_name = cl_args['bolt'] if bolt_name: if bolt_name in bolts: bolts = [bolt_name] else: Log.error('Unknown bolt: \'%s\'' % bolt_name) raise except Exception: return False bolts_result = [] for bolt in bolts: try: metrics = utils.get_component_metrics(bolt, cluster, env, topology, role) stat, header = to_table(metrics) bolts_result.append((bolt, stat, header)) except Exception: return False for i, (bolt, stat, header) in enumerate(bolts_result): if i != 0: print('') print('\'%s\' metrics:' % bolt) print(tabulate(stat, headers=header)) return True
def extract_common_args(command, parser, cl_args): try: # do not pop like cli because ``topologies`` subcommand still needs it cluster_role_env = cl_args['cluster/[role]/[env]'] config_path = cl_args['config_path'] except KeyError: # if some of the arguments are not found, print error and exit subparser = utils.get_subparser(parser, command) print(subparser.format_help()) return dict() cluster = utils.get_heron_cluster(cluster_role_env) config_path = utils.get_heron_cluster_conf_dir(cluster, config_path) new_cl_args = dict() try: cluster_tuple = utils.parse_cluster_role_env(cluster_role_env, config_path) new_cl_args['cluster'] = cluster_tuple[0] new_cl_args['role'] = cluster_tuple[1] new_cl_args['environ'] = cluster_tuple[2] new_cl_args['config_path'] = config_path except Exception as e: Log.error("Unable to get valid topology location: %s" % str(e)) return dict() cl_args.update(new_cl_args) return cl_args
def submit_fatjar(cl_args, unknown_args, tmp_dir): # execute main of the topology to create the topology definition topology_file = cl_args['topology-file-name'] try: execute.heron_class( cl_args['topology-class-name'], utils.get_heron_libs(jars.topology_jars()), extra_jars = [topology_file], args = tuple(unknown_args), javaDefines = cl_args['javaDefines']) except Exception as ex: Log.error("Unable to execute topology main class") return False try: launch_topologies(cl_args, topology_file, tmp_dir) except Exception as ex: return False finally: shutil.rmtree(tmp_dir) return True
def get_topology_info(*args): instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_topology_info(*args)) except Exception as ex: Log.error(str(ex)) raise
def register_watch(self, callback): """ Returns the UUID with which the watch is registered. This UUID can be used to unregister the watch. Returns None if watch could not be registered. The argument 'callback' must be a function that takes exactly one argument, the topology on which the watch was triggered. Note that the watch will be unregistered in case it raises any Exception the first time. This callback is also called at the time of registration. """ RETRY_COUNT = 5 # Retry in case UID is previously # generated, just in case... for _ in range(RETRY_COUNT): # Generate a random UUID. uid = uuid.uuid4() if uid not in self.watches: Log.info("Registering a watch with uid: " + str(uid)) try: callback(self) except Exception as e: Log.error("Caught exception while triggering callback: " + str(e)) Log.debug(traceback.format_exc()) return None self.watches[uid] = callback return uid return None
def extract_common_args(command, parser, cl_args): try: cluster_role_env = cl_args.pop('cluster/[role]/[env]') config_path = cl_args['config_path'] override_config_file = utils.parse_override_config(cl_args['config_property']) except KeyError: # if some of the arguments are not found, print error and exit subparser = utils.get_subparser(parser, command) print(subparser.format_help()) return dict() cluster = utils.get_heron_cluster(cluster_role_env) config_path = utils.get_heron_cluster_conf_dir(cluster, config_path) if not os.path.isdir(config_path): Log.error("Config path cluster directory does not exist: %s" % config_path) return dict() new_cl_args = dict() try: cluster_tuple = utils.parse_cluster_role_env(cluster_role_env, config_path) new_cl_args['cluster'] = cluster_tuple[0] new_cl_args['role'] = cluster_tuple[1] new_cl_args['environ'] = cluster_tuple[2] new_cl_args['config_path'] = config_path new_cl_args['override_config_file'] = override_config_file except Exception as e: Log.error("Argument cluster/[role]/[env] is not correct: %s" % str(e)) return dict() cl_args.update(new_cl_args) return cl_args
def run_metrics(command, parser, cl_args, unknown_args): """ run metrics subcommand """ cluster, role, env = cl_args["cluster"], cl_args["role"], cl_args["environ"] topology = cl_args["topology-name"] try: result = utils.get_topology_info(cluster, env, topology, role) spouts = result["physical_plan"]["spouts"].keys() bolts = result["physical_plan"]["bolts"].keys() components = spouts + bolts cname = cl_args["component"] if cname: if cname in components: components = [cname] else: Log.error("Unknown component: '%s'" % cname) raise except Exception: return False cresult = [] for comp in components: try: metrics = utils.get_component_metrics(comp, cluster, env, topology, role) stat, header = to_table(metrics) cresult.append((comp, stat, header)) except: return False for i, (comp, stat, header) in enumerate(cresult): if i != 0: print "" print "'%s' metrics:" % comp print tabulate(stat, headers=header) return True
def get_clusters(): instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_clusters()) except Exception as ex: Log.error('Error: %s' % str(ex)) Log.error('Failed to retrive clusters') raise
def get_topology_metrics(*args): instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_comp_metrics(*args)) except Exception as ex: Log.error(str(ex)) Log.error("Failed to retrive metrics of component \'%s\'" % args[3]) raise
def get_cluster_topologies(cluster): instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_cluster_topologies(cluster)) except Exception as ex: Log.error(str(ex)) Log.error('Failed to retrive topologies running in cluster \'%s\'' % cluster) raise
def get_cluster_role_env_topologies(cluster, role, env): instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_cluster_role_env_topologies(cluster, role, env)) except Exception as ex: Log.error(str(ex)) Log.error('Failed to retrive topologies running in cluster' '\'%s\' submitted by %s under environment %s' % (cluster, role, env)) raise
def get_logical_plan(cluster, env, topology, role): instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_logical_plan(cluster, env, topology, role)) except Exception as ex: Log.error('Error: %s' % str(ex)) Log.error('Failed to retrive logical plan info of topology \'%s\'' % ('/'.join([cluster, role, env, topology]))) raise
def check_release_file_exists(): release_file = get_heron_release_file() # if the file does not exist and is not a file if not os.path.isfile(release_file): Log.error("%s file not found: %s" % release_file) return False return True
def check_release_file_exists(): release_file = get_heron_release_file() # if the file does not exist and is not a file if not os.path.isfile(release_file): Log.error("Required file not found: %s" % release_file) return False return True
def parse_cluster_role_env(cluster_role_env, config_path): """Parse cluster/[role]/[environ], supply default, if not provided, not required""" parts = cluster_role_env.split('/')[:3] Log.info("Using config file under %s" % config_path) if not os.path.isdir(config_path): Log.error("Config path cluster directory does not exist: %s" % config_path) raise Exception("Invalid config path") # if cluster/role/env is not completely provided, check further if len(parts) < 3: cli_conf_file = os.path.join(config_path, CLIENT_YAML) # if client conf doesn't exist, use default value if not os.path.isfile(cli_conf_file): if len(parts) == 1: parts.append(getpass.getuser()) if len(parts) == 2: parts.append(ENVIRON) else: cli_confs = {} with open(cli_conf_file, 'r') as conf_file: tmp_confs = yaml.load(conf_file) # the return value of yaml.load can be None if conf_file is an empty file if tmp_confs is not None: cli_confs = tmp_confs else: print "Failed to read: %s due to it is empty" % ( CLIENT_YAML) # if role is required but not provided, raise exception if len(parts) == 1: if (IS_ROLE_REQUIRED in cli_confs) and ( cli_confs[IS_ROLE_REQUIRED] is True): raise Exception( "role required but not provided (cluster/role/env = %s). See %s in %s" % (cluster_role_env, IS_ROLE_REQUIRED, CLIENT_YAML)) else: parts.append(getpass.getuser()) # if environ is required but not provided, raise exception if len(parts) == 2: if (IS_ENV_REQUIRED in cli_confs) and (cli_confs[IS_ENV_REQUIRED] is True): raise Exception( "environ required but not provided (cluster/role/env = %s). See %s in %s" % (cluster_role_env, IS_ENV_REQUIRED, CLIENT_YAML)) else: parts.append(ENVIRON) # if cluster or role or environ is empty, print if len(parts[0]) == 0 or len(parts[1]) == 0 or len(parts[2]) == 0: print "Failed to parse" sys.exit(1) return (parts[0], parts[1], parts[2])
def check_release_file_exists(): """Check if the release.yaml file exists""" release_file = get_heron_release_file() # if the file does not exist and is not a file if not os.path.isfile(release_file): Log.error("Required file not found: %s" % release_file) return False return True
def get_cluster_role_topologies(cluster, role): """Synced API call to get topologies under a cluster submitted by a role""" instance = tornado.ioloop.IOLoop.instance() try: return instance.run_sync(lambda: API.get_cluster_role_topologies(cluster, role)) except Exception as ex: Log.error(str(ex)) Log.error('Failed to retrive topologies running in cluster' '\'%s\' submitted by %s' % (cluster, role)) raise
def get_clusters(): """Synced API call to get all cluster names""" instance = tornado.ioloop.IOLoop.instance() # pylint: disable=unnecessary-lambda try: return instance.run_sync(lambda: API.get_clusters()) except Exception as ex: Log.error('Error: %s' % str(ex)) Log.error('Failed to retrive clusters') raise
def parse_topo_loc(cl_args): try: topo_loc = cl_args['cluster/[role]/[env]'].split('/') topo_loc.append(cl_args['topology-name']) if len(topo_loc) != 4: raise return topo_loc except Exception: Log.error('Error: invalid topology location') raise
def parse_topo_loc(cl_args): """ parse topology location """ try: topo_loc = cl_args['cluster/[role]/[env]'].split('/') topo_loc.append(cl_args['topology-name']) if len(topo_loc) != 4: raise return topo_loc except Exception: Log.error('Error: invalid topology location') raise
def run(command, parser, cl_args, unknown_args): """ run command """ try: clusters = utils.get_clusters() except: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False print 'Available clusters:' for cluster in clusters: print ' %s' % cluster return True
def run(command, parser, cl_args, unknown_args): ''' Submits the topology to the scheduler * Depending on the topology file name extension, we treat the file as a fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz). * We upload the topology file to the packer, update zookeeper and launch scheduler jobs representing that topology * You can see your topology in Heron UI :param command: :param parser: :param cl_args: :param unknown_args: :return: ''' # get the topology file name topology_file = cl_args['topology-file-name'] # check to see if the topology file exists if not os.path.isfile(topology_file): Log.error("Topology jar|tar|pex file %s does not exist" % topology_file) return False # check if it is a valid file type jar_type = topology_file.endswith(".jar") tar_type = topology_file.endswith(".tar") or topology_file.endswith(".tar.gz") pex_type = topology_file.endswith(".pex") if not jar_type and not tar_type and not pex_type: Log.error("Unknown file type. Please use .tar or .tar.gz or .jar or .pex file") return False # create a temporary directory for topology definition file tmp_dir = tempfile.mkdtemp() # if topology needs to be launched in deactivated state, do it so if cl_args['deploy_deactivated']: initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED) else: initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING) # set the tmp dir and deactivated state in global options opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir) opts.set_config('cmdline.topology.initial.state', initial_state) # check the extension of the file name to see if it is tar/jar file. if jar_type: return submit_fatjar(cl_args, unknown_args, tmp_dir) elif tar_type: return submit_tar(cl_args, unknown_args, tmp_dir) elif pex_type: return submit_pex(cl_args, unknown_args, tmp_dir) return False
def run(command, parser, cl_args, unknown_args): """ run command """ try: clusters = utils.get_clusters() except: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False print 'Available clusters:' for cluster in clusters: print ' %s' % cluster return True
def run(command, parser, cl_args, unknown_args): location = cl_args['cluster/[role]/[env]'].split('/') if len(location) == 1: return show_cluster(*location) elif len(location) == 2: return show_cluster_role(*location) elif len(location) == 3: return show_cluster_role_env(*location) else: Log.error('Invalid topologies selection') return False
def parse_topo_loc(cl_args): """ parse topology location """ try: topo_loc = cl_args["cluster/[role]/[env]"].split("/") topo_name = cl_args["topology-name"] topo_loc.append(topo_name) if len(topo_loc) != 4: raise return topo_loc except Exception: Log.error("Invalid topology location") raise
def run(command, parser, cl_args, unknown_args): """ run command """ location = cl_args['cluster/[role]/[env]'].split('/') if len(location) == 1: return show_cluster(cl_args, *location) elif len(location) == 2: return show_cluster_role(cl_args, *location) elif len(location) == 3: return show_cluster_role_env(cl_args, *location) else: Log.error('Invalid topologies selection') return False
def parse_cluster_role_env(cluster_role_env, config_path): """Parse cluster/[role]/[environ], supply default, if not provided, not required""" parts = cluster_role_env.split('/')[:3] Log.info("Using config file under %s" % config_path) if not os.path.isdir(config_path): Log.error("Config path cluster directory does not exist: %s" % config_path) raise Exception("Invalid config path") # if cluster/role/env is not completely provided, check further if len(parts) < 3: cli_conf_file = os.path.join(config_path, CLIENT_YAML) # if client conf doesn't exist, use default value if not os.path.isfile(cli_conf_file): if len(parts) == 1: parts.append(getpass.getuser()) if len(parts) == 2: parts.append(ENVIRON) else: cli_confs = {} with open(cli_conf_file, 'r') as conf_file: tmp_confs = yaml.load(conf_file) # the return value of yaml.load can be None if conf_file is an empty file if tmp_confs is not None: cli_confs = tmp_confs else: print "Failed to read: %s due to it is empty" % (CLIENT_YAML) # if role is required but not provided, raise exception if len(parts) == 1: if (IS_ROLE_REQUIRED in cli_confs) and (cli_confs[IS_ROLE_REQUIRED] is True): raise Exception( "role required but not provided (cluster/role/env = %s). See %s in %s" % (cluster_role_env, IS_ROLE_REQUIRED, CLIENT_YAML)) else: parts.append(getpass.getuser()) # if environ is required but not provided, raise exception if len(parts) == 2: if (IS_ENV_REQUIRED in cli_confs) and (cli_confs[IS_ENV_REQUIRED] is True): raise Exception( "environ required but not provided (cluster/role/env = %s). See %s in %s" % (cluster_role_env, IS_ENV_REQUIRED, CLIENT_YAML)) else: parts.append(ENVIRON) # if cluster or role or environ is empty, print if len(parts[0]) == 0 or len(parts[1]) == 0 or len(parts[2]) == 0: print "Failed to parse" sys.exit(1) return (parts[0], parts[1], parts[2])
def run(command, parser, cl_args, unknown_args, action): ''' helper function to take action on topologies :param command: :param parser: :param cl_args: :param unknown_args: :param action: description of action taken :return: ''' try: topology_name = cl_args['topology-name'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", utils.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", utils.get_heron_release_file(), "--topology_name", topology_name, "--command", command, ] if opts.verbose(): new_args.append("--verbose") lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) except Exception: Log.error('Failed to %s \'%s\'' % (action, topology_name)) return False Log.info('Successfully %s \'%s\'' % (action, topology_name)) return True
def run(command, parser, cl_args, unknown_args): ''' Submits the topology to the scheduler * Depending on the topology file name extension, we treat the file as a fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz). * We upload the topology file to the packer, update zookeeper and launch scheduler jobs representing that topology * You can see your topology in Heron UI :param command: :param parser: :param cl_args: :param unknown_args: :return: ''' # get the topology file name topology_file = cl_args['topology-file-name'] # check to see if the topology file exists if not os.path.isfile(topology_file): Log.error("Topology jar|tar file %s does not exist" % topology_file) return False # check if it is a valid file type jar_type = topology_file.endswith(".jar") tar_type = topology_file.endswith(".tar") or topology_file.endswith( ".tar.gz") if not jar_type and not tar_type: Log.error("Unknown file type. Please use .tar or .tar.gz or .jar file") return False # create a temporary directory for topology definition file tmp_dir = tempfile.mkdtemp() # if topology needs to be launched in deactivated state, do it so if cl_args['deploy_deactivated']: initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED) else: initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING) # set the tmp dir and deactivated state in global options opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir) opts.set_config('cmdline.topology.initial.state', initial_state) # check the extension of the file name to see if it is tar/jar file. if jar_type: return submit_fatjar(cl_args, unknown_args, tmp_dir) elif tar_type: return submit_tar(cl_args, unknown_args, tmp_dir) return False
def check_java_home_set(): """Check if the java home set""" # check if environ variable is set if "JAVA_HOME" not in os.environ: Log.error("JAVA_HOME not set") return False # check if the value set is correct java_path = get_java_path() if os.path.isfile(java_path) and os.access(java_path, os.X_OK): return True Log.error("JAVA_HOME/bin/java either does not exist or not an executable") return False
def check_java_home_set(): # check if environ variable is set if "JAVA_HOME" not in os.environ: Log.error("JAVA_HOME not set") return False # check if the value set is correct java_path = get_java_path() if os.path.isfile(java_path) and os.access(java_path, os.X_OK): return True Log.error("JAVA_HOME/bin/java either does not exist or not an executable") return False
def show_cluster(cluster): try: result = utils.get_cluster_topologies(cluster) if not result: Log.error('Unknown cluster \'%s\'' % cluster) return False result = result[cluster] except Exception: return False table, header, rest_count = to_table(result) print('Topologies running in cluster \'%s\'' % cluster) if rest_count: print(' with %d more...' % rest_count) print(tabulate(table, headers=header)) return True
def show_cluster_role(cluster, role): try: result = utils.get_cluster_role_topologies(cluster, role) if not result: Log.error('Unknown cluster/role \'%s\'' % '/'.join([cluster, role])) return False result = result[cluster] except Exception: return False table, header, rest_count = to_table(result) print('Topologies running in cluster \'%s\' submitted by \'%s\':' % (cluster, role)) if rest_count: print(' with %d more...' % rest_count) print(tabulate(table, headers=header)) return True
def run_containers(command, parser, cl_args, unknown_args): """ run containers subcommand """ cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args[ 'environ'] topology = cl_args['topology-name'] container_id = cl_args['id'] try: result = utils.get_topology_info(cluster, env, topology, role) except: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False containers = result['physical_plan']['stmgrs'] all_bolts, all_spouts = set(), set() for _, bolts in result['physical_plan']['bolts'].items(): all_bolts = all_bolts | set(bolts) for _, spouts in result['physical_plan']['spouts'].items(): all_spouts = all_spouts | set(spouts) stmgrs = containers.keys() stmgrs.sort() if container_id is not None: try: normalized_cid = container_id - 1 if normalized_cid < 0: raise stmgrs = [stmgrs[normalized_cid]] except: Log.error('Invalid container id: %d' % container_id) return False table = [] for sid, name in enumerate(stmgrs): cid = sid + 1 host = containers[name]["host"] port = containers[name]["port"] pid = containers[name]["pid"] instances = containers[name]["instance_ids"] bolt_nums = len( [instance for instance in instances if instance in all_bolts]) spout_nums = len( [instance for instance in instances if instance in all_spouts]) table.append( [cid, host, port, pid, bolt_nums, spout_nums, len(instances)]) headers = [ "container", "host", "port", "pid", "#bolt", "#spout", "#instance" ] sys.stdout.flush() print tabulate(table, headers=headers) return True
def show_cluster(cl_args, cluster): ''' print topologies information to stdout ''' try: result = utils.get_cluster_topologies(cluster) if not result: Log.error('Unknown cluster \'%s\'' % cluster) return False result = result[cluster] except Exception: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False table, header, rest_count = to_table(result) print 'Topologies running in cluster \'%s\'' % cluster if rest_count: print ' with %d more...' % rest_count print tabulate(table, headers=header) return True
def show_cluster(cl_args, cluster): ''' print topologies information to stdout ''' try: result = tracker_access.get_cluster_topologies(cluster) if not result: Log.error('No topologies in cluster \'%s\'' % cluster) return False result = result[cluster] except Exception: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False table, header, rest_count = to_table(result) print 'Topologies running in cluster \'%s\'' % cluster if rest_count: print ' with %d more...' % rest_count print tabulate(table, headers=header) return True
def run(command, parser, args, unknown_args): # get the command for detailed help command_help = args['help-command'] # if no command is provided, just print main help if command_help == 'help': parser.print_help() return True # get the subparser for the specific command subparser = utils.get_subparser(parser, command_help) if subparser: print(subparser.format_help()) return True else: Log.error("Unknown subcommand \'%s\'" % command_help) return False
def show_cluster_role(cl_args, cluster, role): ''' print topologies information to stdout ''' try: result = tracker_access.get_cluster_role_topologies(cluster, role) if not result: Log.error('Unknown cluster/role \'%s\'' % '/'.join([cluster, role])) return False result = result[cluster] except Exception: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False table, header, rest_count = to_table(result) print 'Topologies running in cluster \'%s\' submitted by \'%s\':' % (cluster, role) if rest_count: print ' with %d more...' % rest_count print tabulate(table, headers=header) return True
def run(command, parser, cl_args, unknown_args): try: topology_name = cl_args['topology-name'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", utils.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", utils.get_heron_release_file(), "--topology_name", topology_name, "--command", command, ] if opts.verbose(): new_args.append("--verbose") lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) except Exception as ex: print 'Error: %s' % str(ex) Log.error('Failed to activate topology \'%s\'' % topology_name) return False Log.info('Successfully activated topology \'%s\'' % topology_name) return True
def run(command, parser, args, unknown_args): """ run command """ # get the command for detailed help command_help = args['help-command'] # if no command is provided, just print main help if command_help == 'help': parser.print_help() return True # get the subparser for the specific command subparser = utils.get_subparser(parser, command_help) if subparser: print subparser.format_help() return True else: Log.error("Unknown subcommand \'%s\'" % command_help) return False
def show_cluster_role_env(cl_args, cluster, role, env): ''' print topologies information to stdout ''' try: result = utils.get_cluster_role_env_topologies(cluster, role, env) if not result: Log.error('Unknown cluster/role/env \'%s\'' % '/'.join([cluster, role, env])) return False result = result[cluster] except Exception: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False table, header, rest_count = to_table(result) print 'Topologies running in cluster \'%s\', submitted by \'%s\', and\ under environment \'%s\':' % (cluster, role, env) if rest_count: print ' with %d more...' % rest_count print tabulate(table, headers=header) return True
def run_bolts(command, parser, cl_args, unknown_args): """ run bolts subcommand """ cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args[ 'environ'] topology = cl_args['topology-name'] try: result = utils.get_topology_info(cluster, env, topology, role) bolts = result['physical_plan']['bolts'].keys() bolt_name = cl_args['bolt'] if bolt_name: if bolt_name in bolts: bolts = [bolt_name] else: Log.error('Unknown bolt: \'%s\'' % bolt_name) raise except Exception: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False bolts_result = [] for bolt in bolts: try: metrics = utils.get_component_metrics(bolt, cluster, env, topology, role) stat, header = to_table(metrics) bolts_result.append((bolt, stat, header)) except Exception: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False for i, (bolt, stat, header) in enumerate(bolts_result): if i != 0: print '' print '\'%s\' metrics:' % bolt print tabulate(stat, headers=header) return True
def submit_fatjar(cl_args, unknown_args, tmp_dir): ''' We use the packer to make a package for the jar and dump it to a well-known location. We then run the main method of class with the specified arguments. We pass arguments as heron.options. This will run the jar file with the topology_class_name. The submitter inside will write out the topology defn file to a location that we specify. Then we write the topology defn file to a well known location. We then write to appropriate places in zookeeper and launch the scheduler jobs :param cl_args: :param unknown_args: :param tmp_dir: :return: ''' # execute main of the topology to create the topology definition topology_file = cl_args['topology-file-name'] try: execute.heron_class(cl_args['topology-class-name'], utils.get_heron_libs(jars.topology_jars()), extra_jars=[topology_file], args=tuple(unknown_args), java_defines=cl_args['topology_main_jvm_property']) except Exception: Log.error("Unable to execute topology main class") return False try: launch_topologies(cl_args, topology_file, tmp_dir) except Exception: return False finally: shutil.rmtree(tmp_dir) return True
def run(cl_args, compo_type): """ run command """ cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args[ 'environ'] topology = cl_args['topology-name'] spouts_only, bolts_only = cl_args['spout'], cl_args['bolt'] try: components = utils.get_logical_plan(cluster, env, topology, role) topo_info = utils.get_topology_info(cluster, env, topology, role) table, header = to_table(components, topo_info) if spouts_only == bolts_only: print tabulate(table, headers=header) elif spouts_only: table, header = filter_spouts(table, header) print tabulate(table, headers=header) else: table, header = filter_bolts(table, header) print tabulate(table, headers=header) return True except: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False
def run(command, parser, cl_args, unknown_args): ''' :param command: :param parser: :param cl_args: :param unknown_args: :return: ''' try: topology_name = cl_args['topology-name'] container_id = cl_args['container-id'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", utils.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", utils.get_heron_release_file(), "--topology_name", topology_name, "--command", command, "--container_id", str(container_id) ] lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) except Exception as ex: print 'Error: %s' % str(ex) Log.error('Failed to restart topology \'%s\'' % topology_name) return False Log.info('Successfully restarted topology \'%s\'' % topology_name) return True
def run(command, parser, cl_args, unknown_args): # get the topology file name topology_file = cl_args['topology-file-name'] # check to see if the topology file exists if not os.path.isfile(topology_file): Log.error("Topology jar|tar file %s does not exist" % topology_file) return False # check if it is a valid file type jar_type = topology_file.endswith(".jar") tar_type = topology_file.endswith(".tar") or topology_file.endswith(".tar.gz") if not jar_type and not tar_type: Log.error("Unknown file type. Please use .tar or .tar.gz or .jar file") return False # create a temporary directory for topology definition file tmp_dir = tempfile.mkdtemp() # if topology needs to be launched in deactivated state, do it so if cl_args['deploy_deactivated']: initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED) else: initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING) # set the tmp dir and deactivated state in global options opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir) opts.set_config('cmdline.topology.initial.state', initial_state) # check the extension of the file name to see if it is tar/jar file. if jar_type: return submit_fatjar(cl_args, unknown_args, tmp_dir) elif tar_type: return submit_tar(cl_args, unknown_args, tmp_dir) return False
def run_metrics(command, parser, cl_args, unknown_args): """ run metrics subcommand """ cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args[ 'environ'] topology = cl_args['topology-name'] try: result = utils.get_topology_info(cluster, env, topology, role) spouts = result['physical_plan']['spouts'].keys() bolts = result['physical_plan']['bolts'].keys() components = spouts + bolts cname = cl_args['component'] if cname: if cname in components: components = [cname] else: Log.error('Unknown component: \'%s\'' % cname) raise except Exception: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False cresult = [] for comp in components: try: metrics = utils.get_component_metrics(comp, cluster, env, topology, role) except: Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"]) return False stat, header = to_table(metrics) cresult.append((comp, stat, header)) for i, (comp, stat, header) in enumerate(cresult): if i != 0: print '' print '\'%s\' metrics:' % comp print tabulate(stat, headers=header) return True