def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file, topology_name): ''' Launch a topology given topology jar, its definition file and configurations :param cl_args: :param tmp_dir: :param topology_file: :param topology_defn_file: :return: ''' # get the normalized path for topology.tar.gz topology_pkg_path = config.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz')) # get the release yaml file release_yaml_file = config.get_heron_release_file() # create a tar package with the cluster configuration and generated config files config_path = cl_args['config_path'] tar_pkg_files = [topology_file, topology_defn_file] generated_config_files = [release_yaml_file, cl_args['override_config_file']] config.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files) # pass the args to submitter main args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", config_path, "--override_config_file", cl_args['override_config_file'], "--release_file", release_yaml_file, "--topology_package", topology_pkg_path, "--topology_defn", topology_defn_file, "--topology_bin", topology_file # pex file if pex specified ] if Log.getEffectiveLevel() == logging.DEBUG: args.append("--verbose") lib_jars = config.get_heron_libs( jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars() ) extra_jars = cl_args['extra_launch_classpath'].split(':') # invoke the submitter to submit and launch the topology main_class = 'com.twitter.heron.scheduler.SubmitterMain' resp = execute.heron_class( class_name=main_class, lib_jars=lib_jars, extra_jars=extra_jars, args=args, java_defines=[]) err_context = "Failed to launch topology '%s'" % topology_name succ_context = "Successfully launched topology '%s'" % topology_name resp.add_context(err_context, succ_context) return resp
def add_additional_args(parsers): ''' add additional parameters to parser ''' for parser in parsers: cli_args.add_verbose(parser) cli_args.add_config(parser) parser.add_argument('--heron-dir', default=config.get_heron_dir(), help='Path to Heron home directory')
def add_config(parser): """ add config """ # the default config path default_config_path = config.get_heron_conf_dir() parser.add_argument( '--config-path', metavar='(a string; path to cluster config; default: "' + default_config_path + '")', default=os.path.join(config.get_heron_dir(), default_config_path)) return parser
def add_additional_args(parsers): ''' add additional parameters to parser ''' for parser in parsers: cli_args.add_verbose(parser) cli_args.add_config(parser) parser.add_argument( '--heron-dir', default=config.get_heron_dir(), help='Path to Heron home directory')
def run(command, cl_args, action, extra_args=[], extra_lib_jars=[]): ''' helper function to take action on topologies :param command: :param cl_args: :param action: description of action taken :return: ''' topology_name = cl_args['topology-name'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--submit_user", cl_args['submit_user'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, ] new_args += extra_args lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) lib_jars += extra_lib_jars if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") # invoke the runtime manager to kill the topology result = execute.heron_class( 'com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) err_msg = "Failed to %s: %s" % (action, topology_name) succ_msg = "Successfully %s: %s" % (action, topology_name) result.add_context(err_msg, succ_msg) return result
def run(command, parser, cl_args, unknown_args, action): ''' helper function to take action on topologies :param command: :param parser: :param cl_args: :param unknown_args: :param action: description of action taken :return: ''' try: topology_name = cl_args['topology-name'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, ] if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) except Exception: Log.error('Failed to %s \'%s\'' % (action, topology_name)) return False Log.info('Successfully %s \'%s\'' % (action, topology_name)) return True
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file): ''' Launch a topology given topology jar, its definition file and configurations :param cl_args: :param tmp_dir: :param topology_file: :param topology_defn_file: :return: ''' # get the normalized path for topology.tar.gz topology_pkg_path = config.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz')) # get the release yaml file release_yaml_file = config.get_heron_release_file() # create a tar package with the cluster configuration and generated config files config_path = cl_args['config_path'] tar_pkg_files = [topology_file, topology_defn_file] generated_config_files = [release_yaml_file, cl_args['override_config_file']] config.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files) # pass the args to submitter main args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", config_path, "--override_config_file", cl_args['override_config_file'], "--release_file", release_yaml_file, "--topology_package", topology_pkg_path, "--topology_defn", topology_defn_file, "--topology_bin", topology_file # pex file if pex specified ] if Log.getEffectiveLevel() == logging.DEBUG: args.append("--verbose") lib_jars = config.get_heron_libs( jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars() ) # invoke the submitter to submit and launch the topology execute.heron_class( 'com.twitter.heron.scheduler.SubmitterMain', lib_jars, extra_jars=[], args=args, java_defines=[] )
def download(uri, cluster): tmp_dir = tempfile.mkdtemp() cmd_downloader = config.get_heron_bin_dir() + "/heron-downloader.sh" cmd_uri = "-u " + uri cmd_destination = "-f " + tmp_dir cmd_heron_root = "-d " + config.get_heron_dir() cmd_heron_config = "-p " + config.get_heron_cluster_conf_dir(cluster, config.get_heron_conf_dir()) cmd_mode = "-m local" cmd = [cmd_downloader, cmd_uri, cmd_destination, cmd_heron_root, cmd_heron_config, cmd_mode] Log.debug("download uri command: %s", cmd) subprocess.call(cmd) suffix = (".jar", ".tar", ".tar.gz", ".pex", ".dylib", ".so") for f in os.listdir(tmp_dir): if f.endswith(suffix): return os.path.join(tmp_dir, f)
def run(command, parser, cl_args, unknown_args): """ run the update command """ topology_name = cl_args['topology-name'] try: new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, "--component_parallelism", ','.join(cl_args['component_parallelism']), ] if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars() + jars.packing_jars()) # invoke the runtime manager to kill the topology execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) except Exception as ex: Log.error('Failed to update topology \'%s\': %s', topology_name, traceback.format_exc(ex)) return False Log.info('Successfully updated topology \'%s\'' % topology_name) return True
def run(command, parser, cl_args, unknown_args, action): ''' helper function to take action on topologies :param command: :param parser: :param cl_args: :param unknown_args: :param action: description of action taken :return: ''' try: topology_name = cl_args['topology-name'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, ] if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class( 'com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args ) except Exception: Log.error('Failed to %s \'%s\'' % (action, topology_name)) return False Log.info('Successfully executed %s \'%s\'' % (action, topology_name)) return True
def add_config(parser): ''' :param parser: :return: ''' # the default config path default_config_path = config.get_heron_conf_dir() parser.add_argument( '--config-path', metavar='(a string; path to cluster config; default: "' + default_config_path + '")', default=os.path.join(config.get_heron_dir(), default_config_path)) parser.add_argument( '--config-property', metavar='(key=value; a config key and its value; default: [])', action='append', default=[]) return parser
def run_direct(command, cl_args, action, extra_args=[], extra_lib_jars=[]): ''' helper function to take action on topologies :param command: :param cl_args: :param action: description of action taken :return: ''' topology_name = cl_args['topology-name'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--submit_user", cl_args['submit_user'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, ] new_args += extra_args lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) lib_jars += extra_lib_jars if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") # invoke the runtime manager to kill the topology result = execute.heron_class( 'com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args ) err_msg = "Failed to %s: %s" % (action, topology_name) succ_msg = "Successfully %s: %s" % (action, topology_name) result.add_context(err_msg, succ_msg) return result
def run(command, parser, cl_args, unknown_args): ''' :param command: :param parser: :param cl_args: :param unknown_args: :return: ''' topology_name = cl_args['topology-name'] try: container_id = cl_args['container-id'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, "--container_id", str(container_id) ] lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class( 'com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args ) except Exception as ex: Log.debug(traceback.format_exc(ex)) Log.error('Failed to restart topology \'%s\'' % topology_name) return False Log.info('Successfully restarted topology \'%s\'' % topology_name) return True
def add_config(parser): ''' :param parser: :return: ''' # the default config path default_config_path = config.get_heron_conf_dir() parser.add_argument( '--config-path', default=os.path.join(config.get_heron_dir(), default_config_path), help='Path to cluster configuration files') parser.add_argument( '--config-property', metavar='PROPERTY=VALUE', action='append', default=[], help='Configuration properties that overrides default options') return parser
def add_config(parser): ''' :param parser: :return: ''' # the default config path default_config_path = config.get_heron_conf_dir() parser.add_argument('--config-path', default=os.path.join(config.get_heron_dir(), default_config_path), help='Path to cluster configuration files') parser.add_argument( '--config-property', metavar='PROPERTY=VALUE', action='append', default=[], help='Configuration properties that overrides default options') return parser
def run(command, parser, cl_args, unknown_args): ''' :param command: :param parser: :param cl_args: :param unknown_args: :return: ''' topology_name = cl_args['topology-name'] try: container_id = cl_args['container-id'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, "--container_id", str(container_id) ] lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) except Exception as ex: Log.debug(traceback.format_exc(ex)) Log.error('Failed to restart topology \'%s\'' % topology_name) return False Log.info('Successfully restarted topology \'%s\'' % topology_name) return True
def run(command, parser, cl_args, unknown_args): """ run the update command """ topology_name = cl_args['topology-name'] try: new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, "--component_parallelism", ','.join(cl_args['component_parallelism']), ] if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") lib_jars = config.get_heron_libs( jars.scheduler_jars() + jars.statemgr_jars() + jars.packing_jars() ) # invoke the runtime manager to kill the topology execute.heron_class( 'com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args ) except Exception as ex: Log.error('Failed to update topology \'%s\': %s', topology_name, traceback.format_exc(ex)) return False Log.info('Successfully updated topology \'%s\'' % topology_name) return True
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file, topology_name): ''' Launch a topology given topology jar, its definition file and configurations :param cl_args: :param tmp_dir: :param topology_file: :param topology_defn_file: :return: ''' # get the normalized path for topology.tar.gz topology_pkg_path = config.normalized_class_path( os.path.join(tmp_dir, 'topology.tar.gz')) # get the release yaml file release_yaml_file = config.get_heron_release_file() # create a tar package with the cluster configuration and generated config files config_path = cl_args['config_path'] tar_pkg_files = [topology_file, topology_defn_file] generated_config_files = [ release_yaml_file, cl_args['override_config_file'] ] config.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files) # pass the args to submitter main args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--submit_user", cl_args['submit_user'], "--heron_home", config.get_heron_dir(), "--config_path", config_path, "--override_config_file", cl_args['override_config_file'], "--release_file", release_yaml_file, "--topology_package", topology_pkg_path, "--topology_defn", topology_defn_file, "--topology_bin", os.path.basename(topology_file) # pex file if pex specified ] if Log.getEffectiveLevel() == logging.DEBUG: args.append("--verbose") if cl_args["dry_run"]: args.append("--dry_run") if "dry_run_format" in cl_args: args += ["--dry_run_format", cl_args["dry_run_format"]] lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars()) extra_jars = cl_args['extra_launch_classpath'].split(':') # invoke the submitter to submit and launch the topology main_class = 'com.twitter.heron.scheduler.SubmitterMain' res = execute.heron_class(class_name=main_class, lib_jars=lib_jars, extra_jars=extra_jars, args=args, java_defines=[]) err_context = "Failed to launch topology '%s'" % topology_name if cl_args["dry_run"]: err_context += " in dry-run mode" succ_context = "Successfully launched topology '%s'" % topology_name if cl_args["dry_run"]: succ_context += " in dry-run mode" res.add_context(err_context, succ_context) return res