def heron_tar(class_name, topology_tar, arguments, tmpdir_root, java_defines): ''' :param class_name: :param topology_tar: :param arguments: :param tmpdir_root: :param java_defines: :return: ''' # Extract tar to a tmp folder. tmpdir = tempfile.mkdtemp(dir=tmpdir_root, prefix='tmp') with contextlib.closing(tarfile.open(topology_tar)) as tar: tar.extractall(path=tmpdir) # A tar generated by pants has all dependency jars under libs/ # in addition to the topology jar at top level. Pants keeps # filename for jar and tar the same except for extension. topology_jar = os.path.basename(topology_tar).replace(".tar.gz", "").replace(".tar", "") + ".jar" extra_jars = [ os.path.join(tmpdir, "heron-instance.jar"), os.path.join(tmpdir, topology_jar), os.path.join(tmpdir, "*"), os.path.join(tmpdir, "libs/*") ] lib_jars = config.get_heron_libs(jars.topology_jars()) # Now execute the class return heron_class(class_name, lib_jars, extra_jars, arguments, java_defines)
def heron_tar(class_name, topology_tar, arguments, tmpdir_root, java_defines): ''' :param class_name: :param topology_tar: :param arguments: :param tmpdir_root: :param java_defines: :return: ''' # Extract tar to a tmp folder. tmpdir = tempfile.mkdtemp(dir=tmpdir_root, prefix='tmp') with contextlib.closing(tarfile.open(topology_tar)) as tar: tar.extractall(path=tmpdir) # A tar generated by pants has all dependency jars under libs/ # in addition to the topology jar at top level. Pants keeps # filename for jar and tar the same except for extension. topology_jar = os.path.basename(topology_tar).replace( ".tar.gz", "").replace(".tar", "") + ".jar" extra_jars = [ os.path.join(tmpdir, "heron-instance.jar"), os.path.join(tmpdir, topology_jar), os.path.join(tmpdir, "*"), os.path.join(tmpdir, "libs/*") ] lib_jars = config.get_heron_libs(jars.topology_jars()) # Now execute the class return heron_class(class_name, lib_jars, extra_jars, arguments, java_defines)
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file, topology_name): ''' Launch a topology given topology jar, its definition file and configurations :param cl_args: :param tmp_dir: :param topology_file: :param topology_defn_file: :return: ''' # get the normalized path for topology.tar.gz topology_pkg_path = config.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz')) # get the release yaml file release_yaml_file = config.get_heron_release_file() # create a tar package with the cluster configuration and generated config files config_path = cl_args['config_path'] tar_pkg_files = [topology_file, topology_defn_file] generated_config_files = [release_yaml_file, cl_args['override_config_file']] config.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files) # pass the args to submitter main args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", config_path, "--override_config_file", cl_args['override_config_file'], "--release_file", release_yaml_file, "--topology_package", topology_pkg_path, "--topology_defn", topology_defn_file, "--topology_bin", topology_file # pex file if pex specified ] if Log.getEffectiveLevel() == logging.DEBUG: args.append("--verbose") lib_jars = config.get_heron_libs( jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars() ) extra_jars = cl_args['extra_launch_classpath'].split(':') # invoke the submitter to submit and launch the topology main_class = 'com.twitter.heron.scheduler.SubmitterMain' resp = execute.heron_class( class_name=main_class, lib_jars=lib_jars, extra_jars=extra_jars, args=args, java_defines=[]) err_context = "Failed to launch topology '%s'" % topology_name succ_context = "Successfully launched topology '%s'" % topology_name resp.add_context(err_context, succ_context) return resp
def run(command, cl_args, action, extra_args=[], extra_lib_jars=[]): ''' helper function to take action on topologies :param command: :param cl_args: :param action: description of action taken :return: ''' topology_name = cl_args['topology-name'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--submit_user", cl_args['submit_user'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, ] new_args += extra_args lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) lib_jars += extra_lib_jars if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") # invoke the runtime manager to kill the topology result = execute.heron_class( 'com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) err_msg = "Failed to %s: %s" % (action, topology_name) succ_msg = "Successfully %s: %s" % (action, topology_name) result.add_context(err_msg, succ_msg) return result
def run(command, parser, cl_args, unknown_args, action): ''' helper function to take action on topologies :param command: :param parser: :param cl_args: :param unknown_args: :param action: description of action taken :return: ''' try: topology_name = cl_args['topology-name'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, ] if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) except Exception: Log.error('Failed to %s \'%s\'' % (action, topology_name)) return False Log.info('Successfully %s \'%s\'' % (action, topology_name)) return True
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file): ''' Launch a topology given topology jar, its definition file and configurations :param cl_args: :param tmp_dir: :param topology_file: :param topology_defn_file: :return: ''' # get the normalized path for topology.tar.gz topology_pkg_path = config.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz')) # get the release yaml file release_yaml_file = config.get_heron_release_file() # create a tar package with the cluster configuration and generated config files config_path = cl_args['config_path'] tar_pkg_files = [topology_file, topology_defn_file] generated_config_files = [release_yaml_file, cl_args['override_config_file']] config.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files) # pass the args to submitter main args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", config_path, "--override_config_file", cl_args['override_config_file'], "--release_file", release_yaml_file, "--topology_package", topology_pkg_path, "--topology_defn", topology_defn_file, "--topology_bin", topology_file # pex file if pex specified ] if Log.getEffectiveLevel() == logging.DEBUG: args.append("--verbose") lib_jars = config.get_heron_libs( jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars() ) # invoke the submitter to submit and launch the topology execute.heron_class( 'com.twitter.heron.scheduler.SubmitterMain', lib_jars, extra_jars=[], args=args, java_defines=[] )
def run(command, parser, cl_args, unknown_args): """ run the update command """ topology_name = cl_args['topology-name'] try: new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, "--component_parallelism", ','.join(cl_args['component_parallelism']), ] if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars() + jars.packing_jars()) # invoke the runtime manager to kill the topology execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) except Exception as ex: Log.error('Failed to update topology \'%s\': %s', topology_name, traceback.format_exc(ex)) return False Log.info('Successfully updated topology \'%s\'' % topology_name) return True
def run(command, parser, cl_args, unknown_args, action): ''' helper function to take action on topologies :param command: :param parser: :param cl_args: :param unknown_args: :param action: description of action taken :return: ''' try: topology_name = cl_args['topology-name'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, ] if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class( 'com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args ) except Exception: Log.error('Failed to %s \'%s\'' % (action, topology_name)) return False Log.info('Successfully executed %s \'%s\'' % (action, topology_name)) return True
def run_direct(command, cl_args, action, extra_args=[], extra_lib_jars=[]): ''' helper function to take action on topologies :param command: :param cl_args: :param action: description of action taken :return: ''' topology_name = cl_args['topology-name'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--submit_user", cl_args['submit_user'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, ] new_args += extra_args lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) lib_jars += extra_lib_jars if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") # invoke the runtime manager to kill the topology result = execute.heron_class( 'com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args ) err_msg = "Failed to %s: %s" % (action, topology_name) succ_msg = "Successfully %s: %s" % (action, topology_name) result.add_context(err_msg, succ_msg) return result
def run(command, parser, cl_args, unknown_args): ''' :param command: :param parser: :param cl_args: :param unknown_args: :return: ''' topology_name = cl_args['topology-name'] try: container_id = cl_args['container-id'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, "--container_id", str(container_id) ] lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class( 'com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args ) except Exception as ex: Log.debug(traceback.format_exc(ex)) Log.error('Failed to restart topology \'%s\'' % topology_name) return False Log.info('Successfully restarted topology \'%s\'' % topology_name) return True
def submit_fatjar(cl_args, unknown_args, tmp_dir): ''' We use the packer to make a package for the jar and dump it to a well-known location. We then run the main method of class with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS. This will run the jar file with the topology_class_name. The submitter inside will write out the topology defn file to a location that we specify. Then we write the topology defn file to a well known location. We then write to appropriate places in zookeeper and launch the scheduler jobs :param cl_args: :param unknown_args: :param tmp_dir: :return: ''' # execute main of the topology to create the topology definition topology_file = cl_args['topology-file-name'] main_class = cl_args['topology-class-name'] res = execute.heron_class( class_name=main_class, lib_jars=config.get_heron_libs(jars.topology_jars()), extra_jars=[topology_file], args=tuple(unknown_args), java_defines=cl_args['topology_main_jvm_property']) result.render(res) if not res.is_successful(): err_context = "Failed to create topology definition \ file when executing class '%s' of file '%s'" % (main_class, topology_file) res.add_context(err_context) return res results = launch_topologies(cl_args, topology_file, tmp_dir) return results
def submit_fatjar(cl_args, unknown_args, tmp_dir): ''' We use the packer to make a package for the jar and dump it to a well-known location. We then run the main method of class with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS. This will run the jar file with the topology_class_name. The submitter inside will write out the topology defn file to a location that we specify. Then we write the topology defn file to a well known location. We then write to appropriate places in zookeeper and launch the scheduler jobs :param cl_args: :param unknown_args: :param tmp_dir: :return: ''' # execute main of the topology to create the topology definition topology_file = cl_args['topology-file-name'] try: execute.heron_class( cl_args['topology-class-name'], config.get_heron_libs(jars.topology_jars()), extra_jars=[topology_file], args=tuple(unknown_args), java_defines=cl_args['topology_main_jvm_property']) except Exception as ex: Log.debug(traceback.format_exc(ex)) Log.error("Unable to execute topology main class") return False try: launch_topologies(cl_args, topology_file, tmp_dir) except Exception as ex: return False finally: shutil.rmtree(tmp_dir) return True
def submit_fatjar(cl_args, unknown_args, tmp_dir): ''' We use the packer to make a package for the jar and dump it to a well-known location. We then run the main method of class with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS. This will run the jar file with the topology_class_name. The submitter inside will write out the topology defn file to a location that we specify. Then we write the topology defn file to a well known location. We then write to appropriate places in zookeeper and launch the scheduler jobs :param cl_args: :param unknown_args: :param tmp_dir: :return: ''' # execute main of the topology to create the topology definition topology_file = cl_args['topology-file-name'] main_class = cl_args['topology-class-name'] res = execute.heron_class( class_name=main_class, lib_jars=config.get_heron_libs(jars.topology_jars()), extra_jars=[topology_file], args=tuple(unknown_args), java_defines=cl_args['topology_main_jvm_property']) result.render(res) if not res.is_successful(): err_context = ("Failed to create topology definition " \ "file when executing class '%s' of file '%s'") % (main_class, topology_file) res.add_context(err_context) return res results = launch_topologies(cl_args, topology_file, tmp_dir) return results
def run(command, parser, cl_args, unknown_args): ''' :param command: :param parser: :param cl_args: :param unknown_args: :return: ''' topology_name = cl_args['topology-name'] try: container_id = cl_args['container-id'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, "--container_id", str(container_id) ] lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) except Exception as ex: Log.debug(traceback.format_exc(ex)) Log.error('Failed to restart topology \'%s\'' % topology_name) return False Log.info('Successfully restarted topology \'%s\'' % topology_name) return True
def run(command, parser, cl_args, unknown_args): """ run the update command """ topology_name = cl_args['topology-name'] try: new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, "--component_parallelism", ','.join(cl_args['component_parallelism']), ] if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") lib_jars = config.get_heron_libs( jars.scheduler_jars() + jars.statemgr_jars() + jars.packing_jars() ) # invoke the runtime manager to kill the topology execute.heron_class( 'com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args ) except Exception as ex: Log.error('Failed to update topology \'%s\': %s', topology_name, traceback.format_exc(ex)) return False Log.info('Successfully updated topology \'%s\'' % topology_name) return True
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file, topology_name): ''' Launch a topology given topology jar, its definition file and configurations :param cl_args: :param tmp_dir: :param topology_file: :param topology_defn_file: :return: ''' # get the normalized path for topology.tar.gz topology_pkg_path = config.normalized_class_path( os.path.join(tmp_dir, 'topology.tar.gz')) # get the release yaml file release_yaml_file = config.get_heron_release_file() # create a tar package with the cluster configuration and generated config files config_path = cl_args['config_path'] tar_pkg_files = [topology_file, topology_defn_file] generated_config_files = [ release_yaml_file, cl_args['override_config_file'] ] config.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files) # pass the args to submitter main args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--submit_user", cl_args['submit_user'], "--heron_home", config.get_heron_dir(), "--config_path", config_path, "--override_config_file", cl_args['override_config_file'], "--release_file", release_yaml_file, "--topology_package", topology_pkg_path, "--topology_defn", topology_defn_file, "--topology_bin", os.path.basename(topology_file) # pex file if pex specified ] if Log.getEffectiveLevel() == logging.DEBUG: args.append("--verbose") if cl_args["dry_run"]: args.append("--dry_run") if "dry_run_format" in cl_args: args += ["--dry_run_format", cl_args["dry_run_format"]] lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars()) extra_jars = cl_args['extra_launch_classpath'].split(':') # invoke the submitter to submit and launch the topology main_class = 'com.twitter.heron.scheduler.SubmitterMain' res = execute.heron_class(class_name=main_class, lib_jars=lib_jars, extra_jars=extra_jars, args=args, java_defines=[]) err_context = "Failed to launch topology '%s'" % topology_name if cl_args["dry_run"]: err_context += " in dry-run mode" succ_context = "Successfully launched topology '%s'" % topology_name if cl_args["dry_run"]: succ_context += " in dry-run mode" res.add_context(err_context, succ_context) return res