def run(command, parser, known_args, unknown_args): """ run command """ release_file = config.get_heron_release_file() with open(release_file) as release_info: for line in release_info: print line, return True
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file, topology_name): ''' Launch a topology given topology jar, its definition file and configurations :param cl_args: :param tmp_dir: :param topology_file: :param topology_defn_file: :return: ''' # get the normalized path for topology.tar.gz topology_pkg_path = config.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz')) # get the release yaml file release_yaml_file = config.get_heron_release_file() # create a tar package with the cluster configuration and generated config files config_path = cl_args['config_path'] tar_pkg_files = [topology_file, topology_defn_file] generated_config_files = [release_yaml_file, cl_args['override_config_file']] config.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files) # pass the args to submitter main args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", config_path, "--override_config_file", cl_args['override_config_file'], "--release_file", release_yaml_file, "--topology_package", topology_pkg_path, "--topology_defn", topology_defn_file, "--topology_bin", topology_file # pex file if pex specified ] if Log.getEffectiveLevel() == logging.DEBUG: args.append("--verbose") lib_jars = config.get_heron_libs( jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars() ) extra_jars = cl_args['extra_launch_classpath'].split(':') # invoke the submitter to submit and launch the topology main_class = 'com.twitter.heron.scheduler.SubmitterMain' resp = execute.heron_class( class_name=main_class, lib_jars=lib_jars, extra_jars=extra_jars, args=args, java_defines=[]) err_context = "Failed to launch topology '%s'" % topology_name succ_context = "Successfully launched topology '%s'" % topology_name resp.add_context(err_context, succ_context) return resp
def run(command, parser, cl_args, unknown_args, action): ''' helper function to take action on topologies :param command: :param parser: :param cl_args: :param unknown_args: :param action: description of action taken :return: ''' try: topology_name = cl_args['topology-name'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, ] if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) except Exception: Log.error('Failed to %s \'%s\'' % (action, topology_name)) return False Log.info('Successfully %s \'%s\'' % (action, topology_name)) return True
def run(command, cl_args, action, extra_args=[], extra_lib_jars=[]): ''' helper function to take action on topologies :param command: :param cl_args: :param action: description of action taken :return: ''' topology_name = cl_args['topology-name'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--submit_user", cl_args['submit_user'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, ] new_args += extra_args lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) lib_jars += extra_lib_jars if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") # invoke the runtime manager to kill the topology result = execute.heron_class( 'com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) err_msg = "Failed to %s: %s" % (action, topology_name) succ_msg = "Successfully %s: %s" % (action, topology_name) result.add_context(err_msg, succ_msg) return result
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file): ''' Launch a topology given topology jar, its definition file and configurations :param cl_args: :param tmp_dir: :param topology_file: :param topology_defn_file: :return: ''' # get the normalized path for topology.tar.gz topology_pkg_path = config.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz')) # get the release yaml file release_yaml_file = config.get_heron_release_file() # create a tar package with the cluster configuration and generated config files config_path = cl_args['config_path'] tar_pkg_files = [topology_file, topology_defn_file] generated_config_files = [release_yaml_file, cl_args['override_config_file']] config.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files) # pass the args to submitter main args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", config_path, "--override_config_file", cl_args['override_config_file'], "--release_file", release_yaml_file, "--topology_package", topology_pkg_path, "--topology_defn", topology_defn_file, "--topology_bin", topology_file # pex file if pex specified ] if Log.getEffectiveLevel() == logging.DEBUG: args.append("--verbose") lib_jars = config.get_heron_libs( jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars() ) # invoke the submitter to submit and launch the topology execute.heron_class( 'com.twitter.heron.scheduler.SubmitterMain', lib_jars, extra_jars=[], args=args, java_defines=[] )
def run(command, parser, args, unknown_args): ''' :param command: :param parser: :param args: :param unknown_args: :return: ''' release_file = config.get_heron_release_file() with open(release_file) as release_info: for line in release_info: print line, return True
def run(command, parser, cl_args, unknown_args): """ run the update command """ topology_name = cl_args['topology-name'] try: new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, "--component_parallelism", ','.join(cl_args['component_parallelism']), ] if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars() + jars.packing_jars()) # invoke the runtime manager to kill the topology execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) except Exception as ex: Log.error('Failed to update topology \'%s\': %s', topology_name, traceback.format_exc(ex)) return False Log.info('Successfully updated topology \'%s\'' % topology_name) return True
def run(command, parser, cl_args, unknown_args, action): ''' helper function to take action on topologies :param command: :param parser: :param cl_args: :param unknown_args: :param action: description of action taken :return: ''' try: topology_name = cl_args['topology-name'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, ] if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class( 'com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args ) except Exception: Log.error('Failed to %s \'%s\'' % (action, topology_name)) return False Log.info('Successfully executed %s \'%s\'' % (action, topology_name)) return True
def run(command, parser, cl_args, unknown_args): ''' :param command: :param parser: :param cl_args: :param unknown_args: :return: ''' topology_name = cl_args['topology-name'] try: container_id = cl_args['container-id'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, "--container_id", str(container_id) ] lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class( 'com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args ) except Exception as ex: Log.debug(traceback.format_exc(ex)) Log.error('Failed to restart topology \'%s\'' % topology_name) return False Log.info('Successfully restarted topology \'%s\'' % topology_name) return True
def run_direct(command, cl_args, action, extra_args=[], extra_lib_jars=[]): ''' helper function to take action on topologies :param command: :param cl_args: :param action: description of action taken :return: ''' topology_name = cl_args['topology-name'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--submit_user", cl_args['submit_user'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, ] new_args += extra_args lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) lib_jars += extra_lib_jars if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") # invoke the runtime manager to kill the topology result = execute.heron_class( 'com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args ) err_msg = "Failed to %s: %s" % (action, topology_name) succ_msg = "Successfully %s: %s" % (action, topology_name) result.add_context(err_msg, succ_msg) return result
def run(command, parser, cl_args, unknown_args): ''' :param command: :param parser: :param cl_args: :param unknown_args: :return: ''' topology_name = cl_args['topology-name'] try: container_id = cl_args['container-id'] new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, "--container_id", str(container_id) ] lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars()) # invoke the runtime manager to kill the topology execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args) except Exception as ex: Log.debug(traceback.format_exc(ex)) Log.error('Failed to restart topology \'%s\'' % topology_name) return False Log.info('Successfully restarted topology \'%s\'' % topology_name) return True
def run(command, parser, cl_args, unknown_args): """ run the update command """ topology_name = cl_args['topology-name'] try: new_args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--heron_home", config.get_heron_dir(), "--config_path", cl_args['config_path'], "--override_config_file", cl_args['override_config_file'], "--release_file", config.get_heron_release_file(), "--topology_name", topology_name, "--command", command, "--component_parallelism", ','.join(cl_args['component_parallelism']), ] if Log.getEffectiveLevel() == logging.DEBUG: new_args.append("--verbose") lib_jars = config.get_heron_libs( jars.scheduler_jars() + jars.statemgr_jars() + jars.packing_jars() ) # invoke the runtime manager to kill the topology execute.heron_class( 'com.twitter.heron.scheduler.RuntimeManagerMain', lib_jars, extra_jars=[], args=new_args ) except Exception as ex: Log.error('Failed to update topology \'%s\': %s', topology_name, traceback.format_exc(ex)) return False Log.info('Successfully updated topology \'%s\'' % topology_name) return True
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file, topology_name): ''' Launch a topology given topology jar, its definition file and configurations :param cl_args: :param tmp_dir: :param topology_file: :param topology_defn_file: :return: ''' # get the normalized path for topology.tar.gz topology_pkg_path = config.normalized_class_path( os.path.join(tmp_dir, 'topology.tar.gz')) # get the release yaml file release_yaml_file = config.get_heron_release_file() # create a tar package with the cluster configuration and generated config files config_path = cl_args['config_path'] tar_pkg_files = [topology_file, topology_defn_file] generated_config_files = [ release_yaml_file, cl_args['override_config_file'] ] config.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files) # pass the args to submitter main args = [ "--cluster", cl_args['cluster'], "--role", cl_args['role'], "--environment", cl_args['environ'], "--submit_user", cl_args['submit_user'], "--heron_home", config.get_heron_dir(), "--config_path", config_path, "--override_config_file", cl_args['override_config_file'], "--release_file", release_yaml_file, "--topology_package", topology_pkg_path, "--topology_defn", topology_defn_file, "--topology_bin", os.path.basename(topology_file) # pex file if pex specified ] if Log.getEffectiveLevel() == logging.DEBUG: args.append("--verbose") if cl_args["dry_run"]: args.append("--dry_run") if "dry_run_format" in cl_args: args += ["--dry_run_format", cl_args["dry_run_format"]] lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars()) extra_jars = cl_args['extra_launch_classpath'].split(':') # invoke the submitter to submit and launch the topology main_class = 'com.twitter.heron.scheduler.SubmitterMain' res = execute.heron_class(class_name=main_class, lib_jars=lib_jars, extra_jars=extra_jars, args=args, java_defines=[]) err_context = "Failed to launch topology '%s'" % topology_name if cl_args["dry_run"]: err_context += " in dry-run mode" succ_context = "Successfully launched topology '%s'" % topology_name if cl_args["dry_run"]: succ_context += " in dry-run mode" res.add_context(err_context, succ_context) return res
def run(command, parser, cl_args, unknown_args): ''' Submits the topology to the scheduler * Depending on the topology file name extension, we treat the file as a fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz). * We upload the topology file to the packer, update zookeeper and launch scheduler jobs representing that topology * You can see your topology in Heron UI :param command: :param parser: :param cl_args: :param unknown_args: :return: ''' Log.debug("Submit Args %s", cl_args) # get the topology file name topology_file = cl_args['topology-file-name'] if urlparse.urlparse(topology_file).scheme: cl_args['topology-file-name'] = download(topology_file, cl_args['cluster']) topology_file = cl_args['topology-file-name'] Log.debug("download uri to local file: %s", topology_file) # check to see if the topology file exists if not os.path.isfile(topology_file): err_context = "Topology file '%s' does not exist" % topology_file return SimpleResult(Status.InvocationError, err_context) # check if it is a valid file type jar_type = topology_file.endswith(".jar") tar_type = topology_file.endswith(".tar") or topology_file.endswith(".tar.gz") pex_type = topology_file.endswith(".pex") cpp_type = topology_file.endswith(".dylib") or topology_file.endswith(".so") if not (jar_type or tar_type or pex_type or cpp_type): _, ext_name = os.path.splitext(topology_file) err_context = "Unknown file type '%s'. Please use .tar "\ "or .tar.gz or .jar or .pex or .dylib or .so file"\ % ext_name return SimpleResult(Status.InvocationError, err_context) # check if extra launch classpath is provided and if it is validate if cl_args['extra_launch_classpath']: valid_classpath = classpath.valid_java_classpath(cl_args['extra_launch_classpath']) if not valid_classpath: err_context = "One of jar or directory in extra launch classpath does not exist: %s" % \ cl_args['extra_launch_classpath'] return SimpleResult(Status.InvocationError, err_context) # create a temporary directory for topology definition file tmp_dir = tempfile.mkdtemp() opts.cleaned_up_files.append(tmp_dir) # if topology needs to be launched in deactivated state, do it so if cl_args['deploy_deactivated']: initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED) else: initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING) # set the tmp dir and deactivated state in global options opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir) opts.set_config('cmdline.topology.initial.state', initial_state) opts.set_config('cmdline.topology.role', cl_args['role']) opts.set_config('cmdline.topology.environment', cl_args['environ']) # Use CLI release yaml file if the release_yaml_file config is empty if not cl_args['release_yaml_file']: cl_args['release_yaml_file'] = config.get_heron_release_file() # check the extension of the file name to see if it is tar/jar file. if jar_type: return submit_fatjar(cl_args, unknown_args, tmp_dir) elif tar_type: return submit_tar(cl_args, unknown_args, tmp_dir) elif cpp_type: return submit_cpp(cl_args, unknown_args, tmp_dir) else: return submit_pex(cl_args, unknown_args, tmp_dir)