def launch_topology_server(cl_args, topology_file, topology_defn_file, topology_name): ''' Launch a topology given topology jar, its definition file and configurations :param cl_args: :param topology_file: :param topology_defn_file: :param topology_name: :return: ''' service_apiurl = cl_args['service_url'] + rest.ROUTE_SIGNATURES['submit'][1] service_method = rest.ROUTE_SIGNATURES['submit'][0] data = dict( name=topology_name, cluster=cl_args['cluster'], role=cl_args['role'], environment=cl_args['environ'], user=cl_args['submit_user'], ) Log.info("" + str(cl_args)) overrides = dict() if 'config_property' in cl_args: overrides = config.parse_override_config(cl_args['config_property']) if overrides: data.update(overrides) if cl_args['dry_run']: data["dry_run"] = True if cl_args['verbose_gc']: data['verbose_gc'] = True files = dict( definition=open(topology_defn_file, 'rb'), topology=open(topology_file, 'rb'), ) err_ctxt = "Failed to launch topology '%s' %s" % (topology_name, launch_mode_msg(cl_args)) succ_ctxt = "Successfully launched topology '%s' %s" % ( topology_name, launch_mode_msg(cl_args)) try: r = service_method(service_apiurl, data=data, files=files) ok = r.status_code is requests.codes.ok created = r.status_code is requests.codes.created s = Status.Ok if created or ok else Status.HeronError if s is Status.HeronError: Log.error(r.json().get( 'message', "Unknown error from API server %d" % r.status_code)) elif ok: # this case happens when we request a dry_run print(r.json().get("response")) except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err: Log.error(err) return SimpleResult(Status.HeronError, err_ctxt, succ_ctxt) return SimpleResult(s, err_ctxt, succ_ctxt)
def run_server(command, cl_args, action, extra_args=dict()): ''' helper function to take action on topologies using REST API :param command: :param cl_args: :param action: description of action taken :return: ''' topology_name = cl_args['topology-name'] service_endpoint = cl_args['service_url'] apiroute = rest.ROUTE_SIGNATURES[command][1] % ( cl_args['cluster'], cl_args['role'], cl_args['environ'], topology_name) service_apiurl = service_endpoint + apiroute service_method = rest.ROUTE_SIGNATURES[command][0] # convert the dictionary to a list of tuples data = flatten_args(extra_args) err_msg = "Failed to %s: %s" % (action, topology_name) succ_msg = "Successfully %s: %s" % (action, topology_name) try: r = service_method(service_apiurl, data=data) s = Status.Ok if r.status_code == requests.codes.ok else Status.HeronError if r.status_code != requests.codes.ok: Log.error(r.json().get( 'message', "Unknown error from API server %d" % r.status_code)) except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err: Log.error(err) return SimpleResult(Status.HeronError, err_msg, succ_msg) return SimpleResult(s, err_msg, succ_msg)
def launch_topologies(cl_args, topology_file, tmp_dir): ''' Launch topologies :param cl_args: :param topology_file: :param tmp_dir: :return: list(Responses) ''' # the submitter would have written the .defn file to the tmp_dir defn_files = glob.glob(tmp_dir + '/*.defn') if len(defn_files) == 0: return SimpleResult(Status.HeronError, "No topologies found under %s" % tmp_dir) results = [] for defn_file in defn_files: # load the topology definition from the file topology_defn = topology_pb2.Topology() try: handle = open(defn_file, "rb") topology_defn.ParseFromString(handle.read()) handle.close() except Exception as e: err_context = "Cannot load topology definition '%s': %s" % ( defn_file, e) return SimpleResult(Status.HeronError, err_context) # launch the topology mode = " in dry-run mode" if cl_args['dry_run'] else '' Log.info("Launching topology: \'%s\'%s", topology_defn.name, mode) res = launch_a_topology(cl_args, tmp_dir, topology_file, defn_file, topology_defn.name) results.append(res) return results
def heron_pex(topology_pex, topology_class_name, args=None): Log.debug("Importing %s from %s", topology_class_name, topology_pex) if topology_class_name == '-': # loading topology by running its main method (if __name__ == "__main__") heron_env = os.environ.copy() heron_env['HERON_OPTIONS'] = opts.get_heron_config() cmd = [topology_pex] if args is not None: cmd.extend(args) Log.debug("Invoking class using command: ``%s''", ' '.join(cmd)) Log.debug('Heron options: {%s}', str(heron_env['HERON_OPTIONS'])) # invoke the command with subprocess and print error message, if any process = subprocess.Popen(cmd, env=heron_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1) # todo(rli): improve python topology submission workflow return ProcessResult(process) else: try: # loading topology from Topology's subclass (no main method) # to support specifying the name of topology Log.debug("args: %s", args) if args is not None and isinstance(args, (list, tuple)) and len(args) > 0: opts.set_config('cmdline.topology.name', args[0]) os.environ["HERON_OPTIONS"] = opts.get_heron_config() Log.debug("Heron options: {%s}", os.environ["HERON_OPTIONS"]) pex_loader.load_pex(topology_pex) topology_class = pex_loader.import_and_get_class(topology_pex, topology_class_name) topology_class.write() return SimpleResult(Status.Ok) except Exception as ex: Log.debug(traceback.format_exc()) err_context = "Topology %s failed to be loaded from the given pex: %s" %\ (topology_class_name, ex) return SimpleResult(Status.HeronError, err_context)
def run(command, parser, cl_args, unknown_args): ''' :param command: :param parser: :param args: :param unknown_args: :return: ''' cluster = cl_args['cluster'] # server mode if cluster: config_file = config.heron_rc_file() client_confs = {} # Read the cluster definition, if not found client_confs = cdefs.read_server_mode_cluster_definition( cluster, cl_args, config_file) if not client_confs[cluster]: Log.error( 'Neither service url nor %s cluster definition in %s file', cluster, config_file) return SimpleResult(Status.HeronError) # if cluster definition exists, but service_url is not set, it is an error if not 'service_url' in client_confs[cluster]: Log.error('No service url for %s cluster in %s', cluster, config_file) sys.exit(1) service_endpoint = cl_args['service_url'] service_apiurl = service_endpoint + rest.ROUTE_SIGNATURES[command][1] service_method = rest.ROUTE_SIGNATURES[command][0] try: r = service_method(service_apiurl) if r.status_code != requests.codes.ok: Log.error(r.json().get( 'message', f"Unknown error from API server {r.status_code}")) sorted_items = sorted(list(r.json().items()), key=lambda tup: tup[0]) for key, value in sorted_items: print(f"{key} : {value}") except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err: Log.error(err) return SimpleResult(Status.HeronError) else: config.print_build_info() return SimpleResult(Status.Ok)
def launch_topologies(cl_args, topology_file, tmp_dir): ''' Launch topologies :param cl_args: :param topology_file: :param tmp_dir: :return: list(Responses) ''' # the submitter would have written the .defn file to the tmp_dir defn_files = glob.glob(tmp_dir + '/*.defn') if len(defn_files) == 0: return SimpleResult(Status.HeronError, "No topologies found under %s" % tmp_dir) results = [] for defn_file in defn_files: # load the topology definition from the file topology_defn = topology_pb2.Topology() try: handle = open(defn_file, "rb") topology_defn.ParseFromString(handle.read()) handle.close() except Exception as e: err_context = "Cannot load topology definition '%s': %s" % ( defn_file, e) return SimpleResult(Status.HeronError, err_context) # log topology and components configurations Log.debug("Topology config: %s", topology_defn.topology_config) Log.debug("Component config:") for spout in topology_defn.spouts: Log.debug("%s => %s", spout.comp.name, spout.comp.config) for bolt in topology_defn.bolts: Log.debug("%s => %s", bolt.comp.name, bolt.comp.config) # launch the topology Log.info("Launching topology: \'%s\'%s", topology_defn.name, launch_mode_msg(cl_args)) # check if we have to do server or direct based deployment if cl_args['deploy_mode'] == config.SERVER_MODE: res = launch_topology_server(cl_args, topology_file, defn_file, topology_defn.name) else: res = launch_a_topology(cl_args, tmp_dir, topology_file, defn_file, topology_defn.name) results.append(res) return results
def run(command, parser, cl_args, unknown_args): """ run the update command """ Log.debug("Update Args: %s", cl_args) # Build jar list extra_lib_jars = jars.packing_jars() action = "update topology%s" % (' in dry-run mode' if cl_args["dry_run"] else '') # Build extra args dict_extra_args = {} try: dict_extra_args = build_extra_args_dict(cl_args) except Exception as err: return SimpleResult(Status.InvocationError, err.message) # Execute if cl_args['deploy_mode'] == config.SERVER_MODE: return cli_helper.run_server(command, cl_args, action, dict_extra_args) else: # Convert extra argument to commandline format and then execute list_extra_args = convert_args_dict_to_list(dict_extra_args) return cli_helper.run_direct(command, cl_args, action, list_extra_args, extra_lib_jars)
def heron_class(class_name, lib_jars, extra_jars=None, args=None, java_defines=None): ''' Execute a heron class given the args and the jars needed for class path :param class_name: :param lib_jars: :param extra_jars: :param args: :param java_defines: :return: ''' # default optional params to empty list if not provided if extra_jars is None: extra_jars = [] if args is None: args = [] if java_defines is None: java_defines = [] # Format all java -D options that need to be passed while running # the class locally. java_opts = ['-D' + opt for opt in java_defines] java_path = config.get_java_path() if java_path is None: err_context = "Unable to find java command" return SimpleResult(Status.InvocationError, err_context) # Construct the command line for the sub process to run # Because of the way Python execute works, # the java opts must be passed as part of the list all_args = [java_path, "-client", "-Xmx1g"] + \ java_opts + \ ["-cp", config.get_classpath(extra_jars + lib_jars)] all_args += [class_name] + list(args) # set heron_config environment variable heron_env = os.environ.copy() heron_env['HERON_OPTIONS'] = opts.get_heron_config() # print the verbose message Log.debug("Invoking class using command: `%s`", ' '.join(shlex.quote(a) for a in all_args)) Log.debug("Heron options: {%s}", str(heron_env["HERON_OPTIONS"])) # invoke the command with subprocess and print error message, if any # pylint: disable=consider-using-with process = subprocess.Popen(all_args, env=heron_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, bufsize=1) # stdout message has the information Java program sends back # stderr message has extra information, such as debugging message return ProcessResult(process)
def _unset(cl_args): cluster, prop = cl_args['cluster'], cl_args['property'] if cliconfig.is_valid_property(prop): cliconfig.unset_property(cluster, prop) print("Cleared property [%s] for cluster %s" % (prop, cluster)) else: print("Error: Unknown property [%s] for cluster %s" % (prop, cluster)) return SimpleResult(Status.Ok)
def _set(cl_args): cluster, prop, value = cl_args['cluster'], cl_args['property'], cl_args['value'] if cliconfig.is_valid_property(prop): cliconfig.set_property(cluster, prop, value) print("Updated property [%s] for cluster %s" % (prop, cluster)) else: print("Error: Unknown property [%s] for cluster %s" % (prop, cluster)) return SimpleResult(Status.Ok)
def run(command, parser, args, unknown_args): ''' :param command: :param parser: :param args: :param unknown_args: :return: ''' config.print_build_info() return SimpleResult(Status.Ok)
def _list(cl_args): cluster = cl_args['cluster'] config = cliconfig.cluster_config(cluster) if config: for k, v in config.items(): print("%s = %s" % (str(k), str(v))) else: print("No config for cluster %s" % cluster) return SimpleResult(Status.Ok)
def _list(cl_args): cluster = cl_args['cluster'] config = cliconfig.cluster_config(cluster) if config: for k, v in list(config.items()): print(f"{str(k)} = {str(v)}") else: print(f"No config for cluster {cluster}") return SimpleResult(Status.Ok)
def run(command, parser, args, unknown_args): ''' :param command: :param parser: :param args: :param unknown_args: :return: ''' # get the command for detailed help command_help = args['help-command'] # if no command is provided, just print main help if command_help == 'help': parser.print_help() return SimpleResult(Status.Ok) # get the subparser for the specific command subparser = config.get_subparser(parser, command_help) if subparser: print(subparser.format_help()) return SimpleResult(Status.Ok) Log.error("Unknown subcommand \'%s\'", command_help) return SimpleResult(Status.InvocationError)
def run(command, parser, cl_args, unknown_args): ''' runs parser ''' action = cl_args["action"] action_type = cl_args["type"] if action == ACTION.SET: if action_type == SET.ZOOKEEPERS: call_editor(get_role_definition_file(SET.ZOOKEEPERS, cl_args)) update_zookeeper_config_files(cl_args) elif action_type == SET.SLAVES: call_editor(get_role_definition_file(SET.SLAVES, cl_args)) update_slave_config_files(cl_args) elif action_type == SET.MASTERS: call_editor(get_role_definition_file(SET.MASTERS, cl_args)) update_master_config_files(cl_args) else: raise ValueError("Invalid set type %s" % action_type) elif action == ACTION.CLUSTER: if action_type == CLUSTER.START: start_cluster(cl_args) elif action_type == CLUSTER.STOP: if check_sure( cl_args, "Are you sure you want to stop the cluster?" " This will terminate everything running in " "the cluster and remove any scheduler state."): stop_cluster(cl_args) else: raise ValueError("Invalid cluster action %s" % action_type) elif action == ACTION.TEMPLATE: update_zookeeper_config_files(cl_args) update_master_config_files(cl_args) update_slave_config_files(cl_args) else: raise ValueError("Invalid action %s" % action) return SimpleResult(Status.Ok)
def run(command, parser, cl_args, unknown_args): ''' runs parser ''' action = cl_args["action"] if action == Action.SET: call_editor(get_inventory_file(cl_args)) update_config_files(cl_args) elif action == Action.CLUSTER: action_type = cl_args["type"] if action_type == Cluster.START: start_cluster(cl_args) elif action_type == Cluster.STOP: if check_sure( cl_args, "Are you sure you want to stop the cluster?" " This will terminate everything running in " "the cluster and remove any scheduler state."): stop_cluster(cl_args) else: raise ValueError("Invalid cluster action %s" % action_type) elif action == Action.TEMPLATE: update_config_files(cl_args) elif action == Action.GET: action_type = cl_args["type"] if action_type == Get.SERVICE_URL: print get_service_url(cl_args) elif action_type == Get.HERON_UI_URL: print get_heron_ui_url(cl_args) elif action_type == Get.HERON_TRACKER_URL: print get_heron_tracker_url(cl_args) else: raise ValueError("Invalid get action %s" % action_type) elif action == Action.INFO: print_cluster_info(cl_args) else: raise ValueError("Invalid action %s" % action) return SimpleResult(Status.Ok)
def run(command, parser, cl_args, unknown_args): ''' Submits the topology to the scheduler * Depending on the topology file name extension, we treat the file as a fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz). * We upload the topology file to the packer, update zookeeper and launch scheduler jobs representing that topology * You can see your topology in Heron UI :param command: :param parser: :param cl_args: :param unknown_args: :return: ''' # get the topology file name topology_file = cl_args['topology-file-name'] # check to see if the topology file exists if not os.path.isfile(topology_file): err_context = "Topology file '%s' does not exist" % topology_file return SimpleResult(Status.InvocationError, err_context) # check if it is a valid file type jar_type = topology_file.endswith(".jar") tar_type = topology_file.endswith(".tar") or topology_file.endswith( ".tar.gz") pex_type = topology_file.endswith(".pex") if not jar_type and not tar_type and not pex_type: ext_name = os.path.splitext(topology_file) err_context = "Unknown file type '%s'. Please use .tar or .tar.gz or .jar or .pex file"\ % ext_name return SimpleResult(Status.InvocationError, err_context) # check if extra launch classpath is provided and if it is validate if cl_args['extra_launch_classpath']: valid_classpath = classpath.valid_java_classpath( cl_args['extra_launch_classpath']) if not valid_classpath: err_context = "One of jar or directory in extra launch classpath does not exist: %s" % \ cl_args['extra_launch_classpath'] return SimpleResult(Status.InvocationError, err_context) # create a temporary directory for topology definition file tmp_dir = tempfile.mkdtemp() opts.cleaned_up_files.append(tmp_dir) # if topology needs to be launched in deactivated state, do it so if cl_args['deploy_deactivated']: initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED) else: initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING) # set the tmp dir and deactivated state in global options opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir) opts.set_config('cmdline.topology.initial.state', initial_state) # check the extension of the file name to see if it is tar/jar file. if jar_type: return submit_fatjar(cl_args, unknown_args, tmp_dir) elif tar_type: return submit_tar(cl_args, unknown_args, tmp_dir) else: return submit_pex(cl_args, unknown_args, tmp_dir)
def run(command, parser, cl_args, unknown_args): ''' Submits the topology to the scheduler * Depending on the topology file name extension, we treat the file as a fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz). * We upload the topology file to the packer, update zookeeper and launch scheduler jobs representing that topology * You can see your topology in Heron UI :param command: :param parser: :param cl_args: :param unknown_args: :return: ''' Log.debug("Submit Args %s", cl_args) # get the topology file name topology_file = cl_args['topology-file-name'] if urlparse.urlparse(topology_file).scheme: cl_args['topology-file-name'] = download(topology_file, cl_args['cluster']) topology_file = cl_args['topology-file-name'] Log.debug("download uri to local file: %s", topology_file) # check to see if the topology file exists if not os.path.isfile(topology_file): err_context = "Topology file '%s' does not exist" % topology_file return SimpleResult(Status.InvocationError, err_context) # check if it is a valid file type jar_type = topology_file.endswith(".jar") tar_type = topology_file.endswith(".tar") or topology_file.endswith(".tar.gz") pex_type = topology_file.endswith(".pex") cpp_type = topology_file.endswith(".dylib") or topology_file.endswith(".so") if not (jar_type or tar_type or pex_type or cpp_type): _, ext_name = os.path.splitext(topology_file) err_context = "Unknown file type '%s'. Please use .tar "\ "or .tar.gz or .jar or .pex or .dylib or .so file"\ % ext_name return SimpleResult(Status.InvocationError, err_context) # check if extra launch classpath is provided and if it is validate if cl_args['extra_launch_classpath']: valid_classpath = classpath.valid_java_classpath(cl_args['extra_launch_classpath']) if not valid_classpath: err_context = "One of jar or directory in extra launch classpath does not exist: %s" % \ cl_args['extra_launch_classpath'] return SimpleResult(Status.InvocationError, err_context) # create a temporary directory for topology definition file tmp_dir = tempfile.mkdtemp() opts.cleaned_up_files.append(tmp_dir) # if topology needs to be launched in deactivated state, do it so if cl_args['deploy_deactivated']: initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED) else: initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING) # set the tmp dir and deactivated state in global options opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir) opts.set_config('cmdline.topology.initial.state', initial_state) opts.set_config('cmdline.topology.role', cl_args['role']) opts.set_config('cmdline.topology.environment', cl_args['environ']) # Use CLI release yaml file if the release_yaml_file config is empty if not cl_args['release_yaml_file']: cl_args['release_yaml_file'] = config.get_heron_release_file() # check the extension of the file name to see if it is tar/jar file. if jar_type: return submit_fatjar(cl_args, unknown_args, tmp_dir) elif tar_type: return submit_tar(cl_args, unknown_args, tmp_dir) elif cpp_type: return submit_cpp(cl_args, unknown_args, tmp_dir) else: return submit_pex(cl_args, unknown_args, tmp_dir)