def launch_topology_server(cl_args, topology_file, topology_defn_file, topology_name): ''' Launch a topology given topology jar, its definition file and configurations :param cl_args: :param topology_file: :param topology_defn_file: :param topology_name: :return: ''' service_apiurl = cl_args['service_url'] + rest.ROUTE_SIGNATURES['submit'][1] service_method = rest.ROUTE_SIGNATURES['submit'][0] data = dict( name=topology_name, cluster=cl_args['cluster'], role=cl_args['role'], environment=cl_args['environ'], user=cl_args['submit_user'], ) Log.info("" + str(cl_args)) overrides = dict() if 'config_property' in cl_args: overrides = config.parse_override_config(cl_args['config_property']) if overrides: data.update(overrides) if cl_args['dry_run']: data["dry_run"] = True if cl_args['verbose_gc']: data['verbose_gc'] = True files = dict( definition=open(topology_defn_file, 'rb'), topology=open(topology_file, 'rb'), ) err_ctxt = "Failed to launch topology '%s' %s" % (topology_name, launch_mode_msg(cl_args)) succ_ctxt = "Successfully launched topology '%s' %s" % ( topology_name, launch_mode_msg(cl_args)) try: r = service_method(service_apiurl, data=data, files=files) ok = r.status_code is requests.codes.ok created = r.status_code is requests.codes.created s = Status.Ok if created or ok else Status.HeronError if s is Status.HeronError: Log.error(r.json().get( 'message', "Unknown error from API server %d" % r.status_code)) elif ok: # this case happens when we request a dry_run print(r.json().get("response")) except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err: Log.error(err) return SimpleResult(Status.HeronError, err_ctxt, succ_ctxt) return SimpleResult(s, err_ctxt, succ_ctxt)
def direct_deployment_mode(command, parser, cluster, cl_args): ''' check the direct deployment mode for the given cluster if it is valid return the valid set of args :param command: :param parser: :param cluster: :param cl_args: :return: ''' cluster = cl_args['cluster'] try: config_path = cl_args['config_path'] override_config_file = config.parse_override_config( cl_args['config_property']) except KeyError: # if some of the arguments are not found, print error and exit subparser = config.get_subparser(parser, command) print(subparser.format_help()) return dict() # check if the cluster config directory exists if not cdefs.check_direct_mode_cluster_definition(cluster, config_path): Log.error("Cluster config directory \'%s\' does not exist", config_path) return dict() config_path = config.get_heron_cluster_conf_dir(cluster, config_path) if not os.path.isdir(config_path): Log.error("Cluster config directory \'%s\' does not exist", config_path) return dict() Log.info("Using cluster definition in %s" % config_path) try: cluster_role_env = (cl_args['cluster'], cl_args['role'], cl_args['environ']) config.direct_mode_cluster_role_env(cluster_role_env, config_path) cluster_tuple = config.defaults_cluster_role_env(cluster_role_env) except Exception as ex: Log.error("Argument cluster/[role]/[env] is not correct: %s", str(ex)) return dict() new_cl_args = dict() new_cl_args['cluster'] = cluster_tuple[0] new_cl_args['role'] = cluster_tuple[1] new_cl_args['environ'] = cluster_tuple[2] new_cl_args['config_path'] = config_path new_cl_args['override_config_file'] = override_config_file new_cl_args['deploy_mode'] = config.DIRECT_MODE cl_args.update(new_cl_args) return cl_args
def launch_topology_server(cl_args, topology_file, topology_defn_file, topology_name): ''' Launch a topology given topology jar, its definition file and configurations :param cl_args: :param topology_file: :param topology_defn_file: :param topology_name: :return: ''' service_apiurl = cl_args['service_url'] + rest.ROUTE_SIGNATURES['submit'][1] service_method = rest.ROUTE_SIGNATURES['submit'][0] data = dict( name=topology_name, cluster=cl_args['cluster'], role=cl_args['role'], environment=cl_args['environ'], user=cl_args['submit_user'], ) Log.info("" + str(cl_args)) overrides = dict() if 'config_property' in cl_args: overrides = config.parse_override_config(cl_args['config_property']) if overrides: data.update(overrides) if cl_args['dry_run']: data["dry_run"] = True files = dict( definition=open(topology_defn_file, 'rb'), topology=open(topology_file, 'rb'), ) err_ctxt = "Failed to launch topology '%s' %s" % (topology_name, launch_mode_msg(cl_args)) succ_ctxt = "Successfully launched topology '%s' %s" % (topology_name, launch_mode_msg(cl_args)) try: r = service_method(service_apiurl, data=data, files=files) ok = r.status_code is requests.codes.ok created = r.status_code is requests.codes.created s = Status.Ok if created or ok else Status.HeronError if s is Status.HeronError: Log.error(r.json().get('message', "Unknown error from API server %d" % r.status_code)) elif ok: # this case happens when we request a dry_run print(r.json().get("response")) except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err: Log.error(err) return SimpleResult(Status.HeronError, err_ctxt, succ_ctxt) return SimpleResult(s, err_ctxt, succ_ctxt)
def extract_common_args(command, parser, cl_args): ''' Extract all the common args for all commands :param command: :param parser: :param cl_args: :return: ''' try: cluster_role_env = cl_args.pop('cluster/[role]/[env]') config_path = cl_args['config_path'] override_config_file = config.parse_override_config( cl_args['config_property']) except KeyError: # if some of the arguments are not found, print error and exit subparser = config.get_subparser(parser, command) print subparser.format_help() return dict() cluster = config.get_heron_cluster(cluster_role_env) config_path = config.get_heron_cluster_conf_dir(cluster, config_path) if not os.path.isdir(config_path): Log.error("Config path cluster directory does not exist: %s", config_path) return dict() new_cl_args = dict() try: cluster_tuple = config.parse_cluster_role_env(cluster_role_env, config_path) new_cl_args['cluster'] = cluster_tuple[0] new_cl_args['role'] = cluster_tuple[1] new_cl_args['environ'] = cluster_tuple[2] new_cl_args['submit_user'] = getpass.getuser() new_cl_args['config_path'] = config_path new_cl_args['override_config_file'] = override_config_file except Exception as ex: Log.error("Argument cluster/[role]/[env] is not correct: %s", str(ex)) return dict() cl_args.update(new_cl_args) return cl_args
def extract_common_args(command, parser, cl_args): ''' Extract all the common args for all commands :param command: :param parser: :param cl_args: :return: ''' try: cluster_role_env = cl_args.pop('cluster/[role]/[env]') config_path = cl_args['config_path'] override_config_file = config.parse_override_config(cl_args['config_property']) except KeyError: # if some of the arguments are not found, print error and exit subparser = config.get_subparser(parser, command) print subparser.format_help() return dict() cluster = config.get_heron_cluster(cluster_role_env) config_path = config.get_heron_cluster_conf_dir(cluster, config_path) if not os.path.isdir(config_path): Log.error("Config path cluster directory does not exist: %s", config_path) return dict() new_cl_args = dict() try: cluster_tuple = config.parse_cluster_role_env(cluster_role_env, config_path) new_cl_args['cluster'] = cluster_tuple[0] new_cl_args['role'] = cluster_tuple[1] new_cl_args['environ'] = cluster_tuple[2] new_cl_args['config_path'] = config_path new_cl_args['override_config_file'] = override_config_file except Exception as ex: Log.error("Argument cluster/[role]/[env] is not correct: %s", str(ex)) return dict() cl_args.update(new_cl_args) return cl_args