def launch_topology_server(cl_args, topology_file, topology_defn_file,
topology_name):
'''
Launch a topology given topology jar, its definition file and configurations
:param cl_args:
:param topology_file:
:param topology_defn_file:
:param topology_name:
:return:
'''
service_apiurl = cl_args['service_url'] + rest.ROUTE_SIGNATURES['submit'][1]
service_method = rest.ROUTE_SIGNATURES['submit'][0]
data = dict(
name=topology_name,
cluster=cl_args['cluster'],
role=cl_args['role'],
environment=cl_args['environ'],
user=cl_args['submit_user'],
)
Log.info("" + str(cl_args))
overrides = dict()
if 'config_property' in cl_args:
overrides = config.parse_override_config(cl_args['config_property'])
if overrides:
data.update(overrides)
if cl_args['dry_run']:
data["dry_run"] = True
if cl_args['verbose_gc']:
data['verbose_gc'] = True
files = dict(
definition=open(topology_defn_file, 'rb'),
topology=open(topology_file, 'rb'),
)
err_ctxt = "Failed to launch topology '%s' %s" % (topology_name,
launch_mode_msg(cl_args))
succ_ctxt = "Successfully launched topology '%s' %s" % (
topology_name, launch_mode_msg(cl_args))
try:
r = service_method(service_apiurl, data=data, files=files)
ok = r.status_code is requests.codes.ok
created = r.status_code is requests.codes.created
s = Status.Ok if created or ok else Status.HeronError
if s is Status.HeronError:
Log.error(r.json().get(
'message', "Unknown error from API server %d" % r.status_code))
elif ok:
# this case happens when we request a dry_run
print(r.json().get("response"))
except (requests.exceptions.ConnectionError,
requests.exceptions.HTTPError) as err:
Log.error(err)
return SimpleResult(Status.HeronError, err_ctxt, succ_ctxt)
return SimpleResult(s, err_ctxt, succ_ctxt)
def direct_deployment_mode(command, parser, cluster, cl_args):
'''
check the direct deployment mode for the given cluster
if it is valid return the valid set of args
:param command:
:param parser:
:param cluster:
:param cl_args:
:return:
'''
cluster = cl_args['cluster']
try:
config_path = cl_args['config_path']
override_config_file = config.parse_override_config(
cl_args['config_property'])
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = config.get_subparser(parser, command)
print(subparser.format_help())
return dict()
# check if the cluster config directory exists
if not cdefs.check_direct_mode_cluster_definition(cluster, config_path):
Log.error("Cluster config directory \'%s\' does not exist",
config_path)
return dict()
config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
if not os.path.isdir(config_path):
Log.error("Cluster config directory \'%s\' does not exist",
config_path)
return dict()
Log.info("Using cluster definition in %s" % config_path)
try:
cluster_role_env = (cl_args['cluster'], cl_args['role'],
cl_args['environ'])
config.direct_mode_cluster_role_env(cluster_role_env, config_path)
cluster_tuple = config.defaults_cluster_role_env(cluster_role_env)
except Exception as ex:
Log.error("Argument cluster/[role]/[env] is not correct: %s", str(ex))
return dict()
new_cl_args = dict()
new_cl_args['cluster'] = cluster_tuple[0]
new_cl_args['role'] = cluster_tuple[1]
new_cl_args['environ'] = cluster_tuple[2]
new_cl_args['config_path'] = config_path
new_cl_args['override_config_file'] = override_config_file
new_cl_args['deploy_mode'] = config.DIRECT_MODE
cl_args.update(new_cl_args)
return cl_args
def launch_topology_server(cl_args, topology_file, topology_defn_file, topology_name):
'''
Launch a topology given topology jar, its definition file and configurations
:param cl_args:
:param topology_file:
:param topology_defn_file:
:param topology_name:
:return:
'''
service_apiurl = cl_args['service_url'] + rest.ROUTE_SIGNATURES['submit'][1]
service_method = rest.ROUTE_SIGNATURES['submit'][0]
data = dict(
name=topology_name,
cluster=cl_args['cluster'],
role=cl_args['role'],
environment=cl_args['environ'],
user=cl_args['submit_user'],
)
Log.info("" + str(cl_args))
overrides = dict()
if 'config_property' in cl_args:
overrides = config.parse_override_config(cl_args['config_property'])
if overrides:
data.update(overrides)
if cl_args['dry_run']:
data["dry_run"] = True
files = dict(
definition=open(topology_defn_file, 'rb'),
topology=open(topology_file, 'rb'),
)
err_ctxt = "Failed to launch topology '%s' %s" % (topology_name, launch_mode_msg(cl_args))
succ_ctxt = "Successfully launched topology '%s' %s" % (topology_name, launch_mode_msg(cl_args))
try:
r = service_method(service_apiurl, data=data, files=files)
ok = r.status_code is requests.codes.ok
created = r.status_code is requests.codes.created
s = Status.Ok if created or ok else Status.HeronError
if s is Status.HeronError:
Log.error(r.json().get('message', "Unknown error from API server %d" % r.status_code))
elif ok:
# this case happens when we request a dry_run
print(r.json().get("response"))
except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
Log.error(err)
return SimpleResult(Status.HeronError, err_ctxt, succ_ctxt)
return SimpleResult(s, err_ctxt, succ_ctxt)
def extract_common_args(command, parser, cl_args):
'''
Extract all the common args for all commands
:param command:
:param parser:
:param cl_args:
:return:
'''
try:
cluster_role_env = cl_args.pop('cluster/[role]/[env]')
config_path = cl_args['config_path']
override_config_file = config.parse_override_config(
cl_args['config_property'])
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = config.get_subparser(parser, command)
print subparser.format_help()
return dict()
cluster = config.get_heron_cluster(cluster_role_env)
config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
if not os.path.isdir(config_path):
Log.error("Config path cluster directory does not exist: %s",
config_path)
return dict()
new_cl_args = dict()
try:
cluster_tuple = config.parse_cluster_role_env(cluster_role_env,
config_path)
new_cl_args['cluster'] = cluster_tuple[0]
new_cl_args['role'] = cluster_tuple[1]
new_cl_args['environ'] = cluster_tuple[2]
new_cl_args['submit_user'] = getpass.getuser()
new_cl_args['config_path'] = config_path
new_cl_args['override_config_file'] = override_config_file
except Exception as ex:
Log.error("Argument cluster/[role]/[env] is not correct: %s", str(ex))
return dict()
cl_args.update(new_cl_args)
return cl_args
def extract_common_args(command, parser, cl_args):
'''
Extract all the common args for all commands
:param command:
:param parser:
:param cl_args:
:return:
'''
try:
cluster_role_env = cl_args.pop('cluster/[role]/[env]')
config_path = cl_args['config_path']
override_config_file = config.parse_override_config(cl_args['config_property'])
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = config.get_subparser(parser, command)
print subparser.format_help()
return dict()
cluster = config.get_heron_cluster(cluster_role_env)
config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
if not os.path.isdir(config_path):
Log.error("Config path cluster directory does not exist: %s", config_path)
return dict()
new_cl_args = dict()
try:
cluster_tuple = config.parse_cluster_role_env(cluster_role_env, config_path)
new_cl_args['cluster'] = cluster_tuple[0]
new_cl_args['role'] = cluster_tuple[1]
new_cl_args['environ'] = cluster_tuple[2]
new_cl_args['config_path'] = config_path
new_cl_args['override_config_file'] = override_config_file
except Exception as ex:
Log.error("Argument cluster/[role]/[env] is not correct: %s", str(ex))
return dict()
cl_args.update(new_cl_args)
return cl_args
