def extract_common_args(command, parser, cl_args):
""" extract common args """
try:
# do not pop like cli because ``topologies`` subcommand still needs it
cluster_role_env = cl_args['cluster/[role]/[env]']
config_path = cl_args['config_path']
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = config.get_subparser(parser, command)
print subparser.format_help()
return dict()
cluster = config.get_heron_cluster(cluster_role_env)
config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
new_cl_args = dict()
try:
cluster_tuple = config.parse_cluster_role_env(cluster_role_env, config_path)
new_cl_args['cluster'] = cluster_tuple[0]
new_cl_args['role'] = cluster_tuple[1]
new_cl_args['environ'] = cluster_tuple[2]
new_cl_args['config_path'] = config_path
except Exception as e:
Log.error("Unable to get valid topology location: %s" % str(e))
return dict()
cl_args.update(new_cl_args)
return cl_args
def extract_common_args(command, parser, cl_args):
""" extract common args """
try:
# do not pop like cli because ``topologies`` subcommand still needs it
cluster_role_env = cl_args['cluster/[role]/[env]']
config_path = cl_args['config_path']
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = config.get_subparser(parser, command)
print(subparser.format_help())
return dict()
cluster = config.get_heron_cluster(cluster_role_env)
config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
new_cl_args = dict()
try:
cluster_tuple = config.parse_cluster_role_env(cluster_role_env, config_path)
new_cl_args['cluster'] = cluster_tuple[0]
new_cl_args['role'] = cluster_tuple[1]
new_cl_args['environ'] = cluster_tuple[2]
new_cl_args['config_path'] = config_path
except Exception as e:
Log.error("Unable to get valid topology location: %s", str(e))
return dict()
cl_args.update(new_cl_args)
return cl_args
def direct_deployment_mode(command, parser, cluster, cl_args):
'''
check the direct deployment mode for the given cluster
if it is valid return the valid set of args
:param command:
:param parser:
:param cluster:
:param cl_args:
:return:
'''
cluster = cl_args['cluster']
try:
config_path = cl_args['config_path']
override_config_file = config.parse_override_config_and_write_file(
cl_args['config_property'])
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = config.get_subparser(parser, command)
print(subparser.format_help())
return dict()
# check if the cluster config directory exists
if not cdefs.check_direct_mode_cluster_definition(cluster, config_path):
Log.error("Cluster config directory \'%s\' does not exist",
config_path)
return dict()
config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
if not os.path.isdir(config_path):
Log.error("Cluster config directory \'%s\' does not exist",
config_path)
return dict()
Log.info("Using cluster definition in %s" % config_path)
try:
cluster_role_env = (cl_args['cluster'], cl_args['role'],
cl_args['environ'])
config.direct_mode_cluster_role_env(cluster_role_env, config_path)
cluster_tuple = config.defaults_cluster_role_env(cluster_role_env)
except Exception as ex:
Log.error("Argument cluster/[role]/[env] is not correct: %s", str(ex))
return dict()
new_cl_args = dict()
new_cl_args['cluster'] = cluster_tuple[0]
new_cl_args['role'] = cluster_tuple[1]
new_cl_args['environ'] = cluster_tuple[2]
new_cl_args['config_path'] = config_path
new_cl_args['override_config_file'] = override_config_file
new_cl_args['deploy_mode'] = config.DIRECT_MODE
cl_args.update(new_cl_args)
return cl_args
def check_direct_mode_cluster_definition(cluster, config_path):
'''
Check the cluster definition for direct mode
:param cluster:
:param config_path:
:return:
'''
config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
if not os.path.isdir(config_path):
return False
return True
def check_direct_mode_cluster_definition(cluster, config_path):
'''
Check the cluster definition for direct mode
:param cluster:
:param config_path:
:return:
'''
config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
if not os.path.isdir(config_path):
return False
return True
def check_direct_mode_cluster_definition(cluster, config_path):
'''
Check the cluster definition for direct mode
:param cluster:
:param config_path:
:return:
'''
config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
if not os.path.isdir(config_path):
Log.error("Cluster config directory \'%s\' does not exist",
config_path)
return False
return True
def direct_deployment_mode(command, parser, cluster, cl_args):
'''
check the direct deployment mode for the given cluster
if it is valid return the valid set of args
:param command:
:param parser:
:param cluster:
:param cl_args:
:return:
'''
cluster = cl_args['cluster']
try:
config_path = cl_args['config_path']
override_config_file = config.parse_override_config_and_write_file(cl_args['config_property'])
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = config.get_subparser(parser, command)
print(subparser.format_help())
return dict()
# check if the cluster config directory exists
if not cdefs.check_direct_mode_cluster_definition(cluster, config_path):
Log.error("Cluster config directory \'%s\' does not exist", config_path)
return dict()
config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
if not os.path.isdir(config_path):
Log.error("Cluster config directory \'%s\' does not exist", config_path)
return dict()
Log.info("Using cluster definition in %s" % config_path)
try:
cluster_role_env = (cl_args['cluster'], cl_args['role'], cl_args['environ'])
config.direct_mode_cluster_role_env(cluster_role_env, config_path)
cluster_tuple = config.defaults_cluster_role_env(cluster_role_env)
except Exception as ex:
Log.error("Argument cluster/[role]/[env] is not correct: %s", str(ex))
return dict()
new_cl_args = dict()
new_cl_args['cluster'] = cluster_tuple[0]
new_cl_args['role'] = cluster_tuple[1]
new_cl_args['environ'] = cluster_tuple[2]
new_cl_args['config_path'] = config_path
new_cl_args['override_config_file'] = override_config_file
new_cl_args['deploy_mode'] = config.DIRECT_MODE
cl_args.update(new_cl_args)
return cl_args
def download(uri, cluster):
tmp_dir = tempfile.mkdtemp()
cmd_downloader = config.get_heron_bin_dir() + "/heron-downloader.sh"
cmd_uri = "-u " + uri
cmd_destination = "-f " + tmp_dir
cmd_heron_root = "-d " + config.get_heron_dir()
cmd_heron_config = "-p " + config.get_heron_cluster_conf_dir(cluster, config.get_heron_conf_dir())
cmd_mode = "-m local"
cmd = [cmd_downloader, cmd_uri, cmd_destination, cmd_heron_root, cmd_heron_config, cmd_mode]
Log.debug("download uri command: %s", cmd)
subprocess.call(cmd)
suffix = (".jar", ".tar", ".tar.gz", ".pex", ".dylib", ".so")
for f in os.listdir(tmp_dir):
if f.endswith(suffix):
return os.path.join(tmp_dir, f)
def download(uri, cluster):
tmp_dir = tempfile.mkdtemp()
cmd_downloader = config.get_heron_bin_dir() + "/heron-downloader.sh"
cmd_uri = "-u " + uri
cmd_destination = "-f " + tmp_dir
cmd_heron_root = "-d " + config.get_heron_dir()
cmd_heron_config = "-p " + config.get_heron_cluster_conf_dir(cluster, config.get_heron_conf_dir())
cmd_mode = "-m local"
cmd = [cmd_downloader, cmd_uri, cmd_destination, cmd_heron_root, cmd_heron_config, cmd_mode]
Log.debug("download uri command: %s", cmd)
subprocess.call(cmd)
suffix = (".jar", ".tar", ".tar.gz", ".pex", ".dylib", ".so")
for f in os.listdir(tmp_dir):
if f.endswith(suffix):
return os.path.join(tmp_dir, f)
def containers(
config_path: str,
cre: str,
tracker_url: str,
topology: str,
container_id: int,
) -> None:
tracker.tracker_url = tracker_url
cluster = config.get_heron_cluster(cre)
cluster_config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
cluster, role, environment = config.parse_cluster_role_env(cre, cluster_config_path)
physicalplan.run_containers(
cluster=cluster,
role=role,
environment=environment,
container_id=container_id,
topology=topology,
)
def extract_common_args(command, parser, cl_args):
'''
Extract all the common args for all commands
:param command:
:param parser:
:param cl_args:
:return:
'''
try:
cluster_role_env = cl_args.pop('cluster/[role]/[env]')
config_path = cl_args['config_path']
override_config_file = config.parse_override_config(
cl_args['config_property'])
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = config.get_subparser(parser, command)
print subparser.format_help()
return dict()
cluster = config.get_heron_cluster(cluster_role_env)
config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
if not os.path.isdir(config_path):
Log.error("Config path cluster directory does not exist: %s",
config_path)
return dict()
new_cl_args = dict()
try:
cluster_tuple = config.parse_cluster_role_env(cluster_role_env,
config_path)
new_cl_args['cluster'] = cluster_tuple[0]
new_cl_args['role'] = cluster_tuple[1]
new_cl_args['environ'] = cluster_tuple[2]
new_cl_args['submit_user'] = getpass.getuser()
new_cl_args['config_path'] = config_path
new_cl_args['override_config_file'] = override_config_file
except Exception as ex:
Log.error("Argument cluster/[role]/[env] is not correct: %s", str(ex))
return dict()
cl_args.update(new_cl_args)
return cl_args
def logical_plan(
config_path: str,
cre: str,
topology: str,
component_type: str,
tracker_url: str,
) -> None:
"""Show logical plan information for the given topology."""
tracker.tracker_url = tracker_url
cluster = config.get_heron_cluster(cre)
cluster_config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
cluster, role, environment = config.parse_cluster_role_env(cre, cluster_config_path)
logicalplan.run(
component_type=component_type,
cluster=cluster,
role=role,
environment=environment,
topology=topology,
)
def metrics(
config_path: str,
cre: str,
tracker_url: str,
topology: str,
component: str,
) -> None:
define("tracker_url", tracker_url)
cluster = config.get_heron_cluster(cre)
cluster_config_path = config.get_heron_cluster_conf_dir(
cluster, config_path)
cluster, role, environment = config.parse_cluster_role_env(
cre, cluster_config_path)
physicalplan.run_metrics(
cluster=cluster,
role=role,
environment=environment,
component=component,
topology=topology,
)
def extract_common_args(command, parser, cl_args):
'''
Extract all the common args for all commands
:param command:
:param parser:
:param cl_args:
:return:
'''
try:
cluster_role_env = cl_args.pop('cluster/[role]/[env]')
config_path = cl_args['config_path']
override_config_file = config.parse_override_config(cl_args['config_property'])
except KeyError:
# if some of the arguments are not found, print error and exit
subparser = config.get_subparser(parser, command)
print subparser.format_help()
return dict()
cluster = config.get_heron_cluster(cluster_role_env)
config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
if not os.path.isdir(config_path):
Log.error("Config path cluster directory does not exist: %s", config_path)
return dict()
new_cl_args = dict()
try:
cluster_tuple = config.parse_cluster_role_env(cluster_role_env, config_path)
new_cl_args['cluster'] = cluster_tuple[0]
new_cl_args['role'] = cluster_tuple[1]
new_cl_args['environ'] = cluster_tuple[2]
new_cl_args['config_path'] = config_path
new_cl_args['override_config_file'] = override_config_file
except Exception as ex:
Log.error("Argument cluster/[role]/[env] is not correct: %s", str(ex))
return dict()
cl_args.update(new_cl_args)
return cl_args
