def template_apiserver_hcl(cl_args, masters, zookeepers):
"""
template apiserver.hcl
"""
single_master = masters[0]
apiserver_config_template = "%s/standalone/templates/apiserver.template.hcl" \
% cl_args["config_path"]
apiserver_config_actual = "%s/standalone/resources/apiserver.hcl" % cl_args[
"config_path"]
replacements = {
"<heron_apiserver_hostname>":
'"%s"' % get_hostname(single_master, cl_args),
"<heron_apiserver_executable>":
'"%s/heron-apiserver"' % config.get_heron_bin_dir()
if is_self(single_master) else '"%s/.heron/bin/heron-apiserver"' %
get_remote_home(single_master, cl_args),
"<zookeeper_host:zookeeper_port>":
",".join(
['%s' % zk if ":" in zk else '%s:2181' % zk for zk in zookeepers]),
"<scheduler_uri>":
"http://%s:4646" % single_master
}
template_file(apiserver_config_template, apiserver_config_actual,
replacements)
def template_heron_tools_hcl(cl_args, masters, zookeepers):
'''
template heron tools
'''
heron_tools_hcl_template = "%s/standalone/templates/heron_tools.template.hcl" \
% cl_args["config_path"]
heron_tools_hcl_actual = "%s/standalone/resources/heron_tools.hcl" \
% cl_args["config_path"]
single_master = masters[0]
template_file(heron_tools_hcl_template, heron_tools_hcl_actual,
{
"<zookeeper_host:zookeeper_port>": ",".join(
['%s' % zk if ":" in zk else '%s:2181' % zk for zk in zookeepers]),
"<heron_tracker_executable>": '"%s/heron-tracker"' % config.get_heron_bin_dir(),
"<heron_tools_hostname>": '"%s"' % get_hostname(single_master, cl_args),
"<heron_ui_executable>": '"%s/heron-ui"' % config.get_heron_bin_dir()
})
def template_heron_tools_hcl(cl_args, masters, zookeepers):
'''
template heron tools
'''
heron_tools_hcl_template = "%s/standalone/templates/heron_tools.template.hcl" \
% cl_args["config_path"]
heron_tools_hcl_actual = "%s/standalone/resources/heron_tools.hcl" \
% cl_args["config_path"]
single_master = masters[0]
template_file(heron_tools_hcl_template, heron_tools_hcl_actual,
{
"<zookeeper_host:zookeeper_port>": ",".join(
['%s' % zk if ":" in zk else '%s:2181' % zk for zk in zookeepers]),
"<heron_tracker_executable>": '"%s/heron-tracker"' % config.get_heron_bin_dir(),
"<heron_tools_hostname>": '"%s"' % get_hostname(single_master, cl_args),
"<heron_ui_executable>": '"%s/heron-ui"' % config.get_heron_bin_dir()
})
def update_slave_config_files(cl_args):
'''
update/template config files related to slave servers
'''
roles = read_and_parse_roles(cl_args)
slaves = list(roles[SET.SLAVES])
if not slaves:
return
Log.debug("Templating files for slaves...")
# update apiserver location
single_slave = slaves[0]
uploader_config_template = "%s/standalone/templates/uploader.template.yaml" \
% cl_args["config_path"]
with open(uploader_config_template, 'r') as tf:
file_contents = tf.read()
new_file_contents = file_contents.replace(
"<http_uploader_uri>",
"http://%s:9000/api/v1/file/upload" % single_slave)
uploader_config_actual = "%s/standalone/uploader.yaml" % cl_args[
"config_path"]
with open(uploader_config_actual, 'w') as tf:
tf.write(new_file_contents)
tf.truncate()
# Api server nomad job def
apiserver_config_template = "%s/standalone/templates/apiserver.template.hcl" \
% cl_args["config_path"]
with open(apiserver_config_template, 'r') as tf:
file_contents = tf.read()
new_file_contents = file_contents.replace(
"<heron_apiserver_hostname>",
'"%s"' % get_hostname(single_slave, cl_args))
if is_self(single_slave):
new_file_contents = new_file_contents.replace(
"<heron_apiserver_executable>",
'"%s/heron-apiserver"' % config.get_heron_bin_dir())
else:
new_file_contents = new_file_contents.replace(
"<heron_apiserver_executable>",
'"%s/.heron/bin/heron-apiserver"' %
get_remote_home(single_slave, cl_args))
apiserver_config_actual = "%s/standalone/resources/apiserver.hcl" % cl_args[
"config_path"]
with open(apiserver_config_actual, 'w') as tf:
tf.write(new_file_contents)
tf.truncate()
def download(uri, cluster):
tmp_dir = tempfile.mkdtemp()
cmd_downloader = config.get_heron_bin_dir() + "/heron-downloader.sh"
cmd_uri = "-u " + uri
cmd_destination = "-f " + tmp_dir
cmd_heron_root = "-d " + config.get_heron_dir()
cmd_heron_config = "-p " + config.get_heron_cluster_conf_dir(cluster, config.get_heron_conf_dir())
cmd_mode = "-m local"
cmd = [cmd_downloader, cmd_uri, cmd_destination, cmd_heron_root, cmd_heron_config, cmd_mode]
Log.debug("download uri command: %s", cmd)
subprocess.call(cmd)
suffix = (".jar", ".tar", ".tar.gz", ".pex", ".dylib", ".so")
for f in os.listdir(tmp_dir):
if f.endswith(suffix):
return os.path.join(tmp_dir, f)
def download(uri, cluster):
tmp_dir = tempfile.mkdtemp()
cmd_downloader = config.get_heron_bin_dir() + "/heron-downloader.sh"
cmd_uri = "-u " + uri
cmd_destination = "-f " + tmp_dir
cmd_heron_root = "-d " + config.get_heron_dir()
cmd_heron_config = "-p " + config.get_heron_cluster_conf_dir(cluster, config.get_heron_conf_dir())
cmd_mode = "-m local"
cmd = [cmd_downloader, cmd_uri, cmd_destination, cmd_heron_root, cmd_heron_config, cmd_mode]
Log.debug("download uri command: %s", cmd)
subprocess.call(cmd)
suffix = (".jar", ".tar", ".tar.gz", ".pex", ".dylib", ".so")
for f in os.listdir(tmp_dir):
if f.endswith(suffix):
return os.path.join(tmp_dir, f)
def template_apiserver_hcl(cl_args, masters, zookeepers):
"""
template apiserver.hcl
"""
single_master = masters[0]
apiserver_config_template = "%s/standalone/templates/apiserver.template.hcl" \
% cl_args["config_path"]
apiserver_config_actual = "%s/standalone/resources/apiserver.hcl" % cl_args["config_path"]
replacements = {
"<heron_apiserver_hostname>": '"%s"' % get_hostname(single_master, cl_args),
"<heron_apiserver_executable>": '"%s/heron-apiserver"'
% config.get_heron_bin_dir()
if is_self(single_master)
else '"%s/.heron/bin/heron-apiserver"'
% get_remote_home(single_master, cl_args),
"<zookeeper_host:zookeeper_port>": ",".join(
['%s' % zk if ":" in zk else '%s:2181' % zk for zk in zookeepers]),
"<scheduler_uri>": "http://%s:4646" % single_master
}
template_file(apiserver_config_template, apiserver_config_actual, replacements)
def get_nomad_path(cl_args): ''' get path to nomad binary ''' return "%s/heron-nomad" % config.get_heron_bin_dir()
def get_nomad_path(cl_args):
'''
get path to nomad binary
'''
return "%s/heron-nomad" % config.get_heron_bin_dir()
