def template_apiserver_hcl(cl_args, masters, zookeepers): """ template apiserver.hcl """ single_master = masters[0] apiserver_config_template = "%s/standalone/templates/apiserver.template.hcl" \ % cl_args["config_path"] apiserver_config_actual = "%s/standalone/resources/apiserver.hcl" % cl_args[ "config_path"] replacements = { "<heron_apiserver_hostname>": '"%s"' % get_hostname(single_master, cl_args), "<heron_apiserver_executable>": '"%s/heron-apiserver"' % config.get_heron_bin_dir() if is_self(single_master) else '"%s/.heron/bin/heron-apiserver"' % get_remote_home(single_master, cl_args), "<zookeeper_host:zookeeper_port>": ",".join( ['%s' % zk if ":" in zk else '%s:2181' % zk for zk in zookeepers]), "<scheduler_uri>": "http://%s:4646" % single_master } template_file(apiserver_config_template, apiserver_config_actual, replacements)
def template_heron_tools_hcl(cl_args, masters, zookeepers): ''' template heron tools ''' heron_tools_hcl_template = "%s/standalone/templates/heron_tools.template.hcl" \ % cl_args["config_path"] heron_tools_hcl_actual = "%s/standalone/resources/heron_tools.hcl" \ % cl_args["config_path"] single_master = masters[0] template_file(heron_tools_hcl_template, heron_tools_hcl_actual, { "<zookeeper_host:zookeeper_port>": ",".join( ['%s' % zk if ":" in zk else '%s:2181' % zk for zk in zookeepers]), "<heron_tracker_executable>": '"%s/heron-tracker"' % config.get_heron_bin_dir(), "<heron_tools_hostname>": '"%s"' % get_hostname(single_master, cl_args), "<heron_ui_executable>": '"%s/heron-ui"' % config.get_heron_bin_dir() })
def template_heron_tools_hcl(cl_args, masters, zookeepers): ''' template heron tools ''' heron_tools_hcl_template = "%s/standalone/templates/heron_tools.template.hcl" \ % cl_args["config_path"] heron_tools_hcl_actual = "%s/standalone/resources/heron_tools.hcl" \ % cl_args["config_path"] single_master = masters[0] template_file(heron_tools_hcl_template, heron_tools_hcl_actual, { "<zookeeper_host:zookeeper_port>": ",".join( ['%s' % zk if ":" in zk else '%s:2181' % zk for zk in zookeepers]), "<heron_tracker_executable>": '"%s/heron-tracker"' % config.get_heron_bin_dir(), "<heron_tools_hostname>": '"%s"' % get_hostname(single_master, cl_args), "<heron_ui_executable>": '"%s/heron-ui"' % config.get_heron_bin_dir() })
def update_slave_config_files(cl_args): ''' update/template config files related to slave servers ''' roles = read_and_parse_roles(cl_args) slaves = list(roles[SET.SLAVES]) if not slaves: return Log.debug("Templating files for slaves...") # update apiserver location single_slave = slaves[0] uploader_config_template = "%s/standalone/templates/uploader.template.yaml" \ % cl_args["config_path"] with open(uploader_config_template, 'r') as tf: file_contents = tf.read() new_file_contents = file_contents.replace( "<http_uploader_uri>", "http://%s:9000/api/v1/file/upload" % single_slave) uploader_config_actual = "%s/standalone/uploader.yaml" % cl_args[ "config_path"] with open(uploader_config_actual, 'w') as tf: tf.write(new_file_contents) tf.truncate() # Api server nomad job def apiserver_config_template = "%s/standalone/templates/apiserver.template.hcl" \ % cl_args["config_path"] with open(apiserver_config_template, 'r') as tf: file_contents = tf.read() new_file_contents = file_contents.replace( "<heron_apiserver_hostname>", '"%s"' % get_hostname(single_slave, cl_args)) if is_self(single_slave): new_file_contents = new_file_contents.replace( "<heron_apiserver_executable>", '"%s/heron-apiserver"' % config.get_heron_bin_dir()) else: new_file_contents = new_file_contents.replace( "<heron_apiserver_executable>", '"%s/.heron/bin/heron-apiserver"' % get_remote_home(single_slave, cl_args)) apiserver_config_actual = "%s/standalone/resources/apiserver.hcl" % cl_args[ "config_path"] with open(apiserver_config_actual, 'w') as tf: tf.write(new_file_contents) tf.truncate()
def download(uri, cluster): tmp_dir = tempfile.mkdtemp() cmd_downloader = config.get_heron_bin_dir() + "/heron-downloader.sh" cmd_uri = "-u " + uri cmd_destination = "-f " + tmp_dir cmd_heron_root = "-d " + config.get_heron_dir() cmd_heron_config = "-p " + config.get_heron_cluster_conf_dir(cluster, config.get_heron_conf_dir()) cmd_mode = "-m local" cmd = [cmd_downloader, cmd_uri, cmd_destination, cmd_heron_root, cmd_heron_config, cmd_mode] Log.debug("download uri command: %s", cmd) subprocess.call(cmd) suffix = (".jar", ".tar", ".tar.gz", ".pex", ".dylib", ".so") for f in os.listdir(tmp_dir): if f.endswith(suffix): return os.path.join(tmp_dir, f)
def download(uri, cluster): tmp_dir = tempfile.mkdtemp() cmd_downloader = config.get_heron_bin_dir() + "/heron-downloader.sh" cmd_uri = "-u " + uri cmd_destination = "-f " + tmp_dir cmd_heron_root = "-d " + config.get_heron_dir() cmd_heron_config = "-p " + config.get_heron_cluster_conf_dir(cluster, config.get_heron_conf_dir()) cmd_mode = "-m local" cmd = [cmd_downloader, cmd_uri, cmd_destination, cmd_heron_root, cmd_heron_config, cmd_mode] Log.debug("download uri command: %s", cmd) subprocess.call(cmd) suffix = (".jar", ".tar", ".tar.gz", ".pex", ".dylib", ".so") for f in os.listdir(tmp_dir): if f.endswith(suffix): return os.path.join(tmp_dir, f)
def template_apiserver_hcl(cl_args, masters, zookeepers): """ template apiserver.hcl """ single_master = masters[0] apiserver_config_template = "%s/standalone/templates/apiserver.template.hcl" \ % cl_args["config_path"] apiserver_config_actual = "%s/standalone/resources/apiserver.hcl" % cl_args["config_path"] replacements = { "<heron_apiserver_hostname>": '"%s"' % get_hostname(single_master, cl_args), "<heron_apiserver_executable>": '"%s/heron-apiserver"' % config.get_heron_bin_dir() if is_self(single_master) else '"%s/.heron/bin/heron-apiserver"' % get_remote_home(single_master, cl_args), "<zookeeper_host:zookeeper_port>": ",".join( ['%s' % zk if ":" in zk else '%s:2181' % zk for zk in zookeepers]), "<scheduler_uri>": "http://%s:4646" % single_master } template_file(apiserver_config_template, apiserver_config_actual, replacements)
def get_nomad_path(cl_args): ''' get path to nomad binary ''' return "%s/heron-nomad" % config.get_heron_bin_dir()
def get_nomad_path(cl_args): ''' get path to nomad binary ''' return "%s/heron-nomad" % config.get_heron_bin_dir()