def configure():
# config
context = conf.copy()
context['id'] = get_slave_id(env.host_string)
context['zookeeper_server'] = [{'name' : name} for name in env.hostnames[0:conf['num_instances']]]
destination = "%s/%s" % (PATH, "config")
process_template("kafka", "server.properties.mustache", context, destination)
def configure():
# config
context = conf.copy()
context['zookeeper'] = [{'server': address} for address in env.hostnames[0:conf['num_zookeeper_instances']]]
context['master'] = env.master
context['supervisor_slots'] = [{'slot': 6700 + i} for i in range(conf['num_supervisor_slots'])]
destination = "%s/%s" % (PATH, "conf")
process_template("storm", "storm.yaml.mustache", context, destination)
def configure():
# config
context = conf.copy()
context['server'] = [{'i': i, 'name': name} for i, name in enumerate(env.hostnames[0:conf['num_instances']])]
destination = "%s/%s" % (PATH, "conf")
process_template("zookeeper", "zoo.cfg.mustache", context, destination)
# data dir
run("rm -rf %s" % conf['data_dir'])
run("mkdir -p %s" % conf['data_dir'])
def configure():
# config
context = conf.copy()
context['id'] = get_slave_id(env.host_string)
context['zookeeper_server'] = [{
'name': name
} for name in env.hostnames[0:conf['num_instances']]]
destination = "%s/%s" % (PATH, "config")
process_template("kafka", "server.properties.mustache", context,
destination)
def configure():
# config
context = conf.copy()
context['server'] = [{
'i': i,
'name': name
} for i, name in enumerate(env.hostnames[0:conf['num_instances']])]
destination = "%s/%s" % (PATH, "conf")
process_template("zookeeper", "zoo.cfg.mustache", context, destination)
# data dir
run("rm -rf %s" % conf['data_dir'])
run("mkdir -p %s" % conf['data_dir'])
def configure():
# config
context = conf.copy()
context['zookeeper'] = [{
'server': address
} for address in env.hostnames[0:conf['num_zookeeper_instances']]]
context['master'] = env.master
context['supervisor_slots'] = [{
'slot': 6700 + i
} for i in range(conf['num_supervisor_slots'])]
destination = "%s/%s" % (PATH, "conf")
process_template("storm", "storm.yaml.mustache", context, destination)
def configure():
context = conf.copy()
context['java_home'] = find_java_home()
context['master'] = env.master
# get hadoop conf environment variable
context['hadoop_conf_path'] = run("echo $HADOOP_CONF_DIR")
destination = get_flink_dist_path() + "/conf"
process_template("flink", "flink-conf.yaml.mustache", context, destination)
slaves = '\n'.join(env.slaves)
context2 = {'slaves' : slaves}
process_template("flink", "slaves.mustache", context2, destination)
# update the PATH variable
run("echo export PATH=$PATH:'%s'/bin >> %s" % (PATH, "~/.profile"))
def configure():
set_java_home("%s/etc/hadoop/hadoop-env.sh" % conf['path'])
# configure hdfs
context = conf.copy()
context['master'] = env.master
context['namenode_path'] = "%s/%s" % (conf['data_path'], namenode_dir)
context['datanode_path'] = "%s/%s" % (conf['data_path'], datanode_dir)
destination = conf['path'] + "/etc/hadoop/"
process_template("hadoop", "hdfs-site.xml.mustache", context, destination)
slaves = '\n'.join(env.slaves)
context['slaves'] = slaves
process_template("hadoop", "slaves.mustache", context, destination)
format_hdfs_master()
# configure YARN
process_template("hadoop", "core-site.xml.mustache", context, destination)
process_template("hadoop", "yarn-site.xml.mustache", context, destination)
# configure MapReduce v2
process_template("hadoop", "mapred-site.xml.mustache", context, destination)
def configure():
set_java_home("%s/etc/hadoop/hadoop-env.sh" % PATH)
# configure hdfs
context = conf.copy()
context['master'] = env.master
context['namenode_path'] = "%s/%s" % (conf['data_path'], namenode_dir)
context['datanode_path'] = "%s/%s" % (conf['data_path'], datanode_dir)
destination = PATH + "/etc/hadoop/"
process_template("hadoop", "hdfs-site.xml.mustache", context, destination)
slaves = '\n'.join(env.slaves)
context['slaves'] = slaves
process_template("hadoop", "slaves.mustache", context, destination)
# delete all data and ensure consistent state
execute(format_hdfs_master)
execute(delete_data_slaves)
# configure YARN
process_template("hadoop", "core-site.xml.mustache", context, destination)
process_template("hadoop", "yarn-site.xml.mustache", context, destination)
# configure MapReduce v2
process_template("hadoop", "mapred-site.xml.mustache", context, destination)
def configure():
# build tez
with cd(PATH):
run("mvn clean package -DskipTests=true -Dmaven.javadoc.skip=true > /dev/null")
# copy to HDFS
mkdir_hdfs("/tez")
hdfs_path = "/tez/tarball.tar.gz"
copy_to_hdfs(get_tez_tarball_path("tez-*SNAPSHOT.tar.gz"), hdfs_path)
# configure tez
destination = PATH
process_template(module="tez",
template="tez-site.xml.mustache",
context={'hdfs_path' : hdfs_path},
destination=destination)
# configure client
run("rm -rf '%s'" % conf['path_client'])
tarball_location = get_tez_tarball_path("tez*-minimal.tar.gz")
run("mkdir -p %s" % conf['path_client'])
run("tar -xzf %s -C %s" % (tarball_location, conf['path_client']))
run("echo 'export TEZ_CONF_DIR=%s' >> ~/.profile" % path)
run("echo 'export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${TEZ_CONF_DIR}:${TEZ_JARS}/*:${TEZ_JARS}/lib/*' >> ~/.profile")
def configure():
set_java_home("%s/etc/hadoop/hadoop-env.sh" % PATH)
# configure hdfs
context = conf.copy()
context['master'] = env.master
context['namenode_path'] = "%s/%s" % (conf['data_path'], namenode_dir)
context['datanode_path'] = "%s/%s" % (conf['data_path'], datanode_dir)
destination = PATH + "/etc/hadoop/"
process_template("hadoop", "hdfs-site.xml.mustache", context, destination)
slaves = '\n'.join(env.slaves)
context['slaves'] = slaves
process_template("hadoop", "slaves.mustache", context, destination)
# delete all data and ensure consistent state
# TODO check if we really need to format (in case of resume)
execute(format_hdfs_master)
execute(delete_data_slaves)
# configure YARN
process_template("hadoop", "core-site.xml.mustache", context, destination)
process_template("hadoop", "yarn-site.xml.mustache", context, destination)
# configure MapReduce v2
process_template("hadoop", "mapred-site.xml.mustache", context, destination)
def configure():
# build tez
with cd(PATH):
run("mvn clean package -DskipTests=true -Dmaven.javadoc.skip=true > /dev/null")
# copy to HDFS
mkdir_hdfs("/tez")
hdfs_path = "/tez/tarball.tar.gz"
copy_to_hdfs(get_tez_tarball_path("tez-*SNAPSHOT.tar.gz"), hdfs_path)
# configure tez
destination = PATH
process_template(module="tez",
template="tez-site.xml.mustache",
context={'hdfs_path' : hdfs_path},
destination=destination)
# configure client
run("rm -rf '%s'" % conf['path_client'])
tarball_location = get_tez_tarball_path("tez*-minimal.tar.gz")
run("mkdir -p %s" % conf['path_client'])
run("tar -xzf %s -C %s" % (tarball_location, conf['path_client']))
run("echo 'export TEZ_CONF_DIR=%s' >> ~/.profile" % PATH)
run("echo 'export TEZ_JARS=%s' >> ~/.profile" % conf['path_client'])
run("echo 'export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${TEZ_CONF_DIR}:${TEZ_JARS}/*:${TEZ_JARS}/lib/*' >> ~/.profile")
