def bootstrap(self, tar_file):
# 0. Check that required packages are present
required_packages = "openjdk-7-jre openjdk-7-jdk"
check_packages = TaktukRemote("dpkg -s " + required_packages,
self.hosts)
for p in check_packages.processes:
p.nolog_exit_code = p.nolog_error = True
check_packages.run()
if not check_packages.ok:
logger.info("Packages not installed, trying to install")
install_packages = TaktukRemote(
"export DEBIAN_MASTER=noninteractive ; " +
"apt-get update && apt-get install -y --force-yes " +
required_packages, self.hosts).run()
if not install_packages.ok:
logger.error("Unable to install the packages")
get_java_home = SshProcess(
'echo $(readlink -f /usr/bin/javac | '
'sed "s:/bin/javac::")', self.master)
get_java_home.run()
self.java_home = get_java_home.stdout.strip()
logger.info("All required packages are present")
# 1. Copy Hive tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_dirs = TaktukRemote(
"rm -rf " + self.base_dir + " " + self.conf_dir + " " +
self.warehouse_dir + " " + self.logs_dir, self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hosts)
SequentialActions([rm_dirs, put_tar, tar_xf]).run()
# 2. Move installation to base dir
logger.info("Create installation directories")
mv_base_dir = TaktukRemote(
"mv /tmp/" + os.path.basename(tar_file).replace(".tar.gz", "") +
" " + self.base_dir, self.hosts)
mkdirs = TaktukRemote(
"mkdir -p " + self.conf_dir + " && mkdir -p " + self.warehouse_dir,
self.hosts)
chmods = TaktukRemote(
"chmod g+w " + self.base_dir + " && chmod g+w " + self.conf_dir +
" && chmod g+w " + self.warehouse_dir, self.hosts)
SequentialActions([mv_base_dir, mkdirs, chmods]).run()
# 3. Specify environment variables
command = "cat >> " + self.conf_dir + "/hive-env.sh << EOF\n"
command += "JAVA_HOME=" + self.java_home + "\n"
command += "HIVE_HOME=" + self.base_dir + "\n"
command += "HIVE_CONF_DIR=" + self.conf_dir + "\n"
command += "HADOOP_HOME=" + self.hc.base_dir + "\n"
command += "EOF\n"
command += "chmod +x " + self.conf_dir + "/hive-env.sh"
action = Remote(command, self.hosts)
action.run()
def change_conf(self, params, conf_file=None, default_file=MR_CONF_FILE):
"""Modify Hadoop configuration. This method copies the configuration
files from the first host of each g5k cluster conf dir into a local
temporary dir, do all the changes in place and broadcast the new
configuration files to all hosts.
Args:
params (dict of str:str):
The parameters to be changed in the form key:value.
conf_file (str, optional):
The file where parameters should be set. If not specified, all
files are checked for the parameter name and the parameter is set
in the file where the property is found. If not found, the
parameter is set in the default file.
default_file (str, optional): The default conf file where to set the
parameter if not found. Only applies when conf_file is not set.
"""
for cluster in self.hw.get_clusters():
hosts = cluster.get_hosts()
# Copy conf files from first host in the cluster
action = Remote("ls " + self.conf_dir + "/*.xml", [hosts[0]])
action.run()
output = action.processes[0].stdout
remote_conf_files = []
for f in output.split():
remote_conf_files.append(os.path.join(self.conf_dir, f))
tmp_dir = "/tmp/mliroz_temp_hadoop/"
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
action = Get([hosts[0]], remote_conf_files, tmp_dir)
action.run()
# Do replacements in temp file
if conf_file:
f = os.path.join(tmp_dir, conf_file)
for name, value in params.iteritems():
replace_in_xml_file(f, name, value, True)
else:
temp_conf_files = [os.path.join(tmp_dir, f) for f in
os.listdir(tmp_dir)]
for name, value in params.iteritems():
for f in temp_conf_files:
if replace_in_xml_file(f, name, value):
break
else:
# Property not found - add it in MR_CONF_FILE
logger.info("Parameter with name " + name + " has not "
"been found in any conf file. Setting it "
"in " + default_file)
f = os.path.join(tmp_dir, default_file)
replace_in_xml_file(f, name, value, True)
# Copy back the files to all hosts
self._copy_conf(tmp_dir, hosts)
def change_conf(self, params, conf_file=None, default_file=MR_CONF_FILE):
"""Modify Hadoop configuration. This method copies the configuration
files from the first host of each g5k cluster conf dir into a local
temporary dir, do all the changes in place and broadcast the new
configuration files to all hosts.
Args:
params (dict of str:str):
The parameters to be changed in the form key:value.
conf_file (str, optional):
The file where parameters should be set. If not specified, all
files are checked for the parameter name and the parameter is set
in the file where the property is found. If not found, the
parameter is set in the default file.
default_file (str, optional): The default conf file where to set the
parameter if not found. Only applies when conf_file is not set.
"""
for cluster in self.hw.get_clusters():
hosts = cluster.get_hosts()
# Copy conf files from first host in the cluster
action = Remote("ls " + self.conf_dir + "/*.xml", [hosts[0]])
action.run()
output = action.processes[0].stdout
remote_conf_files = []
for f in output.split():
remote_conf_files.append(os.path.join(self.conf_dir, f))
tmp_dir = "/tmp/mliroz_temp_hadoop/"
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
action = Get([hosts[0]], remote_conf_files, tmp_dir)
action.run()
# Do replacements in temp file
if conf_file:
f = os.path.join(tmp_dir, conf_file)
for name, value in params.iteritems():
replace_in_xml_file(f, name, value, True)
else:
temp_conf_files = [os.path.join(tmp_dir, f) for f in
os.listdir(tmp_dir)]
for name, value in params.iteritems():
for f in temp_conf_files:
if replace_in_xml_file(f, name, value):
break
else:
# Property not found - add it in MR_CONF_FILE
logger.info("Parameter with name " + name + " has not "
"been found in any conf file. Setting it "
"in " + default_file)
f = os.path.join(tmp_dir, default_file)
replace_in_xml_file(f, name, value, True)
# Copy back the files to all hosts
self._copy_conf(tmp_dir, hosts)
def _set_common_params(self, params, conf_dir, default_tuning=False):
"""Replace common parameters. Some user-specified values are
overwritten.
Args:
params (str):
Already defined parameters over all the clusters.
conf_dir (str):
The path of the directory with the configuration files.
default_tuning (bool, optional):
Whether to use automatic tuning based on some best practices or
leave the default parameters.
"""
defs_file = conf_dir + "/spark-defaults.conf"
# spark-env.sh
command = "cat >> " + self.conf_dir + "/spark-env.sh << EOF\n"
command += "SPARK_MASTER_PORT=" + str(self.port) + "\n"
command += "EOF\n"
action = Remote(command, self.hosts)
action.run()
# Get already set parameters
global_params = params["global"]
exec_mem = global_params["exec_mem"]
exec_cores = global_params["exec_cores"]
total_execs = global_params["total_execs"]
# Log parameters
if self.evs_log_dir:
write_in_props_file(defs_file,
"spark.eventLog.enabled", "true",
create_if_absent=True,
override=True)
write_in_props_file(defs_file,
"spark.eventLog.dir", self.evs_log_dir,
create_if_absent=True,
override=True)
write_in_props_file(defs_file,
"spark.logConf", "true",
create_if_absent=True,
override=False)
if default_tuning:
write_in_props_file(defs_file,
"spark.executor.memory", "%dm" % exec_mem,
create_if_absent=True,
override=False)
write_in_props_file(defs_file,
"spark.executor.cores", exec_cores,
create_if_absent=True,
override=False)
write_in_props_file(defs_file,
"spark.executor.instances", total_execs,
create_if_absent=True,
override=False)
def bootstrap(self, tar_file):
# 0. Check that required packages are present
required_packages = "openjdk-7-jre openjdk-7-jdk"
check_packages = TaktukRemote("dpkg -s " + required_packages,
self.hosts)
for p in check_packages.processes:
p.nolog_exit_code = p.nolog_error = True
check_packages.run()
if not check_packages.ok:
logger.info("Packages not installed, trying to install")
install_packages = TaktukRemote(
"export DEBIAN_MASTER=noninteractive ; " +
"apt-get update && apt-get install -y --force-yes " +
required_packages, self.hosts).run()
if not install_packages.ok:
logger.error("Unable to install the packages")
get_java_home = SshProcess('echo $(readlink -f /usr/bin/javac | '
'sed "s:/bin/javac::")', self.master)
get_java_home.run()
self.java_home = get_java_home.stdout.strip()
logger.info("All required packages are present")
# 1. Copy hadoop tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_dirs = TaktukRemote("rm -rf " + self.base_dir +
" " + self.conf_dir,
self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hosts)
SequentialActions([rm_dirs, put_tar, tar_xf]).run()
# 2. Move installation to base dir
logger.info("Create installation directories")
mv_base_dir = TaktukRemote(
"mv /tmp/" + os.path.basename(tar_file).replace(".tgz", "") + " " +
self.base_dir,
self.hosts)
mkdirs = TaktukRemote("mkdir -p " + self.conf_dir, self.hosts)
chmods = TaktukRemote("chmod g+w " + self.base_dir +
" && chmod g+w " + self.conf_dir,
self.hosts)
SequentialActions([mv_base_dir, mkdirs, chmods]).run()
# 3. Specify environment variables
command = "cat >> " + self.conf_dir + "/spark-env.sh << EOF\n"
command += "JAVA_HOME=" + self.java_home + "\n"
command += "SPARK_LOG_DIR=" + self.logs_dir + "\n"
if self.hc:
command += "HADOOP_CONF_DIR=" + self.hc.conf_dir + "\n"
if self.mode == YARN_MODE:
command += "YARN_CONF_DIR=" + self.hc.conf_dir + "\n"
command += "EOF\n"
command += "chmod +x " + self.conf_dir + "/spark-env.sh"
action = Remote(command, self.hosts)
action.run()
def get_conf(self, param_names):
params = {}
remaining_param_names = param_names[:]
# Copy conf files from first host in the cluster
action = Remote("ls " + self.conf_dir + "/*.xml", [self.hosts[0]])
action.run()
output = action.processes[0].stdout
remote_conf_files = []
for f in output.split():
remote_conf_files.append(os.path.join(self.conf_dir, f))
tmp_dir = "/tmp/mliroz_temp_hadoop/"
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
action = Get([self.hosts[0]], remote_conf_files, tmp_dir)
action.run()
# Do replacements in temp file
temp_conf_files = [os.path.join(tmp_dir, f) for f in
os.listdir(tmp_dir)]
for f in temp_conf_files:
fparams = get_xml_params(f, remaining_param_names)
for p in fparams:
if fparams[p]:
params[p] = fparams[p]
remaining_param_names.remove(p)
return params
def _initialize_conf(self):
"""Merge locally-specified configuration files with default files
from the distribution."""
action = Remote("cp " + os.path.join(self.conf_dir,
SPARK_CONF_FILE + ".template ") +
os.path.join(self.conf_dir, SPARK_CONF_FILE),
self.hosts)
action.run()
if os.path.exists(self.local_base_conf_dir):
base_conf_files = [os.path.join(self.local_base_conf_dir, f)
for f in os.listdir(self.local_base_conf_dir)]
for f in base_conf_files:
shutil.copy(f, self.init_conf_dir)
else:
logger.warn(
"Local conf dir does not exist. Using default configuration")
base_conf_files = []
missing_conf_files = self.conf_mandatory_files
for f in base_conf_files:
f_base_name = os.path.basename(f)
if f_base_name in missing_conf_files:
missing_conf_files.remove(f_base_name)
logger.info("Copying missing conf files from master: " + str(
missing_conf_files))
remote_missing_files = [os.path.join(self.conf_dir, f)
for f in missing_conf_files]
action = Get([self.master], remote_missing_files, self.init_conf_dir)
action.run()
def check_java_version(java_major_version, hosts):
tr = Remote("java -version 2>&1 | grep version", hosts)
tr.run()
for p in tr.processes:
match = re.match('.*[^.0-9]1\.([0-9]+).[0-9].*', p.stdout)
version = int(match.group(1))
if java_major_version > version:
msg = "Java 1.%d+ required" % java_major_version
return False
return True
def clean_history(self):
"""Remove history."""
logger.info("Cleaning history")
restart = False
if self.running:
logger.warn("The cluster needs to be stopped before cleaning.")
self.stop()
restart = True
action = Remote("rm -rf " + self.logs_dir + "/history", [self.master])
action.run()
if restart:
self.start()
def clean_logs(self):
"""Remove all Hadoop logs."""
logger.info("Cleaning logs")
restart = False
if self.running:
logger.warn("The cluster needs to be stopped before cleaning.")
self.stop()
restart = True
action = Remote("rm -rf " + self.logs_dir + "/*", self.hosts)
action.run()
if restart:
self.start()
def clean_logs(self):
"""Remove all Hive logs."""
logger.info("Cleaning logs")
restart = False
if self.running:
logger.warn("The cluster needs to be stopped before cleaning.")
self.stop()
restart = True
action = Remote("rm -rf " + self.logs_dir + "/* ", self.hosts)
action.run()
if restart:
self.start()
def clean_history(self):
"""Remove history."""
logger.info("Cleaning history")
restart = False
if self.running:
logger.warn("The cluster needs to be stopped before cleaning.")
self.stop()
restart = True
action = Remote("rm -rf " + self.logs_dir + "/history",
[self.master])
action.run()
if restart:
self.start()
def change_conf(self, params):
"""Modify Hadoop configuration. This method copies the configuration
files from the first host of each g5k cluster conf dir into a local
temporary dir, do all the changes in place and broadcast the new
configuration files to all hosts.
Args:
params (dict of str:str):
The parameters to be changed in the form key:value.
"""
for g5k_cluster in self.host_clusters:
hosts = self.host_clusters[g5k_cluster]
# Copy conf files from first host in the cluster
action = Remote("ls " + self.conf_dir + "/*.xml", [hosts[0]])
action.run()
output = action.processes[0].stdout
remote_conf_files = []
for f in output.split():
remote_conf_files.append(os.path.join(self.conf_dir, f))
tmp_dir = "/tmp/mliroz_temp_hadoop/"
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
action = Get([hosts[0]], remote_conf_files, tmp_dir)
action.run()
# Do replacements in temp file
temp_conf_files = [os.path.join(tmp_dir, f) for f in
os.listdir(tmp_dir)]
for name, value in params.iteritems():
for f in temp_conf_files:
if replace_in_xml_file(f, name, value):
break
else:
# Property not found - provisionally add it in MR_CONF_FILE
f = os.path.join(tmp_dir, MR_CONF_FILE)
replace_in_xml_file(f, name, value, True)
# Copy back the files to all hosts
self._copy_conf(tmp_dir, hosts)
def change_conf(self, params):
"""Modify Hadoop configuration. This method copies the configuration
files from the first host of each g5k cluster conf dir into a local
temporary dir, do all the changes in place and broadcast the new
configuration files to all hosts.
Args:
params (dict of str:str):
The parameters to be changed in the form key:value.
"""
for g5k_cluster in self.host_clusters:
hosts = self.host_clusters[g5k_cluster]
# Copy conf files from first host in the cluster
action = Remote("ls " + self.conf_dir + "/*.xml", [hosts[0]])
action.run()
output = action.processes[0].stdout
remote_conf_files = []
for f in output.split():
remote_conf_files.append(os.path.join(self.conf_dir, f))
tmp_dir = "/tmp/mliroz_temp_hadoop/"
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
action = Get([hosts[0]], remote_conf_files, tmp_dir)
action.run()
# Do replacements in temp file
temp_conf_files = [os.path.join(tmp_dir, f) for f in
os.listdir(tmp_dir)]
for name, value in params.iteritems():
for f in temp_conf_files:
if replace_in_xml_file(f, name, value):
break
else:
# Property not found - provisionally add it in MR_CONF_FILE
f = os.path.join(tmp_dir, MR_CONF_FILE)
replace_in_xml_file(f, name, value, True)
# Copy back the files to all hosts
self._copy_conf(tmp_dir, hosts)
def _configure_servers(self, hosts=None):
"""Configure servers and host-dependant parameters.
Args:
hosts (list of Host, optional):
The list of hosts to take into account in the configuration. If
not specified, all the hosts of the Spark cluster are used. The
first host of this list is always used as the reference.
"""
if not hosts:
hosts = self.hosts
host_attrs = get_host_attributes(hosts[0])
num_cores = host_attrs[u'architecture'][u'smt_size']
total_memory_mb = (int(host_attrs[u'main_memory'][u'ram_size']) /
(1024 * 1024))
memory_per_worker = int(0.75 * total_memory_mb)
memory_per_task = int(memory_per_worker / num_cores)
# Set memory for each worker
command = "cat >> " + self.conf_dir + "/spark-env.sh << EOF\n"
command += "SPARK_MASTER_PORT=" + str(self.port) + "\n"
command += "SPARK_WORKER_MEMORY=" + str(memory_per_worker) + "m\n"
command += "EOF\n"
action = Remote(command, self.hosts)
action.run()
# Default parameters
driver_mem = "1g"
executor_mem = str(memory_per_task) + "m"
with open(self.temp_conf_dir + "/spark-defaults.conf", "a") \
as defaults_file:
defaults_file.write("spark.executor.memory\t" + executor_mem + "\n")
defaults_file.write("spark.driver.memory\t" + driver_mem + "\n")
# defaults_file.write("spark.driver.maxResultSize\t1g\n")
defaults_file.write("spark.logConf\ttrue\n")
# defaults_file.write("spark.python.worker.memory\t512m")
if self.evs_log_dir:
defaults_file.write("spark.eventLog.enabled\ttrue\n")
defaults_file.write("spark.eventLog.dir\t" +
self.evs_log_dir + "\n")
def clean_data(self):
"""Remove all data created by Hadoop (including filesystem)."""
if self.running:
logger.warn("The cluster needs to be stopped before cleaning.")
self.stop()
logger.info("Cleaning MongoDB data")
restart = False
if self.running:
self.stop()
restart = True
action = Remote("rm -rf " + self.data_dir + "/*", self.hosts)
action.run()
if restart:
self.start()
def _configure_servers(self, hosts=None):
"""Configure servers and host-dependant parameters.
Args:
hosts (list of Host, optional):
The list of hosts to take into account in the configuration. If
not specified, all the hosts of the Spark cluster are used. The
first host of this list is always used as the reference.
"""
if not hosts:
hosts = self.hosts
host_attrs = get_host_attributes(hosts[0])
num_cores = host_attrs[u'architecture'][u'smt_size']
total_memory_mb = (int(host_attrs[u'main_memory'][u'ram_size']) /
(1024 * 1024))
memory_per_worker = int(0.75 * total_memory_mb)
memory_per_task = int(memory_per_worker / num_cores)
# Set memory for each worker
command = "cat >> " + self.conf_dir + "/spark-env.sh << EOF\n"
command += "SPARK_MASTER_PORT=" + str(self.port) + "\n"
command += "SPARK_WORKER_MEMORY=" + str(memory_per_worker) + "m\n"
command += "EOF\n"
action = Remote(command, self.hosts)
action.run()
# Default parameters
driver_mem = "1g"
executor_mem = str(memory_per_task) + "m"
with open(self.temp_conf_dir + "/spark-defaults.conf", "a") \
as defaults_file:
defaults_file.write("spark.executor.memory\t" + executor_mem +
"\n")
defaults_file.write("spark.driver.memory\t" + driver_mem + "\n")
# defaults_file.write("spark.driver.maxResultSize\t1g\n")
defaults_file.write("spark.logConf\ttrue\n")
# defaults_file.write("spark.python.worker.memory\t512m")
defaults_file.write("spark.eventLog.enabled\ttrue\n")
defaults_file.write("spark.eventLog.dir\t" + self.event_log_dir +
"\n")
def clean_data(self):
"""Remove all data created by Hadoop (including filesystem)."""
if self.running:
logger.warn("The cluster needs to be stopped before cleaning.")
self.stop()
logger.info("Cleaning hadoop data")
restart = False
if self.running:
self.stop()
restart = True
action = Remote("rm -rf " + self.hadoop_temp_dir + " /tmp/hadoop-" +
getpass.getuser() + "-*", self.hosts)
action.run()
if restart:
self.start()
def clean_data(self):
"""Remove all data created by Hadoop (including filesystem)."""
if self.running:
logger.warn("The cluster needs to be stopped before cleaning.")
self.stop()
logger.info("Cleaning hadoop data")
restart = False
if self.running:
self.stop()
restart = True
action = Remote("rm -rf " + self.hadoop_temp_dir + " /tmp/hadoop-" +
getpass.getuser() + "-*", self.hosts)
action.run()
if restart:
self.start()
def bootstrap(self, tar_file):
"""Install MongoDB in all cluster nodes from the specified tgz file.
Args:
tar_file (str):
The file containing MongoDB binaries.
"""
# 1. Copy mongo tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_files = TaktukRemote("rm -rf " + self.base_dir +
" " + self.conf_dir +
" " + self.data_dir +
" " + self.logs_dir,
self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote("tar xf /tmp/" + os.path.basename(tar_file) +
" -C /tmp", self.hosts)
SequentialActions([rm_files, put_tar, tar_xf]).run()
# 2. Move installation to base dir
logger.info("Create installation directories")
action = Remote(
"mv /tmp/" +
os.path.basename(tar_file).replace(".tgz", "") + " " +
self.base_dir,
self.hosts)
action.run()
# 3 Create other dirs
mkdirs = TaktukRemote("mkdir -p " + self.data_dir +
" && mkdir -p " + self.conf_dir +
" && mkdir -p " + self.logs_dir +
" && touch " + os.path.join(self.conf_dir,
CONF_FILE),
self.hosts)
mkdirs.run()
# 4. Generate initial configuration
self._initialize_conf()
def _get_conf_files(self, host):
action = Remote("ls " + self.conf_dir + "/*.xml", [host])
action.run()
output = action.processes[0].stdout
remote_conf_files = []
for f in output.split():
remote_conf_files.append(os.path.join(self.conf_dir, f))
tmp_dir = "/tmp/mliroz_temp_hadoop/"
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
action = Get([host], remote_conf_files, tmp_dir)
action.run()
temp_conf_files = [os.path.join(tmp_dir, f) for f in
os.listdir(tmp_dir)]
return temp_conf_files
def _get_conf_files(self, host):
action = Remote("ls " + self.conf_dir + "/*.xml", [host])
action.run()
output = action.processes[0].stdout
remote_conf_files = []
for f in output.split():
remote_conf_files.append(os.path.join(self.conf_dir, f))
tmp_dir = "/tmp/mliroz_temp_hadoop/"
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
action = Get([host], remote_conf_files, tmp_dir)
action.run()
temp_conf_files = [os.path.join(tmp_dir, f) for f in
os.listdir(tmp_dir)]
return temp_conf_files
def _initialize_conf(self):
"""Merge locally-specified configuration files with default files
from the distribution."""
action = Remote(
"cp " +
os.path.join(self.conf_dir, SPARK_CONF_FILE + ".template ") +
os.path.join(self.conf_dir, SPARK_CONF_FILE), self.hosts)
action.run()
if os.path.exists(self.local_base_conf_dir):
base_conf_files = [
os.path.join(self.local_base_conf_dir, f)
for f in os.listdir(self.local_base_conf_dir)
]
for f in base_conf_files:
shutil.copy(f, self.init_conf_dir)
else:
logger.warn(
"Local conf dir does not exist. Using default configuration")
base_conf_files = []
missing_conf_files = self.conf_mandatory_files
for f in base_conf_files:
f_base_name = os.path.basename(f)
if f_base_name in missing_conf_files:
missing_conf_files.remove(f_base_name)
logger.info("Copying missing conf files from master: " +
str(missing_conf_files))
remote_missing_files = [
os.path.join(self.conf_dir, f) for f in missing_conf_files
]
action = Get([self.master], remote_missing_files, self.init_conf_dir)
action.run()
def bootstrap(self, tar_file):
# 1. Remove used dirs if existing
action = Remote("rm -rf " + self.base_dir, self.hc.hosts)
action.run()
action = Remote("rm -rf " + self.conf_dir, self.hc.hosts)
action.run()
# 1. Copy Mahout tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
action = Put(self.hc.hosts, [tar_file], "/tmp")
action.run()
action = Remote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hc.hosts)
action.run()
# 2. Move installation to base dir
logger.info("Create installation directories")
action = Remote(
"mv /tmp/" + os.path.basename(tar_file).replace(".tar.gz", "") +
" " + self.base_dir, self.hc.hosts)
action.run()
# 3 Create other dirs
action = Remote("mkdir -p " + self.conf_dir, self.hc.hosts)
action.run()
# 4. Include libraries in Hadoop's classpath
list_dirs = SshProcess("ls -1 " + self.base_dir + "/*.jar",
self.hc.master)
list_dirs.run()
libs = " ".join(list_dirs.stdout.splitlines())
action = Remote("cp " + libs + " " + self.hc.base_dir + "/lib",
self.hc.hosts)
action.run()
initialized = True # No need to call initialize()
def bootstrap(self, tar_file):
"""Install Hadoop in all cluster nodes from the specified tar.gz file.
Args:
tar_file (str):
The file containing Hadoop binaries.
"""
# 0. Check requirements
java_major_version = 7
if not check_java_version(java_major_version, self.hosts):
msg = "Java 1.%d+ required" % java_major_version
logger.error(msg)
raise HadoopException(msg)
self.java_home = get_java_home(self.master)
# 1. Copy hadoop tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_dirs = TaktukRemote(
"rm -rf " + self.base_dir + " " + self.conf_dir + " " +
self.logs_dir + " " + self.hadoop_temp_dir, self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hosts)
rm_tar = TaktukRemote("rm /tmp/" + os.path.basename(tar_file),
self.hosts)
SequentialActions([rm_dirs, put_tar, tar_xf, rm_tar]).run()
# 2. Move installation to base dir and create other dirs
logger.info("Create installation directories")
mv_base_dir = TaktukRemote(
"mv /tmp/" + os.path.basename(tar_file).replace(".tar.gz", "") +
" " + self.base_dir, self.hosts)
mkdirs = TaktukRemote(
"mkdir -p " + self.conf_dir + " && mkdir -p " + self.logs_dir +
" && mkdir -p " + self.hadoop_temp_dir, self.hosts)
chmods = TaktukRemote(
"chmod g+w " + self.base_dir + " && chmod g+w " + self.conf_dir +
" && chmod g+w " + self.logs_dir + " && chmod g+w " +
self.hadoop_temp_dir, self.hosts)
SequentialActions([mv_base_dir, mkdirs, chmods]).run()
# 4. Specify environment variables
command = "cat >> " + self.conf_dir + "/hadoop-env.sh << EOF\n"
command += "export JAVA_HOME=" + self.java_home + "\n"
command += "export HADOOP_LOG_DIR=" + self.logs_dir + "\n"
command += "HADOOP_HOME_WARN_SUPPRESS=\"TRUE\"\n"
command += "EOF"
action = Remote(command, self.hosts)
action.run()
# 5. Check version (cannot do it before)
if not self._check_version_compliance():
return False
# 6. Generate initial configuration
self._initialize_conf()
return True
def bootstrap(self, tar_file):
"""Install Hadoop in all cluster nodes from the specified tar.gz file.
Args:
tar_file (str):
The file containing Hadoop binaries.
"""
# 0. Check that required packages are present
required_packages = "openjdk-7-jre openjdk-7-jdk"
check_packages = TaktukRemote("dpkg -s " + required_packages,
self.hosts)
for p in check_packages.processes:
p.nolog_exit_code = p.nolog_error = True
check_packages.run()
if not check_packages.ok:
logger.info("Packages not installed, trying to install")
install_packages = TaktukRemote(
"export DEBIAN_MASTER=noninteractive ; " +
"apt-get update && apt-get install -y --force-yes " +
required_packages, self.hosts).run()
if not install_packages.ok:
logger.error("Unable to install the packages")
get_java_home = SshProcess('echo $(readlink -f /usr/bin/javac | '
'sed "s:/bin/javac::")', self.master)
get_java_home.run()
self.java_home = get_java_home.stdout.strip()
logger.info("All required packages are present")
# 1. Copy hadoop tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_dirs = Remote("rm -rf " + self.base_dir +
" " + self.conf_dir +
" " + self.logs_dir +
" " + self.hadoop_temp_dir,
self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hosts)
SequentialActions([rm_dirs, put_tar, tar_xf]).run()
# 2. Move installation to base dir and create other dirs
logger.info("Create installation directories")
mv_base_dir = TaktukRemote(
"mv /tmp/" +
os.path.basename(tar_file).replace(".tar.gz", "") + " " +
self.base_dir,
self.hosts)
mkdirs = TaktukRemote("mkdir -p " + self.conf_dir +
" && mkdir -p " + self.logs_dir +
" && mkdir -p " + self.hadoop_temp_dir,
self.hosts)
chmods = TaktukRemote("chmod g+w " + self.base_dir +
" && chmod g+w " + self.conf_dir +
" && chmod g+w " + self.logs_dir +
" && chmod g+w " + self.hadoop_temp_dir,
self.hosts)
SequentialActions([mv_base_dir, mkdirs, chmods]).run()
# 4. Specify environment variables
command = "cat >> " + self.conf_dir + "/hadoop-env.sh << EOF\n"
command += "export JAVA_HOME=" + self.java_home + "\n"
command += "export HADOOP_LOG_DIR=" + self.logs_dir + "\n"
command += "HADOOP_HOME_WARN_SUPPRESS=\"TRUE\"\n"
command += "EOF"
action = Remote(command, self.hosts)
action.run()
# 5. Check version
return self._check_version_compliance()
def uncompress(file_name, host):
if file_name.endswith("tar.gz"):
decompression = Remote("tar xf " + file_name, [host])
decompression.run()
base_name = os.path.basename(file_name[:-7])
dir_name = os.path.dirname(file_name[:-7])
new_name = dir_name + "/data-" + base_name
action = Remote("mv " + file_name[:-7] + " " + new_name, [host])
action.run()
elif file_name.endswith("gz"):
decompression = Remote("gzip -d " + file_name, [host])
decompression.run()
base_name = os.path.basename(file_name[:-3])
dir_name = os.path.dirname(file_name[:-3])
new_name = dir_name + "/data-" + base_name
action = Remote("mv " + file_name[:-3] + " " + new_name, [host])
action.run()
elif file_name.endswith("zip"):
decompression = Remote("unzip " + file_name, [host])
decompression.run()
base_name = os.path.basename(file_name[:-4])
dir_name = os.path.dirname(file_name[:-4])
new_name = dir_name + "/data-" + base_name
action = Remote("mv " + file_name[:-4] + " " + new_name, [host])
action.run()
elif file_name.endswith("bz2"):
decompression = Remote("bzip2 -d " + file_name, [host])
decompression.run()
base_name = os.path.basename(file_name[:-4])
dir_name = os.path.dirname(file_name[:-4])
new_name = dir_name + "/data-" + base_name
action = Remote("mv " + file_name[:-4] + " " + new_name, [host])
action.run()
else:
logger.warn("Unknown extension")
return file_name
return new_name
def bootstrap(self, tar_file):
# 1. Remove used dirs if existing
action = Remote("rm -rf " + self.base_dir, self.hc.hosts)
action.run()
action = Remote("rm -rf " + self.conf_dir, self.hc.hosts)
action.run()
# 1. Copy Mahout tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
action = Put(self.hc.hosts, [tar_file], "/tmp")
action.run()
action = Remote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hc.hosts)
action.run()
# 2. Move installation to base dir
logger.info("Create installation directories")
action = Remote(
"mv /tmp/" +
os.path.basename(tar_file).replace(".tar.gz", "") + " " +
self.base_dir,
self.hc.hosts)
action.run()
# 3 Create other dirs
action = Remote("mkdir -p " + self.conf_dir, self.hc.hosts)
action.run()
# 4. Include libraries in Hadoop's classpath
list_dirs = SshProcess("ls -1 " + self.base_dir + "/*.jar",
self.hc.master)
list_dirs.run()
libs = " ".join(list_dirs.stdout.splitlines())
action = Remote("cp " + libs + " " + self.hc.base_dir + "/lib",
self.hc.hosts)
action.run()
initialized = True # No need to call initialize()
def load(self, hc, dest, desired_size=None):
"""Load the dataset in the given dfs folder by copying it from the
local folder.
Args:
hc (HadoopCluster):
The Hadoop cluster where to deploy the dataset.
dest (str):
The dfs destination folder.
desired_size (int, optional):
The size of the data to be copied. If indicated only the first files
of the dataset up to the given size are copied, if not, the whole
dataset is transferred.
"""
dataset_files = [
os.path.join(self.local_path, f)
for f in os.listdir(self.local_path)
]
hosts = hc.hosts
# Define and create temp dir
tmp_dir = "/tmp" + dest
action_remove = Remote("rm -rf " + tmp_dir, hosts)
action_remove.run()
action_create = Remote("mkdir -p " + tmp_dir, hosts)
action_create.run()
# Generate list of files to copy
if desired_size:
all_files_to_copy = []
dataset_files.sort()
real_size = 0
while real_size < desired_size:
if dataset_files:
all_files_to_copy.append(dataset_files[0])
real_size += os.path.getsize(dataset_files[0])
del dataset_files[0]
else:
logger.warn("Dataset files do not fill up to desired size "
"(real size = " + str(real_size) + ")")
break
else:
real_size = 0
all_files_to_copy = dataset_files
for f in all_files_to_copy:
real_size += os.path.getsize(f)
# Assign files to hosts
files_per_host = [[]] * len(hosts)
for idx in range(0, len(hosts)):
files_per_host[idx] = all_files_to_copy[idx::len(hosts)]
# Create threads and launch them
logger.info("Loading dataset in parallel into " + str(len(hosts)) +
" hosts")
if not hc.running:
hc.start()
class SizeCollector:
size = 0
lock = threading.Lock()
def __init__(self):
pass
def increment(self, qty):
self.lock.acquire()
try:
self.size += qty
finally:
self.lock.release()
def copy_function(host, files_to_copy, collector=None):
action = Put([host], files_to_copy, tmp_dir)
action.run()
local_final_size = 0
for f in files_to_copy:
src_file = os.path.join(tmp_dir, os.path.basename(f))
if self.pre_load_function:
src_file = self.pre_load_function(src_file, host)
action = SshProcess("du -b " + src_file + "| cut -f1",
host)
action.run()
local_final_size += int(action.stdout.strip())
hc.execute(
"fs -put " + src_file + " " +
os.path.join(dest, os.path.basename(src_file)), host, True,
False)
if collector:
collector.increment(local_final_size)
if self.pre_load_function:
final_size = SizeCollector()
else:
final_size = None
threads = []
for idx, h in enumerate(hosts):
if files_per_host[idx]:
t = threading.Thread(target=copy_function,
args=(h, files_per_host[idx], final_size))
t.start()
threads.append(t)
# Wait for the threads to finish
for t in threads:
t.join()
logger.info("Loading completed: real local size = " + str(real_size) +
", final remote size = " + str(final_size.size))
self.deployments[hc, desired_size] = dest
def bootstrap(self, tar_file):
"""Install Hadoop in all cluster nodes from the specified tar.gz file.
Args:
tar_file (str):
The file containing Hadoop binaries.
"""
# 0. Check requirements
java_major_version = 7
if not check_java_version(java_major_version, self.hosts):
msg = "Java 1.%d+ required" % java_major_version
logger.error(msg)
raise HadoopException(msg)
self.java_home = get_java_home(self.master)
# 1. Copy hadoop tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_dirs = TaktukRemote("rm -rf " + self.base_dir +
" " + self.conf_dir +
" " + self.logs_dir +
" " + self.hadoop_temp_dir,
self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hosts)
rm_tar = TaktukRemote(
"rm /tmp/" + os.path.basename(tar_file),
self.hosts)
SequentialActions([rm_dirs, put_tar, tar_xf, rm_tar]).run()
# 2. Move installation to base dir and create other dirs
logger.info("Create installation directories")
mv_base_dir = TaktukRemote(
"mv /tmp/" +
os.path.basename(tar_file).replace(".tar.gz", "") + " " +
self.base_dir,
self.hosts)
mkdirs = TaktukRemote("mkdir -p " + self.conf_dir +
" && mkdir -p " + self.logs_dir +
" && mkdir -p " + self.hadoop_temp_dir,
self.hosts)
chmods = TaktukRemote("chmod g+w " + self.base_dir +
" && chmod g+w " + self.conf_dir +
" && chmod g+w " + self.logs_dir +
" && chmod g+w " + self.hadoop_temp_dir,
self.hosts)
SequentialActions([mv_base_dir, mkdirs, chmods]).run()
# 4. Specify environment variables
command = "cat >> " + self.conf_dir + "/hadoop-env.sh << EOF\n"
command += "export JAVA_HOME=" + self.java_home + "\n"
command += "export HADOOP_LOG_DIR=" + self.logs_dir + "\n"
command += "HADOOP_HOME_WARN_SUPPRESS=\"TRUE\"\n"
command += "EOF"
action = Remote(command, self.hosts)
action.run()
# 5. Check version (cannot do it before)
if not self._check_version_compliance():
return False
# 6. Generate initial configuration
self._initialize_conf()
return True
def uncompress(file_name, host):
if file_name.endswith("tar.gz"):
decompression = Remote("tar xf " + file_name, [host])
decompression.run()
base_name = os.path.basename(file_name[:-7])
dir_name = os.path.dirname(file_name[:-7])
new_name = dir_name + "/data-" + base_name
action = Remote("mv " + file_name[:-7] + " " + new_name, [host])
action.run()
elif file_name.endswith("gz"):
decompression = Remote("gzip -d " + file_name, [host])
decompression.run()
base_name = os.path.basename(file_name[:-3])
dir_name = os.path.dirname(file_name[:-3])
new_name = dir_name + "/data-" + base_name
action = Remote("mv " + file_name[:-3] + " " + new_name, [host])
action.run()
elif file_name.endswith("zip"):
decompression = Remote("unzip " + file_name, [host])
decompression.run()
base_name = os.path.basename(file_name[:-4])
dir_name = os.path.dirname(file_name[:-4])
new_name = dir_name + "/data-" + base_name
action = Remote("mv " + file_name[:-4] + " " + new_name, [host])
action.run()
elif file_name.endswith("bz2"):
decompression = Remote("bzip2 -d " + file_name, [host])
decompression.run()
base_name = os.path.basename(file_name[:-4])
dir_name = os.path.dirname(file_name[:-4])
new_name = dir_name + "/data-" + base_name
action = Remote("mv " + file_name[:-4] + " " + new_name, [host])
action.run()
else:
logger.warn("Unknown extension")
return file_name
return new_name
def bootstrap(self, tar_file):
"""Install Hadoop in all cluster nodes from the specified tar.gz file.
Args:
tar_file (str):
The file containing Hadoop binaries.
"""
# 0. Check that required packages are present
required_packages = "openjdk-7-jre openjdk-7-jdk"
check_packages = TaktukRemote("dpkg -s " + required_packages,
self.hosts)
for p in check_packages.processes:
p.nolog_exit_code = p.nolog_error = True
check_packages.run()
if not check_packages.ok:
logger.info("Packages not installed, trying to install")
install_packages = TaktukRemote(
"export DEBIAN_MASTER=noninteractive ; " +
"apt-get update && apt-get install -y --force-yes " +
required_packages, self.hosts).run()
if not install_packages.ok:
logger.error("Unable to install the packages")
get_java_home = SshProcess('echo $(readlink -f /usr/bin/javac | '
'sed "s:/bin/javac::")', self.master)
get_java_home.run()
self.java_home = get_java_home.stdout.strip()
logger.info("All required packages are present")
# 1. Copy hadoop tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_dirs = Remote("rm -rf " + self.base_dir +
" " + self.conf_dir +
" " + self.logs_dir +
" " + self.hadoop_temp_dir,
self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hosts)
SequentialActions([rm_dirs, put_tar, tar_xf]).run()
# 2. Move installation to base dir and create other dirs
logger.info("Create installation directories")
mv_base_dir = TaktukRemote(
"mv /tmp/" +
os.path.basename(tar_file).replace(".tar.gz", "") + " " +
self.base_dir,
self.hosts)
mkdirs = TaktukRemote("mkdir -p " + self.conf_dir +
" && mkdir -p " + self.logs_dir +
" && mkdir -p " + self.hadoop_temp_dir,
self.hosts)
chmods = TaktukRemote("chmod g+w " + self.base_dir +
" && chmod g+w " + self.conf_dir +
" && chmod g+w " + self.logs_dir +
" && chmod g+w " + self.hadoop_temp_dir,
self.hosts)
SequentialActions([mv_base_dir, mkdirs, chmods]).run()
# 4. Specify environment variables
command = "cat >> " + self.conf_dir + "/hadoop-env.sh << EOF\n"
command += "export JAVA_HOME=" + self.java_home + "\n"
command += "export HADOOP_LOG_DIR=" + self.logs_dir + "\n"
command += "HADOOP_HOME_WARN_SUPPRESS=\"TRUE\"\n"
command += "EOF"
action = Remote(command, self.hosts)
action.run()
# 5. Check version
return self._check_version_compliance()
def bootstrap(self, tar_file):
# 0. Check requirements
java_major_version = 7
if not check_java_version(java_major_version, self.hosts):
msg = "Java 1.%d+ required" % java_major_version
logger.error(msg)
raise SparkException(msg)
self.java_home = get_java_home(self.master)
# 1. Copy hadoop tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_dirs = TaktukRemote("rm -rf " + self.base_dir + " " + self.conf_dir,
self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hosts)
rm_tar = TaktukRemote("rm /tmp/" + os.path.basename(tar_file),
self.hosts)
SequentialActions([rm_dirs, put_tar, tar_xf, rm_tar]).run()
# 2. Move installation to base dir
logger.info("Create installation directories")
mv_base_dir = TaktukRemote(
"mv /tmp/" + os.path.basename(tar_file).replace(".tgz", "") + " " +
self.base_dir, self.hosts)
mkdirs = TaktukRemote(
"mkdir -p " + self.conf_dir + " && mkdir -p " + self.logs_dir,
self.hosts)
chmods = TaktukRemote(
"chmod g+w " + self.base_dir + " && chmod g+w " + self.conf_dir +
" && chmod g+w " + self.logs_dir, self.hosts)
SequentialActions([mv_base_dir, mkdirs, chmods]).run()
# 2.1. Create spark-events dir
if self.evs_log_dir:
if self.evs_log_dir.startswith("file://") or \
"://" not in self.evs_log_dir:
mk_evs_dir = TaktukRemote(
"mkdir -p " + self.evs_log_dir + " && chmod g+w " +
self.evs_log_dir, self.hosts)
mk_evs_dir.run()
elif self.evs_log_dir.startswith("hdfs://"):
self.hc.execute("fs -mkdir -p " + self.evs_log_dir)
# 3. Specify environment variables
env_file = self.conf_dir + "/spark-env.sh"
command = "cat >> " + env_file + " << EOF\n"
command += "JAVA_HOME=" + self.java_home + "\n"
command += "SPARK_LOG_DIR=" + self.logs_dir + "\n"
if self.hc:
command += "HADOOP_CONF_DIR=" + self.hc.conf_dir + "\n"
if self.mode == YARN_MODE:
command += "YARN_CONF_DIR=" + self.hc.conf_dir + "\n"
command += "EOF\n"
command += "echo SPARK_PUBLIC_DNS=$(hostname) >> " + env_file
command += " && chmod +x " + env_file
action = Remote(command, self.hosts)
action.run()
# 4. Generate initial configuration
self._initialize_conf()
def check_packages(packages, hosts):
tr = Remote("dpkg -s " + packages, hosts)
for p in tr.processes:
p.nolog_exit_code = p.nolog_error = True
tr.run()
return tr.ok
def _set_common_params(self, params, conf_dir, default_tuning=False):
"""Replace common parameters. Some user-specified values are
overwritten.
Args:
params (str):
Already defined parameters over all the clusters.
conf_dir (str):
The path of the directory with the configuration files.
default_tuning (bool, optional):
Whether to use automatic tuning based on some best practices or
leave the default parameters.
"""
defs_file = conf_dir + "/spark-defaults.conf"
# spark-env.sh
command = "cat >> " + self.conf_dir + "/spark-env.sh << EOF\n"
command += "SPARK_MASTER_PORT=" + str(self.port) + "\n"
command += "EOF\n"
action = Remote(command, self.hosts)
action.run()
# Get already set parameters
global_params = params["global"]
exec_mem = global_params["exec_mem"]
exec_cores = global_params["exec_cores"]
total_execs = global_params["total_execs"]
# Log parameters
if self.evs_log_dir:
write_in_props_file(defs_file,
"spark.eventLog.enabled",
"true",
create_if_absent=True,
override=True)
write_in_props_file(defs_file,
"spark.eventLog.dir",
self.evs_log_dir,
create_if_absent=True,
override=True)
write_in_props_file(defs_file,
"spark.logConf",
"true",
create_if_absent=True,
override=False)
if default_tuning:
write_in_props_file(defs_file,
"spark.executor.memory",
"%dm" % exec_mem,
create_if_absent=True,
override=False)
write_in_props_file(defs_file,
"spark.executor.cores",
exec_cores,
create_if_absent=True,
override=False)
write_in_props_file(defs_file,
"spark.executor.instances",
total_execs,
create_if_absent=True,
override=False)
def bootstrap(self, tar_file):
# 0. Check requirements
java_major_version = 7
if not check_java_version(java_major_version, self.hosts):
msg = "Java 1.%d+ required" % java_major_version
logger.error(msg)
raise SparkException(msg)
self.java_home = get_java_home(self.master)
# 1. Copy hadoop tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_dirs = TaktukRemote("rm -rf " + self.base_dir +
" " + self.conf_dir,
self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hosts)
rm_tar = TaktukRemote(
"rm /tmp/" + os.path.basename(tar_file),
self.hosts)
SequentialActions([rm_dirs, put_tar, tar_xf, rm_tar]).run()
# 2. Move installation to base dir
logger.info("Create installation directories")
mv_base_dir = TaktukRemote(
"mv /tmp/" + os.path.basename(tar_file).replace(".tgz", "") + " " +
self.base_dir,
self.hosts)
mkdirs = TaktukRemote("mkdir -p " + self.conf_dir +
" && mkdir -p " + self.logs_dir,
self.hosts)
chmods = TaktukRemote("chmod g+w " + self.base_dir +
" && chmod g+w " + self.conf_dir +
" && chmod g+w " + self.logs_dir,
self.hosts)
SequentialActions([mv_base_dir, mkdirs, chmods]).run()
# 2.1. Create spark-events dir
if self.evs_log_dir:
if self.evs_log_dir.startswith("file://") or \
"://" not in self.evs_log_dir:
mk_evs_dir = TaktukRemote("mkdir -p " + self.evs_log_dir +
" && chmod g+w " + self.evs_log_dir,
self.hosts)
mk_evs_dir.run()
elif self.evs_log_dir.startswith("hdfs://"):
self.hc.execute("fs -mkdir -p " + self.evs_log_dir)
# 3. Specify environment variables
env_file = self.conf_dir + "/spark-env.sh"
command = "cat >> " + env_file + " << EOF\n"
command += "JAVA_HOME=" + self.java_home + "\n"
command += "SPARK_LOG_DIR=" + self.logs_dir + "\n"
if self.hc:
command += "HADOOP_CONF_DIR=" + self.hc.conf_dir + "\n"
if self.mode == YARN_MODE:
command += "YARN_CONF_DIR=" + self.hc.conf_dir + "\n"
command += "EOF\n"
command += "echo SPARK_PUBLIC_DNS=$(hostname) >> " + env_file
command += " && chmod +x " + env_file
action = Remote(command, self.hosts)
action.run()
# 4. Generate initial configuration
self._initialize_conf()