def bootstrap(self, tar_file):
# 0. Check that required packages are present
required_packages = "openjdk-7-jre openjdk-7-jdk"
check_packages = TaktukRemote("dpkg -s " + required_packages,
self.hosts)
for p in check_packages.processes:
p.nolog_exit_code = p.nolog_error = True
check_packages.run()
if not check_packages.ok:
logger.info("Packages not installed, trying to install")
install_packages = TaktukRemote(
"export DEBIAN_MASTER=noninteractive ; " +
"apt-get update && apt-get install -y --force-yes " +
required_packages, self.hosts).run()
if not install_packages.ok:
logger.error("Unable to install the packages")
get_java_home = SshProcess('echo $(readlink -f /usr/bin/javac | '
'sed "s:/bin/javac::")', self.master)
get_java_home.run()
self.java_home = get_java_home.stdout.strip()
logger.info("All required packages are present")
# 1. Copy hadoop tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_dirs = TaktukRemote("rm -rf " + self.base_dir +
" " + self.conf_dir,
self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hosts)
SequentialActions([rm_dirs, put_tar, tar_xf]).run()
# 2. Move installation to base dir
logger.info("Create installation directories")
mv_base_dir = TaktukRemote(
"mv /tmp/" + os.path.basename(tar_file).replace(".tgz", "") + " " +
self.base_dir,
self.hosts)
mkdirs = TaktukRemote("mkdir -p " + self.conf_dir, self.hosts)
chmods = TaktukRemote("chmod g+w " + self.base_dir +
" && chmod g+w " + self.conf_dir,
self.hosts)
SequentialActions([mv_base_dir, mkdirs, chmods]).run()
# 3. Specify environment variables
command = "cat >> " + self.conf_dir + "/spark-env.sh << EOF\n"
command += "JAVA_HOME=" + self.java_home + "\n"
command += "SPARK_LOG_DIR=" + self.logs_dir + "\n"
if self.hc:
command += "HADOOP_CONF_DIR=" + self.hc.conf_dir + "\n"
if self.mode == YARN_MODE:
command += "YARN_CONF_DIR=" + self.hc.conf_dir + "\n"
command += "EOF\n"
command += "chmod +x " + self.conf_dir + "/spark-env.sh"
action = Remote(command, self.hosts)
action.run()
def _start_disk_copy(self, disks=None):
""" """
disks_copy = []
if not disks:
disks = self.backing_files
for bf in disks:
logger.info('Treating ' + style.emph(bf))
logger.debug("Checking frontend disk vs host disk")
raw_disk = '/tmp/orig_' + bf.split('/')[-1]
f_disk = Process('md5sum -b ' + bf).run()
disk_hash = f_disk.stdout.split(' ')[0]
cmd = 'if [ -f ' + raw_disk + ' ]; ' + \
'then md5sum -b ' + raw_disk + '; fi'
h_disk = self.fact.get_remote(cmd, self.hosts).run()
disk_ok = True
for p in h_disk.processes:
if p.stdout.split(' ')[0] != disk_hash:
disk_ok = False
break
if disk_ok:
logger.info("Disk " + style.emph(bf) +
" is already present, skipping copy")
else:
disks_copy.append(
self.fact.get_fileput(self.hosts, [bf],
remote_location="/tmp"))
if len(disks_copy) > 0:
self.copy_actions = ParallelActions(disks_copy).start()
else:
self.copy_actions = Remote('ls', self.hosts[0]).run()
def _set_common_params(self, params, conf_dir, default_tuning=False):
"""Replace common parameters. Some user-specified values are
overwritten.
Args:
params (str):
Already defined parameters over all the clusters.
conf_dir (str):
The path of the directory with the configuration files.
default_tuning (bool, optional):
Whether to use automatic tuning based on some best practices or
leave the default parameters.
"""
defs_file = conf_dir + "/spark-defaults.conf"
# spark-env.sh
command = "cat >> " + self.conf_dir + "/spark-env.sh << EOF\n"
command += "SPARK_MASTER_PORT=" + str(self.port) + "\n"
command += "EOF\n"
action = Remote(command, self.hosts)
action.run()
# Get already set parameters
global_params = params["global"]
exec_mem = global_params["exec_mem"]
exec_cores = global_params["exec_cores"]
total_execs = global_params["total_execs"]
# Log parameters
if self.evs_log_dir:
write_in_props_file(defs_file,
"spark.eventLog.enabled", "true",
create_if_absent=True,
override=True)
write_in_props_file(defs_file,
"spark.eventLog.dir", self.evs_log_dir,
create_if_absent=True,
override=True)
write_in_props_file(defs_file,
"spark.logConf", "true",
create_if_absent=True,
override=False)
if default_tuning:
write_in_props_file(defs_file,
"spark.executor.memory", "%dm" % exec_mem,
create_if_absent=True,
override=False)
write_in_props_file(defs_file,
"spark.executor.cores", exec_cores,
create_if_absent=True,
override=False)
write_in_props_file(defs_file,
"spark.executor.instances", total_execs,
create_if_absent=True,
override=False)
def _start_disk_copy(self, disks=None, backing_file_dir='/tmp'):
""" """
disks_copy = []
if not disks:
disks = self.backing_files
for bf in disks:
logger.info('Treating ' + style.emph(bf))
logger.debug("Checking frontend disk vs host disk")
raw_disk = '%s/orig_' % backing_file_dir + bf.split('/')[-1]
f_disk = Process('md5sum -b ' + bf).run()
disk_hash = f_disk.stdout.split(' ')[0]
cmd = 'if [ -f ' + raw_disk + ' ]; ' + \
'then md5sum -b ' + raw_disk + '; fi'
h_disk = self.fact.get_remote(cmd, self.hosts).run()
disk_ok = True
for p in h_disk.processes:
if p.stdout.split(' ')[0] != disk_hash:
disk_ok = False
break
if disk_ok:
logger.info("Disk " + style.emph(bf) +
" is already present, skipping copy")
else:
disks_copy.append(self.fact.get_fileput(self.hosts, [bf],
remote_location="%s" % backing_file_dir))
if len(disks_copy) > 0:
self.copy_actions = ParallelActions(disks_copy).start()
else:
self.copy_actions = Remote('ls', self.hosts[0]).run()
def bootstrap(self, tar_file):
# 0. Check that required packages are present
required_packages = "openjdk-7-jre openjdk-7-jdk"
check_packages = TaktukRemote("dpkg -s " + required_packages,
self.hosts)
for p in check_packages.processes:
p.nolog_exit_code = p.nolog_error = True
check_packages.run()
if not check_packages.ok:
logger.info("Packages not installed, trying to install")
install_packages = TaktukRemote(
"export DEBIAN_MASTER=noninteractive ; " +
"apt-get update && apt-get install -y --force-yes " +
required_packages, self.hosts).run()
if not install_packages.ok:
logger.error("Unable to install the packages")
get_java_home = SshProcess(
'echo $(readlink -f /usr/bin/javac | '
'sed "s:/bin/javac::")', self.master)
get_java_home.run()
self.java_home = get_java_home.stdout.strip()
logger.info("All required packages are present")
# 1. Copy Hive tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_dirs = TaktukRemote(
"rm -rf " + self.base_dir + " " + self.conf_dir + " " +
self.warehouse_dir + " " + self.logs_dir, self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hosts)
SequentialActions([rm_dirs, put_tar, tar_xf]).run()
# 2. Move installation to base dir
logger.info("Create installation directories")
mv_base_dir = TaktukRemote(
"mv /tmp/" + os.path.basename(tar_file).replace(".tar.gz", "") +
" " + self.base_dir, self.hosts)
mkdirs = TaktukRemote(
"mkdir -p " + self.conf_dir + " && mkdir -p " + self.warehouse_dir,
self.hosts)
chmods = TaktukRemote(
"chmod g+w " + self.base_dir + " && chmod g+w " + self.conf_dir +
" && chmod g+w " + self.warehouse_dir, self.hosts)
SequentialActions([mv_base_dir, mkdirs, chmods]).run()
# 3. Specify environment variables
command = "cat >> " + self.conf_dir + "/hive-env.sh << EOF\n"
command += "JAVA_HOME=" + self.java_home + "\n"
command += "HIVE_HOME=" + self.base_dir + "\n"
command += "HIVE_CONF_DIR=" + self.conf_dir + "\n"
command += "HADOOP_HOME=" + self.hc.base_dir + "\n"
command += "EOF\n"
command += "chmod +x " + self.conf_dir + "/hive-env.sh"
action = Remote(command, self.hosts)
action.run()
def check_java_version(java_major_version, hosts):
tr = Remote("java -version 2>&1 | grep version", hosts)
tr.run()
for p in tr.processes:
match = re.match('.*[^.0-9]1\.([0-9]+).[0-9].*', p.stdout)
version = int(match.group(1))
if java_major_version > version:
msg = "Java 1.%d+ required" % java_major_version
return False
return True
def clean_logs(self):
"""Remove all Hive logs."""
logger.info("Cleaning logs")
restart = False
if self.running:
logger.warn("The cluster needs to be stopped before cleaning.")
self.stop()
restart = True
action = Remote("rm -rf " + self.logs_dir + "/* ", self.hosts)
action.run()
if restart:
self.start()
def clean_history(self):
"""Remove history."""
logger.info("Cleaning history")
restart = False
if self.running:
logger.warn("The cluster needs to be stopped before cleaning.")
self.stop()
restart = True
action = Remote("rm -rf " + self.logs_dir + "/history", [self.master])
action.run()
if restart:
self.start()
def clean_logs(self):
"""Remove all Hadoop logs."""
logger.info("Cleaning logs")
restart = False
if self.running:
logger.warn("The cluster needs to be stopped before cleaning.")
self.stop()
restart = True
action = Remote("rm -rf " + self.logs_dir + "/*", self.hosts)
action.run()
if restart:
self.start()
def clean_history(self):
"""Remove history."""
logger.info("Cleaning history")
restart = False
if self.running:
logger.warn("The cluster needs to be stopped before cleaning.")
self.stop()
restart = True
action = Remote("rm -rf " + self.logs_dir + "/history",
[self.master])
action.run()
if restart:
self.start()
def change_conf(self, params, conf_file=None, default_file=MR_CONF_FILE):
"""Modify Hadoop configuration. This method copies the configuration
files from the first host of each g5k cluster conf dir into a local
temporary dir, do all the changes in place and broadcast the new
configuration files to all hosts.
Args:
params (dict of str:str):
The parameters to be changed in the form key:value.
conf_file (str, optional):
The file where parameters should be set. If not specified, all
files are checked for the parameter name and the parameter is set
in the file where the property is found. If not found, the
parameter is set in the default file.
default_file (str, optional): The default conf file where to set the
parameter if not found. Only applies when conf_file is not set.
"""
for cluster in self.hw.get_clusters():
hosts = cluster.get_hosts()
# Copy conf files from first host in the cluster
action = Remote("ls " + self.conf_dir + "/*.xml", [hosts[0]])
action.run()
output = action.processes[0].stdout
remote_conf_files = []
for f in output.split():
remote_conf_files.append(os.path.join(self.conf_dir, f))
tmp_dir = "/tmp/mliroz_temp_hadoop/"
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
action = Get([hosts[0]], remote_conf_files, tmp_dir)
action.run()
# Do replacements in temp file
if conf_file:
f = os.path.join(tmp_dir, conf_file)
for name, value in params.iteritems():
replace_in_xml_file(f, name, value, True)
else:
temp_conf_files = [os.path.join(tmp_dir, f) for f in
os.listdir(tmp_dir)]
for name, value in params.iteritems():
for f in temp_conf_files:
if replace_in_xml_file(f, name, value):
break
else:
# Property not found - add it in MR_CONF_FILE
logger.info("Parameter with name " + name + " has not "
"been found in any conf file. Setting it "
"in " + default_file)
f = os.path.join(tmp_dir, default_file)
replace_in_xml_file(f, name, value, True)
# Copy back the files to all hosts
self._copy_conf(tmp_dir, hosts)
def clean_data(self):
"""Remove all data created by Hadoop (including filesystem)."""
if self.running:
logger.warn("The cluster needs to be stopped before cleaning.")
self.stop()
logger.info("Cleaning MongoDB data")
restart = False
if self.running:
self.stop()
restart = True
action = Remote("rm -rf " + self.data_dir + "/*", self.hosts)
action.run()
if restart:
self.start()
def _configure_servers(self, hosts=None):
"""Configure servers and host-dependant parameters.
Args:
hosts (list of Host, optional):
The list of hosts to take into account in the configuration. If
not specified, all the hosts of the Spark cluster are used. The
first host of this list is always used as the reference.
"""
if not hosts:
hosts = self.hosts
host_attrs = get_host_attributes(hosts[0])
num_cores = host_attrs[u'architecture'][u'smt_size']
total_memory_mb = (int(host_attrs[u'main_memory'][u'ram_size']) /
(1024 * 1024))
memory_per_worker = int(0.75 * total_memory_mb)
memory_per_task = int(memory_per_worker / num_cores)
# Set memory for each worker
command = "cat >> " + self.conf_dir + "/spark-env.sh << EOF\n"
command += "SPARK_MASTER_PORT=" + str(self.port) + "\n"
command += "SPARK_WORKER_MEMORY=" + str(memory_per_worker) + "m\n"
command += "EOF\n"
action = Remote(command, self.hosts)
action.run()
# Default parameters
driver_mem = "1g"
executor_mem = str(memory_per_task) + "m"
with open(self.temp_conf_dir + "/spark-defaults.conf", "a") \
as defaults_file:
defaults_file.write("spark.executor.memory\t" + executor_mem + "\n")
defaults_file.write("spark.driver.memory\t" + driver_mem + "\n")
# defaults_file.write("spark.driver.maxResultSize\t1g\n")
defaults_file.write("spark.logConf\ttrue\n")
# defaults_file.write("spark.python.worker.memory\t512m")
if self.evs_log_dir:
defaults_file.write("spark.eventLog.enabled\ttrue\n")
defaults_file.write("spark.eventLog.dir\t" +
self.evs_log_dir + "\n")
def _configure_servers(self, hosts=None):
"""Configure servers and host-dependant parameters.
Args:
hosts (list of Host, optional):
The list of hosts to take into account in the configuration. If
not specified, all the hosts of the Spark cluster are used. The
first host of this list is always used as the reference.
"""
if not hosts:
hosts = self.hosts
host_attrs = get_host_attributes(hosts[0])
num_cores = host_attrs[u'architecture'][u'smt_size']
total_memory_mb = (int(host_attrs[u'main_memory'][u'ram_size']) /
(1024 * 1024))
memory_per_worker = int(0.75 * total_memory_mb)
memory_per_task = int(memory_per_worker / num_cores)
# Set memory for each worker
command = "cat >> " + self.conf_dir + "/spark-env.sh << EOF\n"
command += "SPARK_MASTER_PORT=" + str(self.port) + "\n"
command += "SPARK_WORKER_MEMORY=" + str(memory_per_worker) + "m\n"
command += "EOF\n"
action = Remote(command, self.hosts)
action.run()
# Default parameters
driver_mem = "1g"
executor_mem = str(memory_per_task) + "m"
with open(self.temp_conf_dir + "/spark-defaults.conf", "a") \
as defaults_file:
defaults_file.write("spark.executor.memory\t" + executor_mem +
"\n")
defaults_file.write("spark.driver.memory\t" + driver_mem + "\n")
# defaults_file.write("spark.driver.maxResultSize\t1g\n")
defaults_file.write("spark.logConf\ttrue\n")
# defaults_file.write("spark.python.worker.memory\t512m")
defaults_file.write("spark.eventLog.enabled\ttrue\n")
defaults_file.write("spark.eventLog.dir\t" + self.event_log_dir +
"\n")
def clean_data(self):
"""Remove all data created by Hadoop (including filesystem)."""
if self.running:
logger.warn("The cluster needs to be stopped before cleaning.")
self.stop()
logger.info("Cleaning hadoop data")
restart = False
if self.running:
self.stop()
restart = True
action = Remote("rm -rf " + self.hadoop_temp_dir + " /tmp/hadoop-" +
getpass.getuser() + "-*", self.hosts)
action.run()
if restart:
self.start()
def clean_data(self):
"""Remove all data created by Hadoop (including filesystem)."""
if self.running:
logger.warn("The cluster needs to be stopped before cleaning.")
self.stop()
logger.info("Cleaning hadoop data")
restart = False
if self.running:
self.stop()
restart = True
action = Remote("rm -rf " + self.hadoop_temp_dir + " /tmp/hadoop-" +
getpass.getuser() + "-*", self.hosts)
action.run()
if restart:
self.start()
def bootstrap(self, tar_file):
"""Install MongoDB in all cluster nodes from the specified tgz file.
Args:
tar_file (str):
The file containing MongoDB binaries.
"""
# 1. Copy mongo tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_files = TaktukRemote("rm -rf " + self.base_dir +
" " + self.conf_dir +
" " + self.data_dir +
" " + self.logs_dir,
self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote("tar xf /tmp/" + os.path.basename(tar_file) +
" -C /tmp", self.hosts)
SequentialActions([rm_files, put_tar, tar_xf]).run()
# 2. Move installation to base dir
logger.info("Create installation directories")
action = Remote(
"mv /tmp/" +
os.path.basename(tar_file).replace(".tgz", "") + " " +
self.base_dir,
self.hosts)
action.run()
# 3 Create other dirs
mkdirs = TaktukRemote("mkdir -p " + self.data_dir +
" && mkdir -p " + self.conf_dir +
" && mkdir -p " + self.logs_dir +
" && touch " + os.path.join(self.conf_dir,
CONF_FILE),
self.hosts)
mkdirs.run()
# 4. Generate initial configuration
self._initialize_conf()
def change_conf(self, params, conf_file=None, default_file=MR_CONF_FILE):
"""Modify Hadoop configuration. This method copies the configuration
files from the first host of each g5k cluster conf dir into a local
temporary dir, do all the changes in place and broadcast the new
configuration files to all hosts.
Args:
params (dict of str:str):
The parameters to be changed in the form key:value.
conf_file (str, optional):
The file where parameters should be set. If not specified, all
files are checked for the parameter name and the parameter is set
in the file where the property is found. If not found, the
parameter is set in the default file.
default_file (str, optional): The default conf file where to set the
parameter if not found. Only applies when conf_file is not set.
"""
for cluster in self.hw.get_clusters():
hosts = cluster.get_hosts()
# Copy conf files from first host in the cluster
action = Remote("ls " + self.conf_dir + "/*.xml", [hosts[0]])
action.run()
output = action.processes[0].stdout
remote_conf_files = []
for f in output.split():
remote_conf_files.append(os.path.join(self.conf_dir, f))
tmp_dir = "/tmp/mliroz_temp_hadoop/"
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
action = Get([hosts[0]], remote_conf_files, tmp_dir)
action.run()
# Do replacements in temp file
if conf_file:
f = os.path.join(tmp_dir, conf_file)
for name, value in params.iteritems():
replace_in_xml_file(f, name, value, True)
else:
temp_conf_files = [os.path.join(tmp_dir, f) for f in
os.listdir(tmp_dir)]
for name, value in params.iteritems():
for f in temp_conf_files:
if replace_in_xml_file(f, name, value):
break
else:
# Property not found - add it in MR_CONF_FILE
logger.info("Parameter with name " + name + " has not "
"been found in any conf file. Setting it "
"in " + default_file)
f = os.path.join(tmp_dir, default_file)
replace_in_xml_file(f, name, value, True)
# Copy back the files to all hosts
self._copy_conf(tmp_dir, hosts)
def _initialize_conf(self):
"""Merge locally-specified configuration files with default files
from the distribution."""
action = Remote("cp " + os.path.join(self.conf_dir,
SPARK_CONF_FILE + ".template ") +
os.path.join(self.conf_dir, SPARK_CONF_FILE),
self.hosts)
action.run()
if os.path.exists(self.local_base_conf_dir):
base_conf_files = [os.path.join(self.local_base_conf_dir, f)
for f in os.listdir(self.local_base_conf_dir)]
for f in base_conf_files:
shutil.copy(f, self.init_conf_dir)
else:
logger.warn(
"Local conf dir does not exist. Using default configuration")
base_conf_files = []
missing_conf_files = self.conf_mandatory_files
for f in base_conf_files:
f_base_name = os.path.basename(f)
if f_base_name in missing_conf_files:
missing_conf_files.remove(f_base_name)
logger.info("Copying missing conf files from master: " + str(
missing_conf_files))
remote_missing_files = [os.path.join(self.conf_dir, f)
for f in missing_conf_files]
action = Get([self.master], remote_missing_files, self.init_conf_dir)
action.run()
def get_conf(self, param_names):
params = {}
remaining_param_names = param_names[:]
# Copy conf files from first host in the cluster
action = Remote("ls " + self.conf_dir + "/*.xml", [self.hosts[0]])
action.run()
output = action.processes[0].stdout
remote_conf_files = []
for f in output.split():
remote_conf_files.append(os.path.join(self.conf_dir, f))
tmp_dir = "/tmp/mliroz_temp_hadoop/"
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
action = Get([self.hosts[0]], remote_conf_files, tmp_dir)
action.run()
# Do replacements in temp file
temp_conf_files = [os.path.join(tmp_dir, f) for f in
os.listdir(tmp_dir)]
for f in temp_conf_files:
fparams = get_xml_params(f, remaining_param_names)
for p in fparams:
if fparams[p]:
params[p] = fparams[p]
remaining_param_names.remove(p)
return params
def change_conf(self, params):
"""Modify Hadoop configuration. This method copies the configuration
files from the first host of each g5k cluster conf dir into a local
temporary dir, do all the changes in place and broadcast the new
configuration files to all hosts.
Args:
params (dict of str:str):
The parameters to be changed in the form key:value.
"""
for g5k_cluster in self.host_clusters:
hosts = self.host_clusters[g5k_cluster]
# Copy conf files from first host in the cluster
action = Remote("ls " + self.conf_dir + "/*.xml", [hosts[0]])
action.run()
output = action.processes[0].stdout
remote_conf_files = []
for f in output.split():
remote_conf_files.append(os.path.join(self.conf_dir, f))
tmp_dir = "/tmp/mliroz_temp_hadoop/"
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
action = Get([hosts[0]], remote_conf_files, tmp_dir)
action.run()
# Do replacements in temp file
temp_conf_files = [os.path.join(tmp_dir, f) for f in
os.listdir(tmp_dir)]
for name, value in params.iteritems():
for f in temp_conf_files:
if replace_in_xml_file(f, name, value):
break
else:
# Property not found - provisionally add it in MR_CONF_FILE
f = os.path.join(tmp_dir, MR_CONF_FILE)
replace_in_xml_file(f, name, value, True)
# Copy back the files to all hosts
self._copy_conf(tmp_dir, hosts)
def _get_conf_files(self, host):
action = Remote("ls " + self.conf_dir + "/*.xml", [host])
action.run()
output = action.processes[0].stdout
remote_conf_files = []
for f in output.split():
remote_conf_files.append(os.path.join(self.conf_dir, f))
tmp_dir = "/tmp/mliroz_temp_hadoop/"
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
action = Get([host], remote_conf_files, tmp_dir)
action.run()
temp_conf_files = [os.path.join(tmp_dir, f) for f in
os.listdir(tmp_dir)]
return temp_conf_files
def change_conf(self, params):
"""Modify Hadoop configuration. This method copies the configuration
files from the first host of each g5k cluster conf dir into a local
temporary dir, do all the changes in place and broadcast the new
configuration files to all hosts.
Args:
params (dict of str:str):
The parameters to be changed in the form key:value.
"""
for g5k_cluster in self.host_clusters:
hosts = self.host_clusters[g5k_cluster]
# Copy conf files from first host in the cluster
action = Remote("ls " + self.conf_dir + "/*.xml", [hosts[0]])
action.run()
output = action.processes[0].stdout
remote_conf_files = []
for f in output.split():
remote_conf_files.append(os.path.join(self.conf_dir, f))
tmp_dir = "/tmp/mliroz_temp_hadoop/"
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
action = Get([hosts[0]], remote_conf_files, tmp_dir)
action.run()
# Do replacements in temp file
temp_conf_files = [os.path.join(tmp_dir, f) for f in
os.listdir(tmp_dir)]
for name, value in params.iteritems():
for f in temp_conf_files:
if replace_in_xml_file(f, name, value):
break
else:
# Property not found - provisionally add it in MR_CONF_FILE
f = os.path.join(tmp_dir, MR_CONF_FILE)
replace_in_xml_file(f, name, value, True)
# Copy back the files to all hosts
self._copy_conf(tmp_dir, hosts)
def _initialize_conf(self):
"""Merge locally-specified configuration files with default files
from the distribution."""
action = Remote(
"cp " +
os.path.join(self.conf_dir, SPARK_CONF_FILE + ".template ") +
os.path.join(self.conf_dir, SPARK_CONF_FILE), self.hosts)
action.run()
if os.path.exists(self.local_base_conf_dir):
base_conf_files = [
os.path.join(self.local_base_conf_dir, f)
for f in os.listdir(self.local_base_conf_dir)
]
for f in base_conf_files:
shutil.copy(f, self.init_conf_dir)
else:
logger.warn(
"Local conf dir does not exist. Using default configuration")
base_conf_files = []
missing_conf_files = self.conf_mandatory_files
for f in base_conf_files:
f_base_name = os.path.basename(f)
if f_base_name in missing_conf_files:
missing_conf_files.remove(f_base_name)
logger.info("Copying missing conf files from master: " +
str(missing_conf_files))
remote_missing_files = [
os.path.join(self.conf_dir, f) for f in missing_conf_files
]
action = Get([self.master], remote_missing_files, self.init_conf_dir)
action.run()
def _get_conf_files(self, host):
action = Remote("ls " + self.conf_dir + "/*.xml", [host])
action.run()
output = action.processes[0].stdout
remote_conf_files = []
for f in output.split():
remote_conf_files.append(os.path.join(self.conf_dir, f))
tmp_dir = "/tmp/mliroz_temp_hadoop/"
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
action = Get([host], remote_conf_files, tmp_dir)
action.run()
temp_conf_files = [os.path.join(tmp_dir, f) for f in
os.listdir(tmp_dir)]
return temp_conf_files
def uncompress(file_name, host):
if file_name.endswith("tar.gz"):
decompression = Remote("tar xf " + file_name, [host])
decompression.run()
base_name = os.path.basename(file_name[:-7])
dir_name = os.path.dirname(file_name[:-7])
new_name = dir_name + "/data-" + base_name
action = Remote("mv " + file_name[:-7] + " " + new_name, [host])
action.run()
elif file_name.endswith("gz"):
decompression = Remote("gzip -d " + file_name, [host])
decompression.run()
base_name = os.path.basename(file_name[:-3])
dir_name = os.path.dirname(file_name[:-3])
new_name = dir_name + "/data-" + base_name
action = Remote("mv " + file_name[:-3] + " " + new_name, [host])
action.run()
elif file_name.endswith("zip"):
decompression = Remote("unzip " + file_name, [host])
decompression.run()
base_name = os.path.basename(file_name[:-4])
dir_name = os.path.dirname(file_name[:-4])
new_name = dir_name + "/data-" + base_name
action = Remote("mv " + file_name[:-4] + " " + new_name, [host])
action.run()
elif file_name.endswith("bz2"):
decompression = Remote("bzip2 -d " + file_name, [host])
decompression.run()
base_name = os.path.basename(file_name[:-4])
dir_name = os.path.dirname(file_name[:-4])
new_name = dir_name + "/data-" + base_name
action = Remote("mv " + file_name[:-4] + " " + new_name, [host])
action.run()
else:
logger.warn("Unknown extension")
return file_name
return new_name
def bootstrap(self, tar_file):
# 0. Check requirements
java_major_version = 7
if not check_java_version(java_major_version, self.hosts):
msg = "Java 1.%d+ required" % java_major_version
logger.error(msg)
raise SparkException(msg)
self.java_home = get_java_home(self.master)
# 1. Copy hadoop tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_dirs = TaktukRemote("rm -rf " + self.base_dir + " " + self.conf_dir,
self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hosts)
rm_tar = TaktukRemote("rm /tmp/" + os.path.basename(tar_file),
self.hosts)
SequentialActions([rm_dirs, put_tar, tar_xf, rm_tar]).run()
# 2. Move installation to base dir
logger.info("Create installation directories")
mv_base_dir = TaktukRemote(
"mv /tmp/" + os.path.basename(tar_file).replace(".tgz", "") + " " +
self.base_dir, self.hosts)
mkdirs = TaktukRemote(
"mkdir -p " + self.conf_dir + " && mkdir -p " + self.logs_dir,
self.hosts)
chmods = TaktukRemote(
"chmod g+w " + self.base_dir + " && chmod g+w " + self.conf_dir +
" && chmod g+w " + self.logs_dir, self.hosts)
SequentialActions([mv_base_dir, mkdirs, chmods]).run()
# 2.1. Create spark-events dir
if self.evs_log_dir:
if self.evs_log_dir.startswith("file://") or \
"://" not in self.evs_log_dir:
mk_evs_dir = TaktukRemote(
"mkdir -p " + self.evs_log_dir + " && chmod g+w " +
self.evs_log_dir, self.hosts)
mk_evs_dir.run()
elif self.evs_log_dir.startswith("hdfs://"):
self.hc.execute("fs -mkdir -p " + self.evs_log_dir)
# 3. Specify environment variables
env_file = self.conf_dir + "/spark-env.sh"
command = "cat >> " + env_file + " << EOF\n"
command += "JAVA_HOME=" + self.java_home + "\n"
command += "SPARK_LOG_DIR=" + self.logs_dir + "\n"
if self.hc:
command += "HADOOP_CONF_DIR=" + self.hc.conf_dir + "\n"
if self.mode == YARN_MODE:
command += "YARN_CONF_DIR=" + self.hc.conf_dir + "\n"
command += "EOF\n"
command += "echo SPARK_PUBLIC_DNS=$(hostname) >> " + env_file
command += " && chmod +x " + env_file
action = Remote(command, self.hosts)
action.run()
# 4. Generate initial configuration
self._initialize_conf()
def bootstrap(self, tar_file):
"""Install Hadoop in all cluster nodes from the specified tar.gz file.
Args:
tar_file (str):
The file containing Hadoop binaries.
"""
# 0. Check that required packages are present
required_packages = "openjdk-7-jre openjdk-7-jdk"
check_packages = TaktukRemote("dpkg -s " + required_packages,
self.hosts)
for p in check_packages.processes:
p.nolog_exit_code = p.nolog_error = True
check_packages.run()
if not check_packages.ok:
logger.info("Packages not installed, trying to install")
install_packages = TaktukRemote(
"export DEBIAN_MASTER=noninteractive ; " +
"apt-get update && apt-get install -y --force-yes " +
required_packages, self.hosts).run()
if not install_packages.ok:
logger.error("Unable to install the packages")
get_java_home = SshProcess('echo $(readlink -f /usr/bin/javac | '
'sed "s:/bin/javac::")', self.master)
get_java_home.run()
self.java_home = get_java_home.stdout.strip()
logger.info("All required packages are present")
# 1. Copy hadoop tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_dirs = Remote("rm -rf " + self.base_dir +
" " + self.conf_dir +
" " + self.logs_dir +
" " + self.hadoop_temp_dir,
self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hosts)
SequentialActions([rm_dirs, put_tar, tar_xf]).run()
# 2. Move installation to base dir and create other dirs
logger.info("Create installation directories")
mv_base_dir = TaktukRemote(
"mv /tmp/" +
os.path.basename(tar_file).replace(".tar.gz", "") + " " +
self.base_dir,
self.hosts)
mkdirs = TaktukRemote("mkdir -p " + self.conf_dir +
" && mkdir -p " + self.logs_dir +
" && mkdir -p " + self.hadoop_temp_dir,
self.hosts)
chmods = TaktukRemote("chmod g+w " + self.base_dir +
" && chmod g+w " + self.conf_dir +
" && chmod g+w " + self.logs_dir +
" && chmod g+w " + self.hadoop_temp_dir,
self.hosts)
SequentialActions([mv_base_dir, mkdirs, chmods]).run()
# 4. Specify environment variables
command = "cat >> " + self.conf_dir + "/hadoop-env.sh << EOF\n"
command += "export JAVA_HOME=" + self.java_home + "\n"
command += "export HADOOP_LOG_DIR=" + self.logs_dir + "\n"
command += "HADOOP_HOME_WARN_SUPPRESS=\"TRUE\"\n"
command += "EOF"
action = Remote(command, self.hosts)
action.run()
# 5. Check version
return self._check_version_compliance()
def bootstrap(self, tar_file):
# 1. Remove used dirs if existing
action = Remote("rm -rf " + self.base_dir, self.hc.hosts)
action.run()
action = Remote("rm -rf " + self.conf_dir, self.hc.hosts)
action.run()
# 1. Copy Mahout tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
action = Put(self.hc.hosts, [tar_file], "/tmp")
action.run()
action = Remote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hc.hosts)
action.run()
# 2. Move installation to base dir
logger.info("Create installation directories")
action = Remote(
"mv /tmp/" + os.path.basename(tar_file).replace(".tar.gz", "") +
" " + self.base_dir, self.hc.hosts)
action.run()
# 3 Create other dirs
action = Remote("mkdir -p " + self.conf_dir, self.hc.hosts)
action.run()
# 4. Include libraries in Hadoop's classpath
list_dirs = SshProcess("ls -1 " + self.base_dir + "/*.jar",
self.hc.master)
list_dirs.run()
libs = " ".join(list_dirs.stdout.splitlines())
action = Remote("cp " + libs + " " + self.hc.base_dir + "/lib",
self.hc.hosts)
action.run()
initialized = True # No need to call initialize()
def load(self, hc, dest, desired_size=None):
"""Load the dataset in the given dfs folder by copying it from the
local folder.
Args:
hc (HadoopCluster):
The Hadoop cluster where to deploy the dataset.
dest (str):
The dfs destination folder.
desired_size (int, optional):
The size of the data to be copied. If indicated only the first files
of the dataset up to the given size are copied, if not, the whole
dataset is transferred.
"""
dataset_files = [
os.path.join(self.local_path, f)
for f in os.listdir(self.local_path)
]
hosts = hc.hosts
# Define and create temp dir
tmp_dir = "/tmp" + dest
action_remove = Remote("rm -rf " + tmp_dir, hosts)
action_remove.run()
action_create = Remote("mkdir -p " + tmp_dir, hosts)
action_create.run()
# Generate list of files to copy
if desired_size:
all_files_to_copy = []
dataset_files.sort()
real_size = 0
while real_size < desired_size:
if dataset_files:
all_files_to_copy.append(dataset_files[0])
real_size += os.path.getsize(dataset_files[0])
del dataset_files[0]
else:
logger.warn("Dataset files do not fill up to desired size "
"(real size = " + str(real_size) + ")")
break
else:
real_size = 0
all_files_to_copy = dataset_files
for f in all_files_to_copy:
real_size += os.path.getsize(f)
# Assign files to hosts
files_per_host = [[]] * len(hosts)
for idx in range(0, len(hosts)):
files_per_host[idx] = all_files_to_copy[idx::len(hosts)]
# Create threads and launch them
logger.info("Loading dataset in parallel into " + str(len(hosts)) +
" hosts")
if not hc.running:
hc.start()
class SizeCollector:
size = 0
lock = threading.Lock()
def __init__(self):
pass
def increment(self, qty):
self.lock.acquire()
try:
self.size += qty
finally:
self.lock.release()
def copy_function(host, files_to_copy, collector=None):
action = Put([host], files_to_copy, tmp_dir)
action.run()
local_final_size = 0
for f in files_to_copy:
src_file = os.path.join(tmp_dir, os.path.basename(f))
if self.pre_load_function:
src_file = self.pre_load_function(src_file, host)
action = SshProcess("du -b " + src_file + "| cut -f1",
host)
action.run()
local_final_size += int(action.stdout.strip())
hc.execute(
"fs -put " + src_file + " " +
os.path.join(dest, os.path.basename(src_file)), host, True,
False)
if collector:
collector.increment(local_final_size)
if self.pre_load_function:
final_size = SizeCollector()
else:
final_size = None
threads = []
for idx, h in enumerate(hosts):
if files_per_host[idx]:
t = threading.Thread(target=copy_function,
args=(h, files_per_host[idx], final_size))
t.start()
threads.append(t)
# Wait for the threads to finish
for t in threads:
t.join()
logger.info("Loading completed: real local size = " + str(real_size) +
", final remote size = " + str(final_size.size))
self.deployments[hc, desired_size] = dest
def check_packages(packages, hosts):
tr = Remote("dpkg -s " + packages, hosts)
for p in tr.processes:
p.nolog_exit_code = p.nolog_error = True
tr.run()
return tr.ok
class vm5k_deployment():
""" Base class to control a deployment of hosts and virtual machines on
Grid'5000. It helps to deploy a wheezy-x64-base environment,
to install and configure libvirt from testing repository, and to deploy
virtual machines.
The base run() method allows to setup automatically the hosts and
virtual machines, using the value of the object.
"""
def __init__(self,
infile=None,
resources=None,
hosts=None,
ip_mac=None,
vlan=None,
env_name=None,
env_file=None,
vms=None,
distribution=None,
outdir=None):
""":param infile: an XML file that describe the topology of the
deployment
:param resources: a dict whose keys are Grid'5000 sites and values are
dict, whose keys are hosts and ip_mac, where hosts is a list of
execo.Host and ip_mac is a list of tuple (ip, mac).
:param env_name: name of the Kadeploy environment
:param env_file: path to the Kadeploy environment file
:params vms: dict defining the virtual machines
:params distribution: how to distribute the vms on the hosts
(``round-robin`` , ``concentrated``, ``random``)
:params outdir: directory to store the deployment files
"""
# set a factory for the deployment that use taktuk and chainput
self.fact = ActionFactory(remote_tool=TAKTUK,
fileput_tool=CHAINPUT,
fileget_tool=TAKTUK)
self.kavlan = None if not vlan else vlan
self.kavlan_site = None
if env_name is not None:
self.env_file = None
if ':' not in env_name:
self.env_name, self.env_user = env_name, None
else:
self.env_user, self.env_name = env_name.split(':')
else:
if env_file is not None:
self.env_name = None
self.env_user = None
self.env_file = env_file
else:
self.env_name = 'vm5k'
self.env_user = '******'
self.env_file = None
if outdir:
self.outdir = outdir
else:
self.outdir = 'vm5k_' + strftime("%Y%m%d_%H%M%S_%z")
self.copy_actions = None
self.state = Element('vm5k')
self._define_elements(infile, resources, hosts, vms, ip_mac,
distribution)
logger.info('%s %s %s %s %s %s %s %s', len(self.sites),
style.emph('sites'), len(self.clusters),
style.user1('clusters'), len(self.hosts),
style.host('hosts'), len(self.vms), style.vm('vms'))
# PUBLIC METHODS
def run(self):
"""Launch the deployment and configuration of hosts and virtual
machines: hosts_deployment, packages_mamangement, configure_service_node
configure_libvirt, deploy_vms"""
try:
print_step('HOSTS DEPLOYMENT')
self.hosts_deployment()
print_step('MANAGING PACKAGES')
self.packages_management()
print_step('CONFIGURING SERVICE NODE')
self.configure_service_node()
print_step('CONFIGURING LIBVIRT')
self.configure_libvirt()
print_step('VIRTUAL MACHINES')
self.deploy_vms()
finally:
self.get_state()
def hosts_deployment(self, max_tries=1, check_deploy=True, conf_ssh=True):
"""Deploy the hosts using kadeploy, configure ssh for taktuk execution
and launch backing file disk copy"""
self._launch_kadeploy(max_tries, check_deploy)
if conf_ssh:
self._configure_ssh()
def packages_management(self,
upgrade=True,
other_packages=None,
launch_disk_copy=True,
apt_cacher=False):
"""Configure APT to use testing repository,
perform upgrade and install required packages. Finally start
kvm module"""
self._configure_apt()
if upgrade:
self._upgrade_hosts()
self._install_packages(other_packages=other_packages,
launch_disk_copy=launch_disk_copy)
if apt_cacher:
setup_aptcacher_server(self.hosts)
# Post configuration to load KVM
self.fact.get_remote(
'modprobe kvm; modprobe kvm-intel; modprobe kvm-amd ; ' + \
'chown root:kvm /dev/kvm ;', self.hosts).run()
def configure_service_node(self):
"""Setup automatically a DNS server to access virtual machines by id
and also install a DHCP server if kavlan is used"""
if self.kavlan:
service = 'DNS/DHCP'
dhcp = True
else:
service = 'DNS'
dhcp = False
service_node = get_fastest_host(self.hosts)
logger.info('Setting up %s on %s', style.emph(service),
style.host(service_node.split('.')[0]))
clients = list(self.hosts)
clients.remove(service_node)
dnsmasq_server(service_node, clients, self.vms, dhcp)
def configure_libvirt(self, bridge='br0', libvirt_conf=None):
"""Enable a bridge if needed on the remote hosts, configure libvirt
with a bridged network for the virtual machines, and restart service.
"""
self._enable_bridge()
self._libvirt_check_service()
self._libvirt_uniquify()
self._libvirt_bridged_network(bridge)
logger.info('Restarting %s', style.emph('libvirt'))
self.fact.get_remote('service libvirtd restart', self.hosts).run()
def deploy_vms(self,
clean_disks=False,
disk_location='one',
apt_cacher=False):
"""Destroy the existing VMS, create the virtual disks, install the vms
start them and wait until they have rebooted"""
logger.info('Destroying existing virtual machines')
destroy_vms(self.hosts, undefine=True)
if clean_disks:
self._remove_existing_disks()
logger.info('Creating the virtual disks ')
self._create_backing_file()
if disk_location == 'one':
logger.info('Create disk on each nodes')
create_disks(self.vms).run()
elif disk_location == 'all':
logger.info('Create all disks on all nodes')
create_disks_all_hosts(self.vms, self.hosts).run()
logger.info('Installing the virtual machines')
install_vms(self.vms).run()
logger.info('Starting the virtual machines')
self.boot_time = Timer()
start_vms(self.vms).run()
logger.info('Waiting for VM to boot ...')
wait_vms_have_started(self.vms, self.hosts[0])
activate_vms(self.vms)
self._update_vms_xml()
if apt_cacher:
configure_apt_proxy(self.vms)
def get_state(self, name=None, output=True, mode='compact', plot=False):
""" """
if not name:
name = 'vm5k_' + strftime('%Y%m%d_%H%M%S', localtime())
if output:
output = self.outdir + '/' + name + '.xml'
f = open(output, 'w')
f.write(prettify(self.state))
f.close()
if mode == 'compact':
log = self._print_state_compact()
logger.info('State %s', log)
# PRIVATE METHODS
def _launch_kadeploy(self, max_tries=1, check_deploy=True):
"""Create a execo_g5k.Deployment object, launch the deployment and
return a tuple (deployed_hosts, undeployed_hosts)
"""
logger.info('Deploying %s hosts \n%s', len(self.hosts),
hosts_list(self.hosts))
deployment = Deployment(
hosts=[Host(canonical_host_name(host)) for host in self.hosts],
env_file=self.env_file,
env_name=self.env_name,
user=self.env_user,
vlan=self.kavlan)
# Activate kadeploy output log if log level is debug
if logger.getEffectiveLevel() <= 10:
stdout = [sys.stdout]
stderr = [sys.stderr]
else:
stdout = None
stderr = None
deployed_hosts, undeployed_hosts = deploy(
deployment,
stdout_handlers=stdout,
stderr_handlers=stderr,
num_tries=max_tries,
check_deployed_command=check_deploy)
deployed_hosts = list(deployed_hosts)
undeployed_hosts = list(undeployed_hosts)
# Renaming hosts if a kavlan is used
if self.kavlan:
for i, host in enumerate(deployed_hosts):
deployed_hosts[i] = get_kavlan_host_name(host, self.kavlan)
for i, host in enumerate(undeployed_hosts):
undeployed_hosts[i] = get_kavlan_host_name(host, self.kavlan)
logger.info('Deployed %s hosts \n%s', len(deployed_hosts),
hosts_list(deployed_hosts))
cr = '\n' if len(undeployed_hosts) > 0 else ''
logger.info('Failed %s hosts %s%s', len(undeployed_hosts), cr,
hosts_list(undeployed_hosts))
self._update_hosts_state(deployed_hosts, undeployed_hosts)
return deployed_hosts, undeployed_hosts
def _configure_ssh(self):
if self.fact.remote_tool == 2:
# Configuring SSH with precopy of id_rsa and id_rsa.pub keys on all
# host to allow TakTuk connection
taktuk_conf = ('-s', '-S', '$HOME/.ssh/id_rsa:$HOME/.ssh/id_rsa,' +
'$HOME/.ssh/id_rsa.pub:$HOME/.ssh')
else:
taktuk_conf = ('-s', )
conf_ssh = self.fact.get_remote(
'echo "Host *" >> /root/.ssh/config ;' +
'echo " StrictHostKeyChecking no" >> /root/.ssh/config; ',
self.hosts,
connection_params={
'taktuk_options': taktuk_conf
}).run()
self._actions_hosts(conf_ssh)
def _start_disk_copy(self, disks=None):
""" """
disks_copy = []
if not disks:
disks = self.backing_files
for bf in disks:
logger.info('Treating ' + style.emph(bf))
logger.debug("Checking frontend disk vs host disk")
raw_disk = '/tmp/orig_' + bf.split('/')[-1]
f_disk = Process('md5sum -b ' + bf).run()
disk_hash = f_disk.stdout.split(' ')[0]
cmd = 'if [ -f ' + raw_disk + ' ]; ' + \
'then md5sum -b ' + raw_disk + '; fi'
h_disk = self.fact.get_remote(cmd, self.hosts).run()
disk_ok = True
for p in h_disk.processes:
if p.stdout.split(' ')[0] != disk_hash:
disk_ok = False
break
if disk_ok:
logger.info("Disk " + style.emph(bf) +
" is already present, skipping copy")
else:
disks_copy.append(
self.fact.get_fileput(self.hosts, [bf],
remote_location="/tmp"))
if len(disks_copy) > 0:
self.copy_actions = ParallelActions(disks_copy).start()
else:
self.copy_actions = Remote('ls', self.hosts[0]).run()
def _create_backing_file(self, disks=None):
""" """
if not self.copy_actions:
self._start_disk_copy(disks)
if not self.copy_actions.ended:
logger.info("Waiting for the end of the disks copy")
self.copy_actions.wait()
if isinstance(self.copy_actions, ParallelActions):
mv_actions = []
for act in self.copy_actions.actions:
fname = act.local_files[0].split('/')[-1]
mv_actions.append(
self.fact.get_remote(
"mv /tmp/" + fname + " /tmp/orig_" + fname,
self.hosts))
mv = ParallelActions(mv_actions).run()
if not disks:
disks = self.backing_files
for bf in disks:
raw_disk = '/tmp/orig_' + bf.split('/')[-1]
to_disk = '/tmp/' + bf.split('/')[-1]
self.fact.get_remote('cp ' + raw_disk + ' ' + to_disk,
self.hosts).run()
logger.info('Copying ssh key on ' + to_disk + ' ...')
cmd = 'modprobe nbd max_part=16; ' + \
'qemu-nbd --connect=/dev/nbd0 ' + to_disk + \
' ; sleep 3 ; partprobe /dev/nbd0 ; ' + \
'part=`fdisk -l /dev/nbd0 |grep dev|grep Linux| grep -v swap|cut -f 1 -d " "` ; ' + \
'mount $part /mnt ; mkdir -p /mnt/root/.ssh ; ' + \
'cat /root/.ssh/authorized_keys >> /mnt/root/.ssh/authorized_keys ; ' + \
'cp -r /root/.ssh/id_rsa* /mnt/root/.ssh/ ;' + \
'umount /mnt; qemu-nbd -d /dev/nbd0'
logger.detail(cmd)
copy_on_vm_base = self.fact.get_remote(cmd, self.hosts).run()
self._actions_hosts(copy_on_vm_base)
def _remove_existing_disks(self, hosts=None):
"""Remove all img and qcow2 file from /tmp directory """
logger.info('Removing existing disks')
if hosts is None:
hosts = self.hosts
remove = self.fact.get_remote('rm -f /tmp/*.img; rm -f /tmp/*.qcow2',
self.hosts).run()
self._actions_hosts(remove)
def _libvirt_check_service(self):
""" """
logger.detail('Checking libvirt service name')
cmd = "if [ ! -e /etc/init.d/libvirtd ]; " + \
" then if [ -e /etc/init.d/libvirt-bin ]; " + \
" then ln -s /etc/init.d/libvirt-bin /etc/init.d/libvirtd; " + \
" else echo 1; " + \
" fi; " + \
"else echo 0; fi"
check_libvirt = self.fact.get_remote(cmd, self.hosts).run()
self._actions_hosts(check_libvirt)
def _libvirt_uniquify(self):
logger.detail('Making libvirt host unique')
cmd = 'uuid=`uuidgen` ' + \
'&& sed -i "s/.*host_uuid.*/host_uuid=\\"${uuid}\\"/g" ' + \
'/etc/libvirt/libvirtd.conf ' + \
'&& service libvirtd restart'
logger.debug(cmd)
self.fact.get_remote(cmd, self.hosts).run()
def _libvirt_bridged_network(self, bridge):
logger.detail('Configuring libvirt network')
# Creating an XML file describing the network
root = Element('network')
name = SubElement(root, 'name')
name.text = 'default'
SubElement(root, 'forward', attrib={'mode': 'bridge'})
SubElement(root, 'bridge', attrib={'name': bridge})
fd, network_xml = mkstemp(dir='/tmp/', prefix='create_br_')
f = fdopen(fd, 'w')
f.write(prettify(root))
f.close()
logger.debug('Destroying existing network')
destroy = self.fact.get_remote(
'virsh net-destroy default; ' + 'virsh net-undefine default',
self.hosts)
put = TaktukPut(self.hosts, [network_xml], remote_location='/root/')
start = self.fact.get_remote(
'virsh net-define /root/' + \
network_xml.split('/')[-1] + ' ; ' + \
'virsh net-start default; virsh net-autostart default;',
self.hosts)
netconf = SequentialActions([destroy, put, start]).run()
self._actions_hosts(netconf)
# Hosts configuration
def _enable_bridge(self, name='br0'):
"""We need a bridge to have automatic DHCP configuration for the VM."""
logger.detail('Configuring the bridge')
hosts_br = self._get_bridge(self.hosts)
nobr_hosts = []
for host, br in hosts_br.iteritems():
if br is None:
logger.debug('No bridge on host %s', style.host(host))
nobr_hosts.append(host)
elif br != name:
logger.debug('Wrong bridge on host %s, destroying it',
style.host(host))
SshProcess('ip link set ' + br + ' down ; brctl delbr ' + br,
host).run()
nobr_hosts.append(host)
else:
logger.debug('Bridge %s is present on host %s',
style.emph('name'), style.host(host))
nobr_hosts = map(lambda x: x.address
if isinstance(x, Host) else x, nobr_hosts)
if len(nobr_hosts) > 0:
logger.debug('Creating bridge on %s', hosts_list(nobr_hosts))
script = 'export br_if=`ip route |grep default |cut -f 5 -d " "`; \n' + \
'ifdown $br_if ; \n' + \
'sed -i "s/$br_if inet dhcp/$br_if inet manual/g" /etc/network/interfaces ; \n' + \
'sed -i "s/auto $br_if//g" /etc/network/interfaces ; \n' + \
'echo " " >> /etc/network/interfaces ; \n' + \
'echo "auto ' + name + '" >> /etc/network/interfaces ; \n' + \
'echo "iface ' + name + ' inet dhcp" >> /etc/network/interfaces ; \n' + \
'echo " bridge_ports $br_if" >> /etc/network/interfaces ; \n' + \
'echo " bridge_stp off" >> /etc/network/interfaces ; \n' + \
'echo " bridge_maxwait 0" >> /etc/network/interfaces ; \n' + \
'echo " bridge_fd 0" >> /etc/network/interfaces ; \n' + \
'ifup ' + name
fd, br_script = mkstemp(dir='/tmp/', prefix='create_br_')
f = fdopen(fd, 'w')
f.write(script)
f.close()
self.fact.get_fileput(nobr_hosts, [br_script]).run()
self.fact.get_remote('nohup sh ' + br_script.split('/')[-1],
nobr_hosts).run()
logger.debug('Waiting for network restart')
if_up = False
nmap_tries = 0
while (not if_up) and nmap_tries < 20:
sleep(20)
nmap_tries += 1
nmap = Process('nmap ' +
' '.join([host for host in nobr_hosts]) +
' -p 22').run()
for line in nmap.stdout.split('\n'):
if 'Nmap done' in line:
if_up = line.split()[2] == line.split()[5].replace(
'(', '')
logger.debug('Network has been restarted')
logger.detail('All hosts have the bridge %s', style.emph(name))
def _get_bridge(self, hosts):
""" """
logger.debug('Retrieving bridge on hosts %s',
", ".join([host for host in hosts]))
cmd = "brctl show |grep -v 'bridge name' | awk '{ print $1 }' |head -1"
bridge_exists = self.fact.get_remote(cmd, hosts)
bridge_exists.nolog_exit_code = True
bridge_exists.run()
hosts_br = {}
for p in bridge_exists.processes:
stdout = p.stdout.strip()
if len(stdout) == 0:
hosts_br[p.host] = None
else:
hosts_br[p.host] = stdout
return hosts_br
def _configure_apt(self):
"""Create the sources.list file """
logger.detail('Configuring APT')
# Create sources.list file
fd, tmpsource = mkstemp(dir='/tmp/', prefix='sources.list_')
f = fdopen(fd, 'w')
f.write('deb http://ftp.debian.org/debian wheezy main contrib non-free\n' + \
'deb http://ftp.debian.org/debian wheezy-backports main contrib non-free\n' + \
'deb http://security.debian.org/ wheezy/updates main contrib non-free\n')
f.close()
# Create preferences file
fd, tmppref = mkstemp(dir='/tmp/', prefix='preferences_')
f = fdopen(fd, 'w')
f.write('Package: * \nPin: release a=wheezy \nPin-Priority: 900\n\n' + \
'Package: * \nPin: release a=wheezy-backports \nPin-Priority: 875\n\n')
f.close()
# Create apt.conf file
fd, tmpaptconf = mkstemp(dir='/tmp/', prefix='apt.conf_')
f = fdopen(fd, 'w')
f.write('APT::Acquire::Retries=20;\n')
f.close()
TaktukPut(self.hosts, [tmpsource, tmppref, tmpaptconf],
remote_location='/etc/apt/').run()
cmd = 'cd /etc/apt && ' + \
'mv ' + tmpsource.split('/')[-1] + ' sources.list &&' + \
'mv ' + tmppref.split('/')[-1] + ' preferences &&' + \
'mv ' + tmpaptconf.split('/')[-1] + ' apt.conf'
apt_conf = self.fact.get_remote(cmd, self.hosts).run()
self._actions_hosts(apt_conf)
Local('rm ' + tmpsource + ' ' + tmppref + ' ' + tmpaptconf).run()
def _upgrade_hosts(self):
"""Dist upgrade performed on all hosts"""
logger.info('Upgrading packages')
cmd = "echo 'debconf debconf/frontend select noninteractive' | debconf-set-selections ; " + \
"echo 'debconf debconf/priority select critical' | debconf-set-selections ; " + \
"export DEBIAN_MASTER=noninteractive ; apt-get update ; " + \
"apt-get dist-upgrade -y --force-yes -o Dpkg::Options::='--force-confdef' " + \
"-o Dpkg::Options::='--force-confold' "
upgrade = self.fact.get_remote(cmd, self.hosts).run()
self._actions_hosts(upgrade)
def _install_packages(self, other_packages=None, launch_disk_copy=True):
"""Installation of required packages on the hosts"""
base_packages = 'uuid-runtime bash-completion taktuk locate htop init-system-helpers netcat-traditional'
logger.info('Installing base packages \n%s', style.emph(base_packages))
cmd = 'export DEBIAN_MASTER=noninteractive ; apt-get update && apt-get ' + \
'install -y --force-yes --no-install-recommends ' + base_packages
install_base = self.fact.get_remote(cmd, self.hosts).run()
self._actions_hosts(install_base)
if launch_disk_copy:
self._start_disk_copy()
libvirt_packages = 'libvirt-bin virtinst python2.7 python-pycurl python-libxml2 qemu-kvm nmap libgmp10'
logger.info('Installing libvirt packages \n%s',
style.emph(libvirt_packages))
cmd = 'export DEBIAN_MASTER=noninteractive ; apt-get update && apt-get install -y --force-yes '+\
'-o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -t wheezy-backports '+\
libvirt_packages
install_libvirt = self.fact.get_remote(cmd, self.hosts).run()
self._actions_hosts(install_libvirt)
if other_packages:
self._other_packages(other_packages)
def _other_packages(self, other_packages=None):
"""Installation of packages"""
other_packages = other_packages.replace(',', ' ')
logger.info('Installing extra packages \n%s',
style.emph(other_packages))
cmd = 'export DEBIAN_MASTER=noninteractive ; ' + \
'apt-get update && apt-get install -y --force-yes ' + \
other_packages
install_extra = self.fact.get_remote(cmd, self.hosts).run()
self._actions_hosts(install_extra)
# State related methods
def _define_elements(self,
infile=None,
resources=None,
hosts=None,
vms=None,
ip_mac=None,
distribution=None):
"""Create the list of sites, clusters, hosts, vms and check
correspondance between infile and resources"""
if not ip_mac:
self._get_ip_mac(resources)
else:
self.ip_mac = ip_mac
if not hosts:
self._get_resources_elements(resources)
else:
self.hosts = hosts if not self.kavlan else \
map(lambda h: get_kavlan_host_name(h, self.kavlan),
hosts)
self.sites = []
self.clusters = []
if not infile:
self.vms = vms
if len(filter(lambda v: v['host'] is None, self.vms)) > 0:
self.distribution = distribution if distribution \
else 'round-robin'
else:
self.distribution = None
else:
xml = parse(infile)
if self._check_xml_elements(xml, resources):
self.vms = self._get_xml_vms(xml)
self.distribution = None
else:
exit()
self._add_xml_elements()
if self.vms:
if self.distribution:
distribute_vms(self.vms, self.hosts, self.distribution)
self._set_vms_ip_mac()
self._add_xml_vms()
else:
self.vms = []
self.backing_files = list(set([vm['backing_file'] for vm in self.vms]))
def _get_ip_mac(self, resources):
""" """
if len(resources.keys()) == 1:
# mono site
self.ip_mac = resources[resources.keys()[0]]['ip_mac']
self.kavlan = resources[resources.keys()[0]]['kavlan']
elif 'global' in resources:
# multi site in a global kavlan
self.ip_mac = resources['global']['ip_mac']
self.kavlan = resources['global']['kavlan']
self.kavlan_site = resources['global']['site']
else:
# multi site in prod network
self.ip_mac = {
site: resource['ip_mac']
for site, resource in resources.iteritems()
}
if isinstance(self.ip_mac, list) and len(self.ip_mac) == 0:
logger.error('No ip_range given in the resources')
exit()
elif isinstance(self.ip_mac, dict):
for ip_mac in self.ip_mac.itervalues():
if len(ip_mac) == 0:
logger.error('No ip_range given in the resources')
exit()
def _get_resources_elements(self, resources=None):
""" """
self.sites = sorted(
[site for site in resources.keys() if site != 'global'])
self.hosts = []
for site in self.sites:
if self.kavlan:
self.hosts += map(
lambda host: get_kavlan_host_name(host, self.kavlan),
resources[site]['hosts'])
else:
self.hosts += resources[site]['hosts']
self.hosts.sort(key=lambda host: (host.split('.', 1)[0].split('-')[
0], int(host.split('.', 1)[0].split('-')[1])))
self.clusters = list(
set([get_host_cluster(host) for host in self.hosts]))
self.clusters.sort()
def _check_xml_elements(self, xml, resources, strict=False):
sites, clusters, hosts = self._get_xml_elements(xml)
ok = True
if not sites == self.sites:
logger.error('List of sites from resources differs from infile' + \
'\n resource %s \n infile %s', self.sites, sites)
ok = False
if not clusters == self.clusters:
logger.error('List of clusters from resources differs from infile' + \
'\n resource %s \n infile %s', self.clusters, clusters)
ok = False
if strict:
if not hosts == self.hosts:
logger.error('List of hosts from resources differs from infile' + \
'\n resource %s \n infile %s', self.hosts, hosts)
ok = False
else:
res_hosts = {}
for host in self.hosts:
cluster = get_host_cluster(host)
if cluster in res_hosts:
res_hosts[cluster] += 1
else:
res_hosts[cluster] = 1
xml_hosts = {}
for host in hosts:
cluster = get_host_cluster(host)
if cluster in xml_hosts:
xml_hosts[cluster] += 1
else:
xml_hosts[cluster] = 1
if not res_hosts == xml_hosts:
logger.error('List of hosts from resources differs from infile' + \
'\n resource %s \n infile %s', self.hosts, hosts)
ok = False
else:
for i in range(len(hosts)):
el_host = xml.find(".//host/[@id='" + hosts[i] + "']")
el_host.attrib['id'] = self.hosts[i]
return ok
def _get_xml_elements(self, xml):
""" """
sites = sorted([site.get('id') for site in xml.findall('./site')])
clusters = sorted(
[cluster.get('id') for cluster in xml.findall('.//cluster')])
hosts = sorted([host.get('id') for host in xml.findall('.//host')],
key=lambda host:
(host.split('.', 1)[0].split('-')[0],
int(host.split('.', 1)[0].split('-')[1])))
return sites, clusters, hosts
def _get_xml_vms(self, xml):
"""Define the list of VMs from the infile """
def _default_xml_value(key):
return default_vm[key] if key not in vm.attrib else vm.get(key)
vms = []
for host in xml.findall('.//host'):
for vm in host.findall('.//vm'):
vms.append({
'id': vm.get('id'),
'n_cpu': int(_default_xml_value('n_cpu')),
'cpuset': _default_xml_value('cpuset'),
'mem': int(_default_xml_value('mem')),
'hdd': int(_default_xml_value('hdd')),
'backing_file': _default_xml_value('backing_file'),
'real_file': _default_xml_value('real_file'),
'host': host.get('id'),
'state': 'KO'
})
return vms
def _set_vms_ip_mac(self):
"""Not finished """
if isinstance(self.ip_mac, dict):
i_vm = {site: 0 for site in self.sites}
for vm in self.vms:
vm_site = get_host_site(vm['host'])
vm['ip'], vm['mac'] = self.ip_mac[vm_site][i_vm[vm_site]]
i_vm[vm_site] += 1
else:
i_vm = 0
for vm in self.vms:
if 'mac' not in vm:
vm['ip'], vm['mac'] = self.ip_mac[i_vm]
i_vm += 1
def _add_xml_elements(self):
"""Add sites, clusters, hosts to self.state """
_state = self.state
logger.debug('Initial state \n %s', prettify(_state))
for site in self.sites:
SubElement(_state, 'site', attrib={'id': site})
else:
el_site = SubElement(_state, 'site', attrib={'id': 'unknown'})
logger.debug('Sites added \n %s', prettify(_state))
for cluster in self.clusters:
el_site = _state.find("./site[@id='" + get_cluster_site(cluster) \
+ "']")
SubElement(el_site, 'cluster', attrib={'id': cluster})
else:
el_cluster = SubElement(el_site,
'cluster',
attrib={'id': 'unknown'})
logger.debug('Clusters added \n %s', prettify(_state))
hosts_attr = get_CPU_RAM_FLOPS(self.hosts)
for host in self.hosts:
if host in get_g5k_hosts():
el_cluster = _state.find(".//cluster/[@id='" +
get_host_cluster(host) + "']")
SubElement(el_cluster,
'host',
attrib={
'id': host,
'state': 'Undeployed',
'cpu': str(hosts_attr[host]['CPU']),
'mem': str(hosts_attr[host]['RAM'])
})
logger.debug('Hosts added \n %s', prettify(_state))
def _add_xml_vms(self):
"""Add vms distributed on hosts to self.state """
for vm in self.vms:
host = self.state.find(".//host/[@id='" + vm['host'] + "']")
SubElement(host,
'vm',
attrib={
'id': vm['id'],
'ip': vm['ip'],
'mac': vm['mac'],
'mem': str(vm['mem']),
'n_cpu': str(vm['n_cpu']),
'cpuset': vm['cpuset'],
'hdd': str(vm['hdd']),
'backing_file': vm['backing_file'],
'real_file': str(vm['real_file']),
'state': vm['state']
})
def _print_state_compact(self):
"""Display in a compact form the distribution of vms on hosts."""
dist = {}
max_len_host = 0
for host in self.hosts:
if len(host.split('.')[0]) > max_len_host:
max_len_host = len(host.split('.')[0])
for vm in self.vms:
host = vm['host'].split('.')[0]
if len(host) > max_len_host:
max_len_host = len(host)
if host not in dist.keys():
dist[host] = {vm['id']: vm['state']}
else:
dist[host][vm['id']] = vm['state']
log = ''
for host in sorted(self.hosts,
key=lambda x: (x.split('.')[0].split('-')[0],
int(x.split('.')[0].split('-')[1]))):
host = host.split('.')[0]
if host not in dist:
dist[host] = {}
log += '\n' + style.host(host) + ' '.ljust(max_len_host + 2 - len(host)) + \
str(len(dist[host].keys())) + ' '
try:
vms = sorted(dist[host].keys(),
key=lambda x:
(x.split('.')[0].split('-')[0],
int(x.split('.')[0].split('-')[1])))
except:
vms = sorted(dist[host].keys())
pass
for vm in vms:
if dist[host][vm] == 'OK':
log += style.OK(vm)
elif dist[host][vm] == 'KO':
log += style.KO(vm)
else:
log += style.Unknown(vm)
log += ' '
return log
def _update_vms_xml(self):
for vm in self.vms:
self.state.find(".//vm/[@id='" + vm['id'] + "']").set(
'state', vm['state'])
def _update_hosts_state(self, hosts_ok, hosts_ko):
""" """
for host in hosts_ok:
if host:
if isinstance(host, Host):
host = host.address
self.state.find(".//host/[@id='" + host + "']").set(
'state', 'OK')
for host in hosts_ko:
if host:
if isinstance(host, Host):
host = host.address
self.state.find(".//host/[@id='" + host + "']").set(
'state', 'KO')
self.hosts.remove(host)
if len(self.hosts) == 0:
logger.error('No hosts available, because %s are KO',
hosts_list(hosts_ko))
exit()
if self.vms:
distribute_vms(self.vms, self.hosts, self.distribution)
self._set_vms_ip_mac()
def _actions_hosts(self, action):
hosts_ok, hosts_ko = [], []
for p in action.processes:
if p.ok:
hosts_ok.append(p.host)
else:
logger.warn('%s is KO', p.host)
hosts_ko.append(p.host)
hosts_ok, hosts_ko = list(set(hosts_ok)), list(set(hosts_ko))
self._update_hosts_state(hosts_ok, hosts_ko)
def bootstrap(self, tar_file):
# 0. Check requirements
java_major_version = 7
if not check_java_version(java_major_version, self.hosts):
msg = "Java 1.%d+ required" % java_major_version
logger.error(msg)
raise SparkException(msg)
self.java_home = get_java_home(self.master)
# 1. Copy hadoop tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_dirs = TaktukRemote("rm -rf " + self.base_dir +
" " + self.conf_dir,
self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hosts)
rm_tar = TaktukRemote(
"rm /tmp/" + os.path.basename(tar_file),
self.hosts)
SequentialActions([rm_dirs, put_tar, tar_xf, rm_tar]).run()
# 2. Move installation to base dir
logger.info("Create installation directories")
mv_base_dir = TaktukRemote(
"mv /tmp/" + os.path.basename(tar_file).replace(".tgz", "") + " " +
self.base_dir,
self.hosts)
mkdirs = TaktukRemote("mkdir -p " + self.conf_dir +
" && mkdir -p " + self.logs_dir,
self.hosts)
chmods = TaktukRemote("chmod g+w " + self.base_dir +
" && chmod g+w " + self.conf_dir +
" && chmod g+w " + self.logs_dir,
self.hosts)
SequentialActions([mv_base_dir, mkdirs, chmods]).run()
# 2.1. Create spark-events dir
if self.evs_log_dir:
if self.evs_log_dir.startswith("file://") or \
"://" not in self.evs_log_dir:
mk_evs_dir = TaktukRemote("mkdir -p " + self.evs_log_dir +
" && chmod g+w " + self.evs_log_dir,
self.hosts)
mk_evs_dir.run()
elif self.evs_log_dir.startswith("hdfs://"):
self.hc.execute("fs -mkdir -p " + self.evs_log_dir)
# 3. Specify environment variables
env_file = self.conf_dir + "/spark-env.sh"
command = "cat >> " + env_file + " << EOF\n"
command += "JAVA_HOME=" + self.java_home + "\n"
command += "SPARK_LOG_DIR=" + self.logs_dir + "\n"
if self.hc:
command += "HADOOP_CONF_DIR=" + self.hc.conf_dir + "\n"
if self.mode == YARN_MODE:
command += "YARN_CONF_DIR=" + self.hc.conf_dir + "\n"
command += "EOF\n"
command += "echo SPARK_PUBLIC_DNS=$(hostname) >> " + env_file
command += " && chmod +x " + env_file
action = Remote(command, self.hosts)
action.run()
# 4. Generate initial configuration
self._initialize_conf()
def bootstrap(self, tar_file):
"""Install Hadoop in all cluster nodes from the specified tar.gz file.
Args:
tar_file (str):
The file containing Hadoop binaries.
"""
# 0. Check that required packages are present
required_packages = "openjdk-7-jre openjdk-7-jdk"
check_packages = TaktukRemote("dpkg -s " + required_packages,
self.hosts)
for p in check_packages.processes:
p.nolog_exit_code = p.nolog_error = True
check_packages.run()
if not check_packages.ok:
logger.info("Packages not installed, trying to install")
install_packages = TaktukRemote(
"export DEBIAN_MASTER=noninteractive ; " +
"apt-get update && apt-get install -y --force-yes " +
required_packages, self.hosts).run()
if not install_packages.ok:
logger.error("Unable to install the packages")
get_java_home = SshProcess('echo $(readlink -f /usr/bin/javac | '
'sed "s:/bin/javac::")', self.master)
get_java_home.run()
self.java_home = get_java_home.stdout.strip()
logger.info("All required packages are present")
# 1. Copy hadoop tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_dirs = Remote("rm -rf " + self.base_dir +
" " + self.conf_dir +
" " + self.logs_dir +
" " + self.hadoop_temp_dir,
self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hosts)
SequentialActions([rm_dirs, put_tar, tar_xf]).run()
# 2. Move installation to base dir and create other dirs
logger.info("Create installation directories")
mv_base_dir = TaktukRemote(
"mv /tmp/" +
os.path.basename(tar_file).replace(".tar.gz", "") + " " +
self.base_dir,
self.hosts)
mkdirs = TaktukRemote("mkdir -p " + self.conf_dir +
" && mkdir -p " + self.logs_dir +
" && mkdir -p " + self.hadoop_temp_dir,
self.hosts)
chmods = TaktukRemote("chmod g+w " + self.base_dir +
" && chmod g+w " + self.conf_dir +
" && chmod g+w " + self.logs_dir +
" && chmod g+w " + self.hadoop_temp_dir,
self.hosts)
SequentialActions([mv_base_dir, mkdirs, chmods]).run()
# 4. Specify environment variables
command = "cat >> " + self.conf_dir + "/hadoop-env.sh << EOF\n"
command += "export JAVA_HOME=" + self.java_home + "\n"
command += "export HADOOP_LOG_DIR=" + self.logs_dir + "\n"
command += "HADOOP_HOME_WARN_SUPPRESS=\"TRUE\"\n"
command += "EOF"
action = Remote(command, self.hosts)
action.run()
# 5. Check version
return self._check_version_compliance()
def uncompress(file_name, host):
if file_name.endswith("tar.gz"):
decompression = Remote("tar xf " + file_name, [host])
decompression.run()
base_name = os.path.basename(file_name[:-7])
dir_name = os.path.dirname(file_name[:-7])
new_name = dir_name + "/data-" + base_name
action = Remote("mv " + file_name[:-7] + " " + new_name, [host])
action.run()
elif file_name.endswith("gz"):
decompression = Remote("gzip -d " + file_name, [host])
decompression.run()
base_name = os.path.basename(file_name[:-3])
dir_name = os.path.dirname(file_name[:-3])
new_name = dir_name + "/data-" + base_name
action = Remote("mv " + file_name[:-3] + " " + new_name, [host])
action.run()
elif file_name.endswith("zip"):
decompression = Remote("unzip " + file_name, [host])
decompression.run()
base_name = os.path.basename(file_name[:-4])
dir_name = os.path.dirname(file_name[:-4])
new_name = dir_name + "/data-" + base_name
action = Remote("mv " + file_name[:-4] + " " + new_name, [host])
action.run()
elif file_name.endswith("bz2"):
decompression = Remote("bzip2 -d " + file_name, [host])
decompression.run()
base_name = os.path.basename(file_name[:-4])
dir_name = os.path.dirname(file_name[:-4])
new_name = dir_name + "/data-" + base_name
action = Remote("mv " + file_name[:-4] + " " + new_name, [host])
action.run()
else:
logger.warn("Unknown extension")
return file_name
return new_name
class vm5k_deployment():
""" Base class to control a deployment of hosts and virtual machines on
Grid'5000. It helps to deploy a wheezy-x64-base environment,
to install and configure libvirt from testing repository, and to deploy
virtual machines.
The base run() method allows to setup automatically the hosts and
virtual machines, using the value of the object.
"""
def __init__(self, infile=None, resources=None, hosts=None,
ip_mac=None, vlan=None,
env_name=None, env_file=None, vms=None,
distribution=None, outdir=None):
""":param infile: an XML file that describe the topology of the
deployment
:param resources: a dict whose keys are Grid'5000 sites and values are
dict, whose keys are hosts and ip_mac, where hosts is a list of
execo.Host and ip_mac is a list of tuple (ip, mac).
:param env_name: name of the Kadeploy environment
:param env_file: path to the Kadeploy environment file
:params vms: dict defining the virtual machines
:params distribution: how to distribute the vms on the hosts
(``round-robin`` , ``concentrated``, ``random``)
:params outdir: directory to store the deployment files
"""
# set a factory for the deployment that use taktuk and chainput
self.fact = ActionFactory(remote_tool=TAKTUK,
fileput_tool=CHAINPUT,
fileget_tool=TAKTUK)
self.kavlan = None if not vlan else vlan
self.kavlan_site = None
if env_name is not None:
self.env_file = None
if ':' not in env_name:
self.env_name, self.env_user = env_name, None
else:
self.env_user, self.env_name = env_name.split(':')
else:
if env_file is not None:
self.env_name = None
self.env_user = None
self.env_file = env_file
else:
self.env_name = 'vm5k'
self.env_user = '******'
self.env_file = None
if outdir:
self.outdir = outdir
else:
self.outdir = 'vm5k_' + strftime("%Y%m%d_%H%M%S_%z")
self.copy_actions = None
self.state = Element('vm5k')
self._define_elements(infile, resources, hosts, vms, ip_mac,
distribution)
self.debian_name = self.env_name.split('-')[0]
#logger.info('XXXX %s || %s || %s' % (self.env_file, self.env_name, self.env_user))
logger.info('%s %s %s %s %s %s %s %s',
len(self.sites), style.emph('sites'),
len(self.clusters), style.user1('clusters'),
len(self.hosts), style.host('hosts'),
len(self.vms), style.vm('vms'))
# PUBLIC METHODS
def run(self):
"""Launch the deployment and configuration of hosts and virtual
machines: hosts_deployment, packages_mamangement, configure_service_node
configure_libvirt, deploy_vms"""
try:
print_step('HOSTS DEPLOYMENT')
self.hosts_deployment()
print_step('MANAGING PACKAGES')
self.packages_management()
print_step('CONFIGURING SERVICE NODE')
self.configure_service_node()
print_step('CONFIGURING LIBVIRT')
self.configure_libvirt()
print_step('VIRTUAL MACHINES')
self.deploy_vms()
finally:
self.get_state()
def hosts_deployment(self, max_tries=1, check_deploy=True,
conf_ssh=True):
"""Deploy the hosts using kadeploy, configure ssh for taktuk execution
and launch backing file disk copy"""
self._launch_kadeploy(max_tries, check_deploy)
if conf_ssh:
self._configure_ssh()
def packages_management(self, upgrade=True, other_packages=None,
launch_disk_copy=True, apt_cacher=False):
"""Configure APT to use testing repository,
perform upgrade and install required packages. Finally start
kvm module"""
self._configure_apt()
if upgrade:
self._upgrade_hosts()
self._install_packages(other_packages=other_packages,
launch_disk_copy=launch_disk_copy)
if apt_cacher:
setup_aptcacher_server(self.hosts)
# Post configuration to load KVM
self.fact.get_remote(
'modprobe kvm; modprobe kvm-intel; modprobe kvm-amd ; ' + \
'chown root:kvm /dev/kvm ;', self.hosts).run()
def configure_service_node(self):
"""Setup automatically a DNS server to access virtual machines by id
and also install a DHCP server if kavlan is used"""
if self.kavlan:
service = 'DNS/DHCP'
dhcp = True
else:
service = 'DNS'
dhcp = False
service_node = get_fastest_host(self.hosts)
logger.info('Setting up %s on %s', style.emph(service),
style.host(service_node.split('.')[0]))
clients = list(self.hosts)
clients.remove(service_node)
dnsmasq_server(service_node, clients, self.vms, dhcp)
def configure_libvirt(self, bridge='br0', libvirt_conf=None):
"""Enable a bridge if needed on the remote hosts, configure libvirt
with a bridged network for the virtual machines, and restart service.
"""
print 'Start configuring libvirt'
self._enable_bridge()
self._libvirt_check_service()
self._libvirt_uniquify()
self._libvirt_bridged_network(bridge)
logger.info('Restarting %s', style.emph('libvirt'))
self.fact.get_remote('service libvirtd restart', self.hosts).run()
def deploy_vms(self, clean_disks=False, disk_location='one',
apt_cacher=False):
"""Destroy the existing VMS, create the virtual disks, install the vms
start them and wait until they have rebooted"""
logger.info('Destroying existing virtual machines')
destroy_vms(self.hosts, undefine=True)
if clean_disks:
self._remove_existing_disks()
logger.info('Creating the virtual disks ')
self._create_backing_file()
if disk_location == 'one':
logger.info('Create disk on each nodes')
create_disks(self.vms).run()
elif disk_location == 'all':
logger.info('Create all disks on all nodes')
create_disks_all_hosts(self.vms, self.hosts).run()
logger.info('Installing the virtual machines')
install_vms(self.vms).run()
logger.info('Starting the virtual machines')
self.boot_time = Timer()
start_vms(self.vms).run()
logger.info('Waiting for VM to boot ...')
wait_vms_have_started(self.vms, self.hosts[0])
activate_vms(self.vms)
self._update_vms_xml()
if apt_cacher:
configure_apt_proxy(self.vms)
def get_state(self, name=None, output=True, mode='compact', plot=False):
""" """
if not name:
name = 'vm5k_' + strftime('%Y%m%d_%H%M%S', localtime())
if output:
output = self.outdir + '/' + name + '.xml'
f = open(output, 'w')
f.write(prettify(self.state))
f.close()
if mode == 'compact':
log = self._print_state_compact()
logger.info('State %s', log)
# PRIVATE METHODS
def _launch_kadeploy(self, max_tries=1, check_deploy=True):
"""Create a execo_g5k.Deployment object, launch the deployment and
return a tuple (deployed_hosts, undeployed_hosts)
"""
logger.info('Deploying %s hosts \n%s', len(self.hosts),
hosts_list(self.hosts))
deployment = Deployment(hosts=[Host(canonical_host_name(host))
for host in self.hosts],
env_file=self.env_file,
env_name=self.env_name,
user=self.env_user,
vlan=self.kavlan)
# Activate kadeploy output log if log level is debug
if logger.getEffectiveLevel() <= 10:
stdout = [sys.stdout]
stderr = [sys.stderr]
else:
stdout = None
stderr = None
deployed_hosts, undeployed_hosts = deploy(deployment,
stdout_handlers=stdout,
stderr_handlers=stderr,
num_tries=max_tries,
check_deployed_command=check_deploy)
deployed_hosts = list(deployed_hosts)
undeployed_hosts = list(undeployed_hosts)
# Renaming hosts if a kavlan is used
if self.kavlan:
for i, host in enumerate(deployed_hosts):
deployed_hosts[i] = get_kavlan_host_name(host, self.kavlan)
for i, host in enumerate(undeployed_hosts):
undeployed_hosts[i] = get_kavlan_host_name(host, self.kavlan)
logger.info('Deployed %s hosts \n%s', len(deployed_hosts),
hosts_list(deployed_hosts))
cr = '\n' if len(undeployed_hosts) > 0 else ''
logger.info('Failed %s hosts %s%s', len(undeployed_hosts), cr,
hosts_list(undeployed_hosts))
self._update_hosts_state(deployed_hosts, undeployed_hosts)
return deployed_hosts, undeployed_hosts
def _configure_ssh(self):
if self.fact.remote_tool == 2:
# Configuring SSH with precopy of id_rsa and id_rsa.pub keys on all
# host to allow TakTuk connection
taktuk_conf = ('-s', '-S',
'$HOME/.ssh/id_rsa:$HOME/.ssh/id_rsa,' +
'$HOME/.ssh/id_rsa.pub:$HOME/.ssh')
else:
taktuk_conf = ('-s', )
conf_ssh = self.fact.get_remote('echo "Host *" >> /root/.ssh/config ;' +
'echo " StrictHostKeyChecking no" >> /root/.ssh/config; ',
self.hosts,
connection_params={'taktuk_options': taktuk_conf}).run()
self._actions_hosts(conf_ssh)
def _start_disk_copy(self, disks=None, backing_file_dir='/tmp'):
""" """
disks_copy = []
if not disks:
disks = self.backing_files
for bf in disks:
logger.info('Treating ' + style.emph(bf))
logger.debug("Checking frontend disk vs host disk")
raw_disk = '%s/orig_' % backing_file_dir + bf.split('/')[-1]
f_disk = Process('md5sum -b ' + bf).run()
disk_hash = f_disk.stdout.split(' ')[0]
cmd = 'if [ -f ' + raw_disk + ' ]; ' + \
'then md5sum -b ' + raw_disk + '; fi'
h_disk = self.fact.get_remote(cmd, self.hosts).run()
disk_ok = True
for p in h_disk.processes:
if p.stdout.split(' ')[0] != disk_hash:
disk_ok = False
break
if disk_ok:
logger.info("Disk " + style.emph(bf) +
" is already present, skipping copy")
else:
disks_copy.append(self.fact.get_fileput(self.hosts, [bf],
remote_location="%s" % backing_file_dir))
if len(disks_copy) > 0:
self.copy_actions = ParallelActions(disks_copy).start()
else:
self.copy_actions = Remote('ls', self.hosts[0]).run()
def _create_backing_file(self, disks=None, backing_file_dir='/tmp'):
""" """
if not self.copy_actions:
self._start_disk_copy(disks)
if not self.copy_actions.ended:
logger.info("Waiting for the end of the disks copy")
self.copy_actions.wait()
if isinstance(self.copy_actions, ParallelActions):
mv_actions = []
for act in self.copy_actions.actions:
fname = act.local_files[0].split('/')[-1]
mv_actions.append(self.fact.get_remote("mv %s/" % backing_file_dir + fname +
" %s/orig_" % backing_file_dir + fname,
self.hosts))
mv = ParallelActions(mv_actions).run()
if not disks:
disks = self.backing_files
for bf in disks:
raw_disk = '%s/orig_' % backing_file_dir + bf.split('/')[-1]
to_disk = '%s/' % backing_file_dir + bf.split('/')[-1]
self.fact.get_remote('cp ' + raw_disk + ' ' + to_disk, self.hosts).run()
logger.info('Copying ssh key on ' + to_disk + ' ...')
cmd = 'modprobe nbd max_part=16; ' + \
'qemu-nbd --connect=/dev/nbd0 ' + to_disk + \
' ; sleep 3 ; partprobe /dev/nbd0 ; ' + \
'part=`fdisk -l /dev/nbd0 |grep dev|grep Linux| grep -v swap|cut -f 1 -d " "` ; ' + \
'mount $part /mnt ; mkdir -p /mnt/root/.ssh ; ' + \
'cat /root/.ssh/authorized_keys >> /mnt/root/.ssh/authorized_keys ; ' + \
'cp -r /root/.ssh/id_rsa* /mnt/root/.ssh/ ;' + \
'umount /mnt; qemu-nbd -d /dev/nbd0'
logger.detail(cmd)
copy_on_vm_base = self.fact.get_remote(cmd, self.hosts).run()
self._actions_hosts(copy_on_vm_base)
def _remove_existing_disks(self, hosts=None):
"""Remove all img and qcow2 file from /tmp directory """
logger.info('Removing existing disks')
if hosts is None:
hosts = self.hosts
remove = self.fact.get_remote('rm -f /tmp/*.img; rm -f /tmp/*.qcow2',
self.hosts).run()
self._actions_hosts(remove)
def _libvirt_check_service(self):
""" """
logger.detail('Checking libvirt service name')
cmd = "if [ ! -e /etc/init.d/libvirtd ]; " + \
" then if [ -e /etc/init.d/libvirt-bin ]; " + \
" then ln -s /etc/init.d/libvirt-bin /etc/init.d/libvirtd; " + \
" else echo 1; " + \
" fi; " + \
"else echo 0; fi"
check_libvirt = self.fact.get_remote(cmd, self.hosts).run()
self._actions_hosts(check_libvirt)
def _libvirt_uniquify(self):
logger.detail('Making libvirt host unique')
cmd = 'uuid=`uuidgen` ' + \
'&& sed -i "s/.*host_uuid.*/host_uuid=\\"${uuid}\\"/g" ' + \
'/etc/libvirt/libvirtd.conf ' + \
'&& service libvirtd restart'
logger.debug(cmd)
self.fact.get_remote(cmd, self.hosts).run()
def _libvirt_bridged_network(self, bridge):
logger.detail('Configuring libvirt network')
# Creating an XML file describing the network
root = Element('network')
name = SubElement(root, 'name')
name.text = 'default'
SubElement(root, 'forward', attrib={'mode': 'bridge'})
SubElement(root, 'bridge', attrib={'name': bridge})
fd, network_xml = mkstemp(dir='/tmp/', prefix='create_br_')
f = fdopen(fd, 'w')
f.write(prettify(root))
f.close()
logger.debug('Destroying existing network')
destroy = self.fact.get_remote('virsh net-destroy default; ' +
'virsh net-undefine default',
self.hosts)
put = TaktukPut(self.hosts, [network_xml],
remote_location='/root/')
start = self.fact.get_remote(
'virsh net-define /root/' + \
network_xml.split('/')[-1] + ' ; ' + \
'virsh net-start default; virsh net-autostart default;',
self.hosts)
netconf = SequentialActions([destroy, put, start]).run()
self._actions_hosts(netconf)
# Hosts configuration
def _enable_bridge(self, name='br0'):
"""We need a bridge to have automatic DHCP configuration for the VM."""
print('Configuring the bridge')
hosts_br = self._get_bridge(self.hosts)
nobr_hosts = []
for host, br in hosts_br.iteritems():
if br is None:
print('No bridge on host %s' % style.host(host))
nobr_hosts.append(host)
elif br != name:
print('Wrong bridge on host %s, destroying it' %
style.host(host))
SshProcess('ip link set ' + br + ' down ; brctl delbr ' + br,
host).run()
nobr_hosts.append(host)
else:
print('Bridge is present on host')
# print('Bridge %s is present on host %s' %
# style.emph('name'), style.host(host))
nobr_hosts = map(lambda x: x.address if isinstance(x, Host) else x,
nobr_hosts)
if len(nobr_hosts) > 0:
print('Creating bridge on %s' % hosts_list(nobr_hosts))
# 'ifdown $br_if ; \n' + \
# 'sed -i "s/$br_if inet dhcp/$br_if inet manual/g" /etc/network/interfaces ; \n' + \
# 'sed -i "s/auto $br_if//g" /etc/network/interfaces ; \n' + \
script = 'export br_if=`ip route |grep default |cut -f 5 -d " "`; \n' + \
'export ip_df=`ip route |grep default |cut -f 3 -d " "`; \n' + \
'export ip_net=`ip route | grep link | grep -v '+ name + '| cut -f 1 -d " "`; \n' +\
'echo " " >> /etc/network/interfaces ; \n' + \
'echo "auto ' + name + '" >> /etc/network/interfaces ; \n' + \
'echo "iface ' + name + ' inet dhcp" >> /etc/network/interfaces ; \n' + \
'echo " bridge_ports $br_if" >> /etc/network/interfaces ; \n' + \
'echo " bridge_stp off" >> /etc/network/interfaces ; \n' + \
'echo " bridge_maxwait 0" >> /etc/network/interfaces ; \n' + \
'echo " bridge_fd 0" >> /etc/network/interfaces ; \n' + \
'ifup ' + name + ' ; \n' + \
'ip route delete default ; \n' + \
'ip route delete $ip_net dev $br_if ; \n' + \
'ip route add default via $ip_df dev br0 ; \n' + \
'sed -i "s/$br_if inet dhcp/$br_if inet manual/g" /etc/network/interfaces ; \n' + \
'sed -i "s/auto $br_if//g" /etc/network/interfaces ; \n' + \
'ifconfig $br_if 0.0.0.0'
fd, br_script = mkstemp(dir='/tmp/', prefix='create_br_')
f = fdopen(fd, 'w')
f.write(script)
f.close()
self.fact.get_fileput(nobr_hosts, [br_script]).run()
self.fact.get_remote('nohup sh ' + br_script.split('/')[-1],
nobr_hosts).run()
print('Waiting for network restart')
if_up = False
nmap_tries = 0
while (not if_up) and nmap_tries < 20:
sleep(20)
nmap_tries += 1
nmap = Process('nmap ' +
' '.join([host for host in nobr_hosts]) +
' -p 22').run()
for line in nmap.stdout.split('\n'):
if 'Nmap done' in line:
if_up = line.split()[2] == line.split()[5].replace('(',
'')
print('Network has been restarted')
print('All hosts have the bridge %s' % style.emph(name))
def _get_bridge(self, hosts):
""" """
print('Retrieving bridge on hosts %s' %
", ".join([host for host in hosts]))
cmd = "brctl show |grep -v 'bridge name' | awk '{ print $1 }' |head -1"
bridge_exists = self.fact.get_remote(cmd, hosts)
bridge_exists.nolog_exit_code = True
bridge_exists.run()
hosts_br = {}
for p in bridge_exists.processes:
stdout = p.stdout.strip()
if len(stdout) == 0:
hosts_br[p.host] = None
else:
hosts_br[p.host] = stdout
return hosts_br
def _configure_apt(self):
"""Create the sources.list file """
logger.detail('Configuring APT')
# Create sources.list file
fd, tmpsource = mkstemp(dir='/tmp/', prefix='sources.list_')
f = fdopen(fd, 'w')
f.write('deb http://ftp.debian.org/debian %s main contrib non-free\n' % self.debian_name + \
'deb http://ftp.debian.org/debian %s-backports main contrib non-free\n' % self.debian_name+ \
'deb http://security.debian.org/ %s/updates main contrib non-free\n' % self.debian_name)
f.close()
# Create preferences file
fd, tmppref = mkstemp(dir='/tmp/', prefix='preferences_')
f = fdopen(fd, 'w')
f.write('Package: * \nPin: release a=%s \nPin-Priority: 900\n\n' % self.debian_name + \
'Package: * \nPin: release a=%s-backports \nPin-Priority: 875\n\n' % self.debian_name)
f.close()
# Create apt.conf file
fd, tmpaptconf = mkstemp(dir='/tmp/', prefix='apt.conf_')
f = fdopen(fd, 'w')
f.write('APT::Acquire::Retries=20;\n')
f.close()
TaktukPut(self.hosts, [tmpsource, tmppref, tmpaptconf],
remote_location='/etc/apt/').run()
cmd = 'cd /etc/apt && ' + \
'mv ' + tmpsource.split('/')[-1] + ' sources.list &&' + \
'mv ' + tmppref.split('/')[-1] + ' preferences &&' + \
'mv ' + tmpaptconf.split('/')[-1] + ' apt.conf'
apt_conf = self.fact.get_remote(cmd, self.hosts).run()
self._actions_hosts(apt_conf)
Local('rm ' + tmpsource + ' ' + tmppref + ' ' + tmpaptconf).run()
def _upgrade_hosts(self):
"""Dist upgrade performed on all hosts"""
logger.info('Upgrading packages')
cmd = "echo 'debconf debconf/frontend select noninteractive' | debconf-set-selections ; " + \
"echo 'debconf debconf/priority select critical' | debconf-set-selections ; " + \
"export DEBIAN_MASTER=noninteractive ; apt-get update ; " + \
"apt-get dist-upgrade -y --force-yes -o Dpkg::Options::='--force-confdef' " + \
"-o Dpkg::Options::='--force-confold' "
upgrade = self.fact.get_remote(cmd, self.hosts).run()
self._actions_hosts(upgrade)
def _install_packages(self, other_packages=None, launch_disk_copy=True):
"""Installation of required packages on the hosts"""
base_packages = 'uuid-runtime bash-completion taktuk locate htop init-system-helpers netcat-traditional'
logger.info('Installing base packages \n%s', style.emph(base_packages))
cmd = 'export DEBIAN_MASTER=noninteractive ; apt-get update && apt-get ' + \
'install -y --force-yes --no-install-recommends ' + base_packages
install_base = self.fact.get_remote(cmd, self.hosts).run()
self._actions_hosts(install_base)
if launch_disk_copy:
self._start_disk_copy()
libvirt_packages = 'libvirt-bin virtinst python2.7 python-pycurl python-libxml2 qemu-kvm nmap libgmp10'
logger.info('Installing libvirt packages \n%s',
style.emph(libvirt_packages))
cmd = 'export DEBIAN_MASTER=noninteractive ; apt-get update && apt-get install -y --force-yes '+\
'-o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -t %s-backports ' % self.debian_name+\
libvirt_packages
install_libvirt = self.fact.get_remote(cmd, self.hosts).run()
self._actions_hosts(install_libvirt)
if other_packages:
self._other_packages(other_packages)
def _other_packages(self, other_packages=None):
"""Installation of packages"""
other_packages = other_packages.replace(',', ' ')
logger.info('Installing extra packages \n%s',
style.emph(other_packages))
cmd = 'export DEBIAN_MASTER=noninteractive ; ' + \
'apt-get update && apt-get install -y --force-yes ' + \
other_packages
install_extra = self.fact.get_remote(cmd, self.hosts).run()
self._actions_hosts(install_extra)
# State related methods
def _define_elements(self, infile=None, resources=None,
hosts=None, vms=None, ip_mac=None,
distribution=None):
"""Create the list of sites, clusters, hosts, vms and check
correspondance between infile and resources"""
if not ip_mac:
self._get_ip_mac(resources)
else:
self.ip_mac = ip_mac
if not hosts:
self._get_resources_elements(resources)
else:
self.hosts = hosts if not self.kavlan else \
map(lambda h: get_kavlan_host_name(h, self.kavlan),
hosts)
self.sites = []
self.clusters = []
if not infile:
self.vms = vms
if len(filter(lambda v: v['host'] is None, self.vms)) > 0:
self.distribution = distribution if distribution \
else 'round-robin'
else:
self.distribution = None
else:
xml = parse(infile)
if self._check_xml_elements(xml, resources):
self.vms = self._get_xml_vms(xml)
self.distribution = None
else:
exit()
self._add_xml_elements()
if self.vms:
if self.distribution:
distribute_vms(self.vms, self.hosts, self.distribution)
self._set_vms_ip_mac()
self._add_xml_vms()
else:
self.vms = []
self.backing_files = list(set([vm['backing_file'] for vm in self.vms]))
def _get_ip_mac(self, resources):
""" """
if len(resources.keys()) == 1:
# mono site
self.ip_mac = resources[resources.keys()[0]]['ip_mac']
self.kavlan = resources[resources.keys()[0]]['kavlan']
elif 'global' in resources:
# multi site in a global kavlan
self.ip_mac = resources['global']['ip_mac']
self.kavlan = resources['global']['kavlan']
self.kavlan_site = resources['global']['site']
else:
# multi site in prod network
self.ip_mac = {site: resource['ip_mac']
for site, resource in resources.iteritems()}
if isinstance(self.ip_mac, list) and len(self.ip_mac) == 0:
logger.error('No ip_range given in the resources')
exit()
elif isinstance(self.ip_mac, dict):
for ip_mac in self.ip_mac.itervalues():
if len(ip_mac) == 0:
logger.error('No ip_range given in the resources')
exit()
def _get_resources_elements(self, resources=None):
""" """
self.sites = sorted([site for site in resources.keys()
if site != 'global'])
self.hosts = []
for site in self.sites:
if self.kavlan:
self.hosts += map(lambda host: get_kavlan_host_name(host,
self.kavlan), resources[site]['hosts'])
else:
self.hosts += resources[site]['hosts']
self.hosts.sort(key=lambda host: (host.split('.', 1)[0].split('-')[0],
int(host.split('.', 1)[0].split('-')[1])))
self.clusters = list(set([get_host_cluster(host)
for host in self.hosts]))
self.clusters.sort()
def _check_xml_elements(self, xml, resources, strict=False):
sites, clusters, hosts = self._get_xml_elements(xml)
ok = True
if not sites == self.sites:
logger.error('List of sites from resources differs from infile' + \
'\n resource %s \n infile %s', self.sites, sites)
ok = False
if not clusters == self.clusters:
logger.error('List of clusters from resources differs from infile' + \
'\n resource %s \n infile %s', self.clusters, clusters)
ok = False
if strict:
if not hosts == self.hosts:
logger.error('List of hosts from resources differs from infile' + \
'\n resource %s \n infile %s', self.hosts, hosts)
ok = False
else:
res_hosts = {}
for host in self.hosts:
cluster = get_host_cluster(host)
if cluster in res_hosts:
res_hosts[cluster] += 1
else:
res_hosts[cluster] = 1
xml_hosts = {}
for host in hosts:
cluster = get_host_cluster(host)
if cluster in xml_hosts:
xml_hosts[cluster] += 1
else:
xml_hosts[cluster] = 1
if not res_hosts == xml_hosts:
logger.error('List of hosts from resources differs from infile' + \
'\n resource %s \n infile %s', self.hosts, hosts)
ok = False
else:
for i in range(len(hosts)):
el_host = xml.find(".//host/[@id='" + hosts[i] + "']")
el_host.attrib['id'] = self.hosts[i]
return ok
def _get_xml_elements(self, xml):
""" """
sites = sorted([site.get('id') for site in xml.findall('./site')])
clusters = sorted([cluster.get('id')
for cluster in xml.findall('.//cluster')])
hosts = sorted([host.get('id') for host in xml.findall('.//host')],
key=lambda host: (host.split('.', 1)[0].split('-')[0],
int(host.split('.', 1)[0].split('-')[1])))
return sites, clusters, hosts
def _get_xml_vms(self, xml):
"""Define the list of VMs from the infile """
def _default_xml_value(key):
return default_vm[key] if key not in vm.attrib else vm.get(key)
vms = []
for host in xml.findall('.//host'):
for vm in host.findall('.//vm'):
vms.append({'id': vm.get('id'),
'n_cpu': int(_default_xml_value('n_cpu')),
'cpuset': _default_xml_value('cpuset'),
'mem': int(_default_xml_value('mem')),
'hdd': int(_default_xml_value('hdd')),
'backing_file': _default_xml_value('backing_file'),
'real_file': _default_xml_value('real_file'),
'host': host.get('id'),
'state': 'KO'})
return vms
def _set_vms_ip_mac(self):
"""Not finished """
if isinstance(self.ip_mac, dict):
i_vm = {site: 0 for site in self.sites}
for vm in self.vms:
vm_site = get_host_site(vm['host'])
vm['ip'], vm['mac'] = self.ip_mac[vm_site][i_vm[vm_site]]
i_vm[vm_site] += 1
else:
i_vm = 0
for vm in self.vms:
if 'mac' not in vm:
vm['ip'], vm['mac'] = self.ip_mac[i_vm]
i_vm += 1
def _add_xml_elements(self):
"""Add sites, clusters, hosts to self.state """
_state = self.state
logger.debug('Initial state \n %s', prettify(_state))
for site in self.sites:
SubElement(_state, 'site', attrib={'id': site})
else:
el_site = SubElement(_state, 'site', attrib={'id': 'unknown'})
logger.debug('Sites added \n %s', prettify(_state))
for cluster in self.clusters:
el_site = _state.find("./site[@id='" + get_cluster_site(cluster) \
+ "']")
SubElement(el_site, 'cluster', attrib={'id': cluster})
else:
el_cluster = SubElement(el_site, 'cluster', attrib={'id': 'unknown'})
logger.debug('Clusters added \n %s', prettify(_state))
print 'xxxxxxx', self.hosts
hosts_attr = get_CPU_RAM_FLOPS(self.hosts)
for host in self.hosts:
if host in get_g5k_hosts():
el_cluster = _state.find(".//cluster/[@id='" +
get_host_cluster(host) + "']")
SubElement(el_cluster, 'host', attrib={'id': host,
'state': 'Undeployed',
'cpu': str(hosts_attr[host]['CPU']),
'mem': str(hosts_attr[host]['RAM'])})
logger.debug('Hosts added \n %s', prettify(_state))
def _add_xml_vms(self):
"""Add vms distributed on hosts to self.state """
for vm in self.vms:
host = self.state.find(".//host/[@id='" + vm['host'] + "']")
SubElement(host, 'vm', attrib={'id': vm['id'],
'ip': vm['ip'],
'mac': vm['mac'],
'mem': str(vm['mem']),
'n_cpu': str(vm['n_cpu']),
'cpuset': vm['cpuset'],
'hdd': str(vm['hdd']),
'backing_file': vm['backing_file'],
'real_file': str(vm['real_file']),
'state': vm['state']})
def _print_state_compact(self):
"""Display in a compact form the distribution of vms on hosts."""
dist = {}
max_len_host = 0
for host in self.hosts:
if len(host.split('.')[0]) > max_len_host:
max_len_host = len(host.split('.')[0])
for vm in self.vms:
host = vm['host'].split('.')[0]
if len(host) > max_len_host:
max_len_host = len(host)
if host not in dist.keys():
dist[host] = {vm['id']: vm['state']}
else:
dist[host][vm['id']] = vm['state']
log = ''
for host in sorted(self.hosts, key=lambda x: (x.split('.')[0].split('-')[0],
int(x.split('.')[0].split('-')[1]))):
host = host.split('.')[0]
if host not in dist:
dist[host] = {}
log += '\n' + style.host(host) + ' '.ljust(max_len_host + 2 - len(host)) + \
str(len(dist[host].keys())) + ' '
try:
vms = sorted(dist[host].keys(), key=lambda x: (x.split('.')[0].split('-')[0],
int(x.split('.')[0].split('-')[1])))
except:
vms = sorted(dist[host].keys())
pass
for vm in vms:
if dist[host][vm] == 'OK':
log += style.OK(vm)
elif dist[host][vm] == 'KO':
log += style.KO(vm)
else:
log += style.Unknown(vm)
log += ' '
return log
def _update_vms_xml(self):
for vm in self.vms:
self.state.find(".//vm/[@id='" + vm['id'] + "']").set('state',
vm['state'])
def _update_hosts_state(self, hosts_ok, hosts_ko):
""" """
for host in hosts_ok:
if host:
if isinstance(host, Host):
host = host.address
self.state.find(".//host/[@id='" + host + "']").set('state',
'OK')
for host in hosts_ko:
if host:
if isinstance(host, Host):
host = host.address
self.state.find(".//host/[@id='" + host + "']").set('state',
'KO')
self.hosts.remove(host)
if len(self.hosts) == 0:
logger.error('No hosts available, because %s are KO',
hosts_list(hosts_ko))
exit()
if self.vms:
distribute_vms(self.vms, self.hosts, self.distribution)
self._set_vms_ip_mac()
def _actions_hosts(self, action):
hosts_ok, hosts_ko = [], []
for p in action.processes:
if p.ok:
hosts_ok.append(p.host)
else:
logger.warn('%s is KO', p.host)
hosts_ko.append(p.host)
hosts_ok, hosts_ko = list(set(hosts_ok)), list(set(hosts_ko))
self._update_hosts_state(hosts_ok, hosts_ko)
def bootstrap(self, tar_file):
"""Install Hadoop in all cluster nodes from the specified tar.gz file.
Args:
tar_file (str):
The file containing Hadoop binaries.
"""
# 0. Check requirements
java_major_version = 7
if not check_java_version(java_major_version, self.hosts):
msg = "Java 1.%d+ required" % java_major_version
logger.error(msg)
raise HadoopException(msg)
self.java_home = get_java_home(self.master)
# 1. Copy hadoop tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_dirs = TaktukRemote(
"rm -rf " + self.base_dir + " " + self.conf_dir + " " +
self.logs_dir + " " + self.hadoop_temp_dir, self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hosts)
rm_tar = TaktukRemote("rm /tmp/" + os.path.basename(tar_file),
self.hosts)
SequentialActions([rm_dirs, put_tar, tar_xf, rm_tar]).run()
# 2. Move installation to base dir and create other dirs
logger.info("Create installation directories")
mv_base_dir = TaktukRemote(
"mv /tmp/" + os.path.basename(tar_file).replace(".tar.gz", "") +
" " + self.base_dir, self.hosts)
mkdirs = TaktukRemote(
"mkdir -p " + self.conf_dir + " && mkdir -p " + self.logs_dir +
" && mkdir -p " + self.hadoop_temp_dir, self.hosts)
chmods = TaktukRemote(
"chmod g+w " + self.base_dir + " && chmod g+w " + self.conf_dir +
" && chmod g+w " + self.logs_dir + " && chmod g+w " +
self.hadoop_temp_dir, self.hosts)
SequentialActions([mv_base_dir, mkdirs, chmods]).run()
# 4. Specify environment variables
command = "cat >> " + self.conf_dir + "/hadoop-env.sh << EOF\n"
command += "export JAVA_HOME=" + self.java_home + "\n"
command += "export HADOOP_LOG_DIR=" + self.logs_dir + "\n"
command += "HADOOP_HOME_WARN_SUPPRESS=\"TRUE\"\n"
command += "EOF"
action = Remote(command, self.hosts)
action.run()
# 5. Check version (cannot do it before)
if not self._check_version_compliance():
return False
# 6. Generate initial configuration
self._initialize_conf()
return True
def bootstrap(self, tar_file):
"""Install Hadoop in all cluster nodes from the specified tar.gz file.
Args:
tar_file (str):
The file containing Hadoop binaries.
"""
# 0. Check requirements
java_major_version = 7
if not check_java_version(java_major_version, self.hosts):
msg = "Java 1.%d+ required" % java_major_version
logger.error(msg)
raise HadoopException(msg)
self.java_home = get_java_home(self.master)
# 1. Copy hadoop tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_dirs = TaktukRemote("rm -rf " + self.base_dir +
" " + self.conf_dir +
" " + self.logs_dir +
" " + self.hadoop_temp_dir,
self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hosts)
rm_tar = TaktukRemote(
"rm /tmp/" + os.path.basename(tar_file),
self.hosts)
SequentialActions([rm_dirs, put_tar, tar_xf, rm_tar]).run()
# 2. Move installation to base dir and create other dirs
logger.info("Create installation directories")
mv_base_dir = TaktukRemote(
"mv /tmp/" +
os.path.basename(tar_file).replace(".tar.gz", "") + " " +
self.base_dir,
self.hosts)
mkdirs = TaktukRemote("mkdir -p " + self.conf_dir +
" && mkdir -p " + self.logs_dir +
" && mkdir -p " + self.hadoop_temp_dir,
self.hosts)
chmods = TaktukRemote("chmod g+w " + self.base_dir +
" && chmod g+w " + self.conf_dir +
" && chmod g+w " + self.logs_dir +
" && chmod g+w " + self.hadoop_temp_dir,
self.hosts)
SequentialActions([mv_base_dir, mkdirs, chmods]).run()
# 4. Specify environment variables
command = "cat >> " + self.conf_dir + "/hadoop-env.sh << EOF\n"
command += "export JAVA_HOME=" + self.java_home + "\n"
command += "export HADOOP_LOG_DIR=" + self.logs_dir + "\n"
command += "HADOOP_HOME_WARN_SUPPRESS=\"TRUE\"\n"
command += "EOF"
action = Remote(command, self.hosts)
action.run()
# 5. Check version (cannot do it before)
if not self._check_version_compliance():
return False
# 6. Generate initial configuration
self._initialize_conf()
return True
def _set_common_params(self, params, conf_dir, default_tuning=False):
"""Replace common parameters. Some user-specified values are
overwritten.
Args:
params (str):
Already defined parameters over all the clusters.
conf_dir (str):
The path of the directory with the configuration files.
default_tuning (bool, optional):
Whether to use automatic tuning based on some best practices or
leave the default parameters.
"""
defs_file = conf_dir + "/spark-defaults.conf"
# spark-env.sh
command = "cat >> " + self.conf_dir + "/spark-env.sh << EOF\n"
command += "SPARK_MASTER_PORT=" + str(self.port) + "\n"
command += "EOF\n"
action = Remote(command, self.hosts)
action.run()
# Get already set parameters
global_params = params["global"]
exec_mem = global_params["exec_mem"]
exec_cores = global_params["exec_cores"]
total_execs = global_params["total_execs"]
# Log parameters
if self.evs_log_dir:
write_in_props_file(defs_file,
"spark.eventLog.enabled",
"true",
create_if_absent=True,
override=True)
write_in_props_file(defs_file,
"spark.eventLog.dir",
self.evs_log_dir,
create_if_absent=True,
override=True)
write_in_props_file(defs_file,
"spark.logConf",
"true",
create_if_absent=True,
override=False)
if default_tuning:
write_in_props_file(defs_file,
"spark.executor.memory",
"%dm" % exec_mem,
create_if_absent=True,
override=False)
write_in_props_file(defs_file,
"spark.executor.cores",
exec_cores,
create_if_absent=True,
override=False)
write_in_props_file(defs_file,
"spark.executor.instances",
total_execs,
create_if_absent=True,
override=False)
def bootstrap(self, tar_file):
# 1. Remove used dirs if existing
action = Remote("rm -rf " + self.base_dir, self.hc.hosts)
action.run()
action = Remote("rm -rf " + self.conf_dir, self.hc.hosts)
action.run()
# 1. Copy Mahout tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
action = Put(self.hc.hosts, [tar_file], "/tmp")
action.run()
action = Remote(
"tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp",
self.hc.hosts)
action.run()
# 2. Move installation to base dir
logger.info("Create installation directories")
action = Remote(
"mv /tmp/" +
os.path.basename(tar_file).replace(".tar.gz", "") + " " +
self.base_dir,
self.hc.hosts)
action.run()
# 3 Create other dirs
action = Remote("mkdir -p " + self.conf_dir, self.hc.hosts)
action.run()
# 4. Include libraries in Hadoop's classpath
list_dirs = SshProcess("ls -1 " + self.base_dir + "/*.jar",
self.hc.master)
list_dirs.run()
libs = " ".join(list_dirs.stdout.splitlines())
action = Remote("cp " + libs + " " + self.hc.base_dir + "/lib",
self.hc.hosts)
action.run()
initialized = True # No need to call initialize()
