def run_xp(self):
"""Iterate over the parameters and execute the bench"""
while len(self.sweeper.get_remaining()) > 0:
comb = self.sweeper.get_next()
if comb['n_core'] > get_host_attributes(comb['cluster']+'-1')['architecture']['smt_size'] * self.n_nodes:
self.sweeper.skip(comb)
continue
logger.info('Processing new combination %s' % (comb,))
site = get_cluster_site(comb['cluster'])
jobs = oarsub([(OarSubmission(resources = "{cluster='" + comb['cluster']+"'}/nodes=" + str(self.n_nodes),
job_type = 'allow_classic_ssh',
walltime ='0:10:00'),
site)])
if jobs[0][0]:
try:
wait_oar_job_start(*jobs[0])
nodes = get_oar_job_nodes(*jobs[0])
bench_cmd = 'mpirun -H %s -n %i %s ~/NPB3.3-MPI/bin/lu.%s.%i' % (
",".join([node.address for node in nodes]),
comb['n_core'],
get_mpi_opts(comb['cluster']),
comb['size'],
comb['n_core'])
lu_bench = SshProcess(bench_cmd, nodes[0])
lu_bench.stdout_handlers.append(self.result_dir + '/' + slugify(comb) + '.out')
lu_bench.run()
if lu_bench.ok:
logger.info("comb ok: %s" % (comb,))
self.sweeper.done(comb)
continue
finally:
oardel(jobs)
logger.info("comb NOT ok: %s" % (comb,))
self.sweeper.cancel(comb)
def takeMetric(
path,
startTime,
endTime,
metric=['cpu', 'mem', 'disk', 'swap', 'network']):
opt = ''
cmd_template_sar = (
"sar -f /var/log/sysstat/sa* -{opt} -s {startTime} -e {endTime}"
)
for met in metric:
if met == 'cpu':
opt = 'u'
elif met == 'mem':
opt = 'r'
elif met == 'disk':
opt = 'dp'
elif met == 'swap':
opt = 'S'
elif met == 'network':
opt = 'n DEV'
cmd = cmd_template_sar.format(opt=opt,
startTime=startTime,
endTime=endTime)
for host in self.cluster:
hE = SshProcess(cmd,
host,
connection_params={'user': '******'})
hE.run()
stdMetric = host + '-' + met + '.txt'
with open(os.path.join(path, stdMetric),
"w") as sout:
sout.write(hE.stdout)
def get_memory_and_cores(self, host):
"""Obtain the total available memory in MB and number of cores of the
given host.
Supported systems include Linux and Max OS X.. In linux it uses nproc
and /proc/meminfo to obtain the informtion. In Max OS X it uses
system_profiler.
Args:
host (Host): The host to query.
Return:
tuple of (int, int): A tuple containing the host available memory in
MB and its number of cores.
"""
linux_cmd = "nproc && " \
"cat /proc/meminfo | grep MemTotal | awk '{ print $2,$3 }'"
max_cmd = "system_profiler SPHardwareDataType | grep Cores | " \
"awk '{ print $NF }' && " \
"system_profiler SPHardwareDataType | grep Memory | " \
"awk '{ print $2,$3 }'"
undef_str = "?"
undef_cmd = "echo '" + undef_str + "'"
command = 'if [ $(uname) == "Linux" ]; then %s; ' \
'elif [ $(uname) == "Darwin" ]; then %s; ' \
'else %s; fi' % \
(linux_cmd, max_cmd, undef_cmd)
proc = SshProcess(command, host)
proc.run()
out = proc.stdout
if out == undef_str:
return None
else:
units = {
"kb": lambda x: int(x) // 1024,
"mb": lambda x: int(x),
"gb": lambda x: int(x) * 1024,
}
(cores_str, mem_str) = out.splitlines()
cores = int(cores_str)
(num, unit) = mem_str.split()
mem = units[unit.lower()](num)
return mem, cores
def bootstrap(self, tar_file):
"""Install Cassandra in all cluster nodes from the specified tar.gz file.
Args:
tar_file (str):
The file containing Cassandra binaries.
"""
# 0. Check that required packages are present
required_packages = "openjdk-7-jre openjdk-7-jdk"
check_packages = TaktukRemote("dpkg -s " + required_packages, self.hosts)
for p in check_packages.processes:
p.nolog_exit_code = p.nolog_error = True
check_packages.run()
if not check_packages.ok:
logger.info("Packages not installed, trying to install")
install_packages = TaktukRemote(
"export DEBIAN_MASTER=noninteractive ; "
+ "apt-get update && apt-get install -y --force-yes "
+ required_packages,
self.hosts,
).run()
if not install_packages.ok:
logger.error("Unable to install the packages")
get_java_home = SshProcess("echo $(readlink -f /usr/bin/javac | " 'sed "s:/bin/javac::")', self.master)
get_java_home.run()
self.java_home = get_java_home.stdout.strip()
logger.info("All required packages are present")
# 1. Copy hadoop tar file and uncompress
logger.info("Copy " + tar_file + " to hosts and uncompress")
rm_dirs = TaktukRemote("rm -rf " + self.base_dir + " " + self.conf_dir + " " + self.logs_dir, self.hosts)
put_tar = TaktukPut(self.hosts, [tar_file], "/tmp")
tar_xf = TaktukRemote("tar xf /tmp/" + os.path.basename(tar_file) + " -C /tmp", self.hosts)
SequentialActions([rm_dirs, put_tar, tar_xf]).run()
# 2. Move installation to base dir and create other dirs
logger.info("Create installation directories")
mv_base_dir = TaktukRemote(
"mv /tmp/" + os.path.basename(tar_file).replace(".tar.gz", "") + " " + self.base_dir, self.hosts
)
mkdirs = TaktukRemote("mkdir -p " + self.conf_dir + " && mkdir -p " + self.logs_dir, self.hosts)
chmods = TaktukRemote(
"chmod g+w " + self.base_dir + " && chmod g+w " + self.conf_dir + " && chmod g+w " + self.logs_dir,
self.hosts,
)
SequentialActions([mv_base_dir, mkdirs, chmods]).run()
def runMpi(cmd):
act = SshProcess(cmd,
master,
connection_params={'user': '******'},
shell=True)
act.run()
if not os.path.exists(curPath):
os.makedirs(curPath)
with open(os.path.join(curPath, "stdout.txt"),
"a+") as sout, open(
os.path.join(curPath, "stderr.txt"),
"w") as serr:
sout.write(act.stdout)
serr.write(act.stderr)
return act.ok
def import_from_kaenv(self, env, remote=None):
"""import a kadeploy environment from a kadeploy database"""
env = re.match(
r"^(?P<name>[-_.\w]+)(?:@(?P<user>[_.\w]+))?(:?:(?P<version>[_.\w]+))?$",
env).groupdict("")
if env['user']:
env['user'] = "******" + env['user']
if env['version']:
env['version'] = " --env-version " + env['version']
kaenv_cmd = "kaenv3{user}{version} -p {name}".format(**env)
if remote:
remote = re.match(
r"^(?:(?P<user>[-_.\w]+)@)?(?P<address>[-_.\w]+)(?::(?P<port>\d{1,5}))?$",
remote).groupdict()
p = SshProcess(kaenv_cmd, Host(**remote))
else:
p = Process(kaenv_cmd, shell=True)
p.run()
self.desc = yaml.load(p.stdout)
return self
def run(self):
"""Run the experiment"""
already_configured = self.options.already_configured
reservation_job_id = int(self.options.reservation_id) \
if self.options.reservation_id is not None else None
is_a_test = self.options.is_a_test
if is_a_test:
logger.warn('THIS IS A TEST! This run will use only a few '
'resources')
# make the result folder writable for all
os.chmod(self.result_dir, 0o777)
# Import configuration
with open(self.args[0]) as config_file:
config = json.load(config_file)
# backup configuration
copy(self.args[0], self.result_dir)
site = config["grid5000_site"]
resources = config["resources"]
nb_experiment_nodes = config["nb_experiment_nodes"]
walltime = str(config["walltime"])
env_name = config["kadeploy_env_name"]
workloads = config["workloads"]
# check if workloads exists (Suppose that the same NFS mount point
# is present on the remote and the local environment
for workload_file in workloads:
with open(workload_file):
pass
# copy the workloads files to the results dir
copy(workload_file, self.result_dir)
# define the workloads parameters
self.parameters = {
'workload_filename': workloads
}
logger.info('Workloads: {}'.format(workloads))
# define the iterator over the parameters combinations
self.sweeper = ParamSweeper(os.path.join(self.result_dir, "sweeps"),
sweep(self.parameters))
# Due to previous (using -c result_dir) run skip some combination
logger.info('Skipped parameters:' +
'{}'.format(str(self.sweeper.get_skipped())))
logger.info('Number of parameters combinations {}'.format(
str(len(self.sweeper.get_remaining()))))
logger.info('combinations {}'.format(
str(self.sweeper.get_remaining())))
if reservation_job_id is not None:
jobs = [(reservation_job_id, site)]
else:
jobs = oarsub([(OarSubmission(resources=resources,
job_type='deploy',
walltime=walltime), site)])
job_id, site = jobs[0]
if job_id:
try:
logger.info("waiting job start %s on %s" % (job_id, site))
wait_oar_job_start(
job_id, site, prediction_callback=prediction_callback)
logger.info("getting nodes of %s on %s" % (job_id, site))
nodes = get_oar_job_nodes(job_id, site)
# sort the nodes
nodes = sorted(nodes, key=lambda node: node.address)
# get only the necessary nodes under the switch
if nb_experiment_nodes > len(nodes):
raise RuntimeError('The number of given node in the '
'reservation ({}) do not match the '
'requested resources '
'({})'.format(len(nodes),
nb_experiment_nodes))
nodes = nodes[:nb_experiment_nodes]
logger.info("deploying nodes: {}".format(str(nodes)))
deployed, undeployed = deploy(
Deployment(nodes, env_name=env_name),
check_deployed_command=already_configured)
if undeployed:
logger.warn(
"NOT deployed nodes: {}".format(str(undeployed)))
raise RuntimeError('Deployement failed')
if not already_configured:
# install OAR
install_cmd = "apt-get update; apt-get install -y "
node_packages = "oar-node"
logger.info(
"installing OAR nodes: {}".format(str(nodes[1:])))
install_oar_nodes = Remote(
install_cmd + node_packages,
nodes[1:],
connection_params={'user': '******'})
install_oar_nodes.start()
server_packages = ("oar-server oar-server-pgsql oar-user "
"oar-user-pgsql postgresql python3-pip "
"libjson-perl postgresql-server-dev-all")
install_oar_sched_cmd = """
mkdir -p /opt/oar_sched; \
cd /opt/oar_sched; \
git clone https://github.com/oar-team/oar3.git; \
cd oar3; \
git checkout dce942bebc2; \
pip3 install -e .; \
cd /usr/lib/oar/schedulers; \
ln -s /usr/local/bin/kamelot; \
pip3 install psycopg2
"""
logger.info("installing OAR server node: {}".format(str(nodes[0])))
install_master = SshProcess(install_cmd + server_packages +
";" + install_oar_sched_cmd, nodes[0],
connection_params={'user': '******'})
install_master.run()
install_oar_nodes.wait()
if not install_master.ok:
Report(install_master)
configure_oar_cmd = """
sed -i \
-e 's/^\(DB_TYPE\)=.*/\\1="Pg"/' \
-e 's/^\(DB_HOSTNAME\)=.*/\\1="localhost"/' \
-e 's/^\(DB_PORT\)=.*/\\1="5432"/' \
-e 's/^\(DB_BASE_PASSWD\)=.*/\\1="oar"/' \
-e 's/^\(DB_BASE_LOGIN\)=.*/\\1="oar"/' \
-e 's/^\(DB_BASE_PASSWD_RO\)=.*/\\1="oar_ro"/' \
-e 's/^\(DB_BASE_LOGIN_RO\)=.*/\\1="oar_ro"/' \
-e 's/^\(SERVER_HOSTNAME\)=.*/\\1="localhost"/' \
-e 's/^\(SERVER_PORT\)=.*/\\1="16666"/' \
-e 's/^\(LOG_LEVEL\)\=\"2\"/\\1\=\"3\"/' \
-e 's#^\(LOG_FILE\)\=.*#\\1="{result_dir}/oar.log"#' \
-e 's/^\(JOB_RESOURCE_MANAGER_PROPERTY_DB_FIELD\=\"cpuset\".*\)/#\\1/' \
-e 's/^#\(CPUSET_PATH\=\"\/oar\".*\)/\\1/' \
-e 's/^\(FINAUD_FREQUENCY\)\=.*/\\1="0"/' \
/etc/oar/oar.conf
""".format(result_dir=self.result_dir)
configure_oar = Remote(configure_oar_cmd, nodes,
connection_params={'user': '******'})
configure_oar.run()
logger.info("OAR is configured on all nodes")
# Configure server
create_db = "oar-database --create --db-is-local"
config_oar_sched = ("oarnotify --remove-queue default;"
"oarnotify --add-queue default,1,kamelot")
start_oar = "systemctl start oar-server.service"
logger.info(
"configuring OAR database: {}".format(str(nodes[0])))
config_master = SshProcess(create_db + ";" + config_oar_sched + ";" + start_oar,
nodes[0],
connection_params={'user': '******'})
config_master.run()
# propagate SSH keys
logger.info("configuring OAR SSH")
oar_key = "/tmp/.ssh"
Process('rm -rf ' + oar_key).run()
Process('scp -o BatchMode=yes -o PasswordAuthentication=no '
'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null '
'-o ConnectTimeout=20 -rp -o User=root ' +
nodes[0].address + ":/var/lib/oar/.ssh"
' ' + oar_key).run()
# Get(nodes[0], "/var/lib/oar/.ssh", [oar_key], connection_params={'user': '******'}).run()
Put(nodes[1:], [oar_key], "/var/lib/oar/", connection_params={'user': '******'}).run()
add_resources_cmd = """
oarproperty -a cpu || true; \
oarproperty -a core || true; \
oarproperty -c -a host || true; \
oarproperty -a mem || true; \
"""
for node in nodes[1:]:
add_resources_cmd = add_resources_cmd + "oarnodesetting -a -h {node} -p host={node} -p cpu=1 -p core=4 -p cpuset=0 -p mem=16; \\\n".format(node=node.address)
add_resources = SshProcess(add_resources_cmd, nodes[0],
connection_params={'user': '******'})
add_resources.run()
if add_resources.ok:
logger.info("oar is now configured!")
else:
raise RuntimeError("error in the OAR configuration: Abort!")
# TODO backup de la config de OAR
# Do the replay
logger.info('begining the replay')
while len(self.sweeper.get_remaining()) > 0:
combi = self.sweeper.get_next()
workload_file = os.path.basename(combi['workload_filename'])
oar_replay = SshProcess(script_path + "/oar_replay.py " +
combi['workload_filename'] + " " +
self.result_dir + " oar_gant_" +
workload_file,
nodes[0])
oar_replay.stdout_handlers.append(self.result_dir + '/' +
workload_file + '.out')
logger.info("replaying workload: {}".format(combi))
oar_replay.run()
if oar_replay.ok:
logger.info("Replay workload OK: {}".format(combi))
self.sweeper.done(combi)
else:
logger.info("Replay workload NOT OK: {}".format(combi))
self.sweeper.cancel(combi)
raise RuntimeError("error in the OAR replay: Abort!")
except:
traceback.print_exc()
ipdb.set_trace()
finally:
if is_a_test:
ipdb.set_trace()
if reservation_job_id is None:
logger.info("delete job: {}".format(jobs))
oardel(jobs)
def is_job_running(self, job_id=None):
""" """
get_state = SshProcess('qstat -f ' + str(job_id), jobserver)
get_state.ignore_exit_code = True
get_state.run()
return get_state.ok
def get_java_home(host):
proc = SshProcess('echo $(readlink -f /usr/bin/javac | '
'sed "s:/bin/javac::")', host)
proc.run()
return proc.stdout.strip()
def get_java_home(host):
proc = SshProcess(
'echo $(readlink -f /usr/bin/javac | '
'sed "s:/bin/javac::")', host)
proc.run()
return proc.stdout.strip()
def run(self):
"""Run the experiment"""
already_configured = self.options.already_configured
reservation_job_id = int(self.options.reservation_id) \
if self.options.reservation_id is not None else None
is_a_test = self.options.is_a_test
if is_a_test:
logger.warn('THIS IS A TEST! This run will use only a few '
'resources')
# make the result folder writable for all
os.chmod(self.result_dir, 0o777)
# Import configuration
with open(self.args[0]) as config_file:
config = json.load(config_file)
# backup configuration
copy(self.args[0], self.result_dir)
site = config["grid5000_site"]
resources = config["resources"]
nb_experiment_nodes = config["nb_experiment_nodes"]
walltime = str(config["walltime"])
env_name = config["kadeploy_env_name"]
workloads = config["workloads"]
# check if workloads exists (Suppose that the same NFS mount point
# is present on the remote and the local environment
for workload_file in workloads:
with open(workload_file):
pass
# copy the workloads files to the results dir
copy(workload_file, self.result_dir)
# define the workloads parameters
self.parameters = {'workload_filename': workloads}
logger.info('Workloads: {}'.format(workloads))
# define the iterator over the parameters combinations
self.sweeper = ParamSweeper(os.path.join(self.result_dir, "sweeps"),
sweep(self.parameters))
# Due to previous (using -c result_dir) run skip some combination
logger.info('Skipped parameters:' +
'{}'.format(str(self.sweeper.get_skipped())))
logger.info('Number of parameters combinations {}'.format(
str(len(self.sweeper.get_remaining()))))
logger.info('combinations {}'.format(str(
self.sweeper.get_remaining())))
if reservation_job_id is not None:
jobs = [(reservation_job_id, site)]
else:
jobs = oarsub([(OarSubmission(resources=resources,
job_type='deploy',
walltime=walltime), site)])
job_id, site = jobs[0]
if job_id:
try:
logger.info("waiting job start %s on %s" % (job_id, site))
wait_oar_job_start(job_id,
site,
prediction_callback=prediction_callback)
logger.info("getting nodes of %s on %s" % (job_id, site))
nodes = get_oar_job_nodes(job_id, site)
# sort the nodes
nodes = sorted(nodes, key=lambda node: node.address)
# get only the necessary nodes under the switch
if nb_experiment_nodes > len(nodes):
raise RuntimeError('The number of given node in the '
'reservation ({}) do not match the '
'requested resources '
'({})'.format(len(nodes),
nb_experiment_nodes))
nodes = nodes[:nb_experiment_nodes]
logger.info("deploying nodes: {}".format(str(nodes)))
deployed, undeployed = deploy(
Deployment(nodes, env_name=env_name),
check_deployed_command=already_configured)
if undeployed:
logger.warn("NOT deployed nodes: {}".format(
str(undeployed)))
raise RuntimeError('Deployement failed')
if not already_configured:
# install OAR
install_cmd = "apt-get update; apt-get install -y "
node_packages = "oar-node"
logger.info("installing OAR nodes: {}".format(
str(nodes[1:])))
install_oar_nodes = Remote(
install_cmd + node_packages,
nodes[1:],
connection_params={'user': '******'})
install_oar_nodes.start()
server_packages = (
"oar-server oar-server-pgsql oar-user "
"oar-user-pgsql postgresql python3-pip "
"libjson-perl postgresql-server-dev-all")
install_oar_sched_cmd = """
mkdir -p /opt/oar_sched; \
cd /opt/oar_sched; \
git clone https://github.com/oar-team/oar3.git; \
cd oar3; \
git checkout dce942bebc2; \
pip3 install -e .; \
cd /usr/lib/oar/schedulers; \
ln -s /usr/local/bin/kamelot; \
pip3 install psycopg2
"""
logger.info("installing OAR server node: {}".format(
str(nodes[0])))
install_master = SshProcess(
install_cmd + server_packages + ";" +
install_oar_sched_cmd,
nodes[0],
connection_params={'user': '******'})
install_master.run()
install_oar_nodes.wait()
if not install_master.ok:
Report(install_master)
configure_oar_cmd = """
sed -i \
-e 's/^\(DB_TYPE\)=.*/\\1="Pg"/' \
-e 's/^\(DB_HOSTNAME\)=.*/\\1="localhost"/' \
-e 's/^\(DB_PORT\)=.*/\\1="5432"/' \
-e 's/^\(DB_BASE_PASSWD\)=.*/\\1="oar"/' \
-e 's/^\(DB_BASE_LOGIN\)=.*/\\1="oar"/' \
-e 's/^\(DB_BASE_PASSWD_RO\)=.*/\\1="oar_ro"/' \
-e 's/^\(DB_BASE_LOGIN_RO\)=.*/\\1="oar_ro"/' \
-e 's/^\(SERVER_HOSTNAME\)=.*/\\1="localhost"/' \
-e 's/^\(SERVER_PORT\)=.*/\\1="16666"/' \
-e 's/^\(LOG_LEVEL\)\=\"2\"/\\1\=\"3\"/' \
-e 's#^\(LOG_FILE\)\=.*#\\1="{result_dir}/oar.log"#' \
-e 's/^\(JOB_RESOURCE_MANAGER_PROPERTY_DB_FIELD\=\"cpuset\".*\)/#\\1/' \
-e 's/^#\(CPUSET_PATH\=\"\/oar\".*\)/\\1/' \
-e 's/^\(FINAUD_FREQUENCY\)\=.*/\\1="0"/' \
/etc/oar/oar.conf
""".format(result_dir=self.result_dir)
configure_oar = Remote(configure_oar_cmd,
nodes,
connection_params={'user': '******'})
configure_oar.run()
logger.info("OAR is configured on all nodes")
# Configure server
create_db = "oar-database --create --db-is-local"
config_oar_sched = (
"oarnotify --remove-queue default;"
"oarnotify --add-queue default,1,kamelot")
start_oar = "systemctl start oar-server.service"
logger.info("configuring OAR database: {}".format(
str(nodes[0])))
config_master = SshProcess(
create_db + ";" + config_oar_sched + ";" + start_oar,
nodes[0],
connection_params={'user': '******'})
config_master.run()
# propagate SSH keys
logger.info("configuring OAR SSH")
oar_key = "/tmp/.ssh"
Process('rm -rf ' + oar_key).run()
Process(
'scp -o BatchMode=yes -o PasswordAuthentication=no '
'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null '
'-o ConnectTimeout=20 -rp -o User=root ' +
nodes[0].address + ":/var/lib/oar/.ssh"
' ' + oar_key).run()
# Get(nodes[0], "/var/lib/oar/.ssh", [oar_key], connection_params={'user': '******'}).run()
Put(nodes[1:], [oar_key],
"/var/lib/oar/",
connection_params={
'user': '******'
}).run()
add_resources_cmd = """
oarproperty -a cpu || true; \
oarproperty -a core || true; \
oarproperty -c -a host || true; \
oarproperty -a mem || true; \
"""
for node in nodes[1:]:
add_resources_cmd = add_resources_cmd + "oarnodesetting -a -h {node} -p host={node} -p cpu=1 -p core=4 -p cpuset=0 -p mem=16; \\\n".format(
node=node.address)
add_resources = SshProcess(
add_resources_cmd,
nodes[0],
connection_params={'user': '******'})
add_resources.run()
if add_resources.ok:
logger.info("oar is now configured!")
else:
raise RuntimeError(
"error in the OAR configuration: Abort!")
# TODO backup de la config de OAR
# Do the replay
logger.info('begining the replay')
while len(self.sweeper.get_remaining()) > 0:
combi = self.sweeper.get_next()
workload_file = os.path.basename(
combi['workload_filename'])
oar_replay = SshProcess(
script_path + "/oar_replay.py " +
combi['workload_filename'] + " " + self.result_dir +
" oar_gant_" + workload_file, nodes[0])
oar_replay.stdout_handlers.append(self.result_dir + '/' +
workload_file + '.out')
logger.info("replaying workload: {}".format(combi))
oar_replay.run()
if oar_replay.ok:
logger.info("Replay workload OK: {}".format(combi))
self.sweeper.done(combi)
else:
logger.info("Replay workload NOT OK: {}".format(combi))
self.sweeper.cancel(combi)
raise RuntimeError("error in the OAR replay: Abort!")
except:
traceback.print_exc()
ipdb.set_trace()
finally:
if is_a_test:
ipdb.set_trace()
if reservation_job_id is None:
logger.info("delete job: {}".format(jobs))
oardel(jobs)
def start(self):
"""Start MongoDB server."""
self._check_initialization()
logger.info("Starting MongoDB")
if self.running:
logger.warn("MongoDB was already started")
return
# Start nodes
procs = []
for h in self.hosts:
mongo_command = (NUMA_PREFIX + " " +
self.bin_dir + "/mongod "
" --fork "
" --config " + os.path.join(self.conf_dir,
CONF_FILE) +
" --bind_ip " + h.address +
" --port " + str(self.md_port))
logger.debug(mongo_command)
proc = SshProcess(mongo_command, h)
proc.start()
procs.append(proc)
finished_ok = True
for p in procs:
p.wait()
if not p.finished_ok:
finished_ok = False
if not finished_ok:
logger.warn("Error while starting MongoDB")
return
else:
self.running = True
# Start replication
if self.do_replication:
logger.info("Configuring replication")
mongo_command = "rs.initiate();"
mongo_command += ';'.join(
'rs.add("' + h.address + ':' + str(self.md_port) + '")'
for h in self.hosts)
logger.debug(mongo_command)
proc = TaktukRemote(self.bin_dir + "/mongo "
"--eval '" + mongo_command + "' " +
self.master.address,
[self.master])
proc.run()
if not proc.finished_ok:
logger.warn("Not able to start replication")
if self.do_sharding:
if not self.initialized_sharding:
logger.info("Configuring sharding")
time.sleep(2)
mongo_command = (
'rs.initiate({'
'_id : "%s",'
'configsvr : true,'
'members : [%s]})' % (
self.rs_name,
",".join('{ _id : %d, host : "%s:%d" }' %
(_id, h.address, self.md_port)
for (_id, h) in enumerate(self.hosts))
)
)
logger.debug(mongo_command)
proc = SshProcess(self.bin_dir + "/mongo " +
"--eval '" + mongo_command + "' " +
self.master.address,
self.master)
proc.run()
if proc.finished_ok:
self.initialized_sharding = True
else:
logger.warn("Not able to configure sharding")
logger.info("Starting sharding servers")
mongo_command = (
NUMA_PREFIX + " " +
self.bin_dir + "/mongos"
" --configdb " + self.rs_name + "/" +
",".join('%s:%d' % (h.address, self.md_port)
for h in self.hosts) +
" --bind_ip " + self.master.address +
" --port " + str(self.ms_port) +
" --fork"
" --logpath " + self.logs_dir + "/mongos.log"
" --pidfilepath " + self.mongos_pid_file
)
logger.debug(mongo_command)
start_ms = TaktukRemote(mongo_command, [self.master])
start_ms.run()