def add_munin_plugins(hosts, plugins):
"""Create a symbolic link to activate plugins """
cmd = '; '.join([
'ln -s /usr/share/munin/plugins/ /etc/munin/plugins' + plugin
for plugin in plugins
])
TaktukRemote(cmd, hosts)
def activate_vms(vms, dest='lyon.grid5000.fr'):
"""Connect locally on every host and on all VMS to ping a host
and update ARP tables"""
logger.info('Executing ping from virtual machines on hosts')
cmd = "VMS=`virsh list | grep -v State | grep -v -e '----' | awk '{print $2}'`; " + \
"for VM in $VMS; do " + \
" ssh $VM \"ping -c 3 " + dest + " \"; " + \
"done"
logger.debug('Launching ping probes to update ARP tables with %s', cmd)
activate = TaktukRemote(cmd, list(set([vm['host'] for vm in vms])))
for p in activate.processes:
p.ignore_exit_code = p.nolog_exit_code = True
if logger.getEffectiveLevel() <= 10:
p.stdout_handlers.append(sys.stdout)
activate.run()
return activate.ok
def activate_vms(vms, dest='lyon.grid5000.fr'):
"""Connect locally on every host and on all VMS to ping a host
and update ARP tables"""
logger.info('Executing ping from virtual machines on hosts')
cmd = "VMS=`virsh list | grep -v State | grep -v -e '----' | awk '{print $2}'`; " + \
"for VM in $VMS; do " + \
" ssh $VM \"ping -c 3 " + dest + " \"; " + \
"done"
logger.debug('Launching ping probes to update ARP tables with %s', cmd)
activate = TaktukRemote(cmd, list(set([vm['host'] for vm in vms])))
for p in activate.processes:
p.ignore_exit_code = p.nolog_exit_code = True
if logger.getEffectiveLevel() <= 10:
p.stdout_handlers.append(sys.stdout)
activate.run()
return activate.ok
def dnsmasq_server(server, clients=None, vms=None, dhcp=True):
"""Configure a DHCP server with dnsmasq
:param server: host where the server will be installed
:param clients: list of hosts that will be declared in dnsmasq
:param vms: list of virtual machines
"""
logger.debug('Installing and configuring a DNS/DHCP server on %s', server)
test_running = Process('nmap ' + server + ' -p 53 | grep domain')
test_running.shell = True
test_running.run()
if 'open' in test_running.stdout:
logger.info('DNS server already running, updating configuration')
else:
cmd = 'killall dnsmasq; export DEBIAN_MASTER=noninteractive ; ' + \
'apt-get update ; apt-get -y purge dnsmasq-base ; ' + \
'apt-get install -t wheezy -o Dpkg::Options::="--force-confdef" ' + \
'-o Dpkg::Options::="--force-confnew" ' + \
'-y dnsmasq; echo 1 > /proc/sys/net/ipv4/ip_forward '
SshProcess(cmd, server).run()
sites = list(
set([
get_host_site(client)
for client in clients if get_host_site(client)
] + [get_host_site(server)]))
add_vms(vms, server)
if clients:
kill_dnsmasq = TaktukRemote('killall dnsmasq', clients)
for p in kill_dnsmasq.processes:
p.ignore_exit_code = p.nolog_exit_code = True
kill_dnsmasq.run()
resolv_conf(server, clients, sites)
if dhcp:
sysctl_conf(server, vms)
dhcp_conf(server, vms, sites)
logger.debug('Restarting service ...')
cmd = 'service dnsmasq stop ; rm /var/lib/misc/dnsmasq.leases ; ' + \
'service dnsmasq start',
SshProcess(cmd, server).run()
def dnsmasq_server(server, clients=None, vms=None, dhcp=True):
"""Configure a DHCP server with dnsmasq
:param server: host where the server will be installed
:param clients: list of hosts that will be declared in dnsmasq
:param vms: list of virtual machines
"""
logger.debug('Installing and configuring a DNS/DHCP server on %s', server)
test_running = Process('nmap ' + server + ' -p 53 | grep domain')
test_running.shell = True
test_running.run()
if 'open' in test_running.stdout:
logger.info('DNS server already running, updating configuration')
else:
cmd = 'killall dnsmasq; export DEBIAN_MASTER=noninteractive ; ' + \
'apt-get update ; apt-get -y purge dnsmasq-base ; ' + \
'apt-get install -t wheezy -o Dpkg::Options::="--force-confdef" ' + \
'-o Dpkg::Options::="--force-confnew" ' + \
'-y dnsmasq; echo 1 > /proc/sys/net/ipv4/ip_forward '
SshProcess(cmd, server).run()
sites = list(set([get_host_site(client) for client in clients
if get_host_site(client)] + [get_host_site(server)]))
add_vms(vms, server)
if clients:
kill_dnsmasq = TaktukRemote('killall dnsmasq', clients)
for p in kill_dnsmasq.processes:
p.ignore_exit_code = p.nolog_exit_code = True
kill_dnsmasq.run()
resolv_conf(server, clients, sites)
if dhcp:
sysctl_conf(server, vms)
dhcp_conf(server, vms, sites)
logger.debug('Restarting service ...')
cmd = 'service dnsmasq stop ; rm /var/lib/misc/dnsmasq.leases ; ' + \
'service dnsmasq start',
SshProcess(cmd, server).run()
def migrate_vm(vm, host):
""" Migrate a VM to an host """
if vm['host'] is None:
raise NameError
return None
else:
src = vm['host']
# Check that the disk is here
test_disk = TaktukRemote('ls /tmp/' + vm['id'] + '.qcow2', [host]).run()
if not test_disk.ok:
vm['host'] = host
create_disk_on_dest = create_disks([vm]).run()
if not create_disk_on_dest:
raise ActionsFailed, [create_disk_on_dest]
cmd = 'virsh --connect qemu:///system migrate ' + vm['id'] + \
' --live --copy-storage-inc qemu+ssh://' + host + "/system' "
return TaktukRemote(cmd, [src])
def start_vms(vms):
""" Return an action to start the VMs on the hosts """
hosts_cmds = {}
for vm in vms:
cmd = 'virsh --connect qemu:///system start ' + vm['id'] + ' ; '
hosts_cmds[vm['host']] = cmd if not vm['host'] in hosts_cmds \
else hosts_cmds[vm['host']] + cmd
logger.debug(pformat(hosts_cmds))
return TaktukRemote('{{hosts_cmds.values()}}', list(hosts_cmds.keys()))
def resolv_conf(server, clients, sites):
"""Generate the resolv.conf with dhcp parameters and put it on the server
"""
fd, resolv = mkstemp(dir='/tmp/', prefix='resolv_')
f = fdopen(fd, 'w')
f.write('domain grid5000.fr\nsearch grid5000.fr ' +
' '.join([site + '.grid5000.fr' for site in sites]) +
'\nnameserver ' + get_server_ip(server))
f.close()
TaktukPut(clients, [resolv], remote_location='/etc/').run()
TaktukRemote('cd /etc && cp ' + resolv.split('/')[-1] + ' resolv.conf',
clients).run()
Process('rm ' + resolv).run()
def setup_aptcacher_server(hosts, base_dir='/tmp/apt-cacher-ng'):
"""Install and configure apt-cacher on one server"""
hosts = map(Host, hosts)
logger.info('Installing apt-cacher on %s',
','.join([style.host(host.address) for host in hosts]))
logger.detail('Package')
package = TaktukRemote(
'export DEBIAN_MASTER=noninteractive ; apt-get update ; ' +
'apt-get install -o Dpkg::Options::="--force-confdef" -o ' +
'Dpkg::Options::="--force-confnew" -y apt-cacher-ng', hosts).run()
if not package.ok:
logger.error('Unable to install apt-cacher-ng on %s')
return
logger.detail('Directory creation')
log_dir = base_dir + '/log'
cache_dir = base_dir + '/cache'
mkdirs = TaktukRemote(
'mkdir -p ' + log_dir + '; mkdir -p ' + cache_dir +
'; chown -R apt-cacher-ng:apt-cacher-ng ' + base_dir, hosts).run()
if not mkdirs.ok:
logger.error('Unable to create the directories')
return
cmd = 'sed -i "s#/var/cache/apt-cacher-ng#' + cache_dir + \
'#g" /etc/apt-cacher-ng/acng.conf ;' + \
'sed -i "s#/var/log/apt-cacher-ng#' + log_dir + '#g" ' + \
'/etc/apt-cacher-ng/acng.conf ;' + \
'sed -i "s/3142/9999/g" /etc/apt-cacher-ng/acng.conf ; ' + \
'sed -i "s?#Proxy: http://www-proxy.example.net:80?Proxy: ' + \
'http://proxy:3128?g" /etc/apt-cacher-ng/acng.conf ; ' + \
'service apt-cacher-ng restart'
configure = TaktukRemote(cmd, hosts).run()
if not configure.ok:
logger.error('Unable to configure and restart the service')
return
logger.info('apt-cacher-ng up and running on %s',
','.join([style.host(host.address) for host in hosts]))
def create_disks(vms):
""" Return an action to create the disks for the VMs on the hosts"""
logger.detail(', '.join([vm['id'] for vm in sorted(vms)]))
hosts_cmds = {}
for vm in vms:
if vm['real_file']:
cmd = cmd_disk_real(vm)
else:
cmd = cmd_disk_qcow2(vm)
logger.detail(vm['id'] + ': ' + cmd)
hosts_cmds[vm['host']] = cmd if not vm['host'] in hosts_cmds \
else hosts_cmds[vm['host']] + cmd
logger.debug(pformat(hosts_cmds.values()))
return TaktukRemote('{{hosts_cmds.values()}}', list(hosts_cmds.keys()))
def list_vm(hosts, not_running=False):
""" Return the list of VMs on hosts using a disk which keys are the hosts and
value are list of VM id"""
cmd = 'virsh --connect qemu:///system list'
if not_running:
cmd += ' --all'
logger.debug('Listing Virtual machines on ' + pformat(hosts))
list_vm = TaktukRemote(cmd, hosts).run()
hosts_vms = {host: [] for host in hosts}
for p in list_vm.processes:
lines = p.stdout.split('\n')
for line in lines:
if 'running' in line or 'shut off' in line:
std = line.split()
hosts_vms[p.host.address].append({'id': std[1]})
logger.debug(pformat(hosts_vms))
return hosts_vms
def configure_apt_proxy(vms):
"""Override apt proxy-guess with server as proxy"""
hosts_vms = {}
for vm in vms:
if not vm['host'] in hosts_vms:
hosts_vms[vm['host']] = []
hosts_vms[vm['host']].append(vm['ip'])
conf = []
for server, clients in hosts_vms.iteritems():
server = Host(server)
logger.detail('Configuring %s as APT proxy for %s',
style.host(server.address), ','.join(clients))
conf.append(
TaktukRemote(
' echo \'Acquire::http::Proxy \"http://' + server.address +
':9999" ; \' > /etc/apt/apt.conf.d/proxy-guess', clients))
ParallelActions(conf).run()
def destroy_vms(hosts, undefine=False):
"""Destroy all the VM on the hosts"""
cmds = []
hosts_with_vms = []
hosts_vms = list_vm(hosts, not_running=True)
for host, vms in hosts_vms.iteritems():
if len(vms) > 0:
if not undefine:
cmds.append('; '.join('virsh destroy ' + vm['id']
for vm in vms))
else:
cmds.append('; '.join('virsh destroy ' + vm['id'] + '; '
'virsh undefine ' + vm['id']
for vm in vms))
hosts_with_vms.append(host)
if len(cmds) > 0:
TaktukRemote('{{cmds}}', hosts_with_vms).run()
def create_disks_all_hosts(vms, hosts):
"""Create a temporary file containing the vms disks creation commands
upload it and run it on the hosts"""
fd, vms_disks = tempfile.mkstemp(dir='/tmp/', prefix='vms_disks_')
f = fdopen(fd, 'w')
for vm in vms:
if vm['real_file']:
vm_cmd = cmd_disk_real(vm)
else:
vm_cmd = cmd_disk_qcow2(vm)
f.write('\n' + vm_cmd)
f.close()
return SequentialActions([
ChainPut(hosts, [vms_disks]),
TaktukRemote('sh ' + vms_disks.split('/')[-1], hosts),
Local('rm ' + vms_disks)
])
def install_vms(vms):
""" Return an action to install the VM on the hosts"""
logger.detail(', '.join([vm['id'] for vm in sorted(vms)]))
hosts_cmds = {}
for vm in vms:
cmd = 'virt-install -d --import --connect qemu:///system ' + \
'--nographics --noautoconsole --noreboot --name=' + vm['id'] + ' '\
'--network network=default,mac=' + vm['mac'] + ' --ram=' + \
str(vm['mem']) + ' --disk path=/tmp/' + vm['id'] + \
'.qcow2,device=disk,bus=virtio,format=qcow2,size=' + \
str(vm['hdd']) + ',cache=none ' + \
'--vcpus=' + str(vm['n_cpu']) + ' --cpuset=' + vm['cpuset']
if vm['tap']:
cmd += '--network tap,script=no,ifname=' + vm['tap']
cmd += ' ; '
hosts_cmds[vm['host']] = cmd if not vm['host'] in hosts_cmds \
else hosts_cmds[vm['host']] + cmd
return TaktukRemote('{{hosts_cmds.values()}}', list(hosts_cmds.keys()))
# optmodes.add_argument('--prepare-and-run',
# T)
logger.setLevel('INFO')
default_connection_params['user'] = '******'
vms = {}
f = open('vms.list')
for line in f:
ip, vm = line.strip().split('\t')
vms[vm] = ip
f.close()
logger.info('Installing cpulimit on all VMs')
apt = TaktukRemote('apt-get install -y cpulimit', vms.values()).run()
logger.info('Copying memtouch on all vms')
copy_stress = TaktukPut(vms.values(), ['memtouch-with-busyloop3']).run()
logger.info('Killing all memtouch processes')
kill_stress = TaktukRemote('killall memtouch-with-busyloop3', vms.values())
for p in kill_stress.processes:
p.shell = True
p.nolog_exit_code = ignore_exit_code = True
kill_stress.run()
logger.info('Starting memtouch process')
cmd = './memtouch-with-busyloop3 --cmd-makeload ' +\
'--cpu-speed 304408.621872 --mem-speed 63235516.087661 128 128'
stress = TaktukRemote(cmd, vms.values()).start()
vms = {}
f = open('vms.list')
for line in f:
ip, vm = line.strip().split('\t')
vms[vm] = ip
f.close()
logger.info('Installing cpulimit on all VMs')
apt = TaktukRemote('apt-get install -y cpulimit', vms.values()).run()
logger.info('Copying memtouch on all vms')
copy_stress = TaktukPut(vms.values(), ['memtouch-with-busyloop3']).run()
logger.info('Killing all memtouch processes')
kill_stress = TaktukRemote('killall memtouch-with-busyloop3', vms.values())
for p in kill_stress.processes:
p.shell = True
p.nolog_exit_code = ignore_exit_code = True
kill_stress.run()
logger.info('Starting memtouch process')
cmd = './memtouch-with-busyloop3 --cmd-makeload ' +\
'--cpu-speed 304408.621872 --mem-speed 63235516.087661 128 128'
stress = TaktukRemote(cmd, vms.values()).start()
all_started = False
while not all_started:
all_started = True
for p in stress.processes:
if not p.started:
def wait_vms_have_started(vms, restart=True):
"""Scan port 22 on all vms, distributed on hosts"""
# Creating file with list of VMs ip
fd, tmpfile = tempfile.mkstemp(prefix='vmips')
f = fdopen(fd, 'w')
for vm in vms:
f.write(vm['ip'] + '\n')
f.close()
# getting the list of host
hosts = list(set([vm['host'] for vm in vms]))
hosts.sort()
# Pushing file on all hosts
TaktukPut(hosts, [tmpfile]).run()
logger.debug(pformat(hosts))
# Splitting nmap scan
n_vm_scan = ceil(len(vms) / len(hosts)) + 1
cmds = []
for i in range(len(hosts)):
start = str(int(i * n_vm_scan))
end = str(int((i + 1) * n_vm_scan))
cmds.append("awk 'NR>=" + start + " && NR<" + end + "' " +
tmpfile.split('/')[-1] + " > nmap_file ; " +
"nmap -v -oG - -i nmap_file -p 22")
logger.debug('%s', pformat(cmds))
nmap = TaktukRemote('{{cmds}}', hosts)
nmap_tries = 0
all_up = False
started_vms = []
old_started = started_vms[:]
while (not all_up) and nmap_tries < 10:
sleep(15)
logger.detail('nmap_tries %s', nmap_tries)
nmap.run()
for p in nmap.processes:
for line in p.stdout.split('\n'):
if 'Status' in line:
split_line = line.split(' ')
ip = split_line[1]
state = split_line[3].strip()
if state == 'Up':
vm = [vm for vm in vms if vm['ip'] == ip]
if len(vm) > 0:
vm[0]['state'] = 'OK'
started_vms = [vm for vm in vms if vm['state'] == 'OK']
all_up = len(started_vms) == len(vms)
if started_vms != old_started:
old_started = started_vms
else:
if restart:
restart_vms([vm for vm in vms if vm['state'] == 'KO'])
nmap_tries += 1
if nmap_tries == 1:
activate_vms([vm for vm in vms if vm['state'] == 'KO'])
if not all_up:
logger.info(
str(nmap_tries) + ': ' + str(len(started_vms)) + '/' +
str(len(vms)))
nmap.reset()
TaktukRemote('rm ' + tmpfile.split('/')[-1], hosts).run()
Process('rm ' + tmpfile).run()
if all_up:
logger.info('All VM have been started')
return True
else:
logger.error('All VM have not been started')
return False
run_dir = sys.argv[1]
if not run_dir:
logger.error('No directory specified')
exit()
vms = []
f = open(run_dir + '/vms.list')
for line in f:
tmp = line.split()
vms.append({'id': tmp[1], 'ip': tmp[0]})
f.close()
# Measuring boot_duration
now = time.time()
get_uptime = TaktukRemote('cat /proc/uptime', [vm['ip']
for vm in vms]).run()
boot_time = {}
for p in get_uptime.processes:
boot_time[p.host.address] = now - float(p.stdout.strip().split(' ')[0])
get_ssh_up = TaktukRemote('grep listening /var/log/auth.log' + \
' |grep 0.0.0.0|awk \'{print $1" "$2" "$3}\'',
[vm['ip'] for vm in vms]).run()
boot_duration = []
for p in get_ssh_up.processes:
ssh_up = time.mktime(datetime.datetime.strptime('2014 ' + \
p.stdout.strip(), "%Y %b %d %H:%M:%S").timetuple())
boot_duration.append(ssh_up - boot_time[p.host.address])
# Calculating stats
def rm_qcow2_disks(hosts):
"""Removing qcow2 disks located in /tmp"""
logger.debug('Removing existing disks')
TaktukRemote('rm -f /tmp/*.qcow2', hosts).run()
def wait_vms_have_started(vms, restart=True):
"""Scan port 22 on all vms, distributed on hosts"""
# Creating file with list of VMs ip
fd, tmpfile = tempfile.mkstemp(prefix='vmips')
f = fdopen(fd, 'w')
for vm in vms:
f.write(vm['ip'] + '\n')
f.close()
# getting the list of host
hosts = list(set([vm['host'] for vm in vms]))
hosts.sort()
# Pushing file on all hosts
TaktukPut(hosts, [tmpfile]).run()
logger.debug(pformat(hosts))
# Splitting nmap scan
n_vm_scan = ceil(len(vms) / len(hosts)) + 1
cmds = []
for i in range(len(hosts)):
start = str(int(i * n_vm_scan))
end = str(int((i + 1) * n_vm_scan))
cmds.append("awk 'NR>=" + start + " && NR<" + end +
"' " + tmpfile.split('/')[-1] + " > nmap_file ; "
+ "nmap -v -oG - -i nmap_file -p 22")
logger.debug('%s', pformat(cmds))
nmap = TaktukRemote('{{cmds}}', hosts)
nmap_tries = 0
all_up = False
started_vms = []
old_started = started_vms[:]
while (not all_up) and nmap_tries < 10:
sleep(15)
logger.detail('nmap_tries %s', nmap_tries)
nmap.run()
for p in nmap.processes:
for line in p.stdout.split('\n'):
if 'Status' in line:
split_line = line.split(' ')
ip = split_line[1]
state = split_line[3].strip()
if state == 'Up':
vm = [vm for vm in vms if vm['ip'] == ip]
if len(vm) > 0:
vm[0]['state'] = 'OK'
started_vms = [vm for vm in vms if vm['state'] == 'OK']
all_up = len(started_vms) == len(vms)
if started_vms != old_started:
old_started = started_vms
else:
if restart:
restart_vms([vm for vm in vms if vm['state'] == 'KO'])
nmap_tries += 1
if nmap_tries == 1:
activate_vms([vm for vm in vms if vm['state'] == 'KO'])
if not all_up:
logger.info(str(nmap_tries) + ': ' + str(len(started_vms)) + '/' +
str(len(vms)))
nmap.reset()
TaktukRemote('rm ' + tmpfile.split('/')[-1], hosts).run()
Process('rm ' + tmpfile).run()
if all_up:
logger.info('All VM have been started')
return True
else:
logger.error('All VM have not been started')
return False
'n_cpu': int(_default_xml_value('n_cpu')),
'cpuset': _default_xml_value('cpuset'),
'mem': int(_default_xml_value('mem')),
'hdd': int(_default_xml_value('hdd')),
'backing_file': _default_xml_value('backing_file'),
'ip': _default_xml_value('ip'),
'mac': _default_xml_value('mac'),
'host': host.get('id')})
while True:
logger.detail('Cleaning all VMS from XML file')
for el_host in state.findall('.//host'):
for vm in el_host.findall('./vm'):
el_host.remove(vm)
logger.info('Retrieving VMS position and load')
get_vms_load = TaktukRemote("get_cpu_consumptions.sh",
hosts).run()
vms_loads = {}
hosts_vms = {host: [] for host in hosts}
for p in get_vms_load.processes:
for line in p.stdout.strip().split('\n'):
logger.detail(p.host.address)
tmp_load = line.split(' ')
logger.detail(tmp_load)
try:
vms_loads[tmp_load[0]] = float(tmp_load[1]) + float(tmp_load[2]) + float(tmp_load[-1])
except:
vms_loads[tmp_load[0]] = 0
hosts_vms[p.host.address].append(tmp_load[0])
logger.detail(hosts_vms)
for host, vms_list in hosts_vms.iteritems():