def install(self, option=''):
with api.warn_only():
if env.node['package_manager'] == 'yum':
self.result = run('rpm -q {0}'.format(self.package_name))
if not self.result.return_code == 0:
if self.path:
self.result = sudo('yum install {0} -y {1}'.format(self.path, option))
else:
self.result = sudo('yum install {0} -y {1}'.format(
self.package_name, option))
elif env.node['package_manager'] == 'apt':
self.result = run('dpkg -l {0} | grep "^ii "'.format(self.package_name))
if not self.result.return_code == 0:
if self.path:
self.result = sudo('apt-get install {0} -y {1}'.format(self.path, option))
else:
self.result = sudo('apt-get install {0} -y {1}'.format(
self.package_name, option))
else:
self.unsupport()
print self.result
if self.result.return_code != 0:
msg = 'Failed install {0}.'.format(self.package_name)
log.error(msg)
raise Exception(msg)
return self
def update_stats(self):
local_dstat_csv = os.path.join(self.stats_dir, '{0}_dstat.csv'.format(env.host))
dstat_csv = CONF._remote_stats_dir + '/dstat.csv'
local_wrk_csv = os.path.join(self.stats_dir, '{0}_wrk.csv'.format(env.host))
wrk_out = CONF._remote_stats_dir + '/wrk.out'
if filer.exists(dstat_csv):
result = run('tail -n +6 {0}'.format(dstat_csv))
with open(local_dstat_csv, 'w') as f:
f.write(str(result))
if filer.exists(wrk_out):
result = run('cat {0}'.format(wrk_out))
lines = str(result).split('\n')
options = lines[1].strip().split(' ')
threads = options[0]
connections = options[3]
t_latency = re.sub(r' +', ' ', lines[3].strip()).split(' ')[1:]
t_rps = re.sub(r' +', ' ', lines[4].strip()).split(' ')[1:]
rps = re.sub(r' +', ' ', lines[6].strip()).split(' ')[1]
tps = re.sub(r' +', ' ', lines[7].strip()).split(' ')[1]
csv = 'threads,connections,rps,tps,t_latency_avg,t_latency_stdev,t_latency_max,t_latency_stdev,t_rps_avg,t_rps_stdev,t_rps_max,t_rps_stdev\n'
csv += '{t},{c},{rps},{tps},{t_latency},{t_rps}'.format(
t=threads, c=connections, rps=rps, tps=tps, t_latency=','.join(t_latency), t_rps=','.join(t_rps)
)
with open(local_wrk_csv, 'w') as f:
f.write(csv)
def create_image(self, image_name, image_url, disk_format='qcow2', container_format='bare',
property='is_public=True'):
self.init()
result = self.cmd(
"image-list 2>/dev/null | grep ' {0} ' | awk '{{print $2}}'".format(image_name))
if len(result) > 0:
return result
image_file = '/tmp/{0}'.format(image_name)
if not filer.exists(image_file):
run('wget {0} -O {1}'.format(image_url, image_file))
create_cmd = 'image-create --name "{0}" --disk-format {1}' \
+ ' --container-format {2}' \
+ ' --property "{3}"' \
+ ' --file {4}'
result = self.cmd(create_cmd.format(
image_name, disk_format, container_format, property, image_file))
result = self.cmd(
"image-list 2>/dev/null | grep ' {0} ' | awk '{{print $2}}'".format(image_name))
if len(result) > 0:
return result
raise Exception('Failed Create Image: {0}'.format(image_name))
def setup(self):
"""
easy_install, pipをインストールします。
また、pipはパッケージインストール時にソースからコンパイルするため、
いくつかのdevelパッケージをインストールしておきます。
"""
self.init()
git.setup()
self.install_packages()
with api.warn_only():
result = run('which easy_install')
if result.return_code != 0:
sudo('sh -c "cd /tmp/ && wget https://bootstrap.pypa.io/ez_setup.py -O - | python"')
with api.warn_only():
result = run('which pip')
if result.return_code != 0:
sudo('easy_install pip')
if self.prefix != '/usr':
sudo('pip install virtualenv')
if not filer.exists(self.prefix):
sudo('virtualenv {0} --system-site-packages'.format(self.prefix))
def setup():
filer.mkdir('/tmp/fabkit')
filer.file(src='file_sample.txt', dest='/tmp/fabkit/file_sample.txt',
override=True)
filer.template(src='template_sample.txt',
dest='/tmp/fabkit/template_sample.txt', data={'msg': 'test'})
run('diff /tmp/fabkit/file_sample.txt /tmp/fabkit/template_sample.txt')
def mkdir(dest, is_local=False, owner='root:root', mode='775', use_sudo=True):
cmd_mkdir = 't={0} && mkdir -p $t'.format(dest)
if is_local or env.is_local:
cmd(cmd_mkdir)
elif use_sudo:
sudo('{0} && chmod {1} $t && chown {2} $t'.format(cmd_mkdir, mode, owner))
else:
run('{0} $t'.format(cmd_mkdir, mode, owner))
def setup():
filer.mkdir('/tmp/fabkit')
filer.file(src='file_sample.txt',
dest='/tmp/fabkit/file_sample.txt',
override=True)
filer.template(src='template_sample.txt',
dest='/tmp/fabkit/template_sample.txt',
data={'msg': 'test'})
run('diff /tmp/fabkit/file_sample.txt /tmp/fabkit/template_sample.txt')
def setup():
print '{0}: hello'.format(env.host)
print env.cluster
run('hostname')
sudo('hostname')
Package('memcached').install()
Service('memcached').start()
# run('touch /tmp/test')
filer.file('/etc/memcached.conf')
filer.template('/etc/memcached.conf')
def mkdir(dest, is_local=False, owner='root:root', mode='775', use_sudo=True):
cmd_mkdir = 't={0} && mkdir -p $t'.format(dest)
if is_local or env.is_local:
if use_sudo:
sudo("sh -c '{0} && chmod {1} $t && chown {2} $t'".format(
cmd_mkdir, mode, owner))
else:
cmd(cmd_mkdir)
elif use_sudo:
sudo('{0} && chmod {1} $t && chown {2} $t'.format(
cmd_mkdir, mode, owner))
else:
run('{0} $t'.format(cmd_mkdir, mode, owner))
def setup():
Package('python-crypto').uninstall()
Package('python-devel').install()
Package('libevent-devel').install()
Package('libxml2-devel').install()
Package('libxslt-devel').install()
with api.warn_only():
if run('which easy_install').return_code != 0:
sudo('wget https://bootstrap.pypa.io/ez_setup.py -O - | python')
if run('which pip').return_code != 0:
sudo('easy_install pip')
sudo('pip install fabric==1.10.1')
sudo('pip install pyyaml==3.11')
sudo('pip install jinja2==2.7.3')
sudo('pip install Django==1.6.5')
def add(name, group=None):
with api.warn_only():
passwd = run('cat /etc/passwd | grep ^{0}:'.format(name))
if passwd.return_code != 0:
sudo('useradd {0} -M -s/bin/false'.format(name))
if group is not None:
sudo('gpasswd -add {0} {1}'.format(name, group))
def set_os():
with api.warn_only():
result = run('cat /etc/os-release')
if result.return_code == 0:
# CentOS(Test: Ubuntu 14.10, Centos Linux 7 (Core))
re_search = re.search('PRETTY_NAME="(.+)"', result)
os = re_search.group(1)
env.node['os'] = os
if re_ubuntu16.match(os):
env.node['package_manager'] = 'apt'
env.node['service_manager'] = 'systemd'
env.node['init_manager'] = 'systemd'
elif re_ubuntu.match(os):
env.node['package_manager'] = 'apt'
env.node['service_manager'] = 'service'
env.node['init_manager'] = 'update-rc.d'
elif re_centos.match(os):
env.node['package_manager'] = 'yum'
env.node['service_manager'] = 'systemd'
env.node['init_manager'] = 'systemd'
env.path += ':/sbin'
else:
result = run('cat /etc/centos-release')
if result.return_code == 0:
# CentOS(Test: CentOS 6.5)
re_search = re.search('release ([0-9.]+) ', result)
os = 'CentOS {0}'.format(re_search.group(1))
env.node['os'] = os
env.node['package_manager'] = 'yum'
env.node['service_manager'] = 'service'
env.node['init_manager'] = 'chkconfig'
env.path += ':/sbin'
else:
if run('which yum').return_code == 0:
env.node['package_manager'] = 'yum'
env.node['service_manager'] = 'service'
env.node['init_manager'] = 'chkconfig'
env.path += ':/sbin'
run('env')
if 'os' in env.node:
return True
else:
return False
def run_remote():
remote = env.remote_map[env.host]
root, dirname = CONF._repo_dir.rsplit('/', 1)
storage_dir = os.path.join(dirname, CONF._storage_dir.rsplit('/', 1)[1])
repo_tar = '{0}/fabrepo.tar.gz'.format(CONF._tmp_dir)
remote_repo = '{0}/fabrepo'.format(CONF._remote_tmp_dir)
remote_repo_tar = '{0}/fabrepo.tar.gz'.format(CONF._remote_tmp_dir)
local('cd {0} && tar -zcf {1} {2} --exclude .git --exclude {3}'.format(
root, repo_tar, dirname, storage_dir))
scp(repo_tar, remote_repo_tar)
run('rm -rf $HOME/fabric-repo')
run('cd {0} && tar -xf fabrepo.tar.gz -C $HOME'.format(CONF._remote_tmp_dir, remote_repo))
task_name = ''
if env.is_setup:
task_name = 'setup'
elif env.is_check:
task_name = 'check'
elif env.is_manage:
task_name = 'manage:{0}'.format(','.join(env.func_names))
cluster_map = {}
with api.shell_env(password=env.password):
for cluster in remote['clusters']:
run('cd $HOME/{0} && fab node:{1}/{2},yes {3} -u $USER -p $password'.format(
dirname, cluster, remote['host_pattern'], task_name, env.password))
with api.warn_only():
yaml_str = run('cat $HOME/{0}/nodes/{1}/__cluster.yml'.format(dirname, cluster))
cluster_map[cluster] = yaml.load(yaml_str)
return cluster_map
def set_os():
with api.warn_only():
result = run('cat /etc/os-release')
if result.return_code == 0:
# CentOS(Test: Ubuntu 14.10, Centos Linux 7 (Core))
re_search = re.search('PRETTY_NAME="(.+)"', result)
os = re_search.group(1)
env.node['os'] = os
if re_ubuntu16.match(os):
env.node['package_manager'] = 'apt'
env.node['service_manager'] = 'systemd'
env.node['init_manager'] = 'systemd'
elif re_ubuntu.match(os):
env.node['package_manager'] = 'apt'
env.node['service_manager'] = 'service'
env.node['init_manager'] = 'update-rc.d'
elif re_centos.match(os):
env.node['package_manager'] = 'yum'
env.node['service_manager'] = 'systemd'
env.node['init_manager'] = 'systemd'
env.path += ':/sbin'
else:
result = run('cat /etc/centos-release')
if result.return_code == 0:
# CentOS(Test: CentOS 6.5)
re_search = re.search('release ([0-9.]+) ', result)
os = 'CentOS {0}'.format(re_search.group(1))
env.node['os'] = os
env.node['package_manager'] = 'yum'
env.node['service_manager'] = 'service'
env.node['init_manager'] = 'chkconfig'
env.path += ':/sbin'
else:
if run('which yum').return_code == 0:
env.node['package_manager'] = 'yum'
env.node['service_manager'] = 'service'
env.node['init_manager'] = 'chkconfig'
env.path += ':/sbin'
run('env')
if 'os' in env.node:
return True
else:
return False
def oscmd(cmd):
keystone = env.cluster['keystone']
with api.shell_env(
OS_USERNAME='******',
OS_PASSWORD=keystone['admin_password'],
OS_TENANT_NAME='admin',
OS_AUTH_URL=keystone['services']['keystone']['adminurl'],
):
return run(cmd)
def create(self):
data = self.data
sudo('modprobe kvm')
sudo('modprobe kvm_intel')
for i, vm in enumerate(data['libvirt_vms']):
instance_dir = os.path.join(self.instances_dir, vm['name'])
filer.mkdir(instance_dir)
image_path = '{0}/vm.img'.format(instance_dir)
vm['image_path'] = image_path
src_image_path = self.wget_src_image(vm)
if not filer.exists(image_path):
sudo('cp {0} {1}'.format(src_image_path, image_path))
sudo('qemu-img resize {0} {1}G'.format(image_path, vm.get('disk_size', 10)))
configiso_path = self.create_configiso(vm, instance_dir)
vm['configiso_path'] = configiso_path
vm['mac'] = self.get_random_mac()
domain_xml = self.create_domain_xml(vm, instance_dir)
sudo("sed -i 's/^Defaults.*requiretty/# Defaults requiretty/' /etc/sudoers")
sudo("virsh net-update default add ip-dhcp-host "
"\"<host mac='{0}' name='{1}' ip='{2}' />\"".format(
vm['mac'], vm['name'], vm['ip']))
sudo('virsh define {0}'.format(domain_xml))
sudo('chown -R root:root {0}'.format(instance_dir))
sudo('virsh start {0}'.format(vm['name']))
for vm in data['libvirt_vms']:
while True:
with api.warn_only():
if run('nmap -p 22 {0} | grep open'.format(vm['ip'])):
break
time.sleep(5)
sudo("iptables -R FORWARD 1 -o virbr0 -s 0.0.0.0/0"
" -d 192.168.122.0/255.255.255.0 -j ACCEPT")
for vm in data['libvirt_vms']:
for port in vm.get('ports', []):
sudo("iptables -t nat -A PREROUTING -p tcp"
" --dport {0[1]} -j DNAT --to {1}:{0[0]}".format(
port, vm['ip']))
for ip in data['iptables']:
for port in ip.get('ports', []):
sudo("iptables -t nat -A PREROUTING -p tcp"
" --dport {0[1]} -j DNAT --to {1}:{0[0]}".format(
port, ip['ip']))
time.sleep(5)
def install(self, option=''):
with api.warn_only():
if env.node['package_manager'] == 'yum':
self.result = run('rpm -q {0}'.format(self.package_name))
if not self.result.return_code == 0:
if self.path:
self.result = sudo('yum install {0} -y {1}'.format(
self.path, option))
else:
self.result = sudo('yum install {0} -y {1}'.format(
self.package_name, option))
elif env.node['package_manager'] == 'apt':
splited_name = self.package_name.split('=')
if len(splited_name) > 1:
package_name = splited_name[0]
version = splited_name[1]
else:
package_name = self.package_name
version = ''
self.result = run('dpkg -l {0} | grep "^ii *{1}"'.format(
package_name, version))
if not self.result.return_code == 0:
if self.path:
self.result = sudo(
'RUNLEVEL=1 apt-get install {0} -y {1}'.format(
self.path, option))
else:
self.result = sudo(
'RUNLEVEL=1 apt-get install {0} -y {1}'.format(
self.package_name, option))
else:
self.unsupport()
if self.result.return_code != 0:
msg = 'Failed install {0}.'.format(self.package_name)
log.error(msg)
raise Exception(msg)
return self
def cmd(self, cmd, use_admin_token=True):
self.init()
# create users, roles, services
if use_admin_token:
endpoint = '{0}/v2.0'.format(self.data['admin_endpoint'])
with api.shell_env(
OS_SERVICE_TOKEN=self.data['admin_token'],
OS_SERVICE_ENDPOINT=endpoint,
OS_TOKEN=self.data['admin_token'],
OS_URL=endpoint,
):
return run('openstack {0}'.format(cmd))
else:
return utils.oscmd('openstack {0}'.format(cmd))
def sync(url, branch='master', dest=None, user=None):
if not dest:
name = url.rsplit('/', 1)[1]
dest = os.path.join(tmp_dest, name)
if not filer.exists(dest):
with api.cd('/tmp'):
expect(
'git clone -b {0} {1} {2}'.format(branch, url, dest),
expects=[
[
'Are you sure you want to continue connecting (yes/no)?',
'yes\\n'
],
],
user=user)
else:
if user is not None:
sudo('cd {0} && git pull'.format(dest), user=user)
else:
run('cd {0} && git pull'.format(dest))
return dest
def install(self, option=''):
with api.warn_only():
if env.node['package_manager'] == 'yum':
self.result = run('rpm -q {0}'.format(self.package_name))
if not self.result.return_code == 0:
if self.path:
self.result = sudo('yum install {0} -y {1}'.format(self.path, option))
else:
self.result = sudo('yum install {0} -y {1}'.format(
self.package_name, option))
elif env.node['package_manager'] == 'apt':
splited_name = self.package_name.split('=')
if len(splited_name) > 1:
package_name = splited_name[0]
version = splited_name[1]
else:
package_name = self.package_name
version = ''
self.result = run('dpkg -l {0} | grep "^ii *{1}"'.format(package_name, version))
if not self.result.return_code == 0:
if self.path:
self.result = sudo('RUNLEVEL=1 apt-get install {0} -y {1}'.format(
self.path, option))
else:
self.result = sudo('RUNLEVEL=1 apt-get install {0} -y {1}'.format(
self.package_name, option))
else:
self.unsupport()
if self.result.return_code != 0:
msg = 'Failed install {0}.'.format(self.package_name)
log.error(msg)
raise Exception(msg)
return self
def setup(self):
data = self.init()
run('hostname')
self.install_packages()
filer.template('/tmp/pdns.sql')
sudo('mysql -uroot pdns -e "show tables;" | grep domain || '
'mysql -uroot pdns < /tmp/pdns.sql')
if env.host in data['master']['hosts']:
data['is_master'] = True
if env.host in data['slave']['hosts']:
data['is_slave'] = True
filer.template('/etc/pdns/pdns.conf', data=data)
self.start_services().enable_services()
if not filer.exists('/var/www/html/poweradmin'):
sudo('cd /tmp/ && '
'wget http://downloads.sourceforge.net/project/poweradmin/poweradmin-2.1.7.tgz && '
'tar xf poweradmin-2.1.7.tgz && '
'mv poweradmin-2.1.7 /var/www/html/poweradmin && '
'chown -R apache:apache /var/www/html/poweradmin')
def set_ip():
with api.warn_only():
result = run('ip r')
if result.return_code == 0:
devs = re.findall(
'([0-9./]+) +dev +([a-zA-Z0-9\-]+) +proto +kernel +scope +link +src +([0-9.]+)',
result)
default = re.findall(
'default +via +([0-9.]+) +dev +([a-zA-Z0-9\-]+)', result)
dev = default[0][1].split(' ')[0]
ips = {
'default': {
'ip': default[0][0],
'dev': dev,
}
}
for dev in devs:
ip_data = {
'subnet': dev[0],
'dev': dev[1],
'ip': dev[2],
}
ips[dev[0]] = ip_data
ips[dev[1]] = ip_data
ips[dev[0].split('.')[0]] = ip_data
ips['default_dev'] = ips[ips['default']['dev']]
env.node['ip'] = ips
return True
else:
return False
def set_ip():
with api.warn_only():
result = run('ip r')
if result.return_code == 0:
devs = re.findall(
'([0-9./]+) +dev +([a-zA-Z0-9\-]+) +proto +kernel +scope +link +src +([0-9.]+)',
result)
default = re.findall(
'default +via +([0-9.]+) +dev +([a-zA-Z0-9\-]+)', result)
dev = default[0][1].split(' ')[0]
ips = {
'default': {
'ip': default[0][0],
'dev': dev,
}
}
for dev in devs:
ip_data = {
'subnet': dev[0],
'dev': dev[1],
'ip': dev[2],
}
ips[dev[0]] = ip_data
ips[dev[1]] = ip_data
ips[dev[0].split('.')[0]] = ip_data
ips['default_dev'] = ips[ips['default']['dev']]
env.node['ip'] = ips
return True
else:
return False
def pip_show(self, package_name):
"""
pip show [package_name] の結果をパースして、タプル形式で返します。
"""
with api.warn_only():
result = run('pip show {0}'.format(package_name))
if result == '':
return None
RE_NAME = re.compile('Name: (.+)\r')
RE_VERSION = re.compile('Version: (.+)\r')
finded_name = RE_NAME.findall(result)
if len(finded_name) == 0:
return None
name = finded_name[0]
finded_version = RE_VERSION.findall(result)
if len(finded_version) == 0:
return None
version = finded_version[0]
return (name, version)
def setup(self):
data = self.init()
node_os = env.node['os']
if re.match('Ubuntu 14.*', node_os):
with api.warn_only():
sudo('apt-get update -y')
self.install_packages()
self.start_services()
sudo('modprobe kvm')
sudo('modprobe kvm_intel')
for i, vm in enumerate(data['libvirt_vms']):
template_data = {
'user': CONF.test.user,
'password': CONF.test.password,
'vm': vm,
'gateway': data['libvirt']['gateway'],
'netmask': data['libvirt']['netmask'],
}
vm_dir = '/var/lib/libvirt/images/{0}'.format(vm['name'])
image_path = '{0}/vm.img'.format(vm_dir)
metadata_path = '{0}/meta-data'.format(vm_dir)
userdata_path = '{0}/user-data'.format(vm_dir)
configiso_path = '{0}/config.iso'.format(vm_dir)
src_image = vm['src_image'].rsplit('/', 1)[1]
src_image_path = '/var/lib/libvirt/images/{0}'.format(src_image)
src_image_format = 'qcow2'
if src_image_path[-3:] == '.xz':
src_image_path = src_image_path[:-3]
src_image_format = 'xz'
if not filer.exists(src_image_path):
sudo('cd /var/lib/libvirt/images/ && wget {0}'.format(vm['src_image']))
if src_image_format == 'xz':
sudo('cd /var/lib/libvirt/images/ && xz -d {0}'.format(src_image))
with api.warn_only():
sudo("virsh list --all | grep {0} && virsh destroy {0}"
" && virsh undefine {0}".format(vm['name']))
sudo('rm -rf {0}'.format(vm_dir))
filer.mkdir(vm_dir)
if not filer.exists(image_path):
sudo('cp {0} {1}'.format(src_image_path, image_path))
sudo('qemu-img resize {0} {1}G'.format(image_path, vm.get('disk_size', 10)))
filer.template(metadata_path, src='meta-data', data=template_data)
filer.template(userdata_path, src=vm['template'], data=template_data)
if not filer.exists(configiso_path):
sudo('genisoimage -o {0} -V cidata -r -J {1} {2}'.format(
configiso_path, metadata_path, userdata_path))
sudo("sed -i 's/^Defaults.*requiretty/# Defaults requiretty/' /etc/sudoers")
vm['uuid'] = str(uuid.uuid1())
vm['image_path'] = image_path
vm['configiso_path'] = configiso_path
vm['tap'] = 'tap{0}'.format(i)
vm['mac'] = self.get_random_mac()
domain_xml = '/tmp/domain-{0}.xml'.format(vm['name'])
filer.template(domain_xml, src='domain.xml', data=vm)
with api.warn_only():
sudo("virsh net-update default delete ip-dhcp-host \"`virsh net-dumpxml default | grep '{0}' | sed -e 's/^ *//'`\"".format(vm['ip']))
sudo("virsh net-update default add ip-dhcp-host "
"\"<host mac='{0}' name='{1}' ip='{2}' />\"".format(
vm['mac'], vm['name'], vm['ip']))
sudo('virsh define {0}'.format(domain_xml))
sudo('virsh start {0}'.format(vm['name']))
# sudo("virt-install"
# " --connect=qemu:///system"
# " --name={name} --vcpus={vcpus} --ram={ram}"
# " --accelerate --hvm --virt-type=kvm"
# " --cpu host"
# " --network bridge=virbr0,model=virtio"
# " --disk {image_path},format=qcow2 --import"
# " --disk {configiso_path},device=cdrom"
# " --nographics &".format(
# name=vm['name'],
# vcpus=vm['vcpus'],
# ram=vm['ram'],
# image_path=image_path,
# configiso_path=configiso_path,
# ip=vm['ip'],
# ), pty=False) # ), pty=False)
for vm in data['libvirt_vms']:
while True:
with api.warn_only():
if run('nmap -p 22 {0} | grep open'.format(vm['ip'])):
break
time.sleep(5)
sudo("iptables -R FORWARD 1 -o virbr0 -s 0.0.0.0/0"
" -d 192.168.122.0/255.255.255.0 -j ACCEPT")
for vm in data['libvirt_vms']:
for port in vm.get('ports', []):
sudo("iptables -t nat -A PREROUTING -p tcp"
" --dport {0[1]} -j DNAT --to {1}:{0[0]}".format(
port, vm['ip']))
for ip in data['iptables']:
for port in ip.get('ports', []):
sudo("iptables -t nat -A PREROUTING -p tcp"
" --dport {0[1]} -j DNAT --to {1}:{0[0]}".format(
port, ip['ip']))
def setup_network_bridge(self):
data = self.init()
if 'linuxbridge' in data['ml2']['mechanism_drivers']:
sudo('modprobe bridge')
elif 'openvswitch' in data['ml2']['mechanism_drivers']:
sudo('modprobe -r bridge')
Service('openvswitch').start().enable()
sudo('ovs-vsctl br-exists {0} || ovs-vsctl add-br {0}'.format(
data['ovs']['integration_bridge']))
filer.template(
'/etc/sysconfig/network-scripts/ifcfg-{0}'.format(
data['ovs']['integration_bridge']),
src='network/ovs-ifcfg-br.j2',
data=data)
for mapping in data['ovs']['bridge_mappings']:
pair = mapping.split(':')
ovs_interface = pair[1]
sudo('ovs-vsctl br-exists {0} || ovs-vsctl add-br {0}'.format(ovs_interface))
for mapping in data['ovs']['physical_interface_mappings']:
pair = mapping.split(':')
ovs_interface = pair[0]
physical_interface = pair[1]
backup_default_dev_file = '/etc/sysconfig/network-scripts/bk-ifcfg-defualt'
if filer.exists(backup_default_dev_file):
default = run('cat {0}'.format(backup_default_dev_file))
dev, ip, subnet, gateway = default.split(':')
data['default_dev'] = {
'dev': dev,
'ip': ip,
'subnet': subnet,
'gateway': gateway,
}
else:
sudo("echo '{0[dev]}:{0[ip]}:{0[subnet]}:{1}' > {2}".format(
env.node['ip']['default_dev'],
env.node['ip']['default']['ip'],
backup_default_dev_file))
data['default_dev'] = env.node['ip']['default_dev']
data['default_dev']['gateway'] = env.node['ip']['default']['ip']
data['default_dev']['netmask'] = self.cidr(
data['default_dev']['subnet'].split('/')[1])
if physical_interface == data['default_dev']['dev']:
# create backup for default interface
data['ovs_interface'] = ovs_interface
filer.template(
'/etc/sysconfig/network-scripts/ifcfg-{0}'.format(physical_interface),
src='network/ovs-ifcfg-flat.j2',
data=data)
filer.template(
'/etc/sysconfig/network-scripts/ifcfg-{0}'.format(ovs_interface),
src='network/ovs-ifcfg-br-flat.j2',
data=data)
result = sudo('ovs-vsctl list-ports {0}'.format(ovs_interface))
if result.find(data['default_dev']['dev']) == -1:
with api.warn_only():
api.reboot(180)
def get_site_packages(self):
return run('{0}/bin/python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()"'.format(self.prefix)) # noqa
def setup(self):
data = self.init()
if self.is_tag('package'):
self.python.setup()
self.install_packages()
self.python.setup_package(**self.package)
sudo('modprobe tun') # for vhost_net
self.setup_network_bridge()
if self.is_tag('conf'):
filer.template(
'/etc/sudoers.d/neutron',
data=data,
src='sudoers.j2',
)
if filer.template(
'/etc/neutron/neutron.conf',
src='{0}/neutron.conf.j2'.format(data['version']),
data=data,
):
self.handlers['restart_neutron-*'] = True
if filer.template(
'/etc/neutron/plugins/ml2/ml2_conf.ini',
src='{0}/ml2_conf.ini.j2'.format(data['version']),
data=data,
):
self.handlers['restart_neutron-*'] = True
if self.data['version'] == 'kilo':
linuxbridge_conf = '/etc/neutron/plugins/linuxbridge/linuxbridge_conf.ini'
elif self.data['version'] in ['liberty', 'mitaka', 'master']:
linuxbridge_conf = '/etc/neutron/plugins/ml2/linuxbridge_agent.ini'
if filer.template(
linuxbridge_conf,
src='{0}/linuxbridge_conf.ini.j2'.format(data['version']),
data=data,
):
self.handlers['restart_neutron-*'] = True
if self.data['version'] == 'kilo':
ovs_conf = '/etc/neutron/plugins/openvswitch/ovs_neutron_plugin.ini'
elif self.data['version'] in ['liberty', 'mitaka', 'master']:
ovs_conf = '/etc/neutron/plugins/ml2/openvswitch_agent.ini'
if 'openvswitch' in self.data['ml2']['mechanism_drivers']:
if filer.template(
ovs_conf,
src='{0}/ovs_neutron_plugin.ini.j2'.format(data['version']),
data=data,
):
self.handlers['restart_neutron-*'] = True
if filer.template(
'/etc/neutron/l3_agent.ini',
src='{0}/l3_agent.ini.j2'.format(data['version']),
data=data,
):
self.handlers['restart_neutron-*'] = True
else:
sudo('echo '' > {0}'.format(ovs_conf))
if filer.template(
'/etc/neutron/dhcp_agent.ini',
src='{0}/dhcp_agent.ini.j2'.format(data['version']),
data=data,
):
self.handlers['restart_neutron-*'] = True
if filer.template(
'/etc/neutron/metadata_agent.ini',
src='{0}/metadata_agent.ini.j2'.format(data['version']),
data=data,
):
self.handlers['restart_neutron-*'] = True
# lbaas
if filer.template(
'/etc/neutron/services_lbaas.conf',
src='{0}/services_lbaas.conf.j2'.format(data['version']),
data=data,
):
self.handlers['restart_neutron-*'] = True
if filer.template(
'/etc/neutron/neutron_lbaas.conf',
src='{0}/neutron_lbaas.conf.j2'.format(data['version']),
data=data,
):
self.handlers['restart_neutron-*'] = True
if filer.template(
'/etc/neutron/lbaas_agent.ini',
src='{0}/lbaas_agent.ini.j2'.format(data['version']),
data=data,
):
self.handlers['restart_neutron-*'] = True
if self.is_tag('data') and env.host == env.hosts[0]:
if data['is_master']:
option = '--config-file /etc/neutron/neutron.conf'
run('{0}/bin/neutron-db-manage {1} upgrade head'.format(self.prefix, option))
run('{0}/bin/neutron-db-manage {1} --service lbaas upgrade head'.format(
self.prefix, option))
if self.is_tag('service'):
self.enable_services().start_services(pty=False)
self.exec_handlers()
if self.is_tag('data') and env.host == env.hosts[0]:
if data['is_master']:
time.sleep(5)
self.create_nets()
self.create_routers()
return 0
def setup(self):
run('hostname')
def setup():
run('echo "hello world" > ~/hello')
return {'status': 0, 'msg': 'success hello world'}
def touch():
run('touch ~/hello')
def check():
if run('cat ~/hello').find('hello world') == -1:
return {'status': 100, 'msg': 'not found'}
else:
return {'status': 0, 'msg': 'success check hello world'}
def setup():
run('cd ~/ && git clone [email protected]:syunkitada/home.git')
def setup(self):
self.install_packages()
self.start_services().enable_services()
repo = '/home/{0}/fabkit-repo'.format(env.user)
filer.mkdir(repo, use_sudo=False)
git.setup()
git.sync('https://github.com/fabrickit/fabkit.git',
dest='{0}/fabfile'.format(repo))
python = Python('/opt/fabkit')
python.setup()
python.install(
requirements='{0}/fabfile/requirements.txt'.format(repo))
run('cd {0} && /opt/fabkit/bin/fab genconfig:fabfile.ini &&'
' sed -i "/^\[web\]/,/^\[/s/#hostname =.*/hostname = */g" fabfile.ini'
.format(repo))
data = {
'port': 80,
'repo': repo,
'user': env.user,
'group': env.user,
'python_path': python.get_site_packages(),
'processes': 5,
'threads': 1,
}
run('cd {0}/fabfile/core/webapp/ &&'
' /opt/fabkit/bin/python manage.py migrate &&'
' echo "from django.contrib.auth.models import User;'
' User.objects.create_superuser(\'admin\', \'admin@localhost\', \'admin\')"'
' | /opt/fabkit/bin/python manage.py shell &&'
' /opt/fabkit/bin/python manage.py collectstatic --noinput'.format(
repo))
sudo('chmod 755 /home/{0}'.format(env.user))
if re.match('CentOS .*', env.node['os']):
log_prefix = '/var/log/httpd/{0}'.format(env.user)
data['error_log'] = '{0}-error.log'.format(log_prefix)
data['custom_log'] = '{0}-access.log'.format(log_prefix)
if filer.template(src='httpd.conf',
dest='/etc/httpd/conf.d/{0}_httpd.conf'.format(
env.user),
data=data):
self.handlers['restart_httpd'] = True
elif re.match('Ubuntu .*', env.node['os']):
log_prefix = '/var/log/apache2/{0}'.format(env.user)
data['error_log'] = '{0}-error.log'.format(log_prefix)
data['custom_log'] = '{0}-access.log'.format(log_prefix)
if filer.template(
src='httpd.conf',
dest='/etc/apache2/sites-enabled/{0}_httpd.conf'.format(
env.user),
data=data):
self.handlers['restart_apache2'] = True
self.exec_handlers()
# install
sudo('npm install -g coffee-script'.format(repo))
sudo('cd {0}/fabfile/core/webapp && npm install'.format(repo))
# install node packages for develop
sudo('npm install -g grunt-cli'.format(repo))
sudo(
'cd {0}/fabfile/core/webapp/node_chat && npm install'.format(repo))
def setup(self):
self.install_packages()
self.start_services().enable_services()
repo = '/home/{0}/fabkit-repo'.format(env.user)
filer.mkdir(repo, use_sudo=False)
git.setup()
git.sync('https://github.com/fabrickit/fabkit.git',
dest='{0}/fabfile'.format(repo))
python = Python('/opt/fabkit')
python.setup()
python.install(
requirements='{0}/fabfile/requirements.txt'.format(repo))
run('cd {0} && /opt/fabkit/bin/fab genconfig:fabfile.ini &&'
' sed -i "/^\[web\]/,/^\[/s/#hostname =.*/hostname = */g" fabfile.ini'.format(repo))
data = {
'port': 80,
'repo': repo,
'user': env.user,
'group': env.user,
'python_path': python.get_site_packages(),
'processes': 5,
'threads': 1,
}
run('cd {0}/fabfile/core/webapp/ &&'
' /opt/fabkit/bin/python manage.py migrate &&'
' echo "from django.contrib.auth.models import User;'
' User.objects.create_superuser(\'admin\', \'admin@localhost\', \'admin\')"'
' | /opt/fabkit/bin/python manage.py shell &&'
' /opt/fabkit/bin/python manage.py collectstatic --noinput'.format(repo))
sudo('chmod 755 /home/{0}'.format(env.user))
if re.match('CentOS .*', env.node['os']):
log_prefix = '/var/log/httpd/{0}'.format(env.user)
data['error_log'] = '{0}-error.log'.format(log_prefix)
data['custom_log'] = '{0}-access.log'.format(log_prefix)
if filer.template(src='httpd.conf',
dest='/etc/httpd/conf.d/{0}_httpd.conf'.format(env.user),
data=data):
self.handlers['restart_httpd'] = True
elif re.match('Ubuntu .*', env.node['os']):
log_prefix = '/var/log/apache2/{0}'.format(env.user)
data['error_log'] = '{0}-error.log'.format(log_prefix)
data['custom_log'] = '{0}-access.log'.format(log_prefix)
if filer.template(src='httpd.conf',
dest='/etc/apache2/sites-enabled/{0}_httpd.conf'.format(env.user),
data=data):
self.handlers['restart_apache2'] = True
self.exec_handlers()
# install
sudo('npm install -g coffee-script'.format(repo))
sudo('cd {0}/fabfile/core/webapp && npm install'.format(repo))
# install node packages for develop
sudo('npm install -g grunt-cli'.format(repo))
sudo('cd {0}/fabfile/core/webapp/node_chat && npm install'.format(repo))
def create(self):
data = self.data
sudo('modprobe kvm')
sudo('modprobe kvm_intel')
network = CONF.network.libvirt_net.split(':')
bridge = network[0]
brctl_show = sudo('brctl show')
if brctl_show.find(bridge) == -1:
sudo('brctl addbr {0}'.format(bridge))
ip_network = IPNetwork(network[1])
gateway_ip = '{0}/{1}'.format(ip_network.ip + 1, ip_network.prefixlen)
dhcp_ip = '{0}/{1}'.format(ip_network.ip + 2, ip_network.prefixlen)
bridge_info = sudo('ip addr show dev {0}'.format(bridge))
if bridge_info.find(gateway_ip) == -1:
sudo('ip addr add {0} dev {1}'.format(gateway_ip, bridge))
if bridge_info.find('DOWN') != -1:
sudo('ip link set {0} up'.format(bridge))
sudo('ip route add 10.0.0.0/8 via {0}'.format(ip_network.ip + 1))
network_seg = "{0}/{1}".format(ip_network.ip, ip_network.netmask)
ip_netns = sudo('ip netns show')
dhcp_netns = 'dhcp-{0}'.format(bridge)
dhcp_veth_br = 'ns-{0}'.format(bridge)
dhcp_veth = 'veth-{0}'.format(bridge)
if ip_netns.find(dhcp_netns):
sudo('ip netns add {0}'.format(dhcp_netns))
if brctl_show.find(dhcp_veth_br) == -1:
sudo('ip link add {0} type veth peer name {1}'.format(dhcp_veth_br, dhcp_veth))
sudo('brctl addif {0} {1}'.format(bridge, dhcp_veth_br))
sudo('ip link set {0} up'.format(dhcp_veth_br))
sudo('ip link set {0} netns {1}'.format(dhcp_veth, dhcp_netns))
sudo('ip netns exec {0} ip addr add dev {1} {2}'.format(dhcp_netns, dhcp_veth, dhcp_ip))
sudo('ip netns exec {0} ip link set {1} up'.format(dhcp_netns, dhcp_veth))
# ss_ln = sudo('ip netns exec {0} ss -ln'.format(dhcp_netns))
# if ss_ln.find('*:67') == -1:
# sudo('ip netns exec {0} dnsmasq -p 0 --dhcp-range 172.16.100.3,172.16.100.254,12h'.format( # noqa
# dhcp_netns, ip_network[3], ip_network[-2]))
ss_ln = sudo('ss -ln'.format(dhcp_netns))
if ss_ln.find('*:67') == -1:
sudo('dnsmasq -p 0 --dhcp-range=172.16.100.3,172.16.100.254')
for i, vm in enumerate(data['libvirt_vms']):
instance_dir = os.path.join(self.instances_dir, vm['name'])
filer.mkdir(instance_dir)
vm['bridge'] = bridge
vm['hostname'] = '{0}.{1}'.format(vm['name'], CONF.network.domain)
image_path = '{0}/vm.img'.format(instance_dir)
vm['image_path'] = image_path
src_image_path = self.wget_src_image(vm)
if not filer.exists(image_path):
sudo('cp {0} {1}'.format(src_image_path, image_path))
sudo('qemu-img resize {0} {1}G'.format(image_path, vm.get('disk_size', 10)))
if 'disk_cache' not in vm:
vm['disk_cache'] = 'none'
elif vm['disk_cache'] not in ['none', 'writethrough', 'writeback',
'directsync', 'unsafe', 'default']:
raise Exception('Invalid disk_cache: {0}'.format(vm['disk_cache']))
configiso_path = self.create_configiso(vm, instance_dir)
vm['configiso_path'] = configiso_path
alias_index = 0
pci_slot_num = 2
for port in vm['ports']:
mac = self.get_random_mac()
port['mac'] = mac
port['tap'] = 'tap{0}'.format(mac.replace(':', ''))
port['pci_slot'] = '0x0{0}'.format(pci_slot_num)
pci_slot_num += 1
port['alias_name'] = 'net{0}'.format(alias_index)
alias_index += 1
vm['memballoon'] = {
'pci_slot': '0x0{0}'.format(pci_slot_num)
}
domain_xml = self.create_domain_xml(vm, instance_dir)
sudo("sed -i 's/^Defaults.*requiretty/# Defaults requiretty/' /etc/sudoers")
for port in vm['ports']:
if port['ip'] == 'none':
continue
# sudo("virsh net-update {3} add ip-dhcp-host "
# "\"<host mac='{0}' name='{1}' ip='{2}' />\"".format(
# port['mac'], vm['name'], port['ip'], bridge))
sudo('virsh define {0}'.format(domain_xml))
sudo('chown -R root:root {0}'.format(instance_dir))
sudo('virsh start {0}'.format(vm['name']))
nat_table = sudo("iptables -t nat -L")
if nat_table.find(network_seg) == -1:
# sudo("iptables -R FORWARD 1 -o {0} -s {1}"
# " -d 0.0.0.0/0 -j ACCEPT".format(bridge, network_seg))
sudo("iptables -t filter -A FORWARD -s 0.0.0.0/0 -d {0} -j ACCEPT".format(network_seg))
sudo("iptables -t filter -A FORWARD -d 0.0.0.0/0 -s {0} -j ACCEPT".format(network_seg))
nat_table = sudo("iptables -t nat -L")
if nat_table.find(network_seg) == -1:
sudo("iptables -t nat -A POSTROUTING -p TCP -s {0} ! -d {0} -j MASQUERADE --to-ports 1024-65535".format(
network_seg))
sudo("iptables -t nat -A POSTROUTING -p UDP -s {0} ! -d {0} -j MASQUERADE --to-ports 1024-65535".format(
network_seg))
sudo("iptables -t nat -A POSTROUTING -s {0} ! -d {0} -j MASQUERADE".format(
network_seg))
sudo("iptables -t nat -A POSTROUTING -s {0} -d 255.255.255.255 -j RETURN".format(
network_seg))
sudo("iptables -t nat -A POSTROUTING -s {0} -d base-address.mcast.net/24 -j RETURN".format(
network_seg))
for vm in data['libvirt_vms']:
self.pdns.create_record(vm['name'], CONF.network.domain, 'A', vm['ports'][0]['ip'])
while True:
with api.warn_only():
if run('nmap -p 22 {0} | grep open'.format(vm['ports'][0]['ip'])):
break
time.sleep(5)
for ip in data.get('iptables', {}):
for port in ip.get('ports', []):
if ip['ip'] == 'none':
continue
sudo("iptables -t nat -A PREROUTING -p tcp"
" --dport {0[1]} -j DNAT --to {1}:{0[0]}".format(
port, ip['ip']))
time.sleep(60)