def do_rabbit_addusers(cname):
self = facility.get_component(cname)
pwd = cmd_quote(util.get_keymgr()(self.name, 'openstack'))
localsh.run("""rabbitmqctl add_user openstack {passwd} ||
rabbitmqctl change_password openstack {passwd} &&
rabbitmqctl set_permissions -p / openstack ".*" ".*" ".*"
""".format(passwd=pwd))
def create_empty_disk(dst, size, fmt='qcow2'):
if fmt == 'raw':
localsh.run("truncate -s {size} {dst}".format(
fmt=fmt, dst=dst))
else:
localsh.run("qemu-img create -f {fmt} '{dst}' '{size}'".format(
fmt=fmt, dst=dst, size=size))
def do_selinux():
localsh.run("""
setenforce 0 # please report the detected issues!
setsebool -P httpd_can_network_connect on
setsebool -P httpd_use_openstack on
setsebool -P haproxy_connect_any=1
""")
def handle_schema(self, schema, user, passwd, pre_sync_script_dir=None):
# BUG? two grant some cases makes mariadb not authentice non 'localhost'
# users until restart , flush privileges does not helps
# GRANT ALL PRIVILEGES ON {schema}.* TO '{user}'@'localhost' \
# IDENTIFIED BY '{passwd}';
sql = r"""CREATE SCHEMA IF NOT EXISTS {schema};
GRANT ALL PRIVILEGES ON {schema}.* TO '{user}'@'%' \
IDENTIFIED BY '{passwd}';
SELECT IF(count(*) = 0, CONCAT('FREE','_FOR','_ALL'), 'FULL')
FROM INFORMATION_SCHEMA.TABLES WHERE table_schema='{schema}';""".format(
schema=schema, user=user,
# $ for shell, the others for mysql
passwd=passwd.replace('\\', '\\\\').replace("'", r"\'").replace('$', r'\$')
)
retry = 1024 # wating for mariadb become ready
while True:
try:
if pre_sync_script_dir: # NOT TESTED
script = ("if mysql -u root <<EOF\n | grep FREE_FOR_ALL &&"
" [ -f {dir}/{schema}.sql] then\n{sql}\nEOF\n"
"mysql -u root <{dir}/{schema}.sql; fi".format(
dir=pre_sync_script_dir, schema=schema))
else:
script = 'mysql -u root <<EOF\n{sql}\nEOF\n'.format(
sql=sql)
break
except util.NonZeroExitCode:
if retry:
time.sleep(0.2)
retry -= 1
else:
raise
# the merged version was too confusing to debug
localsh.run(script)
def do_swift_service_start(cname):
self = facility.get_component(cname)
tasks.local_os_service_start_by_component(self)
# NOTE: other service will be started implictly
selected_services = set(self.get_enabled_services_from_component())
if selected_services.intersection(s_store):
localsh.run('systemctl start rsyncd')
def files_to_iso(filemap, config_image):
# filemap is target,source pairs
pathspec = ' '.join(('='.join((target, source)).join(("'", "'"))
for (target, source) in filemap))
# use real shell escape ? single=False
localsh.run("mkisofs -graft-points -o '{config_image}' "
"-V cidata -r -J --quiet {pathspec}".format(
pathspec=pathspec, config_image=config_image))
def do_fernet_init(cname):
self = facility.get_component(cname)
self.have_content()
localsh.run("""
mkdir -p /etc/keystone/fernet-keys # replace with install
chown keystone:keystone /etc/keystone/fernet-keys
chmod 770 /etc/keystone/fernet-keys
keystone-manage fernet_setup --keystone-user keystone --keystone-group keystone
""")
def do_proxy(cname, cfg):
self = facility.get_component(cname)
self.have_content()
self.file_path('/etc/systemd/system/haproxy.service.d')
self.file_ini('/etc/systemd/system/haproxy.service.d/limits.conf',
self.etc_systemd_system_haproxy_service_d_limits_conf())
self.file_haproxy('/etc/haproxy/haproxy.cfg', cfg)
localsh.run(
'systemctl daemon-reload && systemctl reload-or-restart haproxy')
def pip_install(targets):
# target either a 'package' or '-r req.txt', '-e project', input is iterable
ensure_requirements()
pkgutils.ensure_compose()
try:
PIP_LOCK.acquire()
localsh.run('pip3 install {targets}'.format(
targets=' '.join(targets)))
finally:
PIP_LOCK.release()
def update(cls):
retry = 5
LOG.info("Updating packages ..")
while retry:
try:
localsh.run(cls.update_cmd)
retry = 0
except Exception:
retry -= 1
if not retry:
raise
def pip_install_req(targets):
# target either a 'package' or '-r req.txt', '-e project', input is iterable
ensure_requirements()
pkgutils.ensure_compose()
r_dir = req_dir()
try:
PIP_LOCK.acquire()
localsh.run('pip3 install -c {req_dir}/upper-constraints.txt {targets}'.format(
req_dir=r_dir, targets=' '.join(targets)))
finally:
PIP_LOCK.release()
def install(cls, pkgs):
retry = 5
LOG.info("Installing packages ..")
pkgs = cls.pkg_mapping(pkgs)
while retry:
try:
localsh.run(cls.install_cmd + ' '.join(pkgs))
retry = 0
except Exception:
retry -= 1
if not retry:
raise
def local_os_service_start_by_component(*args, update_cfg=False):
to_start = []
for comp in args:
if not update_cfg:
comp.have_content()
enabled = comp.get_enabled_services_from_component()
ds = comp.deploy_source
for s in enabled:
service = comp.services[s]
if service[
'deploy_mode'] == 'standalone': # TODO make soure the options can be different for component instance
to_start.append(
service['unit_name'][ds]) # TODO: handle offset
localsh.run('systemctl start %s' % (' '.join(to_start)))
def ensure_git():
global SYSTEM_HAS_GIT
if SYSTEM_HAS_GIT:
return
try:
ENSURE_GIT_LOCK.acquire()
if SYSTEM_HAS_GIT:
return
if not localsh.test("git --version"):
pkgutils.get_pkgmgr().install({'git'})
localsh.run("git --version")
SYSTEM_HAS_GIT = True
finally:
ENSURE_GIT_LOCK.release()
def do_retrycmd_after_content(cname, cmd):
self = get_component(cname)
self.have_content()
retry = 30
while True:
try:
localsh.run(cmd)
except Exception:
if retry == 0:
raise
else:
break
time.sleep(0.2)
retry -= 1
def create_backed_qcow2(src, dst, size='10G', bfmt='raw'):
# the args are not shell escaped
if size:
s = util.human_byte_to_int(size)
if bfmt == 'raw':
image_size = os.path.getsize(src)
else:
image_size = get_virtual_size(src)
if image_size > s:
size = image_size
localsh.run("qemu-img create -f qcow2 -o 'backing_fmt={bfmt},"
"backing_file={src}' '{dst}' '{size}'".format(
src=src, dst=dst, size=size, bfmt=bfmt))
else:
localsh.run("qemu-img create -f qcow2 -o 'backing_fmt={bfmt},"
"backing_file={src}' '{dst}'".format(
src=src, dst=dst, bfmt=bfmt))
def do_ensure_flavors(cname):
localsh.run(
util.userrc_script('admin') + """
available_flavors=$(nova flavor-list)
retry=30
while ! available_flavors=$(nova flavor-list) ; do
((retry--))
if [[ retry == 0 ]]; then
break;
fi
done
if [[ ! ( $available_flavors =~ 'm1.nano' ) ]]; then
openstack flavor create --id 42 --ram 64 --disk 1 --vcpus 1 m1.nano
fi
if [[ ! ( $available_flavors =~ 'm1.micro' ) ]]; then
openstack flavor create --id 84 --ram 128 --disk 1 --vcpus 1 m1.micro
fi """)
def do_create_clustr_user(cname):
self = facility.get_component(cname)
passwd = util.get_keymgr()(self.name, 'clustercheckuser')
pwd = passwd.replace('\\', '\\\\').replace("'", r"\'").replace('$', r'\$')
sql = "GRANT PROCESS ON *.* TO 'clustercheckuser'@'localhost' IDENTIFIED BY '{pwd}'".format(pwd=pwd)
# $ for shell, the others for mysql
retry = 1024 # wating for mariadb become ready
while True:
try:
script = 'mysql -u root <<EOF\n{sql}\nEOF\n'.format(sql=sql)
localsh.run(script)
break
except util.NonZeroExitCode:
if retry:
time.sleep(0.2)
retry -= 1
else:
raise
def group(name, gid=None, gpasswd=None):
try:
g = grp.getgrnam(name)
if gid and g[2] != gid:
LOG.warning("Group '{name}' already exists"
" with gid:{real_gid}, not with {wanted_gid}".format(name=name,
real_gid=g[2], wanted_gid=gid))
# TODO: remove pass in case of empty ?
if gpasswd:
if not check_hash(gpasswd, g[1]):
localsh.run("groupmod -p '{passwd_hash}' '{name}'".format(
name=name,
passwd_hash=passwd_to_hash(gpasswd)))
return 1
return 0
except KeyError:
pass
if (gid):
try:
g = grp.getgrgid(gid)
if g[2] != gid:
LOG.warning("Group '{name}' already exists"
" with gid: {real_gid}", name=name,
real_gid=g[2])
except KeyError:
pass
if gpasswd:
passwd_opt = ''.join(("-p '", passwd_to_hash(gpasswd), "'"))
else:
passwd_opt = ''
if gid:
gid_opt = '-g ' + str(gid)
else:
gid_opt = ''
localsh.run("groupadd -f {gid_opt} {passwd_opt} '{name}'".format(
gid_opt=gid_opt,
passwd_opt=passwd_opt, name=name))
return 1
def do_dummy_public_net(cname):
# guest net hack
# 192.0.2.1 expected to be configured on an interface
localsh.run(
util.userrc_script('admin') + """
(
retry=30
while ! neutron net-create public --router:external=True --is-default=True --provider:network_type flat --provider:physical_network extnet ; do
((retry--))
if [[ retry == 0 ]]; then
break;
fi
done
FLOATING_IP_CIDR=${FLOATING_IP_CIDR:-"192.0.2.0/24"}
FLOATING_IP_START=${FLOATING_IP_START:-"192.0.2.32"}
FLOATING_IP_END=${FLOATING_IP_END:-"192.0.2.196"}
EXTERNAL_NETWORK_GATEWAY=${EXTERNAL_NETWORK_GATEWAY:-"192.0.2.1"}
neutron subnet-create --name ext-subnet --allocation-pool start=$FLOATING_IP_START,end=$FLOATING_IP_END --disable-dhcp --gateway $EXTERNAL_NETWORK_GATEWAY public $FLOATING_IP_CIDR
# for auto allocation test
openstack subnet pool create --share --default --pool-prefix 192.0.3.0/24 --default-prefix-length 26 shared-default
openstack subnet pool create --share --default --pool-prefix 2001:db8:8000::/48 --default-prefix-length 64 default-v6
)""")
def do_rabbit_start(cname):
self = facility.get_component(cname)
self.have_content()
retry = 128
# TODO: use state file, or vallet/key_mgr
self.file_plain('/var/lib/rabbitmq/.erlang.cookie',
'NETTIQETJNDTXLRUSANA',
owner='rabbitmq',
mode=0o600)
while True:
try:
if self.changed: # TODO: rolling bounce
action = 'reload-or-restart'
else:
action = 'start'
localsh.run("systemctl {} rabbitmq-server".format(action))
break
except util.NonZeroExitCode:
LOG.warn('Check the RABBIT systemd deps!')
time.sleep(0.5)
if not retry:
raise
retry -= 1
def do_swift_deploy_demo_local(cname):
self = facility.get_component(cname)
# prepare swift
# this is from the all in script, it needs to be completly rewritten
object_ip = self.get_addr_for(self.get_this_inv(),
'backing_object',
net_attr='swift_object_network')
# replica_ip = self.get_addr_for(self.get_this_inv(), 'replication',
# net_attr='swift_object_replica_network')
self.have_content()
script = """
INSTALLER_DATA_DIR="%s"
BACKING_IP="%s"
mkdir $INSTALLER_DATA_DIR/swift
cd $INSTALLER_DATA_DIR/swift
# old demo only script!
for ring in account container object; do
swift-ring-builder "$ring.builder" create 10 1 1 # 2^10 partiotions, 1 replicas (no replication), 1 hour move limit
done
# device is the name of directory in the /srv/node , normally it is a mounted xfs
swift-ring-builder account.builder add --region 1 --zone 1 --ip "$BACKING_IP" --port 6202 --device disk1 --weight 100
swift-ring-builder container.builder add --region 1 --zone 1 --ip "$BACKING_IP" --port 6201 --device disk1 --weight 100
swift-ring-builder object.builder add --region 1 --zone 1 --ip "$BACKING_IP" --port 6200 --device disk1 --weight 100
# update the ring file and copy to ALL SWIFT STORAGE SERVERS
# it should be rsync-d or scp -ed not cp -d, (or remote copied by the script itself)
for ring in account container object; do
swift-ring-builder $ring.builder rebalance
cp "$ring.ring.gz" /etc/swift/ # TODO: use install
done
""" % ('/tmp', object_ip)
# we would need to use the inventory ips, and iterate over the full map
localsh.run(script)
def do_dummy_netconfig(cname):
if util.get_distro()['family'] != 'debian':
osrv = 'openvswitch.service'
else:
osrv = 'openvswitch-switch.service'
localsh.run('systemctl start ' + osrv)
# TODO switch to os-net-config
# wait (no --no-wait)
localsh.run('ovs-vsctl --may-exist add-br br-ex')
# add ip to external bridge instead of adding a phyisical if
localsh.run("""
ifconfig br-ex 192.0.2.1
ip link set br-ex up
ROUTE_TO_INTERNET=$(ip route get 8.8.8.8)
OBOUND_DEV=$(echo ${ROUTE_TO_INTERNET#*dev} | awk '{print $1}')
iptables -t nat -A POSTROUTING -o $OBOUND_DEV -j MASQUERADE
tee /proc/sys/net/ipv4/ip_forward <<<1 >/dev/null
""")
def create_workspace():
# consider adding other groups
root = get_path() + os.path.sep
dirs = ['downloads', 'library',
'live', 'cd', 'log', 'keys']
for d in dirs:
os.makedirs(root + d, exist_ok=True)
# is priv key exists
base_key_path = get_path("keys")
priv_key = base_key_path + SSH_PRIVATE_KEY_PATH_REL
pub_keys = base_key_path + SSH_PUBLIC_KEY_LIST_PATH_REL
if not os.path.isfile(priv_key):
localsh.run("ssh-keygen -t rsa -b 4096 -P '' -f '{path}'".format(
path=priv_key))
if not os.path.isfile(pub_keys):
localsh.run("ssh-keygen -y -f '{private'} > '{public}'".format(
private=priv_key,
public=pub_keys))
non_root = queury_non_root()
if non_root:
localsh.run("chown {non_root} '{priv}'".format(non_root=non_root,
priv=priv_key))
def do_rabbitmq_reset_join(cname, leader):
localsh.run("""rabbitmqctl stop_app
rabbitmqctl reset
rabbitmqctl join_cluster {leader}
rabbitmqctl start_app
""".format(leader='rabbit@' + leader))
def do_ovs(cname):
localsh.run('systemctl start openvswitch.service')
def do_httpd_restart(cname):
self = facility.get_component(cname)
self.have_content()
srv_name = 'httpd' if util.get_distro(
)['family'] == 'redhat' else 'apache2'
localsh.run("systemctl reload-or-restart " + srv_name)
def do_keystone_init(cname):
self = facility.get_component(cname)
self.have_content()
localsh.run("keystone-manage bootstrap --bootstrap-password %s" %
cmd_quote(util.get_keymgr()(self.name, 'admin@default')))
def do_selinux_permissive():
localsh.run("""
setenforce 0 || true # please report the detected issues!""")
def do_memcached_service_start(cname):
self = facility.get_component(cname)
self.have_content()
localsh.run('systemctl start memcached')
