def get_machine_numbers(service):
juju_units = mojo_utils.get_juju_units(service=service)
machine_numbers = []
for unit in juju_units:
machine_numbers.append(mojo_utils.convert_unit_to_machinename(unit))
machine_numbers.sort()
return machine_numbers
def main(argv):
cert_script = mojo_utils.get_mojo_file('checksum_keystone_certs.py')
remote_script = '/home/ubuntu/checksum_keystone_certs.py'
hashes = set()
for unit in mojo_utils.get_juju_units(service='keystone'):
mojo_utils.remote_upload(unit, cert_script, remote_script)
hashes.add(mojo_utils.remote_run(unit, remote_script)[0].strip())
if len(hashes) != 1:
raise Exception('Keystone cert mismatch')
def rabbit_status():
juju_units = mojo_utils.get_juju_units(service='rabbitmq-server')
machine_numbers = get_machine_numbers('rabbitmq-server')
for unit in juju_units:
units = rabbit_unit_status(unit)
units.sort()
if machine_numbers == units:
logging.info('Rabbit status on %s look good' % (unit))
else:
msg = 'Mismatch on rabbit status for on unit {}'.format(unit)
raise Exception(msg)
def main(argv):
logging.basicConfig(level=logging.INFO)
parser = argparse.ArgumentParser()
parser.add_argument("units", nargs="*")
options = parser.parse_args()
unit_args = mojo_utils.parse_mojo_arg(options, 'units', multiargs=True)
for unitreq in unit_args:
if ':' in unitreq:
service, unitcount = unitreq.split(':')
units = mojo_utils.get_juju_units(service=service)
i = 0
for unit in units:
if i >= int(unitcount):
break
mojo_utils.delete_unit(unit)
i = i + 1
elif '/' in unitreq:
units = mojo_utils.get_juju_units(service=unitreq.split('/')[0])
if unitreq in units:
mojo_utils.delete_unit(unitreq)
def main(argv):
cli_utils.setup_logging()
parser = argparse.ArgumentParser()
parser.add_argument("units", nargs="*")
options = parser.parse_args()
unit_args = cli_utils.parse_arg(options, 'units', multiargs=True)
for unitreq in unit_args:
if ':' in unitreq:
application, unitcount = unitreq.split(':')
units = mojo_utils.get_juju_units(application)
i = 0
for unit in units:
if i >= int(unitcount):
break
mojo_utils.delete_unit(unit)
i = i + 1
elif '/' in unitreq:
units = mojo_utils.get_juju_units(unitreq.split('/')[0])
if unitreq in units:
mojo_utils.delete_unit(unitreq)
def main(argv):
# Tempest expects an admin project in the admin domain that the admin user
# has admin role on so make sure that exists (pre-17.02)
keystone_v3_domain_setup()
expected_results = mojo_utils.get_mojo_config(
'tempest_expected_results.yaml')['smoke']
tempest_unit = mojo_utils.get_juju_units(service='tempest')
action_id = mojo_utils.action_run(tempest_unit[0],
'run-tempest',
timeout=18000)
action_output = mojo_utils.action_get_output(action_id)
logging.debug(action_output)
actual_results = action_output['results']
result_matrix = {
'failed': {
'on_more': raise_tempest_fail,
'on_less': report_success
},
'skipped': {
'on_more': warn,
'on_less': report_success
},
'expected-fail': {
'on_more': raise_tempest_fail,
'on_less': report_success
},
'unexpected-success': {
'on_more': report_success,
'on_less': warn
},
'passed': {
'on_more': report_success,
'on_less': raise_tempest_fail
}
}
for result_type, expected in expected_results.items():
actual = actual_results[result_type]
msg = "Number of tests {} was {} expected {}".format(
result_type, actual, expected)
if int(actual) > expected:
result_matrix[result_type]['on_more'](msg)
elif int(actual) == expected:
report_success(msg)
else:
result_matrix[result_type]['on_less'](msg)
def check_crm_status(service):
juju_units = mojo_utils.get_juju_units(service=service)
if not juju_units:
return
cmd = 'which crm_mon || echo "Not Found"'
output = mojo_utils.remote_run(juju_units[0], remote_cmd=cmd)
if output[0].rstrip() == "Not Found":
return
for unit in juju_units:
mach_nums = get_machine_numbers(service)
crm_online = unit_crm_online(unit)
if mach_nums == crm_online:
logging.info('Service %s status on %s look good' % (service, unit))
else:
logging.info('%s != %s' % (str(mach_nums), str(crm_online)))
msg = ('Mismatch on crm status for service {} '
'on unit {}'.format(service, unit))
raise Exception(msg)
def main(argv):
addr = mojo_utils.juju_get('nova-cloud-controller', 'vip')
# The second vip is the os-internal-network one
addr = addr.split(' ')[1]
for unit in mojo_utils.get_juju_units(service='nova-cloud-controller'):
for conf, entry, url in [[
'/etc/nova/nova.conf', 'neutron_url',
"http://%s:9696" % (addr)
],
[
'/etc/neutron/neutron.conf', 'nova_url',
"http://%s:8774/v2" % (addr)
]]:
out = mojo_utils.remote_run(unit, 'grep %s %s' % (entry, conf))
if url not in out[0]:
raise Exception("Did not get expected %s in %s (got='%s', "
"wanted='%s')" % (entry, conf, out, url))
logging.info("%s checked and good." % (conf))
def main(argv):
mojo_utils.setup_logging()
parser = argparse.ArgumentParser()
parser.add_argument('--peer-application', '-a',
help='BGP Peer application name. Default: quagga',
default='quagga')
options = parser.parse_args()
peer_application_name = mojo_utils.parse_mojo_arg(options,
'peer_application')
overcloud_novarc = mojo_utils.get_overcloud_auth()
os_version = mojo_os_utils.get_current_os_versions('keystone')['keystone']
# Keystone policy.json shipped the charm with liberty requires a domain
# scoped token. Bug #1649106
if os_version == 'liberty':
scope = 'DOMAIN'
else:
scope = 'PROJECT'
keystone_session = mojo_os_utils.get_keystone_session(overcloud_novarc,
scope=scope)
neutronc = mojo_os_utils.get_neutron_session_client(keystone_session)
# Run show ip route on BGP peer
peer_unit = mojo_utils.get_juju_units(service=peer_application_name)[0]
logging.info("Checking routes on BGP peer {}".format(peer_unit))
routes = mojo_utils.remote_run(peer_unit,
remote_cmd="vtysh -c 'show ip route'")[0]
logging.debug(routes)
# Check for expected advertised routes
private_cidr = neutronc.list_subnets(
name='private_subnet')['subnets'][0]['cidr']
floating_ip_cidr = "{}/32".format(
neutronc.list_floatingips()['floatingips'][0]['floating_ip_address'])
assert private_cidr in routes, ("Private subnet CIDR, {}, not advertised "
"to BGP peer".format(private_cidr))
logging.info("Private subnet CIDR, {}, found in routing table"
.format(private_cidr))
assert floating_ip_cidr in routes, ("Floating IP, {}, not advertised "
"to BGP peer".format(floating_ip_cidr))
logging.info("Floating IP CIDR, {}, found in routing table"
.format(floating_ip_cidr))
def main(argv):
cli_utils.setup_logging()
parser = argparse.ArgumentParser()
parser.add_argument('-r',
'--resolver',
help='Resolver address. '
'Usually designate-bind address.',
required=False)
parser.add_argument('-d',
'--domain_name',
help='DNS Domain Name. '
'Must end in a .',
default='mojo.serverstack.')
parser.add_argument('-e',
'--email',
help='Email address',
default='*****@*****.**')
parser.add_argument('-b',
'--bind-service',
help='Bind Service Name',
default='designate-bind')
options = parser.parse_args()
bind_service_name = cli_utils.parse_arg(options, 'bind_service')
resolver = cli_utils.parse_arg(options, 'resolver')
bind_unit = mojo_utils.get_juju_units(bind_service_name)[0]
if not resolver:
resolver = mojo_utils.get_juju_unit_ip(bind_unit)
domain_name = cli_utils.parse_arg(options, 'domain_name')
email = cli_utils.parse_arg(options, 'email')
nameserver = 'ns1.{}'.format(domain_name)
logging.info('Setting up designate {} {}'.format(nameserver, resolver))
os_version = openstack_utils.get_current_os_versions(
'keystone')['keystone']
try:
cacert = os.path.join(os.environ.get('MOJO_LOCAL_DIR'), 'cacert.pem')
os.stat(cacert)
except FileNotFoundError:
cacert = None
keystone_session = openstack_utils.get_overcloud_keystone_session(
verify=cacert)
neutronc = openstack_utils.get_neutron_session_client(keystone_session)
if os_version >= 'queens':
designatec = mojo_os_utils.get_designate_session_client(
keystone_session, client_version='2')
zone = mojo_os_utils.create_or_return_zone(designatec, domain_name,
email)
mojo_os_utils.create_or_return_recordset(designatec, zone['id'], 'www',
'A', [resolver])
else:
designatec = mojo_os_utils.get_designate_session_client(
keystone_session, client_version='1')
if not mojo_os_utils.get_designate_server_id(designatec, nameserver):
logging.info('Creating server {}'.format(nameserver))
server = Server(name=nameserver)
server_id = designatec.servers.create(server)
assert (server_id is not None)
else:
logging.info('{} server already exists.'.format(nameserver))
domain_id = mojo_os_utils.get_designate_domain_id(
designatec, domain_name)
if not domain_id:
logging.info('Creating domain {}'.format(domain_name))
domain = Domain(name=domain_name, email=email)
domain_id = designatec.domains.create(domain)
assert (domain_id is not None)
else:
logging.info('{} domain already exists.'.format(domain_name))
if not mojo_os_utils.get_designate_record_id(designatec, domain_id,
nameserver):
logging.info('Creating NS record {}'.format(nameserver))
ns_record = Record(name=nameserver, type="A", data=resolver)
record_id = designatec.records.create(domain_id, ns_record)
assert (record_id is not None)
else:
logging.info('{} record already exists.'.format(nameserver))
logging.info('Update network to use domain {}'.format(domain_name))
net_uuid = openstack_utils.get_net_uuid(neutronc, 'private')
mojo_os_utils.update_network_dns(neutronc, net_uuid, domain_name)
def main(argv):
for unit in mojo_utils.get_juju_units(service='neutron-gateway'):
cmd = 'sudo service neutron-plugin-openvswitch-agent restart'
mojo_utils.remote_run(unit, remote_cmd=cmd)
if line == "":
continue
hashsum, filename = line.split()
ring_data[filename] = hashsum
return ring_data
def verify_ring_data(ring_data):
ring_dict = next(iter(ring_data.values()))
for unit in ring_data.keys():
if ring_data[unit] != ring_dict:
return False
return True
sp_units = mojo_utils.get_juju_units('swift-proxy')
ring_data = {}
for unit in sp_units:
cmd = 'ls -1 /etc/swift/*{.builder,.ring.gz,arse} 2>/dev/null ' \
'| xargs -l md5sum'
out = juju_utils.remote_run(unit, remote_cmd=cmd)
ring_data[unit] = process_ring_info(out)
if verify_ring_data(ring_data):
logging.info('Ring data consistent accross proxies')
sys.exit(0)
else:
logging.error('Ring data inconsistent accross proxies')
sys.exit(1)
continue
hashsum, filename = line.split()
ring_data[filename] = hashsum
return ring_data
def verify_ring_data(ring_data):
ring_dict = ring_data.itervalues().next()
for unit in ring_data.iterkeys():
if ring_data[unit] != ring_dict:
return False
return True
juju_status = mojo_utils.get_juju_status(service='swift-proxy')
sp_units = mojo_utils.get_juju_units(juju_status=juju_status)
ring_data = {}
for unit in sp_units:
cmd = 'ls -1 /etc/swift/*{.builder,.ring.gz,arse} 2>/dev/null ' \
'| xargs -l md5sum'
out, err = mojo_utils.remote_run(unit, remote_cmd=cmd)
ring_data[unit] = process_ring_info(out)
if verify_ring_data(ring_data):
logging.info('Ring data consistent accross proxies')
sys.exit(0)
else:
logging.error('Ring data inconsistent accross proxies')
sys.exit(1)
def run_check():
juju_units = mojo_utils.get_juju_units()
remote_runs(juju_units)
remote_runs(juju_units)
def main(argv):
mojo_utils.setup_logging()
parser = argparse.ArgumentParser()
parser.add_argument('-r', '--resolver',
help='Resolver address. '
'Usually designate-bind address.',
required=False)
parser.add_argument('-d', '--domain_name', help='DNS Domain Name. '
'Must end in a .',
default='mojo.serverstack.')
parser.add_argument('-e', '--email', help='Email address',
default='*****@*****.**')
parser.add_argument('-b', '--bind-service', help='Bind Service Name',
default='designate-bind')
options = parser.parse_args()
bind_service_name = mojo_utils.parse_mojo_arg(options, 'bind_service')
resolver = mojo_utils.parse_mojo_arg(options, 'resolver')
bind_unit = mojo_utils.get_juju_units(service=bind_service_name)[0]
if not resolver:
resolver = mojo_utils.get_juju_unit_ip(bind_unit)
domain_name = mojo_utils.parse_mojo_arg(options, 'domain_name')
email = mojo_utils.parse_mojo_arg(options, 'email')
nameserver = 'ns1.{}'.format(domain_name)
logging.info('Setting up designate {} {}'.format(nameserver, resolver))
overcloud_novarc = mojo_utils.get_overcloud_auth()
os_version = mojo_os_utils.get_current_os_versions('keystone')['keystone']
# Keystone policy.json shipped the charm with liberty requires a domain
# scoped token. Bug #1649106
if os_version == 'liberty':
scope = 'DOMAIN'
else:
scope = 'PROJECT'
keystone_session = mojo_os_utils.get_keystone_session(overcloud_novarc,
scope=scope)
neutronc = mojo_os_utils.get_neutron_session_client(keystone_session)
if os_version >= 'queens':
designatec = mojo_os_utils.get_designate_session_client(
keystone_session,
client_version='2')
zone = mojo_os_utils.create_or_return_zone(
designatec,
domain_name,
email)
rs = mojo_os_utils.create_or_return_recordset(
designatec,
zone['id'],
'www',
'A',
[resolver])
else:
designatec = mojo_os_utils.get_designate_session_client(
keystone_session,
client_version='1')
if not mojo_os_utils.get_designate_server_id(designatec, nameserver):
logging.info('Creating server {}'.format(nameserver))
server = Server(name=nameserver)
server_id = designatec.servers.create(server)
assert(server_id is not None)
else:
logging.info('{} server already exists.'.format(nameserver))
domain_id = mojo_os_utils.get_designate_domain_id(
designatec,
domain_name)
if not domain_id:
logging.info('Creating domain {}'.format(domain_name))
domain = Domain(name=domain_name, email=email)
domain_id = designatec.domains.create(domain)
assert(domain_id is not None)
else:
logging.info('{} domain already exists.'.format(domain_name))
if not mojo_os_utils.get_designate_record_id(designatec, domain_id,
nameserver):
logging.info('Creating NS record {}'.format(nameserver))
ns_record = Record(
name=nameserver,
type="A",
data=resolver)
record_id = designatec.records.create(domain_id, ns_record)
assert(record_id is not None)
else:
logging.info('{} record already exists.'.format(nameserver))
logging.info('Update network to use domain {}'.format(domain_name))
net_uuid = mojo_os_utils.get_net_uuid(neutronc, 'private')
mojo_os_utils.update_network_dns(neutronc, net_uuid, domain_name)
