def check_flavors():
orphan_flavors = []
clients = mwopenstackclients.clients("/etc/novaobserver.yaml")
allprojects = clients.allprojects()
allprojectslist = [project.name for project in allprojects]
# these will always be weird; don't check them
for project in SpecialProjects:
allprojectslist.remove(project)
for project in allprojectslist:
novaclient = clients.novaclient(project)
flavors = novaclient.flavors.list()
for flavor in flavors:
orphan_flavors.append(flavor.name)
for key in flavor.get_keys():
if key.startswith("aggregate_instance_extra_specs"):
orphan_flavors.remove(flavor.name)
break
if orphan_flavors:
return (
WARNING,
"Some flavors are not assigned to aggregates: " + ", ".join(orphan_flavors),
)
return OK, "All flavors are assigned to aggregates"
def __init__(self, instance_id, region):
self.instance_id = instance_id
self.osclients = mwopenstackclients.clients()
self.novaclient = self.osclients.novaclient()
self.refresh_instance()
nova = self.osclients.novaclient()
flavors = nova.flavors.list()
flavoriddict = {f.id: f.name for f in flavors}
flavornamedict = {f.name: f.id for f in flavors}
projects = self.osclients.allprojects()
for project in projects:
nova_per_project = self.osclients.novaclient(project.id)
project_flavors = nova_per_project.flavors.list()
for project_flavor in project_flavors:
if project_flavor.id not in flavoriddict:
flavornamedict[project_flavor.name] = project_flavor.id
flavoriddict[project_flavor.id] = project_flavor.name
source_flavor_name = flavoriddict[self.instance.flavor["id"]]
dest_flavor_name = flavor_map[source_flavor_name]
self.dest_flavor_id = flavornamedict[dest_flavor_name]
logging.warning("We will convert to flavor %s (%s)" %
(dest_flavor_name, self.dest_flavor_id))
def __init__(self):
clients = mwopenstackclients.clients()
services = clients.keystoneclient().services.list()
serviceid = [s.id for s in services if s.type == 'dns'][0]
endpoints = clients.keystoneclient().endpoints.list(serviceid)
self.url = [e.url for e in endpoints if e.interface == 'public'][0]
session = clients.session()
self.token = session.get_token()
def main():
clients = mwopenstackclients.clients('/etc/novaobserver.yaml')
instances = clients.allinstances(projectid='admin-monitoring')
print("%s instances in the admin-monitoring project" % len(instances))
if len(instances) >= CRIT_LEAKS:
sys.exit(CRITICAL)
elif len(instances) >= WARN_LEAKS:
sys.exit(WARNING)
sys.exit(OK)
def check_roles(user, requiredroles, all_projects=True):
clients = mwopenstackclients.clients('/etc/novaobserver.yaml')
keystoneclient = clients.keystoneclient()
# test observer roles
roledict = {role.name: role.id for role in keystoneclient.roles.list()}
roledictdecode = {
role.id: role.name
for role in keystoneclient.roles.list()
}
requiredset = set()
for role in requiredroles:
requiredset.add(roledict[role])
assignments = keystoneclient.role_assignments.list(user=user)
assignmentdict = {}
for assignment in assignments:
if 'project' not in assignment.scope:
continue
project = assignment.scope['project']['id']
if project not in assignmentdict:
assignmentdict[project] = set()
assignmentdict[project].add(assignment.role['id'])
for project in assignmentdict:
if project in SpecialProjects:
continue
if assignmentdict[project] != requiredset:
# Make a human-readable list of role names for reporting purposes
namelist = [
roledictdecode[roleid] for roleid in assignmentdict[project]
]
rstring = ("In %s, user %s should have roles %s but has %s" %
(project, user, requiredroles, namelist))
return (CRITICAL, rstring)
if all_projects:
allprojects = clients.allprojects()
allprojectslist = [project.name for project in allprojects]
# these will always be weird; don't check them
for project in SpecialProjects:
allprojectslist.remove(project)
leftovers = set(allprojectslist) - set(assignmentdict.keys())
if leftovers:
rstring = ("Roles for %s are not set in these projects: %s" %
(user, leftovers))
return (WARNING, rstring)
rstring = ("%s has the correct roles in all projects." % user)
return (OK, rstring)
def main():
clients = mwopenstackclients.clients('/etc/novaobserver.yaml')
allprojects = clients.allprojects()
allprojectslist = [project.name for project in allprojects]
requiredprojects = ['admin', 'observer']
for project in requiredprojects:
if project not in allprojectslist:
print("%s project missing from Keystone." % project)
sys.exit(WARNING)
for project in allprojects:
if project.id != project.name:
print("Keystone project with name %s has mismatched key %s" %
(project.name, project.id))
sys.exit(WARNING)
print "Keystone projects exist and have matching names and ids."
sys.exit(OK)
def check_instance_ages(project, days_to_nag, days_to_delete):
clients = mwopenstackclients.clients(envfile='/etc/novaadmin.yaml')
keystone = clients.keystoneclient(project=project)
for instance in clients.allinstances(projectid=project, allregions=True):
created = datetime.datetime.strptime(instance.created,
"%Y-%m-%dT%H:%M:%SZ")
age = (datetime.datetime.now() - created).days
user = keystone.users.get(instance.user_id)
print(instance.name, user.email, age)
if days_to_delete:
if age > days_to_delete:
print("Deleting %s.%s because it is more than %s days old" %
(instance.name, project, days_to_delete))
clients.novaclient(project=project).servers.delete(instance.id)
continue
if age > days_to_nag:
print(
"Sending warning email about %s.%s because it is %s days old" %
(instance.name, project, age))
send_nag_email(user.email, project, instance.name, age,
days_to_delete)
This also queries nova directly in order to compare actual on-disk usage
with potential allocated space.
"""
import mwopenstackclients
import subprocess
import os.path
import collections
ONEGIG = 1024 * 1024 * 1024
ONEMEG = 1024 * 1024
clients = mwopenstackclients.clients()
hosts = clients.novaclient().hosts.list()
computenodedict = {}
flavors = clients.novaclient().flavors.list()
flavordict = {f.id: f.disk for f in flavors}
# extras that our query can't pick up
flavordict['7'] = '80'
flavordict['101'] = '40'
flavordict['bb5bf060-cdbb-4448-b436-a015ae2d4aaf'] = '160'
flavordict['8af1f1cc-d95f-4380-bf10-bcfa0321b10f'] = '60'
flavordict['2d59cc0d-538c-4bbd-b975-8e696a4f7207'] = '80'
flavordict['deea3460-069e-44c7-98ca-ae30bb0de772'] = '80'
flavordict['cc0f1723-38d7-42da-aa2c-cef28d5f4250'] = '300'
def check_flavors():
orphan_flavors = []
ceph_flavors_without_bytes_sec = []
ceph_flavors_without_read_iops = []
ceph_flavors_without_write_iops = []
clients = mwopenstackclients.clients("/etc/novaobserver.yaml")
allprojects = clients.allprojects()
allprojectslist = [project.name for project in allprojects]
# these will always be weird; don't check them
for project in SpecialProjects:
allprojectslist.remove(project)
for project in allprojectslist:
novaclient = clients.novaclient(project)
flavors = novaclient.flavors.list()
for flavor in flavors:
orphan_flavors.append(flavor.name)
keys = flavor.get_keys()
for key in keys:
if key.startswith("aggregate_instance_extra_specs"):
orphan_flavors.remove(flavor.name)
if "ceph" in key:
# Make sure this is throttled properly
ceph_flavors_without_bytes_sec.append(flavor.name)
ceph_flavors_without_read_iops.append(flavor.name)
ceph_flavors_without_write_iops.append(flavor.name)
for throttlekey in keys:
if throttlekey.startswith(
"quota:disk_total_bytes_sec"):
ceph_flavors_without_bytes_sec.remove(
flavor.name)
elif throttlekey.startswith(
"quota:disk_read_iops_sec"):
ceph_flavors_without_read_iops.remove(
flavor.name)
elif throttlekey.startswith(
"quota:disk_write_iops_sec"):
ceph_flavors_without_write_iops.remove(
flavor.name)
break
errstring = ""
if orphan_flavors:
errstring += ("Some flavors are not assigned to aggregates: " +
", ".join(orphan_flavors) + "\n")
if ceph_flavors_without_bytes_sec:
errstring += ("Some ceph flavors lack quota:disk_total_bytes_sec: " +
", ".join(ceph_flavors_without_bytes_sec) + "\n")
if ceph_flavors_without_read_iops:
errstring += ("Some ceph flavors lack quota:disk_read_iops_sec: " +
", ".join(ceph_flavors_without_read_iops) + "\n")
if ceph_flavors_without_write_iops:
errstring += ("Some ceph flavors lack quota:disk_write_iops_sec: " +
", ".join(ceph_flavors_without_write_iops) + "\n")
if errstring:
return (CRITICAL, errstring)
return OK, "All flavors are assigned to aggregates"
if __name__ == '__main__':
argparser = argparse.ArgumentParser()
argparser.add_argument("--test-if-up-to-date",
action="store_true",
help="Succeed only if no changes would be done")
args = argparser.parse_args()
with open('/etc/shinkengen.yaml') as f:
config = yaml.safe_load(f)
with open('/etc/novaobserver.yaml') as n:
nova_observer = yaml.safe_load(n)
observer_pass = nova_observer['OS_PASSWORD']
clients = mwopenstackclients.clients('/etc/novaobserver.yaml')
for project in config['projects']:
instances = clients.allinstances(projectid=project, allregions=True)
host_configs = []
for instance in instances:
ip = ""
for value in instance.addresses.values():
for ipentry in value:
if ipentry['OS-EXT-IPS:type'] == 'fixed':
ip = str(ipentry['addr'])
if not ip:
# This VM is broken in some serious way; skip.
next
fqdn = '.'.join(
#!/usr/bin/python
import sys
import mwopenstackclients
certname = sys.argv[1]
clients = mwopenstackclients.clients(envfile='/etc/novaobserver.yaml')
pieces = certname.split('.')
if len(pieces) != 4:
sys.exit('certname %s is formatted incorrectly' % certname)
if pieces[2] != 'eqiad':
sys.exit('certname %s is for an invalid site' % certname)
if pieces[3] != 'wmflabs':
sys.exit('certname %s does not end with wmflabs' % certname)
certhostname = pieces[0]
certproject = pieces[1]
projects = [project.id for project in clients.allprojects()]
if certproject not in projects:
sys.exit('certname %s is not for a real project' % certname)
# the cert name will always be lowercase. So we need to lower()
# the instance name for proper comparison
instances = [
instance.name.lower()
for instance in clients.novaclient(certproject).servers.list()
#!/usr/bin/python3
# Dump fqdns for all VMs, sorted by project and hypervisor
import mwopenstackclients
clients = mwopenstackclients.clients(envfile="/etc/novaadmin.yaml")
hypervisors = clients.novaclient().services.list()
hvnames = sorted([hv.host for hv in hypervisors])
for hv in hvnames:
print("\n== %s ==\n" % hv)
servers = clients.novaclient().servers.list(search_opts={
"all_tenants": True,
"host": hv
})
projdict = {}
for server in servers:
if server.tenant_id not in projdict:
projdict[server.tenant_id] = [server]
else:
projdict[server.tenant_id].append(server)
for project in sorted(projdict):
for server in projdict[project]:
print("%s.%s.eqiad1.wikimedia.cloud (%s)" %
(server.name, project, server.id))
