def get_hostgroups(classifier, observer_pass, regions):
hostgroups = {name: [] for name in classifier.values()}
for region in regions:
client = novaclient.Client(
"2.0",
session=KeystoneSession(auth=KeystonePassword(
auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
username="******",
password=observer_pass,
project_name="tools",
user_domain_name="default",
project_domain_name="default",
)),
region_name=region,
)
for instance in client.servers.list():
name = instance.name
if name.startswith("tools-puppetmaster"):
# To avoid chicken/egg strangeness, the tools puppetmaster
# is not itself managed by the tools puppetmaster. That
# means clush keys aren't set up there.
continue
for prefix in classifier:
if name.startswith("tools-" + prefix):
role = classifier[prefix]
hostgroups[role].append(name + ".tools.eqiad.wmflabs")
return hostgroups
def get_keystone_session(project):
return KeystoneSession(auth=KeystonePassword(
auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
username="******",
password=open('novaobserver_password').read(),
project_name=project,
user_domain_name='default',
project_domain_name='default'))
def get_projects_with_nfs(mounts_config, observer_pass, auth_url):
"""
Get populated project objects that need NFS exports
:param mounts_config: dict
:returns: list
"""
projects = []
# Special one-off session just to grab the list of regions
session = KeystoneSession(auth=KeystonePassword(
auth_url=auth_url,
username="******",
password=observer_pass,
project_name="observer",
user_domain_name="default",
project_domain_name="default",
))
keystoneclient = keystone_client.Client(session=session,
interface="public")
region_recs = keystoneclient.regions.list()
regions = [region.id for region in region_recs]
server_vols = mounts_config["volumes_served"]
for name, config in mounts_config["private"].items():
if "mounts" in config:
mounts = [
k for k, v in config["mounts"].items()
if k in server_vols and v
]
if len(mounts) == 0:
# Skip project if it has no private mounts
logging.debug("skipping exports for %s, no private mounts",
name)
continue
else:
continue
ips = get_instance_ips(name, observer_pass, regions, auth_url)
if ips:
vol = "misc"
if name == "tools":
vol = "tools"
project = Project(name, config["gid"], ips, vol)
projects.append(project)
logging.debug("project %s has %s instances", name,
len(project.instance_ips))
else:
logging.warning("project %s has no instances; skipping.", name)
# Validate that there are no duplicate gids
gids = [p.gid for p in projects]
if len(set(gids)) != len(gids):
logging.error("duplicate GIDs found in project config, aborting")
sys.exit(1)
logging.warning("found %s projects requiring private mounts",
len(projects))
return projects
def get_projects_with_nfs(mounts_config, observer_pass):
"""
Get populated project objects that need NFS exports
:param mounts_config: dict
:returns: list
"""
projects = []
# Special one-off session just to grab the list of regions
session = KeystoneSession(auth=KeystonePassword(
auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
username="******",
password=observer_pass,
project_name='observer',
user_domain_name='default',
project_domain_name='default'))
keystoneclient = keystone_client.Client(session=session,
interface='public')
region_recs = keystoneclient.regions.list()
regions = [region.id for region in region_recs]
server_vols = mounts_config['volumes_served']
for name, config in mounts_config['private'].items():
if 'mounts' in config:
mounts = [
k for k, v in config['mounts'].items()
if k in server_vols and v
]
if len(mounts) == 0:
# Skip project if it has no private mounts
logging.debug('skipping exports for %s, no private mounts',
name)
continue
else:
continue
ips = get_instance_ips(name, observer_pass, regions)
if ips:
project = Project(name, config['gid'], ips, mounts)
projects.append(project)
logging.debug('project %s has %s instances', name,
len(project.instance_ips))
else:
logging.warning('project %s has no instances; skipping.', name)
# Validate that there are no duplicate gids
gids = [p.gid for p in projects]
if len(set(gids)) != len(gids):
logging.error('duplicate GIDs found in project config, aborting')
sys.exit(1)
logging.warning("found %s projects requiring private mounts",
len(projects))
return projects
def get_keystone_session(project_name):
with open('/etc/novaobserver.yaml') as n:
nova_observer = yaml.safe_load(n)
observer_pass = nova_observer['OS_PASSWORD']
return KeystoneSession(auth=KeystonePassword(
auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
username="******",
password=observer_pass,
project_name=project_name,
user_domain_name='default',
project_domain_name='default'))
def get_regions(observer_pass):
client = keystone_client.Client(
session=KeystoneSession(auth=KeystonePassword(
auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
username="******",
password=observer_pass,
project_name="observer",
user_domain_name="default",
project_domain_name="default",
)),
interface="public",
)
return [region.id for region in client.regions.list()]
def get_instance_ips(project, observer_pass, regions):
"""
Return a list of Instance internal IPs for a given project
This uses the Nova API to fetch this data
"""
session = KeystoneSession(auth=KeystonePassword(
auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
username="******",
password=observer_pass,
project_name=project,
user_domain_name='default',
project_domain_name='default'))
ips = []
for region in regions:
try:
client = novaclient.Client(
"2.0",
session=session,
region_name=region,
)
for instance in client.servers.list():
# Only provide internal IPs!
if 'public' in instance.addresses:
# This is a nova-network instance
for ip in instance.addresses['public']:
if ip['OS-EXT-IPS:type'] == 'fixed' and is_valid_ipv4(
ip['addr']):
ips.append(str(ip['addr']))
else:
# This is probably a neutron instance. Export all fixed
# addresses (probably there's only one)
for value in instance.addresses.values():
for ip in value:
if ip['OS-EXT-IPS:type'] == 'fixed' and is_valid_ipv4(
ip['addr']):
ips.append(str(ip['addr']))
except keystoneauth1.exceptions.http.Unauthorized:
logging.error("Failed to get server list for project %s."
" Maybe the project was deleted." % project)
ips = []
return ips
def get_instance_ips(project, observer_pass, regions, auth_url):
"""
Return a list of Instance internal IPs for a given project
This uses the Nova API to fetch this data
"""
session = KeystoneSession(auth=KeystonePassword(
auth_url=auth_url,
username="******",
password=observer_pass,
project_name=project,
user_domain_name="default",
project_domain_name="default",
))
ips = []
for region in regions:
try:
client = novaclient.Client("2.0",
session=session,
region_name=region)
for instance in client.servers.list():
# Only provide internal IPs!
if "public" in instance.addresses:
# This is a nova-network instance
for ip in instance.addresses["public"]:
if ip["OS-EXT-IPS:type"] == "fixed" and is_valid_ipv4(
ip["addr"]):
ips.append(str(ip["addr"]))
else:
# This is probably a neutron instance. Export all fixed
# addresses (probably there's only one)
for value in instance.addresses.values():
for ip in value:
if ip["OS-EXT-IPS:type"] == "fixed" and is_valid_ipv4(
ip["addr"]):
ips.append(str(ip["addr"]))
except keystoneauth1.exceptions.http.Unauthorized:
logging.error("Failed to get server list for project %s."
" Maybe the project was deleted." % project)
raise
return ips
def email_admins(subject, msg):
keystone_session = KeystoneSession(auth=KeystonePassword(
auth_url=nova_observer_config['OS_AUTH_URL'],
username=nova_observer_config['OS_USERNAME'],
password=nova_observer_config['OS_PASSWORD'],
project_name=nova_observer_config['OS_PROJECT_NAME'],
user_domain_name='default',
project_domain_name='default'))
keystoneclient = keystone_client.Client(session=keystone_session,
interface='public')
roleid = None
for r in keystoneclient.roles.list():
if r.name == 'projectadmin':
roleid = r.id
break
assert roleid is not None
for ra in keystoneclient.role_assignments.list(project=project_name,
role=roleid):
dn = 'uid={},ou=people,{}'.format(ra.user['id'], ldap_config['basedn'])
_email_member(dn, subject, msg)
#!/usr/bin/python3
from keystoneclient.session import Session as KeystoneSession
from keystoneclient.auth.identity.v3 import Password as KeystonePassword
from novaclient import client as novaclient
client = novaclient.Client(
"2.0",
session=KeystoneSession(auth=KeystonePassword(
auth_url="http://{host}:{port}/v3".format(
host="cloudcontrol1003.wikimedia.org", port=5000),
username="******",
password=open('novaobserver_password').read(
), # public password for guest 'novaobserver' account
project_name='deployment-prep',
user_domain_name='default',
project_domain_name='default')))
for instance in sorted(client.servers.list(),
key=lambda instance: instance.name):
print(instance.name)
