def get_hostgroups(classifier, observer_pass, regions):
hostgroups = {name: [] for name in classifier.values()}
for region in regions:
client = novaclient.Client(
"2.0",
session=KeystoneSession(auth=KeystonePassword(
auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
username="******",
password=observer_pass,
project_name="tools",
user_domain_name="default",
project_domain_name="default",
)),
region_name=region,
)
for instance in client.servers.list():
name = instance.name
if name.startswith("tools-puppetmaster"):
# To avoid chicken/egg strangeness, the tools puppetmaster
# is not itself managed by the tools puppetmaster. That
# means clush keys aren't set up there.
continue
for prefix in classifier:
if name.startswith("tools-" + prefix):
role = classifier[prefix]
hostgroups[role].append(name + ".tools.eqiad.wmflabs")
return hostgroups
def get_keystone_session(project):
return KeystoneSession(auth=KeystonePassword(
auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
username="******",
password=open('novaobserver_password').read(),
project_name=project,
user_domain_name='default',
project_domain_name='default'))
def get_projects_with_nfs(mounts_config, observer_pass, auth_url):
"""
Get populated project objects that need NFS exports
:param mounts_config: dict
:returns: list
"""
projects = []
# Special one-off session just to grab the list of regions
session = KeystoneSession(auth=KeystonePassword(
auth_url=auth_url,
username="******",
password=observer_pass,
project_name="observer",
user_domain_name="default",
project_domain_name="default",
))
keystoneclient = keystone_client.Client(session=session,
interface="public")
region_recs = keystoneclient.regions.list()
regions = [region.id for region in region_recs]
server_vols = mounts_config["volumes_served"]
for name, config in mounts_config["private"].items():
if "mounts" in config:
mounts = [
k for k, v in config["mounts"].items()
if k in server_vols and v
]
if len(mounts) == 0:
# Skip project if it has no private mounts
logging.debug("skipping exports for %s, no private mounts",
name)
continue
else:
continue
ips = get_instance_ips(name, observer_pass, regions, auth_url)
if ips:
vol = "misc"
if name == "tools":
vol = "tools"
project = Project(name, config["gid"], ips, vol)
projects.append(project)
logging.debug("project %s has %s instances", name,
len(project.instance_ips))
else:
logging.warning("project %s has no instances; skipping.", name)
# Validate that there are no duplicate gids
gids = [p.gid for p in projects]
if len(set(gids)) != len(gids):
logging.error("duplicate GIDs found in project config, aborting")
sys.exit(1)
logging.warning("found %s projects requiring private mounts",
len(projects))
return projects
def get_projects_with_nfs(mounts_config, observer_pass):
"""
Get populated project objects that need NFS exports
:param mounts_config: dict
:returns: list
"""
projects = []
# Special one-off session just to grab the list of regions
session = KeystoneSession(auth=KeystonePassword(
auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
username="******",
password=observer_pass,
project_name='observer',
user_domain_name='default',
project_domain_name='default'))
keystoneclient = keystone_client.Client(session=session,
interface='public')
region_recs = keystoneclient.regions.list()
regions = [region.id for region in region_recs]
server_vols = mounts_config['volumes_served']
for name, config in mounts_config['private'].items():
if 'mounts' in config:
mounts = [
k for k, v in config['mounts'].items()
if k in server_vols and v
]
if len(mounts) == 0:
# Skip project if it has no private mounts
logging.debug('skipping exports for %s, no private mounts',
name)
continue
else:
continue
ips = get_instance_ips(name, observer_pass, regions)
if ips:
project = Project(name, config['gid'], ips, mounts)
projects.append(project)
logging.debug('project %s has %s instances', name,
len(project.instance_ips))
else:
logging.warning('project %s has no instances; skipping.', name)
# Validate that there are no duplicate gids
gids = [p.gid for p in projects]
if len(set(gids)) != len(gids):
logging.error('duplicate GIDs found in project config, aborting')
sys.exit(1)
logging.warning("found %s projects requiring private mounts",
len(projects))
return projects
def get_keystone_session(project_name):
with open('/etc/novaobserver.yaml') as n:
nova_observer = yaml.safe_load(n)
observer_pass = nova_observer['OS_PASSWORD']
return KeystoneSession(auth=KeystonePassword(
auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
username="******",
password=observer_pass,
project_name=project_name,
user_domain_name='default',
project_domain_name='default'))
def get_regions(observer_pass):
client = keystone_client.Client(
session=KeystoneSession(auth=KeystonePassword(
auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
username="******",
password=observer_pass,
project_name="observer",
user_domain_name="default",
project_domain_name="default",
)),
interface="public",
)
return [region.id for region in client.regions.list()]
def get_instance_ips(project, observer_pass, regions):
"""
Return a list of Instance internal IPs for a given project
This uses the Nova API to fetch this data
"""
session = KeystoneSession(auth=KeystonePassword(
auth_url="http://cloudcontrol1003.wikimedia.org:5000/v3",
username="******",
password=observer_pass,
project_name=project,
user_domain_name='default',
project_domain_name='default'))
ips = []
for region in regions:
try:
client = novaclient.Client(
"2.0",
session=session,
region_name=region,
)
for instance in client.servers.list():
# Only provide internal IPs!
if 'public' in instance.addresses:
# This is a nova-network instance
for ip in instance.addresses['public']:
if ip['OS-EXT-IPS:type'] == 'fixed' and is_valid_ipv4(
ip['addr']):
ips.append(str(ip['addr']))
else:
# This is probably a neutron instance. Export all fixed
# addresses (probably there's only one)
for value in instance.addresses.values():
for ip in value:
if ip['OS-EXT-IPS:type'] == 'fixed' and is_valid_ipv4(
ip['addr']):
ips.append(str(ip['addr']))
except keystoneauth1.exceptions.http.Unauthorized:
logging.error("Failed to get server list for project %s."
" Maybe the project was deleted." % project)
ips = []
return ips
def get_instance_ips(project, observer_pass, regions, auth_url):
"""
Return a list of Instance internal IPs for a given project
This uses the Nova API to fetch this data
"""
session = KeystoneSession(auth=KeystonePassword(
auth_url=auth_url,
username="******",
password=observer_pass,
project_name=project,
user_domain_name="default",
project_domain_name="default",
))
ips = []
for region in regions:
try:
client = novaclient.Client("2.0",
session=session,
region_name=region)
for instance in client.servers.list():
# Only provide internal IPs!
if "public" in instance.addresses:
# This is a nova-network instance
for ip in instance.addresses["public"]:
if ip["OS-EXT-IPS:type"] == "fixed" and is_valid_ipv4(
ip["addr"]):
ips.append(str(ip["addr"]))
else:
# This is probably a neutron instance. Export all fixed
# addresses (probably there's only one)
for value in instance.addresses.values():
for ip in value:
if ip["OS-EXT-IPS:type"] == "fixed" and is_valid_ipv4(
ip["addr"]):
ips.append(str(ip["addr"]))
except keystoneauth1.exceptions.http.Unauthorized:
logging.error("Failed to get server list for project %s."
" Maybe the project was deleted." % project)
raise
return ips
def drvCreateCloudClient(self, credentials):
cloudConfig = self.getTargetConfiguration()
server = cloudConfig['name']
port = cloudConfig['nova_port']
projectName = cloudConfig['project_name']
try:
session = KeystoneSession()
def authenticate(self, **kwargs):
secure = kwargs.pop('secure')
authUrl = self._authUrl(server, port, secure=secure)
keystoneCli = KeystoneClient(self.KEYSTONE_API_VERSION,
tenant_name=projectName,
auth_url=authUrl,
username=credentials['username'],
password=credentials['password'],
session=session)
auth = v2_auth.Password(keystoneCli.auth_url,
username=credentials['username'],
password=credentials['password'])
session.auth = auth
keystoneCli.authenticate()
auth.auth_ref = keystoneCli.auth_ref
return keystoneCli
keystoneCli = self._secureToInsecureFallback(authenticate)
novaCli = self.NovaClientClass(auth_token=keystoneCli.auth_token,
project_id=projectName,
auth_url=keystoneCli.auth_url,
session=session)
endpoint = session.get_endpoint(service_type="image")
glanceCli = GlanceClient(self.GLANCE_CLIENT_VERSION,
endpoint=endpoint,
project_id=projectName,
token=keystoneCli.auth_token,
session=session)
clients = ConsolidatedClient(keystoneCli, novaCli, glanceCli)
except Exception, e:
raise errors.PermissionDenied(
message="Error initializing client: %s" % (e, ))
def email_admins(subject, msg):
keystone_session = KeystoneSession(auth=KeystonePassword(
auth_url=nova_observer_config['OS_AUTH_URL'],
username=nova_observer_config['OS_USERNAME'],
password=nova_observer_config['OS_PASSWORD'],
project_name=nova_observer_config['OS_PROJECT_NAME'],
user_domain_name='default',
project_domain_name='default'))
keystoneclient = keystone_client.Client(session=keystone_session,
interface='public')
roleid = None
for r in keystoneclient.roles.list():
if r.name == 'projectadmin':
roleid = r.id
break
assert roleid is not None
for ra in keystoneclient.role_assignments.list(project=project_name,
role=roleid):
dn = 'uid={},ou=people,{}'.format(ra.user['id'], ldap_config['basedn'])
_email_member(dn, subject, msg)
#!/usr/bin/python3
from keystoneclient.session import Session as KeystoneSession
from keystoneclient.auth.identity.v3 import Password as KeystonePassword
from novaclient import client as novaclient
client = novaclient.Client(
"2.0",
session=KeystoneSession(auth=KeystonePassword(
auth_url="http://{host}:{port}/v3".format(
host="cloudcontrol1003.wikimedia.org", port=5000),
username="******",
password=open('novaobserver_password').read(
), # public password for guest 'novaobserver' account
project_name='deployment-prep',
user_domain_name='default',
project_domain_name='default')))
for instance in sorted(client.servers.list(),
key=lambda instance: instance.name):
print(instance.name)
help='Exit with 0 if there are no changes and 1 if there are changes. Do not write to file',
action='store_true'
)
LUA_LINE_TEMPLATE = 'aliasmapping["{public}"] = "{private}" -- {name}\n'
args = argparser.parse_args()
config = yaml.safe_load(args.config_file)
auth = KeystonePassword(
auth_url=config['nova_api_url'],
username=config['username'],
password=config['password'],
tenant_name=config['admin_project_name']
)
keystoneClient = KeystoneClient(session=KeystoneSession(auth=auth), endpoint=config['nova_api_url'])
projects = []
for tenant in keystoneClient.tenants.list():
projects.append(tenant.name)
aliases = {}
for project in projects:
client = novaclient.Client(
"1.1",
config['username'],
config['password'],
project,
config['nova_api_url']
)
argparser.add_argument(
'--check-changes-only',
help=
'Exit with 0 if there are no changes and 1 if there are changes. Do not write to file',
action='store_true')
LUA_LINE_TEMPLATE = '{table}["{key}"] = "{value}" -- {comment}\n'
args = argparser.parse_args()
config = yaml.safe_load(args.config_file)
auth = KeystonePassword(auth_url=config['nova_api_url'],
username=config['username'],
password=config['password'],
tenant_name=config['admin_project_name'])
keystoneClient = KeystoneClient(session=KeystoneSession(auth=auth),
endpoint=config['nova_api_url'])
projects = []
for tenant in keystoneClient.tenants.list():
projects.append(tenant.name)
aliases = {}
for project in projects:
client = novaclient.Client("1.1", config['username'], config['password'],
project, config['nova_api_url'])
for server in client.servers.list():
serverAddresses = {}
try:
private = [