def __init__(self, project, common_config, source_config, dest_config):
self.source_config = source_config
self.dest_config = dest_config
self.common_config = common_config
source_auth = KeystonePassword(
auth_url=self.common_config['keystone_url'],
username=self.common_config['user'],
password=self.common_config['password'],
user_domain_name='Default',
project_domain_name='Default',
project_name=project)
source_session = keystone_session.Session(auth=source_auth)
self.source_novaclient = client.Client(
'2', session=source_session, region_name=source_config['region'])
dest_auth = KeystonePassword(
auth_url=self.common_config['keystone_url'],
username=self.common_config['user'],
password=self.common_config['password'],
user_domain_name='Default',
project_domain_name='Default',
project_name=project)
self.dest_session = keystone_session.Session(auth=dest_auth)
self.dest_neutronclient = neutronclient.Client(
session=self.dest_session, region_name=dest_config['region'])
def make_destination_vm(self, name, image_id, flavor_id):
# we need a project-scoped session to create the VM in the right project
create_auth = KeystonePassword(
auth_url=self.common_config['keystone_url'],
username=self.common_config['user'],
password=self.common_config['password'],
user_domain_name='Default',
project_domain_name='Default',
project_name=self.project_id)
create_session = keystone_session.Session(auth=create_auth)
self.create_novaclient = novaclient.Client(
'2',
session=create_session,
region_name=self.dest_config['region'])
nics = [{"net-id": self.dest_config['network_id'], "v4-fixed-ip": ''}]
inst = self.create_novaclient.servers.create(name,
image_id,
flavor_id,
nics=nics)
self.dest_instance_id = inst.id
self.refresh_instance(False)
self.wait_for_status('ACTIVE', source=False)
self.dest_host = self.dest_instance._info['OS-EXT-SRV-ATTR:host']
print("dest_instance_id: %s on %s" %
(self.dest_instance_id, self.dest_host))
def get_hostgroups(classifier, observer_pass, regions):
hostgroups = {name: [] for name in classifier.values()}
for region in regions:
client = novaclient.Client(
"2.0",
session=KeystoneSession(
auth=KeystonePassword(
auth_url="http://openstack.eqiad1.wikimediacloud.org:5000/v3",
username="******",
password=observer_pass,
project_name="tools",
user_domain_name="default",
project_domain_name="default",
)
),
region_name=region,
)
for instance in client.servers.list():
name = instance.name
if name.startswith("tools-puppetmaster"):
# To avoid chicken/egg strangeness, the tools puppetmaster
# is not itself managed by the tools puppetmaster. That
# means clush keys aren't set up there.
continue
for prefix in classifier:
if name.startswith("tools-" + prefix):
role = classifier[prefix]
hostgroups[role].append(name + ".tools.eqiad1.wikimedia.cloud")
return hostgroups
def create_keystone_session(self):
try:
auth = KeystonePassword(**self.__keystoneCredentials)
session = KeystoneSession(auth=auth, verify=self.__certificatesPath)
session.get_project_id()
return session
except Exception as error:
raise Exception('Connection to Keystone failed: {}'.format(error))
def get_projects_with_nfs(mounts_config, observer_pass, auth_url):
"""
Get populated project objects that need NFS exports
:param mounts_config: dict
:returns: list
"""
projects = []
# Special one-off session just to grab the list of regions
session = KeystoneSession(auth=KeystonePassword(
auth_url=auth_url,
username="******",
password=observer_pass,
project_name="observer",
user_domain_name="default",
project_domain_name="default",
))
keystoneclient = keystone_client.Client(session=session,
interface="public")
region_recs = keystoneclient.regions.list()
regions = [region.id for region in region_recs]
server_vols = mounts_config["volumes_served"]
for name, config in mounts_config["private"].items():
if "mounts" in config:
mounts = [
k for k, v in config["mounts"].items()
if k in server_vols and v
]
if len(mounts) == 0:
# Skip project if it has no private mounts
logging.debug("skipping exports for %s, no private mounts",
name)
continue
else:
continue
ips = get_instance_ips(name, observer_pass, regions, auth_url)
if ips:
vol = "misc"
if name == "tools":
vol = "tools"
project = Project(name, config["gid"], ips, vol)
projects.append(project)
logging.debug("project %s has %s instances", name,
len(project.instance_ips))
else:
logging.warning("project %s has no instances; skipping.", name)
# Validate that there are no duplicate gids
gids = [p.gid for p in projects]
if len(set(gids)) != len(gids):
logging.error("duplicate GIDs found in project config, aborting")
sys.exit(1)
logging.warning("found %s projects requiring private mounts",
len(projects))
return projects
def _get_keystone_session(self, project_name='admin'):
auth = KeystonePassword(
auth_url=cfg.CONF['keystone_authtoken'].www_authenticate_uri,
username=cfg.CONF['keystone_authtoken'].username,
password=cfg.CONF['keystone_authtoken'].password,
user_domain_name='Default',
project_domain_name='Default',
project_name=project_name)
return(keystone_session.Session(auth=auth))
def get_keystone_session(project_name):
with open('/etc/novaobserver.yaml') as n:
nova_observer = yaml.safe_load(n)
observer_pass = nova_observer['OS_PASSWORD']
return KeystoneSession(auth=KeystonePassword(
auth_url="http://openstack.eqiad1.wikimediacloud.org:5000/v3",
username="******",
password=observer_pass,
project_name=project_name,
user_domain_name='default',
project_domain_name='default'))
def session(self, project=None):
if not project:
project = self.project
if project not in self.sessions:
auth = KeystonePassword(auth_url=self.url,
username=self.username,
password=self.password,
user_domain_name='Default',
project_domain_name='Default',
project_name=project)
self.sessions[project] = keystone_session.Session(auth=auth)
return self.sessions[project]
def get_regions(observer_pass):
client = keystone_client.Client(
session=KeystoneSession(
auth=KeystonePassword(
auth_url="http://openstack.eqiad1.wikimediacloud.org:5000/v3",
username="******",
password=observer_pass,
project_name="observer",
user_domain_name="default",
project_domain_name="default",
)
),
interface="public",
)
return [region.id for region in client.regions.list()]
def get_instance_ips(project, observer_pass, regions, auth_url):
"""
Return a list of Instance internal IPs for a given project
This uses the Nova API to fetch this data
"""
session = KeystoneSession(auth=KeystonePassword(
auth_url=auth_url,
username="******",
password=observer_pass,
project_name=project,
user_domain_name="default",
project_domain_name="default",
))
ips = []
for region in regions:
try:
client = novaclient.Client("2.0",
session=session,
region_name=region)
for instance in client.servers.list():
# Only provide internal IPs!
if "public" in instance.addresses:
# This is a nova-network instance
for ip in instance.addresses["public"]:
if ip["OS-EXT-IPS:type"] == "fixed" and is_valid_ipv4(
ip["addr"]):
ips.append(str(ip["addr"]))
else:
# This is probably a neutron instance. Export all fixed
# addresses (probably there's only one)
for value in instance.addresses.values():
for ip in value:
if ip["OS-EXT-IPS:type"] == "fixed" and is_valid_ipv4(
ip["addr"]):
ips.append(str(ip["addr"]))
except Unauthorized:
logging.error("Failed to get server list for project %s."
" Maybe the project was deleted." % project)
raise
return ips
def email_admins(subject, msg):
keystone_session = KeystoneSession(auth=KeystonePassword(
auth_url=nova_observer_config["OS_AUTH_URL"],
username=nova_observer_config["OS_USERNAME"],
password=nova_observer_config["OS_PASSWORD"],
project_name=nova_observer_config["OS_PROJECT_NAME"],
user_domain_name="default",
project_domain_name="default",
))
keystoneclient = keystone_client.Client(session=keystone_session,
interface="public")
roleid = None
for r in keystoneclient.roles.list():
if r.name == "projectadmin":
roleid = r.id
break
assert roleid is not None
for ra in keystoneclient.role_assignments.list(project=project_name,
role=roleid):
dn = "uid={},ou=people,{}".format(ra.user["id"], ldap_config["basedn"])
_email_member(dn, subject, msg)
def main():
argparser = argparse.ArgumentParser()
argparser.add_argument('--debug',
help='Turn on debug logging',
action='store_true')
argparser.add_argument(
'--project',
default='admin-monitoring',
help='Set project to test creation for',
)
argparser.add_argument(
'--keyfile',
default='',
help='Path to SSH key file for verification',
)
argparser.add_argument(
'--bastion-ip',
default='',
help='IP of bastion to use for ssh tests',
)
argparser.add_argument(
'--user',
default='',
help='Set username (Expected to be the same across all backends)',
)
argparser.add_argument(
'--puppetmaster',
default='',
help='fqdn of the cloud frontend puppetmaster',
)
argparser.add_argument(
'--prepend',
default='test-create',
help='String to add to beginning of instance names',
)
argparser.add_argument(
'--max-pool',
default=1,
type=int,
help='Allow this many instances',
)
argparser.add_argument('--preserve-leaks',
help='Never delete failed VMs',
action='store_true')
argparser.add_argument(
'--keystone-url',
default="http://openstack.eqiad1.wikimediacloud.org:35357/v3",
help='Auth url for token and service discovery',
)
argparser.add_argument(
'--interval',
default=600,
type=int,
help='Seconds delay for daemon (default: 600 [10m])',
)
argparser.add_argument(
'--creation-timeout',
default=180,
type=int,
help='Allow this long for creation to succeed.',
)
argparser.add_argument(
'--ssh-timeout',
default=180,
type=int,
help='Allow this long for SSH to succeed.',
)
argparser.add_argument(
'--puppet-timeout',
default=120,
type=int,
help='Allow this long for Puppet to succeed.',
)
argparser.add_argument(
'--deletion-timeout',
default=120,
type=int,
help='Allow this long for delete to succeed.',
)
argparser.add_argument(
'--image',
default='debian-10.0-buster',
help='Image to use',
)
argparser.add_argument(
'--flavor',
default='m1.small',
help='Flavor to use',
)
argparser.add_argument('--skip-puppet',
help='Turn off Puppet validation',
action='store_true')
argparser.add_argument('--skip-dns',
help='Turn off DNS validation',
action='store_true')
argparser.add_argument(
'--dns-resolvers',
help='Comma separated list of nameservers',
default='208.80.154.143,208.80.154.24',
)
argparser.add_argument('--skip-ssh',
help='Turn off basic SSH validation',
action='store_true')
argparser.add_argument('--pause-for-deletion',
help='Wait for user input before deletion',
action='store_true')
argparser.add_argument('--skip-deletion',
help='Leave instance behind',
action='store_true')
argparser.add_argument(
'--virthost',
default=None,
help='Specify a particular host to launch on, e.g. labvirt1001. Default'
'behavior is to use the standard scheduling pool.',
)
argparser.add_argument(
'--adhoc-command',
default='',
help='Specify a command over SSH prior to deletion',
)
argparser.add_argument(
'--network',
default='',
help='Specify a Neutron network for VMs',
)
argparser.add_argument(
'--statsd',
default='statsd.eqiad.wmnet',
help='Send statistics to statsd endpoint',
)
args = argparser.parse_args()
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.DEBUG if args.debug else logging.INFO)
if args.adhoc_command and args.skip_ssh:
logging.error("cannot skip SSH with adhoc command specified")
sys.exit(1)
try:
with open(args.keyfile, 'r') as f:
f.read()
except OSError:
logging.error("keyfile {} cannot be read".format(args.keyfile))
sys.exit(1)
pw = os.environ.get('OS_PASSWORD')
region = os.environ.get('OS_REGION_NAME')
user = os.environ.get('OS_USERNAME') or args.user
project = os.environ.get('OS_PROJECT_ID') or args.project
if not all([user, pw, project]):
logging.error('Set username and password environment variables')
sys.exit(1)
def stat(metric, value):
metric_prepend = 'cloudvps.novafullstack.{}'.format(
socket.gethostname())
submit_stat(args.statsd, 8125, metric_prepend, metric, value)
while True:
loop_start = round(time.time(), 2)
auth = KeystonePassword(auth_url=args.keystone_url,
username=user,
password=pw,
user_domain_name='Default',
project_domain_name='Default',
project_name=project)
sess = keystone_session.Session(auth=auth)
nova_conn = nova_client.Client('2', session=sess, region_name=region)
prepend = args.prepend
date = int(datetime.today().strftime('%Y%m%d%H%M%S'))
name = '{}-{}'.format(prepend, date)
exist = nova_conn.servers.list()
logging.debug(exist)
prependinstances = [
server for server in exist if server.human_id.startswith(prepend)
]
pexist_count = len(prependinstances)
stat('instances.count', pexist_count)
stat('instances.max', args.max_pool)
# If we're pushing up against max_pool, delete the oldest server
if not args.preserve_leaks and pexist_count >= args.max_pool - 1:
logging.warning("There are {} leaked instances with prepend {}; "
"cleaning up".format(pexist_count, prepend))
servers = sorted(prependinstances,
key=lambda server: server.human_id)
servers[0].delete()
if pexist_count >= args.max_pool:
# If the cleanup in the last two cycles didn't get us anywhere,
# best to just bail out so we stop trampling on the API.
logging.error(
"max server(s) with prepend {} -- skipping creation".format(
prepend))
continue
cimage = nova_conn.glance.find_image(args.image)
cflavor = nova_conn.flavors.find(name=args.flavor)
try:
vc, server = verify_create(nova_conn, name, cimage, cflavor,
args.creation_timeout, args.network,
args.virthost)
stat('verify.creation', vc)
if 'public' in server.addresses:
addr = server.addresses['public'][0]['addr']
if not addr.startswith('10.'):
raise Exception("Bad address of {}".format(addr))
else:
addr = server.addresses['lan-flat-cloudinstances2b'][0]['addr']
if not addr.startswith('172.'):
raise Exception("Bad address of {}".format(addr))
if not args.skip_dns:
host = '{}.{}.eqiad1.wikimedia.cloud'.format(
server.name, server.tenant_id)
dnsd = args.dns_resolvers.split(',')
vdns = verify_dns(host, dnsd, timeout=2.0)
stat('verify.dns', vdns)
if not args.skip_ssh:
vs = verify_ssh(addr, user, args.keyfile, args.bastion_ip,
args.ssh_timeout)
stat('verify.ssh', vs)
if args.adhoc_command:
sshout = run_remote(addr,
user,
args.keyfile,
args.bastion_ip,
args.adhoc_command,
debug=args.debug)
logging.debug(sshout)
if not args.skip_puppet:
ps, puppetrun = verify_puppet(addr, user, args.keyfile,
args.bastion_ip,
args.puppet_timeout)
stat('verify.puppet', ps)
categories = ['changes', 'events', 'resources', 'time']
for d in categories:
for k, v in puppetrun[d].items():
stat('puppet.{}.{}'.format(d, k), v)
if args.pause_for_deletion:
logging.info("Pausing for deletion")
get_verify('continue with deletion', 'Not a valid response',
['y'])
if not args.skip_deletion:
vd = verify_deletion(nova_conn, server, args.deletion_timeout)
if not args.pause_for_deletion:
stat('verify.deletion', vd)
loop_end = time.time()
stat('verify.fullstack', round(loop_end - loop_start, 2))
if not args.skip_dns:
host = '{}.{}.eqiad1.wikimedia.cloud'.format(
server.name, server.tenant_id)
dnsd = args.dns_resolvers.split(',')
vdns = verify_dns_cleanup(host, dnsd, timeout=60.0)
stat('verify.dns-cleanup', vdns)
if not args.interval:
return
stat('verify.success', 1)
except: # noqa: E722
logging.exception("{} failed, leaking".format(name))
stat('verify.success', 0)
time.sleep(args.interval)
def __init__(self, instance_id, common_config, source_config, dest_config):
self.dnsdomain = common_config['dnsdomain']
self.instance_id = instance_id
self.source_config = source_config
self.dest_config = dest_config
self.common_config = common_config
source_auth = KeystonePassword(
auth_url=self.common_config['keystone_url'],
username=self.common_config['user'],
password=self.common_config['password'],
user_domain_name='Default',
project_domain_name='Default',
project_name='admin')
source_session = keystone_session.Session(auth=source_auth)
self.source_novaclient = novaclient.Client(
'2', session=source_session, region_name=source_config['region'])
self.refresh_instance()
self.project_id = self.source_instance.tenant_id
self.user_id = self.source_instance.user_id
project_auth = KeystonePassword(
auth_url=self.common_config['keystone_url'],
username=self.common_config['user'],
password=self.common_config['password'],
user_domain_name='Default',
project_domain_name='Default',
project_name=self.project_id)
project_session = keystone_session.Session(auth=project_auth)
self.designateclient = designateclient.Client(
session=project_session, region_name=source_config['region'])
self.novaclient_projectscope = novaclient.Client(
'2', session=project_session, region_name=dest_config['region'])
wmflabs_auth = KeystonePassword(
auth_url=self.common_config['keystone_url'],
username=self.common_config['user'],
password=self.common_config['password'],
user_domain_name='Default',
project_domain_name='Default',
project_name='wmflabsdotorg')
wmflabs_session = keystone_session.Session(auth=wmflabs_auth)
self.wmflabsdesignateclient = designateclient.Client(
session=wmflabs_session, region_name=source_config['region'])
dest_auth = KeystonePassword(
auth_url=self.common_config['keystone_url'],
username=self.common_config['user'],
password=self.common_config['password'],
user_domain_name='Default',
project_domain_name='Default',
project_name='admin')
self.dest_session = keystone_session.Session(auth=dest_auth)
self.dest_novaclient = novaclient.Client(
'2', session=self.dest_session, region_name=dest_config['region'])
self.dest_neutronclient = neutronclient.Client(
session=self.dest_session, region_name=dest_config['region'])
self.dest_keystoneclient = keystoneclient.Client(
session=self.dest_session, region_name=dest_config['region'])
self.proxy_endpoint = self.get_proxy_endpoint(self.dest_keystoneclient,
dest_config['region'])
exit(1)
config = ScriptConfig(args.datacenter, args.destination,
args.mysql_password, args.nova_db_server,
args.nova_db, args.cleanup, args.leak)
logging.basicConfig(format="%(filename)s: %(levelname)s: %(message)s",
level=logging.INFO,
stream=sys.stdout)
sshargs = [
"ssh", "-i", "/root/.ssh/compute-hosts-key",
"nova@%s" % config.destination_fqdn, "true"
]
r = subprocess.call(sshargs)
if r:
logging.error(
"remote execution failed; this whole enterprise is doomed.")
exit(1)
auth = KeystonePassword(auth_url=args.nova_url,
username=args.nova_user,
password=args.nova_pass,
user_domain_name='Default',
project_domain_name='Default',
project_name='admin')
session = keystone_session.Session(auth=auth)
instance = NovaInstance(session, args.instanceid, region=args.region)
instance.migrate(config)
logging.shutdown()