def __init__(self, name, os_type, os_version, status=None, user='******',
logfile=None):
self.name = name
self.shortname = decanonicalize_hostname(self.name)
self.os_type = os_type
self.os_version = os_version
self.status = status or query.get_status(self.name)
self.config_path = None
self.user_path = None
self.user = user
self.logfile = logfile
self.host = decanonicalize_hostname(self.status['vm_host']['name'])
self.executable = downburst_executable()
def nuke_targets(targets_dict, owner):
targets = targets_dict.get('targets')
if not targets:
log.info("No locked machines. Not nuking anything")
return
to_nuke = []
for target in targets:
to_nuke.append(misc.decanonicalize_hostname(target))
target_file = tempfile.NamedTemporaryFile(delete=False, mode='w+t')
target_file.write(yaml.safe_dump(targets_dict))
target_file.close()
log.info("Nuking machines: " + str(to_nuke))
nuke_args = [
'teuthology-nuke', '-t', target_file.name, '--unlock', '-r', '--owner',
owner
]
proc = subprocess.Popen(nuke_args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
for line in iter(proc.stdout.readline, b''):
line = line.replace(b'\r', b'').replace(b'\n', b'')
log.info(ensure_str(line))
sys.stdout.flush()
os.unlink(target_file.name)
def create_if_vm(ctx, machine_name, _downburst=None):
"""
Use downburst to create a virtual machine
:param _downburst: Only used for unit testing.
"""
if _downburst:
status_info = _downburst.status
else:
status_info = teuthology.lock.query.get_status(machine_name)
shortname = decanonicalize_hostname(machine_name)
machine_type = status_info['machine_type']
os_type = get_distro(ctx)
os_version = get_distro_version(ctx)
if not teuthology.lock.query.is_vm(status=status_info):
return False
if machine_type in cloud.get_types():
return cloud.get_provisioner(
machine_type,
shortname,
os_type,
os_version,
conf=getattr(ctx, 'config', dict()),
).create()
has_config = hasattr(ctx, 'config') and ctx.config is not None
if has_config and 'downburst' in ctx.config:
log.warning('Usage of a custom downburst config has been deprecated.')
dbrst = _downburst or \
downburst.Downburst(name=machine_name, os_type=os_type,
os_version=os_version, status=status_info,
logfile=_logfile(ctx, shortname))
return dbrst.create()
def nuke_targets(targets_dict, owner, save_logs=False):
targets = targets_dict.get('targets')
if not targets:
log.info("No locked machines. Not nuking anything")
return
to_nuke = []
for target in targets:
to_nuke.append(misc.decanonicalize_hostname(target))
target_file = tempfile.NamedTemporaryFile(delete=False, mode='w+t')
target_file.write(yaml.safe_dump(targets_dict))
target_file.close()
log.info("Nuking machines: " + str(to_nuke))
nuke_args = ['teuthology-nuke', '-t', target_file.name, '--owner', owner]
if save_logs:
nuke_args.extend(['--no-reboot', '--keep-logs'])
else:
nuke_args.extend(['--reboot-all', '--unlock'])
proc = subprocess.Popen(nuke_args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
for line in proc.stdout:
line = line.replace(b'\r', b'').replace(b'\n', b'')
log.info(line.decode())
sys.stdout.flush()
os.unlink(target_file.name)
def __init__(
self, provider, name, os_type=None, os_version=None,
conf=None, user='******',
):
if isinstance(provider, basestring):
provider = teuthology.provision.cloud.get_provider(provider)
self.provider = provider
self.name = decanonicalize_hostname(name)
self.hostname = canonicalize_hostname(name, user=None)
self.os_type = os_type
self.os_version = os_version
self.user = user
def destroy_if_vm(ctx,
machine_name,
user=None,
description=None,
_downburst=None):
"""
Use downburst to destroy a virtual machine
Return False only on vm downburst failures.
:param _downburst: Only used for unit testing.
"""
if _downburst:
status_info = _downburst.status
else:
status_info = teuthology.lock.query.get_status(machine_name)
if not status_info or not teuthology.lock.query.is_vm(status=status_info):
return True
if user is not None and user != status_info['locked_by']:
msg = "Tried to destroy {node} as {as_user} but it is locked " + \
"by {locked_by}"
log.error(
msg.format(node=machine_name,
as_user=user,
locked_by=status_info['locked_by']))
return False
if (description is not None and description != status_info['description']):
msg = "Tried to destroy {node} with description {desc_arg} " + \
"but it is locked with description {desc_lock}"
log.error(
msg.format(node=machine_name,
desc_arg=description,
desc_lock=status_info['description']))
return False
machine_type = status_info.get('machine_type')
shortname = decanonicalize_hostname(machine_name)
if machine_type == 'openstack':
return openstack.ProvisionOpenStack().destroy(shortname)
elif machine_type in cloud.get_types():
return cloud.get_provisioner(machine_type, shortname, None,
None).destroy()
dbrst = _downburst or \
downburst.Downburst(name=machine_name, os_type=None,
os_version=None, status=status_info,
logfile=_logfile(ctx, shortname))
return dbrst.destroy()
def __init__(
self,
provider,
name,
os_type=None,
os_version=None,
conf=None,
user='******',
):
if isinstance(provider, basestring):
provider = teuthology.provision.cloud.get_provider(provider)
self.provider = provider
self.name = decanonicalize_hostname(name)
self.hostname = canonicalize_hostname(name, user=None)
self.os_type = os_type
self.os_version = os_version
self.user = user
def stale_openstack_nodes(ctx, instances, locked_nodes):
names = set([i['Name'] for i in instances.values()])
for (name, node) in locked_nodes.items():
name = decanonicalize_hostname(name)
if node['machine_type'] != 'openstack':
continue
if (name not in names
and locked_since_seconds(node) > OPENSTACK_DELAY):
log.info("stale-openstack: unlocking node {name} unlocked"
" because it was created {created}"
" seconds ago which is older than {delay}"
" and it has no instance".format(
name=name,
created=locked_since_seconds(node),
delay=OPENSTACK_DELAY))
if not ctx.dry_run:
unlock_one(ctx, name, node['locked_by'])
continue
log.debug("stale-openstack: node " + name + " OK")
def test_decanonicalize_hostname_otherlab(self):
config.lab_domain = 'example.com'
host = '*****@*****.**'
result = misc.decanonicalize_hostname(host)
assert result == 'box1'
def test_decanonicalize_hostname_nouser(self):
host = 'box1.front.sepia.ceph.com'
result = misc.decanonicalize_hostname(host)
assert result == 'box1'
def test_decanonicalize_hostname_full_other_user(self):
config.lab_domain = 'example.com'
host = '*****@*****.**'
result = misc.decanonicalize_hostname(host)
assert result == 'box1'
def test_decanonicalize_hostname_nodomain(self):
config.lab_domain = ''
host = 'ubuntu@box2'
result = misc.decanonicalize_hostname(host)
assert result == 'box2'
def main(ctx):
if ctx.verbose:
teuthology.log.setLevel(logging.DEBUG)
set_config_attr(ctx)
ret = 0
user = ctx.owner
machines = [
misc.canonicalize_hostname(m, user=False) for m in ctx.machines
]
machines_to_update = []
if ctx.targets:
try:
with open(ctx.targets) as f:
g = yaml.safe_load_all(f)
for new in g:
if 'targets' in new:
for t in new['targets'].iterkeys():
machines.append(t)
except IOError as e:
raise argparse.ArgumentTypeError(str(e))
if ctx.f:
assert ctx.lock or ctx.unlock, \
'-f is only supported by --lock and --unlock'
if machines:
assert ctx.lock or ctx.unlock or ctx.list or ctx.list_targets \
or ctx.update or ctx.brief, \
'machines cannot be specified with that operation'
else:
if ctx.lock:
log.error("--lock requires specific machines passed as arguments")
else:
# This condition might never be hit, but it's not clear.
assert ctx.num_to_lock or ctx.list or ctx.list_targets or \
ctx.summary or ctx.brief, \
'machines must be specified for that operation'
if ctx.all:
assert ctx.list or ctx.list_targets or ctx.brief, \
'--all can only be used with --list, --list-targets, and --brief'
assert ctx.owner is None, \
'--all and --owner are mutually exclusive'
assert not machines, \
'--all and listing specific machines are incompatible'
if ctx.num_to_lock:
assert ctx.machine_type, \
'must specify machine type to lock'
if ctx.brief or ctx.list or ctx.list_targets:
assert ctx.desc is None, '--desc does nothing with --list/--brief'
# we may need to update host keys for vms. Don't do it for
# every vm; however, update any vms included in the list given
# to the CLI (machines), or any owned by the specified owner or
# invoking user if no machines are specified.
vmachines = []
statuses = query.get_statuses(machines)
owner = ctx.owner or misc.get_user()
for machine in statuses:
if query.is_vm(status=machine) and machine['locked'] and \
(machines or machine['locked_by'] == owner):
vmachines.append(machine['name'])
if vmachines:
log.info("updating host keys for %s", ' '.join(sorted(vmachines)))
keys.do_update_keys(vmachines, _raise=False)
# get statuses again to refresh any updated keys
statuses = query.get_statuses(machines)
if statuses:
statuses = util.winnow(statuses, ctx.machine_type, 'machine_type')
if not machines and ctx.owner is None and not ctx.all:
ctx.owner = misc.get_user()
statuses = util.winnow(statuses, ctx.owner, 'locked_by')
statuses = util.winnow(statuses, ctx.status, 'up',
lambda s: s['up'] == (ctx.status == 'up'))
statuses = util.winnow(
statuses, ctx.locked, 'locked', lambda s: s['locked'] ==
(ctx.locked == 'true'))
statuses = util.winnow(statuses, ctx.desc, 'description')
statuses = util.winnow(statuses, ctx.desc_pattern, 'description',
lambda s: s['description'] and \
ctx.desc_pattern in s['description'])
if ctx.json_query:
statuses = util.json_matching_statuses(ctx.json_query,
statuses)
statuses = util.winnow(statuses, ctx.os_type, 'os_type')
statuses = util.winnow(statuses, ctx.os_version, 'os_version')
# When listing, only show the vm_host's name, not every detail
for s in statuses:
if not query.is_vm(status=s):
continue
# with an OpenStack API, there is no host for a VM
if s['vm_host'] is None:
continue
vm_host_name = s.get('vm_host', dict())['name']
if vm_host_name:
s['vm_host'] = vm_host_name
if ctx.list:
print json.dumps(statuses, indent=4)
elif ctx.brief:
for s in sorted(statuses, key=lambda s: s.get('name')):
locked = "un" if s['locked'] == 0 else " "
mo = re.match('\w+@(\w+?)\..*', s['name'])
host = mo.group(1) if mo else s['name']
print '{host} {locked}locked {owner} "{desc}"'.format(
locked=locked,
host=host,
owner=s['locked_by'],
desc=s['description'])
else:
frag = {'targets': {}}
for f in statuses:
frag['targets'][f['name']] = f['ssh_pub_key']
print yaml.safe_dump(frag, default_flow_style=False)
else:
log.error('error retrieving lock statuses')
ret = 1
elif ctx.summary:
do_summary(ctx)
return 0
elif ctx.lock:
if not util.vps_version_or_type_valid(ctx.machine_type, ctx.os_type,
ctx.os_version):
log.error('Invalid os-type or version detected -- lock failed')
return 1
reimage_types = teuthology.provision.fog.get_types()
reimage_machines = list()
updatekeys_machines = list()
for machine in machines:
resp = ops.lock_one(machine, user, ctx.desc)
if resp.ok:
machine_status = resp.json()
machine_type = machine_status['machine_type']
if not resp.ok:
ret = 1
if not ctx.f:
return ret
elif not query.is_vm(machine, machine_status):
if machine_type in reimage_types:
# Reimage in parallel just below here
reimage_machines.append(machine)
# Update keys last
updatekeys_machines = list()
else:
machines_to_update.append(machine)
teuthology.provision.create_if_vm(
ctx,
misc.canonicalize_hostname(machine),
)
with teuthology.parallel.parallel() as p:
for machine in reimage_machines:
p.spawn(teuthology.provision.reimage, ctx, machine)
for machine in updatekeys_machines:
keys.do_update_keys([machine])
elif ctx.unlock:
if ctx.owner is None and user is None:
user = misc.get_user()
# If none of them are vpm, do them all in one shot
if not filter(query.is_vm, machines):
res = ops.unlock_many(machines, user)
return 0 if res else 1
for machine in machines:
if not ops.unlock_one(ctx, machine, user):
ret = 1
if not ctx.f:
return ret
else:
machines_to_update.append(machine)
elif ctx.num_to_lock:
result = ops.lock_many(ctx, ctx.num_to_lock, ctx.machine_type, user,
ctx.desc, ctx.os_type, ctx.os_version, ctx.arch)
if not result:
ret = 1
else:
machines_to_update = result.keys()
if ctx.machine_type == 'vps':
shortnames = ' '.join([
misc.decanonicalize_hostname(name)
for name in result.keys()
])
if len(result) < ctx.num_to_lock:
log.error("Locking failed.")
for machine in result:
ops.unlock_one(ctx, machine, user)
ret = 1
else:
log.info("Successfully Locked:\n%s\n" % shortnames)
log.info("Unable to display keys at this time (virtual " +
"machines are booting).")
log.info(
"Please run teuthology-lock --list-targets %s once " +
"these machines come up.", shortnames)
else:
print yaml.safe_dump(dict(targets=result),
default_flow_style=False)
elif ctx.update:
assert ctx.desc is not None or ctx.status is not None, \
'you must specify description or status to update'
assert ctx.owner is None, 'only description and status may be updated'
machines_to_update = machines
if ctx.desc is not None or ctx.status is not None:
for machine in machines_to_update:
ops.update_lock(machine, ctx.desc, ctx.status)
return ret
def main(ctx):
if ctx.verbose:
teuthology.log.setLevel(logging.DEBUG)
set_config_attr(ctx)
ret = 0
user = ctx.owner
machines = [misc.canonicalize_hostname(m, user=False)
for m in ctx.machines]
machines_to_update = []
if ctx.targets:
try:
with file(ctx.targets) as f:
g = yaml.safe_load_all(f)
for new in g:
if 'targets' in new:
for t in new['targets'].iterkeys():
machines.append(t)
except IOError as e:
raise argparse.ArgumentTypeError(str(e))
if ctx.f:
assert ctx.lock or ctx.unlock, \
'-f is only supported by --lock and --unlock'
if machines:
assert ctx.lock or ctx.unlock or ctx.list or ctx.list_targets \
or ctx.update or ctx.brief, \
'machines cannot be specified with that operation'
else:
if ctx.lock:
log.error("--lock requires specific machines passed as arguments")
else:
# This condition might never be hit, but it's not clear.
assert ctx.num_to_lock or ctx.list or ctx.list_targets or \
ctx.summary or ctx.brief, \
'machines must be specified for that operation'
if ctx.all:
assert ctx.list or ctx.list_targets or ctx.brief, \
'--all can only be used with --list, --list-targets, and --brief'
assert ctx.owner is None, \
'--all and --owner are mutually exclusive'
assert not machines, \
'--all and listing specific machines are incompatible'
if ctx.num_to_lock:
assert ctx.machine_type, \
'must specify machine type to lock'
if ctx.brief or ctx.list or ctx.list_targets:
assert ctx.desc is None, '--desc does nothing with --list/--brief'
# we may need to update host keys for vms. Don't do it for
# every vm; however, update any vms included in the list given
# to the CLI (machines), or any owned by the specified owner or
# invoking user if no machines are specified.
vmachines = []
statuses = query.get_statuses(machines)
owner = ctx.owner or misc.get_user()
for machine in statuses:
if query.is_vm(status=machine) and machine['locked'] and \
(machines or machine['locked_by'] == owner):
vmachines.append(machine['name'])
if vmachines:
log.info("updating host keys for %s", ' '.join(sorted(vmachines)))
keys.do_update_keys(vmachines, _raise=False)
# get statuses again to refresh any updated keys
statuses = query.get_statuses(machines)
if statuses:
statuses = util.winnow(statuses, ctx.machine_type, 'machine_type')
if not machines and ctx.owner is None and not ctx.all:
ctx.owner = misc.get_user()
statuses = util.winnow(statuses, ctx.owner, 'locked_by')
statuses = util.winnow(statuses, ctx.status, 'up',
lambda s: s['up'] == (ctx.status == 'up'))
statuses = util.winnow(statuses, ctx.locked, 'locked',
lambda s: s['locked'] == (ctx.locked == 'true'))
statuses = util.winnow(statuses, ctx.desc, 'description')
statuses = util.winnow(statuses, ctx.desc_pattern, 'description',
lambda s: s['description'] and \
ctx.desc_pattern in s['description'])
if ctx.json_query:
statuses = util.json_matching_statuses(ctx.json_query, statuses)
statuses = util.winnow(statuses, ctx.os_type, 'os_type')
statuses = util.winnow(statuses, ctx.os_version, 'os_version')
# When listing, only show the vm_host's name, not every detail
for s in statuses:
if not query.is_vm(status=s):
continue
# with an OpenStack API, there is no host for a VM
if s['vm_host'] is None:
continue
vm_host_name = s.get('vm_host', dict())['name']
if vm_host_name:
s['vm_host'] = vm_host_name
if ctx.list:
print json.dumps(statuses, indent=4)
elif ctx.brief:
for s in sorted(statuses, key=lambda s: s.get('name')):
locked = "un" if s['locked'] == 0 else " "
mo = re.match('\w+@(\w+?)\..*', s['name'])
host = mo.group(1) if mo else s['name']
print '{host} {locked}locked {owner} "{desc}"'.format(
locked=locked, host=host,
owner=s['locked_by'], desc=s['description'])
else:
frag = {'targets': {}}
for f in statuses:
frag['targets'][f['name']] = f['ssh_pub_key']
print yaml.safe_dump(frag, default_flow_style=False)
else:
log.error('error retrieving lock statuses')
ret = 1
elif ctx.summary:
do_summary(ctx)
return 0
elif ctx.lock:
if not util.vps_version_or_type_valid(
ctx.machine_type, ctx.os_type, ctx.os_version):
log.error('Invalid os-type or version detected -- lock failed')
return 1
reimage_types = teuthology.provision.fog.get_types()
reimage_machines = list()
updatekeys_machines = list()
for machine in machines:
resp = ops.lock_one(machine, user, ctx.desc)
if resp.ok:
machine_status = resp.json()
machine_type = machine_status['machine_type']
if not resp.ok:
ret = 1
if not ctx.f:
return ret
elif not query.is_vm(machine, machine_status):
if machine_type in reimage_types:
# Reimage in parallel just below here
reimage_machines.append(machine)
# Update keys last
updatekeys_machines = list()
else:
machines_to_update.append(machine)
teuthology.provision.create_if_vm(
ctx,
misc.canonicalize_hostname(machine),
)
with teuthology.parallel.parallel() as p:
for machine in reimage_machines:
p.spawn(teuthology.provision.reimage, ctx, machine)
for machine in updatekeys_machines:
keys.do_update_keys([machine])
elif ctx.unlock:
if ctx.owner is None and user is None:
user = misc.get_user()
# If none of them are vpm, do them all in one shot
if not filter(query.is_vm, machines):
res = ops.unlock_many(machines, user)
return 0 if res else 1
for machine in machines:
if not ops.unlock_one(ctx, machine, user):
ret = 1
if not ctx.f:
return ret
else:
machines_to_update.append(machine)
elif ctx.num_to_lock:
result = ops.lock_many(ctx, ctx.num_to_lock, ctx.machine_type, user,
ctx.desc, ctx.os_type, ctx.os_version, ctx.arch)
if not result:
ret = 1
else:
machines_to_update = result.keys()
if ctx.machine_type == 'vps':
shortnames = ' '.join(
[misc.decanonicalize_hostname(name) for name in
result.keys()]
)
if len(result) < ctx.num_to_lock:
log.error("Locking failed.")
for machine in result:
ops.unlock_one(ctx, machine, user)
ret = 1
else:
log.info("Successfully Locked:\n%s\n" % shortnames)
log.info(
"Unable to display keys at this time (virtual " +
"machines are booting).")
log.info(
"Please run teuthology-lock --list-targets %s once " +
"these machines come up.",
shortnames)
else:
print yaml.safe_dump(
dict(targets=result),
default_flow_style=False)
elif ctx.update:
assert ctx.desc is not None or ctx.status is not None, \
'you must specify description or status to update'
assert ctx.owner is None, 'only description and status may be updated'
machines_to_update = machines
if ctx.desc is not None or ctx.status is not None:
for machine in machines_to_update:
ops.update_lock(machine, ctx.desc, ctx.status)
return ret
