def __init__(self, name, os_type, os_version, status=None, user='******', logfile=None): self.name = name self.shortname = decanonicalize_hostname(self.name) self.os_type = os_type self.os_version = os_version self.status = status or query.get_status(self.name) self.config_path = None self.user_path = None self.user = user self.logfile = logfile self.host = decanonicalize_hostname(self.status['vm_host']['name']) self.executable = downburst_executable()
def nuke_targets(targets_dict, owner): targets = targets_dict.get('targets') if not targets: log.info("No locked machines. Not nuking anything") return to_nuke = [] for target in targets: to_nuke.append(misc.decanonicalize_hostname(target)) target_file = tempfile.NamedTemporaryFile(delete=False, mode='w+t') target_file.write(yaml.safe_dump(targets_dict)) target_file.close() log.info("Nuking machines: " + str(to_nuke)) nuke_args = [ 'teuthology-nuke', '-t', target_file.name, '--unlock', '-r', '--owner', owner ] proc = subprocess.Popen(nuke_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for line in iter(proc.stdout.readline, b''): line = line.replace(b'\r', b'').replace(b'\n', b'') log.info(ensure_str(line)) sys.stdout.flush() os.unlink(target_file.name)
def create_if_vm(ctx, machine_name, _downburst=None): """ Use downburst to create a virtual machine :param _downburst: Only used for unit testing. """ if _downburst: status_info = _downburst.status else: status_info = teuthology.lock.query.get_status(machine_name) shortname = decanonicalize_hostname(machine_name) machine_type = status_info['machine_type'] os_type = get_distro(ctx) os_version = get_distro_version(ctx) if not teuthology.lock.query.is_vm(status=status_info): return False if machine_type in cloud.get_types(): return cloud.get_provisioner( machine_type, shortname, os_type, os_version, conf=getattr(ctx, 'config', dict()), ).create() has_config = hasattr(ctx, 'config') and ctx.config is not None if has_config and 'downburst' in ctx.config: log.warning('Usage of a custom downburst config has been deprecated.') dbrst = _downburst or \ downburst.Downburst(name=machine_name, os_type=os_type, os_version=os_version, status=status_info, logfile=_logfile(ctx, shortname)) return dbrst.create()
def nuke_targets(targets_dict, owner, save_logs=False): targets = targets_dict.get('targets') if not targets: log.info("No locked machines. Not nuking anything") return to_nuke = [] for target in targets: to_nuke.append(misc.decanonicalize_hostname(target)) target_file = tempfile.NamedTemporaryFile(delete=False, mode='w+t') target_file.write(yaml.safe_dump(targets_dict)) target_file.close() log.info("Nuking machines: " + str(to_nuke)) nuke_args = ['teuthology-nuke', '-t', target_file.name, '--owner', owner] if save_logs: nuke_args.extend(['--no-reboot', '--keep-logs']) else: nuke_args.extend(['--reboot-all', '--unlock']) proc = subprocess.Popen(nuke_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for line in proc.stdout: line = line.replace(b'\r', b'').replace(b'\n', b'') log.info(line.decode()) sys.stdout.flush() os.unlink(target_file.name)
def __init__( self, provider, name, os_type=None, os_version=None, conf=None, user='******', ): if isinstance(provider, basestring): provider = teuthology.provision.cloud.get_provider(provider) self.provider = provider self.name = decanonicalize_hostname(name) self.hostname = canonicalize_hostname(name, user=None) self.os_type = os_type self.os_version = os_version self.user = user
def destroy_if_vm(ctx, machine_name, user=None, description=None, _downburst=None): """ Use downburst to destroy a virtual machine Return False only on vm downburst failures. :param _downburst: Only used for unit testing. """ if _downburst: status_info = _downburst.status else: status_info = teuthology.lock.query.get_status(machine_name) if not status_info or not teuthology.lock.query.is_vm(status=status_info): return True if user is not None and user != status_info['locked_by']: msg = "Tried to destroy {node} as {as_user} but it is locked " + \ "by {locked_by}" log.error( msg.format(node=machine_name, as_user=user, locked_by=status_info['locked_by'])) return False if (description is not None and description != status_info['description']): msg = "Tried to destroy {node} with description {desc_arg} " + \ "but it is locked with description {desc_lock}" log.error( msg.format(node=machine_name, desc_arg=description, desc_lock=status_info['description'])) return False machine_type = status_info.get('machine_type') shortname = decanonicalize_hostname(machine_name) if machine_type == 'openstack': return openstack.ProvisionOpenStack().destroy(shortname) elif machine_type in cloud.get_types(): return cloud.get_provisioner(machine_type, shortname, None, None).destroy() dbrst = _downburst or \ downburst.Downburst(name=machine_name, os_type=None, os_version=None, status=status_info, logfile=_logfile(ctx, shortname)) return dbrst.destroy()
def stale_openstack_nodes(ctx, instances, locked_nodes): names = set([i['Name'] for i in instances.values()]) for (name, node) in locked_nodes.items(): name = decanonicalize_hostname(name) if node['machine_type'] != 'openstack': continue if (name not in names and locked_since_seconds(node) > OPENSTACK_DELAY): log.info("stale-openstack: unlocking node {name} unlocked" " because it was created {created}" " seconds ago which is older than {delay}" " and it has no instance".format( name=name, created=locked_since_seconds(node), delay=OPENSTACK_DELAY)) if not ctx.dry_run: unlock_one(ctx, name, node['locked_by']) continue log.debug("stale-openstack: node " + name + " OK")
def test_decanonicalize_hostname_otherlab(self): config.lab_domain = 'example.com' host = '*****@*****.**' result = misc.decanonicalize_hostname(host) assert result == 'box1'
def test_decanonicalize_hostname_nouser(self): host = 'box1.front.sepia.ceph.com' result = misc.decanonicalize_hostname(host) assert result == 'box1'
def test_decanonicalize_hostname_full_other_user(self): config.lab_domain = 'example.com' host = '*****@*****.**' result = misc.decanonicalize_hostname(host) assert result == 'box1'
def test_decanonicalize_hostname_nodomain(self): config.lab_domain = '' host = 'ubuntu@box2' result = misc.decanonicalize_hostname(host) assert result == 'box2'
def main(ctx): if ctx.verbose: teuthology.log.setLevel(logging.DEBUG) set_config_attr(ctx) ret = 0 user = ctx.owner machines = [ misc.canonicalize_hostname(m, user=False) for m in ctx.machines ] machines_to_update = [] if ctx.targets: try: with open(ctx.targets) as f: g = yaml.safe_load_all(f) for new in g: if 'targets' in new: for t in new['targets'].iterkeys(): machines.append(t) except IOError as e: raise argparse.ArgumentTypeError(str(e)) if ctx.f: assert ctx.lock or ctx.unlock, \ '-f is only supported by --lock and --unlock' if machines: assert ctx.lock or ctx.unlock or ctx.list or ctx.list_targets \ or ctx.update or ctx.brief, \ 'machines cannot be specified with that operation' else: if ctx.lock: log.error("--lock requires specific machines passed as arguments") else: # This condition might never be hit, but it's not clear. assert ctx.num_to_lock or ctx.list or ctx.list_targets or \ ctx.summary or ctx.brief, \ 'machines must be specified for that operation' if ctx.all: assert ctx.list or ctx.list_targets or ctx.brief, \ '--all can only be used with --list, --list-targets, and --brief' assert ctx.owner is None, \ '--all and --owner are mutually exclusive' assert not machines, \ '--all and listing specific machines are incompatible' if ctx.num_to_lock: assert ctx.machine_type, \ 'must specify machine type to lock' if ctx.brief or ctx.list or ctx.list_targets: assert ctx.desc is None, '--desc does nothing with --list/--brief' # we may need to update host keys for vms. Don't do it for # every vm; however, update any vms included in the list given # to the CLI (machines), or any owned by the specified owner or # invoking user if no machines are specified. vmachines = [] statuses = query.get_statuses(machines) owner = ctx.owner or misc.get_user() for machine in statuses: if query.is_vm(status=machine) and machine['locked'] and \ (machines or machine['locked_by'] == owner): vmachines.append(machine['name']) if vmachines: log.info("updating host keys for %s", ' '.join(sorted(vmachines))) keys.do_update_keys(vmachines, _raise=False) # get statuses again to refresh any updated keys statuses = query.get_statuses(machines) if statuses: statuses = util.winnow(statuses, ctx.machine_type, 'machine_type') if not machines and ctx.owner is None and not ctx.all: ctx.owner = misc.get_user() statuses = util.winnow(statuses, ctx.owner, 'locked_by') statuses = util.winnow(statuses, ctx.status, 'up', lambda s: s['up'] == (ctx.status == 'up')) statuses = util.winnow( statuses, ctx.locked, 'locked', lambda s: s['locked'] == (ctx.locked == 'true')) statuses = util.winnow(statuses, ctx.desc, 'description') statuses = util.winnow(statuses, ctx.desc_pattern, 'description', lambda s: s['description'] and \ ctx.desc_pattern in s['description']) if ctx.json_query: statuses = util.json_matching_statuses(ctx.json_query, statuses) statuses = util.winnow(statuses, ctx.os_type, 'os_type') statuses = util.winnow(statuses, ctx.os_version, 'os_version') # When listing, only show the vm_host's name, not every detail for s in statuses: if not query.is_vm(status=s): continue # with an OpenStack API, there is no host for a VM if s['vm_host'] is None: continue vm_host_name = s.get('vm_host', dict())['name'] if vm_host_name: s['vm_host'] = vm_host_name if ctx.list: print json.dumps(statuses, indent=4) elif ctx.brief: for s in sorted(statuses, key=lambda s: s.get('name')): locked = "un" if s['locked'] == 0 else " " mo = re.match('\w+@(\w+?)\..*', s['name']) host = mo.group(1) if mo else s['name'] print '{host} {locked}locked {owner} "{desc}"'.format( locked=locked, host=host, owner=s['locked_by'], desc=s['description']) else: frag = {'targets': {}} for f in statuses: frag['targets'][f['name']] = f['ssh_pub_key'] print yaml.safe_dump(frag, default_flow_style=False) else: log.error('error retrieving lock statuses') ret = 1 elif ctx.summary: do_summary(ctx) return 0 elif ctx.lock: if not util.vps_version_or_type_valid(ctx.machine_type, ctx.os_type, ctx.os_version): log.error('Invalid os-type or version detected -- lock failed') return 1 reimage_types = teuthology.provision.fog.get_types() reimage_machines = list() updatekeys_machines = list() for machine in machines: resp = ops.lock_one(machine, user, ctx.desc) if resp.ok: machine_status = resp.json() machine_type = machine_status['machine_type'] if not resp.ok: ret = 1 if not ctx.f: return ret elif not query.is_vm(machine, machine_status): if machine_type in reimage_types: # Reimage in parallel just below here reimage_machines.append(machine) # Update keys last updatekeys_machines = list() else: machines_to_update.append(machine) teuthology.provision.create_if_vm( ctx, misc.canonicalize_hostname(machine), ) with teuthology.parallel.parallel() as p: for machine in reimage_machines: p.spawn(teuthology.provision.reimage, ctx, machine) for machine in updatekeys_machines: keys.do_update_keys([machine]) elif ctx.unlock: if ctx.owner is None and user is None: user = misc.get_user() # If none of them are vpm, do them all in one shot if not filter(query.is_vm, machines): res = ops.unlock_many(machines, user) return 0 if res else 1 for machine in machines: if not ops.unlock_one(ctx, machine, user): ret = 1 if not ctx.f: return ret else: machines_to_update.append(machine) elif ctx.num_to_lock: result = ops.lock_many(ctx, ctx.num_to_lock, ctx.machine_type, user, ctx.desc, ctx.os_type, ctx.os_version, ctx.arch) if not result: ret = 1 else: machines_to_update = result.keys() if ctx.machine_type == 'vps': shortnames = ' '.join([ misc.decanonicalize_hostname(name) for name in result.keys() ]) if len(result) < ctx.num_to_lock: log.error("Locking failed.") for machine in result: ops.unlock_one(ctx, machine, user) ret = 1 else: log.info("Successfully Locked:\n%s\n" % shortnames) log.info("Unable to display keys at this time (virtual " + "machines are booting).") log.info( "Please run teuthology-lock --list-targets %s once " + "these machines come up.", shortnames) else: print yaml.safe_dump(dict(targets=result), default_flow_style=False) elif ctx.update: assert ctx.desc is not None or ctx.status is not None, \ 'you must specify description or status to update' assert ctx.owner is None, 'only description and status may be updated' machines_to_update = machines if ctx.desc is not None or ctx.status is not None: for machine in machines_to_update: ops.update_lock(machine, ctx.desc, ctx.status) return ret
def main(ctx): if ctx.verbose: teuthology.log.setLevel(logging.DEBUG) set_config_attr(ctx) ret = 0 user = ctx.owner machines = [misc.canonicalize_hostname(m, user=False) for m in ctx.machines] machines_to_update = [] if ctx.targets: try: with file(ctx.targets) as f: g = yaml.safe_load_all(f) for new in g: if 'targets' in new: for t in new['targets'].iterkeys(): machines.append(t) except IOError as e: raise argparse.ArgumentTypeError(str(e)) if ctx.f: assert ctx.lock or ctx.unlock, \ '-f is only supported by --lock and --unlock' if machines: assert ctx.lock or ctx.unlock or ctx.list or ctx.list_targets \ or ctx.update or ctx.brief, \ 'machines cannot be specified with that operation' else: if ctx.lock: log.error("--lock requires specific machines passed as arguments") else: # This condition might never be hit, but it's not clear. assert ctx.num_to_lock or ctx.list or ctx.list_targets or \ ctx.summary or ctx.brief, \ 'machines must be specified for that operation' if ctx.all: assert ctx.list or ctx.list_targets or ctx.brief, \ '--all can only be used with --list, --list-targets, and --brief' assert ctx.owner is None, \ '--all and --owner are mutually exclusive' assert not machines, \ '--all and listing specific machines are incompatible' if ctx.num_to_lock: assert ctx.machine_type, \ 'must specify machine type to lock' if ctx.brief or ctx.list or ctx.list_targets: assert ctx.desc is None, '--desc does nothing with --list/--brief' # we may need to update host keys for vms. Don't do it for # every vm; however, update any vms included in the list given # to the CLI (machines), or any owned by the specified owner or # invoking user if no machines are specified. vmachines = [] statuses = query.get_statuses(machines) owner = ctx.owner or misc.get_user() for machine in statuses: if query.is_vm(status=machine) and machine['locked'] and \ (machines or machine['locked_by'] == owner): vmachines.append(machine['name']) if vmachines: log.info("updating host keys for %s", ' '.join(sorted(vmachines))) keys.do_update_keys(vmachines, _raise=False) # get statuses again to refresh any updated keys statuses = query.get_statuses(machines) if statuses: statuses = util.winnow(statuses, ctx.machine_type, 'machine_type') if not machines and ctx.owner is None and not ctx.all: ctx.owner = misc.get_user() statuses = util.winnow(statuses, ctx.owner, 'locked_by') statuses = util.winnow(statuses, ctx.status, 'up', lambda s: s['up'] == (ctx.status == 'up')) statuses = util.winnow(statuses, ctx.locked, 'locked', lambda s: s['locked'] == (ctx.locked == 'true')) statuses = util.winnow(statuses, ctx.desc, 'description') statuses = util.winnow(statuses, ctx.desc_pattern, 'description', lambda s: s['description'] and \ ctx.desc_pattern in s['description']) if ctx.json_query: statuses = util.json_matching_statuses(ctx.json_query, statuses) statuses = util.winnow(statuses, ctx.os_type, 'os_type') statuses = util.winnow(statuses, ctx.os_version, 'os_version') # When listing, only show the vm_host's name, not every detail for s in statuses: if not query.is_vm(status=s): continue # with an OpenStack API, there is no host for a VM if s['vm_host'] is None: continue vm_host_name = s.get('vm_host', dict())['name'] if vm_host_name: s['vm_host'] = vm_host_name if ctx.list: print json.dumps(statuses, indent=4) elif ctx.brief: for s in sorted(statuses, key=lambda s: s.get('name')): locked = "un" if s['locked'] == 0 else " " mo = re.match('\w+@(\w+?)\..*', s['name']) host = mo.group(1) if mo else s['name'] print '{host} {locked}locked {owner} "{desc}"'.format( locked=locked, host=host, owner=s['locked_by'], desc=s['description']) else: frag = {'targets': {}} for f in statuses: frag['targets'][f['name']] = f['ssh_pub_key'] print yaml.safe_dump(frag, default_flow_style=False) else: log.error('error retrieving lock statuses') ret = 1 elif ctx.summary: do_summary(ctx) return 0 elif ctx.lock: if not util.vps_version_or_type_valid( ctx.machine_type, ctx.os_type, ctx.os_version): log.error('Invalid os-type or version detected -- lock failed') return 1 reimage_types = teuthology.provision.fog.get_types() reimage_machines = list() updatekeys_machines = list() for machine in machines: resp = ops.lock_one(machine, user, ctx.desc) if resp.ok: machine_status = resp.json() machine_type = machine_status['machine_type'] if not resp.ok: ret = 1 if not ctx.f: return ret elif not query.is_vm(machine, machine_status): if machine_type in reimage_types: # Reimage in parallel just below here reimage_machines.append(machine) # Update keys last updatekeys_machines = list() else: machines_to_update.append(machine) teuthology.provision.create_if_vm( ctx, misc.canonicalize_hostname(machine), ) with teuthology.parallel.parallel() as p: for machine in reimage_machines: p.spawn(teuthology.provision.reimage, ctx, machine) for machine in updatekeys_machines: keys.do_update_keys([machine]) elif ctx.unlock: if ctx.owner is None and user is None: user = misc.get_user() # If none of them are vpm, do them all in one shot if not filter(query.is_vm, machines): res = ops.unlock_many(machines, user) return 0 if res else 1 for machine in machines: if not ops.unlock_one(ctx, machine, user): ret = 1 if not ctx.f: return ret else: machines_to_update.append(machine) elif ctx.num_to_lock: result = ops.lock_many(ctx, ctx.num_to_lock, ctx.machine_type, user, ctx.desc, ctx.os_type, ctx.os_version, ctx.arch) if not result: ret = 1 else: machines_to_update = result.keys() if ctx.machine_type == 'vps': shortnames = ' '.join( [misc.decanonicalize_hostname(name) for name in result.keys()] ) if len(result) < ctx.num_to_lock: log.error("Locking failed.") for machine in result: ops.unlock_one(ctx, machine, user) ret = 1 else: log.info("Successfully Locked:\n%s\n" % shortnames) log.info( "Unable to display keys at this time (virtual " + "machines are booting).") log.info( "Please run teuthology-lock --list-targets %s once " + "these machines come up.", shortnames) else: print yaml.safe_dump( dict(targets=result), default_flow_style=False) elif ctx.update: assert ctx.desc is not None or ctx.status is not None, \ 'you must specify description or status to update' assert ctx.owner is None, 'only description and status may be updated' machines_to_update = machines if ctx.desc is not None or ctx.status is not None: for machine in machines_to_update: ops.update_lock(machine, ctx.desc, ctx.status) return ret