def lock_many(ctx, num, machine_type, user=None, description=None, os_type=None, os_version=None, arch=None): if user is None: user = misc.get_user() if not util.vps_version_or_type_valid(ctx.machine_type, os_type, os_version): log.error('Invalid os-type or version detected -- lock failed') return # In the for loop below we can safely query for all bare-metal machine_type # values at once. So, if we're being asked for 'plana,mira,burnupi', do it # all in one shot. If we are passed 'plana,mira,burnupi,vps', do one query # for 'plana,mira,burnupi' and one for 'vps' machine_types_list = misc.get_multi_machine_types(machine_type) if machine_types_list == ['vps']: machine_types = machine_types_list elif machine_types_list == ['openstack']: return lock_many_openstack(ctx, num, machine_type, user=user, description=description, arch=arch) elif 'vps' in machine_types_list: machine_types_non_vps = list(machine_types_list) machine_types_non_vps.remove('vps') machine_types_non_vps = '|'.join(machine_types_non_vps) machine_types = [machine_types_non_vps, 'vps'] else: machine_types_str = '|'.join(machine_types_list) machine_types = [ machine_types_str, ] for machine_type in machine_types: uri = os.path.join(config.lock_server, 'nodes', 'lock_many', '') data = dict( locked_by=user, count=num, machine_type=machine_type, description=description, ) # Only query for os_type/os_version if non-vps and non-libcloud, since # in that case we just create them. vm_types = ['vps'] + teuthology.provision.cloud.get_types() reimage_types = teuthology.provision.get_reimage_types() if machine_type not in vm_types + reimage_types: if os_type: data['os_type'] = os_type if os_version: data['os_version'] = os_version if arch: data['arch'] = arch log.debug("lock_many request: %s", repr(data)) response = requests.post( uri, data=json.dumps(data), headers={'content-type': 'application/json'}, ) if response.ok: machines = { misc.canonicalize_hostname(machine['name']): machine['ssh_pub_key'] for machine in response.json() } log.debug('locked {machines}'.format( machines=', '.join(machines.keys()))) if machine_type in vm_types: ok_machs = {} update_nodes(machines, True) for machine in machines: if teuthology.provision.create_if_vm(ctx, machine): ok_machs[machine] = machines[machine] else: log.error('Unable to create virtual machine: %s', machine) unlock_one(ctx, machine, user) ok_machs = do_update_keys(list(ok_machs.keys()))[1] update_nodes(ok_machs) return ok_machs elif machine_type in reimage_types: reimaged = dict() console_log_conf = dict( logfile_name='{shortname}_reimage.log', remotes=[ teuthology.orchestra.remote.Remote(machine) for machine in machines ], ) with console_log.task(ctx, console_log_conf): update_nodes(reimaged, True) with teuthology.parallel.parallel() as p: for machine in machines: p.spawn(teuthology.provision.reimage, ctx, machine, machine_type) reimaged[machine] = machines[machine] reimaged = do_update_keys(reimaged.keys())[1] update_nodes(reimaged) return reimaged return machines elif response.status_code == 503: log.error('Insufficient nodes available to lock %d %s nodes.', num, machine_type) log.error(response.text) else: log.error('Could not lock %d %s nodes, reason: unknown.', num, machine_type) return []
def lock_machines(ctx, config): """ Lock machines. Called when the teuthology run finds and locks new machines. This is not called if the one has teuthology-locked machines and placed those keys in the Targets section of a yaml file. """ log.info('Locking machines...') assert isinstance(config[0], int), 'config[0] must be an integer' machine_type = config[1] machine_types = teuthology.get_multi_machine_types(machine_type) how_many = config[0] while True: # make sure there are enough machines up machines = lock.list_locks() if machines is None: if ctx.block: log.warn('error listing machines, trying again') time.sleep(20) continue else: assert 0, 'error listing machines' is_up = lambda machine: machine['up'] and machine['type'] in machine_types # noqa num_up = len(filter(is_up, machines)) assert num_up >= how_many, 'not enough machines are up' # make sure there are machines for non-automated jobs to run is_up_and_free = lambda machine: machine['up'] and machine['locked'] == 0 and machine['type'] in machine_types # noqa up_and_free = filter(is_up_and_free, machines) num_free = len(up_and_free) if num_free < 6 and ctx.owner.startswith('scheduled'): if ctx.block: log.info( 'waiting for more machines to be free (need %s see %s)...', how_many, num_free, ) time.sleep(10) continue else: assert 0, 'not enough machines free' newly_locked = lock.lock_many(ctx, how_many, machine_type, ctx.owner, ctx.archive) if len(newly_locked) == how_many: vmlist = [] for lmach in newly_locked: if teuthology.is_vm(lmach): vmlist.append(lmach) if vmlist: log.info('Waiting for virtual machines to come up') keyscan_out = '' loopcount = 0 while len(keyscan_out.splitlines()) != len(vmlist): loopcount += 1 time.sleep(10) keyscan_out, current_locks = lock.keyscan_check(ctx, vmlist) log.info('virtual machine is still unavailable') if loopcount == 40: loopcount = 0 log.info('virtual machine(s) still not up, ' + 'recreating unresponsive ones.') for guest in vmlist: if guest not in keyscan_out: log.info('recreating: ' + guest) lock.destroy_if_vm(ctx, 'ubuntu@' + guest) lock.create_if_vm(ctx, 'ubuntu@' + guest) if lock.update_keys(ctx, keyscan_out, current_locks): log.info("Error in virtual machine keys") newscandict = {} for dkey in newly_locked.iterkeys(): stats = lockstatus.get_status(ctx, dkey) newscandict[dkey] = stats['sshpubkey'] ctx.config['targets'] = newscandict else: ctx.config['targets'] = newly_locked # FIXME: Ugh. log.info('\n '.join(['Locked targets:', ] + yaml.safe_dump(ctx.config['targets'], default_flow_style=False).splitlines())) break elif not ctx.block: assert 0, 'not enough machines are available' log.warn('Could not lock enough machines, waiting...') time.sleep(10) try: yield finally: if ctx.config.get('unlock_on_failure', False) or \ ctx.summary.get('success', False): log.info('Unlocking machines...') for machine in ctx.config['targets'].iterkeys(): lock.unlock_one(ctx, machine, ctx.owner)
def lock_machines(ctx, config): """ Lock machines. Called when the teuthology run finds and locks new machines. This is not called if the one has teuthology-locked machines and placed those keys in the Targets section of a yaml file. """ log.info('Locking machines...') assert isinstance(config[0], int), 'config[0] must be an integer' machine_type = config[1] machine_types = teuthology.get_multi_machine_types(machine_type) how_many = config[0] while True: # make sure there are enough machines up machines = lock.list_locks() if machines is None: if ctx.block: log.warn('error listing machines, trying again') time.sleep(20) continue else: assert 0, 'error listing machines' is_up = lambda machine: machine['up'] and machine['type'] in machine_types # noqa num_up = len(filter(is_up, machines)) assert num_up >= how_many, 'not enough machines are up' # make sure there are machines for non-automated jobs to run is_up_and_free = lambda machine: machine['up'] and machine['locked'] == 0 and machine['type'] in machine_types # noqa up_and_free = filter(is_up_and_free, machines) num_free = len(up_and_free) if num_free < 6 and ctx.owner.startswith('scheduled'): if ctx.block: log.info( 'waiting for more machines to be free (need %s see %s)...', how_many, num_free, ) time.sleep(10) continue else: assert 0, 'not enough machines free' newly_locked = lock.lock_many(ctx, how_many, machine_type, ctx.owner, ctx.archive) if len(newly_locked) == how_many: vmlist = [] for lmach in newly_locked: if teuthology.is_vm(lmach): vmlist.append(lmach) if vmlist: log.info('Waiting for virtual machines to come up') keyscan_out = '' loopcount = 0 while len(keyscan_out.splitlines()) != len(vmlist): loopcount += 1 time.sleep(10) keyscan_out, current_locks = lock.keyscan_check(ctx, vmlist) log.info('virtual machine is still unavailable') if loopcount == 40: loopcount = 0 log.info('virtual machine(s) still not up, ' + 'recreating unresponsive ones.') for guest in vmlist: if guest not in keyscan_out: log.info('recreating: ' + guest) lock.destroy_if_vm(ctx, 'ubuntu@' + guest) lock.create_if_vm(ctx, 'ubuntu@' + guest) if lock.update_keys(ctx, keyscan_out, current_locks): log.info("Error in virtual machine keys") newscandict = {} for dkey in newly_locked.iterkeys(): stats = lockstatus.get_status(ctx, dkey) newscandict[dkey] = stats['sshpubkey'] ctx.config['targets'] = newscandict else: ctx.config['targets'] = newly_locked # FIXME: Ugh. log.info('\n '.join(['Locked targets:', ] + yaml.safe_dump(ctx.config['targets'], default_flow_style=False).splitlines())) break elif not ctx.block: assert 0, 'not enough machines are available' log.warn('Could not lock enough machines, waiting...') time.sleep(10) try: yield finally: if ctx.config.get('unlock_on_failure', False) or \ ctx.summary.get('success', False): log.info('Unlocking machines...') for machine in ctx.config['targets'].iterkeys(): lock.unlock_one(ctx, machine, ctx.owner)
def lock_many(ctx, num, machine_type, user=None, description=None, os_type=None, os_version=None, arch=None): if user is None: user = misc.get_user() if not util.vps_version_or_type_valid( ctx.machine_type, os_type, os_version ): log.error('Invalid os-type or version detected -- lock failed') return # In the for loop below we can safely query for all bare-metal machine_type # values at once. So, if we're being asked for 'plana,mira,burnupi', do it # all in one shot. If we are passed 'plana,mira,burnupi,vps', do one query # for 'plana,mira,burnupi' and one for 'vps' machine_types_list = misc.get_multi_machine_types(machine_type) if machine_types_list == ['vps']: machine_types = machine_types_list elif machine_types_list == ['openstack']: return lock_many_openstack(ctx, num, machine_type, user=user, description=description, arch=arch) elif 'vps' in machine_types_list: machine_types_non_vps = list(machine_types_list) machine_types_non_vps.remove('vps') machine_types_non_vps = '|'.join(machine_types_non_vps) machine_types = [machine_types_non_vps, 'vps'] else: machine_types_str = '|'.join(machine_types_list) machine_types = [machine_types_str, ] for machine_type in machine_types: uri = os.path.join(config.lock_server, 'nodes', 'lock_many', '') data = dict( locked_by=user, count=num, machine_type=machine_type, description=description, ) # Only query for os_type/os_version if non-vps and non-libcloud, since # in that case we just create them. vm_types = ['vps'] + teuthology.provision.cloud.get_types() reimage_types = teuthology.provision.fog.get_types() if machine_type not in vm_types + reimage_types: if os_type: data['os_type'] = os_type if os_version: data['os_version'] = os_version if arch: data['arch'] = arch log.debug("lock_many request: %s", repr(data)) response = requests.post( uri, data=json.dumps(data), headers={'content-type': 'application/json'}, ) if response.ok: machines = {misc.canonicalize_hostname(machine['name']): machine['ssh_pub_key'] for machine in response.json()} log.debug('locked {machines}'.format( machines=', '.join(machines.keys()))) if machine_type in vm_types: ok_machs = {} for machine in machines: if teuthology.provision.create_if_vm(ctx, machine): ok_machs[machine] = machines[machine] else: log.error('Unable to create virtual machine: %s', machine) unlock_one(ctx, machine, user) ok_machs = keys.do_update_keys(ok_machs.keys())[1] return ok_machs elif machine_type in reimage_types: reimaged = dict() console_log_conf = dict( logfile_name='{shortname}_reimage.log', remotes=[teuthology.orchestra.remote.Remote(machine) for machine in machines], ) with console_log.task( ctx, console_log_conf): with teuthology.parallel.parallel() as p: for machine in machines: p.spawn(teuthology.provision.reimage, ctx, machine) reimaged[machine] = machines[machine] reimaged = keys.do_update_keys(reimaged.keys())[1] return reimaged return machines elif response.status_code == 503: log.error('Insufficient nodes available to lock %d %s nodes.', num, machine_type) log.error(response.text) else: log.error('Could not lock %d %s nodes, reason: unknown.', num, machine_type) return []