def lock_many(ctx,
num,
machine_type,
user=None,
description=None,
os_type=None,
os_version=None,
arch=None):
if user is None:
user = misc.get_user()
if not util.vps_version_or_type_valid(ctx.machine_type, os_type,
os_version):
log.error('Invalid os-type or version detected -- lock failed')
return
# In the for loop below we can safely query for all bare-metal machine_type
# values at once. So, if we're being asked for 'plana,mira,burnupi', do it
# all in one shot. If we are passed 'plana,mira,burnupi,vps', do one query
# for 'plana,mira,burnupi' and one for 'vps'
machine_types_list = misc.get_multi_machine_types(machine_type)
if machine_types_list == ['vps']:
machine_types = machine_types_list
elif machine_types_list == ['openstack']:
return lock_many_openstack(ctx,
num,
machine_type,
user=user,
description=description,
arch=arch)
elif 'vps' in machine_types_list:
machine_types_non_vps = list(machine_types_list)
machine_types_non_vps.remove('vps')
machine_types_non_vps = '|'.join(machine_types_non_vps)
machine_types = [machine_types_non_vps, 'vps']
else:
machine_types_str = '|'.join(machine_types_list)
machine_types = [
machine_types_str,
]
for machine_type in machine_types:
uri = os.path.join(config.lock_server, 'nodes', 'lock_many', '')
data = dict(
locked_by=user,
count=num,
machine_type=machine_type,
description=description,
)
# Only query for os_type/os_version if non-vps and non-libcloud, since
# in that case we just create them.
vm_types = ['vps'] + teuthology.provision.cloud.get_types()
reimage_types = teuthology.provision.get_reimage_types()
if machine_type not in vm_types + reimage_types:
if os_type:
data['os_type'] = os_type
if os_version:
data['os_version'] = os_version
if arch:
data['arch'] = arch
log.debug("lock_many request: %s", repr(data))
response = requests.post(
uri,
data=json.dumps(data),
headers={'content-type': 'application/json'},
)
if response.ok:
machines = {
misc.canonicalize_hostname(machine['name']):
machine['ssh_pub_key']
for machine in response.json()
}
log.debug('locked {machines}'.format(
machines=', '.join(machines.keys())))
if machine_type in vm_types:
ok_machs = {}
update_nodes(machines, True)
for machine in machines:
if teuthology.provision.create_if_vm(ctx, machine):
ok_machs[machine] = machines[machine]
else:
log.error('Unable to create virtual machine: %s',
machine)
unlock_one(ctx, machine, user)
ok_machs = do_update_keys(list(ok_machs.keys()))[1]
update_nodes(ok_machs)
return ok_machs
elif machine_type in reimage_types:
reimaged = dict()
console_log_conf = dict(
logfile_name='{shortname}_reimage.log',
remotes=[
teuthology.orchestra.remote.Remote(machine)
for machine in machines
],
)
with console_log.task(ctx, console_log_conf):
update_nodes(reimaged, True)
with teuthology.parallel.parallel() as p:
for machine in machines:
p.spawn(teuthology.provision.reimage, ctx, machine,
machine_type)
reimaged[machine] = machines[machine]
reimaged = do_update_keys(reimaged.keys())[1]
update_nodes(reimaged)
return reimaged
return machines
elif response.status_code == 503:
log.error('Insufficient nodes available to lock %d %s nodes.', num,
machine_type)
log.error(response.text)
else:
log.error('Could not lock %d %s nodes, reason: unknown.', num,
machine_type)
return []
def lock_machines(ctx, config):
"""
Lock machines. Called when the teuthology run finds and locks
new machines. This is not called if the one has teuthology-locked
machines and placed those keys in the Targets section of a yaml file.
"""
log.info('Locking machines...')
assert isinstance(config[0], int), 'config[0] must be an integer'
machine_type = config[1]
machine_types = teuthology.get_multi_machine_types(machine_type)
how_many = config[0]
while True:
# make sure there are enough machines up
machines = lock.list_locks()
if machines is None:
if ctx.block:
log.warn('error listing machines, trying again')
time.sleep(20)
continue
else:
assert 0, 'error listing machines'
is_up = lambda machine: machine['up'] and machine['type'] in machine_types # noqa
num_up = len(filter(is_up, machines))
assert num_up >= how_many, 'not enough machines are up'
# make sure there are machines for non-automated jobs to run
is_up_and_free = lambda machine: machine['up'] and machine['locked'] == 0 and machine['type'] in machine_types # noqa
up_and_free = filter(is_up_and_free, machines)
num_free = len(up_and_free)
if num_free < 6 and ctx.owner.startswith('scheduled'):
if ctx.block:
log.info(
'waiting for more machines to be free (need %s see %s)...',
how_many,
num_free,
)
time.sleep(10)
continue
else:
assert 0, 'not enough machines free'
newly_locked = lock.lock_many(ctx, how_many, machine_type, ctx.owner,
ctx.archive)
if len(newly_locked) == how_many:
vmlist = []
for lmach in newly_locked:
if teuthology.is_vm(lmach):
vmlist.append(lmach)
if vmlist:
log.info('Waiting for virtual machines to come up')
keyscan_out = ''
loopcount = 0
while len(keyscan_out.splitlines()) != len(vmlist):
loopcount += 1
time.sleep(10)
keyscan_out, current_locks = lock.keyscan_check(ctx,
vmlist)
log.info('virtual machine is still unavailable')
if loopcount == 40:
loopcount = 0
log.info('virtual machine(s) still not up, ' +
'recreating unresponsive ones.')
for guest in vmlist:
if guest not in keyscan_out:
log.info('recreating: ' + guest)
lock.destroy_if_vm(ctx, 'ubuntu@' + guest)
lock.create_if_vm(ctx, 'ubuntu@' + guest)
if lock.update_keys(ctx, keyscan_out, current_locks):
log.info("Error in virtual machine keys")
newscandict = {}
for dkey in newly_locked.iterkeys():
stats = lockstatus.get_status(ctx, dkey)
newscandict[dkey] = stats['sshpubkey']
ctx.config['targets'] = newscandict
else:
ctx.config['targets'] = newly_locked
# FIXME: Ugh.
log.info('\n '.join(['Locked targets:', ] + yaml.safe_dump(ctx.config['targets'], default_flow_style=False).splitlines()))
break
elif not ctx.block:
assert 0, 'not enough machines are available'
log.warn('Could not lock enough machines, waiting...')
time.sleep(10)
try:
yield
finally:
if ctx.config.get('unlock_on_failure', False) or \
ctx.summary.get('success', False):
log.info('Unlocking machines...')
for machine in ctx.config['targets'].iterkeys():
lock.unlock_one(ctx, machine, ctx.owner)
def lock_machines(ctx, config):
"""
Lock machines. Called when the teuthology run finds and locks
new machines. This is not called if the one has teuthology-locked
machines and placed those keys in the Targets section of a yaml file.
"""
log.info('Locking machines...')
assert isinstance(config[0], int), 'config[0] must be an integer'
machine_type = config[1]
machine_types = teuthology.get_multi_machine_types(machine_type)
how_many = config[0]
while True:
# make sure there are enough machines up
machines = lock.list_locks()
if machines is None:
if ctx.block:
log.warn('error listing machines, trying again')
time.sleep(20)
continue
else:
assert 0, 'error listing machines'
is_up = lambda machine: machine['up'] and machine['type'] in machine_types # noqa
num_up = len(filter(is_up, machines))
assert num_up >= how_many, 'not enough machines are up'
# make sure there are machines for non-automated jobs to run
is_up_and_free = lambda machine: machine['up'] and machine['locked'] == 0 and machine['type'] in machine_types # noqa
up_and_free = filter(is_up_and_free, machines)
num_free = len(up_and_free)
if num_free < 6 and ctx.owner.startswith('scheduled'):
if ctx.block:
log.info(
'waiting for more machines to be free (need %s see %s)...',
how_many,
num_free,
)
time.sleep(10)
continue
else:
assert 0, 'not enough machines free'
newly_locked = lock.lock_many(ctx, how_many, machine_type, ctx.owner,
ctx.archive)
if len(newly_locked) == how_many:
vmlist = []
for lmach in newly_locked:
if teuthology.is_vm(lmach):
vmlist.append(lmach)
if vmlist:
log.info('Waiting for virtual machines to come up')
keyscan_out = ''
loopcount = 0
while len(keyscan_out.splitlines()) != len(vmlist):
loopcount += 1
time.sleep(10)
keyscan_out, current_locks = lock.keyscan_check(ctx,
vmlist)
log.info('virtual machine is still unavailable')
if loopcount == 40:
loopcount = 0
log.info('virtual machine(s) still not up, ' +
'recreating unresponsive ones.')
for guest in vmlist:
if guest not in keyscan_out:
log.info('recreating: ' + guest)
lock.destroy_if_vm(ctx, 'ubuntu@' + guest)
lock.create_if_vm(ctx, 'ubuntu@' + guest)
if lock.update_keys(ctx, keyscan_out, current_locks):
log.info("Error in virtual machine keys")
newscandict = {}
for dkey in newly_locked.iterkeys():
stats = lockstatus.get_status(ctx, dkey)
newscandict[dkey] = stats['sshpubkey']
ctx.config['targets'] = newscandict
else:
ctx.config['targets'] = newly_locked
# FIXME: Ugh.
log.info('\n '.join(['Locked targets:', ] + yaml.safe_dump(ctx.config['targets'], default_flow_style=False).splitlines()))
break
elif not ctx.block:
assert 0, 'not enough machines are available'
log.warn('Could not lock enough machines, waiting...')
time.sleep(10)
try:
yield
finally:
if ctx.config.get('unlock_on_failure', False) or \
ctx.summary.get('success', False):
log.info('Unlocking machines...')
for machine in ctx.config['targets'].iterkeys():
lock.unlock_one(ctx, machine, ctx.owner)
def lock_many(ctx, num, machine_type, user=None, description=None,
os_type=None, os_version=None, arch=None):
if user is None:
user = misc.get_user()
if not util.vps_version_or_type_valid(
ctx.machine_type,
os_type,
os_version
):
log.error('Invalid os-type or version detected -- lock failed')
return
# In the for loop below we can safely query for all bare-metal machine_type
# values at once. So, if we're being asked for 'plana,mira,burnupi', do it
# all in one shot. If we are passed 'plana,mira,burnupi,vps', do one query
# for 'plana,mira,burnupi' and one for 'vps'
machine_types_list = misc.get_multi_machine_types(machine_type)
if machine_types_list == ['vps']:
machine_types = machine_types_list
elif machine_types_list == ['openstack']:
return lock_many_openstack(ctx, num, machine_type,
user=user,
description=description,
arch=arch)
elif 'vps' in machine_types_list:
machine_types_non_vps = list(machine_types_list)
machine_types_non_vps.remove('vps')
machine_types_non_vps = '|'.join(machine_types_non_vps)
machine_types = [machine_types_non_vps, 'vps']
else:
machine_types_str = '|'.join(machine_types_list)
machine_types = [machine_types_str, ]
for machine_type in machine_types:
uri = os.path.join(config.lock_server, 'nodes', 'lock_many', '')
data = dict(
locked_by=user,
count=num,
machine_type=machine_type,
description=description,
)
# Only query for os_type/os_version if non-vps and non-libcloud, since
# in that case we just create them.
vm_types = ['vps'] + teuthology.provision.cloud.get_types()
reimage_types = teuthology.provision.fog.get_types()
if machine_type not in vm_types + reimage_types:
if os_type:
data['os_type'] = os_type
if os_version:
data['os_version'] = os_version
if arch:
data['arch'] = arch
log.debug("lock_many request: %s", repr(data))
response = requests.post(
uri,
data=json.dumps(data),
headers={'content-type': 'application/json'},
)
if response.ok:
machines = {misc.canonicalize_hostname(machine['name']):
machine['ssh_pub_key'] for machine in response.json()}
log.debug('locked {machines}'.format(
machines=', '.join(machines.keys())))
if machine_type in vm_types:
ok_machs = {}
for machine in machines:
if teuthology.provision.create_if_vm(ctx, machine):
ok_machs[machine] = machines[machine]
else:
log.error('Unable to create virtual machine: %s',
machine)
unlock_one(ctx, machine, user)
ok_machs = keys.do_update_keys(ok_machs.keys())[1]
return ok_machs
elif machine_type in reimage_types:
reimaged = dict()
console_log_conf = dict(
logfile_name='{shortname}_reimage.log',
remotes=[teuthology.orchestra.remote.Remote(machine)
for machine in machines],
)
with console_log.task(
ctx, console_log_conf):
with teuthology.parallel.parallel() as p:
for machine in machines:
p.spawn(teuthology.provision.reimage, ctx, machine)
reimaged[machine] = machines[machine]
reimaged = keys.do_update_keys(reimaged.keys())[1]
return reimaged
return machines
elif response.status_code == 503:
log.error('Insufficient nodes available to lock %d %s nodes.',
num, machine_type)
log.error(response.text)
else:
log.error('Could not lock %d %s nodes, reason: unknown.',
num, machine_type)
return []