Esempio n. 1
0
def lock_machines(ctx, config):
    """
    Lock machines.  Called when the teuthology run finds and locks
    new machines.  This is not called if the one has teuthology-locked
    machines and placed those keys in the Targets section of a yaml file.
    """
    log.info('Locking machines...')
    assert isinstance(config[0], int), 'config[0] must be an integer'
    machine_type = config[1]
    machine_types = teuthology.get_multi_machine_types(machine_type)
    how_many = config[0]

    while True:
        # make sure there are enough machines up
        machines = lock.list_locks()
        if machines is None:
            if ctx.block:
                log.warn('error listing machines, trying again')
                time.sleep(20)
                continue
            else:
                assert 0, 'error listing machines'

        is_up = lambda machine: machine['up'] and machine['type'] in machine_types  # noqa
        num_up = len(filter(is_up, machines))
        assert num_up >= how_many, 'not enough machines are up'

        # make sure there are machines for non-automated jobs to run
        is_up_and_free = lambda machine: machine['up'] and machine['locked'] == 0 and machine['type'] in machine_types  # noqa
        up_and_free = filter(is_up_and_free, machines)
        num_free = len(up_and_free)
        if num_free < 6 and ctx.owner.startswith('scheduled'):
            if ctx.block:
                log.info(
                    'waiting for more machines to be free (need %s see %s)...',
                    how_many,
                    num_free,
                )
                time.sleep(10)
                continue
            else:
                assert 0, 'not enough machines free'

        newly_locked = lock.lock_many(ctx, how_many, machine_type, ctx.owner,
                                      ctx.archive)
        if len(newly_locked) == how_many:
            vmlist = []
            for lmach in newly_locked:
                if teuthology.is_vm(lmach):
                    vmlist.append(lmach)
            if vmlist:
                log.info('Waiting for virtual machines to come up')
                keyscan_out = ''
                loopcount = 0
                while len(keyscan_out.splitlines()) != len(vmlist):
                    loopcount += 1
                    time.sleep(10)
                    keyscan_out, current_locks = lock.keyscan_check(ctx,
                                                                    vmlist)
                    log.info('virtual machine is still unavailable')
                    if loopcount == 40:
                        loopcount = 0
                        log.info('virtual machine(s) still not up, ' +
                                 'recreating unresponsive ones.')
                        for guest in vmlist:
                            if guest not in keyscan_out:
                                log.info('recreating: ' + guest)
                                lock.destroy_if_vm(ctx, 'ubuntu@' + guest)
                                lock.create_if_vm(ctx, 'ubuntu@' + guest)
                if lock.update_keys(ctx, keyscan_out, current_locks):
                    log.info("Error in virtual machine keys")
                newscandict = {}
                for dkey in newly_locked.iterkeys():
                    stats = lockstatus.get_status(ctx, dkey)
                    newscandict[dkey] = stats['sshpubkey']
                ctx.config['targets'] = newscandict
            else:
                ctx.config['targets'] = newly_locked
            # FIXME: Ugh.
            log.info('\n  '.join(['Locked targets:', ] + yaml.safe_dump(ctx.config['targets'], default_flow_style=False).splitlines()))
            break
        elif not ctx.block:
            assert 0, 'not enough machines are available'

        log.warn('Could not lock enough machines, waiting...')
        time.sleep(10)
    try:
        yield
    finally:
        if ctx.config.get('unlock_on_failure', False) or \
           ctx.summary.get('success', False):
            log.info('Unlocking machines...')
            for machine in ctx.config['targets'].iterkeys():
                lock.unlock_one(ctx, machine, ctx.owner)
Esempio n. 2
0
def lock_machines(ctx, config):
    """
    Lock machines.  Called when the teuthology run finds and locks
    new machines.  This is not called if the one has teuthology-locked
    machines and placed those keys in the Targets section of a yaml file.
    """
    log.info('Locking machines...')
    assert isinstance(config[0], int), 'config[0] must be an integer'
    machine_type = config[1]
    machine_types = teuthology.get_multi_machine_types(machine_type)
    how_many = config[0]

    while True:
        # make sure there are enough machines up
        machines = lock.list_locks()
        if machines is None:
            if ctx.block:
                log.warn('error listing machines, trying again')
                time.sleep(20)
                continue
            else:
                assert 0, 'error listing machines'

        is_up = lambda machine: machine['up'] and machine['type'] in machine_types  # noqa
        num_up = len(filter(is_up, machines))
        assert num_up >= how_many, 'not enough machines are up'

        # make sure there are machines for non-automated jobs to run
        is_up_and_free = lambda machine: machine['up'] and machine['locked'] == 0 and machine['type'] in machine_types  # noqa
        up_and_free = filter(is_up_and_free, machines)
        num_free = len(up_and_free)
        if num_free < 6 and ctx.owner.startswith('scheduled'):
            if ctx.block:
                log.info(
                    'waiting for more machines to be free (need %s see %s)...',
                    how_many,
                    num_free,
                )
                time.sleep(10)
                continue
            else:
                assert 0, 'not enough machines free'

        newly_locked = lock.lock_many(ctx, how_many, machine_type, ctx.owner,
                                      ctx.archive)
        if len(newly_locked) == how_many:
            vmlist = []
            for lmach in newly_locked:
                if teuthology.is_vm(lmach):
                    vmlist.append(lmach)
            if vmlist:
                log.info('Waiting for virtual machines to come up')
                keyscan_out = ''
                loopcount = 0
                while len(keyscan_out.splitlines()) != len(vmlist):
                    loopcount += 1
                    time.sleep(10)
                    keyscan_out, current_locks = lock.keyscan_check(ctx,
                                                                    vmlist)
                    log.info('virtual machine is still unavailable')
                    if loopcount == 40:
                        loopcount = 0
                        log.info('virtual machine(s) still not up, ' +
                                 'recreating unresponsive ones.')
                        for guest in vmlist:
                            if guest not in keyscan_out:
                                log.info('recreating: ' + guest)
                                lock.destroy_if_vm(ctx, 'ubuntu@' + guest)
                                lock.create_if_vm(ctx, 'ubuntu@' + guest)
                if lock.update_keys(ctx, keyscan_out, current_locks):
                    log.info("Error in virtual machine keys")
                newscandict = {}
                for dkey in newly_locked.iterkeys():
                    stats = lockstatus.get_status(ctx, dkey)
                    newscandict[dkey] = stats['sshpubkey']
                ctx.config['targets'] = newscandict
            else:
                ctx.config['targets'] = newly_locked
            # FIXME: Ugh.
            log.info('\n  '.join(['Locked targets:', ] + yaml.safe_dump(ctx.config['targets'], default_flow_style=False).splitlines()))
            break
        elif not ctx.block:
            assert 0, 'not enough machines are available'

        log.warn('Could not lock enough machines, waiting...')
        time.sleep(10)
    try:
        yield
    finally:
        if ctx.config.get('unlock_on_failure', False) or \
           ctx.summary.get('success', False):
            log.info('Unlocking machines...')
            for machine in ctx.config['targets'].iterkeys():
                lock.unlock_one(ctx, machine, ctx.owner)
Esempio n. 3
0
def lock_machines(ctx, config):
    log.info("Locking machines...")
    assert isinstance(config[0], int), "config[0] must be an integer"
    machine_type = config[1]
    how_many = config[0]

    while True:
        # make sure there are enough machines up
        machines = lock.list_locks()
        if machines is None:
            if ctx.block:
                log.warn("error listing machines, trying again")
                time.sleep(20)
                continue
            else:
                assert 0, "error listing machines"

        is_up = lambda machine: machine["up"] and machine["type"] == machine_type  # noqa
        num_up = len(filter(is_up, machines))
        assert num_up >= how_many, "not enough machines are up"

        # make sure there are machines for non-automated jobs to run
        is_up_and_free = (
            lambda machine: machine["up"] and machine["locked"] == 0 and machine["type"] == machine_type
        )  # noqa
        up_and_free = filter(is_up_and_free, machines)
        num_free = len(up_and_free)
        if num_free < 6 and ctx.owner.startswith("scheduled"):
            if ctx.block:
                log.info("waiting for more machines to be free (need %s see %s)...", how_many, num_free)
                time.sleep(10)
                continue
            else:
                assert 0, "not enough machines free"

        newly_locked = lock.lock_many(ctx, how_many, machine_type, ctx.owner, ctx.archive)
        if len(newly_locked) == how_many:
            vmlist = []
            for lmach in newly_locked:
                if lock.create_if_vm(ctx, lmach):
                    vmlist.append(lmach)
            if vmlist:
                log.info("Waiting for virtual machines to come up")
                keyscan_out = ""
                loopcount = 0
                while len(keyscan_out.splitlines()) != len(vmlist):
                    loopcount += 1
                    time.sleep(10)
                    keyscan_out, current_locks = lock.keyscan_check(ctx, vmlist)
                    log.info("virtual machine is stil unavailable")
                    if loopcount == 40:
                        loopcount = 0
                        log.info("virtual machine(s) still not up, " + "recreating unresponsive ones.")
                        for guest in vmlist:
                            if guest not in keyscan_out:
                                log.info("recreating: " + guest)
                                lock.destroy_if_vm(ctx, "ubuntu@" + guest)
                                lock.create_if_vm(ctx, "ubuntu@" + guest)
                if lock.update_keys(ctx, keyscan_out, current_locks):
                    log.info("Error in virtual machine keys")
                newscandict = {}
                for dkey in newly_locked.iterkeys():
                    stats = lockstatus.get_status(ctx, dkey)
                    newscandict[dkey] = stats["sshpubkey"]
                ctx.config["targets"] = newscandict
            else:
                ctx.config["targets"] = newly_locked
            # FIXME: Ugh.
            log.info(
                "\n  ".join(
                    ["Locked targets:"] + yaml.safe_dump(ctx.config["targets"], default_flow_style=False).splitlines()
                )
            )
            break
        elif not ctx.block:
            assert 0, "not enough machines are available"

        log.warn("Could not lock enough machines, waiting...")
        time.sleep(10)
    try:
        yield
    finally:
        if ctx.summary.get("success", False):
            log.info("Unlocking machines...")
            for machine in ctx.config["targets"].iterkeys():
                lock.unlock_one(ctx, machine, ctx.owner)
Esempio n. 4
0
def lock_machines(ctx, config):
    log.info('Locking machines...')
    assert isinstance(config[0], int), 'config must be an integer'
    machine_type = config[1]
    config = config[0]

    while True:
        # make sure there are enough machines up
        machines = lock.list_locks(ctx)
        if machines is None:
            if ctx.block:
                log.warn('error listing machines, trying again')
                time.sleep(20)
                continue
            else:
                assert 0, 'error listing machines'
        num_up = len(filter(lambda machine: machine['up'] and machine['type'] == machine_type, machines))
        assert num_up >= config, 'not enough machines are up'

        # make sure there are machines for non-automated jobs to run
        num_free = len(filter(
                lambda machine: machine['up'] and machine['locked'] == 0 and machine['type'] == machine_type,
                machines
                ))
        if num_free < 6 and ctx.owner.startswith('scheduled'):
            if ctx.block:
                log.info('waiting for more machines to be free...')
                time.sleep(10)
                continue
            else:
                assert 0, 'not enough machines free'

        newly_locked = lock.lock_many(ctx, config, machine_type, ctx.owner, ctx.archive)
        if len(newly_locked) == config:
            vmlist = []
            for lmach in newly_locked:
                if lock.create_if_vm(ctx,lmach):
                    vmlist.append(lmach)
            if vmlist:
                log.info('Waiting for virtual machines to come up')
                keyscan_out = ''
                while len(keyscan_out.splitlines()) != len(vmlist):
                    time.sleep(10)
                    keyscan_out, current_locks = lock.keyscan_check(ctx, vmlist)
                    log.info('virtual machine is stil unavailable')
                if lock.update_keys(ctx, keyscan_out, current_locks):
                    log.info("Error in virtual machine keys")
                newscandict = {}
                for dkey in newly_locked.iterkeys():
                    stats = lockstatus.get_status(ctx, dkey)
                    newscandict[dkey] = stats['sshpubkey']
                ctx.config['targets'] = newscandict
            else:
                ctx.config['targets'] = newly_locked
            log.info('\n  '.join(['Locked targets:', ] + yaml.safe_dump(ctx.config['targets'], default_flow_style=False).splitlines()))
            break
        elif not ctx.block:
            assert 0, 'not enough machines are available'

        log.warn('Could not lock enough machines, waiting...')
        time.sleep(10)
    try:
        yield
    finally:
        if ctx.summary.get('success', False):
            log.info('Unlocking machines...')
            for machine in ctx.config['targets'].iterkeys():
                lock.unlock(ctx, machine, ctx.owner)
Esempio n. 5
0
def lock_machines(ctx, config):
    log.info('Locking machines...')
    assert isinstance(config[0], int), 'config must be an integer'
    machine_type = config[1]
    config = config[0]

    while True:
        # make sure there are enough machines up
        machines = lock.list_locks(ctx)
        if machines is None:
            if ctx.block:
                log.warn('error listing machines, trying again')
                time.sleep(20)
                continue
            else:
                assert 0, 'error listing machines'
        num_up = len(
            filter(
                lambda machine: machine['up'] and machine['type'] ==
                machine_type, machines))
        assert num_up >= config, 'not enough machines are up'

        # make sure there are machines for non-automated jobs to run
        num_free = len(
            filter(
                lambda machine: machine['up'] and machine['locked'] == 0 and
                machine['type'] == machine_type, machines))
        if num_free < 6 and ctx.owner.startswith('scheduled'):
            if ctx.block:
                log.info('waiting for more machines to be free...')
                time.sleep(10)
                continue
            else:
                assert 0, 'not enough machines free'

        newly_locked = lock.lock_many(ctx, config, machine_type, ctx.owner,
                                      ctx.archive)
        if len(newly_locked) == config:
            vmlist = []
            for lmach in newly_locked:
                if lock.create_if_vm(ctx, lmach):
                    vmlist.append(lmach)
            if vmlist:
                log.info('Waiting for virtual machines to come up')
                keyscan_out = ''
                loopcount = 0
                while len(keyscan_out.splitlines()) != len(vmlist):
                    loopcount += 1
                    time.sleep(10)
                    keyscan_out, current_locks = lock.keyscan_check(
                        ctx, vmlist)
                    log.info('virtual machine is stil unavailable')
                    if loopcount == 40:
                        loopcount = 0
                        log.info(
                            'virtual machine(s) still not up, recreating unresponsive ones.'
                        )
                        for guest in vmlist:
                            if guest not in keyscan_out:
                                log.info('recreating: ' + guest)
                                lock.destroy_if_vm(ctx, 'ubuntu@' + guest)
                                lock.create_if_vm(ctx, 'ubuntu@' + guest)
                if lock.update_keys(ctx, keyscan_out, current_locks):
                    log.info("Error in virtual machine keys")
                newscandict = {}
                for dkey in newly_locked.iterkeys():
                    stats = lockstatus.get_status(ctx, dkey)
                    newscandict[dkey] = stats['sshpubkey']
                ctx.config['targets'] = newscandict
            else:
                ctx.config['targets'] = newly_locked
            log.info('\n  '.join([
                'Locked targets:',
            ] + yaml.safe_dump(ctx.config['targets'],
                               default_flow_style=False).splitlines()))
            break
        elif not ctx.block:
            assert 0, 'not enough machines are available'

        log.warn('Could not lock enough machines, waiting...')
        time.sleep(10)
    try:
        yield
    finally:
        if ctx.summary.get('success', False):
            log.info('Unlocking machines...')
            for machine in ctx.config['targets'].iterkeys():
                lock.unlock(ctx, machine, ctx.owner)