コード例 #1
0
def processKitten(options, remoteEnv, job):
    dNow  = datetime.datetime.now()
    dDate = dNow.strftime('%Y-%m-%d')
    dHour = dNow.strftime('%H')
    r     = {}

    if job is not None:
        if job in remoteEnv.hosts:
            info = remoteEnv.hosts[job]
            if info['environment'] == options.environ:
                if not info['enabled'] and not options.force:
                    if options.verbose:
                        log.info('%s not enabled, skipping' % job)
                elif len(info['notes']) > 0 and not options.force:
                    if options.verbose:
                        log.info('%s has a slavealloc notes field, skipping' % job)
                else:
                    log.info(job)
                    host = remoteEnv.getHost(job)
                    if host is None:
                        log.error('unknown host for %s' % job)
                    else:
                        r = remoteEnv.check(host, indent='    ', dryrun=options.dryrun, verbose=options.verbose)

                        if host.farm != 'ec2':
                            d = remoteEnv.rebootIfNeeded(host, lastSeen=r['lastseen'], indent='    ', dryrun=options.dryrun, verbose=options.verbose)
                            for s in ['reboot', 'recovery', 'ipmi', 'pdu']:
                                r[s] = d[s]
                            r['output'] += d['output']

                        r['host'] = host
                        hostKey   = 'kittenherder:%s.%s:%s' % (dDate, dHour, job)
                        for key in r:
                            db.hset(hostKey, key, r[key])
                        db.expire(hostKey, _keyExpire)

                        # all this because json cannot dumps() the timedelta object
                        td = r['lastseen']
                        if td is not None:
                            secs             = td.seconds
                            hours, remainder = divmod(secs, 3600)
                            minutes, seconds = divmod(remainder, 60)
                            r['lastseen']    = { 'hours':    hours,
                                                 'minutes':  minutes,
                                                 'seconds':  seconds,
                                                 'relative': relative(td),
                                                 'since':    secs,
                                               }
            else:
                if options.verbose:
                    log.info('%s not in requested environment %s (%s), skipping' % (job, options.environ, info['environment']))
        else:
            if options.verbose:
                log.error('%s not listed in slavealloc, skipping' % job, exc_info=True)

    return r
コード例 #2
0
def processKitten(options, remoteEnv, job):
    dNow  = datetime.datetime.now()
    dDate = dNow.strftime('%Y-%m-%d')
    dHour = dNow.strftime('%H')
    r     = {}

    if job is not None:
        if job in remoteEnv.hosts:
            info = remoteEnv.hosts[job]
            if info['environment'] == options.environ:
                if not info['enabled'] and not options.force:
                    if options.verbose:
                        log.info('%s not enabled, skipping' % job)
                elif len(info['notes']) > 0 and 'tegra' not in job and not options.force:
                    if options.verbose:
                        log.info('%s has a slavealloc notes field, skipping' % job)
                else:
                    log.info(job)
                    host = remoteEnv.getHost(job)
                    if host is None:
                        log.error('unknown host for %s' % job)
                    else:
                        r = remoteEnv.check(host, indent='    ', dryrun=options.dryrun, verbose=options.verbose)
                        if host.farm != 'ec2':
                            d = remoteEnv.rebootIfNeeded(host, lastSeen=r['lastseen'], indent='    ', dryrun=options.dryrun, verbose=options.verbose)
                            for s in ['reboot', 'recovery', 'ipmi', 'pdu']:
                                r[s] = d[s]
                            r['output'] += d['output']

                        r['host'] = host
                        hostKey   = 'kittenherder:%s.%s:%s' % (dDate, dHour, job)
                        for key in r:
                            db.hset(hostKey, key, r[key])
                        db.expire(hostKey, _keyExpire)

                        # all this because json cannot dumps() the timedelta object
                        td = r['lastseen']
                        if td is not None:
                            secs             = td.seconds
                            hours, remainder = divmod(secs, 3600)
                            minutes, seconds = divmod(remainder, 60)
                            r['lastseen']    = { 'hours':    hours,
                                                 'minutes':  minutes,
                                                 'seconds':  seconds,
                                                 'relative': relative(td),
                                                 'since':    secs,
                                               }
            else:
                if options.verbose:
                    log.info('%s not in requested environment %s (%s), skipping' % (job, options.environ, info['environment']))
        else:
            if options.verbose:
                log.error('%s not listed in slavealloc, skipping' % job, exc_info=True)

    return r
コード例 #3
0
ファイル: kittenherder.py プロジェクト: ccooper/briar-patch
def processKittens(options, jobs, results):
    remoteEnv = releng.remote.RemoteEnvironment(options.tools, db=db)
    dNow      = datetime.datetime.now()
    dDate     = dNow.strftime('%Y-%m-%d')
    dHour     = dNow.strftime('%H')

    while True:
        try:
            job = jobs.get(False)
        except Empty:
            job = None

        if job is not None:
            r = {}
            if job in remoteEnv.hosts:
                info = remoteEnv.hosts[job]
                if info['environment'] == options.environ:
                    if not info['enabled'] and not options.force:
                        if options.verbose:
                            log.info('%s not enabled, skipping' % job)
                    elif len(info['notes']) > 0 and not options.force:
                        if options.verbose:
                            log.info('%s has a slavealloc notes field, skipping' % job)
                    else:
                        log.info(job)
                        host = remoteEnv.getHost(job)
                        if host is None:
                            log.error('unknown host for %s' % job)
                        else:
                            r = remoteEnv.check(host, indent='    ', dryrun=options.dryrun, verbose=options.verbose)
                            d = remoteEnv.rebootIfNeeded(host, lastSeen=r['lastseen'], indent='    ', dryrun=options.dryrun, verbose=options.verbose)

                            for s in ['reboot', 'recovery', 'ipmi', 'pdu']:
                                r[s] = d[s]
                            r['output'] += d['output']

                            hostKey = 'kittenherder:%s.%s:%s' % (dDate, dHour, job)
                            for key in r:
                                db.hset(hostKey, key, r[key])
                            db.expire(hostKey, _keyExpire)

                            # all this because json cannot dumps() the timedelta object
                            td = r['lastseen']
                            if td is not None:
                                secs             = td.seconds
                                hours, remainder = divmod(secs, 3600)
                                minutes, seconds = divmod(remainder, 60)
                                r['lastseen']    = { 'hours':    hours,
                                                     'minutes':  minutes,
                                                     'seconds':  seconds,
                                                     'relative': relative(td),
                                                     'since':    secs,
                                                   }
                            log.info('%s: %s' % (job, json.dumps(r)))

                            if (host.farm == 'ec2') and (r['reboot'] or r['recovery']):
                                log.info('shutting down ec2 instance')
                                try:
                                    conn = connect_to_region(host.info['region'],
                                                             aws_access_key_id=getPassword('aws_access_key_id'),
                                                             aws_secret_access_key=getPassword('aws_secret_access_key'))
                                    conn.stop_instances(instance_ids=[host.info['id'],])
                                except:
                                    log.error('unable to stop ec2 instance %s [%s]' % (job, host.info['id']), exc_info=True)
                else:
                    if options.verbose:
                        log.info('%s not in requested environment %s (%s), skipping' % (job, options.environ, info['environment']))
            else:
                if options.verbose:
                    log.error('%s not listed in slavealloc, skipping' % job, exc_info=True)

            results.put((job, r))
コード例 #4
0
def processKittens(options, jobs, results):
    remoteEnv = releng.remote.RemoteEnvironment(options.tools, db=db)
    dNow = datetime.datetime.now()
    dDate = dNow.strftime('%Y-%m-%d')
    dHour = dNow.strftime('%H')

    while True:
        try:
            job = jobs.get(False)
        except Empty:
            job = None

        if job is not None:
            r = {}
            if job in remoteEnv.hosts:
                info = remoteEnv.hosts[job]
                if info['environment'] == options.environ:
                    if not info['enabled'] and not options.force:
                        if options.verbose:
                            log.info('%s not enabled, skipping' % job)
                    elif len(info['notes']) > 0 and not options.force:
                        if options.verbose:
                            log.info(
                                '%s has a slavealloc notes field, skipping' %
                                job)
                    else:
                        log.info(job)
                        host = remoteEnv.getHost(job)
                        if host is None:
                            log.error('unknown host for %s' % job)
                        else:
                            r = remoteEnv.check(host,
                                                indent='    ',
                                                dryrun=options.dryrun,
                                                verbose=options.verbose)
                            d = remoteEnv.rebootIfNeeded(
                                host,
                                lastSeen=r['lastseen'],
                                indent='    ',
                                dryrun=options.dryrun,
                                verbose=options.verbose)

                            for s in ['reboot', 'recovery', 'ipmi', 'pdu']:
                                r[s] = d[s]
                            r['output'] += d['output']

                            hostKey = 'kittenherder:%s.%s:%s' % (dDate, dHour,
                                                                 job)
                            for key in r:
                                db.hset(hostKey, key, r[key])
                            db.expire(hostKey, _keyExpire)

                            # all this because json cannot dumps() the timedelta object
                            td = r['lastseen']
                            if td is not None:
                                secs = td.seconds
                                hours, remainder = divmod(secs, 3600)
                                minutes, seconds = divmod(remainder, 60)
                                r['lastseen'] = {
                                    'hours': hours,
                                    'minutes': minutes,
                                    'seconds': seconds,
                                    'relative': relative(td),
                                    'since': secs,
                                }
                            log.info('%s: %s' % (job, json.dumps(r)))

                            if (host.farm == 'ec2') and (r['reboot']
                                                         or r['recovery']):
                                log.info('shutting down ec2 instance')
                                try:
                                    conn = connect_to_region(
                                        host.info['region'],
                                        aws_access_key_id=getPassword(
                                            'aws_access_key_id'),
                                        aws_secret_access_key=getPassword(
                                            'aws_secret_access_key'))
                                    conn.stop_instances(instance_ids=[
                                        host.info['id'],
                                    ])
                                except:
                                    log.error(
                                        'unable to stop ec2 instance %s [%s]' %
                                        (job, host.info['id']),
                                        exc_info=True)
                else:
                    if options.verbose:
                        log.info(
                            '%s not in requested environment %s (%s), skipping'
                            % (job, options.environ, info['environment']))
            else:
                if options.verbose:
                    log.error('%s not listed in slavealloc, skipping' % job,
                              exc_info=True)

            results.put((job, r))
コード例 #5
0
def check(kitten):
    s = '%s: ' % kitten

    info = remoteEnv.hosts[kitten]

    if info['enabled']:
        s += 'enabled'
    else:
        # hopefully short term hack until tegras are
        # handled properly by slavealloc...
        if 'tegra' in kitten:
            s += 'enabled'
        else:
            s += 'DISABLED'

    host = remoteEnv.getHost(kitten)

    print s
    print '%12s: %s' % ('farm',       host.farm)
    print '%12s: %s' % ('colo',       info['datacenter'])
    print '%12s: %s' % ('distro',     info['distro'])
    print '%12s: %s' % ('pool',       info['pool'])
    print '%12s: %s' % ('trustlevel', info['trustlevel'])
    print '%12s: %s' % ('master',     info['current_master'])
    print '%12s: %s' % ('fqdn',       host.fqdn)
    print '%12s: %s' % ('PDU?',       host.hasPDU)
    print '%12s: %s' % ('IPMI?',      host.hasIPMI)

    if len(info['notes']) > 0:
        print '%12s: %s' % ('note', info['notes'])

    if not options.info:
        r = remoteEnv.check(host, dryrun=options.dryrun, verbose=options.verbose, indent='    ', reboot=options.reboot)

        for key in ('reachable', 'buildbot', 'tacfile', 'lastseen', 'master'):
            if key in r:
                s = r[key]
                if key == 'lastseen':
                    if r[key] is None:
                        s = 'unknown'
                    else:
                        s = relative(r[key])
                if key == 'master':
                    if r[key] is None:
                        s = info['current_master']
                    else:
                        if len(r[key]) > 0:
                            s = r[key][0]
                        else:
                            s = r[key]
                print '%12s: %s' % (key, s)

        if 'tacfile' in r and r['tacfile'].lower() != 'not found' and 'master' in r:
            if r['master'] is not None and len(r['master']) > 0:
                m = r['master'][0]
            else:
                m = ''

            master         = remoteEnv.findMaster(m)
            current_master = remoteEnv.findMaster(info['current_master'])
            if master is not None and current_master is not None and master['masterid'] != current_master['masterid']:
                print '%12s: current master is different than buildbot.tac master [%s]' % ('error', m)

        if options.reboot:
            if 'reboot' in r and r['reboot']:
                s = '%12s: via ' % 'reboot'
                if 'ipmi' in r and r['ipmi']:
                    s += 'IPMI'
                elif 'pdu' in r and r['pdu']:
                    s += 'PDU'
            elif 'recovery' in r and r['recovery']:
                s = 'recovery needed, could not reboot host'
            else:
                s = '%12s\n%s' % ('reboot requested but not performed', r['output'])
            print s

        if options.stop:
            print host.graceful_shutdown()
コード例 #6
0
ファイル: kitten.py プロジェクト: catlee/briar-patch
def check(kitten):
    s = '%s: ' % kitten

    info = remoteEnv.hosts[kitten]
    if info['enabled']:
        s += 'enabled'
    else:
        # hopefully short term hack until tegras are
        # handled properly by slavealloc...
        if 'tegra' in kitten:
            s += 'enabled'
        else:
            s += 'DISABLED'

    print s
    print '%12s: %s' % ('colo',       info['datacenter'])
    print '%12s: %s' % ('distro',     info['distro'])
    print '%12s: %s' % ('pool',       info['pool'])
    print '%12s: %s' % ('trustlevel', info['trustlevel'])

    if len(info['notes']) > 0:
        print '%12s: %s' % ('note', info['notes'])


    if options.info:
        print '%12s: %s' % ('master', info['current_master'])
    else:
        host = remoteEnv.getHost(kitten)
        r    = remoteEnv.check(host, dryrun=options.dryrun, verbose=options.verbose, indent='    ', reboot=options.reboot)

        for key in ('fqdn', 'reachable', 'buildbot', 'tacfile', 'lastseen', 'master'):
            s = r[key]
            if key == 'lastseen':
                if r[key] is None:
                    s = 'unknown'
                else:
                    s = relative(r[key])
            if key == 'master':
                if r[key] is None:
                    s = info['current_master']
                else:
                    if len(r[key]) > 0:
                        s = r[key][0]
                    else:
                        s = r[key]
            print '%12s: %s' % (key, s)

        if 'master' in r:
            if r['master'] is not None and len(r['master']) > 0:
                m = r['master'][0]
            else:
                m = ''

            master         = remoteEnv.findMaster(m)
            current_master = remoteEnv.findMaster(info['current_master'])
            if master['masterid'] != current_master['masterid']:
                print '%12s: current master is different than buildbot.tac master [%s]' % ('error', m)

        print '%12s: %s' % ('IPMI?', host.hasIPMI)

        if options.stop:
            print host.graceful_shutdown()