def processKitten(options, remoteEnv, job):
dNow = datetime.datetime.now()
dDate = dNow.strftime('%Y-%m-%d')
dHour = dNow.strftime('%H')
r = {}
if job is not None:
if job in remoteEnv.hosts:
info = remoteEnv.hosts[job]
if info['environment'] == options.environ:
if not info['enabled'] and not options.force:
if options.verbose:
log.info('%s not enabled, skipping' % job)
elif len(info['notes']) > 0 and not options.force:
if options.verbose:
log.info('%s has a slavealloc notes field, skipping' % job)
else:
log.info(job)
host = remoteEnv.getHost(job)
if host is None:
log.error('unknown host for %s' % job)
else:
r = remoteEnv.check(host, indent=' ', dryrun=options.dryrun, verbose=options.verbose)
if host.farm != 'ec2':
d = remoteEnv.rebootIfNeeded(host, lastSeen=r['lastseen'], indent=' ', dryrun=options.dryrun, verbose=options.verbose)
for s in ['reboot', 'recovery', 'ipmi', 'pdu']:
r[s] = d[s]
r['output'] += d['output']
r['host'] = host
hostKey = 'kittenherder:%s.%s:%s' % (dDate, dHour, job)
for key in r:
db.hset(hostKey, key, r[key])
db.expire(hostKey, _keyExpire)
# all this because json cannot dumps() the timedelta object
td = r['lastseen']
if td is not None:
secs = td.seconds
hours, remainder = divmod(secs, 3600)
minutes, seconds = divmod(remainder, 60)
r['lastseen'] = { 'hours': hours,
'minutes': minutes,
'seconds': seconds,
'relative': relative(td),
'since': secs,
}
else:
if options.verbose:
log.info('%s not in requested environment %s (%s), skipping' % (job, options.environ, info['environment']))
else:
if options.verbose:
log.error('%s not listed in slavealloc, skipping' % job, exc_info=True)
return r
def processKitten(options, remoteEnv, job):
dNow = datetime.datetime.now()
dDate = dNow.strftime('%Y-%m-%d')
dHour = dNow.strftime('%H')
r = {}
if job is not None:
if job in remoteEnv.hosts:
info = remoteEnv.hosts[job]
if info['environment'] == options.environ:
if not info['enabled'] and not options.force:
if options.verbose:
log.info('%s not enabled, skipping' % job)
elif len(info['notes']) > 0 and 'tegra' not in job and not options.force:
if options.verbose:
log.info('%s has a slavealloc notes field, skipping' % job)
else:
log.info(job)
host = remoteEnv.getHost(job)
if host is None:
log.error('unknown host for %s' % job)
else:
r = remoteEnv.check(host, indent=' ', dryrun=options.dryrun, verbose=options.verbose)
if host.farm != 'ec2':
d = remoteEnv.rebootIfNeeded(host, lastSeen=r['lastseen'], indent=' ', dryrun=options.dryrun, verbose=options.verbose)
for s in ['reboot', 'recovery', 'ipmi', 'pdu']:
r[s] = d[s]
r['output'] += d['output']
r['host'] = host
hostKey = 'kittenherder:%s.%s:%s' % (dDate, dHour, job)
for key in r:
db.hset(hostKey, key, r[key])
db.expire(hostKey, _keyExpire)
# all this because json cannot dumps() the timedelta object
td = r['lastseen']
if td is not None:
secs = td.seconds
hours, remainder = divmod(secs, 3600)
minutes, seconds = divmod(remainder, 60)
r['lastseen'] = { 'hours': hours,
'minutes': minutes,
'seconds': seconds,
'relative': relative(td),
'since': secs,
}
else:
if options.verbose:
log.info('%s not in requested environment %s (%s), skipping' % (job, options.environ, info['environment']))
else:
if options.verbose:
log.error('%s not listed in slavealloc, skipping' % job, exc_info=True)
return r
def processKittens(options, jobs, results):
remoteEnv = releng.remote.RemoteEnvironment(options.tools, db=db)
dNow = datetime.datetime.now()
dDate = dNow.strftime('%Y-%m-%d')
dHour = dNow.strftime('%H')
while True:
try:
job = jobs.get(False)
except Empty:
job = None
if job is not None:
r = {}
if job in remoteEnv.hosts:
info = remoteEnv.hosts[job]
if info['environment'] == options.environ:
if not info['enabled'] and not options.force:
if options.verbose:
log.info('%s not enabled, skipping' % job)
elif len(info['notes']) > 0 and not options.force:
if options.verbose:
log.info('%s has a slavealloc notes field, skipping' % job)
else:
log.info(job)
host = remoteEnv.getHost(job)
if host is None:
log.error('unknown host for %s' % job)
else:
r = remoteEnv.check(host, indent=' ', dryrun=options.dryrun, verbose=options.verbose)
d = remoteEnv.rebootIfNeeded(host, lastSeen=r['lastseen'], indent=' ', dryrun=options.dryrun, verbose=options.verbose)
for s in ['reboot', 'recovery', 'ipmi', 'pdu']:
r[s] = d[s]
r['output'] += d['output']
hostKey = 'kittenherder:%s.%s:%s' % (dDate, dHour, job)
for key in r:
db.hset(hostKey, key, r[key])
db.expire(hostKey, _keyExpire)
# all this because json cannot dumps() the timedelta object
td = r['lastseen']
if td is not None:
secs = td.seconds
hours, remainder = divmod(secs, 3600)
minutes, seconds = divmod(remainder, 60)
r['lastseen'] = { 'hours': hours,
'minutes': minutes,
'seconds': seconds,
'relative': relative(td),
'since': secs,
}
log.info('%s: %s' % (job, json.dumps(r)))
if (host.farm == 'ec2') and (r['reboot'] or r['recovery']):
log.info('shutting down ec2 instance')
try:
conn = connect_to_region(host.info['region'],
aws_access_key_id=getPassword('aws_access_key_id'),
aws_secret_access_key=getPassword('aws_secret_access_key'))
conn.stop_instances(instance_ids=[host.info['id'],])
except:
log.error('unable to stop ec2 instance %s [%s]' % (job, host.info['id']), exc_info=True)
else:
if options.verbose:
log.info('%s not in requested environment %s (%s), skipping' % (job, options.environ, info['environment']))
else:
if options.verbose:
log.error('%s not listed in slavealloc, skipping' % job, exc_info=True)
results.put((job, r))
def processKittens(options, jobs, results):
remoteEnv = releng.remote.RemoteEnvironment(options.tools, db=db)
dNow = datetime.datetime.now()
dDate = dNow.strftime('%Y-%m-%d')
dHour = dNow.strftime('%H')
while True:
try:
job = jobs.get(False)
except Empty:
job = None
if job is not None:
r = {}
if job in remoteEnv.hosts:
info = remoteEnv.hosts[job]
if info['environment'] == options.environ:
if not info['enabled'] and not options.force:
if options.verbose:
log.info('%s not enabled, skipping' % job)
elif len(info['notes']) > 0 and not options.force:
if options.verbose:
log.info(
'%s has a slavealloc notes field, skipping' %
job)
else:
log.info(job)
host = remoteEnv.getHost(job)
if host is None:
log.error('unknown host for %s' % job)
else:
r = remoteEnv.check(host,
indent=' ',
dryrun=options.dryrun,
verbose=options.verbose)
d = remoteEnv.rebootIfNeeded(
host,
lastSeen=r['lastseen'],
indent=' ',
dryrun=options.dryrun,
verbose=options.verbose)
for s in ['reboot', 'recovery', 'ipmi', 'pdu']:
r[s] = d[s]
r['output'] += d['output']
hostKey = 'kittenherder:%s.%s:%s' % (dDate, dHour,
job)
for key in r:
db.hset(hostKey, key, r[key])
db.expire(hostKey, _keyExpire)
# all this because json cannot dumps() the timedelta object
td = r['lastseen']
if td is not None:
secs = td.seconds
hours, remainder = divmod(secs, 3600)
minutes, seconds = divmod(remainder, 60)
r['lastseen'] = {
'hours': hours,
'minutes': minutes,
'seconds': seconds,
'relative': relative(td),
'since': secs,
}
log.info('%s: %s' % (job, json.dumps(r)))
if (host.farm == 'ec2') and (r['reboot']
or r['recovery']):
log.info('shutting down ec2 instance')
try:
conn = connect_to_region(
host.info['region'],
aws_access_key_id=getPassword(
'aws_access_key_id'),
aws_secret_access_key=getPassword(
'aws_secret_access_key'))
conn.stop_instances(instance_ids=[
host.info['id'],
])
except:
log.error(
'unable to stop ec2 instance %s [%s]' %
(job, host.info['id']),
exc_info=True)
else:
if options.verbose:
log.info(
'%s not in requested environment %s (%s), skipping'
% (job, options.environ, info['environment']))
else:
if options.verbose:
log.error('%s not listed in slavealloc, skipping' % job,
exc_info=True)
results.put((job, r))
def check(kitten):
s = '%s: ' % kitten
info = remoteEnv.hosts[kitten]
if info['enabled']:
s += 'enabled'
else:
# hopefully short term hack until tegras are
# handled properly by slavealloc...
if 'tegra' in kitten:
s += 'enabled'
else:
s += 'DISABLED'
host = remoteEnv.getHost(kitten)
print s
print '%12s: %s' % ('farm', host.farm)
print '%12s: %s' % ('colo', info['datacenter'])
print '%12s: %s' % ('distro', info['distro'])
print '%12s: %s' % ('pool', info['pool'])
print '%12s: %s' % ('trustlevel', info['trustlevel'])
print '%12s: %s' % ('master', info['current_master'])
print '%12s: %s' % ('fqdn', host.fqdn)
print '%12s: %s' % ('PDU?', host.hasPDU)
print '%12s: %s' % ('IPMI?', host.hasIPMI)
if len(info['notes']) > 0:
print '%12s: %s' % ('note', info['notes'])
if not options.info:
r = remoteEnv.check(host, dryrun=options.dryrun, verbose=options.verbose, indent=' ', reboot=options.reboot)
for key in ('reachable', 'buildbot', 'tacfile', 'lastseen', 'master'):
if key in r:
s = r[key]
if key == 'lastseen':
if r[key] is None:
s = 'unknown'
else:
s = relative(r[key])
if key == 'master':
if r[key] is None:
s = info['current_master']
else:
if len(r[key]) > 0:
s = r[key][0]
else:
s = r[key]
print '%12s: %s' % (key, s)
if 'tacfile' in r and r['tacfile'].lower() != 'not found' and 'master' in r:
if r['master'] is not None and len(r['master']) > 0:
m = r['master'][0]
else:
m = ''
master = remoteEnv.findMaster(m)
current_master = remoteEnv.findMaster(info['current_master'])
if master is not None and current_master is not None and master['masterid'] != current_master['masterid']:
print '%12s: current master is different than buildbot.tac master [%s]' % ('error', m)
if options.reboot:
if 'reboot' in r and r['reboot']:
s = '%12s: via ' % 'reboot'
if 'ipmi' in r and r['ipmi']:
s += 'IPMI'
elif 'pdu' in r and r['pdu']:
s += 'PDU'
elif 'recovery' in r and r['recovery']:
s = 'recovery needed, could not reboot host'
else:
s = '%12s\n%s' % ('reboot requested but not performed', r['output'])
print s
if options.stop:
print host.graceful_shutdown()
def check(kitten):
s = '%s: ' % kitten
info = remoteEnv.hosts[kitten]
if info['enabled']:
s += 'enabled'
else:
# hopefully short term hack until tegras are
# handled properly by slavealloc...
if 'tegra' in kitten:
s += 'enabled'
else:
s += 'DISABLED'
print s
print '%12s: %s' % ('colo', info['datacenter'])
print '%12s: %s' % ('distro', info['distro'])
print '%12s: %s' % ('pool', info['pool'])
print '%12s: %s' % ('trustlevel', info['trustlevel'])
if len(info['notes']) > 0:
print '%12s: %s' % ('note', info['notes'])
if options.info:
print '%12s: %s' % ('master', info['current_master'])
else:
host = remoteEnv.getHost(kitten)
r = remoteEnv.check(host, dryrun=options.dryrun, verbose=options.verbose, indent=' ', reboot=options.reboot)
for key in ('fqdn', 'reachable', 'buildbot', 'tacfile', 'lastseen', 'master'):
s = r[key]
if key == 'lastseen':
if r[key] is None:
s = 'unknown'
else:
s = relative(r[key])
if key == 'master':
if r[key] is None:
s = info['current_master']
else:
if len(r[key]) > 0:
s = r[key][0]
else:
s = r[key]
print '%12s: %s' % (key, s)
if 'master' in r:
if r['master'] is not None and len(r['master']) > 0:
m = r['master'][0]
else:
m = ''
master = remoteEnv.findMaster(m)
current_master = remoteEnv.findMaster(info['current_master'])
if master['masterid'] != current_master['masterid']:
print '%12s: current master is different than buildbot.tac master [%s]' % ('error', m)
print '%12s: %s' % ('IPMI?', host.hasIPMI)
if options.stop:
print host.graceful_shutdown()