def derive_machine_info(*args):
'''
derives machine information and stores it in redis.
args:
datetime_from - when to start from (optional)
default = utc_now() - 24hours
format: YYYY-MM-DD.HH
datetime_to - when to run until (optional, NA without datetime_from)
default = utc_now()
format: YYYY-MM-DD.HH
'''
current = datetime.datetime.utcnow()
last = current - datetime.timedelta(hours=24)
if len(args) is 2:
current = parse_dtime(args[1])
if len(args) > 0:
last = parse_dtime(args[0])
if current < last:
last, current = current, last
# get all these once and reuse them
# to reduce the risk of timeout during the while loop
redis_source = redis_client('briar-patch')
redis_store = redis_client('bramble')
slave_names = get_all_slave_names()
while current > last:
retry_hourly_info_forever(current, redis_source, redis_store,
slave_names)
current -= datetime.timedelta(hours=1)
def derive_machine_info(*args):
'''
derives machine information and stores it in redis.
args:
datetime_from - when to start from (optional)
default = utc_now() - 24hours
format: YYYY-MM-DD.HH
datetime_to - when to run until (optional, NA without datetime_from)
default = utc_now()
format: YYYY-MM-DD.HH
'''
current = datetime.datetime.utcnow()
last = current - datetime.timedelta(hours=24)
if len(args) is 2:
current = parse_dtime(args[1])
if len(args) > 0:
last = parse_dtime(args[0])
if current < last:
last, current = current, last
# get all these once and reuse them
# to reduce the risk of timeout during the while loop
redis_source = redis_client('briar-patch')
redis_store = redis_client('bramble')
slave_names = get_all_slave_names()
while current > last:
retry_hourly_info_forever(current, redis_source, redis_store,
slave_names)
current -= datetime.timedelta(hours=1)
def handle(self, *dates, **options):
redis = redis_client('default')
redis_store = redis_client('store')
if not dates:
dates = [(datetime.datetime.utcnow()
.strftime('%Y-%m-%d.%H'))]
for date in dates:
assert re.findall('\.\d{1,2}$', date) # must end in .HH or .H
slave_names = get_all_slave_names()
assert slave_names
assert len(slave_names) == len(set(slave_names))
for date in dates:
logging.debug("Processing machine counts for %s" % date)
process_date(date, slave_names, redis, redis_store,
dry_run=options['dryrun'],
quiet=not int(options['verbosity']),
)
def fetch_machine_info(dt, redis_store=None):
'''
returns a list of machine data from the redis store for the provided
datetime, dt
'''
if not redis_store:
redis_store = redis_client('bramble')
machines = []
for slave in redis_store.smembers(make_info_pool_key(dt)):
machine_key = make_info_key(dt, slave)
machine_info = redis_store.hgetall(machine_key)
machine_info.update({
'name': slave,
'datetime': format_date(dt),
})
machines.append(machine_info)
return machines
def build_machine_info(dt, redis_source=None, redis_store=None,
slave_names=None):
"""
derives slave machine information from briar-patch and stores it in the
bramble data store.
creates a set, machine:yyyy-mm-dd.hh, containing all the slave names for
the provided datetime, dt
creates a hash machine:yyyy-mm-dd.hh.slavename, for each slave in the
provided list of slave names
"""
if not redis_source:
redis_source = redis_client('briar-patch')
if not redis_store:
redis_store = redis_client('bramble')
if not slave_names:
slave_names = get_all_slave_names()
_builds_key = make_builds_key(dt)
if not redis_source.exists(_builds_key):
logger.warning("No build data for %s", _builds_key)
machine_day = make_info_pool_key(dt)
# put all slaves into hashes hash scaffolding
for slave in slave_names:
redis_store.hmset(make_info_key(dt, slave), {
'scheduler': '',
'master': '',
'platform': '',
'successes': 0,
'failures': 0,
})
redis_store.sadd(machine_day, slave)
for build in redis_source.smembers(_builds_key):
if 'None' in build:
continue
type_, builduid = build.split(':')
_jobs_key = 'build:%s' % builduid
for job_key in redis_source.smembers(_jobs_key):
job = redis_source.hgetall(job_key)
slave = job['slave']
# create the machine key and machine value objects
machine_key = make_info_key(dt, slave)
if not redis_store.sismember(machine_day, machine_key):
logger.warning("%s not in master slave list, skipping",
machine_key)
continue
else:
logger.warning("%s is in master slave list")
redis_source.hmset(machine_key, {
'scheduler': job['scheduler'],
'master': job['master'],
'platform': job['platform'],
})
if job['results'] is "0":
redis_source.hincrby(machine_key, 'successes')
else:
redis_source.hincrby(machine_key, 'failures')
def derive_hourly_info(date, redis_source=None, redis_store=None,
slave_names=None):
"""
derives slave machine information from briar-patch and stores it in the
bramble data store.
creates a set, machine:yyyy-mm-dd.hh, containing all the slave names for
the provided date
creates a hash machine:yyyy-mm-dd.hh.slavename, for each slave in the
provided list of slave names
"""
if not redis_source:
redis_source = redis_client('briar-patch')
if not redis_store:
redis_store = redis_client('bramble')
if not slave_names:
slave_names = get_all_slave_names()
# generate a key for today's machine info pool
pool_key = make_info_pool_key(date)
# clean out any previously calculated data
for slave in redis_store.smembers(pool_key):
redis_store.delete(make_info_key(date, slave))
redis_store.delete(pool_key)
# put all slaves into the pool, add hash scaffolding
for slave in slave_names:
redis_store.sadd(pool_key, slave)
redis_store.hmset(make_info_key(date, slave), {
'scheduler': '',
'master': '',
'platform': '',
'successes': 0,
'failures': 0,
})
_builds_key = make_builds_key(date)
if not redis_source.exists(_builds_key):
logger.warning("No build data for %s", _builds_key)
return
logger.warning("deriving data for %s", _builds_key)
for build in redis_source.smembers(_builds_key):
if 'None' in build:
continue
type_, builduid = build.split(':')
_jobs_key = make_builds_key(builduid)
for job_key in redis_source.smembers(_jobs_key):
job = redis_source.hgetall(job_key)
slave = job['slave']
# create the machine key and machine value objects
machine_key = make_info_key(date, slave)
if not redis_store.exists(machine_key):
logger.warning("%s not in master slave list, skipping",
machine_key)
continue
try:
for key in ['scheduler', 'master', 'platform']:
redis_store.hmset(machine_key, {key: job[key]})
except KeyError:
logger.warning("%s not found in latest job for slave %s",
key, slave)
if job['results'] is "0":
redis_store.hincrby(machine_key, 'successes')
else:
redis_store.hincrby(machine_key, 'failures')
