def test_from_result():
""" Test from_result """
redis_cli = MockRedis()
results = {'results': [{'wubba-lubba': 'dub-dub'}, {'thats-the-way': 'the-news-goes'}]}
kq = KQuery(redis_cli)
kq.query = 'wat'
ret_vals = MTS.from_result(results, redis_cli, kq)
assert isinstance(ret_vals, GeneratorType)
ctr = 0
for mts in ret_vals:
assert isinstance(mts, MTS)
assert mts.result == results['results'][ctr]
assert mts.expiry == 10800
assert mts.cache_type == 'mts'
assert mts.query_mask == 'wat'
ctr += 1
assert redis_cli.set_call_count == 0 and redis_cli.get_call_count == 0
def test_from_result():
""" Test from_result """
redis_cli = MockRedis()
results = {
'results': [{
'wubba-lubba': 'dub-dub'
}, {
'thats-the-way': 'the-news-goes'
}]
}
kq = KQuery(redis_cli)
kq.query = 'wat'
ret_vals = MTS.from_result(results, redis_cli, kq)
assert isinstance(ret_vals, GeneratorType)
ctr = 0
for mts in ret_vals:
assert isinstance(mts, MTS)
assert mts.result == results['results'][ctr]
assert mts.expiry == 10800
assert mts.cache_type == 'mts'
assert mts.query_mask == 'wat'
ctr += 1
assert redis_cli.set_call_count == 0 and redis_cli.get_call_count == 0
def cold(config, redis_client, kquery, kairos_time_range):
""" Cold / Miss, with chunking.
:param config: dict, 'tscached' level from config file.
:param redis_client: redis.StrictRedis
:param kquery: kquery.KQuery object
:param kairos_time_range: dict, time range from HTTP request payload
:return: dict, with keys sample_size (int) and results (list of dicts).
"""
chunked_ranges = get_chunked_time_ranges(config, kairos_time_range)
results = kquery.proxy_to_kairos_chunked(
config['kairosdb']['kairosdb_host'],
config['kairosdb']['kairosdb_port'], chunked_ranges)
logging.info('KQuery is COLD - using %d chunks' % len(results))
# Merge everything together as they come out - in chunked order - from the result.
mts_lookup = {}
ndx = len(
results) - 1 # Results come out newest to eldest, so count backwards.
while ndx >= 0:
for mts in MTS.from_result(results[ndx]['queries'][0], redis_client,
kquery):
# Almost certainly a null result. Empty data should not be included in mts_lookup.
if not mts.result or len(mts.result['values']) == 0:
logging.debug(
'cache_calls.cold: got an empty chunked mts response')
continue
if not mts_lookup.get(mts.get_key()):
mts_lookup[mts.get_key()] = mts
else:
# So, we could use merge_at_end, but it throws away beginning/ending values because of
# partial windowing. But since we force align_start_time, we don't have that worry here.
mts_lookup[
mts.get_key()].result['values'] += mts.result['values']
ndx -= 1
# Accumulate the full KQuery response as the Redis operations are being queued up.
response_kquery = {'results': [], 'sample_size': 0}
pipeline = redis_client.pipeline()
for mts in mts_lookup.values():
if len(mts.result['values']) == 0:
continue
kquery.add_mts(mts)
pipeline.set(mts.get_key(), json.dumps(mts.result), ex=mts.expiry)
logging.debug('Cold: Writing %d points to MTS: %s' %
(len(mts.result['values']), mts.get_key()))
response_kquery = mts.build_response(kairos_time_range,
response_kquery,
trim=False)
# Handle a fully empty set of MTS. Bail out before we upsert.
if len(mts_lookup) == 0:
kquery.query['values'] = []
s = {}
s['name'] = kquery.query['name']
s['tags'] = {}
s['values'] = []
response_kquery['results'].append(s)
logging.info(
'Received probable incorrect query; no results. Not caching!')
return response_kquery
# Execute the MTS Redis pipeline, then set the KQuery to its full new value.
try:
result = pipeline.execute()
success_count = len(filter(lambda x: x is True, result))
logging.info("MTS write pipeline: %d of %d successful" %
(success_count, len(result)))
start_time = chunked_ranges[-1][0]
end_time = chunked_ranges[0][1]
kquery.upsert(start_time, end_time)
except redis.exceptions.RedisError as e:
# We want to eat this Redis exception, because in a catastrophe this becones a straight proxy.
logging.error('RedisError: ' + e.message)
return response_kquery
def warm(config, redis_client, kquery, kairos_time_range, range_needed):
""" Warm / Stale
config: nested dict loaded from the 'tscached' section of a yaml file.
redis_client: redis.StrictRedis
kquery: KQuery, generated from the client's request. get_cached was already called.
kairos_time_range: dict, contents some subset of '{start,end}_{relative,absolute}'
range_needed: describes kairos data needed to make cache complete for this request.
3-tuple (datetime start, datetime end, const<str>[FETCH_BEFORE, FETCH_AFTER])
"""
logging.info('KQuery is WARM')
expected_resolution = config['data'].get('expected_resolution', 10000)
time_dict = {
'start_absolute':
int(range_needed[0].strftime('%s')) * 1000 - expected_resolution,
'end_absolute': int(range_needed[1].strftime('%s')) * 1000,
}
new_kairos_result = kquery.proxy_to_kairos(
config['kairosdb']['kairosdb_host'],
config['kairosdb']['kairosdb_port'], time_dict)
response_kquery = {'results': [], 'sample_size': 0}
# Initial KQuery, and each MTS, can be slightly different on start/end. We need to get the min/max.
start_times = [
datetime.datetime.fromtimestamp(
float(kquery.cached_data.get('earliest_data')))
]
end_times = [
datetime.datetime.fromtimestamp(
float(kquery.cached_data.get('last_add_data')))
]
cached_mts = {} # redis key to MTS
# pull in cached MTS, put them in a lookup table
# TODO expected_resolution should be passed in
for mts in MTS.from_cache(kquery.cached_data.get('mts_keys', []),
redis_client):
kquery.add_mts(mts) # we want to write these back eventually
cached_mts[mts.get_key()] = mts
# loop over newly returned MTS. if they already existed, merge/write. if not, just write.
pipeline = redis_client.pipeline()
sign = False
for mts in MTS.from_result(new_kairos_result['queries'][0], redis_client,
kquery):
old_mts = cached_mts.get(mts.get_key())
if not old_mts: # This MTS just started reporting and isn't yet in the cache (cold behavior).
if len(mts.result['values']) > 0:
sign = True
kquery.add_mts(mts)
pipeline.set(mts.get_key(),
json.dumps(mts.result),
ex=mts.expiry)
response_kquery = mts.build_response(kairos_time_range,
response_kquery,
trim=False)
else:
if range_needed[2] == FETCH_AFTER:
end_times.append(range_needed[1])
old_mts.merge_at_end(mts)
# This seems the only case where too-old data should be removed.
expiry = old_mts.ttl_expire()
if expiry:
start_times.append(expiry)
elif range_needed[2] == FETCH_BEFORE:
start_times.append(range_needed[0])
old_mts.merge_at_beginning(mts)
else:
logging.error(
"WARM is not equipped for this range_needed attrib: %s" %
range_needed[2])
return response_kquery
sign = True
if len(old_mts.result['values']) > 0:
pipeline.set(old_mts.get_key(),
json.dumps(old_mts.result),
ex=old_mts.expiry)
response_kquery = old_mts.build_response(kairos_time_range,
response_kquery)
if not sign:
for mts in cached_mts.itervalues():
response_kquery = mts.build_response(kairos_time_range,
response_kquery)
try:
result = pipeline.execute()
success_count = len(filter(lambda x: x is True, result))
logging.info("MTS write pipeline: %d of %d successful" %
(success_count, len(result)))
kquery.upsert(min(start_times), max(end_times))
except redis.exceptions.RedisError as e:
# Sneaky edge case where Redis fails after reading but before writing. Still return data!
logging.error('RedisError: ' + e.message)
return response_kquery
def cold(config, redis_client, kquery, kairos_time_range):
""" Cold / Miss, with chunking.
:param config: dict, 'tscached' level from config file.
:param redis_client: redis.StrictRedis
:param kquery: kquery.KQuery object
:param kairos_time_range: dict, time range from HTTP request payload
:return: dict, with keys sample_size (int) and results (list of dicts).
"""
chunked_ranges = get_chunked_time_ranges(config, kairos_time_range)
results = kquery.proxy_to_kairos_chunked(config['kairosdb']['host'], config['kairosdb']['port'],
chunked_ranges)
logging.info('KQuery is COLD - using %d chunks' % len(results))
# Merge everything together as they come out - in chunked order - from the result.
mts_lookup = {}
ndx = len(results) - 1 # Results come out newest to eldest, so count backwards.
while ndx >= 0:
for mts in MTS.from_result(results[ndx]['queries'][0], redis_client, kquery):
# Almost certainly a null result. Empty data should not be included in mts_lookup.
if not mts.result or len(mts.result['values']) == 0:
logging.debug('cache_calls.cold: got an empty chunked mts response')
continue
if not mts_lookup.get(mts.get_key()):
mts_lookup[mts.get_key()] = mts
else:
# So, we could use merge_at_end, but it throws away beginning/ending values because of
# partial windowing. But since we force align_start_time, we don't have that worry here.
mts_lookup[mts.get_key()].result['values'] += mts.result['values']
ndx -= 1
# Accumulate the full KQuery response as the Redis operations are being queued up.
response_kquery = {'results': [], 'sample_size': 0}
pipeline = redis_client.pipeline()
for mts in mts_lookup.values():
kquery.add_mts(mts)
pipeline.set(mts.get_key(), json.dumps(mts.result), ex=mts.expiry)
logging.debug('Cold: Writing %d points to MTS: %s' % (len(mts.result['values']), mts.get_key()))
response_kquery = mts.build_response(kairos_time_range, response_kquery, trim=False)
# Handle a fully empty set of MTS. Bail out before we upsert.
if len(mts_lookup) == 0:
kquery.query['values'] = []
response_kquery['results'].append(kquery.query)
logging.info('Received probable incorrect query; no results. Not caching!')
return response_kquery
# Execute the MTS Redis pipeline, then set the KQuery to its full new value.
try:
result = pipeline.execute()
success_count = len(filter(lambda x: x is True, result))
logging.info("MTS write pipeline: %d of %d successful" % (success_count, len(result)))
start_time = chunked_ranges[-1][0]
end_time = chunked_ranges[0][1]
kquery.upsert(start_time, end_time)
except redis.exceptions.RedisError as e:
# We want to eat this Redis exception, because in a catastrophe this becones a straight proxy.
logging.error('RedisError: ' + e.message)
return response_kquery
def warm(config, redis_client, kquery, kairos_time_range, range_needed):
""" Warm / Stale
config: nested dict loaded from the 'tscached' section of a yaml file.
redis_client: redis.StrictRedis
kquery: KQuery, generated from the client's request. get_cached was already called.
kairos_time_range: dict, contents some subset of '{start,end}_{relative,absolute}'
range_needed: describes kairos data needed to make cache complete for this request.
3-tuple (datetime start, datetime end, const<str>[FETCH_BEFORE, FETCH_AFTER])
"""
logging.info('KQuery is WARM')
expected_resolution = config['data'].get('expected_resolution', 10000)
time_dict = {
'start_absolute': int(range_needed[0].strftime('%s')) * 1000 - expected_resolution,
'end_absolute': int(range_needed[1].strftime('%s')) * 1000,
}
new_kairos_result = kquery.proxy_to_kairos(config['kairosdb']['host'], config['kairosdb']['port'],
time_dict)
response_kquery = {'results': [], 'sample_size': 0}
# Initial KQuery, and each MTS, can be slightly different on start/end. We need to get the min/max.
start_times = [datetime.datetime.fromtimestamp(float(kquery.cached_data.get('earliest_data')))]
end_times = [datetime.datetime.fromtimestamp(float(kquery.cached_data.get('last_add_data')))]
cached_mts = {} # redis key to MTS
# pull in cached MTS, put them in a lookup table
# TODO expected_resolution should be passed in
for mts in MTS.from_cache(kquery.cached_data.get('mts_keys', []), redis_client):
kquery.add_mts(mts) # we want to write these back eventually
cached_mts[mts.get_key()] = mts
# loop over newly returned MTS. if they already existed, merge/write. if not, just write.
pipeline = redis_client.pipeline()
for mts in MTS.from_result(new_kairos_result['queries'][0], redis_client, kquery):
old_mts = cached_mts.get(mts.get_key())
if not old_mts: # This MTS just started reporting and isn't yet in the cache (cold behavior).
kquery.add_mts(mts)
pipeline.set(mts.get_key(), json.dumps(mts.result), ex=mts.expiry)
response_kquery = mts.build_response(kairos_time_range, response_kquery, trim=False)
else:
if range_needed[2] == FETCH_AFTER:
end_times.append(range_needed[1])
old_mts.merge_at_end(mts)
# This seems the only case where too-old data should be removed.
expiry = old_mts.ttl_expire()
if expiry:
start_times.append(expiry)
elif range_needed[2] == FETCH_BEFORE:
start_times.append(range_needed[0])
old_mts.merge_at_beginning(mts)
else:
logging.error("WARM is not equipped for this range_needed attrib: %s" % range_needed[2])
return response_kquery
pipeline.set(old_mts.get_key(), json.dumps(old_mts.result), ex=old_mts.expiry)
response_kquery = old_mts.build_response(kairos_time_range, response_kquery)
try:
result = pipeline.execute()
success_count = len(filter(lambda x: x is True, result))
logging.info("MTS write pipeline: %d of %d successful" % (success_count, len(result)))
kquery.upsert(min(start_times), max(end_times))
except redis.exceptions.RedisError as e:
# Sneaky edge case where Redis fails after reading but before writing. Still return data!
logging.error('RedisError: ' + e.message)
return response_kquery