def hot(redis_client, kquery, kairos_time_range):
""" Hot / Hit """
logging.info("KQuery is HOT")
response_kquery = {'results': [], 'sample_size': 0}
for mts in MTS.from_cache(kquery.cached_data.get('mts_keys', []), redis_client):
response_kquery = mts.build_response(kairos_time_range, response_kquery)
# Handle a fully empty set of MTS: hand back the expected query with no values.
if len(response_kquery['results']) == 0:
kquery.query['values'] = []
response_kquery['results'].append(kquery.query)
return response_kquery
def hot(redis_client, kquery, kairos_time_range):
""" Hot / Hit """
logging.info("KQuery is HOT")
response_kquery = {'results': [], 'sample_size': 0}
for mts in MTS.from_cache(kquery.cached_data.get('mts_keys', []),
redis_client):
response_kquery = mts.build_response(kairos_time_range,
response_kquery)
# Handle a fully empty set of MTS: hand back the expected query with no values.
if len(response_kquery['results']) == 0:
kquery.query['values'] = []
response_kquery['results'].append(kquery.query)
return response_kquery
def test_from_cache():
redis_cli = MockRedis()
keys = ['key1', 'key2', 'key3']
ret_vals = list(MTS.from_cache(keys, redis_cli))
assert redis_cli.derived_pipeline.pipe_get_call_count == 3
assert redis_cli.derived_pipeline.execute_count == 1
ctr = 0
for mts in ret_vals:
assert isinstance(mts, MTS)
assert mts.result == {'hello': 'goodbye'}
assert mts.expiry == 10800
assert mts.redis_key == keys[ctr]
ctr += 1
assert redis_cli.set_call_count == 0 and redis_cli.get_call_count == 0
def test_from_cache():
redis_cli = MockRedis()
keys = ['key1', 'key2', 'key3']
ret_vals = list(MTS.from_cache(keys, redis_cli))
assert redis_cli.derived_pipeline.pipe_get_call_count == 3
assert redis_cli.derived_pipeline.execute_count == 1
ctr = 0
for mts in ret_vals:
assert isinstance(mts, MTS)
assert mts.result == {'hello': 'goodbye'}
assert mts.expiry == 10800
assert mts.redis_key == keys[ctr]
ctr += 1
assert redis_cli.set_call_count == 0 and redis_cli.get_call_count == 0
def warm(config, redis_client, kquery, kairos_time_range, range_needed):
""" Warm / Stale
config: nested dict loaded from the 'tscached' section of a yaml file.
redis_client: redis.StrictRedis
kquery: KQuery, generated from the client's request. get_cached was already called.
kairos_time_range: dict, contents some subset of '{start,end}_{relative,absolute}'
range_needed: describes kairos data needed to make cache complete for this request.
3-tuple (datetime start, datetime end, const<str>[FETCH_BEFORE, FETCH_AFTER])
"""
logging.info('KQuery is WARM')
expected_resolution = config['data'].get('expected_resolution', 10000)
time_dict = {
'start_absolute':
int(range_needed[0].strftime('%s')) * 1000 - expected_resolution,
'end_absolute': int(range_needed[1].strftime('%s')) * 1000,
}
new_kairos_result = kquery.proxy_to_kairos(
config['kairosdb']['kairosdb_host'],
config['kairosdb']['kairosdb_port'], time_dict)
response_kquery = {'results': [], 'sample_size': 0}
# Initial KQuery, and each MTS, can be slightly different on start/end. We need to get the min/max.
start_times = [
datetime.datetime.fromtimestamp(
float(kquery.cached_data.get('earliest_data')))
]
end_times = [
datetime.datetime.fromtimestamp(
float(kquery.cached_data.get('last_add_data')))
]
cached_mts = {} # redis key to MTS
# pull in cached MTS, put them in a lookup table
# TODO expected_resolution should be passed in
for mts in MTS.from_cache(kquery.cached_data.get('mts_keys', []),
redis_client):
kquery.add_mts(mts) # we want to write these back eventually
cached_mts[mts.get_key()] = mts
# loop over newly returned MTS. if they already existed, merge/write. if not, just write.
pipeline = redis_client.pipeline()
sign = False
for mts in MTS.from_result(new_kairos_result['queries'][0], redis_client,
kquery):
old_mts = cached_mts.get(mts.get_key())
if not old_mts: # This MTS just started reporting and isn't yet in the cache (cold behavior).
if len(mts.result['values']) > 0:
sign = True
kquery.add_mts(mts)
pipeline.set(mts.get_key(),
json.dumps(mts.result),
ex=mts.expiry)
response_kquery = mts.build_response(kairos_time_range,
response_kquery,
trim=False)
else:
if range_needed[2] == FETCH_AFTER:
end_times.append(range_needed[1])
old_mts.merge_at_end(mts)
# This seems the only case where too-old data should be removed.
expiry = old_mts.ttl_expire()
if expiry:
start_times.append(expiry)
elif range_needed[2] == FETCH_BEFORE:
start_times.append(range_needed[0])
old_mts.merge_at_beginning(mts)
else:
logging.error(
"WARM is not equipped for this range_needed attrib: %s" %
range_needed[2])
return response_kquery
sign = True
if len(old_mts.result['values']) > 0:
pipeline.set(old_mts.get_key(),
json.dumps(old_mts.result),
ex=old_mts.expiry)
response_kquery = old_mts.build_response(kairos_time_range,
response_kquery)
if not sign:
for mts in cached_mts.itervalues():
response_kquery = mts.build_response(kairos_time_range,
response_kquery)
try:
result = pipeline.execute()
success_count = len(filter(lambda x: x is True, result))
logging.info("MTS write pipeline: %d of %d successful" %
(success_count, len(result)))
kquery.upsert(min(start_times), max(end_times))
except redis.exceptions.RedisError as e:
# Sneaky edge case where Redis fails after reading but before writing. Still return data!
logging.error('RedisError: ' + e.message)
return response_kquery
def warm(config, redis_client, kquery, kairos_time_range, range_needed):
""" Warm / Stale
config: nested dict loaded from the 'tscached' section of a yaml file.
redis_client: redis.StrictRedis
kquery: KQuery, generated from the client's request. get_cached was already called.
kairos_time_range: dict, contents some subset of '{start,end}_{relative,absolute}'
range_needed: describes kairos data needed to make cache complete for this request.
3-tuple (datetime start, datetime end, const<str>[FETCH_BEFORE, FETCH_AFTER])
"""
logging.info('KQuery is WARM')
expected_resolution = config['data'].get('expected_resolution', 10000)
time_dict = {
'start_absolute': int(range_needed[0].strftime('%s')) * 1000 - expected_resolution,
'end_absolute': int(range_needed[1].strftime('%s')) * 1000,
}
new_kairos_result = kquery.proxy_to_kairos(config['kairosdb']['host'], config['kairosdb']['port'],
time_dict)
response_kquery = {'results': [], 'sample_size': 0}
# Initial KQuery, and each MTS, can be slightly different on start/end. We need to get the min/max.
start_times = [datetime.datetime.fromtimestamp(float(kquery.cached_data.get('earliest_data')))]
end_times = [datetime.datetime.fromtimestamp(float(kquery.cached_data.get('last_add_data')))]
cached_mts = {} # redis key to MTS
# pull in cached MTS, put them in a lookup table
# TODO expected_resolution should be passed in
for mts in MTS.from_cache(kquery.cached_data.get('mts_keys', []), redis_client):
kquery.add_mts(mts) # we want to write these back eventually
cached_mts[mts.get_key()] = mts
# loop over newly returned MTS. if they already existed, merge/write. if not, just write.
pipeline = redis_client.pipeline()
for mts in MTS.from_result(new_kairos_result['queries'][0], redis_client, kquery):
old_mts = cached_mts.get(mts.get_key())
if not old_mts: # This MTS just started reporting and isn't yet in the cache (cold behavior).
kquery.add_mts(mts)
pipeline.set(mts.get_key(), json.dumps(mts.result), ex=mts.expiry)
response_kquery = mts.build_response(kairos_time_range, response_kquery, trim=False)
else:
if range_needed[2] == FETCH_AFTER:
end_times.append(range_needed[1])
old_mts.merge_at_end(mts)
# This seems the only case where too-old data should be removed.
expiry = old_mts.ttl_expire()
if expiry:
start_times.append(expiry)
elif range_needed[2] == FETCH_BEFORE:
start_times.append(range_needed[0])
old_mts.merge_at_beginning(mts)
else:
logging.error("WARM is not equipped for this range_needed attrib: %s" % range_needed[2])
return response_kquery
pipeline.set(old_mts.get_key(), json.dumps(old_mts.result), ex=old_mts.expiry)
response_kquery = old_mts.build_response(kairos_time_range, response_kquery)
try:
result = pipeline.execute()
success_count = len(filter(lambda x: x is True, result))
logging.info("MTS write pipeline: %d of %d successful" % (success_count, len(result)))
kquery.upsert(min(start_times), max(end_times))
except redis.exceptions.RedisError as e:
# Sneaky edge case where Redis fails after reading but before writing. Still return data!
logging.error('RedisError: ' + e.message)
return response_kquery
