def test_get_chunked_time_ranges_last_12h():
kairos_timing = {'start_relative': {'unit': 'hours', 'value': '12'}}
now = datetime.datetime.now()
results = get_chunked_time_ranges({'chunking': {'chunk_length': 1800}}, kairos_timing)
assert len(results) == 6
for i in xrange(len(results)):
offset = datetime.timedelta(minutes=(i * 120))
assert results[i][1] == now - offset
assert results[i][0] == now - offset - datetime.timedelta(minutes=120, seconds=-1)
def test_get_chunked_time_ranges_last_12h():
kairos_timing = {'start_relative': {'unit': 'hours', 'value': '12'}}
now = datetime.datetime.now()
results = get_chunked_time_ranges({'chunking': {
'chunk_length': 1800
}}, kairos_timing)
assert len(results) == 6
for i in xrange(len(results)):
offset = datetime.timedelta(minutes=(i * 120))
assert results[i][1] == now - offset
assert results[i][0] == now - offset - datetime.timedelta(minutes=120,
seconds=-1)
def test_get_chunked_time_ranges_last_1h_clock_drift():
""" Most tests have time frozen, so the clock doesn't drift during execution.
This one purposely does not. It could flake if it takes over 2s to execute.
"""
kairos_timing = {'start_relative': {'unit': 'hours', 'value': '1'}}
now = datetime.datetime.now()
results = get_chunked_time_ranges({'chunking': {'chunk_length': 1800}}, kairos_timing)
assert len(results) == 2
for i in xrange(len(results) - 1):
offset = datetime.timedelta(minutes=(i * 30))
end_diff = results[i][1] - (now - offset)
assert end_diff < datetime.timedelta(seconds=2)
begin_diff = results[i][0] - (now - offset - datetime.timedelta(minutes=30))
assert begin_diff < datetime.timedelta(seconds=2)
def test_get_chunked_time_ranges_last_1h_clock_drift():
""" Most tests have time frozen, so the clock doesn't drift during execution.
This one purposely does not. It could flake if it takes over 2s to execute.
"""
kairos_timing = {'start_relative': {'unit': 'hours', 'value': '1'}}
now = datetime.datetime.now()
results = get_chunked_time_ranges({'chunking': {
'chunk_length': 1800
}}, kairos_timing)
assert len(results) == 2
for i in xrange(len(results) - 1):
offset = datetime.timedelta(minutes=(i * 30))
end_diff = results[i][1] - (now - offset)
assert end_diff < datetime.timedelta(seconds=2)
begin_diff = results[i][0] - (now - offset -
datetime.timedelta(minutes=30))
assert begin_diff < datetime.timedelta(seconds=2)
def cold(config, redis_client, kquery, kairos_time_range):
""" Cold / Miss, with chunking.
:param config: dict, 'tscached' level from config file.
:param redis_client: redis.StrictRedis
:param kquery: kquery.KQuery object
:param kairos_time_range: dict, time range from HTTP request payload
:return: dict, with keys sample_size (int) and results (list of dicts).
"""
chunked_ranges = get_chunked_time_ranges(config, kairos_time_range)
results = kquery.proxy_to_kairos_chunked(
config['kairosdb']['kairosdb_host'],
config['kairosdb']['kairosdb_port'], chunked_ranges)
logging.info('KQuery is COLD - using %d chunks' % len(results))
# Merge everything together as they come out - in chunked order - from the result.
mts_lookup = {}
ndx = len(
results) - 1 # Results come out newest to eldest, so count backwards.
while ndx >= 0:
for mts in MTS.from_result(results[ndx]['queries'][0], redis_client,
kquery):
# Almost certainly a null result. Empty data should not be included in mts_lookup.
if not mts.result or len(mts.result['values']) == 0:
logging.debug(
'cache_calls.cold: got an empty chunked mts response')
continue
if not mts_lookup.get(mts.get_key()):
mts_lookup[mts.get_key()] = mts
else:
# So, we could use merge_at_end, but it throws away beginning/ending values because of
# partial windowing. But since we force align_start_time, we don't have that worry here.
mts_lookup[
mts.get_key()].result['values'] += mts.result['values']
ndx -= 1
# Accumulate the full KQuery response as the Redis operations are being queued up.
response_kquery = {'results': [], 'sample_size': 0}
pipeline = redis_client.pipeline()
for mts in mts_lookup.values():
if len(mts.result['values']) == 0:
continue
kquery.add_mts(mts)
pipeline.set(mts.get_key(), json.dumps(mts.result), ex=mts.expiry)
logging.debug('Cold: Writing %d points to MTS: %s' %
(len(mts.result['values']), mts.get_key()))
response_kquery = mts.build_response(kairos_time_range,
response_kquery,
trim=False)
# Handle a fully empty set of MTS. Bail out before we upsert.
if len(mts_lookup) == 0:
kquery.query['values'] = []
s = {}
s['name'] = kquery.query['name']
s['tags'] = {}
s['values'] = []
response_kquery['results'].append(s)
logging.info(
'Received probable incorrect query; no results. Not caching!')
return response_kquery
# Execute the MTS Redis pipeline, then set the KQuery to its full new value.
try:
result = pipeline.execute()
success_count = len(filter(lambda x: x is True, result))
logging.info("MTS write pipeline: %d of %d successful" %
(success_count, len(result)))
start_time = chunked_ranges[-1][0]
end_time = chunked_ranges[0][1]
kquery.upsert(start_time, end_time)
except redis.exceptions.RedisError as e:
# We want to eat this Redis exception, because in a catastrophe this becones a straight proxy.
logging.error('RedisError: ' + e.message)
return response_kquery
def test_get_chunked_time_ranges_last_15m():
kairos_timing = {'start_relative': {'unit': 'minutes', 'value': '15'}}
now = datetime.datetime.now()
then = now - datetime.timedelta(minutes=15)
assert get_chunked_time_ranges({'chunking': {}}, kairos_timing) == [(then, now)]
def cold(config, redis_client, kquery, kairos_time_range):
""" Cold / Miss, with chunking.
:param config: dict, 'tscached' level from config file.
:param redis_client: redis.StrictRedis
:param kquery: kquery.KQuery object
:param kairos_time_range: dict, time range from HTTP request payload
:return: dict, with keys sample_size (int) and results (list of dicts).
"""
chunked_ranges = get_chunked_time_ranges(config, kairos_time_range)
results = kquery.proxy_to_kairos_chunked(config['kairosdb']['host'], config['kairosdb']['port'],
chunked_ranges)
logging.info('KQuery is COLD - using %d chunks' % len(results))
# Merge everything together as they come out - in chunked order - from the result.
mts_lookup = {}
ndx = len(results) - 1 # Results come out newest to eldest, so count backwards.
while ndx >= 0:
for mts in MTS.from_result(results[ndx]['queries'][0], redis_client, kquery):
# Almost certainly a null result. Empty data should not be included in mts_lookup.
if not mts.result or len(mts.result['values']) == 0:
logging.debug('cache_calls.cold: got an empty chunked mts response')
continue
if not mts_lookup.get(mts.get_key()):
mts_lookup[mts.get_key()] = mts
else:
# So, we could use merge_at_end, but it throws away beginning/ending values because of
# partial windowing. But since we force align_start_time, we don't have that worry here.
mts_lookup[mts.get_key()].result['values'] += mts.result['values']
ndx -= 1
# Accumulate the full KQuery response as the Redis operations are being queued up.
response_kquery = {'results': [], 'sample_size': 0}
pipeline = redis_client.pipeline()
for mts in mts_lookup.values():
kquery.add_mts(mts)
pipeline.set(mts.get_key(), json.dumps(mts.result), ex=mts.expiry)
logging.debug('Cold: Writing %d points to MTS: %s' % (len(mts.result['values']), mts.get_key()))
response_kquery = mts.build_response(kairos_time_range, response_kquery, trim=False)
# Handle a fully empty set of MTS. Bail out before we upsert.
if len(mts_lookup) == 0:
kquery.query['values'] = []
response_kquery['results'].append(kquery.query)
logging.info('Received probable incorrect query; no results. Not caching!')
return response_kquery
# Execute the MTS Redis pipeline, then set the KQuery to its full new value.
try:
result = pipeline.execute()
success_count = len(filter(lambda x: x is True, result))
logging.info("MTS write pipeline: %d of %d successful" % (success_count, len(result)))
start_time = chunked_ranges[-1][0]
end_time = chunked_ranges[0][1]
kquery.upsert(start_time, end_time)
except redis.exceptions.RedisError as e:
# We want to eat this Redis exception, because in a catastrophe this becones a straight proxy.
logging.error('RedisError: ' + e.message)
return response_kquery
def test_get_chunked_time_ranges_last_15m():
kairos_timing = {'start_relative': {'unit': 'minutes', 'value': '15'}}
now = datetime.datetime.now()
then = now - datetime.timedelta(minutes=15)
assert get_chunked_time_ranges({'chunking': {}},
kairos_timing) == [(then, now)]
