def caql(self, query, start, period, count):
"""
Fetch data using CAQL.
Returns a map: slot_name => list
Limitations:
- slots_names are currently output[i]
- For histogram output only a single slot is returned
"""
start = _fix_time(start, period)
params = {
"query": query,
"period": period,
"start": int(start),
"end": int(start + count * period),
}
res = self._api.api_call("GET", "/caql", params=params)["_data"]
if _caql_infer_type(res) == "histogram":
# In this case, we have only a single output metric and res looks like:
# res = [[1467892920, 60, {'1.2e+02': 1, '2': 1, '1': 1}], ... ]
return {
"time": [row[0] for row in res],
"output[0]": [Circllhist.from_dict(row[2]) for row in res],
}
else:
# In the numeric case res looks like this:
# res = [[1467892920, [1, 2, 3]], [1467892980, [1, 2, 3]], ... ]
out = {}
width = len(res[0][1])
out["time"] = [row[0] for row in res]
for i in range(width):
out["output[{}]".format(i)] = [row[1][i] for row in res]
return out
def _extend(kind, count, lst):
if kind == "histogram":
lst += [Circllhist() for i in range(count - len(lst))]
return lst
else:
for i, y in enumerate(lst):
if not y:
lst[i] = NAN
lst += [NAN] * (count - len(lst))
return lst
def test_hist(self):
h = Circllhist()
h.insert(123,3)
h.insert_intscale(1,1)
self.assertEqual(h.count(), 4)
self.assertEqual(h.bin_count(), 2)
self.assertAlmostEqual(h.sum(), 385.5)
self.assertAlmostEqual(h.mean(), 96.375)
self.assertAlmostEqual(h.quantile(0.5), 123.333, 1)
self.assertTrue(str(h))
g = Circllhist.from_dict(h.to_dict())
self.assertEqual(h.sum(), g.sum())
h.merge(g)
f = Circllhist.from_b64(h.to_b64())
self.assertEqual(h.sum(), f.sum())
h.clear()
self.assertEqual(h.count(), 0)
def test_hist(self):
h = Circllhist()
h.insert(123, 3)
h.insert_intscale(1, 1)
self.assertEqual(h.count(), 4)
self.assertEqual(h.bin_count(), 2)
self.assertAlmostEqual(h.sum(), 384.87619047)
self.assertAlmostEqual(h.mean(), 96.2190476)
self.assertAlmostEqual(h.quantile(0.5), 122.5, 1)
self.assertAlmostEqual(h.quantile(0.5, qtype=7), 122.5, 1)
self.assertTrue(str(h))
g = Circllhist.from_dict(h.to_dict())
self.assertEqual(h.sum(), g.sum())
h.merge(g)
print(h.to_b64())
f = Circllhist.from_b64(h.to_b64())
self.assertEqual(h.sum(), f.sum())
h.clear()
self.assertEqual(h.count(), 0)
def fmt(rec):
return _hist2kind(Circllhist.from_dict(rec[2]), kind)
def fmt(rec):
# raise Exception("Can't fetch histogram data from a numeric metric")
h = Circllhist()
if rec[kind]:
h.insert(rec[kind])
return h
def test_hist(self):
h = Circllhist()
h.insert(123, 3)
h.insert_intscale(1, 1)
self.assertEqual(h.count(), 4)
self.assertEqual(h.bin_count(), 2)
self.assertAlmostEqual(h.sum(), 385.5)
self.assertAlmostEqual(h.mean(), 96.375)
self.assertAlmostEqual(h.quantile(0.5), 123.333, 1)
self.assertTrue(str(h))
g = Circllhist.from_dict(h.to_dict())
self.assertEqual(h.sum(), g.sum())
h.merge(g)
f = Circllhist.from_b64(h.to_b64())
self.assertEqual(h.sum(), f.sum())
h.clear()
self.assertEqual(h.count(), 0)
def caql(self,
query,
start,
period,
count,
convert_hists=True,
explain=False):
"""
Fetch data using CAQL.
Args:
- query (str): the CAQL query string
- start (int/datetime): starttime of the query.
Either UNIX timestamp in seconds or datetime object
- period (int): period of data to fetch
- count (int): number of datapoints to fetch
- convert_hists (boolean, optional): Convert returned histograms to
Circllhist objects. Requires Circllhist to be available.
Returns:
res (dict): result in DF4 format. Example::
{
"head" : { count = 60, start = ..., period = 60 }
"meta" : [ {kind:"numeric", label:"some_metric"}, {...}, ... ] -- per metric metadata
"data" : [ [1,1,1,1,...], [2,2,2,2,...] ] -- per metric data
}
"""
if isinstance(start, datetime):
start = start.timestamp()
if not start % period == 0:
new_start = math.floor(start / period)
warnings.warn(
"start parameter {} is not divisible by period {}. Using {} instead."
.format(start, period, new_start))
start = new_start
params = {
"explain": explain,
"query": query,
"period": int(period),
"start": int(start),
"end": int(start + count * period),
"format": "DF4"
}
res = self._caql_request(params)
# In the case of 0 output metrics, res['meta']/res['data'] might be None
if not res['meta']: res['meta'] = []
if not res['data']: res['data'] = []
assert (len(res['meta']) == len(res['data']))
if not convert_hists:
return res
#
# Convert histogram JSON values to Circllhist objects.
#
if not Circllhist:
raise ImportError("Circllhist not available")
for i in range(len(res['meta'])):
if res['meta'][i]['kind'] == "histogram":
res['data'][i] = [
Circllhist.from_dict(h) for h in res['data'][i]
]
return res