def report_group(project, tablename, datestring1, datestring2, value_key):
# get data, from datalogger, or dataloggerhelper
datalogger = DataLogger(BASEDIR, project, tablename)
dataloggerweb = DataLoggerWeb(DATALOGGER_URL)
print "loading data"
starttime = time.time()
# tsa1 = datalogger.load_tsa(datestring1)
tsa1 = dataloggerweb.get_tsa(project, tablename, datestring1)
tsa1 = datalogger.group_by(datestring1, tsa1, ("hostname",), lambda a, b: (a + b) / 2)
# tsa2 = datalogger.load_tsa(datestring2)
tsa2 = dataloggerweb.get_tsa(project, tablename, datestring2)
tsa2 = datalogger.group_by(datestring2, tsa2, ("hostname",), lambda a, b: (a + b) / 2)
print "Duration load %f" % (time.time() - starttime)
starttime = time.time()
cm = CorrelationMatrixTime(tsa1, tsa2, value_key)
print "TOP most differing keys between %s and %s" % (datestring1, datestring2)
for key, coefficient in sorted(cm.items(), key=lambda items: items[1], reverse=True)[:20]:
print key, coefficient
def get_tsa_adv(self, args):
"""
return exported TimeseriesArray json formatted
"""
group_funcs = {
"avg" : lambda a, b: (a+b)/2,
"min" : min,
"max" : max,
"sum" : lambda a, b: a+b,
}
logging.info(args)
project, tablename, datestring, groupkeys_enc, group_func_name, index_pattern_enc = args
groupkeys_dec = eval(base64.b64decode(groupkeys_enc)) # should be tuple
logging.info("groupkeys_dec: %s", groupkeys_dec)
index_pattern = base64.b64decode(index_pattern_enc)
if index_pattern == "None":
index_pattern = None
logging.info("index_pattern: %s", index_pattern)
assert group_func_name in group_funcs.keys()
datalogger = DataLogger(basedir, project, tablename)
tsa = None
# gete data
if groupkeys_dec is not None:
logging.info("groupkeys is %s", groupkeys_dec)
groupkeys = tuple([unicode(key_value) for key_value in groupkeys_dec])
tsa1 = datalogger.load_tsa(datestring, index_pattern=index_pattern)
tsa = datalogger.group_by(datestring, tsa1, groupkeys, group_funcs[group_func_name])
else:
logging.info("groupkeys is None, fallback to get ungrouped tsa")
tsa = datalogger.load_tsa(datestring, index_pattern=index_pattern)
logging.info(tsa.keys()[0])
web.header('Content-type', 'text/html')
# you must not set this option, according to
# http://stackoverflow.com/questions/11866333/ioerror-when-trying-to-serve-file
# web.header('Transfer-Encoding','chunked')
yield "[" + json.dumps(tsa.export().next())
for chunk in tsa.export():
#logging.info("yielding %s", chunk)
yield "," + json.dumps(chunk)
yield "]"
def get_hc_daily_data(self, args):
"""
get values(min 1) from TimeseriesArray to use for highcharts graphing
parameters:
/project/tablename/datestring/index_key/value_keynames/index_keyname
<b>poject</b> <str> defines which project to use
<b>tablename</b> <str> defines which tablename to use
<b>datestring</b> <str> in form of YYYY-MM-DD to define whih day to use
<b>index_key</b> base64 encoded tuple, defines which Timeseries to use, ex. (u'srvcl14db2.tilak.cc', u'DB2', u'ablagsys', u'data only')
<b>value_keynames</b> json encoded list of value_keynames to show in graph
each value_keyname will be a separate highchart line
<b>index_keynam</b> json encoded <str> or null
if given, the data will be grouped on this given index_keyname
if hostname is given the above example will be gruped by hostname=u'srvcl14db2.tilak.cc'
and all possible Timeseries will be summed up
return data json encoded like this
[
{ name : "timeseries value_name 1",
data : [[ts, value], ...]
},
{ name : "timeseries value name 2",
data : [[ts, value], ...]
}
...
]
this structure could already be used in highcharts.data
"""
assert len(args) == 6
project, tablename, datestring, index_key_b64, value_keynames_str, index_keyname_str = args
# key_str should be a tuple string, convert to unicode tuple
index_key = tuple([unicode(key_value) for key_value in eval(base64.b64decode(index_key_b64))])
value_keynames = ()
if json.loads(value_keynames_str) is not None:
value_keynames = tuple(json.loads(value_keynames_str))
index_keyname = ()
if json.loads(index_keyname_str) is not None:
index_keyname = (json.loads(index_keyname_str),)
logging.info("project : %s", project)
logging.info("tablename : %s", tablename)
logging.info("datestring : %s", datestring)
logging.info("index_key : %s", index_key)
logging.info("value_keynames : %s", value_keynames)
logging.info("index_keyname : %s", index_keyname)
datalogger = DataLogger(basedir, project, tablename)
index_key_dict = dict(zip(datalogger.index_keynames, index_key))
# build filter if any group_by is given
filterkeys = index_key_dict # default
if len(index_keyname) > 0:
filterkeys = {}
for key in index_keyname:
filterkeys[key] = index_key_dict[key]
logging.info("using filterkeys: %s", filterkeys)
tsa = datalogger.load_tsa(datestring, filterkeys=filterkeys)
logging.info("got tsa with %d keys", len(tsa))
# grouping stuff if necessary
data = None # holds finally calculated data
stats = None # holds tsstats informations
if len(index_keyname) > 0:
# grouping by key named
logging.info("generating new key for left possible keys in grouped tsa")
new_key = tuple((index_key_dict[key] for key in index_keyname))
logging.info("key after grouping would be %s", new_key)
logging.info("grouping tsa by %s", index_keyname)
new_tsa = datalogger.group_by(datestring, tsa, index_keyname, group_func=lambda a, b: a + b)
tsa = new_tsa
data = tsa[new_key].dump_dict()
stats = tsa[new_key].stats.get_stats()
else:
# not grouping, simple
data = tsa[index_key].dump_dict()
stats = tsa[index_key].stats.get_stats()
# holds return data
logging.info("data keys : %s", data[data.keys()[0]].keys())
# get in highcharts shape
result = {
"stats" : stats,
"data" : [], # holds highchart data
}
for value_keyname in value_keynames:
# its important to sort by timestamp, to not confuse
# highcharts
result["data"].append(
{
"name" : value_keyname,
"data" : tuple(((ts * 1000, row_dict[value_keyname]) for ts, row_dict in sorted(data.items())))
}
)
return json.dumps(result)
def get_chart_data_ungrouped(self, args):
"""
get values from RAW Archive
parameters:
/<str>project/<str>tablename/<str>datestring/<str>key/<str>value_keys/<str>datetype/<str>group_str
keyids=hostname:srvszp2orb.tilak.cc means
this is only useful if keyids are unique
return data like this:
[
{
name: "name of this series" usually this is the counter name
data : [[ts, value], ...]
},
...
]
"""
assert len(args) == 7
project, tablename, datestring, keys_str, value_keys_str, datatype_str, group_str = args
# key_str should be a tuple string, convert to unicode tuple
keys = tuple([unicode(key_value) for key_value in eval(base64.b64decode(keys_str))])
value_keys = ()
if json.loads(value_keys_str) is not None:
value_keys = tuple(json.loads(value_keys_str))
datatype = json.loads(datatype_str)
group_by = ()
if json.loads(group_str) is not None:
group_by = (json.loads(group_str),)
logging.info("project : %s", project)
logging.info("tablename : %s", tablename)
logging.info("datestring : %s", datestring)
logging.info("keys : %s", keys)
logging.info("value_keys : %s", value_keys)
logging.info("datatype : %s", datatype)
logging.info("group_by : %s", group_by)
datalogger = DataLogger(basedir, project, tablename)
keys_dict = dict(zip(datalogger.index_keynames, keys))
# build filter if any group_by is given
filterkeys = keys_dict # default
if len(group_by) > 0:
filterkeys = {}
for key in group_by:
filterkeys[key] = keys_dict[key]
logging.info("useing filterkeys: %s", filterkeys)
tsa = datalogger.load_tsa(datestring, filterkeys=filterkeys)
logging.info("got tsa with %d keys", len(tsa))
# is there something to calculate, lets do it
if datatype != u"absolute":
new_value_keys = []
for value_key in value_keys:
new_value_key = None
if datatype == "derive":
new_value_key = "%s_d" % value_key
logging.info("deriving %s to %s", value_key, new_value_key)
tsa.add_derive_col(value_key, new_value_key)
elif datatype == "per_s":
new_value_key = "%s_s" % value_key
logging.info("deriving %s to %s", value_key, new_value_key)
tsa.add_per_s_col(value_key, new_value_key)
tsa.remove_col(value_key)
new_value_keys.append(new_value_key)
value_keys = new_value_keys
#logging.info(tsa.get_value_keys())
# grouping stuff if necessary
data = None # holds finally calculated data
stats = None
if len(group_by) > 0:
logging.info("generating new key for left possible keys in grouped tsa")
key_dict = dict(zip(datalogger.index_keynames, keys))
new_key = tuple((key_dict[key] for key in group_by))
logging.info("key after grouping would be %s", new_key)
logging.info("grouping tsa by %s", group_by)
new_tsa = datalogger.group_by(datestring, tsa, group_by, group_func=lambda a, b: a + b)
#new_tsa = tsa.get_group_by_tsa(group_by, group_func=lambda a: sum(a))
tsa = new_tsa
data = tsa[new_key].dump_dict()
stats = tsa[new_key].stats.htmltable()
else:
data = tsa[keys].dump_dict()
stats = tsa[keys].stats.htmltable()
result = {
"stats" : stats,
"data" : [],
}
# holds return data
logging.info("data keys : %s", data[data.keys()[0]].keys())
for value_key in value_keys:
# ist important to sort by timestamp, to not confuse
# highcharts
result["data"].append(
{
"name" : value_key,
"data" : tuple(((ts * 1000, row_dict[value_key]) for ts, row_dict in sorted(data.items())))
}
)
return json.dumps(result)
