def find_report_instance_by_dt(self, dt, tags=None):
"""Find a report instance with a creation datetime matching ``dt`` datetime as close as
possible."""
ri_prev = self.fetch_prev_instance(util.max_uuid_with_dt(dt), tags)
ri_next = self.fetch_next_instance(util.min_uuid_with_dt(dt), tags)
return min([ri_prev, ri_next], key=lambda ri: abs((ri.created - dt).total_seconds())
if ri else sys.maxint)
def test_range__text_table_drawer(self):
tile, pr, rd = self.test_range__chart_range_drawer()
tc = tile.get_tile_config()
self.assertEqual('ChartRangeDrawer', tc['tile_options']['drawer_type'])
tc['tile_options']['drawer_type'] = 'TextTableDrawer'
tile = tile.insert_similar(tc)
self.assertEqual('TextTableDrawer', tc['tile_options']['drawer_type'])
data = tile.get_tile_data()
data = tile.get_new_tile_data(pr.report_instance.report_instance_id)
self.assertIn('series_data_as_rows', data)
self.assertEqual(2, len(data['series_data']))
self.assertEqual([], data['series_data'][0]['data_points'])
self.assertEqual([], data['series_data'][1]['data_points'])
data = tile.get_new_tile_data(rd.instances[-1].report_instance_id)
self.assertIn('series_data_as_rows', data)
self.assertEqual(2, len(data['series_data']))
self.assertEqual(0, len(data['series_data'][0]['data_points']))
self.assertEqual(1, len(data['series_data'][1]['data_points']))
self.assertEqual(128, data['series_data'][1]['data_points'][0].value)
data = tile.get_new_tile_data(rd.instances[-2].report_instance_id)
self.assertIn('series_data_as_rows', data)
self.assertEqual(2, len(data['series_data']))
self.assertEqual(0, len(data['series_data'][0]['data_points']))
self.assertEqual(1, len(data['series_data'][1]['data_points']))
self.assertEqual(128, data['series_data'][1]['data_points'][0].value)
data = tile.get_new_tile_data(rd.instances[-3].report_instance_id)
self.assertIn('series_data_as_rows', data)
self.assertEqual(2, len(data['series_data']))
self.assertEqual(1, len(data['series_data'][0]['data_points']))
self.assertEqual(1, len(data['series_data'][1]['data_points']))
self.assertEqual(265, data['series_data'][0]['data_points'][0].value)
self.assertEqual(128, data['series_data'][1]['data_points'][0].value)
data = tile.get_new_tile_data(rd.instances[-4].report_instance_id)
self.assertIn('series_data_as_rows', data)
self.assertEqual(2, len(data['series_data']))
self.assertEqual(2, len(data['series_data'][0]['data_points']))
self.assertEqual(1, len(data['series_data'][1]['data_points']))
self.assertEqual(220, data['series_data'][0]['data_points'][0].value)
self.assertEqual(265, data['series_data'][0]['data_points'][1].value)
self.assertEqual(128, data['series_data'][1]['data_points'][0].value)
self.assertEqual(
tile.get_new_tile_data(
util.min_uuid_with_dt(rd.instances[0].created)),
tile.get_new_tile_data(None))
def fill_new_tile_data(self,
data,
after_report_instance_id,
limit=None,
fetch_params={}):
if not after_report_instance_id:
after_report_instance_id = util.min_uuid_with_dt(
datetime.datetime.utcnow() - \
datetime.timedelta(seconds=self.tile_options['seconds_back']))
self._set_series_data(data,
after=after_report_instance_id,
limit=limit)
def insert_series_values(series_def, report, from_dt, to_dt, after=None, limit=None):
assert after or (from_dt is not None and to_dt is not None)
log.debug('insert_series_values report_id=%s sd.from_dt=%s sd.to_dt=%s from_dt=%s'
'to_dt=%s after=%s limit=%s', report.report_id, series_def.from_dt,
series_def.to_dt, from_dt, to_dt, after, limit)
instances_it = report.fetch_instances_iter(after=after,
from_dt=from_dt if not after else None,
to_dt=to_dt if not after else None,
limit=limit or mqeconfig.MAX_SERIES_POINTS,
tags=series_def.tags,
columns=['report_instance_id', 'ri_data'])
info = dict(oldest_rid_fetched=None,
newest_rid_fetched=None,
count=0)
def rows_it():
for ri in instances_it:
if info['oldest_rid_fetched'] is None:
info['oldest_rid_fetched'] = ri.report_instance_id
info['newest_rid_fetched'] = ri.report_instance_id
info['count'] += 1
cell = series_def.series_spec.get_cell(ri)
if cell:
row = dict(report_instance_id=ri.report_instance_id,
json_value=serialize.mjson(cell.value))
header = ri.table.header(cell.colno)
if header:
row['header'] = header
yield row
c.dao.SeriesValueDAO.insert_multi(series_def.series_id, rows_it())
if info['count'] == 0:
return
log.info('Inserted %d series values report_name=%r series_id=%s',
info['count'], report.report_name, series_def.series_id)
# from_rid stores minimal uuid from dt for which we fetched instances,
# while to_rid stores an actual latest report_instance_id in the series.
# However, generally it's not expected to_rid can always be a real report_instance_id
if from_dt is not None:
oldest_rid_stored = util.min_uuid_with_dt(from_dt)
else:
oldest_rid_stored = info['oldest_rid_fetched']
if series_def.from_rid is None or \
util.uuid_lt(oldest_rid_stored, series_def.from_rid):
log.debug('Updating series_def_id=%s from_rid_dt=%s', series_def.series_id,
util.datetime_from_uuid1(oldest_rid_stored))
series_def.update_from_rid(oldest_rid_stored)
if series_def.to_rid is None or \
util.uuid_lt(series_def.to_rid, info['newest_rid_fetched']):
log.debug('Updating series_def_id=%s to_rid_dt=%s', series_def.series_id,
util.datetime_from_uuid1(info['newest_rid_fetched']))
series_def.update_to_rid(info['newest_rid_fetched'])
def _delete_ris(self, owner_id, report_id, tags, ris, update_counters):
qs = []
tags_days = set()
all_tags_subsets = set()
with cursor() as cur:
for ri in ris:
tags_powerset = util.powerset(ri['all_tags'])
cur.execute("""DELETE FROM report_instance WHERE report_id=?
AND tags IN {in_p} AND report_instance_id=?""".format(in_p=in_params(tags_powerset)),
[report_id] + tags_powerset + [ri['report_instance_id']])
day = util.datetime_from_uuid1(ri['report_instance_id']).date()
for tags_subset in tags_powerset:
tags_days.add((tuple(tags_subset), day))
all_tags_subsets.add(tuple(tags_subset))
if update_counters:
total_diskspace = sum(self._compute_ri_diskspace(ri) for ri in ris)
cur.execute("""UPDATE report
SET report_instance_count = report_instance_count - ?
WHERE report_id=?""",
[len(ris), report_id])
cur.execute("""UPDATE report
SET report_instance_diskspace = report_instance_diskspace - ?
WHERE report_id=?""",
[total_diskspace, report_id])
cur.execute("""UPDATE report_data_for_owner
SET report_instance_count=report_instance_count - ?
WHERE owner_id=?""",
[len(ris), owner_id])
cur.execute("""UPDATE report_data_for_owner
SET report_instance_diskspace=report_instance_diskspace - ?
WHERE owner_id=?""",
[total_diskspace, owner_id])
### Delete days for which report instances no longer exist
for day_tags, day in tags_days:
cur.execute("""SELECT report_instance_id FROM report_instance
WHERE report_id=? AND tags=? AND
report_instance_id > ? AND report_instance_id < ?
LIMIT 1""",
[report_id, list(day_tags),
util.min_uuid_with_dt(datetime.datetime.combine(day,
datetime.datetime.min.time())),
util.max_uuid_with_dt(datetime.datetime.combine(day,
datetime.datetime.max.time()))])
if not cur.fetchall():
cur.execute("""DELETE FROM report_instance_day
WHERE report_id=? AND tags=? AND day=?""",
[report_id, list(day_tags), day])
### Delete tags for which report instances no longer exist
tags_present = set()
for tags, _ in tags_days:
for tag in tags:
tags_present.add(tag)
for tag in tags_present:
cur.execute("""SELECT report_id FROM report_instance_day
WHERE report_id=? AND tags=?
LIMIT 1""",
[report_id, [tag]])
if cur.fetchall():
continue
cur.execute("""DELETE FROM report_tag
WHERE report_id=? AND tag=?""",
[report_id, tag])
return len(ris), [list(ts) for ts in all_tags_subsets]
def insert_series_values(series_def,
report,
from_dt,
to_dt,
after=None,
limit=None):
assert after or (from_dt is not None and to_dt is not None)
log.debug(
'insert_series_values report_id=%s sd.from_dt=%s sd.to_dt=%s from_dt=%s'
'to_dt=%s after=%s limit=%s', report.report_id, series_def.from_dt,
series_def.to_dt, from_dt, to_dt, after, limit)
instances = report.fetch_instances(
after=after,
from_dt=from_dt if not after else None,
to_dt=to_dt if not after else None,
limit=limit or mqeconfig.MAX_SERIES_POINTS,
tags=series_def.tags,
columns=['report_instance_id', 'ri_data'])
if not instances:
return
data = []
for ri in instances:
cell = series_def.series_spec.get_cell(ri)
if cell:
row = dict(report_instance_id=ri.report_instance_id,
json_value=serialize.mjson(cell.value))
header = ri.table.header(cell.colno)
if header:
row['header'] = header
data.append(row)
log.info(
'Inserting %d series values from %d instances report_name=%r series_id=%s',
len(data), len(instances), report.report_name, series_def.series_id)
c.dao.SeriesValueDAO.insert_multi(series_def.series_id, data)
oldest_rid_fetched = instances[0].report_instance_id
newest_rid_fetched = instances[-1].report_instance_id
# from_rid stores minimal uuid from dt for which we fetched instances,
# while to_rid stores an actual latest report_instance_id in the series.
# However, generally it's not expected to_rid can always be a real report_instance_id
if from_dt is not None:
oldest_rid_stored = util.min_uuid_with_dt(from_dt)
else:
oldest_rid_stored = oldest_rid_fetched
if series_def.from_rid is None or \
util.uuid_lt(oldest_rid_stored, series_def.from_rid):
log.debug('Updating series_def_id=%s from_rid_dt=%s',
series_def.series_id,
util.datetime_from_uuid1(oldest_rid_stored))
series_def.update_from_rid(oldest_rid_stored)
if series_def.to_rid is None or \
util.uuid_lt(series_def.to_rid, newest_rid_fetched):
log.debug('Updating series_def_id=%s to_rid_dt=%s',
series_def.series_id,
util.datetime_from_uuid1(newest_rid_fetched))
series_def.update_to_rid(newest_rid_fetched)
def test_range__chart_range_drawer(self):
rd = new_report_data('points', tags=['ip:192.168.1.1'])
tile_config = {
'tw_type':
'Range',
'tags': ['ip:192.168.1.1'],
'series_spec_list': [
dataseries.SeriesSpec(2, 0, dict(op='eq', args=['monique'])),
],
'tile_options': {
'seconds_back':
600,
'tile_title':
'Points by user',
'sscs':
dataseries.SeriesSpec(2, 0, dict(op='eq', args=['monique'])),
}
}
tile_config['tile_options'][
'tpcreator_uispec'] = tpcreator.suggested_tpcreator_uispec(
tile_config['tags'])
tile = Tile.insert(rd.owner_id, rd.report.report_id, rd.dashboard_id,
tile_config)
layouts.place_tile(tile)
self.assertEqual(tile.tile_options['drawer_type'], 'ChartRangeDrawer')
d = [
OrderedDict([('user_name', 'robert3'), ('is_active', True),
('points', 128)])
]
pr = rd.report.process_input(json.dumps(d),
tags=['ip:192.168.1.1'],
extra_ri_data={'ed': 88})
tile = rd.only_tile_from_layout()
data = tile.get_new_tile_data(pr.report_instance.report_instance_id)
self.assertIn('extra_options', data)
self.assertEqual(2, len(data['series_data']))
self.assertEqual([], data['series_data'][0]['data_points'])
self.assertEqual([], data['series_data'][1]['data_points'])
data = tile.get_new_tile_data(rd.instances[-1].report_instance_id)
self.assertIn('extra_options', data)
self.assertEqual(2, len(data['series_data']))
self.assertEqual(0, len(data['series_data'][0]['data_points']))
self.assertEqual(1, len(data['series_data'][1]['data_points']))
self.assertEqual(128, data['series_data'][1]['data_points'][0].value)
data = tile.get_new_tile_data(rd.instances[-2].report_instance_id)
self.assertIn('extra_options', data)
self.assertEqual(2, len(data['series_data']))
self.assertEqual(0, len(data['series_data'][0]['data_points']))
self.assertEqual(1, len(data['series_data'][1]['data_points']))
self.assertEqual(128, data['series_data'][1]['data_points'][0].value)
data = tile.get_new_tile_data(rd.instances[-3].report_instance_id)
self.assertIn('extra_options', data)
self.assertEqual(2, len(data['series_data']))
self.assertEqual(1, len(data['series_data'][0]['data_points']))
self.assertEqual(1, len(data['series_data'][1]['data_points']))
self.assertEqual(265, data['series_data'][0]['data_points'][0].value)
self.assertEqual(128, data['series_data'][1]['data_points'][0].value)
data = tile.get_new_tile_data(rd.instances[-4].report_instance_id)
self.assertIn('extra_options', data)
self.assertEqual(2, len(data['series_data']))
self.assertEqual(2, len(data['series_data'][0]['data_points']))
self.assertEqual(1, len(data['series_data'][1]['data_points']))
self.assertEqual(220, data['series_data'][0]['data_points'][0].value)
self.assertEqual(265, data['series_data'][0]['data_points'][1].value)
self.assertEqual(128, data['series_data'][1]['data_points'][0].value)
self.assertEqual(
tile.get_new_tile_data(
util.min_uuid_with_dt(rd.instances[0].created)),
tile.get_new_tile_data(None))
return tile, pr, rd
