def find_report_instance_by_dt(self, dt, tags=None): """Find a report instance with a creation datetime matching ``dt`` datetime as close as possible.""" ri_prev = self.fetch_prev_instance(util.max_uuid_with_dt(dt), tags) ri_next = self.fetch_next_instance(util.min_uuid_with_dt(dt), tags) return min([ri_prev, ri_next], key=lambda ri: abs((ri.created - dt).total_seconds()) if ri else sys.maxint)
def test_range__text_table_drawer(self): tile, pr, rd = self.test_range__chart_range_drawer() tc = tile.get_tile_config() self.assertEqual('ChartRangeDrawer', tc['tile_options']['drawer_type']) tc['tile_options']['drawer_type'] = 'TextTableDrawer' tile = tile.insert_similar(tc) self.assertEqual('TextTableDrawer', tc['tile_options']['drawer_type']) data = tile.get_tile_data() data = tile.get_new_tile_data(pr.report_instance.report_instance_id) self.assertIn('series_data_as_rows', data) self.assertEqual(2, len(data['series_data'])) self.assertEqual([], data['series_data'][0]['data_points']) self.assertEqual([], data['series_data'][1]['data_points']) data = tile.get_new_tile_data(rd.instances[-1].report_instance_id) self.assertIn('series_data_as_rows', data) self.assertEqual(2, len(data['series_data'])) self.assertEqual(0, len(data['series_data'][0]['data_points'])) self.assertEqual(1, len(data['series_data'][1]['data_points'])) self.assertEqual(128, data['series_data'][1]['data_points'][0].value) data = tile.get_new_tile_data(rd.instances[-2].report_instance_id) self.assertIn('series_data_as_rows', data) self.assertEqual(2, len(data['series_data'])) self.assertEqual(0, len(data['series_data'][0]['data_points'])) self.assertEqual(1, len(data['series_data'][1]['data_points'])) self.assertEqual(128, data['series_data'][1]['data_points'][0].value) data = tile.get_new_tile_data(rd.instances[-3].report_instance_id) self.assertIn('series_data_as_rows', data) self.assertEqual(2, len(data['series_data'])) self.assertEqual(1, len(data['series_data'][0]['data_points'])) self.assertEqual(1, len(data['series_data'][1]['data_points'])) self.assertEqual(265, data['series_data'][0]['data_points'][0].value) self.assertEqual(128, data['series_data'][1]['data_points'][0].value) data = tile.get_new_tile_data(rd.instances[-4].report_instance_id) self.assertIn('series_data_as_rows', data) self.assertEqual(2, len(data['series_data'])) self.assertEqual(2, len(data['series_data'][0]['data_points'])) self.assertEqual(1, len(data['series_data'][1]['data_points'])) self.assertEqual(220, data['series_data'][0]['data_points'][0].value) self.assertEqual(265, data['series_data'][0]['data_points'][1].value) self.assertEqual(128, data['series_data'][1]['data_points'][0].value) self.assertEqual( tile.get_new_tile_data( util.min_uuid_with_dt(rd.instances[0].created)), tile.get_new_tile_data(None))
def fill_new_tile_data(self, data, after_report_instance_id, limit=None, fetch_params={}): if not after_report_instance_id: after_report_instance_id = util.min_uuid_with_dt( datetime.datetime.utcnow() - \ datetime.timedelta(seconds=self.tile_options['seconds_back'])) self._set_series_data(data, after=after_report_instance_id, limit=limit)
def insert_series_values(series_def, report, from_dt, to_dt, after=None, limit=None): assert after or (from_dt is not None and to_dt is not None) log.debug('insert_series_values report_id=%s sd.from_dt=%s sd.to_dt=%s from_dt=%s' 'to_dt=%s after=%s limit=%s', report.report_id, series_def.from_dt, series_def.to_dt, from_dt, to_dt, after, limit) instances_it = report.fetch_instances_iter(after=after, from_dt=from_dt if not after else None, to_dt=to_dt if not after else None, limit=limit or mqeconfig.MAX_SERIES_POINTS, tags=series_def.tags, columns=['report_instance_id', 'ri_data']) info = dict(oldest_rid_fetched=None, newest_rid_fetched=None, count=0) def rows_it(): for ri in instances_it: if info['oldest_rid_fetched'] is None: info['oldest_rid_fetched'] = ri.report_instance_id info['newest_rid_fetched'] = ri.report_instance_id info['count'] += 1 cell = series_def.series_spec.get_cell(ri) if cell: row = dict(report_instance_id=ri.report_instance_id, json_value=serialize.mjson(cell.value)) header = ri.table.header(cell.colno) if header: row['header'] = header yield row c.dao.SeriesValueDAO.insert_multi(series_def.series_id, rows_it()) if info['count'] == 0: return log.info('Inserted %d series values report_name=%r series_id=%s', info['count'], report.report_name, series_def.series_id) # from_rid stores minimal uuid from dt for which we fetched instances, # while to_rid stores an actual latest report_instance_id in the series. # However, generally it's not expected to_rid can always be a real report_instance_id if from_dt is not None: oldest_rid_stored = util.min_uuid_with_dt(from_dt) else: oldest_rid_stored = info['oldest_rid_fetched'] if series_def.from_rid is None or \ util.uuid_lt(oldest_rid_stored, series_def.from_rid): log.debug('Updating series_def_id=%s from_rid_dt=%s', series_def.series_id, util.datetime_from_uuid1(oldest_rid_stored)) series_def.update_from_rid(oldest_rid_stored) if series_def.to_rid is None or \ util.uuid_lt(series_def.to_rid, info['newest_rid_fetched']): log.debug('Updating series_def_id=%s to_rid_dt=%s', series_def.series_id, util.datetime_from_uuid1(info['newest_rid_fetched'])) series_def.update_to_rid(info['newest_rid_fetched'])
def _delete_ris(self, owner_id, report_id, tags, ris, update_counters): qs = [] tags_days = set() all_tags_subsets = set() with cursor() as cur: for ri in ris: tags_powerset = util.powerset(ri['all_tags']) cur.execute("""DELETE FROM report_instance WHERE report_id=? AND tags IN {in_p} AND report_instance_id=?""".format(in_p=in_params(tags_powerset)), [report_id] + tags_powerset + [ri['report_instance_id']]) day = util.datetime_from_uuid1(ri['report_instance_id']).date() for tags_subset in tags_powerset: tags_days.add((tuple(tags_subset), day)) all_tags_subsets.add(tuple(tags_subset)) if update_counters: total_diskspace = sum(self._compute_ri_diskspace(ri) for ri in ris) cur.execute("""UPDATE report SET report_instance_count = report_instance_count - ? WHERE report_id=?""", [len(ris), report_id]) cur.execute("""UPDATE report SET report_instance_diskspace = report_instance_diskspace - ? WHERE report_id=?""", [total_diskspace, report_id]) cur.execute("""UPDATE report_data_for_owner SET report_instance_count=report_instance_count - ? WHERE owner_id=?""", [len(ris), owner_id]) cur.execute("""UPDATE report_data_for_owner SET report_instance_diskspace=report_instance_diskspace - ? WHERE owner_id=?""", [total_diskspace, owner_id]) ### Delete days for which report instances no longer exist for day_tags, day in tags_days: cur.execute("""SELECT report_instance_id FROM report_instance WHERE report_id=? AND tags=? AND report_instance_id > ? AND report_instance_id < ? LIMIT 1""", [report_id, list(day_tags), util.min_uuid_with_dt(datetime.datetime.combine(day, datetime.datetime.min.time())), util.max_uuid_with_dt(datetime.datetime.combine(day, datetime.datetime.max.time()))]) if not cur.fetchall(): cur.execute("""DELETE FROM report_instance_day WHERE report_id=? AND tags=? AND day=?""", [report_id, list(day_tags), day]) ### Delete tags for which report instances no longer exist tags_present = set() for tags, _ in tags_days: for tag in tags: tags_present.add(tag) for tag in tags_present: cur.execute("""SELECT report_id FROM report_instance_day WHERE report_id=? AND tags=? LIMIT 1""", [report_id, [tag]]) if cur.fetchall(): continue cur.execute("""DELETE FROM report_tag WHERE report_id=? AND tag=?""", [report_id, tag]) return len(ris), [list(ts) for ts in all_tags_subsets]
def insert_series_values(series_def, report, from_dt, to_dt, after=None, limit=None): assert after or (from_dt is not None and to_dt is not None) log.debug( 'insert_series_values report_id=%s sd.from_dt=%s sd.to_dt=%s from_dt=%s' 'to_dt=%s after=%s limit=%s', report.report_id, series_def.from_dt, series_def.to_dt, from_dt, to_dt, after, limit) instances = report.fetch_instances( after=after, from_dt=from_dt if not after else None, to_dt=to_dt if not after else None, limit=limit or mqeconfig.MAX_SERIES_POINTS, tags=series_def.tags, columns=['report_instance_id', 'ri_data']) if not instances: return data = [] for ri in instances: cell = series_def.series_spec.get_cell(ri) if cell: row = dict(report_instance_id=ri.report_instance_id, json_value=serialize.mjson(cell.value)) header = ri.table.header(cell.colno) if header: row['header'] = header data.append(row) log.info( 'Inserting %d series values from %d instances report_name=%r series_id=%s', len(data), len(instances), report.report_name, series_def.series_id) c.dao.SeriesValueDAO.insert_multi(series_def.series_id, data) oldest_rid_fetched = instances[0].report_instance_id newest_rid_fetched = instances[-1].report_instance_id # from_rid stores minimal uuid from dt for which we fetched instances, # while to_rid stores an actual latest report_instance_id in the series. # However, generally it's not expected to_rid can always be a real report_instance_id if from_dt is not None: oldest_rid_stored = util.min_uuid_with_dt(from_dt) else: oldest_rid_stored = oldest_rid_fetched if series_def.from_rid is None or \ util.uuid_lt(oldest_rid_stored, series_def.from_rid): log.debug('Updating series_def_id=%s from_rid_dt=%s', series_def.series_id, util.datetime_from_uuid1(oldest_rid_stored)) series_def.update_from_rid(oldest_rid_stored) if series_def.to_rid is None or \ util.uuid_lt(series_def.to_rid, newest_rid_fetched): log.debug('Updating series_def_id=%s to_rid_dt=%s', series_def.series_id, util.datetime_from_uuid1(newest_rid_fetched)) series_def.update_to_rid(newest_rid_fetched)
def test_range__chart_range_drawer(self): rd = new_report_data('points', tags=['ip:192.168.1.1']) tile_config = { 'tw_type': 'Range', 'tags': ['ip:192.168.1.1'], 'series_spec_list': [ dataseries.SeriesSpec(2, 0, dict(op='eq', args=['monique'])), ], 'tile_options': { 'seconds_back': 600, 'tile_title': 'Points by user', 'sscs': dataseries.SeriesSpec(2, 0, dict(op='eq', args=['monique'])), } } tile_config['tile_options'][ 'tpcreator_uispec'] = tpcreator.suggested_tpcreator_uispec( tile_config['tags']) tile = Tile.insert(rd.owner_id, rd.report.report_id, rd.dashboard_id, tile_config) layouts.place_tile(tile) self.assertEqual(tile.tile_options['drawer_type'], 'ChartRangeDrawer') d = [ OrderedDict([('user_name', 'robert3'), ('is_active', True), ('points', 128)]) ] pr = rd.report.process_input(json.dumps(d), tags=['ip:192.168.1.1'], extra_ri_data={'ed': 88}) tile = rd.only_tile_from_layout() data = tile.get_new_tile_data(pr.report_instance.report_instance_id) self.assertIn('extra_options', data) self.assertEqual(2, len(data['series_data'])) self.assertEqual([], data['series_data'][0]['data_points']) self.assertEqual([], data['series_data'][1]['data_points']) data = tile.get_new_tile_data(rd.instances[-1].report_instance_id) self.assertIn('extra_options', data) self.assertEqual(2, len(data['series_data'])) self.assertEqual(0, len(data['series_data'][0]['data_points'])) self.assertEqual(1, len(data['series_data'][1]['data_points'])) self.assertEqual(128, data['series_data'][1]['data_points'][0].value) data = tile.get_new_tile_data(rd.instances[-2].report_instance_id) self.assertIn('extra_options', data) self.assertEqual(2, len(data['series_data'])) self.assertEqual(0, len(data['series_data'][0]['data_points'])) self.assertEqual(1, len(data['series_data'][1]['data_points'])) self.assertEqual(128, data['series_data'][1]['data_points'][0].value) data = tile.get_new_tile_data(rd.instances[-3].report_instance_id) self.assertIn('extra_options', data) self.assertEqual(2, len(data['series_data'])) self.assertEqual(1, len(data['series_data'][0]['data_points'])) self.assertEqual(1, len(data['series_data'][1]['data_points'])) self.assertEqual(265, data['series_data'][0]['data_points'][0].value) self.assertEqual(128, data['series_data'][1]['data_points'][0].value) data = tile.get_new_tile_data(rd.instances[-4].report_instance_id) self.assertIn('extra_options', data) self.assertEqual(2, len(data['series_data'])) self.assertEqual(2, len(data['series_data'][0]['data_points'])) self.assertEqual(1, len(data['series_data'][1]['data_points'])) self.assertEqual(220, data['series_data'][0]['data_points'][0].value) self.assertEqual(265, data['series_data'][0]['data_points'][1].value) self.assertEqual(128, data['series_data'][1]['data_points'][0].value) self.assertEqual( tile.get_new_tile_data( util.min_uuid_with_dt(rd.instances[0].created)), tile.get_new_tile_data(None)) return tile, pr, rd