def _should_expire_tile(tile, min_valid, optimize_check): max_dt = None if optimize_check: latest_report_instance_id = tile.report.fetch_latest_instance_id(tile.tags) if latest_report_instance_id: max_dt = util.datetime_from_uuid1(latest_report_instance_id) else: tile_data = tile.tilewidget.get_tile_data() last_points = [sd['data_points'][-1] for sd in tile_data['series_data'] if sd['data_points']] if last_points: max_dt = max(p.dt for p in last_points) if max_dt is None: # if there's no data, use the tile's creation datetime max_dt = util.datetime_from_uuid1(tile.tile_id) return max_dt < min_valid
def report_instance_for_viewer(): report_name = request.get_json()['report_name'] tags = request.get_json().get('tags') curr_report_instance_id = request.get_json()['curr_report_instance_id'] direction = request.get_json().get('direction') search_date = request.get_json().get('search_date') check_access(lambda: auth.access_profile()) report = Report.select_by_name(auth.logged_owner_id(), report_name) if not curr_report_instance_id: curr_report_instance_id = report.fetch_latest_instance_id(tags) if not curr_report_instance_id: return error() if not direction: if search_date is not None: ri = report.find_report_instance_by_dt(search_date, tags) else: ri = report.fetch_single_instance(curr_report_instance_id) elif direction == 'next': ri = report.fetch_next_instance(curr_report_instance_id, tags) elif direction == 'prev': ri = report.fetch_prev_instance(curr_report_instance_id, tags) else: return error('Wrong direction') res = {} res['report_id'] = report.report_id res['report_has_tags'] = report.has_tags() if ri: res['html_newest_table'] = get_template_attribute('m.html', 'table_as_html_table')(ri.table) res['created_raw'] = datetime_from_uuid1(ri.report_instance_id) res['created'] = format_datetime(datetime_from_uuid1(ri.report_instance_id)) res['tags'] = Markup(' '.join('<span class="selected-tag-name clickable">%s</span>' % tag for tag in ri.all_tags)) res['curr_report_instance_id'] = ri.report_instance_id res['has_next'] = report.fetch_next_instance(ri.report_instance_id, tags) is not None res['has_prev'] = report.fetch_prev_instance(ri.report_instance_id, tags) is not None else: res['html_newest_table'] = '' res['created_raw'] = '' res['created'] = '' res['tags'] = '' res['curr_report_instance_id'] = None res['has_next'] = False res['has_prev'] = False return success(result=res)
def _should_expire_tile(tile, max_seconds_without_data): tile_data = tile.tilewidget.get_tile_data(limit=1) last_points = [ sd['data_points'][-1] for sd in tile_data['series_data'] if sd['data_points'] ] if not last_points: max_dt = util.datetime_from_uuid1(tile.tile_id) else: max_dt = max(p.dt for p in last_points) min_valid = datetime.datetime.utcnow() - datetime.timedelta( seconds=max_seconds_without_data) if max_dt >= min_valid: return False return True
def test_expire_tiles_without_data_losing_sscreated(self): rd = ReportData('r') ss = dataseries.SeriesSpec(0, -1, dict(op='eq', args=['0'])) tile_config1 = { 'series_spec_list': [ss], 'tags': ['p1:10'], 'tile_options': { 'tpcreator_uispec': tpcreator.suggested_tpcreator_uispec(['p1:10']), 'sscs': ss } } tile1 = Tile.insert(rd.owner_id, rd.report.report_id, rd.dashboard_id, tile_config1) place_tile(tile1) #rd.report.process_input('1', tags=['p1:10']) rd.report.process_input('1\n2\n', tags=['p1:11']) rd.report.process_input('1\n2\n3\n', tags=['p1:12']) self.assertEqual(3, len(rd.layout().layout_dict)) self.assertEqual(3, len(rd.get_tile_by_tags(['p1:12']).series_specs())) tile1_created_ago = datetime.datetime.utcnow( ) - util.datetime_from_uuid1(tile1.tile_id) tiles.expire_tiles_without_data( rd.layout().tile_dict.keys(), tile1_created_ago.total_seconds() - 0.00001, rd.layout().layout_id) self.assertEqual(2, len(rd.layout().layout_dict)) self.assertTrue(rd.layout_has_tags([['p1:11'], ['p1:12']])) master_tile = rd.get_tile_by_tags(['p1:11']) self.assertTrue(master_tile.is_master_tile()) self.assertEqual(master_tile.tile_id, rd.get_tile_by_tags(['p1:12']).get_master_tile_id()) self.assertEqual(3, len(rd.get_tile_by_tags(['p1:12']).series_specs()))
def insert_series_values(series_def, report, from_dt, to_dt, after=None, limit=None): assert after or (from_dt is not None and to_dt is not None) log.debug('insert_series_values report_id=%s sd.from_dt=%s sd.to_dt=%s from_dt=%s' 'to_dt=%s after=%s limit=%s', report.report_id, series_def.from_dt, series_def.to_dt, from_dt, to_dt, after, limit) instances_it = report.fetch_instances_iter(after=after, from_dt=from_dt if not after else None, to_dt=to_dt if not after else None, limit=limit or mqeconfig.MAX_SERIES_POINTS, tags=series_def.tags, columns=['report_instance_id', 'ri_data']) info = dict(oldest_rid_fetched=None, newest_rid_fetched=None, count=0) def rows_it(): for ri in instances_it: if info['oldest_rid_fetched'] is None: info['oldest_rid_fetched'] = ri.report_instance_id info['newest_rid_fetched'] = ri.report_instance_id info['count'] += 1 cell = series_def.series_spec.get_cell(ri) if cell: row = dict(report_instance_id=ri.report_instance_id, json_value=serialize.mjson(cell.value)) header = ri.table.header(cell.colno) if header: row['header'] = header yield row c.dao.SeriesValueDAO.insert_multi(series_def.series_id, rows_it()) if info['count'] == 0: return log.info('Inserted %d series values report_name=%r series_id=%s', info['count'], report.report_name, series_def.series_id) # from_rid stores minimal uuid from dt for which we fetched instances, # while to_rid stores an actual latest report_instance_id in the series. # However, generally it's not expected to_rid can always be a real report_instance_id if from_dt is not None: oldest_rid_stored = util.min_uuid_with_dt(from_dt) else: oldest_rid_stored = info['oldest_rid_fetched'] if series_def.from_rid is None or \ util.uuid_lt(oldest_rid_stored, series_def.from_rid): log.debug('Updating series_def_id=%s from_rid_dt=%s', series_def.series_id, util.datetime_from_uuid1(oldest_rid_stored)) series_def.update_from_rid(oldest_rid_stored) if series_def.to_rid is None or \ util.uuid_lt(series_def.to_rid, info['newest_rid_fetched']): log.debug('Updating series_def_id=%s to_rid_dt=%s', series_def.series_id, util.datetime_from_uuid1(info['newest_rid_fetched'])) series_def.update_to_rid(info['newest_rid_fetched'])
def to_dt(self): """A :class:`~datetime.datetime` up to which this data series has data""" if self.to_rid is None: return None return util.datetime_from_uuid1(self.to_rid)
def insert(self, owner_id, report_id, report_instance_id, tags, ri_data, input_string, extra_ri_data, custom_created): created = util.datetime_from_uuid1(report_instance_id) with cursor() as cur: first_row = None tags_powerset = util.powerset(tags[:mqeconfig.MAX_TAGS]) for tags_subset in tags_powerset: row = dict(report_id=report_id, tags=tags_subset, report_instance_id=report_instance_id, ri_data=ri_data, input_string=input_string, all_tags=tags, extra_ri_data=extra_ri_data) if first_row is None: first_row = row cur.execute(*insert('report_instance', row)) cur.execute( """INSERT OR IGNORE INTO report_instance_day (report_id, tags, day) VALUES (?, ?, ?)""", [report_id, tags_subset, created.date()]) if first_row: # report counts cur.execute( """UPDATE report SET report_instance_count = report_instance_count + 1 WHERE report_id=?""", [report_id]) diskspace = self._compute_ri_diskspace(first_row) cur.execute( """UPDATE report SET report_instance_diskspace = report_instance_diskspace + ? WHERE report_id=?""", [diskspace, report_id]) # owner counts cur.execute( """SELECT 1 FROM report_data_for_owner WHERE owner_id=?""", [owner_id]) if not cur.fetchone(): try: cur.execute( """INSERT INTO report_data_for_owner (owner_id) VALUES (?)""", [owner_id]) except sqlite3.IntegrityError: pass cur.execute( """UPDATE report_data_for_owner SET report_instance_count=report_instance_count+1 WHERE owner_id=?""", [owner_id]) cur.execute( """UPDATE report_data_for_owner SET report_instance_diskspace=report_instance_diskspace+? WHERE owner_id=?""", [diskspace, owner_id]) for tag in tags: cur.execute( """INSERT OR IGNORE INTO report_tag (report_id, tag) VALUES (?, ?)""", [report_id, tag]) return first_row
def _delete_ris(self, owner_id, report_id, tags, ris, update_counters): qs = [] tags_days = set() all_tags_subsets = set() with cursor() as cur: for ri in ris: tags_powerset = util.powerset(ri['all_tags']) cur.execute("""DELETE FROM report_instance WHERE report_id=? AND tags IN {in_p} AND report_instance_id=?""".format(in_p=in_params(tags_powerset)), [report_id] + tags_powerset + [ri['report_instance_id']]) day = util.datetime_from_uuid1(ri['report_instance_id']).date() for tags_subset in tags_powerset: tags_days.add((tuple(tags_subset), day)) all_tags_subsets.add(tuple(tags_subset)) if update_counters: total_diskspace = sum(self._compute_ri_diskspace(ri) for ri in ris) cur.execute("""UPDATE report SET report_instance_count = report_instance_count - ? WHERE report_id=?""", [len(ris), report_id]) cur.execute("""UPDATE report SET report_instance_diskspace = report_instance_diskspace - ? WHERE report_id=?""", [total_diskspace, report_id]) cur.execute("""UPDATE report_data_for_owner SET report_instance_count=report_instance_count - ? WHERE owner_id=?""", [len(ris), owner_id]) cur.execute("""UPDATE report_data_for_owner SET report_instance_diskspace=report_instance_diskspace - ? WHERE owner_id=?""", [total_diskspace, owner_id]) ### Delete days for which report instances no longer exist for day_tags, day in tags_days: cur.execute("""SELECT report_instance_id FROM report_instance WHERE report_id=? AND tags=? AND report_instance_id > ? AND report_instance_id < ? LIMIT 1""", [report_id, list(day_tags), util.min_uuid_with_dt(datetime.datetime.combine(day, datetime.datetime.min.time())), util.max_uuid_with_dt(datetime.datetime.combine(day, datetime.datetime.max.time()))]) if not cur.fetchall(): cur.execute("""DELETE FROM report_instance_day WHERE report_id=? AND tags=? AND day=?""", [report_id, list(day_tags), day]) ### Delete tags for which report instances no longer exist tags_present = set() for tags, _ in tags_days: for tag in tags: tags_present.add(tag) for tag in tags_present: cur.execute("""SELECT report_id FROM report_instance_day WHERE report_id=? AND tags=? LIMIT 1""", [report_id, [tag]]) if cur.fetchall(): continue cur.execute("""DELETE FROM report_tag WHERE report_id=? AND tag=?""", [report_id, tag]) return len(ris), [list(ts) for ts in all_tags_subsets]
def dt(self): return util.datetime_from_uuid1(self.rid)
def insert_series_values(series_def, report, from_dt, to_dt, after=None, limit=None): assert after or (from_dt is not None and to_dt is not None) log.debug( 'insert_series_values report_id=%s sd.from_dt=%s sd.to_dt=%s from_dt=%s' 'to_dt=%s after=%s limit=%s', report.report_id, series_def.from_dt, series_def.to_dt, from_dt, to_dt, after, limit) instances = report.fetch_instances( after=after, from_dt=from_dt if not after else None, to_dt=to_dt if not after else None, limit=limit or mqeconfig.MAX_SERIES_POINTS, tags=series_def.tags, columns=['report_instance_id', 'ri_data']) if not instances: return data = [] for ri in instances: cell = series_def.series_spec.get_cell(ri) if cell: row = dict(report_instance_id=ri.report_instance_id, json_value=serialize.mjson(cell.value)) header = ri.table.header(cell.colno) if header: row['header'] = header data.append(row) log.info( 'Inserting %d series values from %d instances report_name=%r series_id=%s', len(data), len(instances), report.report_name, series_def.series_id) c.dao.SeriesValueDAO.insert_multi(series_def.series_id, data) oldest_rid_fetched = instances[0].report_instance_id newest_rid_fetched = instances[-1].report_instance_id # from_rid stores minimal uuid from dt for which we fetched instances, # while to_rid stores an actual latest report_instance_id in the series. # However, generally it's not expected to_rid can always be a real report_instance_id if from_dt is not None: oldest_rid_stored = util.min_uuid_with_dt(from_dt) else: oldest_rid_stored = oldest_rid_fetched if series_def.from_rid is None or \ util.uuid_lt(oldest_rid_stored, series_def.from_rid): log.debug('Updating series_def_id=%s from_rid_dt=%s', series_def.series_id, util.datetime_from_uuid1(oldest_rid_stored)) series_def.update_from_rid(oldest_rid_stored) if series_def.to_rid is None or \ util.uuid_lt(series_def.to_rid, newest_rid_fetched): log.debug('Updating series_def_id=%s to_rid_dt=%s', series_def.series_id, util.datetime_from_uuid1(newest_rid_fetched)) series_def.update_to_rid(newest_rid_fetched)
def created(self): """Creation datetime""" return util.datetime_from_uuid1(self.report_instance_id)
def process_input(self, input_string, tags=None, created=None, input_type='any', ip_options={}, force_header=None, extra_ri_data=None, handle_tpcreator=True, handle_sscreator=True): """Process an input string - parse it into a table and create a report instance belonging to the report. :param str|unicode input_string: the input string :param list tags: a list of string tags attached to the report instance :param ~datetime.datetime created: an explicit creation datetime of the report instance ( default: the current datetime) :param str input_type: input type (see :func:`mqetables.parseany.parse_input`) :param dict ip_options: extra parser options (see :func:`mqetables.parsing.InputParser`) :param force_header: a list of header rows indexes to set as a header (defaults to auto-detection) :param extra_ri_data: a custom JSON-serializable document attached to the report instance :param handle_tpcreator: whether to handle TPCreator for the created report instance by calling :func:`~mqe.tpcreator.handle_tpcreator` :param handle_sscreator: whether to handle SSCS by calling :func:`~mqe.sscreator.handle_sscreator` :return: an :class:`InputProcessingResult` """ assert isinstance(input_string, (str, unicode)) # disallow 'created' in the future now = datetime.datetime.utcnow() if created is not None and created.tzinfo: created = util.make_tz_naive(created) if created is not None and created.year < 2000: raise ValueError('created cannot be before the year 2000') if created is not None and created < now: report_instance_id = util.uuid_with_dt(created) custom_created = True else: custom_created = False report_instance_id = gen_timeuuid() created = util.datetime_from_uuid1(report_instance_id) if tags is None: tags = [] parsing_result = parseany.parse_input(input_string, input_type, ip_options) table = mqeconfig.get_table_from_parsing_result(parsing_result) if table is None: return InputProcessingResult(None, parsing_result) if force_header is not None: log.debug('Overwriting header detection due to force_header') table.header_idxs = [ i for i in force_header if util.valid_index(table.num_rows, i) ] table.header_idxs_source = parsing.HEADER_IDXS_SOURCE_USER ri_data_dict = { 'table': table, } result_desc = self._get_result_desc(parsing_result) if result_desc: ri_data_dict['result_desc'] = result_desc report_instance_row = c.dao.ReportInstanceDAO.insert( owner_id=self.owner_id, report_id=self.report_id, report_instance_id=report_instance_id, tags=tags, ri_data=serialize.mjson(ri_data_dict), input_string=parsing_result.input_string, extra_ri_data=serialize.mjson(extra_ri_data) if extra_ri_data else None, custom_created=custom_created) report_instance = ReportInstance(report_instance_row) log.info( 'Created new report instance report_id=%s report_name=%r tags=%s ' 'report_instance_id=%s created=%s', self.report_id, self.report_name, tags, report_instance_id, report_instance.created) if tags and handle_tpcreator: from mqe import tpcreator tpcreator.handle_tpcreator(self.owner_id, self.report_id, report_instance) if handle_sscreator: from mqe import sscreator sscreator.handle_sscreator(self.owner_id, self.report_id, report_instance) if custom_created: from mqe import dataseries dataseries.clear_series_defs(self.report_id, util.powerset(tags)) return InputProcessingResult(report_instance, parsing_result)