def delete(self, owner_id, report_id): row = self.select(report_id) if not row: log.warn('No report row %s', report_id) return qs = [] qs.append( bind("""DELETE FROM mqe.report WHERE report_id=?""", [report_id])) qs.append( bind( """DELETE FROM mqe.report_by_owner_id_report_name WHERE owner_id=? AND report_name=?""", [owner_id, row['report_name']])) for prefix in util.iter_prefixes(row['report_name'], include_empty=True): qs.append( bind( """DELETE FROM mqe.report_name_by_owner_id_report_name_prefix WHERE owner_id=? AND report_name_prefix=? AND report_name=?""", [owner_id, prefix, row['report_name']])) qs.append( bind("""DELETE FROM mqe.report_tag WHERE report_id=?""", [report_id])) c.cass.execute_parallel(qs)
def delete(self, user_id): row = self.select(user_id) if not row: return c.cass.execute( batch( bind("""DELETE FROM mqe.user WHERE user_id=?""", [row['user_id']]), bind("""DELETE FROM mqe.user_by_email WHERE email=?""", [row['email']]), ))
def delete_multi(self, tile_list): qs = [] for tile in tile_list: if not tile: continue qs.append( bind( """DELETE FROM mqe.tile WHERE dashboard_id=? AND tile_id=?""", [tile.dashboard_id, tile.tile_id])) qs.append( bind( """UPDATE mqe.tile_count SET count=count-1 WHERE owner_id=?""", [tile.owner_id])) c.cass.execute_parallel(qs)
def select_id_or_insert_multi(self, report_id, tags_series_spec_list): select_qs = [] for (tags, series_spec) in tags_series_spec_list: select_qs.append( bind( """SELECT series_id FROM mqe.series_def_by_series_spec WHERE report_id=? AND tags_repr=? AND series_spec=?""", [ report_id, tags_repr_from_tags(tags), serialize.mjson(series_spec) ])) select_qs_res = c.cass.execute_parallel(select_qs) res = [ rows[0]['series_id'] if rows else None for rows in select_qs_res ] to_insert_idxs = [ i for i in xrange(len(tags_series_spec_list)) if res[i] is None ] to_insert_data = [tags_series_spec_list[i] for i in to_insert_idxs] insert_res = self.insert_multi(report_id, to_insert_data) for i in xrange(len(to_insert_idxs)): res[to_insert_idxs[i]] = insert_res[i] return res
def invalidate_precomputed_instances_data(report_id, tags_powerset, first_valid_dt): log.debug('Invalidating precomputed %s %s %s', report_id, tags_powerset, first_valid_dt) all_tags_reprs = [tags_repr_from_tags(ts) for ts in tags_powerset] series_rows = c.cass.execute( """SELECT * FROM mqe.series_def WHERE report_id=? AND tags_repr IN ?""", [report_id, all_tags_reprs]) qs = [] for row in series_rows: if row['from_rid'] is None: continue if cassandra.util.datetime_from_uuid1(row['from_rid']) < first_valid_dt and \ cassandra.util.datetime_from_uuid1(row['to_rid']) < first_valid_dt: log.debug('Clearing series_id=%s', row['series_id']) new_from_rid = None new_to_rid = None elif cassandra.util.datetime_from_uuid1(row['from_rid']) >= first_valid_dt and \ cassandra.util.datetime_from_uuid1(row['to_rid']) >= first_valid_dt: continue else: log.debug('Updating from_rid_dt=%s', first_valid_dt) new_from_rid = cassandra.util.min_uuid_from_time(first_valid_dt) new_to_rid = row['to_rid'] qs.append( bind( """UPDATE mqe.series_def SET from_rid=?, to_rid=? WHERE report_id=? AND tags_repr=? AND series_id=?""", [ new_from_rid, new_to_rid, report_id, row['tags_repr'], row['series_id'] ])) if new_from_rid is None: qs.append( bind("""DELETE FROM mqe.series_value WHERE series_id=?""", [row['series_id']])) else: qs.append( bind( """DELETE FROM mqe.series_value WHERE series_id=? AND report_instance_id < ?""", [row['series_id'], new_from_rid])) c.cass.execute_parallel(qs)
def clear_all_series_defs(self, report_id, tags_powerset): all_tags_reprs = [tags_repr_from_tags(ts) for ts in tags_powerset] series_rows = c.cass.execute( """SELECT series_id, from_rid, to_rid, tags_repr FROM mqe.series_def WHERE report_id=? AND tags_repr IN ?""", [report_id, all_tags_reprs]) qs = [] for row in series_rows: if row['from_rid'] is not None or row['to_rid'] is not None: qs.append( bind( """UPDATE mqe.series_def SET from_rid=NULL, to_rid=NULL WHERE report_id=? AND tags_repr=? AND series_id=?""", [report_id, row['tags_repr'], row['series_id']])) qs.append( bind("""DELETE FROM mqe.series_value WHERE series_id=?""", [row['series_id']])) c.cass.execute_parallel(qs)
def select_multi(self, report_id, tags_series_id_list): qs = [] for tags, series_id in tags_series_id_list: qs.append( bind( """SELECT * FROM mqe.series_def WHERE report_id=? AND tags_repr=? AND series_id=?""", [report_id, tags_repr_from_tags(tags), series_id])) res = c.cass.execute_parallel(qs) res = [postprocess_tags(rows[0]) if rows else None for rows in res] return res
def insert_multi(self, owner_id, dashboard_id, tile_options_list): qs = [] res = [] for tile_options in tile_options_list: row = dict(dashboard_id=dashboard_id, tile_id=gen_timeuuid(), tile_options=tile_options) qs.append(insert('mqe.tile', row)) qs.append( bind( """UPDATE mqe.tile_count SET count=count+1 WHERE owner_id=?""", [owner_id])) res.append(row) c.cass.execute_parallel(qs) return res
def delete(self, owner_id, report_id, report_instance_id): ri = self.select(report_id, report_instance_id, None) if not ri: return False diskspace = self._compute_ri_diskspace(ri) qs = [] qs.append( bind( """DELETE FROM mqe.report_instance_metadata WHERE report_id=? AND day=? AND report_instance_id=?""", [report_id, ri['day'], report_instance_id])) for tags_subset in util.powerset(ri['all_tags']): tags_repr = tags_repr_from_tags(tags_subset) qs.append( bind( """DELETE FROM mqe.report_instance WHERE report_id=? AND day=? AND tags_repr=? AND report_instance_id=?""", [report_id, ri['day'], tags_repr, report_instance_id])) qs.append( bind( """UPDATE mqe.report_instance_diskspace SET bytes=bytes-? WHERE report_id=? AND tags_repr=?""", [diskspace, report_id, tags_repr])) qs.append( bind( """UPDATE mqe.report_instance_count SET count=count-1 WHERE report_id=? AND tags_repr=?""", [report_id, tags_repr])) qs.append( bind( """UPDATE mqe.report_instance_count_for_owner SET count=count-1 WHERE owner_id=?""", [owner_id])) qs.append( bind( """UPDATE mqe.report_instance_diskspace_for_owner SET bytes=bytes-? WHERE owner_id=?""", [diskspace, owner_id])) # Can't delete from mqe.report_tag - tag might be used by other report instances #for tag in ri.all_tags: # prefixes = list(util.iter_prefixes(tag, include_empty=True)) # for p in prefixes: # qs.append(bind("""DELETE FROM mqe.report_tag # WHERE report_id=? AND tag_prefix=? AND tag=?""", # [report_id, p, tag])) c.cass.execute_parallel(qs) return True
def _delete_ris(self, owner_id, report_id, tags, ris, update_counters): qs = [] count_by_tags_repr = defaultdict(int) diskspace_by_tags_repr = defaultdict(int) tags_reprs_days = set() for ri in ris: qs.append( bind( """DELETE FROM mqe.report_instance_metadata WHERE report_id=? AND day=? AND report_instance_id=?""", [report_id, ri['day'], ri['report_instance_id']])) for tags_subset in util.powerset(ri['all_tags']): tags_repr = tags_repr_from_tags(tags_subset) qs.append( bind( """DELETE FROM mqe.report_instance WHERE report_id=? AND day=? AND tags_repr=? AND report_instance_id=?""", [ report_id, ri['day'], tags_repr, ri['report_instance_id'] ])) count_by_tags_repr[tags_repr] += 1 if update_counters: diskspace_by_tags_repr[ tags_repr] += self._compute_ri_diskspace(ri) tags_reprs_days.add((tags_repr, ri['day'])) if update_counters: qs.append( bind( """UPDATE mqe.report_instance_count_for_owner SET count=count-? WHERE owner_id=?""", [count_by_tags_repr[''], owner_id])) qs.append( bind( """UPDATE mqe.report_instance_diskspace_for_owner SET bytes=bytes-? WHERE owner_id=?""", [diskspace_by_tags_repr[''], owner_id])) for tags_repr, count in count_by_tags_repr.iteritems(): qs.append( bind( """UPDATE mqe.report_instance_count SET count=count-? WHERE report_id=? AND tags_repr=?""", [count, report_id, tags_repr])) for tags_repr, bytes in diskspace_by_tags_repr.iteritems(): qs.append( bind( """UPDATE mqe.report_instance_diskspace SET bytes=bytes-? WHERE report_id=? AND tags_repr=?""", [bytes, report_id, tags_repr])) c.cass.execute_parallel(qs) ### Delete days for which report instances no longer exist days_qs = {} for tags_repr, day in tags_reprs_days: days_qs[(tags_repr, day)] = bind( """SELECT report_instance_id FROM mqe.report_instance WHERE report_id=? AND day=? AND tags_repr=? LIMIT 1""", [report_id, day, tags_repr]) days_res = c.cass.execute_parallel(days_qs) qs = [] for (tags_repr, day), rows in days_res.iteritems(): if rows: continue qs.append( bind( """DELETE FROM mqe.report_instance_day WHERE report_id=? AND tags_repr=? AND day=?""", [report_id, tags_repr, day])) log.info('Deleting %s days', len(qs)) c.cass.execute_parallel(qs) ### Delete tags for which report instances no longer exist tags_present = set() for tags_repr, _ in tags_reprs_days: for tag in tags_from_tags_repr(tags_repr): tags_present.add(tag) qs = [] for tag in tags_present: rows = c.cass.execute( """SELECT tags_repr FROM mqe.report_instance_day WHERE report_id=? AND tags_repr=? LIMIT 1""", [report_id, tags_repr_from_tags([tag])]) if rows: continue for p in util.iter_prefixes(tag, include_empty=True): qs.append( bind( """DELETE FROM mqe.report_tag WHERE report_id=? AND tag_prefix=? AND tag=?""", [report_id, p, tag])) log.info('Deleting %s tag rows', len(qs)) c.cass.execute_parallel(qs) return len(ris), [tags_from_tags_repr(tr) for tr in count_by_tags_repr]
def insert(self, owner_id, report_id, report_instance_id, tags, ri_data, input_string, extra_ri_data, custom_created): created = util.datetime_from_uuid1(report_instance_id) day = day_text(created) all_tags_repr = tags_repr_from_tags(tags) qs = [] metadata_row = dict( report_id=report_id, day=day, report_instance_id=report_instance_id, all_tags_repr=all_tags_repr, inserted=datetime.datetime.utcnow(), ) if extra_ri_data: metadata_row['extra_ri_data'] = extra_ri_data qs.append(insert('mqe.report_instance_metadata', metadata_row)) first_row = None tags_powerset = util.powerset(tags[:mqeconfig.MAX_TAGS]) for tags_subset in tags_powerset: tags_repr = tags_repr_from_tags(tags_subset) row = dict(report_id=report_id, day=day, tags_repr=tags_repr, report_instance_id=report_instance_id, ri_data=ri_data, input_string=input_string, all_tags_repr=all_tags_repr) if first_row is None: first_row = row qs.append( insert('mqe.report_instance', row, COLUMN_RENAMES['report_instance'])) if not c.cass.execute( """SELECT day FROM mqe.report_instance_day WHERE report_id=? AND tags_repr=? AND day=?""", [report_id, tags_repr, day]): qs.append( insert( 'mqe.report_instance_day', dict(report_id=report_id, tags_repr=tags_repr, day=day))) qs.append( bind( """UPDATE mqe.report_instance_count SET count=count+1 WHERE report_id=? AND tags_repr=?""", [report_id, tags_repr])) diskspace = self._compute_ri_diskspace(row) qs.append( bind( """UPDATE mqe.report_instance_diskspace SET bytes=bytes+? WHERE report_id=? AND tags_repr=?""", [diskspace, report_id, tags_repr])) ### queries for all tags qs.append( bind( """UPDATE mqe.report_instance_count_for_owner SET count=count+1 WHERE owner_id=?""", [owner_id])) if first_row: diskspace = self._compute_ri_diskspace(first_row) qs.append( bind( """UPDATE mqe.report_instance_diskspace_for_owner SET bytes=bytes+? WHERE owner_id=?""", [diskspace, owner_id])) # avoid reinserting the same tag multiple times tag_rows = c.cass.execute( """SELECT tag FROM mqe.report_tag WHERE report_id=? AND tag_prefix='' AND tag IN ?""", [report_id, tags]) tags_from_rows = {row['tag'] for row in tag_rows} for tag in tags: if tag in tags_from_rows: continue for p in util.iter_prefixes(tag, include_empty=True): qs.append( insert('mqe.report_tag', dict(report_id=report_id, tag_prefix=p, tag=tag))) c.cass.execute_parallel(qs) return postprocess_tags(first_row)