def delete(self, owner_id, report_id):
row = self.select(report_id)
if not row:
log.warn('No report row %s', report_id)
return
qs = []
qs.append(
bind("""DELETE FROM mqe.report WHERE report_id=?""", [report_id]))
qs.append(
bind(
"""DELETE FROM mqe.report_by_owner_id_report_name
WHERE owner_id=? AND report_name=?""",
[owner_id, row['report_name']]))
for prefix in util.iter_prefixes(row['report_name'],
include_empty=True):
qs.append(
bind(
"""DELETE FROM mqe.report_name_by_owner_id_report_name_prefix
WHERE owner_id=? AND report_name_prefix=? AND report_name=?""",
[owner_id, prefix, row['report_name']]))
qs.append(
bind("""DELETE FROM mqe.report_tag WHERE report_id=?""",
[report_id]))
c.cass.execute_parallel(qs)
def delete(self, user_id):
row = self.select(user_id)
if not row:
return
c.cass.execute(
batch(
bind("""DELETE FROM mqe.user WHERE user_id=?""",
[row['user_id']]),
bind("""DELETE FROM mqe.user_by_email WHERE email=?""",
[row['email']]),
))
def delete_multi(self, tile_list):
qs = []
for tile in tile_list:
if not tile:
continue
qs.append(
bind(
"""DELETE FROM mqe.tile WHERE dashboard_id=? AND tile_id=?""",
[tile.dashboard_id, tile.tile_id]))
qs.append(
bind(
"""UPDATE mqe.tile_count SET count=count-1 WHERE owner_id=?""",
[tile.owner_id]))
c.cass.execute_parallel(qs)
def select_id_or_insert_multi(self, report_id, tags_series_spec_list):
select_qs = []
for (tags, series_spec) in tags_series_spec_list:
select_qs.append(
bind(
"""SELECT series_id FROM mqe.series_def_by_series_spec
WHERE report_id=? AND tags_repr=? AND series_spec=?""",
[
report_id,
tags_repr_from_tags(tags),
serialize.mjson(series_spec)
]))
select_qs_res = c.cass.execute_parallel(select_qs)
res = [
rows[0]['series_id'] if rows else None for rows in select_qs_res
]
to_insert_idxs = [
i for i in xrange(len(tags_series_spec_list)) if res[i] is None
]
to_insert_data = [tags_series_spec_list[i] for i in to_insert_idxs]
insert_res = self.insert_multi(report_id, to_insert_data)
for i in xrange(len(to_insert_idxs)):
res[to_insert_idxs[i]] = insert_res[i]
return res
def invalidate_precomputed_instances_data(report_id, tags_powerset,
first_valid_dt):
log.debug('Invalidating precomputed %s %s %s', report_id, tags_powerset,
first_valid_dt)
all_tags_reprs = [tags_repr_from_tags(ts) for ts in tags_powerset]
series_rows = c.cass.execute(
"""SELECT * FROM mqe.series_def
WHERE report_id=? AND tags_repr IN ?""",
[report_id, all_tags_reprs])
qs = []
for row in series_rows:
if row['from_rid'] is None:
continue
if cassandra.util.datetime_from_uuid1(row['from_rid']) < first_valid_dt and \
cassandra.util.datetime_from_uuid1(row['to_rid']) < first_valid_dt:
log.debug('Clearing series_id=%s', row['series_id'])
new_from_rid = None
new_to_rid = None
elif cassandra.util.datetime_from_uuid1(row['from_rid']) >= first_valid_dt and \
cassandra.util.datetime_from_uuid1(row['to_rid']) >= first_valid_dt:
continue
else:
log.debug('Updating from_rid_dt=%s', first_valid_dt)
new_from_rid = cassandra.util.min_uuid_from_time(first_valid_dt)
new_to_rid = row['to_rid']
qs.append(
bind(
"""UPDATE mqe.series_def SET from_rid=?, to_rid=?
WHERE report_id=? AND tags_repr=? AND series_id=?""",
[
new_from_rid, new_to_rid, report_id, row['tags_repr'],
row['series_id']
]))
if new_from_rid is None:
qs.append(
bind("""DELETE FROM mqe.series_value WHERE series_id=?""",
[row['series_id']]))
else:
qs.append(
bind(
"""DELETE FROM mqe.series_value WHERE series_id=? AND report_instance_id < ?""",
[row['series_id'], new_from_rid]))
c.cass.execute_parallel(qs)
def clear_all_series_defs(self, report_id, tags_powerset):
all_tags_reprs = [tags_repr_from_tags(ts) for ts in tags_powerset]
series_rows = c.cass.execute(
"""SELECT series_id, from_rid, to_rid, tags_repr
FROM mqe.series_def
WHERE report_id=? AND tags_repr IN ?""",
[report_id, all_tags_reprs])
qs = []
for row in series_rows:
if row['from_rid'] is not None or row['to_rid'] is not None:
qs.append(
bind(
"""UPDATE mqe.series_def SET from_rid=NULL, to_rid=NULL
WHERE report_id=? AND tags_repr=? AND series_id=?""",
[report_id, row['tags_repr'], row['series_id']]))
qs.append(
bind("""DELETE FROM mqe.series_value WHERE series_id=?""",
[row['series_id']]))
c.cass.execute_parallel(qs)
def select_multi(self, report_id, tags_series_id_list):
qs = []
for tags, series_id in tags_series_id_list:
qs.append(
bind(
"""SELECT * FROM mqe.series_def
WHERE report_id=? AND tags_repr=? AND series_id=?""",
[report_id,
tags_repr_from_tags(tags), series_id]))
res = c.cass.execute_parallel(qs)
res = [postprocess_tags(rows[0]) if rows else None for rows in res]
return res
def insert_multi(self, owner_id, dashboard_id, tile_options_list):
qs = []
res = []
for tile_options in tile_options_list:
row = dict(dashboard_id=dashboard_id,
tile_id=gen_timeuuid(),
tile_options=tile_options)
qs.append(insert('mqe.tile', row))
qs.append(
bind(
"""UPDATE mqe.tile_count SET count=count+1 WHERE owner_id=?""",
[owner_id]))
res.append(row)
c.cass.execute_parallel(qs)
return res
def delete(self, owner_id, report_id, report_instance_id):
ri = self.select(report_id, report_instance_id, None)
if not ri:
return False
diskspace = self._compute_ri_diskspace(ri)
qs = []
qs.append(
bind(
"""DELETE FROM mqe.report_instance_metadata
WHERE report_id=? AND day=? AND report_instance_id=?""",
[report_id, ri['day'], report_instance_id]))
for tags_subset in util.powerset(ri['all_tags']):
tags_repr = tags_repr_from_tags(tags_subset)
qs.append(
bind(
"""DELETE FROM mqe.report_instance WHERE report_id=? AND day=? AND tags_repr=? AND report_instance_id=?""",
[report_id, ri['day'], tags_repr, report_instance_id]))
qs.append(
bind(
"""UPDATE mqe.report_instance_diskspace SET bytes=bytes-?
WHERE report_id=? AND tags_repr=?""",
[diskspace, report_id, tags_repr]))
qs.append(
bind(
"""UPDATE mqe.report_instance_count SET count=count-1
WHERE report_id=? AND tags_repr=?""",
[report_id, tags_repr]))
qs.append(
bind(
"""UPDATE mqe.report_instance_count_for_owner SET count=count-1
WHERE owner_id=?""", [owner_id]))
qs.append(
bind(
"""UPDATE mqe.report_instance_diskspace_for_owner SET bytes=bytes-?
WHERE owner_id=?""", [diskspace, owner_id]))
# Can't delete from mqe.report_tag - tag might be used by other report instances
#for tag in ri.all_tags:
# prefixes = list(util.iter_prefixes(tag, include_empty=True))
# for p in prefixes:
# qs.append(bind("""DELETE FROM mqe.report_tag
# WHERE report_id=? AND tag_prefix=? AND tag=?""",
# [report_id, p, tag]))
c.cass.execute_parallel(qs)
return True
def _delete_ris(self, owner_id, report_id, tags, ris, update_counters):
qs = []
count_by_tags_repr = defaultdict(int)
diskspace_by_tags_repr = defaultdict(int)
tags_reprs_days = set()
for ri in ris:
qs.append(
bind(
"""DELETE FROM mqe.report_instance_metadata
WHERE report_id=? AND day=? AND report_instance_id=?""",
[report_id, ri['day'], ri['report_instance_id']]))
for tags_subset in util.powerset(ri['all_tags']):
tags_repr = tags_repr_from_tags(tags_subset)
qs.append(
bind(
"""DELETE FROM mqe.report_instance
WHERE report_id=? AND day=? AND tags_repr=?
AND report_instance_id=?""", [
report_id, ri['day'], tags_repr,
ri['report_instance_id']
]))
count_by_tags_repr[tags_repr] += 1
if update_counters:
diskspace_by_tags_repr[
tags_repr] += self._compute_ri_diskspace(ri)
tags_reprs_days.add((tags_repr, ri['day']))
if update_counters:
qs.append(
bind(
"""UPDATE mqe.report_instance_count_for_owner
SET count=count-?
WHERE owner_id=?""",
[count_by_tags_repr[''], owner_id]))
qs.append(
bind(
"""UPDATE mqe.report_instance_diskspace_for_owner
SET bytes=bytes-?
WHERE owner_id=?""",
[diskspace_by_tags_repr[''], owner_id]))
for tags_repr, count in count_by_tags_repr.iteritems():
qs.append(
bind(
"""UPDATE mqe.report_instance_count
SET count=count-?
WHERE report_id=? AND tags_repr=?""",
[count, report_id, tags_repr]))
for tags_repr, bytes in diskspace_by_tags_repr.iteritems():
qs.append(
bind(
"""UPDATE mqe.report_instance_diskspace
SET bytes=bytes-?
WHERE report_id=? AND tags_repr=?""",
[bytes, report_id, tags_repr]))
c.cass.execute_parallel(qs)
### Delete days for which report instances no longer exist
days_qs = {}
for tags_repr, day in tags_reprs_days:
days_qs[(tags_repr, day)] = bind(
"""SELECT report_instance_id FROM mqe.report_instance
WHERE report_id=? AND day=? AND tags_repr=?
LIMIT 1""",
[report_id, day, tags_repr])
days_res = c.cass.execute_parallel(days_qs)
qs = []
for (tags_repr, day), rows in days_res.iteritems():
if rows:
continue
qs.append(
bind(
"""DELETE FROM mqe.report_instance_day
WHERE report_id=? AND tags_repr=? AND day=?""",
[report_id, tags_repr, day]))
log.info('Deleting %s days', len(qs))
c.cass.execute_parallel(qs)
### Delete tags for which report instances no longer exist
tags_present = set()
for tags_repr, _ in tags_reprs_days:
for tag in tags_from_tags_repr(tags_repr):
tags_present.add(tag)
qs = []
for tag in tags_present:
rows = c.cass.execute(
"""SELECT tags_repr FROM mqe.report_instance_day
WHERE report_id=? AND tags_repr=?
LIMIT 1""",
[report_id, tags_repr_from_tags([tag])])
if rows:
continue
for p in util.iter_prefixes(tag, include_empty=True):
qs.append(
bind(
"""DELETE FROM mqe.report_tag
WHERE report_id=? AND tag_prefix=? AND tag=?""",
[report_id, p, tag]))
log.info('Deleting %s tag rows', len(qs))
c.cass.execute_parallel(qs)
return len(ris), [tags_from_tags_repr(tr) for tr in count_by_tags_repr]
def insert(self, owner_id, report_id, report_instance_id, tags, ri_data,
input_string, extra_ri_data, custom_created):
created = util.datetime_from_uuid1(report_instance_id)
day = day_text(created)
all_tags_repr = tags_repr_from_tags(tags)
qs = []
metadata_row = dict(
report_id=report_id,
day=day,
report_instance_id=report_instance_id,
all_tags_repr=all_tags_repr,
inserted=datetime.datetime.utcnow(),
)
if extra_ri_data:
metadata_row['extra_ri_data'] = extra_ri_data
qs.append(insert('mqe.report_instance_metadata', metadata_row))
first_row = None
tags_powerset = util.powerset(tags[:mqeconfig.MAX_TAGS])
for tags_subset in tags_powerset:
tags_repr = tags_repr_from_tags(tags_subset)
row = dict(report_id=report_id,
day=day,
tags_repr=tags_repr,
report_instance_id=report_instance_id,
ri_data=ri_data,
input_string=input_string,
all_tags_repr=all_tags_repr)
if first_row is None:
first_row = row
qs.append(
insert('mqe.report_instance', row,
COLUMN_RENAMES['report_instance']))
if not c.cass.execute(
"""SELECT day FROM mqe.report_instance_day
WHERE report_id=? AND tags_repr=? AND day=?""",
[report_id, tags_repr, day]):
qs.append(
insert(
'mqe.report_instance_day',
dict(report_id=report_id, tags_repr=tags_repr,
day=day)))
qs.append(
bind(
"""UPDATE mqe.report_instance_count SET count=count+1
WHERE report_id=? AND tags_repr=?""",
[report_id, tags_repr]))
diskspace = self._compute_ri_diskspace(row)
qs.append(
bind(
"""UPDATE mqe.report_instance_diskspace SET bytes=bytes+?
WHERE report_id=? AND tags_repr=?""",
[diskspace, report_id, tags_repr]))
### queries for all tags
qs.append(
bind(
"""UPDATE mqe.report_instance_count_for_owner SET count=count+1
WHERE owner_id=?""", [owner_id]))
if first_row:
diskspace = self._compute_ri_diskspace(first_row)
qs.append(
bind(
"""UPDATE mqe.report_instance_diskspace_for_owner SET bytes=bytes+?
WHERE owner_id=?""", [diskspace, owner_id]))
# avoid reinserting the same tag multiple times
tag_rows = c.cass.execute(
"""SELECT tag FROM mqe.report_tag
WHERE report_id=? AND tag_prefix='' AND tag IN ?""",
[report_id, tags])
tags_from_rows = {row['tag'] for row in tag_rows}
for tag in tags:
if tag in tags_from_rows:
continue
for p in util.iter_prefixes(tag, include_empty=True):
qs.append(
insert('mqe.report_tag',
dict(report_id=report_id, tag_prefix=p, tag=tag)))
c.cass.execute_parallel(qs)
return postprocess_tags(first_row)
