def delete(self, owner_id, report_id):
row = self.select(report_id)
if not row:
log.warn('No report row %s', report_id)
return
qs = []
qs.append(
bind("""DELETE FROM mqe.report WHERE report_id=?""", [report_id]))
qs.append(
bind(
"""DELETE FROM mqe.report_by_owner_id_report_name
WHERE owner_id=? AND report_name=?""",
[owner_id, row['report_name']]))
for prefix in util.iter_prefixes(row['report_name'],
include_empty=True):
qs.append(
bind(
"""DELETE FROM mqe.report_name_by_owner_id_report_name_prefix
WHERE owner_id=? AND report_name_prefix=? AND report_name=?""",
[owner_id, prefix, row['report_name']]))
qs.append(
bind("""DELETE FROM mqe.report_tag WHERE report_id=?""",
[report_id]))
c.cass.execute_parallel(qs)
def insert(self, owner_id, report_name):
log.debug('Trying to insert new report %r', report_name)
report_id = gen_timeuuid()
def insert_report_name():
return c.cass.execute(
"""INSERT INTO mqe.report_by_owner_id_report_name (owner_id, report_name, report_id)
VALUES (?, ?, ?)
IF NOT EXISTS""",
[owner_id, report_name, report_id])
lwt_res = execute_lwt(insert_report_name)
if lwt_res == False:
log.info('Race condition in creating a new report %r', report_name)
return None
elif lwt_res == None:
rows = c.cass.execute(
"""SELECT report_id FROM mqe.report_by_owner_id_report_name
WHERE owner_id=? AND report_name=? /* SERIAL */""",
[owner_id, report_name], ConsistencyLevel.SERIAL)
if not rows or rows[0]['report_id'] != report_id:
log.info(
'Race condition in creating a new report when lwt_res==None: %r',
report_name)
return None
row = {
'report_id': report_id,
'report_name': report_name,
'owner_id': owner_id
}
c.cass.execute(insert('mqe.report', row))
log.info('Inserted new report report_id=%s report_name=%r', report_id,
report_name)
qs = []
for prefix in util.iter_prefixes(report_name, include_empty=True):
qs.append(
insert(
'mqe.report_name_by_owner_id_report_name_prefix',
dict(owner_id=owner_id,
report_name_prefix=prefix,
report_name=report_name)))
c.cass.execute_parallel(qs)
return row
def _delete_ris(self, owner_id, report_id, tags, ris, update_counters):
qs = []
count_by_tags_repr = defaultdict(int)
diskspace_by_tags_repr = defaultdict(int)
tags_reprs_days = set()
for ri in ris:
qs.append(
bind(
"""DELETE FROM mqe.report_instance_metadata
WHERE report_id=? AND day=? AND report_instance_id=?""",
[report_id, ri['day'], ri['report_instance_id']]))
for tags_subset in util.powerset(ri['all_tags']):
tags_repr = tags_repr_from_tags(tags_subset)
qs.append(
bind(
"""DELETE FROM mqe.report_instance
WHERE report_id=? AND day=? AND tags_repr=?
AND report_instance_id=?""", [
report_id, ri['day'], tags_repr,
ri['report_instance_id']
]))
count_by_tags_repr[tags_repr] += 1
if update_counters:
diskspace_by_tags_repr[
tags_repr] += self._compute_ri_diskspace(ri)
tags_reprs_days.add((tags_repr, ri['day']))
if update_counters:
qs.append(
bind(
"""UPDATE mqe.report_instance_count_for_owner
SET count=count-?
WHERE owner_id=?""",
[count_by_tags_repr[''], owner_id]))
qs.append(
bind(
"""UPDATE mqe.report_instance_diskspace_for_owner
SET bytes=bytes-?
WHERE owner_id=?""",
[diskspace_by_tags_repr[''], owner_id]))
for tags_repr, count in count_by_tags_repr.iteritems():
qs.append(
bind(
"""UPDATE mqe.report_instance_count
SET count=count-?
WHERE report_id=? AND tags_repr=?""",
[count, report_id, tags_repr]))
for tags_repr, bytes in diskspace_by_tags_repr.iteritems():
qs.append(
bind(
"""UPDATE mqe.report_instance_diskspace
SET bytes=bytes-?
WHERE report_id=? AND tags_repr=?""",
[bytes, report_id, tags_repr]))
c.cass.execute_parallel(qs)
### Delete days for which report instances no longer exist
days_qs = {}
for tags_repr, day in tags_reprs_days:
days_qs[(tags_repr, day)] = bind(
"""SELECT report_instance_id FROM mqe.report_instance
WHERE report_id=? AND day=? AND tags_repr=?
LIMIT 1""",
[report_id, day, tags_repr])
days_res = c.cass.execute_parallel(days_qs)
qs = []
for (tags_repr, day), rows in days_res.iteritems():
if rows:
continue
qs.append(
bind(
"""DELETE FROM mqe.report_instance_day
WHERE report_id=? AND tags_repr=? AND day=?""",
[report_id, tags_repr, day]))
log.info('Deleting %s days', len(qs))
c.cass.execute_parallel(qs)
### Delete tags for which report instances no longer exist
tags_present = set()
for tags_repr, _ in tags_reprs_days:
for tag in tags_from_tags_repr(tags_repr):
tags_present.add(tag)
qs = []
for tag in tags_present:
rows = c.cass.execute(
"""SELECT tags_repr FROM mqe.report_instance_day
WHERE report_id=? AND tags_repr=?
LIMIT 1""",
[report_id, tags_repr_from_tags([tag])])
if rows:
continue
for p in util.iter_prefixes(tag, include_empty=True):
qs.append(
bind(
"""DELETE FROM mqe.report_tag
WHERE report_id=? AND tag_prefix=? AND tag=?""",
[report_id, p, tag]))
log.info('Deleting %s tag rows', len(qs))
c.cass.execute_parallel(qs)
return len(ris), [tags_from_tags_repr(tr) for tr in count_by_tags_repr]
def insert(self, owner_id, report_id, report_instance_id, tags, ri_data,
input_string, extra_ri_data, custom_created):
created = util.datetime_from_uuid1(report_instance_id)
day = day_text(created)
all_tags_repr = tags_repr_from_tags(tags)
qs = []
metadata_row = dict(
report_id=report_id,
day=day,
report_instance_id=report_instance_id,
all_tags_repr=all_tags_repr,
inserted=datetime.datetime.utcnow(),
)
if extra_ri_data:
metadata_row['extra_ri_data'] = extra_ri_data
qs.append(insert('mqe.report_instance_metadata', metadata_row))
first_row = None
tags_powerset = util.powerset(tags[:mqeconfig.MAX_TAGS])
for tags_subset in tags_powerset:
tags_repr = tags_repr_from_tags(tags_subset)
row = dict(report_id=report_id,
day=day,
tags_repr=tags_repr,
report_instance_id=report_instance_id,
ri_data=ri_data,
input_string=input_string,
all_tags_repr=all_tags_repr)
if first_row is None:
first_row = row
qs.append(
insert('mqe.report_instance', row,
COLUMN_RENAMES['report_instance']))
if not c.cass.execute(
"""SELECT day FROM mqe.report_instance_day
WHERE report_id=? AND tags_repr=? AND day=?""",
[report_id, tags_repr, day]):
qs.append(
insert(
'mqe.report_instance_day',
dict(report_id=report_id, tags_repr=tags_repr,
day=day)))
qs.append(
bind(
"""UPDATE mqe.report_instance_count SET count=count+1
WHERE report_id=? AND tags_repr=?""",
[report_id, tags_repr]))
diskspace = self._compute_ri_diskspace(row)
qs.append(
bind(
"""UPDATE mqe.report_instance_diskspace SET bytes=bytes+?
WHERE report_id=? AND tags_repr=?""",
[diskspace, report_id, tags_repr]))
### queries for all tags
qs.append(
bind(
"""UPDATE mqe.report_instance_count_for_owner SET count=count+1
WHERE owner_id=?""", [owner_id]))
if first_row:
diskspace = self._compute_ri_diskspace(first_row)
qs.append(
bind(
"""UPDATE mqe.report_instance_diskspace_for_owner SET bytes=bytes+?
WHERE owner_id=?""", [diskspace, owner_id]))
# avoid reinserting the same tag multiple times
tag_rows = c.cass.execute(
"""SELECT tag FROM mqe.report_tag
WHERE report_id=? AND tag_prefix='' AND tag IN ?""",
[report_id, tags])
tags_from_rows = {row['tag'] for row in tag_rows}
for tag in tags:
if tag in tags_from_rows:
continue
for p in util.iter_prefixes(tag, include_empty=True):
qs.append(
insert('mqe.report_tag',
dict(report_id=report_id, tag_prefix=p, tag=tag)))
c.cass.execute_parallel(qs)
return postprocess_tags(first_row)