def select_id_or_insert_multi(self, report_id, tags_series_spec_list):
res = []
with cursor() as cur:
for (tags, series_spec) in tags_series_spec_list:
cur.execute(
"""SELECT series_id FROM series_def
WHERE series_spec=? AND report_id=? AND tags=?""",
[serialize.mjson(series_spec), report_id, tags])
row = cur.fetchone()
if row:
res.append(row['series_id'])
continue
series_id = gen_timeuuid()
cur.execute(
"""INSERT INTO series_def (report_id, tags, series_id, series_spec, from_rid, to_rid)
SELECT ?, ?, ?, ?, ?, ? WHERE NOT EXISTS
(SELECT 1 FROM series_def
WHERE report_id=? AND tags=? AND series_id=?)""",
[
report_id, tags, series_id,
serialize.mjson(series_spec), None, None, report_id,
tags, series_id
])
if cur.lastrowid:
res.append(series_id)
continue
cur.execute(
"""SELECT series_id FROM series_def
WHERE series_spec=? AND report_id=? AND tags=?""",
[serialize.mjson(series_spec), report_id, tags])
res.append(cur.fetchone()['series_id'])
return res
def insert_multi(self, report_id, tags_series_spec_list):
res = []
with cursor() as cur:
for tags, series_spec in tags_series_spec_list:
series_id = gen_timeuuid()
cur.execute(*insert('series_def', dict(
report_id=report_id,
tags=tags,
series_id=series_id,
series_spec=serialize.mjson(series_spec))))
res.append(series_id)
return res
def insert_multi(self, owner_id, dashboard_id, tile_options_list):
qs = """INSERT INTO tile (dashboard_id, tile_id, tile_options)
VALUES (?, ?, ?)"""
params_list = []
res = []
for tile_options in tile_options_list:
row = dict(dashboard_id=dashboard_id,
tile_id=gen_timeuuid(),
tile_options=tile_options)
params_list.append([row['dashboard_id'], row['tile_id'], row['tile_options']])
res.append(row)
with cursor() as cur:
cur.executemany(qs, params_list)
return res
def select_or_insert(self, owner_id, report_name):
row = self.select_by_name(owner_id, report_name)
if row:
return False, row
with cursor() as cur:
report_id = gen_timeuuid()
cur.execute("""INSERT INTO report (report_id, report_name, owner_id)
SELECT ?, ?, ? WHERE NOT EXISTS
(SELECT 1 FROM report WHERE owner_id=? AND report_name=?)""",
[report_id, report_name, owner_id, owner_id, report_name])
if cur.lastrowid:
return True, {'owner_id': owner_id, 'report_name': report_name,
'report_id': report_id}
return False, self.select_by_name(owner_id, report_name)
def insert_multi(self, owner_id, dashboard_id, tile_options_list):
qs = []
res = []
for tile_options in tile_options_list:
row = dict(dashboard_id=dashboard_id,
tile_id=gen_timeuuid(),
tile_options=tile_options)
qs.append(insert('mqe.tile', row))
qs.append(
bind(
"""UPDATE mqe.tile_count SET count=count+1 WHERE owner_id=?""",
[owner_id]))
res.append(row)
c.cass.execute_parallel(qs)
return res
def insert(self, owner_id, report_name):
log.debug('Trying to insert new report %r', report_name)
report_id = gen_timeuuid()
def insert_report_name():
return c.cass.execute(
"""INSERT INTO mqe.report_by_owner_id_report_name (owner_id, report_name, report_id)
VALUES (?, ?, ?)
IF NOT EXISTS""",
[owner_id, report_name, report_id])
lwt_res = execute_lwt(insert_report_name)
if lwt_res == False:
log.info('Race condition in creating a new report %r', report_name)
return None
elif lwt_res == None:
rows = c.cass.execute(
"""SELECT report_id FROM mqe.report_by_owner_id_report_name
WHERE owner_id=? AND report_name=? /* SERIAL */""",
[owner_id, report_name], ConsistencyLevel.SERIAL)
if not rows or rows[0]['report_id'] != report_id:
log.info(
'Race condition in creating a new report when lwt_res==None: %r',
report_name)
return None
row = {
'report_id': report_id,
'report_name': report_name,
'owner_id': owner_id
}
c.cass.execute(insert('mqe.report', row))
log.info('Inserted new report report_id=%s report_name=%r', report_id,
report_name)
qs = []
for prefix in util.iter_prefixes(report_name, include_empty=True):
qs.append(
insert(
'mqe.report_name_by_owner_id_report_name_prefix',
dict(owner_id=owner_id,
report_name_prefix=prefix,
report_name=report_name)))
c.cass.execute_parallel(qs)
return row
def insert_multi(self, report_id, tags_series_spec_list):
if not tags_series_spec_list:
return []
qs = []
res = []
for tags, series_spec in tags_series_spec_list:
series_id = gen_timeuuid()
qs.append(
insert(
'mqe.series_def',
dict(report_id=report_id,
tags_repr=tags_repr_from_tags(tags),
series_id=series_id,
series_spec=serialize.mjson(series_spec))))
qs.append(
insert(
'mqe.series_def_by_series_spec',
dict(report_id=report_id,
tags_repr=tags_repr_from_tags(tags),
series_spec=serialize.mjson(series_spec),
series_id=series_id)))
res.append(series_id)
c.cass.execute_parallel(qs)
return res
def set(self, owner_id=None, dashboard_id=None, old_layout_id=None):
"""Set a new layout definition for the dashboard (replacing the existing one), using
the current content of the :attr:`layout_dict`. The parameters are optional - if not
specified, the current values of :attr:`owner_id`, :attr:`dashboard_id` and
:attr:`layout_id` are used.
:param owner_id: the owner ID of the dashboard
:param dashboard_id: the dashboard's ID
:param old_layout_id: ``None`` if this should be a new layout definition
for the dashboard, ``layout_id`` of the existing layout otherwise
:return: a ``layout_id`` of a newly set layout if the operation was successful,
``None`` otherwise (ie. when the passed ``old_layout_id`` didn't match the
version in the database)
"""
owner_id = owner_id or self.owner_id
if not owner_id:
raise ValueError('owner_id not set in Layout and not passed as an argument')
dashboard_id = dashboard_id or self.dashboard_id
if not dashboard_id:
raise ValueError('dashboard_id not set in Layout and not passed as an argument')
old_layout_id = old_layout_id or self.layout_id
# a layout def is a layout_dict serialized as a list of items. The list is
# sorted by tile creation time (but this assumption should not be generally made).
new_layout_def = serialize.mjson(sorted(self.layout_dict.items(),
key=lambda (tile_id, vo): tile_id.time))
# Merge old layout_props with new data
old_layout_props_row = c.dao.LayoutDAO.select(owner_id, dashboard_id, ['layout_props'])
if not old_layout_props_row and old_layout_id:
return None
if old_layout_props_row and old_layout_props_row['layout_props']:
old_layout_props = serialize.json_loads(old_layout_props_row['layout_props'])
else:
old_layout_props = {'by_tile_id': []}
by_tile_id = {}
old_by_tile_id = dict(old_layout_props['by_tile_id'])
tile_ids_to_fetch = []
for tile_id in self.layout_dict:
if tile_id in old_by_tile_id:
by_tile_id[tile_id] = old_by_tile_id[tile_id]
elif tile_id in self._included_tiles:
by_tile_id[tile_id] = self.props_of_tile(self._included_tiles[tile_id])
else:
tile_ids_to_fetch.append(tile_id)
tile_dict = Tile.select_multi(dashboard_id, tile_ids_to_fetch)
for tile_id, tile in tile_dict.items():
by_tile_id[tile.tile_id] = self.props_of_tile(tile)
# Compute data for sscreator and tpcreator
sscs_data = set()
master_data = set()
for props in by_tile_id.values():
if props.get('sscs'):
#sscs_data.add((props['report_id'], tuple(props['tags'])))
sscs_data.add(props['report_id'])
if props.get('is_master'):
master_data.add(props['report_id'])
new_layout_props = serialize.mjson({'by_tile_id': by_tile_id.items()})
# Set the new layout
new_layout_id = gen_timeuuid()
res = c.dao.LayoutDAO.set(owner_id, dashboard_id, old_layout_id, new_layout_id,
new_layout_def, new_layout_props)
if not res:
log.info('Setting new layout failed')
return None
# Insert layout_by_report for sscs and tpcreator
c.dao.LayoutDAO.insert_layout_by_report_multi(owner_id, sscs_data, [], 'sscs',
dashboard_id, new_layout_id)
c.dao.LayoutDAO.insert_layout_by_report_multi(owner_id, master_data, [], 'tpcreator',
dashboard_id, new_layout_id)
self.layout_id = new_layout_id
return new_layout_id
def process_input(self,
input_string,
tags=None,
created=None,
input_type='any',
ip_options={},
force_header=None,
extra_ri_data=None,
handle_tpcreator=True,
handle_sscreator=True):
"""Process an input string - parse it into a table and create a report instance belonging
to the report.
:param str|unicode input_string: the input string
:param list tags: a list of string tags attached to the report instance
:param ~datetime.datetime created: an explicit creation datetime of the report instance (
default: the current datetime)
:param str input_type: input type (see :func:`mqetables.parseany.parse_input`)
:param dict ip_options: extra parser options (see :func:`mqetables.parsing.InputParser`)
:param force_header: a list of header rows indexes to set as a header (defaults to
auto-detection)
:param extra_ri_data: a custom JSON-serializable document attached to the report instance
:param handle_tpcreator: whether to handle TPCreator for the created report instance
by calling :func:`~mqe.tpcreator.handle_tpcreator`
:param handle_sscreator: whether to handle SSCS by calling :func:`~mqe.sscreator.handle_sscreator`
:return: an :class:`InputProcessingResult`
"""
assert isinstance(input_string, (str, unicode))
# disallow 'created' in the future
now = datetime.datetime.utcnow()
if created is not None and created.tzinfo:
created = util.make_tz_naive(created)
if created is not None and created.year < 2000:
raise ValueError('created cannot be before the year 2000')
if created is not None and created < now:
report_instance_id = util.uuid_with_dt(created)
custom_created = True
else:
custom_created = False
report_instance_id = gen_timeuuid()
created = util.datetime_from_uuid1(report_instance_id)
if tags is None:
tags = []
parsing_result = parseany.parse_input(input_string, input_type,
ip_options)
table = mqeconfig.get_table_from_parsing_result(parsing_result)
if table is None:
return InputProcessingResult(None, parsing_result)
if force_header is not None:
log.debug('Overwriting header detection due to force_header')
table.header_idxs = [
i for i in force_header if util.valid_index(table.num_rows, i)
]
table.header_idxs_source = parsing.HEADER_IDXS_SOURCE_USER
ri_data_dict = {
'table': table,
}
result_desc = self._get_result_desc(parsing_result)
if result_desc:
ri_data_dict['result_desc'] = result_desc
report_instance_row = c.dao.ReportInstanceDAO.insert(
owner_id=self.owner_id,
report_id=self.report_id,
report_instance_id=report_instance_id,
tags=tags,
ri_data=serialize.mjson(ri_data_dict),
input_string=parsing_result.input_string,
extra_ri_data=serialize.mjson(extra_ri_data)
if extra_ri_data else None,
custom_created=custom_created)
report_instance = ReportInstance(report_instance_row)
log.info(
'Created new report instance report_id=%s report_name=%r tags=%s '
'report_instance_id=%s created=%s', self.report_id,
self.report_name, tags, report_instance_id,
report_instance.created)
if tags and handle_tpcreator:
from mqe import tpcreator
tpcreator.handle_tpcreator(self.owner_id, self.report_id,
report_instance)
if handle_sscreator:
from mqe import sscreator
sscreator.handle_sscreator(self.owner_id, self.report_id,
report_instance)
if custom_created:
from mqe import dataseries
dataseries.clear_series_defs(self.report_id, util.powerset(tags))
return InputProcessingResult(report_instance, parsing_result)
