def select_id_or_insert_multi(self, report_id, tags_series_spec_list): res = [] with cursor() as cur: for (tags, series_spec) in tags_series_spec_list: cur.execute( """SELECT series_id FROM series_def WHERE series_spec=? AND report_id=? AND tags=?""", [serialize.mjson(series_spec), report_id, tags]) row = cur.fetchone() if row: res.append(row['series_id']) continue series_id = gen_timeuuid() cur.execute( """INSERT INTO series_def (report_id, tags, series_id, series_spec, from_rid, to_rid) SELECT ?, ?, ?, ?, ?, ? WHERE NOT EXISTS (SELECT 1 FROM series_def WHERE report_id=? AND tags=? AND series_id=?)""", [ report_id, tags, series_id, serialize.mjson(series_spec), None, None, report_id, tags, series_id ]) if cur.lastrowid: res.append(series_id) continue cur.execute( """SELECT series_id FROM series_def WHERE series_spec=? AND report_id=? AND tags=?""", [serialize.mjson(series_spec), report_id, tags]) res.append(cur.fetchone()['series_id']) return res
def insert_multi(self, report_id, tags_series_spec_list): res = [] with cursor() as cur: for tags, series_spec in tags_series_spec_list: series_id = gen_timeuuid() cur.execute(*insert('series_def', dict( report_id=report_id, tags=tags, series_id=series_id, series_spec=serialize.mjson(series_spec)))) res.append(series_id) return res
def insert_multi(self, owner_id, dashboard_id, tile_options_list): qs = """INSERT INTO tile (dashboard_id, tile_id, tile_options) VALUES (?, ?, ?)""" params_list = [] res = [] for tile_options in tile_options_list: row = dict(dashboard_id=dashboard_id, tile_id=gen_timeuuid(), tile_options=tile_options) params_list.append([row['dashboard_id'], row['tile_id'], row['tile_options']]) res.append(row) with cursor() as cur: cur.executemany(qs, params_list) return res
def select_or_insert(self, owner_id, report_name): row = self.select_by_name(owner_id, report_name) if row: return False, row with cursor() as cur: report_id = gen_timeuuid() cur.execute("""INSERT INTO report (report_id, report_name, owner_id) SELECT ?, ?, ? WHERE NOT EXISTS (SELECT 1 FROM report WHERE owner_id=? AND report_name=?)""", [report_id, report_name, owner_id, owner_id, report_name]) if cur.lastrowid: return True, {'owner_id': owner_id, 'report_name': report_name, 'report_id': report_id} return False, self.select_by_name(owner_id, report_name)
def insert_multi(self, owner_id, dashboard_id, tile_options_list): qs = [] res = [] for tile_options in tile_options_list: row = dict(dashboard_id=dashboard_id, tile_id=gen_timeuuid(), tile_options=tile_options) qs.append(insert('mqe.tile', row)) qs.append( bind( """UPDATE mqe.tile_count SET count=count+1 WHERE owner_id=?""", [owner_id])) res.append(row) c.cass.execute_parallel(qs) return res
def insert(self, owner_id, report_name): log.debug('Trying to insert new report %r', report_name) report_id = gen_timeuuid() def insert_report_name(): return c.cass.execute( """INSERT INTO mqe.report_by_owner_id_report_name (owner_id, report_name, report_id) VALUES (?, ?, ?) IF NOT EXISTS""", [owner_id, report_name, report_id]) lwt_res = execute_lwt(insert_report_name) if lwt_res == False: log.info('Race condition in creating a new report %r', report_name) return None elif lwt_res == None: rows = c.cass.execute( """SELECT report_id FROM mqe.report_by_owner_id_report_name WHERE owner_id=? AND report_name=? /* SERIAL */""", [owner_id, report_name], ConsistencyLevel.SERIAL) if not rows or rows[0]['report_id'] != report_id: log.info( 'Race condition in creating a new report when lwt_res==None: %r', report_name) return None row = { 'report_id': report_id, 'report_name': report_name, 'owner_id': owner_id } c.cass.execute(insert('mqe.report', row)) log.info('Inserted new report report_id=%s report_name=%r', report_id, report_name) qs = [] for prefix in util.iter_prefixes(report_name, include_empty=True): qs.append( insert( 'mqe.report_name_by_owner_id_report_name_prefix', dict(owner_id=owner_id, report_name_prefix=prefix, report_name=report_name))) c.cass.execute_parallel(qs) return row
def insert_multi(self, report_id, tags_series_spec_list): if not tags_series_spec_list: return [] qs = [] res = [] for tags, series_spec in tags_series_spec_list: series_id = gen_timeuuid() qs.append( insert( 'mqe.series_def', dict(report_id=report_id, tags_repr=tags_repr_from_tags(tags), series_id=series_id, series_spec=serialize.mjson(series_spec)))) qs.append( insert( 'mqe.series_def_by_series_spec', dict(report_id=report_id, tags_repr=tags_repr_from_tags(tags), series_spec=serialize.mjson(series_spec), series_id=series_id))) res.append(series_id) c.cass.execute_parallel(qs) return res
def set(self, owner_id=None, dashboard_id=None, old_layout_id=None): """Set a new layout definition for the dashboard (replacing the existing one), using the current content of the :attr:`layout_dict`. The parameters are optional - if not specified, the current values of :attr:`owner_id`, :attr:`dashboard_id` and :attr:`layout_id` are used. :param owner_id: the owner ID of the dashboard :param dashboard_id: the dashboard's ID :param old_layout_id: ``None`` if this should be a new layout definition for the dashboard, ``layout_id`` of the existing layout otherwise :return: a ``layout_id`` of a newly set layout if the operation was successful, ``None`` otherwise (ie. when the passed ``old_layout_id`` didn't match the version in the database) """ owner_id = owner_id or self.owner_id if not owner_id: raise ValueError('owner_id not set in Layout and not passed as an argument') dashboard_id = dashboard_id or self.dashboard_id if not dashboard_id: raise ValueError('dashboard_id not set in Layout and not passed as an argument') old_layout_id = old_layout_id or self.layout_id # a layout def is a layout_dict serialized as a list of items. The list is # sorted by tile creation time (but this assumption should not be generally made). new_layout_def = serialize.mjson(sorted(self.layout_dict.items(), key=lambda (tile_id, vo): tile_id.time)) # Merge old layout_props with new data old_layout_props_row = c.dao.LayoutDAO.select(owner_id, dashboard_id, ['layout_props']) if not old_layout_props_row and old_layout_id: return None if old_layout_props_row and old_layout_props_row['layout_props']: old_layout_props = serialize.json_loads(old_layout_props_row['layout_props']) else: old_layout_props = {'by_tile_id': []} by_tile_id = {} old_by_tile_id = dict(old_layout_props['by_tile_id']) tile_ids_to_fetch = [] for tile_id in self.layout_dict: if tile_id in old_by_tile_id: by_tile_id[tile_id] = old_by_tile_id[tile_id] elif tile_id in self._included_tiles: by_tile_id[tile_id] = self.props_of_tile(self._included_tiles[tile_id]) else: tile_ids_to_fetch.append(tile_id) tile_dict = Tile.select_multi(dashboard_id, tile_ids_to_fetch) for tile_id, tile in tile_dict.items(): by_tile_id[tile.tile_id] = self.props_of_tile(tile) # Compute data for sscreator and tpcreator sscs_data = set() master_data = set() for props in by_tile_id.values(): if props.get('sscs'): #sscs_data.add((props['report_id'], tuple(props['tags']))) sscs_data.add(props['report_id']) if props.get('is_master'): master_data.add(props['report_id']) new_layout_props = serialize.mjson({'by_tile_id': by_tile_id.items()}) # Set the new layout new_layout_id = gen_timeuuid() res = c.dao.LayoutDAO.set(owner_id, dashboard_id, old_layout_id, new_layout_id, new_layout_def, new_layout_props) if not res: log.info('Setting new layout failed') return None # Insert layout_by_report for sscs and tpcreator c.dao.LayoutDAO.insert_layout_by_report_multi(owner_id, sscs_data, [], 'sscs', dashboard_id, new_layout_id) c.dao.LayoutDAO.insert_layout_by_report_multi(owner_id, master_data, [], 'tpcreator', dashboard_id, new_layout_id) self.layout_id = new_layout_id return new_layout_id
def process_input(self, input_string, tags=None, created=None, input_type='any', ip_options={}, force_header=None, extra_ri_data=None, handle_tpcreator=True, handle_sscreator=True): """Process an input string - parse it into a table and create a report instance belonging to the report. :param str|unicode input_string: the input string :param list tags: a list of string tags attached to the report instance :param ~datetime.datetime created: an explicit creation datetime of the report instance ( default: the current datetime) :param str input_type: input type (see :func:`mqetables.parseany.parse_input`) :param dict ip_options: extra parser options (see :func:`mqetables.parsing.InputParser`) :param force_header: a list of header rows indexes to set as a header (defaults to auto-detection) :param extra_ri_data: a custom JSON-serializable document attached to the report instance :param handle_tpcreator: whether to handle TPCreator for the created report instance by calling :func:`~mqe.tpcreator.handle_tpcreator` :param handle_sscreator: whether to handle SSCS by calling :func:`~mqe.sscreator.handle_sscreator` :return: an :class:`InputProcessingResult` """ assert isinstance(input_string, (str, unicode)) # disallow 'created' in the future now = datetime.datetime.utcnow() if created is not None and created.tzinfo: created = util.make_tz_naive(created) if created is not None and created.year < 2000: raise ValueError('created cannot be before the year 2000') if created is not None and created < now: report_instance_id = util.uuid_with_dt(created) custom_created = True else: custom_created = False report_instance_id = gen_timeuuid() created = util.datetime_from_uuid1(report_instance_id) if tags is None: tags = [] parsing_result = parseany.parse_input(input_string, input_type, ip_options) table = mqeconfig.get_table_from_parsing_result(parsing_result) if table is None: return InputProcessingResult(None, parsing_result) if force_header is not None: log.debug('Overwriting header detection due to force_header') table.header_idxs = [ i for i in force_header if util.valid_index(table.num_rows, i) ] table.header_idxs_source = parsing.HEADER_IDXS_SOURCE_USER ri_data_dict = { 'table': table, } result_desc = self._get_result_desc(parsing_result) if result_desc: ri_data_dict['result_desc'] = result_desc report_instance_row = c.dao.ReportInstanceDAO.insert( owner_id=self.owner_id, report_id=self.report_id, report_instance_id=report_instance_id, tags=tags, ri_data=serialize.mjson(ri_data_dict), input_string=parsing_result.input_string, extra_ri_data=serialize.mjson(extra_ri_data) if extra_ri_data else None, custom_created=custom_created) report_instance = ReportInstance(report_instance_row) log.info( 'Created new report instance report_id=%s report_name=%r tags=%s ' 'report_instance_id=%s created=%s', self.report_id, self.report_name, tags, report_instance_id, report_instance.created) if tags and handle_tpcreator: from mqe import tpcreator tpcreator.handle_tpcreator(self.owner_id, self.report_id, report_instance) if handle_sscreator: from mqe import sscreator sscreator.handle_sscreator(self.owner_id, self.report_id, report_instance) if custom_created: from mqe import dataseries dataseries.clear_series_defs(self.report_id, util.powerset(tags)) return InputProcessingResult(report_instance, parsing_result)