def get_series_values_after(series_def, report, after,
limit=mqeconfig.MAX_SERIES_POINTS_IN_TILE,
latest_instance_id=None):
"""Retrieves a list of :class:`SeriesValue` created after the specified report instance ID
(``after``). The function inserts new data series values if they haven't been already created.
:param SeriesDef series_def: a series definition for which to get data
:param ~mqe.report.Report report: a report for which to get data
:param ~uuid.UUID after: a ``report_instance_id`` that specifies the starting point to fetch
(and possibly create) data series values.
:param int limit: the limit of the series values to fetch/create
:param latest_instance_id: (optional) a latest report instance ID of the report and tags.
Passing this parameter ensures multiple calls of the functions for the same report
will return consistent data (ie. coming from the same report instances).
:return: a list of :class:`SeriesValue` objects in the order of creation time of the corresponding report instances
"""
if series_def['from_rid'] is None or series_def['to_rid'] is None:
insert_after = after
elif util.uuid_lt(after, series_def['from_rid']):
insert_after = after
else:
insert_after = series_def['to_rid']
insert_series_values(series_def, report, None, None, after=insert_after)
if latest_instance_id:
max_report_instance_id = util.uuid_for_next_dt(latest_instance_id)
else:
max_report_instance_id = None
rows = c.dao.SeriesValueDAO.select_multi(series_def.series_id, after, max_report_instance_id, limit)
log.debug('Selected %d series_values after series_id=%s report_name=%r',
len(rows) if isinstance(rows, list) else '?',
series_def.series_id, report.report_name)
return list(reversed([SeriesValue(row) for row in rows]))
def get_series_values_after(series_def,
report,
after,
limit=mqeconfig.MAX_SERIES_POINTS_IN_TILE):
"""Retrieves a list of :class:`SeriesValue` created after the specified report instance ID
(``after``). The function inserts new data series values if they haven't been already created.
:param SeriesDef series_def: a series definition for which to get data
:param ~mqe.report.Report report: a report for which to get data
:param ~uuid.UUID after: a ``report_instance_id`` that specifies the starting point to fetch
(and possibly create) data series values.
:param int limit: the limit of the series values to fetch/create
:return: a list of :class:`SeriesValue` objects in the order of creation time of the corresponding report instances
"""
if series_def['from_rid'] is None or series_def['to_rid'] is None:
insert_after = after
elif util.uuid_lt(after, series_def['from_rid']):
insert_after = after
else:
insert_after = series_def['to_rid']
insert_series_values(series_def, report, None, None, after=insert_after)
rows = c.dao.SeriesValueDAO.select_multi(series_def.series_id, after, None,
limit)
log.debug('Selected %d series_values after series_id=%s report_name=%r',
len(rows), series_def.series_id, report.report_name)
return list(reversed([SeriesValue(row) for row in rows]))
def get_series_values(series_def,
report,
from_dt,
to_dt,
limit=mqeconfig.MAX_SERIES_POINTS_IN_TILE,
latest_instance_id=None):
"""Retrieves a list of :class:`SeriesValue` objects for a given time range.
The function inserts new data series values if they haven't been already created
for the requested time period.
:param SeriesDef series_def: a series definition for which to get data
:param ~mqe.report.Report report: a report for which to get data
:param ~datetime.datetime from_dt: starting datetime
:param ~datetime.datetime to_dt: ending datetime
:param int limit: the limit of the series values to fetch/create
:param latest_instance_id: (optional) a latest report instance ID of the report and tags
(if not passed, the value will be fetched)
:return: a list of :class:`SeriesValue` objects in the order of creation time of the corresponding report instances
"""
assert from_dt is not None and to_dt is not None
if series_def.from_dt is None or series_def.to_dt is None:
insert_series_values(series_def, report, from_dt, to_dt, limit=limit)
else:
if from_dt < series_def.from_dt:
insert_series_values(series_def,
report,
from_dt,
prev_dt(series_def.from_dt),
limit=limit)
if not latest_instance_id:
latest_instance_id = report.fetch_latest_instance_id(
series_def.tags)
if latest_instance_id is not None \
and util.uuid_lt(series_def['to_rid'], latest_instance_id) \
and to_dt >= series_def.to_dt:
insert_series_values(series_def,
report,
None,
None,
after=series_def['to_rid'],
limit=limit)
min_report_instance_id = util.uuid_for_prev_dt(util.uuid_with_dt(from_dt))
max_report_instance_id = util.uuid_for_next_dt(util.uuid_with_dt(to_dt))
rows = c.dao.SeriesValueDAO.select_multi(series_def.series_id,
min_report_instance_id,
max_report_instance_id, limit)
log.debug('Selected %d series_values by dates series_id=%s report_name=%r',
len(rows), series_def.series_id, report.report_name)
return list(reversed([SeriesValue(row) for row in rows]))
def fill_new_tile_data(self, data, after_report_instance_id):
ri = self._fetch_ri()
if not ri:
return
if after_report_instance_id and \
(ri.report_instance_id == after_report_instance_id or \
util.uuid_lt(ri.report_instance_id, after_report_instance_id)):
data['series_data'] = [
self._empty_series_dict(i)
for i in xrange(len(self.tile.tile_options['series_configs']))
]
return
self._set_series_data(data, ri)
def insert_series_values(series_def, report, from_dt, to_dt, after=None, limit=None):
assert after or (from_dt is not None and to_dt is not None)
log.debug('insert_series_values report_id=%s sd.from_dt=%s sd.to_dt=%s from_dt=%s'
'to_dt=%s after=%s limit=%s', report.report_id, series_def.from_dt,
series_def.to_dt, from_dt, to_dt, after, limit)
instances_it = report.fetch_instances_iter(after=after,
from_dt=from_dt if not after else None,
to_dt=to_dt if not after else None,
limit=limit or mqeconfig.MAX_SERIES_POINTS,
tags=series_def.tags,
columns=['report_instance_id', 'ri_data'])
info = dict(oldest_rid_fetched=None,
newest_rid_fetched=None,
count=0)
def rows_it():
for ri in instances_it:
if info['oldest_rid_fetched'] is None:
info['oldest_rid_fetched'] = ri.report_instance_id
info['newest_rid_fetched'] = ri.report_instance_id
info['count'] += 1
cell = series_def.series_spec.get_cell(ri)
if cell:
row = dict(report_instance_id=ri.report_instance_id,
json_value=serialize.mjson(cell.value))
header = ri.table.header(cell.colno)
if header:
row['header'] = header
yield row
c.dao.SeriesValueDAO.insert_multi(series_def.series_id, rows_it())
if info['count'] == 0:
return
log.info('Inserted %d series values report_name=%r series_id=%s',
info['count'], report.report_name, series_def.series_id)
# from_rid stores minimal uuid from dt for which we fetched instances,
# while to_rid stores an actual latest report_instance_id in the series.
# However, generally it's not expected to_rid can always be a real report_instance_id
if from_dt is not None:
oldest_rid_stored = util.min_uuid_with_dt(from_dt)
else:
oldest_rid_stored = info['oldest_rid_fetched']
if series_def.from_rid is None or \
util.uuid_lt(oldest_rid_stored, series_def.from_rid):
log.debug('Updating series_def_id=%s from_rid_dt=%s', series_def.series_id,
util.datetime_from_uuid1(oldest_rid_stored))
series_def.update_from_rid(oldest_rid_stored)
if series_def.to_rid is None or \
util.uuid_lt(series_def.to_rid, info['newest_rid_fetched']):
log.debug('Updating series_def_id=%s to_rid_dt=%s', series_def.series_id,
util.datetime_from_uuid1(info['newest_rid_fetched']))
series_def.update_to_rid(info['newest_rid_fetched'])
def insert_series_values(series_def,
report,
from_dt,
to_dt,
after=None,
limit=None):
assert after or (from_dt is not None and to_dt is not None)
log.debug(
'insert_series_values report_id=%s sd.from_dt=%s sd.to_dt=%s from_dt=%s'
'to_dt=%s after=%s limit=%s', report.report_id, series_def.from_dt,
series_def.to_dt, from_dt, to_dt, after, limit)
instances = report.fetch_instances(
after=after,
from_dt=from_dt if not after else None,
to_dt=to_dt if not after else None,
limit=limit or mqeconfig.MAX_SERIES_POINTS,
tags=series_def.tags,
columns=['report_instance_id', 'ri_data'])
if not instances:
return
data = []
for ri in instances:
cell = series_def.series_spec.get_cell(ri)
if cell:
row = dict(report_instance_id=ri.report_instance_id,
json_value=serialize.mjson(cell.value))
header = ri.table.header(cell.colno)
if header:
row['header'] = header
data.append(row)
log.info(
'Inserting %d series values from %d instances report_name=%r series_id=%s',
len(data), len(instances), report.report_name, series_def.series_id)
c.dao.SeriesValueDAO.insert_multi(series_def.series_id, data)
oldest_rid_fetched = instances[0].report_instance_id
newest_rid_fetched = instances[-1].report_instance_id
# from_rid stores minimal uuid from dt for which we fetched instances,
# while to_rid stores an actual latest report_instance_id in the series.
# However, generally it's not expected to_rid can always be a real report_instance_id
if from_dt is not None:
oldest_rid_stored = util.min_uuid_with_dt(from_dt)
else:
oldest_rid_stored = oldest_rid_fetched
if series_def.from_rid is None or \
util.uuid_lt(oldest_rid_stored, series_def.from_rid):
log.debug('Updating series_def_id=%s from_rid_dt=%s',
series_def.series_id,
util.datetime_from_uuid1(oldest_rid_stored))
series_def.update_from_rid(oldest_rid_stored)
if series_def.to_rid is None or \
util.uuid_lt(series_def.to_rid, newest_rid_fetched):
log.debug('Updating series_def_id=%s to_rid_dt=%s',
series_def.series_id,
util.datetime_from_uuid1(newest_rid_fetched))
series_def.update_to_rid(newest_rid_fetched)