def existence_timeseries(self, **kwargs):
"""
The utilisation graphs use this data to show an "overview" graph of the
historical trend of system numbers. The user can select date ranges
from this overview. So we need to accept the same system filtering
params as the utilisation_timeseries method, but no date range.
"""
reports_session = create_session(bind=get_reports_engine())
try:
systems = self._systems_for_timeseries(reports_session, **kwargs)
# build a cumulative frequency type of thing
count = 0
cum_freqs = {}
for date_added, in systems.values(System.date_added):
count += 1
cum_freqs[js_datetime(date_added.replace(hour=0, minute=0, second=0, microsecond=0))] = count
cum_freqs[js_datetime(datetime.datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0))] = count
finally:
reports_session.close()
return dict(cum_freqs=sorted(cum_freqs.items()))
def metrics_loop(*args, **kwargs):
# bind thread local session to reports_engine
metrics_session = create_session(bind=get_reports_engine())
session.registry.set(metrics_session)
while running:
start = time.time()
try:
session.begin()
recipe_count_metrics()
system_count_metrics()
dirty_job_metrics()
system_command_metrics()
except Exception:
log.exception('Exception in metrics loop')
finally:
session.close()
end = time.time()
duration = end - start
if duration >= 30.0:
log.debug("Metrics collection took %d seconds", duration)
time.sleep(max(30.0 - duration, 5.0))
def metrics_loop(*args, **kwargs):
# bind thread local session to reports_engine
metrics_session = create_session(bind=get_reports_engine())
session.registry.set(metrics_session)
while running:
start = time.time()
try:
session.begin()
recipe_count_metrics()
system_count_metrics()
dirty_job_metrics()
system_command_metrics()
except Exception:
log.exception('Exception in metrics loop')
finally:
session.close()
end = time.time()
duration = end - start
if duration >= 30.0:
log.debug("Metrics collection took %d seconds", duration)
time.sleep(max(30.0 - duration, 5.0))
def utilisation_timeseries(self, start=None, end=None, resolution=70,
tg_format='json', tg_errors=None, **kwargs):
if tg_errors:
raise cherrypy.HTTPError(status=400, message=repr(tg_errors))
retval = dict(manual=[], recipe=[], idle_automated=[], idle_manual=[],
idle_broken=[], idle_removed=[])
reports_session = create_session(bind=get_reports_engine())
try:
systems = self._systems_for_timeseries(reports_session, **kwargs)
if not start:
start = systems.value(func.min(System.date_added)) or datetime.datetime(2009, 1, 1)
if not end:
end = datetime.datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
dts = list(dt.replace(microsecond=0) for dt in
datetime_range(start, end, step=(end - start) / resolution))
for dt in dts:
reserved_query = systems.join('reservations')\
.filter(and_(
Reservation.start_time <= dt,
or_(Reservation.finish_time >= dt, Reservation.finish_time == None)))\
.group_by(Reservation.type)\
.values(Reservation.type, func.count(System.id))
reserved = dict(reserved_query)
for reservation_type in ['recipe', 'manual']:
retval[reservation_type].append(reserved.get(reservation_type, 0))
idle_query = systems\
.filter(System.date_added <= dt)\
.filter(not_(System.id.in_(select([Reservation.system_id]).where(and_(
Reservation.start_time <= dt,
or_(Reservation.finish_time >= dt, Reservation.finish_time == None))))))\
.join('status_durations')\
.filter(and_(
SystemStatusDuration.start_time <= dt,
or_(SystemStatusDuration.finish_time >= dt, SystemStatusDuration.finish_time == None)))\
.group_by(SystemStatusDuration.status)\
.values(SystemStatusDuration.status, func.count(System.id))
idle = dict(idle_query)
for status in SystemStatus:
retval['idle_%s' % status.value.lower()].append(idle.get(status, 0))
finally:
reports_session.close()
if tg_format == 'json':
cherrypy.response.headers['Content-Type'] = 'application/json'
return jsonify.encode(dict((k, zip((js_datetime(dt) for dt in dts), v))
for k, v in retval.items()))
elif tg_format == 'csv':
cherrypy.response.headers['Content-Type'] = 'text/csv'
stringio = StringIO()
csv_writer = csv.writer(stringio)
# metadata
csv_writer.writerow(['Date generated', str(datetime.datetime.utcnow()
.replace(microsecond=0).isoformat()) + 'Z'])
csv_writer.writerow(['Generated by', identity.current.user])
csv_writer.writerow(['URL', absolute_url('/reports/utilisation_timeseries?' + cherrypy.request.query_string)])
csv_writer.writerow([])
# header
csv_writer.writerow(['timestamp', 'manual', 'recipe', 'idle (automated)',
'idle (manual)', 'idle (broken)'])
# data
csv_writer.writerows(zip(dts, retval['manual'], retval['recipe'],
retval['idle_automated'], retval['idle_manual'], retval['idle_broken']))
return stringio.getvalue()
else:
raise cherrypy.HTTPError(status=400, message='Unrecognised tg_format %r' % tg_format)