def recompute_performance_summaries():
"""Recompute performance summaries for all units"""
from dcmetrometrics.eles.models import Unit
units = Unit.objects.no_cache()
start = datetime.now()
n = units.count()
GARBAGE_COLLECT_INTERVAL = 10
jwriter = JSONWriter(WWW_DIR)
for i, unit in enumerate(units):
INFO("Computing performance summary for unit %s: %i of %i (%.2f%%)"%(unit.unit_id, i, n, 100.0*i/n))
unit.compute_performance_summary(save = True)
if i%GARBAGE_COLLECT_INTERVAL == 0:
DEBUG("Running garbage collector after iteration over units.")
count = gc.collect()
DEBUG("Garbage collect returned %i"%count)
jwriter.write_unit(unit)
# Write the station directory
jwriter.write_station_directory()
elapsed = (datetime.now() - start).total_seconds()
print "%.2f seconds elapsed"%elapsed
def write_json():
"""Generate all json files.
"""
from dcmetrometrics.common.JSONifier import JSONWriter
import os
jwriter = JSONWriter(WWW_DIR)
# jwriter = JSONWriter(basedir = os.path.join('client', 'app'))
from dcmetrometrics.eles.models import Unit
num_units = Unit.objects.no_cache().count()
for i, unit in enumerate(Unit.objects.no_cache()):
logger.info('Writing unit %i of %i: %s'%(i, num_units, unit.unit_id))
jwriter.write_unit(unit)
# Write the station directory
jwriter.write_station_directory()
# Write the recent updates
jwriter.write_recent_updates()
for report in SystemServiceReport.objects.no_cache():
logger.info("Writing system service report for day %s"%report.day)
jwriter.write_daily_system_service_report(report = report)
elapsed = (datetime.now() - start).total_seconds()
print "%.2f seconds elapsed"%elapsed
def compute_daily_service_reports(start_day = None, end_day = None, force_min_start_day = None):
"""
Compute daily service reports for all units, and write json.
"""
if not start_day:
start_day = date(2013, 6, 1)
if not end_day:
end_day = date.today() # exclusive
assert(end_day > start_day)
num_units = Unit.objects.no_cache().count()
sys.stderr.write("Have %i units\n"%num_units)
GARBAGE_COLLECT_INTERVAL = 20
min_start_day = start_day
for i, unit in enumerate(Unit.objects.no_cache()):
INFO('Computing daily service report unit %s\n (%i of %i)'%(unit.unit_id, i, num_units))
if i%GARBAGE_COLLECT_INTERVAL == 0:
DEBUG("Running garbage collector after iteration over units.")
count = gc.collect()
DEBUG("Garbage collect returned %i"%count)
unit_statuses = sorted(unit.get_statuses(), key = attrgetter('time'), reverse = True)
if not unit_statuses:
continue
last_status = unit_statuses[0]
if start_day is not None:
unit_start_day = min(start_day, getLastOpenTime(last_status.time).date())
else:
unit_start_day = getLastOpenTime(last_status.time).date()
# Override the unit_start_day with force_min_start_day
if force_min_start_day:
unit_start_day = force_min_start_day
unit.compute_daily_service_reports(start_day = unit_start_day, last_day = end_day,
statuses = unit_statuses, save = True)
min_start_day = min(min_start_day, unit_start_day)
jwriter = JSONWriter(WWW_DIR)
for day in gen_days(min_start_day, end_day):
INFO('Computing system service report for day %s'%(day))
report = SystemServiceReport.compute_for_day(day, save = True)
jwriter.write_daily_system_service_report(report = report)
def recompute_performance_summaries():
"""Recompute performance summaries for all units"""
from dcmetrometrics.eles.models import Unit
units = Unit.objects.no_cache()
start = datetime.now()
n = units.count()
GARBAGE_COLLECT_INTERVAL = 10
jwriter = JSONWriter(WWW_DIR)
for i, unit in enumerate(units):
INFO("Computing performance summary for unit %s: %i of %i (%.2f%%)" %
(unit.unit_id, i, n, 100.0 * i / n))
unit.compute_performance_summary(save=True)
if i % GARBAGE_COLLECT_INTERVAL == 0:
DEBUG("Running garbage collector after iteration over units.")
count = gc.collect()
DEBUG("Garbage collect returned %i" % count)
jwriter.write_unit(unit)
# Write the station directory
jwriter.write_station_directory()
elapsed = (datetime.now() - start).total_seconds()
print "%.2f seconds elapsed" % elapsed
def write_json():
jwriter = JSONWriter(WWW_DIR)
jwriter.write_hotcars()
jwriter.write_hotcars_by_day()
def compute_daily_service_reports(start_day = None, end_day = None, force_min_start_day = None):
"""
Compute daily service reports for all units, and write json.
- start_day: The starting day for which to compute. This can be overridden by long running outages.
Computation will be unit dependent, depending on when its current outage goes back to.
- end_day: The last day to compute, exclusive. By default this is today.
- force_min_start_day: Force a minimum starting day for all computation. This overrides start_day or
whatever the current outage suggest to do.
"""
if not start_day:
start_day = date(2013, 6, 1)
if not end_day:
end_day = date.today() # exclusive
assert(end_day > start_day)
num_units = Unit.objects.no_cache().count()
sys.stderr.write("Have %i units\n"%num_units)
GARBAGE_COLLECT_INTERVAL = 20
min_start_day = force_min_start_day if force_min_start_day else start_day
# This long loop was raising Exceptions like:
# pymongo.errors.OperationFailure: cursor id 'XXXX' not valid at server
# Solution is to not timeout the cursor and explicitly close it ourselves
units = Unit.objects.timeout(False).no_cache()
for i, unit in enumerate(units):
INFO('Computing daily service report unit %s\n (%i of %i)'%(unit.unit_id, i, num_units))
if i%GARBAGE_COLLECT_INTERVAL == 0:
DEBUG("Running garbage collector after iteration over units.")
count = gc.collect()
DEBUG("Garbage collect returned %i"%count)
unit_statuses = sorted(unit.get_statuses(), key = attrgetter('time'), reverse = True)
if not unit_statuses:
continue
last_status = unit_statuses[0]
# If the last status is non-operational, consider making updates back to
# the start of the outage.
unit_start_day = start_day
if last_status.symptom_category != "ON":
if start_day is not None:
unit_start_day = min(start_day, getLastOpenTime(last_status.time).date())
else:
unit_start_day = getLastOpenTime(last_status.time).date()
# Override the unit_start_day with force_min_start_day
if force_min_start_day:
unit_start_day = force_min_start_day
unit.compute_daily_service_reports(start_day = unit_start_day, last_day = end_day,
statuses = unit_statuses, save = True)
min_start_day = min(min_start_day, unit_start_day)
if force_min_start_day:
min_start_day = force_min_start_day
# We are done with the units query set, so close the cursor.
units._cursor.close()
jwriter = JSONWriter(WWW_DIR)
for day in gen_days(min_start_day, end_day):
INFO('Computing system service report for day %s'%(day))
report = SystemServiceReport.compute_for_day(day, save = True)
jwriter.write_daily_system_service_report(report = report)
def write_json(): jwriter = JSONWriter(WWW_DIR) jwriter.write_hotcars() jwriter.write_hotcars_by_day()
def compute_daily_service_reports(start_day=None,
end_day=None,
force_min_start_day=None):
"""
Compute daily service reports for all units, and write json.
"""
if not start_day:
start_day = date(2013, 6, 1)
if not end_day:
end_day = date.today() # exclusive
assert (end_day > start_day)
num_units = Unit.objects.no_cache().count()
sys.stderr.write("Have %i units\n" % num_units)
GARBAGE_COLLECT_INTERVAL = 20
min_start_day = start_day
for i, unit in enumerate(Unit.objects.no_cache()):
INFO('Computing daily service report unit %s\n (%i of %i)' %
(unit.unit_id, i, num_units))
if i % GARBAGE_COLLECT_INTERVAL == 0:
DEBUG("Running garbage collector after iteration over units.")
count = gc.collect()
DEBUG("Garbage collect returned %i" % count)
unit_statuses = sorted(unit.get_statuses(),
key=attrgetter('time'),
reverse=True)
if not unit_statuses:
continue
last_status = unit_statuses[0]
if start_day is not None:
unit_start_day = min(start_day,
getLastOpenTime(last_status.time).date())
else:
unit_start_day = getLastOpenTime(last_status.time).date()
# Override the unit_start_day with force_min_start_day
if force_min_start_day:
unit_start_day = force_min_start_day
unit.compute_daily_service_reports(start_day=unit_start_day,
last_day=end_day,
statuses=unit_statuses,
save=True)
min_start_day = min(min_start_day, unit_start_day)
jwriter = JSONWriter(WWW_DIR)
for day in gen_days(min_start_day, end_day):
INFO('Computing system service report for day %s' % (day))
report = SystemServiceReport.compute_for_day(day, save=True)
jwriter.write_daily_system_service_report(report=report)
