def compute_daily_service_reports(start_day = None, end_day = None):
"""
Compute daily service reports for all units
"""
if not start_day:
start_day = date(2013, 6, 1)
if not end_day:
end_day = date.today()
assert(end_day > start_day)
num_units = Unit.objects.no_cache().count()
sys.stderr.write("Have %i units\n"%num_units)
GARBAGE_COLLECT_INTERVAL = 20
for i, unit in enumerate(Unit.objects.no_cache()):
INFO('Computing daily service report unit %s\n (%i of %i)'%(unit.unit_id, i, num_units))
if i%GARBAGE_COLLECT_INTERVAL == 0:
DEBUG("Running garbage collector after iteration over units.")
count = gc.collect()
DEBUG("Garbage collect returned %i"%count)
unit_statuses = unit.get_statuses()
unit.compute_daily_service_reports(start_day = start_day, last_day = end_day,
statuses = unit_statuses, save = True)
for day in gen_days(start_day, end_day):
SystemServiceReport.compute_for_day(day, save = True)
def compute_daily_service_reports(start_day = None, end_day = None, force_min_start_day = None):
"""
Compute daily service reports for all units, and write json.
"""
if not start_day:
start_day = date(2013, 6, 1)
if not end_day:
end_day = date.today() # exclusive
assert(end_day > start_day)
num_units = Unit.objects.no_cache().count()
sys.stderr.write("Have %i units\n"%num_units)
GARBAGE_COLLECT_INTERVAL = 20
min_start_day = start_day
for i, unit in enumerate(Unit.objects.no_cache()):
INFO('Computing daily service report unit %s\n (%i of %i)'%(unit.unit_id, i, num_units))
if i%GARBAGE_COLLECT_INTERVAL == 0:
DEBUG("Running garbage collector after iteration over units.")
count = gc.collect()
DEBUG("Garbage collect returned %i"%count)
unit_statuses = sorted(unit.get_statuses(), key = attrgetter('time'), reverse = True)
if not unit_statuses:
continue
last_status = unit_statuses[0]
if start_day is not None:
unit_start_day = min(start_day, getLastOpenTime(last_status.time).date())
else:
unit_start_day = getLastOpenTime(last_status.time).date()
# Override the unit_start_day with force_min_start_day
if force_min_start_day:
unit_start_day = force_min_start_day
unit.compute_daily_service_reports(start_day = unit_start_day, last_day = end_day,
statuses = unit_statuses, save = True)
min_start_day = min(min_start_day, unit_start_day)
jwriter = JSONWriter(WWW_DIR)
for day in gen_days(min_start_day, end_day):
INFO('Computing system service report for day %s'%(day))
report = SystemServiceReport.compute_for_day(day, save = True)
jwriter.write_daily_system_service_report(report = report)
def compute_daily_service_reports(start_day = None, end_day = None, force_min_start_day = None):
"""
Compute daily service reports for all units, and write json.
- start_day: The starting day for which to compute. This can be overridden by long running outages.
Computation will be unit dependent, depending on when its current outage goes back to.
- end_day: The last day to compute, exclusive. By default this is today.
- force_min_start_day: Force a minimum starting day for all computation. This overrides start_day or
whatever the current outage suggest to do.
"""
if not start_day:
start_day = date(2013, 6, 1)
if not end_day:
end_day = date.today() # exclusive
assert(end_day > start_day)
num_units = Unit.objects.no_cache().count()
sys.stderr.write("Have %i units\n"%num_units)
GARBAGE_COLLECT_INTERVAL = 20
min_start_day = force_min_start_day if force_min_start_day else start_day
# This long loop was raising Exceptions like:
# pymongo.errors.OperationFailure: cursor id 'XXXX' not valid at server
# Solution is to not timeout the cursor and explicitly close it ourselves
units = Unit.objects.timeout(False).no_cache()
for i, unit in enumerate(units):
INFO('Computing daily service report unit %s\n (%i of %i)'%(unit.unit_id, i, num_units))
if i%GARBAGE_COLLECT_INTERVAL == 0:
DEBUG("Running garbage collector after iteration over units.")
count = gc.collect()
DEBUG("Garbage collect returned %i"%count)
unit_statuses = sorted(unit.get_statuses(), key = attrgetter('time'), reverse = True)
if not unit_statuses:
continue
last_status = unit_statuses[0]
# If the last status is non-operational, consider making updates back to
# the start of the outage.
unit_start_day = start_day
if last_status.symptom_category != "ON":
if start_day is not None:
unit_start_day = min(start_day, getLastOpenTime(last_status.time).date())
else:
unit_start_day = getLastOpenTime(last_status.time).date()
# Override the unit_start_day with force_min_start_day
if force_min_start_day:
unit_start_day = force_min_start_day
unit.compute_daily_service_reports(start_day = unit_start_day, last_day = end_day,
statuses = unit_statuses, save = True)
min_start_day = min(min_start_day, unit_start_day)
if force_min_start_day:
min_start_day = force_min_start_day
# We are done with the units query set, so close the cursor.
units._cursor.close()
jwriter = JSONWriter(WWW_DIR)
for day in gen_days(min_start_day, end_day):
INFO('Computing system service report for day %s'%(day))
report = SystemServiceReport.compute_for_day(day, save = True)
jwriter.write_daily_system_service_report(report = report)
def compute_daily_service_reports(start_day=None,
end_day=None,
force_min_start_day=None):
"""
Compute daily service reports for all units, and write json.
"""
if not start_day:
start_day = date(2013, 6, 1)
if not end_day:
end_day = date.today() # exclusive
assert (end_day > start_day)
num_units = Unit.objects.no_cache().count()
sys.stderr.write("Have %i units\n" % num_units)
GARBAGE_COLLECT_INTERVAL = 20
min_start_day = start_day
for i, unit in enumerate(Unit.objects.no_cache()):
INFO('Computing daily service report unit %s\n (%i of %i)' %
(unit.unit_id, i, num_units))
if i % GARBAGE_COLLECT_INTERVAL == 0:
DEBUG("Running garbage collector after iteration over units.")
count = gc.collect()
DEBUG("Garbage collect returned %i" % count)
unit_statuses = sorted(unit.get_statuses(),
key=attrgetter('time'),
reverse=True)
if not unit_statuses:
continue
last_status = unit_statuses[0]
if start_day is not None:
unit_start_day = min(start_day,
getLastOpenTime(last_status.time).date())
else:
unit_start_day = getLastOpenTime(last_status.time).date()
# Override the unit_start_day with force_min_start_day
if force_min_start_day:
unit_start_day = force_min_start_day
unit.compute_daily_service_reports(start_day=unit_start_day,
last_day=end_day,
statuses=unit_statuses,
save=True)
min_start_day = min(min_start_day, unit_start_day)
jwriter = JSONWriter(WWW_DIR)
for day in gen_days(min_start_day, end_day):
INFO('Computing system service report for day %s' % (day))
report = SystemServiceReport.compute_for_day(day, save=True)
jwriter.write_daily_system_service_report(report=report)