def compute_daily_service_reports(start_day = None, end_day = None): """ Compute daily service reports for all units """ if not start_day: start_day = date(2013, 6, 1) if not end_day: end_day = date.today() assert(end_day > start_day) num_units = Unit.objects.no_cache().count() sys.stderr.write("Have %i units\n"%num_units) GARBAGE_COLLECT_INTERVAL = 20 for i, unit in enumerate(Unit.objects.no_cache()): INFO('Computing daily service report unit %s\n (%i of %i)'%(unit.unit_id, i, num_units)) if i%GARBAGE_COLLECT_INTERVAL == 0: DEBUG("Running garbage collector after iteration over units.") count = gc.collect() DEBUG("Garbage collect returned %i"%count) unit_statuses = unit.get_statuses() unit.compute_daily_service_reports(start_day = start_day, last_day = end_day, statuses = unit_statuses, save = True) for day in gen_days(start_day, end_day): SystemServiceReport.compute_for_day(day, save = True)
def compute_daily_service_reports(start_day = None, end_day = None, force_min_start_day = None): """ Compute daily service reports for all units, and write json. """ if not start_day: start_day = date(2013, 6, 1) if not end_day: end_day = date.today() # exclusive assert(end_day > start_day) num_units = Unit.objects.no_cache().count() sys.stderr.write("Have %i units\n"%num_units) GARBAGE_COLLECT_INTERVAL = 20 min_start_day = start_day for i, unit in enumerate(Unit.objects.no_cache()): INFO('Computing daily service report unit %s\n (%i of %i)'%(unit.unit_id, i, num_units)) if i%GARBAGE_COLLECT_INTERVAL == 0: DEBUG("Running garbage collector after iteration over units.") count = gc.collect() DEBUG("Garbage collect returned %i"%count) unit_statuses = sorted(unit.get_statuses(), key = attrgetter('time'), reverse = True) if not unit_statuses: continue last_status = unit_statuses[0] if start_day is not None: unit_start_day = min(start_day, getLastOpenTime(last_status.time).date()) else: unit_start_day = getLastOpenTime(last_status.time).date() # Override the unit_start_day with force_min_start_day if force_min_start_day: unit_start_day = force_min_start_day unit.compute_daily_service_reports(start_day = unit_start_day, last_day = end_day, statuses = unit_statuses, save = True) min_start_day = min(min_start_day, unit_start_day) jwriter = JSONWriter(WWW_DIR) for day in gen_days(min_start_day, end_day): INFO('Computing system service report for day %s'%(day)) report = SystemServiceReport.compute_for_day(day, save = True) jwriter.write_daily_system_service_report(report = report)
def compute_daily_service_reports(start_day = None, end_day = None, force_min_start_day = None): """ Compute daily service reports for all units, and write json. - start_day: The starting day for which to compute. This can be overridden by long running outages. Computation will be unit dependent, depending on when its current outage goes back to. - end_day: The last day to compute, exclusive. By default this is today. - force_min_start_day: Force a minimum starting day for all computation. This overrides start_day or whatever the current outage suggest to do. """ if not start_day: start_day = date(2013, 6, 1) if not end_day: end_day = date.today() # exclusive assert(end_day > start_day) num_units = Unit.objects.no_cache().count() sys.stderr.write("Have %i units\n"%num_units) GARBAGE_COLLECT_INTERVAL = 20 min_start_day = force_min_start_day if force_min_start_day else start_day # This long loop was raising Exceptions like: # pymongo.errors.OperationFailure: cursor id 'XXXX' not valid at server # Solution is to not timeout the cursor and explicitly close it ourselves units = Unit.objects.timeout(False).no_cache() for i, unit in enumerate(units): INFO('Computing daily service report unit %s\n (%i of %i)'%(unit.unit_id, i, num_units)) if i%GARBAGE_COLLECT_INTERVAL == 0: DEBUG("Running garbage collector after iteration over units.") count = gc.collect() DEBUG("Garbage collect returned %i"%count) unit_statuses = sorted(unit.get_statuses(), key = attrgetter('time'), reverse = True) if not unit_statuses: continue last_status = unit_statuses[0] # If the last status is non-operational, consider making updates back to # the start of the outage. unit_start_day = start_day if last_status.symptom_category != "ON": if start_day is not None: unit_start_day = min(start_day, getLastOpenTime(last_status.time).date()) else: unit_start_day = getLastOpenTime(last_status.time).date() # Override the unit_start_day with force_min_start_day if force_min_start_day: unit_start_day = force_min_start_day unit.compute_daily_service_reports(start_day = unit_start_day, last_day = end_day, statuses = unit_statuses, save = True) min_start_day = min(min_start_day, unit_start_day) if force_min_start_day: min_start_day = force_min_start_day # We are done with the units query set, so close the cursor. units._cursor.close() jwriter = JSONWriter(WWW_DIR) for day in gen_days(min_start_day, end_day): INFO('Computing system service report for day %s'%(day)) report = SystemServiceReport.compute_for_day(day, save = True) jwriter.write_daily_system_service_report(report = report)
def compute_daily_service_reports(start_day=None, end_day=None, force_min_start_day=None): """ Compute daily service reports for all units, and write json. """ if not start_day: start_day = date(2013, 6, 1) if not end_day: end_day = date.today() # exclusive assert (end_day > start_day) num_units = Unit.objects.no_cache().count() sys.stderr.write("Have %i units\n" % num_units) GARBAGE_COLLECT_INTERVAL = 20 min_start_day = start_day for i, unit in enumerate(Unit.objects.no_cache()): INFO('Computing daily service report unit %s\n (%i of %i)' % (unit.unit_id, i, num_units)) if i % GARBAGE_COLLECT_INTERVAL == 0: DEBUG("Running garbage collector after iteration over units.") count = gc.collect() DEBUG("Garbage collect returned %i" % count) unit_statuses = sorted(unit.get_statuses(), key=attrgetter('time'), reverse=True) if not unit_statuses: continue last_status = unit_statuses[0] if start_day is not None: unit_start_day = min(start_day, getLastOpenTime(last_status.time).date()) else: unit_start_day = getLastOpenTime(last_status.time).date() # Override the unit_start_day with force_min_start_day if force_min_start_day: unit_start_day = force_min_start_day unit.compute_daily_service_reports(start_day=unit_start_day, last_day=end_day, statuses=unit_statuses, save=True) min_start_day = min(min_start_day, unit_start_day) jwriter = JSONWriter(WWW_DIR) for day in gen_days(min_start_day, end_day): INFO('Computing system service report for day %s' % (day)) report = SystemServiceReport.compute_for_day(day, save=True) jwriter.write_daily_system_service_report(report=report)