def compute_daily_service_reports(start_day = None, end_day = None):
  """
  Compute daily service reports for all units
  """

  if not start_day:
    start_day = date(2013, 6, 1)

  if not end_day:
    end_day = date.today()

  assert(end_day > start_day)

  num_units = Unit.objects.no_cache().count()
  sys.stderr.write("Have %i units\n"%num_units)

  GARBAGE_COLLECT_INTERVAL = 20

  for i, unit in enumerate(Unit.objects.no_cache()):

    INFO('Computing daily service report unit %s\n (%i of %i)'%(unit.unit_id, i, num_units))

    if i%GARBAGE_COLLECT_INTERVAL == 0:
      DEBUG("Running garbage collector after iteration over units.")
      count = gc.collect()
      DEBUG("Garbage collect returned %i"%count)

    unit_statuses = unit.get_statuses()
    unit.compute_daily_service_reports(start_day = start_day, last_day = end_day,
      statuses = unit_statuses, save = True)


  for day in gen_days(start_day, end_day):
    SystemServiceReport.compute_for_day(day, save = True)
def compute_daily_service_reports(start_day = None, end_day = None, force_min_start_day = None):
  """
  Compute daily service reports for all units, and write json.
  """

  if not start_day:
    start_day = date(2013, 6, 1)

  if not end_day:
    end_day = date.today() # exclusive

  assert(end_day > start_day)

  num_units = Unit.objects.no_cache().count()
  sys.stderr.write("Have %i units\n"%num_units)

  GARBAGE_COLLECT_INTERVAL = 20

  min_start_day = start_day

  for i, unit in enumerate(Unit.objects.no_cache()):

    INFO('Computing daily service report unit %s\n (%i of %i)'%(unit.unit_id, i, num_units))

    if i%GARBAGE_COLLECT_INTERVAL == 0:
      DEBUG("Running garbage collector after iteration over units.")
      count = gc.collect()
      DEBUG("Garbage collect returned %i"%count)

    unit_statuses = sorted(unit.get_statuses(), key = attrgetter('time'), reverse = True)

    if not unit_statuses:
      continue

    last_status = unit_statuses[0]

    if start_day is not None:
      unit_start_day = min(start_day, getLastOpenTime(last_status.time).date())
    else:
      unit_start_day = getLastOpenTime(last_status.time).date()

    # Override the unit_start_day with force_min_start_day
    if force_min_start_day:
      unit_start_day = force_min_start_day

    unit.compute_daily_service_reports(start_day = unit_start_day, last_day = end_day,
      statuses = unit_statuses, save = True)

    min_start_day = min(min_start_day, unit_start_day)

  jwriter = JSONWriter(WWW_DIR)
  for day in gen_days(min_start_day, end_day):
    INFO('Computing system service report for day %s'%(day))
    report = SystemServiceReport.compute_for_day(day, save = True)
    jwriter.write_daily_system_service_report(report = report)
def compute_daily_service_reports(start_day = None, end_day = None, force_min_start_day = None):
  """
  Compute daily service reports for all units, and write json.
   - start_day: The starting day for which to compute. This can be overridden by long running outages. 
      Computation will be unit dependent, depending on when its current outage goes back to.
   - end_day: The last day to compute, exclusive. By default this is today.
   - force_min_start_day: Force a minimum starting day for all computation. This overrides start_day or
     whatever the current outage suggest to do.
  """

  if not start_day:
    start_day = date(2013, 6, 1)

  if not end_day:
    end_day = date.today() # exclusive

  assert(end_day > start_day)

  num_units = Unit.objects.no_cache().count()
  sys.stderr.write("Have %i units\n"%num_units)

  GARBAGE_COLLECT_INTERVAL = 20

  min_start_day = force_min_start_day if force_min_start_day else start_day

  # This long loop was raising Exceptions like:
  # pymongo.errors.OperationFailure: cursor id 'XXXX' not valid at server
  # Solution is to not timeout the cursor and explicitly close it ourselves
  units = Unit.objects.timeout(False).no_cache()

  for i, unit in enumerate(units):

    INFO('Computing daily service report unit %s\n (%i of %i)'%(unit.unit_id, i, num_units))

    if i%GARBAGE_COLLECT_INTERVAL == 0:
      DEBUG("Running garbage collector after iteration over units.")
      count = gc.collect()
      DEBUG("Garbage collect returned %i"%count)

    unit_statuses = sorted(unit.get_statuses(), key = attrgetter('time'), reverse = True)

    if not unit_statuses:
      continue

    last_status = unit_statuses[0]

    # If the last status is non-operational, consider making updates back to
    # the start of the outage.
    unit_start_day = start_day
    if last_status.symptom_category != "ON":
      if start_day is not None:
        unit_start_day = min(start_day, getLastOpenTime(last_status.time).date())
      else:
        unit_start_day = getLastOpenTime(last_status.time).date()

    # Override the unit_start_day with force_min_start_day
    if force_min_start_day:
      unit_start_day = force_min_start_day

    unit.compute_daily_service_reports(start_day = unit_start_day, last_day = end_day,
      statuses = unit_statuses, save = True)

    min_start_day = min(min_start_day, unit_start_day)

  if force_min_start_day:
    min_start_day = force_min_start_day

  # We are done with the units query set, so close the cursor.
  units._cursor.close()


  jwriter = JSONWriter(WWW_DIR)
  for day in gen_days(min_start_day, end_day):
    INFO('Computing system service report for day %s'%(day))
    report = SystemServiceReport.compute_for_day(day, save = True)
    jwriter.write_daily_system_service_report(report = report)
Exemplo n.º 4
0
def compute_daily_service_reports(start_day=None,
                                  end_day=None,
                                  force_min_start_day=None):
    """
  Compute daily service reports for all units, and write json.
  """

    if not start_day:
        start_day = date(2013, 6, 1)

    if not end_day:
        end_day = date.today()  # exclusive

    assert (end_day > start_day)

    num_units = Unit.objects.no_cache().count()
    sys.stderr.write("Have %i units\n" % num_units)

    GARBAGE_COLLECT_INTERVAL = 20

    min_start_day = start_day

    for i, unit in enumerate(Unit.objects.no_cache()):

        INFO('Computing daily service report unit %s\n (%i of %i)' %
             (unit.unit_id, i, num_units))

        if i % GARBAGE_COLLECT_INTERVAL == 0:
            DEBUG("Running garbage collector after iteration over units.")
            count = gc.collect()
            DEBUG("Garbage collect returned %i" % count)

        unit_statuses = sorted(unit.get_statuses(),
                               key=attrgetter('time'),
                               reverse=True)

        if not unit_statuses:
            continue

        last_status = unit_statuses[0]

        if start_day is not None:
            unit_start_day = min(start_day,
                                 getLastOpenTime(last_status.time).date())
        else:
            unit_start_day = getLastOpenTime(last_status.time).date()

        # Override the unit_start_day with force_min_start_day
        if force_min_start_day:
            unit_start_day = force_min_start_day

        unit.compute_daily_service_reports(start_day=unit_start_day,
                                           last_day=end_day,
                                           statuses=unit_statuses,
                                           save=True)

        min_start_day = min(min_start_day, unit_start_day)

    jwriter = JSONWriter(WWW_DIR)
    for day in gen_days(min_start_day, end_day):
        INFO('Computing system service report for day %s' % (day))
        report = SystemServiceReport.compute_for_day(day, save=True)
        jwriter.write_daily_system_service_report(report=report)