def denormalize_unit_statuses(): """ Denormalized UnitStatus records by adding status and symptom fields to the UnitStatus. Also compute end_time and metro_open_time. """ num_units = Unit.objects.no_cache().count() sys.stderr.write("Have %i units\n"%num_units) for i, unit in enumerate(Unit.objects.no_cache()): sys.stderr.write('Processing unit %s\n (%i of %i)'%(unit.unit_id, i, num_units)) unit_statuses = UnitStatus.objects(unit = unit).no_cache().order_by('-time') num_statuses = unit_statuses.count() unit_statuses.select_related() sys.stderr.write('\tHave %i statuses\n'%num_statuses) status_after = None for status in unit_statuses: status.denormalize() if status_after: status.end_time = status_after.time status.compute_metro_open_time() status.save() status_after = status
def fix_all_end_times_and_merge_consecutive(): """ Fix all end_time and metro_open_time. """ num_units = Unit.objects.no_cache().count() sys.stderr.write("Have %i units\n" % num_units) GARBAGE_COLLECT_INTERVAL = 20 for i, unit in enumerate(Unit.objects.no_cache()): INFO('Processing unit %s\n (%i of %i)' % (unit.unit_id, i, num_units)) if i % GARBAGE_COLLECT_INTERVAL == 0: DEBUG("Running garbage collector after iteration over units.") count = gc.collect() DEBUG("Garbage collect returned %i" % count) unit_statuses = [s for s in UnitStatus.objects(unit=unit)] ret = filter_consecutive_statuses(unit_statuses) DEBUG("Have %i statuses for unit %s to delete" % (len(ret.delete), unit.unit_id)) DEBUG("Have %i statuses for unit %s to keep" % (len(ret.keep), unit.unit_id)) for s in ret.delete: DEBUG("Deleting unit status %s" % (s.pk)) s.delete() del ret.delete unit_statuses = ret.keep unit_statuses = sorted(unit_statuses, key=attrgetter('time'), reverse=True) status_after = None for status in unit_statuses: if status_after: assert (status_after.symptom_description != status.symptom_description) status.end_time = status_after.time status.compute_metro_open_time() status.save() status_after = status
def delete_last_n_statuses(n): """Delete the last n statuses""" from dcmetrometrics.common import metroTimes from dcmetrometrics.eles.models import UnitStatus statuses = UnitStatus.objects().order_by('-time').limit(n) units = set(s.unit for s in statuses) print "Deleting %i recent statuses"%len(statuses) for s in statuses: s.delete() for u in units: print "computing key statuses for unit: %s"%u.unit_id u.compute_key_statuses(save=True)
def delete_recent_statuses(time_delta = timedelta(hours = 2)): """Delete statuses from the past two days. Recompute keystatuses for affected units""" from dcmetrometrics.common import metroTimes from dcmetrometrics.eles.models import UnitStatus t0 = metroTimes.utcnow() - time_delta statuses = UnitStatus.objects(time__gt = t0) units = set(s.unit for s in statuses) print "Deleting %i recent statuses"%len(statuses) for s in statuses: s.delete() for u in units: print "computing key statuses for unit: %s"%u.unit_id u.compute_key_statuses(save=True)
def fix_all_end_times_and_merge_consecutive(): """ Fix all end_time and metro_open_time. """ num_units = Unit.objects.no_cache().count() sys.stderr.write("Have %i units\n"%num_units) GARBAGE_COLLECT_INTERVAL = 20 for i, unit in enumerate(Unit.objects.no_cache()): INFO('Processing unit %s\n (%i of %i)'%(unit.unit_id, i, num_units)) if i%GARBAGE_COLLECT_INTERVAL == 0: DEBUG("Running garbage collector after iteration over units.") count = gc.collect() DEBUG("Garbage collect returned %i"%count) unit_statuses = [s for s in UnitStatus.objects(unit = unit)] ret = filter_consecutive_statuses(unit_statuses) DEBUG("Have %i statuses for unit %s to delete"%(len(ret.delete), unit.unit_id)) DEBUG("Have %i statuses for unit %s to keep"%(len(ret.keep), unit.unit_id)) for s in ret.delete: DEBUG("Deleting unit status %s"%(s.pk)) s.delete() del ret.delete unit_statuses = ret.keep unit_statuses = sorted(unit_statuses, key = attrgetter('time'), reverse = True) status_after = None for status in unit_statuses: if status_after: assert(status_after.symptom_description != status.symptom_description) status.end_time = status_after.time status.compute_metro_open_time() status.save() status_after = status