def denormalize_unit_statuses():
"""
Denormalized UnitStatus records by adding status and
symptom fields to the UnitStatus.
Also compute end_time and metro_open_time.
"""
num_units = Unit.objects.no_cache().count()
sys.stderr.write("Have %i units\n"%num_units)
for i, unit in enumerate(Unit.objects.no_cache()):
sys.stderr.write('Processing unit %s\n (%i of %i)'%(unit.unit_id, i, num_units))
unit_statuses = UnitStatus.objects(unit = unit).no_cache().order_by('-time')
num_statuses = unit_statuses.count()
unit_statuses.select_related()
sys.stderr.write('\tHave %i statuses\n'%num_statuses)
status_after = None
for status in unit_statuses:
status.denormalize()
if status_after:
status.end_time = status_after.time
status.compute_metro_open_time()
status.save()
status_after = status
def fix_all_end_times_and_merge_consecutive():
"""
Fix all end_time and metro_open_time.
"""
num_units = Unit.objects.no_cache().count()
sys.stderr.write("Have %i units\n" % num_units)
GARBAGE_COLLECT_INTERVAL = 20
for i, unit in enumerate(Unit.objects.no_cache()):
INFO('Processing unit %s\n (%i of %i)' % (unit.unit_id, i, num_units))
if i % GARBAGE_COLLECT_INTERVAL == 0:
DEBUG("Running garbage collector after iteration over units.")
count = gc.collect()
DEBUG("Garbage collect returned %i" % count)
unit_statuses = [s for s in UnitStatus.objects(unit=unit)]
ret = filter_consecutive_statuses(unit_statuses)
DEBUG("Have %i statuses for unit %s to delete" %
(len(ret.delete), unit.unit_id))
DEBUG("Have %i statuses for unit %s to keep" %
(len(ret.keep), unit.unit_id))
for s in ret.delete:
DEBUG("Deleting unit status %s" % (s.pk))
s.delete()
del ret.delete
unit_statuses = ret.keep
unit_statuses = sorted(unit_statuses,
key=attrgetter('time'),
reverse=True)
status_after = None
for status in unit_statuses:
if status_after:
assert (status_after.symptom_description !=
status.symptom_description)
status.end_time = status_after.time
status.compute_metro_open_time()
status.save()
status_after = status
def delete_last_n_statuses(n):
"""Delete the last n statuses"""
from dcmetrometrics.common import metroTimes
from dcmetrometrics.eles.models import UnitStatus
statuses = UnitStatus.objects().order_by('-time').limit(n)
units = set(s.unit for s in statuses)
print "Deleting %i recent statuses"%len(statuses)
for s in statuses:
s.delete()
for u in units:
print "computing key statuses for unit: %s"%u.unit_id
u.compute_key_statuses(save=True)
def delete_recent_statuses(time_delta = timedelta(hours = 2)):
"""Delete statuses from the past two days.
Recompute keystatuses for affected units"""
from dcmetrometrics.common import metroTimes
from dcmetrometrics.eles.models import UnitStatus
t0 = metroTimes.utcnow() - time_delta
statuses = UnitStatus.objects(time__gt = t0)
units = set(s.unit for s in statuses)
print "Deleting %i recent statuses"%len(statuses)
for s in statuses:
s.delete()
for u in units:
print "computing key statuses for unit: %s"%u.unit_id
u.compute_key_statuses(save=True)
def fix_all_end_times_and_merge_consecutive():
"""
Fix all end_time and metro_open_time.
"""
num_units = Unit.objects.no_cache().count()
sys.stderr.write("Have %i units\n"%num_units)
GARBAGE_COLLECT_INTERVAL = 20
for i, unit in enumerate(Unit.objects.no_cache()):
INFO('Processing unit %s\n (%i of %i)'%(unit.unit_id, i, num_units))
if i%GARBAGE_COLLECT_INTERVAL == 0:
DEBUG("Running garbage collector after iteration over units.")
count = gc.collect()
DEBUG("Garbage collect returned %i"%count)
unit_statuses = [s for s in UnitStatus.objects(unit = unit)]
ret = filter_consecutive_statuses(unit_statuses)
DEBUG("Have %i statuses for unit %s to delete"%(len(ret.delete), unit.unit_id))
DEBUG("Have %i statuses for unit %s to keep"%(len(ret.keep), unit.unit_id))
for s in ret.delete:
DEBUG("Deleting unit status %s"%(s.pk))
s.delete()
del ret.delete
unit_statuses = ret.keep
unit_statuses = sorted(unit_statuses, key = attrgetter('time'), reverse = True)
status_after = None
for status in unit_statuses:
if status_after:
assert(status_after.symptom_description != status.symptom_description)
status.end_time = status_after.time
status.compute_metro_open_time()
status.save()
status_after = status
