def test_process_raw_data_old_timestamp(self):
deployment = self.mox.CreateMockAnything()
when = '2013-1-25T13:38:23.123'
dict = {
'_context_timestamp': when,
}
args = ('monitor.info', dict)
json_args = json.dumps(args)
old_info_handler = views.HANDLERS['monitor.info']
views.HANDLERS['monitor.info'] = lambda key, mess: {'host': 'api'}
raw_values = {
'deployment': deployment,
'when': utils.decimal_utc(datetime.datetime.strptime(when, "%Y-%m-%dT%H:%M:%S.%f")),
'host': 'api',
'routing_key': 'monitor.info',
'json': json_args
}
raw = self.mox.CreateMockAnything()
views.STACKDB.create_rawdata(**raw_values).AndReturn(raw)
views.STACKDB.save(raw)
self.mox.StubOutWithMock(views, "aggregate_lifecycle")
views.aggregate_lifecycle(raw)
self.mox.StubOutWithMock(views, "aggregate_usage")
views.aggregate_usage(raw, dict)
self.mox.ReplayAll()
views.process_raw_data(deployment, args, json_args)
self.mox.VerifyAll()
views.HANDLERS['monitor.info'] = old_info_handler
def add_past_usage(raws):
count = raws.count()
processed = 0
print "%s events to be processed" % count
last_update = datetime.datetime.utcnow()
for raw in raws:
json_dict = json.loads(raw.json)
views.aggregate_usage(raw, json_dict[1])
processed += 1
if processed % 50 == 0:
next_update = last_update + datetime.timedelta(seconds=30)
if datetime.datetime.utcnow() > next_update:
m = (processed, count - processed, (float(processed) / count) * 100)
print "%s processed, %s to go, %.2f percent done" % m
last_update = datetime.datetime.utcnow()
print "completed processing %s events" % count
def add_past_usage(raws):
count = raws.count()
processed = 0
print "%s events to be processed" % count
last_update = datetime.datetime.utcnow()
for raw in raws:
json_dict = json.loads(raw.json)
views.aggregate_usage(raw, json_dict[1])
processed += 1
if processed % 50 == 0:
next_update = last_update + datetime.timedelta(seconds=30)
if datetime.datetime.utcnow() > next_update:
m = (processed, count - processed,
(float(processed) / count)*100)
print "%s processed, %s to go, %.2f percent done" % m
last_update = datetime.datetime.utcnow()
print "completed processing %s events" % count
def populate_usage(raw, body):
if not usage_already_exists(raw):
views.aggregate_usage(raw, body)
def populate_usage(raw, body):
if not usage_already_exists(raw):
views.aggregate_usage(raw, body)