def _verify_for_delete(exist, delete=None, delete_type="InstanceDeletes"):
if not delete and exist.delete:
# We know we have a delete and we have it's id
delete = exist.delete
elif not delete:
if exist.deleted_at:
# We received this exists before the delete, go find it
deletes = models.InstanceDeletes.find(exist.instance, dt.dt_from_decimal(exist.launched_at))
if deletes.count() == 1:
delete = deletes[0]
else:
query = {"instance": exist.instance, "launched_at": exist.launched_at}
raise NotFound(delete_type, query)
else:
# We don't know if this is supposed to have a delete or not.
# Thus, we need to check if we have a delete for this instance.
# We need to be careful though, since we could be verifying an
# exist event that we got before the delete. So, we restrict the
# search to only deletes before this exist's audit period ended.
# If we find any, we fail validation
launched_at = dt.dt_from_decimal(exist.launched_at)
deleted_at_max = dt.dt_from_decimal(exist.audit_period_ending)
deletes = models.InstanceDeletes.find(exist.instance, launched_at, deleted_at_max)
if deletes.count() > 0:
reason = "Found %s for non-delete exist" % delete_type
raise VerificationException(reason)
if delete:
if not base_verifier._verify_date_field(delete.launched_at, exist.launched_at, same_second=True):
raise FieldMismatch("launched_at", exist.launched_at, delete.launched_at, exist.instance)
if not base_verifier._verify_date_field(delete.deleted_at, exist.deleted_at, same_second=True):
raise FieldMismatch("deleted_at", exist.deleted_at, delete.deleted_at, exist.instance)
def test_do_list_usage_exists_with_instance(self):
fake_request = self.mox.CreateMockAnything()
fake_request.GET = {"instance": INSTANCE_ID_1}
results = self.mox.CreateMockAnything()
models.InstanceExists.objects.filter(instance=INSTANCE_ID_1).AndReturn(results)
usage = self.mox.CreateMockAnything()
usage.instance = INSTANCE_ID_1
usage.launched_at = utils.decimal_utc()
usage.deleted_at = usage.launched_at + 10
usage.instance_type_id = 1
usage.message_id = "someid"
usage.status = "pending"
results.__iter__().AndReturn([usage].__iter__())
self.mox.ReplayAll()
resp = stacky_server.do_list_usage_exists(fake_request)
self.assertEqual(resp.status_code, 200)
resp_json = json.loads(resp.content)
self.assertEqual(len(resp_json), 2)
self.assertEqual(
resp_json[0], ["UUID", "Launched At", "Deleted At", "Instance Type Id", "Message ID", "Status"]
)
self.assertEqual(resp_json[1][0], INSTANCE_ID_1)
launch_time_str = dt.dt_from_decimal(usage.launched_at)
self.assertEqual(resp_json[1][1], str(launch_time_str))
delete_time_str = dt.dt_from_decimal(usage.deleted_at)
self.assertEqual(resp_json[1][2], str(delete_time_str))
self.mox.VerifyAll()
def test_do_list_usage_exists_with_instance(self):
fake_request = self.mox.CreateMockAnything()
fake_request.GET = {'instance': INSTANCE_ID_1}
results = self.mox.CreateMockAnything()
models.InstanceExists.objects.filter(instance=INSTANCE_ID_1)\
.AndReturn(results)
usage = self.mox.CreateMockAnything()
usage.instance = INSTANCE_ID_1
usage.launched_at = utils.decimal_utc()
usage.deleted_at = usage.launched_at + 10
usage.instance_type_id = 1
usage.message_id = 'someid'
usage.status = 'pending'
results.__iter__().AndReturn([usage].__iter__())
self.mox.ReplayAll()
resp = stacky_server.do_list_usage_exists(fake_request)
self.assertEqual(resp.status_code, 200)
resp_json = json.loads(resp.content)
self.assertEqual(len(resp_json), 2)
self.assertEqual(resp_json[0], [
"UUID", "Launched At", "Deleted At", "Instance Type Id",
"Message ID", "Status"
])
self.assertEqual(resp_json[1][0], INSTANCE_ID_1)
launch_time_str = dt.dt_from_decimal(usage.launched_at)
self.assertEqual(resp_json[1][1], str(launch_time_str))
delete_time_str = dt.dt_from_decimal(usage.deleted_at)
self.assertEqual(resp_json[1][2], str(delete_time_str))
self.mox.VerifyAll()
def _audit_launches_to_exists(launches, exists, beginning):
fails = []
for (instance, launches) in launches.items():
if instance in exists:
for expected in launches:
found = False
for actual in exists[instance]:
if int(expected['launched_at']) == \
int(actual['launched_at']):
# HACK (apmelton): Truncate the decimal because we may not
# have the milliseconds.
found = True
if not found:
rec = False
if reconciler:
args = (expected['id'], beginning)
rec = reconciler.missing_exists_for_instance(*args)
launched_at = dt.dt_from_decimal(expected['launched_at'])
msg = "Couldn't find exists for launch (%s, %s)"
msg = msg % (instance, launched_at)
cell, compute = cell_and_compute(instance, launched_at)
fails.append(['Launch', expected['id'], msg,
'Y' if rec else 'N', cell, compute])
else:
rec = False
if reconciler:
args = (launches[0]['id'], beginning)
rec = reconciler.missing_exists_for_instance(*args)
msg = "No exists for instance (%s)" % instance
launched_at = dt.dt_from_decimal(launches[0]['launched_at'])
cell, compute = cell_and_compute(instance, launched_at)
fails.append(['-', msg, 'Y' if rec else 'N',
cell, compute])
return fails
def _verify_for_delete(exist, delete=None, delete_type="InstanceDeletes"):
if not delete and exist.delete:
# We know we have a delete and we have it's id
delete = exist.delete
elif not delete:
if exist.deleted_at:
# We received this exists before the delete, go find it
deletes = models.InstanceDeletes.find(
exist.instance, dt.dt_from_decimal(exist.launched_at))
if deletes.count() == 1:
delete = deletes[0]
else:
query = {
'instance': exist.instance,
'launched_at': exist.launched_at
}
raise NotFound(delete_type, query)
else:
# We don't know if this is supposed to have a delete or not.
# Thus, we need to check if we have a delete for this instance.
# We need to be careful though, since we could be verifying an
# exist event that we got before the delete. So, we restrict the
# search to only deletes before this exist's audit period ended.
# If we find any, we fail validation
launched_at = dt.dt_from_decimal(exist.launched_at)
deleted_at_max = dt.dt_from_decimal(exist.audit_period_ending)
deletes = models.InstanceDeletes.find(exist.instance, launched_at,
deleted_at_max)
if deletes.count() > 0:
reason = 'Found %s for non-delete exist' % delete_type
raise VerificationException(reason)
if delete:
if not base_verifier._verify_date_field(
delete.launched_at, exist.launched_at, same_second=True):
raise FieldMismatch('launched_at', {
'name': 'exists',
'value': exist.launched_at
}, {
'name': 'deletes',
'value': delete.launched_at
}, exist.instance)
if not base_verifier._verify_date_field(
delete.deleted_at, exist.deleted_at, same_second=True):
raise FieldMismatch('deleted_at', {
'name': 'exists',
'value': exist.deleted_at
}, {
'name': 'deletes',
'value': delete.deleted_at
}, exist.instance)
def test_do_list_usage_launches_with_instance(self):
fake_request = self.mox.CreateMockAnything()
fake_request.GET = {'instance': INSTANCE_ID_1}
results = self.mox.CreateMockAnything()
models.InstanceUsage.objects.filter(instance=INSTANCE_ID_1)\
.AndReturn(results)
usage = self.mox.CreateMockAnything()
usage.instance = INSTANCE_ID_1
usage.launched_at = utils.decimal_utc()
usage.instance_type_id = 1
results.__iter__().AndReturn([usage].__iter__())
self.mox.ReplayAll()
resp = stacky_server.do_list_usage_launches(fake_request)
self.assertEqual(resp.status_code, 200)
resp_json = json.loads(resp.content)
self.assertEqual(len(resp_json), 2)
self.assertEqual(resp_json[0], ["UUID", "Launched At",
"Instance Type Id"])
self.assertEqual(resp_json[1][0], INSTANCE_ID_1)
time_str = dt.dt_from_decimal(usage.launched_at)
self.assertEqual(resp_json[1][1], str(time_str))
self.assertEqual(resp_json[1][2], 1)
self.mox.VerifyAll()
def test_verify_for_launch_late_ambiguous_usage(self):
exist = self.mox.CreateMockAnything()
exist.usage = None
exist.instance = INSTANCE_ID_1
launched_at = decimal.Decimal('1.1')
exist.launched_at = launched_at
exist.instance_type_id = 2
results = self.mox.CreateMockAnything()
models.InstanceUsage.objects.filter(
instance=INSTANCE_ID_1).AndReturn(results)
results.count().AndReturn(1)
models.InstanceUsage.find(
INSTANCE_ID_1, dt.dt_from_decimal(launched_at)).AndReturn(results)
results.count().AndReturn(2)
self.mox.ReplayAll()
with self.assertRaises(AmbiguousResults) as ar:
nova_verifier._verify_for_launch(exist)
exception = ar.exception
self.assertEqual(exception.object_type, 'InstanceUsage')
search_params = {'instance': INSTANCE_ID_1,
'launched_at': decimal.Decimal('1.1')}
self.assertEqual(exception.search_params, search_params)
self.mox.VerifyAll()
def _post_process_raw_data(rows, highlight=None):
for row in rows:
if "error" in row.routing_key:
row.is_error = True
if highlight and row.id == int(highlight):
row.highlight = True
row.fwhen = dt.dt_from_decimal(row.when)
def test_do_request(self):
fake_request = self.mox.CreateMockAnything()
fake_request.GET = {"request_id": REQUEST_ID_1}
raw = self._create_raw()
results = self.mox.CreateMockAnything()
models.RawData.objects.filter(request_id=REQUEST_ID_1).AndReturn(results)
results.order_by("when").AndReturn(results)
results.__iter__().AndReturn([raw].__iter__())
self.mox.ReplayAll()
resp = stacky_server.do_request(fake_request)
self.assertEqual(resp.status_code, 200)
json_resp = json.loads(resp.content)
self.assertEqual(len(json_resp), 2)
self.assertEqual(json_resp[0], ["#", "?", "When", "Deployment", "Event", "Host", "State", "State'", "Task'"])
self.assertEqual(json_resp[1][0], 1)
self.assertEqual(json_resp[1][1], u" ")
self.assertEqual(json_resp[1][2], str(dt.dt_from_decimal(raw.when)))
self.assertEqual(json_resp[1][3], u"deployment")
self.assertEqual(json_resp[1][4], u"test.start")
self.assertEqual(json_resp[1][5], u"example.com")
self.assertEqual(json_resp[1][6], u"active")
self.assertEqual(json_resp[1][7], None)
self.assertEqual(json_resp[1][8], None)
self.mox.VerifyAll()
def test_do_uuid(self):
fake_request = self.mox.CreateMockAnything()
fake_request.GET = {'uuid': INSTANCE_ID_1}
result = self.mox.CreateMockAnything()
models.RawData.objects.select_related().AndReturn(result)
result.filter(instance=INSTANCE_ID_1).AndReturn(result)
result.order_by('when').AndReturn(result)
raw = self._create_raw()
result.__iter__().AndReturn([raw].__iter__())
self.mox.ReplayAll()
resp = stacky_server.do_uuid(fake_request)
self.assertEqual(resp.status_code, 200)
json_resp = json.loads(resp.content)
self.assertEqual(len(json_resp), 2)
header = [
"#", "?", "When", "Deployment", "Event", "Host", "State", "State'",
"Task'"
]
self.assertEqual(json_resp[0], header)
datetime = dt.dt_from_decimal(raw.when)
body = [
1, " ",
str(datetime), "deployment", "test.start", "example.com", "active",
None, None
]
self.assertEqual(json_resp[1], body)
self.mox.VerifyAll()
def _verify_with_reconciled_data(exist):
if not exist.launched_at:
raise VerificationException("Exists without a launched_at")
query = models.InstanceReconcile.objects.filter(instance=exist.instance)
if query.count() > 0:
recs = models.InstanceReconcile.find(exist.instance,
dt.dt_from_decimal((
exist.launched_at)))
search_query = {'instance': exist.instance,
'launched_at': exist.launched_at}
count = recs.count()
if count > 1:
raise AmbiguousResults('InstanceReconcile', search_query)
elif count == 0:
raise NotFound('InstanceReconcile', search_query)
reconcile = recs[0]
else:
raise NotFound('InstanceReconcile', {'instance': exist.instance})
_verify_for_launch(exist, launch=reconcile,
launch_type="InstanceReconcile")
delete = None
if reconcile.deleted_at is not None:
delete = reconcile
_verify_for_delete(exist, delete=delete, delete_type="InstanceReconcile")
def test_verify_with_reconciled_data_deleted(self):
exists = self.mox.CreateMockAnything()
exists.instance = INSTANCE_ID_1
launched_at = decimal.Decimal('1.1')
deleted_at = decimal.Decimal('2.1')
exists.launched_at = launched_at
exists.deleted_at = deleted_at
results = self.mox.CreateMockAnything()
models.InstanceReconcile.objects.filter(instance=INSTANCE_ID_1)\
.AndReturn(results)
results.count().AndReturn(1)
launched_at = dt.dt_from_decimal(decimal.Decimal('1.1'))
recs = self.mox.CreateMockAnything()
models.InstanceReconcile.find(INSTANCE_ID_1, launched_at).AndReturn(recs)
recs.count().AndReturn(1)
reconcile = self.mox.CreateMockAnything()
reconcile.deleted_at = deleted_at
recs[0].AndReturn(reconcile)
self.mox.StubOutWithMock(nova_verifier, '_verify_for_launch')
nova_verifier._verify_for_launch(exists, launch=reconcile,
launch_type='InstanceReconcile')
self.mox.StubOutWithMock(nova_verifier, '_verify_for_delete')
nova_verifier._verify_for_delete(exists, delete=reconcile,
delete_type='InstanceReconcile')
self.mox.ReplayAll()
nova_verifier._verify_with_reconciled_data(exists)
self.mox.VerifyAll()
def test_do_request(self):
fake_request = self.mox.CreateMockAnything()
fake_request.GET = {'request_id': REQUEST_ID_1}
raw = self._create_raw()
results = self.mox.CreateMockAnything()
models.RawData.objects.filter(
request_id=REQUEST_ID_1).AndReturn(results)
results.order_by('when').AndReturn(results)
results.__iter__().AndReturn([raw].__iter__())
self.mox.ReplayAll()
resp = stacky_server.do_request(fake_request)
self.assertEqual(resp.status_code, 200)
json_resp = json.loads(resp.content)
self.assertEqual(len(json_resp), 2)
self.assertEqual(json_resp[0], [
"#", "?", "When", "Deployment", "Event", "Host", "State", "State'",
"Task'"
])
self.assertEqual(json_resp[1][0], 1)
self.assertEqual(json_resp[1][1], u' ')
self.assertEqual(json_resp[1][2], str(dt.dt_from_decimal(raw.when)))
self.assertEqual(json_resp[1][3], u'deployment')
self.assertEqual(json_resp[1][4], u'test.start')
self.assertEqual(json_resp[1][5], u'example.com')
self.assertEqual(json_resp[1][6], u'active')
self.assertEqual(json_resp[1][7], None)
self.assertEqual(json_resp[1][8], None)
self.mox.VerifyAll()
def _verify_with_reconciled_data(exist):
if not exist.launched_at:
raise VerificationException("Exists without a launched_at")
query = models.InstanceReconcile.objects.filter(instance=exist.instance)
if query.count() > 0:
recs = models.InstanceReconcile.find(
exist.instance, dt.dt_from_decimal((exist.launched_at)))
search_query = {
'instance': exist.instance,
'launched_at': exist.launched_at
}
count = recs.count()
if count > 1:
raise AmbiguousResults('InstanceReconcile', search_query)
elif count == 0:
raise NotFound('InstanceReconcile', search_query)
reconcile = recs[0]
else:
raise NotFound('InstanceReconcile', {'instance': exist.instance})
_verify_for_launch(exist,
launch=reconcile,
launch_type="InstanceReconcile")
delete = None
if reconcile.deleted_at is not None:
delete = reconcile
_verify_for_delete(exist, delete=delete, delete_type="InstanceReconcile")
def _post_process_raw_data(rows, highlight=None):
for row in rows:
if "error" in row.routing_key:
row.is_error = True
if highlight and row.id == int(highlight):
row.highlight = True
row.fwhen = dt.dt_from_decimal(row.when)
def _verify_for_launch(exist):
if exist.usage:
launch = exist.usage
else:
if models.InstanceUsage.objects\
.filter(instance=exist.instance).count() > 0:
launches = _find_launch(exist.instance,
dt.dt_from_decimal(exist.launched_at))
count = launches.count()
query = {
'instance': exist.instance,
'launched_at': exist.launched_at
}
if count > 1:
raise AmbiguousResults('InstanceUsage', query)
elif count == 0:
raise NotFound('InstanceUsage', query)
launch = launches[0]
else:
raise NotFound('InstanceUsage', {'instance': exist.instance})
if not _verify_date_field(launch.launched_at, exist.launched_at,
same_second=True):
raise FieldMismatch('launched_at', exist.launched_at,
launch.launched_at)
if launch.instance_type_id != exist.instance_type_id:
raise FieldMismatch('instance_type_id', exist.instance_type_id,
launch.instance_type_id)
def _convert_model(model, extra_values_func=None):
model_dict = model_to_dict(model)
for key in model_dict:
if isinstance(model_dict[key], decimal.Decimal):
model_dict[key] = str(dt.dt_from_decimal(model_dict[key]))
if extra_values_func:
model_dict.update(extra_values_func(model))
return model_dict
def _convert_model(model, extra_values_func=None):
model_dict = model_to_dict(model)
for key in model_dict:
if isinstance(model_dict[key], decimal.Decimal):
model_dict[key] = str(dt.dt_from_decimal(model_dict[key]))
if extra_values_func:
model_dict.update(extra_values_func(model))
return model_dict
def test_verify_for_delete_non_delete(self):
launched_at = decimal.Decimal('1.1')
deleted_at = decimal.Decimal('1.1')
exist = self.mox.CreateMockAnything()
exist.delete = None
exist.instance = INSTANCE_ID_1
exist.launched_at = launched_at
exist.deleted_at = None
exist.audit_period_ending = deleted_at
results = self.mox.CreateMockAnything()
models.InstanceDeletes.find(
INSTANCE_ID_1, dt.dt_from_decimal(launched_at),
dt.dt_from_decimal(deleted_at)).AndReturn(results)
results.count().AndReturn(0)
self.mox.ReplayAll()
nova_verifier._verify_for_delete(exist)
self.mox.VerifyAll()
def _post_process_raw_data(rows, highlight=None):
deployments = models.Deployment.objects.all()
dep_dict = dict((dep.id, dep) for dep in deployments)
for row in rows:
if "error" in row.routing_key:
row.is_error = True
if highlight and row.id == int(highlight):
row.highlight = True
row.fwhen = dt.dt_from_decimal(row.when)
row.deployment = dep_dict[row.deployment_id]
def _verify_for_delete(exist):
delete = None
if exist.delete:
# We know we have a delete and we have it's id
delete = exist.delete
else:
if exist.deleted_at:
# We received this exists before the delete, go find it
deletes = _find_delete(exist.instance,
dt.dt_from_decimal(exist.launched_at))
if deletes.count() == 1:
delete = deletes[0]
else:
query = {
'instance': exist.instance,
'launched_at': exist.launched_at
}
raise NotFound('InstanceDelete', query)
else:
# We don't know if this is supposed to have a delete or not.
# Thus, we need to check if we have a delete for this instance.
# We need to be careful though, since we could be verifying an
# exist event that we got before the delete. So, we restrict the
# search to only deletes before the time this exist was sent.
# If we find any, we fail validation
deletes = _find_delete(exist.instance,
dt.dt_from_decimal(exist.launched_at),
dt.dt_from_decimal(exist.raw.when))
if deletes.count() > 0:
reason = 'Found InstanceDeletes for non-delete exist'
raise VerificationException(reason)
if delete:
if not _verify_date_field(delete.launched_at, exist.launched_at,
same_second=True):
raise FieldMismatch('launched_at', exist.launched_at,
delete.launched_at)
if not _verify_date_field(delete.deleted_at, exist.deleted_at,
same_second=True):
raise FieldMismatch('deleted_at', exist.deleted_at,
delete.deleted_at)
def _post_process_raw_data(rows, highlight=None):
deployments = models.Deployment.objects.all()
dep_dict = dict((dep.id, dep) for dep in deployments)
for row in rows:
if "error" in row.routing_key:
row.is_error = True
if highlight and row.id == int(highlight):
row.highlight = True
row.fwhen = dt.dt_from_decimal(row.when)
row.deployment = dep_dict[row.deployment_id]
def test_do_list_usage_deletes(self):
fake_request = self.mox.CreateMockAnything()
fake_request.GET = {}
results = self.mox.CreateMockAnything()
models.InstanceDeletes.objects.all().AndReturn(results)
usage = self.mox.CreateMockAnything()
usage.instance = INSTANCE_ID_1
usage.launched_at = utils.decimal_utc()
usage.deleted_at = usage.launched_at + 10
results.__iter__().AndReturn([usage].__iter__())
self.mox.ReplayAll()
resp = stacky_server.do_list_usage_deletes(fake_request)
self.assertEqual(resp.status_code, 200)
resp_json = json.loads(resp.content)
self.assertEqual(len(resp_json), 2)
self.assertEqual(resp_json[0], ["UUID", "Launched At", "Deleted At"])
self.assertEqual(resp_json[1][0], INSTANCE_ID_1)
launch_time_str = dt.dt_from_decimal(usage.launched_at)
self.assertEqual(resp_json[1][1], str(launch_time_str))
delete_time_str = dt.dt_from_decimal(usage.deleted_at)
self.assertEqual(resp_json[1][2], str(delete_time_str))
self.mox.VerifyAll()
def test_do_list_usage_deletes(self):
fake_request = self.mox.CreateMockAnything()
fake_request.GET = {}
results = self.mox.CreateMockAnything()
models.InstanceDeletes.objects.all().AndReturn(results)
usage = self.mox.CreateMockAnything()
usage.instance = INSTANCE_ID_1
usage.launched_at = utils.decimal_utc()
usage.deleted_at = usage.launched_at + 10
results.__iter__().AndReturn([usage].__iter__())
self.mox.ReplayAll()
resp = stacky_server.do_list_usage_deletes(fake_request)
self.assertEqual(resp.status_code, 200)
resp_json = json.loads(resp.content)
self.assertEqual(len(resp_json), 2)
self.assertEqual(resp_json[0], ["UUID", "Launched At", "Deleted At"])
self.assertEqual(resp_json[1][0], INSTANCE_ID_1)
launch_time_str = dt.dt_from_decimal(usage.launched_at)
self.assertEqual(resp_json[1][1], str(launch_time_str))
delete_time_str = dt.dt_from_decimal(usage.deleted_at)
self.assertEqual(resp_json[1][2], str(delete_time_str))
self.mox.VerifyAll()
def test_verify_for_delete_non_delete_found_deletes(self):
exist = self.mox.CreateMockAnything()
exist.delete = None
exist.instance = INSTANCE_ID_1
launched_at = decimal.Decimal('1.1')
deleted_at = decimal.Decimal('1.3')
exist.launched_at = launched_at
exist.deleted_at = None
exist.audit_period_ending = deleted_at
results = self.mox.CreateMockAnything()
models.InstanceDeletes.find(
INSTANCE_ID_1, dt.dt_from_decimal(launched_at),
dt.dt_from_decimal(deleted_at)).AndReturn(results)
results.count().AndReturn(1)
self.mox.ReplayAll()
try:
nova_verifier._verify_for_delete(exist)
self.fail()
except VerificationException, ve:
msg = 'Found InstanceDeletes for non-delete exist'
self.assertEqual(ve.reason, msg)
def _assert_on_show(self, values, raw):
self.assertEqual(len(values), 12)
self.assertEqual(values[0], ["Key", "Value"])
self.assertEqual(values[1], ["#", raw.id])
self.assertEqual(values[2], ["When", str(dt.dt_from_decimal(raw.when))])
self.assertEqual(values[3], ["Deployment", raw.deployment.name])
self.assertEqual(values[4], ["Category", raw.routing_key])
self.assertEqual(values[5], ["Publisher", raw.publisher])
self.assertEqual(values[6], ["State", raw.state])
self.assertEqual(values[7], ["Event", raw.event])
self.assertEqual(values[8], ["Service", raw.service])
self.assertEqual(values[9], ["Host", raw.host])
self.assertEqual(values[10], ["UUID", raw.instance])
self.assertEqual(values[11], ["Req ID", raw.request_id])
def _assert_on_show(self, values, raw):
self.assertEqual(len(values), 12)
self.assertEqual(values[0], ["Key", "Value"])
self.assertEqual(values[1], ["#", raw.id])
self.assertEqual(values[2],
["When", str(dt.dt_from_decimal(raw.when))])
self.assertEqual(values[3], ["Deployment", raw.deployment.name])
self.assertEqual(values[4], ["Category", raw.routing_key])
self.assertEqual(values[5], ["Publisher", raw.publisher])
self.assertEqual(values[6], ["State", raw.state])
self.assertEqual(values[7], ["Event", raw.event])
self.assertEqual(values[8], ["Service", raw.service])
self.assertEqual(values[9], ["Host", raw.host])
self.assertEqual(values[10], ["UUID", raw.instance])
self.assertEqual(values[11], ["Req ID", raw.request_id])
def test_verify_delete_when_no_delete(self):
exist = self.mox.CreateMockAnything()
exist.delete = None
exist.uuid = IMAGE_UUID_1
exist.deleted_at = None
audit_period_ending = decimal.Decimal("1.2")
exist.audit_period_ending = audit_period_ending
results = self.mox.CreateMockAnything()
models.ImageDeletes.find(IMAGE_UUID_1, dt.dt_from_decimal(audit_period_ending)).AndReturn(results)
results.count().AndReturn(0)
self.mox.ReplayAll()
glance_verifier._verify_for_delete(exist)
self.mox.VerifyAll()
def _verify_for_launch(exist, launch=None, launch_type="InstanceUsage"):
if not launch and exist.usage:
launch = exist.usage
elif not launch:
if models.InstanceUsage.objects.filter(instance=exist.instance).count() > 0:
launches = models.InstanceUsage.find(exist.instance, dt.dt_from_decimal(exist.launched_at))
count = launches.count()
query = {"instance": exist.instance, "launched_at": exist.launched_at}
if count > 1:
raise AmbiguousResults(launch_type, query)
elif count == 0:
raise NotFound(launch_type, query)
launch = launches[0]
else:
raise NotFound(launch_type, {"instance": exist.instance})
_verify_field_mismatch(exist, launch)
def test_verify_delete_when_no_delete(self):
exist = self.mox.CreateMockAnything()
exist.delete = None
exist.uuid = IMAGE_UUID_1
exist.deleted_at = None
audit_period_ending = decimal.Decimal('1.2')
exist.audit_period_ending = audit_period_ending
results = self.mox.CreateMockAnything()
models.ImageDeletes.find(
IMAGE_UUID_1,
dt.dt_from_decimal(audit_period_ending)).AndReturn(results)
results.count().AndReturn(0)
self.mox.ReplayAll()
glance_verifier._verify_for_delete(exist)
self.mox.VerifyAll()
def test_verify_delete_found_delete_when_exist_deleted_at_is_none(self):
exist = self.mox.CreateMockAnything()
exist.delete = None
exist.uuid = IMAGE_UUID_1
audit_period_ending = decimal.Decimal("1.3")
exist.deleted_at = None
exist.audit_period_ending = audit_period_ending
results = self.mox.CreateMockAnything()
models.ImageDeletes.find(IMAGE_UUID_1, dt.dt_from_decimal(audit_period_ending)).AndReturn(results)
results.count().AndReturn(1)
self.mox.ReplayAll()
with self.assertRaises(VerificationException) as ve:
glance_verifier._verify_for_delete(exist)
exception = ve.exception
self.assertEqual(exception.reason, "Found ImageDeletes for non-delete exist")
self.mox.VerifyAll()
def test_verify_for_delete_found_delete(self):
exist = self.mox.CreateMockAnything()
exist.delete = None
exist.instance = INSTANCE_ID_1
exist.launched_at = decimal.Decimal('1.1')
exist.deleted_at = decimal.Decimal('5.1')
launched_at = decimal.Decimal('1.1')
results = self.mox.CreateMockAnything()
models.InstanceDeletes.find(INSTANCE_ID_1, dt.dt_from_decimal(
launched_at)).AndReturn(results)
results.count().AndReturn(1)
delete = self.mox.CreateMockAnything()
delete.launched_at = decimal.Decimal('1.1')
delete.deleted_at = decimal.Decimal('5.1')
results.__getitem__(0).AndReturn(delete)
self.mox.ReplayAll()
nova_verifier._verify_for_delete(exist)
self.mox.VerifyAll()
def test_verify_with_reconciled_data_reconcile_not_found(self):
exists = self.mox.CreateMockAnything()
exists.instance = INSTANCE_ID_1
launched_at = decimal.Decimal('1.1')
deleted_at = decimal.Decimal('2.1')
exists.launched_at = launched_at
exists.deleted_at = deleted_at
results = self.mox.CreateMockAnything()
models.InstanceReconcile.objects.filter(instance=INSTANCE_ID_1)\
.AndReturn(results)
results.count().AndReturn(1)
launched_at = dt.dt_from_decimal(decimal.Decimal('1.1'))
recs = self.mox.CreateMockAnything()
models.InstanceReconcile.find(INSTANCE_ID_1, launched_at).AndReturn(recs)
recs.count().AndReturn(0)
self.mox.ReplayAll()
with self.assertRaises(NotFound) as cm:
nova_verifier._verify_with_reconciled_data(exists)
exception = cm.exception
self.assertEquals(exception.object_type, 'InstanceReconcile')
self.mox.VerifyAll()
def test_verify_delete_found_delete_when_exist_deleted_at_is_none(self):
exist = self.mox.CreateMockAnything()
exist.delete = None
exist.uuid = IMAGE_UUID_1
audit_period_ending = decimal.Decimal('1.3')
exist.deleted_at = None
exist.audit_period_ending = audit_period_ending
results = self.mox.CreateMockAnything()
models.ImageDeletes.find(
IMAGE_UUID_1,
dt.dt_from_decimal(audit_period_ending)).AndReturn(results)
results.count().AndReturn(1)
self.mox.ReplayAll()
with self.assertRaises(VerificationException) as ve:
glance_verifier._verify_for_delete(exist)
exception = ve.exception
self.assertEqual(exception.reason,
'Found ImageDeletes for non-delete exist')
self.mox.VerifyAll()
def _verify_for_launch(exist):
if exist.usage:
launch = exist.usage
else:
if models.InstanceUsage.objects\
.filter(instance=exist.instance).count() > 0:
launches = _find_launch(exist.instance,
dt.dt_from_decimal(exist.launched_at))
count = launches.count()
query = {
'instance': exist.instance,
'launched_at': exist.launched_at
}
if count > 1:
raise AmbiguousResults('InstanceUsage', query)
elif count == 0:
raise NotFound('InstanceUsage', query)
launch = launches[0]
else:
raise NotFound('InstanceUsage', {'instance': exist.instance})
_verify_field_mismatch(exist, launch)
def test_verify_for_launch_late_usage(self):
exist = self.mox.CreateMockAnything()
exist.usage = None
exist.instance = INSTANCE_ID_1
launched_at = decimal.Decimal('1.1')
exist.launched_at = launched_at
exist.instance_type_id = 2
results = self.mox.CreateMockAnything()
models.InstanceUsage.objects.filter(instance=INSTANCE_ID_1)\
.AndReturn(results)
results.count().AndReturn(2)
models.InstanceUsage.find(INSTANCE_ID_1, dt.dt_from_decimal(
launched_at)).AndReturn(results)
results.count().AndReturn(1)
usage = self.mox.CreateMockAnything()
results.__getitem__(0).AndReturn(usage)
usage.launched_at = decimal.Decimal('1.1')
usage.instance_type_id = 2
self.mox.ReplayAll()
nova_verifier._verify_for_launch(exist)
self.mox.VerifyAll()
def test_do_uuid(self):
fake_request = self.mox.CreateMockAnything()
fake_request.GET = {"uuid": INSTANCE_ID_1}
result = self.mox.CreateMockAnything()
models.RawData.objects.select_related().AndReturn(result)
result.filter(instance=INSTANCE_ID_1).AndReturn(result)
result.order_by("when").AndReturn(result)
raw = self._create_raw()
result.__iter__().AndReturn([raw].__iter__())
self.mox.ReplayAll()
resp = stacky_server.do_uuid(fake_request)
self.assertEqual(resp.status_code, 200)
json_resp = json.loads(resp.content)
self.assertEqual(len(json_resp), 2)
header = ["#", "?", "When", "Deployment", "Event", "Host", "State", "State'", "Task'"]
self.assertEqual(json_resp[0], header)
datetime = dt.dt_from_decimal(raw.when)
body = [1, " ", str(datetime), "deployment", "test.start", "example.com", "active", None, None]
self.assertEqual(json_resp[1], body)
self.mox.VerifyAll()
def details(request, deployment_id, column, row_id):
deployment_id = int(deployment_id)
c = _default_context(request, deployment_id)
row = models.RawData.objects.get(pk=row_id)
value = getattr(row, column)
rows = models.RawData.objects
if deployment_id:
rows = rows.filter(deployment=deployment_id)
if column != 'when':
rows = rows.filter(**{column: value})
else:
when = dt.dt_from_decimal(value)
from_time = when - datetime.timedelta(minutes=1)
to_time = when + datetime.timedelta(minutes=1)
from_time_dec = dt.dt_to_decimal(from_time)
to_time_dec = dt.dt_to_decimal(to_time)
rows = rows.filter(when__range=(from_time_dec, to_time_dec))
rows = rows.order_by('-when')[:200]
_post_process_raw_data(rows, highlight=row_id)
c['rows'] = rows
c['allow_expansion'] = True
c['show_absolute_time'] = True
return render_to_response('rows.html', c)
def details(request, deployment_id, column, row_id):
deployment_id = int(deployment_id)
c = _default_context(request, deployment_id)
row = models.RawData.objects.get(pk=row_id)
value = getattr(row, column)
rows = models.RawData.objects.select_related()
if deployment_id:
rows = rows.filter(deployment=deployment_id)
if column != 'when':
rows = rows.filter(**{column:value})
else:
when = dt.dt_from_decimal(value)
from_time = when - datetime.timedelta(minutes=1)
to_time = when + datetime.timedelta(minutes=1)
from_time_dec = dt.dt_to_decimal(from_time)
to_time_dec = dt.dt_to_decimal(to_time)
rows = rows.filter(when__range=(from_time_dec, to_time_dec))
rows = rows.order_by('-when')[:200]
_post_process_raw_data(rows, highlight=row_id)
c['rows'] = rows
c['allow_expansion'] = True
c['show_absolute_time'] = True
return render_to_response('rows.html', c)
def _exists_extra_values(exist):
values = {'received': str(dt.dt_from_decimal(exist.raw.when))}
return values
def test_decimal_to_datetime(self):
expected_decimal = decimal.Decimal('1356093296.123')
expected_datetime = datetime.datetime.utcfromtimestamp(
expected_decimal)
actual_datetime = datetime_to_decimal.dt_from_decimal(expected_decimal)
self.assertEqual(actual_datetime, expected_datetime)
def print_status(event, completed, errored, total):
out = (event, completed, errored, total - (completed + errored))
print "%s: %s completed, %s errored, %s remaining" % out
def find_start_decimal():
usage = models.InstanceUsage.objects.all().order_by('id')[0]
usage_raw = models.RawData.objects.filter(request_id=usage.request_id)\
.order_by('when')[0]
delete = models.InstanceDeletes.objects.all().order_by('id')[0]
# Start a day after receiving the first usage, just to be safe
return min(usage_raw.when, delete.raw.when) + (60 * 60 * 24)
start_decimal = find_start_decimal()
print "Starting from date %s" % dt.dt_from_decimal(start_decimal)
for event in events:
start = datetime.datetime.utcnow()
raws = models.RawData.objects.filter(event=event, when__lte=start_decimal)
total = raws.count()
completed = 0
errored = 0
loc = 0
print_status(event, completed, errored, total)
update_interval = datetime.timedelta(seconds=30)
next_update = start + update_interval
while loc < total:
new_loc = loc + 500
failed_request['platform'] = image_type.readable(image_type_num)
tenant_issues[tenant] = tenant_issues.get(tenant, 0) + 1
if err_id:
err = models.RawData.objects.get(id=err_id)
queue, body = json.loads(err.json)
payload = body['payload']
# Add error information to failed request report
failed_request['event_id'] = err.id
failed_request['tenant'] = err.tenant
failed_request['service'] = err.service
failed_request['host'] = err.host
failed_request['deployment'] = err.deployment.name
failed_request['event'] = err.event
failed_request['when'] = str(dt.dt_from_decimal(err.when))
# Track failed event counts
event_counts[err.event] = event_counts.get(err.event, 0) + 1
exc = payload.get('exception')
if exc:
# group the messages ...
failed_request['exception'] = exc
# assemble message from exception and generalize
message_str = mask_msg(build_exc_msg(exc))
# count exception messages
exception_counts[message_str] = exception_counts.get(
message_str, 0) + 1
image_type_num)
tenant_issues[tenant] = tenant_issues.get(tenant, 0) + 1
if err_id:
err = models.RawData.objects.get(id=err_id)
queue, body = json.loads(err.json)
payload = body['payload']
# Add error information to failed request report
failed_request['event_id'] = err.id
failed_request['tenant'] = err.tenant
failed_request['service'] = err.service
failed_request['host'] = err.host
failed_request['deployment'] = err.deployment.name
failed_request['event'] = err.event
failed_request['when'] = str(dt.dt_from_decimal(err.when))
# Track failed event counts
event_counts[err.event] = event_counts.get(err.event,
0) + 1
exc = payload.get('exception')
if exc:
# group the messages ...
failed_request['exception'] = exc
# assemble message from exception and generalize
message_str = mask_msg(build_exc_msg(exc))
# count exception messages
exception_counts[message_str] = exception_counts.get(
message_str, 0) + 1
def print_status(event, completed, errored, total):
out = (event, completed, errored, total - (completed + errored))
print "%s: %s completed, %s errored, %s remaining" % out
def find_start_decimal():
usage = models.InstanceUsage.objects.all().order_by('id')[0]
usage_raw = models.RawData.objects.filter(request_id=usage.request_id)\
.order_by('when')[0]
delete = models.InstanceDeletes.objects.all().order_by('id')[0]
# Start a day after receiving the first usage, just to be safe
return min(usage_raw.when, delete.raw.when) + (60 * 60 * 24)
start_decimal = find_start_decimal()
print "Starting from date %s" % dt.dt_from_decimal(start_decimal)
for event in events:
start = datetime.datetime.utcnow()
raws = models.RawData.objects.filter(event=event, when__lte=start_decimal)
total = raws.count()
completed = 0
errored = 0
loc = 0
print_status(event, completed, errored, total)
update_interval = datetime.timedelta(seconds=30)
next_update = start + update_interval
while loc < total:
new_loc = loc + 500
def _exists_extra_values(exist):
values = {'received': str(dt.dt_from_decimal(exist.raw.when))}
return values
def test_decimal_to_datetime(self):
expected_decimal = decimal.Decimal('1356093296.123')
expected_datetime = datetime.datetime.utcfromtimestamp(expected_decimal)
actual_datetime = datetime_to_decimal.dt_from_decimal(expected_decimal)
self.assertEqual(actual_datetime, expected_datetime)
