def save_exists(self, raw):
if self.created_at:
values = {
'uuid': self.uuid,
'audit_period_beginning': self.audit_period_beginning,
'audit_period_ending': self.audit_period_ending,
'owner': self.owner,
'size': self.size,
'raw': raw
}
created_at_range = (self.created_at, self.created_at+1)
usage = db.get_image_usage(
uuid=self.uuid, created_at__range=created_at_range)
values['usage'] = usage
values['created_at'] = self.created_at
if self.deleted_at:
delete = db.get_image_delete(
uuid=self.uuid, created_at__range=created_at_range)
values['delete'] = delete
values['deleted_at'] = self.deleted_at
db.create_image_exists(**values)
else:
stacklog.warn("Ignoring exists without created_at. GlanceRawData(%s)"
% raw.id)
def _log_api_exception(cls, ex, request):
line1 = "Exception: %s - %s - %s" % (cls.__name__, ex.status, ex.message)
line2 = "Request: %s - %s" % (request.method, request.path)
line3 = "Body: %s" % request.body
msg = "%s\n%s\n%s" % (line1, line2, line3)
if 400 <= ex.status < 500:
stacklog.warn(msg)
else:
stacklog.error(msg)
def _log_api_exception(cls, ex, request):
line1 = "Exception: %s - %s - %s" % (cls.__name__, ex.status, ex.message)
line2 = "Request: %s - %s" % (request.method, request.path)
line3 = "Body: %s" % request.body
msg = "%s\n%s\n%s" % (line1, line2, line3)
if 400 <= ex.status < 500:
stacklog.warn(msg)
else:
stacklog.error(msg)
def _safe_get(Model, **kwargs):
object = None
query = Model.objects.filter(**kwargs)
count = query.count()
if count > 1:
stacklog.warn('Multiple records found for %s get.' % Model.__name__)
object = query[0]
elif count < 1:
stacklog.warn('No records found for %s get.' % Model.__name__)
else:
object = query[0]
return object
def test_save_image_exists_without_created_at(self):
raw = self.mox.CreateMockAnything()
raw.id = 1
audit_period_beginning = "2013-05-20 17:31:57.939614"
audit_period_ending = "2013-06-20 17:31:57.939614"
size = 123
uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb"
body = {
"event_type": "image.exists",
"timestamp": "2013-06-20 18:31:57.939614",
"publisher_id":
"glance-api01-r2961.global.preprod-ord.ohthree.com",
"payload": {
"audit_period_beginning":
audit_period_beginning,
"audit_period_ending":
audit_period_ending,
"owner":
TENANT_ID_1,
"images": [{
"created_at": None,
"id": uuid,
"size": size,
"status": "saving",
"properties": {
"instance_uuid": INSTANCE_ID_1
},
"deleted_at": None,
}, {
"created_at": None,
"id": uuid,
"size": size,
"status": "saving",
"properties": {
"instance_uuid": INSTANCE_ID_1
},
"deleted_at": None,
}]
}
}
deployment = "1"
routing_key = "glance_monitor.info"
json_body = json.dumps([routing_key, body])
self.mox.StubOutWithMock(stacklog, 'warn')
stacklog.warn("Ignoring exists without created_at. GlanceRawData(1)")
stacklog.warn("Ignoring exists without created_at. GlanceRawData(1)")
self.mox.ReplayAll()
notification = GlanceNotification(body, deployment, routing_key,
json_body)
notification.save_exists(raw)
self.mox.VerifyAll()
def _process_exists(raw, notification):
instance_id = notification.instance
launched_at_str = notification.launched_at
if launched_at_str is not None and launched_at_str != '':
launched_at = utils.str_time_to_unix(notification.launched_at)
launched_range = (launched_at, launched_at + 1)
usage = STACKDB.get_instance_usage(instance=instance_id,
launched_at__range=launched_range)
values = {}
values['message_id'] = notification.message_id
values['instance'] = instance_id
values['launched_at'] = launched_at
beginning = utils.str_time_to_unix(notification.audit_period_beginning)
values['audit_period_beginning'] = beginning
ending = utils.str_time_to_unix(notification.audit_period_ending)
values['audit_period_ending'] = ending
values['instance_type_id'] = notification.instance_type_id
values['instance_flavor_id'] = notification.instance_flavor_id
if usage:
values['usage'] = usage
values['raw'] = raw
values['tenant'] = notification.tenant
values['rax_options'] = notification.rax_options
values['os_architecture'] = notification.os_architecture
values['os_version'] = notification.os_version
values['os_distro'] = notification.os_distro
values['bandwidth_public_out'] = notification.bandwidth_public_out
deleted_at = notification.deleted_at
if deleted_at and deleted_at != '':
# We only want to pre-populate the 'delete' if we know this is in
# fact an exist event for a deleted instance. Otherwise, there
# is a chance we may populate it for a previous period's exist.
filter = {
'instance': instance_id,
'launched_at__range': launched_range
}
delete = STACKDB.get_instance_delete(**filter)
deleted_at = utils.str_time_to_unix(deleted_at)
values['deleted_at'] = deleted_at
if delete:
values['delete'] = delete
exists = STACKDB.create_instance_exists(**values)
STACKDB.save(exists)
else:
stacklog.warn("Ignoring exists without launched_at. RawData(%s)" %
raw.id)
def save_exists(self, raw):
if isinstance(self.payload, dict):
audit_period_beginning = self.payload.get(
'audit_period_beginning', None)
audit_period_beginning = audit_period_beginning and\
utils.str_time_to_unix(audit_period_beginning)
audit_period_ending = self.payload.get(
'audit_period_ending', None)
audit_period_ending = audit_period_ending and \
utils.str_time_to_unix(audit_period_ending)
message_id = self.message_id
images = self.payload.get('images', [])
else:
stacklog.warn("Received exists with invalid payload "
"GlanceRawData(%s)" % raw.id)
audit_period_beginning = None
audit_period_ending = None
images = []
for image in images:
created_at = image['created_at']
created_at = created_at and utils.str_time_to_unix(created_at)
uuid = image['id']
deleted_at = image['deleted_at']
deleted_at = deleted_at and utils.str_time_to_unix(deleted_at)
if created_at:
values = {
'uuid': uuid,
'audit_period_beginning': audit_period_beginning,
'audit_period_ending': audit_period_ending,
'owner': self.owner,
'size': image['size'],
'raw': raw,
'message_id': message_id
}
usage = db.get_image_usage(uuid=uuid)
values['usage'] = usage
values['created_at'] = created_at
if deleted_at:
delete = db.get_image_delete(uuid=uuid)
values['delete'] = delete
values['deleted_at'] = deleted_at
db.create_image_exists(**values)
else:
stacklog.warn("Ignoring exists without created_at. GlanceRawData(%s)"
% raw.id)
def save_exists(self, raw):
if isinstance(self.payload, dict):
audit_period_beginning = self.payload.get('audit_period_beginning',
None)
audit_period_beginning = audit_period_beginning and\
utils.str_time_to_unix(audit_period_beginning)
audit_period_ending = self.payload.get('audit_period_ending', None)
audit_period_ending = audit_period_ending and \
utils.str_time_to_unix(audit_period_ending)
message_id = self.message_id
images = self.payload.get('images', [])
else:
stacklog.warn("Received exists with invalid payload "
"GlanceRawData(%s)" % raw.id)
audit_period_beginning = None
audit_period_ending = None
images = []
for image in images:
created_at = image['created_at']
created_at = created_at and utils.str_time_to_unix(created_at)
uuid = image['id']
deleted_at = image['deleted_at']
deleted_at = deleted_at and utils.str_time_to_unix(deleted_at)
if created_at:
values = {
'uuid': uuid,
'audit_period_beginning': audit_period_beginning,
'audit_period_ending': audit_period_ending,
'owner': self.owner,
'size': image['size'],
'raw': raw,
'message_id': message_id
}
usage = db.get_image_usage(uuid=uuid)
values['usage'] = usage
values['created_at'] = created_at
if deleted_at:
delete = db.get_image_delete(uuid=uuid)
values['delete'] = delete
values['deleted_at'] = deleted_at
db.create_image_exists(**values)
else:
stacklog.warn(
"Ignoring exists without created_at. GlanceRawData(%s)" %
raw.id)
def test_save_image_exists_without_created_at(self):
raw = self.mox.CreateMockAnything()
raw.id = 1
audit_period_beginning = "2013-05-20 17:31:57.939614"
audit_period_ending = "2013-06-20 17:31:57.939614"
size = 123
uuid = "2df2ccf6-bc1b-4853-aab0-25fda346b3bb"
body = {
"event_type": "image.exists",
"timestamp": "2013-06-20 18:31:57.939614",
"publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com",
"payload": {
"audit_period_beginning": audit_period_beginning,
"audit_period_ending": audit_period_ending,
"owner": TENANT_ID_1,
"images":
[
{
"created_at": None,
"id": uuid,
"size": size,
"status": "saving",
"properties": {"instance_uuid": INSTANCE_ID_1},
"deleted_at": None,
},
{
"created_at": None,
"id": uuid,
"size": size,
"status": "saving",
"properties": {"instance_uuid": INSTANCE_ID_1},
"deleted_at": None,
}
]
}
}
deployment = "1"
routing_key = "glance_monitor.info"
json_body = json.dumps([routing_key, body])
self.mox.StubOutWithMock(stacklog, 'warn')
stacklog.warn("Ignoring exists without created_at. GlanceRawData(1)")
stacklog.warn("Ignoring exists without created_at. GlanceRawData(1)")
self.mox.ReplayAll()
notification = GlanceNotification(body, deployment, routing_key,
json_body)
notification.save_exists(raw)
self.mox.VerifyAll()
def _process_exists(raw, body):
payload = body['payload']
instance_id = payload['instance_id']
launched_at_str = payload.get('launched_at')
if launched_at_str is not None and launched_at_str != '':
launched_at = utils.str_time_to_unix(payload['launched_at'])
launched_range = (launched_at, launched_at+1)
usage = STACKDB.get_instance_usage(instance=instance_id,
launched_at__range=launched_range)
values = {}
values['message_id'] = body['message_id']
values['instance'] = instance_id
values['launched_at'] = launched_at
beginning = utils.str_time_to_unix(payload['audit_period_beginning'])
values['audit_period_beginning'] = beginning
ending = utils.str_time_to_unix(payload['audit_period_ending'])
values['audit_period_ending'] = ending
values['instance_type_id'] = payload['instance_type_id']
if usage:
values['usage'] = usage
values['raw'] = raw
values['tenant'] = payload['tenant_id']
image_meta = payload.get('image_meta', {})
values['rax_options'] = image_meta.get('com.rackspace__1__options', '')
os_arch = image_meta.get('org.openstack__1__architecture', '')
values['os_architecture'] = os_arch
os_version = image_meta.get('org.openstack__1__os_version', '')
values['os_version'] = os_version
values['os_distro'] = image_meta.get('org.openstack__1__os_distro', '')
deleted_at = payload.get('deleted_at')
if deleted_at and deleted_at != '':
# We only want to pre-populate the 'delete' if we know this is in
# fact an exist event for a deleted instance. Otherwise, there
# is a chance we may populate it for a previous period's exist.
filter = {'instance': instance_id,
'launched_at__range': launched_range}
delete = STACKDB.get_instance_delete(**filter)
deleted_at = utils.str_time_to_unix(deleted_at)
values['deleted_at'] = deleted_at
if delete:
values['delete'] = delete
exists = STACKDB.create_instance_exists(**values)
STACKDB.save(exists)
else:
stacklog.warn("Ignoring exists without launched_at. RawData(%s)" % raw.id)
def _process_exists(raw, notification):
instance_id = notification.instance
launched_at_str = notification.launched_at
if launched_at_str is not None and launched_at_str != '':
launched_at = utils.str_time_to_unix(notification.launched_at)
launched_range = (launched_at, launched_at+1)
usage = STACKDB.get_instance_usage(instance=instance_id,
launched_at__range=launched_range)
values = {}
values['message_id'] = notification.message_id
values['instance'] = instance_id
values['launched_at'] = launched_at
beginning = utils.str_time_to_unix(notification.audit_period_beginning)
values['audit_period_beginning'] = beginning
ending = utils.str_time_to_unix(notification.audit_period_ending)
values['audit_period_ending'] = ending
values['instance_type_id'] = notification.instance_type_id
values['instance_flavor_id'] = notification.instance_flavor_id
if usage:
values['usage'] = usage
values['raw'] = raw
values['tenant'] = notification.tenant
values['rax_options'] = notification.rax_options
values['os_architecture'] = notification.os_architecture
values['os_version'] = notification.os_version
values['os_distro'] = notification.os_distro
values['bandwidth_public_out'] = notification.bandwidth_public_out
deleted_at = notification.deleted_at
if deleted_at and deleted_at != '':
# We only want to pre-populate the 'delete' if we know this is in
# fact an exist event for a deleted instance. Otherwise, there
# is a chance we may populate it for a previous period's exist.
filter = {'instance': instance_id,
'launched_at__range': launched_range}
delete = STACKDB.get_instance_delete(**filter)
deleted_at = utils.str_time_to_unix(deleted_at)
values['deleted_at'] = deleted_at
if delete:
values['delete'] = delete
exists = STACKDB.create_instance_exists(**values)
STACKDB.save(exists)
else:
stacklog.warn("Ignoring exists without launched_at. RawData(%s)" % raw.id)
def _process_exists(raw, body):
payload = body['payload']
instance_id = payload['instance_id']
launched_at_str = payload.get('launched_at')
if launched_at_str is not None and launched_at_str != '':
launched_at = utils.str_time_to_unix(payload['launched_at'])
launched_range = (launched_at, launched_at + 1)
usage = STACKDB.get_instance_usage(instance=instance_id,
launched_at__range=launched_range)
values = {}
values['message_id'] = body['message_id']
values['instance'] = instance_id
values['launched_at'] = launched_at
beginning = utils.str_time_to_unix(payload['audit_period_beginning'])
values['audit_period_beginning'] = beginning
ending = utils.str_time_to_unix(payload['audit_period_ending'])
values['audit_period_ending'] = ending
values['instance_type_id'] = payload['instance_type_id']
if usage:
values['usage'] = usage
values['raw'] = raw
values['tenant'] = payload['tenant_id']
deleted_at = payload.get('deleted_at')
if deleted_at and deleted_at != '':
# We only want to pre-populate the 'delete' if we know this is in
# fact an exist event for a deleted instance. Otherwise, there
# is a chance we may populate it for a previous period's exist.
filter = {
'instance': instance_id,
'launched_at__range': launched_range
}
delete = STACKDB.get_instance_delete(**filter)
deleted_at = utils.str_time_to_unix(deleted_at)
values['deleted_at'] = deleted_at
if delete:
values['delete'] = delete
exists = STACKDB.create_instance_exists(**values)
STACKDB.save(exists)
else:
stacklog.warn("Ignoring exists without launched_at. RawData(%s)" %
raw.id)
def test_save_exists_should_log_warning_when_payload_is_invalid(self):
raw = self.mox.CreateMockAnything()
raw.id = 1
body = {
"event_type": "image.exists",
"publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com",
"payload": []
}
deployment = "1"
routing_key = "glance_monitor.info"
json_body = json.dumps([routing_key, body])
self.mox.StubOutWithMock(stacklog, 'warn')
stacklog.warn("Received exists with invalid payload GlanceRawData(1)")
self.mox.ReplayAll()
notification = GlanceNotification(body, deployment, routing_key,
json_body)
notification.save_exists(raw)
self.mox.VerifyAll()
def test_save_exists_should_log_warning_when_payload_is_invalid(self):
raw = self.mox.CreateMockAnything()
raw.id = 1
body = {
"event_type": "image.exists",
"publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com",
"payload": []
}
deployment = "1"
routing_key = "glance_monitor.info"
json_body = json.dumps([routing_key, body])
self.mox.StubOutWithMock(stacklog, 'warn')
stacklog.warn("Received exists with invalid payload GlanceRawData(1)")
self.mox.ReplayAll()
notification = GlanceNotification(body, deployment, routing_key,
json_body)
notification.save_exists(raw)
self.mox.VerifyAll()
