def audit_for_period(beginning, ending, ums=False, ums_offset=0): beginning_decimal = dt.dt_to_decimal(beginning) ending_decimal = dt.dt_to_decimal(ending) if ums: verifier_audit_func = functools.partial( usage_audit._verifier_audit_for_day_ums, ums_offset=ums_offset ) else: verifier_audit_func = usage_audit._verifier_audit_for_day (verify_summary, verify_detail) = verifier_audit_func(beginning_decimal, ending_decimal, models.InstanceExists) detail, new_count, old_count = _launch_audit_for_period(beginning_decimal, ending_decimal) summary = { 'verifier': verify_summary, 'launch_summary': { 'new_launches': new_count, 'old_launches': old_count, 'failures': len(detail) }, } details = { 'exist_fails': verify_detail, 'launch_fails': detail, } return summary, details
def audit_for_period(beginning, ending): beginning_decimal = dt.dt_to_decimal(beginning) ending_decimal = dt.dt_to_decimal(ending) image_event_counts = __get_image_activate_count(beginning_decimal, ending_decimal) return image_event_counts
def find(instance, launched): start = launched - datetime.timedelta(microseconds=launched.microsecond) end = start + datetime.timedelta(microseconds=999999) params = {'instance': instance, 'launched_at__gte': dt.dt_to_decimal(start), 'launched_at__lte': dt.dt_to_decimal(end)} return InstanceReconcile.objects.filter(**params)
def audit_for_period(beginning, ending): beginning_decimal = dt.dt_to_decimal(beginning) ending_decimal = dt.dt_to_decimal(ending) (verify_summary, verify_detail) = _verifier_audit_for_day(beginning_decimal, ending_decimal, models.ImageExists) detail, new_count, old_count = _image_audit_for_period(beginning_decimal, ending_decimal) summary = { 'verifier': verify_summary, 'image_summary': { 'new_images': new_count, 'old_images': old_count, 'failures': len(detail) }, } details = { 'exist_fails': verify_detail, 'image_fails': detail, } return summary, details
def audit_for_period(beginning, ending): beginning_decimal = dt.dt_to_decimal(beginning) ending_decimal = dt.dt_to_decimal(ending) (verify_summary, verify_detail) = _verifier_audit_for_day(beginning_decimal, ending_decimal) detail, new_count, old_count = _launch_audit_for_period(beginning_decimal, ending_decimal) summary = { 'verifier': verify_summary, 'launch_summary': { 'new_launches': new_count, 'old_launches': old_count, 'failures': len(detail) }, } details = { 'exist_fails': verify_detail, 'launch_fails': detail, } return summary, details
def __audit_for_instance_exists(beginning, ending): beginning_decimal = dt.dt_to_decimal(beginning) ending_decimal = dt.dt_to_decimal(ending) instance_exists = __get_instance_exists(beginning_decimal, ending_decimal) total_bw = reduce(lambda x, y: x + y.bandwidth_public_out, instance_exists, 0) report = {"total_public_outbound_bandwidth": total_bw} return report
def _get_verified_exists(self): start = dt.dt_to_decimal(self.start) end = dt.dt_to_decimal(self.end) return models.InstanceExists.objects.filter( status=models.InstanceExists.VERIFIED, audit_period_beginning__gte=start, audit_period_beginning__lte=end, audit_period_ending__gte=start, audit_period_ending__lte=end)
def find(instance, launched, deleted_max=None): start = launched - datetime.timedelta(microseconds=launched.microsecond) end = start + datetime.timedelta(microseconds=999999) params = {'instance': instance, 'launched_at__gte': dt.dt_to_decimal(start), 'launched_at__lte': dt.dt_to_decimal(end)} if deleted_max: params['deleted_at__lte'] = dt.dt_to_decimal(deleted_max) return InstanceDeletes.objects.filter(**params)
def _delete_for_instance(instance): delete = { 'instance': instance['uuid'], 'deleted_at': dt.dt_to_decimal(instance.get('deleted_at')), } launched_at = instance.get('launched_at') if launched_at is not None: delete['launched_at'] = dt.dt_to_decimal(launched_at) return delete
def _list_exists(received_max=None, received_min=None, status=None): params = {} if received_max: params['raw__when__lte'] = dt.dt_to_decimal(received_max) if received_min: params['raw__when__gt'] = dt.dt_to_decimal(received_min) if status: params['status'] = status return models.InstanceExists.objects.select_related()\ .filter(**params).order_by('id')
def __audit_for_instance_exists(beginning, ending): beginning_decimal = dt.dt_to_decimal(beginning) ending_decimal = dt.dt_to_decimal(ending) instance_exists = __get_instance_exists(beginning_decimal, ending_decimal) total_bw = reduce(lambda x, y: x + y.bandwidth_public_out, instance_exists, 0) report = { 'total_public_outbound_bandwidth': total_bw, } return report
def audit_for_period(beginning, ending): beginning_decimal = dt.dt_to_decimal(beginning) ending_decimal = dt.dt_to_decimal(ending) (verify_summary, verify_detail) = _verifier_audit_for_day(beginning_decimal, ending_decimal) detail, new_count, old_count = _launch_audit_for_period(beginning_decimal, ending_decimal) summary = { "verifier": verify_summary, "launch_summary": {"new_launches": new_count, "old_launches": old_count, "failures": len(detail)}, } details = {"exist_fails": verify_detail, "launch_fails": detail} return summary, details
def test_create_raw_data_should_populate_rawdata_and_rawdata_imagemeta(self): deployment = db.get_or_create_deployment('deployment1')[0] kwargs = { 'deployment': deployment, 'when': dt_to_decimal(datetime.utcnow()), 'tenant': '1', 'json': '{}', 'routing_key': 'monitor.info', 'state': 'verifying', 'old_state': 'pending', 'old_task': '', 'task': '', 'image_type': 1, 'publisher': '', 'event': 'compute.instance.exists', 'service': '', 'host': '', 'instance': '1234-5678-9012-3456', 'request_id': '1234', 'os_architecture': 'x86', 'os_version': '1', 'os_distro': 'windows', 'rax_options': '2'} rawdata = db.create_rawdata(**kwargs) for field in get_model_fields(RawData): if field.name != 'id': self.assertEquals(getattr(rawdata, field.name), kwargs[field.name]) raw_image_meta = RawDataImageMeta.objects.all()[0] self.assertEquals(raw_image_meta.raw, rawdata) self.assertEquals(raw_image_meta.os_architecture, kwargs['os_architecture']) self.assertEquals(raw_image_meta.os_version, kwargs['os_version']) self.assertEquals(raw_image_meta.os_distro, kwargs['os_distro']) self.assertEquals(raw_image_meta.rax_options, kwargs['rax_options'])
def process_raw_data(deployment, args, json_args): """This is called directly by the worker to add the event to the db.""" db.reset_queries() routing_key, body = args record = None handler = HANDLERS.get(routing_key, None) if handler: values = handler(routing_key, body) if not values: return record values['deployment'] = deployment try: when = body['timestamp'] except KeyError: when = body['_context_timestamp'] # Old way of doing it try: try: when = datetime.datetime.strptime(when, "%Y-%m-%d %H:%M:%S.%f") except ValueError: # Old way of doing it when = datetime.datetime.strptime(when, "%Y-%m-%dT%H:%M:%S.%f") except Exception, e: pass values['when'] = dt.dt_to_decimal(when) values['routing_key'] = routing_key values['json'] = json_args record = STACKDB.create_rawdata(**values) STACKDB.save(record) aggregate_lifecycle(record) aggregate_usage(record)
def test_group_exists_with_date_status_in_audit_period_by_owner_rawid(self): end_max = datetime.utcnow() status = 'pending' exist1 = self.mox.CreateMockAnything() exist1.owner = "owner1" exist1.raw_id = "1" exist2 = self.mox.CreateMockAnything() exist2.owner = "owner2" exist2.raw_id = "2" exist3 = self.mox.CreateMockAnything() exist3.owner = "owner1" exist3.raw_id = "1" exist4 = self.mox.CreateMockAnything() exist4.owner = "owner1" exist4.raw_id = "3" ordered_results = [exist1, exist3, exist4, exist2] unordered_results = self.mox.CreateMockAnything() related_results = self.mox.CreateMockAnything() self.mox.StubOutWithMock(ImageExists.objects, 'select_related') ImageExists.objects.select_related().AndReturn(related_results) related_results.filter( audit_period_ending__lte=dt.dt_to_decimal(end_max), status=status).AndReturn(unordered_results) unordered_results.order_by('owner').AndReturn(ordered_results) self.mox.ReplayAll() results = ImageExists.find_and_group_by_owner_and_raw_id(end_max, status) self.mox.VerifyAll() self.assertEqual(results, {'owner1-1': [exist1, exist3], 'owner1-3': [exist4], 'owner2-2': [exist2]})
def test_verify_for_range_with_callback(self): callback = self.mox.CreateMockAnything() pool = self.mox.CreateMockAnything() when_max = datetime.datetime.utcnow() results = self.mox.CreateMockAnything() models.InstanceExists.objects.select_related().AndReturn(results) models.InstanceExists.PENDING = 'pending' models.InstanceExists.VERIFYING = 'verifying' filters = { 'audit_period_ending__lte': dt.dt_to_decimal(when_max), 'status': 'pending' } results.filter(**filters).AndReturn(results) results.order_by('id').AndReturn(results) results.count().AndReturn(2) exist1 = self.mox.CreateMockAnything() exist2 = self.mox.CreateMockAnything() results.__getslice__(0, 1000).AndReturn(results) results.__iter__().AndReturn([exist1, exist2].__iter__()) exist1.save() exist2.save() pool.apply_async(dbverifier._verify, args=(exist1,), callback=callback) pool.apply_async(dbverifier._verify, args=(exist2,), callback=callback) self.mox.ReplayAll() dbverifier.verify_for_range(pool, when_max, callback=callback) self.assertEqual(exist1.status, 'verifying') self.assertEqual(exist2.status, 'verifying') self.mox.VerifyAll()
def test_verify_for_range_with_callback(self): callback = self.mox.CreateMockAnything() pool = self.mox.CreateMockAnything() when_max = datetime.datetime.utcnow() results = self.mox.CreateMockAnything() models.InstanceExists.objects.select_related().AndReturn(results) models.InstanceExists.PENDING = 'pending' models.InstanceExists.VERIFYING = 'verifying' filters = { 'audit_period_ending__lte': dt.dt_to_decimal(when_max), 'status': 'pending' } results.filter(**filters).AndReturn(results) results.order_by('id').AndReturn(results) results.count().AndReturn(2) exist1 = self.mox.CreateMockAnything() exist2 = self.mox.CreateMockAnything() results.__getslice__(0, 1000).AndReturn(results) results.__iter__().AndReturn([exist1, exist2].__iter__()) exist1.save() exist2.save() self.pool.apply_async(dbverifier._verify, args=(exist1,), callback=callback) self.pool.apply_async(dbverifier._verify, args=(exist2,), callback=callback) self.mox.ReplayAll() self.verifier.verify_for_range(when_max, callback=callback) self.assertEqual(exist1.status, 'verifying') self.assertEqual(exist2.status, 'verifying') self.mox.VerifyAll()
def find(ending_max, status): params = { 'audit_period_ending__lte': dt.dt_to_decimal(ending_max), 'status': status } return InstanceExists.objects.select_related()\ .filter(**params).order_by('id')
def _usage_for_instance(instance, task=None): usage = { 'instance': instance['uuid'], 'tenant': instance['project_id'], 'instance_type_id': instance.get('instance_type_id'), } instance_type = get_instance_type(instance.get('instance_type_id')) usage['instance_flavor_id'] = instance_type['flavorid'] metadata = get_metadata(instance['uuid']) usage['os_architecture'] = metadata.get( 'image_org.openstack__1__architecture') usage['os_distro'] = metadata.get('image_org.openstack__1__os_distro') usage['os_version'] = metadata.get('image_org.openstack__1__os_version') usage['rax_options'] = metadata.get('image_com.rackspace__1__options') launched_at = instance.get('launched_at') if launched_at is not None: usage['launched_at'] = dt.dt_to_decimal(launched_at) if task is not None: usage['task'] = task return usage
def _list_exists(ending_max=None, status=None): params = {} if ending_max: params['audit_period_ending__lte'] = dt.dt_to_decimal(ending_max) if status: params['status'] = status return models.InstanceExists.objects.select_related()\ .filter(**params).order_by('id')
def str_time_to_unix(when): if 'Z' in when: when = _try_parse(when, ["%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%dT%H:%M:%S.%fZ"]) elif 'T' in when: when = _try_parse(when, ["%Y-%m-%dT%H:%M:%S.%f", "%Y-%m-%dT%H:%M:%S"]) else: when = _try_parse(when, ["%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S"]) return dt.dt_to_decimal(when)
def _usage_for_image(image): return { 'uuid': image.id, 'owner': image.owner, 'created_at': dt.dt_to_decimal(image.created_at), 'owner': image.owner, 'size': image.size, 'last_raw_id': None }
def store(self, json_report): report = models.JsonReport(json=json_report, created=dt.dt_to_decimal( datetime.datetime.utcnow()), period_start=self.start, period_end=self.end, version=1, name='instance hours') report.save()
def get_verified_count(request): try: audit_period_beginning = datetime.strptime( request.GET.get("audit_period_beginning"), "%Y-%m-%d") audit_period_ending = datetime.strptime( request.GET.get("audit_period_ending"), "%Y-%m-%d") service = request.GET.get("service", "nova") rawdata = _rawdata_factory(service) filters = { 'when__gte': dt.dt_to_decimal(audit_period_beginning), 'when__lte': dt.dt_to_decimal(audit_period_ending), 'event': "compute.instance.exists.verified" } return {'count': rawdata.filter(**filters).count()} except KeyError and TypeError: raise BadRequestException(message="Invalid/absent query parameter") except ValueError: raise BadRequestException(message="Invalid format for date (Correct " "format should be %YYYY-%mm-%dd)")
def test_find_delete_should_return_delete_issued_before_given_time(self): delete = self.mox.CreateMockAnything() deleted_max = datetime.utcnow() self.mox.StubOutWithMock(ImageDeletes.objects, 'filter') ImageDeletes.objects.filter( uuid=IMAGE_UUID_1, deleted_at__lte=dt.dt_to_decimal(deleted_max)).AndReturn(delete) self.mox.ReplayAll() self.assertEquals(ImageDeletes.find(IMAGE_UUID_1, deleted_max), delete) self.mox.VerifyAll()
def __store_report_in_db(start, end, report): values = { 'json': __make_json_report(report), 'created': dt.dt_to_decimal(datetime.datetime.utcnow()), 'period_start': start, 'period_end': end, 'version': 1, 'name': 'nova compute periodic usage' } report = models.JsonReport(**values) report.save()
def __store_report_in_db(start, end, report): values = { 'json': __make_json_report(report), 'created': dt.dt_to_decimal(datetime.datetime.utcnow()), 'period_start': start, 'period_end': end, 'version': 1, 'name': 'public outbound bandwidth' } report = models.JsonReport(**values) report.save()
def test_find_delete_should_return_delete_issued_before_given_time(self): delete = self.mox.CreateMockAnything() deleted_max = datetime.utcnow() self.mox.StubOutWithMock(ImageDeletes.objects, 'filter') ImageDeletes.objects.filter( uuid=IMAGE_UUID_1, deleted_at__lte=dt.dt_to_decimal(deleted_max)).AndReturn(delete) self.mox.ReplayAll() self.assertEquals(ImageDeletes.find( IMAGE_UUID_1, deleted_max), delete) self.mox.VerifyAll()
def __store_report_in_db(start, end, report): values = { "json": __make_json_report(report), "created": dt.dt_to_decimal(datetime.datetime.utcnow()), "period_start": start, "period_end": end, "version": 1, "name": "public outbound bandwidth", } report = models.JsonReport(**values) report.save()
def __store_results(start, end, report): values = { 'json': __make_json_report(report), 'created': dt.dt_to_decimal(datetime.datetime.utcnow()), 'period_start': start, 'period_end': end, 'version': 1, 'name': 'image events audit' } report = models.JsonReport(**values) report.save()
def __store_results(start, end, report): values = { "json": __make_json_report(report), "created": dt.dt_to_decimal(datetime.datetime.utcnow()), "period_start": start, "period_end": end, "version": 1, "name": "image events audit", } report = models.JsonReport(**values) report.save()
def store_results(start, end, summary, details): values = { "json": make_json_report(summary, details), "created": dt.dt_to_decimal(datetime.datetime.utcnow()), "period_start": start, "period_end": end, "version": 3, "name": "nova usage audit", } report = models.JsonReport(**values) report.save()
def store_results(start, end, summary, details): values = { 'json': make_json_report(summary, details), 'created': dt.dt_to_decimal(datetime.datetime.utcnow()), 'period_start': start, 'period_end': end, 'version': 7, 'name': 'nova usage audit' } report = models.JsonReport(**values) report.save()
def latest_raw(request, deployment_id): """This is the 2sec ticker that updates the Recent Activity box.""" deployment_id = int(deployment_id) c = _default_context(request, deployment_id) then = datetime.datetime.utcnow() - datetime.timedelta(hours=1) thend = dt.dt_to_decimal(then) query = models.RawData.objects.select_related().filter(when__gt=thend) if deployment_id > 0: query = query.filter(deployment=deployment_id) rows = query.order_by('-when')[:20] _post_process_raw_data(rows) c['rows'] = rows return render_to_response('host_status.html', c)
def find_and_group_by_owner_and_raw_id(ending_max, status): params = {'audit_period_ending__lte': dt.dt_to_decimal(ending_max), 'status': status} ordered_exists = ImageExists.objects.select_related().\ filter(**params).order_by('owner') result = {} for exist in ordered_exists: key = "%s-%s" % (exist.owner, exist.raw_id) if key in result: result[key].append(exist) else: result[key] = [exist] return result
def latest_raw(request, deployment_id): """This is the 2sec ticker that updates the Recent Activity box.""" deployment_id = int(deployment_id) c = _default_context(request, deployment_id) then = datetime.datetime.utcnow() - datetime.timedelta(hours=1) thend = dt.dt_to_decimal(then) query = models.RawData.objects.filter(when__gt=thend) if deployment_id > 0: query = query.filter(deployment=deployment_id) rows = query.order_by('-when')[:20] _post_process_raw_data(rows) c['rows'] = rows return render_to_response('host_status.html', c)
def fix_chunk(hours, length): now = datetime.datetime.utcnow() start = now - datetime.timedelta(hours=hours+length) end = now - datetime.timedelta(hours=hours) dstart = dt.dt_to_decimal(start) dend = dt.dt_to_decimal(end) done = 0 updated = 0 block = 0 print "Hours ago (%d to %d) %d - %d" % (hours + length, hours, dstart, dend) updates = models.RawData.objects.filter(event='compute.instance.update', when__gt=dstart, when__lte=dend)\ .only('task', 'image_type', 'json') for raw in updates: queue, body = json.loads(raw.json) payload = body.get('payload', {}) task = payload.get('new_task_state', None) if task != None and task != 'None': states[task] = states.get(task, 0) + 1 raw.task = task raw.image_type = image_type.get_numeric_code(payload, raw.image_type) updated += 1 raw.save() done += 1 if done >= 10000: block += 1 done = 0 print "# 10k blocks processed: %d (events %d)" % \ (block, updated) updated = 0 for kv in states.iteritems(): print "%s = %d" % kv
def audit_for_period(beginning, ending): beginning_decimal = dt.dt_to_decimal(beginning) ending_decimal = dt.dt_to_decimal(ending) verifier_report = _verifier_audit_for_period(beginning_decimal, ending_decimal) detail, new_count, old_count = _launch_audit_for_period(beginning_decimal, ending_decimal) summary = { 'verifier': verifier_report, 'launch_fails': { 'total_failures': len(detail), 'new_launches': new_count, 'old_launches': old_count } } details = { 'launch_fails': detail } return summary, details
def test_create_image_delete_should_populate_image_delete(self): _, rawdata = self._create_glance_rawdata() kwargs = { 'uuid': '1', 'raw': rawdata, 'deleted_at': dt_to_decimal(datetime.utcnow()) } db.create_image_delete(**kwargs) image_delete = ImageDeletes.objects.all()[0] for field in get_model_fields(ImageDeletes): if field.name != 'id': self.assertEquals(getattr(image_delete, field.name), kwargs[field.name])
def details(request, deployment_id, column, row_id): deployment_id = int(deployment_id) c = _default_context(request, deployment_id) row = models.RawData.objects.get(pk=row_id) value = getattr(row, column) rows = models.RawData.objects.select_related() if deployment_id: rows = rows.filter(deployment=deployment_id) if column != 'when': rows = rows.filter(**{column:value}) else: when = dt.dt_from_decimal(value) from_time = when - datetime.timedelta(minutes=1) to_time = when + datetime.timedelta(minutes=1) from_time_dec = dt.dt_to_decimal(from_time) to_time_dec = dt.dt_to_decimal(to_time) rows = rows.filter(when__range=(from_time_dec, to_time_dec)) rows = rows.order_by('-when')[:200] _post_process_raw_data(rows, highlight=row_id) c['rows'] = rows c['allow_expansion'] = True c['show_absolute_time'] = True return render_to_response('rows.html', c)
def _usage_for_instance(instance, task=None): usage = { 'instance': instance['uuid'], 'tenant': instance['project_id'], 'instance_type_id': instance.get('instance_type_id'), } launched_at = instance.get('launched_at') if launched_at is not None: usage['launched_at'] = dt.dt_to_decimal(launched_at) if task is not None: usage['task'] = task return usage
def details(request, deployment_id, column, row_id): deployment_id = int(deployment_id) c = _default_context(request, deployment_id) row = models.RawData.objects.get(pk=row_id) value = getattr(row, column) rows = models.RawData.objects if deployment_id: rows = rows.filter(deployment=deployment_id) if column != 'when': rows = rows.filter(**{column: value}) else: when = dt.dt_from_decimal(value) from_time = when - datetime.timedelta(minutes=1) to_time = when + datetime.timedelta(minutes=1) from_time_dec = dt.dt_to_decimal(from_time) to_time_dec = dt.dt_to_decimal(to_time) rows = rows.filter(when__range=(from_time_dec, to_time_dec)) rows = rows.order_by('-when')[:200] _post_process_raw_data(rows, highlight=row_id) c['rows'] = rows c['allow_expansion'] = True c['show_absolute_time'] = True return render_to_response('rows.html', c)
def test_create_glance_usage_should_populate_image_usage(self): _, rawdata = self._create_glance_rawdata() kwargs = { 'uuid': '1', 'created_at': dt_to_decimal(datetime.utcnow()), 'owner': '1234567', 'size': 12345, 'last_raw': rawdata } db.create_image_usage(**kwargs) usage = ImageUsage.objects.all()[0] for field in get_model_fields(ImageUsage): if field.name != 'id': self.assertEquals(getattr(usage, field.name), kwargs[field.name])