def _get_audit_task_logs(self, context, begin=None, end=None, before=None): """Returns a full log for all instance usage audit tasks on all computes. :param begin: datetime beginning of audit period to get logs for, Defaults to the beginning of the most recently completed audit period prior to the 'before' date. :param end: datetime ending of audit period to get logs for, Defaults to the ending of the most recently completed audit period prior to the 'before' date. :param before: By default we look for the audit period most recently completed before this datetime. Has no effect if both begin and end are specified. """ defbegin, defend = utils.last_completed_audit_period(before=before) if begin is None: begin = defbegin if end is None: end = defend task_logs = self.host_api.task_log_get_all(context, "instance_usage_audit", begin, end) # We do this in this way to include disabled compute services, # which can have instances on them. (mdragon) filters = {"topic": CONF.compute_topic} services = self.host_api.service_get_all(context, filters=filters) hosts = set(serv["host"] for serv in services) seen_hosts = set() done_hosts = set() running_hosts = set() total_errors = 0 total_items = 0 for tlog in task_logs: seen_hosts.add(tlog["host"]) if tlog["state"] == "DONE": done_hosts.add(tlog["host"]) if tlog["state"] == "RUNNING": running_hosts.add(tlog["host"]) total_errors += tlog["errors"] total_items += tlog["task_items"] log = { tl["host"]: dict(state=tl["state"], instances=tl["task_items"], errors=tl["errors"], message=tl["message"]) for tl in task_logs } missing_hosts = hosts - seen_hosts overall_status = "%s hosts done. %s errors." % ( "ALL" if len(done_hosts) == len(hosts) else "%s of %s" % (len(done_hosts), len(hosts)), total_errors, ) return dict( period_beginning=str(begin), period_ending=str(end), num_hosts=len(hosts), num_hosts_done=len(done_hosts), num_hosts_running=len(running_hosts), num_hosts_not_run=len(missing_hosts), hosts_not_run=list(missing_hosts), total_instances=total_items, total_errors=total_errors, overall_status=overall_status, log=log, )
def has_audit_been_run(context, conductor, host, timestamp=None): begin, end = utils.last_completed_audit_period(before=timestamp) task_log = conductor.task_log_get(context, "instance_usage_audit", begin, end, host) if task_log: return True else: return False
def test_hour_with_offset_after_current(self): begin, end = utils.last_completed_audit_period(unit='hour@30') self.assertEquals( begin, datetime.datetime(minute=30, hour=6, day=5, month=3, year=2012)) self.assertEquals( end, datetime.datetime( minute=30, hour=7, day=5, month=3, year=2012))
def notify_usage_exists(context, instance_ref, current_period=False, ignore_missing_network_data=True): """Generates 'exists' notification for an instance for usage auditing purposes. :param current_period: if True, this will generate a usage for the current usage period; if False, this will generate a usage for the previous audit period. :param ignore_missing_network_data: if True, log any exceptions generated while getting network info; if False, raise the exception. """ admin_context = nova.context.get_admin_context(read_deleted='yes') begin, end = utils.last_completed_audit_period() bw = {} if current_period: audit_start = end audit_end = utils.utcnow() else: audit_start = begin audit_end = end if (instance_ref.get('info_cache') and instance_ref['info_cache'].get('network_info')): cached_info = instance_ref['info_cache']['network_info'] nw_info = network_model.NetworkInfo.hydrate(cached_info) else: try: nw_info = network.API().get_instance_nw_info(admin_context, instance_ref) except Exception: LOG.exception('Failed to get nw_info', instance=instance_ref) if ignore_missing_network_data: return raise macs = [vif['address'] for vif in nw_info] uuids = [instance_ref.uuid] bw_usages = db.bw_usage_get_by_uuids(admin_context, uuids, audit_start) bw_usages = [b for b in bw_usages if b.mac in macs] for b in bw_usages: label = 'net-name-not-found-%s' % b['mac'] for vif in nw_info: if vif['address'] == b['mac']: label = vif['network']['label'] break bw[label] = dict(bw_in=b.bw_in, bw_out=b.bw_out) extra_usage_info = dict(audit_period_beginning=str(audit_start), audit_period_ending=str(audit_end), bandwidth=bw) notify_about_instance_usage( context, instance_ref, 'exists', extra_usage_info=extra_usage_info)
def get_audit_task_logs(context, begin=None, end=None, before=None): """Returns a full log for all instance usage audit tasks on all computes. :param begin: datetime beginning of audit period to get logs for, Defaults to the beginning of the most recently completed audit period prior to the 'before' date. :param end: datetime ending of audit period to get logs for, Defaults to the ending of the most recently completed audit period prior to the 'before' date. :param before: By default we look for the audit period most recently completed before this datetime. Has no effect if both begin and end are specified. """ defbegin, defend = utils.last_completed_audit_period(before=before) if begin is None: begin = defbegin if end is None: end = defend task_logs = db.task_log_get_all(context, "instance_usage_audit", begin, end) services = db.service_get_all_by_topic(context, "compute") hosts = set(serv['host'] for serv in services) seen_hosts = set() done_hosts = set() running_hosts = set() total_errors = 0 total_items = 0 for tlog in task_logs: seen_hosts.add(tlog['host']) if tlog['state'] == "DONE": done_hosts.add(tlog['host']) if tlog['state'] == "RUNNING": running_hosts.add(tlog['host']) total_errors += tlog['errors'] total_items += tlog['task_items'] log = dict((tl['host'], dict(state=tl['state'], instances=tl['task_items'], errors=tl['errors'], message=tl['message'])) for tl in task_logs) missing_hosts = hosts - seen_hosts overall_status = "%s hosts done. %s errors." % ( 'ALL' if len(done_hosts) == len(hosts) else "%s of %s" % (len(done_hosts), len(hosts)), total_errors) return dict(period_beginning=str(begin), period_ending=str(end), num_hosts=len(hosts), num_hosts_done=len(done_hosts), num_hosts_running=len(running_hosts), num_hosts_not_run=len(missing_hosts), hosts_not_run=list(missing_hosts), total_instances=total_items, total_errors=total_errors, overall_status=overall_status, log=log)
def test_year_with_offset_after_current(self): begin, end = utils.last_completed_audit_period(unit='year@6') self.assertEqual(begin, datetime.datetime( day=1, month=6, year=2010)) self.assertEqual(end, datetime.datetime( day=1, month=6, year=2011))
def test_day(self): begin, end = utils.last_completed_audit_period(unit='day') self.assertEqual(begin, datetime.datetime( day=4, month=3, year=2012)) self.assertEqual(end, datetime.datetime( day=5, month=3, year=2012))
def test_month_with_offset_after_current(self): begin, end = utils.last_completed_audit_period(unit='month@15') self.assertEqual(begin, datetime.datetime( day=15, month=1, year=2012)) self.assertEqual(end, datetime.datetime( day=15, month=2, year=2012))
def test_year_with_offset_before_current(self): begin, end = utils.last_completed_audit_period(unit='year@2') self.assertEquals(begin, datetime.datetime( day=1, month=2, year=2011)) self.assertEquals(end, datetime.datetime( day=1, month=2, year=2012))
def test_year(self): begin, end = utils.last_completed_audit_period(unit='year') self.assertEqual(begin, datetime.datetime( day=1, month=1, year=2011)) self.assertEqual(end, datetime.datetime( day=1, month=1, year=2012))
def test_month(self): begin, end = utils.last_completed_audit_period(unit='month') self.assertEquals(begin, datetime.datetime( day=1, month=2, year=2012)) self.assertEquals(end, datetime.datetime( day=1, month=3, year=2012))
def test_hour(self): begin, end = utils.last_completed_audit_period(unit='hour') self.assertEquals(begin, datetime.datetime( hour=7, day=5, month=3, year=2012)) self.assertEquals(end, datetime.datetime( hour=8, day=5, month=3, year=2012))
def test_hour(self): begin, end = utils.last_completed_audit_period(unit='hour') self.assertEqual(begin, datetime.datetime( hour=7, day=5, month=3, year=2012)) self.assertEqual(end, datetime.datetime( hour=8, day=5, month=3, year=2012))
def test_day_with_offset_before_current(self): begin, end = utils.last_completed_audit_period(unit='day@6') self.assertEquals(begin, datetime.datetime( hour=6, day=4, month=3, year=2012)) self.assertEquals(end, datetime.datetime( hour=6, day=5, month=3, year=2012))
def test_day_with_offset_after_current(self): begin, end = utils.last_completed_audit_period(unit='day@10') self.assertEqual(begin, datetime.datetime( hour=10, day=3, month=3, year=2012)) self.assertEqual(end, datetime.datetime( hour=10, day=4, month=3, year=2012))
def _get_audit_task_logs(self, context, before=None): """Returns a full log for all instance usage audit tasks on all computes. :param context: Nova request context. :param before: By default we look for the audit period most recently completed before this datetime. Has no effect if both begin and end are specified. """ begin, end = utils.last_completed_audit_period(before=before) task_logs = self.host_api.task_log_get_all(context, "instance_usage_audit", begin, end) # We do this in this way to include disabled compute services, # which can have instances on them. (mdragon) filters = {'topic': CONF.compute_topic} services = self.host_api.service_get_all(context, filters=filters) hosts = set(serv['host'] for serv in services) seen_hosts = set() done_hosts = set() running_hosts = set() total_errors = 0 total_items = 0 for tlog in task_logs: seen_hosts.add(tlog['host']) if tlog['state'] == "DONE": done_hosts.add(tlog['host']) if tlog['state'] == "RUNNING": running_hosts.add(tlog['host']) total_errors += tlog['errors'] total_items += tlog['task_items'] log = {tl['host']: dict(state=tl['state'], instances=tl['task_items'], errors=tl['errors'], message=tl['message']) for tl in task_logs} missing_hosts = hosts - seen_hosts overall_status = "%s hosts done. %s errors." % ( 'ALL' if len(done_hosts) == len(hosts) else "%s of %s" % (len(done_hosts), len(hosts)), total_errors) return dict(period_beginning=str(begin), period_ending=str(end), num_hosts=len(hosts), num_hosts_done=len(done_hosts), num_hosts_running=len(running_hosts), num_hosts_not_run=len(missing_hosts), hosts_not_run=list(missing_hosts), total_instances=total_items, total_errors=total_errors, overall_status=overall_status, log=log)
def test_hour_with_offset_after_current(self): begin, end = utils.last_completed_audit_period(unit='hour@30') self.assertEquals(begin, datetime.datetime( minute=30, hour=6, day=5, month=3, year=2012)) self.assertEquals(end, datetime.datetime( minute=30, hour=7, day=5, month=3, year=2012))
def test_archive_then_purge_all(self): # Enable the generation of task_log records by the instance usage audit # nova-compute periodic task. self.flags(instance_usage_audit=True) compute = self.computes['compute'] server = self._create_server() server_id = server['id'] admin_context = context.get_admin_context() future = timeutils.utcnow() + datetime.timedelta(days=30) with osloutils_fixture.TimeFixture(future): # task_log records are generated by the _instance_usage_audit # periodic task. compute.manager._instance_usage_audit(admin_context) # Audit period defaults to 1 month, the last audit period will # be the previous calendar month. begin, end = nova_utils.last_completed_audit_period() # Verify that we have 1 task_log record per audit period. task_logs = objects.TaskLogList.get_all(admin_context, 'instance_usage_audit', begin, end) self.assertEqual(1, len(task_logs)) self._delete_server(server) results, deleted_ids, archived = db.archive_deleted_rows(max_rows=1000, task_log=True) self.assertEqual([server_id], deleted_ids) lines = [] def status(msg): lines.append(msg) deleted = sqlalchemy_api.purge_shadow_tables(admin_context, None, status_fn=status) self.assertNotEqual(0, deleted) self.assertNotEqual(0, len(lines)) self.assertEqual(sum(results.values()), archived) for line in lines: self.assertIsNotNone( re.match(r'Deleted [1-9][0-9]* rows from .*', line)) # Ensure we purged task_log records. self.assertIn('shadow_task_log', str(lines)) results = self._get_table_counts() # No table should have any rows self.assertFalse(any(results.values()))
def notify_usage_exists(context, instance_ref, current_period=False): """ Generates 'exists' notification for an instance for usage auditing purposes. Generates usage for last completed period, unless 'current_period' is True.""" admin_context = nova.context.get_admin_context(read_deleted='yes') begin, end = utils.last_completed_audit_period() bw = {} if current_period: audit_start = end audit_end = utils.utcnow() else: audit_start = begin audit_end = end if (instance_ref.get('info_cache') and instance_ref['info_cache'].get('network_info')): cached_info = instance_ref['info_cache']['network_info'] nw_info = network_model.NetworkInfo.hydrate(cached_info) else: nw_info = network.API().get_instance_nw_info(admin_context, instance_ref) macs = [vif['address'] for vif in nw_info] uuids = [instance_ref.uuid] bw_usages = db.bw_usage_get_by_uuids(admin_context, uuids, audit_start) bw_usages = [b for b in bw_usages if b.mac in macs] for b in bw_usages: label = 'net-name-not-found-%s' % b['mac'] for vif in nw_info: if vif['address'] == b['mac']: label = vif['network']['label'] break bw[label] = dict(bw_in=b.bw_in, bw_out=b.bw_out) extra_usage_info = dict(audit_period_beginning=str(audit_start), audit_period_ending=str(audit_end), bandwidth=bw) notify_about_instance_usage( context, instance_ref, 'exists', extra_usage_info=extra_usage_info)
def audit_period_bounds(current_period=False): """Get the start and end of the relevant audit usage period :param current_period: if True, this will generate a usage for the current usage period; if False, this will generate a usage for the previous audit period. """ begin, end = utils.last_completed_audit_period() if current_period: audit_start = end audit_end = timeutils.utcnow() else: audit_start = begin audit_end = end return (audit_start, audit_end)
def notify_usage_exists(instance_ref, current_period=False): """ Generates 'exists' notification for an instance for usage auditing purposes. Generates usage for last completed period, unless 'current_period' is True.""" admin_context = context.get_admin_context(read_deleted='yes') begin, end = utils.last_completed_audit_period() bw = {} if current_period: audit_start = end audit_end = utils.utcnow() else: audit_start = begin audit_end = end if (instance_ref.get('info_cache') and instance_ref['info_cache'].get('network_info')): cached_info = instance_ref['info_cache']['network_info'] nw_info = network_model.NetworkInfo.hydrate(cached_info) else: nw_info = network.API().get_instance_nw_info(admin_context, instance_ref) macs = [vif['address'] for vif in nw_info] for b in db.bw_usage_get_by_macs(admin_context, macs, audit_start): label = 'net-name-not-found-%s' % b['mac'] for vif in nw_info: if vif['address'] == b['mac']: label = vif['network']['label'] break bw[label] = dict(bw_in=b.bw_in, bw_out=b.bw_out) usage_info = utils.usage_from_instance(instance_ref, audit_period_beginning=str(audit_start), audit_period_ending=str(audit_end), bandwidth=bw) notifier_api.notify('compute.%s' % FLAGS.host, 'compute.instance.exists', notifier_api.INFO, usage_info)
def notify_usage_exists(context, volume_ref, current_period=False): """ Generates 'exists' notification for a volume for usage auditing purposes. Generates usage for last completed period, unless 'current_period' is True.""" begin, end = utils.last_completed_audit_period() if current_period: audit_start = end audit_end = timeutils.utcnow() else: audit_start = begin audit_end = end extra_usage_info = dict(audit_period_beginning=str(audit_start), audit_period_ending=str(audit_end)) notify_about_volume_usage( context, volume_ref, 'exists', extra_usage_info=extra_usage_info)
def notify_usage_exists(instance_ref, current_period=False): """ Generates 'exists' notification for an instance for usage auditing purposes. Generates usage for last completed period, unless 'current_period' is True.""" admin_context = context.get_admin_context(read_deleted='yes') begin, end = utils.last_completed_audit_period() bw = {} if current_period: audit_start = end audit_end = utils.utcnow() else: audit_start = begin audit_end = end if (instance_ref.get('info_cache') and instance_ref['info_cache'].get('network_info')): cached_info = instance_ref['info_cache']['network_info'] nw_info = network_model.NetworkInfo.hydrate(cached_info) else: nw_info = network.API().get_instance_nw_info(admin_context, instance_ref) macs = [vif['address'] for vif in nw_info] for b in db.bw_usage_get_by_macs(admin_context, macs, audit_start): label = 'net-name-not-found-%s' % b['mac'] for vif in nw_info: if vif['address'] == b['mac']: label = vif['network']['label'] break bw[label] = dict(bw_in=b.bw_in, bw_out=b.bw_out) usage_info = utils.usage_from_instance( instance_ref, audit_period_beginning=str(audit_start), audit_period_ending=str(audit_end), bandwidth=bw) notifier_api.notify('compute.%s' % FLAGS.host, 'compute.instance.exists', notifier_api.INFO, usage_info)
def notify_usage_exists(context, volume_ref, current_period=False): """ Generates 'exists' notification for a volume for usage auditing purposes. Generates usage for last completed period, unless 'current_period' is True.""" begin, end = utils.last_completed_audit_period() if current_period: audit_start = end audit_end = timeutils.utcnow() else: audit_start = begin audit_end = end extra_usage_info = dict(audit_period_beginning=str(audit_start), audit_period_ending=str(audit_end)) notify_about_volume_usage(context, volume_ref, 'exists', extra_usage_info=extra_usage_info)
def test_day(self): begin, end = utils.last_completed_audit_period(unit='day') self.assertEqual(begin, datetime.datetime(day=4, month=3, year=2012)) self.assertEqual(end, datetime.datetime(day=5, month=3, year=2012))
def test_month_with_offset_before_current(self): begin, end = utils.last_completed_audit_period(unit="month@2") self.assertEqual(begin, datetime.datetime(day=2, month=2, year=2012)) self.assertEqual(end, datetime.datetime(day=2, month=3, year=2012))
def notify_usage_exists(context, instance_ref, current_period=False, ignore_missing_network_data=True, system_metadata=None, extra_usage_info=None): """Generates 'exists' notification for an instance for usage auditing purposes. :param current_period: if True, this will generate a usage for the current usage period; if False, this will generate a usage for the previous audit period. :param ignore_missing_network_data: if True, log any exceptions generated while getting network info; if False, raise the exception. :param system_metadata: system_metadata DB entries for the instance, if not None. *NOTE*: Currently unused here in trunk, but needed for potential custom modifications. :param extra_usage_info: Dictionary containing extra values to add or override in the notification if not None. """ admin_context = nova.context.get_admin_context(read_deleted='yes') begin, end = utils.last_completed_audit_period() bw = {} if current_period: audit_start = end audit_end = utils.utcnow() else: audit_start = begin audit_end = end if (instance_ref.get('info_cache') and instance_ref['info_cache'].get('network_info')): cached_info = instance_ref['info_cache']['network_info'] nw_info = network_model.NetworkInfo.hydrate(cached_info) else: try: nw_info = network.API().get_instance_nw_info(admin_context, instance_ref) except Exception: LOG.exception('Failed to get nw_info', instance=instance_ref) if ignore_missing_network_data: return raise macs = [vif['address'] for vif in nw_info] uuids = [instance_ref.uuid] bw_usages = db.bw_usage_get_by_uuids(admin_context, uuids, audit_start) bw_usages = [b for b in bw_usages if b.mac in macs] for b in bw_usages: label = 'net-name-not-found-%s' % b['mac'] for vif in nw_info: if vif['address'] == b['mac']: label = vif['network']['label'] break bw[label] = dict(bw_in=b.bw_in, bw_out=b.bw_out) if system_metadata is None: try: system_metadata = db.instance_system_metadata_get( context, instance_ref.uuid) except exception.NotFound: system_metadata = {} # add image metadata to the notification: image_meta = {} for md_key, md_value in system_metadata.iteritems(): if md_key.startswith('image_'): image_meta[md_key[6:]] = md_value extra_info = dict(audit_period_beginning=str(audit_start), audit_period_ending=str(audit_end), bandwidth=bw, image_meta=image_meta) if extra_usage_info: extra_info.update(extra_usage_info) notify_about_instance_usage(context, instance_ref, 'exists', system_metadata=system_metadata, extra_usage_info=extra_info)
def test_day_with_offset_before_current(self): begin, end = utils.last_completed_audit_period(unit='day@6') self.assertEquals(begin, datetime.datetime(hour=6, day=4, month=3, year=2012)) self.assertEquals(end, datetime.datetime(hour=6, day=5, month=3, year=2012))
def test_month(self): begin, end = utils.last_completed_audit_period(unit='month') self.assertEquals(begin, datetime.datetime(day=1, month=2, year=2012)) self.assertEquals(end, datetime.datetime(day=1, month=3, year=2012))
def test_year_with_offset_before_current(self): begin, end = utils.last_completed_audit_period(unit='year@2') self.assertEquals(begin, datetime.datetime(day=1, month=2, year=2011)) self.assertEquals(end, datetime.datetime(day=1, month=2, year=2012))
def test_year_with_offset_after_current(self): begin, end = utils.last_completed_audit_period(unit='year@6') self.assertEqual(begin, datetime.datetime(day=1, month=6, year=2010)) self.assertEqual(end, datetime.datetime(day=1, month=6, year=2011))
def test_year(self): begin, end = utils.last_completed_audit_period(unit='year') self.assertEqual(begin, datetime.datetime(day=1, month=1, year=2011)) self.assertEqual(end, datetime.datetime(day=1, month=1, year=2012))
def test_month_with_offset_after_current(self): begin, end = utils.last_completed_audit_period(unit='month@15') self.assertEqual(begin, datetime.datetime(day=15, month=1, year=2012)) self.assertEqual(end, datetime.datetime(day=15, month=2, year=2012))
def test_day_with_offset_after_current(self): begin, end = utils.last_completed_audit_period(unit='day@10') self.assertEqual(begin, datetime.datetime(hour=10, day=3, month=3, year=2012)) self.assertEqual(end, datetime.datetime(hour=10, day=4, month=3, year=2012))
def test_hour_with_offset_before_current(self): begin, end = utils.last_completed_audit_period(unit="hour@10") self.assertEqual(begin, datetime.datetime(minute=10, hour=7, day=5, month=3, year=2012)) self.assertEqual(end, datetime.datetime(minute=10, hour=8, day=5, month=3, year=2012))
def notify_usage_exists(context, instance_ref, current_period=False, ignore_missing_network_data=True, system_metadata=None, extra_usage_info=None): """Generates 'exists' notification for an instance for usage auditing purposes. :param current_period: if True, this will generate a usage for the current usage period; if False, this will generate a usage for the previous audit period. :param ignore_missing_network_data: if True, log any exceptions generated while getting network info; if False, raise the exception. :param system_metadata: system_metadata DB entries for the instance, if not None. *NOTE*: Currently unused here in trunk, but needed for potential custom modifications. :param extra_usage_info: Dictionary containing extra values to add or override in the notification if not None. """ admin_context = nova.context.get_admin_context(read_deleted='yes') begin, end = utils.last_completed_audit_period() bw = {} if current_period: audit_start = end audit_end = utils.utcnow() else: audit_start = begin audit_end = end if (instance_ref.get('info_cache') and instance_ref['info_cache'].get('network_info')): cached_info = instance_ref['info_cache']['network_info'] nw_info = network_model.NetworkInfo.hydrate(cached_info) else: try: nw_info = network.API().get_instance_nw_info( admin_context, instance_ref) except Exception: LOG.exception('Failed to get nw_info', instance=instance_ref) if ignore_missing_network_data: return raise macs = [vif['address'] for vif in nw_info] uuids = [instance_ref.uuid] bw_usages = db.bw_usage_get_by_uuids(admin_context, uuids, audit_start) bw_usages = [b for b in bw_usages if b.mac in macs] for b in bw_usages: label = 'net-name-not-found-%s' % b['mac'] for vif in nw_info: if vif['address'] == b['mac']: label = vif['network']['label'] break bw[label] = dict(bw_in=b.bw_in, bw_out=b.bw_out) if system_metadata is None: try: system_metadata = db.instance_system_metadata_get( context, instance_ref.uuid) except exception.NotFound: system_metadata = {} # add image metadata to the notification: image_meta = {} for md_key, md_value in system_metadata.iteritems(): if md_key.startswith('image_'): image_meta[md_key[6:]] = md_value extra_info = dict(audit_period_beginning=str(audit_start), audit_period_ending=str(audit_end), bandwidth=bw, image_meta=image_meta) if extra_usage_info: extra_info.update(extra_usage_info) notify_about_instance_usage(context, instance_ref, 'exists', system_metadata=system_metadata, extra_usage_info=extra_info)
def _get_audit_task_logs(self, context, begin=None, end=None, before=None): """Returns a full log for all instance usage audit tasks on all computes. :param begin: datetime beginning of audit period to get logs for, Defaults to the beginning of the most recently completed audit period prior to the 'before' date. :param end: datetime ending of audit period to get logs for, Defaults to the ending of the most recently completed audit period prior to the 'before' date. :param before: By default we look for the audit period most recently completed before this datetime. Has no effect if both begin and end are specified. """ # NOTE(alex_xu): back-compatible with db layer hard-code admin # permission checks. nova_context.require_admin_context(context) defbegin, defend = utils.last_completed_audit_period(before=before) if begin is None: begin = defbegin if end is None: end = defend task_logs = self.host_api.task_log_get_all(context, "instance_usage_audit", begin, end) # We do this in this way to include disabled compute services, # which can have instances on them. (mdragon) filters = {'topic': CONF.compute_topic} services = self.host_api.service_get_all(context, filters=filters) hosts = set(serv['host'] for serv in services) seen_hosts = set() done_hosts = set() running_hosts = set() total_errors = 0 total_items = 0 for tlog in task_logs: seen_hosts.add(tlog['host']) if tlog['state'] == "DONE": done_hosts.add(tlog['host']) if tlog['state'] == "RUNNING": running_hosts.add(tlog['host']) total_errors += tlog['errors'] total_items += tlog['task_items'] log = {tl['host']: dict(state=tl['state'], instances=tl['task_items'], errors=tl['errors'], message=tl['message']) for tl in task_logs} missing_hosts = hosts - seen_hosts overall_status = "{0!s} hosts done. {1!s} errors.".format( 'ALL' if len(done_hosts) == len(hosts) else "{0!s} of {1!s}".format(len(done_hosts), len(hosts)), total_errors) return dict(period_beginning=str(begin), period_ending=str(end), num_hosts=len(hosts), num_hosts_done=len(done_hosts), num_hosts_running=len(running_hosts), num_hosts_not_run=len(missing_hosts), hosts_not_run=list(missing_hosts), total_instances=total_items, total_errors=total_errors, overall_status=overall_status, log=log)
def _get_audit_task_logs(self, context, begin=None, end=None, before=None): """Returns a full log for all instance usage audit tasks on all computes. :param begin: datetime beginning of audit period to get logs for, Defaults to the beginning of the most recently completed audit period prior to the 'before' date. :param end: datetime ending of audit period to get logs for, Defaults to the ending of the most recently completed audit period prior to the 'before' date. :param before: By default we look for the audit period most recently completed before this datetime. Has no effect if both begin and end are specified. """ # NOTE(alex_xu): back-compatible with db layer hard-code admin # permission checks. nova_context.require_admin_context(context) defbegin, defend = utils.last_completed_audit_period(before=before) if begin is None: begin = defbegin if end is None: end = defend task_logs = self.host_api.task_log_get_all(context, "instance_usage_audit", begin, end) # We do this in this way to include disabled compute services, # which can have instances on them. (mdragon) filters = {'topic': CONF.compute_topic} services = self.host_api.service_get_all(context, filters=filters) hosts = set(serv['host'] for serv in services) seen_hosts = set() done_hosts = set() running_hosts = set() total_errors = 0 total_items = 0 for tlog in task_logs: seen_hosts.add(tlog['host']) if tlog['state'] == "DONE": done_hosts.add(tlog['host']) if tlog['state'] == "RUNNING": running_hosts.add(tlog['host']) total_errors += tlog['errors'] total_items += tlog['task_items'] log = { tl['host']: dict(state=tl['state'], instances=tl['task_items'], errors=tl['errors'], message=tl['message']) for tl in task_logs } missing_hosts = hosts - seen_hosts overall_status = "%s hosts done. %s errors." % ( 'ALL' if len(done_hosts) == len(hosts) else "%s of %s" % (len(done_hosts), len(hosts)), total_errors) return dict(period_beginning=str(begin), period_ending=str(end), num_hosts=len(hosts), num_hosts_done=len(done_hosts), num_hosts_running=len(running_hosts), num_hosts_not_run=len(missing_hosts), hosts_not_run=list(missing_hosts), total_instances=total_items, total_errors=total_errors, overall_status=overall_status, log=log)