Ejemplo n.º 1
0
def setup_meters_config():
    """load the meters definitions from yaml config file."""
    config_file = get_config_file()

    LOG.debug("Hardware snmp meter definition file: %s" % config_file)
    with open(config_file) as cf:
        config = cf.read()

    try:
        meters_config = yaml.safe_load(config)
    except yaml.YAMLError as err:
        if hasattr(err, 'problem_mark'):
            mark = err.problem_mark
            errmsg = (_LE("Invalid YAML syntax in Meter Definitions file "
                      "%(file)s at line: %(line)s, column: %(column)s.")
                      % dict(file=config_file,
                             line=mark.line + 1,
                             column=mark.column + 1))
        else:
            errmsg = (_LE("YAML error reading Meter Definitions file "
                      "%(file)s")
                      % dict(file=config_file))
        LOG.error(errmsg)
        raise

    LOG.info(_LI("Meter Definitions: %s") % meters_config)

    return meters_config
Ejemplo n.º 2
0
    def record_metering_data(self, data):
        if self.target == '':
            # if the target was not set, do not do anything
            LOG.error(_LE('Dispatcher target was not set, no meter will '
                          'be posted. Set the target in the ceilometer.conf '
                          'file.'))
            return

        # We may have receive only one counter on the wire
        if not isinstance(data, list):
            data = [data]

        for meter in data:
            LOG.debug(
                'metering data %(counter_name)s '
                'for %(resource_id)s @ %(timestamp)s: %(counter_volume)s',
                {'counter_name': meter['counter_name'],
                 'resource_id': meter['resource_id'],
                 'timestamp': meter.get('timestamp', 'NO TIMESTAMP'),
                 'counter_volume': meter['counter_volume']})
            try:
                # Every meter should be posted to the target
                res = requests.post(self.target,
                                    data=json.dumps(meter),
                                    headers=self.headers,
                                    timeout=self.timeout)
                LOG.debug('Message posting finished with status code '
                          '%d.', res.status_code)
            except Exception as err:
                LOG.exception(_LE('Failed to record metering data: %s.'), err)
Ejemplo n.º 3
0
    def inspect_perf_events(self, instance, duration=None):
        domain = self._get_domain_not_shut_off_or_raise(instance)

        try:
            stats = self.connection.domainListGetStats(
                [domain], libvirt.VIR_DOMAIN_STATS_PERF)
            perf = stats[0][1]
            return virt_inspector.PerfEventsStats(
                cpu_cycles=perf["perf.cpu_cycles"],
                instructions=perf["perf.instructions"],
                cache_references=perf["perf.cache_references"],
                cache_misses=perf["perf.cache_misses"])
            # NOTE(sileht): KeyError if for libvirt >=2.0.0,<2.3.0, the perf
            # subsystem ws existing but not  these attributes
            # https://github.com/libvirt/libvirt/commit/bae660869de0612bee2a740083fb494c27e3f80c
        except (AttributeError, KeyError) as e:
            msg = _LE('Perf is not supported by current version of libvirt, '
                      'and failed to inspect perf events of '
                      '%(instance_uuid)s, can not get info from libvirt: '
                      '%(error)s') % {
                'instance_uuid': instance.id, 'error': e}
            raise virt_inspector.NoDataException(msg)
        # domainListGetStats might launch an exception if the method or
        # mbmt/mbml perf event is not supported by the underlying hypervisor
        # being used by libvirt.
        except libvirt.libvirtError as e:
            msg = _LE('Failed to inspect perf events of %(instance_uuid)s, '
                      'can not get info from libvirt: %(error)s') % {
                'instance_uuid': instance.id, 'error': e}
            raise virt_inspector.NoDataException(msg)
Ejemplo n.º 4
0
def setup_meters_config():
    """Setup the meters definitions from yaml config file."""
    config_file = get_config_file()
    if config_file is not None:
        LOG.debug(_LE("Meter Definitions configuration file: %s"), config_file)

        with open(config_file) as cf:
            config = cf.read()

        try:
            meters_config = yaml.safe_load(config)
        except yaml.YAMLError as err:
            if hasattr(err, 'problem_mark'):
                mark = err.problem_mark
                errmsg = (_LE("Invalid YAML syntax in Meter Definitions file "
                          "%(file)s at line: %(line)s, column: %(column)s.")
                          % dict(file=config_file,
                                 line=mark.line + 1,
                                 column=mark.column + 1))
            else:
                errmsg = (_LE("YAML error reading Meter Definitions file "
                          "%(file)s")
                          % dict(file=config_file))
            LOG.error(errmsg)
            raise

    else:
        LOG.debug(_LE("No Meter Definitions configuration file found!"
                  " Using default config."))
        meters_config = {}

    LOG.info(_LE("Meter Definitions: %s"), meters_config)

    return meters_config
Ejemplo n.º 5
0
    def __init__(self, definition_cfg):
        self.cfg = definition_cfg
        missing = [field for field in self.REQUIRED_FIELDS if not self.cfg.get(field)]
        if missing:
            raise MeterDefinitionException(_LE("Required fields %s not specified") % missing, self.cfg)
        self._event_type = self.cfg.get("event_type")
        if isinstance(self._event_type, six.string_types):
            self._event_type = [self._event_type]

        if "type" not in self.cfg.get("lookup", []) and self.cfg["type"] not in sample.TYPES:
            raise MeterDefinitionException(_LE("Invalid type %s specified") % self.cfg["type"], self.cfg)

        self._field_getter = {}
        for name, field in self.cfg.items():
            if name in ["event_type", "lookup"] or not field:
                continue
            elif isinstance(field, six.integer_types):
                self._field_getter[name] = field
            elif isinstance(field, dict) and name == "metadata":
                meta = {}
                for key, val in field.items():
                    parts = self.parse_jsonpath(val)
                    meta[key] = functools.partial(self._parse_jsonpath_field, parts)
                self._field_getter["metadata"] = meta
            else:
                parts = self.parse_jsonpath(field)
                self._field_getter[name] = functools.partial(self._parse_jsonpath_field, parts)
Ejemplo n.º 6
0
    def __init__(self, definition_cfg):
        self.cfg = definition_cfg

        self._event_type = self.cfg.get('event_type')
        if not self._event_type:
            raise MeterDefinitionException(
                _LE("Required field event_type not specified"), self.cfg)
        if isinstance(self._event_type, six.string_types):
            self._event_type = [self._event_type]

        if ('type' not in self.cfg.get('lookup', []) and
                self.cfg['type'] not in sample.TYPES):
            raise MeterDefinitionException(
                _LE("Invalid type %s specified") % self.cfg['type'], self.cfg)

        self._field_getter = {}
        for name, field in self.cfg.items():
            if name in ["event_type", "lookup"] or not field:
                continue
            elif isinstance(field, six.integer_types):
                self._field_getter[name] = field
            elif isinstance(field, dict) and name == 'metadata':
                meta = {}
                for key, val in field.items():
                    parts = self.parse_jsonpath(val)
                    meta[key] = functools.partial(self._parse_jsonpath_field,
                                                  parts)
                self._field_getter['metadata'] = meta
            else:
                parts = self.parse_jsonpath(field)
                self._field_getter[name] = functools.partial(
                    self._parse_jsonpath_field, parts)
Ejemplo n.º 7
0
    def __init__(self, definition_cfg, default_archive_policy, plugin_manager):
        self._default_archive_policy = default_archive_policy
        self.cfg = definition_cfg

        for field, field_type in self.MANDATORY_FIELDS.items():
            if field not in self.cfg:
                raise declarative.DefinitionException(
                    _LE("Required field %s not specified") % field, self.cfg)
            if not isinstance(self.cfg[field], field_type):
                raise declarative.DefinitionException(
                    _LE("Required field %(field)s should be a %(type)s") %
                    {'field': field, 'type': field_type}, self.cfg)

        self._attributes = {}
        for name, attr_cfg in self.cfg.get('attributes', {}).items():
            self._attributes[name] = declarative.Definition(name, attr_cfg,
                                                            plugin_manager)

        self.metrics = {}
        for t in self.cfg['metrics']:
            archive_policy = self.cfg.get('archive_policy',
                                          self._default_archive_policy)
            if archive_policy is None:
                self.metrics[t] = {}
            else:
                self.metrics[t] = dict(archive_policy_name=archive_policy)
Ejemplo n.º 8
0
    def __init__(self, conf, parsed_url):
        super(FilePublisher, self).__init__(conf, parsed_url)

        self.publisher_logger = None
        path = parsed_url.path
        if not path:
            LOG.error(_LE('The path for the file publisher is required'))
            return

        rfh = None
        max_bytes = 0
        backup_count = 0
        # Handling other configuration options in the query string
        if parsed_url.query:
            params = urlparse.parse_qs(parsed_url.query)
            if params.get('max_bytes') and params.get('backup_count'):
                try:
                    max_bytes = int(params.get('max_bytes')[0])
                    backup_count = int(params.get('backup_count')[0])
                except ValueError:
                    LOG.error(_LE('max_bytes and backup_count should be '
                                  'numbers.'))
                    return
        # create rotating file handler
        rfh = logging.handlers.RotatingFileHandler(
            path, encoding='utf8', maxBytes=max_bytes,
            backupCount=backup_count)

        self.publisher_logger = logging.Logger('publisher.file')
        self.publisher_logger.propagate = False
        self.publisher_logger.setLevel(logging.INFO)
        rfh.setLevel(logging.INFO)
        self.publisher_logger.addHandler(rfh)
Ejemplo n.º 9
0
    def __init__(self, definition_cfg, plugin_manager):
        self.cfg = definition_cfg
        missing = [field for field in self.REQUIRED_FIELDS if not self.cfg.get(field)]
        if missing:
            raise declarative.MeterDefinitionException(_LE("Required fields %s not specified") % missing, self.cfg)

        self._event_type = self.cfg.get("event_type")
        if isinstance(self._event_type, six.string_types):
            self._event_type = [self._event_type]

        if "type" not in self.cfg.get("lookup", []) and self.cfg["type"] not in sample.TYPES:
            raise declarative.MeterDefinitionException(_LE("Invalid type %s specified") % self.cfg["type"], self.cfg)

        self._fallback_user_id = declarative.Definition("user_id", "_context_user_id|_context_user", plugin_manager)
        self._fallback_project_id = declarative.Definition(
            "project_id", "_context_tenant_id|_context_tenant", plugin_manager
        )
        self._attributes = {}
        self._metadata_attributes = {}

        for name in self.SAMPLE_ATTRIBUTES:
            attr_cfg = self.cfg.get(name)
            if attr_cfg:
                self._attributes[name] = declarative.Definition(name, attr_cfg, plugin_manager)
        metadata = self.cfg.get("metadata", {})
        for name in metadata:
            self._metadata_attributes[name] = declarative.Definition(name, metadata[name], plugin_manager)

        # List of fields we expected when multiple meter are in the payload
        self.lookup = self.cfg.get("lookup")
        if isinstance(self.lookup, six.string_types):
            self.lookup = [self.lookup]
Ejemplo n.º 10
0
    def __init__(self, definition_cfg, default_archive_policy,
                 legacy_archive_policy_definition):
        self._default_archive_policy = default_archive_policy
        self._legacy_archive_policy_definition =\
            legacy_archive_policy_definition
        self.cfg = definition_cfg

        for field, field_type in self.MANDATORY_FIELDS.items():
            if field not in self.cfg:
                raise ResourcesDefinitionException(
                    _LE("Required field %s not specified") % field, self.cfg)
            if not isinstance(self.cfg[field], field_type):
                raise ResourcesDefinitionException(
                    _LE("Required field %(field)s should be a %(type)s") %
                    {'field': field, 'type': field_type}, self.cfg)

        self._field_getter = {}
        for name, fval in self.cfg.get('attributes', {}).items():
            if isinstance(fval, six.integer_types):
                self._field_getter[name] = fval
            else:
                try:
                    parts = self.JSONPATH_RW_PARSER.parse(fval)
                except Exception as e:
                    raise ResourcesDefinitionException(
                        _LE("Parse error in JSONPath specification "
                            "'%(jsonpath)s': %(err)s")
                        % dict(jsonpath=fval, err=e), self.cfg)
                self._field_getter[name] = functools.partial(
                    self._parse_jsonpath_field, parts)
Ejemplo n.º 11
0
 def _validate(self):
     for field, field_type in self.MANDATORY_FIELDS.items():
         if field not in self.cfg:
             raise ResourcesDefinitionException(
                 _LE("Required field %s not specified") % field, self.cfg)
         if not isinstance(self.cfg[field], field_type):
             raise ResourcesDefinitionException(
                 _LE("Required field %(field)s should be a %(type)s") %
                 {'field': field, 'type': field_type}, self.cfg)
Ejemplo n.º 12
0
 def _check_required_and_types(expected, definition):
     for field, field_type in expected.items():
         if field not in definition:
             raise declarative.ResourceDefinitionException(
                 _LE("Required field %s not specified") % field, definition)
         if not isinstance(definition[field], field_type):
             raise declarative.ResourceDefinitionException(
                 _LE("Required field %(field)s should be a %(type)s") %
                 {'field': field, 'type': field_type}, definition)
Ejemplo n.º 13
0
 def _catch_extension_load_error(mgr, ep, exc):
     # Extension raising ExtensionLoadError can be ignored,
     # and ignore anything we can't import as a safety measure.
     if isinstance(exc, plugin_base.ExtensionLoadError):
         LOG.exception(_LE("Skip loading extension for %s"), ep.name)
         return
     if isinstance(exc, ImportError):
         LOG.error(_LE("Failed to import extension for %(name)s: "
                       "%(error)s"),
                   {'name': ep.name, 'error': exc})
         return
     raise exc
Ejemplo n.º 14
0
    def _ensure_connection(self):
        if self._producer:
            return

        try:
            self._producer = kafka.KafkaProducer(bootstrap_servers=["%s:%s" % (self._host, self._port)])
        except kafka.errors.KafkaError as e:
            LOG.exception(_LE("Failed to connect to Kafka service: %s"), e)
            raise messaging.DeliveryFailure("Kafka Client is not available, " "please restart Kafka client")
        except Exception as e:
            LOG.exception(_LE("Failed to connect to Kafka service: %s"), e)
            raise messaging.DeliveryFailure("Kafka Client is not available, " "please restart Kafka client")
Ejemplo n.º 15
0
    def get_samples(self, manager, cache, resources=None):
        """Return an iterable of Sample instances from polling the resources.

        :param manager: The service manager invoking the plugin
        :param cache: A dictionary for passing data between plugins
        :param resources: end point to poll data from
        """
        resources = resources or []
        h_cache = cache.setdefault(self.CACHE_KEY, {})
        sample_iters = []

        # Get the meter identifiers to poll
        identifier = self.meter_definition.name

        for resource in resources:
            parsed_url, res, extra_metadata = self._parse_resource(resource)
            if parsed_url is None:
                LOG.error(_LE("Skip invalid resource %s"), resource)
                continue
            ins = self._get_inspector(parsed_url)
            try:
                # Call hardware inspector to poll for the data
                i_cache = h_cache.setdefault(res, {})

                # Prepare inspector parameters and cache it for performance
                param_key = parsed_url.scheme + '.' + identifier
                inspector_param = self.cached_inspector_params.get(param_key)
                if not inspector_param:
                    param = getattr(self.meter_definition,
                                    parsed_url.scheme + '_inspector', {})
                    inspector_param = ins.prepare_params(param)
                    self.cached_inspector_params[param_key] = inspector_param

                if identifier not in i_cache:
                    i_cache[identifier] = list(ins.inspect_generic(
                        host=parsed_url,
                        cache=i_cache,
                        extra_metadata=extra_metadata,
                        param=inspector_param))
                # Generate samples
                if i_cache[identifier]:
                    sample_iters.append(self.generate_samples(
                        parsed_url,
                        i_cache[identifier]))
            except Exception as err:
                LOG.exception(_LE('inspector call failed for %(ident)s '
                                  'host %(host)s: %(err)s'),
                              dict(ident=identifier,
                                   host=parsed_url.hostname,
                                   err=err))
        return itertools.chain(*sample_iters)
Ejemplo n.º 16
0
    def refresh_pipeline(self):
        """Refreshes appropriate pipeline, then delegates to agent."""

        if self.conf.refresh_pipeline_cfg:
            manager = None
            if hasattr(self, 'pipeline_manager'):
                manager = self.pipeline_manager
            elif hasattr(self, 'polling_manager'):
                manager = self.polling_manager
            pipeline_hash = manager.cfg_changed() if manager else None
            if pipeline_hash:
                try:
                    LOG.debug("Pipeline has been refreshed. "
                              "old hash: %(old)s, new hash: %(new)s",
                              {'old': manager.cfg_hash,
                               'new': pipeline_hash})
                    # Pipeline in the notification agent.
                    if hasattr(self, 'pipeline_manager'):
                        self.pipeline_manager = pipeline.setup_pipeline(
                            self.conf)
                    # Polling in the polling agent.
                    elif hasattr(self, 'polling_manager'):
                        self.polling_manager = pipeline.setup_polling(
                            self.conf)
                    self.pipeline_validated = True
                except Exception as err:
                    LOG.exception(_LE('Unable to load changed pipeline: %s')
                                  % err)

        if self.conf.refresh_event_pipeline_cfg:
            # Pipeline in the notification agent.
            manager = (self.event_pipeline_manager
                       if hasattr(self, 'event_pipeline_manager') else None)
            ev_pipeline_hash = manager.cfg_changed()
            if ev_pipeline_hash:
                try:
                    LOG.debug("Event Pipeline has been refreshed. "
                              "old hash: %(old)s, new hash: %(new)s",
                              {'old': manager.cfg_hash,
                               'new': ev_pipeline_hash})
                    self.event_pipeline_manager = (
                        pipeline. setup_event_pipeline(self.conf))
                    self.event_pipeline_validated = True
                except Exception as err:
                    LOG.exception(_LE('Unable to load changed event pipeline:'
                                      ' %s') % err)

        if self.pipeline_validated or self.event_pipeline_validated:
            self.reload_pipeline()
            self.clear_pipeline_validation_status()
Ejemplo n.º 17
0
    def refresh_pipeline(self):
        """Refreshes appropriate pipeline, then delegates to agent."""

        if cfg.CONF.refresh_pipeline_cfg:
            pipeline_hash = self.pipeline_changed()
            if pipeline_hash:
                try:
                    # Pipeline in the notification agent.
                    if hasattr(self, 'pipeline_manager'):
                        self.pipeline_manager = pipeline.setup_pipeline()
                    # Polling in the polling agent.
                    elif hasattr(self, 'polling_manager'):
                        self.polling_manager = pipeline.setup_polling()
                    LOG.debug("Pipeline has been refreshed. "
                              "old hash: %(old)s, new hash: %(new)s",
                              {'old': self.pipeline_hash,
                               'new': pipeline_hash})
                    self.set_pipeline_hash(pipeline_hash)
                    self.pipeline_validated = True
                except Exception as err:
                    LOG.debug("Active pipeline config's hash is %s",
                              self.pipeline_hash)
                    LOG.exception(_LE('Unable to load changed pipeline: %s')
                                  % err)

        if cfg.CONF.refresh_event_pipeline_cfg:
            ev_pipeline_hash = self.pipeline_changed(pipeline.EVENT_TYPE)
            if ev_pipeline_hash:
                try:
                    # Pipeline in the notification agent.
                    if hasattr(self, 'event_pipeline_manager'):
                        self.event_pipeline_manager = (pipeline.
                                                       setup_event_pipeline())

                    LOG.debug("Event Pipeline has been refreshed. "
                              "old hash: %(old)s, new hash: %(new)s",
                              {'old': self.event_pipeline_hash,
                               'new': ev_pipeline_hash})
                    self.set_pipeline_hash(ev_pipeline_hash,
                                           pipeline.EVENT_TYPE)
                    self.event_pipeline_validated = True
                except Exception as err:
                    LOG.debug("Active event pipeline config's hash is %s",
                              self.event_pipeline_hash)
                    LOG.exception(_LE('Unable to load changed event pipeline:'
                                      ' %s') % err)

        if self.pipeline_validated or self.event_pipeline_validated:
            self.reload_pipeline()
            self.clear_pipeline_validation_status()
Ejemplo n.º 18
0
    def record_metering_data(self, data):
        # We may have receive only one counter on the wire
        if not data:
            return
        if not isinstance(data, list):
            data = [data]

        for meter in data:
            LOG.debug(
                'metering data %(counter_name)s '
                'for %(resource_id)s @ %(timestamp)s: %(counter_volume)s',
                {'counter_name': meter['counter_name'],
                 'resource_id': meter['resource_id'],
                 'timestamp': meter.get('timestamp', 'NO TIMESTAMP'),
                 'counter_volume': meter['counter_volume']})
            # Convert the timestamp to a datetime instance.
            # Storage engines are responsible for converting
            # that value to something they can store.
            if meter.get('timestamp'):
                ts = timeutils.parse_isotime(meter['timestamp'])
                meter['timestamp'] = timeutils.normalize_time(ts)
        try:
            self.conn.record_metering_data_batch(data)
        except Exception as err:
            LOG.error(_LE('Failed to record %(len)s: %(err)s.'),
                      {'len': len(data), 'err': err})
            raise
Ejemplo n.º 19
0
    def _load_definitions():
        plugin_manager = extension.ExtensionManager(
            namespace='ceilometer.event.trait_plugin')
        meters_cfg = declarative.load_definitions(
            {}, cfg.CONF.meter.meter_definitions_cfg_file,
            pkg_resources.resource_filename(__name__, "data/meters.yaml"))

        definitions = {}
        for meter_cfg in reversed(meters_cfg['metric']):
            if meter_cfg.get('name') in definitions:
                # skip duplicate meters
                LOG.warning(_LW("Skipping duplicate meter definition %s")
                            % meter_cfg)
                continue
            if (meter_cfg.get('volume') != 1
                    or not cfg.CONF.notification.disable_non_metric_meters):
                try:
                    md = MeterDefinition(meter_cfg, plugin_manager)
                except declarative.DefinitionException as me:
                    errmsg = (_LE("Error loading meter definition : %(err)s")
                              % dict(err=six.text_type(me)))
                    LOG.error(errmsg)
                else:
                    definitions[meter_cfg['name']] = md
        return definitions.values()
Ejemplo n.º 20
0
 def get_samples(self, manager, cache, resources):
     for instance in resources:
         try:
             disk_iops_info = self._populate_cache(
                 self.inspector,
                 cache,
                 instance,
             )
             for disk_iops in self._get_samples(instance,
                                                disk_iops_info):
                 yield disk_iops
         except virt_inspector.InstanceNotFoundException as err:
             # Instance was deleted while getting samples. Ignore it.
             LOG.debug('Exception while getting samples %s', err)
         except ceilometer.NotImplementedError:
             # Selected inspector does not implement this pollster.
             LOG.debug('%(inspector)s does not provide data for '
                       '%(pollster)s',
                       {'inspector': self.inspector.__class__.__name__,
                        'pollster': self.__class__.__name__})
             raise plugin_base.PollsterPermanentError(resources)
         except Exception as err:
             instance_name = util.instance_name(instance)
             LOG.exception(_LE('Ignoring instance %(name)s: %(error)s'),
                           {'name': instance_name, 'error': err})
Ejemplo n.º 21
0
    def record_events(self, event_models):
        """Write the events to database.

        :param event_models: a list of models.Event objects.
        """
        error = None
        for event_model in event_models:
            traits = []
            if event_model.traits:
                for trait in event_model.traits:
                    traits.append({'trait_name': trait.name,
                                   'trait_type': trait.dtype,
                                   'trait_value': trait.value})
            try:
                self.db.event.insert_one(
                    {'_id': event_model.message_id,
                     'event_type': event_model.event_type,
                     'timestamp': event_model.generated,
                     'traits': traits, 'raw': event_model.raw})
            except pymongo.errors.DuplicateKeyError as ex:
                LOG.info(_LI("Duplicate event detected, skipping it: %s") % ex)
            except Exception as ex:
                LOG.exception(_LE("Failed to record event: %s") % ex)
                error = ex
        if error:
            raise error
Ejemplo n.º 22
0
    def __init__(self, conf, cfg, transformer_manager, publisher_manager):
        self.conf = conf
        self.cfg = cfg

        try:
            self.name = cfg['name']
            # It's legal to have no transformer specified
            self.transformer_cfg = cfg.get('transformers') or []
        except KeyError as err:
            raise PipelineException(
                "Required field %s not specified" % err.args[0], cfg)

        if not cfg.get('publishers'):
            raise PipelineException("No publisher specified", cfg)

        self.publishers = []
        for p in cfg['publishers']:
            if '://' not in p:
                # Support old format without URL
                p = p + "://"

            try:
                self.publishers.append(publisher_manager.get(p))
            except Exception:
                LOG.error(_LE("Unable to load publisher %s"), p,
                          exc_info=True)

        self.multi_publish = True if len(self.publishers) > 1 else False
        self.transformers = self._setup_transformers(cfg, transformer_manager)
Ejemplo n.º 23
0
    def record_events(self, events):

        def _build_bulk_index(event_list):
            for ev in event_list:
                traits = {t.name: t.value for t in ev.traits}
                yield {'_op_type': 'create',
                       '_index': '%s_%s' % (self.index_name,
                                            ev.generated.date().isoformat()),
                       '_type': ev.event_type,
                       '_id': ev.message_id,
                       '_source': {'timestamp': ev.generated.isoformat(),
                                   'traits': traits,
                                   'raw': ev.raw}}

        error = None
        for ok, result in helpers.streaming_bulk(
                self.conn, _build_bulk_index(events)):
            if not ok:
                __, result = result.popitem()
                if result['status'] == 409:
                    LOG.info(_LI('Duplicate event detected, skipping it: %s'),
                             result)
                else:
                    LOG.exception(_LE('Failed to record event: %s'), result)
                    error = storage.StorageUnknownWriteError(result)

        if self._refresh_on_write:
            self.conn.indices.refresh(index='%s_*' % self.index_name)
            while self.conn.cluster.pending_tasks(local=True)['tasks']:
                pass
        if error:
            raise error
Ejemplo n.º 24
0
    def record_events(self, events):
        for event in events:
            rd = self._get_resource_definition_from_event(event['event_type'])
            if not rd:
                LOG.debug("No gnocchi definition for event type: %s",
                          event['event_type'])
                continue

            rd, operation = rd
            resource_type = rd.cfg['resource_type']
            resource = rd.event_attributes(event)

            if operation == EVENT_DELETE:
                ended_at = timeutils.utcnow().isoformat()
                resources_to_end = [resource]
                extra_resources = cfg['event_associated_resources'].items()
                for resource_type, filters in extra_resources:
                    resources_to_end.extend(self._gnocchi.search_resource(
                        resource_type, filters['query'] % resource['id']))
                for resource in resources_to_end:
                    try:
                        self._gnocchi.update_resource(resource_type,
                                                      resource['id'],
                                                      {'ended_at': ended_at})
                    except gnocchi_exc.NoSuchResource:
                        LOG.debug(_("Delete event received on unexiting "
                                    "resource (%s), ignore it.") %
                                  resource['id'])
                    except Exception:
                        LOG.error(_LE("Fail to update the resource %s") %
                                  resource, exc_info=True)
Ejemplo n.º 25
0
    def _publish_samples(self, start, samples):
        """Push samples into pipeline for publishing.

        :param start: The first transformer that the sample will be injected.
                      This is mainly for flush() invocation that transformer
                      may emit samples.
        :param samples: Sample list.

        """

        transformed_samples = []
        if not self.transformers:
            transformed_samples = samples
        else:
            for sample in samples:
                LOG.debug(
                    "Pipeline %(pipeline)s: Transform sample "
                    "%(smp)s from %(trans)s transformer", {'pipeline': self,
                                                           'smp': sample,
                                                           'trans': start})
                sample = self._transform_sample(start, sample)
                if sample:
                    transformed_samples.append(sample)

        if transformed_samples:
            for p in self.publishers:
                try:
                    p.publish_samples(transformed_samples)
                except Exception:
                    LOG.error(_LE("Pipeline %(pipeline)s: Continue after "
                                  "error from publisher %(pub)s")
                              % {'pipeline': self, 'pub': p},
                              exc_info=True)
Ejemplo n.º 26
0
 def get_samples(self, manager, cache, resources):
     self._inspection_duration = self._record_poll_time()
     for instance in resources:
         try:
             c_data = self._populate_cache(
                 self.inspector,
                 cache,
                 instance,
             )
             for s in self._get_samples(instance, c_data):
                 yield s
         except virt_inspector.InstanceNotFoundException as err:
             # Instance was deleted while getting samples. Ignore it.
             LOG.debug('Exception while getting samples %s', err)
         except virt_inspector.InstanceShutOffException as e:
             LOG.debug('Instance %(instance_id)s was shut off while '
                       'getting samples of %(pollster)s: %(exc)s',
                       {'instance_id': instance.id,
                        'pollster': self.__class__.__name__, 'exc': e})
         except virt_inspector.NoDataException as e:
             LOG.warning(_LW('Cannot inspect data of %(pollster)s for '
                             '%(instance_id)s, non-fatal reason: %(exc)s'),
                         {'pollster': self.__class__.__name__,
                          'instance_id': instance.id, 'exc': e})
             raise plugin_base.PollsterPermanentError(resources)
         except ceilometer.NotImplementedError:
             # Selected inspector does not implement this pollster.
             LOG.debug('Obtaining memory bandwidth is not implemented'
                       ' for %s', self.inspector.__class__.__name__)
         except Exception as err:
             LOG.exception(_LE('Could not get memory bandwidth for '
                               '%(id)s: %(e)s'), {'id': instance.id,
                                                  'e': err})
Ejemplo n.º 27
0
    def record_metering_data(self, data):
        # We may have receive only one counter on the wire
        if not isinstance(data, list):
            data = [data]

        for meter in data:
            LOG.debug(
                "metering data %(counter_name)s " "for %(resource_id)s @ %(timestamp)s: %(counter_volume)s",
                {
                    "counter_name": meter["counter_name"],
                    "resource_id": meter["resource_id"],
                    "timestamp": meter.get("timestamp", "NO TIMESTAMP"),
                    "counter_volume": meter["counter_volume"],
                },
            )
            if publisher_utils.verify_signature(meter, self.conf.publisher.telemetry_secret):
                try:
                    # Convert the timestamp to a datetime instance.
                    # Storage engines are responsible for converting
                    # that value to something they can store.
                    if meter.get("timestamp"):
                        ts = timeutils.parse_isotime(meter["timestamp"])
                        meter["timestamp"] = timeutils.normalize_time(ts)
                    self.meter_conn.record_metering_data(meter)
                except Exception as err:
                    LOG.exception(_LE("Failed to record metering data: %s"), err)
                    # raise the exception to propagate it up in the chain.
                    raise
            else:
                LOG.warning(_LW("message signature invalid, discarding message: %r"), meter)
Ejemplo n.º 28
0
 def get_connection(purpose):
     try:
         return storage.get_connection_from_config(cfg.CONF, purpose)
     except Exception as err:
         params = {"purpose": purpose, "err": err}
         LOG.exception(_LE("Failed to connect to db, purpose %(purpose)s "
                           "retry later: %(err)s") % params)
Ejemplo n.º 29
0
    def record_events(self, event_models):
        """Write the events to Hbase.

        :param event_models: a list of models.Event objects.
        """
        error = None
        with self.conn_pool.connection() as conn:
            events_table = conn.table(self.EVENT_TABLE)
            for event_model in event_models:
                # Row key consists of timestamp and message_id from
                # models.Event or purposes of storage event sorted by
                # timestamp in the database.
                ts = event_model.generated
                row = hbase_utils.prepare_key(hbase_utils.timestamp(ts, reverse=False), event_model.message_id)
                event_type = event_model.event_type
                traits = {}
                if event_model.traits:
                    for trait in event_model.traits:
                        key = hbase_utils.prepare_key(trait.name, trait.dtype)
                        traits[key] = trait.value
                record = hbase_utils.serialize_entry(traits, event_type=event_type, timestamp=ts, raw=event_model.raw)
                try:
                    events_table.put(row, record)
                except Exception as ex:
                    LOG.exception(_LE("Failed to record event: %s") % ex)
                    error = ex
        if error:
            raise error
Ejemplo n.º 30
0
    def extract_my_subset(self, group_id, iterable, attempt=0):
        """Filters an iterable, returning only objects assigned to this agent.

        We have a list of objects and get a list of active group members from
        `tooz`. We then hash all the objects into buckets and return only
        the ones that hashed into *our* bucket.
        """
        if not group_id:
            return iterable
        if group_id not in self._groups:
            self.join_group(group_id)
        try:
            members = self._get_members(group_id)
            LOG.debug('Members of group: %s, Me: %s', members, self._my_id)
            if self._my_id not in members:
                raise tooz.coordination.MemberNotJoined(group_id, self._my_id)
            hr = utils.HashRing(members)
            filtered = [v for v in iterable
                        if hr.get_node(str(v)) == self._my_id]
            LOG.debug('My subset: %s', [str(f) for f in filtered])
            return filtered
        except tooz.coordination.MemberNotJoined:
            if attempt >= 5:
                raise
            LOG.warning(_LW('Cannot extract tasks because agent failed to '
                            'join group properly. Rejoining group.'))
            self.join_group(group_id)
            return self.extract_my_subset(group_id, iterable, attempt + 1)
        except tooz.coordination.ToozError:
            LOG.exception(_LE('Error getting group membership info from '
                              'coordination backend.'))
            return []
Ejemplo n.º 31
0
    def discover(self, manager, param=None):
        """Discover resources to monitor.

        instance_get_all will return all instances if last_run is None,
        and will return only the instances changed since the last_run time.
        """
        try:
            instances = self.nova_cli.instance_get_all(self.last_run)
        except Exception:
            # NOTE(zqfan): instance_get_all is wrapped and will log exception
            # when there is any error. It is no need to raise it again and
            # print one more time.
            return []

        for instance in instances:
            if getattr(instance, 'OS-EXT-STS:vm_state',
                       None) in ['deleted', 'error']:
                self.instances.pop(instance.id, None)
            else:
                self.instances[instance.id] = instance
        self.last_run = timeutils.utcnow(True).isoformat()

        resources = []
        for instance in self.instances.values():
            try:
                ip_address = self._address(instance, 'addr')
                final_address = self._make_resource_url(ip_address)

                resource = {
                    'resource_id': instance.id,
                    'resource_url': final_address,
                    'mac_addr': self._address(instance,
                                              'OS-EXT-IPS-MAC:mac_addr'),
                    'image_id': instance.image['id'],
                    'flavor_id': instance.flavor['id']
                }

                resources.append(resource)
            except KeyError:
                LOG.error(
                    _LE("Couldn't obtain IP address of "
                        "instance %s") % instance.id)

        return resources
Ejemplo n.º 32
0
    def discover(self, discovery=None, discovery_cache=None):
        resources = []
        discovery = discovery or []
        for url in discovery:
            if discovery_cache is not None and url in discovery_cache:
                resources.extend(discovery_cache[url])
                continue
            name, param = self._parse_discoverer(url)
            discoverer = self._discoverer(name)
            if discoverer:
                try:
                    if discoverer.KEYSTONE_REQUIRED_FOR_SERVICE:
                        service_type = getattr(
                            cfg.CONF.service_types,
                            discoverer.KEYSTONE_REQUIRED_FOR_SERVICE)
                        if not self.keystone.service_catalog.get_endpoints(
                                service_type=service_type):
                            LOG.warning(
                                _LW('Skipping %(name)s, %(service_type)s service '
                                    'is not registered in keystone'), {
                                        'name': name,
                                        'service_type': service_type
                                    })
                            continue

                    discovered = discoverer.discover(self, param)
                    partitioned = self.partition_coordinator.extract_my_subset(
                        self.construct_group_id(discoverer.group_id),
                        discovered)
                    resources.extend(partitioned)
                    if discovery_cache is not None:
                        discovery_cache[url] = partitioned
                except ks_exceptions.ClientException as e:
                    LOG.error(
                        _LE('Skipping %(name)s, keystone issue: '
                            '%(exc)s'), {
                                'name': name,
                                'exc': e
                            })
                except Exception as err:
                    LOG.exception(_('Unable to discover resources: %s') % err)
            else:
                LOG.warning(_('Unknown discovery extension: %s') % name)
        return resources
Ejemplo n.º 33
0
 def get_samples(self, manager, cache, resources):
     self._inspection_duration = self._record_poll_time()
     for instance in resources:
         try:
             c_data = self._populate_cache(
                 self.inspector,
                 cache,
                 instance,
             )
             for s in self._get_samples(instance, c_data):
                 yield s
         except virt_inspector.InstanceNotFoundException as err:
             # Instance was deleted while getting samples. Ignore it.
             LOG.debug('Exception while getting samples %s', err)
         except virt_inspector.InstanceShutOffException as e:
             LOG.debug(
                 'Instance %(instance_id)s was shut off while '
                 'getting samples of %(pollster)s: %(exc)s', {
                     'instance_id': instance.id,
                     'pollster': self.__class__.__name__,
                     'exc': e
                 })
         except virt_inspector.NoDataException as e:
             LOG.warning(
                 _LW('Cannot inspect data of %(pollster)s for '
                     '%(instance_id)s, non-fatal reason: %(exc)s'), {
                         'pollster': self.__class__.__name__,
                         'instance_id': instance.id,
                         'exc': e
                     })
             raise plugin_base.PollsterPermanentError(resources)
         except ceilometer.NotImplementedError:
             # Selected inspector does not implement this pollster.
             LOG.debug(
                 'Obtaining memory bandwidth is not implemented'
                 ' for %s', self.inspector.__class__.__name__)
         except Exception as err:
             LOG.exception(
                 _LE('Could not get memory bandwidth for '
                     '%(id)s: %(e)s'), {
                         'id': instance.id,
                         'e': err
                     })
Ejemplo n.º 34
0
 def parse_fields(self, field, message):
     fval = self.cfg.get(field)
     if not fval:
         return
     if isinstance(fval, six.integer_types):
         return fval
     try:
         parts = jsonpath_rw.parse(fval)
     except Exception as e:
         raise MeterDefinitionException(
             _LE("Parse error in JSONPath specification "
                 "'%(jsonpath)s': %(err)s") % dict(jsonpath=parts, err=e),
             self.cfg)
     values = [
         match.value for match in parts.find(message)
         if match.value is not None
     ]
     if values:
         return values[0]
Ejemplo n.º 35
0
 def _inner():
     try:
         join_req = self._coordinator.join_group(group_id)
         join_req.get()
         LOG.info(_LI('Joined partitioning group %s'), group_id)
     except tooz.coordination.MemberAlreadyExist:
         return
     except tooz.coordination.GroupNotCreated:
         create_grp_req = self._coordinator.create_group(group_id)
         try:
             create_grp_req.get()
         except tooz.coordination.GroupAlreadyExist:
             pass
         raise ErrorJoiningPartitioningGroup()
     except tooz.coordination.ToozError:
         LOG.exception(_LE('Error joining partitioning group %s,'
                           ' re-trying'), group_id)
         raise ErrorJoiningPartitioningGroup()
     self._groups.add(group_id)
Ejemplo n.º 36
0
 def _parse_field(self, field, sample):
     # TODO(sileht): share this with
     # https://review.openstack.org/#/c/197633/
     if not field:
         return
     if isinstance(field, six.integer_types):
         return field
     try:
         parts = jsonpath_rw.parse(field)
     except Exception as e:
         raise ResourcesDefinitionException(
             _LE("Parse error in JSONPath specification "
                 "'%(jsonpath)s': %(err)s") % dict(jsonpath=field, err=e),
             self.cfg)
     values = [
         match.value for match in parts.find(sample)
         if match.value is not None
     ]
     if values:
         return values[0]
Ejemplo n.º 37
0
    def _load_definitions():
        plugin_manager = extension.ExtensionManager(
            namespace='ceilometer.event.trait_plugin')
        meters_cfg = declarative.load_definitions(
            {}, cfg.CONF.meter.meter_definitions_cfg_file,
            pkg_resources.resource_filename(__name__, "data/meters.yaml"))

        definitions = []
        for meter_cfg in reversed(meters_cfg['metric']):
            if (meter_cfg['volume'] != 1
                    or not cfg.CONF.notification.disable_non_metric_meters):
                try:
                    md = MeterDefinition(meter_cfg, plugin_manager)
                except declarative.DefinitionException as me:
                    errmsg = (_LE("Error loading meter definition : %(err)s")
                              % dict(err=six.text_type(me)))
                    LOG.error(errmsg)
                else:
                    definitions.append(md)
        return definitions
Ejemplo n.º 38
0
    def record_events(self, events):
        if not isinstance(events, list):
            events = [events]

        for event in events:
            res = None
            try:
                res = requests.post(self.event_target,
                                    data=event,
                                    headers=self.headers,
                                    timeout=self.timeout)
                res.raise_for_status()
            except Exception:
                error_code = res.status_code if res else 'unknown'
                LOG.exception(
                    _LE('Status Code: %{code}s. Failed to dispatch '
                        'event: %{event}s'), {
                            'code': error_code,
                            'event': event
                        })
Ejemplo n.º 39
0
 def post_event(self, event):
     res = None
     try:
         event_json = json.dumps(event)
         LOG.trace('Event Message: %s', event_json)
         res = requests.post(self.event_target,
                             data=event_json,
                             headers=self.headers,
                             verify=self.verify_ssl,
                             timeout=self.timeout)
         LOG.debug('Event Message posting to %s: status code %d.',
                   self.event_target, res.status_code)
         res.raise_for_status()
     except requests.exceptions.HTTPError:
         LOG.exception(
             _LE('Status Code: %(code)s. '
                 'Failed to dispatch event: %(event)s') % {
                     'code': res.status_code,
                     'event': event_json
                 })
Ejemplo n.º 40
0
    def sample(self, messages):
        """RPC endpoint for notification messages

        When another service sends a notification over the message
        bus, this method receives it.
        """
        goods = []
        for sample in chain.from_iterable(m["payload"] for m in messages):
            if publisher_utils.verify_signature(sample, self.secret):
                goods.append(sample)
            else:
                LOG.warning(
                    _LW('notification signature invalid, '
                        'discarding: %s'), sample)
        try:
            self.dispatcher_manager.map_method(self.method, goods)
        except Exception:
            LOG.exception(
                _LE("Dispatcher failed to handle the notification, "
                    "re-queuing it."))
            return oslo_messaging.NotificationResult.REQUEUE
Ejemplo n.º 41
0
 def gnocchi_project_id(self):
     if self._gnocchi_project_id is not None:
         return self._gnocchi_project_id
     with self._gnocchi_project_id_lock:
         if self._gnocchi_project_id is None:
             try:
                 project = self._ks_client.projects.find(
                     name=self.conf.dispatcher_gnocchi.filter_project)
             except ka_exceptions.NotFound:
                 LOG.warning(_LW('gnocchi project not found in keystone,'
                                 ' ignoring the filter_service_activity '
                                 'option'))
                 self.filter_service_activity = False
                 return None
             except Exception:
                 LOG.exception(_LE('fail to retrieve user of Gnocchi '
                                   'service'))
                 raise
             self._gnocchi_project_id = project.id
             LOG.debug("gnocchi project found: %s", self.gnocchi_project_id)
         return self._gnocchi_project_id
Ejemplo n.º 42
0
    def post_meter(self, meter):
        meter_json = json.dumps(meter)
        res = None
        try:
            LOG.trace('Meter Message: %s', meter_json)
            res = requests.post(self.target,
                                data=meter_json,
                                headers=self.headers,
                                verify=self.verify_ssl,
                                timeout=self.timeout)
            LOG.debug('Meter message posting finished with status code '
                      '%d.', res.status_code)
            res.raise_for_status()

        except requests.exceptions.HTTPError:
            LOG.exception(
                _LE('Status Code: %(code)s. '
                    'Failed to dispatch meter: %(meter)s') % {
                        'code': res.status_code,
                        'meter': meter_json
                    })
Ejemplo n.º 43
0
 def sample(self, ctxt, publisher_id, event_type, payload, metadata):
     events = []
     for ev in payload:
         try:
             events.append(
                 models.Event(
                     message_id=ev['message_id'],
                     event_type=ev['event_type'],
                     generated=timeutils.normalize_time(
                         timeutils.parse_isotime(ev['generated'])),
                     traits=[models.Trait(
                             name, dtype,
                             models.Trait.convert_value(dtype, value))
                             for name, dtype, value in ev['traits']],
                     raw=ev.get('raw', {}))
             )
         except Exception:
             LOG.exception(_LE("Error processing event and it will be "
                               "dropped: %s"), ev)
     return super(EventEndpoint, self).sample(
         ctxt, publisher_id, event_type, events, metadata)
Ejemplo n.º 44
0
    def record_events(self, events):
        if not isinstance(events, list):
            events = [events]

        for event in events:
            if publisher_utils.verify_signature(
                    event, self.conf.publisher.telemetry_secret):
                res = None
                try:
                    res = requests.post(self.event_target, data=event,
                                        headers=self.headers,
                                        timeout=self.timeout)
                    res.raise_for_status()
                except Exception:
                    error_code = res.status_code if res else 'unknown'
                    LOG.exception(_LE('Status Code: %{code}s. Failed to'
                                      'dispatch event: %{event}s'),
                                  {'code': error_code, 'event': event})
            else:
                LOG.warning(_LW(
                    'event signature invalid, discarding event: %s'), event)
Ejemplo n.º 45
0
def load_definitions(config_def):
    if not config_def:
        return []
    meter_defs = {}
    for event_def in reversed(config_def['metric']):
        if event_def.get('name') in meter_defs:
            # skip duplicate meters
            LOG.warning(
                _LW("Skipping duplicate meter definition %s") % event_def)
            continue

        try:
            if (event_def['volume'] != 1
                    or not cfg.CONF.notification.disable_non_metric_meters):
                md = MeterDefinition(event_def)
                meter_defs[event_def['name']] = md
        except MeterDefinitionException as me:
            errmsg = (_LE("Error loading meter definition : %(err)s") %
                      dict(err=me.message))
            LOG.error(errmsg)
    return meter_defs.values()
Ejemplo n.º 46
0
    def _do_post(self, data):
        if not data:
            LOG.debug('Data set is empty!')
            return

        session = requests.Session()
        session.mount(self.target,
                      adapters.HTTPAdapter(max_retries=self.max_retries))

        content = ','.join([jsonutils.dumps(item) for item in data])
        content = '[' + content + ']'

        LOG.debug('Data to be posted by HttpPublisher: %s' % content)

        res = session.post(self.target,
                           data=content,
                           headers=self.headers,
                           timeout=self.timeout)
        if res.status_code >= 300:
            LOG.error(
                _LE('Data post failed with status code %s') % res.status_code)
Ejemplo n.º 47
0
 def get_samples(self, manager, cache, resources):
     self._inspection_duration = self._record_poll_time()
     for instance in resources:
         instance_name = util.instance_name(instance)
         LOG.debug('checking net info for instance %s', instance.id)
         try:
             vnics = self._get_vnics_for_instance(
                 cache,
                 self.inspector,
                 instance,
             )
             for vnic, info in vnics:
                 LOG.debug(self.NET_USAGE_MESSAGE, instance_name, vnic.name,
                           self._get_rx_info(info), self._get_tx_info(info))
                 yield self._get_sample(instance, vnic, info)
         except virt_inspector.InstanceNotFoundException as err:
             # Instance was deleted while getting samples. Ignore it.
             LOG.debug('Exception while getting samples %s', err)
         except virt_inspector.InstanceShutOffException as e:
             LOG.debug(
                 'Instance %(instance_id)s was shut off while '
                 'getting samples of %(pollster)s: %(exc)s', {
                     'instance_id': instance.id,
                     'pollster': self.__class__.__name__,
                     'exc': e
                 })
         except ceilometer.NotImplementedError:
             # Selected inspector does not implement this pollster.
             LOG.debug(
                 '%(inspector)s does not provide data for '
                 '%(pollster)s', {
                     'inspector': self.inspector.__class__.__name__,
                     'pollster': self.__class__.__name__
                 })
             raise plugin_base.PollsterPermanentError(resources)
         except Exception as err:
             LOG.exception(_LE('Ignoring instance %(name)s: %(error)s'), {
                 'name': instance_name,
                 'error': err
             })
Ejemplo n.º 48
0
    def extract_my_subset(self, group_id, iterable):
        """Filters an iterable, returning only objects assigned to this agent.

        We have a list of objects and get a list of active group members from
        `tooz`. We then hash all the objects into buckets and return only
        the ones that hashed into *our* bucket.
        """
        if not group_id:
            return iterable
        if group_id not in self._groups:
            self.join_group(group_id)
        try:
            members = self._get_members(group_id)
            LOG.debug('Members of group %s are: %s, Me: %s', group_id, members,
                      self._my_id)
            if self._my_id not in members:
                LOG.warning(
                    _LW('Cannot extract tasks because agent failed to '
                        'join group properly. Rejoining group.'))
                self.join_group(group_id)
                members = self._get_members(group_id)
                if self._my_id not in members:
                    raise MemberNotInGroupError(group_id, members, self._my_id)
                LOG.debug('Members of group %s are: %s, Me: %s', group_id,
                          members, self._my_id)
            hr = utils.HashRing(members)
            iterable = list(iterable)
            filtered = [
                v for v in iterable
                if hr.get_node(six.text_type(v)) == self._my_id
            ]
            LOG.debug('The universal set: %s, my subset: %s',
                      [six.text_type(f) for f in iterable],
                      [six.text_type(f) for f in filtered])
            return filtered
        except tooz.coordination.ToozError:
            LOG.exception(
                _LE('Error getting group membership info from '
                    'coordination backend.'))
            return []
Ejemplo n.º 49
0
    def record_events(self, events):
        def _build_bulk_index(event_list):
            for ev in event_list:
                traits = {t.name: t.value for t in ev.traits}
                yield {
                    '_op_type':
                    'create',
                    '_index':
                    '%s_%s' %
                    (self.index_name, ev.generated.date().isoformat()),
                    '_type':
                    ev.event_type,
                    '_id':
                    ev.message_id,
                    '_source': {
                        'timestamp': ev.generated.isoformat(),
                        'traits': traits,
                        'raw': ev.raw
                    }
                }

        error = None
        for ok, result in helpers.streaming_bulk(self.conn,
                                                 _build_bulk_index(events)):
            if not ok:
                __, result = result.popitem()
                if result['status'] == 409:
                    LOG.info(
                        _LI('Duplicate event detected, skipping it: %s') %
                        result)
                else:
                    LOG.exception(_LE('Failed to record event: %s') % result)
                    error = storage.StorageUnknownWriteError(result)

        if self._refresh_on_write:
            self.conn.indices.refresh(index='%s_*' % self.index_name)
            while self.conn.cluster.pending_tasks(local=True)['tasks']:
                pass
        if error:
            raise error
Ejemplo n.º 50
0
 def closure(self, *args, **kwargs):
     # NOTE(idegtiarov) options max_retries and retry_interval have been
     # registered in storage.__init__ in oslo_db.options.set_defaults
     # default values for both options are 10.
     max_retries = self.conf.database.max_retries
     retry_interval = self.conf.database.retry_interval
     attempts = 0
     while True:
         try:
             return call(self, *args, **kwargs)
         except pymongo.errors.AutoReconnect as err:
             if 0 <= max_retries <= attempts:
                 LOG.error(_LE('Unable to reconnect to the primary mongodb '
                               'after %(retries)d retries. Giving up.') %
                           {'retries': max_retries})
                 raise
             LOG.warning(_('Unable to reconnect to the primary '
                           'mongodb: %(errmsg)s. Trying again in '
                           '%(retry_interval)d seconds.') %
                         {'errmsg': err, 'retry_interval': retry_interval})
             attempts += 1
             time.sleep(retry_interval)
Ejemplo n.º 51
0
 def get_samples(self, manager, cache, resources):
     self._inspection_duration = self._record_poll_time()
     for instance in resources:
         LOG.debug('Checking resident memory for instance %s',
                   instance.id)
         try:
             memory_info = self.inspector.inspect_memory_resident(
                 instance, self._inspection_duration)
             LOG.debug("RESIDENT MEMORY: %(instance)s %(resident)f",
                       {'instance': instance,
                        'resident': memory_info.resident})
             yield util.make_sample_from_instance(
                 instance,
                 name='memory.resident',
                 type=sample.TYPE_GAUGE,
                 unit='MB',
                 volume=memory_info.resident,
             )
         except virt_inspector.InstanceNotFoundException as err:
             # Instance was deleted while getting samples. Ignore it.
             LOG.debug('Exception while getting samples %s', err)
         except virt_inspector.InstanceShutOffException as e:
             LOG.warn(_LW('Instance %(instance_id)s was shut off while '
                          'getting samples of %(pollster)s: %(exc)s'),
                      {'instance_id': instance.id,
                       'pollster': self.__class__.__name__, 'exc': e})
         except virt_inspector.NoDataException as e:
             LOG.warn(_LW('Cannot inspect data of %(pollster)s for '
                          '%(instance_id)s, non-fatal reason: %(exc)s'),
                      {'pollster': self.__class__.__name__,
                       'instance_id': instance.id, 'exc': e})
         except ceilometer.NotImplementedError:
             # Selected inspector does not implement this pollster.
             LOG.debug('Obtaining Resident Memory is not implemented'
                       ' for %s', self.inspector.__class__.__name__)
         except Exception as err:
             LOG.exception(_LE('Could not get Resident Memory Usage for '
                               '%(id)s: %(e)s'), {'id': instance.id,
                                                  'e': err})
Ejemplo n.º 52
0
 def _do_post(self, data):
     if not data:
         LOG.debug('Data set is empty!')
         return
     data = jsonutils.dumps(data)
     LOG.trace('Message: %s', data)
     try:
         res = self.session.post(self.target,
                                 data=data,
                                 headers=self.headers,
                                 timeout=self.timeout,
                                 verify=self.verify_ssl)
         res.raise_for_status()
         LOG.debug('Message posting to %s: status code %d.', self.target,
                   res.status_code)
     except requests.exceptions.HTTPError:
         LOG.exception(
             _LE('Status Code: %(code)s. '
                 'Failed to dispatch message: %(data)s') % {
                     'code': res.status_code,
                     'data': data
                 })
Ejemplo n.º 53
0
 def _transform_sample(self, start, sample):
     try:
         for transformer in self.transformers[start:]:
             sample = transformer.handle_sample(sample)
             if not sample:
                 LOG.debug(
                     "Pipeline %(pipeline)s: Sample dropped by "
                     "transformer %(trans)s", {
                         'pipeline': self,
                         'trans': transformer
                     })
                 return
         return sample
     except Exception:
         LOG.error(_LE("Pipeline %(pipeline)s: Exit after error "
                       "from transformer %(trans)s "
                       "for %(smp)s") % {
                           'pipeline': self,
                           'trans': transformer,
                           'smp': sample
                       },
                   exc_info=True)
Ejemplo n.º 54
0
    def _load_resources_definitions(cls, conf):
        res_def_file = cls._get_config_file(
            conf, conf.dispatcher_gnocchi.resources_definition_file)
        data = {}
        if res_def_file is not None:
            with open(res_def_file) as data_file:
                try:
                    data = yaml.safe_load(data_file)
                except ValueError:
                    data = {}

        legacy_archive_policies = cls._load_archive_policy(conf)
        resource_defs = []
        for resource in data.get('resources', []):
            try:
                resource_defs.append(
                    ResourcesDefinition(resource,
                                        conf.dispatcher_gnocchi.archive_policy,
                                        legacy_archive_policies))
            except Exception as exc:
                LOG.error(_LE("Failed to load resource due to error %s") % exc)
        return resource_defs
Ejemplo n.º 55
0
 def _process_queue(self, queue):
     current_retry = 0
     while queue:
         data = queue[0]
         try:
             self._send(data)
         except Exception:
             LOG.warn(_LW("Failed to publish %d datum"),
                      sum([len(d) for d in queue]))
             if self.policy == 'queue':
                 return queue
             elif self.policy == 'drop':
                 return []
             current_retry += 1
             if self.current_retry >= self.max_retry:
                 self.local_queue = []
                 LOG.exception(_LE("Failed to retry to send sample data "
                                   "with max_retry times"))
                 raise
         else:
             queue.pop(0)
     return []
Ejemplo n.º 56
0
 def join_group(self, group_id):
     if (not self._coordinator or not self._coordinator.is_started
             or not group_id):
         return
     while True:
         try:
             join_req = self._coordinator.join_group(group_id)
             join_req.get()
             LOG.info(_LI('Joined partitioning group %s'), group_id)
             break
         except tooz.coordination.MemberAlreadyExist:
             return
         except tooz.coordination.GroupNotCreated:
             create_grp_req = self._coordinator.create_group(group_id)
             try:
                 create_grp_req.get()
             except tooz.coordination.GroupAlreadyExist:
                 pass
         except tooz.coordination.ToozError:
             LOG.exception(_LE('Error joining partitioning group %s,'
                               ' re-trying'), group_id)
     self._groups.add(group_id)
Ejemplo n.º 57
0
    def _load_definitions(self):
        plugin_manager = extension.ExtensionManager(
            namespace='ceilometer.event.trait_plugin')
        meters_cfg = declarative.load_definitions(
            self.manager.conf, {},
            self.manager.conf.meter.meter_definitions_cfg_file,
            pkg_resources.resource_filename(__name__, "data/meters.yaml"))

        definitions = {}
        for meter_cfg in reversed(meters_cfg['metric']):
            if meter_cfg.get('name') in definitions:
                # skip duplicate meters
                LOG.warning(
                    _LW("Skipping duplicate meter definition %s") % meter_cfg)
                continue
            try:
                md = MeterDefinition(meter_cfg, plugin_manager)
            except declarative.DefinitionException as e:
                errmsg = _LE("Error loading meter definition: %s")
                LOG.error(errmsg, six.text_type(e))
            else:
                definitions[meter_cfg['name']] = md
        return definitions.values()
Ejemplo n.º 58
0
    def extract_my_subset(self, group_id, iterable):
        """Filters an iterable, returning only objects assigned to this agent.

        We have a list of objects and get a list of active group members from
        `tooz`. We then hash all the objects into buckets and return only
        the ones that hashed into *our* bucket.
        """
        if not group_id:
            return iterable
        if group_id not in self._groups:
            self.join_group(group_id)
        try:
            members = self._get_members(group_id)
            LOG.debug('Members of group: %s', members)
            hr = utils.HashRing(members)
            filtered = [v for v in iterable
                        if hr.get_node(str(v)) == self._my_id]
            LOG.debug('My subset: %s', [str(f) for f in filtered])
            return filtered
        except tooz.coordination.ToozError:
            LOG.exception(_LE('Error getting group membership info from '
                              'coordination backend.'))
            return []
Ejemplo n.º 59
0
    def _publish_samples(self, start, samples):
        """Push samples into pipeline for publishing.

        :param start: The first transformer that the sample will be injected.
                      This is mainly for flush() invocation that transformer
                      may emit samples.
        :param samples: Sample list.

        """

        transformed_samples = []
        if not self.transformers:
            transformed_samples = samples
        else:
            for sample in samples:
                LOG.debug(
                    "Pipeline %(pipeline)s: Transform sample "
                    "%(smp)s from %(trans)s transformer", {
                        'pipeline': self,
                        'smp': sample,
                        'trans': start
                    })
                sample = self._transform_sample(start, sample)
                if sample:
                    transformed_samples.append(sample)

        if transformed_samples:
            for p in self.publishers:
                try:
                    p.publish_samples(transformed_samples)
                except Exception:
                    LOG.error(_LE("Pipeline %(pipeline)s: Continue after "
                                  "error from publisher %(pub)s") % {
                                      'pipeline': self,
                                      'pub': p
                                  },
                              exc_info=True)
Ejemplo n.º 60
0
 def _process_queue(self, queue, policy):
     current_retry = 0
     while queue:
         topic, data = queue[0]
         try:
             self._send(topic, data)
         except DeliveryFailure:
             data = sum([len(m) for __, m in queue])
             if policy == 'queue':
                 LOG.warning(_("Failed to publish %d datapoints, queue "
                               "them"), data)
                 return queue
             elif policy == 'drop':
                 LOG.warning(_("Failed to publish %d datapoints, "
                             "dropping them"), data)
                 return []
             current_retry += 1
             if current_retry >= self.max_retry:
                 LOG.exception(_LE("Failed to retry to send sample data "
                                   "with max_retry times"))
                 raise
         else:
             queue.pop(0)
     return []