def testImportModel(self):
        adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()

        autostack = adapter.createAutostack(self.stackSpec)

        modelSpec = self.getModelSpec("cloudwatch", "CPUUtilization", autostack)
        modelId = adapter.monitorMetric(modelSpec)

        spec = adapter.exportModel(modelId)
        adapter.unmonitorMetric(modelId)

        modelId = adapter.importModel(spec)
        self.validateModel(modelId, modelSpec, autostack)
        with self.engine.connect() as conn:
            metrics = repository.getAutostackMetrics(conn, autostack.uid)
            self.assertEqual(len([metricObj for metricObj in metrics]), 1)

            # Ensure that import can create an autostack if it doesn't exist
            repository.deleteAutostack(conn, autostack.uid)

        adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()

        modelId = adapter.importModel(spec)
        newModelSpec = dict(modelSpec)

        with self.engine.connect() as conn:
            repository.getMetric(conn, modelId)
            autostack = repository.getAutostackFromMetric(conn, modelId)
        self.addCleanup(self._deleteAutostack, autostack.uid)
        newModelSpec["metricSpec"]["autostackId"] = autostack.uid

        self.validateModel(modelId, modelSpec, autostack)
    def testImportModel(self):
        adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()

        autostack = adapter.createAutostack(self.stackSpec)

        modelSpec = self.getModelSpec("cloudwatch", "CPUUtilization",
                                      autostack)
        modelId = adapter.monitorMetric(modelSpec)

        spec = adapter.exportModel(modelId)
        adapter.unmonitorMetric(modelId)

        modelId = adapter.importModel(spec)
        self.validateModel(modelId, modelSpec, autostack)
        with self.engine.connect() as conn:
            metrics = repository.getAutostackMetrics(conn, autostack.uid)
            self.assertEqual(len([metricObj for metricObj in metrics]), 1)

            # Ensure that import can create an autostack if it doesn't exist
            repository.deleteAutostack(conn, autostack.uid)

        adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()

        modelId = adapter.importModel(spec)
        newModelSpec = dict(modelSpec)

        with self.engine.connect() as conn:
            repository.getMetric(conn, modelId)
            autostack = repository.getAutostackFromMetric(conn, modelId)
        self.addCleanup(self._deleteAutostack, autostack.uid)
        newModelSpec["metricSpec"]["autostackId"] = autostack.uid

        self.validateModel(modelId, modelSpec, autostack)
Example #3
0
  def DELETE(self, autostackId, metricId): # pylint: disable=C0103,R0201
    """
      Remove a specific metric from autostack

      ::

          DELETE /_autostacks/{autostackId}/metrics/{metricId}
    """
    try:
      # The if statement makes sure that the metric belongs to the autostack
      with web.ctx.connFactory() as conn:
        autostackObj = repository.getAutostackFromMetric(conn, metricId)

      if autostackId != autostackObj.uid:
        raise InvalidRequestResponse(
          {"result": "Metric=%s does not belong to autostack=%s"
           % (metricId, autostackId)})

      with web.ctx.connFactory() as conn:
        repository.deleteMetric(conn, metricId)

      model_swapper_utils.deleteHTMModel(metricId)
      raise web.HTTPError(status="204 No Content")
    except ObjectNotFoundError:
      raise web.notfound(("Autostack or metric not found: autostack=%s, "
                          "metric=%s") % (autostackId, metricId))
    except web.HTTPError as ex:
      if bool(re.match(r"([45][0-9][0-9])\s?", web.ctx.status)):
        # Log 400-599 status codes as errors, ignoring 200-399
        log.error(str(ex) or repr(ex))
      raise
    except Exception as ex:
      log.exception("DELETE Failed")
      raise web.internalerror(str(ex) or repr(ex))
    def validateModel(self, modelId, modelSpec, autostack):
        self.assertIsNotNone(modelId)

        with self.engine.connect() as conn:
            metricObj = repository.getMetric(conn, modelId, fields=[schema.metric.c.status, schema.metric.c.parameters])

            self.assertIn(metricObj.status, [MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE])
            self.assertEqual(json.loads(metricObj.parameters), modelSpec)
            self.assertEqual(repository.getAutostackFromMetric(conn, modelId).uid, autostack.uid)
    def validateModel(self, modelId, modelSpec, autostack):
        self.assertIsNotNone(modelId)

        with self.engine.connect() as conn:
            metricObj = repository.getMetric(
                conn,
                modelId,
                fields=[schema.metric.c.status, schema.metric.c.parameters])

            self.assertIn(metricObj.status,
                          [MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE])
            self.assertEqual(json.loads(metricObj.parameters), modelSpec)
            self.assertEqual(
                repository.getAutostackFromMetric(conn, modelId).uid,
                autostack.uid)
Example #6
0
def getStatistics(metric):
    """Get aggregate statistics for an Autostack metric.

  The metric must belong to an Autostack or a ValueError will be raised. If AWS
  returns no stats and there is no data in the database then an
  ObjectNotFoundError will be raised.

  :param metric: the Autostack metric to get statistics for
  :type metric: TODO

  :returns: metric statistics
  :rtype: dict {"min": minVal, "max": maxVal}

  :raises: ValueError if the metric doesn't not belong to an Autostack

  :raises: grok.app.exceptions.ObjectNotFoundError if the metric or the
      corresponding autostack doesn't exist; this may happen if it got deleted
      by another process in the meantime.

  :raises: grok.app.exceptions.MetricStatisticsNotReadyError if there are no or
      insufficent samples at this time; this may also happen if the metric and
      its data were deleted by another process in the meantime
  """
    engine = repository.engineFactory()

    if metric.datasource != "autostack":
        raise ValueError(
            "Metric must belong to an Autostack but has datasource=%r" %
            metric.datasource)
    metricGetter = EC2InstanceMetricGetter()
    try:
        with engine.connect() as conn:
            autostack = repository.getAutostackFromMetric(conn, metric.uid)
        instanceMetricList = metricGetter.collectMetricStatistics(
            autostack, metric)
    finally:
        metricGetter.close()

    n = 0
    mins = 0.0
    maxs = 0.0
    for instanceMetric in instanceMetricList:
        assert len(instanceMetric.records) == 1
        metricRecord = instanceMetric.records[0]
        stats = metricRecord.value

        if (not isinstance(stats["min"], numbers.Number)
                or math.isnan(stats["min"])
                or not isinstance(stats["max"], numbers.Number)
                or math.isnan(stats["max"])):
            # Cloudwatch gave us bogus data for this metric so we will exclude it
            continue

        mins += stats["min"]
        maxs += stats["max"]
        n += 1

    if n == 0:
        # Fall back to metric_data when we don't get anything from AWS. This may
        # raise an MetricStatisticsNotReadyError if there is no or not enough data.
        with engine.connect() as conn:
            dbStats = repository.getMetricStats(conn, metric.uid)
        minVal = dbStats["min"]
        maxVal = dbStats["max"]
    else:
        minVal = mins / n
        maxVal = maxs / n

    # Now add the 20% buffer on the range
    buff = (maxVal - minVal) * 0.2
    minVal -= buff
    maxVal += buff

    return {"min": minVal, "max": maxVal}
Example #7
0
    def exportModel(self, metricId):
        """ Export the given model.

    :param metricId: datasource-specific unique metric identifier

    :returns: Model-export specification for the Autostack model
    :rtype: dict

    ::
        {
          "datasource": "autostack",

          "stackSpec": {
            "name": "all_web_servers",  # Autostack name
            "aggSpec": {  # aggregation spec
              "datasource": "cloudwatch",
              "region": "us-west-2",
              "resourceType": "AWS::EC2::Instance"
              "filters": {  # resourceType-specific filter
                "tag:Name":["*test*", "*grok*"],
                "tag:Description":["Blah", "foo"]
              },
            }
          },

          "modelSpec": {
            "datasource": "autostack",

            "metricSpec": {
              "slaveDatasource": "cloudwatch",

              "slaveMetric": {
                # specific to slaveDatasource
                "namespace": "AWS/EC2",
                "metric": "CPUUtilization"
              },

              "period": 300  # aggregation period; seconds
            },

            "modelParams": { # optional; specific to slave metric
              "min": 0,  # optional
              "max": 100  # optional
            }
          }
        }

    """
        with self.connectionFactory() as conn:
            spec = {}
            spec["datasource"] = self._DATASOURCE

            metricObj = repository.getMetric(
                conn, metricId, fields=[schema.metric.c.parameters])
            autostackObj = repository.getAutostackFromMetric(conn, metricId)

        parameters = htmengine.utils.jsonDecode(metricObj.parameters)
        spec["modelSpec"] = parameters
        modelSpec = spec["modelSpec"]
        metricSpec = modelSpec["metricSpec"]
        del metricSpec["autostackId"]

        spec["stackSpec"] = {}
        stackSpec = spec["stackSpec"]
        stackSpec["name"] = autostackObj.name

        # Only supporting cloudwatch / EC2 for now
        stackSpec["aggSpec"] = {}
        aggSpec = stackSpec["aggSpec"]
        aggSpec["datasource"] = "cloudwatch"
        aggSpec["region"] = autostackObj.region
        aggSpec["resourceType"] = "AWS::EC2::Instance"
        aggSpec["filters"] = htmengine.utils.jsonDecode(autostackObj.filters)

        return spec
Example #8
0
def getStatistics(metric):
  """Get aggregate statistics for an Autostack metric.

  The metric must belong to an Autostack or a ValueError will be raised. If AWS
  returns no stats and there is no data in the database then an
  ObjectNotFoundError will be raised.

  :param metric: the Autostack metric to get statistics for
  :type metric: TODO

  :returns: metric statistics
  :rtype: dict {"min": minVal, "max": maxVal}

  :raises: ValueError if the metric doesn't not belong to an Autostack

  :raises: grok.app.exceptions.ObjectNotFoundError if the metric or the
      corresponding autostack doesn't exist; this may happen if it got deleted
      by another process in the meantime.

  :raises: grok.app.exceptions.MetricStatisticsNotReadyError if there are no or
      insufficent samples at this time; this may also happen if the metric and
      its data were deleted by another process in the meantime
  """
  engine = repository.engineFactory()

  if metric.datasource != "autostack":
    raise ValueError(
      "Metric must belong to an Autostack but has datasource=%r"
      % metric.datasource)
  metricGetter = EC2InstanceMetricGetter()
  try:
    with engine.connect() as conn:
      autostack = repository.getAutostackFromMetric(conn, metric.uid)
    instanceMetricList = metricGetter.collectMetricStatistics(autostack, metric)
  finally:
    metricGetter.close()

  n = 0
  mins = 0.0
  maxs = 0.0
  for instanceMetric in instanceMetricList:
    assert len(instanceMetric.records) == 1
    metricRecord = instanceMetric.records[0]
    stats = metricRecord.value

    if (not isinstance(stats["min"], numbers.Number) or
        math.isnan(stats["min"]) or
        not isinstance(stats["max"], numbers.Number) or
        math.isnan(stats["max"])):
      # Cloudwatch gave us bogus data for this metric so we will exclude it
      continue

    mins += stats["min"]
    maxs += stats["max"]
    n += 1

  if n == 0:
    # Fall back to metric_data when we don't get anything from AWS. This may
    # raise an MetricStatisticsNotReadyError if there is no or not enough data.
    with engine.connect() as conn:
      dbStats = repository.getMetricStats(conn, metric.uid)
    minVal = dbStats["min"]
    maxVal = dbStats["max"]
  else:
    minVal = mins / n
    maxVal = maxs / n

  # Now add the 20% buffer on the range
  buff = (maxVal - minVal) * 0.2
  minVal -= buff
  maxVal += buff

  return {"min": minVal,
          "max": maxVal}
Example #9
0
  def exportModel(self, metricId):
    """ Export the given model.

    :param metricId: datasource-specific unique metric identifier

    :returns: Model-export specification for the Autostack model
    :rtype: dict

    ::
        {
          "datasource": "autostack",

          "stackSpec": {
            "name": "all_web_servers",  # Autostack name
            "aggSpec": {  # aggregation spec
              "datasource": "cloudwatch",
              "region": "us-west-2",
              "resourceType": "AWS::EC2::Instance"
              "filters": {  # resourceType-specific filter
                "tag:Name":["*test*", "*grok*"],
                "tag:Description":["Blah", "foo"]
              },
            }
          },

          "modelSpec": {
            "datasource": "autostack",

            "metricSpec": {
              "slaveDatasource": "cloudwatch",

              "slaveMetric": {
                # specific to slaveDatasource
                "namespace": "AWS/EC2",
                "metric": "CPUUtilization"
              },

              "period": 300  # aggregation period; seconds
            },

            "modelParams": { # optional; specific to slave metric
              "min": 0,  # optional
              "max": 100  # optional
            }
          }
        }

    """
    with self.connectionFactory() as conn:
      spec = {}
      spec["datasource"] = self._DATASOURCE

      metricObj = repository.getMetric(conn,
                                       metricId,
                                       fields=[schema.metric.c.parameters])
      autostackObj = repository.getAutostackFromMetric(conn, metricId)

    parameters = htmengine.utils.jsonDecode(metricObj.parameters)
    spec["modelSpec"] = parameters
    modelSpec = spec["modelSpec"]
    metricSpec = modelSpec["metricSpec"]
    del metricSpec["autostackId"]

    spec["stackSpec"] = {}
    stackSpec = spec["stackSpec"]
    stackSpec["name"] = autostackObj.name

    # Only supporting cloudwatch / EC2 for now
    stackSpec["aggSpec"] = {}
    aggSpec = stackSpec["aggSpec"]
    aggSpec["datasource"] = "cloudwatch"
    aggSpec["region"] = autostackObj.region
    aggSpec["resourceType"] = "AWS::EC2::Instance"
    aggSpec["filters"] = htmengine.utils.jsonDecode(autostackObj.filters)

    return spec