コード例 #1
0
    def testExportImport(self):
        metricName = "test-" + uuid.uuid1().hex

        adapter = datasource_adapter_factory.createCustomDatasourceAdapter()

        g_log.info("Creating htmengine custom metric; name=%s", metricName)
        metricId = adapter.createMetric(metricName)
        self.addCleanup(adapter.deleteMetricByName, metricName)

        # Add some data
        # NOTE: we discard the fractional part because it gets eliminated
        # in the database, and we will want to compare against retrieved
        # items later.
        now = datetime.datetime.utcnow().replace(microsecond=0)
        data = [(0, now - datetime.timedelta(minutes=5)), (100, now)]

        with self.engine.connect() as conn:
            repository.addMetricData(conn, metricId, data)

        # Turn on monitoring
        modelSpec = {
            "datasource": "custom",
            "metricSpec": {
                "metric": metricName
            },
        }

        adapter.monitorMetric(modelSpec)

        def checkExportSpec(exportSpec):
            self.assertEqual(exportSpec["datasource"], modelSpec["datasource"])
            self.assertEqual(exportSpec["metricSpec"], modelSpec["metricSpec"])
            self.assertSequenceEqual(exportSpec["data"], data)

        # Export
        exportSpec = adapter.exportModel(metricId)
        checkExportSpec(exportSpec)

        # Delete metric
        adapter.deleteMetricByName(metricName)
        self.checkModelDeleted(metricId)

        # Import
        metricId = adapter.importModel(
            htmengine.utils.jsonDecode(htmengine.utils.jsonEncode(exportSpec)))

        with self.engine.connect() as conn:
            metricObj = repository.getMetric(
                conn,
                metricId,
                fields=[schema.metric.c.parameters, schema.metric.c.status])
        self.assertEqual(metricObj.status, MetricStatus.PENDING_DATA)
        self.assertEqual(json.loads(metricObj.parameters), modelSpec)

        self._validateModelSpec(json.loads(metricObj.parameters))

        # Export again
        exportSpec = adapter.exportModel(metricId)
        checkExportSpec(exportSpec)
コード例 #2
0
  def testActivateModelClassifierEnabled(self):
    """ Test activateModel with classifier enabled in model spec. """
    metricName = "test-" + uuid.uuid1().hex

    adapter = datasource_adapter_factory.createCustomDatasourceAdapter()

    g_log.info("Creating htmengine custom metric; name=%s", metricName)
    metricId = adapter.createMetric(metricName)
    self.addCleanup(adapter.deleteMetricByName, metricName)

    # Turn on monitoring
    modelSpec = {
      "datasource": "custom",
      "metricSpec": {
        "metric": metricName
      },
      "modelParams": {
        "enableClassifier": True
      }
    }

    adapter.monitorMetric(modelSpec)

    with self.engine.connect() as conn:
      metricObj = repository.getMetric(conn,
                                       metricId,
                                       fields=[schema.metric.c.status])
    self.assertEqual(metricObj.status, MetricStatus.PENDING_DATA)

    # Add some data
    data = [
      (0, datetime.datetime.utcnow() - datetime.timedelta(minutes=5)),
      (100, datetime.datetime.utcnow())
    ]
    with self.engine.connect() as conn:
      repository.addMetricData(conn, metricId, data)

    # Activate model
    adapter.activateModel(metricId)

    with self.engine.connect() as conn:
      metricObj = repository.getMetric(conn,
                                       metricId,
                                       fields=[schema.metric.c.status,
                                               schema.metric.c.model_params])
    self.assertIn(metricObj.status, (MetricStatus.CREATE_PENDING,
                                     MetricStatus.ACTIVE))

    self._assertClassifierStatusInModelParams(metricObj.model_params,
                                              classifierEnabled=True)

    g_log.info("Waiting for model to become active")
    self.checkModelIsActive(metricId)

    g_log.info("Waiting at least one model result")
    self.checkModelResultsSize(metricId, 1, atLeast=True)
コード例 #3
0
    def testActivateModelClassifierEnabled(self):
        """ Test activateModel with classifier enabled in model spec. """
        metricName = "test-" + uuid.uuid1().hex

        adapter = datasource_adapter_factory.createCustomDatasourceAdapter()

        g_log.info("Creating htmengine custom metric; name=%s", metricName)
        metricId = adapter.createMetric(metricName)
        self.addCleanup(adapter.deleteMetricByName, metricName)

        # Turn on monitoring
        modelSpec = {
            "datasource": "custom",
            "metricSpec": {
                "metric": metricName
            },
            "modelParams": {
                "enableClassifier": True
            }
        }

        adapter.monitorMetric(modelSpec)

        with self.engine.connect() as conn:
            metricObj = repository.getMetric(conn,
                                             metricId,
                                             fields=[schema.metric.c.status])
        self.assertEqual(metricObj.status, MetricStatus.PENDING_DATA)

        # Add some data
        data = [(0,
                 datetime.datetime.utcnow() - datetime.timedelta(minutes=5)),
                (100, datetime.datetime.utcnow())]
        with self.engine.connect() as conn:
            repository.addMetricData(conn, metricId, data)

        # Activate model
        adapter.activateModel(metricId)

        with self.engine.connect() as conn:
            metricObj = repository.getMetric(
                conn,
                metricId,
                fields=[schema.metric.c.status, schema.metric.c.model_params])
        self.assertIn(metricObj.status,
                      (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE))

        self._assertClassifierStatusInModelParams(metricObj.model_params,
                                                  classifierEnabled=True)

        g_log.info("Waiting for model to become active")
        self.checkModelIsActive(metricId)

        g_log.info("Waiting at least one model result")
        self.checkModelResultsSize(metricId, 1, atLeast=True)
コード例 #4
0
  def testMonitorMetricWithEnoughDataForStats(self):
    """ monitorMetric should create a model when there is enough data rows """
    metricName = "test-" + uuid.uuid1().hex

    adapter = datasource_adapter_factory.createCustomDatasourceAdapter()

    g_log.info("Creating htmengine custom metric; name=%s", metricName)
    metricId = adapter.createMetric(metricName)
    self.addCleanup(adapter.deleteMetricByName, metricName)

    # Add enough data to force activation of model
    data = [
      (offset, datetime.datetime.utcnow() + datetime.timedelta(minutes=offset))
      for offset in xrange(
        0,
        scalar_metric_utils.MODEL_CREATION_RECORD_THRESHOLD * 5,
        5)
    ]
    self.assertEqual(len(data),
                     scalar_metric_utils.MODEL_CREATION_RECORD_THRESHOLD)

    with self.engine.connect() as conn:
      repository.addMetricData(conn, metricId, data)

    # Turn on monitoring
    modelSpec = {
      "datasource": "custom",

      "metricSpec": {
        "metric": metricName
      },
    }

    adapter.monitorMetric(modelSpec)

    with self.engine.connect() as conn:
      metricObj = repository.getMetric(conn,
                                       metricId,
                                       fields=[schema.metric.c.status])

    self.assertIn(metricObj.status, (MetricStatus.CREATE_PENDING,
                                     MetricStatus.ACTIVE))

    g_log.info("Waiting for model to become active")
    self.checkModelIsActive(metricId)

    g_log.info("Waiting at least one model result")
    self.checkModelResultsSize(metricId, 1, atLeast=True)
コード例 #5
0
  def _storeDataSamples(self, data, metricID, conn):
    """ Store the given metric data samples in metric_data table
    :param data: A sequence of data samples; each data sample is a pair:
                  (datetime.datetime, float)
    :param metricID: unique metric id
    :param sqlalchemy.engine.Connection conn: A sqlalchemy connection object

    :returns: a (possibly empty) tuple of ModelInputRow objects corresponding
        to the samples that were stored; ordered by rowid.
    :rtype: tuple of model_swapper_interface.ModelInputRow objects
    """

    if data:
      # repository.addMetricData expects samples as pairs of (value, timestamp)
      data = tuple((value, ts) for (ts, value) in data)

      # Save new metric data in metric table
      rows = repository.addMetricData(conn, metricID, data)

      # Update tail metric data timestamp cache for metrics stored by us
      self._tailInputMetricDataTimestamps[metricID] = rows[-1]["timestamp"]

      # Add newly-stored records to batch for sending to CLA model
      modelInputRows = tuple(
        ModelInputRow(rowID=row["rowid"], data=(timestamp, metricValue,))
        for (metricValue, timestamp), row
        in itertools.izip_longest(data, rows))
    else:
      modelInputRows = tuple()
      self._log.warning("_storeDataSamples called with empty data")

    return modelInputRows
コード例 #6
0
  def _storeDataSamples(self, data, metricID, conn):
    """ Store the given metric data samples in metric_data table
    :param data: A sequence of data samples; each data sample is a pair:
                  (datetime.datetime, float)
    :param metricID: unique metric id
    :param sqlalchemy.engine.Connection conn: A sqlalchemy connection object

    :returns: a (possibly empty) tuple of ModelInputRow objects corresponding
        to the samples that were stored; ordered by rowid.
    :rtype: tuple of model_swapper_interface.ModelInputRow objects
    """

    if data:
      # repository.addMetricData expects samples as pairs of (value, timestamp)
      data = tuple((value, ts) for (ts, value) in data)

      # Save new metric data in metric table
      rows = repository.addMetricData(conn, metricID, data)

      # Update tail metric data timestamp cache for metrics stored by us
      self._tailInputMetricDataTimestamps[metricID] = rows[-1]["timestamp"]

      # Add newly-stored records to batch for sending to CLA model
      modelInputRows = tuple(
        ModelInputRow(rowID=row["rowid"], data=(timestamp, metricValue,))
        for (metricValue, timestamp), row
        in itertools.izip_longest(data, rows))
    else:
      modelInputRows = tuple()
      self._log.warning("_storeDataSamples called with empty data")

    return modelInputRows
コード例 #7
0
    def testMonitorMetricWithEnoughDataForStats(self):
        """ monitorMetric should create a model when there is enough data rows """
        metricName = "test-" + uuid.uuid1().hex

        adapter = datasource_adapter_factory.createCustomDatasourceAdapter()

        g_log.info("Creating htmengine custom metric; name=%s", metricName)
        metricId = adapter.createMetric(metricName)
        self.addCleanup(adapter.deleteMetricByName, metricName)

        # Add enough data to force activation of model
        data = [
            (offset,
             datetime.datetime.utcnow() + datetime.timedelta(minutes=offset))
            for offset in xrange(
                0, scalar_metric_utils.MODEL_CREATION_RECORD_THRESHOLD * 5, 5)
        ]
        self.assertEqual(len(data),
                         scalar_metric_utils.MODEL_CREATION_RECORD_THRESHOLD)

        with self.engine.connect() as conn:
            repository.addMetricData(conn, metricId, data)

        # Turn on monitoring
        modelSpec = {
            "datasource": "custom",
            "metricSpec": {
                "metric": metricName
            },
        }

        adapter.monitorMetric(modelSpec)

        with self.engine.connect() as conn:
            metricObj = repository.getMetric(conn,
                                             metricId,
                                             fields=[schema.metric.c.status])

        self.assertIn(metricObj.status,
                      (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE))

        g_log.info("Waiting for model to become active")
        self.checkModelIsActive(metricId)

        g_log.info("Waiting at least one model result")
        self.checkModelResultsSize(metricId, 1, atLeast=True)
コード例 #8
0
  def testExportImportCompleteModelParams(self):
    metricName = "test-" + uuid.uuid1().hex

    adapter = datasource_adapter_factory.createCustomDatasourceAdapter()

    g_log.info("Creating htmengine custom metric; name=%s", metricName)
    metricId = adapter.createMetric(metricName)
    self.addCleanup(adapter.deleteMetricByName, metricName)

    # Add some data
    # NOTE: we discard the fractional part because it gets eliminated
    # in the database, and we will want to compare against retrieved
    # items later.
    now = datetime.datetime.utcnow().replace(microsecond=0)
    data = [
      (0, now - datetime.timedelta(minutes=5)),
      (100, now)
    ]

    with self.engine.connect() as conn:
      repository.addMetricData(conn, metricId, data)

    fileName = "custom_datasource_adapter_test_model_config.json"
    with self._openTestDataFile(fileName) as modelConfigFile:
      modelConfig = json.load(modelConfigFile)

    # Turn on monitoring
    modelSpec = {
      "datasource": "custom",
      "metricSpec": {
        "metric": metricName
      },
      "completeModelParams": {
        "modelConfig": modelConfig,
        "inferenceArgs": {"predictionSteps": [1], "predictedField": "bar",
                          "inputPredictedField": "auto"},
        "timestampFieldName": "foo",
        "valueFieldName": "bar"
      }
    }

    adapter.monitorMetric(modelSpec)

    def checkExportSpec(exportSpec):
      self.assertEqual(exportSpec["datasource"], modelSpec["datasource"])
      self.assertEqual(exportSpec["metricSpec"], modelSpec["metricSpec"])
      self.assertSequenceEqual(exportSpec["data"], data)

    # Export
    exportSpec = adapter.exportModel(metricId)
    checkExportSpec(exportSpec)

    # Delete metric
    adapter.deleteMetricByName(metricName)
    self.checkModelDeleted(metricId)

    # Import
    metricId = adapter.importModel(
      htmengine.utils.jsonDecode(htmengine.utils.jsonEncode(exportSpec)))

    with self.engine.connect() as conn:
      metricObj = repository.getMetric(conn,
                                       metricId,
                                       fields=[schema.metric.c.parameters,
                                               schema.metric.c.status])
    self.assertIn(metricObj.status, (MetricStatus.CREATE_PENDING,
                                     MetricStatus.ACTIVE))
    self.assertEqual(json.loads(metricObj.parameters), modelSpec)

    self._validateModelSpec(json.loads(metricObj.parameters))

    # Export again
    exportSpec = adapter.exportModel(metricId)
    checkExportSpec(exportSpec)
コード例 #9
0
  def testExportImport(self):
    metricName = "test-" + uuid.uuid1().hex

    adapter = datasource_adapter_factory.createCustomDatasourceAdapter()

    g_log.info("Creating htmengine custom metric; name=%s", metricName)
    metricId = adapter.createMetric(metricName)
    self.addCleanup(adapter.deleteMetricByName, metricName)

    # Add some data
    # NOTE: we discard the fractional part because it gets eliminated
    # in the database, and we will want to compare against retrieved
    # items later.
    now = datetime.datetime.utcnow().replace(microsecond=0)
    data = [
      (0, now - datetime.timedelta(minutes=5)),
      (100, now)
    ]

    with self.engine.connect() as conn:
      repository.addMetricData(conn, metricId, data)

    # Turn on monitoring
    modelSpec = {
      "datasource": "custom",

      "metricSpec": {
        "metric": metricName
      },
    }

    adapter.monitorMetric(modelSpec)

    def checkExportSpec(exportSpec):
      self.assertEqual(exportSpec["datasource"], modelSpec["datasource"])
      self.assertEqual(exportSpec["metricSpec"], modelSpec["metricSpec"])
      self.assertSequenceEqual(exportSpec["data"], data)

    # Export
    exportSpec = adapter.exportModel(metricId)
    checkExportSpec(exportSpec)

    # Delete metric
    adapter.deleteMetricByName(metricName)
    self.checkModelDeleted(metricId)

    # Import
    metricId = adapter.importModel(
      htmengine.utils.jsonDecode(htmengine.utils.jsonEncode(exportSpec)))

    with self.engine.connect() as conn:
      metricObj = repository.getMetric(conn,
                                       metricId,
                                       fields=[schema.metric.c.parameters,
                                               schema.metric.c.status])
    self.assertEqual(metricObj.status, MetricStatus.PENDING_DATA)
    self.assertEqual(json.loads(metricObj.parameters), modelSpec)

    self._validateModelSpec(json.loads(metricObj.parameters))

    # Export again
    exportSpec = adapter.exportModel(metricId)
    checkExportSpec(exportSpec)
コード例 #10
0
    def testExportImportCompleteModelParams(self):
        metricName = "test-" + uuid.uuid1().hex

        adapter = datasource_adapter_factory.createCustomDatasourceAdapter()

        g_log.info("Creating htmengine custom metric; name=%s", metricName)
        metricId = adapter.createMetric(metricName)
        self.addCleanup(adapter.deleteMetricByName, metricName)

        # Add some data
        # NOTE: we discard the fractional part because it gets eliminated
        # in the database, and we will want to compare against retrieved
        # items later.
        now = datetime.datetime.utcnow().replace(microsecond=0)
        data = [(0, now - datetime.timedelta(minutes=5)), (100, now)]

        with self.engine.connect() as conn:
            repository.addMetricData(conn, metricId, data)

        fileName = "custom_datasource_adapter_test_model_config.json"
        with self._openTestDataFile(fileName) as modelConfigFile:
            modelConfig = json.load(modelConfigFile)

        # Turn on monitoring
        modelSpec = {
            "datasource": "custom",
            "metricSpec": {
                "metric": metricName
            },
            "completeModelParams": {
                "modelConfig": modelConfig,
                "inferenceArgs": {
                    "predictionSteps": [1],
                    "predictedField": "bar",
                    "inputPredictedField": "auto"
                },
                "timestampFieldName": "foo",
                "valueFieldName": "bar"
            }
        }

        adapter.monitorMetric(modelSpec)

        def checkExportSpec(exportSpec):
            self.assertEqual(exportSpec["datasource"], modelSpec["datasource"])
            self.assertEqual(exportSpec["metricSpec"], modelSpec["metricSpec"])
            self.assertSequenceEqual(exportSpec["data"], data)

        # Export
        exportSpec = adapter.exportModel(metricId)
        checkExportSpec(exportSpec)

        # Delete metric
        adapter.deleteMetricByName(metricName)
        self.checkModelDeleted(metricId)

        # Import
        metricId = adapter.importModel(
            htmengine.utils.jsonDecode(htmengine.utils.jsonEncode(exportSpec)))

        with self.engine.connect() as conn:
            metricObj = repository.getMetric(
                conn,
                metricId,
                fields=[schema.metric.c.parameters, schema.metric.c.status])
        self.assertIn(metricObj.status,
                      (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE))
        self.assertEqual(json.loads(metricObj.parameters), modelSpec)

        self._validateModelSpec(json.loads(metricObj.parameters))

        # Export again
        exportSpec = adapter.exportModel(metricId)
        checkExportSpec(exportSpec)