def event_create(event_id): """If event already exists it recreates it.""" schema = { "type": "object", "definitions": { "traffic": { "type": "object", "properties": { "type": {"enum": ["host", "az", "dc"]}, "value": {"type": "string"} }, "required": ["type", "value"] } }, "properties": { "name": {"type": "string"}, "started_at": {"type": "string"}, "finished_at": {"type": "string"}, "traffic_from": {"$ref": "#/definitions/traffic"}, "traffic_to": {"$ref": "#/definitions/traffic"} }, "required": ["started_at", "name"], "additionalProperties": False } try: data = flask.request.get_json(silent=False, force=True) jsonschema.validate(data, schema) except (ValueError, jsonschema.exceptions.ValidationError) as e: return flask.jsonify({"error": "Bad request: %s" % e}), 400 db.get().event_create(event_id, data) return flask.jsonify({"message": "Event created %s" % event_id}), 201
def metrics_add(): """Stores metrics to elastic.""" # Check just basic schema, let elastic check everything else schema = { "type": "array", "items": {"type": "object"} } try: req_data = flask.request.get_json(silent=False, force=True) jsonschema.validate(req_data, schema) except (ValueError, jsonschema.exceptions.ValidationError) as e: return flask.jsonify({"error": "Bad request: %s" % e}), 400 else: data = {"north-south": [], "east-west": []} for d in req_data: for key in data: if key in d: data[key].append(d[key]) break else: LOG.warning("Ignoring wrong object %s" % json.dumps(d)) # TODO(boris-42): Use pusher here, to reduce amount of quires # from netmet server to elastic, join data from different netmet # clients requests before pushing them to elastic for k, v in data.iteritems(): if v: db.get().metrics_add(k, v) return flask.jsonify({"message": "successfully stored metrics"}), 201
def _job(self): get_conf = db.get().server_config_get is_meshed = lambda cfg: (not cfg or (cfg and not cfg["applied"]) or (cfg and cfg["meshed"])) try: if is_meshed(get_conf()): LOG.info(self.no_changes_msg) else: with eslock.Glock("update_config"): # TODO(boris-42): Alogrithm should be a bit smarter # even if it is meshed try to update all not configured # clients. config = get_conf() if not is_meshed(config): LOG.info(self.new_config_msg) for c in self._mesh(config["config"]): # TODO(boris-42): Run this in parallel self._update_client(c[0], c[1]) db.get().server_config_meshed(config["id"]) else: LOG.info(self.no_changes_msg) except exceptions.GlobalLockException: pass # can't accuire lock, someone else is working on it except Exception: LOG.exception(self.update_failed_msg)
def test_server_config_meshed(self, mock_elastic): db.DB.create("a", ["b"]) db.get().server_config_meshed("id2") mock_elastic.return_value.update.assert_called_once_with( index="netmet_catalog", doc_type="config", id="id2", body={"doc": { "meshed": True }}, refresh="true")
def test_create_mocked_all(self, mock_ensure_elastic, mock_ensure_schema, mock_rollover_data, mock_elastic): elastics = ["elastic"] db.DB.create("own_url", elastics) self.assertIsInstance(db.get(), db.DB) self.assertEqual(db.get().own_url, "own_url") self.assertEqual(db.get().elastic_urls, elastics) mock_ensure_elastic.assert_called_once_with() mock_ensure_schema.assert_called_once_with() mock_rollover_data.assert_called_once_with() mock_elastic.assert_called_once_with(elastics)
def test_server_config_add(self, mock_elastic): db.DB.create("a", ["b"]) db.get().server_config_add({"a": 1}) expected_body = { "config": '{"a": 1}', "applied": False, "meshed": False, "timestamp": mock.ANY } mock_elastic.return_value.index.assert_called_once_with( index="netmet_catalog", doc_type="config", body=expected_body, refresh="true")
def test_server_config_get(self, mock_elastic): config = { "config": json.dumps({"some": "stuff"}), "applied": True, "meshed": "False" } expected_result = { "id": "id", "config": { "some": "stuff" }, "applied": True, "meshed": "False" } query = { "sort": { "timestamp": { "order": "desc" } }, "query": { "term": { "applied": True } } } mock_elastic.return_value.search.side_effect = [{ "hits": { "hits": [{ "_id": "id", "_source": config }] } }, { "hits": { "hits": [] } }] db.DB.create("a", ["b"]) self.assertEqual(expected_result, db.get().server_config_get(only_applied=True)) mock_elastic.return_value.search.assert_called_once_with( index="netmet_catalog", doc_type="config", body=query, size=1) self.assertIsNone(db.get().server_config_get(only_applied=True))
def test_metrics_add(self, mock_elastic): mock_elastic.return_value.bulk.return_value = { "items": [{ "index": { "status": 200 } }, { "index": { "status": 200 } }, { "index": { "status": 500 } }] } db.DB.create("a", ["b"]) doc = {"a": {"b": 1}, "c": 2} expected_bulk = '{"index": {}}\n{"c": 2, "a.b": 1}' self.assertEqual({ 200: 2, 500: 1 }, db.get().metrics_add("east-west", [doc])) mock_elastic.return_value.bulk.assert_called_once_with( index="netmet_data_v2", doc_type="east-west", body=expected_bulk)
def test_event_get_not_found(self, mock_elastic): mock_elastic.return_value.get.return_value = {"found": False} db.DB.create("a", ["b"]) self.assertRaises(exceptions.DBRecordNotFound, db.get().event_get, "some_id2") mock_elastic.return_value.get.assert_called_once_with( index="netmet_events", doc_type="events", id="some_id2")
def __enter__(self): if self.acquired: raise exceptions.GlobalLockException("Lock already in use %s" % self.name) if db.get().lock_acquire(self.name, self.ttl): self.acquired = True else: raise exceptions.GlobalLockException("Can't lock %s" % self.name)
def test_clients_set(self, mock_elastic): fake_catalog = [{"a": 1}, {"b": 2}] expected_body = '{"index": {}}\n{"a": 1}\n{"index": {}}\n{"b": 2}' db.DB.create("a", ["b"]) db.get().clients_set(fake_catalog) mock_elastic.return_value.delete_by_query.assert_called_once_with( index="netmet_catalog", doc_type="clients", body={"query": { "match_all": {} }}) mock_elastic.return_value.bulk.assert_called_once_with( index="netmet_catalog", doc_type="clients", body=expected_body, refresh="true")
def _mesh(self, config): mesh = self.plugins[config["mesher"].keys()[0]].mesh allowed = set(["ip", "port", "host", "hypervisor", "dc", "az"]) clients = [{k: x[k] for k in allowed if k in x} for x in db.get().clients_get()] return mesh(config["mesher"].values()[0], clients, config["external"])
def config_get(): """Returns netmet server configuration.""" server_config = db.get().server_config_get() if not server_config: return flask.jsonify({ "message": "Netmet server has not been setup yet"}), 404 return flask.jsonify(server_config), 200
def test_events_list(self, mock_elastic): mock_elastic.return_value.search.return_value = { "hits": { "hits": [{ "_source": { "a": 1 } }, { "_source": { "b": 2 } }] } } db.DB.create("a", ["b"]) self.assertEqual([{ "a": 1 }, { "b": 2 }], db.get().events_list(10, 20, only_active=True)) expected_query = { "from": 10, "size": 20, "query": { "bool": { "must_not": [{ "term": { "status": "deleted" } }], "should": [{ "range": { "finished_at": { "gt": "now/m" } } }, { "missing": { "field": "finished_at" } }] }, "filter": [{ "range": { "started_at": { "lte": "now/m" } } }] } } mock_elastic.return_value.search.assert_called_once_with( index="netmet_events", body=expected_query)
def test_event_get(self, mock_elastic): mock_elastic.return_value.get.return_value = { "found": True, "_version": 2, "_source": { "a": 1 } } db.DB.create("a", ["b"]) self.assertEqual((2, {"a": 1}), db.get().event_get("some_id")) mock_elastic.return_value.get.assert_called_once_with( index="netmet_events", doc_type="events", id="some_id")
def _job(self): get_conf = db.get().server_config_get is_applied = lambda cfg: not cfg or (cfg and cfg["applied"]) no_changes_msg = "Deployer: no changes in config detected." try: if is_applied(get_conf()): LOG.info(no_changes_msg) else: with eslock.Glock("update_config"): config = get_conf() # Refresh config after lock if not is_applied(config): LOG.info("Deployer detect new config: " "Updating deployment") clients = db.get().clients_get() # TODO(boris-42): Add support of multi drivers new_clients = StaticDeployer().redeploy( config["config"]["deployment"]["static"], clients) db.get().clients_set(new_clients) db.get().server_config_apply(config["id"]) return True else: LOG.info(no_changes_msg) except exceptions.GlobalLockException: pass # can't accuire lock, someone else is working on it except Exception: LOG.exception("Deployer update failed")
def test_event_update(self, mock_event_get, mock_elastic): mock_elastic.return_value.update.side_effect = [{ "result": "updated" }, { "result": "noop" }] mock_event_get.return_value = (2, {}) db.DB.create("a", ["b"]) self.assertTrue(db.get()._event_update("some_id", {"a": 1})) mock_event_get.assert_called_once_with("some_id") mock_elastic.return_value.update.assert_called_once_with( index="netmet_events", doc_type="events", id="some_id", body={"doc": { "a": 1 }}, refresh='true', version=2) self.assertRaises(exceptions.DBConflict, db.get()._event_update, "some_other_id", {"a": 1})
def test_event_upgrade_metrics(self, mock_elastic, mock_event_update, mock_event_get, mock_get_query, mock_get_script): melastic = mock_elastic.return_value mock_event_get.return_value = (1, {"task_id": "some_task2"}) melastic.tasks.get.return_value = {"completed": True} melastic.update_by_query.return_value = {"task": "some_task"} db.DB.create("a", ["b"]) db.get()._event_upgrade_metrics("some_id", "add") mock_event_get.assert_called_once_with("some_id") melastic.tasks.get.assert_called_once_with(task_id="some_task2") mock_event_update.assert_has_calls([ mock.call("some_id", { "task_id": None, "status": "updating" }, version=1), mock.call("some_id", { "task_id": "some_task", "status": "created" }) ]) body = { "query": mock_get_query.return_value, "script": mock_get_script.return_value, } melastic.update_by_query.assert_called_once_with( index="netmet_data_v2*", body=body, conflicts="proceed", wait_for_completion=False, requests_per_second=1000) mock_get_query.assert_called_once_with(mock_event_get.return_value[1], "some_id", "add") mock_get_script.assert_called_once_with("some_id", "add")
def test_clients_get(self, mock_elastic): mock_elastic.return_value.search.return_value = { "hits": { "hits": [{ "_source": { "a": 1 } }, { "_source": { "a": 2 } }] } } db.DB.create("a", ["b"]) self.assertEqual(db.get().clients_get(), [{"a": 1}, {"a": 2}])
def test_event_create(self, mock_elastic, mock_event_upgrade_metrics): db.DB.create("a", ["b"]) data = {"some_data": 1} mock_elastic.return_value.create.return_value = {"created": True} self.assertTrue(db.get().event_create("some_id", data)) mock_elastic.return_value.create.assert_called_once_with( index="netmet_events", doc_type="events", id="some_id", body={ "some_data": 1, "status": "created" }, refresh="true") mock_event_upgrade_metrics.assert_called_once_with("some_id", "add")
def refresh_client(self, host, port): lock_acuired = False attempts = 0 while not lock_acuired and attempts < 3: try: with eslock.Glock("update_config"): config = db.get().server_config_get() if not (config["applied"] and config["meshed"]): return False, 404, "Configuration not found" for c in self._mesh(config["config"]): if c[0]["host"] == host and c[0]["port"] == port: return self._update_client(c[0], c[1]) return False, 404, "Client not found" except exceptions.GlobalLockException: attempts += 1 self._death.wait(0.1) return False, 500, "Couldn't accuire lock"
def clients_list(): """List all hosts.""" return flask.jsonify(db.get().clients_get()), 200
def test_job_inited(self, mock_elastic, mock_rollover_data): db.DB.create("own_url", ["elastics"]) self.assertEqual(1, mock_rollover_data.call_count) db.get()._job() self.assertEqual(2, mock_rollover_data.call_count)
def events_list(): offset = flask.request.args.get('offset', 0) limit = flask.request.args.get('limit', 100) active_only = flask.request.args.get('active_only') return flask.jsonify(db.get().events_list(offset, limit, active_only)), 200
def event_stop(event_id): db.get().event_stop(event_id) return flask.jsonify({"message": "event %s stopped" % event_id}), 200
def event_delete(event_id): db.get().event_delete(event_id) return flask.jsonify({"message": "event %s deleted" % event_id}), 202
def config_set(): """Sets netmet server configuration.""" CONFIG_SCHEMA = { "type": "object", "properties": { "deployment": { "type": "object", "properties": { "static": { "type": "object", "properties": { "clients": { "type": "array", "items": { "type": "object", "properties": { "host": {"type": "string"}, "ip": {"type": "string"}, "port": {"type": "integer"}, "az": {"type": "string"}, "dc": {"type": "string"}, "hypervisor": {"type": "string"} }, "required": ["host", "ip", "az", "dc"], "additionalProperties": False } } }, "required": ["clients"], "additionalProperties": False } }, "required": ["static"], "additionalProperties": False }, "mesher": mesher.Mesher.get_jsonschema(), "external": { "type": "array", "items": { "type": "object", "properties": { "dest": {"type": "string"}, "protocol": {"enum": ["http", "icmp"]}, "period": {"type": "number"}, "timeout": {"type": "number"} }, "required": ["dest", "protocol", "period", "timeout"], "additionalProperties": False } } }, "required": ["deployment", "mesher"], "additionalProperties": False } try: server_config = flask.request.get_json(silent=False, force=True) jsonschema.validate(server_config, CONFIG_SCHEMA) except (ValueError, jsonschema.exceptions.ValidationError) as e: return flask.jsonify({"error": "Bad request: %s" % e}), 400 db.get().server_config_add(server_config) deployer.Deployer.force_update() return flask.jsonify({"message": "Config was updated"}), 201
def __exit__(self, exception_type, exception_value, traceback): if not db.get().lock_release(self.name): logging.warning("Can't release lock %(name)s." % {"name": self.name}) self.acquired = False
def test_get_not_init(self): self.assertIsNone(db.get())