def create_datasources(bus):
"""Create and register datasource services ."""
if cfg.CONF.delete_missing_driver_datasources:
# congress server started with --delete-missing-driver-datasources
bus.delete_missing_driver_datasources()
# create regular (table) data sources
datasources = db_datasources.get_datasources()
services = []
for ds in datasources:
LOG.info("create configured datasource service %s.", ds.name)
try:
service = bus.create_datasource_service(ds)
if service:
bus.register_service(service)
services.append(service)
except exception.DriverNotFound:
LOG.exception("Some datasources could not be loaded, start "
"congress server with "
"--delete-missing-driver-datasources option to "
"clean up stale datasources in DB.")
sys.exit(1)
except Exception:
LOG.exception("datasource %s creation failed. %s service may not "
"be running.", ds.name, ds.driver)
# create json_ingester data sources
if cfg.CONF.json_ingester.enable:
create_json_ingester_datasources(bus)
return services
def create_datasources(bus):
"""Create and register datasource services ."""
if cfg.CONF.delete_missing_driver_datasources:
# congress server started with --delete-missing-driver-datasources
bus.delete_missing_driver_datasources()
datasources = db_datasources.get_datasources()
services = []
for ds in datasources:
LOG.info("create configured datasource service %s.", ds.name)
try:
service = bus.create_datasource_service(ds)
if service:
bus.register_service(service)
services.append(service)
except exception.DriverNotFound:
LOG.exception("Some datasources could not be loaded, start "
"congress server with "
"--delete-missing-driver-datasources option to "
"clean up stale datasources in DB.")
sys.exit(1)
except Exception:
LOG.exception(
"datasource %s creation failed. %s service may not "
"be running.", ds.name, ds.driver)
return services
def get_datasources(cls, filter_secret=False):
"""Return the created datasources."""
results = []
for datasouce_driver in datasources_db.get_datasources():
result = cls.make_datasource_dict(datasouce_driver)
if filter_secret:
hide_fields = cls.get_driver_info(result['driver'])['secret']
for hide_field in hide_fields:
result['config'][hide_field] = "<hidden>"
results.append(result)
return results
def test_get_datasource(self):
id_ = uuidutils.generate_uuid()
datasources.add_datasource(
id_=id_, name="hiya", driver="foo", config="{user: foo}", description="hello", enabled=True
)
sources = datasources.get_datasources()
self.assertEqual(id_, sources[0].id)
self.assertEqual("hiya", sources[0].name)
self.assertEqual("foo", sources[0].driver)
self.assertEqual("hello", sources[0].description)
self.assertEqual('"{user: foo}"', sources[0].config)
self.assertTrue(sources[0].enabled)
def delete_missing_driver_datasources(self):
removed = 0
for datasource in datasources_db.get_datasources():
try:
self.get_driver_info(datasource.driver)
except exception.DriverNotFound:
ds_dict = self.make_datasource_dict(datasource)
self.delete_datasource(ds_dict)
removed = removed + 1
LOG.debug("Deleted datasource with config %s ",
strutils.mask_password(ds_dict))
LOG.info("Datsource cleanup completed, removed %d datasources",
removed)
def delete_missing_driver_datasources(self):
removed = 0
for datasource in datasources_db.get_datasources():
try:
self.get_driver_info(datasource.driver)
except exception.DriverNotFound:
datasources_db.delete_datasource_with_data(datasource.id)
removed = removed+1
LOG.debug("Datasource driver '%s' not found, deleting the "
"datasource '%s' from DB ", datasource.driver,
datasource.name)
LOG.info("Datsource cleanup completed, removed %d datasources",
removed)
def delete_missing_driver_datasources(self):
removed = 0
for datasource in datasources_db.get_datasources():
try:
self.get_driver_info(datasource.driver)
except exception.DriverNotFound:
datasources_db.delete_datasource_with_data(datasource.id)
removed = removed+1
LOG.debug("Datasource driver '%s' not found, deleting the "
"datasource '%s' from DB ", datasource.driver,
datasource.name)
LOG.info("Datsource cleanup completed, removed %d datasources",
removed)
def test_get_datasource(self):
id_ = uuidutils.generate_uuid()
datasources.add_datasource(id_=id_,
name="hiya",
driver="foo",
config={'user': '******'},
description="hello",
enabled=True)
sources = datasources.get_datasources()
self.assertEqual(id_, sources[0].id)
self.assertEqual("hiya", sources[0].name)
self.assertEqual("foo", sources[0].driver)
self.assertEqual("hello", sources[0].description)
self.assertEqual({'user': '******'}, json.loads(sources[0].config))
self.assertTrue(sources[0].enabled)
def test_get_datasource(self):
id_ = uuidutils.generate_uuid()
datasources.add_datasource(id_=id_,
name="hiya",
driver="foo",
config='{user: foo}',
description="hello",
enabled=True)
sources = datasources.get_datasources()
self.assertEqual(id_, sources[0].id)
self.assertEqual("hiya", sources[0].name)
self.assertEqual("foo", sources[0].driver)
self.assertEqual("hello", sources[0].description)
self.assertEqual('"{user: foo}"', sources[0].config)
self.assertEqual(True, sources[0].enabled)
def get_datasources(cls, filter_secret=False):
"""Return the created datasources.
This returns what datasources the database contains, not the
datasources that this server instance is running.
"""
results = []
for datasouce_driver in datasources_db.get_datasources():
result = cls.make_datasource_dict(datasouce_driver)
if filter_secret:
hide_fields = cls.get_driver_info(result['driver'])['secret']
for hide_field in hide_fields:
result['config'][hide_field] = "<hidden>"
results.append(result)
return results
def get_datasources(cls, filter_secret=False):
"""Return the created datasources.
This returns what datasources the database contains, not the
datasources that this server instance is running.
"""
results = []
for datasouce_driver in datasources_db.get_datasources():
result = cls.make_datasource_dict(datasouce_driver)
if filter_secret:
hide_fields = cls.get_driver_info(result['driver'])['secret']
for hide_field in hide_fields:
result['config'][hide_field] = "<hidden>"
results.append(result)
return results
def test_get_datasource_with_encryption(self):
id_ = uuidutils.generate_uuid()
datasources.add_datasource(
id_=id_,
name="hiya",
driver="foo",
config={'user': '******'},
description="hello",
enabled=True,
secret_config_fields=['user'])
sources = datasources.get_datasources()
self.assertEqual(id_, sources[0].id)
self.assertEqual("hiya", sources[0].name)
self.assertEqual("foo", sources[0].driver)
self.assertEqual("hello", sources[0].description)
self.assertEqual({'user': '******'}, json.loads(sources[0].config))
self.assertTrue(sources[0].enabled)
def get_datasources(self, filter_secret=False):
"""Return the created datasources as recorded in the DB.
This returns what datasources the database contains, not the
datasources that this server instance is running.
"""
results = []
for datasource in datasources_db.get_datasources():
result = self.make_datasource_dict(datasource)
if filter_secret:
# driver_info knows which fields should be secret
driver_info = self.get_driver_info(result['driver'])
try:
for hide_field in driver_info['secret']:
result['config'][hide_field] = "<hidden>"
except KeyError:
pass
results.append(result)
return results
def get_datasources(self, filter_secret=False):
"""Return the created datasources as recorded in the DB.
This returns what datasources the database contains, not the
datasources that this server instance is running.
"""
results = []
for datasource in datasources_db.get_datasources():
result = self.make_datasource_dict(datasource)
if filter_secret:
# driver_info knows which fields should be secret
driver_info = self.get_driver_info(result['driver'])
try:
for hide_field in driver_info['secret']:
result['config'][hide_field] = "<hidden>"
except KeyError:
pass
results.append(result)
return results
def create_datasources(bus):
"""Create datasource services and return datasources."""
datasources = db_datasources.get_datasources()
services = []
for ds in datasources:
ds_dict = bus.make_datasource_dict(ds)
if not ds['enabled']:
LOG.info("module %s not enabled, skip loading", ds_dict['name'])
continue
LOG.info("create configured datasource service %s." % ds_dict['name'])
try:
driver_info = bus.get_driver_info(ds_dict['driver'])
service = bus.create_service(
class_path=driver_info['module'],
kwargs={'name': ds_dict['name'], 'args': ds_dict['config']})
bus.register_service(service)
services.append(service)
except Exception:
LOG.exception("datasource %s creation failed." % ds_dict['name'])
return services