def start(self, config):
"""
Method to configure the Authenticator object. This method should be called
after object creation. It is responsible by initializing the selected backend,
as well as the initialization of the database connection.
:param config: dictionary containing the relevant parameters for this object.
"""
self.config = config
try:
if not self.db:
if config["database"]["driver"] == "mongo":
self.db = dbmongo.DbMongo()
self.db.db_connect(config["database"])
elif config["database"]["driver"] == "memory":
self.db = dbmemory.DbMemory()
self.db.db_connect(config["database"])
else:
raise AuthException(
"Invalid configuration param '{}' at '[database]':'driver'"
.format(config["database"]["driver"]))
if not self.backend:
if config["authentication"]["backend"] == "keystone":
self.backend = AuthconnKeystone(
self.config["authentication"])
elif config["authentication"]["backend"] == "internal":
self._internal_tokens_prune()
else:
raise AuthException(
"Unknown authentication backend: {}".format(
config["authentication"]["backend"]))
except Exception as e:
raise AuthException(str(e))
def start(self, config):
"""
Connect to database, filesystem storage, and messaging
:param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
:param config: Configuration of db, storage, etc
:return: None
"""
self.config = config
self.config["process_id"] = get_process_id() # used for HA identity
# check right version of common
if versiontuple(common_version) < versiontuple(min_common_version):
raise NsException(
"Not compatible osm/common version '{}'. Needed '{}' or higher"
.format(common_version, min_common_version))
try:
if not self.db:
if config["database"]["driver"] == "mongo":
self.db = dbmongo.DbMongo()
self.db.db_connect(config["database"])
elif config["database"]["driver"] == "memory":
self.db = dbmemory.DbMemory()
self.db.db_connect(config["database"])
else:
raise NsException(
"Invalid configuration param '{}' at '[database]':'driver'"
.format(config["database"]["driver"]))
if not self.fs:
if config["storage"]["driver"] == "local":
self.fs = fslocal.FsLocal()
self.fs.fs_connect(config["storage"])
elif config["storage"]["driver"] == "mongo":
self.fs = fsmongo.FsMongo()
self.fs.fs_connect(config["storage"])
else:
raise NsException(
"Invalid configuration param '{}' at '[storage]':'driver'"
.format(config["storage"]["driver"]))
if not self.msg:
if config["message"]["driver"] == "local":
self.msg = msglocal.MsgLocal()
self.msg.connect(config["message"])
elif config["message"]["driver"] == "kafka":
self.msg = msgkafka.MsgKafka()
self.msg.connect(config["message"])
else:
raise NsException(
"Invalid configuration param '{}' at '[message]':'driver'"
.format(config["message"]["driver"]))
# TODO load workers to deal with exising database tasks
self.write_lock = Lock()
except (DbException, FsException, MsgException) as e:
raise NsException(str(e), http_code=e.http_code)
def start(self, config):
"""
Method to configure the Authenticator object. This method should be called
after object creation. It is responsible by initializing the selected backend,
as well as the initialization of the database connection.
:param config: dictionary containing the relevant parameters for this object.
"""
self.config = config
try:
if not self.db:
if config["database"]["driver"] == "mongo":
self.db = dbmongo.DbMongo()
self.db.db_connect(config["database"])
elif config["database"]["driver"] == "memory":
self.db = dbmemory.DbMemory()
self.db.db_connect(config["database"])
else:
raise AuthException("Invalid configuration param '{}' at '[database]':'driver'"
.format(config["database"]["driver"]))
if not self.backend:
if config["authentication"]["backend"] == "keystone":
self.backend = AuthconnKeystone(self.config["authentication"])
elif config["authentication"]["backend"] == "internal":
self._internal_tokens_prune()
else:
raise AuthException("Unknown authentication backend: {}"
.format(config["authentication"]["backend"]))
if not self.resources_to_operations_file:
if "resources_to_operations" in config["rbac"]:
self.resources_to_operations_file = config["rbac"]["resources_to_operations"]
else:
for config_file in (__file__[:__file__.rfind("auth.py")] + "resources_to_operations.yml",
"./resources_to_operations.yml"):
if path.isfile(config_file):
self.resources_to_operations_file = config_file
break
if not self.resources_to_operations_file:
raise AuthException("Invalid permission configuration: resources_to_operations file missing")
if not self.roles_to_operations_file:
if "roles_to_operations" in config["rbac"]:
self.roles_to_operations_file = config["rbac"]["roles_to_operations"]
else:
for config_file in (__file__[:__file__.rfind("auth.py")] + "roles_to_operations.yml",
"./roles_to_operations.yml"):
if path.isfile(config_file):
self.roles_to_operations_file = config_file
break
if not self.roles_to_operations_file:
raise AuthException("Invalid permission configuration: roles_to_operations file missing")
except Exception as e:
raise AuthException(str(e))
def run(self):
"""
Start of the thread
:return: None
"""
self.loop = asyncio.new_event_loop()
try:
if not self.db:
if self.config["database"]["driver"] == "mongo":
self.db = dbmongo.DbMongo()
self.db.db_connect(self.config["database"])
elif self.config["database"]["driver"] == "memory":
self.db = dbmemory.DbMemory()
self.db.db_connect(self.config["database"])
else:
raise SubscriptionException(
"Invalid configuration param '{}' at '[database]':'driver'"
.format(self.config["database"]["driver"]))
if not self.msg:
config_msg = self.config["message"].copy()
config_msg["loop"] = self.loop
if config_msg["driver"] == "local":
self.msg = msglocal.MsgLocal()
self.msg.connect(config_msg)
elif config_msg["driver"] == "kafka":
self.msg = msgkafka.MsgKafka()
self.msg.connect(config_msg)
else:
raise SubscriptionException(
"Invalid configuration param '{}' at '[message]':'driver'"
.format(config_msg["driver"]))
except (DbException, MsgException) as e:
raise SubscriptionException(str(e), http_code=e.http_code)
self.logger.debug("Starting")
while not self.to_terminate:
try:
self.loop.run_until_complete(
asyncio.ensure_future(self.start_kafka(), loop=self.loop))
# except asyncio.CancelledError:
# break # if cancelled it should end, breaking loop
except Exception as e:
if not self.to_terminate:
self.logger.exception(
"Exception '{}' at messaging read loop".format(e),
exc_info=True)
self.logger.debug("Finishing")
self._stop()
self.loop.close()
def start(self, config):
"""
Connect to database, filesystem storage, and messaging
:param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
:return: None
"""
self.config = config
# check right version of common
if versiontuple(common_version) < versiontuple(min_common_version):
raise EngineException(
"Not compatible osm/common version '{}'. Needed '{}' or higher"
.format(common_version, min_common_version))
try:
if not self.db:
if config["database"]["driver"] == "mongo":
self.db = dbmongo.DbMongo()
self.db.db_connect(config["database"])
elif config["database"]["driver"] == "memory":
self.db = dbmemory.DbMemory()
self.db.db_connect(config["database"])
else:
raise EngineException(
"Invalid configuration param '{}' at '[database]':'driver'"
.format(config["database"]["driver"]))
if not self.fs:
if config["storage"]["driver"] == "local":
self.fs = fslocal.FsLocal()
self.fs.fs_connect(config["storage"])
else:
raise EngineException(
"Invalid configuration param '{}' at '[storage]':'driver'"
.format(config["storage"]["driver"]))
if not self.msg:
if config["message"]["driver"] == "local":
self.msg = msglocal.MsgLocal()
self.msg.connect(config["message"])
elif config["message"]["driver"] == "kafka":
self.msg = msgkafka.MsgKafka()
self.msg.connect(config["message"])
else:
raise EngineException(
"Invalid configuration param '{}' at '[message]':'driver'"
.format(config["message"]["driver"]))
self.write_lock = Lock()
# create one class per topic
for topic, topic_class in self.map_from_topic_to_class.items():
self.map_topic[topic] = topic_class(self.db, self.fs, self.msg)
except (DbException, FsException, MsgException) as e:
raise EngineException(str(e), http_code=e.http_code)
def __init__(self, config_file, loop=None):
"""
Init, Connect to database, filesystem storage, and messaging
:param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
:return: None
"""
self.db = None
self.msg = None
self.msg_admin = None
self.fs = None
self.pings_not_received = 1
self.consecutive_errors = 0
self.first_start = False
# logging
self.logger = logging.getLogger('lcm')
# get id
self.worker_id = self.get_process_id()
# load configuration
config = self.read_config_file(config_file)
self.config = config
self.config["ro_config"] = {
"ng": config["RO"].get("ng", False),
"uri": config["RO"].get("uri"),
"tenant": config.get("tenant", "osm"),
"logger_name": "lcm.roclient",
"loglevel": config["RO"].get("loglevel", "ERROR"),
}
if not self.config["ro_config"]["uri"]:
if not self.config["ro_config"]["ng"]:
self.config["ro_config"][
"uri"] = "http://{}:{}/openmano".format(
config["RO"]["host"], config["RO"]["port"])
else:
self.config["ro_config"]["uri"] = "http://{}:{}/ro".format(
config["RO"]["host"], config["RO"]["port"])
self.loop = loop or asyncio.get_event_loop()
# logging
log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
log_formatter_simple = logging.Formatter(log_format_simple,
datefmt='%Y-%m-%dT%H:%M:%S')
config["database"]["logger_name"] = "lcm.db"
config["storage"]["logger_name"] = "lcm.fs"
config["message"]["logger_name"] = "lcm.msg"
if config["global"].get("logfile"):
file_handler = logging.handlers.RotatingFileHandler(
config["global"]["logfile"],
maxBytes=100e6,
backupCount=9,
delay=0)
file_handler.setFormatter(log_formatter_simple)
self.logger.addHandler(file_handler)
if not config["global"].get("nologging"):
str_handler = logging.StreamHandler()
str_handler.setFormatter(log_formatter_simple)
self.logger.addHandler(str_handler)
if config["global"].get("loglevel"):
self.logger.setLevel(config["global"]["loglevel"])
# logging other modules
for k1, logname in self.cfg_logger_name.items():
config[k1]["logger_name"] = logname
logger_module = logging.getLogger(logname)
if config[k1].get("logfile"):
file_handler = logging.handlers.RotatingFileHandler(
config[k1]["logfile"],
maxBytes=100e6,
backupCount=9,
delay=0)
file_handler.setFormatter(log_formatter_simple)
logger_module.addHandler(file_handler)
if config[k1].get("loglevel"):
logger_module.setLevel(config[k1]["loglevel"])
self.logger.critical("starting osm/lcm version {} {}".format(
lcm_version, lcm_version_date))
# check version of N2VC
# TODO enhance with int conversion or from distutils.version import LooseVersion
# or with list(map(int, version.split(".")))
if versiontuple(n2vc_version) < versiontuple(min_n2vc_version):
raise LcmException(
"Not compatible osm/N2VC version '{}'. Needed '{}' or higher".
format(n2vc_version, min_n2vc_version))
# check version of common
if versiontuple(common_version) < versiontuple(min_common_version):
raise LcmException(
"Not compatible osm/common version '{}'. Needed '{}' or higher"
.format(common_version, min_common_version))
try:
# TODO check database version
if config["database"]["driver"] == "mongo":
self.db = dbmongo.DbMongo()
self.db.db_connect(config["database"])
elif config["database"]["driver"] == "memory":
self.db = dbmemory.DbMemory()
self.db.db_connect(config["database"])
else:
raise LcmException(
"Invalid configuration param '{}' at '[database]':'driver'"
.format(config["database"]["driver"]))
if config["storage"]["driver"] == "local":
self.fs = fslocal.FsLocal()
self.fs.fs_connect(config["storage"])
elif config["storage"]["driver"] == "mongo":
self.fs = fsmongo.FsMongo()
self.fs.fs_connect(config["storage"])
else:
raise LcmException(
"Invalid configuration param '{}' at '[storage]':'driver'".
format(config["storage"]["driver"]))
# copy message configuration in order to remove 'group_id' for msg_admin
config_message = config["message"].copy()
config_message["loop"] = self.loop
if config_message["driver"] == "local":
self.msg = msglocal.MsgLocal()
self.msg.connect(config_message)
self.msg_admin = msglocal.MsgLocal()
config_message.pop("group_id", None)
self.msg_admin.connect(config_message)
elif config_message["driver"] == "kafka":
self.msg = msgkafka.MsgKafka()
self.msg.connect(config_message)
self.msg_admin = msgkafka.MsgKafka()
config_message.pop("group_id", None)
self.msg_admin.connect(config_message)
else:
raise LcmException(
"Invalid configuration param '{}' at '[message]':'driver'".
format(config["message"]["driver"]))
except (DbException, FsException, MsgException) as e:
self.logger.critical(str(e), exc_info=True)
raise LcmException(str(e))
# contains created tasks/futures to be able to cancel
self.lcm_tasks = TaskRegistry(self.worker_id, self.db, self.logger)
if self.config.get("tsdb") and self.config["tsdb"].get("driver"):
if self.config["tsdb"]["driver"] == "prometheus":
self.prometheus = prometheus.Prometheus(
self.config["tsdb"], self.worker_id, self.db, self.loop)
else:
raise LcmException(
"Invalid configuration param '{}' at '[tsdb]':'driver'".
format(config["tsdb"]["driver"]))
else:
self.prometheus = None
self.ns = ns.NsLcm(self.db, self.msg, self.fs, self.lcm_tasks,
self.config, self.loop, self.prometheus)
self.netslice = netslice.NetsliceLcm(self.db, self.msg, self.fs,
self.lcm_tasks, self.config,
self.loop, self.ns)
self.vim = vim_sdn.VimLcm(self.db, self.msg, self.fs, self.lcm_tasks,
self.config, self.loop)
self.wim = vim_sdn.WimLcm(self.db, self.msg, self.fs, self.lcm_tasks,
self.config, self.loop)
self.sdn = vim_sdn.SdnLcm(self.db, self.msg, self.fs, self.lcm_tasks,
self.config, self.loop)
self.k8scluster = vim_sdn.K8sClusterLcm(self.db, self.msg, self.fs,
self.lcm_tasks, self.config,
self.loop)
self.k8srepo = vim_sdn.K8sRepoLcm(self.db, self.msg, self.fs,
self.lcm_tasks, self.config,
self.loop)
def start(self, config):
"""
Connect to database, filesystem storage, and messaging
:param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
:return: None
"""
self.config = config
# check right version of common
if versiontuple(common_version) < versiontuple(min_common_version):
raise EngineException(
"Not compatible osm/common version '{}'. Needed '{}' or higher"
.format(common_version, min_common_version))
try:
if not self.db:
if config["database"]["driver"] == "mongo":
self.db = dbmongo.DbMongo()
self.db.db_connect(config["database"])
elif config["database"]["driver"] == "memory":
self.db = dbmemory.DbMemory()
self.db.db_connect(config["database"])
else:
raise EngineException(
"Invalid configuration param '{}' at '[database]':'driver'"
.format(config["database"]["driver"]))
if not self.fs:
if config["storage"]["driver"] == "local":
self.fs = fslocal.FsLocal()
self.fs.fs_connect(config["storage"])
elif config["storage"]["driver"] == "mongo":
self.fs = fsmongo.FsMongo()
self.fs.fs_connect(config["storage"])
else:
raise EngineException(
"Invalid configuration param '{}' at '[storage]':'driver'"
.format(config["storage"]["driver"]))
if not self.msg:
if config["message"]["driver"] == "local":
self.msg = msglocal.MsgLocal()
self.msg.connect(config["message"])
elif config["message"]["driver"] == "kafka":
self.msg = msgkafka.MsgKafka()
self.msg.connect(config["message"])
else:
raise EngineException(
"Invalid configuration param '{}' at '[message]':'driver'"
.format(config["message"]["driver"]))
if not self.auth:
if config["authentication"]["backend"] == "keystone":
self.auth = AuthconnKeystone(config["authentication"],
self.db, None)
else:
self.auth = AuthconnInternal(config["authentication"],
self.db, self.token_cache)
if not self.operations:
if "resources_to_operations" in config["rbac"]:
resources_to_operations_file = config["rbac"][
"resources_to_operations"]
else:
possible_paths = (__file__[:__file__.rfind("engine.py")] +
"resources_to_operations.yml",
"./resources_to_operations.yml")
for config_file in possible_paths:
if path.isfile(config_file):
resources_to_operations_file = config_file
break
if not resources_to_operations_file:
raise EngineException(
"Invalid permission configuration: resources_to_operations file missing"
)
with open(resources_to_operations_file, 'r') as f:
resources_to_operations = yaml.load(f, Loader=yaml.Loader)
self.operations = []
for _, value in resources_to_operations[
"resources_to_operations"].items():
if value not in self.operations:
self.operations += [value]
self.write_lock = Lock()
# create one class per topic
for topic, topic_class in self.map_from_topic_to_class.items():
# if self.auth and topic_class in (UserTopicAuth, ProjectTopicAuth):
# self.map_topic[topic] = topic_class(self.db, self.fs, self.msg, self.auth)
if self.auth and topic_class == RoleTopicAuth:
self.map_topic[topic] = topic_class(
self.db, self.fs, self.msg, self.auth, self.operations)
else:
self.map_topic[topic] = topic_class(
self.db, self.fs, self.msg, self.auth)
self.map_topic["pm_jobs"] = PmJobsTopic(
self.db, config["prometheus"].get("host"),
config["prometheus"].get("port"))
except (DbException, FsException, MsgException) as e:
raise EngineException(str(e), http_code=e.http_code)
def start(self, config):
"""
Method to configure the Authenticator object. This method should be called
after object creation. It is responsible by initializing the selected backend,
as well as the initialization of the database connection.
:param config: dictionary containing the relevant parameters for this object.
"""
self.config = config
try:
if not self.db:
if config["database"]["driver"] == "mongo":
self.db = dbmongo.DbMongo()
self.db.db_connect(config["database"])
elif config["database"]["driver"] == "memory":
self.db = dbmemory.DbMemory()
self.db.db_connect(config["database"])
else:
raise AuthException(
"Invalid configuration param '{}' at '[database]':'driver'"
.format(config["database"]["driver"]))
if not self.backend:
if config["authentication"]["backend"] == "keystone":
self.backend = AuthconnKeystone(
self.config["authentication"], self.db,
self.tokens_cache)
elif config["authentication"]["backend"] == "internal":
self.backend = AuthconnInternal(
self.config["authentication"], self.db,
self.tokens_cache)
self._internal_tokens_prune()
else:
raise AuthException(
"Unknown authentication backend: {}".format(
config["authentication"]["backend"]))
if not self.roles_to_operations_file:
if "roles_to_operations" in config["rbac"]:
self.roles_to_operations_file = config["rbac"][
"roles_to_operations"]
else:
possible_paths = (__file__[:__file__.rfind("auth.py")] +
"roles_to_operations.yml",
"./roles_to_operations.yml")
for config_file in possible_paths:
if path.isfile(config_file):
self.roles_to_operations_file = config_file
break
if not self.roles_to_operations_file:
raise AuthException(
"Invalid permission configuration: roles_to_operations file missing"
)
# load role_permissions
def load_role_permissions(method_dict):
for k in method_dict:
if k == "ROLE_PERMISSION":
for method in chain(method_dict.get("METHODS", ()),
method_dict.get("TODO", ())):
permission = method_dict[
"ROLE_PERMISSION"] + method.lower()
if permission not in self.role_permissions:
self.role_permissions.append(permission)
elif k in ("TODO", "METHODS"):
continue
else:
load_role_permissions(method_dict[k])
load_role_permissions(self.valid_methods)
for query_string in self.valid_query_string:
for method in ("get", "put", "patch", "post", "delete"):
permission = query_string.lower() + ":" + method
if permission not in self.role_permissions:
self.role_permissions.append(permission)
except Exception as e:
raise AuthException(str(e))
