def start(self, config):
"""
Connect to database, filesystem storage, and messaging
:param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
:param config: Configuration of db, storage, etc
:return: None
"""
self.config = config
self.config["process_id"] = get_process_id() # used for HA identity
# check right version of common
if versiontuple(common_version) < versiontuple(min_common_version):
raise NsException(
"Not compatible osm/common version '{}'. Needed '{}' or higher"
.format(common_version, min_common_version))
try:
if not self.db:
if config["database"]["driver"] == "mongo":
self.db = dbmongo.DbMongo()
self.db.db_connect(config["database"])
elif config["database"]["driver"] == "memory":
self.db = dbmemory.DbMemory()
self.db.db_connect(config["database"])
else:
raise NsException(
"Invalid configuration param '{}' at '[database]':'driver'"
.format(config["database"]["driver"]))
if not self.fs:
if config["storage"]["driver"] == "local":
self.fs = fslocal.FsLocal()
self.fs.fs_connect(config["storage"])
elif config["storage"]["driver"] == "mongo":
self.fs = fsmongo.FsMongo()
self.fs.fs_connect(config["storage"])
else:
raise NsException(
"Invalid configuration param '{}' at '[storage]':'driver'"
.format(config["storage"]["driver"]))
if not self.msg:
if config["message"]["driver"] == "local":
self.msg = msglocal.MsgLocal()
self.msg.connect(config["message"])
elif config["message"]["driver"] == "kafka":
self.msg = msgkafka.MsgKafka()
self.msg.connect(config["message"])
else:
raise NsException(
"Invalid configuration param '{}' at '[message]':'driver'"
.format(config["message"]["driver"]))
# TODO load workers to deal with exising database tasks
self.write_lock = Lock()
except (DbException, FsException, MsgException) as e:
raise NsException(str(e), http_code=e.http_code)
def run(self):
"""
Start of the thread
:return: None
"""
self.loop = asyncio.new_event_loop()
try:
if not self.db:
if self.config["database"]["driver"] == "mongo":
self.db = dbmongo.DbMongo()
self.db.db_connect(self.config["database"])
elif self.config["database"]["driver"] == "memory":
self.db = dbmemory.DbMemory()
self.db.db_connect(self.config["database"])
else:
raise SubscriptionException(
"Invalid configuration param '{}' at '[database]':'driver'"
.format(self.config["database"]["driver"]))
if not self.msg:
config_msg = self.config["message"].copy()
config_msg["loop"] = self.loop
if config_msg["driver"] == "local":
self.msg = msglocal.MsgLocal()
self.msg.connect(config_msg)
elif config_msg["driver"] == "kafka":
self.msg = msgkafka.MsgKafka()
self.msg.connect(config_msg)
else:
raise SubscriptionException(
"Invalid configuration param '{}' at '[message]':'driver'"
.format(config_msg["driver"]))
except (DbException, MsgException) as e:
raise SubscriptionException(str(e), http_code=e.http_code)
self.logger.debug("Starting")
while not self.to_terminate:
try:
self.loop.run_until_complete(
asyncio.ensure_future(self.start_kafka(), loop=self.loop))
# except asyncio.CancelledError:
# break # if cancelled it should end, breaking loop
except Exception as e:
if not self.to_terminate:
self.logger.exception(
"Exception '{}' at messaging read loop".format(e),
exc_info=True)
self.logger.debug("Finishing")
self._stop()
self.loop.close()
def start(self, config):
"""
Connect to database, filesystem storage, and messaging
:param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
:return: None
"""
self.config = config
# check right version of common
if versiontuple(common_version) < versiontuple(min_common_version):
raise EngineException(
"Not compatible osm/common version '{}'. Needed '{}' or higher"
.format(common_version, min_common_version))
try:
if not self.db:
if config["database"]["driver"] == "mongo":
self.db = dbmongo.DbMongo()
self.db.db_connect(config["database"])
elif config["database"]["driver"] == "memory":
self.db = dbmemory.DbMemory()
self.db.db_connect(config["database"])
else:
raise EngineException(
"Invalid configuration param '{}' at '[database]':'driver'"
.format(config["database"]["driver"]))
if not self.fs:
if config["storage"]["driver"] == "local":
self.fs = fslocal.FsLocal()
self.fs.fs_connect(config["storage"])
else:
raise EngineException(
"Invalid configuration param '{}' at '[storage]':'driver'"
.format(config["storage"]["driver"]))
if not self.msg:
if config["message"]["driver"] == "local":
self.msg = msglocal.MsgLocal()
self.msg.connect(config["message"])
elif config["message"]["driver"] == "kafka":
self.msg = msgkafka.MsgKafka()
self.msg.connect(config["message"])
else:
raise EngineException(
"Invalid configuration param '{}' at '[message]':'driver'"
.format(config["message"]["driver"]))
self.write_lock = Lock()
# create one class per topic
for topic, topic_class in self.map_from_topic_to_class.items():
self.map_topic[topic] = topic_class(self.db, self.fs, self.msg)
except (DbException, FsException, MsgException) as e:
raise EngineException(str(e), http_code=e.http_code)
def __init__(self, config_file, loop=None):
"""
Init, Connect to database, filesystem storage, and messaging
:param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
:return: None
"""
self.db = None
self.msg = None
self.msg_admin = None
self.fs = None
self.pings_not_received = 1
self.consecutive_errors = 0
self.first_start = False
# logging
self.logger = logging.getLogger('lcm')
# get id
self.worker_id = self.get_process_id()
# load configuration
config = self.read_config_file(config_file)
self.config = config
self.config["ro_config"] = {
"ng": config["RO"].get("ng", False),
"uri": config["RO"].get("uri"),
"tenant": config.get("tenant", "osm"),
"logger_name": "lcm.roclient",
"loglevel": config["RO"].get("loglevel", "ERROR"),
}
if not self.config["ro_config"]["uri"]:
if not self.config["ro_config"]["ng"]:
self.config["ro_config"][
"uri"] = "http://{}:{}/openmano".format(
config["RO"]["host"], config["RO"]["port"])
else:
self.config["ro_config"]["uri"] = "http://{}:{}/ro".format(
config["RO"]["host"], config["RO"]["port"])
self.loop = loop or asyncio.get_event_loop()
# logging
log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
log_formatter_simple = logging.Formatter(log_format_simple,
datefmt='%Y-%m-%dT%H:%M:%S')
config["database"]["logger_name"] = "lcm.db"
config["storage"]["logger_name"] = "lcm.fs"
config["message"]["logger_name"] = "lcm.msg"
if config["global"].get("logfile"):
file_handler = logging.handlers.RotatingFileHandler(
config["global"]["logfile"],
maxBytes=100e6,
backupCount=9,
delay=0)
file_handler.setFormatter(log_formatter_simple)
self.logger.addHandler(file_handler)
if not config["global"].get("nologging"):
str_handler = logging.StreamHandler()
str_handler.setFormatter(log_formatter_simple)
self.logger.addHandler(str_handler)
if config["global"].get("loglevel"):
self.logger.setLevel(config["global"]["loglevel"])
# logging other modules
for k1, logname in self.cfg_logger_name.items():
config[k1]["logger_name"] = logname
logger_module = logging.getLogger(logname)
if config[k1].get("logfile"):
file_handler = logging.handlers.RotatingFileHandler(
config[k1]["logfile"],
maxBytes=100e6,
backupCount=9,
delay=0)
file_handler.setFormatter(log_formatter_simple)
logger_module.addHandler(file_handler)
if config[k1].get("loglevel"):
logger_module.setLevel(config[k1]["loglevel"])
self.logger.critical("starting osm/lcm version {} {}".format(
lcm_version, lcm_version_date))
# check version of N2VC
# TODO enhance with int conversion or from distutils.version import LooseVersion
# or with list(map(int, version.split(".")))
if versiontuple(n2vc_version) < versiontuple(min_n2vc_version):
raise LcmException(
"Not compatible osm/N2VC version '{}'. Needed '{}' or higher".
format(n2vc_version, min_n2vc_version))
# check version of common
if versiontuple(common_version) < versiontuple(min_common_version):
raise LcmException(
"Not compatible osm/common version '{}'. Needed '{}' or higher"
.format(common_version, min_common_version))
try:
# TODO check database version
if config["database"]["driver"] == "mongo":
self.db = dbmongo.DbMongo()
self.db.db_connect(config["database"])
elif config["database"]["driver"] == "memory":
self.db = dbmemory.DbMemory()
self.db.db_connect(config["database"])
else:
raise LcmException(
"Invalid configuration param '{}' at '[database]':'driver'"
.format(config["database"]["driver"]))
if config["storage"]["driver"] == "local":
self.fs = fslocal.FsLocal()
self.fs.fs_connect(config["storage"])
elif config["storage"]["driver"] == "mongo":
self.fs = fsmongo.FsMongo()
self.fs.fs_connect(config["storage"])
else:
raise LcmException(
"Invalid configuration param '{}' at '[storage]':'driver'".
format(config["storage"]["driver"]))
# copy message configuration in order to remove 'group_id' for msg_admin
config_message = config["message"].copy()
config_message["loop"] = self.loop
if config_message["driver"] == "local":
self.msg = msglocal.MsgLocal()
self.msg.connect(config_message)
self.msg_admin = msglocal.MsgLocal()
config_message.pop("group_id", None)
self.msg_admin.connect(config_message)
elif config_message["driver"] == "kafka":
self.msg = msgkafka.MsgKafka()
self.msg.connect(config_message)
self.msg_admin = msgkafka.MsgKafka()
config_message.pop("group_id", None)
self.msg_admin.connect(config_message)
else:
raise LcmException(
"Invalid configuration param '{}' at '[message]':'driver'".
format(config["message"]["driver"]))
except (DbException, FsException, MsgException) as e:
self.logger.critical(str(e), exc_info=True)
raise LcmException(str(e))
# contains created tasks/futures to be able to cancel
self.lcm_tasks = TaskRegistry(self.worker_id, self.db, self.logger)
if self.config.get("tsdb") and self.config["tsdb"].get("driver"):
if self.config["tsdb"]["driver"] == "prometheus":
self.prometheus = prometheus.Prometheus(
self.config["tsdb"], self.worker_id, self.db, self.loop)
else:
raise LcmException(
"Invalid configuration param '{}' at '[tsdb]':'driver'".
format(config["tsdb"]["driver"]))
else:
self.prometheus = None
self.ns = ns.NsLcm(self.db, self.msg, self.fs, self.lcm_tasks,
self.config, self.loop, self.prometheus)
self.netslice = netslice.NetsliceLcm(self.db, self.msg, self.fs,
self.lcm_tasks, self.config,
self.loop, self.ns)
self.vim = vim_sdn.VimLcm(self.db, self.msg, self.fs, self.lcm_tasks,
self.config, self.loop)
self.wim = vim_sdn.WimLcm(self.db, self.msg, self.fs, self.lcm_tasks,
self.config, self.loop)
self.sdn = vim_sdn.SdnLcm(self.db, self.msg, self.fs, self.lcm_tasks,
self.config, self.loop)
self.k8scluster = vim_sdn.K8sClusterLcm(self.db, self.msg, self.fs,
self.lcm_tasks, self.config,
self.loop)
self.k8srepo = vim_sdn.K8sRepoLcm(self.db, self.msg, self.fs,
self.lcm_tasks, self.config,
self.loop)
def start(self, config):
"""
Connect to database, filesystem storage, and messaging
:param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
:return: None
"""
self.config = config
# check right version of common
if versiontuple(common_version) < versiontuple(min_common_version):
raise EngineException(
"Not compatible osm/common version '{}'. Needed '{}' or higher"
.format(common_version, min_common_version))
try:
if not self.db:
if config["database"]["driver"] == "mongo":
self.db = dbmongo.DbMongo()
self.db.db_connect(config["database"])
elif config["database"]["driver"] == "memory":
self.db = dbmemory.DbMemory()
self.db.db_connect(config["database"])
else:
raise EngineException(
"Invalid configuration param '{}' at '[database]':'driver'"
.format(config["database"]["driver"]))
if not self.fs:
if config["storage"]["driver"] == "local":
self.fs = fslocal.FsLocal()
self.fs.fs_connect(config["storage"])
elif config["storage"]["driver"] == "mongo":
self.fs = fsmongo.FsMongo()
self.fs.fs_connect(config["storage"])
else:
raise EngineException(
"Invalid configuration param '{}' at '[storage]':'driver'"
.format(config["storage"]["driver"]))
if not self.msg:
if config["message"]["driver"] == "local":
self.msg = msglocal.MsgLocal()
self.msg.connect(config["message"])
elif config["message"]["driver"] == "kafka":
self.msg = msgkafka.MsgKafka()
self.msg.connect(config["message"])
else:
raise EngineException(
"Invalid configuration param '{}' at '[message]':'driver'"
.format(config["message"]["driver"]))
if not self.auth:
if config["authentication"]["backend"] == "keystone":
self.auth = AuthconnKeystone(config["authentication"],
self.db, None)
else:
self.auth = AuthconnInternal(config["authentication"],
self.db, self.token_cache)
if not self.operations:
if "resources_to_operations" in config["rbac"]:
resources_to_operations_file = config["rbac"][
"resources_to_operations"]
else:
possible_paths = (__file__[:__file__.rfind("engine.py")] +
"resources_to_operations.yml",
"./resources_to_operations.yml")
for config_file in possible_paths:
if path.isfile(config_file):
resources_to_operations_file = config_file
break
if not resources_to_operations_file:
raise EngineException(
"Invalid permission configuration: resources_to_operations file missing"
)
with open(resources_to_operations_file, 'r') as f:
resources_to_operations = yaml.load(f, Loader=yaml.Loader)
self.operations = []
for _, value in resources_to_operations[
"resources_to_operations"].items():
if value not in self.operations:
self.operations += [value]
self.write_lock = Lock()
# create one class per topic
for topic, topic_class in self.map_from_topic_to_class.items():
# if self.auth and topic_class in (UserTopicAuth, ProjectTopicAuth):
# self.map_topic[topic] = topic_class(self.db, self.fs, self.msg, self.auth)
if self.auth and topic_class == RoleTopicAuth:
self.map_topic[topic] = topic_class(
self.db, self.fs, self.msg, self.auth, self.operations)
else:
self.map_topic[topic] = topic_class(
self.db, self.fs, self.msg, self.auth)
self.map_topic["pm_jobs"] = PmJobsTopic(
self.db, config["prometheus"].get("host"),
config["prometheus"].get("port"))
except (DbException, FsException, MsgException) as e:
raise EngineException(str(e), http_code=e.http_code)