def test_initialise_with_config_sql_only(self):
yaml = YamlConfigurationFile()
self.assertIsNotNone(yaml)
yaml.load_from_text(
"""
console:
storage:
entities:
users: sql
linked_accounts: sql
links: sql
properties: sql
conversations: sql
categories: sql
stores:
sql:
type: sql
config:
url: sqlite:///:memory
echo: false
encoding: utf-8
create_db: true
drop_all_first: true
""", ConsoleConfiguration(), ".")
bot_config = yaml.get_section("console")
storage_config = StorageConfiguration()
storage_config.load_config_section(yaml, bot_config, ".")
factory = StorageFactory()
factory.load_engines_from_config(storage_config)
self.assertTrue(factory.storage_engine_available("sql"))
self.assertFalse(factory.storage_engine_available("mongo"))
self.assertFalse(factory.storage_engine_available("redis"))
self.assertFalse(factory.storage_engine_available("file"))
self.assertFalse(factory.storage_engine_available("logger"))
self.assertFalse(factory.storage_engine_available("other"))
self.assertIsNone(factory.storage_engine("other"))
self.assertTrue(factory.entity_storage_engine_available("users"))
self.assertIsInstance(factory.entity_storage_engine("users"),
SQLStorageEngine)
self.assertTrue(
factory.entity_storage_engine_available("linked_accounts"))
self.assertIsInstance(factory.entity_storage_engine("linked_accounts"),
SQLStorageEngine)
self.assertTrue(factory.entity_storage_engine_available("links"))
self.assertIsInstance(factory.entity_storage_engine("links"),
SQLStorageEngine)
self.assertTrue(factory.entity_storage_engine_available("properties"))
self.assertIsInstance(factory.entity_storage_engine("properties"),
SQLStorageEngine)
self.assertTrue(
factory.entity_storage_engine_available("conversations"))
self.assertIsInstance(factory.entity_storage_engine("conversations"),
SQLStorageEngine)
self.assertTrue(factory.entity_storage_engine_available("categories"))
self.assertIsInstance(factory.entity_storage_engine("categories"),
SQLStorageEngine)
self.assertFalse(factory.entity_storage_engine_available("other"))
self.assertIsNone(factory.entity_storage_engine("other"))
def test_initialise_with_config_file_only(self):
yaml = YamlConfigurationFile()
self.assertIsNotNone(yaml)
yaml.load_from_text(
"""
console:
storage:
entities:
users: file
linked_accounts: file
links: file
properties: file
conversations: file
categories: file
stores:
file:
type: file
config:
storage_dir: ./storage
properties_storage: ./storage/properties
conversation_storage: ./storage/conversations
""", ConsoleConfiguration(), ".")
bot_config = yaml.get_section("console")
storage_config = StorageConfiguration()
storage_config.load_config_section(yaml, bot_config, ".")
factory = StorageFactory()
factory.load_engines_from_config(storage_config)
self.assertFalse(factory.storage_engine_available("sql"))
self.assertFalse(factory.storage_engine_available("mongo"))
self.assertFalse(factory.storage_engine_available("redis"))
self.assertFalse(factory.storage_engine_available("logger"))
self.assertTrue(factory.storage_engine_available("file"))
self.assertTrue(factory.entity_storage_engine_available("users"))
self.assertIsInstance(factory.entity_storage_engine("users"),
FileStorageEngine)
self.assertTrue(
factory.entity_storage_engine_available("linked_accounts"))
self.assertIsInstance(factory.entity_storage_engine("linked_accounts"),
FileStorageEngine)
self.assertTrue(factory.entity_storage_engine_available("links"))
self.assertIsInstance(factory.entity_storage_engine("links"),
FileStorageEngine)
self.assertTrue(factory.entity_storage_engine_available("properties"))
self.assertIsInstance(factory.entity_storage_engine("properties"),
FileStorageEngine)
self.assertTrue(
factory.entity_storage_engine_available("conversations"))
self.assertIsInstance(factory.entity_storage_engine("conversations"),
FileStorageEngine)
self.assertTrue(factory.entity_storage_engine_available("categories"))
self.assertIsInstance(factory.entity_storage_engine("categories"),
FileStorageEngine)
self.assertFalse(factory.entity_storage_engine_available("other"))
self.assertIsNone(factory.entity_storage_engine("other"))
def test_initialise_with_config_mixed(self):
yaml = YamlConfigurationFile()
self.assertIsNotNone(yaml)
yaml.load_from_text(
"""
console:
storage:
entities:
users: sql
linked_accounts: sql
links: sql
properties: redis
conversations: mongo
categories: sql
stores:
sql:
type: sql
config:
url: sqlite:///:memory
echo: false
encoding: utf-8
create_db: true
drop_all_first: true
mongo:
type: mongo
config:
url: mongodb://localhost:27017/
database: programy
drop_all_first: true
redis:
type: redis
config:
host: localhost
port: 6379
password: null
db: 0
prefix: programy
drop_all_first: True
file:
type: file
config:
storage_dir: ./storage
properties_storage: ./storage/properties
conversation_storage: ./storage/conversations
logger:
type: logger
config:
conversation_logger: conversation
""", ConsoleConfiguration(), ".")
bot_config = yaml.get_section("console")
storage_config = StorageConfiguration()
storage_config.load_config_section(yaml, bot_config, ".")
factory = StorageFactory()
factory.load_engines_from_config(storage_config)
self.assertTrue(factory.storage_engine_available("sql"))
self.assertIsInstance(factory.storage_engine("sql"), SQLStorageEngine)
self.assertTrue(factory.storage_engine_available("mongo"))
self.assertIsInstance(factory.storage_engine("mongo"),
MongoStorageEngine)
self.assertTrue(factory.storage_engine_available("redis"))
self.assertIsInstance(factory.storage_engine("redis"),
RedisStorageEngine)
self.assertTrue(factory.storage_engine_available("file"))
self.assertIsInstance(factory.storage_engine("file"),
FileStorageEngine)
self.assertTrue(factory.storage_engine_available("logger"))
self.assertIsInstance(factory.storage_engine("logger"),
LoggerStorageEngine)
self.assertFalse(factory.storage_engine_available("other"))
self.assertIsNone(factory.storage_engine("other"))
self.assertTrue(factory.entity_storage_engine_available("users"))
self.assertIsInstance(factory.entity_storage_engine("users"),
SQLStorageEngine)
self.assertTrue(
factory.entity_storage_engine_available("linked_accounts"))
self.assertIsInstance(factory.entity_storage_engine("linked_accounts"),
SQLStorageEngine)
self.assertTrue(factory.entity_storage_engine_available("links"))
self.assertIsInstance(factory.entity_storage_engine("links"),
SQLStorageEngine)
self.assertTrue(factory.entity_storage_engine_available("properties"))
self.assertIsInstance(factory.entity_storage_engine("properties"),
RedisStorageEngine)
self.assertTrue(
factory.entity_storage_engine_available("conversations"))
self.assertIsInstance(factory.entity_storage_engine("conversations"),
MongoStorageEngine)
self.assertTrue(factory.entity_storage_engine_available("categories"))
self.assertIsInstance(factory.entity_storage_engine("categories"),
SQLStorageEngine)
self.assertFalse(factory.entity_storage_engine_available("other"))
self.assertIsNone(factory.entity_storage_engine("other"))
def test_initialise_with_config_sql_only(self):
yaml = YamlConfigurationFile()
self.assertIsNotNone(yaml)
yaml.load_from_text(
"""
console:
storage:
entities:
users: mongo
linked_accounts: mongo
links: mongo
properties: mongo
conversations: mongo
categories: mongo
stores:
mongo:
type: mongo
config:
url: mongodb://localhost:27017/
database: programy
drop_all_first: true
""", ConsoleConfiguration(), ".")
bot_config = yaml.get_section("console")
storage_config = StorageConfiguration()
storage_config.load_config_section(yaml, bot_config, ".")
factory = StorageFactory()
factory.load_engines_from_config(storage_config)
self.assertTrue(factory.storage_engine_available("mongo"))
self.assertFalse(factory.storage_engine_available("sql"))
self.assertFalse(factory.storage_engine_available("redis"))
self.assertFalse(factory.storage_engine_available("file"))
self.assertFalse(factory.storage_engine_available("logger"))
self.assertFalse(factory.storage_engine_available("other"))
self.assertIsNone(factory.storage_engine("other"))
self.assertTrue(factory.entity_storage_engine_available("users"))
self.assertIsInstance(factory.entity_storage_engine("users"),
MongoStorageEngine)
self.assertTrue(
factory.entity_storage_engine_available("linked_accounts"))
self.assertIsInstance(factory.entity_storage_engine("linked_accounts"),
MongoStorageEngine)
self.assertTrue(factory.entity_storage_engine_available("links"))
self.assertIsInstance(factory.entity_storage_engine("links"),
MongoStorageEngine)
self.assertTrue(factory.entity_storage_engine_available("properties"))
self.assertIsInstance(factory.entity_storage_engine("properties"),
MongoStorageEngine)
self.assertTrue(
factory.entity_storage_engine_available("conversations"))
self.assertIsInstance(factory.entity_storage_engine("conversations"),
MongoStorageEngine)
self.assertTrue(factory.entity_storage_engine_available("categories"))
self.assertIsInstance(factory.entity_storage_engine("categories"),
MongoStorageEngine)
self.assertFalse(factory.entity_storage_engine_available("other"))
self.assertIsNone(factory.entity_storage_engine("other"))
class BotClient(ResponseLogger):
def __init__(self, id, argument_parser=None):
self._id = id
self._license_keys = LicenseKeys()
self._storage = None
self._scheduler = None
self._email = None
self._trigger_mgr = None
self._configuration = None
self._bot_root = '.'
self._engine_version = None
self._arguments = self.parse_arguments(argument_parser=argument_parser)
self._engine_version = self._arguments.version
self.initiate_logging(self.arguments)
self._subsitutions = Substitutions()
if self.arguments.substitutions is not None:
self._subsitutions.load_substitutions(self.arguments.substitutions)
self.load_configuration(self.arguments, subs=self._subsitutions)
else:
self.load_configuration(self.arguments)
self.parse_configuration()
self.load_storage()
self._bot_factory = BotFactory(self,
self.configuration.client_configuration)
self.load_license_keys()
self.get_license_keys()
self._configuration.client_configuration.check_for_license_keys(
self._license_keys)
self.load_scheduler()
self.load_renderer()
self.load_email()
self.load_trigger_manager()
def ylogger_type(self):
return "client"
@property
def configuration(self):
return self._configuration
@property
def id(self):
return self._id
@property
def arguments(self):
return self._arguments
@property
def license_keys(self):
return self._license_keys
@property
def scheduler(self):
return self._scheduler
@property
def storage_factory(self):
return self._storage
@property
def bot_factory(self):
return self._bot_factory
@property
def renderer(self):
return self._renderer
@property
def trigger_manager(self):
return self._trigger_mgr
@property
def engine_version(self):
return self._engine_version
def get_description(self):
if self.configuration is not None:
if self.configuration.client_configuration is not None:
return self.configuration.client_configuration.description
return "Bot Client"
def add_client_arguments(self, parser=None):
# Nothing to add
return
def parse_configuration(self):
# Nothing to add
return
def parse_args(self, arguments, parsed_args):
# Nothing to add
return
def parse_arguments(self, argument_parser):
client_args = CommandLineClientArguments(self, parser=argument_parser)
client_args.parse_args(self)
return client_args
def load_license_keys(self):
if self.storage_factory.entity_storage_engine_available(
StorageFactory.LICENSE_KEYS) is True:
storage_engine = self.storage_factory.entity_storage_engine(
StorageFactory.LICENSE_KEYS)
keys_store = storage_engine.license_store()
keys_store.load(self._license_keys)
else:
YLogger.warning(self,
"No storage factory setting for license_keys")
def get_license_keys(self):
return
def initiate_logging(self, arguments):
if arguments.logging is not None:
try:
with open(arguments.logging, 'r+',
encoding="utf-8") as yml_data_file:
logging_config = yaml.load(yml_data_file,
Loader=yaml.SafeLoader)
logging.config.dictConfig(logging_config)
YLogger.debug(self, "Now logging under configuration")
except Exception as excep:
YLogger.exception(self,
"Failed to open logging configuration [%s]",
excep, arguments.logging)
else:
print(
"Warning. No logging configuration file defined, using defaults..."
)
def get_client_configuration(self):
"""
By overriding this class in you Configuration file, you can add new configurations
and stil use the dynamic loader capabilities
:return: Client configuration object
"""
raise NotImplementedError(
"You must override this and return a subclassed client configuration"
)
def load_configuration(self, arguments, subs: Substitutions = None):
if arguments.bot_root is None:
if arguments.config_filename is not None:
arguments.bot_root = os.path.dirname(arguments.config_filename)
else:
arguments.bot_root = "."
YLogger.debug(
None, "No bot root argument set, defaulting to [%s]" %
arguments.bot_root)
self._bot_root = arguments.bot_root
if arguments.config_filename is not None:
self._configuration = ConfigurationFactory.load_configuration_from_file(
self.get_client_configuration(), arguments.config_filename,
arguments.config_format, arguments.bot_root, subs)
else:
YLogger.debug(
None, "No configuration file specified, using defaults only !")
self._configuration = ProgramyConfiguration(
self.get_client_configuration())
def load_scheduler(self):
if self.configuration.client_configuration.scheduler is not None:
YLogger.debug(None, "Loading Scheduler")
self._scheduler = ProgramyScheduler(
self, self.configuration.client_configuration.scheduler)
self._scheduler.start()
def load_email(self):
if self._configuration.client_configuration.email is not None:
YLogger.debug(None, "Loading Email Manager")
self._email = EmailSender(
self._configuration.client_configuration.email)
def load_trigger_manager(self):
if self._configuration.client_configuration.triggers is not None:
YLogger.debug(None, "Loading Trigger Manager")
self._trigger_mgr = TriggerManager.load_trigger_manager(
self._configuration.client_configuration.triggers)
def load_storage(self):
self._storage = StorageFactory()
if self.configuration.client_configuration.storage is not None:
YLogger.debug(None, "Loading Storage Factory")
self._storage.load_engines_from_config(
self.configuration.client_configuration.storage)
else:
YLogger.debug(None, "No storage defined!")
def load_renderer(self):
try:
if self.get_client_configuration().renderer is not None:
YLogger.debug(None, "Loading Renderer")
clazz = ClassLoader.instantiate_class(
self.get_client_configuration().renderer.renderer)
self._renderer = clazz(self)
return
except Exception as e:
YLogger.exception(None, "Failed to load config specified renderer",
e)
self._renderer = self.get_default_renderer()
def get_default_renderer(self):
return TextRenderer(self)
def create_client_context(self, userid):
client_context = ClientContext(self, userid)
client_context.bot = self._bot_factory.select_bot()
client_context.brain = client_context.bot._brain_factory.select_brain()
return client_context
def startup(self):
if self._trigger_mgr is not None:
self._trigger_mgr.trigger(event=SystemTriggers.SYSTEM_STARTUP)
def shutdown(self):
if self._trigger_mgr is not None:
self._trigger_mgr.trigger(event=SystemTriggers.SYSTEM_SHUTDOWN)
def process_question(self, client_context, question):
raise NotImplementedError(
"You must override this and implement the logic to create a response to the question"
)
def render_response(self, client_context, response):
raise NotImplementedError(
"You must override this and implement the logic to process the response by rendering"
" using a RCS renderer")
def process_response(self, client_context, response):
raise NotImplementedError(
"You must override this and implement the logic to display the response to the user"
)
def run(self):
raise NotImplementedError(
"You must override this and implement the logic to run the client")
def dump_client_config(self):
filename = '%s/dump_%s_config.yaml' % (self._bot_root, self._id)
if filename is not None:
config_writer = ConfigurationWriter()
try:
config_writer.executing_confiuguration(self._configuration,
filename)
except Exception:
pass
class BotClient(ResponseLogger):
def __init__(self, botid, argument_parser=None):
self._id = botid
self._license_keys = LicenseKeys()
self._storage = None
self._scheduler = None
self._email = None
self._trigger_mgr = None
self._configuration = None
self._ping_responder = None
self._questions = 0
self._arguments = self.parse_arguments(argument_parser=argument_parser)
self.initiate_logging(self.arguments)
self._subsitutions = Substitutions()
if self.arguments.substitutions is not None:
self._subsitutions.load_substitutions(self.arguments.substitutions)
self.load_configuration(self.arguments)
self.parse_configuration()
self.load_storage()
self.load_license_keys()
self.get_license_keys()
self._configuration.client_configuration.check_for_license_keys(
self._license_keys)
self._bot_factory = BotFactory(self,
self.configuration.client_configuration)
self.load_scheduler()
self._renderer = None
self.load_renderer()
self.load_email()
self.load_trigger_manager()
self.load_ping_responder()
self.post_initialise()
def post_initialise(self):
bots = self.bot_factory.bots()
for bot in bots:
bot.post_initialise()
@staticmethod
def ylogger_type():
return "client"
@property
def num_questions(self):
return self._questions
@property
def configuration(self):
return self._configuration
@property
def id(self):
return self._id
@property
def arguments(self):
return self._arguments
@property
def license_keys(self):
return self._license_keys
@property
def scheduler(self):
return self._scheduler
@property
def storage_factory(self):
return self._storage
@property
def bot_factory(self):
return self._bot_factory
@property
def renderer(self):
return self._renderer
@property
def trigger_manager(self):
return self._trigger_mgr
@property
def ping_responder(self):
return self._ping_responder
def get_description(self):
if self.configuration is not None:
if self.configuration.client_configuration is not None:
return self.configuration.client_configuration.description
return "Bot Client"
def get_question_counts(self):
return self.bot_factory.get_question_counts()
def add_client_arguments(self, parser=None):
del parser
# Nothing to add
return
def parse_configuration(self):
# Nothing to add
return
def parse_args(self, arguments, parsed_args):
del arguments
del parsed_args
# Nothing to add
return
def parse_arguments(self, argument_parser):
client_args = CommandLineClientArguments(self, parser=argument_parser)
client_args.parse_args(self)
return client_args
def load_license_keys(self):
if self.storage_factory.entity_storage_engine_available(
StorageFactory.LICENSE_KEYS) is True:
storage_engine = self.storage_factory.entity_storage_engine(
StorageFactory.LICENSE_KEYS)
if storage_engine is not None:
keys_store = storage_engine.license_store()
keys_store.load(self._license_keys)
else:
YLogger.error(self,
"No storage engine available for license_keys")
else:
YLogger.warning(self,
"No storage factory setting for license_keys")
def get_license_keys(self):
return
def initiate_logging(self, arguments):
if arguments.logging is not None:
try:
with open(arguments.logging, 'r+',
encoding="utf-8") as yml_data_file:
logging_config = yaml.load(yml_data_file,
Loader=yaml.FullLoader)
logging.config.dictConfig(logging_config)
YLogger.info(self, "Now logging under configuration")
except Exception as excep:
YLogger.exception(self,
"Failed to open logging configuration [%s]",
excep, arguments.logging)
else:
outputLog(
self,
"Warning. No logging configuration file defined, using defaults..."
)
def get_client_configuration(self):
"""
By overriding this class in you Configuration file, you can add new configurations
and stil use the dynamic loader capabilities
:return: Client configuration object
"""
raise NotImplementedError(
"You must override this and return a subclassed client configuration"
) # pragma: no cover
def load_configuration(self, arguments, subs: Substitutions = None):
if arguments.bot_root is None:
if arguments.config_filename is not None:
arguments.bot_root = os.path.dirname(arguments.config_filename)
else:
arguments.bot_root = "."
outputLog(
self, "No bot root argument set, defaulting to [%s]" %
arguments.bot_root)
if arguments.config_filename is not None:
self._configuration = ConfigurationFactory.load_configuration_from_file(
self.get_client_configuration(), arguments.config_filename,
arguments.config_format, arguments.bot_root, subs)
else:
outputLog(
self, "No configuration file specified, using defaults only !")
self._configuration = ProgramyConfiguration(
self.get_client_configuration())
def load_scheduler(self):
if self.configuration.client_configuration.scheduler is not None:
YLogger.debug(None, "Loading Scheduler")
self._scheduler = ProgramyScheduler(
self, self.configuration.client_configuration.scheduler)
self._scheduler.start()
def load_email(self):
if self._configuration.client_configuration.email is not None:
YLogger.debug(None, "Loading Email Manager")
self._email = EmailSender(
self._configuration.client_configuration.email)
def load_trigger_manager(self):
if self._configuration.client_configuration.triggers is not None:
YLogger.debug(None, "Loading Trigger Manager")
self._trigger_mgr = TriggerManager.load_trigger_manager(
self._configuration.client_configuration.triggers)
def load_storage(self):
self._storage = StorageFactory()
if self.configuration.client_configuration.storage is not None:
YLogger.debug(None, "Loading Storage Factory")
self._storage.load_engines_from_config(
self.configuration.client_configuration.storage)
else:
outputLog(self, "No storage defined!")
def _render_callback(self):
return False
def load_renderer(self):
callback = self._render_callback()
try:
if self._configuration.client_configuration.renderer is not None:
YLogger.debug(None, "Loading Renderer")
clazz = ClassLoader.instantiate_class(
self._configuration.client_configuration.renderer)
if callback is True:
self._renderer = clazz(self)
else:
self._renderer = clazz(None)
return
except Exception as e:
YLogger.exception(None, "Failed to load config specified renderer",
e)
self._renderer = self.get_default_renderer(callback=callback)
def get_default_renderer(self, callback=True):
if callback is True:
return TextRenderer(self)
else:
return TextRenderer(None)
def create_client_context(self, userid):
client_context = ClientContext(self, userid)
client_context.bot = self._bot_factory.select_bot()
client_context.brain = client_context.bot.brain_factory.select_brain()
return client_context
def load_ping_responder(self):
self._ping_responder = PingResponder(self)
self._ping_responder.register_with_healthchecker()
def startup(self):
if self._trigger_mgr is not None:
self._trigger_mgr.trigger(event=SystemTriggers.SYSTEM_STARTUP)
if self._ping_responder is not None:
PingResponder.init_ping_response(self._ping_responder)
def shutdown(self):
if self._trigger_mgr is not None:
self._trigger_mgr.trigger(event=SystemTriggers.SYSTEM_SHUTDOWN)
if self._ping_responder is not None:
self._ping_responder.shutdown_ping_service()
def run(self, app=None):
del app
raise NotImplementedError(
"You must override this and implement the logic to run the client"
) # pragma: no cover
