def test_upload_from_file_add_document_false(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoSpellingStore(engine)
self.assert_upload_from_file_exception(store, verbose=False)
def test_load_oobs(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoOOBStore(engine)
self.assert_load(store)
def test_upload_from_file_verbose_no_corpus(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoSpellingStore(engine)
self.assert_upload_from_file_no_corpus(store, verbose=False)
def test_empty_variables(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoVariableStore(engine)
self.assert_empty_variables(store)
def test_save_learnf(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoLearnfStore(engine)
self.assert_save_learnf(store)
def test_upload_from_file_exception(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoOOBStore(engine)
self.assert_upload_from_file_exception(store)
def test_upload_csv_files_from_directory_with_subdir(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoSetsStore(engine)
self.assert_upload_csv_files_from_directory_with_subdir(store)
def test_linkedaccounts_storage(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoLinkedAccountStore(engine)
self.assert_linkedaccounts_storage(store)
def test_upload_from_file(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoPreProcessorStore(engine)
self.assert_upload_from_file(store, verbose=False)
def test_set_storage(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoSetsStore(engine)
self.assert_set_storage(store)
def test_upload_from_csv_file(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoSetsStore(engine)
self.assert_upload_from_csv_file(store)
def test_upload_from_text(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoPersonStore(engine)
self.assert_upload_from_text(store)
def test_lookup_storage(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoPersonStore(engine)
self.assert_lookup_storage(store)
def test_save_errors(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoErrorsStore(engine)
self.assert_errors(store)
def test_load_exception(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoPreProcessorStore(engine)
self.assert_load_exception(store)
def test_category_by_groupid_storage(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoCategoryStore(engine)
self.assertEqual(store.storage_engine, engine)
self.assert_category_by_groupid_storage(store)
def test_empty_named(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoCategoryStore(engine)
self.assertEqual(store.storage_engine, engine)
self.assert_empty_name(store)
def test_upload_from_directory(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoCategoryStore(engine)
self.assertEqual(store.storage_engine, engine)
self.assert_upload_from_directory(store)
def tests_conversation_storage(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoConversationStore(engine)
self.assertEqual(store.storage_engine, engine)
self.assert_conversation_storage(store)
def test_links_storage(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoLinkStore(engine)
self.assertEqual(store.storage_engine, engine)
self.assert_links_storage(store)
def setUp(self):
config = MongoStorageConfiguration()
self.storage_engine = MongoStorageEngine(config)
self.storage_engine.initialise()
client = TestClient()
self.context = client.create_client_context("TESTUSER")
self.context.brain._security._account_linker = BasicAccountLinkerService(
self.storage_engine)
def test_get_lookup_not_present(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = TestMongoLookupStore(engine)
store.drop()
collection = store.get_lookup()
self.assertEquals({}, collection)
def load_config_section(self, configuration_file, configuration, bot_root, subs: Substitutions = None):
storage = configuration_file.get_section(self._section_name, configuration)
if storage is not None:
entities = configuration_file.get_section("entities", storage)
entity_types = configuration_file.get_child_section_keys("entities", storage)
for entity in entity_types:
entity_config = configuration_file.get_section(entity, entities)
self._entity_store[entity] = entity_config
stores = configuration_file.get_section("stores", storage)
store_names = configuration_file.get_child_section_keys("stores", storage)
for store in store_names:
store_config = configuration_file.get_section(store, stores)
keys = configuration_file.get_keys(store_config)
if 'type' not in keys:
YLogger.error(None, "'type' section missing from client config stores element [%s], ignoring config", store)
continue
if 'config' not in keys:
YLogger.error(None, "'config' section missing from client config stores element [%s], ignoring config", store)
continue
type = configuration_file.get_option(store_config, 'type', subs=subs)
if type == 'sql':
config = SQLStorageConfiguration()
config.load_config_section(configuration_file, store_config, bot_root, subs=subs)
elif type == 'mongo':
config = MongoStorageConfiguration()
config.load_config_section(configuration_file, store_config, bot_root, subs=subs)
elif type == 'redis':
config = RedisStorageConfiguration()
config.load_config_section(configuration_file, store_config, bot_root, subs=subs)
elif type == 'file':
config = FileStorageConfiguration()
config.load_config_section(configuration_file, store_config, bot_root, subs=subs)
elif type == 'logger':
config = LoggerStorageConfiguration()
config.load_config_section(configuration_file, store_config, bot_root, subs=subs)
self._store_configs[store] = config
else:
YLogger.debug(self, "'storage' section missing from client config, using to defaults")
self._entity_store = {}
self.add_default_entities(self._entity_store)
self._store_configs = {}
self.add_default_stores(self._store_configs)
def upload(storetype,
url,
database,
filename,
dirname,
subdir,
extension,
verbose=False):
config = MongoStorageConfiguration()
config.url = url
config.database = database
config.drop_all_first = False
engine = MongoStorageEngine(config)
engine.initialise()
store = Uploader._get_store(storetype, engine)
outputLog(None, "Emptying [%s]" % storetype)
store.empty()
outputLog(None, "Loading [%s]" % storetype)
if filename is not None:
count, success = store.upload_from_file(
filename,
fileformat=Store.TEXT_FORMAT,
commit=True,
verbose=verbose)
outputLog(None, "Lines processed: %d" % count)
outputLog(None, "Entities successful: %d" % success)
elif dirname is not None:
count, success = store.upload_from_directory(directory=dirname,
subdir=subdir,
extension=extension,
commit=True,
verbose=verbose)
outputLog(None, "Lines processed: %d" % count)
outputLog(None, "Entities successful: %d" % success)
else:
raise Exception("You must specify either --file or --dir")
return count, success
def upload(type,
url,
database,
filename,
dirname,
subdir,
extension,
drop_all=False,
verbose=False):
config = MongoStorageConfiguration()
config._url = url
config._database = database
config._drop_all_first = drop_all
engine = MongoStorageEngine(config)
engine.initialise()
store = Uploader._get_store(type, engine)
if drop_all is True:
print("Dropped database")
exit(0)
print("Emptying [%s]" % type)
store.empty()
print("Loading [%s]" % type)
if filename is not None:
count, success = store.upload_from_file(filename,
format=Store.TEXT_FORMAT,
commit=True,
verbose=verbose)
print("Lines processed ", count)
print("Entities successful", success)
elif dirname is not None:
count, success = store.upload_from_directory(directory=dirname,
subdir=subdir,
extension=extension,
commit=True,
verbose=verbose)
print("Lines processed ", count)
print("Entities successful", success)
else:
raise Exception("You must specify either --file or --dir")
def test_upload_from_file_with_exception(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = MongoLicenseKeysStore(engine)
store.empty()
count, success = store.upload_from_file(os.path.dirname(__file__) + os.sep + "data" + os.sep + "licenses" + os.sep + "test_license.keys")
self.assertEquals(0, count)
self.assertEquals(0, success)
def test_get_lookup_present(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = TestMongoLookupStore(engine)
store.drop()
store.add_to_lookup("key1", "value1", overwrite_existing=True)
collection = store.get_lookup()
self.assertIsNotNone(collection)
def test_load_no_collection(self):
config = MongoStorageConfiguration()
engine = MongoStorageEngine(config)
engine.initialise()
store = TestMongoLookupStore(engine)
collection = TestCollection()
store.load(collection)
self.assertEquals(0, len(collection.pairs.keys()))
def test_initialise_with_config(self):
yaml = YamlConfigurationFile()
self.assertIsNotNone(yaml)
yaml.load_from_text("""
mongo:
type: mongo
config:
url: mongodb://localhost:27017/
database: programy
drop_all_first: true
""", ConsoleConfiguration(), ".")
mongo_config = yaml.get_section("mongo")
config = MongoStorageConfiguration()
config.load_config_section(yaml, mongo_config, ".")
self.assertTrue(config.url.startswith('mongodb://localhost:'))
self.assertEqual(config.database, "programy")
self.assertEqual(config.drop_all_first, True)
def add_default_stores(amap):
sql = SQLStorageConfiguration()
amap['sqlite'] = {'type': 'sql',
'config': sql.create_sqlstorage_config()}
mongo = MongoStorageConfiguration()
amap['mongo'] = {'type': 'mongo',
'config': mongo.create_mongostorage_config()}
redis = RedisStorageConfiguration()
amap['redis'] = {'type': 'redis',
'config': redis.create_redisstorage_config()}
file = FileStorageConfiguration()
amap['file'] = {'type': 'file',
'config': file.create_filestorage_config()}
logger = LoggerStorageConfiguration()
amap['logger'] = {'type': 'logger',
'config': logger.create_loggerstorage_config()}
