def test_upload_from_file_add_document_false(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoSpellingStore(engine) self.assert_upload_from_file_exception(store, verbose=False)
def test_load_oobs(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoOOBStore(engine) self.assert_load(store)
def test_upload_from_file_verbose_no_corpus(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoSpellingStore(engine) self.assert_upload_from_file_no_corpus(store, verbose=False)
def test_empty_variables(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoVariableStore(engine) self.assert_empty_variables(store)
def test_save_learnf(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoLearnfStore(engine) self.assert_save_learnf(store)
def test_upload_from_file_exception(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoOOBStore(engine) self.assert_upload_from_file_exception(store)
def test_upload_csv_files_from_directory_with_subdir(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoSetsStore(engine) self.assert_upload_csv_files_from_directory_with_subdir(store)
def test_linkedaccounts_storage(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoLinkedAccountStore(engine) self.assert_linkedaccounts_storage(store)
def test_upload_from_file(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoPreProcessorStore(engine) self.assert_upload_from_file(store, verbose=False)
def test_set_storage(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoSetsStore(engine) self.assert_set_storage(store)
def test_upload_from_csv_file(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoSetsStore(engine) self.assert_upload_from_csv_file(store)
def test_upload_from_text(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoPersonStore(engine) self.assert_upload_from_text(store)
def test_lookup_storage(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoPersonStore(engine) self.assert_lookup_storage(store)
def test_save_errors(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoErrorsStore(engine) self.assert_errors(store)
def test_load_exception(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoPreProcessorStore(engine) self.assert_load_exception(store)
def test_category_by_groupid_storage(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoCategoryStore(engine) self.assertEqual(store.storage_engine, engine) self.assert_category_by_groupid_storage(store)
def test_empty_named(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoCategoryStore(engine) self.assertEqual(store.storage_engine, engine) self.assert_empty_name(store)
def test_upload_from_directory(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoCategoryStore(engine) self.assertEqual(store.storage_engine, engine) self.assert_upload_from_directory(store)
def tests_conversation_storage(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoConversationStore(engine) self.assertEqual(store.storage_engine, engine) self.assert_conversation_storage(store)
def test_links_storage(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoLinkStore(engine) self.assertEqual(store.storage_engine, engine) self.assert_links_storage(store)
def setUp(self): config = MongoStorageConfiguration() self.storage_engine = MongoStorageEngine(config) self.storage_engine.initialise() client = TestClient() self.context = client.create_client_context("TESTUSER") self.context.brain._security._account_linker = BasicAccountLinkerService( self.storage_engine)
def test_get_lookup_not_present(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = TestMongoLookupStore(engine) store.drop() collection = store.get_lookup() self.assertEquals({}, collection)
def load_config_section(self, configuration_file, configuration, bot_root, subs: Substitutions = None): storage = configuration_file.get_section(self._section_name, configuration) if storage is not None: entities = configuration_file.get_section("entities", storage) entity_types = configuration_file.get_child_section_keys("entities", storage) for entity in entity_types: entity_config = configuration_file.get_section(entity, entities) self._entity_store[entity] = entity_config stores = configuration_file.get_section("stores", storage) store_names = configuration_file.get_child_section_keys("stores", storage) for store in store_names: store_config = configuration_file.get_section(store, stores) keys = configuration_file.get_keys(store_config) if 'type' not in keys: YLogger.error(None, "'type' section missing from client config stores element [%s], ignoring config", store) continue if 'config' not in keys: YLogger.error(None, "'config' section missing from client config stores element [%s], ignoring config", store) continue type = configuration_file.get_option(store_config, 'type', subs=subs) if type == 'sql': config = SQLStorageConfiguration() config.load_config_section(configuration_file, store_config, bot_root, subs=subs) elif type == 'mongo': config = MongoStorageConfiguration() config.load_config_section(configuration_file, store_config, bot_root, subs=subs) elif type == 'redis': config = RedisStorageConfiguration() config.load_config_section(configuration_file, store_config, bot_root, subs=subs) elif type == 'file': config = FileStorageConfiguration() config.load_config_section(configuration_file, store_config, bot_root, subs=subs) elif type == 'logger': config = LoggerStorageConfiguration() config.load_config_section(configuration_file, store_config, bot_root, subs=subs) self._store_configs[store] = config else: YLogger.debug(self, "'storage' section missing from client config, using to defaults") self._entity_store = {} self.add_default_entities(self._entity_store) self._store_configs = {} self.add_default_stores(self._store_configs)
def upload(storetype, url, database, filename, dirname, subdir, extension, verbose=False): config = MongoStorageConfiguration() config.url = url config.database = database config.drop_all_first = False engine = MongoStorageEngine(config) engine.initialise() store = Uploader._get_store(storetype, engine) outputLog(None, "Emptying [%s]" % storetype) store.empty() outputLog(None, "Loading [%s]" % storetype) if filename is not None: count, success = store.upload_from_file( filename, fileformat=Store.TEXT_FORMAT, commit=True, verbose=verbose) outputLog(None, "Lines processed: %d" % count) outputLog(None, "Entities successful: %d" % success) elif dirname is not None: count, success = store.upload_from_directory(directory=dirname, subdir=subdir, extension=extension, commit=True, verbose=verbose) outputLog(None, "Lines processed: %d" % count) outputLog(None, "Entities successful: %d" % success) else: raise Exception("You must specify either --file or --dir") return count, success
def upload(type, url, database, filename, dirname, subdir, extension, drop_all=False, verbose=False): config = MongoStorageConfiguration() config._url = url config._database = database config._drop_all_first = drop_all engine = MongoStorageEngine(config) engine.initialise() store = Uploader._get_store(type, engine) if drop_all is True: print("Dropped database") exit(0) print("Emptying [%s]" % type) store.empty() print("Loading [%s]" % type) if filename is not None: count, success = store.upload_from_file(filename, format=Store.TEXT_FORMAT, commit=True, verbose=verbose) print("Lines processed ", count) print("Entities successful", success) elif dirname is not None: count, success = store.upload_from_directory(directory=dirname, subdir=subdir, extension=extension, commit=True, verbose=verbose) print("Lines processed ", count) print("Entities successful", success) else: raise Exception("You must specify either --file or --dir")
def test_upload_from_file_with_exception(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = MongoLicenseKeysStore(engine) store.empty() count, success = store.upload_from_file(os.path.dirname(__file__) + os.sep + "data" + os.sep + "licenses" + os.sep + "test_license.keys") self.assertEquals(0, count) self.assertEquals(0, success)
def test_get_lookup_present(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = TestMongoLookupStore(engine) store.drop() store.add_to_lookup("key1", "value1", overwrite_existing=True) collection = store.get_lookup() self.assertIsNotNone(collection)
def test_load_no_collection(self): config = MongoStorageConfiguration() engine = MongoStorageEngine(config) engine.initialise() store = TestMongoLookupStore(engine) collection = TestCollection() store.load(collection) self.assertEquals(0, len(collection.pairs.keys()))
def test_initialise_with_config(self): yaml = YamlConfigurationFile() self.assertIsNotNone(yaml) yaml.load_from_text(""" mongo: type: mongo config: url: mongodb://localhost:27017/ database: programy drop_all_first: true """, ConsoleConfiguration(), ".") mongo_config = yaml.get_section("mongo") config = MongoStorageConfiguration() config.load_config_section(yaml, mongo_config, ".") self.assertTrue(config.url.startswith('mongodb://localhost:')) self.assertEqual(config.database, "programy") self.assertEqual(config.drop_all_first, True)
def add_default_stores(amap): sql = SQLStorageConfiguration() amap['sqlite'] = {'type': 'sql', 'config': sql.create_sqlstorage_config()} mongo = MongoStorageConfiguration() amap['mongo'] = {'type': 'mongo', 'config': mongo.create_mongostorage_config()} redis = RedisStorageConfiguration() amap['redis'] = {'type': 'redis', 'config': redis.create_redisstorage_config()} file = FileStorageConfiguration() amap['file'] = {'type': 'file', 'config': file.create_filestorage_config()} logger = LoggerStorageConfiguration() amap['logger'] = {'type': 'logger', 'config': logger.create_loggerstorage_config()}