def index(self): """ Create new environment, fill it, and create an IndexAPI """ from docido_sdk.index.config import YamlPullCrawlersIndexingConfig config_yaml = osp.splitext(__file__)[0] + '.yml' with docido_config: docido_config.clear() docido_config.update(Configuration.from_file(config_yaml)) env = Environment() env.temp_dir = tempfile.mkdtemp() test_components = self._setup_test_components(env) pipeline = env[IndexPipelineProvider] env[Elasticsearch] try: # build and provide an IndexAPI env[YamlPullCrawlersIndexingConfig] yield pipeline.get_index_api(None, None, None, None) finally: # Hide from Environment the Component classes defined # for this test only. for test_component in test_components: test_component.unregister() # Remove temporary directory previously created if osp.isdir(env.temp_dir): shutil.rmtree(env.temp_dir)
def index(self): """ Create new environment, fill it, and create an IndexAPI """ from docido_sdk.index.config import YamlPullCrawlersIndexingConfig with unregister_component(YamlPullCrawlersIndexingConfig): # `YamlPullCrawlersIndexingConfig` is hidden from Environment # instances in this context thanks to `unregister_component`. # This is required because this class uses a custom component # implementing `IndexPipelineConfig`, which is also the case of # `YamlPullCrawlersIndexingConfig`. But there must be only one. env = Environment() env.temp_dir = tempfile.mkdtemp() test_components = self._setup_test_components(env) pipeline = env[IndexPipelineProvider] try: # build and provide an IndexAPI yield pipeline.get_index_api(None, None, None) finally: # Hide from Environment the Component classes defined # for this test only. for test_component in test_components: test_component.unregister() # Remove temporary directory previously created if osp.isdir(env.temp_dir): shutil.rmtree(env.temp_dir)
def test_abstract_component_not_registered(self): from docido_sdk.core import ComponentMeta class AbstractComponent(Component): abstract = True class ConcreteComponent(Component): pass self.assertNotIn(AbstractComponent, ComponentMeta._components) env = Environment() env[ConcreteComponent] with self.assertRaises(DocidoError): env[AbstractComponent] self.assertFalse(AbstractComponent in env) self.assertTrue(ConcreteComponent in env) self.assertTrue(env.is_component_enabled(ConcreteComponent)) self.assertTrue(env.is_component_enabled(AbstractComponent))
def test_expect_1_when_0(self): class expect_one_pika(Component): pika = ExtensionPoint(Pika, unique=True) env = Environment() consumer = env[expect_one_pika] with self.assertRaises(Exception) as exc: consumer.pika self.assertEqual(exc.exception.message, "Expected one 'Pika' component, but found 0")
def test_component_with_invalid_constructor(self): class SpuriousComponent(Component): def __init__(self, env, another_parameter): pass env = Environment() with self.assertRaises(DocidoError) as exc: env[SpuriousComponent] error_message = 'Unable to instantiate component <class' self.assertTrue(exc.exception.message.startswith(error_message))
def index(self): """ Create new environment, fill it, and create an IndexAPI """ from docido_sdk.index.config import YamlPullCrawlersIndexingConfig with unregister_component(YamlPullCrawlersIndexingConfig): env = Environment() env.temp_dir = tempfile.mkdtemp() test_components = self._setup_test_components(env) pipeline = env[IndexPipelineProvider] try: # build and provide an IndexAPI yield pipeline.get_index_api(None, None, None) finally: # Hide from Environment the Component classes defined # for this test only. for test_component in test_components: test_component.unregister() # Remove temporary directory previously created if osp.isdir(env.temp_dir): shutil.rmtree(env.temp_dir)
def index(self): """ Create new environment, fill it, and create an IndexAPI """ from docido_sdk.index.config import YamlPullCrawlersIndexingConfig with unregister_component(YamlPullCrawlersIndexingConfig): env = Environment() env.temp_dir = tempfile.mkdtemp() test_components = self._setup_test_components(env) pipeline = env[IndexPipelineProvider] try: # build and provide an IndexAPI yield pipeline.get_index_api(None, None, None, None) finally: # Hide from Environment the Component classes defined # for this test only. for test_component in test_components: test_component.unregister() # Remove temporary directory previously created if osp.isdir(env.temp_dir): shutil.rmtree(env.temp_dir)
def run_crawl(self, cls, *args, **kwargs): with restore_dict_kv(os.environ, 'DOCIDO_CC_RUNS'), \ docido_config, \ self.crawler(cls, *args, **kwargs), \ self.check_crawl(*args, **kwargs): config_prefix = osp.splitext(__file__)[0] os.environ['DOCIDO_CC_RUNS'] = config_prefix + '-runs.yml' config_settings = config_prefix + '-settings.yml' docido_config.update(Configuration.from_file(config_settings)) for c in dcc_run.run([], environment=Environment()): shutil.rmtree(c['crawl_path'])
def test_expect_1_when_more(self): class expect_one_foobar(Component): foobar = ExtensionPoint(Foobar, unique=True) env = Environment() consumer = env[expect_one_foobar] with self.assertRaises(Exception) as exc: consumer.foobar self.assertEqual( exc.exception.message, "Expected one 'Foobar' component, but found 2: " "BarComponent, FooComponent")
def kv(self): from docido_sdk.index.config import YamlPullCrawlersIndexingConfig with unregister_component(YamlPullCrawlersIndexingConfig): env = Environment() env.temp_dir = tempfile.mkdtemp() class ForcePipeline(Component): implements(IndexPipelineConfig) def get_pipeline(self, *args, **kwargs): return [env[LocalKV]] class ForceConfig(Component): implements(IndexAPIConfigurationProvider) def get_index_api_conf(self, service, docido_user_id, account_login, config): return { 'local_storage': { 'documents': { 'path': env.temp_dir, }, 'kv': { 'path': env.temp_dir, }, }, } env[ForcePipeline] env[ForceConfig] pipeline = env[IndexPipelineProvider] try: yield pipeline.get_index_api(None, None, None, None) finally: ForcePipeline.unregister() ForceConfig.unregister() if osp.isdir(env.temp_dir): shutil.rmtree(env.temp_dir)
def kv(self): from docido_sdk.index.config import YamlPullCrawlersIndexingConfig with unregister_component(YamlPullCrawlersIndexingConfig): env = Environment() env.temp_dir = tempfile.mkdtemp() class ForcePipeline(Component): implements(IndexPipelineConfig) def get_pipeline(self): return [env[LocalKV]] class ForceConfig(Component): implements(IndexAPIConfigurationProvider) def get_index_api_conf(self, service, docido_user_id, account_login): return { 'local_storage': { 'documents': { 'path': env.temp_dir, }, 'kv': { 'path': env.temp_dir, }, }, } env[ForcePipeline] env[ForceConfig] pipeline = env[IndexPipelineProvider] try: yield pipeline.get_index_api(None, None, None) finally: ForcePipeline.unregister() ForceConfig.unregister() if osp.isdir(env.temp_dir): shutil.rmtree(env.temp_dir)
def index(self): from docido_sdk.index.config import YamlPullCrawlersIndexingConfig config_yaml = osp.splitext(__file__)[0] + '.yml' with docido_config: docido_config.clear() docido_config.update(Configuration.from_file(config_yaml)) env = Environment() test_components = self._setup_test_components(env) env[IndexPipelineProvider] env[LocalDumbIndex] env[processor.CheckProcessor] try: env[YamlPullCrawlersIndexingConfig] index_builder = env[IndexPipelineProvider] yield index_builder.get_index_api('check-processor-test', 'user2', 'account3', None) finally: for test_component in test_components: test_component.unregister()
def test_build_pipeline(self): env = Environment() env[Processor1Provider] env[Processor2Provider] env[IndexPipelineConfig] pipeline_provider = env[IndexPipelineProvider] index_api = pipeline_provider.get_index_api('service1', 'user1', 'account1', None) config = { 'service': 'service1', 'docido_user_id': 'user1', 'account_login': '******', } self.assertIsNotNone(index_api) self.assertTrue(isinstance(index_api, Processor1)) self.assertEqual(index_api._config, config) self.assertIsNotNone(index_api._parent) self.assertTrue(isinstance(index_api._parent, Processor2)) self.assertEqual(index_api._parent._config, config) self.assertTrue(isinstance(index_api._parent._parent, IndexAPI))