def index(self):
""" Create new environment, fill it, and create an IndexAPI
"""
from docido_sdk.index.config import YamlPullCrawlersIndexingConfig
config_yaml = osp.splitext(__file__)[0] + '.yml'
with docido_config:
docido_config.clear()
docido_config.update(Configuration.from_file(config_yaml))
env = Environment()
env.temp_dir = tempfile.mkdtemp()
test_components = self._setup_test_components(env)
pipeline = env[IndexPipelineProvider]
env[Elasticsearch]
try:
# build and provide an IndexAPI
env[YamlPullCrawlersIndexingConfig]
yield pipeline.get_index_api(None, None, None, None)
finally:
# Hide from Environment the Component classes defined
# for this test only.
for test_component in test_components:
test_component.unregister()
# Remove temporary directory previously created
if osp.isdir(env.temp_dir):
shutil.rmtree(env.temp_dir)
def index(self):
""" Create new environment, fill it, and create an IndexAPI
"""
from docido_sdk.index.config import YamlPullCrawlersIndexingConfig
with unregister_component(YamlPullCrawlersIndexingConfig):
# `YamlPullCrawlersIndexingConfig` is hidden from Environment
# instances in this context thanks to `unregister_component`.
# This is required because this class uses a custom component
# implementing `IndexPipelineConfig`, which is also the case of
# `YamlPullCrawlersIndexingConfig`. But there must be only one.
env = Environment()
env.temp_dir = tempfile.mkdtemp()
test_components = self._setup_test_components(env)
pipeline = env[IndexPipelineProvider]
try:
# build and provide an IndexAPI
yield pipeline.get_index_api(None, None, None)
finally:
# Hide from Environment the Component classes defined
# for this test only.
for test_component in test_components:
test_component.unregister()
# Remove temporary directory previously created
if osp.isdir(env.temp_dir):
shutil.rmtree(env.temp_dir)
def index(self):
""" Create new environment, fill it, and create an IndexAPI
"""
from docido_sdk.index.config import YamlPullCrawlersIndexingConfig
config_yaml = osp.splitext(__file__)[0] + '.yml'
with docido_config:
docido_config.clear()
docido_config.update(Configuration.from_file(config_yaml))
env = Environment()
env.temp_dir = tempfile.mkdtemp()
test_components = self._setup_test_components(env)
pipeline = env[IndexPipelineProvider]
env[Elasticsearch]
try:
# build and provide an IndexAPI
env[YamlPullCrawlersIndexingConfig]
yield pipeline.get_index_api(None, None, None, None)
finally:
# Hide from Environment the Component classes defined
# for this test only.
for test_component in test_components:
test_component.unregister()
# Remove temporary directory previously created
if osp.isdir(env.temp_dir):
shutil.rmtree(env.temp_dir)
def test_abstract_component_not_registered(self):
from docido_sdk.core import ComponentMeta
class AbstractComponent(Component):
abstract = True
class ConcreteComponent(Component):
pass
self.assertNotIn(AbstractComponent, ComponentMeta._components)
env = Environment()
env[ConcreteComponent]
with self.assertRaises(DocidoError):
env[AbstractComponent]
self.assertFalse(AbstractComponent in env)
self.assertTrue(ConcreteComponent in env)
self.assertTrue(env.is_component_enabled(ConcreteComponent))
self.assertTrue(env.is_component_enabled(AbstractComponent))
def test_abstract_component_not_registered(self):
from docido_sdk.core import ComponentMeta
class AbstractComponent(Component):
abstract = True
class ConcreteComponent(Component):
pass
self.assertNotIn(AbstractComponent, ComponentMeta._components)
env = Environment()
env[ConcreteComponent]
with self.assertRaises(DocidoError):
env[AbstractComponent]
self.assertFalse(AbstractComponent in env)
self.assertTrue(ConcreteComponent in env)
self.assertTrue(env.is_component_enabled(ConcreteComponent))
self.assertTrue(env.is_component_enabled(AbstractComponent))
def test_expect_1_when_0(self):
class expect_one_pika(Component):
pika = ExtensionPoint(Pika, unique=True)
env = Environment()
consumer = env[expect_one_pika]
with self.assertRaises(Exception) as exc:
consumer.pika
self.assertEqual(exc.exception.message,
"Expected one 'Pika' component, but found 0")
def test_component_with_invalid_constructor(self):
class SpuriousComponent(Component):
def __init__(self, env, another_parameter):
pass
env = Environment()
with self.assertRaises(DocidoError) as exc:
env[SpuriousComponent]
error_message = 'Unable to instantiate component <class'
self.assertTrue(exc.exception.message.startswith(error_message))
def index(self):
""" Create new environment, fill it, and create an IndexAPI
"""
from docido_sdk.index.config import YamlPullCrawlersIndexingConfig
with unregister_component(YamlPullCrawlersIndexingConfig):
env = Environment()
env.temp_dir = tempfile.mkdtemp()
test_components = self._setup_test_components(env)
pipeline = env[IndexPipelineProvider]
try:
# build and provide an IndexAPI
yield pipeline.get_index_api(None, None, None)
finally:
# Hide from Environment the Component classes defined
# for this test only.
for test_component in test_components:
test_component.unregister()
# Remove temporary directory previously created
if osp.isdir(env.temp_dir):
shutil.rmtree(env.temp_dir)
def index(self):
""" Create new environment, fill it, and create an IndexAPI
"""
from docido_sdk.index.config import YamlPullCrawlersIndexingConfig
with unregister_component(YamlPullCrawlersIndexingConfig):
env = Environment()
env.temp_dir = tempfile.mkdtemp()
test_components = self._setup_test_components(env)
pipeline = env[IndexPipelineProvider]
try:
# build and provide an IndexAPI
yield pipeline.get_index_api(None, None, None, None)
finally:
# Hide from Environment the Component classes defined
# for this test only.
for test_component in test_components:
test_component.unregister()
# Remove temporary directory previously created
if osp.isdir(env.temp_dir):
shutil.rmtree(env.temp_dir)
def run_crawl(self, cls, *args, **kwargs):
with restore_dict_kv(os.environ, 'DOCIDO_CC_RUNS'), \
docido_config, \
self.crawler(cls, *args, **kwargs), \
self.check_crawl(*args, **kwargs):
config_prefix = osp.splitext(__file__)[0]
os.environ['DOCIDO_CC_RUNS'] = config_prefix + '-runs.yml'
config_settings = config_prefix + '-settings.yml'
docido_config.update(Configuration.from_file(config_settings))
for c in dcc_run.run([], environment=Environment()):
shutil.rmtree(c['crawl_path'])
def test_expect_1_when_more(self):
class expect_one_foobar(Component):
foobar = ExtensionPoint(Foobar, unique=True)
env = Environment()
consumer = env[expect_one_foobar]
with self.assertRaises(Exception) as exc:
consumer.foobar
self.assertEqual(
exc.exception.message,
"Expected one 'Foobar' component, but found 2: "
"BarComponent, FooComponent")
def kv(self):
from docido_sdk.index.config import YamlPullCrawlersIndexingConfig
with unregister_component(YamlPullCrawlersIndexingConfig):
env = Environment()
env.temp_dir = tempfile.mkdtemp()
class ForcePipeline(Component):
implements(IndexPipelineConfig)
def get_pipeline(self, *args, **kwargs):
return [env[LocalKV]]
class ForceConfig(Component):
implements(IndexAPIConfigurationProvider)
def get_index_api_conf(self, service, docido_user_id,
account_login, config):
return {
'local_storage': {
'documents': {
'path': env.temp_dir,
},
'kv': {
'path': env.temp_dir,
},
},
}
env[ForcePipeline]
env[ForceConfig]
pipeline = env[IndexPipelineProvider]
try:
yield pipeline.get_index_api(None, None, None, None)
finally:
ForcePipeline.unregister()
ForceConfig.unregister()
if osp.isdir(env.temp_dir):
shutil.rmtree(env.temp_dir)
def kv(self):
from docido_sdk.index.config import YamlPullCrawlersIndexingConfig
with unregister_component(YamlPullCrawlersIndexingConfig):
env = Environment()
env.temp_dir = tempfile.mkdtemp()
class ForcePipeline(Component):
implements(IndexPipelineConfig)
def get_pipeline(self):
return [env[LocalKV]]
class ForceConfig(Component):
implements(IndexAPIConfigurationProvider)
def get_index_api_conf(self, service,
docido_user_id, account_login):
return {
'local_storage': {
'documents': {
'path': env.temp_dir,
},
'kv': {
'path': env.temp_dir,
},
},
}
env[ForcePipeline]
env[ForceConfig]
pipeline = env[IndexPipelineProvider]
try:
yield pipeline.get_index_api(None, None, None)
finally:
ForcePipeline.unregister()
ForceConfig.unregister()
if osp.isdir(env.temp_dir):
shutil.rmtree(env.temp_dir)
def index(self):
from docido_sdk.index.config import YamlPullCrawlersIndexingConfig
config_yaml = osp.splitext(__file__)[0] + '.yml'
with docido_config:
docido_config.clear()
docido_config.update(Configuration.from_file(config_yaml))
env = Environment()
test_components = self._setup_test_components(env)
env[IndexPipelineProvider]
env[LocalDumbIndex]
env[processor.CheckProcessor]
try:
env[YamlPullCrawlersIndexingConfig]
index_builder = env[IndexPipelineProvider]
yield index_builder.get_index_api('check-processor-test',
'user2', 'account3', None)
finally:
for test_component in test_components:
test_component.unregister()
def test_build_pipeline(self):
env = Environment()
env[Processor1Provider]
env[Processor2Provider]
env[IndexPipelineConfig]
pipeline_provider = env[IndexPipelineProvider]
index_api = pipeline_provider.get_index_api('service1', 'user1',
'account1', None)
config = {
'service': 'service1',
'docido_user_id': 'user1',
'account_login': '******',
}
self.assertIsNotNone(index_api)
self.assertTrue(isinstance(index_api, Processor1))
self.assertEqual(index_api._config, config)
self.assertIsNotNone(index_api._parent)
self.assertTrue(isinstance(index_api._parent, Processor2))
self.assertEqual(index_api._parent._config, config)
self.assertTrue(isinstance(index_api._parent._parent, IndexAPI))
