def test_load_model_crawler_model_mapper_maps_types(
crawler_inline_definitions: CrawlinoStore):
m = CrawlinoModel(crawler_inline_definitions)
assert all(
isinstance(x, CMModelsMappersObject)
for x in m.model.mappers.all()) == True
def test_load_model_crawler_model_fields_get(
crawler_inline_definitions: CrawlinoStore):
m = CrawlinoModel(crawler_inline_definitions)
field_obj = m.model.fields.get("url")
assert isinstance(field_obj, CMModelsField)
assert field_obj.name == "url"
assert field_obj.type == "text"
assert field_obj.key == True
def test_load_input_import_not_type(crawler_inline_definitions: File):
m = CrawlinoModel(crawler_inline_definitions)
assert hasattr(m.input, "type")
def test_load_model_crawler_model_mapper_get_mapper(
crawler_inline_definitions: CrawlinoStore):
m = CrawlinoModel(crawler_inline_definitions)
assert m.model.mappers.get("my_database").name == "my_database"
def test_load_model_crawler_model_mapper_check_format(
check_model_mappers_invalid_format: File):
with pytest.raises(CrawlinoFormatError):
CrawlinoModel(check_model_mappers_invalid_format)
def test_load_model_crawler_model_mapper_loaded_maps(
crawler_inline_definitions: CrawlinoStore):
m = CrawlinoModel(crawler_inline_definitions)
assert len(m.model.mappers.all()) == 4
def test_load_model_crawler_model_fields_check_duplicates(
check_model_fields_duplicate_names: File):
with pytest.raises(CrawlinoFormatError):
CrawlinoModel(check_model_fields_duplicate_names)
def test_load_model_crawler_model_name(
crawler_inline_definitions: CrawlinoStore):
m = CrawlinoModel(crawler_inline_definitions)
assert m.model.name == "my_model_1"
def test_load_config_ok(crawler_inline_definitions: File):
m = CrawlinoModel(crawler_inline_definitions)
assert isinstance(m.config, CMConfig)
def test_load_input_import_ok(crawler_import_definitions: File):
m = CrawlinoModel(crawler_import_definitions)
assert isinstance(m.input, CMInput)
def test_input_without_config(input_without_config: File):
with pytest.raises(CrawlinoValueError):
CrawlinoModel(input_without_config)
def test_input_without_type(input_without_type: File):
with pytest.raises(CrawlinoValueError):
CrawlinoModel(input_without_type)
def test_load_input_raises_invalid_valid_types(invalid_types: File):
with pytest.raises(CrawlinoValueError):
CrawlinoModel(invalid_types)
def test_load_input_inline_check_valid_types(crawler_import_definitions: File):
m = CrawlinoModel(crawler_import_definitions)
assert CrawlinoModulesStore.find_module("input", m.input.type) is not None
def test_load_input_inline_not_config(crawler_import_definitions: File):
m = CrawlinoModel(crawler_import_definitions)
assert hasattr(m.input, "config")
def test_sources_import_type_is_list(crawler_inline_definitions: File):
m = CrawlinoModel(crawler_inline_definitions)
assert isinstance(m.sources.get_list(), list)
def test_sources_empty_sources(empty_sources: File):
with pytest.raises(CrawlinoValueError):
CrawlinoModel(empty_sources)
def test_load_config_generic_logger_name(generic_logger_name: File):
m = CrawlinoModel(generic_logger_name)
assert m.config.logger == "crawlino"
def test_load_model_crawler_model_fields_check_type(
crawler_inline_definitions: CrawlinoStore):
m = CrawlinoModel(crawler_inline_definitions)
assert isinstance(m.model.fields, CMModelsFields)
def test_load_config_custom_logger_name_inline(
crawler_inline_definitions: File):
m = CrawlinoModel(crawler_inline_definitions)
assert m.config.logger == "mylogger"
def test_load_model_crawler_model_fields_not_found(
crawler_inline_definitions: CrawlinoStore):
m = CrawlinoModel(crawler_inline_definitions)
with pytest.raises(CrawlinoNotFoundError):
m.model.fields.get("aaaa")
def test_sources_check_each_element_type(crawler_import_definitions: File):
m = CrawlinoModel(crawler_import_definitions)
r = list(isinstance(x, CMSource) for x in m.sources.get_list())
assert len(r) != 0 and all(r) is True
def test_load_model_crawler_model_mapper_check_type(
crawler_inline_definitions: CrawlinoStore):
m = CrawlinoModel(crawler_inline_definitions)
assert isinstance(m.model.mappers, CMModelsMappers)
def test_sources_find_element(crawler_import_definitions: File):
m = CrawlinoModel(crawler_import_definitions)
assert isinstance(m.sources.find("dummySource"), CMSource)
def test_sources_check_is_iterable(crawler_import_definitions: File):
m = CrawlinoModel(crawler_import_definitions)
assert isinstance(m.sources, Iterable)
def test_sources_check_iteration(crawler_import_definitions: File):
m = CrawlinoModel(crawler_import_definitions)
content = list(m.sources)
assert len(content) > 0 and all(isinstance(x, CMSource) for x in content)
def test_load_model_crawler_model_mapper_invalid_mapper(
check_model_mappers_invalid_mapper: File):
with pytest.raises(CrawlinoValueError):
CrawlinoModel(check_model_mappers_invalid_mapper)
def test_sources_import_ok(crawler_import_definitions: File):
m = CrawlinoModel(crawler_import_definitions)
assert isinstance(m.sources, CMSources)
def test_load_model_crawler_model_mapper_get_mapper_non_exists(
crawler_inline_definitions: CrawlinoStore):
m = CrawlinoModel(crawler_inline_definitions)
with pytest.raises(CrawlinoNotFoundError):
m.model.mappers.get("XXXXX")
def test_load_hooks_import_check_hooks_types(crawler_import_definitions: TextIOWrapper):
m = CrawlinoModel(crawler_import_definitions)
v = all(isinstance(x, CMHook) for x in m.hooks.get_list())
assert len(m.hooks.get_list()) > 0 and v is True
