def probe(self, file: str) -> None: # pragma: no cover
config = parse_yaml_file(file)
self.assertIsInstance(config, dict)
for crawler_config in config['crawlers']:
imagecrawler = get_imagecrawler(crawler_config)
self._probe_crawl(imagecrawler)
sleep(0.023) # do not be too greedy
def validate(self, file: str) -> None:
config = parse_yaml_file(file)
self.assertIsInstance(config, dict)
imagecrawlers = list() # type: List[BaseImageCrawler]
for crawler_config in config['crawlers']:
imagecrawler = get_imagecrawler(crawler_config)
self.assertNotIn(imagecrawler, imagecrawlers, msg='Duplicate ImageCrawler')
imagecrawlers.append(imagecrawler)
def test_set_optional_loglevel(self) -> None:
# arrange
file = realpath(
path_join(dirname(__file__), 'configs', 'positive',
'logging_level_missing.yaml'))
# act
config = parse_yaml_file(file)
# assert
self.assertEqual(config['logging']['level'], 'INFO')
def test_set_optional_config(self) -> None:
# arrange
file = realpath(
path_join(dirname(__file__), 'configs', 'positive',
'crawler_config_missing.yaml'))
# act
config = parse_yaml_file(file)
# assert
self.assertEqual(len(config["crawlers"]), 1)
self.assertDictEqual(config["crawlers"][0]["config"], dict())