def get_test_scrapers():
"""Return scrapers that should be tested."""
if "TESTALL" in os.environ:
# test all comics (this will take some time)
scrapers = scraper.get_scrapers()
else:
if 'TESTCOMICS' in os.environ:
scraper_pattern = re.compile(os.environ['TESTCOMICS'])
else:
# Get limited number of scraper tests on Travis builds to make it
# faster
testscrapernames = [
# "classic" _BasicScraper
'AbstruseGoose',
# complex _ParserScraper
'GoComics/CalvinAndHobbes',
# _WordPressScraper
'GrrlPower'
]
scraper_pattern = re.compile('^(' + '|'.join(testscrapernames) +
')$')
scrapers = [
scraperobj for scraperobj in scraper.get_scrapers()
if scraper_pattern.match(scraperobj.name)
]
return scrapers
def get_test_scrapers():
"""Return scrapers that should be tested."""
if "TESTALL" in os.environ:
# test all comics (this will take some time)
scrapers = scraper.get_scrapers()
else:
if 'TESTCOMICS' in os.environ:
scraper_pattern = re.compile(os.environ['TESTCOMICS'])
else:
# Get limited number of scraper tests on Travis builds to make it
# faster
testscrapernames = [
# "classic" _BasicScraper
'AbstruseGoose',
# complex _ParserScraper
'GoComics/CalvinAndHobbes',
# _WordPressScraper
'GrrlPower'
]
scraper_pattern = re.compile('^(' + '|'.join(testscrapernames) +
')$')
scrapers = [
scraperobj for scraperobj in scraper.get_scrapers()
if scraper_pattern.match(scraperobj.name)
]
return scrapers
def generate_comic_testers():
"""For each comic scraper, create a test class."""
g = globals()
# optional: limit number of scraper tests
max_scrapers = 0
if max_scrapers:
scrapers = islice(scraper.get_scrapers(), 0, max_scrapers)
else:
scrapers = scraper.get_scrapers()
for scraperclass in scrapers:
name = 'Test'+scraperclass.__name__
g[name] = make_comic_tester(name, scraperclass=scraperclass)
def get_used_languages():
lang = {}
for scraperobj in get_scrapers():
l = scraperobj.lang
if l not in lang:
lang[l] = scraperobj.language()
return lang
def get_used_languages():
languages = {}
for scraperobj in get_scrapers():
lang = scraperobj.lang
if lang not in languages:
languages[lang] = scraperobj.language()
return languages
def has_comic(name):
cname = name.lower()
for scraperclass in get_scrapers():
lname = scraperclass.get_name().lower()
if lname == cname:
return True
return False
def get_used_languages():
lang = {}
for scraperobj in get_scrapers():
l = scraperobj.lang
if l not in lang:
lang[l] = scraperobj.language()
return lang
def has_creators_comic(name):
cname = "Creators/%s" % name
for scraperclass in get_scrapers():
lname = scraperclass.get_name().lower()
if lname == cname.lower():
return True
return False
def has_comic(name):
cname = ("Creators/%s" % name).lower()
gname = ("GoComics/%s" % name).lower()
for scraperclass in get_scrapers():
lname = scraperclass.get_name().lower()
if lname == cname or lname == gname:
return True
return False
def has_comic(name):
"""Check if comic name already exists."""
names = [("Creators/%s" % name).lower(), ("GoComics/%s" % name).lower(), ("ComicGenesis/%s" % name).lower()]
for scraperobj in get_scrapers():
lname = scraperobj.name.lower()
if lname in names:
return True
return False
def test_names(self):
for scraperobj in scraper.get_scrapers():
name = scraperobj.name
assert name.count('/') <= 1
if '/' in name:
comicname = name.split('/')[1]
else:
comicname = name
assert re.sub("[^0-9a-zA-Z_]", "", comicname) == comicname
def find_dups(self, name):
"""Check if comic name already exists."""
names = [(tmpl % name).lower() for tmpl in self.dup_templates]
if names:
for scraperobj in scraper.get_scrapers():
lname = scraperobj.name.lower()
if lname in names:
return scraperobj.name
return None
def test_names(self):
for scraperclass in scraper.get_scrapers():
name = scraperclass.get_name()
self.assertTrue(name.count('/') <= 1, name)
if '/' in name:
comicname = name.split('/')[1]
else:
comicname = name
self.assertEqual(util.asciify(comicname), comicname)
def find_dups(self, name):
"""Check if comic name already exists."""
names = [(tmpl % name).lower() for tmpl in self.dup_templates]
if names:
for scraperobj in scraper.get_scrapers():
lname = scraperobj.name.lower()
if lname in names:
return scraperobj.name
return None
def test_names(self):
for scraperobj in scraper.get_scrapers():
name = scraperobj.name
assert name.count('/') <= 1
if '/' in name:
comicname = name.split('/')[1]
else:
comicname = name
assert re.sub("[^0-9a-zA-Z_]", "", comicname) == comicname
def get_test_name(line):
classname = line.split('::')[1][4:]
for scraper in get_scrapers():
if scraper.__name__ == classname:
try:
url = scraper.starter()
except Exception:
url = None
return scraper.get_name(), url
raise ValueError("Scraper %r not found" % classname)
def has_comic(name):
"""Check if comic name already exists."""
names = [
("Creators/%s" % name).lower(),
("GoComics/%s" % name).lower(),
]
for scraperobj in get_scrapers():
lname = scraperclass.name.lower()
if lname in names:
return True
return False
def test_get_scrapers(self):
for scraperobj in scraper.get_scrapers():
scraperobj.indexes = ["bla"]
assert scraperobj.url, "missing url in %s" % scraperobj.name