def generate_comic_testers():
"""For each comic scraper, create a test class."""
g = globals()
if "TESTALL" not in os.environ:
# Get limited number of scraper tests on Travis builds.
max_scrapers = 1
scraperclasses = islice(scraper.get_scraperclasses(), 0, max_scrapers)
else:
scraperclasses = scraper.get_scraperclasses()
for scraperclass in scraperclasses:
name = 'Test'+scraperclass.__name__
g[name] = make_comic_tester(name, scraperclass=scraperclass)
def generate_comic_testers():
"""For each comic scraper, create a test class."""
g = globals()
if "TESTALL" not in os.environ:
# Get limited number of scraper tests on Travis builds.
max_scrapers = 1
scraperclasses = islice(scraper.get_scraperclasses(), 0, max_scrapers)
else:
scraperclasses = scraper.get_scraperclasses()
for scraperclass in scraperclasses:
name = 'Test'+scraperclass.__name__
g[name] = make_comic_tester(name, scraperclass=scraperclass)
def get_test_scraperclasses():
"""Return scrapers that should be tested."""
if "TESTALL" in os.environ:
# test all comics (this will take some time)
scraperclasses = scraper.get_scraperclasses()
else:
# Get limited number of scraper tests on Travis builds to make
# it faster
testscrapernames = [
'AbstruseGoose', 'GoComics/CalvinandHobbes', 'xkcd'
]
scraperclasses = [
scraperclass for scraperclass in scraper.get_scraperclasses()
if scraperclass.getName() in testscrapernames
]
return scraperclasses
def main(args):
"""Get scraper descriptions from google results."""
if os.path.isfile(json_file):
result = load_result(json_file)
else:
result = {}
if args:
tofind = args[0]
else:
tofind = None
for scraperclass in sorted(get_scraperclasses(), key=classname):
key = classname(scraperclass)
if tofind and key != tofind:
continue
tofind = None
if '_' in key:
continue
print(key)
if scraperclass.description:
continue
if key in result:
continue
url = get_scraper_url(scraperclass)
print(url)
lang = scraperclass.lang
description = get_description(url, lang)
if description:
print(description)
# store result
module = scraperclass.__module__
result[key] = dict(description=description, module=module, url=url)
save_result(result, json_file)
else:
print("No description found")
return 0
def main(args):
"""Get scraper descriptions from google results."""
if os.path.isfile(json_file):
result = load_result(json_file)
else:
result = {}
if args:
tofind = args[0]
else:
tofind = None
for scraperclass in sorted(get_scraperclasses(), key=classname):
key = classname(scraperclass)
if tofind and key != tofind:
continue
tofind = None
if '_' in key:
continue
print(key)
if scraperclass.description:
continue
if key in result:
continue
url = get_scraper_url(scraperclass)
print(url)
lang = scraperclass.lang
description = get_description(url, lang)
if description:
print(description)
# store result
module = scraperclass.__module__
result[key] = dict(description=description, module=module, url=url)
save_result(result, json_file)
else:
print("No description found")
return 0
def get_used_languages():
lang = {}
for scraperclass in get_scraperclasses():
l = scraperclass.lang
if l not in lang:
lang[l] = scraperclass.language()
return lang
def get_used_languages():
lang = {}
for scraperclass in get_scraperclasses():
l = scraperclass.lang
if l not in lang:
lang[l] = scraperclass.language()
return lang
def has_gocomics_comic(name):
"""Test if comic name already exists."""
cname = "Gocomics/%s" % name
for scraperclass in get_scraperclasses():
lname = scraperclass.getName().lower()
if lname == cname.lower():
return True
return False
def generate_comic_testers():
"""For each comic scraper, create a test class."""
g = globals()
if "TESTALL" in os.environ:
# test all comics (this will take some time)
scraperclasses = scraper.get_scraperclasses()
else:
# Get limited number of scraper tests on Travis builds to make
# it faster
testscrapernames = ['GoComics/CalvinandHobbes']
scraperclasses = [
scraperclass for scraperclass in scraper.get_scraperclasses()
if scraperclass.getName() in testscrapernames
]
for scraperclass in scraperclasses:
name = 'Test' + scraperclass.__name__
g[name] = make_comic_tester(name, scraperclass=scraperclass)
def has_comic(name):
"""Check if comic name already exists."""
cname = name.lower()
for scraperclass in get_scraperclasses():
lname = scraperclass.getName().lower()
if lname == cname:
return True
return False
def has_gocomics_comic(name):
"""Test if comic name already exists."""
cname = "Gocomics/%s" % name
for scraperclass in get_scraperclasses():
lname = scraperclass.getName().lower()
if lname == cname.lower():
return True
return False
def has_comic(name):
"""Check if comic name already exists."""
names = [("Creators/%s" % name).lower(), ("GoComics/%s" % name).lower()]
for scraperclass in get_scraperclasses():
lname = scraperclass.getName().lower()
if lname in names:
return True
return False
def test_names(self):
for scraperclass in scraper.get_scraperclasses():
name = scraperclass.getName()
assert name.count('/') <= 1
if '/' in name:
comicname = name.split('/')[1]
else:
comicname = name
assert util.asciify(comicname) == comicname
def test_names(self):
for scraperclass in scraper.get_scraperclasses():
name = scraperclass.getName()
self.assertTrue(name.count('/') <= 1, name)
if '/' in name:
comicname = name.split('/')[1]
else:
comicname = name
self.assertEqual(util.asciify(comicname), comicname)
def test_names(self):
for scraperclass in scraper.get_scraperclasses():
name = scraperclass.getName()
self.assertTrue(name.count('/') <= 1, name)
if '/' in name:
comicname = name.split('/')[1]
else:
comicname = name
self.assertEqual(util.asciify(comicname), comicname)
def test_names(self):
for scraperclass in scraper.get_scraperclasses():
name = scraperclass.getName()
self.assertTrue(name.count('/') <= 1, name)
if '/' in name:
comicname = name.split('/')[1]
else:
comicname = name
self.assertEqual(util.asciify(comicname), comicname)
self.assertTrue(isinstance(scraperclass.description, text_type), "Invalid description in %s: %r" % (scraperclass, scraperclass.description))
def has_comic(name):
"""Check if comic name already exists."""
names = [
("Creators/%s" % name).lower(),
("GoComics/%s" % name).lower(),
]
for scraperclass in get_scraperclasses():
lname = scraperclass.getName().lower()
if lname in names:
return True
return False
def test_names(self):
for scraperclass in scraper.get_scraperclasses():
name = scraperclass.getName()
self.assertTrue(name.count('/') <= 1, name)
if '/' in name:
comicname = name.split('/')[1]
else:
comicname = name
self.assertEqual(util.asciify(comicname), comicname)
self.assertTrue(
isinstance(scraperclass.description,
text_type), "Invalid description in %s: %r" %
(scraperclass, scraperclass.description))
def get_test_scraperclasses():
"""Return scrapers that should be tested."""
if "TESTALL" in os.environ:
# test all comics (this will take some time)
scraperclasses = scraper.get_scraperclasses()
else:
if 'TESTCOMICS' in os.environ:
scraper_pattern = re.compile(os.environ['TESTCOMICS'])
else:
# Get limited number of scraper tests on Travis builds to make it
# faster
testscrapernames = [
'AbstruseGoose',
'GoComics/CalvinandHobbes',
'xkcd'
]
scraper_pattern = re.compile('|'.join(testscrapernames))
scraperclasses = [
scraperclass for scraperclass in scraper.get_scraperclasses()
if scraper_pattern.match(scraperclass.getName())
]
return scraperclasses
def has_comic(name):
"""Check if comic name already exists."""
names = [
("Creators/%s" % name).lower(),
("DrunkDuck/%s" % name).lower(),
("GoComics/%s" % name).lower(),
("KeenSpot/%s" % name).lower(),
("SmackJeeves/%s" % name).lower(),
("Arcamax/%s" % name).lower(),
]
for scraperclass in get_scraperclasses():
lname = scraperclass.getName().lower()
if lname in names:
return True
return False
def has_comic(name):
"""Check if comic name already exists."""
names = [
("Creators/%s" % name).lower(),
("DrunkDuck/%s" % name).lower(),
("GoComics/%s" % name).lower(),
("KeenSpot/%s" % name).lower(),
("ComicGenesis/%s" % name).lower(),
("SmackJeeves/%s" % name).lower(),
]
for scraperclass in get_scraperclasses():
lname = scraperclass.getName().lower()
if lname in names or lname == name.lower():
return True
return False
def test_get_scraperclasses(self):
for scraperclass in scraper.get_scraperclasses():
scraperclass()
scraperclass(indexes=["bla"])
def test_get_scraperclasses(self):
for scraperclass in scraper.get_scraperclasses():
scraperobj = scraperclass()
scraperobj = scraperclass(indexes=["bla"])
self.assertTrue(scraperobj.url,
"missing url in %s" % scraperobj.getName())
def has_description(classname):
"""Check if scraper with given classname already has a description."""
for scraperclass in get_scraperclasses():
if scraperclass.__name__ == classname:
return scraperclass.description
raise ValueError("Unknown scraper class %s" % classname)
def test_get_scraperclasses(self):
for scraperclass in scraper.get_scraperclasses():
scraperobj = scraperclass()
scraperobj = scraperclass(indexes=["bla"])
self.assertTrue(scraperobj.url,
"missing url in %s" % scraperobj.getName())
def has_description(classname):
"""Check if scraper with given classname already has a description."""
for scraperclass in get_scraperclasses():
if scraperclass.__name__ == classname:
return scraperclass.description
raise ValueError("Unknown scraper class %s" % classname)
def test_get_scraperclasses(self):
for scraperclass in scraper.get_scraperclasses():
scraperclass()
scraperclass(indexes=["bla"])
