def main(args):
"""Get scraper descriptions from google results."""
if os.path.isfile(json_file):
result = load_result(json_file)
else:
result = {}
if args:
tofind = args[0]
else:
tofind = None
for scraperclass in sorted(get_scraperclasses(), key=classname):
key = classname(scraperclass)
if tofind and key != tofind:
continue
tofind = None
if '_' in key:
continue
print(key)
if scraperclass.description:
continue
if key in result:
continue
url = get_scraper_url(scraperclass)
print(url)
lang = scraperclass.lang
description = get_description(url, lang)
if description:
print(description)
# store result
module = scraperclass.__module__
result[key] = dict(description=description, module=module, url=url)
save_result(result, json_file)
else:
print("No description found")
return 0
def main(args):
"""Get scraper descriptions from google results."""
if os.path.isfile(json_file):
result = load_result(json_file)
else:
result = {}
if args:
tofind = args[0]
else:
tofind = None
for scraperclass in sorted(get_scraperclasses(), key=classname):
key = classname(scraperclass)
if tofind and key != tofind:
continue
tofind = None
if '_' in key:
continue
print(key)
if scraperclass.description:
continue
if key in result:
continue
url = get_scraper_url(scraperclass)
print(url)
lang = scraperclass.lang
description = get_description(url, lang)
if description:
print(description)
# store result
module = scraperclass.__module__
result[key] = dict(description=description, module=module, url=url)
save_result(result, json_file)
else:
print("No description found")
return 0
def print_results(min_strips):
"""Print all comics that have at least the given number of minimum comic strips."""
for name, entry in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
path, num = entry
if num >= min_strips:
print("add(%r, %r)" % (str(truncate_name(name)), str(path)))
def print_results(args):
"""Print all comics that have at least the given number of minimum comic strips."""
min_comics, filename = args
with codecs.open(filename, 'a', 'utf-8') as fp:
for name, shortname in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
fp.write(u"add(%r, %r)\n" % (
str(truncate_name(name)), str(shortname))
)
def print_results(args):
"""Print all comics that have at least the given number of minimum comic strips."""
for name, shortname in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
if has_comic(name):
prefix = '#'
else:
prefix = ''
print("%sadd(%r, %r)" % (prefix, str(truncate_name(name)), str(shortname)))
def print_results(args):
"""Print all comics that have at least the given number of minimum comic strips."""
min_comics, filename = args
with codecs.open(filename, 'a', 'utf-8') as fp:
for name, shortname in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
fp.write(u"add(%r, %r)\n" % (
str(truncate_name(name)), str(shortname))
)
def print_results(args):
"""Print all comics that have at least the given number of minimum comic strips."""
for name, shortname in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
if has_comic(name):
prefix = '#'
else:
prefix = ''
print("%sadd(%r, %r)" %
(prefix, str(truncate_name(name)), str(shortname)))
def print_results(args):
"""Print comics."""
min_comics, filename = args
with codecs.open(filename, 'a', 'utf-8') as fp:
for name, url in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
if has_gocomics_comic(name):
prefix = u'# duplicate of gocomics '
else:
prefix = u''
fp.write(u"%sadd(%r, %r)\n" %
(prefix, str(truncate_name(name)), str(url)))
def print_results(args):
"""Print comics."""
min_comics, filename = args
with codecs.open(filename, 'a', 'utf-8') as fp:
for name, path in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
lang = 'Es' if name.lower().endswith('spanish') else ''
if has_gocomics_comic(name):
fp.write(u'# %s has a duplicate in gocomics\n' % truncate_name(name))
else:
fp.write(u"class %s(_Creators%s):\n path = %r\n\n" %
(truncate_name(name), lang, path))
def print_results(args):
"""Print all comics."""
for name, entry in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
url, desc = entry
if has_comic(name):
prefix = '#'
else:
prefix = ''
name = truncate_name(name).encode('utf-8')
url = url.encode('utf-8')
desc = desc.encode('utf-8')
print("%sadd(%r, %r, %r)" % (prefix, name, url, desc))
def print_results(args):
"""Print comics."""
min_comics, filename = args
with codecs.open(filename, 'a', 'utf-8') as fp:
for name, url in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
if has_gocomics_comic(name):
prefix = u'# duplicate of gocomics '
else:
prefix = u''
fp.write(u"%sadd(%r, %r)\n" % (
prefix, str(truncate_name(name)), str(url))
)
def print_results(args):
"""Print all comics."""
min_comics, filename = args
with codecs.open(filename, "a", "utf-8") as fp:
for name, entry in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
url = entry
if has_comic(name):
prefix = u"#"
else:
prefix = u""
name = truncate_name(name)
fp.write(u"%sadd(%r, %r)\n" % (prefix, str(name), str(url)))
def print_results(args):
"""Print all comics."""
for name, entry in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
url, desc = entry
if has_comic(name):
prefix = '#'
else:
prefix = ''
name = truncate_name(name).encode('utf-8')
url = url.encode('utf-8')
desc = desc.encode('utf-8')
print("%sadd(%r, %r, %r)" % (prefix, name, url, desc))
def print_results(args):
"""Print all comics."""
min_comics, filename = args
with codecs.open(filename, 'a', 'utf-8') as fp:
for name, entry in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
url = entry
if has_comic(name):
prefix = u'#'
else:
prefix = u''
name = truncate_name(name)
fp.write(u"%sadd(%r, %r)\n" % (prefix, str(name), str(url)))
def print_results(args):
"""Print all comics that have at least the given number of minimum comic strips."""
min_comics = int(args[0])
for name, entry in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
url, num = entry
if num < min_comics:
continue
url = url.replace("comicgen.com", "comicgenesis.com")
if has_comic(name):
prefix = '#'
else:
prefix = ''
print("%sadd(%r, %r)" % (prefix, str(truncate_name(name)), str(url)))
def print_results(args):
"""Print all comics that have at least the given number of minimum comic strips."""
min_comics = int(args[0])
for name, entry in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
url, num = entry
if num < min_comics:
continue
url = url.replace("comicgen.com", "comicgenesis.com")
if has_comic(name):
prefix = '#'
else:
prefix = ''
print("%sadd(%r, %r)" % (prefix, str(truncate_name(name)), str(url)))
def print_results(args):
"""Print all comics that have at least the given number of minimum comic strips."""
min_comics = int(args[0])
for name, entry in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
url, num, desc, adult, bounce = entry
if num < min_comics:
continue
if has_comic(name):
prefix = '#'
else:
prefix = ''
print(
"%sadd(%r, %r, %r, %s, %s)" %
(prefix, str(truncate_name(name)), str(url), desc, adult, bounce))
def print_results(args):
"""Print all comics that have at least the given number of minimum comic strips."""
min_comics = int(args[0])
for name, entry in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
url, desc, num, genre, active = entry
if num < min_comics:
continue
if has_comic(name):
prefix = '#'
else:
prefix = ''
print("%sadd(%r, %r, %r)" % (
prefix, str(truncate_name(name)), str(url), desc
))
def print_results(args):
"""Print all comics that have at least the given number of minimum comic strips."""
min_comics, filename = args
min_comics = int(min_comics)
with codecs.open(filename, "a", "utf-8") as fp:
for name, entry in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
url, num = entry
if num < min_comics:
continue
url = url.replace("comicgen.com", "comicgenesis.com")
if has_comic(name):
prefix = u"#"
else:
prefix = u""
fp.write(u"%sadd(%r, %r)\n" % (prefix, str(truncate_name(name)), str(url)))
def print_results(args):
"""Print all comics that have at least the given number of minimum comic strips."""
min_comics, filename = args
min_comics = int(min_comics)
with codecs.open(filename, 'a', 'utf-8') as fp:
for name, entry in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
url, desc, num, genre, active = entry
if num < min_comics:
continue
if has_comic(name):
prefix = u'#'
else:
prefix = u''
fp.write(u"%sadd(%r, %r, %r)\n" %
(prefix, str(truncate_name(name)), str(url), desc))
def print_results(args):
"""Print all comics that have at least the given number of minimum comic strips."""
min_comics, filename = args
min_comics = int(min_comics)
with codecs.open(filename, 'a', 'utf-8') as fp:
for name, entry in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
url, num, adult, bounce = entry
if num < min_comics:
continue
if has_comic(name):
prefix = u'#'
else:
prefix = u''
fp.write(u"%sadd(%r, %r, %s, %s)\n" % (
prefix, str(truncate_name(name)), str(url), adult, bounce
))
def print_results(args):
"""Print all comics that have at least the given number of minimum
comic strips."""
min_comics, filename = args
min_comics = int(min_comics)
with codecs.open(filename, 'a', 'utf-8') as fp:
for name, entry in sorted(load_result(json_file).items()):
url, num, active, lang = entry
if name in exclude_comics:
fp.write(u"# %s is excluded\n" % name)
continue
if num < min_comics:
continue
dup = find_dups(name)
if dup is not None:
fp.write(u"# %s has a duplicate in %s\n" % (name, dup))
else:
fp.write(u"class CF%s(_ComicFury):\n url = %r\n\n\n" % (
truncate_name(name), str(url)))
def main(args):
"""Get scraper descriptions from google results."""
if os.path.isfile(json_file):
result = load_result(json_file)
else:
result = {}
for classname, info in sorted(result.items()):
if has_description(classname) or '_' in classname:
continue
if info.get('answer') == 'no':
continue
if not answer(classname, info):
info['answer'] = 'no'
save_result(result, json_file)
continue
filename = info['module'].replace('.', os.sep) + ".py"
encoding = get_encoding(filename)
with codecs.open(filename, 'r', encoding) as f:
with codecs.open(filename + "_", 'w', encoding) as out:
write_description(f, out, classname, info)
os.rename(filename + "_", filename)
return 0
def main(args):
"""Get scraper descriptions from google results."""
if os.path.isfile(json_file):
result = load_result(json_file)
else:
result = {}
for classname, info in sorted(result.items()):
if has_description(classname) or '_' in classname:
continue
if info.get('answer') == 'no':
continue
if not answer(classname, info):
info['answer'] = 'no'
save_result(result, json_file)
continue
filename = info['module'].replace('.', os.sep) + ".py"
encoding = get_encoding(filename)
with codecs.open(filename, 'r', encoding) as f:
with codecs.open(filename+"_", 'w', encoding) as out:
write_description(f, out, classname, info)
os.rename(filename+"_", filename)
return 0
def print_results(args):
"""Print comics."""
for name, url in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
print("add(%r, %r)" % (str(truncate_name(name)), str(url)))
def print_results(args):
"""Print comics."""
for name, url in sorted(load_result(json_file).items()):
if name in exclude_comics:
continue
print("add(%r, %r)" % (str(truncate_name(name)), str(url)))