def download_story(chapters):
"""Takes a list of chapters, returns a list of strings with chapter text.
chapters is a list of tuples (title, url), where url points to a chapter
post directly.
"""
cthread, rlist = [], []
lerr = ""
for i in chapters:
while True:
try:
if get_postnum(i[1]) is None:
n = 0
else:
n = [get_postnum(j['post_url'])
for j in cthread].index(get_postnum(i[1]))
rlist.append((i[0], cthread[n]['text']))
except (ValueError, IndexError):
if lerr == i[1]:
raise
print("Getting thread for URL {}".format(i[1]))
lerr = i[1]
g = forum_archive.make_getter(i[1])
pn = g.get_url_page(i[1])
cthread = g.get_thread(pn)
continue
break
return rlist
def download_story(chapters):
"""Takes a list of chapters, returns a list of strings with chapter text.
chapters is a list of tuples (title, url), where url points to a chapter
post directly.
"""
cthread, rlist = [], []
lerr = ""
for i in chapters:
while True:
try:
if get_postnum(i[1]) is None:
n = 0
else:
n = [get_postnum(j['post_url']) for j in cthread].index(get_postnum(i[1]))
rlist.append((i[0], cthread[n]['text']))
except (ValueError, IndexError):
if lerr == i[1]:
raise
print("Getting thread for URL {}".format(i[1]))
lerr = i[1]
g = forum_archive.make_getter(i[1])
pn = g.get_url_page(i[1])
cthread = g.get_thread(pn)
continue
break
return rlist
def main():
ap = argparse.ArgumentParser(description="Forum-based story downloader/compiler")
g = ap.add_mutually_exclusive_group()
g.add_argument("-u", "--update", help="Update an existing story", action="store_true", default=False)
ap.add_argument("-a", "--author", help="Override author name", default=None)
ap.add_argument("-t", "--thread", action="store_true", help="Download archive thread", default=False)
ap.add_argument("-c", "--credential", help="Log in with credentials", default=None)
ap.add_argument("url", help="Post URL to contents page")
g.add_argument("title", help="Story title in file", default=None, nargs='?')
args = ap.parse_args()
if args.update and args.title:
print("Error: may not provide title when updating", file=sys.stderr)
sys.exit(1)
if not args.update and not args.title:
print("Error: must provide title", file=sys.stderr)
sys.exit(1)
if args.update:
args.title, args.url, cli = read_file(args.url)
if args.credential:
c = args.credential.split(':', 1)
c = {'username': c[0], 'password': c[1]}
else:
c = {}
g = forum_archive.make_getter(args.url, c)
if args.thread:
fp = g.get_thread()
stext = [("Chapter {}".format(i[0]+1), i[1]['text']) for i in enumerate(fp)]
l = []
author = fp[0]['poster_name']
else:
fp = g.get_thread(g.get_url_page(args.url))
cl = [i for i in fp if i['post_url'] == args.url][0]
author = cl['poster_name']
l = list(make_listing(cl['text'], args.url))
ede = os.environ.get('EDITOR', 'vim')
helpstr = """Above the marker is the table of contents from the original file; below is
that derived from the source. Edit the former as desired, then quit. Everything
below the marker will be ignored.
"""
ifstr = to_string(l) if not args.update else cli + '-' * 20 + "\n" + helpstr + to_string(l)
with tempfile.NamedTemporaryFile() as tf:
tf.write(ifstr.encode())
tf.flush()
subprocess.call(ede.split() + [tf.name])
tf.seek(0)
ofstr = tf.read().decode()
if args.update:
ofstr = ofstr.split('-'*20)[0]
l = to_chapters(ofstr)
if not l:
return
stext = download_story(l)
if args.author:
author = args.author
fn = make_filename(args.title) + '.html'
with open(fn, 'w') as of:
compile_story((args.title, author, args.url), stext, l, of)
def setup(bot):
global login_creds, pastebin_api_key, pastebin_user_key
c1 = {'username': bot.config.qqbot.username,
'password': bot.config.qqbot.password }
a = forum_archive.make_getter('https://forum.questionablequesting.com/threads/rules.1/', cred=c1)
login_creds = a.cred
pastebin_api_key = bot.config.qqbot.pastebin_api_key
pastebin_user_key = bot.config.qqbot.pastebin_user_key
def setup(bot):
global login_creds, pastebin_api_key, pastebin_user_key
c1 = {
'username': bot.config.qqbot.username,
'password': bot.config.qqbot.password
}
a = forum_archive.make_getter(
'https://forum.questionablequesting.com/threads/rules.1/', cred=c1)
login_creds = a.cred
pastebin_api_key = bot.config.qqbot.pastebin_api_key
pastebin_user_key = bot.config.qqbot.pastebin_user_key
def get_posts(url, surl=None):
o = plink_re.match(url)
page_start = int(o.group('pnum') or '1')
id_start = o.group('pid') or ''
if surl:
o = plink_re.match(surl)
page_end = int(o.group('pnum') or '1')
id_end = o.group('pid') or ''
else:
page_end = id_end = None
a = forum_archive.make_getter(url, cred=login_creds)
result = a.get_thread((page_start, page_end))
fi = [id_start in i['post_url'] for i in result].index(True)
try:
li = [id_end in i['post_url'] for i in result].index(True)
except (ValueError, TypeError):
li = None
result = result[fi:li]
return result
def get_posts(url, surl=None):
o = plink_re.match(url)
page_start = int(o.group('pnum') or '1')
id_start = o.group('pid') or ''
if surl:
o = plink_re.match(surl)
page_end = int(o.group('pnum') or '1')
id_end = o.group('pid') or ''
else:
page_end = id_end = None
a = forum_archive.make_getter(url, cred=login_creds)
result = a.get_thread((page_start, page_end))
fi = [id_start in i['post_url'] for i in result].index(True)
try:
li = [id_end in i['post_url'] for i in result].index(True)
except (ValueError, TypeError):
li = None
result = result[fi:li]
return result
def main():
ap = argparse.ArgumentParser(
description="Forum-based story downloader/compiler")
g = ap.add_mutually_exclusive_group()
g.add_argument("-u",
"--update",
help="Update an existing story",
action="store_true",
default=False)
ap.add_argument("-a",
"--author",
help="Override author name",
default=None)
ap.add_argument("-t",
"--thread",
action="store_true",
help="Download archive thread",
default=False)
ap.add_argument("-c",
"--credential",
help="Log in with credentials",
default=None)
ap.add_argument("url", help="Post URL to contents page")
g.add_argument("title",
help="Story title in file",
default=None,
nargs='?')
args = ap.parse_args()
if args.update and args.title:
print("Error: may not provide title when updating", file=sys.stderr)
sys.exit(1)
if not args.update and not args.title:
print("Error: must provide title", file=sys.stderr)
sys.exit(1)
if args.update:
args.title, args.url, cli = read_file(args.url)
if args.credential:
c = args.credential.split(':', 1)
c = {'username': c[0], 'password': c[1]}
else:
c = {}
g = forum_archive.make_getter(args.url, c)
if args.thread:
fp = g.get_thread()
stext = [("Chapter {}".format(i[0] + 1), i[1]['text'])
for i in enumerate(fp)]
l = []
author = fp[0]['poster_name']
else:
fp = g.get_thread(g.get_url_page(args.url))
cl = [i for i in fp if i['post_url'] == args.url][0]
author = cl['poster_name']
l = list(make_listing(cl['text'], args.url))
ede = os.environ.get('EDITOR', 'vim')
helpstr = """Above the marker is the table of contents from the original file; below is
that derived from the source. Edit the former as desired, then quit. Everything
below the marker will be ignored.
"""
ifstr = to_string(
l
) if not args.update else cli + '-' * 20 + "\n" + helpstr + to_string(
l)
with tempfile.NamedTemporaryFile() as tf:
tf.write(ifstr.encode())
tf.flush()
subprocess.call(ede.split() + [tf.name])
tf.seek(0)
ofstr = tf.read().decode()
if args.update:
ofstr = ofstr.split('-' * 20)[0]
l = to_chapters(ofstr)
if not l:
return
stext = download_story(l)
if args.author:
author = args.author
fn = make_filename(args.title) + '.html'
with open(fn, 'w') as of:
compile_story((args.title, author, args.url), stext, l, of)
