def download_story(chapters): """Takes a list of chapters, returns a list of strings with chapter text. chapters is a list of tuples (title, url), where url points to a chapter post directly. """ cthread, rlist = [], [] lerr = "" for i in chapters: while True: try: if get_postnum(i[1]) is None: n = 0 else: n = [get_postnum(j['post_url']) for j in cthread].index(get_postnum(i[1])) rlist.append((i[0], cthread[n]['text'])) except (ValueError, IndexError): if lerr == i[1]: raise print("Getting thread for URL {}".format(i[1])) lerr = i[1] g = forum_archive.make_getter(i[1]) pn = g.get_url_page(i[1]) cthread = g.get_thread(pn) continue break return rlist
def main(): ap = argparse.ArgumentParser(description="Forum-based story downloader/compiler") g = ap.add_mutually_exclusive_group() g.add_argument("-u", "--update", help="Update an existing story", action="store_true", default=False) ap.add_argument("-a", "--author", help="Override author name", default=None) ap.add_argument("-t", "--thread", action="store_true", help="Download archive thread", default=False) ap.add_argument("-c", "--credential", help="Log in with credentials", default=None) ap.add_argument("url", help="Post URL to contents page") g.add_argument("title", help="Story title in file", default=None, nargs='?') args = ap.parse_args() if args.update and args.title: print("Error: may not provide title when updating", file=sys.stderr) sys.exit(1) if not args.update and not args.title: print("Error: must provide title", file=sys.stderr) sys.exit(1) if args.update: args.title, args.url, cli = read_file(args.url) if args.credential: c = args.credential.split(':', 1) c = {'username': c[0], 'password': c[1]} else: c = {} g = forum_archive.make_getter(args.url, c) if args.thread: fp = g.get_thread() stext = [("Chapter {}".format(i[0]+1), i[1]['text']) for i in enumerate(fp)] l = [] author = fp[0]['poster_name'] else: fp = g.get_thread(g.get_url_page(args.url)) cl = [i for i in fp if i['post_url'] == args.url][0] author = cl['poster_name'] l = list(make_listing(cl['text'], args.url)) ede = os.environ.get('EDITOR', 'vim') helpstr = """Above the marker is the table of contents from the original file; below is that derived from the source. Edit the former as desired, then quit. Everything below the marker will be ignored. """ ifstr = to_string(l) if not args.update else cli + '-' * 20 + "\n" + helpstr + to_string(l) with tempfile.NamedTemporaryFile() as tf: tf.write(ifstr.encode()) tf.flush() subprocess.call(ede.split() + [tf.name]) tf.seek(0) ofstr = tf.read().decode() if args.update: ofstr = ofstr.split('-'*20)[0] l = to_chapters(ofstr) if not l: return stext = download_story(l) if args.author: author = args.author fn = make_filename(args.title) + '.html' with open(fn, 'w') as of: compile_story((args.title, author, args.url), stext, l, of)
def setup(bot): global login_creds, pastebin_api_key, pastebin_user_key c1 = {'username': bot.config.qqbot.username, 'password': bot.config.qqbot.password } a = forum_archive.make_getter('https://forum.questionablequesting.com/threads/rules.1/', cred=c1) login_creds = a.cred pastebin_api_key = bot.config.qqbot.pastebin_api_key pastebin_user_key = bot.config.qqbot.pastebin_user_key
def setup(bot): global login_creds, pastebin_api_key, pastebin_user_key c1 = { 'username': bot.config.qqbot.username, 'password': bot.config.qqbot.password } a = forum_archive.make_getter( 'https://forum.questionablequesting.com/threads/rules.1/', cred=c1) login_creds = a.cred pastebin_api_key = bot.config.qqbot.pastebin_api_key pastebin_user_key = bot.config.qqbot.pastebin_user_key
def get_posts(url, surl=None): o = plink_re.match(url) page_start = int(o.group('pnum') or '1') id_start = o.group('pid') or '' if surl: o = plink_re.match(surl) page_end = int(o.group('pnum') or '1') id_end = o.group('pid') or '' else: page_end = id_end = None a = forum_archive.make_getter(url, cred=login_creds) result = a.get_thread((page_start, page_end)) fi = [id_start in i['post_url'] for i in result].index(True) try: li = [id_end in i['post_url'] for i in result].index(True) except (ValueError, TypeError): li = None result = result[fi:li] return result
def main(): ap = argparse.ArgumentParser( description="Forum-based story downloader/compiler") g = ap.add_mutually_exclusive_group() g.add_argument("-u", "--update", help="Update an existing story", action="store_true", default=False) ap.add_argument("-a", "--author", help="Override author name", default=None) ap.add_argument("-t", "--thread", action="store_true", help="Download archive thread", default=False) ap.add_argument("-c", "--credential", help="Log in with credentials", default=None) ap.add_argument("url", help="Post URL to contents page") g.add_argument("title", help="Story title in file", default=None, nargs='?') args = ap.parse_args() if args.update and args.title: print("Error: may not provide title when updating", file=sys.stderr) sys.exit(1) if not args.update and not args.title: print("Error: must provide title", file=sys.stderr) sys.exit(1) if args.update: args.title, args.url, cli = read_file(args.url) if args.credential: c = args.credential.split(':', 1) c = {'username': c[0], 'password': c[1]} else: c = {} g = forum_archive.make_getter(args.url, c) if args.thread: fp = g.get_thread() stext = [("Chapter {}".format(i[0] + 1), i[1]['text']) for i in enumerate(fp)] l = [] author = fp[0]['poster_name'] else: fp = g.get_thread(g.get_url_page(args.url)) cl = [i for i in fp if i['post_url'] == args.url][0] author = cl['poster_name'] l = list(make_listing(cl['text'], args.url)) ede = os.environ.get('EDITOR', 'vim') helpstr = """Above the marker is the table of contents from the original file; below is that derived from the source. Edit the former as desired, then quit. Everything below the marker will be ignored. """ ifstr = to_string( l ) if not args.update else cli + '-' * 20 + "\n" + helpstr + to_string( l) with tempfile.NamedTemporaryFile() as tf: tf.write(ifstr.encode()) tf.flush() subprocess.call(ede.split() + [tf.name]) tf.seek(0) ofstr = tf.read().decode() if args.update: ofstr = ofstr.split('-' * 20)[0] l = to_chapters(ofstr) if not l: return stext = download_story(l) if args.author: author = args.author fn = make_filename(args.title) + '.html' with open(fn, 'w') as of: compile_story((args.title, author, args.url), stext, l, of)