def load_stories(self): if self.stories is None: print('Loading story urls for', self.shelf) s = [] for page in range(self.pages): print('Loading page', page, 'out of', self.pages, 'for', self.shelf) soup = self.first_page if page == 0 else bs4.BeautifulSoup(get_page(self.shelf, page + 1, LIST_VIEW), 'lxml') bold_tags = soup(class_="search_results_count")[0]('b') from_ = int(bold_tags[0].string) to = int(bold_tags[1].string) # there are 1-60 stories on the first page which means 60, but 60-1=59 so we add one count = (to - from_) + 1 story_list = soup(class_="story-list")[0]('li') for story in story_list: s.append(story(class_="right")[0].h2.a['href']) self.stories = tuple(s) print(number_objects(len(self.stories), 'url(|s)'), 'loaded for', self.shelf) return self.stories
def main(method='', proxy=None, bookshelves=[], username=None, password=None): util.output.open() try: bookshelves = bookshelves or get_user_shelves(username, password) method = method or input('Choose a analyzer ' +\ str(ALL).replace('[', '(').replace(']', ')') +\ ':') # setup login lenbook = len(bookshelves) print('Connected to FimFiction, analyzing ' + number_objects(lenbook, 'bookshel(f|ves)') + '.') shelves = [] for shelf in bookshelves: obj = shelfmanager.Shelf(int(shelf), username, password) shelves.append(obj) # call the appropriate method globals()[method](shelves) input('Press enter to exit') except SystemExit: pass except KeyboardInterrupt: pass except BaseException as e: # scope things def do_fail(): debug = False try: fail('Error: ' + str(e).encode('ascii', errors='replace').decode('ascii')) except SystemExit: pass except AssertionError: debug = True return debug reraise = do_fail() if reraise: raise finally: util.output.close()