def display_link_prompt(args, urls, titles): """Print URLs and their descriptions alongside a prompt. Keyword arguments: args -- program arguments (dict) urls -- search URLs found (list) titles -- descriptions of search URLs found (list) """ while 1: print('\n{0}'.format(BORDER)) for i in range(len(urls)): link = HTMLParser().unescape(titles[i]) print('{0}. {1}'.format(i+1, link.encode('utf-8') if PY2 else link)) print(BORDER) # Handle link prompt input try: link_input = [inp.strip() for inp in input(': ').split()] if not link_input: continue utils.check_input(link_input) # Check input in case of quit print('\n') exec_prompt_cmd(args, urls, link_input[0], link_input[1:]) except (KeyboardInterrupt, EOFError, ValueError, IndexError): return False
def save(self, page, crawl=False): """Requests and saves a remote page to a local file """ print('Saving '+page) html = self.fetch(page, crawl) content = html.replace('</p>', '\n') content = re.sub(r'<.*?>', ' ', content) content = HTMLParser().unescape(content) content = content.encode('utf8') if self.out_file: with open(self.out_file, 'a') as handle: handle.write(content+'\n') elif self.out_dir: page_key = self.page_key(page) try: os.makedirs(os.path.join(self.out_dir, page_key[0])) except OSError: pass filename = os.path.join(self.out_dir, page_key[0], (page_key[1]+'?'+page_key[2]).replace('/', '_')) with open(filename, 'w') as handle: handle.write(content)