def gather_links(page_url): try: finder = LinkFinder(Spider.base_url, page_url) finder.getAllExternalLinks(page_url) except: print('Error : can not crawl page') return set() return finder.page_internalLink()