Example #1
0
    def _get_links_and_forms_and_store(self, url):
        if Page.query.filter(Page.url == url).first():
            return []

        r = requests.get(url, cookies=self.cookies)
        soup = BeautifulSoup(r.text, "html.parser")

        try:
            page = Page(self.website_id, url)
            page.save_to_db()
            self._get_forms(soup, page.id, url)
        except Exception as e:
            print("Row already exists", e)

        return self._get_links(url, soup)