def add_link(self, page, url, commit=True, status=200):
link_page, created = CrawlPage.get_or_create(self.session, url=url)
if created:
link_page.status = status
if link_page not in page.links:
page.links.append(link_page)
if commit:
self.session.commit()
return link_page
def add_link(self, page, url, commit=True, status=200):
link_page, created = CrawlPage.get_or_create(self.session, url=url)
if created:
link_page.status = status
if link_page not in page.links:
page.links.append(link_page)
if commit:
self.session.commit()
return link_page
def add_page(self, url, status=200, n_redirects=0, is_seed=False, commit=True):
page, created = CrawlPage.get_or_create(self.session, url=url)
if created:
page.is_seed = is_seed
page.status = status
page.n_redirects = n_redirects
if commit:
self.session.commit()
return page
def add_page(self,
url,
status=200,
n_redirects=0,
is_seed=False,
commit=True):
page, created = CrawlPage.get_or_create(self.session, url=url)
if created:
page.is_seed = is_seed
page.status = status
page.n_redirects = n_redirects
if commit:
self.session.commit()
return page
def pages(self):
return [page for page in CrawlPage.query(self.session).all()]
def pages(self):
return [page for page in CrawlPage.query(self.session).all()]
