def check_tag(self, tag=None, title=None, url=None, **kwargs):
if tag is None:
tag = self.make_tag(title=title, url=url, **kwargs)
if not self.ignore_tags:
if CrawlState.check(tag):
log.debug("Skipping %r in %r, tagged as done.", tag, self.source)
raise TagExists()
CrawlState.create(self.source, tag)
db.session.commit()
return tag
def check_tag(self, tag=None, title=None, url=None, **kwargs):
if tag is None:
tag = self.make_tag(title=title, url=url, **kwargs)
if not self.ignore_tags:
if CrawlState.check(tag):
log.debug("Skipping %r in %r, tagged as done.", tag,
self.source)
raise TagExists()
CrawlState.create(self.source, tag)
db.session.commit()
return tag
def flush(source):
""" Reset the crawler state for a given source specification. """
CrawlState.flush(source)
db.session.commit()
def flush(source):
""" Reset the crawler state for a given source specification. """
CrawlState.flush(source)
db.session.commit()
