def crawlers():
crawlers = Crawler.select()
for crawler in crawlers:
if crawler.is_runnable():
# delete crawler before crawling it again:
dq = CrawlerPage.delete().where(CrawlerPage.name == crawler.name)
deleted_count = dq.execute()
pages = CrawlerPage.crawl(crawler)
print("%s job: crawlers processed" % datetime.utcnow())
def crawlers_list():
crawlers = Crawler.select()
return render_template('crawlers_list.html',
current_user=current_user,
crawlers=crawlers)