def add_egg(self, crawler, version, egg_handle):
with session.begin():
crawler = session.query(Crawler).filter_by(id=crawler).first()
crawler.egg = egg_handle.read()
crawler.version = version
session.add(crawler)
def schedule_crawl(self, crawler, spider, schedule):
with session.begin():
crawler = session.query(Crawler).filter_by(id=crawler).first()
spider = session.query(Spider).filter_by(crawler_id=crawler.id, name=spider).first()
crawl_schedule = CrawlSchedule(spider_id=spider.id, crontab=schedule)
session.add(crawl_schedule)
def create_crawler(self, name):
if self.session.query(Crawler).filter_by(id=name).first() is not None:
raise Exception('Crawler already exists')
crawler = Crawler(id=name)
with session.begin():
session.add(crawler)
return crawler
def get_crawler(self, crawler_id):
with session.begin():
return session.query(Crawler).filter_by(id=crawler_id).first()