def sync_spiders(): ''' sync spiders :return: ''' for project in Project.query.all(): spider_instance_list = agent.get_spider_list(project) SpiderInstance.update_spider_instances(spider_instance_list)
def sync_spiders(): ''' sync spiders :return: ''' for project in Project.query.all(): spider_instance_list = agent.get_spider_list(project) SpiderInstance.update_spider_instances(project.id, spider_instance_list) app.logger.debug('[sync_spiders]')
def sync_spiders(): ''' sync spiders :return: ''' for project in Project.query.all(): spider_instance_list = agent.get_spider_list(project) SpiderInstance.update_spider_instances(project.id, spider_instance_list) app.logger.debug('[sync_spiders]')
def sync_spiders(): """ 每隔10s同步scrapyd上的爬虫到数据库 sync spiders :return: """ # 遍历所有的工程 for project in Project.query.all(): # 通过工程名获取scrapyd上的爬虫列表 spider_instance_list = agent.get_spider_list(project) SpiderInstance.update_spider_instances(project.id, spider_instance_list) app.logger.debug('[同步scrapyd上的蜘蛛到系统数据库]')
def sync_spiders(): """ 每隔10s同步scrapyd上的爬虫到数据库 sync spiders :return: """ # 遍历所有的工程 db.session.commit() allprojects = db.session.query(Project).all() for project in allprojects: # 通过工程名获取scrapyd上的爬虫列表 spider_instance_list = agent.get_spider_list(project) SpiderInstance.update_spider_instances(project.id, spider_instance_list) app.logger.debug('[同步scrapyd上的蜘蛛到系统数据库]')
def get(self, project_id): project = Project.find_project_by_id(project_id) return agent.get_spider_list(project) if project else []
def spider_deploy(project_id): project = Project.find_project_by_id(project_id) spider_instance_list = agent.get_spider_list(project) SpiderInstance.update_spider_instances(spider_instance_list) return render_template("spider_deploy.html")
def spider_dashboard(project_id): project = Project.find_project_by_id(project_id) spider_instance_list = agent.get_spider_list(project) if project else [] return render_template("spider_dashboard.html", spider_instance_list=spider_instance_list)