Exemplos de SpiderInstance.update_spider_instances em Python

Exemplo n.º 1

def sync_spiders():
    '''
    sync spiders
    :return:
    '''
    for project in Project.query.all():
        spider_instance_list = agent.get_spider_list(project)
        SpiderInstance.update_spider_instances(spider_instance_list)

Exemplo n.º 2

Arquivo: common.py Projeto: kizzleip/SpiderKeeper

def sync_spiders():
    '''
    sync spiders
    :return:
    '''
    for project in Project.query.all():
        spider_instance_list = agent.get_spider_list(project)
        SpiderInstance.update_spider_instances(project.id, spider_instance_list)
    app.logger.debug('[sync_spiders]')

Exemplo n.º 3

Arquivo: common.py Projeto: rockrockyy/SpiderKeeper

def sync_spiders():
    '''
    sync spiders
    :return:
    '''
    for project in Project.query.all():
        spider_instance_list = agent.get_spider_list(project)
        SpiderInstance.update_spider_instances(project.id, spider_instance_list)
    app.logger.debug('[sync_spiders]')

Exemplo n.º 4

Arquivo: common.py Projeto: taozi926494/crawlplatformBE

def sync_spiders():
    """
    每隔10s同步scrapyd上的爬虫到数据库
    sync spiders
    :return:
    """
    # 遍历所有的工程
    for project in Project.query.all():
        # 通过工程名获取scrapyd上的爬虫列表
        spider_instance_list = agent.get_spider_list(project)
        SpiderInstance.update_spider_instances(project.id, spider_instance_list)
    app.logger.debug('[同步scrapyd上的蜘蛛到系统数据库]')

Exemplo n.º 5

Arquivo: common.py Projeto: yuangongping/crawlplatformBE

def sync_spiders():
    """
    每隔10s同步scrapyd上的爬虫到数据库
    sync spiders
    :return:
    """
    # 遍历所有的工程
    db.session.commit()
    allprojects = db.session.query(Project).all()
    for project in allprojects:
        # 通过工程名获取scrapyd上的爬虫列表
        spider_instance_list = agent.get_spider_list(project)
        SpiderInstance.update_spider_instances(project.id, spider_instance_list)
    app.logger.debug('[同步scrapyd上的蜘蛛到系统数据库]')

Exemplo n.º 6

Arquivo: controller.py Projeto: netconstructor/SpiderKeeper

def spider_deploy(project_id):
    project = Project.find_project_by_id(project_id)
    spider_instance_list = agent.get_spider_list(project)
    SpiderInstance.update_spider_instances(spider_instance_list)
    return render_template("spider_deploy.html")