Exemple #1
0
def project_create():
    project_name = request.form['project_name']
    project = Project()
    project.project_name = project_name
    db.session.add(project)
    db.session.commit()
    return redirect("/project/%s/spider/deploy" % project.id, code=302)
Exemple #2
0
 def post(self):
     project_name = request.form['project_name']
     project = Project()
     project.project_name = project_name
     db.session.add(project)
     db.session.commit()
     return project.to_dict()
Exemple #3
0
 def get_project_list(self):
     data = request("get", self._scrapyd_url() + "/listprojects.json", return_type="json")
     result = []
     if data:
         for project_name in data['projects']:
             project = Project()
             project.project_name = project_name
             result.append(project)
     return result
Exemple #4
0
 def post(self):
     project_name = request.form['project_name']
     project = Project()
     project.project_name = project_name
     try:
         db.session.add(project)
         db.session.commit()
     except:
         db.session.rollback()
         raise
     return project.to_dict()
 def get_project_list(self):
     """
     功能: 获取所有的爬虫工程列表
     :return: 返回工程对象列表
     """
     data = self.scrapyd_api.list_projects()  # 获取scrapyd上的所有工程列表
     result = []
     if data:
         for project_name in data:
             project = Project()  # 实例化工程对象
             project.project_name = project_name
             result.append(project)
     return result
Exemple #6
0
def add_project():
    """
    功能: 创建工程, 上传工程的egg文件, 将工程部署到scrapyd服务器上
    :param: project_name: 工程名称
    :param: project_alias: 工程备注或中文名称
    :param: for_project: 引用工程
    :param: developers: 爬虫项目的开发者
    :param: applicant: 爬虫申请人
    :param: egg_file: 待上传的爬虫项目的egg文件
    :return: 返回数据格式: json, 部署成功,返回success, 否则返回error
    """
    status = 'error'
    project = Project()
    is_msd = request.form.get('is_msd')  # 工程名
    project.is_msd = is_msd
    project.project_name = request.form.get('project_name')  # 工程名
    project.project_alias = request.form.get('project_alias')  # 工程备注
    project.for_project = request.form.get('for_project', None)  # 引用工程
    project.developers = request.form.get('developers', None)  # 开发者
    project.applicant = request.form.get('applicant', None)  # 申请人
    project.date_created = datetime.datetime.now().strptime(
        datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
        '%Y-%m-%d %H:%M:%S')
    project.date_modified = datetime.datetime.now().strptime(
        datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
        '%Y-%m-%d %H:%M:%S')

    if not is_msd:
        return json.dumps({"code": 500, "status": '项目类型不能为空, 项目类型为分布式or单机'})
    # 判断是否有值输入
    if request.form.get('project_name') == '' or request.form.get(
            'project_alias') == '':
        return json.dumps({"code": 200, "status": 'no input!'})

    # 判断工程名是否存在
    existed_project = project.query.filter_by(
        project_name=request.form['project_name']).first()
    # 工程存在则不能保存信息以及部署, 不存在则正常部署
    if existed_project:
        return json.dumps({"code": 200, "status": 'existed'})

    egg_path_dict = {}  # 用于存放egg文件保存路径

    if is_msd == '0':  # 如果为单机爬虫
        egg = request.files['egg']
        if egg:
            filename = secure_filename(egg.filename)  # 获取egg文件名
            dst_egg = os.path.join(tempfile.gettempdir(), filename)  # 拼接文件路径
            egg.save(dst_egg)  # 保存egg文件
            egg_path_dict['egg'] = dst_egg  # 将项目文件路径保存到egg路径字典中
        else:  # 如果有一个没有上传文件
            return json.dumps({"code": 500, "status": ' egg file are must'})
    else:
        # 获取上传文件
        master_egg = request.files['master_egg']
        slave_egg = request.files['slave_egg']
        # 判断表单是否传入文件
        if master_egg and slave_egg:
            master_filename = secure_filename(
                master_egg.filename)  # 获取master文件名
            slave_filename = secure_filename(slave_egg.filename)  # 获取slave文件名
            dst_master_egg = os.path.join(tempfile.gettempdir(),
                                          master_filename)  # 拼接文件路径
            dst_slave_egg = os.path.join(tempfile.gettempdir(),
                                         slave_filename)  # 拼接文件路径
            slave_egg.save(dst_slave_egg)  # 保存slave文件
            master_egg.save(dst_master_egg)  # 保存master文件
            egg_path_dict[
                'master'] = dst_master_egg  # 将master项目文件路径保存到egg路径字典中
            egg_path_dict['slave'] = dst_slave_egg  # 将slave项目文件路径保存到egg路径字典中
        else:  # 如果有一个没有上传文件
            return json.dumps({
                "code": 500,
                "status": 'master and slave egg are must'
            })

    if agent.deploy(project, egg_path_dict, is_msd):
        status = 'success'
        # 部署成功后才将数据保存至数据库
        db.session.add(project)
        db.session.commit()
        return json.dumps({"code": 200, "status": status})
    else:
        return json.dumps({"code": 500, "status": "error", "msg": "部署错误"})