def spider_egg_upload(project_id): project = Project.find_project_by_id(project_id) if 'file' not in request.files: flash('No file part') return redirect(request.referrer) file = request.files['file'] # if user does not select file, browser also # submit a empty part without filename if file.filename == '': flash('No selected file') return redirect(request.referrer) if file: filename = secure_filename(file.filename) dst = os.path.join(tempfile.gettempdir(), filename) file.save(dst) agent.deploy(project, dst) flash('deploy success!') return redirect(request.referrer)
def add_project(): """ 功能: 创建工程, 上传工程的egg文件, 将工程部署到scrapyd服务器上 :param: project_name: 工程名称 :param: project_alias: 工程备注或中文名称 :param: for_project: 引用工程 :param: developers: 爬虫项目的开发者 :param: applicant: 爬虫申请人 :param: egg_file: 待上传的爬虫项目的egg文件 :return: 返回数据格式: json, 部署成功,返回success, 否则返回error """ status = 'error' project = Project() is_msd = request.form.get('is_msd') # 是否为分布式 project.is_msd = is_msd project.project_name = request.form.get('project_name') # 工程名 project.project_alias = request.form.get('project_alias') # 工程备注 project.for_project = request.form.get('for_project', None) # 引用工程 project.developers = request.form.get('developers', None) # 开发者 project.project_cate = request.form.get('pro_type', None) # 爬虫分类 project.applicant = request.form.get('applicant', None) # 申请人 project.date_created = datetime.datetime.now().strptime( datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S') project.date_modified = datetime.datetime.now().strptime( datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S') if not is_msd: return json.dumps({"code": 500, "status": '项目类型不能为空, 项目类型为分布式or单机'}) # 判断是否有值输入 if request.form.get('project_name') == '' or request.form.get('project_alias') == '': return json.dumps({"code": 200, "status": 'no input!'}) # 判断工程名是否存在 existed_project = project.query.filter_by(project_name=request.form['project_name']).first() # 工程存在则不能保存信息以及部署, 不存在则正常部署 if existed_project: return json.dumps({"code": 200, "status": 'existed'}) egg_path_dict = {} # 用于存放egg文件保存路径 if is_msd == '0': # 如果为单机爬虫 egg = request.files['egg'] # 如果是单机部署, 则文件放在某一个从服务器中 if egg: filename = secure_filename(egg.filename) # 获取egg文件名 dst_egg = os.path.join(tempfile.gettempdir(), filename) # 拼接文件路径 egg.save(dst_egg) # 保存egg文件 egg_path_dict['egg'] = dst_egg # 将项目文件路径保存到egg路径字典中 else: # 如果有一个没有上传文件 return json.dumps({"code": 500, "status": ' egg file are must'}) else: # 获取上传文件 master_egg = request.files['master_egg'] slave_egg = request.files['slave_egg'] # 判断表单是否传入文件 if master_egg and slave_egg: master_filename = secure_filename(master_egg.filename) # 获取master文件名 slave_filename = secure_filename(slave_egg.filename) # 获取slave文件名 dst_master_egg = os.path.join(tempfile.gettempdir(), master_filename) # 拼接文件路径 dst_slave_egg = os.path.join(tempfile.gettempdir(), slave_filename) # 拼接文件路径 slave_egg.save(dst_slave_egg) # 保存slave文件 master_egg.save(dst_master_egg) # 保存master文件 egg_path_dict['master'] = dst_master_egg # 将master项目文件路径保存到egg路径字典中 egg_path_dict['slave'] = dst_slave_egg # 将slave项目文件路径保存到egg路径字典中 else: # 如果有一个没有上传文件 return json.dumps({"code": 500, "status": 'master and slave egg are must'}) if agent.deploy(project, egg_path_dict, is_msd): status = 'success' # 部署成功后才将数据保存至数据库 db.session.add(project) db.session.commit() return json.dumps({"code": 200, "status": status}) else: return json.dumps({"code": 500, "status": "error", "msg": "部署错误"})