def spider_egg_upload(project_id):
project = Project.find_project_by_id(project_id)
if 'file' not in request.files:
flash('No file part')
return redirect(request.referrer)
file = request.files['file']
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '':
flash('No selected file')
return redirect(request.referrer)
if file:
filename = secure_filename(file.filename)
dst = os.path.join(tempfile.gettempdir(), filename)
file.save(dst)
agent.deploy(project, dst)
flash('deploy success!')
return redirect(request.referrer)
def add_project():
"""
功能: 创建工程, 上传工程的egg文件, 将工程部署到scrapyd服务器上
:param: project_name: 工程名称
:param: project_alias: 工程备注或中文名称
:param: for_project: 引用工程
:param: developers: 爬虫项目的开发者
:param: applicant: 爬虫申请人
:param: egg_file: 待上传的爬虫项目的egg文件
:return: 返回数据格式: json, 部署成功,返回success, 否则返回error
"""
status = 'error'
project = Project()
is_msd = request.form.get('is_msd') # 是否为分布式
project.is_msd = is_msd
project.project_name = request.form.get('project_name') # 工程名
project.project_alias = request.form.get('project_alias') # 工程备注
project.for_project = request.form.get('for_project', None) # 引用工程
project.developers = request.form.get('developers', None) # 开发者
project.project_cate = request.form.get('pro_type', None) # 爬虫分类
project.applicant = request.form.get('applicant', None) # 申请人
project.date_created = datetime.datetime.now().strptime(
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')
project.date_modified = datetime.datetime.now().strptime(
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')
if not is_msd:
return json.dumps({"code": 500, "status": '项目类型不能为空, 项目类型为分布式or单机'})
# 判断是否有值输入
if request.form.get('project_name') == '' or request.form.get('project_alias') == '':
return json.dumps({"code": 200, "status": 'no input!'})
# 判断工程名是否存在
existed_project = project.query.filter_by(project_name=request.form['project_name']).first()
# 工程存在则不能保存信息以及部署, 不存在则正常部署
if existed_project:
return json.dumps({"code": 200, "status": 'existed'})
egg_path_dict = {} # 用于存放egg文件保存路径
if is_msd == '0': # 如果为单机爬虫
egg = request.files['egg'] # 如果是单机部署, 则文件放在某一个从服务器中
if egg:
filename = secure_filename(egg.filename) # 获取egg文件名
dst_egg = os.path.join(tempfile.gettempdir(), filename) # 拼接文件路径
egg.save(dst_egg) # 保存egg文件
egg_path_dict['egg'] = dst_egg # 将项目文件路径保存到egg路径字典中
else: # 如果有一个没有上传文件
return json.dumps({"code": 500, "status": ' egg file are must'})
else:
# 获取上传文件
master_egg = request.files['master_egg']
slave_egg = request.files['slave_egg']
# 判断表单是否传入文件
if master_egg and slave_egg:
master_filename = secure_filename(master_egg.filename) # 获取master文件名
slave_filename = secure_filename(slave_egg.filename) # 获取slave文件名
dst_master_egg = os.path.join(tempfile.gettempdir(), master_filename) # 拼接文件路径
dst_slave_egg = os.path.join(tempfile.gettempdir(), slave_filename) # 拼接文件路径
slave_egg.save(dst_slave_egg) # 保存slave文件
master_egg.save(dst_master_egg) # 保存master文件
egg_path_dict['master'] = dst_master_egg # 将master项目文件路径保存到egg路径字典中
egg_path_dict['slave'] = dst_slave_egg # 将slave项目文件路径保存到egg路径字典中
else: # 如果有一个没有上传文件
return json.dumps({"code": 500, "status": 'master and slave egg are must'})
if agent.deploy(project, egg_path_dict, is_msd):
status = 'success'
# 部署成功后才将数据保存至数据库
db.session.add(project)
db.session.commit()
return json.dumps({"code": 200, "status": status})
else:
return json.dumps({"code": 500, "status": "error", "msg": "部署错误"})
