def project_create():
project_name = request.form['project_name']
project = Project()
project.project_name = project_name
db.session.add(project)
db.session.commit()
return redirect("/project/%s/spider/deploy" % project.id, code=302)
def post(self):
project_name = request.form['project_name']
project = Project()
project.project_name = project_name
db.session.add(project)
db.session.commit()
return project.to_dict()
def get_project_list(self):
data = request("get", self._scrapyd_url() + "/listprojects.json", return_type="json")
result = []
if data:
for project_name in data['projects']:
project = Project()
project.project_name = project_name
result.append(project)
return result
def post(self):
project_name = request.form['project_name']
project = Project()
project.project_name = project_name
try:
db.session.add(project)
db.session.commit()
except:
db.session.rollback()
raise
return project.to_dict()
def get_project_list(self):
"""
功能: 获取所有的爬虫工程列表
:return: 返回工程对象列表
"""
data = self.scrapyd_api.list_projects() # 获取scrapyd上的所有工程列表
result = []
if data:
for project_name in data:
project = Project() # 实例化工程对象
project.project_name = project_name
result.append(project)
return result
def add_project():
"""
功能: 创建工程, 上传工程的egg文件, 将工程部署到scrapyd服务器上
:param: project_name: 工程名称
:param: project_alias: 工程备注或中文名称
:param: for_project: 引用工程
:param: developers: 爬虫项目的开发者
:param: applicant: 爬虫申请人
:param: egg_file: 待上传的爬虫项目的egg文件
:return: 返回数据格式: json, 部署成功,返回success, 否则返回error
"""
status = 'error'
project = Project()
is_msd = request.form.get('is_msd') # 工程名
project.is_msd = is_msd
project.project_name = request.form.get('project_name') # 工程名
project.project_alias = request.form.get('project_alias') # 工程备注
project.for_project = request.form.get('for_project', None) # 引用工程
project.developers = request.form.get('developers', None) # 开发者
project.applicant = request.form.get('applicant', None) # 申请人
project.date_created = datetime.datetime.now().strptime(
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
'%Y-%m-%d %H:%M:%S')
project.date_modified = datetime.datetime.now().strptime(
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
'%Y-%m-%d %H:%M:%S')
if not is_msd:
return json.dumps({"code": 500, "status": '项目类型不能为空, 项目类型为分布式or单机'})
# 判断是否有值输入
if request.form.get('project_name') == '' or request.form.get(
'project_alias') == '':
return json.dumps({"code": 200, "status": 'no input!'})
# 判断工程名是否存在
existed_project = project.query.filter_by(
project_name=request.form['project_name']).first()
# 工程存在则不能保存信息以及部署, 不存在则正常部署
if existed_project:
return json.dumps({"code": 200, "status": 'existed'})
egg_path_dict = {} # 用于存放egg文件保存路径
if is_msd == '0': # 如果为单机爬虫
egg = request.files['egg']
if egg:
filename = secure_filename(egg.filename) # 获取egg文件名
dst_egg = os.path.join(tempfile.gettempdir(), filename) # 拼接文件路径
egg.save(dst_egg) # 保存egg文件
egg_path_dict['egg'] = dst_egg # 将项目文件路径保存到egg路径字典中
else: # 如果有一个没有上传文件
return json.dumps({"code": 500, "status": ' egg file are must'})
else:
# 获取上传文件
master_egg = request.files['master_egg']
slave_egg = request.files['slave_egg']
# 判断表单是否传入文件
if master_egg and slave_egg:
master_filename = secure_filename(
master_egg.filename) # 获取master文件名
slave_filename = secure_filename(slave_egg.filename) # 获取slave文件名
dst_master_egg = os.path.join(tempfile.gettempdir(),
master_filename) # 拼接文件路径
dst_slave_egg = os.path.join(tempfile.gettempdir(),
slave_filename) # 拼接文件路径
slave_egg.save(dst_slave_egg) # 保存slave文件
master_egg.save(dst_master_egg) # 保存master文件
egg_path_dict[
'master'] = dst_master_egg # 将master项目文件路径保存到egg路径字典中
egg_path_dict['slave'] = dst_slave_egg # 将slave项目文件路径保存到egg路径字典中
else: # 如果有一个没有上传文件
return json.dumps({
"code": 500,
"status": 'master and slave egg are must'
})
if agent.deploy(project, egg_path_dict, is_msd):
status = 'success'
# 部署成功后才将数据保存至数据库
db.session.add(project)
db.session.commit()
return json.dumps({"code": 200, "status": status})
else:
return json.dumps({"code": 500, "status": "error", "msg": "部署错误"})