def regist_server():
# 从数据库中获取主爬虫的服务器并进行注册
machines = Serversmachine.query.all()
for machine in machines:
machine_dict = machine.to_dict()
agent.regist(ScrapydProxy(machine_dict['server_ip']),
machine_dict.get("is_master"))
def addmachine():
"""
功能: 添加服务器
:return: 成功返回success, 失败返回相应的异常
"""
try:
# 实例化Serversmachine类
serversmachine = Serversmachine()
# 给server_ip字段赋值
serversmachine.server_ip = request.form.get('server_ip')
# 给is_master字段赋值
serversmachine.is_master = request.form.get('is_master')
# 给server_status字段赋值
serversmachine.server_status = request.form.get('server_status')
res = requests.get(request.form.get('server_ip'), timeout=1)
if res.status_code == 200:
# 保存数据
db.session.add(serversmachine)
db.session.commit()
agent.regist(ScrapydProxy(request.form.get('server_ip')),
request.form.get('is_master'))
return json.dumps({'code': 200, 'data': 'success'})
else:
json.dumps({"code": 500, "status": "error", "msg": "服务器状态不可用"})
except Exception as e:
return json.dumps({"code": 500, "status": "error", "msg": "服务器状态不可用"})
def post(self):
server = request.form.get('server').strip()
if server and (server not in agent.servers):
agent.regist(ScrapydProxy(server))
# sync update app config
new_servers = app.config.get('SERVERS')
new_servers.append(server)
app.config.update(SERVERS=new_servers)
return redirect(request.referrer, code=302)
def edit_machine():
"""
功能: 编辑服务器的信息
:return: 成功返回'data': 'success', 失败'data': 'error'
"""
try:
# 判断服务器是否可用
res = requests.get(request.form.get('server_ip'), timeout=2)
except:
return json.dumps({"code": 500, "status": "error", "msg": "服务器不可用"})
try:
"""新增"""
if res.status_code == 200:
serversmachine = Serversmachine()
id = request.form.get('id')
machines = serversmachine.query.filter_by(id=id).all()
"""先删除"""
if machines:
for machine in machines:
machine_dict = machine.to_dict()
if machine_dict.get('is_master') == '1':
for master_machine_instance in agent.spider_service_instances_master:
if master_machine_instance._server == machine.server_ip:
agent.spider_service_instances_master.remove(
master_machine_instance)
else:
for slave_machine_instance in agent.spider_service_instances_slave:
if slave_machine_instance._server == machine.server_ip:
agent.spider_service_instances_slave.remove(
slave_machine_instance)
# 删除数据库里的值
db.session.delete(machine)
db.session.commit()
"""保存数据"""
# 实例化Serversmachine类
serversmachine = Serversmachine()
# 给server_ip字段赋值
serversmachine.server_ip = request.form.get('server_ip')
# 给is_master字段赋值
serversmachine.is_master = request.form.get('is_master')
# 给server_status字段赋值
serversmachine.server_status = request.form.get('server_status')
db.session.add(serversmachine)
db.session.commit()
agent.regist(ScrapydProxy(request.form.get('server_ip')),
request.form.get('is_master'))
return json.dumps({'code': 200, 'data': 'success'})
except Exception as e:
return json.dumps({"code": 500, "status": "error", "msg": e})
def regist_server():
if app.config.get('SERVER_TYPE') == 'scrapyd':
for server in app.config.get("SERVERS"):
agent.regist(ScrapydProxy(server))