def web_start(dd, host, web_port):
"""
当service.py为入口时会调用这里
"""
GlobalHelper.init(dd)
# wsgi.WSGIServer((host, web_port), app, backlog=1).serve_forever(3)
app.run(host=host, port=web_port, threaded=True)
def web_start(dd, host, web_port):
"""
当service.py为入口时会调用这里
"""
GlobalHelper.init(dd)
# wsgi.WSGIServer((host, web_port), app, backlog=1).serve_forever(3)
app.run(host=host, port=web_port, threaded=True)
def request_handle(data, address):
request = json.loads(data)
slave_record.add_request_record(address[0])
if 'urls_fail' in request and request['urls_fail']:
slave_record.add_fails_record(address[0], request['urls_fail'])
GlobalHelper.set('salve_record', slave_record.slave_record)
restart_slave_list = GlobalHelper.get('restart_slave_list') or []
if address[0] in restart_slave_list:
# print restart_slave_list
restart_slave_list.remove(address[0])
GlobalHelper.set('restart_slave_list', restart_slave_list)
# print restart_slave_list
# print GlobalHelper.get('restart_slave_list')
return json.dumps({'msg': '该客户端被手动重启中!', 'restart': 1})
if slave_record.slave_record[address[0]]['static'] == '暂停中':
return json.dumps({'msg': '. 该客户端被手动暂停中!'})
if 'init' in request:
projects = SlaveCtrl().code_ctrl()
if not projects:
return json.dumps({'msg': '. 暂无项目, 请访问WEB控制台添加项目!'})
return json.dumps({'msg': '. 获取项目成功!', 'projects': projects})
if 'project_name' not in request:
return json.dumps({'msg': '未设置该项目名称'})
handle = SerHandle(request['project_name'], request, address)
if not handle.get_project():
return json.dumps({'msg': '操作不存在的项目, 将该客户端重启!', 'restart': 1})
if 'urls_parsed' in request and request['urls_parsed']:
handle.urls_parsed()
if 'urls_add' in request and request['urls_add'] and isinstance(request['urls_add'], list):
handle.urls_add()
if 'save' in request and request['save'] and isinstance(request['save'], list):
handle.result_save()
if handle.get_project() and handle.get_project()['static'] == '暂停中':
return json.dumps({'msg': '. 该项目"' + request['project_name'] + '"被手动暂停中!'})
response_url_list = []
if 'get_urls' in request:
response_url_list = handle.get_urls()
msg = ''
if not response_url_list:
msg = ', 目前该项目已无可取的队列.'
# 客户端会通过判断change_time的不同而reload项目
return json.dumps({'msg': msg, 'urls': response_url_list,
'change_time': handle.get_project()['update_time']})
def restart_slave(slave_id):
try:
slave_record = Mongo.get()["slave_record"].find_one({"_id": ObjectId(slave_id)})
if not slave_record:
raise Exception("不存在的记录!")
except:
return jsonify({"success": False, "msg": "不存在的记录!"})
restart_slave_list = GlobalHelper.get("restart_slave_list") or []
restart_slave_list.append(slave_record["ip"])
GlobalHelper.set("restart_slave_list", list(set(restart_slave_list)))
return jsonify({"success": True, "msg": "重启中!"})
def restart_slave(slave_id):
try:
slave_record = Mongo.get()['slave_record'].find_one({'_id': ObjectId(slave_id)})
if not slave_record:
raise Exception('不存在的记录!')
except:
return jsonify({'success': False, 'msg': '不存在的记录!'})
restart_slave_list = GlobalHelper.get('restart_slave_list') or []
restart_slave_list.append(slave_record['ip'])
GlobalHelper.set('restart_slave_list', list(set(restart_slave_list)))
return jsonify({'success': True, 'msg': '重启中!'})
def cli(host, port, wwwport):
click.echo('\r\nService listening on %s:%s' % (
host,
port,
))
web_list = list()
client_connect_list = list()
web_list.append('local: http://127.0.0.1:%s' % wwwport)
client_connect_list.append(
'local: python client.py --host 127.0.0.1 --port %s' % port)
if host == '0.0.0.0':
wan_ip = get_wan_ip()
lan_ip = socket.gethostbyname(socket.gethostname())
if lan_ip != wan_ip:
web_list.append('lan : http://%s:%s' % (lan_ip, wwwport))
client_connect_list.append(
'lan : python client.py --host %s --port %s' % (lan_ip, port))
if wan_ip:
web_list.append(
'wan : http://%s:%s PS: Maybe need to set up port forwarding'
% (wan_ip, wwwport))
client_connect_list.append(
'wan : python client.py --host %s --port %s PS: Maybe need to set up port forwarding'
% (wan_ip, port))
click.echo('\r\nAccess console via:')
for item in web_list:
click.echo(item)
click.echo('\r\nClient connection use:')
for item in client_connect_list:
click.echo(item)
click.echo('\r\nListening ...')
global_process_var = Manager().dict()
GlobalHelper.init(global_process_var)
web_ui = Process(target=web_start,
args=(global_process_var, host, wwwport))
web_ui.start()
global slave_record
slave_record = SlaveRecord.get_instance()
GlobalHelper.set('salve_record', slave_record.slave_record)
socket_server(host, port)
web_ui.join()
def restart_slave(slave_id):
try:
slave_record = Mongo.get()['slave_record'].find_one(
{'_id': ObjectId(slave_id)})
if not slave_record:
raise Exception('不存在的记录!')
except:
return jsonify({'success': False, 'msg': '不存在的记录!'})
restart_slave_list = GlobalHelper.get('restart_slave_list') or []
restart_slave_list.append(slave_record['ip'])
GlobalHelper.set('restart_slave_list', list(set(restart_slave_list)))
return jsonify({'success': True, 'msg': '重启中!'})
def save_project():
form_data = json.loads(request.data) # todo 需要验证表单数据
name_r = re.compile(r"^[0-9a-zA-Z_-]+$")
if not name_r.search(form_data["name"]):
return jsonify({"success": False, "msg": "计划名称必须满足正则规则: ^[0-9a-zA-Z_-]+$ "})
exists_project = list(Mongo.get()["projects"].find({"name": form_data["name"]}, {"_id": 1, "add_time": 1}).limit(1))
if "edit" not in form_data and exists_project:
return jsonify({"success": False, "msg": "计划名称已经存在!"})
# 新增计划或更新计划
data = {
"name": form_data["name"],
"init_url": form_data["init_url"],
"desc": form_data["desc"] if "desc" in form_data else "",
"code": form_data["code"],
"static": "暂停中",
"update_time": int(time.time()),
"add_time": exists_project[0]["add_time"] if exists_project else int(time.time()),
}
Mongo.get()["projects"].update({"name": form_data["name"]}, data, True)
# 当是新计划时的初始化
if "edit" not in form_data:
Mongo.get()["queue_" + form_data["name"]].insert(
{
"url": form_data["init_url"],
"url_md5": md5(form_data["init_url"]),
"flag_time": 0,
"add_time": int(time.time()),
"slave_ip": "0.0.0.0",
}
)
# 在没创建集合前设置索引mongodb会自动创建该集合并赋索引
Mongo.get()["parsed_" + form_data["name"]].ensure_index("url_md5", unique=True)
Mongo.get()["queue_" + form_data["name"]].ensure_index("url_md5", unique=True)
# 有新计划加入, 重启全部slave
restart_slave_list = GlobalHelper.get("restart_slave_list") or []
for slave_record in Mongo.get()["slave_record"].find():
restart_slave_list.append(slave_record["ip"])
GlobalHelper.set("restart_slave_list", list(set(restart_slave_list)))
return jsonify({"success": True, "msg": "保存成功!"})
def save_project():
return jsonify({'success': False, 'msg': '禁止修改!'})
form_data = json.loads(request.data) # todo 需要验证表单数据
name_r = re.compile(r'^[0-9a-zA-Z_-]+$')
if not name_r.search(form_data['name']):
return jsonify({'success': False, 'msg': '计划名称必须满足正则规则: ^[0-9a-zA-Z_-]+$ '})
exists_project = list(Mongo.get()['projects'].find({'name': form_data['name']}, {'_id': 1, 'add_time': 1}).limit(1))
if 'edit' not in form_data and exists_project:
return jsonify({'success': False, 'msg': '计划名称已经存在!'})
# 新增计划或更新计划
data = {
'name': form_data['name'],
'init_url': form_data['init_url'],
'desc': form_data['desc'] if 'desc' in form_data else '',
'code': form_data['code'],
'static': '暂停中',
'update_time': int(time.time()),
'add_time': exists_project[0]['add_time'] if exists_project else int(time.time()),
}
Mongo.get()['projects'].update({'name': form_data['name']}, data, True)
# 当是新计划时的初始化
if 'edit' not in form_data:
Mongo.get()['queue_' + form_data['name']].insert(
{
'url': form_data['init_url'],
'url_md5': md5(form_data['init_url']),
'flag_time': 0,
'add_time': int(time.time()),
'slave_ip': '0.0.0.0'
})
# 在没创建集合前设置索引mongodb会自动创建该集合并赋索引
Mongo.get()['parsed_' + form_data['name']].ensure_index('url_md5', unique=True)
Mongo.get()['queue_' + form_data['name']].ensure_index('url_md5', unique=True)
# 有新计划加入, 重启全部slave
restart_slave_list = GlobalHelper.get('restart_slave_list') or []
for slave_record in Mongo.get()['slave_record'].find():
restart_slave_list.append(slave_record['ip'])
GlobalHelper.set('restart_slave_list', list(set(restart_slave_list)))
return jsonify({'success': True, 'msg': '保存成功!'})
def toggle_slave(slave_id):
try:
slave_record = Mongo.get()['slave_record'].find_one({'_id': ObjectId(slave_id)})
if not slave_record:
raise Exception('不存在的记录!')
except:
return jsonify({'success': False, 'msg': '不存在的记录!'})
slave_record['data']['static'] = '抓取中' if slave_record['data']['static'] == '暂停中' else '暂停中'
try:
Mongo.get()['slave_record'].update({'_id': ObjectId(slave_id)},
{'$set': {'data.static': slave_record['data']['static']}})
global_salve_record = GlobalHelper.get('salve_record')
global_salve_record[slave_record['ip']]['static'] = slave_record['data']['static']
GlobalHelper.set('salve_record', global_salve_record)
except:
print traceback.format_exc()
return jsonify({'success': True, 'msg': '切换成功!'})
def toggle_slave(slave_id):
try:
slave_record = Mongo.get()["slave_record"].find_one({"_id": ObjectId(slave_id)})
if not slave_record:
raise Exception("不存在的记录!")
except:
return jsonify({"success": False, "msg": "不存在的记录!"})
slave_record["data"]["static"] = "抓取中" if slave_record["data"]["static"] == "暂停中" else "暂停中"
try:
Mongo.get()["slave_record"].update(
{"_id": ObjectId(slave_id)}, {"$set": {"data.static": slave_record["data"]["static"]}}
)
global_salve_record = GlobalHelper.get("salve_record")
global_salve_record[slave_record["ip"]]["static"] = slave_record["data"]["static"]
GlobalHelper.set("salve_record", global_salve_record)
except:
print traceback.format_exc()
return jsonify({"success": True, "msg": "切换成功!"})
def cli(host, port, wwwport):
click.echo('\r\nService listening on %s:%s' % (host, port,))
web_list = list()
client_connect_list = list()
web_list.append('local: http://127.0.0.1:%s' % wwwport)
client_connect_list.append('local: python client.py --host 127.0.0.1 --port %s' % port)
if host == '0.0.0.0':
wan_ip = get_wan_ip()
lan_ip = socket.gethostbyname(socket.gethostname())
if lan_ip != wan_ip:
web_list.append('lan : http://%s:%s' % (lan_ip, wwwport))
client_connect_list.append('lan : python client.py --host %s --port %s' % (lan_ip, port))
if wan_ip:
web_list.append('wan : http://%s:%s PS: Maybe need to set up port forwarding' % (wan_ip, wwwport))
client_connect_list.append(
'wan : python client.py --host %s --port %s PS: Maybe need to set up port forwarding' %
(wan_ip, port))
click.echo('\r\nAccess console via:')
for item in web_list:
click.echo(item)
click.echo('\r\nClient connection use:')
for item in client_connect_list:
click.echo(item)
click.echo('\r\nListening ...')
global_process_var = Manager().dict()
GlobalHelper.init(global_process_var)
web_ui = Process(target=web_start, args=(global_process_var, host, wwwport))
web_ui.start()
global slave_record
slave_record = SlaveRecord.get_instance()
GlobalHelper.set('salve_record', slave_record.slave_record)
socket_server(host, port)
web_ui.join()
def toggle_slave(slave_id):
try:
slave_record = Mongo.get()['slave_record'].find_one(
{'_id': ObjectId(slave_id)})
if not slave_record:
raise Exception('不存在的记录!')
except:
return jsonify({'success': False, 'msg': '不存在的记录!'})
slave_record['data'][
'static'] = '抓取中' if slave_record['data']['static'] == '暂停中' else '暂停中'
try:
Mongo.get()['slave_record'].update(
{'_id': ObjectId(slave_id)},
{'$set': {
'data.static': slave_record['data']['static']
}})
global_salve_record = GlobalHelper.get('salve_record')
global_salve_record[
slave_record['ip']]['static'] = slave_record['data']['static']
GlobalHelper.set('salve_record', global_salve_record)
except:
print traceback.format_exc()
return jsonify({'success': True, 'msg': '切换成功!'})
def api_slave():
try:
new_records = {}
salve_records = GlobalHelper.get('salve_record')
if not salve_records:
return jsonify(new_records)
for (key, value) in salve_records.items():
item = dict(value)
item['ip'] = mix_ip(key)
new_records[value['_id']] = item
return jsonify(new_records)
except:
print traceback.format_exc()
def api_slave():
try:
new_records = {}
salve_records = GlobalHelper.get('salve_record')
if not salve_records:
return jsonify(new_records)
for (key, value) in salve_records.items():
item = dict(value)
item['ip'] = mix_ip(key)
new_records[value['_id']] = item
return jsonify(new_records)
except:
print traceback.format_exc()
def request_handle(data, address):
request = json.loads(data)
slave_record.add_request_record(address[0])
if 'urls_fail' in request and request['urls_fail']:
slave_record.add_fails_record(address[0], request['urls_fail'])
GlobalHelper.set('salve_record', slave_record.slave_record)
restart_slave_list = GlobalHelper.get('restart_slave_list') or []
if address[0] in restart_slave_list:
# print restart_slave_list
restart_slave_list.remove(address[0])
GlobalHelper.set('restart_slave_list', restart_slave_list)
# print restart_slave_list
# print GlobalHelper.get('restart_slave_list')
return json.dumps({'msg': '该客户端被手动重启中!', 'restart': 1})
if slave_record.slave_record[address[0]]['static'] == '暂停中':
return json.dumps({'msg': '. 该客户端被手动暂停中!'})
if 'init' in request:
projects = SlaveCtrl().code_ctrl()
if not projects:
return json.dumps({'msg': '. 暂无项目, 请访问WEB控制台添加项目!'})
return json.dumps({'msg': '. 获取项目成功!', 'projects': projects})
if 'project_name' not in request:
return json.dumps({'msg': '未设置该项目名称'})
handle = SerHandle(request['project_name'], request, address)
if not handle.get_project():
return json.dumps({'msg': '操作不存在的项目, 将该客户端重启!', 'restart': 1})
if 'urls_parsed' in request and request['urls_parsed']:
handle.urls_parsed()
if 'urls_add' in request and request['urls_add'] and isinstance(
request['urls_add'], list):
handle.urls_add()
if 'save' in request and request['save'] and isinstance(
request['save'], list):
handle.result_save()
if handle.get_project() and handle.get_project()['static'] == '暂停中':
return json.dumps(
{'msg': '. 该项目"' + request['project_name'] + '"被手动暂停中!'})
response_url_list = []
if 'get_urls' in request:
response_url_list = handle.get_urls()
msg = ''
if not response_url_list:
msg = ', 目前该项目已无可取的队列.'
# 客户端会通过判断change_time的不同而reload项目
return json.dumps({
'msg': msg,
'urls': response_url_list,
'change_time': handle.get_project()['update_time']
})
def save_project():
form_data = json.loads(request.data) # todo 需要验证表单数据
name_r = re.compile(r'^[0-9a-zA-Z_-]+$')
if not name_r.search(form_data['name']):
return jsonify({
'success': False,
'msg': '计划名称必须满足正则规则: ^[0-9a-zA-Z_-]+$ '
})
exists_project = list(Mongo.get()['projects'].find(
{
'name': form_data['name']
}, {
'_id': 1,
'add_time': 1
}).limit(1))
if 'edit' not in form_data and exists_project:
return jsonify({'success': False, 'msg': '计划名称已经存在!'})
# 新增计划或更新计划
data = {
'name':
form_data['name'],
'init_url':
form_data['init_url'],
'desc':
form_data['desc'] if 'desc' in form_data else '',
'code':
form_data['code'],
'static':
'暂停中',
'update_time':
int(time.time()),
'add_time':
exists_project[0]['add_time'] if exists_project else int(time.time()),
}
Mongo.get()['projects'].update({'name': form_data['name']}, data, True)
# 当是新计划时的初始化
if 'edit' not in form_data:
Mongo.get()['queue_' + form_data['name']].insert({
'url':
form_data['init_url'],
'url_md5':
md5(form_data['init_url']),
'flag_time':
0,
'add_time':
int(time.time()),
'slave_ip':
'0.0.0.0'
})
# 在没创建集合前设置索引mongodb会自动创建该集合并赋索引
Mongo.get()['parsed_' + form_data['name']].ensure_index('url_md5',
unique=True)
Mongo.get()['queue_' + form_data['name']].ensure_index('url_md5',
unique=True)
# 有新计划加入, 重启全部slave
restart_slave_list = GlobalHelper.get('restart_slave_list') or []
for slave_record in Mongo.get()['slave_record'].find():
restart_slave_list.append(slave_record['ip'])
GlobalHelper.set('restart_slave_list', list(set(restart_slave_list)))
return jsonify({'success': True, 'msg': '保存成功!'})
