def list_jobs(request):
context = {}
project = request.GET['project']
host_ip = request.GET['host_ip']
host_port = DescrapyConstant.DEFAULT_DSCRAPY_SERVER_PORT
params = {'project':project, 'host_ip':host_ip, 'host_port':host_port}
jc = JobControl()
result = jc.list_jobs(params)
context['result'] = result
return render(request, 'jobs_list.html', context)
def cancel_jobs(request):
jobs = json.loads(request.body)
host_port = DescrapyConstant.DEFAULT_DSCRAPY_SERVER_PORT
params = []
for job in jobs:
param = {'project':job['project'], 'job_id':job['job_id'], 'host_ip':job['host_ip'], 'host_port':host_port}
params.append(param)
jc = JobControl()
result = jc.cancel_jobs(params)
return HttpResponse(result.status_message)
def cancel_job(request):
project = request.GET['project']
job_id = request.GET['job_id']
host_ip = request.GET['host_ip']
host_port = DescrapyConstant.DEFAULT_DSCRAPY_SERVER_PORT
params = []
param = {'project':project, 'job_id':job_id, 'host_ip':host_ip, 'host_port':host_port}
params.append(param)
jc = JobControl()
result = jc.cancel_jobs(params)
return HttpResponse(result.status_message)
def list_jobs_by_servers(request):
jc = JobControl()
spider_name = request.GET['spider_name']
project = request.GET['project']
host_ips = str(request.GET['host_ips'])
params = []
host_ip_list = host_ips.split(",")
for host_ip in host_ip_list:
if host_ip:
param = {'spider_name':spider_name, 'project':project, 'host_ip':host_ip, 'host_port': DescrapyConstant.DEFAULT_DSCRAPY_SERVER_PORT}
params.append(param)
results = jc.list_jobs_by_servers(params)
result_str = ''
for result in results:
result_str += str(result) + "<br>"
return HttpResponse(result_str)
def clone_job(request):
spider = request.GET['spider']
host_ip = request.GET['host_ip']
project =request.GET['project']
host_port = DescrapyConstant.DEFAULT_DSCRAPY_SERVER_PORT
params = {'project':project, 'spider':spider, 'host_ips':host_ip, 'host_port':host_port}
jc = JobControl()
results = jc.start_jobs(params)
message = ''
if results and len(results) > 0:
result = results[0]
if result['status'] == 'ok':
message = '克隆成功'
else:
message = '克隆失败'
return HttpResponse(message)
def ajax_add_job(request):
result_json = ResultJson()
project = request.GET['project_name']
spider = request.GET['spider_name']
host_ip = request.GET['host_ip']
host_port = DescrapyConstant.DEFAULT_DSCRAPY_SERVER_PORT
params = {'project':project, 'spider':spider, 'host_ips':host_ip, 'host_port':host_port}
jc = JobControl()
result = jc.start_jobs(params)
if result and len(result) > 0:
status = result[0]['status']
if status == 'ok':
result_json.set_status_code(ResultJson.STATUS_CODE_SUCCESS)
result_json.set_status_message("运行成功")
else:
result_json.set_status_code(ResultJson.STATUS_CODE_FAIL)
result_json.set_status_message("运行失败")
return JsonResponse(result_json.__dict__, content_type='application/json')
def add_jobs(request):
context = {}
project = request.GET['project_name']
spider = request.GET['spider_name']
host_ips = request.GET['host_ips']
server_multi = request.GET['server_multi']
try:
server_multi = int(server_multi)
if server_multi >= 50:
server_multi = 50
except:
server_multi = 1
host_port = DescrapyConstant.DEFAULT_DSCRAPY_SERVER_PORT
params = {'project':project, 'spider':spider, 'host_ips':host_ips, 'host_port':host_port}
jc = JobControl()
for i in range(0, server_multi):
result_json = jc.start_jobs(params)
message = ''
if result_json and len(result_json) > 0:
success_count = 0
success_server = ''
fail_count = 0
fail_server = ''
for target_info in result_json:
if target_info['status'] == 'ok':
success_count += 1
success_server += target_info['host_ip'] + "\t"
else:
fail_count += 1
fail_server += target_info['host_ip'] + "\t"
message = "成功%s个，位于{%s}服务器上, 失败%s个, 位于{%s}服务器上" %(success_count * server_multi, success_server, fail_count, fail_server)
else:
message = '运行失败!'
sc = SpiderControl()
projects = sc.getAllProjects()
context['result'] = projects
context['message'] = message
return render(request, 'job_add.html', context)
def list_all_jobs(request):
context = {}
jc = JobControl()
result = jc.list_all_jobs()
context['result'] = result
return render(request, 'jobs_list.html', context)