def ajax_add_tasks(request):
result_json = ResultJson()
server = request.POST.get('server')
db = request.POST.get('db')
queue_name = request.POST.get('queue_name')
tasks = request.POST.getlist('tasks')
limit = request.POST.get('limit')
data_type = request.POST.get('data_type')
if not data_type:
data_type = DescrapyConstant.QUEUE_DATA_TYPE_LIST
if not queue_name:
result_json.set_status_code(ResultJson.STATUS_CODE_FAIL)
result_json.set_status_message("队列名称不能为空")
return JsonResponse(result_json.__dict__, content_type='application/json')
if not tasks:
result_json.set_status_code(ResultJson.STATUS_CODE_FAIL)
result_json.set_status_message("任务不能为空")
return JsonResponse(result_json.__dict__, content_type='application/json')
if not server:
server = DescrapyConstant.DEFAULT_DSCRAPY_REDIS_IP
if not db:
db = DescrapyConstant.DEFAULT_DSCRAPY_REDIS_DB
if not limit:
limit = DescrapyConstant.QUEUE_TASKS_LIMIT
data_type = int(data_type)
params = None
if data_type == DescrapyConstant.QUEUE_DATA_TYPE_LIST:
params = {"server": server, "db":db, "queue_name":queue_name, "tasks":tasks, "limit":int(limit), "data_type":data_type}
if data_type == DescrapyConstant.QUEUE_DATA_TYPE_HASH:
params = {"server": server, "db":db, "queue_name":queue_name, "tasks":tasks[0], "limit":int(limit), "data_type":data_type}
tc = TaskControl()
result = tc.submit_tasks(params)
return JsonResponse(result.__dict__, content_type='application/json')
def ajax_random_get_server(request):
result_json = ResultJson()
nc = NodeControl()
nodes = nc.getAllNodes()
server = random.choice(nodes)['host_ip']
result_json.set_data(server)
result_json.set_status_code(ResultJson.STATUS_CODE_SUCCESS)
return JsonResponse(result_json.__dict__, content_type='application/json')
def watch_cpu(self, times=2, is_close=True):
result_json = ResultJson()
data = self.doCmd(DescrapyConstant.WATCH_CPU_CMD % (times), is_close)
if data:
result_json.set_status_code(ResultJson.STATUS_CODE_SUCCESS)
result_json.set_data(data)
else:
result_json.set_status_code(ResultJson.STATUS_CODE_FAIL)
return result_json
def watch_disk(self, is_close=True):
result_json = ResultJson()
data = self.doCmd(DescrapyConstant.WATCH_DISK_CMD, is_close)
if data:
result_json.set_status_code(ResultJson.STATUS_CODE_SUCCESS)
result_json.set_data(data)
else:
result_json.set_status_code(ResultJson.STATUS_CODE_FAIL)
return result_json
def count_cpus(self, is_close=True):
result_json = ResultJson()
data = self.doCmd(DescrapyConstant.COUNT_CPUS, is_close)
if data:
result_json.set_status_code(ResultJson.STATUS_CODE_SUCCESS)
result_json.set_data(data)
else:
result_json.set_status_code(ResultJson.STATUS_CODE_FAIL)
return result_json
def ajax_add_job(request):
result_json = ResultJson()
project = request.GET['project_name']
spider = request.GET['spider_name']
host_ip = request.GET['host_ip']
host_port = DescrapyConstant.DEFAULT_DSCRAPY_SERVER_PORT
params = {'project':project, 'spider':spider, 'host_ips':host_ip, 'host_port':host_port}
jc = JobControl()
result = jc.start_jobs(params)
if result and len(result) > 0:
status = result[0]['status']
if status == 'ok':
result_json.set_status_code(ResultJson.STATUS_CODE_SUCCESS)
result_json.set_status_message("运行成功")
else:
result_json.set_status_code(ResultJson.STATUS_CODE_FAIL)
result_json.set_status_message("运行失败")
return JsonResponse(result_json.__dict__, content_type='application/json')
def ajax_load_node(request):
result_json = ResultJson()
server = request.GET['server']
lc = LinuxClient()
data = {}
if lc.connect(server, DescrapyConstant.DEFAULT_NODE_USER, DescrapyConstant.DEFAULT_NODE_PASSWORD):
memery = lc.doCmd("free -m", False)
lines = memery.split("\n")
if len(lines) > 1:
disk_line = lines[1]
disk_result = disk_line.replace(" ", "|")
disk_reults = disk_result.split("|")
results = []
for disk in disk_reults:
if disk:
if not "/" in disk:
results.append(disk)
memery_total = math.ceil(float(results[1])/1024)
memery_used_rate = str((1 - round((int(results[3]) + int(results[5])) / float(results[1]), 2)) * 100) + "%"
data['memery_total'] = memery_total
data['memery_used_rate'] = memery_used_rate
disk = lc.watch_disk(False).data
lines = disk.split("\n")
if len(lines) > 1:
disk_line = lines[1]
disk_result = disk_line.replace(" ", "|")
disk_reults = disk_result.split("|")
results = []
for disk in disk_reults:
if disk:
if not "/" in disk:
results.append(disk)
data['disk_total'] = results[0] if len(results) > 0 else "unknow"
data['disk_used_rate'] = results[3] if len(results) > 3 else "unknow"
cpus = lc.count_cpus().data
data['cpus'] = cpus
result_json.set_data(data)
result_json.set_status_code(ResultJson.STATUS_CODE_SUCCESS)
else:
result_json.set_status_code(ResultJson.STATUS_CODE_FAIL)
result_json.set_status_message("[%s]用户名或密码错误, 连接失败!" %server)
return JsonResponse(result_json.__dict__, content_type='application/json')
def watch_node(request):
node_ip = request.GET['node_ip']
user_name = request.GET['user_name']
password = request.GET['password']
cmd = request.GET['cmd']
lc = LinuxClient()
result_json = None
if lc.connect(node_ip, user_name, password):
if cmd == "cpu":
result_json = lc.watch_cpu()
if cmd == "disk":
result_json = lc.watch_disk()
if cmd == "memery":
result_json = lc.watch_memery()
else:
result_json = ResultJson()
result_json.set_status_code(ResultJson.STATUS_CODE_FAIL)
result_json.set_status_message("用户名或密码错误, 连接失败!")
return JsonResponse(result_json.__dict__, content_type='application/json')
def ajax_load_cpu_line(request):
server = request.GET['server']
lc = LinuxClient()
result_json = ResultJson()
if lc.connect(server, DescrapyConstant.DEFAULT_NODE_USER, DescrapyConstant.DEFAULT_NODE_PASSWORD):
times = 2
result = lc.watch_cpu(times, False).data
lines = result.split("\n")
cpu_lines = []
for line in lines:
pattern = re.compile(r"^%Cpu.*$")
match = pattern.search(line)
if match:
cpu_lines.append(match.group())
used_rate = 100 - float(cpu_lines[times - 1].split(",")[3].strip().replace(" ", "|").split("|")[0])
date_time = date.format_date(datetime.now())
data = {"date": date_time, "rate": used_rate, "server": server}
result_json.set_data(data)
result_json.set_status_code(ResultJson.STATUS_CODE_SUCCESS)
else:
result_json.set_status_code(ResultJson.STATUS_CODE_FAIL)
result_json.set_status_message("[%s]用户名或密码错误, 连接失败!" %server)
return JsonResponse(result_json.__dict__, content_type='application/json')