def find_debug_log(request):
"""
查看测试执行日志
:param request:
:return:
"""
work_path = os.getcwd()
try:
if request.method == 'GET':
project_name = request.GET.get('project_name')
spider_name = request.GET.get('spider_name')
current_line = int(request.GET.get('current_line'))
project_path = join(PROJECTS_FOLDER, project_name)
os.chdir(project_path)
if not os.path.exists("debug_folder"):
r = Result.success(data='')
return JsonResponse(r)
input_file = open('./debug_folder/logs/{}.log'.format(spider_name), 'r', encoding='utf-8')
lines = input_file.readlines()
input_file.close()
response = []
for line in lines[(current_line - 1):]:
data = {'current_line': current_line, 'data': line}
response.append(data)
current_line = current_line + 1
r = Result.success(response)
return JsonResponse(r)
except Exception as e:
r = Result.fail(e)
return JsonResponse(r)
finally:
os.chdir(work_path)
def script_newest_log(request):
"""
获取脚本最新日志
:param request:
:return:
"""
try:
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
script_id = data.get('script_id')
host_ip = data.get('host_ip')
script = CrawlScript.objects.get(id=script_id)
project_name = script.project_name
spider_name = script.name
job_id = script.job_id
if not job_id:
r = Result.success('暂无日志')
return JsonResponse(r)
url = 'http://{}/logs/{}/{}/{}.log'.format(host_ip, project_name, spider_name, job_id)
response = requests.get(url)
if response.status_code != 200:
r = Result.success('暂无日志')
return JsonResponse(r)
log_content = response.content.decode('utf-8')
r = Result.success({'message': log_content})
return JsonResponse(r)
except Exception as e:
r = Result.fail(e)
return JsonResponse(r)
def find_debug_result(request):
"""
查看测试执行结果
:param request:
:return:
"""
work_path = os.getcwd()
try:
if request.method == 'GET':
project_name = request.GET.get('project_name')
spider_name = request.GET.get('spider_name')
project_path = join(PROJECTS_FOLDER, project_name)
os.chdir(project_path)
if not os.path.exists("debug_folder"):
r = Result.success(data='')
return JsonResponse(r)
input_file = open('./debug_folder/items/{}.json'.format(spider_name))
all_text = input_file.read()
input_file.close()
r = Result.success({'content': all_text})
return JsonResponse(r)
except Exception as e:
r = Result.fail(e)
return JsonResponse(r)
finally:
os.chdir(work_path)
def project_deploy(request, project_name):
"""
发布爬虫工程
:param request: request object
:param project_name: project name
:return: json of deploy result
"""
if request.method == 'POST':
path = os.path.abspath(join(os.getcwd(), PROJECTS_FOLDER))
project_path = join(path, project_name)
# 检索打包egg文件
egg = find_egg(project_path)
if not egg:
r = Result.success("没有打包文件")
return JsonResponse(r)
egg_file = open(join(project_path, egg), 'rb')
data = json.loads(request.body.decode('utf-8'))
node_ids = data["node_ids"]
nodes = CrawlNode.objects.filter(id__in=node_ids)
project = CrawlProject.objects.get(name=project_name)
for node in nodes:
engine = get_engine(node)
engine.add_version(project_name, int(time.time()), egg_file.read())
deployed_at = timezone.now()
CrawlDeploy.objects.filter(
node_id=node.id, project_id=project.id).delete() # 这里逻辑删除
deploy, result = CrawlDeploy.objects.update_or_create(
node_id=node.id,
project_id=project.id,
deployed_at=deployed_at,
description=project.description)
r = Result.success("")
return JsonResponse(r)
def fetch_user_permissions(request):
"""
获取用户菜单权限列表
:param request:
:return:
"""
user_id = request.user_id
user_roles = CrawlUserRoleRel.objects.filter(user_id=user_id, is_deleted=0)
if not user_roles:
return JsonResponse(Result.success(data={}))
permission_tree = build_permission_tree(user_roles)
crawl_redis.set('permission#user#{}'.format(user_id), json.dumps(permission_tree))
r = Result.success(data=permission_tree)
return JsonResponse(r)
def reset_profile_pwd(request, user_id):
"""
重置个人密码
:param user_id:
:param request:
:return:
"""
try:
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
old_pwd = data.get('old_pwd')
new_pwd = data.get('new_pwd')
confirm_pwd = data.get('confirm_pwd')
user = CrawlUser.objects.get(id=user_id)
if confirm_pwd != new_pwd:
raise Exception('两次密码输入不一致')
db_pwd = user.password
if db_pwd != password2md5(old_pwd):
raise Exception('密码不正确')
user.password = password2md5(new_pwd)
user.save()
r = Result.success(None)
return JsonResponse(r)
except Exception as e:
r = Result.fail(e)
return JsonResponse(r)
def login(request):
"""
登录（TODO 使用jwt）
:param request:
:return:
"""
try:
domain = settings.SESSION_COOKIE_DOMAIN
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
username = data.get('username').strip()
password = data.get('password').strip()
user = CrawlUser.objects.get(username=username)
if not user:
raise Exception('用户名或密码不正确')
else:
if password2md5(password) == user.password:
token = jwt_tools.encode_token(user.id, user.username)
r = Result.success(None)
response = JsonResponse(r)
response.set_cookie('dt_token', bytes.decode(token), domain=domain, max_age=60 * 60 * 24 * 30)
response.set_cookie('dt_user_id', user.id, domain=domain, max_age=60 * 60 * 24 * 30)
response.set_cookie('dt_username', user.username, domain=domain, max_age=60 * 60 * 24 * 30)
return response
else:
raise Exception('用户名或密码不正确')
except Exception as e:
r = Result.fail(e)
return JsonResponse(r)
def list_role(request):
"""
角色列表
:param request:
:return:
"""
if request.method == 'GET':
page = request.GET.get('page', 1)
size = request.GET.get('size', 15)
response = []
roles = CrawlRole.objects.filter(is_deleted=0)
relsDict = CrawlPermission.objects.filter(is_deleted=0)
for role in roles:
rels = CrawlRolePermission.objects.filter(is_deleted=0, role_id=role.id)
role_permissions = []
for rel in rels:
permissions = relsDict.get(id=rel.permission_id)
role_permissions.append(model_to_dict(permissions).get('permission_name'))
roleD = model_to_dict(role)
roleD.__setitem__('permission', role_permissions)
response.append(roleD)
r = Result.success(response)
return JsonResponse(r)
def task_by_script_id(request, script_id):
"""
根据脚本 id 获取任务
:param script_id:
:param request:
:return:
"""
work_path = os.getcwd()
try:
if request.method == 'GET':
script = CrawlScript.objects.get(id=script_id)
project = CrawlProject.objects.get(id=script.project_id)
task = CrawlTask.objects.get(id=project.task_id)
path = os.path.abspath(join(work_path, PROJECTS_FOLDER))
script_name = script.name
vo = model_to_dict(task)
vo.__setitem__('path', path)
vo.__setitem__('script_name', script.script_file)
r = Result.success(vo)
return JsonResponse(r)
except Exception as e:
r = Result.fail(e)
return JsonResponse(r)
finally:
os.chdir(work_path)
def list_proxy_ip(request):
"""
所有代理 ip
:param request:
:return:
"""
try:
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
keyword = data.get('keyword')
page = data.get('page', 1)
size = data.get('size', 15)
ip_type = data.get('ip_type')
status = request.GET.get('status')
proxy_ips = CrawlProxyIP.objects.filter(is_deleted=0)
if keyword is not None:
proxy_ips = proxy_ips.filter(ip__icontains=keyword)
if ip_type is not None:
proxy_ips = proxy_ips.filter(ip_type=ip_type)
if status is not None:
proxy_ips = proxy_ips.filter(status=status)
total = proxy_ips.count()
r = Result.success(page_helper(total, page, size, proxy_ips))
return JsonResponse(r)
except Exception as e:
r = Result.fail(e)
return JsonResponse(r)
def edit_user(request, user_id):
"""
修改用户
:param user_id:
:param request: request object
:return: json
"""
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
alert_options = data.get('alert_options')
user = CrawlUser.objects.get(id=user_id)
user.account = data.get('account')
user.mobile = data.get('mobile', '')
user.wx_account = data.get('wx_account')
user.comment = data.get('comment', '')
user.alert_enable = data.get('alert_enable', 0)
user.save()
role_ids = data.get('role_ids')
CrawlUserRoleRel.objects.filter(user_id=user_id).update(is_deleted=1)
for role_id in role_ids:
CrawlUserRoleRel.objects.create(role_id=role_id,
user_id=user_id)
# 权限树写进 redis
user_roles = CrawlUserRoleRel.objects.filter(user_id=user_id)
crawl_redis.set('permission#user#{}'.format(user_id), json.dumps(build_permission_tree(user_roles)))
r = Result.success(None)
return JsonResponse(r)
def project_build(request, project_name):
"""
爬虫工程编译打包
:param request: request object
:param project_name: project name
:return: json
"""
path = os.path.abspath(join(os.getcwd(), PROJECTS_FOLDER))
project_path = join(path, project_name)
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
description = data['description']
build_project(
project_name,
include_data=False if project_name != 'auto_login' else True)
egg = find_egg(project_path)
if not egg:
return JsonResponse(Result.fail("编译打包失败"))
built_at = timezone.now()
if not CrawlProject.objects.filter(name=project_name):
CrawlProject(name=project_name,
description=description,
built_at=built_at,
egg=egg).save()
model = CrawlProject.objects.get(name=project_name)
else:
model = CrawlProject.objects.get(name=project_name, is_deleted=0)
model.built_at = built_at
model.egg = egg
model.description = description
model.save()
data = model_to_dict(model)
r = Result.success(data)
return JsonResponse(r)
def list_scripts(request):
"""
某个爬虫工程节点爬虫脚本分布列表
:param request:
:return:
"""
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
size = data.get('size', 15)
page = data.get('page', 1)
task_name = data.get("task_name")
task_id = data.get("task_id")
script_name = data.get("script_name")
scripts = CrawlScript.objects.filter(is_deleted=0)
if task_id:
scripts = scripts.filter(task_id=task_id)
if script_name:
scripts = scripts.filter(name__contains=script_name)
if task_name:
scripts = scripts.filter(task_name__contains=task_name)
scripts = scripts.order_by("-id")
total = scripts.count()
response = page_helper(total, page, size, scripts)
results = response.get('results')
for result in results:
result.__setitem__('hosts', ','.join(get_hosts_by_script_id(result.get('id'))))
r = Result.success(response)
return JsonResponse(r)
def script_stop(request):
"""
启动脚本
:param request:
:return:
"""
try:
if request.method == 'POST':
data_scripts = json.loads(request.body.decode('utf-8'))
if not data_scripts:
return JsonResponse(Result.fail("没有指定脚本"))
for data_script in data_scripts:
crawl_script = CrawlScript.objects.get(id=data_script["id"])
host_list = get_hosts_by_script_id(crawl_script.id)
for host in host_list:
engine = get_engine_by_ip(host)
args = {
"redis": '{{"host":"{}","port": {},"db":1,"password":"******"}}'.format(db_conf.redis_host,
str(db_conf.redis_port),
db_conf.redis_pwd),
"batch_id": ''
}
jobs = engine.cancel(crawl_script.project_name, crawl_script.name)
r = Result.success(None)
return JsonResponse(r)
except Exception as err:
r = Result.fail(err)
return JsonResponse(r)
def edit_script_cfg(request):
"""
编辑爬虫脚本配置
:param request: request object
:return:
"""
try:
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
spider_name = data['spider_name']
script_name = data['script_name']
apply_to_all = data['applyToAll']
task_id = data['project_id']
script_args = []
for p in data.get('params'):
if isinstance(p['args'], str):
p['args'] = json.loads(p['args'])
script_args.append(p)
if p.get('trigger'):
result, message = scheduler_helper.verify_cron(p.get('trigger'))
if not result:
raise Exception('参数错误: {}'.format(message))
update_kwargs = {
"trigger": data.get('trigger'),
"hosts": data.get('hosts'),
"args": json.dumps(script_args)}
# 批量设置当前任务的所有脚本
if apply_to_all:
crawl_scripts = CrawlScript.objects.filter(task_id=task_id)
crawl_scripts.update(**update_kwargs)
else:
crawl_scripts = CrawlScript.objects.get(name=spider_name, task_id=task_id)
crawl_scripts.trigger = data.get('trigger')
crawl_scripts.hosts = data.get('hosts')
crawl_scripts.args = json.dumps(script_args)
crawl_scripts.save()
if 'params' in data and data['params']:
args = data['params']
# 设置每个爬虫脚本的执行参数，不同调度批次的爬虫运行参数使用md5区分
for arg in args:
if apply_to_all:
for script in crawl_scripts:
v_arg = encrypt_kit.md5(json.dumps(arg))
crawl_redis.set("args#{}#{}".format(script.name, v_arg), json.dumps(arg['args']))
else:
v_arg = encrypt_kit.md5(json.dumps(arg))
crawl_redis.set("args#{}#{}".format(spider_name, v_arg), json.dumps(arg['args']))
r = Result.success("")
return JsonResponse(r)
except Exception as e:
r = Result.fail(e)
return JsonResponse(r)
def node_index(request):
"""
获取节点列表
:param request: request object
:return: client list
"""
if request.method == 'GET':
data = CrawlNode.objects.filter(is_deleted=0).order_by('-id')
r = Result.success(data)
return JsonResponse(r)
def task_create(request):
"""
创建任务
:param request: request object
:return: Bool
"""
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
task = CrawlTask.objects.create(**data)
r = Result.success(model_to_dict(task))
return JsonResponse(r)
def node_info(request, client_id):
"""
获取爬虫节点信息
:param request: request object
:param id: client id
:return: json
"""
if request.method == 'GET':
data = model_to_dict(CrawlNode.objects.get(id=client_id))
r = Result.success(data=data)
return JsonResponse(r)
def list_task_progress(request):
"""
爬虫任务进度
:param request:
:return:
"""
try:
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
keyword = data.get('keyword')
script_name = data.get('script_name')
date = data.get('date')
status = data.get('status')
page = data.get('page', 1)
size = data.get('size', 15)
task_progress = CrawlScriptProgress.objects.filter(is_deleted=0).exclude(script_name='proxy')
condition_date = datetime.datetime.today().strftime('%Y-%m-%d') if date == '' else date
stat_task_progress = task_progress.filter(start_time__gte='{} 00:00:00'.format(condition_date),
start_time__lte='{} 23:59:59'.format(condition_date))
running_cnt = stat_task_progress.filter(status=1).count()
success_cnt = stat_task_progress.filter(status=2).count()
fail_cnt = stat_task_progress.filter(status=-1).count()
if keyword is not None and keyword != '':
task_progress = task_progress.filter(task_name__icontains=keyword)
if script_name is not None and script_name != '':
task_progress = task_progress.filter(script_name__icontains=script_name)
if date is not None and date != '':
task_progress = task_progress.filter(start_time__gte='{} 00:00:00'.format(date),
start_time__lte='{} 23:59:59'.format(date))
if status is not None:
task_progress = task_progress.filter(status__in=status)
task_progress = task_progress.order_by("-id")
total = task_progress.count()
pager = page_helper(total, page, size, task_progress, {'fail_cnt': fail_cnt,
'running_cnt': running_cnt,
'success_cnt': success_cnt})
convert_task_progress = []
results = pager.get('results')
for result in results:
result['run_time'] = time_kit.convert_ms(result.get('run_time'))
result['script_id'] = CrawlScript.objects.get(task_name=result.get('task_name'),
name=result.get('script_name')).id
convert_task_progress.append(result)
pager['results'] = convert_task_progress
r = Result.success(pager)
return JsonResponse(r)
except Exception as e:
r = Result.fail(e)
return JsonResponse(r)
def project_file_delete(request):
"""
删除爬虫工程文件
:param request: request object
:return: result of delete
"""
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
path = join(data['path'], data['label'])
os.remove(path)
return JsonResponse(Result.success(""))
def project_file_rename(request):
"""
重命名爬虫工程文件
:param request: request object
:return: result of rename
"""
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
pre = join(data['path'], data['pre'])
new = join(data['path'], data['new'])
os.rename(pre, new)
return JsonResponse(Result.success(""))
def project_file_create(request):
"""
创建爬虫工程文件
:param request: request object
:return: result of create
"""
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
path = join(data['path'], data['name'])
open(path, 'w', encoding='utf-8').close()
r = Result.success("")
return JsonResponse(r)
def get_profile(request, user_id):
"""
获取个人信息
:param user_id:
:param request:
:return:
"""
if request.method == 'GET':
user = CrawlUser.objects.get(id=user_id)
userD = model_to_dict(user)
userD.__setitem__('created_at', user.created_at)
r = Result.success(userD)
return JsonResponse(r)
def project_file_update(request):
"""
修改爬虫工程文件
:param request: request object
:return: result of update
"""
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
path = join(data['path'], data['label'])
code = data['code']
with open(path, 'w', encoding='utf-8') as f:
f.write(code)
r = Result.success("")
return JsonResponse(r)
def edit_profile(request, user_id):
"""
编辑个人信息（TODO 这里产品原型上缺少了告警设置，告警信息也可以编辑）
:param user_id:
:param request:
:return:
"""
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
CrawlUser.objects.filter(id=user_id).update(account=data.get('account'),
mobile=data.get('mobile'),
wx_account=data.get('wx_account'))
r = Result.success(None)
return JsonResponse(r)
def node_manager(request):
if request.method == 'GET':
page = request.GET.get('page', 1)
size = request.GET.get('size', 15)
nodes = list(CrawlNode.objects.filter(is_deleted=0))
total = len(nodes)
output_nodes = []
for client in nodes:
# node_url = engine_url(client.node_ip, client.node_port)
# node_url = engine_url('120.27.210.65', '6800')
client_d = to_dict(client)
output_nodes.append(client_d)
r = Result.success(page_helper(total, page, size, output_nodes))
return JsonResponse(r)
def node_status(request, node_id):
"""
获取某个爬虫节点的状态
:param request: request object
:param node_id: node_id id
:return: json
"""
if request.method == 'GET':
# get client object
client = CrawlNode.objects.get(id=node_id)
try:
requests.get(engine_url(client.ip, client.port), timeout=3)
return JsonResponse(Result.success(""))
except ConnectionError:
return JsonResponse({'message': 'Connect Error'}, status=500)
def get_proxy_ip(request, proxy_ip_id):
"""
获取一个代理 ip
:param request:
:param proxy_ip_id:
:return:
"""
try:
if request.method == 'GET':
proxy_ip = CrawlProxyIP.objects.get(id=proxy_ip_id)
r = Result.success(model_to_dict(proxy_ip))
return JsonResponse(r)
except Exception as e:
r = Result.fail(e)
return JsonResponse(r)
def project_list(request, node_id):
"""
获取某个node节点上的爬虫工程
:param request: request object
:param node_id: node_id
:return: json
"""
if request.method == 'GET':
client = CrawlNode.objects.get(id=node_id)
engine = get_engine(client)
try:
projects = engine.list_projects()
JsonResponse(Result.success(data=projects))
except ConnectionError:
return JsonResponse(Result.fail())
def create_proxy_ip(request):
"""
创建代理 ip
:param request:
:return:
"""
try:
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
CrawlProxyIP.objects.create(**data)
r = Result.success(None)
return JsonResponse(r)
except Exception as e:
r = Result.fail(e)
return JsonResponse(r)
