def render_html(request):
"""
render html with url
:param request:
:return:
"""
if request.method == 'POST':
data = json.loads(request.body)
print(data)
print(type(data))
url = data.get('url')
js = data.get('js')
if not js:
try:
response = requests.get(url, timeout=5)
response.encoding = response.apparent_encoding
html = process_html(response.text)
return JsonResponse({'result': '1', 'data': html})
except:
return JsonResponse({'result': '0'}, status=500)
if request.method == 'GET':
url = request.GET.get('url')
js = request.GET.get('js', 0)
script = request.GET.get('script')
try:
response = requests.get(url, timeout=5)
response.encoding = response.apparent_encoding
html = process_html(response.text)
return HttpResponse(html)
except:
return HttpResponse(status=500)
def project_parse(request, project_name):
"""
parse project
:param request: request object
:param project_name: project name
:return: requests, items, response
"""
if request.method == 'POST':
print(project_name)
project_path = join(PROJECTS_FOLDER, project_name)
print('Project Path', project_path)
data = json.loads(request.body)
spider_name = data.get('spider')
start = data.get('start')
method = data.get('method', 'get')
headers = data.get('headers', {})
meta = data.get('meta', {})
if start:
requests = get_start_requests(project_path, spider_name)
print(requests)
return JsonResponse({'status': '1', 'result': {'requests': requests}})
else:
url = data.get('url')
callback = data.get('callback')
print(url, project_path, spider_name, callback)
result = parser.get_follow_results(url, project_path, spider_name, callback)
print(result)
result['response']['html'] = process_html(result['response']['html'], dirname(url))
return JsonResponse({'status': '1', 'result': result})
def render_html(request):
"""
render html with url
:param request:
:return:
"""
if request.method == 'GET':
url = request.GET.get('url')
url = unquote(base64.b64decode(url).decode('utf-8'))
js = request.GET.get('js', 0)
script = request.GET.get('script')
response = requests.get(url, timeout=5)
response.encoding = response.apparent_encoding
html = process_html(response.text)
return HttpResponse(html)
def render_html(request):
if request.method == 'POST':
data = json.loads(request.body)
print(data)
print(type(data))
url = data.get('url')
js = data.get('js')
if not js:
try:
response = requests.get(url, timeout=5)
response.encoding = response.apparent_encoding
html = process_html(response.text)
return JsonResponse({'result': '1', 'data': html})
except:
return JsonResponse({'result': '0'}, status=500)
def render_html(request):
"""
render html with url
:param request:
:return:
"""
if request.method == 'GET':
url = request.GET.get('url')
url = unquote(base64.b64decode(url).decode('utf-8'))
print('Decoded', url)
js = request.GET.get('js', 0)
script = request.GET.get('script')
try:
response = requests.get(url, timeout=5)
response.encoding = response.apparent_encoding
html = process_html(response.text)
return HttpResponse(html)
except Exception as e:
return JsonResponse({'message': e.args}, status=500)