def do_add_site(request):
name = request.GET['name']
url = request.GET['start_urls']
rule = request.GET['extract_from']
mysql_conn = mysql_options.mysql_init()
result = mysql_options.add_site(mysql_conn, name, url, rule)
if "username" in request.session:
username = request.session['username']
flag = True
if flag:
myid = []
site_list = mysql_options.get_site_info(mysql_conn)
for list in site_list:
myid.append(list[0])
if result:
addMessage = "<script>alert('添加成功!')</script>"
return render(
request, 'site_management.html', {
'message': addMessage,
'flag': flag,
'site_list': site_list,
'myid': myid
})
else:
addMessage = "<script>alert('添加失败!')</script>"
return render(
request, 'site_management.html', {
'message': addMessage,
'flag': flag,
'site_list': site_list,
'myid': myid
})
else:
return render(request, 'spiders_management.html', {'flag': flag})
def do_run(request):
mysql_conn = mysql_options.mysql_init()
flag = False
if "username" in request.session:
username = request.session['username']
flag = True
if flag:
id = request.GET['run_id']
sit_data = mysql_options.get_site_info_by_id(mysql_conn, id)
try:
spiderserver.spider_server(sit_data, id)
except expression as e:
print "SpiderERROR:" + e
finally:
if sit_data[3] == "jingdongpinglun":
crawling_number = mongodb_options.crawling_jingdong_comments_succeed_num_by_id(
uestc, id)
else:
crawling_number = mongodb_options.crawling_succeed_num_by_id(
uestc, id)
print sit_data
name = sit_data[0]
keyword = sit_data[2]
overtime = time.time()
site_id = id
mysql_options.add_spiderinfo(mysql_conn, name, keyword, overtime,
crawling_number, site_id)
return render(request, 'index.html', {'flag': flag})
else:
return render(request, 'index.html', {'flag': flag})
def crawling_info(request):
crawling_list = []
mysql_conn = mysql_options.mysql_init()
flag = False
# web_info = []
if "username" in request.session:
username = request.session['username']
flag = True
if flag:
crawling_web_info = mysql_options.get_spiderinfo_info(mysql_conn)
for crawling_info in crawling_web_info:
over_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(crawling_info[3]))
crawling_info = (crawling_info[1], crawling_info[2],
crawling_info[4], crawling_info[5], over_time)
crawling_list.append(crawling_info)
# for info in crawling_web_info:
# url = info['url'].replace("?touping=", "")
# info['url'] = url
# web_info.append(info)
# return render(request, 'crawling_info.html', {'login_user': username, 'crawling_web_info': web_info, 'crawling_succeed_num':crawling_succeed_num, 'flag': flag})
return render(
request, 'crawling_info.html', {
'login_user': username,
'crawling_web_info': crawling_list,
'flag': flag
})
else:
return render(request, 'crawling_info.html', {'flag': flag})
def delete_siteall(request):
mysql_conn = mysql_options.mysql_init()
ids = request.GET['del_ids']
# print ids
myid = []
flag = False
if "username" in request.session:
username = request.session['username']
flag = True
if flag:
if ids:
ids = ids.split(',')
# print ids
result = mysql_options.delete_site_by_id(mysql_conn, ids)
site_list = mysql_options.get_site_info(mysql_conn)
for list in site_list:
myid.append(list[0])
if result:
mymessage = "<script>alert('删除成功!')</script>"
return render(
request, 'site_management.html', {
'message': mymessage,
'flag': flag,
'site_list': site_list,
'myid': myid
})
else:
mymessage = "<script>alert('删除失败!')</script>"
return render(
request, 'site_management.html', {
'message': mymessage,
'flag': flag,
'site_list': site_list,
'myid': myid
})
else:
site_list = mysql_options.get_site_info(mysql_conn)
for list in site_list:
myid.append(list[0])
mymessage = "<script>alert('请选择一个站点删除!')</script>"
return render(
request, 'site_management.html', {
'message': mymessage,
'flag': flag,
'site_list': site_list,
'myid': myid
})
else:
return render(request, 'spiders_management.html', {'flag': flag})
def site_management(request):
mysql_conn = mysql_options.mysql_init()
flag = False
myid = []
if "username" in request.session:
username = request.session['username']
flag = True
if flag:
site_list = mysql_options.get_site_info(mysql_conn)
for list in site_list:
myid.append(list[0])
return render(
request, 'site_management.html', {
'login_user': username,
'flag': flag,
'site_list': site_list,
'myid': myid
})
# return render(request, 'site_management.html', {'login_user': username, 'flag': flag})
else:
return render(request, 'site_management.html', {'flag': flag})
def do_update(request):
mysql_conn = mysql_options.mysql_init()
myid = request.GET['site_id']
name = request.GET['name']
url = request.GET['start_urls']
rule = request.GET['extract_from']
# print myid,name,url,rule
result = mysql_options.update_site_info(mysql_conn, myid, name, url, rule)
print result
flag = False
if "username" in request.session:
username = request.session['username']
flag = True
if flag:
myid = []
site_list = mysql_options.get_site_info(mysql_conn)
for list in site_list:
myid.append(list[0])
if result:
updatemessage = "<script>alert('修改成功!')</script>"
return render(
request, 'site_management.html', {
'message': updatemessage,
'flag': flag,
'site_list': site_list,
'myid': myid
})
else:
updatemessage = "<script>alert('修改失败!')</script>"
return render(
request, 'site_management.html', {
'message': updatemessage,
'flag': flag,
'site_list': site_list,
'myid': myid
})
else:
return render(request, 'spiders_management.html', {'flag': flag})