def do_add_site(request): name = request.GET['name'] url = request.GET['start_urls'] rule = request.GET['extract_from'] mysql_conn = mysql_options.mysql_init() result = mysql_options.add_site(mysql_conn, name, url, rule) if "username" in request.session: username = request.session['username'] flag = True if flag: myid = [] site_list = mysql_options.get_site_info(mysql_conn) for list in site_list: myid.append(list[0]) if result: addMessage = "<script>alert('添加成功!')</script>" return render( request, 'site_management.html', { 'message': addMessage, 'flag': flag, 'site_list': site_list, 'myid': myid }) else: addMessage = "<script>alert('添加失败!')</script>" return render( request, 'site_management.html', { 'message': addMessage, 'flag': flag, 'site_list': site_list, 'myid': myid }) else: return render(request, 'spiders_management.html', {'flag': flag})
def do_run(request): mysql_conn = mysql_options.mysql_init() flag = False if "username" in request.session: username = request.session['username'] flag = True if flag: id = request.GET['run_id'] sit_data = mysql_options.get_site_info_by_id(mysql_conn, id) try: spiderserver.spider_server(sit_data, id) except expression as e: print "SpiderERROR:" + e finally: if sit_data[3] == "jingdongpinglun": crawling_number = mongodb_options.crawling_jingdong_comments_succeed_num_by_id( uestc, id) else: crawling_number = mongodb_options.crawling_succeed_num_by_id( uestc, id) print sit_data name = sit_data[0] keyword = sit_data[2] overtime = time.time() site_id = id mysql_options.add_spiderinfo(mysql_conn, name, keyword, overtime, crawling_number, site_id) return render(request, 'index.html', {'flag': flag}) else: return render(request, 'index.html', {'flag': flag})
def crawling_info(request): crawling_list = [] mysql_conn = mysql_options.mysql_init() flag = False # web_info = [] if "username" in request.session: username = request.session['username'] flag = True if flag: crawling_web_info = mysql_options.get_spiderinfo_info(mysql_conn) for crawling_info in crawling_web_info: over_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(crawling_info[3])) crawling_info = (crawling_info[1], crawling_info[2], crawling_info[4], crawling_info[5], over_time) crawling_list.append(crawling_info) # for info in crawling_web_info: # url = info['url'].replace("?touping=", "") # info['url'] = url # web_info.append(info) # return render(request, 'crawling_info.html', {'login_user': username, 'crawling_web_info': web_info, 'crawling_succeed_num':crawling_succeed_num, 'flag': flag}) return render( request, 'crawling_info.html', { 'login_user': username, 'crawling_web_info': crawling_list, 'flag': flag }) else: return render(request, 'crawling_info.html', {'flag': flag})
def delete_siteall(request): mysql_conn = mysql_options.mysql_init() ids = request.GET['del_ids'] # print ids myid = [] flag = False if "username" in request.session: username = request.session['username'] flag = True if flag: if ids: ids = ids.split(',') # print ids result = mysql_options.delete_site_by_id(mysql_conn, ids) site_list = mysql_options.get_site_info(mysql_conn) for list in site_list: myid.append(list[0]) if result: mymessage = "<script>alert('删除成功!')</script>" return render( request, 'site_management.html', { 'message': mymessage, 'flag': flag, 'site_list': site_list, 'myid': myid }) else: mymessage = "<script>alert('删除失败!')</script>" return render( request, 'site_management.html', { 'message': mymessage, 'flag': flag, 'site_list': site_list, 'myid': myid }) else: site_list = mysql_options.get_site_info(mysql_conn) for list in site_list: myid.append(list[0]) mymessage = "<script>alert('请选择一个站点删除!')</script>" return render( request, 'site_management.html', { 'message': mymessage, 'flag': flag, 'site_list': site_list, 'myid': myid }) else: return render(request, 'spiders_management.html', {'flag': flag})
def site_management(request): mysql_conn = mysql_options.mysql_init() flag = False myid = [] if "username" in request.session: username = request.session['username'] flag = True if flag: site_list = mysql_options.get_site_info(mysql_conn) for list in site_list: myid.append(list[0]) return render( request, 'site_management.html', { 'login_user': username, 'flag': flag, 'site_list': site_list, 'myid': myid }) # return render(request, 'site_management.html', {'login_user': username, 'flag': flag}) else: return render(request, 'site_management.html', {'flag': flag})
def do_update(request): mysql_conn = mysql_options.mysql_init() myid = request.GET['site_id'] name = request.GET['name'] url = request.GET['start_urls'] rule = request.GET['extract_from'] # print myid,name,url,rule result = mysql_options.update_site_info(mysql_conn, myid, name, url, rule) print result flag = False if "username" in request.session: username = request.session['username'] flag = True if flag: myid = [] site_list = mysql_options.get_site_info(mysql_conn) for list in site_list: myid.append(list[0]) if result: updatemessage = "<script>alert('修改成功!')</script>" return render( request, 'site_management.html', { 'message': updatemessage, 'flag': flag, 'site_list': site_list, 'myid': myid }) else: updatemessage = "<script>alert('修改失败!')</script>" return render( request, 'site_management.html', { 'message': updatemessage, 'flag': flag, 'site_list': site_list, 'myid': myid }) else: return render(request, 'spiders_management.html', {'flag': flag})