def qzone_friend(request): #用于判断前端页面显示活跃状态 friend_thread = True #获取本机IP IP = get_ip() thread_list = Thread_qq_friend.objects.filter(thread_ip=IP) for thread in thread_list: c = ThreadControl() try: #查看是否处于活跃状态 status = c.is_alive(thread.thread_name) if status: #设置状态为1 thread.thread_status = 1 thread.save() else: #设置状态为0 thread.thread_status = 0 thread.save() except: thread.thread_status = 0 thread.save() return render_to_response("qzone_friend.html", { "thread_list": thread_list, "friend_thread": friend_thread })
def index(request): thread1_status = False msg_active = True IP = get_ip() thread_list = ThreadMsg.objects.filter(thread_ip=IP) for thread in thread_list: c = ThreadControl() try: #查看是否处于活跃状态 status = c.is_alive(thread.thread_name) if status: #设置状态为1 thread.thread_status = 1 thread.save() else: #设置状态为0 thread.thread_status = 0 thread.save() except: thread.thread_status = 0 thread.save() return render_to_response( 'index.html', { 'thread1_status': thread1_status, "msg_active": msg_active, "thread_list": thread_list })
def auditor(request): thread1_status = False auditor_thread = True IP = get_ip() thread_list = Threadauditor.objects.filter(thread_ip=IP) for thread in thread_list: c = ThreadControl() try: #查看是否处于活跃状态 status = c.is_alive(thread.thread_name) if status: #设置状态为1 thread.thread_status = 1 thread.save() else: #设置状态为0 thread.thread_status = 0 thread.save() except: thread.thread_status = 0 thread.save() return render_to_response("auditor.html", { "thread_list": thread_list, "auditor_thread": auditor_thread })
def qzone_info(request): thread_status = False info_active = True IP = get_ip() thread_list = ThreadQzoneInfo.objects.filter(thread_ip=IP) for thread in thread_list: c = ThreadControl() try: #查看是否处于活跃状态 status = c.is_alive(thread.thread_name) if status: #设置状态为1 thread.thread_status = 1 thread.save() else: #设置状态为0 thread.thread_status = 0 thread.save() except: thread.thread_status = 0 thread.save() return render_to_response('qzone_info.html', { "info_active": info_active, "thread_list": thread_list })
def loaddata(c_thread, thread_num, interval): log_name_title = "tencent_wb_msg_" ip = get_ip() base_date = time.strftime("%Y%m%d", time.localtime()) log = log_setting(log_name_title + base_date + ".log") log.info("run......") driver = qq_login() time.sleep(3) if driver == None: log.info("phantomjs error!quit") return 0 else: pass #出队 conn_redis = redis_connect() conn_mongo = connect_mongodb() if conn_redis == 0 or conn_mongo == 0: log.info("redis or mongodb connect error") else: log.info("connect redis ok") log.info("connect mongodb ok") while not c_thread.thread_stop: current_date = time.strftime("%Y%m%d", time.localtime()) if current_date == base_date: pass else: base_date = current_date log = log_setting(log_name_title + base_date + ".log") log.info('Thread:(%s)' % (thread_num)) url = pop_redis_list(conn_redis) #判断队列是否为空 if url == None: log.info("msg queue is NULL") break else: #获取详细信息 msg = get_msg(driver, url, log) # print "load to mongodb" try: load_mongodb(conn_mongo, url, msg) except: rtx('ip', ip + "机器mongodb失败") log.info('ip' + ip + "机器mongodb失败") log.info("mongodb error") break # rtx('IP','正常停止') log.info(thread_num + "quit phantomjs") driver.quit() #rtx提醒 rtx('ip', ip + "机器" + thread_num + "停止运行") log.info('ip' + ip + "机器" + thread_num + "停止运行") #数据库状态更新,根据线程名称 log.info("更新数据库线程状态") thread = ThreadMsg.objects.get(thread_name=thread_num) thread.thread_status = 0 thread.save()
def control_thread(request): th_name = request.POST['id'] control = request.POST['control'] print "thread_name is ", th_name #显示活跃状态 msg_active = True thread = ThreadMsg.objects.get(thread_name=th_name) if control == 'start': rtx('ip', '进程' + str(th_name) + ' 开始采集标签信息') #状态信息 # thread1_status = True c = ThreadControl() # status = 1 #出现错误,则线程不存在,因此启动线程 try: status = c.is_alive(th_name) print "thread is alive? ", status if status: print "thread is alive,caonot start twice!" else: print "start ..........thread1" c.start(th_name, 1) except: print "thread is not alive start!!!" c.start(th_name, 1) thread.thread_status = 1 thread.save() if control == 'stop': # thread1_status = False # status = 0 rtx('ip', '进程' + str(th_name) + ' 采集标签信息即将停止') c = ThreadControl() try: c.stop(th_name) thread.thread_status = 0 thread.save() except: print "not thread alive" IP = get_ip() thread_list = ThreadMsg.objects.filter(thread_ip=IP) return render_to_response( 'index.html', { "thread_name": th_name, "control": control, "thread_list": thread_list, "msg_active": msg_active })
def loaddata(c_thread, thread_num, interval): log_name_title = str(thread_num) + "_tencent_qzone_info_" ip = get_ip() base_date = time.strftime("%Y%m%d", time.localtime()) log = log_setting(log_name_title + base_date + ".log") log.info(thread_num + "run......") driver = qzone_login() time.sleep(3) if driver == None: log.info("phantomjs error!quit") return 0 else: pass #出队 conn_redis = redis_connect() conn_mongo = connect_mongodb() # print "conn_redis",conn_redis # print "conn_mongo",conn_mongo #定义pop的redis名字 redis_list_pop_name = "tencent_qzone_qq_info" redis_list_push_qzone_forbid_name = "tencent_qzone_forbid_qq" if conn_redis == 0 or conn_mongo == 0: log.info("redis or mongodb connect error") else: log.info("connect redis ok") log.info("connect mongodb ok") ip = get_ip() while not c_thread.thread_stop: current_date = time.strftime("%Y%m%d", time.localtime()) if current_date == base_date: pass else: base_date = current_date log = log_setting(log_name_title + base_date + ".log") print 'Thread:(%s) Time:%s\n' % (thread_num, time.ctime()) # log = log_setting() #pop_redis_list(redis_conn,redis_list_name) qq = pop_redis_list(conn_redis, redis_list_pop_name) log.info('Thread:(%s) QQ:%s' % (thread_num, qq)) #判断队列是否为空 if qq == None: log.info("queue is NULL") break else: #获取详细信息 url = "http://user.qzone.qq.com/" + str(qq) + "/profile" info_list = get_info(driver, url, log) # print info_list # msg = get_msg(driver,url) if info_list == 0: #qq放入redis消息队列 push_redis_list_tmp(conn_redis, redis_list_push_qzone_forbid_name, qq) log.info(qq + "请求失败,入队禁止访问消息队列") driver = qzone_login() pass elif info_list == 1: #qq放入redis消息队列 push_redis_list_tmp(conn_redis, redis_list_push_qzone_forbid_name, qq) log.info(qq + "入队禁止访问消息队列") pass else: #存入mongodb log.info("load to mongodb") try: load_mongodb_qzone_info(conn_mongo, qq, info_list) except: rtx('ip', ip + "机器mongodb失败") log.info('ip' + ip + "机器mongodb失败") log.info("mongodb error") break # rtx('IP','正常停止') log.info(thread_num + "quit phantomjs") driver.quit() #rtx提醒 rtx('ip', ip + "机器" + thread_num + "停止运行") log.info('ip' + ip + "机器" + thread_num + "停止运行") #数据库状态更新,根据线程名称 log.info("更新数据库线程状态") thread = ThreadQzoneInfo.objects.get(thread_name=thread_num) thread.thread_status = 0 thread.save()
def loaddata(c_thread, thread_num, interval): print "run......" driver = qzone_login() time.sleep(3) if driver == None: "phantomjs error!quit" return 0 else: pass #连接redis conn_redis = redis_connect() redis_list_name_pop = "tencent_qzone_qq_test" redis_list_name_push = "tencent_qzone_qq_tmp_test" print "conn_redis", conn_redis if conn_redis == None: print "redis connect error" else: while not c_thread.thread_stop: print 'qzone_qq_friend Thread:(%s) Time:%s\n' % (thread_num, time.ctime()) qq = pop_redis_list(conn_redis, redis_list_name_pop) if qq == None: print "queue is NULL" break else: url = "http://user.qzone.qq.com/" + qq + "/mood" print "url", url driver.get(url) try: #等待页面加载完成 frame_element = WebDriverWait(driver, 3).until( EC.presence_of_element_located( (By.ID, "app_canvas_frame"))) print "find frame id" driver.switch_to.frame('app_canvas_frame') try: #等待切换后的元素存在 class_name_element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.CLASS_NAME, "comments_content"))) print "find conment" html = driver.page_source soup = BeautifulSoup(html) print "======" my_set = set() for i in soup.find_all(class_='comments_content'): friend_qq = str(i.find('a')['href'])[25:-6] print friend_qq if friend_qq != qq: my_set.add(friend_qq) print my_set friend_qq_list = list(my_set) print friend_qq_list except: print "not found conment" friend_qq_list = ['0'] except: print "没有权限访问" friend_qq_list = ['-1'] print friend_qq_list #############################################存入mysql print "insert mysql" #获取qq和friend_qq组成的元组,多个 tmp_tuple = get_tuple(qq, friend_qq_list) #插入mysql数据库 print "insert into table " mysql_conn = mysql_connect_local_qq() insert_mysql_qq(mysql_conn, tmp_tuple) #关闭数据库 mysql_conn.close() ############################################存入临时的redis print "put mid redis" push_redis_list_tmp(conn_redis, redis_list_name_push, qq) print "put auditor mid redis" for friend_qq in friend_qq_list: push_redis_list_tmp(conn_redis, redis_list_name_push, friend_qq) print thread_num, "quit phantomjs" driver.quit() #rtx提醒 ip = get_ip() rtx('ip', ip + "机器" + thread_num + "停止运行") #数据库状态更新,根据线程名称 print "更新数据库线程状态" thread = Thread_qq_friend.objects.get(thread_name=thread_num) thread.thread_status = 0 thread.save()
def loaddata(c_thread, thread_num, interval): log_name_title = "tencent_wb_auditor_" base_date = time.strftime("%Y%m%d", time.localtime()) log = log_setting(log_name_title + base_date + ".log") log.info(thread_num + "run......") driver = qq_login() time.sleep(3) if driver == None: log.info("phantomjs error!quit") return 0 else: pass #连接redis conn_redis = redis_connect() #mysql连接 异常返回None #mysql_conn = mysql_connect() # conn_mongo = connect_mongodb() # print "conn_mongo",conn_mongo if conn_redis == None: log.info("redis connect error") else: log.info("connect redis ok") ip = get_ip() while not c_thread.thread_stop: current_date = time.strftime("%Y%m%d", time.localtime()) if current_date == base_date: pass else: base_date = current_date log = log_setting(log_name_title + base_date + ".log") # log.info('Thread:(%s) Time:%s'%(thread_num,time.ctime())) log.info('Thread:(%s)' % (thread_num)) mid = pop_redis_list(conn_redis) if mid == None: log.info("queue is NULL") break else: url = "http://t.qq.com/" + str(mid) log.info("url is: " + url) time.sleep(3) #根据用户的主页url获取收听的所有页面 auditor_page_url_list = get_auditor_page_url_via_url( driver, url) if auditor_page_url_list == None: log.info("page is not personal,login again") driver.quit() driver = qq_login() if driver == None: break else: pass #根据收听的所有页面获取收听者的主页url ################根据已知mid获取所有收听的mid else: mid_list = get_auditor_main_url(driver, auditor_page_url_list) if mid_list == None: continue else: #############################################存入mysql try: log.info("insert mysql") #获取mid和auditor_mid组成的元组,多个 tmp_tuple = get_tuple(mid, mid_list) #插入mysql数据库 print "insert into table " mysql_conn = mysql_connect() insert_mysql(mysql_conn, tmp_tuple) #关闭数据库 mysql_conn.close() except: rtx('ip', ip + "机器mysql出错") log.info('ip' + ip + "机器mysql出错") log.info("insert mysql error") ############################################存入临时的redis try: log.info("put mid redis") push_redis_list_tmp(conn_redis, mid) log.info("put auditor mid redis") for auditor_mid in mid_list: push_redis_list_tmp(conn_redis, auditor_mid) except: rtx('ip', ip + "机器redis出错") log.info('ip' + ip + "机器redis出错") log.info("insert redis error") log.info(thread_num + "quit phantomjs") driver.quit() #rtx提醒 rtx('ip', ip + "机器" + thread_num + "停止运行") log.info('ip' + ip + "机器" + thread_num + "停止运行") #数据库状态更新,根据线程名称 log.info("更新数据库线程状态") thread = Threadauditor.objects.get(thread_name=thread_num) thread.thread_status = 0 thread.save()
def loaddata(c_thread, thread_num, interval): log_name_title = "tencent_wb_auditor_" base_date = time.strftime("%Y%m%d", time.localtime()) log = log_setting(log_name_title + base_date + ".log") log.info(thread_num + "run......") driver = qzone_login() time.sleep(3) if driver == None: log.info("phantomjs error!quit") return 0 else: pass #连接redis conn_redis = redis_connect() redis_list_name_pop = "tencent_qzone_qq" redis_list_name_push = "tencent_qzone_qq_transfer" print "conn_redis", conn_redis if conn_redis == None: log.info("redis connect error") else: log.info("redis connect ok") ip = get_ip() while not c_thread.thread_stop: log.info('qzone_qq_friend Thread:(%s) Time:%s' % (thread_num, time.ctime())) qq = pop_redis_list(conn_redis, redis_list_name_pop) if qq == None: log.info("queue is NULL") break else: url = "http://user.qzone.qq.com/" + qq + "/mood" log.info("url" + url) driver.get(url) try: #等待页面加载完成 frame_element = WebDriverWait(driver, 3).until( EC.presence_of_element_located( (By.ID, "app_canvas_frame"))) log.info("find frame id") driver.switch_to.frame('app_canvas_frame') try: #等待切换后的元素存在 class_name_element = WebDriverWait(driver, 10).until( EC.presence_of_element_located( (By.CLASS_NAME, "comments_content"))) log.info("find conment") html = driver.page_source soup = BeautifulSoup(html) print "======" my_set = set() for i in soup.find_all(class_='comments_content'): friend_qq = str(i.find('a')['href'])[25:-6] print friend_qq if friend_qq != qq: my_set.add(friend_qq) print my_set friend_qq_list = list(my_set) print friend_qq_list except: log.info("not found conment") friend_qq_list = ['0'] except: log.info("没有权限访问") friend_qq_list = ['-1'] print friend_qq_list #############################################存入mysql try: log.info("insert mysql") #获取qq和friend_qq组成的元组,多个 tmp_tuple = get_tuple(qq, friend_qq_list) #插入mysql数据库 print "insert into table " mysql_conn = mysql_connect_qq() insert_mysql_qq(mysql_conn, tmp_tuple) #关闭数据库 mysql_conn.close() except: rtx('ip', ip + "机器QQ空间关系链采集mysql出错") log.info('ip' + ip + "机器QQ空间关系链采集mysql出错") ############################################存入临时的redis try: log.info("put mid redis") push_redis_list_tmp(conn_redis, redis_list_name_push, qq) log.info("put auditor mid redis") for friend_qq in friend_qq_list: push_redis_list_tmp(conn_redis, redis_list_name_push, friend_qq) except: rtx('ip', ip + "机器QQ空间关系链采集redis入队出错") log.info('ip' + ip + "机器QQ空间关系链采集redis入队出错") log.info(thread_num + "quit phantomjs") driver.quit() #rtx提醒 rtx('ip', ip + "机器" + thread_num + "停止运行") log.info('ip' + ip + "机器" + thread_num + "停止运行") #数据库状态更新,根据线程名称 log.info("更新数据库线程状态") thread = Thread_qq_friend.objects.get(thread_name=thread_num) thread.thread_status = 0 thread.save()