def update_cookies(username,cookies): '''更新cookie到数据库''' db = MysqlStore() conn = db.get_connection() timestamp = int(time.time()) cursor = conn.cursor() cursor.execute("insert into cauc_login_cookie_info(USERNAME,SUHB,SUB,SUBP,SUE,SUS,SUP,timestamp) values (%s,%s,%s,%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE SUHB=%s,SUB=%s,SUBP=%s,SUE=%s,SUS=%s,SUP=%s,timestamp=%s",(md5(username),cookies['SUHB'],cookies['SUB'],cookies['SUBP'],cookies['SUE'],cookies['SUS'],cookies['SUP'],timestamp,cookies['SUHB'],cookies['SUB'],cookies['SUBP'],cookies['SUE'],cookies['SUS'],cookies['SUP'],timestamp)) conn.commit() db.close_connection(conn,cursor) logger.info('Update cookies into database...')
def allcookie_fetch(): '''获取所有cookie''' db = MysqlStore() conn = db.get_connection() cursor = conn.cursor(MySQLdb.cursors.DictCursor) nums = cursor.execute('SELECT SUHB,SUB,SUBP,SUE,SUS,SUP,USERNAME FROM cauc_login_cookie_info WHERE (unix_timestamp()-cast(timestamp as signed)) < (%s)',(EXPIRES,)) rows = cursor.fetchallDict() conn.commit() db.close_connection(conn,cursor) return nums,rows
def search_from_keywordDB(self, response): db = MysqlStore() main_url = "http://s.weibo.com/weibo/" getsearchpage = GetSearchpage() for round in range(1): #遍历数据库的轮数 conn = db.get_connection() #选取is_search位为0的关键词 sql1 = "select keyword from cauc_keyword_test_copy where is_search = 0 and is_delete = 0" cursor1 = db.select_operation(conn, sql1) #对is_search位为0的关键词进行爬取 for keyword in cursor1.fetchall(): keyword = keyword[0] logger.info("this is the unsearched keyword:%s", keyword) search_url = main_url + getsearchpage.get_searchurl(keyword) yield Request(url=search_url, cookies=random.choice(COOKIES), meta={ 'search_url': search_url, 'keyword': keyword }, callback=self.parse_total_page) #选取is_search位为1的关键词 sql2 = "select keyword from cauc_keyword_test_copy where is_search = 1 and is_delete = 0" cursor2 = db.select_operation(conn, sql2) #对is_search位为1的关键词进行爬取 for keyword in cursor2.fetchall(): keyword = keyword[0] logger.info("this is the searched keyword:%s", keyword) end_time = get_current_time() #start_time = get_time_by_interval(int(time.time()),3600) #爬取3600秒,即1小时前的内容 start_time = get_time_by_interval(int( time.time()), int(self.interval)) #爬取interval秒前的内容 search_url = main_url + getsearchpage.get_searchurl_time( keyword, start_time, end_time) yield Request(url=search_url, cookies=random.choice(COOKIES), meta={ 'search_url': search_url, 'keyword': keyword }, callback=self.parse_total_page) #更新is_search标志位为1 sql3 = "update cauc_keyword_test_copy set is_search = 1 where is_search = 0 and is_delete = 0" db.update_operation(conn, sql3) db.close_connection(conn)
def set_flag(): '''设置flag''' db = MysqlStore() conn = db.get_connection() cursor = conn.cursor() sql = "update cauc_parameters set param_value = 1 where param_key = 'flag'" n = cursor.execute(sql) conn.commit() db.close_connection(conn,cursor) if n: logger.info('Set flag success!') else: logger.error('Set flag failed,flag is already 1!')
def user_fetch(): '''获取账号信息''' db = MysqlStore() conn = db.get_connection() cursor = conn.cursor(MySQLdb.cursors.DictCursor) sql = 'SELECT id,username,password FROM cauc_login_account_info where is_delete = 0' nums = cursor.execute(sql) users = cursor.fetchallDict() conn.commit() db.close_connection(conn,cursor) if users: logger.info('User fetch success!') return users else: logger.error('There is no user in database!')
def start_getweibo_info(self,response): db = MysqlStore(); #取出没有爬取过的且is_delete=0的重点人员 GetWeibopage.data['page'] = 1; getweibopage = GetWeibopage() #for round in range(1): #遍历数据库的轮数 conn = db.get_connection() sql1 = "select user_id from cauc_warning_man a \ where a.is_search = 0 and a.is_delete = 0" cursor1 = db.select_operation(conn,sql1) for user_id in cursor1.fetchall(): user_id = user_id[0] logger.info("this is the unsearched user_id:%s",user_id) #获取需要爬取的总页面数 start_time = self.start_time;end_time = get_current_time('day') mainpage_url = "http://weibo.com/" + str(user_id) + "?is_ori=1&is_forward=1&is_text=1&is_pic=1&key_word=&start_time=" + start_time + "&end_time=" + end_time + "&is_search=1&is_searchadv=1&" GetWeibopage.data['uid'] = user_id; thirdload_url = mainpage_url + getweibopage.get_thirdloadurl() yield Request(url=thirdload_url,cookies=random.choice(COOKIES),meta={'mainpage_url':mainpage_url,'uid':user_id,'is_search':0},callback=self.parse_total_page) #取出已经爬取过is_search=1的且is_delete=0的预警人员 sql2 = "select user_id from cauc_warning_man a \ where a.is_search = 1 and a.is_delete = 0" cursor2 = db.select_operation(conn,sql2) for user_id in cursor2.fetchall(): user_id = user_id[0] logger.info("this is the searched user_id:%s",user_id) #start_time = get_time_by_interval(int(time.time()),86400,'day');end_time = get_current_time('day') #起始和结束间隔时间为1天(86400s),即过去一天的内容 start_time = get_time_by_interval(int(time.time()),int(self.interval),'day');end_time = get_current_time('day') #起始和结束间隔时间为x天(由interval代表的秒换算而来) mainpage_url = "http://weibo.com/" + str(user_id) + "?is_ori=1&is_forward=1&is_text=1&is_pic=1&key_word=&start_time=" + start_time + "&end_time=" + end_time + "&is_search=1&is_searchadv=1&" GetWeibopage.data['uid'] = user_id; thirdload_url = mainpage_url + getweibopage.get_thirdloadurl() yield Request(url=thirdload_url,cookies=random.choice(COOKIES),meta={'mainpage_url':mainpage_url,'uid':user_id,'is_search':1},callback=self.parse_total_page) #更新is_search标志位为1 sql3 = "update cauc_warning_man set is_search = 1 where is_search = 0 and is_delete = 0" db.update_operation(conn,sql3) db.close_connection(conn)
def search_from_keywordDB(self,response): db = MysqlStore();main_url = "http://s.weibo.com/weibo/" getsearchpage = GetSearchpage() for round in range(1): #遍历数据库的轮数 conn = db.get_connection() #选取is_search位为0的关键词 sql1 = "select keyword from cauc_keyword_test_copy where is_search = 0 and is_delete = 0" cursor1 = db.select_operation(conn,sql1) #对is_search位为0的关键词进行爬取 for keyword in cursor1.fetchall(): keyword = keyword[0] logger.info("this is the unsearched keyword:%s",keyword) search_url = main_url + getsearchpage.get_searchurl(keyword) yield Request(url=search_url,cookies=random.choice(COOKIES),meta={'search_url':search_url,'keyword':keyword},callback=self.parse_total_page) #选取is_search位为1的关键词 sql2 = "select keyword from cauc_keyword_test_copy where is_search = 1 and is_delete = 0" cursor2 = db.select_operation(conn,sql2) #对is_search位为1的关键词进行爬取 for keyword in cursor2.fetchall(): keyword = keyword[0] logger.info("this is the searched keyword:%s",keyword) end_time = get_current_time() #start_time = get_time_by_interval(int(time.time()),3600) #爬取3600秒,即1小时前的内容 start_time = get_time_by_interval(int(time.time()),int(self.interval)) #爬取interval秒前的内容 search_url = main_url + getsearchpage.get_searchurl_time(keyword,start_time,end_time) yield Request(url=search_url,cookies=random.choice(COOKIES),meta={'search_url':search_url,'keyword':keyword},callback=self.parse_total_page) #更新is_search标志位为1 sql3 = "update cauc_keyword_test_copy set is_search = 1 where is_search = 0 and is_delete = 0" db.update_operation(conn,sql3) db.close_connection(conn)