def random_proxy():
try:
redis = RedisDB()
ip_pools = redis.sget(table=redis_key, count=1)
proxy = random.choice(ip_pools)
proxies = {
"http": proxy,
"https": proxy,
}
except Exception as e:
print(e)
proxies = {}
return proxies
class SyncArtice(threading.Thread):
def __init__(self):
super(SyncArtice, self).__init__()
self._es = ES()
self._redis = RedisDB()
self._sync_count = 0
def run(self):
is_show_tip = False
while True:
try:
datas = self.get_data_from_redis(SYNC_STEP)
if not datas:
if not is_show_tip:
print('\n{time} 无数据 休眠... '.format(
time=tools.get_current_date()))
is_show_tip = True
elif self.add_data_to_es(datas):
is_show_tip = False
self._sync_count += len(datas)
tools.print_one_line('已同步 %d 条数据' % self._sync_count)
tools.delay_time(1)
except Exception as e:
log.error(e)
def get_data_from_redis(self, count):
datas = self._redis.sget('news:news_article', count=count)
return_datas = []
for data in datas:
data = eval(data)
release_time = data.get('release_time')
if release_time and len(release_time) == 19:
return_datas.append(data)
return return_datas
def add_data_to_es(self, datas):
return self._es.add_batch(datas,
primary_key='uuid',
table='news_article')