class GetLives():
def __init__(self):
self.get_raw_data = GetRawData()
self.redis_client = RedisClient()
self.lives_off_list = []
def get_rooms(self):
return self.redis_client.get_rooms()
def delete_rooms(self): #直播结束了，就将用户的状态改回0，删除room_id，并添加item_list记录
for each in self.lives_off_list:
room_id = each.split('_', 1)[0]
sec_user_id = each.split('_', 1)[1]
self.redis_client.add_live_users(sec_user_id, 0)
self.redis_client.delete_rooms(room_id)
self.redis_client.add_item_lists(each)
def get_lives(self, room_id):
try:
live_raw_data = self.get_raw_data.get_live(room_id)
except Exception as e:
logger.error('get_live出错-' + e.args[0] + '-room_id-' + room_id)
return None
try:
live_info = self.parse_lives(live_raw_data, room_id)
except Exception as e:
logger.error('parse_lives出错-' + e.args[0] + '-room_id-' + room_id)
return None
self.write_to_file(live_info)
if live_info['status'] == 4: #写入文件成功之后才删除
self.lives_off_list.append(live_info['room_id'] + '_' +
live_info['sec_user_id'])
def parse_lives(self, live_raw_data, room_id):
data = live_raw_data.get('data')
live_info = {}
live_info['room_id'] = room_id
live_info['start_time'] = data.get('create_time') #直播开始时间
live_info['like_count'] = data.get('like_count')
live_info['share_url'] = data.get('share_url')
live_info['title'] = data.get('title')
live_info['status'] = data.get('status') #2为正在直播，4为直播结束
live_info['viewer_count'] = data.get('user_count') #实时观看人数
live_info['cover_url'] = data.get('cover').get('url_list')[0]
owner = data.get('owner')
live_info['avatar_url'] = owner.get('avatar_large').get('url_list')[0]
live_info['city'] = owner.get('city')
live_info['follower_count'] = owner.get('follow_info').get(
'follower_count')
live_info['gender'] = owner.get('gender')
live_info['short_id'] = owner.get('short_id') #主播短id
live_info['id'] = owner.get('id_str') #主播长id
live_info['nickname'] = owner.get('nickname')
live_info['signature'] = owner.get('signature')
live_info['short_id'] = owner.get('short_id')
live_info['sec_user_id'] = owner.get('sec_uid')
#live_info['mobile'] = owner.get('telephone') #主播手机
live_info['ticket_count'] = owner.get('ticket_count') #主播总音浪
live_info['create_time'] = str(int(time.time())) #记录时间
stats = data.get('stats')
live_info['fan_ticket'] = stats.get('fan_ticket') #本场收入音浪
live_info['follow_count'] = stats.get('follow_count') #本场关注
live_info['gift_count'] = stats.get('gift_uv_count') #本场获得礼物
live_info['total_viewer'] = stats.get('total_user') #总观看人数
return live_info
def write_to_file(self, live_info):
today = time.strftime('%Y-%m-%d', time.localtime())
today = today.replace('-', '')
with open(FILE_DIRECTORY + '/' + 'lives' + '_' + today + '.txt',
'a',
encoding='utf-8') as file:
file.write(json.dumps(live_info, ensure_ascii=False) + '\n')
def run(self):
all_room_ids = self.get_rooms()
logger.info('此前正在直播的直播间数量：' + str(len(all_room_ids)))
batch_size = 150 #本地200可以，云手机200不行
for batch_limit in range(0, len(all_room_ids), batch_size):
start = batch_limit
stop = min(batch_limit + batch_size, len(all_room_ids))
logger.info('待查看并获取的此前正在直播的直播间-' + str(start + 1) + '-' +
str(stop))
tasks = [
gevent.spawn(self.get_lives, room_id)
for room_id in all_room_ids[start:stop]
]
gevent.joinall(tasks)
logger.info('新发现已结束的直播数量-' + str(len(self.lives_off_list)))
self.delete_rooms()
self.lives_off_list.clear()
class CheckRooms():
def __init__(self):
self.get_raw_data = GetRawData()
self.redis_client = RedisClient()
self.lives_on_list = []
def get_rooms(self):
return self.redis_client.get_rooms(0, 0)
def change_room_status(self):
for each in self.lives_on_list:
self.redis_client.add_rooms(each, 1)
def check_room(self, room_id):
try:
room_raw_data = self.get_raw_data.get_live(room_id)
except Exception as e:
logger.error('get_live出错-' + e.args[0] + '-room_id-' + room_id)
return None
try:
owner = room_raw_data.get('data').get('owner')
follower_count = owner.get('follow_info').get('follower_count')
sec_user_id = owner.get('sec_uid')
except Exception as e:
logger.error('解析room_raw_data出错-' + e.args[0] + '-room_id-' +
room_id)
return None
if follower_count < 10000:
self.redis_client.delete_users(sec_user_id)
self.redis_client.delete_rooms(room_id)
else:
status = room_raw_data.get('data').get('status')
if status == 2:
try: #判断该场直播是否带货
item_list_raw_data = self.get_raw_data.get_item_list(
sec_user_id, room_id)
except Exception as e:
logger.error('get_item_list出错' + e.args[0] +
'-sec_user_id和room_id-' + sec_user_id + '-' +
room_id)
return None
if len(item_list_raw_data.get('promotions')) != 0:
self.lives_on_list.append(room_id)
def run(self):
all_room_ids = self.get_rooms()
logger.info('此前未在直播的直播间数量：' + str(len(all_room_ids)))
batch_size = 200
for batch_limit in range(0, len(all_room_ids), batch_size):
start = batch_limit
stop = min(batch_limit + batch_size, len(all_room_ids))
logger.info('待查看的此前未在直播的直播间-' + str(start + 1) + '-' + str(stop))
tasks = [
gevent.spawn(self.check_room, room_id)
for room_id in all_room_ids[start:stop]
]
gevent.joinall(tasks)
logger.info('新发现开始的直播数量-' + str(len(self.lives_on_list)))
self.change_room_status()
self.lives_on_list.clear()
