def parse_play_device_distribution_detail(self, response):
device_distribution_item = DeviceDistributionItem()
# 播放设备分布key_name
key_name = "play_device_distribution"
device_distribution_item['key_name'] = key_name
jsondata = json.loads(response.text)
code = jsondata['code']
if code == 'A00000':
# 剧集名称
name = jsondata['names'][0]
device_distribution_item['name'] = name
# 伪造一个唯一key，插入mongodb之前先做删除，再做插入
iqiyipipline = IqiyiPipeline()
iqiyipipline.remove_item(device_distribution_item, key_name, name)
# 剧集aid
device_distribution_item['aid'] = jsondata['ids'][0]
# 具体数据
data = jsondata['data'][0]
for device_distribution in data:
dev_names = device_distribution
dev_name = dev_names['name']
value = dev_names['value']
if dev_name == '移动端':
print("移动端：" + str(value))
device_distribution_item[
'mobile_terminal_distribution'] = value
else:
print("PC端：" + str(value))
device_distribution_item['pc_device_distribution'] = value
yield device_distribution_item
def parse_view_of_public_opinion_detail(self, response):
view_of_public_opinion = ViewOfPublicOpinionItem()
# key_name
key_name = "sentiments"
view_of_public_opinion['key_name'] = key_name
jsondata = json.loads(response.text)
code = jsondata['code']
if code == 'A00000':
aid = jsondata['data'][0]['albumID']
view_of_public_opinion['aid'] = aid
str_aid = str(aid)
# 伪造一个唯一key，插入mongodb之前先做删除，再做插入
iqiyipipline = IqiyiPipeline()
iqiyipipline.remove_aid_item(view_of_public_opinion, key_name,
str_aid)
view_of_public_opinion['data'] = jsondata['data'][0]['sentiments']
yield view_of_public_opinion
def parse_star_look_detail(self, response):
star_look_item = StarLookItem()
# key_name
key_name = "star_look"
star_look_item['key_name'] = key_name
jsondata = json.loads(response.text)
code = jsondata['code']
if code == 'A00000':
# 剧集名称
name = jsondata['names'][0]
star_look_item['name'] = name
# 伪造一个唯一key，插入mongodb之前先做删除，再做插入
iqiyipipline = IqiyiPipeline()
iqiyipipline.remove_item(star_look_item, key_name, name)
# 剧集aid
star_look_item['aid'] = jsondata['ids'][0]
# 具体数据
star_look_item['data'] = jsondata['data'][0]
yield star_look_item
def parse_city_play_data_detail(self, response):
city_play_data_item = CityPlayDataItem()
# 播放设备分布key_name
key_name = "city_play_data"
city_play_data_item['key_name'] = key_name
jsondata = json.loads(response.text)
code = jsondata['code']
if code == 'A00000':
# 剧集名称
name = jsondata['names'][0]
city_play_data_item['name'] = name
# 伪造一个唯一key，插入mongodb之前先做删除，再做插入
iqiyipipline = IqiyiPipeline()
iqiyipipline.remove_item(city_play_data_item, key_name, name)
# 剧集aid
city_play_data_item['aid'] = jsondata['ids'][0]
# 具体数据
city_play_data_item['data'] = jsondata['data'][0]
yield city_play_data_item
def parse_video_play_trend_all_days_detail(self, response):
play_trend_all_days_item = PlayTrendAllDaysItem()
# key_name
key_name = "play_trend_all_days"
play_trend_all_days_item['key_name'] = key_name
jsondata = json.loads(response.text)
code = jsondata['code']
if code == 'A00000':
# 剧集名称
name = jsondata['names'][0]
play_trend_all_days_item['name'] = name
# 伪造一个唯一key，插入mongodb之前先做删除，再做插入
iqiyipipline = IqiyiPipeline()
iqiyipipline.remove_item(play_trend_all_days_item, key_name, name)
# 剧集aid
play_trend_all_days_item['aid'] = jsondata['ids'][0]
# 具体数据
# 播放量
play_trend_all_days_item['data'] = jsondata['details'][0]['data']
# 播放日期
play_trend_all_days_item['playtime'] = jsondata['playtime']
yield play_trend_all_days_item
def parse_audience_portrait_detail(self, response):
jsondata = json.loads(response.text)
code = jsondata['code']
if code == 'A00000':
# 剧集名称
name = jsondata['names'][0]
# 剧集aid
aid = jsondata['ids'][0]
str_aid = str(aid)
iqiyipipline = IqiyiPipeline()
# ----------------受众性别 start------------------------
audience_sex = AudienceSexItem()
key_name = "audience_sex"
audience_sex['key_name'] = key_name
# 伪造一个唯一key，插入mongodb之前先做删除，再做插入
iqiyipipline.remove_item(audience_sex, key_name, name)
audience_sex['name'] = name
audience_sex['aid'] = aid
# 具体数据
audience_sex['sex_data_name'] = jsondata['data']['genderLabels']
audience_sex['sex_data_value'] = jsondata['data']['details'][0][
str_aid]['gender']
yield audience_sex
# ----------------受众年龄 start------------------------
audience_age = AudienceAgeItem()
key_name = "audience_age"
audience_age['key_name'] = key_name
# 伪造一个唯一key，插入mongodb之前先做删除，再做插入
iqiyipipline.remove_item(audience_age, key_name, name)
audience_age['name'] = name
audience_age['aid'] = aid
# 具体数据
audience_age['age_data_name'] = jsondata['data']['ageLabels']
audience_age['age_data_value'] = jsondata['data']['details'][0][
str_aid]['age']
yield audience_age
# ----------------受众星座 start------------------------
audience_constellation = AudienceConstellationItem()
key_name = "audience_constellation"
audience_constellation['key_name'] = key_name
# 伪造一个唯一key，插入mongodb之前先做删除，再做插入
iqiyipipline.remove_item(audience_constellation, key_name, name)
audience_constellation['name'] = name
audience_constellation['aid'] = aid
# 具体数据
audience_constellation['constellation_data_name'] = jsondata[
'data']['constellationLabels']
audience_constellation['constellation_data_value'] = jsondata[
'data']['details'][0][str_aid]['constellation']
yield audience_constellation
# ----------------受众学历 start------------------------
audience_education = AudienceEducationItem()
key_name = "audience_education"
audience_education['key_name'] = key_name
# 伪造一个唯一key，插入mongodb之前先做删除，再做插入
iqiyipipline.remove_item(audience_education, key_name, name)
audience_education['name'] = name
audience_education['aid'] = aid
# 具体数据
audience_education['education_data_name'] = jsondata['data'][
'educationLabels']
audience_education['education_data_value'] = jsondata['data'][
'details'][0][str_aid]['education']
yield audience_education
# ----------------受众兴趣 start------------------------
audience_interest = AudienceInterestItem()
key_name = "audience_interest"
audience_interest['key_name'] = key_name
# 伪造一个唯一key，插入mongodb之前先做删除，再做插入
iqiyipipline.remove_item(audience_interest, key_name, name)
audience_interest['name'] = name
audience_interest['aid'] = aid
# 具体数据
audience_interest['interest_data_name'] = jsondata['data'][
'interestLabels']
audience_interest['interest_data_value'] = jsondata['data'][
'details'][0][str_aid]['interest']
yield audience_interest