def parse_play_device_distribution_detail(self, response): device_distribution_item = DeviceDistributionItem() # 播放设备分布key_name key_name = "play_device_distribution" device_distribution_item['key_name'] = key_name jsondata = json.loads(response.text) code = jsondata['code'] if code == 'A00000': # 剧集名称 name = jsondata['names'][0] device_distribution_item['name'] = name # 伪造一个唯一key,插入mongodb之前先做删除,再做插入 iqiyipipline = IqiyiPipeline() iqiyipipline.remove_item(device_distribution_item, key_name, name) # 剧集aid device_distribution_item['aid'] = jsondata['ids'][0] # 具体数据 data = jsondata['data'][0] for device_distribution in data: dev_names = device_distribution dev_name = dev_names['name'] value = dev_names['value'] if dev_name == '移动端': print("移动端:" + str(value)) device_distribution_item[ 'mobile_terminal_distribution'] = value else: print("PC端:" + str(value)) device_distribution_item['pc_device_distribution'] = value yield device_distribution_item
def parse_view_of_public_opinion_detail(self, response): view_of_public_opinion = ViewOfPublicOpinionItem() # key_name key_name = "sentiments" view_of_public_opinion['key_name'] = key_name jsondata = json.loads(response.text) code = jsondata['code'] if code == 'A00000': aid = jsondata['data'][0]['albumID'] view_of_public_opinion['aid'] = aid str_aid = str(aid) # 伪造一个唯一key,插入mongodb之前先做删除,再做插入 iqiyipipline = IqiyiPipeline() iqiyipipline.remove_aid_item(view_of_public_opinion, key_name, str_aid) view_of_public_opinion['data'] = jsondata['data'][0]['sentiments'] yield view_of_public_opinion
def parse_star_look_detail(self, response): star_look_item = StarLookItem() # key_name key_name = "star_look" star_look_item['key_name'] = key_name jsondata = json.loads(response.text) code = jsondata['code'] if code == 'A00000': # 剧集名称 name = jsondata['names'][0] star_look_item['name'] = name # 伪造一个唯一key,插入mongodb之前先做删除,再做插入 iqiyipipline = IqiyiPipeline() iqiyipipline.remove_item(star_look_item, key_name, name) # 剧集aid star_look_item['aid'] = jsondata['ids'][0] # 具体数据 star_look_item['data'] = jsondata['data'][0] yield star_look_item
def parse_city_play_data_detail(self, response): city_play_data_item = CityPlayDataItem() # 播放设备分布key_name key_name = "city_play_data" city_play_data_item['key_name'] = key_name jsondata = json.loads(response.text) code = jsondata['code'] if code == 'A00000': # 剧集名称 name = jsondata['names'][0] city_play_data_item['name'] = name # 伪造一个唯一key,插入mongodb之前先做删除,再做插入 iqiyipipline = IqiyiPipeline() iqiyipipline.remove_item(city_play_data_item, key_name, name) # 剧集aid city_play_data_item['aid'] = jsondata['ids'][0] # 具体数据 city_play_data_item['data'] = jsondata['data'][0] yield city_play_data_item
def parse_video_play_trend_all_days_detail(self, response): play_trend_all_days_item = PlayTrendAllDaysItem() # key_name key_name = "play_trend_all_days" play_trend_all_days_item['key_name'] = key_name jsondata = json.loads(response.text) code = jsondata['code'] if code == 'A00000': # 剧集名称 name = jsondata['names'][0] play_trend_all_days_item['name'] = name # 伪造一个唯一key,插入mongodb之前先做删除,再做插入 iqiyipipline = IqiyiPipeline() iqiyipipline.remove_item(play_trend_all_days_item, key_name, name) # 剧集aid play_trend_all_days_item['aid'] = jsondata['ids'][0] # 具体数据 # 播放量 play_trend_all_days_item['data'] = jsondata['details'][0]['data'] # 播放日期 play_trend_all_days_item['playtime'] = jsondata['playtime'] yield play_trend_all_days_item
def parse_audience_portrait_detail(self, response): jsondata = json.loads(response.text) code = jsondata['code'] if code == 'A00000': # 剧集名称 name = jsondata['names'][0] # 剧集aid aid = jsondata['ids'][0] str_aid = str(aid) iqiyipipline = IqiyiPipeline() # ----------------受众性别 start------------------------ audience_sex = AudienceSexItem() key_name = "audience_sex" audience_sex['key_name'] = key_name # 伪造一个唯一key,插入mongodb之前先做删除,再做插入 iqiyipipline.remove_item(audience_sex, key_name, name) audience_sex['name'] = name audience_sex['aid'] = aid # 具体数据 audience_sex['sex_data_name'] = jsondata['data']['genderLabels'] audience_sex['sex_data_value'] = jsondata['data']['details'][0][ str_aid]['gender'] yield audience_sex # ----------------受众年龄 start------------------------ audience_age = AudienceAgeItem() key_name = "audience_age" audience_age['key_name'] = key_name # 伪造一个唯一key,插入mongodb之前先做删除,再做插入 iqiyipipline.remove_item(audience_age, key_name, name) audience_age['name'] = name audience_age['aid'] = aid # 具体数据 audience_age['age_data_name'] = jsondata['data']['ageLabels'] audience_age['age_data_value'] = jsondata['data']['details'][0][ str_aid]['age'] yield audience_age # ----------------受众星座 start------------------------ audience_constellation = AudienceConstellationItem() key_name = "audience_constellation" audience_constellation['key_name'] = key_name # 伪造一个唯一key,插入mongodb之前先做删除,再做插入 iqiyipipline.remove_item(audience_constellation, key_name, name) audience_constellation['name'] = name audience_constellation['aid'] = aid # 具体数据 audience_constellation['constellation_data_name'] = jsondata[ 'data']['constellationLabels'] audience_constellation['constellation_data_value'] = jsondata[ 'data']['details'][0][str_aid]['constellation'] yield audience_constellation # ----------------受众学历 start------------------------ audience_education = AudienceEducationItem() key_name = "audience_education" audience_education['key_name'] = key_name # 伪造一个唯一key,插入mongodb之前先做删除,再做插入 iqiyipipline.remove_item(audience_education, key_name, name) audience_education['name'] = name audience_education['aid'] = aid # 具体数据 audience_education['education_data_name'] = jsondata['data'][ 'educationLabels'] audience_education['education_data_value'] = jsondata['data'][ 'details'][0][str_aid]['education'] yield audience_education # ----------------受众兴趣 start------------------------ audience_interest = AudienceInterestItem() key_name = "audience_interest" audience_interest['key_name'] = key_name # 伪造一个唯一key,插入mongodb之前先做删除,再做插入 iqiyipipline.remove_item(audience_interest, key_name, name) audience_interest['name'] = name audience_interest['aid'] = aid # 具体数据 audience_interest['interest_data_name'] = jsondata['data'][ 'interestLabels'] audience_interest['interest_data_value'] = jsondata['data'][ 'details'][0][str_aid]['interest'] yield audience_interest