def parse_effect_recipe_list(self, response): log.msg('[PARSE][recipe_list] url=%s' %(response.url), log.INFO) category = response.request._msj_category for item in self.recipe_list_parser.parse(response): if isinstance(item, PageItem): url = item['url'] headers = {'X-Forwarded-For': random_ip()} request = Request(url, callback=self.parse_effect_recipe_list, headers=headers) request._msj_category = category yield request log.msg('[ENQUEUE][recipes_list] category=%s url=%s' % (category, url), log.INFO) continue yield CategoryRecipePairItem(category=category, recipe=item['name']) url = item['url'] name = item['name'] headers = {'X-Forwarded-For': random_ip()} request = Request(url, callback=self.parse_food_recipe, headers=headers) request._msj_category = category yield request log.msg('[PARSE][food_recipe] category=%s url=%s' % (category, url ), log.INFO)
def parse_material_list(self, response): log.msg('[PARSE][material list] url=%s' % (response.url,), log.INFO) if response.url == self.food_material_category_list_url: log.msg('[PARSE][ignore] url=%s' % (response.url,), log.WARNING) return for item in self.material_list_parser.parse(response): if isinstance(item, PageItem): url = item['url'] category = response.request._msj_category headers = {'X-Forwarded-For': random_ip()} request = Request(url, callback=self.parse_material_list, headers=headers) request._msj_category = category yield request #self.enqueue_request(request) log.msg('[ENQUEUE][material_list] category=%s url=%s' % (category, url), log.INFO) continue if isinstance(item, MaterialCategoryEntryItem): url = item['url'] category = item['category'] headers = {'X-Forwarded-For': random_ip()} request = Request(url, callback=self.parse_material_list, headers=headers) request._msj_category = category yield request log.msg('[ENQUEUE][material_list] category=%s url=%s' % (category, url), log.INFO) continue url = item['url'] name = item['name'] category = response.request._msj_category yield CategoryMaterialPairItem(category=category, material=name) log.msg('[PIPE][category_material_pair] category=%s material=%s' % (category, name), log.INFO) headers = {'X-Forwarded-For': random_ip()} request = Request(url, callback=self.parse_food_material, headers=headers) request._msj_category = category yield request #self.enqueue_request(request) log.msg('[ENQUEUE][food_material] category=%s url=%s' % (category, url), log.INFO)
def parse_material_category_list(self, response): log.msg('[PARSE][category_list] url=%s' % (response.url,), log.INFO) for item in self.material_category_list_parser.parse(response): url = item['url'] category = item['category'] headers = {'X-Forwarded-For': random_ip()} request = Request(url, callback=self.parse_material_list, headers=headers) request._msj_category = category yield request #self.enqueue_request(request) log.msg('[ENQUEUE][material_list] category=%s url=%s' % (category, url), log.INFO)
def parse_recipe_category_list(self, response): log.msg('[PARSE][recipe_category_list url=%s' % (response.url,),log.INFO) for item in self.recipe_category_list_parser.parse(response): url = item['url'] classification = item['classification'] category = item['category'] yield RecipeClassificationCategoryPairItem( classification=classification, category=category) callback = self.parse_recipe_list if getattr(response, '_msj_is_effect', False): callback = self.parse_effect_detail_and_recipe_list headers = {'X-Farwarded-For': random_ip()} request = Request(url, callback=callback, headers=headers) request._msj_category = category yield request log.msg('[ENQUEUE][material_list] category=%s url=%s' % (category, url,), log.INFO)