def parse_effect_recipe_list(self, response):
log.msg('[PARSE][recipe_list] url=%s' %(response.url), log.INFO)
category = response.request._msj_category
for item in self.recipe_list_parser.parse(response):
if isinstance(item, PageItem):
url = item['url']
headers = {'X-Forwarded-For': random_ip()}
request = Request(url, callback=self.parse_effect_recipe_list,
headers=headers)
request._msj_category = category
yield request
log.msg('[ENQUEUE][recipes_list] category=%s url=%s' %
(category, url), log.INFO)
continue
yield CategoryRecipePairItem(category=category, recipe=item['name'])
url = item['url']
name = item['name']
headers = {'X-Forwarded-For': random_ip()}
request = Request(url, callback=self.parse_food_recipe,
headers=headers)
request._msj_category = category
yield request
log.msg('[PARSE][food_recipe] category=%s url=%s' %
(category, url ), log.INFO)
def parse_material_list(self, response):
log.msg('[PARSE][material list] url=%s' % (response.url,), log.INFO)
if response.url == self.food_material_category_list_url:
log.msg('[PARSE][ignore] url=%s' % (response.url,), log.WARNING)
return
for item in self.material_list_parser.parse(response):
if isinstance(item, PageItem):
url = item['url']
category = response.request._msj_category
headers = {'X-Forwarded-For': random_ip()}
request = Request(url, callback=self.parse_material_list,
headers=headers)
request._msj_category = category
yield request
#self.enqueue_request(request)
log.msg('[ENQUEUE][material_list] category=%s url=%s' %
(category, url), log.INFO)
continue
if isinstance(item, MaterialCategoryEntryItem):
url = item['url']
category = item['category']
headers = {'X-Forwarded-For': random_ip()}
request = Request(url, callback=self.parse_material_list,
headers=headers)
request._msj_category = category
yield request
log.msg('[ENQUEUE][material_list] category=%s url=%s' %
(category, url), log.INFO)
continue
url = item['url']
name = item['name']
category = response.request._msj_category
yield CategoryMaterialPairItem(category=category, material=name)
log.msg('[PIPE][category_material_pair] category=%s material=%s' %
(category, name), log.INFO)
headers = {'X-Forwarded-For': random_ip()}
request = Request(url, callback=self.parse_food_material,
headers=headers)
request._msj_category = category
yield request
#self.enqueue_request(request)
log.msg('[ENQUEUE][food_material] category=%s url=%s' %
(category, url), log.INFO)
def parse_material_category_list(self, response):
log.msg('[PARSE][category_list] url=%s' % (response.url,), log.INFO)
for item in self.material_category_list_parser.parse(response):
url = item['url']
category = item['category']
headers = {'X-Forwarded-For': random_ip()}
request = Request(url, callback=self.parse_material_list,
headers=headers)
request._msj_category = category
yield request
#self.enqueue_request(request)
log.msg('[ENQUEUE][material_list] category=%s url=%s' %
(category, url), log.INFO)
def parse_recipe_category_list(self, response):
log.msg('[PARSE][recipe_category_list url=%s' % (response.url,),log.INFO)
for item in self.recipe_category_list_parser.parse(response):
url = item['url']
classification = item['classification']
category = item['category']
yield RecipeClassificationCategoryPairItem(
classification=classification, category=category)
callback = self.parse_recipe_list
if getattr(response, '_msj_is_effect', False):
callback = self.parse_effect_detail_and_recipe_list
headers = {'X-Farwarded-For': random_ip()}
request = Request(url, callback=callback, headers=headers)
request._msj_category = category
yield request
log.msg('[ENQUEUE][material_list] category=%s url=%s' %
(category, url,), log.INFO)
