def process_item(self, item, spider):
obj = ScrapyItem.objects.filter(deal_title=item['deal_title']).first()
if not obj:
scrapy_item = ScrapyItem()
scrapy_item.unique_id = self.unique_id
scrapy_item.web_source = item['web_source']
scrapy_item.deal_title = item['deal_title']
scrapy_item.deal_image_url = item['deal_image_url']
scrapy_item.deal_percentage = item['deal_percentage']
scrapy_item.save()
return item
def process_item(self, item, spider):
scrapy_item = ScrapyItem()
scrapy_item.unique_id = self.unique_id
scrapy_item.title = item['title']
scrapy_item.contents = item['contents']
scrapy_item.published_date = item['published_date']
scrapy_item.views = item['views']
scrapy_item.recommends = item['recommends']
scrapy_item.url = item['url']
scrapy_item.category = item['category']
scrapy_item.save()
return item
def close_spider(self, spider):
# And here we are saving our crawled data with django models.
item = ScrapyItem()
item.unique_id = self.unique_id
item.data = json.dumps(self.items)
item.save()