示例#1
0
    def save_item(self, item_data, update_max_id=False, front_page=False):
        """
		Compile and save item to the database
		Each item is committed separately for better fault tolerance
		"""

        item = None
        # If item was lost
        if isinstance(item_data, LostItem):
            item = item_data
            db.session.add(item)
            debug_print(
                "Lost %s because %s" % (item_data.id, item_data.reason), '\n')
        else:
            if item_data.get('type', None) in ('story', 'comment', 'poll',
                                               'job', None):
                debug_print("Saving %d" % item_data['id'], '\n')
                compiled_data = self.compile_item_data(item_data, front_page)
                item = Item.create_or_update(compiled_data)
            else:
                debug_print(
                    "Skipping %s %d" % (item_data['type'], item_data['id']),
                    '\n')

        if update_max_id:
            Status.set_max_item_id(item_data.id if isinstance(
                item_data, LostItem) else item_data['id'])

        db.session.commit()

        return item
示例#2
0
文件: scraper.py 项目: raquo/hnapp
	def save_item(self, item_data, update_max_id=False, front_page=False):
		"""
		Compile and save item to the database
		Each item is committed separately for better fault tolerance
		"""
		
		item = None
		# If item was lost
		if isinstance(item_data, LostItem):
			if not item_data.id:
				return None
			item = item_data
			db.session.add(item)
			debug_print("Lost %s because %s" % (item_data.id, item_data.reason), '\n')
		else:
			if 'id' not in item_data:
				debug_print('Skipping item because empty id', '\n')
				return None
			if item_data.get('type', None) in ('story', 'comment', 'poll', 'job', None):
				debug_print("Saving %d" % item_data['id'], '\n')
				compiled_data = self.compile_item_data(item_data, front_page)
				item = Item.create_or_update(compiled_data)
			else:
				debug_print("Skipping %s %d" % (item_data['type'], item_data['id']), '\n')
		
		if update_max_id:
			Status.set_max_item_id(item_data.id if isinstance(item_data, LostItem) else item_data['id'])
		
		db.session.commit()
		
		return item