def check_null(item: Item, logger: Logger, pipeline_class, no_check_keys: list = None): if no_check_keys: null_keys = [ key for key, value in item.items() if value is None and key not in set(no_check_keys) ] else: null_keys = [key for key, value in item.items() if not value] if null_keys: logger.error( f"##### ERROR: {', '.join(null_keys)} are null in {pipeline_class}#####" ) raise DropItem(str(item))
def process_item(self, item: scrapy.Item, spider: scrapy.Spider) -> List: for k, v in item.items(): spider.log('{}: {}'.format(k, v), logging.INFO) return item