def check_null(item: Item,
logger: Logger,
pipeline_class,
no_check_keys: list = None):
if no_check_keys:
null_keys = [
key for key, value in item.items()
if value is None and key not in set(no_check_keys)
]
else:
null_keys = [key for key, value in item.items() if not value]
if null_keys:
logger.error(
f"##### ERROR: {', '.join(null_keys)} are null in {pipeline_class}#####"
)
raise DropItem(str(item))
def process_item(self, item: scrapy.Item, spider: scrapy.Spider) -> List:
for k, v in item.items():
spider.log('{}: {}'.format(k, v), logging.INFO)
return item
