def _process_input_value(self, field_name, value): proc = self.get_input_processor(field_name) _proc = proc proc = wrap_loader_context(proc, self.context) try: return proc(value) except Exception as e: raise ValueError( "Error with input processor %s: field=%r value=%r " "error='%s: %s'" % (_proc.__class__.__name__, field_name, value, type(e).__name__, str(e)))
def wrap_loader_context(function, context): """Wrap functions that receive loader_context to contain the context "pre-loaded" and expose a interface that receives only one argument """ warnings.warn( "scrapy.loader.common.wrap_loader_context has moved to a new library." "Please update your reference to itemloaders.common.wrap_loader_context", ScrapyDeprecationWarning, stacklevel=2 ) return common.wrap_loader_context(function, context)
def get_output_value(self, field_name): """ Return the collected values parsed using the output processor, for the given field. This method doesn't populate or modify the item at all. """ proc = self.get_output_processor(field_name) proc = wrap_loader_context(proc, self.context) value = self._values.get(field_name, []) try: return proc(value) except Exception as e: raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" % (field_name, value, type(e).__name__, str(e)))
def __call__(self, value, loader_context=None): if loader_context: context = ChainMap(loader_context, self.default_loader_context) else: context = self.default_loader_context wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions] for func in wrapped_funcs: if value is None and self.stop_on_none: break try: value = func(value) except Exception as e: raise ValueError("Error in Compose with " "%s value=%r error='%s: %s'" % (str(func), value, type(e).__name__, str(e))) return value
def __call__(self, value, loader_context=None): values = arg_to_iter(value) if loader_context: context = ChainMap(loader_context, self.default_loader_context) else: context = self.default_loader_context wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions] for func in wrapped_funcs: next_values = [] for v in values: try: next_values += arg_to_iter(func(v)) except Exception as e: raise ValueError("Error in MapCompose with " "%s value=%r error='%s: %s'" % (str(func), value, type(e).__name__, str(e))) values = next_values return values
def get_value(self, value, *processors, **kw): """ Process the given ``value`` by the given ``processors`` and keyword arguments. Available keyword arguments: :param re: a regular expression to use for extracting data from the given value using :meth:`~parsel.utils.extract_regex` method, applied before processors :type re: str or compiled regex Examples: >>> from itemloaders import ItemLoader >>> from itemloaders.processors import TakeFirst >>> loader = ItemLoader() >>> loader.get_value('name: foo', TakeFirst(), str.upper, re='name: (.+)') 'FOO' """ regex = kw.get('re', None) if regex: value = arg_to_iter(value) value = flatten(extract_regex(regex, x) for x in value) for proc in processors: if value is None: break _proc = proc proc = wrap_loader_context(proc, self.context) try: value = proc(value) except Exception as e: raise ValueError( "Error with processor %s value=%r error='%s: %s'" % (_proc.__class__.__name__, value, type(e).__name__, str(e))) return value