def _add_value(self, field_name, value):
value = arg_to_iter(value)
processed_value = self._process_input_value(field_name, value)
if processed_value:
self._values.setdefault(field_name, [])
if type(arg_to_iter(processed_value)) == list:
self._values[field_name] += arg_to_iter(processed_value)
else:
self._values[field_name].append(arg_to_iter(processed_value))
def process_digit(values, only_digit=True):
# Функция для приведения сток к числовым переменным
# Если в строке содержаться недопустимые для чисел символы
# Не останавливает выполнение программы. В случае исключения
# сохраняет None если only_digit=True или исходную строку.
# Всегда возвращает список
values = arg_to_iter(values)
for i, value in enumerate(values):
value = value.replace(' ', '').replace(',', '.')
if '.' in value:
try:
values[i] = float(value)
except Exception as e:
if only_digit:
print(
"Error in ProductsparserItem with %s value=%r error='%s: %s'"
%
('process_digit', value, type(e).__name__, str(e)))
values[i] = None
else:
try:
values[i] = int(value)
except Exception as e:
if only_digit:
print(
"Error in ProductsparserItem with %s value=%r error='%s: %s'"
%
('process_digit', value, type(e).__name__, str(e)))
values[i] = None
return values
def __call__(self, value, loader_context=None):
values = arg_to_iter(value)
if loader_context:
context = ChainMap(loader_context, self.default_loader_context)
else:
context = self.default_loader_context
wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
for func in wrapped_funcs:
next_values = []
for v in values:
try:
next_values += arg_to_iter(func(v))
except Exception as e:
raise ValueError("Error in MapCompose with "
"%s value=%r error='%s: %s'" %
(str(func), value, type(e).__name__,
str(e)))
values = next_values
return values
def __init__(self, item=None, selector=None, parent=None, **context):
self.selector = selector
context.update(selector=selector)
if item is None:
item = self.default_item_class()
self._local_item = item
context['item'] = item
self.context = context
self.parent = parent
self._local_values = defaultdict(list)
# values from initial item
for field_name, value in ItemAdapter(item).items():
self._values[field_name] += arg_to_iter(value)
def get_value(self, value, *processors, **kw):
"""
Process the given ``value`` by the given ``processors`` and keyword
arguments.
Available keyword arguments:
:param re: a regular expression to use for extracting data from the
given value using :meth:`~parsel.utils.extract_regex` method,
applied before processors
:type re: str or compiled regex
Examples:
>>> from itemloaders import ItemLoader
>>> from itemloaders.processors import TakeFirst
>>> loader = ItemLoader()
>>> loader.get_value('name: foo', TakeFirst(), str.upper, re='name: (.+)')
'FOO'
"""
regex = kw.get('re', None)
if regex:
value = arg_to_iter(value)
value = flatten(extract_regex(regex, x) for x in value)
for proc in processors:
if value is None:
break
_proc = proc
proc = wrap_loader_context(proc, self.context)
try:
value = proc(value)
except Exception as e:
raise ValueError(
"Error with processor %s value=%r error='%s: %s'" %
(_proc.__class__.__name__, value, type(e).__name__,
str(e)))
return value
def _get_cssvalues(self, csss, **kw):
self._check_selector_method()
csss = arg_to_iter(csss)
return flatten(self.selector.css(css).getall() for css in csss)
def _get_xpathvalues(self, xpaths, **kw):
self._check_selector_method()
xpaths = arg_to_iter(xpaths)
return flatten(self.selector.xpath(xpath).getall() for xpath in xpaths)
def _add_value(self, field_name, value):
value = arg_to_iter(value)
processed_value = self._process_input_value(field_name, value)
if processed_value:
self._values.setdefault(field_name, [])
self._values[field_name] += arg_to_iter(processed_value)
def test_arg_to_iter(self):
assert hasattr(arg_to_iter(None), '__iter__')
assert hasattr(arg_to_iter(100), '__iter__')
assert hasattr(arg_to_iter('lala'), '__iter__')
assert hasattr(arg_to_iter([1, 2, 3]), '__iter__')
assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__')
self.assertEqual(list(arg_to_iter(None)), [])
self.assertEqual(list(arg_to_iter('lala')), ['lala'])
self.assertEqual(list(arg_to_iter(100)), [100])
self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c'])
self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3])
self.assertEqual(list(arg_to_iter({'a': 1})), [{'a': 1}])
