def process_item_parser(self, item_key: str, data: DataBag):
item_parser = self._item_parsers[item_key]
if item_parser is None:
return None
if isinstance(item_parser, (str, bool, float, int, list, dict)):
return item_parser
elif isinstance(item_parser, models.ItemModel):
data = data.copy(item_parser.model_manager)
return item_parser.parse_item(data)
elif isinstance(item_parser, BaseParser):
return item_parser.parse(data)
return item_parser(data)
def variants_data(data: DataBag, source: str):
original_variants_data: dict = data[source]
total_variants = len(original_variants_data)
for variant_key, variant_multi_data in original_variants_data.items():
data_copy = data.copy()
variant_data = variant_multi_data[0]
data_copy[source] = variant_data
data_copy["{}_variants".format(source)] = variant_multi_data
data_copy["{}_variants_len".format(source)] = total_variants
data_copy["{}_key".format(source)] = variant_key
yield data_copy
def parse(self, data: DataBag) -> Iterator[DataBag]:
source_data = data[self._source]
if self._debug_raw:
print(source_data)
if self._process_source_data:
source_data = self._process_source_data(source_data)
transformed_data = self._process_data(source_data)
if self._debug:
print(transformed_data)
if self._multi:
for iter_transformed_data in transformed_data:
data_copy = data.copy()
yield self._transformed_data_to_data(iter_transformed_data,
data_copy)
else:
yield self._transformed_data_to_data(transformed_data, data)