Example #1
0
 def load_field(self, field_name, loader, response):
     """
     Generic method to load fields with the item loader
     """
     try:
         method_name = 'load_' + field_name
         # if it is defined a custom method to load the field, call it
         # else, call the generic load_field method
         if (hasattr(self, method_name) and
             callable(getattr(self, method_name))):
             getattr(self, method_name)(loader, response)
         else:
             xpath = getattr(self, field_name + '_xpath', False)
             if not xpath:
                 loader.add_value(field_name, MISSING_VALUE)
             else:
                 loader.add_xpath(field_name, xpath)
     except Exception as e:
         error_level = STATUS_UNKNOWN
         if field_name in CRITICAL_FIELDS:
             error_level = STATUS_CRITICAL
         else:
             error_level = STATUS_WARNING
         raise_missing_value(self.name,
                             field_name=field_name,
                             url=response.url,
                             exception=e,
                             level=error_level,
                             message="Could not load field '%s'" % field_name)
Example #2
0
def parse_discount(text, loader_context):
    try:
        return parse_float_discount(text)
    except Exception as e:
        raise_missing_value(spider_name=loader_context['spider_name'],
                            field_name='discount',
                            url=loader_context['url'],
                            exception=e,
                            level=STATUS_CRITICAL,
                            message="Could not parse discount from '%s'" %
                            text,
                            category=PARSING_ERROR)
        return None
Example #3
0
def parse_price_currency(text, loader_context):
    try:
        data = parse_float_price(text, loader_context['country'],
                                 loader_context['decimal_mark'])
        return data
    except Exception as e:
        raise_missing_value(spider_name=loader_context['spider_name'],
                            field_name='price',
                            url=loader_context['url'],
                            exception=e,
                            level=STATUS_CRITICAL,
                            message="Could not parse price from '%s'." % text,
                            category=PARSING_ERROR)
        return None
Example #4
0
def parse_discount(text, loader_context):
    try:
        return parse_float_discount(text)
    except Exception as e:
        raise_missing_value(
            spider_name=loader_context['spider_name'],
            field_name='discount',
            url=loader_context['url'],
            exception=e,
            level=STATUS_CRITICAL,
            message="Could not parse discount from '%s'" % text,
            category=PARSING_ERROR
        )
        return None
Example #5
0
def parse_sold_count(text, loader_context):
    try:
        text = text.replace(",", "").replace(".", "")
        n = parse_int(text)
        return n
    except Exception as e:
        raise_missing_value(spider_name=loader_context['spider_name'],
                            field_name='sold_count',
                            url=loader_context['url'],
                            exception=e,
                            level=STATUS_CRITICAL,
                            message="Could not parse sold count from '%s'" %
                            text,
                            category=PARSING_ERROR)
        return None
Example #6
0
def parse_price_currency(text, loader_context):
    try:
        data = parse_float_price(text, loader_context['country'],
                                 loader_context['decimal_mark'])
        return data
    except Exception as e:
        raise_missing_value(
            spider_name=loader_context['spider_name'],
            field_name='price',
            url=loader_context['url'],
            exception=e,
            level=STATUS_CRITICAL,
            message="Could not parse price from '%s'." % text,
            category=PARSING_ERROR
        )
        return None
Example #7
0
def parse_sold_count(text, loader_context):
    try:
        text = text.replace(",", "").replace(".", "")
        n = parse_int(text)
        return n
    except Exception as e:
        raise_missing_value(
            spider_name=loader_context['spider_name'],
            field_name='sold_count',
            url=loader_context['url'],
            exception=e,
            level=STATUS_CRITICAL,
            message="Could not parse sold count from '%s'" % text,
            category=PARSING_ERROR
        )
        return None