def __init__(self, stats, prefix=None):
self.analyzer = StatsAnalyzer(stats=stats,
prefix=prefix
or STATS_DEFAULT_VALIDATION_PREFIX)
# items
items_count = sum(self.analyzer.search("items$").values())
items_with_errors_count = sum(
self.analyzer.search("items/errors$").values())
items_dropped_count = sum(
self.analyzer.search("items/dropped$").values())
self.items = ItemsInfo(
items_count=items_count,
items_with_errors=items_with_errors_count,
items_dropped=items_dropped_count,
)
# errors & fields
fields_count = sum(self.analyzer.search("fields$").values())
self.errors = ErrorsInfo(items_count)
self.fields = FieldErrorsInfo(fields_count=fields_count,
items_count=items_count)
field_errors = self.analyzer.search("fields/errors/([^/]+)$",
include_matches=True)
for _, error in field_errors.values():
field_errors_per_field = self.analyzer.search(
"fields/errors/%s/([^/]+)$" % error, include_matches=True)
for count, field in field_errors_per_field.values():
self.errors.add_values(key=error, subkey=field, value=count)
self.fields.add_values(key=field, subkey=error, value=count)
def __init__(self, stats):
self._stats_analyzer = StatsAnalyzer(stats=stats)
self.count = self._stats_analyzer.search(DOWNLOADER_RESPONSE_COUNT + "$").get(
DOWNLOADER_RESPONSE_COUNT, 0
)
# all status codes
self.all = DictPercentCounter(total=self.count)
self._add_status_codes(pattern=None, target=self.all)
# 1xx. informational
self.informational = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_INFORMATIONAL, target=self.informational
)
# 2xx. successful
self.successful = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_SUCCESSFUL, target=self.successful
)
# 3xx. redirections
self.redirections = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_REDIRECTIONS, target=self.redirections
)
# 4xx. bad requests
self.bad_requests = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_BAD_REQUESTS, target=self.bad_requests
)
# 5xx. internal server errors
self.internal_server_errors = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_INTERNAL_SERVER_ERRORS,
target=self.internal_server_errors,
)
# >= 6xx. others
self.others = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_OTHERS, target=self.others
)
# errors (4xx + 5xx)
self.errors = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_ERRORS, target=self.errors
)
class ResponsesInfo(object):
def __init__(self, stats):
self._stats_analyzer = StatsAnalyzer(stats=stats)
self.count = self._stats_analyzer.search(DOWNLOADER_RESPONSE_COUNT +
'$').get(
DOWNLOADER_RESPONSE_COUNT,
0)
# all status codes
self.all = DictPercentCounter(total=self.count)
self._add_status_codes(pattern=None, target=self.all)
# 1xx. informational
self.informational = DictPercentCounter(total=self.count)
self._add_status_codes(pattern=DOWNLOADER_STATUS_CODES_INFORMATIONAL,
target=self.informational)
# 2xx. successful
self.successful = DictPercentCounter(total=self.count)
self._add_status_codes(pattern=DOWNLOADER_STATUS_CODES_SUCCESSFUL,
target=self.successful)
# 3xx. redirections
self.redirections = DictPercentCounter(total=self.count)
self._add_status_codes(pattern=DOWNLOADER_STATUS_CODES_REDIRECTIONS,
target=self.redirections)
# 4xx. bad requests
self.bad_requests = DictPercentCounter(total=self.count)
self._add_status_codes(pattern=DOWNLOADER_STATUS_CODES_BAD_REQUESTS,
target=self.bad_requests)
# 5xx. internal server errors
self.internal_server_errors = DictPercentCounter(total=self.count)
self._add_status_codes(
pattern=DOWNLOADER_STATUS_CODES_INTERNAL_SERVER_ERRORS,
target=self.internal_server_errors)
# >= 6xx. others
self.others = DictPercentCounter(total=self.count)
self._add_status_codes(pattern=DOWNLOADER_STATUS_CODES_OTHERS,
target=self.others)
# errors (4xx + 5xx)
self.errors = DictPercentCounter(total=self.count)
self._add_status_codes(pattern=DOWNLOADER_STATUS_CODES_ERRORS,
target=self.errors)
def _add_status_codes(self, pattern, target):
for code, counter in self._get_response_codes(pattern).items():
target.add_value(code, counter.count)
def _get_response_codes(self, codes=None):
codes = codes or ['[^/]+']
return_codes = {}
for code in codes:
return_codes.update(self._get_response_code(code))
return return_codes
def _get_response_code(self, code):
return dict([(code, PercentCounter(count, self.count))
for count, code in self._stats_analyzer.search(
pattern=DOWNLOADER_RESPONSE_STATUS + ('(%s)$' % code),
include_matches=True).values()])