def create_figures(self, items, items_dicts):
jf = JsonFields(self.schema)
tagged_fields = jf.tagged
no_of_validated_items = len(items.df.index)
dup_items_result = duplicate_rules.check_items(items.df, tagged_fields)
no_of_checked_duplicated_items = dup_items_result.items_count
no_of_duplicated_items = dup_items_result.err_items_count
dup_skus_result = duplicate_rules.check_uniqueness(
items.df, tagged_fields)
no_of_checked_skus_items = dup_skus_result.items_count
no_of_duplicated_skus = dup_skus_result.err_items_count
price_was_now_result = price_rules.compare_was_now(
items.df, tagged_fields)
no_of_price_warns = price_was_now_result.err_items_count
no_of_checked_price_items = price_was_now_result.items_count
garbage_symbols_result = garbage_symbols(items)
crawlera_user = api.get_crawlera_user(items.job)
no_of_validation_warnings = self.report.results.get(
"JSON Schema Validation").get_errors_count()
quality_estimation, field_accuracy = generate_quality_estimation(
items.job,
crawlera_user,
no_of_validation_warnings,
no_of_duplicated_items,
no_of_checked_duplicated_items,
no_of_duplicated_skus,
no_of_checked_skus_items,
no_of_price_warns,
no_of_validated_items,
tested=True,
garbage_symbols=garbage_symbols_result,
)
cleaned_df = self.drop_service_columns(items.df)
self.score_table(quality_estimation, field_accuracy)
self.job_summary_table(items.job)
self.rules_summary_table(
cleaned_df,
no_of_validation_warnings,
tagged_fields.get("name_field", ""),
tagged_fields.get("product_url_field", ""),
no_of_checked_duplicated_items,
no_of_duplicated_items,
tagged_fields.get("unique", []),
no_of_checked_skus_items,
no_of_duplicated_skus,
tagged_fields.get("product_price_field", ""),
tagged_fields.get("product_price_was_field", ""),
no_of_checked_price_items,
no_of_price_warns,
garbage_symbols=garbage_symbols_result,
)
self.scraped_fields_coverage(items.job.key, cleaned_df)
self.coverage_by_categories(cleaned_df, tagged_fields)
def run_customized_rules(self, items, tagged_fields):
self.save_result(price_rules.compare_was_now(items.df, tagged_fields))
self.save_result(duplicate_rules.find_by_tags(items.df, tagged_fields))
self.save_result(
category_rules.get_coverage_per_category(
items.df,
tagged_fields.get("category", []) + self.schema.enums))
def run_customized_rules(self, items, tagged_fields):
self.save_result(price_rules.compare_was_now(items.df, tagged_fields))
self.save_result(
duplicate_rules.check_uniqueness(items.df, tagged_fields))
self.save_result(duplicate_rules.check_items(items.df, tagged_fields))
self.save_result(
category_rules.get_coverage_per_category(
items.df, tagged_fields.get("category", [])))
def test_compare_was_now(data, tagged_fields, expected_messages):
df = pd.DataFrame(data)
assert_results_equal(
p.compare_was_now(df, tagged_fields),
create_result("Compare Price Was And Now",
expected_messages,
items_count=len(df)),
)
def test_compare_was_now(data, tagged_fields, expected_messages, expected_errors):
df = pd.DataFrame(data)
result = p.compare_was_now(df, tagged_fields)
assert result == create_result(
"Compare Price Was And Now",
expected_messages,
expected_errors,
items_count=len(df),
)
def create_figures(self, items: JobItems):
dups = self.report.results.get(
"Duplicates", duplicate_rules.find_by_tags(items.df, self.schema.tags)
)
price_was_now_result = price_rules.compare_was_now(items.df, self.schema.tags)
no_of_price_warns = price_was_now_result.err_items_count
no_of_checked_price_items = price_was_now_result.items_count
crawlera_user = api.get_crawlera_user(items.job)
validation_errors = self.report.results.get(
"JSON Schema Validation",
schema_rules.validate(
self.schema.raw, raw_items=items.raw, keys=items.df.index, fast=False
),
).get_errors_count()
garbage_symbols_result = self.report.results.get(
"Garbage Symbols", garbage_symbols(items.df)
)
quality_estimation, field_accuracy = generate_quality_estimation(
items.job,
crawlera_user,
validation_errors,
dups.err_items_count,
dups.items_count,
no_of_price_warns,
no_of_checked_price_items,
tested=True,
garbage_symbols=garbage_symbols_result,
)
self.score_table(quality_estimation, field_accuracy)
self.job_summary_table(items.job)
self.rules_summary_table(
items.df,
validation_errors,
self.schema.tags.get("name_field", ""),
self.schema.tags.get("product_url_field", ""),
dups.err_items_count,
dups.items_count,
self.schema.tags.get("product_price_field", ""),
self.schema.tags.get("product_price_was_field", ""),
no_of_checked_price_items,
no_of_price_warns,
garbage_symbols=garbage_symbols_result,
)
self.scraped_fields_coverage(items.df)
self.coverage_by_categories(items.df, self.schema.tags)
def create_figures(self, items):
tagged_fields = Tags().get(self.schema)
dup_items_result = duplicate_rules.check_items(items.df, tagged_fields)
no_of_checked_duplicated_items = dup_items_result.items_count
no_of_duplicated_items = dup_items_result.err_items_count
dup_skus_result = duplicate_rules.check_uniqueness(
items.df, tagged_fields)
no_of_checked_skus_items = dup_skus_result.items_count
no_of_duplicated_skus = dup_skus_result.err_items_count
price_was_now_result = price_rules.compare_was_now(
items.df, tagged_fields)
no_of_price_warns = price_was_now_result.err_items_count
no_of_checked_price_items = price_was_now_result.items_count
crawlera_user = api.get_crawlera_user(items.job)
validation_errors = self.report.results.get(
"JSON Schema Validation",
schema_rules.validate(self.schema, raw_items=items.raw,
fast=False),
).get_errors_count()
garbage_symbols_result = self.report.results.get(
"Garbage Symbols", garbage_symbols(items))
quality_estimation, field_accuracy = generate_quality_estimation(
items.job,
crawlera_user,
validation_errors,
no_of_duplicated_items,
no_of_checked_duplicated_items,
no_of_duplicated_skus,
no_of_checked_skus_items,
no_of_price_warns,
no_of_checked_price_items,
tested=True,
garbage_symbols=garbage_symbols_result,
)
self.score_table(quality_estimation, field_accuracy)
self.job_summary_table(items.job)
self.rules_summary_table(
items.df,
validation_errors,
tagged_fields.get("name_field", ""),
tagged_fields.get("product_url_field", ""),
no_of_checked_duplicated_items,
no_of_duplicated_items,
tagged_fields.get("unique", []),
no_of_checked_skus_items,
no_of_duplicated_skus,
tagged_fields.get("product_price_field", ""),
tagged_fields.get("product_price_was_field", ""),
no_of_checked_price_items,
no_of_price_warns,
garbage_symbols=garbage_symbols_result,
)
self.scraped_fields_coverage(items.df)
self.coverage_by_categories(items.df, tagged_fields)
