def check_items(df: pd.DataFrame, tagged_fields: Dict[str, List[str]]) -> Result: """Check for items with the same name and url""" name_fields = tagged_fields.get("name_field") url_fields = tagged_fields.get("product_url_field") result = Result("Duplicated Items") if not name_fields or not url_fields: result.add_info( "'name_field' and 'product_url_field' tags were not found in schema" ) else: result.items_count = len(df) errors = {} name_field = name_fields[0] url_field = url_fields[0] df = df[[name_field, url_field, "_key"]] duplicates = df[df[[name_field, url_field]].duplicated(keep=False)] if duplicates.empty: return result result.err_items_count = len(duplicates) for _, d in duplicates.groupby([name_field, url_field]): msg = ( f"same '{d[name_field].iloc[0]}' name and '{d[url_field].iloc[0]}' url" ) errors[msg] = list(d["_key"]) result.add_error( f"{len(duplicates)} duplicate(s) with same name and url", errors=errors) return result
def find_by_unique(df: pd.DataFrame, tagged_fields: TaggedFields) -> Result: """Verify if each item field tagged with `unique` is unique. Returns: A result containing field names and keys for non unique items """ unique_fields = tagged_fields.get("unique", []) result = Result("Duplicates By **unique** Tag") if not unique_fields: result.add_info(Outcome.SKIPPED) return result err_keys = set() for field in unique_fields: result.items_count = df[field].count() duplicates = df[df.duplicated(field, keep=False)][[field]] errors = {} for _, d in duplicates.groupby([field]): keys = list(d.index) msg = f"same '{d[field].iloc[0]}' `{field}`" errors[msg] = keys err_keys = err_keys.union(keys) if not duplicates.empty: result.add_error( f"{field} contains {len(duplicates[field].unique())} duplicated value(s)", errors=errors, ) result.err_items_count = len(err_keys) return result
def check_uniqueness(df: pd.DataFrame, tagged_fields: Dict[str, List[str]]) -> Result: """Verify if each item field tagged with `unique` is unique. Returns: A result containing field names and keys for non unique items """ unique_fields = tagged_fields.get("unique", []) result = Result("Uniqueness") if not unique_fields: result.add_info("'unique' tag was not found in schema") return result err_keys = set() for field in unique_fields: result.items_count = df[field].count() duplicates = df[df[field].duplicated(keep=False)][[field, "_key"]] errors = {} for _, d in duplicates.groupby([field]): keys = list(d["_key"]) msg = f"same '{d[field].iloc[0]}' {field}" errors[msg] = keys err_keys = err_keys.union(keys) if not duplicates.empty: result.add_error( f"'{field}' contains {len(duplicates[field].unique())} duplicated value(s)", errors=errors, ) result.err_items_count = len(err_keys) return result
def compare_was_now(df: pd.DataFrame, tagged_fields: Dict[str, List[str]]): """Compare price_was and price_now tagged fields""" price_was_fields = tagged_fields.get("product_price_was_field") price_fields = tagged_fields.get("product_price_field") items_number = len(df.index) result = Result("Compare Price Was And Now") if (price_was_fields and price_was_fields[0] in df.columns and price_fields and price_fields[0] in df.columns): price_field = price_fields[0] price_was_field = price_was_fields[0] prices = df.copy() prices[price_was_field] = prices[price_was_field].astype(float) prices[price_field] = prices[price_field].astype(float) df_prices_less = pd.DataFrame( prices[prices[price_was_field] < prices[price_field]], columns=["_key", price_was_field, price_field], ) price_less_percent = "{:.2%}".format( len(df_prices_less) / items_number) if not df_prices_less.empty: error = f"Past price is less than current for {len(df_prices_less)} items" result.add_error( f"{price_less_percent} ({len(df_prices_less)}) of " f"items with {price_was_field} < {price_field}", detailed=f"{error}:\n{list(df_prices_less['_key'])}", ) df_prices_equals = pd.DataFrame( prices[prices[price_was_field] == prices[price_field]], columns=["_key", price_was_field, price_field], ) price_equal_percent = "{:.2%}".format( len(df_prices_equals) / items_number) if not df_prices_equals.empty: result.add_warning( (f"{price_equal_percent} ({len(df_prices_equals)}) " f"of items with {price_was_field} = {price_field}"), detailed=(f"Prices equal for {len(df_prices_equals)} items:\n" f"{list(df_prices_equals['_key'])}"), ) result.err_items_count = len(df_prices_equals) + len(df_prices_less) result.items_count = len(df.index) else: result.add_info( "product_price_field or product_price_was_field tags were not " "found in schema") return result
def compare_was_now(df: pd.DataFrame, tagged_fields: TaggedFields): """Compare price_was and price_now tagged fields""" price_was_fields = tagged_fields.get("product_price_was_field") price_fields = tagged_fields.get("product_price_field") items_number = len(df.index) result = Result("Compare Price Was And Now") if not price_was_fields or not price_fields: result.add_info(Outcome.SKIPPED) return result price_field = price_fields[0] price_was_field = price_was_fields[0] prices = df.copy() prices[price_was_field] = prices[price_was_field].astype(float) prices[price_field] = prices[price_field].astype(float) df_prices_less = pd.DataFrame( prices[prices[price_was_field] < prices[price_field]], columns=[price_was_field, price_field], ) price_less_percent = "{:.2%}".format(len(df_prices_less) / items_number) if not df_prices_less.empty: error = f"Past price is less than current for {len(df_prices_less)} items" result.add_error( f"{price_less_percent} ({len(df_prices_less)}) of " f"items with {price_was_field} < {price_field}", detailed=f"{error}:\n{list(df_prices_less.index)}", ) df_prices_equals = pd.DataFrame( prices[prices[price_was_field] == prices[price_field]], columns=[price_was_field, price_field], ) price_equal_percent = "{:.2%}".format(len(df_prices_equals) / items_number) if not df_prices_equals.empty: result.add_warning( ( f"{price_equal_percent} ({len(df_prices_equals)}) " f"of items with {price_was_field} = {price_field}" ), detailed=( f"Prices equal for {len(df_prices_equals)} items:\n" f"{list(df_prices_equals.index)}" ), ) result.err_items_count = len(df_prices_equals) + len(df_prices_less) result.items_count = len(df.index) return result
def create_result( rule_name, messages, stats=None, err_items_count=None, items_count=None ): result = Result(rule_name) for level, messages in messages.items(): for message in messages: result.add_message(level, *message) if stats: result.stats = stats if err_items_count: result.err_items_count = err_items_count if items_count: result.items_count = items_count return result
def create_result(rule_name, messages, err_items_count=None, checked_fields=None, items_count=None): result = Result(rule_name) for level, messages in messages.items(): for message in messages: result.add_message(level, *message) if err_items_count: result.err_items_count = err_items_count if checked_fields: result.checked_fields = checked_fields if items_count: result.items_count = items_count return result
def garbage_symbols(df: pd.DataFrame) -> Result: """Find unwanted symbols in `np.object` columns. Returns: A result containing item keys per field which contained any trash symbol """ garbage = ( r"(?P<spaces>^\s|\s$)" r"|(?P<html_entities>&[a-zA-Z]{2,}?;|&#\d*?;)" r"|(?P<css>[.#@][^\d{}#.\s][^{}#.]+?{(?:[^:;{}]+?:[^:;{}]+?;)+?\s*?})" r"|(?P<html_tags></??(?:h\d|b|u|i|div|ul|ol|li|table|tbody|th|tr|td|p|a|br|img|sup|SUP|" r"blockquote)\s*?/??>|<!--|-->)") errors = {} row_keys = set() rule_result = Result("Garbage Symbols", items_count=len(df)) for column in tqdm_notebook(df.select_dtypes([np.object]).columns, desc="Garbage Symbols"): matches = df[column].apply(str).str.extractall(garbage, flags=re.IGNORECASE) if not matches.empty: error_keys = df.loc[matches.unstack().index.values].index bad_texts = matches.stack().value_counts().index.sort_values( ).tolist() # escape backslashes for markdown repr, `\n > \\n` bad_texts = [ f"'{codecs.encode(bx, 'unicode_escape').decode()[:20]}'" for bx in bad_texts ] error = (f"{len(error_keys)/len(df)*100:.1f}% of '{column}' " f"values contain `{', '.join(bad_texts)}`") errors[error] = list(error_keys) row_keys = row_keys.union(error_keys) if errors: rule_result.add_error( f"{len(row_keys)/len(df) * 100:.1f}% ({len(row_keys)}) items affected", errors=errors, ) rule_result.err_items_count = len(row_keys) return rule_result
def garbage_symbols(items: Items) -> Result: """Find unwanted symbols in `np.object` columns. Returns: A result containing item keys per field which contained any trash symbol """ garbage = ( r"(?P<spaces>^\s|\s$)" r"|(?P<html_entities>&|®)" r"|(?P<css>(?:(?:\.|#)[^#. ]+\s*){.+})" r"|(?P<html_tags></?(h\d|b|u|i|div|ul|ol|li|table|tbody|th|tr|td|p|a|br|img|sup|SUP|" r"blockquote)\s*/?>|<!--|-->)") errors = {} row_keys = set() rule_result = Result("Garbage Symbols", items_count=items.size) for column in items.flat_df.select_dtypes([np.object]): matches = items.flat_df[column].str.extractall(garbage, flags=re.IGNORECASE) matches = matches[["spaces", "html_entities", "css", "html_tags"]] if not matches.empty: error_keys = items.flat_df.iloc[ matches.unstack().index.values]["_key"] original_column = items.get_origin_column_name(column) bad_texts = matches.stack().value_counts().index.sort_values( ).tolist() error = ( f"{len(error_keys)/items.size*100:.1f}% of '{original_column}' " f"values contain {[t[:20] for t in bad_texts]}") errors[error] = list(error_keys) row_keys = row_keys.union(error_keys) if errors: rule_result.add_error( f"{len(row_keys)/items.size * 100:.1f}% ({len(row_keys)}) items affected", errors=errors, ) rule_result.err_items_count = len(row_keys) return rule_result
def find_by(df: pd.DataFrame, columns: List[str]) -> Result: """Compare items rows in `df` by `columns` Returns: Any duplicates """ result = Result(f"Duplicates") result.items_count = len(df) df = df.dropna(subset=columns, how="all") duplicates = df[df.duplicated(columns, keep=False)][columns] if duplicates.empty: return result result.err_items_count = len(duplicates) errors = {} for _, d in duplicates.groupby(columns): msgs = [f"'{d[c].iloc[0]}' `{c}`" for c in columns] errors[f"same {', '.join(msgs)}"] = list(d.index) result.add_error( f"{len(duplicates)} duplicate(s) with same {', '.join(columns)}", errors=errors) return result