def _parse_renames( renames: Dict[str, str], table_columns: List[str], *, settings: Settings) -> Tuple[Dict[str, str], List[i18n.I18nMessage]]: """ Convert `renames` into a valid mapping for `table_columns`, plus warnings. Ignore any renames to "". That column name is not allowed. Return a minimal and valid dict from old colname to new colname. `renames` is a dict mapping old colname to new colname. It may contain missing origin column names and it may duplicate destination column names. The logic to handle this: do _all_ the user's renames at once, and then queue extra renames for columns that end up with duplicate names. Those extra renames are handled left-to-right (the order of `table_columns` matters). """ # "renames.get(c) or c" means: # * If renames[c] exists and is "", return c # * If renames[c] does not exist, return c # * If renames[c] exists and is _not_ "", return renames[c] nix_colnames = [c for c in table_columns if (renames.get(c) or c) != c] nix_colnames_set = frozenset(nix_colnames) existing_colnames = [c for c in table_columns if c not in nix_colnames_set] try_new_colnames = [ renames[c] for c in table_columns if c in nix_colnames_set ] new_colnames, errors = gen_unique_clean_colnames_and_warn( try_new_colnames, existing_names=existing_colnames, settings=settings) return {k: v for k, v in zip(nix_colnames, new_colnames)}, errors
def test_gen_and_warn_multiple(): assert gen_unique_clean_colnames_and_warn( [ "A Column", "B Column", "ab", "ab", "ab", "", ], settings=MockSettings(6), )[1] == ([ cjwmodule_i18n_message( "util.colnames.warnings.default", { "n_columns": 1, "first_colname": "Column 6" }, ), cjwmodule_i18n_message( "util.colnames.warnings.truncated", { "n_columns": 2, "first_colname": "A Colu", "n_bytes": 6 }, ), cjwmodule_i18n_message( "util.colnames.warnings.numbered", { "n_columns": 2, "first_colname": "ab 2" }, ), ])
def test_gen_and_warn_default(): assert gen_unique_clean_colnames_and_warn(["", ""])[1] == ([ cjwmodule_i18n_message( "util.colnames.warnings.default", { "n_columns": 2, "first_colname": "Column 1" }, ), ])
def test_gen_and_warn_numbered(): assert gen_unique_clean_colnames_and_warn(["A", "A", "A"])[1] == ([ cjwmodule_i18n_message( "util.colnames.warnings.numbered", { "n_columns": 2, "first_colname": "A 2" }, ), ])
def test_gen_and_warn_ascii_cleaned(): assert gen_unique_clean_colnames_and_warn(["ab\n"])[1] == ([ cjwmodule_i18n_message( "util.colnames.warnings.ascii_cleaned", { "n_columns": 1, "first_colname": "ab" }, ), ])
def test_gen_and_warn_unicode_fixed(): assert gen_unique_clean_colnames_and_warn(["ab\ud800cd"])[1] == ([ cjwmodule_i18n_message( "util.colnames.warnings.unicode_fixed", { "n_columns": 1, "first_colname": "ab�cd" }, ), ])
def test_gen_and_warn_truncated(): assert gen_unique_clean_colnames_and_warn( ["A Column", "B Column"], settings=MockSettings(4))[1] == ([ cjwmodule_i18n_message( "util.colnames.warnings.truncated", { "n_columns": 2, "first_colname": "A Co", "n_bytes": 4 }, ), ])
def _gen_colnames_and_warn( first_colname: str, first_column: pd.Series, settings: Settings ) -> GenColnamesResult: """ Generate transposed-table column names. If `first_colname` is empty, `column.name` is the first output column. If both are empty, auto-generate the column name (and warn). Warn if ASCII-cleaning names, renaming duplicates, truncating names or auto-generating names. Assume `first_column` is text without nulls. """ input_names = [first_colname or first_column.name] input_names.extend(list(first_column.values)) names, warnings = gen_unique_clean_colnames_and_warn(input_names, settings=settings) return GenColnamesResult(names, warnings)
def test_gen_and_warn_no_warnings(): assert gen_unique_clean_colnames_and_warn(["A", "A 1", "A 2"])[1] == []
def long_to_wide( table: pd.DataFrame, key_colnames: List[str], variable_colname: str, settings: Settings, ) -> pd.DataFrame: warnings = [] varcol = table[variable_colname] if varcol.dtype != object and not hasattr(varcol, "cat"): # Convert to str, in-place warnings.append({ "message": i18n.trans( "long_to_wide.badColumn.notText.message", 'Column "{column_name}" was auto-converted to Text ' "because column names must be text.", {"column_name": variable_colname}, ), "quickFixes": [{ "text": i18n.trans( "long_to_wide.badColumn.notText.quick_fix.text", 'Convert "{column_name}" to text', {"column_name": variable_colname}, ), "action": "prependModule", "args": ["converttotext", { "colnames": [variable_colname] }], }], }) na = varcol.isnull() varcol = varcol.astype(str) varcol[na] = np.nan table[variable_colname] = varcol # Remove empty values, in-place. Empty column headers aren't allowed. # https://www.pivotaltracker.com/story/show/162648330 empty = varcol.isin([np.nan, None, ""]) n_empty = np.count_nonzero(empty) if n_empty: warnings.append( i18n.trans( "long_to_wide.badRows.emptyColumnHeaders.warning", "{n_rows, plural, " ' one {# row with empty "{column_name}" was removed.}' ' other {# rows with empty "{column_name}" were removed.}' "}", { "n_rows": n_empty, "column_name": variable_colname }, )) table = table[~empty] table.set_index(key_colnames + [variable_colname], inplace=True, drop=True) if np.any(table.index.duplicated()): return i18n.trans( "long_to_wide.error.repeatedVariables", "Some variables are repeated. Please add Row columns to uniquely " "identify each record.", ) if len(table.columns) == 0: return i18n.trans( "long_to_wide.error.noValueColumn", "There is no Value column. " "All but one table column must be a Row or Column variable.", ) if len(table.columns) > 1: return i18n.trans( "long_to_wide.error.tooManyValueColumns", "There are too many Value columns. " "All but one table column must be a Row or Column variable. " "Please drop extra columns before reshaping.", ) value_series = table[table.columns[0]] table = table.unstack() table.columns, colname_warnings = gen_unique_clean_colnames_and_warn( [col[-1] for col in table.columns.values], settings=settings) warnings.extend(colname_warnings) table.reset_index(inplace=True) for colname in list(table.columns): series = table[colname] if hasattr(series, "cat"): # Remove unused categories series.cat.remove_unused_categories(inplace=True) elif hasattr(value_series, "cat"): # Pandas 0.25, at least, nixes Categorical outright. Restore it: # if it makes sense for the input `value_series` to be Categorical, # then surely it makes sense for the _outputs_ because they are # subsets of the input. table[colname] = series.astype("category") if warnings: return (table, warnings) else: return table