# because it varies within household. ("recently bought this house", cla.MeanBounds(0, 0.01)), ("female head", cla.MeanBounds(0.25, 0.55)), ("seguro de riesgos laborales", cla.MeanBounds(0.3, 0.6)) ]: assert test.test(hh[c]) if True: # IO log = "starting\n" # hh = oio.readStage(com.subsample, "households_1_agg_plus." + com.strategy_year_suffix) ppl = oio.readStage(com.subsample, "people_3_income_taxish." + com.strategy_year_suffix) hh["edu-max"] = util.interpretCategorical(hh["edu-max"], edu_key.values()) ppl["edu"] = util.interpretCategorical(ppl["edu"], edu_key.values()) test_const_within_group( # TODO ? move this test to the tests of person data gs=["household"], cs=defs.cols_const_within_hh, d=hh) test_indices(hh=hh, ppl=ppl) test_income_ranks(hh=hh, ppl=ppl) test_sums(hh=hh, ppl=ppl) test_bools(hh=hh, ppl=ppl) com_tests.test_quantiles(df=hh) oio.test_write(com.subsample, "households_1_agg_plus", log)
assert not go([True, True]) def test_next_request(): cols = [ "user email", "user", "subsample", "completed", "time completed", "time requested" ] df = pd.DataFrame([ ["1", "1", 100, False, 99, 99], ["2", "2", 100, False, 6, 6], ["3", "3", 100, True, 7, 7], ["4", "4", 100, True, 8, 8], ], columns=cols) assert r.next_request(df)["user"] == "2" if True: test_memory_permits_another_run() test_delete_oldest_request() test_at_least_one_result_is_old() test_uniquify_requests() test_unexecuted_requests_exist() test_next_request() # oio.test_write( 1, # PTIFALL: Uses no data, so always writes to recip-1/ "requests", "")
assert util.near( 0, 1, tol_frac = 0, tol_abs = 2 ) assert not util.near( 0, 1, tol_frac = 0, tol_abs = 1/2 ) assert util.near( 20, 21, tol_frac = 0.1, tol_abs = 0 ) assert not util.near( 20, 23, tol_frac = 0.1, tol_abs = 0 ) def test_tuple_by_threshold(): sched = [(0,"a","b")] for income in [-1,0,1,1e11]: assert util.tuple_by_threshold( income, sched ) == sched[0] sched = [ (0,1,2), (10,"whatever","something") ] for income in [(-1,0,1,9)]: assert util.tuple_by_threshold( income, sched ) == sched[0] for income in [(10,11,1e11)]: assert util.tuple_by_threshold( income, sched ) == sched[1] def test_util_pad_column_as_int(): c = pd.Series( [2, "2","2.0",np.nan] ) assert pd.Series.equals( util.pad_column_as_int ( 4, c ) , pd.Series( ["0002","0002","0002",np.nan] ) ) if True: # run tests log = "starting\n" test_near() test_util_pad_column_as_int() oio.test_write( cl.subsample , "common_util" , log )
assert set( df.columns ) == set( [code_column_name, 'vat', 'vat frac'] ) for c in df.columns: if c == code_column_name: assert df[c].dtype == "int64" else: assert df[c].dtype == "float64" # The "vat" and "vat frac" columns might have a few missing values. # The others should have none. assert ( ( len( non_null_part( df["vat"] ) ) / len( df ) ) > ( fraction_that_should_be_non_null - tolerance ) ) assert ( len( non_null_part( df["vat"] ) ) == len( non_null_part( df["vat frac"] ) ) ) df = df.drop( columns = ["vat", "vat frac"] ) for c in df.columns: assert len( df[ ~ pd.isnull( df[c] ) ] ) == len( df ) if True: # run tests log = "starting\n" test_vat_file( "vat_coicop_brief", "coicop", 1031 / 1051 ) test_vat_file( "vat_cap_c_brief", "25-broad-categs", 20 / 25 ) oio.test_write( cl.subsample , "vat_rates" , log )
for (c, t) in [("recently bought this house", cla.InSet({True, False})), ("recently bought this house", cla.CoversRange(0, 1)), ("recently bought this house", cla.MeanBounds(0, 0.01)), ("recently bought this house", cla.MissingAtMost(0)), ("estrato", cla.InRange(0, 6)), ("estrato", cla.CoversRange(0, 3)), ("estrato", cla.MeanBounds(1.5, 2.5)), ("estrato", cla.MissingAtMost(0.02))]: assert t.test(bs[c]) if True: # run tests log = "starting\n" bs = oio.readStage( 1 # PITFALL: For buildings, we always use the full sample. , 'buildings', dtype={"estrato": 'float64'} # If subsample is so small that there are no missing values, # "estrato" will by default be read as "int64". ) test_types(bs) test_nullity(bs) test_ranges(bs) assert (unique(bs.columns)) for ss in com.valid_subsamples: # PITFALL: Looping over subsample sizes because this program # always uses the full sample. # If it works, it works for all subsamples. oio.test_write(ss, "build_buildings", log)
# But it's surprising, because for subsample = 10, # the reality is much less than the expectation. assert (set(df.columns) == set(Purchase_2_Columns_missing.all_columns())) # coicop and 25-broad-categs are each individually missing substantially, # but exactly one of them is always present assert len(df[(~pd.isnull(df["coicop"])) & (~pd.isnull(df["25-broad-categs"]))]) == 0 assert len(df[(pd.isnull(df["coicop"])) | (pd.isnull(df["25-broad-categs"]))]) == len(df) for c in Purchase_2_Columns_missing.never: assert (len(df[pd.isnull(df[c])]) == 0) for c in Purchase_2_Columns_missing.slightly: assert ((len(df[pd.isnull(df[c])]) / len(df)) < 0.03) for c in Purchase_2_Columns_missing.very: assert ((len(df[pd.isnull(df[c])]) / len(df)) < 0.25) return log if True: # IO log = "starting\n" ps = oio.readStage(com.subsample, "purchases_2_vat." + com.strategy_suffix) log += test_ranges(ps) log += test_output(ps) oio.test_write(com.subsample, "build_purchases_2_vat", log)
( "value, non-purchase", [ cla.MeanBounds (1e6,1e7), cla.CoversRange (0 ,1e6), cla.InRange (0 ,3.3e9), cla.MissingAtMost (0) ] ), ( "value, purchase", [ cla.MeanBounds (1e6 ,5e6), cla.CoversRange (1e2 ,4e7), # TODO ? This minimum is nuts. cla.InRange (0 ,2e8), cla.MissingAtMost (0) ] ), ( "value, spending", [ cla.MeanBounds (1e6 ,5e6), cla.CoversRange (1e2 ,4e7), # TODO ? This minimum is nuts. cla.InRange (0 ,2e8), cla.MissingAtMost(0) ] ) ]: for t in ts: assert t.test( sums[c] ) assert sums["household"].is_unique assert util.near( len(sums), num_households / com.subsample, tol_frac = 1/5 ) oio.test_write( com.subsample, "build_purchase_sums", "It worked." )
# TODO : extend to all the old variables def test_means(ppl: pd.DataFrame) -> None: for (col, theMin, theMax) in [ ("used savings", 0.005, 0.05), ("empleado", 0.20, 0.6), ("desempleado", 0.03, 0.12), ("in labor force", 0.25, 0.6), ]: x = ppl[col].mean() assert (x >= theMin) & (x <= theMax) if True: # run tests log = "starting\n" # unit tests test_count_num_matches_in_space_separated_list() # integration tests ppl = oio.readStage(com.subsample, 'people_1') ppl["edu"] = util.interpretCategorical(ppl["edu"], files.edu_key.values()) test_ranges(ppl) test_upper_bound_on_fraction_missing(ppl) test_means(ppl) assert util.near(len(ppl), num_people / com.subsample, tol_frac=1 / 5) oio.test_write(com.subsample, "people_main", log)
if True: for k,v in { "vat / purchase value" : cl.InRange( 0, 0.3 ), # The special motorcycle tax, abusivelyed lump into the VAT table, # means the max "vat" is 0.27 rather than 0.19. "vat / income" : cl.InRange( 0, np.inf ), "purchase value / income" : cl.InRange( 0, np.inf ) }.items(): assert v.test( merge[k] ) for k,v in { # These bounds could be tighter, # but the 1/1000 subsample has a small range. "vat / purchase value" : cl.CoversRange( 0, 0.1 ), "vat / income" : cl.CoversRange( 0, np.inf ), "purchase value / income" : cl.CoversRange( 0.2, np.inf ) }.items(): assert v.test( merge[k] ) for k,v in { "vat / purchase value" : cl.MeanBounds( 2.5e-2, 6e-2 ), "vat / income" : cl.MeanBounds( np.inf, np.inf ), "purchase value / income" : cl.MeanBounds( np.inf, np.inf ) }.items(): assert v.test( merge[k] ) for c in new_cols: assert cl.MissingAtMost( 0.01 ) . test( merge[c] ) oio.test_write( com.subsample, "households_2_purchases", "It worked." )
log += "Very few missing quantity values." assert ((1e-5) > (len(df[pd.isnull(df["quantity"])]) / len(df))) log += "Very few negative quantity values." assert ((1e-5) > (len(df[df["quantity"] <= 0]) / len(df))) log += "Negative quantity purchases are for very little money." assert (df[df["quantity"] < 0]["value"] < 1e4).all() log += "Very few purchases with a frequency of \"never\"." assert ((1e-5) > (len(df[df["per month"] > 10]) / len(df))) log += "Those few frequency=\"never\" purchases are for very little money." assert (df[df["per month"] > 10]["value"] < 1e4).all() return log if True: # run the tests log = "starting\n" # unit tests log += test_drop_if_coicop_or_value_invalid() log += test_drop_absurdly_big_expenditures() # integration test df = oio.readStage(com.subsample, 'purchases_1') log += test_output(df) oio.test_write(com.subsample, "purchases_correct", log)
assert near( reg.taxable(r), r[reg.gravable_pre] * (1 - 0.325) ) # The dependent exemption knocks off 10% from what's left. r["claims dependent (labor income tax)"] = True assert near( reg.taxable(r), r[reg.gravable_pre] * (1 - 0.325) * 0.9 ) if True: # For high earners (20000 muvt is around 50 million / month) # and you have no dependents, 5040 muvt is exempted. r = pd.Series( { "claims dependent (labor income tax)" : False, reg.gravable_pre : 20000 * muvt } ) assert near( reg.taxable(r), r[reg.gravable_pre] - 5040 * muvt ) # The dependent exemption knocks off another 32 muvt. r["claims dependent (labor income tax)"] = True assert near( reg.taxable(r), r[reg.gravable_pre] - 5072 * muvt ) if True: test_most_income_tax() test_taxable() log = str( datetime.datetime.now() ) for ss in common.valid_subsamples: # PITFALL: Looping over subsample sizes because this program # uses no data. # If it works, it works for all subsamples. oio.test_write( ss , "regime_r2018" , log )
}))) def test_File(): f = cla.File("sassafrass", "sassafrass.csv", [("ugly input.csv", "dirt", "beautiful output.csv", "gold")]) assert (cla.name_map(f.col_specs) == { "ugly input.csv": "beautiful output.csv" }) assert (cla.input_map(f.col_specs) == {"ugly input.csv": "dirt"}) assert (cla.output_map(f.col_specs) == {"beautiful output.csv": "gold"}) if True: # run the tests log = "starting\n" test_Correction() test_File() test_MeanBounds() test_Property_subclasses() test_re_c() test_re_digits() test_re_gt1c() test_re_gt1p() test_re_nonNumeric() test_re_p() test_re_white() test_stringProperties() oio.test_write(com.subsample, "build_classes", log)
# import python.build.classes as cla import python.build.output_io as oio import python.common.common as common import python.common.misc as misc def test_coicop_data(df: pd.DataFrame): for t in [cla.InRange(1e6, 2e7), cla.CoversRange(2e6, 1e7)]: assert t.test(df["coicop"]) for col in ["vat", "vat, min", "vat, max"]: assert (cla.InRange(0, 1).test(df[col])) def test_capitulo_c_data(df: pd.DataFrame): for t in [cla.InRange(1, 25), cla.CoversRange(1, 25)]: assert t.test(df["CODE"]) for col in ["vat", "vat, min", "vat, max"]: assert (cla.InRange(0, 1).test(df[col])) if True: test_coicop_data(misc.read_csv_or_xlsx("config/vat/vat_by_coicop.csv")) test_capitulo_c_data( misc.read_csv_or_xlsx("config/vat/vat_by_capitulo_c.csv")) oio.test_write( 1, # PTIFALL: Uses no sample-size-dependent data, # so always writes to recip-1/ "rate_input", "")
assert util.unique(out.columns) assert util.unique(new_cols) assert set.intersection(cols1, new_cols) == set() assert set.union(cols1, new_cols) == cols2 assert set.difference(cols2, cols1) == new_cols assert len(in_rows) == len(out) assert util.near(len(out), misc.num_people / com.subsample, tol_frac=1 / 5) per_cell_spec = { "age-decile": cl.InRange(0, 9), "income-decile": cl.InRange(0, 9), "female head": cl.InRange(0, 1) } per_column_spec = { "age-decile": cl.CoversRange(0, 9), "income-decile": cl.CoversRange(0, 9), "female head": cl.CoversRange(0, 1) } for k, v in per_cell_spec.items(): assert v.test(out[k]) for k, v in per_column_spec.items(): assert v.test(out[k]) oio.test_write(com.subsample, "people_2_buildings", "It worked.")
assert near(t(employee, 100 * min_wage), 0.04 * 25 * min_wage) def test_mk_cesantias_y_primas_employer(): t = sf.mk_cesantias_y_primas_employer if True: # for contractors, always 0 assert near(t(contractor, 0), 0) assert near(t(contractor, 1000 * min_wage), 0) if True: # for employees assert near(t(employee, 0.5 * min_wage), 0) assert near(t(employee, 1.1 * min_wage), (2.12 / 12) * 1.1 * min_wage) assert near(t(employee, 12 * min_wage), (2.12 / 12) * 12 * min_wage) assert near(t(employee, 14 * min_wage), 0) assert near(t(employee, 100 * min_wage), 0) if True: log = str(datetime.datetime.now()) test_mk_pension() test_mk_pension_employer() test_mk_salud() test_mk_salud_employer() test_mk_solidaridad() test_mk_parafiscales_employer() test_mk_cajas_de_compensacion_employer() test_mk_cesantias_y_primas_employer() for ss in common.valid_subsamples: # PITFALL: Looping over subsample sizes because this program # doesn't use any data. If it works, it works for all subsamples. oio.test_write(ss, "build_ss_functions", log)
import python.build.output_io as oio import python.build.purchases.articulos as articulos import python.build.purchases.capitulo_c as capitulo_c # import python.build.purchases.medios as medios import python.build.purchases.nice_purchases as nice_purchases import python.common.util as util import python.common.common as common def test_purchase_inputs(): for f in (articulos.files # + medios.files + capitulo_c.files + nice_purchases.files): df = common.retrieve_file( f, subsample=1) # PITFALL: Always the full sample. assert util.unique(df.columns) acc = {} for c in df.columns: acc.update([(c, cla.stringProperties(df[c]))]) assert acc[c] == cla.input_map(f.col_specs)[c] if True: # run tests log = "starting\n" test_purchase_inputs() for ss in common.valid_subsamples: # PITFALL: Looping over subsample sizes because this program # always uses the full sample. # If it works, it works for all subsamples. oio.test_write(subsample=ss, filename="purchase_inputs", content=log)
htemp = (hh2[["household"] + same_ratio].rename(columns=same_ratio_hh_dict)) ptemp = ps4.merge(htemp[["household"] + same_ratio_hh_list], on="household") ptemp = ( # To make sure none of the ratios is infinite. ptemp[(ptemp["income"] > 0) & (ptemp["value, purchase"] > 0)]) for c in same_ratio: assert (ptemp[c + "-hh"] > 0.99 * ptemp[c] - 1).all() assert (ptemp[c + "-hh"] < 1.01 * ptemp[c] + 1).all() log = log + "\n" + c + "has the same values in ps4 as in hh2." del (htemp, ptemp) for (c, m) in [ ("in labor force", 0), ("share", 0), ("one", 0), ("income-decile", 0), ("income-percentile", 0), ("vat / purchase value", 0.1), ("vat / income", 0.1), ("purchase value / income", 0.1), ]: # PITFALL: Income and purcahse value is zero for far more rows of people_4 than for households_2. # The reason for the first is that many households have positive income even though some earners in the household don't. # The reason for the second is that "value, purchase" is zero for anyone with zero income in a household with positive income. # (If everyone in the household has zero income then purchase value is instead divided equally among earners.) assert ps4[c].isnull().mean() <= m log = log + "\n" + c + " is missing no more than " + str(100 * m) + "%." com_tests.test_quantiles(df=ps4) oio.test_write(com.subsample, "people_4_post_households", log)