def test_main_api(spark, sqlite_con_1):
settings = {
"link_type": "dedupe_only",
"comparison_columns": [{
"col_name": "surname"
}, {
"col_name": "mob"
}],
"blocking_rules": ["l.mob = r.mob", "l.surname = r.surname"],
"max_iterations": 2
}
settings = complete_settings_dict(settings, spark=None)
dfpd = pd.read_sql("select * from test1", sqlite_con_1)
df = spark.createDataFrame(dfpd)
linker = Splink(settings, spark, df=df)
df_e = linker.get_scored_comparisons()
linker.save_model_as_json("saved_model.json", overwrite=True)
linker_2 = load_from_json("saved_model.json", spark=spark, df=df)
df_e = linker_2.get_scored_comparisons()
from splink.intuition import intuition_report
params = linker.params
row_dict = df_e.toPandas().sample(1).to_dict(orient="records")[0]
print(intuition_report(row_dict, params))
linker.params._print_m_u_probs()
def test_main_api(spark):
rows = [
{
"unique_id": 1,
"mob": 10,
"surname": "Linacre"
},
{
"unique_id": 2,
"mob": 10,
"surname": "Linacre"
},
{
"unique_id": 3,
"mob": 10,
"surname": "Linacer"
},
{
"unique_id": 4,
"mob": 7,
"surname": "Smith"
},
{
"unique_id": 5,
"mob": 8,
"surname": "Smith"
},
{
"unique_id": 6,
"mob": 8,
"surname": "Smith"
},
{
"unique_id": 7,
"mob": 8,
"surname": "Jones"
},
]
df = spark.createDataFrame(Row(**x) for x in rows)
settings = {
"link_type": "dedupe_only",
"comparison_columns": [{
"col_name": "surname"
}, {
"col_name": "mob"
}],
"blocking_rules": ["l.mob = r.mob", "l.surname = r.surname"],
"max_iterations": 1,
}
linker = Splink(settings, df, spark)
df_e = linker.get_scored_comparisons()
linker.save_model_as_json("saved_model.json", overwrite=True)
linker_2 = load_from_json("saved_model.json", df, spark=spark)
df_e = linker_2.get_scored_comparisons()
model = linker.model
row_dict = df_e.toPandas().sample(1).to_dict(orient="records")[0]
intuition_report(row_dict, model)
bayes_factor_chart(row_dict, model)