def is_conjunct_filterable(self, conjunct, rule_name):
# a conjunct is filterable if it uses
# a filterable sim function (jaccard, cosine, dice, ...),
# an allowed operator (<, <=),
is_auto_gen, sim_fn, l_attr, r_attr, l_tok, r_tok, op, th = parse_conjunct(
conjunct, self.rule_ft[rule_name])
if is_auto_gen != True:
# conjunct not filterable as the feature is not auto generated
return False
if sim_fn == 'lev_dist':
if op == '>' or op == '>=':
return True
else:
# conjunct not filterable due to unsupported operator
return False
if l_tok != r_tok:
# conjunct not filterable because left and right tokenizers mismatch
return False
if sim_fn not in self.filterable_sim_fns:
# conjunct not filterable due to unsupported sim_fn
return False
if op not in self.allowed_ops:
# conjunct not filterable due to unsupported operator
return False
# conjunct is filterable
return True
def is_conjunct_filterable(self, conjunct, rule_name):
# a conjunct is filterable if it uses
# a filterable sim function (jaccard, cosine, dice, ...),
# an allowed operator (<, <=),
is_auto_gen, sim_fn, l_attr, r_attr, l_tok, r_tok, op, th = parse_conjunct(
conjunct, self.rule_ft[rule_name])
if is_auto_gen != True:
# conjunct not filterable as the feature is not auto generated
return False
if sim_fn == 'lev_dist':
if op == '>' or op == '>=':
return True
else:
# conjunct not filterable due to unsupported operator
return False
if l_tok != r_tok:
# conjunct not filterable because left and right tokenizers mismatch
return False
if sim_fn not in self.filterable_sim_fns:
# conjunct not filterable due to unsupported sim_fn
return False
if op not in self.allowed_ops:
# conjunct not filterable due to unsupported operator
return False
# conjunct is filterable
return True
def apply_filterable_rule(self, rule_name, l_df, r_df, l_key, r_key,
l_output_attrs, r_output_attrs, l_output_prefix,
r_output_prefix, verbose, show_progress,
n_chunks):
candset = None
conjunct_list = self.rule_str[rule_name]
for conjunct in conjunct_list:
is_auto_gen, sim_fn, l_attr, r_attr, l_tok, r_tok, op, th = parse_conjunct(
conjunct, self.rule_ft[rule_name])
if l_tok == 'dlm_dc0':
tokenizer = WhitespaceTokenizer(return_set=True)
elif l_tok == 'qgm_3':
tokenizer = QgramTokenizer(qval=3, return_set=True)
if sim_fn == 'jaccard':
join_fn = ssj.jaccard_join
elif sim_fn == 'cosine':
join_fn = ssj.cosine_join
elif sim_fn == 'dice':
join_fn = ssj.dice_join
elif sim_fn == 'overlap_coeff':
join_fn = ssj.overlap_coefficient_join
elif sim_fn == 'lev_dist':
join_fn = ssj.edit_distance_join
if join_fn == ssj.edit_distance_join:
comp_op = '<='
if op == '>=':
comp_op = '<'
else:
comp_op = '>='
if op == '<=':
comp_op = '>'
ssj.dataframe_column_to_str(l_df, l_attr, inplace=True)
ssj.dataframe_column_to_str(r_df, r_attr, inplace=True)
if join_fn == ssj.edit_distance_join:
c_df = join_fn(l_df, r_df, l_key, r_key, l_attr, r_attr,
float(th), comp_op, True, l_output_attrs,
r_output_attrs, l_output_prefix,
r_output_prefix, False, n_chunks, show_progress)
else:
c_df = join_fn(l_df, r_df,
l_key, r_key, l_attr, r_attr, tokenizer,
float(th), comp_op, True, True, l_output_attrs,
r_output_attrs, l_output_prefix,
r_output_prefix, False, n_chunks, show_progress)
if candset is not None:
# union the candset of this conjunct with the existing candset
candset = pd.concat([candset, c_df]).drop_duplicates(
[l_output_prefix + l_key,
r_output_prefix + r_key]).reset_index(drop=True)
else:
# candset from the first conjunct of the rule
candset = c_df
return candset
def apply_filterable_rule(self, rule_name, l_df, r_df, l_key, r_key,
l_output_attrs, r_output_attrs,
l_output_prefix, r_output_prefix,
verbose, show_progress, n_jobs):
candset = None
conjunct_list = self.rule_str[rule_name]
for conjunct in conjunct_list:
is_auto_gen, sim_fn, l_attr, r_attr, l_tok, r_tok, op, th = parse_conjunct(
conjunct, self.rule_ft[rule_name])
if l_tok == 'dlm_dc0':
tokenizer = WhitespaceTokenizer(return_set=True)
elif l_tok == 'qgm_3':
tokenizer = QgramTokenizer(qval=3, return_set=True)
if sim_fn == 'jaccard':
join_fn = ssj.jaccard_join
elif sim_fn == 'cosine':
join_fn = ssj.cosine_join
elif sim_fn == 'dice':
join_fn = ssj.dice_join
elif sim_fn == 'overlap_coeff':
join_fn = ssj.overlap_coefficient_join
elif sim_fn == 'lev_dist':
join_fn = ssj.edit_distance_join
if join_fn == ssj.edit_distance_join:
comp_op = '<='
if op == '>=':
comp_op = '<'
else:
comp_op = '>='
if op == '<=':
comp_op = '>'
ssj.dataframe_column_to_str(l_df, l_attr, inplace=True)
ssj.dataframe_column_to_str(r_df, r_attr, inplace=True)
if join_fn == ssj.edit_distance_join:
c_df = join_fn(l_df, r_df, l_key, r_key, l_attr, r_attr,
float(th), comp_op, True, l_output_attrs,
r_output_attrs, l_output_prefix,
r_output_prefix, False, n_jobs, show_progress)
else:
c_df = join_fn(l_df, r_df, l_key, r_key, l_attr, r_attr,
tokenizer, float(th), comp_op, True, True,
l_output_attrs, r_output_attrs,
l_output_prefix,
r_output_prefix, False, n_jobs, show_progress)
if candset is not None:
# union the candset of this conjunct with the existing candset
candset = pd.concat([candset, c_df]).drop_duplicates(
[l_output_prefix + l_key,
r_output_prefix + r_key]).reset_index(drop=True)
else:
# candset from the first conjunct of the rule
candset = c_df
return candset
def get_attrs_to_project(self, l_key, r_key, l_output_attrs,
r_output_attrs):
l_proj_attrs = [l_key]
r_proj_attrs = [r_key]
if l_output_attrs:
l_proj_attrs.extend(
[c for c in l_output_attrs if c not in l_proj_attrs])
if r_output_attrs:
r_proj_attrs.extend(
[c for c in r_output_attrs if c not in r_proj_attrs])
for rule_name, conjunct_list in six.iteritems(self.rule_str):
for conjunct in conjunct_list:
is_auto_gen, sim_fn, l_attr, r_attr, l_tok, r_tok, op, th = parse_conjunct(
conjunct, self.rule_ft[rule_name])
if l_attr not in l_proj_attrs:
l_proj_attrs.append(l_attr)
if r_attr not in r_proj_attrs:
r_proj_attrs.append(r_attr)
return l_proj_attrs, r_proj_attrs
def get_attrs_to_project(self, l_key, r_key, l_output_attrs,
r_output_attrs):
l_proj_attrs = [l_key]
r_proj_attrs = [r_key]
if l_output_attrs:
l_proj_attrs.extend(
[c for c in l_output_attrs if c not in l_proj_attrs])
if r_output_attrs:
r_proj_attrs.extend(
[c for c in r_output_attrs if c not in r_proj_attrs])
for rule_name, conjunct_list in six.iteritems(self.rule_str):
for conjunct in conjunct_list:
is_auto_gen, sim_fn, l_attr, r_attr, l_tok, r_tok, op, th = parse_conjunct(
conjunct, self.rule_ft[rule_name])
if l_attr not in l_proj_attrs:
l_proj_attrs.append(l_attr)
if r_attr not in r_proj_attrs:
r_proj_attrs.append(r_attr)
return l_proj_attrs, r_proj_attrs
def execute(self, input_table, label_column, inplace=True, verbose=False):
""" Executes the rules of the match trigger for a table of matcher
results.
Args:
input_table (DataFrame): The input table of type pandas DataFrame
containing tuple pairs and labels from matching (defaults to None).
label_column (string): The attribute name where the predictions
are stored in the input table (defaults to None).
inplace (boolean): A flag to indicate whether the append needs to be
done inplace (defaults to True).
verbose (boolean): A flag to indicate whether the debug information
should be logged (defaults to False).
Returns:
A DataFrame with predictions updated.
Examples:
>>> import py_entitymatching as em
>>> mt = em.MatchTrigger()
>>> A = em.read_csv_metadata('path_to_csv_dir/table_A.csv', key='id')
>>> B = em.read_csv_metadata('path_to_csv_dir/table_B.csv', key='id')
>>> match_f = em.get_features_for_matching(A, B)
>>> rule = ['title_title_lev_sim(ltuple, rtuple) > 0.7']
>>> mt.add_cond_rule(rule, match_f)
>>> mt.add_cond_status(True)
>>> mt.add_action(1)
>>> # The table H is a table with prediction labels generated from matching
>>> mt.execute(input_table=H, label_column='predicted_labels', inplace=False)
"""
# Validate input parameters
# # We expect the table to be of type pandas DataFrame
validate_object_type(input_table, pd.DataFrame, 'Input table')
# # We expect the target_attr to be of type string if not None
if label_column is not None and not isinstance(label_column, str):
logger.error('Input target_attr must be a string.')
raise AssertionError('Input target_attr must be a string.')
# # We expect the inplace to be of type boolean
validate_object_type(inplace, bool, 'Input inplace')
# # We expect the verbose to be of type boolean
validate_object_type(verbose, bool, 'Input append')
# Validate that there are some rules
assert len(self.rules.keys()) > 0, 'There are no rules to apply'
# # get metadata
key, fk_ltable, fk_rtable, ltable, rtable, l_key, r_key = cm.get_metadata_for_candset(
input_table, logger, verbose)
# # validate metadata
cm._validate_metadata_for_candset(input_table, key, fk_ltable,
fk_rtable, ltable, rtable, l_key,
r_key, logger, verbose)
assert ltable is not None, 'Left table is not set'
assert rtable is not None, 'Right table is not set'
assert label_column in input_table.columns, 'Label column not in the input table'
# Parse conjuncts to validate that the features are in the feature table
for rule in self.rule_conjunct_list:
for conjunct in self.rule_conjunct_list[rule]:
parse_conjunct(conjunct, self.rule_ft[rule])
if inplace == False:
table = input_table.copy()
else:
table = input_table
# set the index and store it in l_tbl/r_tbl
l_tbl = ltable.set_index(l_key, drop=False)
r_tbl = rtable.set_index(r_key, drop=False)
# keep track of valid ids
y = []
column_names = list(input_table.columns)
lid_idx = column_names.index(fk_ltable)
rid_idx = column_names.index(fk_rtable)
label_idx = column_names.index(label_column)
idx = 0
for row in input_table.itertuples(index=False):
if row[label_idx] != self.value_to_set:
l_row = l_tbl.loc[row[lid_idx]]
r_row = r_tbl.loc[row[rid_idx]]
res = self.apply_rules(l_row, r_row)
if res == self.cond_status:
table.iat[idx, label_idx] = self.value_to_set
idx += 1
return table
def predict(self, table=None, target_attr=None, append=False, inplace=True):
"""Predict interface for the matcher.
A point to note is all the input parameters have a default value of
None.
Args:
table (DataFrame): The input candidate set of type pandas DataFrame
containing tuple pairs (defaults to None).
target_attr (string): The attribute name where the predictions
need to be stored in the input table (defaults to None).
append (boolean): A flag to indicate whether the predictions need
to be appended in the input DataFrame (defaults to False).
return_probs (boolean): A flag to indicate where the prediction probabilities
need to be returned (defaults to False). If set to True, returns the
probability if the pair was a match.
inplace (boolean): A flag to indicate whether the append needs to be
done inplace (defaults to True).
Returns:
An array of predictions or a DataFrame with predictions updated.
Examples:
>>> import py_entitymatching as em
>>> brm = em.BooleanRuleMatcher()
>>> A = em.read_csv_metadata('path_to_csv_dir/table_A.csv', key='id')
>>> B = em.read_csv_metadata('path_to_csv_dir/table_B.csv', key='id')
>>> match_f = em.get_features_for_matching(A, B)
>>> rule = ['address_address_lev(ltuple, rtuple) > 6']
>>> brm.add_rule(rule, match_f)
>>> # The table S is a cand set generated by the blocking and then labeling phases
>>> brm.predict(S, target_attr='pred_label', append=True)
"""
# Validate input parameters
# # We expect the table to be of type pandas DataFrame
validate_object_type(table, pd.DataFrame, 'Input table')
# # We expect the target_attr to be of type string if not None
if target_attr is not None and not isinstance(target_attr, str):
logger.error('Input target_attr must be a string.')
raise AssertionError('Input target_attr must be a string.')
# # We expect the append to be of type boolean
validate_object_type(append, bool, 'Input append')
# # We expect the inplace to be of type boolean
validate_object_type(inplace, bool, 'Input inplace')
# # get metadata
key, fk_ltable, fk_rtable, ltable, rtable, l_key, r_key = cm.get_metadata_for_candset(
table, logger, False)
# # validate metadata
cm._validate_metadata_for_candset(table, key, fk_ltable, fk_rtable,
ltable, rtable, l_key, r_key,
logger, False)
# Validate that there are some rules
assert len(self.rules.keys()) > 0, 'There are no rules to apply'
# Parse conjuncts to validate that the features are in the feature table
for rule in self.rule_conjunct_list:
for conjunct in self.rule_conjunct_list[rule]:
parse_conjunct(conjunct, self.rule_ft[rule])
if table is not None:
y = self._predict_candset(table)
if target_attr is not None and append is True:
if inplace == True:
table[target_attr] = y
return table
else:
tbl = table.copy()
tbl[target_attr] = y
return tbl
else:
return y
else:
raise SyntaxError('The arguments supplied does not match the signatures supported !!!')
def predict(self,
table=None,
target_attr=None,
append=False,
inplace=True):
"""Predict interface for the matcher.
A point to note is all the input parameters have a default value of
None.
Args:
table (DataFrame): The input candidate set of type pandas DataFrame
containing tuple pairs (defaults to None).
target_attr (string): The attribute name where the predictions
need to be stored in the input table (defaults to None).
append (boolean): A flag to indicate whether the predictions need
to be appended in the input DataFrame (defaults to False).
return_probs (boolean): A flag to indicate where the prediction probabilities
need to be returned (defaults to False). If set to True, returns the
probability if the pair was a match.
inplace (boolean): A flag to indicate whether the append needs to be
done inplace (defaults to True).
Returns:
An array of predictions or a DataFrame with predictions updated.
Examples:
>>> import py_entitymatching as em
>>> brm = em.BooleanRuleMatcher()
>>> A = em.read_csv_metadata('path_to_csv_dir/table_A.csv', key='id')
>>> B = em.read_csv_metadata('path_to_csv_dir/table_B.csv', key='id')
>>> match_f = em.get_features_for_matching(A, B)
>>> rule = ['address_address_lev(ltuple, rtuple) > 6']
>>> brm.add_rule(rule, match_f)
>>> # The table S is a cand set generated by the blocking and then labeling phases
>>> brm.predict(S, target_attr='pred_label', append=True)
"""
# Validate input parameters
# # We expect the table to be of type pandas DataFrame
validate_object_type(table, pd.DataFrame, 'Input table')
# # We expect the target_attr to be of type string if not None
if target_attr is not None and not isinstance(target_attr, str):
logger.error('Input target_attr must be a string.')
raise AssertionError('Input target_attr must be a string.')
# # We expect the append to be of type boolean
validate_object_type(append, bool, 'Input append')
# # We expect the inplace to be of type boolean
validate_object_type(inplace, bool, 'Input inplace')
# # get metadata
key, fk_ltable, fk_rtable, ltable, rtable, l_key, r_key = cm.get_metadata_for_candset(
table, logger, False)
# # validate metadata
cm._validate_metadata_for_candset(table, key, fk_ltable, fk_rtable,
ltable, rtable, l_key, r_key, logger,
False)
# Validate that there are some rules
assert len(self.rules.keys()) > 0, 'There are no rules to apply'
# Parse conjuncts to validate that the features are in the feature table
for rule in self.rule_conjunct_list:
for conjunct in self.rule_conjunct_list[rule]:
parse_conjunct(conjunct, self.rule_ft[rule])
if table is not None:
y = self._predict_candset(table)
if target_attr is not None and append is True:
if inplace == True:
table[target_attr] = y
return table
else:
tbl = table.copy()
tbl[target_attr] = y
return tbl
else:
return y
else:
raise SyntaxError(
'The arguments supplied does not match the signatures supported !!!'
)