def filter_tables(self, ltable, rtable,
l_key_attr, r_key_attr,
l_filter_attr, r_filter_attr,
l_out_attrs=None, r_out_attrs=None,
l_out_prefix='l_', r_out_prefix='r_',
out_sim_score=False, n_jobs=1, show_progress=True):
"""Finds candidate matching pairs of strings from the input tables using
overlap filtering technique.
Args:
ltable (DataFrame): left input table.
rtable (DataFrame): right input table.
l_key_attr (string): key attribute in left table.
r_key_attr (string): key attribute in right table.
l_filter_attr (string): attribute in left table on which the filter
should be applied.
r_filter_attr (string): attribute in right table on which the filter
should be applied.
l_out_attrs (list): list of attribute names from the left table to
be included in the output table (defaults to None).
r_out_attrs (list): list of attribute names from the right table to
be included in the output table (defaults to None).
l_out_prefix (string): prefix to be used for the attribute names
coming from the left table, in the output table
(defaults to 'l\_').
r_out_prefix (string): prefix to be used for the attribute names
coming from the right table, in the output table
(defaults to 'r\_').
out_sim_score (boolean): flag to indicate whether the overlap score
should be included in the output table (defaults to True).
Setting this flag to True will add a column named '_sim_score'
in the output table. This column will contain the overlap scores
for the tuple pairs in the output.
n_jobs (int): number of parallel jobs to use for the computation
(defaults to 1). If -1 is given, all CPUs are used. If 1 is
given, no parallel computing code is used at all, which is
useful for debugging. For n_jobs below -1,
(n_cpus + 1 + n_jobs) are used (where n_cpus is the total
number of CPUs in the machine). Thus for n_jobs = -2, all CPUs
but one are used. If (n_cpus + 1 + n_jobs) becomes less than 1,
then no parallel computing code will be used (i.e., equivalent
to the default).
show_progress (boolean): flag to indicate whether task progress
should be displayed to the user (defaults to True).
Returns:
An output table containing tuple pairs that survive the filter
(DataFrame).
"""
# check if the input tables are dataframes
validate_input_table(ltable, 'left table')
validate_input_table(rtable, 'right table')
# check if the key attributes and filter attributes exist
validate_attr(l_key_attr, ltable.columns,
'key attribute', 'left table')
validate_attr(r_key_attr, rtable.columns,
'key attribute', 'right table')
validate_attr(l_filter_attr, ltable.columns,
'attribute', 'left table')
validate_attr(r_filter_attr, rtable.columns,
'attribute', 'right table')
# check if the filter attributes are not of numeric type
validate_attr_type(l_filter_attr, ltable[l_filter_attr].dtype,
'attribute', 'left table')
validate_attr_type(r_filter_attr, rtable[r_filter_attr].dtype,
'attribute', 'right table')
# check if the output attributes exist
validate_output_attrs(l_out_attrs, ltable.columns,
r_out_attrs, rtable.columns)
# check if the key attributes are unique and do not contain
# missing values
validate_key_attr(l_key_attr, ltable, 'left table')
validate_key_attr(r_key_attr, rtable, 'right table')
# remove redundant attrs from output attrs.
l_out_attrs = remove_redundant_attrs(l_out_attrs, l_key_attr)
r_out_attrs = remove_redundant_attrs(r_out_attrs, r_key_attr)
# get attributes to project.
l_proj_attrs = get_attrs_to_project(l_out_attrs,
l_key_attr, l_filter_attr)
r_proj_attrs = get_attrs_to_project(r_out_attrs,
r_key_attr, r_filter_attr)
# Do a projection on the input dataframes to keep only the required
# attributes. Then, remove rows with missing value in filter attribute
# from the input dataframes. Then, convert the resulting dataframes
# into ndarray.
ltable_array = convert_dataframe_to_array(ltable, l_proj_attrs,
l_filter_attr)
rtable_array = convert_dataframe_to_array(rtable, r_proj_attrs,
r_filter_attr)
# computes the actual number of jobs to launch.
n_jobs = min(get_num_processes_to_launch(n_jobs), len(rtable_array))
if n_jobs <= 1:
# if n_jobs is 1, do not use any parallel code.
output_table = _filter_tables_split(
ltable_array, rtable_array,
l_proj_attrs, r_proj_attrs,
l_key_attr, r_key_attr,
l_filter_attr, r_filter_attr,
self,
l_out_attrs, r_out_attrs,
l_out_prefix, r_out_prefix,
out_sim_score, show_progress)
else:
# if n_jobs is above 1, split the right table into n_jobs splits and
# filter each right table split with the whole of left table in a
# separate process.
r_splits = split_table(rtable_array, n_jobs)
results = Parallel(n_jobs=n_jobs)(delayed(_filter_tables_split)(
ltable_array, r_splits[job_index],
l_proj_attrs, r_proj_attrs,
l_key_attr, r_key_attr,
l_filter_attr, r_filter_attr,
self,
l_out_attrs, r_out_attrs,
l_out_prefix, r_out_prefix,
out_sim_score,
(show_progress and (job_index==n_jobs-1)))
for job_index in range(n_jobs))
output_table = pd.concat(results)
# If allow_missing flag is set, then compute all pairs with missing
# value in at least one of the filter attributes and then add it to the
# output obtained from applying the filter.
if self.allow_missing:
missing_pairs = get_pairs_with_missing_value(
ltable, rtable,
l_key_attr, r_key_attr,
l_filter_attr, r_filter_attr,
l_out_attrs, r_out_attrs,
l_out_prefix, r_out_prefix,
out_sim_score, show_progress)
output_table = pd.concat([output_table, missing_pairs])
# add an id column named '_id' to the output table.
output_table.insert(0, '_id', range(0, len(output_table)))
return output_table
def dice_join(ltable,
rtable,
l_key_attr,
r_key_attr,
l_join_attr,
r_join_attr,
tokenizer,
threshold,
comp_op='>=',
allow_empty=True,
allow_missing=False,
l_out_attrs=None,
r_out_attrs=None,
l_out_prefix='l_',
r_out_prefix='r_',
out_sim_score=True,
n_jobs=1,
show_progress=True):
"""Join two tables using Dice similarity measure.
For two sets X and Y, the Dice similarity score between them is given by:
:math:`dice(X, Y) = \\frac{2 * |X \\cap Y|}{|X| + |Y|}`
In the case where both X and Y are empty sets, we define their Dice
score to be 1.
Finds tuple pairs from left table and right table such that the Dice
similarity between the join attributes satisfies the condition on input
threshold. For example, if the comparison operator is '>=', finds tuple
pairs whose Dice similarity between the strings that are the values of
the join attributes is greater than or equal to the input threshold, as
specified in "threshold".
Args:
ltable (DataFrame): left input table.
rtable (DataFrame): right input table.
l_key_attr (string): key attribute in left table.
r_key_attr (string): key attribute in right table.
l_join_attr (string): join attribute in left table.
r_join_attr (string): join attribute in right table.
tokenizer (Tokenizer): tokenizer to be used to tokenize join
attributes.
threshold (float): Dice similarity threshold to be satisfied.
comp_op (string): comparison operator. Supported values are '>=', '>'
and '=' (defaults to '>=').
allow_empty (boolean): flag to indicate whether tuple pairs with empty
set of tokens in both the join attributes should be included in the
output (defaults to True).
allow_missing (boolean): flag to indicate whether tuple pairs with
missing value in at least one of the join attributes should be
included in the output (defaults to False). If this flag is set to
True, a tuple in ltable with missing value in the join attribute
will be matched with every tuple in rtable and vice versa.
l_out_attrs (list): list of attribute names from the left table to be
included in the output table (defaults to None).
r_out_attrs (list): list of attribute names from the right table to be
included in the output table (defaults to None).
l_out_prefix (string): prefix to be used for the attribute names coming
from the left table, in the output table (defaults to 'l\_').
r_out_prefix (string): prefix to be used for the attribute names coming
from the right table, in the output table (defaults to 'r\_').
out_sim_score (boolean): flag to indicate whether similarity score
should be included in the output table (defaults to True). Setting
this flag to True will add a column named '_sim_score' in the
output table. This column will contain the similarity scores for the
tuple pairs in the output.
n_jobs (int): number of parallel jobs to use for the computation
(defaults to 1). If -1 is given, all CPUs are used. If 1 is given,
no parallel computing code is used at all, which is useful for
debugging. For n_jobs below -1, (n_cpus + 1 + n_jobs) are used
(where n_cpus is the total number of CPUs in the machine). Thus for
n_jobs = -2, all CPUs but one are used. If (n_cpus + 1 + n_jobs)
becomes less than 1, then no parallel computing code will be used
(i.e., equivalent to the default).
show_progress (boolean): flag to indicate whether task progress should
be displayed to the user (defaults to True).
Returns:
An output table containing tuple pairs that satisfy the join
condition (DataFrame).
"""
# check if the input tables are dataframes
validate_input_table(ltable, 'left table')
validate_input_table(rtable, 'right table')
# check if the key attributes and join attributes exist
validate_attr(l_key_attr, ltable.columns, 'key attribute', 'left table')
validate_attr(r_key_attr, rtable.columns, 'key attribute', 'right table')
validate_attr(l_join_attr, ltable.columns, 'join attribute', 'left table')
validate_attr(r_join_attr, rtable.columns, 'join attribute', 'right table')
# check if the join attributes are not of numeric type
validate_attr_type(l_join_attr, ltable[l_join_attr].dtype,
'join attribute', 'left table')
validate_attr_type(r_join_attr, rtable[r_join_attr].dtype,
'join attribute', 'right table')
# check if the input tokenizer is valid
validate_tokenizer(tokenizer)
# check if the input threshold is valid
validate_threshold(threshold, 'DICE')
# check if the comparison operator is valid
validate_comp_op_for_sim_measure(comp_op, 'DICE')
# check if the output attributes exist
validate_output_attrs(l_out_attrs, ltable.columns, r_out_attrs,
rtable.columns)
# check if the key attributes are unique and do not contain missing values
validate_key_attr(l_key_attr, ltable, 'left table')
validate_key_attr(r_key_attr, rtable, 'right table')
# set return_set flag of tokenizer to be True, in case it is set to False
revert_tokenizer_return_set_flag = False
if not tokenizer.get_return_set():
tokenizer.set_return_set(True)
revert_tokenizer_return_set_flag = True
# remove redundant attrs from output attrs.
l_out_attrs = remove_redundant_attrs(l_out_attrs, l_key_attr)
r_out_attrs = remove_redundant_attrs(r_out_attrs, r_key_attr)
# get attributes to project.
l_proj_attrs = get_attrs_to_project(l_out_attrs, l_key_attr, l_join_attr)
r_proj_attrs = get_attrs_to_project(r_out_attrs, r_key_attr, r_join_attr)
# Do a projection on the input dataframes to keep only the required
# attributes. Then, remove rows with missing value in join attribute from
# the input dataframes. Then, convert the resulting dataframes into ndarray.
ltable_array = convert_dataframe_to_array(ltable, l_proj_attrs,
l_join_attr)
rtable_array = convert_dataframe_to_array(rtable, r_proj_attrs,
r_join_attr)
# computes the actual number of jobs to launch.
n_jobs = min(get_num_processes_to_launch(n_jobs), len(rtable_array))
if n_jobs <= 1:
# if n_jobs is 1, do not use any parallel code.
output_table = set_sim_join(ltable_array, rtable_array, l_proj_attrs,
r_proj_attrs, l_key_attr, r_key_attr,
l_join_attr, r_join_attr, tokenizer,
'DICE', threshold, comp_op, allow_empty,
l_out_attrs, r_out_attrs, l_out_prefix,
r_out_prefix, out_sim_score, show_progress)
else:
# if n_jobs is above 1, split the right table into n_jobs splits and
# join each right table split with the whole of left table in a separate
# process.
r_splits = split_table(rtable_array, n_jobs)
results = Parallel(n_jobs=n_jobs)(delayed(set_sim_join)(
ltable_array, r_splits[job_index], l_proj_attrs, r_proj_attrs,
l_key_attr, r_key_attr, l_join_attr, r_join_attr, tokenizer,
'DICE', threshold, comp_op, allow_empty, l_out_attrs, r_out_attrs,
l_out_prefix, r_out_prefix, out_sim_score, (
show_progress and (job_index == n_jobs - 1)))
for job_index in range(n_jobs))
output_table = pd.concat(results)
# If allow_missing flag is set, then compute all pairs with missing value in
# at least one of the join attributes and then add it to the output
# obtained from the join.
if allow_missing:
missing_pairs = get_pairs_with_missing_value(
ltable, rtable, l_key_attr, r_key_attr, l_join_attr, r_join_attr,
l_out_attrs, r_out_attrs, l_out_prefix, r_out_prefix,
out_sim_score, show_progress)
output_table = pd.concat([output_table, missing_pairs])
# add an id column named '_id' to the output table.
output_table.insert(0, '_id', range(0, len(output_table)))
# revert the return_set flag of tokenizer, in case it was modified.
if revert_tokenizer_return_set_flag:
tokenizer.set_return_set(False)
return output_table
def edit_distance_join(ltable, rtable,
l_key_attr, r_key_attr,
l_join_attr, r_join_attr,
threshold, comp_op='<=',
allow_missing=False,
l_out_attrs=None, r_out_attrs=None,
l_out_prefix='l_', r_out_prefix='r_',
out_sim_score=True, n_jobs=1, show_progress=True,
tokenizer=QgramTokenizer(qval=2)):
"""Join two tables using edit distance measure.
Finds tuple pairs from left table and right table such that the edit
distance between the join attributes satisfies the condition on input
threshold. For example, if the comparison operator is '<=', finds tuple
pairs whose edit distance between the strings that are the values of
the join attributes is less than or equal to the input threshold, as
specified in "threshold".
Note:
Currently, this method only computes an approximate join result. This is
because, to perform the join we transform an edit distance measure
between strings into an overlap measure between qgrams of the strings.
Hence, we need at least one qgram to be in common between two input
strings, to appear in the join output. For smaller strings, where all
qgrams of the strings differ, we cannot process them.
This method implements a simplified version of the algorithm proposed in
`Ed-Join: An Efficient Algorithm for Similarity Joins With Edit Distance
Constraints (Chuan Xiao, Wei Wang and Xuemin Lin), VLDB 08
<http://www.vldb.org/pvldb/1/1453957.pdf>`_.
Args:
ltable (DataFrame): left input table.
rtable (DataFrame): right input table.
l_key_attr (string): key attribute in left table.
r_key_attr (string): key attribute in right table.
l_join_attr (string): join attribute in left table.
r_join_attr (string): join attribute in right table.
threshold (float): edit distance threshold to be satisfied.
comp_op (string): comparison operator. Supported values are '<=', '<'
and '=' (defaults to '<=').
allow_missing (boolean): flag to indicate whether tuple pairs with
missing value in at least one of the join attributes should be
included in the output (defaults to False). If this flag is set to
True, a tuple in ltable with missing value in the join attribute
will be matched with every tuple in rtable and vice versa.
l_out_attrs (list): list of attribute names from the left table to be
included in the output table (defaults to None).
r_out_attrs (list): list of attribute names from the right table to be
included in the output table (defaults to None).
l_out_prefix (string): prefix to be used for the attribute names coming
from the left table, in the output table (defaults to 'l\_').
r_out_prefix (string): prefix to be used for the attribute names coming
from the right table, in the output table (defaults to 'r\_').
out_sim_score (boolean): flag to indicate whether the edit distance
score should be included in the output table (defaults to True).
Setting this flag to True will add a column named '_sim_score' in
the output table. This column will contain the edit distance scores
for the tuple pairs in the output.
n_jobs (int): number of parallel jobs to use for the computation
(defaults to 1). If -1 is given, all CPUs are used. If 1 is given,
no parallel computing code is used at all, which is useful for
debugging. For n_jobs below -1, (n_cpus + 1 + n_jobs) are used
(where n_cpus is the total number of CPUs in the machine). Thus for
n_jobs = -2, all CPUs but one are used. If (n_cpus + 1 + n_jobs)
becomes less than 1, then no parallel computing code will be used
(i.e., equivalent to the default).
show_progress (boolean): flag to indicate whether task progress should
be displayed to the user (defaults to True).
tokenizer (Tokenizer): tokenizer to be used to tokenize the join
attributes during filtering, when edit distance measure is
transformed into an overlap measure. This must be a q-gram tokenizer
(defaults to 2-gram tokenizer).
Returns:
An output table containing tuple pairs that satisfy the join
condition (DataFrame).
"""
# check if the input tables are dataframes
validate_input_table(ltable, 'left table')
validate_input_table(rtable, 'right table')
# check if the key attributes and join attributes exist
validate_attr(l_key_attr, ltable.columns,
'key attribute', 'left table')
validate_attr(r_key_attr, rtable.columns,
'key attribute', 'right table')
validate_attr(l_join_attr, ltable.columns,
'join attribute', 'left table')
validate_attr(r_join_attr, rtable.columns,
'join attribute', 'right table')
# check if the join attributes are not of numeric type
validate_attr_type(l_join_attr, ltable[l_join_attr].dtype,
'join attribute', 'left table')
validate_attr_type(r_join_attr, rtable[r_join_attr].dtype,
'join attribute', 'right table')
# check if the input tokenizer is valid for edit distance measure. Only
# qgram tokenizer can be used for edit distance.
validate_tokenizer_for_sim_measure(tokenizer, 'EDIT_DISTANCE')
# check if the input threshold is valid
validate_threshold(threshold, 'EDIT_DISTANCE')
# check if the comparison operator is valid
validate_comp_op_for_sim_measure(comp_op, 'EDIT_DISTANCE')
# check if the output attributes exist
validate_output_attrs(l_out_attrs, ltable.columns,
r_out_attrs, rtable.columns)
# check if the key attributes are unique and do not contain missing values
validate_key_attr(l_key_attr, ltable, 'left table')
validate_key_attr(r_key_attr, rtable, 'right table')
# convert threshold to integer (incase if it is float)
threshold = int(floor(threshold))
# set return_set flag of tokenizer to be False, in case it is set to True
revert_tokenizer_return_set_flag = False
if tokenizer.get_return_set():
tokenizer.set_return_set(False)
revert_tokenizer_return_set_flag = True
# remove redundant attrs from output attrs.
l_out_attrs = remove_redundant_attrs(l_out_attrs, l_key_attr)
r_out_attrs = remove_redundant_attrs(r_out_attrs, r_key_attr)
# get attributes to project.
l_proj_attrs = get_attrs_to_project(l_out_attrs, l_key_attr, l_join_attr)
r_proj_attrs = get_attrs_to_project(r_out_attrs, r_key_attr, r_join_attr)
# Do a projection on the input dataframes to keep only the required
# attributes. Then, remove rows with missing value in join attribute from
# the input dataframes. Then, convert the resulting dataframes into ndarray.
ltable_array = convert_dataframe_to_array(ltable, l_proj_attrs, l_join_attr)
rtable_array = convert_dataframe_to_array(rtable, r_proj_attrs, r_join_attr)
# computes the actual number of jobs to launch.
n_jobs = min(get_num_processes_to_launch(n_jobs), len(rtable_array))
if n_jobs <= 1:
# if n_jobs is 1, do not use any parallel code.
output_table = _edit_distance_join_split(
ltable_array, rtable_array,
l_proj_attrs, r_proj_attrs,
l_key_attr, r_key_attr,
l_join_attr, r_join_attr,
tokenizer, threshold, comp_op,
l_out_attrs, r_out_attrs,
l_out_prefix, r_out_prefix,
out_sim_score, show_progress)
else:
# if n_jobs is above 1, split the right table into n_jobs splits and
# join each right table split with the whole of left table in a separate
# process.
r_splits = split_table(rtable_array, n_jobs)
results = Parallel(n_jobs=n_jobs)(delayed(_edit_distance_join_split)(
ltable_array, r_splits[job_index],
l_proj_attrs, r_proj_attrs,
l_key_attr, r_key_attr,
l_join_attr, r_join_attr,
tokenizer, threshold, comp_op,
l_out_attrs, r_out_attrs,
l_out_prefix, r_out_prefix,
out_sim_score,
(show_progress and (job_index==n_jobs-1)))
for job_index in range(n_jobs))
output_table = pd.concat(results)
# If allow_missing flag is set, then compute all pairs with missing value in
# at least one of the join attributes and then add it to the output
# obtained from the join.
if allow_missing:
missing_pairs = get_pairs_with_missing_value(
ltable, rtable,
l_key_attr, r_key_attr,
l_join_attr, r_join_attr,
l_out_attrs, r_out_attrs,
l_out_prefix, r_out_prefix,
out_sim_score, show_progress)
output_table = pd.concat([output_table, missing_pairs])
# add an id column named '_id' to the output table.
output_table.insert(0, '_id', range(0, len(output_table)))
# revert the return_set flag of tokenizer, in case it was modified.
if revert_tokenizer_return_set_flag:
tokenizer.set_return_set(True)
return output_table
def edit_distance_join(ltable,
rtable,
l_key_attr,
r_key_attr,
l_join_attr,
r_join_attr,
threshold,
comp_op='<=',
allow_missing=False,
l_out_attrs=None,
r_out_attrs=None,
l_out_prefix='l_',
r_out_prefix='r_',
out_sim_score=True,
n_jobs=1,
show_progress=True,
tokenizer=QgramTokenizer(qval=2)):
"""Join two tables using edit distance measure.
Finds tuple pairs from left table and right table such that the edit
distance between the join attributes satisfies the condition on input
threshold. For example, if the comparison operator is '<=', finds tuple
pairs whose edit distance between the strings that are the values of
the join attributes is less than or equal to the input threshold, as
specified in "threshold".
Note:
Currently, this method only computes an approximate join result. This is
because, to perform the join we transform an edit distance measure
between strings into an overlap measure between qgrams of the strings.
Hence, we need at least one qgram to be in common between two input
strings, to appear in the join output. For smaller strings, where all
qgrams of the strings differ, we cannot process them.
This method implements a simplified version of the algorithm proposed in
`Ed-Join: An Efficient Algorithm for Similarity Joins With Edit Distance
Constraints (Chuan Xiao, Wei Wang and Xuemin Lin), VLDB 08
<http://www.vldb.org/pvldb/1/1453957.pdf>`_.
Args:
ltable (DataFrame): left input table.
rtable (DataFrame): right input table.
l_key_attr (string): key attribute in left table.
r_key_attr (string): key attribute in right table.
l_join_attr (string): join attribute in left table.
r_join_attr (string): join attribute in right table.
threshold (float): edit distance threshold to be satisfied.
comp_op (string): comparison operator. Supported values are '<=', '<'
and '=' (defaults to '<=').
allow_missing (boolean): flag to indicate whether tuple pairs with
missing value in at least one of the join attributes should be
included in the output (defaults to False). If this flag is set to
True, a tuple in ltable with missing value in the join attribute
will be matched with every tuple in rtable and vice versa.
l_out_attrs (list): list of attribute names from the left table to be
included in the output table (defaults to None).
r_out_attrs (list): list of attribute names from the right table to be
included in the output table (defaults to None).
l_out_prefix (string): prefix to be used for the attribute names coming
from the left table, in the output table (defaults to 'l\_').
r_out_prefix (string): prefix to be used for the attribute names coming
from the right table, in the output table (defaults to 'r\_').
out_sim_score (boolean): flag to indicate whether the edit distance
score should be included in the output table (defaults to True).
Setting this flag to True will add a column named '_sim_score' in
the output table. This column will contain the edit distance scores
for the tuple pairs in the output.
n_jobs (int): number of parallel jobs to use for the computation
(defaults to 1). If -1 is given, all CPUs are used. If 1 is given,
no parallel computing code is used at all, which is useful for
debugging. For n_jobs below -1, (n_cpus + 1 + n_jobs) are used
(where n_cpus is the total number of CPUs in the machine). Thus for
n_jobs = -2, all CPUs but one are used. If (n_cpus + 1 + n_jobs)
becomes less than 1, then no parallel computing code will be used
(i.e., equivalent to the default).
show_progress (boolean): flag to indicate whether task progress should
be displayed to the user (defaults to True).
tokenizer (Tokenizer): tokenizer to be used to tokenize the join
attributes during filtering, when edit distance measure is
transformed into an overlap measure. This must be a q-gram tokenizer
(defaults to 2-gram tokenizer).
Returns:
An output table containing tuple pairs that satisfy the join
condition (DataFrame).
"""
# check if the input tables are dataframes
validate_input_table(ltable, 'left table')
validate_input_table(rtable, 'right table')
# check if the key attributes and join attributes exist
validate_attr(l_key_attr, ltable.columns, 'key attribute', 'left table')
validate_attr(r_key_attr, rtable.columns, 'key attribute', 'right table')
validate_attr(l_join_attr, ltable.columns, 'join attribute', 'left table')
validate_attr(r_join_attr, rtable.columns, 'join attribute', 'right table')
# check if the join attributes are not of numeric type
validate_attr_type(l_join_attr, ltable[l_join_attr].dtype,
'join attribute', 'left table')
validate_attr_type(r_join_attr, rtable[r_join_attr].dtype,
'join attribute', 'right table')
# check if the input tokenizer is valid for edit distance measure. Only
# qgram tokenizer can be used for edit distance.
validate_tokenizer_for_sim_measure(tokenizer, 'EDIT_DISTANCE')
# check if the input threshold is valid
validate_threshold(threshold, 'EDIT_DISTANCE')
# check if the comparison operator is valid
validate_comp_op_for_sim_measure(comp_op, 'EDIT_DISTANCE')
# check if the output attributes exist
validate_output_attrs(l_out_attrs, ltable.columns, r_out_attrs,
rtable.columns)
# check if the key attributes are unique and do not contain missing values
validate_key_attr(l_key_attr, ltable, 'left table')
validate_key_attr(r_key_attr, rtable, 'right table')
# convert threshold to integer (incase if it is float)
threshold = int(floor(threshold))
# set return_set flag of tokenizer to be False, in case it is set to True
revert_tokenizer_return_set_flag = False
if tokenizer.get_return_set():
tokenizer.set_return_set(False)
revert_tokenizer_return_set_flag = True
# remove redundant attrs from output attrs.
l_out_attrs = remove_redundant_attrs(l_out_attrs, l_key_attr)
r_out_attrs = remove_redundant_attrs(r_out_attrs, r_key_attr)
# get attributes to project.
l_proj_attrs = get_attrs_to_project(l_out_attrs, l_key_attr, l_join_attr)
r_proj_attrs = get_attrs_to_project(r_out_attrs, r_key_attr, r_join_attr)
# Do a projection on the input dataframes to keep only the required
# attributes. Then, remove rows with missing value in join attribute from
# the input dataframes. Then, convert the resulting dataframes into ndarray.
ltable_array = convert_dataframe_to_array(ltable, l_proj_attrs,
l_join_attr)
rtable_array = convert_dataframe_to_array(rtable, r_proj_attrs,
r_join_attr)
# computes the actual number of jobs to launch.
n_jobs = min(get_num_processes_to_launch(n_jobs), len(rtable_array))
if n_jobs <= 1:
# if n_jobs is 1, do not use any parallel code.
output_table = _edit_distance_join_split(
ltable_array, rtable_array, l_proj_attrs, r_proj_attrs, l_key_attr,
r_key_attr, l_join_attr, r_join_attr, tokenizer, threshold,
comp_op, l_out_attrs, r_out_attrs, l_out_prefix, r_out_prefix,
out_sim_score, show_progress)
else:
# if n_jobs is above 1, split the right table into n_jobs splits and
# join each right table split with the whole of left table in a separate
# process.
r_splits = split_table(rtable_array, n_jobs)
results = Parallel(n_jobs=n_jobs)(delayed(_edit_distance_join_split)(
ltable_array, r_splits[job_index], l_proj_attrs, r_proj_attrs,
l_key_attr, r_key_attr, l_join_attr, r_join_attr, tokenizer,
threshold, comp_op, l_out_attrs, r_out_attrs, l_out_prefix,
r_out_prefix, out_sim_score, (
show_progress and (job_index == n_jobs - 1)))
for job_index in range(n_jobs))
output_table = pd.concat(results)
# If allow_missing flag is set, then compute all pairs with missing value in
# at least one of the join attributes and then add it to the output
# obtained from the join.
if allow_missing:
missing_pairs = get_pairs_with_missing_value(
ltable, rtable, l_key_attr, r_key_attr, l_join_attr, r_join_attr,
l_out_attrs, r_out_attrs, l_out_prefix, r_out_prefix,
out_sim_score, show_progress)
output_table = pd.concat([output_table, missing_pairs])
# add an id column named '_id' to the output table.
output_table.insert(0, '_id', range(0, len(output_table)))
# revert the return_set flag of tokenizer, in case it was modified.
if revert_tokenizer_return_set_flag:
tokenizer.set_return_set(True)
return output_table
def dice_join_py(ltable, rtable,
l_key_attr, r_key_attr,
l_join_attr, r_join_attr,
tokenizer, threshold, comp_op='>=',
allow_empty=True, allow_missing=False,
l_out_attrs=None, r_out_attrs=None,
l_out_prefix='l_', r_out_prefix='r_',
out_sim_score=True, n_jobs=1, show_progress=True):
"""Join two tables using Dice similarity measure.
For two sets X and Y, the Dice similarity score between them is given by:
:math:`dice(X, Y) = \\frac{2 * |X \\cap Y|}{|X| + |Y|}`
In the case where both X and Y are empty sets, we define their Dice
score to be 1.
Finds tuple pairs from left table and right table such that the Dice
similarity between the join attributes satisfies the condition on input
threshold. For example, if the comparison operator is '>=', finds tuple
pairs whose Dice similarity between the strings that are the values of
the join attributes is greater than or equal to the input threshold, as
specified in "threshold".
Args:
ltable (DataFrame): left input table.
rtable (DataFrame): right input table.
l_key_attr (string): key attribute in left table.
r_key_attr (string): key attribute in right table.
l_join_attr (string): join attribute in left table.
r_join_attr (string): join attribute in right table.
tokenizer (Tokenizer): tokenizer to be used to tokenize join
attributes.
threshold (float): Dice similarity threshold to be satisfied.
comp_op (string): comparison operator. Supported values are '>=', '>'
and '=' (defaults to '>=').
allow_empty (boolean): flag to indicate whether tuple pairs with empty
set of tokens in both the join attributes should be included in the
output (defaults to True).
allow_missing (boolean): flag to indicate whether tuple pairs with
missing value in at least one of the join attributes should be
included in the output (defaults to False). If this flag is set to
True, a tuple in ltable with missing value in the join attribute
will be matched with every tuple in rtable and vice versa.
l_out_attrs (list): list of attribute names from the left table to be
included in the output table (defaults to None).
r_out_attrs (list): list of attribute names from the right table to be
included in the output table (defaults to None).
l_out_prefix (string): prefix to be used for the attribute names coming
from the left table, in the output table (defaults to 'l\_').
r_out_prefix (string): prefix to be used for the attribute names coming
from the right table, in the output table (defaults to 'r\_').
out_sim_score (boolean): flag to indicate whether similarity score
should be included in the output table (defaults to True). Setting
this flag to True will add a column named '_sim_score' in the
output table. This column will contain the similarity scores for the
tuple pairs in the output.
n_jobs (int): number of parallel jobs to use for the computation
(defaults to 1). If -1 is given, all CPUs are used. If 1 is given,
no parallel computing code is used at all, which is useful for
debugging. For n_jobs below -1, (n_cpus + 1 + n_jobs) are used
(where n_cpus is the total number of CPUs in the machine). Thus for
n_jobs = -2, all CPUs but one are used. If (n_cpus + 1 + n_jobs)
becomes less than 1, then no parallel computing code will be used
(i.e., equivalent to the default).
show_progress (boolean): flag to indicate whether task progress should
be displayed to the user (defaults to True).
Returns:
An output table containing tuple pairs that satisfy the join
condition (DataFrame).
"""
# check if the input tables are dataframes
validate_input_table(ltable, 'left table')
validate_input_table(rtable, 'right table')
# check if the key attributes and join attributes exist
validate_attr(l_key_attr, ltable.columns,
'key attribute', 'left table')
validate_attr(r_key_attr, rtable.columns,
'key attribute', 'right table')
validate_attr(l_join_attr, ltable.columns,
'join attribute', 'left table')
validate_attr(r_join_attr, rtable.columns,
'join attribute', 'right table')
# check if the join attributes are not of numeric type
validate_attr_type(l_join_attr, ltable[l_join_attr].dtype,
'join attribute', 'left table')
validate_attr_type(r_join_attr, rtable[r_join_attr].dtype,
'join attribute', 'right table')
# check if the input tokenizer is valid
validate_tokenizer(tokenizer)
# check if the input threshold is valid
validate_threshold(threshold, 'DICE')
# check if the comparison operator is valid
validate_comp_op_for_sim_measure(comp_op, 'DICE')
# check if the output attributes exist
validate_output_attrs(l_out_attrs, ltable.columns,
r_out_attrs, rtable.columns)
# check if the key attributes are unique and do not contain missing values
validate_key_attr(l_key_attr, ltable, 'left table')
validate_key_attr(r_key_attr, rtable, 'right table')
# set return_set flag of tokenizer to be True, in case it is set to False
revert_tokenizer_return_set_flag = False
if not tokenizer.get_return_set():
tokenizer.set_return_set(True)
revert_tokenizer_return_set_flag = True
# remove redundant attrs from output attrs.
l_out_attrs = remove_redundant_attrs(l_out_attrs, l_key_attr)
r_out_attrs = remove_redundant_attrs(r_out_attrs, r_key_attr)
# get attributes to project.
l_proj_attrs = get_attrs_to_project(l_out_attrs, l_key_attr, l_join_attr)
r_proj_attrs = get_attrs_to_project(r_out_attrs, r_key_attr, r_join_attr)
# Do a projection on the input dataframes to keep only the required
# attributes. Then, remove rows with missing value in join attribute from
# the input dataframes. Then, convert the resulting dataframes into ndarray.
ltable_array = convert_dataframe_to_array(ltable, l_proj_attrs, l_join_attr)
rtable_array = convert_dataframe_to_array(rtable, r_proj_attrs, r_join_attr)
# computes the actual number of jobs to launch.
n_jobs = min(get_num_processes_to_launch(n_jobs), len(rtable_array))
if n_jobs <= 1:
# if n_jobs is 1, do not use any parallel code.
output_table = set_sim_join(ltable_array, rtable_array,
l_proj_attrs, r_proj_attrs,
l_key_attr, r_key_attr,
l_join_attr, r_join_attr,
tokenizer, 'DICE',
threshold, comp_op, allow_empty,
l_out_attrs, r_out_attrs,
l_out_prefix, r_out_prefix,
out_sim_score, show_progress)
else:
# if n_jobs is above 1, split the right table into n_jobs splits and
# join each right table split with the whole of left table in a separate
# process.
r_splits = split_table(rtable_array, n_jobs)
results = Parallel(n_jobs=n_jobs)(delayed(set_sim_join)(
ltable_array, r_splits[job_index],
l_proj_attrs, r_proj_attrs,
l_key_attr, r_key_attr,
l_join_attr, r_join_attr,
tokenizer, 'DICE',
threshold, comp_op, allow_empty,
l_out_attrs, r_out_attrs,
l_out_prefix, r_out_prefix,
out_sim_score,
(show_progress and (job_index==n_jobs-1)))
for job_index in range(n_jobs))
output_table = pd.concat(results)
# If allow_missing flag is set, then compute all pairs with missing value in
# at least one of the join attributes and then add it to the output
# obtained from the join.
if allow_missing:
missing_pairs = get_pairs_with_missing_value(
ltable, rtable,
l_key_attr, r_key_attr,
l_join_attr, r_join_attr,
l_out_attrs, r_out_attrs,
l_out_prefix, r_out_prefix,
out_sim_score, show_progress)
output_table = pd.concat([output_table, missing_pairs])
# add an id column named '_id' to the output table.
output_table.insert(0, '_id', range(0, len(output_table)))
# revert the return_set flag of tokenizer, in case it was modified.
if revert_tokenizer_return_set_flag:
tokenizer.set_return_set(False)
return output_table
def block_tables(self, ltable, rtable, l_overlap_attr, r_overlap_attr,
rem_stop_words=False, q_val=None, word_level=True, overlap_size=1,
l_output_attrs=None, r_output_attrs=None,
l_output_prefix='ltable_', r_output_prefix='rtable_',
allow_missing=False, verbose=False, show_progress=True,
n_ltable_chunks=1, n_rtable_chunks=1):
"""
WARNING THIS COMMAND IS EXPERIMENTAL AND NOT TESTED. USE AT YOUR OWN RISK.
Blocks two tables based on the overlap of token sets of attribute
values. Finds tuple pairs from left and right tables such that the overlap
between (a) the set of tokens obtained by tokenizing the value of
attribute l_overlap_attr of a tuple from the left table, and (b) the
set of tokens obtained by tokenizing the value of attribute
r_overlap_attr of a tuple from the right table, is above a certain
threshold.
Args:
ltable (DataFrame): The left input table.
rtable (DataFrame): The right input table.
l_overlap_attr (string): The overlap attribute in left table.
r_overlap_attr (string): The overlap attribute in right table.
rem_stop_words (boolean): A flag to indicate whether stop words
(e.g., a, an, the) should be removed from the token sets of the
overlap attribute values (defaults to False).
q_val (int): The value of q to use if the overlap attributes
values are to be tokenized as qgrams (defaults to None).
word_level (boolean): A flag to indicate whether the overlap
attributes should be tokenized as words (i.e, using whitespace
as delimiter) (defaults to True).
overlap_size (int): The minimum number of tokens that must
overlap (defaults to 1).
l_output_attrs (list): A list of attribute names from the left
table to be included in the output candidate set (defaults
to None).
r_output_attrs (list): A list of attribute names from the right
table to be included in the output candidate set (defaults
to None).
l_output_prefix (string): The prefix to be used for the attribute names
coming from the left table in the output
candidate set (defaults to 'ltable\_').
r_output_prefix (string): The prefix to be used for the attribute names
coming from the right table in the output
candidate set (defaults to 'rtable\_').
allow_missing (boolean): A flag to indicate whether tuple pairs
with missing value in at least one of the
blocking attributes should be included in
the output candidate set (defaults to
False). If this flag is set to True, a
tuple in ltable with missing value in the
blocking attribute will be matched with
every tuple in rtable and vice versa.
verbose (boolean): A flag to indicate whether the debug
information should be logged (defaults to False).
show_progress (boolean): A flag to indicate whether progress should
be displayed to the user (defaults to True).
n_ltable_chunks (int): The number of partitions to split the left table (
defaults to 1). If it is set to -1, then the number of
partitions is set to the number of cores in the
machine.
n_rtable_chunks (int): The number of partitions to split the right table (
defaults to 1). If it is set to -1, then the number of
partitions is set to the number of cores in the
machine.
Returns:
A candidate set of tuple pairs that survived blocking (DataFrame).
Raises:
AssertionError: If `ltable` is not of type pandas
DataFrame.
AssertionError: If `rtable` is not of type pandas
DataFrame.
AssertionError: If `l_overlap_attr` is not of type string.
AssertionError: If `r_overlap_attr` is not of type string.
AssertionError: If `l_output_attrs` is not of type of
list.
AssertionError: If `r_output_attrs` is not of type of
list.
AssertionError: If the values in `l_output_attrs` is not of type
string.
AssertionError: If the values in `r_output_attrs` is not of type
string.
AssertionError: If `l_output_prefix` is not of type
string.
AssertionError: If `r_output_prefix` is not of type
string.
AssertionError: If `q_val` is not of type int.
AssertionError: If `word_level` is not of type boolean.
AssertionError: If `overlap_size` is not of type int.
AssertionError: If `verbose` is not of type
boolean.
AssertionError: If `allow_missing` is not of type boolean.
AssertionError: If `show_progress` is not of type
boolean.
AssertionError: If `n_ltable_chunks` is not of type
int.
AssertionError: If `n_rtable_chunks` is not of type
int.
AssertionError: If `l_overlap_attr` is not in the ltable
columns.
AssertionError: If `r_block_attr` is not in the rtable columns.
AssertionError: If `l_output_attrs` are not in the ltable.
AssertionError: If `r_output_attrs` are not in the rtable.
SyntaxError: If `q_val` is set to a valid value and
`word_level` is set to True.
SyntaxError: If `q_val` is set to None and
`word_level` is set to False.
Examples:
>>> from py_entitymatching.dask.dask_overlap_blocker import DaskOverlapBlocker
>>> A = em.read_csv_metadata('path_to_csv_dir/table_A.csv', key='ID')
>>> B = em.read_csv_metadata('path_to_csv_dir/table_B.csv', key='ID')
>>> ob = DaskOverlapBlocker()
# Use all cores
# # Use word-level tokenizer
>>> C1 = ob.block_tables(A, B, 'address', 'address', l_output_attrs=['name'], r_output_attrs=['name'], word_level=True, overlap_size=1, n_ltable_chunks=-1, n_rtable_chunks=-1)
# # Use q-gram tokenizer
>>> C2 = ob.block_tables(A, B, 'address', 'address', l_output_attrs=['name'], r_output_attrs=['name'], word_level=False, q_val=2, n_ltable_chunks=-1, n_rtable_chunks=-1)
# # Include all possible missing values
>>> C3 = ob.block_tables(A, B, 'address', 'address', l_output_attrs=['name'], r_output_attrs=['name'], allow_missing=True, n_ltable_chunks=-1, n_rtable_chunks=-1)
"""
logger.warning(
"WARNING THIS COMMAND IS EXPERIMENTAL AND NOT TESTED. USE AT YOUR OWN "
"RISK.")
# Input validations
self.validate_types_params_tables(ltable, rtable, l_output_attrs,
r_output_attrs, l_output_prefix,
r_output_prefix, verbose, n_ltable_chunks, n_rtable_chunks)
self.validate_types_other_params(l_overlap_attr, r_overlap_attr,
rem_stop_words, q_val, word_level, overlap_size)
self.validate_allow_missing(allow_missing)
self.validate_show_progress(show_progress)
self.validate_overlap_attrs(ltable, rtable, l_overlap_attr, r_overlap_attr)
self.validate_output_attrs(ltable, rtable, l_output_attrs, r_output_attrs)
self.validate_word_level_qval(word_level, q_val)
log_info(logger, 'Required metadata: ltable key, rtable key', verbose)
l_key, r_key = cm.get_keys_for_ltable_rtable(ltable, rtable, logger, verbose)
# validate metadata
cm._validate_metadata_for_table(ltable, l_key, 'ltable', logger, verbose)
cm._validate_metadata_for_table(rtable, r_key, 'rtable', logger, verbose)
# validate input table chunks
validate_object_type(n_ltable_chunks, int, 'Parameter n_ltable_chunks')
validate_object_type(n_rtable_chunks, int,
'Parameter n_rtable_chunks')
validate_chunks(n_ltable_chunks)
validate_chunks(n_rtable_chunks)
if n_ltable_chunks == -1:
n_ltable_chunks = multiprocessing.cpu_count()
ltable_chunks = pd.np.array_split(ltable, n_ltable_chunks)
# preprocess/tokenize ltable
if word_level == True:
tokenizer = WhitespaceTokenizer(return_set=True)
else:
tokenizer = QgramTokenizer(qval=q_val, return_set=True)
preprocessed_tokenized_ltbl = []
# Construct DAG for preprocessing/tokenizing ltable chunks
start_row_id = 0
for i in range(len(ltable_chunks)):
result = delayed(self.process_tokenize_block_attr)(ltable_chunks[i][
l_overlap_attr],
start_row_id,
rem_stop_words, tokenizer)
preprocessed_tokenized_ltbl.append(result)
start_row_id += len(ltable_chunks[i])
preprocessed_tokenized_ltbl = delayed(wrap)(preprocessed_tokenized_ltbl)
# Execute the DAG
if show_progress:
with ProgressBar():
logger.info('Preprocessing/tokenizing ltable')
preprocessed_tokenized_ltbl_vals = preprocessed_tokenized_ltbl.compute(
scheduler="processes", num_workers=multiprocessing.cpu_count())
else:
preprocessed_tokenized_ltbl_vals = preprocessed_tokenized_ltbl.compute(
scheduler="processes", num_workers=multiprocessing.cpu_count())
ltable_processed_dict = {}
for i in range(len(preprocessed_tokenized_ltbl_vals)):
ltable_processed_dict.update(preprocessed_tokenized_ltbl_vals[i])
# build inverted index
inverted_index = self.build_inverted_index(ltable_processed_dict)
if n_rtable_chunks == -1:
n_rtable_chunks = multiprocessing.cpu_count()
rtable_chunks = pd.np.array_split(rtable, n_rtable_chunks)
# Construct the DAG for probing
probe_result = []
start_row_id = 0
for i in range(len(rtable_chunks)):
result = delayed(self.probe)(rtable_chunks[i][r_overlap_attr],
inverted_index, start_row_id, rem_stop_words,
tokenizer, overlap_size)
probe_result.append(result)
start_row_id += len(rtable_chunks[i])
probe_result = delayed(wrap)(probe_result)
# Execute the DAG for probing
if show_progress:
with ProgressBar():
logger.info('Probing using rtable')
probe_result = probe_result.compute(scheduler="processes",
num_workers=multiprocessing.cpu_count())
else:
probe_result = probe_result.compute(scheduler="processes",
num_workers=multiprocessing.cpu_count())
# construct a minimal dataframe that can be used to add more attributes
flat_list = [item for sublist in probe_result for item in sublist]
tmp = pd.DataFrame(flat_list, columns=['fk_ltable_rid', 'fk_rtable_rid'])
fk_ltable = ltable.iloc[tmp.fk_ltable_rid][l_key].values
fk_rtable = rtable.iloc[tmp.fk_rtable_rid][r_key].values
id_vals = list(range(len(flat_list)))
candset = pd.DataFrame.from_dict(
{'_id': id_vals, l_output_prefix+l_key: fk_ltable, r_output_prefix+r_key: fk_rtable})
# set the properties for the candidate set
cm.set_key(candset, '_id')
cm.set_fk_ltable(candset, 'ltable_'+l_key)
cm.set_fk_rtable(candset, 'rtable_'+r_key)
cm.set_ltable(candset, ltable)
cm.set_rtable(candset, rtable)
ret_candset = gh.add_output_attributes(candset, l_output_attrs=l_output_attrs,
r_output_attrs=r_output_attrs,
l_output_prefix=l_output_prefix,
r_output_prefix=r_output_prefix,
validate=False)
# handle missing values
if allow_missing:
missing_value_pairs = get_pairs_with_missing_value(ltable, rtable, l_key,
r_key, l_overlap_attr,
r_overlap_attr,
l_output_attrs,
r_output_attrs,
l_output_prefix,
r_output_prefix, False, False)
missing_value_pairs.insert(0, '_id', range(len(ret_candset),
len(ret_candset)+len(missing_value_pairs)))
if len(missing_value_pairs) > 0:
ret_candset = pd.concat([ret_candset, missing_value_pairs], ignore_index=True, sort=False)
cm.set_key(ret_candset, '_id')
cm.set_fk_ltable(ret_candset, 'ltable_' + l_key)
cm.set_fk_rtable(ret_candset, 'rtable_' + r_key)
cm.set_ltable(ret_candset, ltable)
cm.set_rtable(ret_candset, rtable)
# Return the final candidate set to user.
return ret_candset
def block_tables(self, ltable, rtable, l_overlap_attr, r_overlap_attr,
rem_stop_words=False, q_val=None, word_level=True, overlap_size=1,
l_output_attrs=None, r_output_attrs=None,
l_output_prefix='ltable_', r_output_prefix='rtable_',
allow_missing=False, verbose=False, show_progress=True,
n_ltable_chunks=1, n_rtable_chunks=1):
"""
WARNING THIS COMMAND IS EXPERIMENTAL AND NOT TESTED. USE AT YOUR OWN RISK.
Blocks two tables based on the overlap of token sets of attribute
values. Finds tuple pairs from left and right tables such that the overlap
between (a) the set of tokens obtained by tokenizing the value of
attribute l_overlap_attr of a tuple from the left table, and (b) the
set of tokens obtained by tokenizing the value of attribute
r_overlap_attr of a tuple from the right table, is above a certain
threshold.
Args:
ltable (DataFrame): The left input table.
rtable (DataFrame): The right input table.
l_overlap_attr (string): The overlap attribute in left table.
r_overlap_attr (string): The overlap attribute in right table.
rem_stop_words (boolean): A flag to indicate whether stop words
(e.g., a, an, the) should be removed from the token sets of the
overlap attribute values (defaults to False).
q_val (int): The value of q to use if the overlap attributes
values are to be tokenized as qgrams (defaults to None).
word_level (boolean): A flag to indicate whether the overlap
attributes should be tokenized as words (i.e, using whitespace
as delimiter) (defaults to True).
overlap_size (int): The minimum number of tokens that must
overlap (defaults to 1).
l_output_attrs (list): A list of attribute names from the left
table to be included in the output candidate set (defaults
to None).
r_output_attrs (list): A list of attribute names from the right
table to be included in the output candidate set (defaults
to None).
l_output_prefix (string): The prefix to be used for the attribute names
coming from the left table in the output
candidate set (defaults to 'ltable\_').
r_output_prefix (string): The prefix to be used for the attribute names
coming from the right table in the output
candidate set (defaults to 'rtable\_').
allow_missing (boolean): A flag to indicate whether tuple pairs
with missing value in at least one of the
blocking attributes should be included in
the output candidate set (defaults to
False). If this flag is set to True, a
tuple in ltable with missing value in the
blocking attribute will be matched with
every tuple in rtable and vice versa.
verbose (boolean): A flag to indicate whether the debug
information should be logged (defaults to False).
show_progress (boolean): A flag to indicate whether progress should
be displayed to the user (defaults to True).
n_ltable_chunks (int): The number of partitions to split the left table (
defaults to 1). If it is set to -1, then the number of
partitions is set to the number of cores in the
machine.
n_rtable_chunks (int): The number of partitions to split the right table (
defaults to 1). If it is set to -1, then the number of
partitions is set to the number of cores in the
machine.
Returns:
A candidate set of tuple pairs that survived blocking (DataFrame).
Raises:
AssertionError: If `ltable` is not of type pandas
DataFrame.
AssertionError: If `rtable` is not of type pandas
DataFrame.
AssertionError: If `l_overlap_attr` is not of type string.
AssertionError: If `r_overlap_attr` is not of type string.
AssertionError: If `l_output_attrs` is not of type of
list.
AssertionError: If `r_output_attrs` is not of type of
list.
AssertionError: If the values in `l_output_attrs` is not of type
string.
AssertionError: If the values in `r_output_attrs` is not of type
string.
AssertionError: If `l_output_prefix` is not of type
string.
AssertionError: If `r_output_prefix` is not of type
string.
AssertionError: If `q_val` is not of type int.
AssertionError: If `word_level` is not of type boolean.
AssertionError: If `overlap_size` is not of type int.
AssertionError: If `verbose` is not of type
boolean.
AssertionError: If `allow_missing` is not of type boolean.
AssertionError: If `show_progress` is not of type
boolean.
AssertionError: If `n_ltable_chunks` is not of type
int.
AssertionError: If `n_rtable_chunks` is not of type
int.
AssertionError: If `l_overlap_attr` is not in the ltable
columns.
AssertionError: If `r_block_attr` is not in the rtable columns.
AssertionError: If `l_output_attrs` are not in the ltable.
AssertionError: If `r_output_attrs` are not in the rtable.
SyntaxError: If `q_val` is set to a valid value and
`word_level` is set to True.
SyntaxError: If `q_val` is set to None and
`word_level` is set to False.
Examples:
>>> from py_entitymatching.dask.dask_overlap_blocker import DaskOverlapBlocker
>>> A = em.read_csv_metadata('path_to_csv_dir/table_A.csv', key='ID')
>>> B = em.read_csv_metadata('path_to_csv_dir/table_B.csv', key='ID')
>>> ob = DaskOverlapBlocker()
# Use all cores
# # Use word-level tokenizer
>>> C1 = ob.block_tables(A, B, 'address', 'address', l_output_attrs=['name'], r_output_attrs=['name'], word_level=True, overlap_size=1, n_ltable_chunks=-1, n_rtable_chunks=-1)
# # Use q-gram tokenizer
>>> C2 = ob.block_tables(A, B, 'address', 'address', l_output_attrs=['name'], r_output_attrs=['name'], word_level=False, q_val=2, n_ltable_chunks=-1, n_rtable_chunks=-1)
# # Include all possible missing values
>>> C3 = ob.block_tables(A, B, 'address', 'address', l_output_attrs=['name'], r_output_attrs=['name'], allow_missing=True, n_ltable_chunks=-1, n_rtable_chunks=-1)
"""
logger.warning(
"WARNING THIS COMMAND IS EXPERIMENTAL AND NOT TESTED. USE AT YOUR OWN "
"RISK.")
# Input validations
self.validate_types_params_tables(ltable, rtable, l_output_attrs,
r_output_attrs, l_output_prefix,
r_output_prefix, verbose, n_ltable_chunks, n_rtable_chunks)
self.validate_types_other_params(l_overlap_attr, r_overlap_attr,
rem_stop_words, q_val, word_level, overlap_size)
self.validate_allow_missing(allow_missing)
self.validate_show_progress(show_progress)
self.validate_overlap_attrs(ltable, rtable, l_overlap_attr, r_overlap_attr)
self.validate_output_attrs(ltable, rtable, l_output_attrs, r_output_attrs)
self.validate_word_level_qval(word_level, q_val)
log_info(logger, 'Required metadata: ltable key, rtable key', verbose)
l_key, r_key = cm.get_keys_for_ltable_rtable(ltable, rtable, logger, verbose)
# validate metadata
cm._validate_metadata_for_table(ltable, l_key, 'ltable', logger, verbose)
cm._validate_metadata_for_table(rtable, r_key, 'rtable', logger, verbose)
# validate input table chunks
validate_object_type(n_ltable_chunks, int, 'Parameter n_ltable_chunks')
validate_object_type(n_rtable_chunks, int,
'Parameter n_rtable_chunks')
validate_chunks(n_ltable_chunks)
validate_chunks(n_rtable_chunks)
if n_ltable_chunks == -1:
n_ltable_chunks = multiprocessing.cpu_count()
ltable_chunks = pd.np.array_split(ltable, n_ltable_chunks)
# preprocess/tokenize ltable
if word_level == True:
tokenizer = WhitespaceTokenizer(return_set=True)
else:
tokenizer = QgramTokenizer(qval=q_val, return_set=True)
preprocessed_tokenized_ltbl = []
# Construct DAG for preprocessing/tokenizing ltable chunks
start_row_id = 0
for i in range(len(ltable_chunks)):
result = delayed(self.process_tokenize_block_attr)(ltable_chunks[i][
l_overlap_attr],
start_row_id,
rem_stop_words, tokenizer)
preprocessed_tokenized_ltbl.append(result)
start_row_id += len(ltable_chunks[i])
preprocessed_tokenized_ltbl = delayed(wrap)(preprocessed_tokenized_ltbl)
# Execute the DAG
if show_progress:
with ProgressBar():
logger.info('Preprocessing/tokenizing ltable')
preprocessed_tokenized_ltbl_vals = preprocessed_tokenized_ltbl.compute(
scheduler="processes", num_workers=multiprocessing.cpu_count())
else:
preprocessed_tokenized_ltbl_vals = preprocessed_tokenized_ltbl.compute(
scheduler="processes", num_workers=multiprocessing.cpu_count())
ltable_processed_dict = {}
for i in range(len(preprocessed_tokenized_ltbl_vals)):
ltable_processed_dict.update(preprocessed_tokenized_ltbl_vals[i])
# build inverted index
inverted_index = self.build_inverted_index(ltable_processed_dict)
if n_rtable_chunks == -1:
n_rtable_chunks = multiprocessing.cpu_count()
rtable_chunks = pd.np.array_split(rtable, n_rtable_chunks)
# Construct the DAG for probing
probe_result = []
start_row_id = 0
for i in range(len(rtable_chunks)):
result = delayed(self.probe)(rtable_chunks[i][r_overlap_attr],
inverted_index, start_row_id, rem_stop_words,
tokenizer, overlap_size)
probe_result.append(result)
start_row_id += len(rtable_chunks[i])
probe_result = delayed(wrap)(probe_result)
# Execute the DAG for probing
if show_progress:
with ProgressBar():
logger.info('Probing using rtable')
probe_result = probe_result.compute(scheduler="processes",
num_workers=multiprocessing.cpu_count())
else:
probe_result = probe_result.compute(scheduler="processes",
num_workers=multiprocessing.cpu_count())
# construct a minimal dataframe that can be used to add more attributes
flat_list = [item for sublist in probe_result for item in sublist]
tmp = pd.DataFrame(flat_list, columns=['fk_ltable_rid', 'fk_rtable_rid'])
fk_ltable = ltable.iloc[tmp.fk_ltable_rid][l_key].values
fk_rtable = rtable.iloc[tmp.fk_rtable_rid][r_key].values
id_vals = list(range(len(flat_list)))
candset = pd.DataFrame.from_dict(
{'_id': id_vals, l_output_prefix+l_key: fk_ltable, r_output_prefix+r_key: fk_rtable})
# set the properties for the candidate set
cm.set_key(candset, '_id')
cm.set_fk_ltable(candset, 'ltable_'+l_key)
cm.set_fk_rtable(candset, 'rtable_'+r_key)
cm.set_ltable(candset, ltable)
cm.set_rtable(candset, rtable)
ret_candset = gh.add_output_attributes(candset, l_output_attrs=l_output_attrs,
r_output_attrs=r_output_attrs,
l_output_prefix=l_output_prefix,
r_output_prefix=r_output_prefix,
validate=False)
# handle missing values
if allow_missing:
missing_value_pairs = get_pairs_with_missing_value(ltable, rtable, l_key,
r_key, l_overlap_attr,
r_overlap_attr,
l_output_attrs,
r_output_attrs,
l_output_prefix,
r_output_prefix, False, False)
missing_value_pairs.insert(0, '_id', range(len(ret_candset),
len(ret_candset)+len(missing_value_pairs)))
if len(missing_value_pairs) > 0:
ret_candset = pd.concat([ret_candset, missing_value_pairs], ignore_index=True, sort=False)
cm.set_key(ret_candset, '_id')
cm.set_fk_ltable(ret_candset, 'ltable_' + l_key)
cm.set_fk_rtable(ret_candset, 'rtable_' + r_key)
cm.set_ltable(ret_candset, ltable)
cm.set_rtable(ret_candset, rtable)
# Return the final candidate set to user.
return ret_candset
