def compare(self, comp_func, labels_a, labels_b, *args, **kwargs):
"""Compare two records.
Core method to compare record pairs. This method takes a function and
data from both records in the record pair. The data is compared with
the compare function. The built-in methods also use this function.
Example
-------
>>> comp = recordlinkage.Compare(PAIRS, DATAFRAME1, DATAFRAME2)
>>> comp.exact('first_name', 'name')
>>> # same as
>>> comp.compare(recordlinkage._compare_exact, 'first_name', 'name')
Parameters
----------
comp_func : function
A comparison function. This function can be a built-in function or
a user defined comparison function.
labels_a : label, pandas.Series, pandas.DataFrame
The labels, Series or DataFrame to compare.
labels_b : label, pandas.Series, pandas.DataFrame
The labels, Series or DataFrame to compare.
name : label
The name of the feature and the name of the column.
store : bool, default True
Store the result in the dataframe.
Returns
-------
pandas.Series
A pandas series with the result of comparing each record pair.
"""
if len(self.pairs) == 0:
raise ValueError(
"need at least one record pair"
)
# the name and store arguments
name = kwargs.pop('name', None)
store = kwargs.pop('store', True)
labels_a = listify(labels_a)
labels_b = listify(labels_b)
data_a = []
for label_a in labels_a:
# the label is a numpy or pandas object
if is_numpy_like(label_a) or is_pandas_like(label_a):
data_a.append(label_a)
# check requested labels (for better error messages)
elif label_a not in self.df_a.columns:
raise KeyError("label '%s' is not found in the first"
"dataframe" % label_a)
else:
if self.low_memory:
df_a_label = self._loc2(self.df_a[label_a], self.pairs, 0)
data_a.append(df_a_label)
# not low memory
else:
if self._df_a_indexed is None:
self._df_a_indexed = self._loc2(
self.df_a, self.pairs, 0)
data_a.append(self._df_a_indexed[label_a])
data_a = tuple(data_a)
data_b = []
for label_b in labels_b:
# the label is a numpy or pandas object
if is_numpy_like(label_b) or is_pandas_like(label_b):
data_b.append(label_b)
# check requested labels (for better error messages)
elif label_b not in self.df_b.columns:
raise KeyError("label '%s' is not found in the second"
"dataframe" % label_b)
else:
if self.low_memory:
df_b_label = self._loc2(self.df_b[label_b], self.pairs, 1)
data_b.append(df_b_label)
# not low memory
else:
if self._df_b_indexed is None:
self._df_b_indexed = self._loc2(
self.df_b, self.pairs, 1)
data_b.append(self._df_b_indexed[label_b])
data_b = tuple(data_b)
if self.njobs > 1:
jobs = []
chunk_size = np.ceil(self.njobs / len(self.pairs))
# each job
for i in range(0, self.njobs):
# The data arguments
args_a = tuple(df_a_indexed.loc[i*chunk_size:(i+1)*chunk_size, da] for da in labels_a)
args_b = tuple(df_b_indexed.loc[i*chunk_size:(i+1)*chunk_size, db] for db in labels_b)
p = mp.Process(target=comp_func, args=args_a + args_b + args, kwargs=kwargs)
jobs.append(p)
for proc in jobs:
# Start the process
p.start()
proc.join()
# merge parts
c = pandas.concat(jobs, axis=0, copy=False)
else:
# # The data arguments
# args_a = tuple(df_a_indexed.loc[:, da] for da in labels_a)
# args_b = tuple(df_b_indexed.loc[:, db] for db in labels_b)
# Compute the comparison
c = comp_func(*tuple(data_a + data_b + args), **kwargs)
# if a pandas series is returned, overwrite the index. The
# returned index can be different than the MultiIndex passed to
# the compare function.
if isinstance(c, pandas.Series):
c.index = self.vectors.index
# append column to Compare.vectors
if store:
name_or_id = name if name else len(self.vectors.columns)
self.vectors[name_or_id] = c
return self.vectors[name_or_id].rename(name)
def _union(self, objs, index=None, column_i=0):
"""Make a union of the features.
The term 'union' is based on the terminology of scikit-learn.
"""
feat_conc = []
for feat, label in objs:
# result is tuple of results
if isinstance(feat, tuple):
if label is None:
label = [None] * len(feat)
partial_result = self._union(zip(feat, label),
column_i=column_i)
feat_conc.append(partial_result)
column_i = column_i + partial_result.shape[1]
# result is pandas.Series.
elif isinstance(feat, pandas.Series):
feat.reset_index(drop=True, inplace=True)
if label is None:
label = column_i
feat.rename(label, inplace=True)
feat_conc.append(feat)
column_i = column_i + 1
# result is pandas.DataFrame
elif isinstance(feat, pandas.DataFrame):
feat.reset_index(drop=True, inplace=True)
if label is None:
label = np.arange(column_i, column_i + feat.shape[1])
feat.columns = label
feat_conc.append(feat)
column_i = column_i + feat.shape[1]
# result is numpy 1d array
elif is_numpy_like(feat) and len(feat.shape) == 1:
if label is None:
label = column_i
f = pandas.Series(feat, name=label, copy=False)
feat_conc.append(f)
column_i = column_i + 1
# result is numpy 2d array
elif is_numpy_like(feat) and len(feat.shape) == 2:
if label is None:
label = np.arange(column_i, column_i + feat.shape[1])
feat_df = pandas.DataFrame(feat, columns=label, copy=False)
if label is None:
feat_df.columns = [None for _ in range(feat_df.shape[1])]
feat_conc.append(feat_df)
column_i = column_i + feat.shape[1]
# other results are not (yet) supported
else:
raise ValueError("expected numpy.ndarray or "
"pandas object to be returned, "
"got '{}'".format(feat.__class__.__name__))
result = pandas.concat(feat_conc, axis=1, copy=False)
if index is not None:
result.set_index(index, inplace=True)
return result
def compare(self, comp_func, labels_a, labels_b, *args, **kwargs):
"""[DEPRECATED] Compare two records.
Core method to compare record pairs. This method takes a function and
data from both records in the record pair. The data is compared with
the compare function. The built-in methods also use this function.
Example
-------
>>> comp = recordlinkage.Compare(PAIRS, DATAFRAME1, DATAFRAME2)
>>> comp.exact('first_name', 'name')
>>> # same as
>>> comp.compare(recordlinkage._compare_exact, 'first_name', 'name')
Parameters
----------
comp_func : function
A comparison function. This function can be a built-in function or
a user defined comparison function.
labels_a : label, pandas.Series, pandas.DataFrame
The labels, Series or DataFrame to compare.
labels_b : label, pandas.Series, pandas.DataFrame
The labels, Series or DataFrame to compare.
name : label
The name of the feature and the name of the column.
store : bool, default True
Store the result in the dataframe.
Returns
-------
pandas.Series
A pandas series with the result of comparing each record pair.
"""
if isinstance(comp_func, pandas.MultiIndex):
raise ValueError("see new api documentation: "
"use method 'compute' instead of 'compare'")
if len(self.pairs) == 0:
raise ValueError("need at least one record pair")
# the name and store arguments
name = kwargs.pop('name', None)
store = kwargs.pop('store', True)
labels_a = listify(labels_a)
labels_b = listify(labels_b)
data_a = []
for label_a in labels_a:
# the label is a numpy or pandas object
if is_numpy_like(label_a) or is_pandas_like(label_a):
data_a.append(label_a)
# check requested labels (for better error messages)
elif label_a not in self.df_a.columns:
raise KeyError("label '{}' is not found in the first"
"dataframe".format(label_a))
else:
if self.low_memory:
df_a_label = self._loc2(self.df_a[label_a], self.pairs, 0)
data_a.append(df_a_label)
# not low memory
else:
if self._df_a_indexed is None:
self._df_a_indexed = self._loc2(
self.df_a, self.pairs, 0)
data_a.append(self._df_a_indexed[label_a])
data_a = tuple(data_a)
data_b = []
for label_b in labels_b:
# the label is a numpy or pandas object
if is_numpy_like(label_b) or is_pandas_like(label_b):
data_b.append(label_b)
# check requested labels (for better error messages)
elif label_b not in self.df_b.columns:
raise KeyError("label '{}' is not found in the second"
"dataframe".format(label_b))
else:
if self.low_memory:
df_b_label = self._loc2(self.df_b[label_b], self.pairs, 1)
data_b.append(df_b_label)
# not low memory
else:
if self._df_b_indexed is None:
self._df_b_indexed = self._loc2(
self.df_b, self.pairs, 1)
data_b.append(self._df_b_indexed[label_b])
data_b = tuple(data_b)
# Compute the comparison
c = comp_func(*tuple(data_a + data_b + args), **kwargs)
# if a pandas series is returned, overwrite the index. The
# returned index can be different than the MultiIndex passed to
# the compare function.
if isinstance(c, pandas.Series):
c.index = self.vectors.index
# append column to Compare.vectors
if store:
name_or_id = name if name else len(self.vectors.columns)
self.vectors[name_or_id] = c
return self.vectors[name_or_id].rename(name)