def _learnBlocking(self, eta, epsilon):
confident_nonduplicates = training.semiSupervisedNonDuplicates(self.data_sample, self.data_model)
self.training_pairs[0].extend(confident_nonduplicates)
predicate_functions = (
predicates.wholeFieldPredicate,
predicates.tokenFieldPredicate,
predicates.commonIntegerPredicate,
predicates.sameThreeCharStartPredicate,
predicates.sameFiveCharStartPredicate,
predicates.sameSevenCharStartPredicate,
predicates.nearIntegersPredicate,
predicates.commonFourGram,
predicates.commonSixGram,
)
tfidf_thresholds = [0.2, 0.4, 0.6, 0.8]
full_string_records = {}
fields = self.data_model["fields"].keys()
for pair in self.data_sample[0:2000]:
for k, v in pair:
full_string_records[k] = " ".join(v[field] for field in fields)
df_index = tfidf.documentFrequency(full_string_records)
learned_predicates = blocking.blockTraining(
self.training_pairs, predicate_functions, self.data_model, tfidf_thresholds, df_index, eta, epsilon
)
return learned_predicates
def _learnBlocking(self, data_d, eta, epsilon):
confident_nonduplicates = blocking.semiSupervisedNonDuplicates(self.data_d,
self.data_model)
self.training_pairs[0].extend(confident_nonduplicates)
predicate_functions = (predicates.wholeFieldPredicate,
predicates.tokenFieldPredicate,
predicates.commonIntegerPredicate,
predicates.sameThreeCharStartPredicate,
predicates.sameFiveCharStartPredicate,
predicates.sameSevenCharStartPredicate,
predicates.nearIntegersPredicate,
predicates.commonFourGram,
predicates.commonSixGram,
)
tfidf_thresholds = [0.2, 0.4, 0.6, 0.8]
full_string_records = {}
for k, v in data_d.iteritems() :
document = ''
for field in self.data_model['fields'].keys() :
document += v[field]
document += ' '
full_string_records[k] = document
self.df_index = tfidf.documentFrequency(full_string_records)
blocker = blocking.Blocking(self.training_pairs,
predicate_functions,
self.data_model,
tfidf_thresholds,
self.df_index,
eta,
epsilon
)
learned_predicates = blocker.trainBlocking()
return learned_predicates
