def testIsNotNone(self): # - - - - - - - - - - - - - - - - - - - - - - - - -
"""Test 'check_is_not_none' function."""
assert (auxiliary.check_is_not_none('TestArgument','hello') == None)
assert (auxiliary.check_is_not_none('TestArgument',1) == None)
assert (auxiliary.check_is_not_none('TestArgument',0) == None)
assert (auxiliary.check_is_not_none('TestArgument',-111) == None)
assert (auxiliary.check_is_not_none('TestArgument',0.555) == None)
assert (auxiliary.check_is_not_none('TestArgument',{}) == None)
assert (auxiliary.check_is_not_none('TestArgument',[]) == None)
def testIsNotNone(self): # - - - - - - - - - - - - - - - - - - - - - - - - -
"""Test 'check_is_not_none' function."""
assert (auxiliary.check_is_not_none('TestArgument','hello') == None)
assert (auxiliary.check_is_not_none('TestArgument',1) == None)
assert (auxiliary.check_is_not_none('TestArgument',0) == None)
assert (auxiliary.check_is_not_none('TestArgument',-111) == None)
assert (auxiliary.check_is_not_none('TestArgument',0.555) == None)
assert (auxiliary.check_is_not_none('TestArgument',{}) == None)
assert (auxiliary.check_is_not_none('TestArgument',[]) == None)
def testIsNotNone(self): # - - - - - - - - - - - - - - - - - - - - - - - - -
"""Test 'check_is_not_none' function."""
assert auxiliary.check_is_not_none("TestArgument", "hello") == None
assert auxiliary.check_is_not_none("TestArgument", 1) == None
assert auxiliary.check_is_not_none("TestArgument", 0) == None
assert auxiliary.check_is_not_none("TestArgument", -111) == None
assert auxiliary.check_is_not_none("TestArgument", 0.555) == None
assert auxiliary.check_is_not_none("TestArgument", {}) == None
assert auxiliary.check_is_not_none("TestArgument", []) == None
def testIsNotNone(
self): # - - - - - - - - - - - - - - - - - - - - - - - - -
"""Test 'check_is_not_none' function."""
assert auxiliary.check_is_not_none("TestArgument", "hello")
assert auxiliary.check_is_not_none("TestArgument", 1)
assert auxiliary.check_is_not_none("TestArgument", 0)
assert auxiliary.check_is_not_none("TestArgument", -111)
assert auxiliary.check_is_not_none("TestArgument", 0.555)
assert auxiliary.check_is_not_none("TestArgument", {})
assert auxiliary.check_is_not_none("TestArgument", [])
def SaveMatchDataSet(match_set, dataset1, id_field1, new_dataset_name1,
dataset2=None, id_field2=None, new_dataset_name2=None):
"""Save the original data set(s) with an additional field (attribute) that
contains match identifiers.
This functions creates unique match identifiers (one for each matched pair
of record identifiers in the given match set), and inserts them into a new
attribute (field) of a data set(s) which will be written.
If the record identifier field is not one of the fields in the input data
set, then additionally such a field will be added to the output data set
(with the name of the record identifier from the input data set).
Currently the output data set(s) to be written will be CSV type data sets.
Match identifiers as or the form 'mid00001', 'mid0002', etc. with the
number of digits depending upon the total number of matches in the match
set. If a record is involved in several matches, then the match
identifiers will be separated by a semi-colon (;).
Only one new data set will be created for deduplication, and two new data
sets for linkage.
For a deduplication, it is assumed that the second data set is set to
None.
"""
auxiliary.check_is_set('match_set', match_set)
auxiliary.check_is_not_none('dataset1', dataset1)
auxiliary.check_is_string('id_field1', id_field1)
auxiliary.check_is_string('new_dataset_name1', new_dataset_name1)
if (dataset2 != None): # A linkage, check second set of parameters
auxiliary.check_is_not_none('dataset2', dataset2)
auxiliary.check_is_string('id_field2', id_field2)
auxiliary.check_is_string('new_dataset_name2', new_dataset_name2)
do_link = True
else:
do_link = False
match_rec_id_list = list(match_set) # Make a list so it can be sorted
match_rec_id_list.sort()
if (len(match_set) > 0):
num_digit = max(1,int(math.ceil(math.log(len(match_set), 10))))
else:
num_digit = 1
mid_count = 1 # Counter for match identifiers
# Generate a dictionary with record identifiers as keys and lists of match
# identifiers as values
#
match_id_dict1 = {} # For first data set
match_id_dict2 = {} # For second data set, not required for deduplication
for rec_id_tuple in match_rec_id_list:
rec_id1, rec_id2 = rec_id_tuple
mid_count_str = '%s' % (mid_count)
this_mid = 'mid%s' % (mid_count_str.zfill(num_digit))
rec_id1_mid_list = match_id_dict1.get(rec_id1, [])
rec_id1_mid_list.append(this_mid)
match_id_dict1[rec_id1] = rec_id1_mid_list
if (do_link == True): # Do the same for second data set
rec_id2_mid_list = match_id_dict2.get(rec_id2, [])
rec_id2_mid_list.append(this_mid)
match_id_dict2[rec_id2] = rec_id2_mid_list
else: # Same dicionary for deduplication
rec_id2_mid_list = match_id_dict1.get(rec_id2, [])
rec_id2_mid_list.append(this_mid)
match_id_dict1[rec_id2] = rec_id2_mid_list
mid_count += 1
# Now initialise new data set(s) for output based on input data set(s) - - -
# First need to generate field list from input data set
#
if (dataset1.dataset_type == 'CSV'):
new_dataset1_field_list = dataset1.field_list[:] # Make a copy of list
last_col_index = new_dataset1_field_list[-1][1]+1
elif (dataset1.dataset_type == 'COL'):
new_dataset1_field_list = []
col_index = 0
for (field, col_width) in dataset1.field_list:
new_dataset1_field_list.append((field, col_index))
col_index += 1
last_col_index = col_index
# Check if the record identifier is not a normal input field (in which case
# it has to be written into the output data set as well)
#
rec_ident_name = dataset1.rec_ident
add_rec_ident = True
for (field_name, field_data) in dataset1.field_list:
if (field_name == rec_ident_name):
add_rec_ident = False
break
if (add_rec_ident == True): # Put record identifier into first column
new_dataset1_field_list.append((rec_ident_name, last_col_index))
last_col_index += 1
# Append match id field
#
new_dataset1_field_list.append((id_field1, last_col_index))
new_dataset1_description = dataset1.description+' with match identifiers'
new_dataset1 = dataset.DataSetCSV(description=new_dataset1_description,
access_mode='write',
rec_ident=dataset1.rec_ident,
header_line=True,
write_header=True,
strip_fields = dataset1.strip_fields,
miss_val = dataset1.miss_val,
field_list = new_dataset1_field_list,
delimiter = dataset1.delimiter,
file_name = new_dataset_name1)
# Read all records, add match identifiers and write into new data set
#
for (rec_id, rec_list) in dataset1.readall():
if (add_rec_ident == True): # Add record identifier
rec_list.append(rec_id)
mid_list = match_id_dict1.get(rec_id, [])
mid_str = ';'.join(mid_list)
rec_list.append(mid_str)
new_dataset1.write({rec_id:rec_list})
new_dataset1.finalise()
if (do_link == True): # Second data set for linkage only - - - - - - - - - -
if (dataset2.dataset_type == 'CSV'):
new_dataset2_field_list = dataset2.field_list[:] # Make a copy of list
last_col_index = new_dataset2_field_list[-1][1]+1
elif (dataset2.dataset_type == 'COL'):
new_dataset2_field_list = []
col_index = 0
for (field, col_width) in dataset2.field_list:
new_dataset2_field_list.append((field, col_index))
col_index += 1
last_col_index = col_index
# Check if the record identifier is not an normal input field (in which
# case it has to be written into the output data set as well)
#
rec_ident_name = dataset2.rec_ident
add_rec_ident = True
for (field_name, field_data) in dataset2.field_list:
if (field_name == rec_ident_name):
add_rec_ident = False
break
if (add_rec_ident == True): # Put record identifier into first column
new_dataset2_field_list.append((rec_ident_name, last_col_index))
last_col_index += 1
# Append match id field
#
new_dataset2_field_list.append((id_field2, last_col_index))
new_dataset2_description = dataset2.description+' with match identifiers'
new_dataset2 = dataset.DataSetCSV(description=new_dataset2_description,
access_mode='write',
rec_ident=dataset2.rec_ident,
header_line=True,
write_header=True,
strip_fields = dataset2.strip_fields,
miss_val = dataset2.miss_val,
field_list = new_dataset2_field_list,
file_name = new_dataset_name2)
# Read all records, add match identifiers and write into new data set
#
for (rec_id, rec_list) in dataset2.readall():
if (add_rec_ident == True): # Add record identifier
rec_list.append(rec_id)
mid_list = match_id_dict2.get(rec_id, [])
mid_str = ';'.join(mid_list)
rec_list.append(mid_str)
new_dataset2.write({rec_id:rec_list})
new_dataset2.finalise()
def SaveMatchDataSet(match_set,
dataset1,
id_field1,
new_dataset_name1,
dataset2=None,
id_field2=None,
new_dataset_name2=None):
"""Save the original data set(s) with an additional field (attribute) that
contains match identifiers.
This functions creates unique match identifiers (one for each matched pair
of record identifiers in the given match set), and inserts them into a new
attribute (field) of a data set(s) which will be written.
If the record identifier field is not one of the fields in the input data
set, then additionally such a field will be added to the output data set
(with the name of the record identifier from the input data set).
Currently the output data set(s) to be written will be CSV type data sets.
Match identifiers as or the form 'mid00001', 'mid0002', etc. with the
number of digits depending upon the total number of matches in the match
set. If a record is involved in several matches, then the match
identifiers will be separated by a semi-colon (;).
Only one new data set will be created for deduplication, and two new data
sets for linkage.
For a deduplication, it is assumed that the second data set is set to
None.
"""
auxiliary.check_is_set('match_set', match_set)
auxiliary.check_is_not_none('dataset1', dataset1)
auxiliary.check_is_string('id_field1', id_field1)
auxiliary.check_is_string('new_dataset_name1', new_dataset_name1)
if (dataset2 != None): # A linkage, check second set of parameters
auxiliary.check_is_not_none('dataset2', dataset2)
auxiliary.check_is_string('id_field2', id_field2)
auxiliary.check_is_string('new_dataset_name2', new_dataset_name2)
do_link = True
else:
do_link = False
match_rec_id_list = list(match_set) # Make a list so it can be sorted
match_rec_id_list.sort()
if (len(match_set) > 0):
num_digit = max(1, int(math.ceil(math.log(len(match_set), 10))))
else:
num_digit = 1
mid_count = 1 # Counter for match identifiers
# Generate a dictionary with record identifiers as keys and lists of match
# identifiers as values
#
match_id_dict1 = {} # For first data set
match_id_dict2 = {} # For second data set, not required for deduplication
for rec_id_tuple in match_rec_id_list:
rec_id1, rec_id2 = rec_id_tuple
mid_count_str = '%s' % (mid_count)
this_mid = 'mid%s' % (mid_count_str.zfill(num_digit))
rec_id1_mid_list = match_id_dict1.get(rec_id1, [])
rec_id1_mid_list.append(this_mid)
match_id_dict1[rec_id1] = rec_id1_mid_list
if (do_link == True): # Do the same for second data set
rec_id2_mid_list = match_id_dict2.get(rec_id2, [])
rec_id2_mid_list.append(this_mid)
match_id_dict2[rec_id2] = rec_id2_mid_list
else: # Same dicionary for deduplication
rec_id2_mid_list = match_id_dict1.get(rec_id2, [])
rec_id2_mid_list.append(this_mid)
match_id_dict1[rec_id2] = rec_id2_mid_list
mid_count += 1
# Now initialise new data set(s) for output based on input data set(s) - - -
# First need to generate field list from input data set
#
if (dataset1.dataset_type == 'CSV'):
new_dataset1_field_list = dataset1.field_list[:] # Make a copy of list
last_col_index = new_dataset1_field_list[-1][1] + 1
elif (dataset1.dataset_type == 'COL'):
new_dataset1_field_list = []
col_index = 0
for (field, col_width) in dataset1.field_list:
new_dataset1_field_list.append((field, col_index))
col_index += 1
last_col_index = col_index
# Check if the record identifier is not a normal input field (in which case
# it has to be written into the output data set as well)
#
rec_ident_name = dataset1.rec_ident
add_rec_ident = True
for (field_name, field_data) in dataset1.field_list:
if (field_name == rec_ident_name):
add_rec_ident = False
break
if (add_rec_ident == True): # Put record identifier into first column
new_dataset1_field_list.append((rec_ident_name, last_col_index))
last_col_index += 1
# Append match id field
#
new_dataset1_field_list.append((id_field1, last_col_index))
new_dataset1_description = dataset1.description + ' with match identifiers'
new_dataset1 = dataset.DataSetCSV(description=new_dataset1_description,
access_mode='write',
rec_ident=dataset1.rec_ident,
header_line=True,
write_header=True,
strip_fields=dataset1.strip_fields,
miss_val=dataset1.miss_val,
field_list=new_dataset1_field_list,
delimiter=dataset1.delimiter,
file_name=new_dataset_name1)
# Read all records, add match identifiers and write into new data set
#
for (rec_id, rec_list) in dataset1.readall():
if (add_rec_ident == True): # Add record identifier
rec_list.append(rec_id)
mid_list = match_id_dict1.get(rec_id, [])
mid_str = ';'.join(mid_list)
rec_list.append(mid_str)
new_dataset1.write({rec_id: rec_list})
new_dataset1.finalise()
if (do_link == True
): # Second data set for linkage only - - - - - - - - - -
if (dataset2.dataset_type == 'CSV'):
new_dataset2_field_list = dataset2.field_list[:] # Make a copy of list
last_col_index = new_dataset2_field_list[-1][1] + 1
elif (dataset2.dataset_type == 'COL'):
new_dataset2_field_list = []
col_index = 0
for (field, col_width) in dataset2.field_list:
new_dataset2_field_list.append((field, col_index))
col_index += 1
last_col_index = col_index
# Check if the record identifier is not an normal input field (in which
# case it has to be written into the output data set as well)
#
rec_ident_name = dataset2.rec_ident
add_rec_ident = True
for (field_name, field_data) in dataset2.field_list:
if (field_name == rec_ident_name):
add_rec_ident = False
break
if (add_rec_ident == True): # Put record identifier into first column
new_dataset2_field_list.append((rec_ident_name, last_col_index))
last_col_index += 1
# Append match id field
#
new_dataset2_field_list.append((id_field2, last_col_index))
new_dataset2_description = dataset2.description + ' with match identifiers'
new_dataset2 = dataset.DataSetCSV(description=new_dataset2_description,
access_mode='write',
rec_ident=dataset2.rec_ident,
header_line=True,
write_header=True,
strip_fields=dataset2.strip_fields,
miss_val=dataset2.miss_val,
field_list=new_dataset2_field_list,
file_name=new_dataset_name2)
# Read all records, add match identifiers and write into new data set
#
for (rec_id, rec_list) in dataset2.readall():
if (add_rec_ident == True): # Add record identifier
rec_list.append(rec_id)
mid_list = match_id_dict2.get(rec_id, [])
mid_str = ';'.join(mid_list)
rec_list.append(mid_str)
new_dataset2.write({rec_id: rec_list})
new_dataset2.finalise()
def pairs_completeness(weight_vec_dict, dataset1, dataset2, get_id_funct,
match_check_funct):
"""Pairs completeness is measured as
pc = Nm / M
with Nm (<= M) being the number of correctly classified truly matched
record pairs in the blocked comparison space, and M the total number of
true matches.
If both data sets are the same a deduplication is assumed, otherwise a
linkage.
The arguments that have to be set when this method is called are:
weight_vec_dict A dictionary containing weight vectors.
dataset1 The initialised first data set object.
dataset2 The initialised second data set object.
get_id_funct This has to be a function (or method), assumed to
have argument a record (assumed to be a list fo field
values), and returns the record identifier from that
record.
match_check_funct This has to be a function (or method), assumed to
have as arguments the two record identifiers of a
record pair and its weight vector, and returns True
if the record pair is from a true match, or False
otherwise. Thus, 'match_check_funct' is of the form:
match_flag = match_check_funct(rec_id1, rec_id2,
weight_vec)
"""
auxiliary.check_is_dictionary('weight_vec_dict', weight_vec_dict)
auxiliary.check_is_not_none('dataset1', dataset1)
auxiliary.check_is_not_none('dataset2', dataset2)
auxiliary.check_is_function_or_method('get_id_funct', get_id_funct)
auxiliary.check_is_function_or_method('match_check_funct',
match_check_funct)
# Check if a deduplication will be done or a linkage - - - - - - - - - - - -
#
if (dataset1 == dataset2):
do_dedup = True
else:
do_dedup = False
logging.info('')
logging.info('Calculate pairs completeness:')
logging.info(' Data set 1: %s (containing %d records)' % \
(dataset1.description, dataset1.num_records))
if (do_dedup == True):
logging.info(' Data sets are the same: Deduplication')
else:
logging.info(' Data set 2: %s (containing %d records)' % \
(dataset2.description, dataset2.num_records))
logging.info(' Data sets differ: Linkage')
logging.info(' Number of record pairs in weight vector dictionary: %d' % \
(len(weight_vec_dict)))
num_all_true_matches = 0 # Count the total number of all true matches
# For a deduplication only process data set 1 - - - - - - - - - - - - - - - -
#
if (do_dedup == True):
# Build a dictionary with entity identifiers as keys and a list of their
# record identifier (rec_ident) as values
#
entity_ident_dict = {}
for (rec_ident, rec) in dataset1.readall():
ent_id = get_id_funct(rec)
this_rec_list = entity_ident_dict.get(ent_id, [])
this_rec_list.append(rec_ident)
entity_ident_dict[ent_id] = this_rec_list
logging.info(' Number of unique entity identifiers in data set 1: %d' % \
(len(entity_ident_dict)))
for (ent_id, rec_list) in entity_ident_dict.iteritems():
num_this_rec = len(rec_list)
if (num_this_rec > 1):
num_all_true_matches += num_this_rec * (num_this_rec - 1) / 2
# More efficent version: Only count number of matches ber record don't
# store them
#
entity_ident_dict2 = {}
for (rec_ident, rec) in dataset1.readall():
ent_id = get_id_funct(rec)
ent_id_count = entity_ident_dict2.get(ent_id, 0) + 1
entity_ident_dict2[ent_id] = ent_id_count
assert sum(entity_ident_dict2.values()) == dataset1.num_records
tm = 0 # Total number of true matches (without indexing)
for (ent_id, ent_count) in entity_ident_dict2.iteritems():
tm += ent_count * (ent_count - 1) / 2
assert num_all_true_matches == tm
else: # For a linkage - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Build two dictionaries with entity identifiers as keys and a list of
# their record identifier (rec_ident) as values
#
entity_ident_dict1 = {}
entity_ident_dict2 = {}
for (rec_ident, rec) in dataset1.readall():
ent_id = get_id_funct(rec)
this_rec_list = entity_ident_dict1.get(ent_id, [])
this_rec_list.append(rec_ident)
entity_ident_dict1[ent_id] = this_rec_list
logging.info(' Number of unique entity identifiers in data set 1: %d' % \
(len(entity_ident_dict1)))
for (rec_ident, rec) in dataset2.readall():
ent_id = get_id_funct(rec)
this_rec_list = entity_ident_dict2.get(ent_id, [])
this_rec_list.append(rec_ident)
entity_ident_dict2[ent_id] = this_rec_list
logging.info(' Number of unique entity identifiers in data set 2: %d' % \
(len(entity_ident_dict2)))
# Now calculate total true match number (loop over smaller dict)
#
if (len(entity_ident_dict1) < len(entity_ident_dict2)):
for (ent_id1, rec_list1) in entity_ident_dict1.iteritems():
if (ent_id1 in entity_ident_dict2):
rec_list2 = entity_ident_dict2[ent_id1]
num_all_true_matches += len(rec_list1) * len(rec_list2)
else:
for (ent_id2, rec_list2) in entity_ident_dict2.iteritems():
if (ent_id2 in entity_ident_dict1):
rec_list1 = entity_ident_dict1[ent_id2]
num_all_true_matches += len(rec_list1) * len(rec_list2)
# More efficent version: Only count number of matches ber record don't
# store them
#
entity_ident_dict3 = {}
entity_ident_dict4 = {}
for (rec_ident, rec) in dataset1.readall():
ent_id = get_id_funct(rec)
ent_id_count = entity_ident_dict3.get(ent_id, 0) + 1
entity_ident_dict3[ent_id] = ent_id_count
for (rec_ident, rec) in dataset2.readall():
ent_id = get_id_funct(rec)
ent_id_count = entity_ident_dict4.get(ent_id, 0) + 1
entity_ident_dict4[ent_id] = ent_id_count
assert sum(entity_ident_dict3.values()) == dataset1.num_records
assert sum(entity_ident_dict4.values()) == dataset2.num_records
tm = 0 # Total number of true matches (without indexing)
if (len(entity_ident_dict3) < len(entity_ident_dict4)):
for (ent_id, ent_count) in entity_ident_dict3.iteritems():
if ent_id in entity_ident_dict4:
tm += ent_count * entity_ident_dict4[ent_id]
else:
for (ent_id, ent_count) in entity_ident_dict4.iteritems():
if ent_id in entity_ident_dict3:
tm += ent_count * entity_ident_dict3[ent_id]
assert num_all_true_matches == tm
logging.info(' Number of all true matches: %d' % (num_all_true_matches))
# Get number of true matches in weight vector dictionary - - - - - - - - - -
#
num_true_matches = 0
num_false_matches = 0
for (rec_id_tuple, this_vec) in weight_vec_dict.iteritems():
if (match_check_funct(rec_id_tuple[0], rec_id_tuple[1],
this_vec) == True):
num_true_matches += 1
else:
num_false_matches += 1
assert len(weight_vec_dict) == num_true_matches + num_false_matches
logging.info(' Number of true and false matches in weight vector ' + \
'dictionary: %d / %d' % (num_true_matches,num_false_matches))
if (num_all_true_matches > 0):
pc = float(num_true_matches) / float(num_all_true_matches)
logging.info(' Pairs completeness: %.4f%%' %
(100.0 * pc)) # As percentage
else:
pc = 0.0
logging.info(' No true matches - cannot calculate pairs completeness')
assert pc <= 1.0, pc
return pc
def pairs_completeness(weight_vec_dict, dataset1, dataset2, get_id_funct,
match_check_funct):
"""Pairs completeness is measured as
pc = Nm / M
with Nm (<= M) being the number of correctly classified truly matched
record pairs in the blocked comparison space, and M the total number of
true matches.
If both data sets are the same a deduplication is assumed, otherwise a
linkage.
The arguments that have to be set when this method is called are:
weight_vec_dict A dictionary containing weight vectors.
dataset1 The initialised first data set object.
dataset2 The initialised second data set object.
get_id_funct This has to be a function (or method), assumed to
have argument a record (assumed to be a list fo field
values), and returns the record identifier from that
record.
match_check_funct This has to be a function (or method), assumed to
have as arguments the two record identifiers of a
record pair and its weight vector, and returns True
if the record pair is from a true match, or False
otherwise. Thus, 'match_check_funct' is of the form:
match_flag = match_check_funct(rec_id1, rec_id2,
weight_vec)
"""
auxiliary.check_is_dictionary('weight_vec_dict', weight_vec_dict)
auxiliary.check_is_not_none('dataset1', dataset1)
auxiliary.check_is_not_none('dataset2', dataset2)
auxiliary.check_is_function_or_method('get_id_funct', get_id_funct)
auxiliary.check_is_function_or_method('match_check_funct', match_check_funct)
# Check if a deduplication will be done or a linkage - - - - - - - - - - - -
#
if (dataset1 == dataset2):
do_dedup = True
else:
do_dedup = False
logging.info('')
logging.info('Calculate pairs completeness:')
logging.info(' Data set 1: %s (containing %d records)' % \
(dataset1.description, dataset1.num_records))
if (do_dedup == True):
logging.info(' Data sets are the same: Deduplication')
else:
logging.info(' Data set 2: %s (containing %d records)' % \
(dataset2.description, dataset2.num_records))
logging.info(' Data sets differ: Linkage')
logging.info(' Number of record pairs in weight vector dictionary: %d' % \
(len(weight_vec_dict)))
num_all_true_matches = 0 # Count the total number of all true matches
# For a deduplication only process data set 1 - - - - - - - - - - - - - - - -
#
if (do_dedup == True):
# Build a dictionary with entity identifiers as keys and a list of their
# record identifier (rec_ident) as values
#
entity_ident_dict = {}
for (rec_ident, rec) in dataset1.readall():
ent_id = get_id_funct(rec)
this_rec_list = entity_ident_dict.get(ent_id, [])
this_rec_list.append(rec_ident)
entity_ident_dict[ent_id] = this_rec_list
logging.info(' Number of unique entity identifiers in data set 1: %d' % \
(len(entity_ident_dict)))
for (ent_id, rec_list) in entity_ident_dict.iteritems():
num_this_rec = len(rec_list)
if (num_this_rec > 1):
num_all_true_matches += num_this_rec*(num_this_rec-1)/2
# More efficent version: Only count number of matches ber record don't
# store them
#
entity_ident_dict2 = {}
for (rec_ident, rec) in dataset1.readall():
ent_id = get_id_funct(rec)
ent_id_count = entity_ident_dict2.get(ent_id, 0) + 1
entity_ident_dict2[ent_id] = ent_id_count
assert sum(entity_ident_dict2.values()) == dataset1.num_records
tm = 0 # Total number of true matches (without indexing)
for (ent_id, ent_count) in entity_ident_dict2.iteritems():
tm += ent_count*(ent_count-1)/2
assert num_all_true_matches == tm
else: # For a linkage - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Build two dictionaries with entity identifiers as keys and a list of
# their record identifier (rec_ident) as values
#
entity_ident_dict1 = {}
entity_ident_dict2 = {}
for (rec_ident, rec) in dataset1.readall():
ent_id = get_id_funct(rec)
this_rec_list = entity_ident_dict1.get(ent_id, [])
this_rec_list.append(rec_ident)
entity_ident_dict1[ent_id] = this_rec_list
logging.info(' Number of unique entity identifiers in data set 1: %d' % \
(len(entity_ident_dict1)))
for (rec_ident, rec) in dataset2.readall():
ent_id = get_id_funct(rec)
this_rec_list = entity_ident_dict2.get(ent_id, [])
this_rec_list.append(rec_ident)
entity_ident_dict2[ent_id] = this_rec_list
logging.info(' Number of unique entity identifiers in data set 2: %d' % \
(len(entity_ident_dict2)))
# Now calculate total true match number (loop over smaller dict)
#
if (len(entity_ident_dict1) < len(entity_ident_dict2)):
for (ent_id1, rec_list1) in entity_ident_dict1.iteritems():
if (ent_id1 in entity_ident_dict2):
rec_list2 = entity_ident_dict2[ent_id1]
num_all_true_matches += len(rec_list1) * len(rec_list2)
else:
for (ent_id2, rec_list2) in entity_ident_dict2.iteritems():
if (ent_id2 in entity_ident_dict1):
rec_list1 = entity_ident_dict1[ent_id2]
num_all_true_matches += len(rec_list1) * len(rec_list2)
# More efficent version: Only count number of matches ber record don't
# store them
#
entity_ident_dict3 = {}
entity_ident_dict4 = {}
for (rec_ident, rec) in dataset1.readall():
ent_id = get_id_funct(rec)
ent_id_count = entity_ident_dict3.get(ent_id, 0) + 1
entity_ident_dict3[ent_id] = ent_id_count
for (rec_ident, rec) in dataset2.readall():
ent_id = get_id_funct(rec)
ent_id_count = entity_ident_dict4.get(ent_id, 0) + 1
entity_ident_dict4[ent_id] = ent_id_count
assert sum(entity_ident_dict3.values()) == dataset1.num_records
assert sum(entity_ident_dict4.values()) == dataset2.num_records
tm = 0 # Total number of true matches (without indexing)
if (len(entity_ident_dict3) < len(entity_ident_dict4)):
for (ent_id, ent_count) in entity_ident_dict3.iteritems():
if ent_id in entity_ident_dict4:
tm += ent_count*entity_ident_dict4[ent_id]
else:
for (ent_id, ent_count) in entity_ident_dict4.iteritems():
if ent_id in entity_ident_dict3:
tm += ent_count*entity_ident_dict3[ent_id]
assert num_all_true_matches == tm
logging.info(' Number of all true matches: %d' % (num_all_true_matches))
# Get number of true matches in weight vector dictionary - - - - - - - - - -
#
num_true_matches = 0
num_false_matches = 0
for (rec_id_tuple, this_vec) in weight_vec_dict.iteritems():
if (match_check_funct(rec_id_tuple[0], rec_id_tuple[1], this_vec) == True):
num_true_matches += 1
else:
num_false_matches += 1
assert len(weight_vec_dict) == num_true_matches+num_false_matches
logging.info(' Number of true and false matches in weight vector ' + \
'dictionary: %d / %d' % (num_true_matches,num_false_matches))
if (num_all_true_matches > 0):
pc = float(num_true_matches) / float(num_all_true_matches)
logging.info(' Pairs completeness: %.4f%%' % (100.0*pc)) # As percentage
else:
pc = 0.0
logging.info(' No true matches - cannot calculate pairs completeness')
assert pc <= 1.0, pc
return pc
