def test_get_rdr_match_values(self, mock_query, mock_response):
# pre conditions
mock_query.return_value = {}
mock_response.return_value = [
{
consts.PERSON_ID_FIELD: 1,
consts.STRING_VALUE_FIELD: 'saLLy',
},
{
consts.PERSON_ID_FIELD: 2,
consts.STRING_VALUE_FIELD: 'Rudy'
},
{
consts.PERSON_ID_FIELD: 3,
consts.STRING_VALUE_FIELD: 'MaTiLdA'
},
]
# test
actual = reader.get_rdr_match_values('project-foo', 'rdr-bar',
'table-oye', 12345)
# postconditions
expected = {1: 'saLLy', 2: 'Rudy', 3: 'MaTiLdA'}
self.assertEqual(actual, expected)
self.assertEqual(mock_query.call_count, 1)
self.assertEqual(mock_response.call_count, 1)
self.assertEqual(
mock_query.assert_called_with(
consts.PPI_OBSERVATION_VALUES.format(project='project-foo',
dataset='rdr-bar',
table='table-oye',
field_value=12345)), None)
def _compare_birth_dates(project, validation_dataset, pii_dataset, site,
concept_id_pii, pii_tables):
"""
Compare birth dates for people.
Converts birthdates and birth_datetimes to calendar objects. Converts
the calendar objects back to strings with the same format and compares
these strings.
:param project: project to search for the datasets
:param validation_dataset: the auto generated match validation dataset
created in this module. queried to get the gender value
:param pii_dataset: dataset created from submitted hpo sites. the pii tables
:param site: string identifier of hpo
:param concept_id_pii: integer value of concept id for concept in the rdr_dataset
:return: updated match_values dictionary
"""
match_values = {}
table_name = site + consts.EHR_PERSON_TABLE_SUFFIX
if table_name in pii_tables:
pii_birthdates = readers.get_rdr_match_values(project,
validation_dataset,
consts.ID_MATCH_TABLE,
concept_id_pii)
try:
ehr_birthdates = readers.get_ehr_person_values(
project, pii_dataset, table_name, consts.BIRTH_DATETIME_FIELD)
except (oauth2client.client.HttpAccessTokenRefreshError,
googleapiclient.errors.HttpError):
LOGGER.exception("Unable to read PII for: %s\tdata field:\t%s",
site, consts.BIRTH_DATETIME_FIELD)
raise
# compare birth_datetime from ppi info to ehr info and record results.
for person_id, ehr_birthdate in ehr_birthdates.items():
rdr_birthdate = pii_birthdates.get(person_id)
ehr_birthdate = ehr_birthdates.get(person_id)
if rdr_birthdate is None or ehr_birthdate is None:
match_values[person_id] = consts.MISSING
elif isinstance(rdr_birthdate, str) and isinstance(
ehr_birthdate, str):
# convert values to datetime objects
rdr_date = parse(rdr_birthdate)
ehr_date = parse(ehr_birthdate)
# convert datetime objects to Year/month/day strings and compare
rdr_string = rdr_date.strftime(consts.DATE_FORMAT)
ehr_string = ehr_date.strftime(consts.DATE_FORMAT)
match_str = consts.MATCH if rdr_string == ehr_string else consts.MISMATCH
match_values[person_id] = match_str
else:
match_values[person_id] = consts.MISMATCH
else:
raise RuntimeError('Table {} doesnt exist.'.format(table_name))
return match_values
def _compare_phone_numbers(
project,
rdr_dataset,
pii_dataset,
hpo,
concept_id,
pii_field,
pii_tables
):
"""
Compare the digit based phone numbers from PII and Observation tables.
:param project: project to search for the datasets
:param rdr_dataset: contains datasets from the rdr group
:param pii_dataset: dataset created from submitted hpo sites. the pii tables
:param hpo: string identifier of hpo
:param concept_id: integer value of concept id for concept in the rdr_dataset
:param pii_field: string value of field name with data matching the
concept_id. used to extract the correct values from the pii tables
:return: A match_values dictionary.
"""
match_values = {}
table_name = hpo + consts.PII_PHONE_TABLE
if table_name in pii_tables:
phone_numbers = readers.get_rdr_match_values(
project, rdr_dataset, consts.ID_MATCH_TABLE, concept_id
)
try:
pii_phone_numbers = readers.get_pii_values(
project,
pii_dataset,
hpo,
consts.PII_PHONE_TABLE,
pii_field
)
except (oauth2client.client.HttpAccessTokenRefreshError,
googleapiclient.errors.HttpError):
LOGGER.exception(
"Unable to read PII for: %s\tdata field:\t%s", hpo, pii_field
)
raise
for person_id, pii_number in pii_phone_numbers:
rdr_phone = phone_numbers.get(person_id)
if rdr_phone is None or pii_number is None:
match_str = consts.MISSING
else:
rdr_phone = normalizer.normalize_phone(rdr_phone)
pii_number = normalizer.normalize_phone(pii_number)
match_str = consts.MATCH if rdr_phone == pii_number else consts.MISMATCH
match_values[person_id] = match_str
else:
raise RuntimeError('Table {} doesnt exist.'.format(table_name))
return match_values
def _compare_birth_dates(
project,
validation_dataset,
pii_dataset,
site,
concept_id_pii
):
"""
Compare birth dates for people.
Converts birthdates and birth_datetimes to calendar objects. Converts
the calendar objects back to strings with the same format and compares
these strings.
:param project: project to search for the datasets
:param validation_dataset: the auto generated match validation dataset
created in this module. queried to get the gender value
:param pii_dataset: dataset created from submitted hpo sites. the pii tables
:param hpo: string identifier of hpo
:param concept_id_pii: integer value of concept id for concept in the rdr_dataset
:return: updated match_values dictionary
"""
match_values = {}
pii_birthdates = readers.get_rdr_match_values(
project, validation_dataset, consts.ID_MATCH_TABLE, concept_id_pii
)
ehr_birthdates = readers.get_ehr_person_values(
project,
pii_dataset,
site + consts.EHR_PERSON_TABLE_SUFFIX,
consts.BIRTH_DATETIME_FIELD
)
# compare birth_datetime from ppi info to ehr info and record results.
for person_id, ehr_birthdate in ehr_birthdates.iteritems():
rdr_birthdate = pii_birthdates.get(person_id)
ehr_birthdate = ehr_birthdates.get(person_id)
if rdr_birthdate is None or ehr_birthdate is None:
match_values[person_id] = consts.MISSING
elif isinstance(rdr_birthdate, str) and isinstance(ehr_birthdate, str):
# convert values to datetime objects
rdr_date = parse(rdr_birthdate)
ehr_date = parse(ehr_birthdate)
# convert datetime objects to Year/month/day strings and compare
rdr_string = rdr_date.strftime(consts.DATE)
ehr_string = ehr_date.strftime(consts.DATE)
match_str = consts.MATCH if rdr_string == ehr_string else consts.MISMATCH
match_values[person_id] = match_str
else:
match_values[person_id] = consts.MISMATCH
return match_values
def _compare_genders(project, validation_dataset, pii_dataset, hpo,
concept_id_pii, pii_tables):
"""
Compare genders for people.
Converts birthdates and birth_datetimes to calendar objects. Converts
the calendar objects back to strings with the same format and compares
these strings.
:param project: project to search for the datasets
:param validation_dataset: the auto generated match validation dataset
created in this module. queried to get the gender value
:param pii_dataset: dataset created from submitted hpo sites. the pii tables
:param hpo: string identifier of hpo
:param concept_id_pii: integer value of concept id for concept in the rdr_dataset
:return: updated match_values dictionary
"""
match_values = {}
table_name = hpo + consts.EHR_PERSON_TABLE_SUFFIX
if table_name in pii_tables:
pii_genders = readers.get_rdr_match_values(project, validation_dataset,
consts.ID_MATCH_TABLE,
concept_id_pii)
try:
ehr_genders = readers.get_ehr_person_values(
project, pii_dataset, table_name, consts.GENDER_FIELD)
except (oauth2client.client.HttpAccessTokenRefreshError,
googleapiclient.errors.HttpError):
LOGGER.exception(
f"Unable to read PII for: {hpo}\tdata field:\t{consts.GENDER_FIELD}"
)
raise
# compare gender from ppi info to ehr info and record results.
for person_id, ehr_gender in ehr_genders.items():
rdr_gender = pii_genders.get(person_id, '')
ehr_gender = consts.SEX_CONCEPT_IDS.get(ehr_gender, '')
if rdr_gender is None or ehr_gender is None:
match_str = consts.MISSING
else:
rdr_gender = rdr_gender.lower()
ehr_gender = ehr_gender.lower()
match_str = consts.MATCH if rdr_gender == ehr_gender else consts.MISMATCH
match_values[person_id] = match_str
else:
raise RuntimeError('Table {} doesnt exist.'.format(table_name))
return match_values
def _compare_zip_codes(
project,
validation_dataset,
rdr_dataset,
pii_dataset,
hpo,
concept_id,
pii_field
):
"""
Compare email addresses from hpo PII table and OMOP observation table.
:param project: project to search for the datasets
:param validation_dataset: the auto generated match validation dataset
created in this module. queried to get the location value to identify
a location field
:param rdr_dataset: contains datasets from the rdr group
:param pii_dataset: dataset created from submitted hpo sites. the pii tables
:param hpo: string identifier of hpo
:param concept_id: integer value of concept id for concept in the rdr_dataset
:param pii_field: string value of field name with data matching the
concept_id. used to extract the correct values from the pii tables
:return: a match_value dictionary.
"""
match_values = {}
zip_codes = readers.get_rdr_match_values(
project, validation_dataset, consts.ID_MATCH_TABLE, concept_id
)
pii_zip_codes = readers.get_location_pii(
project,
rdr_dataset,
pii_dataset,
hpo,
consts.PII_ADDRESS_TABLE,
pii_field
)
for person_id, pii_zip_code in pii_zip_codes:
rdr_zip = zip_codes.get(person_id)
if rdr_zip is None or pii_zip_code is None:
match_str = consts.MISSING
else:
rdr_zip = normalizer.normalize_zip(rdr_zip)
pii_zip = normalizer.normalize_zip(pii_zip_code)
match_str = consts.MATCH if rdr_zip == pii_zip else consts.MISMATCH
match_values[person_id] = match_str
return match_values
def _compare_zip_codes(project, validation_dataset, rdr_dataset, pii_dataset,
hpo, concept_id, pii_field, pii_tables):
"""
Compare zip codes from hpo PII table and OMOP observation table.
:param project: project to search for the datasets
:param validation_dataset: the auto generated match validation dataset
created in this module. queried to get the location value to identify
a location field
:param rdr_dataset: contains datasets from the rdr group
:param pii_dataset: dataset created from submitted hpo sites. the pii tables
:param hpo: string identifier of hpo
:param concept_id: integer value of concept id for concept in the rdr_dataset
:param pii_field: string value of field name with data matching the
concept_id. used to extract the correct values from the pii tables
:return: a match_value dictionary.
"""
match_values = {}
table_name = hpo + consts.PII_ADDRESS_TABLE
if table_name in pii_tables:
zip_codes = readers.get_rdr_match_values(project, validation_dataset,
consts.ID_MATCH_TABLE,
concept_id)
try:
pii_zip_codes = readers.get_location_pii(project, rdr_dataset,
pii_dataset, hpo,
consts.PII_ADDRESS_TABLE,
pii_field)
except (oauth2client.client.HttpAccessTokenRefreshError,
googleapiclient.errors.HttpError):
LOGGER.exception(
f"Unable to read PII for: {hpo}\tdata field:\t{pii_field}")
raise
for person_id, pii_zip_code in pii_zip_codes:
rdr_zip = zip_codes.get(person_id)
if rdr_zip is None or pii_zip_code is None:
match_str = consts.MISSING
else:
rdr_zip = normalizer.normalize_zip(rdr_zip)
pii_zip = normalizer.normalize_zip(pii_zip_code)
match_str = consts.MATCH if rdr_zip == pii_zip else consts.MISMATCH
match_values[person_id] = match_str
else:
raise RuntimeError('Table {} doesnt exist.'.format(table_name))
return match_values
def _compare_genders(
project,
validation_dataset,
pii_dataset,
hpo,
concept_id_pii
):
"""
Compare genders for people.
Converts birthdates and birth_datetimes to calendar objects. Converts
the calendar objects back to strings with the same format and compares
these strings.
:param project: project to search for the datasets
:param validation_dataset: the auto generated match validation dataset
created in this module. queried to get the gender value
:param pii_dataset: dataset created from submitted hpo sites. the pii tables
:param hpo: string identifier of hpo
:param concept_id_pii: integer value of concept id for concept in the rdr_dataset
:return: updated match_values dictionary
"""
match_values = {}
pii_genders = readers.get_rdr_match_values(
project, validation_dataset, consts.ID_MATCH_TABLE, concept_id_pii
)
ehr_genders = readers.get_ehr_person_values(
project,
pii_dataset,
hpo + consts.EHR_PERSON_TABLE_SUFFIX,
consts.GENDER_FIELD
)
# compare gender from ppi info to ehr info and record results.
for person_id, ehr_gender in ehr_genders.iteritems():
rdr_gender = pii_genders.get(person_id, '')
ehr_gender = consts.SEX_CONCEPT_IDS.get(ehr_gender, '')
rdr_gender = rdr_gender.lower()
ehr_gender = ehr_gender.lower()
if rdr_gender is None or ehr_gender is None:
match_str = consts.MISSING
else:
match_str = consts.MATCH if rdr_gender == ehr_gender else consts.MISMATCH
match_values[person_id] = match_str
return match_values
def _compare_name_fields(project, rdr_dataset, pii_dataset, hpo, concept_id,
pii_field, pii_tables):
"""
For an hpo, compare all first, middle, and last name fields to omop settings.
This compares a site's name field values found in their uploaded PII
tables with the values in the OMOP observation table.
:param project: project to search for the datasets
:param rdr_dataset: contains datasets from the rdr group
:param pii_dataset: dataset created from submitted hpo sites. the pii tables
:param hpo: string identifier of hpo
:param concept_id: integer value of concept id for concept in the rdr_dataset
:param pii_field: string value of field name with data matching the
concept_id. used to extract the correct values from the pii tables
:return: a match_values dictionary.
"""
match_values = {}
table_name = hpo + consts.PII_NAME_TABLE
if table_name in pii_tables:
rdr_names = readers.get_rdr_match_values(project, rdr_dataset,
consts.ID_MATCH_TABLE,
concept_id)
try:
pii_names = readers.get_pii_values(project, pii_dataset, hpo,
consts.PII_NAME_TABLE,
pii_field)
except (oauth2client.client.HttpAccessTokenRefreshError,
googleapiclient.errors.HttpError):
LOGGER.exception(
f"Unable to read PII for: {hpo}\tdata field:\t{pii_field}")
raise
for person_id, pii_name in pii_names:
rdr_name = rdr_names.get(person_id)
if rdr_name is None or pii_name is None:
match_str = consts.MISSING
else:
pii_name = normalizer.normalize_name(pii_name)
rdr_name = normalizer.normalize_name(rdr_name)
match_str = consts.MATCH if rdr_name == pii_name else consts.MISMATCH
match_values[person_id] = match_str
else:
raise RuntimeError('Table {} doesnt exist.'.format(table_name))
return match_values
def _compare_name_fields(
project,
rdr_dataset,
pii_dataset,
hpo,
concept_id,
pii_field
):
"""
For an hpo, compare all first, middle, and last name fields to omop settings.
This compares a site's name field values found in their uploaded PII
tables with the values in the OMOP observation table.
:param project: project to search for the datasets
:param rdr_dataset: contains datasets from the rdr group
:param pii_dataset: dataset created from submitted hpo sites. the pii tables
:param hpo: string identifier of hpo
:param concept_id: integer value of concept id for concept in the rdr_dataset
:param pii_field: string value of field name with data matching the
concept_id. used to extract the correct values from the pii tables
:return: a match_values dictionary.
"""
match_values = {}
rdr_names = readers.get_rdr_match_values(
project, rdr_dataset, consts.ID_MATCH_TABLE, concept_id
)
pii_names = readers.get_pii_values(
project,
pii_dataset,
hpo,
consts.PII_NAME_TABLE,
pii_field
)
for person_id, pii_name in pii_names:
rdr_name = rdr_names.get(person_id)
if rdr_name is None or pii_name is None:
match_str = consts.MISSING
else:
pii_name = normalizer.normalize_name(pii_name)
rdr_name = normalizer.normalize_name(rdr_name)
match_str = consts.MATCH if rdr_name == pii_name else consts.MISMATCH
match_values[person_id] = match_str
return match_values
def _compare_email_addresses(project, rdr_dataset, pii_dataset, hpo,
concept_id, pii_field, pii_tables):
"""
Compare email addresses from hpo PII table and OMOP observation table.
:param project: project to search for the datasets
:param rdr_dataset: contains datasets from the rdr group
:param pii_dataset: dataset created from submitted hpo sites. the pii tables
:param hpo: string identifier of hpo
:param concept_id: integer value of concept id for concept in the rdr_dataset
:param pii_field: string value of field name with data matching the
concept_id. used to extract the correct values from the pii tables
:return: a match_value dictionary.
"""
match_values = {}
table_name = hpo + consts.PII_EMAIL_TABLE
if table_name in pii_tables:
email_addresses = readers.get_rdr_match_values(project, rdr_dataset,
consts.ID_MATCH_TABLE,
concept_id)
try:
pii_emails = readers.get_pii_values(project, pii_dataset, hpo,
consts.PII_EMAIL_TABLE,
pii_field)
except (oauth2client.client.HttpAccessTokenRefreshError,
googleapiclient.errors.HttpError):
LOGGER.exception(
f"Unable to read PII for: {hpo}\tdata field:\t{pii_field}")
raise
for person_id, pii_email in pii_emails:
rdr_email = email_addresses.get(person_id)
if rdr_email is None or pii_email is None:
match_str = consts.MISSING
else:
rdr_email = normalizer.normalize_email(rdr_email)
pii_email = normalizer.normalize_email(pii_email)
match_str = consts.MATCH if rdr_email == pii_email else consts.MISMATCH
match_values[person_id] = match_str
else:
raise RuntimeError('Table {} doesnt exist.'.format(table_name))
return match_values
def _compare_email_addresses(
project,
rdr_dataset,
pii_dataset,
hpo,
concept_id,
pii_field
):
"""
Compare email addresses from hpo PII table and OMOP observation table.
:param project: project to search for the datasets
:param rdr_dataset: contains datasets from the rdr group
:param pii_dataset: dataset created from submitted hpo sites. the pii tables
:param hpo: string identifier of hpo
:param concept_id: integer value of concept id for concept in the rdr_dataset
:param pii_field: string value of field name with data matching the
concept_id. used to extract the correct values from the pii tables
:return: a match_value dictionary.
"""
match_values = {}
email_addresses = readers.get_rdr_match_values(
project, rdr_dataset, consts.ID_MATCH_TABLE, concept_id
)
pii_emails = readers.get_pii_values(
project,
pii_dataset,
hpo,
consts.PII_EMAIL_TABLE,
pii_field
)
for person_id, pii_email in pii_emails:
rdr_email = email_addresses.get(person_id)
if rdr_email is None or pii_email is None:
match_str = consts.MISSING
else:
rdr_email = normalizer.normalize_email(rdr_email)
pii_email = normalizer.normalize_email(pii_email)
match_str = consts.MATCH if rdr_email == pii_email else consts.MISMATCH
match_values[person_id] = match_str
return match_values
def _compare_phone_numbers(
project,
rdr_dataset,
pii_dataset,
hpo,
concept_id,
pii_field
):
"""
Compare the digit based phone numbers from PII and Observation tables.
:param project: project to search for the datasets
:param rdr_dataset: contains datasets from the rdr group
:param pii_dataset: dataset created from submitted hpo sites. the pii tables
:param hpo: string identifier of hpo
:param concept_id: integer value of concept id for concept in the rdr_dataset
:param pii_field: string value of field name with data matching the
concept_id. used to extract the correct values from the pii tables
:return: A match_values dictionary.
"""
match_values = {}
phone_numbers = readers.get_rdr_match_values(
project, rdr_dataset, consts.ID_MATCH_TABLE, concept_id
)
pii_phone_numbers = readers.get_pii_values(
project,
pii_dataset,
hpo,
consts.PII_PHONE_TABLE,
pii_field
)
for person_id, pii_number in pii_phone_numbers:
rdr_phone = phone_numbers.get(person_id)
if rdr_phone is None or pii_number is None:
match_str = consts.MISSING
else:
rdr_phone = normalizer.normalize_phone(rdr_phone)
pii_number = normalizer.normalize_phone(pii_number)
match_str = consts.MATCH if rdr_phone == pii_number else consts.MISMATCH
match_values[person_id] = match_str
return match_values
def _compare_states(project, validation_dataset, rdr_dataset, pii_dataset, hpo,
concept_id, pii_field):
"""
Compare email addresses from hpo PII table and OMOP observation table.
:param project: project to search for the datasets
:param validation_dataset: the auto generated match validation dataset
created in this module. queried to get the location value to identify
a location field
:param rdr_dataset: contains datasets from the rdr group
:param pii_dataset: dataset created from submitted hpo sites. the pii tables
:param hpo: string identifier of hpo
:param concept_id: integer value of concept id for concept in the rdr_dataset
:param pii_field: string value of field name with data matching the
concept_id. used to extract the correct values from the pii tables
:return: a match_value dictionary.
"""
match_values = {}
states = readers.get_rdr_match_values(project, validation_dataset,
consts.ID_MATCH_TABLE, concept_id)
try:
pii_states = readers.get_location_pii(project, rdr_dataset,
pii_dataset, hpo,
consts.PII_ADDRESS_TABLE,
pii_field)
except (oauth2client.client.HttpAccessTokenRefreshError,
googleapiclient.errors.HttpError) as exception:
LOGGER.exception("Unable to read PII for: %s\tdata field:\t%s", hpo,
pii_field)
return match_values, exception
for person_id, pii_state in pii_states:
rdr_state = states.get(person_id)
if rdr_state is None or pii_state is None:
match_str = consts.MISSING
else:
rdr_state = normalizer.normalize_state(rdr_state)
pii_state = normalizer.normalize_state(pii_state)
match_str = consts.MATCH if rdr_state == pii_state else consts.MISMATCH
match_values[person_id] = match_str
return match_values, None
def _compare_street_addresses(project, validation_dataset, rdr_dataset,
pii_dataset, hpo, concept_id_one, concept_id_two,
field_one, field_two, pii_tables):
"""
Compare the components of the standard address field.
Individually compares the address one, address two, city, state, and zip
fields of an address. Compares address one and address two as distinct
fields and if they do not match, then combines the fields and compares as
a single field. Both are either set as a match or not match.
:param project: project to search for the datasets
:param validation_dataset: the auto generated match validation dataset
created in this module. queried to get the location value to identify
a location field
:param rdr_dataset: contains datasets from the rdr group
:param pii_dataset: dataset created from submitted hpo sites. the pii tables
:param hpo: string identifier of hpo
:param concept_id_one: integer value of concept id for concept in the rdr_dataset
:param concept_id_two: integer value of concept id for concept in the rdr_dataset
:param field_one: string value of field name with data matching the
concept_id. used to extract the correct values from the pii tables
:param field_two: string value of field name with data matching the
concept_id. used to extract the correct values from the pii tables
:param hpo: hpo site name used to download pii from the site's pii table
:return: a match_values dictionary.
"""
address_one_match_values = {}
address_two_match_values = {}
table_name = hpo + consts.PII_ADDRESS_TABLE
if table_name in pii_tables:
rdr_address_ones = readers.get_rdr_match_values(
project, validation_dataset, consts.ID_MATCH_TABLE, concept_id_one)
rdr_address_twos = readers.get_rdr_match_values(
project, validation_dataset, consts.ID_MATCH_TABLE, concept_id_two)
try:
pii_street_ones = readers.get_location_pii(
project, rdr_dataset, pii_dataset, hpo,
consts.PII_ADDRESS_TABLE, field_one)
except (oauth2client.client.HttpAccessTokenRefreshError,
googleapiclient.errors.HttpError):
LOGGER.exception(
f"Unable to read PII for: {hpo}\tdata field:\t{field_one}")
raise
try:
pii_street_twos = readers.get_location_pii(
project, rdr_dataset, pii_dataset, hpo,
consts.PII_ADDRESS_TABLE, field_two)
except (oauth2client.client.HttpAccessTokenRefreshError,
googleapiclient.errors.HttpError):
LOGGER.exception(
f"Unable to read PII for: {hpo}\tdata field:\t{field_two}")
raise
pii_street_addresses = {}
for person_id, street in pii_street_ones:
pii_street_addresses[person_id] = [person_id, street]
for person_id, street in pii_street_twos:
current_value = pii_street_addresses.get(person_id, [])
if current_value == []:
current_value = [person_id, '', street]
else:
current_value.append(street)
pii_street_addresses[person_id] = current_value
for person_id, addresses in pii_street_addresses.items():
pii_addr_one = addresses[1]
pii_addr_two = addresses[2]
rdr_addr_one = normalizer.normalize_street(
rdr_address_ones.get(person_id))
pii_addr_one = normalizer.normalize_street(pii_addr_one)
rdr_addr_two = normalizer.normalize_street(
rdr_address_twos.get(person_id))
pii_addr_two = normalizer.normalize_street(pii_addr_two)
# easy case, fields 1 and 2 from both sources match exactly
if rdr_addr_one == pii_addr_one and rdr_addr_two == pii_addr_two:
address_one_match_values[person_id] = consts.MATCH
address_two_match_values[person_id] = consts.MATCH
else:
# convert two fields to one field and store as a list of strings
full_rdr_street = rdr_addr_one + ' ' + rdr_addr_two
full_pii_street = pii_addr_one + ' ' + pii_addr_two
full_rdr_street_list = full_rdr_street.split()
full_pii_street_list = full_pii_street.split()
# check top see if each item in one list is in the other list and
# set match results from that
missing_rdr = _compare_address_lists(full_rdr_street_list,
full_pii_street_list)
missing_pii = _compare_address_lists(full_pii_street_list,
full_rdr_street_list)
if (missing_rdr + missing_pii) > 0:
address_one_match_values[person_id] = consts.MISMATCH
address_two_match_values[person_id] = consts.MISMATCH
else:
address_one_match_values[person_id] = consts.MATCH
address_two_match_values[person_id] = consts.MATCH
else:
raise RuntimeError('Table {} doesnt exist.'.format(table_name))
return address_one_match_values, address_two_match_values
