def test_normalize_field_dobm(self):
self.assertEqual(Normalize.normalize_field('dobm', '1'), Normalize.hash_sha_256('01'))
self.assertEqual(Normalize.normalize_field('dobm', '9'), Normalize.hash_sha_256('09'))
self.assertEqual(Normalize.normalize_field('dobm', '02'), Normalize.hash_sha_256('02'))
self.assertEqual(Normalize.normalize_field('dobm', '12'), Normalize.hash_sha_256('12'))
self.assertEqual(Normalize.normalize_field('dobm', ''), None)
self.assertEqual(Normalize.normalize_field('dobm', None), None)
def test_normalize_field_dobm_errors(self):
with self.assertRaisesRegex(ValueError, "Invalid format for dobm: '13'"):
Normalize.normalize_field('dobm', '13')
with self.assertRaisesRegex(ValueError, "Invalid format for dobm: '444'"):
Normalize.normalize_field('dobm', '444')
with self.assertRaisesRegex(ValueError, "Invalid format for dobm: '0'"):
Normalize.normalize_field('dobm', '0')
with self.assertRaisesRegex(ValueError, "Invalid format for dobm: '0a'"):
Normalize.normalize_field('dobm', '0a')
with self.assertRaisesRegex(ValueError, "Invalid format for dobm: '-1'"):
Normalize.normalize_field('dobm', '-1')
def test_normalize_validates_email_throws_errors(self):
emails = [
'a',
'abc',
'a@b',
'@b.c',
'.c',
'@',
]
for email in emails:
with self.assertRaisesRegex(
TypeError,
"Invalid email format for the passed email:%s" % email):
Normalize.normalize_field('em', email)
def normalize(self):
normalized_payload = {
'value': self.value,
'currency': Normalize.normalize_field('currency', self.currency),
'content_name': self.content_name,
'content_category': self.content_category,
'content_ids': self.content_ids,
'content_type': self.content_type,
'order_id': self.order_id,
'predicted_ltv': self.predicted_ltv,
'num_items': self.num_items,
'status': self.status,
'search_string': self.search_string,
}
if self.contents is not None:
contents = []
for content in self.contents:
if content is not None:
contents.append(content.normalize())
normalized_payload['contents'] = contents
# Append the custom_properties to the custom_data normalized payload.
if self.custom_properties:
for key in self.custom_properties:
if key in normalized_payload.keys():
raise Exception('Duplicate key in custom_properties:"' + key + '". Please make sure the keys defined in the custom_properties are not already available in standard custom_data property list.')
normalized_payload[key] = self.custom_properties[key]
normalized_payload = {k: v for k, v in normalized_payload.items() if v is not None}
return normalized_payload
def normalize(self):
normalized_payload = {
'value': self.value,
'currency': Normalize.normalize_field('currency', self.currency),
'content_name': self.content_name,
'content_category': self.content_category,
'content_ids': self.content_ids,
'content_type': self.content_type,
'order_id': self.order_id,
'predicted_ltv': self.predicted_ltv,
'num_items': self.num_items,
'status': self.status,
}
if self.contents is not None:
contents = []
for content in self.contents:
if content is not None:
contents.append(content.normalize())
normalized_payload['contents'] = contents
normalized_payload: dict = {
k: v
for k, v in normalized_payload.items() if v is not None
}
return normalized_payload
def normalize(self):
normalized_payload = {
'em':
self.hash_sha_256(Normalize.normalize_field('em', self.email)),
'ph':
self.hash_sha_256(Normalize.normalize_field('ph', self.phone)),
'db':
self.hash_sha_256(
Normalize.normalize_field('db', self.date_of_birth)),
'ln':
self.hash_sha_256(Normalize.normalize_field('ln', self.last_name)),
'fn':
self.hash_sha_256(Normalize.normalize_field('fn',
self.first_name)),
'ct':
self.hash_sha_256(Normalize.normalize_field('ct', self.city)),
'st':
self.hash_sha_256(Normalize.normalize_field('st', self.state)),
'zp':
self.hash_sha_256(Normalize.normalize_field('zp', self.zip_code)),
'country':
self.hash_sha_256(
Normalize.normalize_field('country', self.country_code)),
'external_id':
self.external_id,
'client_ip_address':
self.client_ip_address,
'client_user_agent':
self.client_user_agent,
'fbc':
self.fbc,
'fbp':
self.fbp,
'subscription_id':
self.subscription_id,
'fb_login_id':
self.fb_login_id,
}
if self.gender is not None:
normalized_payload['ge'] = self.hash_sha_256(
Normalize.normalize_field('ge', self.gender.value))
normalized_payload = {
k: v
for k, v in normalized_payload.items() if v is not None
}
return normalized_payload
def test_normalize_field_doby(self):
self.assertEqual(Normalize.normalize_field('doby', '2000'),
Normalize.hash_sha_256('2000'))
self.assertEqual(Normalize.normalize_field('doby', '0000'),
Normalize.hash_sha_256('0000'))
self.assertEqual(Normalize.normalize_field('doby', '9999'),
Normalize.hash_sha_256('9999'))
self.assertEqual(Normalize.normalize_field('doby', ''), None)
self.assertEqual(Normalize.normalize_field('doby', None), None)
def __normalize_list(self, field_name, value_list):
"""Dedup, hash and normalize the given list.
:type: field_name: str
:type value_list: list[str]
:rtype: dict
"""
if field_name is None or not value_list:
return None
normalized_list = list(map(lambda val: Normalize.normalize_field(field_name, val), value_list))
return self.__dedup_list(normalized_list)
def test_normalize_dobm(self):
self.assertEqual(Normalize.normalize_field('dobm', '1'), '01')
self.assertEqual(Normalize.normalize_field('dobm', '9'), '09')
self.assertEqual(Normalize.normalize_field('dobm', '02'), '02')
self.assertEqual(Normalize.normalize_field('dobm', '12'), '12')
self.assertEqual(Normalize.normalize_field('dobm', ''), None)
self.assertEqual(Normalize.normalize_field('dobm', None), None)
def test_normalize_it_normalizes_and_hashes(self):
initial_state = {
'f5first': 'First',
'f5last': 'Last',
'fi': 'A',
'dobd': '01',
'dobm': '02',
'doby': '2000',
'lead_id': 'lead-id-3',
}
user_data = UserData(
f5first=initial_state['f5first'],
f5last=initial_state['f5last'],
fi=initial_state['fi'],
dobd=initial_state['dobd'],
dobm=initial_state['dobm'],
doby=initial_state['doby'],
lead_id=initial_state['lead_id'],
)
actual = user_data.normalize()
expected = {
'f5first':
Normalize.hash_sha_256(initial_state['f5first'].lower()),
'f5last': Normalize.hash_sha_256(initial_state['f5last'].lower()),
'fi': Normalize.hash_sha_256(initial_state['fi'].lower()),
'dobd': Normalize.hash_sha_256(initial_state['dobd']),
'dobm': Normalize.hash_sha_256(initial_state['dobm']),
'doby': Normalize.hash_sha_256(initial_state['doby']),
'lead_id': initial_state['lead_id'],
}
self.assertEqual(actual, expected)
def test_normalize_field_doby_errors(self):
with self.assertRaisesRegex(ValueError, "Invalid format for doby: '19999'"):
Normalize.normalize_field('doby', '19999')
with self.assertRaisesRegex(ValueError, "Invalid format for doby: '1'"):
Normalize.normalize_field('doby', '1')
with self.assertRaisesRegex(ValueError, "Invalid format for doby: '-1'"):
Normalize.normalize_field('doby', '-1')
def test_normalize_field_fi(self):
self.assertEqual(Normalize.normalize_field('fi', 'ABC'),
Normalize.hash_sha_256('a'))
self.assertEqual(Normalize.normalize_field('fi', 'A'),
Normalize.hash_sha_256('a'))
self.assertEqual(Normalize.normalize_field('fi', ''), None)
self.assertEqual(Normalize.normalize_field('fi', None), None)
def test_normalize_field_f5first(self):
self.assertEqual(Normalize.normalize_field('f5first', 'George'),
Normalize.hash_sha_256('georg'))
self.assertEqual(Normalize.normalize_field('f5first', 'John'),
Normalize.hash_sha_256('john'))
self.assertEqual(Normalize.normalize_field('f5first', ''), None)
self.assertEqual(Normalize.normalize_field('f5first', None), None)
def test_normalize_field_f5last(self):
self.assertEqual(Normalize.normalize_field('f5last', 'Washington'),
Normalize.hash_sha_256('washi'))
self.assertEqual(Normalize.normalize_field('f5last', 'Adams'),
Normalize.hash_sha_256('adams'))
self.assertEqual(Normalize.normalize_field('f5last', ''), None)
self.assertEqual(Normalize.normalize_field('f5last', None), None)
def normalize(self):
normalized_payload = {'em': self.__normalize_list('em', self.emails),
'ph': self.__normalize_list('ph', self.phones),
'db': self.__normalize_list('db', self.dates_of_birth),
'ln': self.__normalize_list('ln', self.last_names),
'fn': self.__normalize_list('fn', self.first_names),
'ct': self.__normalize_list('ct', self.cities),
'st': self.__normalize_list('st', self.states),
'zp': self.__normalize_list('zp', self.zip_codes),
'country': self.__normalize_list('country', self.country_codes),
'external_id': self.__dedup_list(self.external_ids),
'client_ip_address': self.client_ip_address,
'client_user_agent': self.client_user_agent,
'fbc': self.fbc,
'fbp': self.fbp,
'subscription_id': self.subscription_id,
'fb_login_id': self.fb_login_id,
'lead_id': self.lead_id,
'f5first': Normalize.normalize_field('f5first', self.f5first),
'f5last': Normalize.normalize_field('f5last', self.f5last),
'fi': Normalize.normalize_field('fi', self.fi),
'dobd': Normalize.normalize_field('dobd', self.dobd),
'dobm': Normalize.normalize_field('dobm', self.dobm),
'doby': Normalize.normalize_field('doby', self.doby),
}
if self.genders:
normalized_payload['ge'] = self.__normalize_list('ge', list(map(lambda g: g.value, self.genders)))
normalized_payload = {k: v for k, v in normalized_payload.items() if v is not None}
return normalized_payload
def test_normalize_validates_email_then_hashes(self):
emails = [
'[email protected]',
'*****@*****.**',
'*****@*****.**',
'"a b"@c.d',
'a\@[email protected]',
"f`[email protected]",
"fo'*****@*****.**",
"f\'o\'*****@*****.**",
]
for email in emails:
self.assertEqual(len(Normalize.normalize_field('em', email)), 64)
def test_normalize_dobd_errors(self):
with self.assertRaisesRegex(ValueError,
"Invalid format for dobd: '32'"):
Normalize.normalize_field('dobd', '32')
with self.assertRaisesRegex(ValueError,
"Invalid format for dobd: '0'"):
Normalize.normalize_field('dobd', '0')
with self.assertRaisesRegex(ValueError,
"Invalid format for dobd: '-1'"):
Normalize.normalize_field('dobd', '-1')
def test_normalize_it_normalizes_and_hashes(self):
initial_state = {
'emails': ['*****@*****.**', '*****@*****.**', '*****@*****.**'],
'phones': ['12345678912', '12062072008', '12062072008'],
'genders': [Gender.MALE, Gender.FEMALE, Gender.FEMALE],
'dates_of_birth': ['19900101', '19660202', '19660202'],
'last_names': ['smith', 'brown', 'brown'],
'first_names': ['joe', 'mary', 'mary'],
'cities': ['seattle', 'sanfrancisco', 'sanfrancisco'],
'states': ['ca', 'wa', 'wa'],
'country_codes': ['us', 'ca', 'ca'],
'zip_codes': ['98001', '12345', '12345'],
'external_ids': ['123', '456', '456'],
'f5first': 'First',
'f5last': 'Last',
'fi': 'A',
'dobd': '01',
'dobm': '02',
'doby': '2000',
'lead_id': 'lead-id-3',
}
user_data = UserData(
f5first=initial_state['f5first'],
f5last=initial_state['f5last'],
fi=initial_state['fi'],
dobd=initial_state['dobd'],
dobm=initial_state['dobm'],
doby=initial_state['doby'],
lead_id=initial_state['lead_id'],
)
user_data.emails = initial_state['emails']
user_data.phones = initial_state['phones']
user_data.genders = initial_state['genders']
user_data.dates_of_birth = initial_state['dates_of_birth']
user_data.last_names = initial_state['last_names']
user_data.first_names = initial_state['first_names']
user_data.cities = initial_state['cities']
user_data.states = initial_state['states']
user_data.country_codes = initial_state['country_codes']
user_data.zip_codes = initial_state['zip_codes']
user_data.external_ids = initial_state['external_ids']
actual = user_data.normalize()
expected = {
'em':
self.__hash_list(initial_state['emails'][0:2]),
'ph':
self.__hash_list(initial_state['phones'][0:2]),
'ge':
self.__hash_list(
list(map(lambda g: g.value, initial_state['genders'][0:2]))),
'db':
self.__hash_list(initial_state['dates_of_birth'][0:2]),
'ln':
self.__hash_list(initial_state['last_names'][0:2]),
'fn':
self.__hash_list(initial_state['first_names'][0:2]),
'ct':
self.__hash_list(initial_state['cities'][0:2]),
'st':
self.__hash_list(initial_state['states'][0:2]),
'zp':
self.__hash_list(initial_state['zip_codes'][0:2]),
'country':
self.__hash_list(initial_state['country_codes'][0:2]),
'external_id':
initial_state['external_ids'][0:2],
'f5first':
Normalize.hash_sha_256(initial_state['f5first'].lower()),
'f5last':
Normalize.hash_sha_256(initial_state['f5last'].lower()),
'fi':
Normalize.hash_sha_256(initial_state['fi'].lower()),
'dobd':
Normalize.hash_sha_256(initial_state['dobd']),
'dobm':
Normalize.hash_sha_256(initial_state['dobm']),
'doby':
Normalize.hash_sha_256(initial_state['doby']),
'lead_id':
initial_state['lead_id'],
}
not_equal_items = {
k: actual[k]
for k in actual
if k not in expected or sorted(actual[k]) != sorted(expected[k])
}
self.assertEqual(len(not_equal_items), 0)
self.assertEqual(len(actual), len(expected))
def test_normalize_f5first(self):
self.assertEqual(Normalize.normalize_field('f5first', 'George'),
'georg')
self.assertEqual(Normalize.normalize_field('f5first', 'John'), 'john')
self.assertEqual(Normalize.normalize_field('f5first', ''), None)
self.assertEqual(Normalize.normalize_field('f5first', None), None)
def test_normalize_field_skip_hashing_it_does_not_hash(self):
value = ' USD '
self.assertEqual(
Normalize.normalize_field_skip_hashing('currency', value), 'usd')
def test_hash_sha_256_hashes(self):
value = '2000'
expected_hashed_value = hashlib.sha256(
value.encode('utf-8')).hexdigest()
self.assertEqual(Normalize.hash_sha_256(value), expected_hashed_value)
def test_normalize_field_it_skips_hashing_when_already_hashed_after_generic_normalizing(
self):
value = " %s " % ('a' * 64)
self.assertEqual(Normalize.normalize_field('f5last', value), 'a' * 64)
def test_normalize_field_it_skips_hashing_when_already_hashed(self):
value = 'a' * 64
self.assertEqual(Normalize.normalize_field('f5last', value), value)
def test_normalize_fi(self):
self.assertEqual(Normalize.normalize_field('fi', 'ABC'), 'a')
self.assertEqual(Normalize.normalize_field('fi', 'A'), 'a')
self.assertEqual(Normalize.normalize_field('fi', ''), None)
self.assertEqual(Normalize.normalize_field('fi', None), None)
def __hash_list(self, value_list):
return list(map(lambda val: Normalize.hash_sha_256(val), value_list))
def test_normalize_doby(self):
self.assertEqual(Normalize.normalize_field('doby', '2000'), '2000')
self.assertEqual(Normalize.normalize_field('doby', '0000'), '0000')
self.assertEqual(Normalize.normalize_field('doby', '9999'), '9999')
self.assertEqual(Normalize.normalize_field('doby', ''), None)
self.assertEqual(Normalize.normalize_field('doby', None), None)