def test_pipe_record_fpe(record_and_meta_2):
xf_fpe = FpeFloatConfig(secret='2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94', radix=10,
labels=['latitude'])
data_paths = [DataPath(input='latitude', xforms=xf_fpe),
DataPath(input='*')]
xf = DataTransformPipeline(data_paths)
rf = DataRestorePipeline(data_paths)
check = xf.transform_record(record_and_meta_2)
assert check['record'] == {'summary': 'Alex Watson <*****@*****.**> works at Gretel. Alexander Ehrath '
'used to work at Qualcomm.',
'dni': 'He loves 8.8.8.8 for DNS',
'city': 'San Diego',
'state': 'California', 'stuff': 'nothing labeled here',
'latitude': 124.10051071657566}
check = rf.transform_record(check)
assert check['record'] == {'summary': 'Alex Watson <*****@*****.**> works at Gretel. Alexander Ehrath '
'used to work at Qualcomm.',
'dni': 'He loves 8.8.8.8 for DNS',
'city': 'San Diego',
'state': 'California', 'stuff': 'nothing labeled here',
'latitude': 112.221}
check = xf.transform_record(check)
assert check['record'] == {'summary': 'Alex Watson <*****@*****.**> works at Gretel. Alexander Ehrath '
'used to work at Qualcomm.',
'dni': 'He loves 8.8.8.8 for DNS',
'city': 'San Diego',
'state': 'California', 'stuff': 'nothing labeled here',
'latitude': 124.10051071657566}
def test_record_fpe_base62():
rec = {'latitude': -70.783, 'longitude': -112.221, 'credit_card': '4123567891234567', 'the_dude': 100000000,
'the_hotness': "This is some awesome text with UPPER and lower case characters.",
"the_sci_notation": 1.23E-7}
numbers_xf = [FpeStringConfig(
secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10)]
float_xf = [FpeFloatConfig(
secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10,
float_precision=3)]
cc_xf = [FormatConfig(pattern=r'\s+', replacement=''),
FpeStringConfig(
secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10)]
text_xf = [
FpeStringConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=62)]
data_paths = [DataPath(input='credit_card', xforms=cc_xf),
DataPath(input='longitude', xforms=float_xf),
DataPath(input='latitude', xforms=float_xf),
DataPath(input='the_dude', xforms=numbers_xf),
DataPath(input='the_sci_notation', xforms=float_xf),
DataPath(input='the_hotness', xforms=text_xf)
]
xf = DataTransformPipeline(data_paths)
rf = DataRestorePipeline(data_paths)
xf_payload = xf.transform_record(rec)
check = xf_payload.get('credit_card')
assert check == '5931468769662449'
check = rf.transform_record(xf_payload)
assert check == rec
def test_record_fpe_precision():
rec = {'latitude': -70.783, 'longitude': -112.221, 'credit_card': '4123567891234567', 'the_dude': 100000000,
'the_hotness': "convertme", "the_sci_notation": 1.23E-7}
int_xf = FpeStringConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10)
num1_xf = FpeFloatConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10,
float_precision=1)
num2_xf = FpeFloatConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10,
float_precision=0)
num3_xf = FpeFloatConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10,
float_precision=1)
num4_xf = FpeStringConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=36)
data_paths = [
DataPath(input='credit_card', xforms=int_xf),
DataPath(input='latitude', xforms=num1_xf),
DataPath(input='the_dude', xforms=int_xf),
DataPath(input='longitude', xforms=num2_xf),
DataPath(input='the_sci_notation', xforms=num3_xf),
DataPath(input='the_hotness', xforms=num4_xf),
DataPath(input='*')
]
xf = DataTransformPipeline(data_paths)
rf = DataRestorePipeline(data_paths)
xf_payload = xf.transform_record(rec)
check = xf_payload.get('credit_card')
assert check == '5931468769662449'
check = xf_payload.get('longitude')
assert check == -112.2929577756414
check = xf_payload.get('latitude')
assert check == -70.78143312456855
check = xf_payload.get('the_hotness')
assert check == '2qjuxg7ju'
check = xf_payload.get('the_dude')
assert check == 128994144
check = xf_payload.get('the_sci_notation')
assert check == 1.2342967235924508e-07
check = rf.transform_record(xf_payload)
assert check == rec
def test_record_output_map_and_schemas():
rec = {'a': 1.23, 'b': 2.34, 'c': 3.45, 'd': 4.56, 'e': 5.67}
rec2 = {'f': 1.23, 'b': 2.34, 'c': 3.45, 'd': 4.56, 'e': 5.67}
test_payloads = [(rec, record_key) for record_key in RECORD_KEYS]
test_payloads.append((rec, None))
for payload, record_key in test_payloads:
xf_list = FpeFloatConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10)
data_paths = [
DataPath(input='a', output='x'),
DataPath(input='b', output='y'),
DataPath(input='c', xforms=xf_list, output='z'),
DataPath(input='d', xforms=xf_list),
DataPath(input='e', xforms=xf_list),
DataPath(input='*')
]
xf = DataTransformPipeline(data_paths)
rf = DataRestorePipeline(data_paths)
xf_payload = xf.transform_record(payload)
xf_record = xf_payload.get(record_key) or xf_payload
check = xf_record.get('x')
assert check == 1.23
check = xf_record.get('y')
assert check == 2.34
check = xf_record.get('z')
assert check == 3.590038584114511
check = xf_record.get('d')
assert check == 7.002521213914073
check = xf_record.get('e')
assert check == 4.9570355284951875
check = rf.transform_record(xf_payload)
check = check.get(record_key) or check
assert check == rec
# test multiple names mapping to the same output field
xf_list = FpeStringConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10)
data_paths = [
DataPath(input='a', xforms=xf_list, output='x'),
DataPath(input='f', xforms=xf_list, output='x'),
DataPath(input='b', xforms=xf_list, output='y'),
DataPath(input='c', xforms=xf_list, output='z'),
DataPath(input='*')
]
xf = DataTransformPipeline(data_paths)
xf_payload = xf.transform_record(rec)
xf_payload2 = xf.transform_record(rec2)
xf_record = xf_payload.get(record_key) or xf_payload
xf_record2 = xf_payload2.get(record_key) or xf_payload2
assert xf_record == xf_record2
def test_record_zero_fpe():
rec = {'latitude': 0.0, 'longitude': -0.0, 'credit_card': '4123567891234567', 'the_dude': 100000000,
'the_hotness': "convertme", "the_sci_notation": 1.23E-7}
numbers_xf = [FpeStringConfig(
secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10)]
float_xf = [FpeFloatConfig(
secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10,
float_precision=3)]
text_xf = [
FpeStringConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=36)]
data_paths = [
DataPath(input='credit_card', xforms=numbers_xf),
DataPath(input='latitude', xforms=float_xf),
DataPath(input='longitude', xforms=float_xf),
DataPath(input='the_dude', xforms=numbers_xf),
DataPath(input='the_sci_notation', xforms=float_xf),
DataPath(input='the_hotness', xforms=text_xf)
]
xf = DataTransformPipeline(data_paths)
rf = DataRestorePipeline(data_paths)
xf_payload = xf.transform_record(rec)
check = xf_payload.get('credit_card')
assert check == '5931468769662449'
check = xf_payload.get('longitude')
assert check == -1.32547939979e-312
check = xf_payload.get('latitude')
assert check == 1.32547939979e-312
check = xf_payload.get('the_hotness')
assert check == '2qjuxg7ju'
check = xf_payload.get('the_dude')
assert check == 128994144
check = xf_payload.get('the_sci_notation')
assert check == 1.229570610794763e-07
check = rf.transform_record(xf_payload)
assert check == rec
def test_conditional_transformer(records_conditional):
xf_fpe = FpeFloatConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10)
xf_consent = ConditionalConfig(conditional_value=FieldRef('user_consent'), regex=r"['1']",
true_xform=xf_fpe,
false_xform=RedactWithLabelConfig())
data_paths_encrypt = [DataPath(input='lon', xforms=xf_fpe),
DataPath(input='lat', xforms=xf_fpe),
DataPath(input='*')
]
data_paths_decrypt = [DataPath(input='lon', xforms=xf_consent),
DataPath(input='lat', xforms=xf_consent),
DataPath(input='*')
]
xf_encrypt = DataTransformPipeline(data_paths_encrypt)
xf_decrypt = DataRestorePipeline(data_paths_decrypt)
check_aw = xf_encrypt.transform_record(records_conditional[0])
check_ae = xf_encrypt.transform_record(records_conditional[1])
assert check_ae['record']['lat'] == 50.65564864394322
assert check_ae['record']['lon'] == 191.8142181740291
assert check_aw['record']['lat'] == 77.00217823076872
assert check_aw['record']['lon'] == 254.0404040486477
check_aw = xf_decrypt.transform_record(check_aw)
check_ae = xf_decrypt.transform_record(check_ae)
assert check_ae['record']['lat'] == 'LATITUDE'
assert check_ae['record']['lon'] == 'LONGITUDE'
assert check_aw['record']['lat'] == 112.22134
assert check_aw['record']['lon'] == 135.76433
xf_fpe = FpeFloatConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10)
xf_consent = ConditionalConfig(conditional_value=FieldRef('user_consent'), regex=r"['1']",
true_xform=xf_fpe)
data_paths_encrypt = [DataPath(input='lon', xforms=xf_fpe),
DataPath(input='lat', xforms=xf_fpe),
DataPath(input='*')
]
data_paths_decrypt = [DataPath(input='lon', xforms=xf_consent),
DataPath(input='lat', xforms=xf_consent),
DataPath(input='*')
]
xf_encrypt = DataTransformPipeline(data_paths_encrypt)
xf_decrypt = DataRestorePipeline(data_paths_decrypt)
check_aw = xf_encrypt.transform_record(records_conditional[0])
check_ae = xf_encrypt.transform_record(records_conditional[1])
assert check_ae['record']['lat'] == 50.65564864394322
assert check_ae['record']['lon'] == 191.8142181740291
assert check_aw['record']['lat'] == 77.00217823076872
assert check_aw['record']['lon'] == 254.0404040486477
check_aw = xf_decrypt.transform_record(check_aw)
check_ae = xf_decrypt.transform_record(check_ae)
assert check_ae['record']['lat'] == 50.65564864394322
assert check_ae['record']['lon'] == 191.8142181740291
assert check_aw['record']['lat'] == 112.22134
assert check_aw['record']['lon'] == 135.76433
xf_fpe = FpeFloatConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10)
xf_consent = ConditionalConfig(conditional_value=FieldRef('user_consent'), regex=r"['1']",
false_xform=xf_fpe)
data_paths_decrypt = [DataPath(input='lon', xforms=xf_consent),
DataPath(input='lat', xforms=xf_consent),
DataPath(input='*')
]
xf_decrypt = DataRestorePipeline(data_paths_decrypt)
check_aw = xf_decrypt.transform_record(check_aw)
check_ae = xf_decrypt.transform_record(check_ae)
assert check_ae['record']['lat'] == 35.659491
assert check_ae['record']['lon'] == 139.72785
assert check_aw['record']['lat'] == 112.22134
assert check_aw['record']['lon'] == 135.76433
"user_consent": "0",
"user_id": "0063",
},
{
"address": "123 Operation Blvd, San Diego, CA 92121",
"lat": 32.880088,
"lon": -117.198943,
"name": "Beavis Smith",
"user_consent": "1",
"user_id": "0078",
},
]
xf_fpe = FpeFloatConfig(
secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94",
radix=10,
float_precision=2,
)
xf_consent = ConditionalConfig(
conditional_value=FieldRef("user_consent"),
regex=r"['1']",
true_xform=xf_fpe,
false_xform=RedactWithCharConfig(char="0"),
)
data_paths_encrypt = [
DataPath(input="lon", xforms=xf_fpe),
DataPath(input="lat", xforms=xf_fpe),
DataPath(input="*"),
]