def test_pipe_record_fpe(record_and_meta_2): xf_fpe = FpeFloatConfig(secret='2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94', radix=10, labels=['latitude']) data_paths = [DataPath(input='latitude', xforms=xf_fpe), DataPath(input='*')] xf = DataTransformPipeline(data_paths) rf = DataRestorePipeline(data_paths) check = xf.transform_record(record_and_meta_2) assert check['record'] == {'summary': 'Alex Watson <*****@*****.**> works at Gretel. Alexander Ehrath ' 'used to work at Qualcomm.', 'dni': 'He loves 8.8.8.8 for DNS', 'city': 'San Diego', 'state': 'California', 'stuff': 'nothing labeled here', 'latitude': 124.10051071657566} check = rf.transform_record(check) assert check['record'] == {'summary': 'Alex Watson <*****@*****.**> works at Gretel. Alexander Ehrath ' 'used to work at Qualcomm.', 'dni': 'He loves 8.8.8.8 for DNS', 'city': 'San Diego', 'state': 'California', 'stuff': 'nothing labeled here', 'latitude': 112.221} check = xf.transform_record(check) assert check['record'] == {'summary': 'Alex Watson <*****@*****.**> works at Gretel. Alexander Ehrath ' 'used to work at Qualcomm.', 'dni': 'He loves 8.8.8.8 for DNS', 'city': 'San Diego', 'state': 'California', 'stuff': 'nothing labeled here', 'latitude': 124.10051071657566}
def test_record_fpe_base62(): rec = {'latitude': -70.783, 'longitude': -112.221, 'credit_card': '4123567891234567', 'the_dude': 100000000, 'the_hotness': "This is some awesome text with UPPER and lower case characters.", "the_sci_notation": 1.23E-7} numbers_xf = [FpeStringConfig( secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10)] float_xf = [FpeFloatConfig( secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10, float_precision=3)] cc_xf = [FormatConfig(pattern=r'\s+', replacement=''), FpeStringConfig( secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10)] text_xf = [ FpeStringConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=62)] data_paths = [DataPath(input='credit_card', xforms=cc_xf), DataPath(input='longitude', xforms=float_xf), DataPath(input='latitude', xforms=float_xf), DataPath(input='the_dude', xforms=numbers_xf), DataPath(input='the_sci_notation', xforms=float_xf), DataPath(input='the_hotness', xforms=text_xf) ] xf = DataTransformPipeline(data_paths) rf = DataRestorePipeline(data_paths) xf_payload = xf.transform_record(rec) check = xf_payload.get('credit_card') assert check == '5931468769662449' check = rf.transform_record(xf_payload) assert check == rec
def test_record_fpe_precision(): rec = {'latitude': -70.783, 'longitude': -112.221, 'credit_card': '4123567891234567', 'the_dude': 100000000, 'the_hotness': "convertme", "the_sci_notation": 1.23E-7} int_xf = FpeStringConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10) num1_xf = FpeFloatConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10, float_precision=1) num2_xf = FpeFloatConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10, float_precision=0) num3_xf = FpeFloatConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10, float_precision=1) num4_xf = FpeStringConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=36) data_paths = [ DataPath(input='credit_card', xforms=int_xf), DataPath(input='latitude', xforms=num1_xf), DataPath(input='the_dude', xforms=int_xf), DataPath(input='longitude', xforms=num2_xf), DataPath(input='the_sci_notation', xforms=num3_xf), DataPath(input='the_hotness', xforms=num4_xf), DataPath(input='*') ] xf = DataTransformPipeline(data_paths) rf = DataRestorePipeline(data_paths) xf_payload = xf.transform_record(rec) check = xf_payload.get('credit_card') assert check == '5931468769662449' check = xf_payload.get('longitude') assert check == -112.2929577756414 check = xf_payload.get('latitude') assert check == -70.78143312456855 check = xf_payload.get('the_hotness') assert check == '2qjuxg7ju' check = xf_payload.get('the_dude') assert check == 128994144 check = xf_payload.get('the_sci_notation') assert check == 1.2342967235924508e-07 check = rf.transform_record(xf_payload) assert check == rec
def test_record_output_map_and_schemas(): rec = {'a': 1.23, 'b': 2.34, 'c': 3.45, 'd': 4.56, 'e': 5.67} rec2 = {'f': 1.23, 'b': 2.34, 'c': 3.45, 'd': 4.56, 'e': 5.67} test_payloads = [(rec, record_key) for record_key in RECORD_KEYS] test_payloads.append((rec, None)) for payload, record_key in test_payloads: xf_list = FpeFloatConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10) data_paths = [ DataPath(input='a', output='x'), DataPath(input='b', output='y'), DataPath(input='c', xforms=xf_list, output='z'), DataPath(input='d', xforms=xf_list), DataPath(input='e', xforms=xf_list), DataPath(input='*') ] xf = DataTransformPipeline(data_paths) rf = DataRestorePipeline(data_paths) xf_payload = xf.transform_record(payload) xf_record = xf_payload.get(record_key) or xf_payload check = xf_record.get('x') assert check == 1.23 check = xf_record.get('y') assert check == 2.34 check = xf_record.get('z') assert check == 3.590038584114511 check = xf_record.get('d') assert check == 7.002521213914073 check = xf_record.get('e') assert check == 4.9570355284951875 check = rf.transform_record(xf_payload) check = check.get(record_key) or check assert check == rec # test multiple names mapping to the same output field xf_list = FpeStringConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10) data_paths = [ DataPath(input='a', xforms=xf_list, output='x'), DataPath(input='f', xforms=xf_list, output='x'), DataPath(input='b', xforms=xf_list, output='y'), DataPath(input='c', xforms=xf_list, output='z'), DataPath(input='*') ] xf = DataTransformPipeline(data_paths) xf_payload = xf.transform_record(rec) xf_payload2 = xf.transform_record(rec2) xf_record = xf_payload.get(record_key) or xf_payload xf_record2 = xf_payload2.get(record_key) or xf_payload2 assert xf_record == xf_record2
def test_record_zero_fpe(): rec = {'latitude': 0.0, 'longitude': -0.0, 'credit_card': '4123567891234567', 'the_dude': 100000000, 'the_hotness': "convertme", "the_sci_notation": 1.23E-7} numbers_xf = [FpeStringConfig( secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10)] float_xf = [FpeFloatConfig( secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10, float_precision=3)] text_xf = [ FpeStringConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=36)] data_paths = [ DataPath(input='credit_card', xforms=numbers_xf), DataPath(input='latitude', xforms=float_xf), DataPath(input='longitude', xforms=float_xf), DataPath(input='the_dude', xforms=numbers_xf), DataPath(input='the_sci_notation', xforms=float_xf), DataPath(input='the_hotness', xforms=text_xf) ] xf = DataTransformPipeline(data_paths) rf = DataRestorePipeline(data_paths) xf_payload = xf.transform_record(rec) check = xf_payload.get('credit_card') assert check == '5931468769662449' check = xf_payload.get('longitude') assert check == -1.32547939979e-312 check = xf_payload.get('latitude') assert check == 1.32547939979e-312 check = xf_payload.get('the_hotness') assert check == '2qjuxg7ju' check = xf_payload.get('the_dude') assert check == 128994144 check = xf_payload.get('the_sci_notation') assert check == 1.229570610794763e-07 check = rf.transform_record(xf_payload) assert check == rec
def test_conditional_transformer(records_conditional): xf_fpe = FpeFloatConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10) xf_consent = ConditionalConfig(conditional_value=FieldRef('user_consent'), regex=r"['1']", true_xform=xf_fpe, false_xform=RedactWithLabelConfig()) data_paths_encrypt = [DataPath(input='lon', xforms=xf_fpe), DataPath(input='lat', xforms=xf_fpe), DataPath(input='*') ] data_paths_decrypt = [DataPath(input='lon', xforms=xf_consent), DataPath(input='lat', xforms=xf_consent), DataPath(input='*') ] xf_encrypt = DataTransformPipeline(data_paths_encrypt) xf_decrypt = DataRestorePipeline(data_paths_decrypt) check_aw = xf_encrypt.transform_record(records_conditional[0]) check_ae = xf_encrypt.transform_record(records_conditional[1]) assert check_ae['record']['lat'] == 50.65564864394322 assert check_ae['record']['lon'] == 191.8142181740291 assert check_aw['record']['lat'] == 77.00217823076872 assert check_aw['record']['lon'] == 254.0404040486477 check_aw = xf_decrypt.transform_record(check_aw) check_ae = xf_decrypt.transform_record(check_ae) assert check_ae['record']['lat'] == 'LATITUDE' assert check_ae['record']['lon'] == 'LONGITUDE' assert check_aw['record']['lat'] == 112.22134 assert check_aw['record']['lon'] == 135.76433 xf_fpe = FpeFloatConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10) xf_consent = ConditionalConfig(conditional_value=FieldRef('user_consent'), regex=r"['1']", true_xform=xf_fpe) data_paths_encrypt = [DataPath(input='lon', xforms=xf_fpe), DataPath(input='lat', xforms=xf_fpe), DataPath(input='*') ] data_paths_decrypt = [DataPath(input='lon', xforms=xf_consent), DataPath(input='lat', xforms=xf_consent), DataPath(input='*') ] xf_encrypt = DataTransformPipeline(data_paths_encrypt) xf_decrypt = DataRestorePipeline(data_paths_decrypt) check_aw = xf_encrypt.transform_record(records_conditional[0]) check_ae = xf_encrypt.transform_record(records_conditional[1]) assert check_ae['record']['lat'] == 50.65564864394322 assert check_ae['record']['lon'] == 191.8142181740291 assert check_aw['record']['lat'] == 77.00217823076872 assert check_aw['record']['lon'] == 254.0404040486477 check_aw = xf_decrypt.transform_record(check_aw) check_ae = xf_decrypt.transform_record(check_ae) assert check_ae['record']['lat'] == 50.65564864394322 assert check_ae['record']['lon'] == 191.8142181740291 assert check_aw['record']['lat'] == 112.22134 assert check_aw['record']['lon'] == 135.76433 xf_fpe = FpeFloatConfig(secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10) xf_consent = ConditionalConfig(conditional_value=FieldRef('user_consent'), regex=r"['1']", false_xform=xf_fpe) data_paths_decrypt = [DataPath(input='lon', xforms=xf_consent), DataPath(input='lat', xforms=xf_consent), DataPath(input='*') ] xf_decrypt = DataRestorePipeline(data_paths_decrypt) check_aw = xf_decrypt.transform_record(check_aw) check_ae = xf_decrypt.transform_record(check_ae) assert check_ae['record']['lat'] == 35.659491 assert check_ae['record']['lon'] == 139.72785 assert check_aw['record']['lat'] == 112.22134 assert check_aw['record']['lon'] == 135.76433
"user_consent": "0", "user_id": "0063", }, { "address": "123 Operation Blvd, San Diego, CA 92121", "lat": 32.880088, "lon": -117.198943, "name": "Beavis Smith", "user_consent": "1", "user_id": "0078", }, ] xf_fpe = FpeFloatConfig( secret="2B7E151628AED2A6ABF7158809CF4F3CEF4359D8D580AA4F7F036D6F04FC6A94", radix=10, float_precision=2, ) xf_consent = ConditionalConfig( conditional_value=FieldRef("user_consent"), regex=r"['1']", true_xform=xf_fpe, false_xform=RedactWithCharConfig(char="0"), ) data_paths_encrypt = [ DataPath(input="lon", xforms=xf_fpe), DataPath(input="lat", xforms=xf_fpe), DataPath(input="*"), ]