def test_process_json(self): # Stress-testing combination of unpack methods with contrived table from hell fake_response = [ { 'id': 1, 'Simple List Col': ['one', 'two', 'three'], 'Mixed List Col': [None, 2, 'three'], 'Spotty List Col': [1, 2, 3], 'Multidim List Col': [[1, 2], [None, 'two'], []], 'Nested List Col': [ {'A': 1, 'B': 'one'}, {'A': 2, 'B': 'two'}, {'A': 3, 'B': 'three'}], 'Simple Dict Col': {'one': 1, 'two': 2, 'three': 3}, 'Nested Dict Col': {'A': 1, 'B': ['two', 2], 'C': [None, 3, 'three']} }, { 'id': 2, 'Simple List Col': ['four', 'five', 'six'], 'Mixed List Col': ['four', None, 6], 'Spotty List Col': [], 'Multidim List Col': [[3, None], [], ['three', 'four']], 'Nested List Col': [ {'A': 4, 'B': 'four'}, {'A': 5, 'B': 'five'}, {'A': 6, 'B': 'six'}], 'Simple Dict Col': {'one': 'I', 'two': 'II', 'three': 'III'}, 'Nested Dict Col': {'A': ['one'], 'B': [], 'C': 3}, }, { 'id': 3, 'Simple List Col': ['seven', 'eight', 'nine'], 'Mixed List Col': [7, 'eight', None], 'Spotty List Col': None, 'Multidim List Col': [['five', 6], [None]], 'Nested List Col': [ {'A': 7, 'B': 'seven'}, {'A': 8, 'B': 'eight'}, {'A': 9, 'B': 'nine'}], 'Simple Dict Col': {'one': 'x', 'two': 'xx', 'three': 'xxx'}, 'Nested Dict Col': {'A': None, 'B': 2, 'C': [None, 3, 'three']} } ] fake_response_tables = {} table_names = ['fake_Nested List Col', 'fake'] fake_response_tables['fake_Nested List Col'] = Table([ {"id": 1, "Nested List Col_A": 1, "Nested List Col_B": "one"}, {"id": 1, "Nested List Col_A": 2, "Nested List Col_B": "two"}, {"id": 1, "Nested List Col_A": 3, "Nested List Col_B": "three"}, {"id": 2, "Nested List Col_A": 4, "Nested List Col_B": "four"}, {"id": 2, "Nested List Col_A": 5, "Nested List Col_B": "five"}, {"id": 2, "Nested List Col_A": 6, "Nested List Col_B": "six"}, {"id": 3, "Nested List Col_A": 7, "Nested List Col_B": "seven"}, {"id": 3, "Nested List Col_A": 8, "Nested List Col_B": "eight"}, {"id": 3, "Nested List Col_A": 9, "Nested List Col_B": "nine"} ]) fake_response_tables['fake'] = Table([ {"id": 1, "Simple List Col": ["one", "two", "three"], "Mixed List Col": [None, 2, "three"], "Spotty List Col": [1, 2, 3], "Multidim List Col": [[1, 2], [None, "two"], []], "Simple Dict Col_one": 1, "Simple Dict Col_three": 3, "Simple Dict Col_two": 2, "Nested Dict Col_A": 1, "Nested Dict Col_B": ["two", 2], "Nested Dict Col_C": [None, 3, "three"]}, {"id": 2, "Simple List Col": ["four", "five", "six"], "Mixed List Col": ["four", None, 6], "Spotty List Col": [], "Multidim List Col": [[3, None], [], ["three", "four"]], "Simple Dict Col_one": "I", "Simple Dict Col_three": "III", "Simple Dict Col_two": "II", "Nested Dict Col_A": ["one"], "Nested Dict Col_B": [], "Nested Dict Col_C": 3}, {"id": 3, "Simple List Col": ["seven", "eight", "nine"], "Mixed List Col": [7, "eight", None], "Spotty List Col": [None], "Multidim List Col": [["five", 6], [None]], "Simple Dict Col_one": "x", "Simple Dict Col_three": "xxx", "Simple Dict Col_two": "xx", "Nested Dict Col_A": None, "Nested Dict Col_B": 2, "Nested Dict Col_C": [None, 3, "three"]} ]) fake_processed = self.cp.process_json(fake_response, 'fake') self.assertTrue([f['name'] for f in fake_processed] == table_names) for tbl in table_names: assert_matching_tables( [f['tbl'] for f in fake_processed if f['name'] == tbl][0], fake_response_tables[tbl] ) fake_tidy = self.cp.process_json(fake_response, 'fake', tidy=0) self.assertTrue(len(fake_tidy) == len(fake_response[0])-1)
def test_get_table(live_sftp, simple_table): # noqa F811 file_util.create_temp_file() tbl = live_sftp.get_table(CSV_PATH) assert_matching_tables(tbl, simple_table)
def test_paginate_request(self, m): # Anonymized real output with nested columns self.blob = [ { 'id': 78757050, 'name': 'Person One', 'prefix': None, 'first_name': 'Person', 'middle_name': None, 'last_name': 'One', 'suffix': None, 'address': { 'street': None, 'city': 'CityA', 'state': 'StateI', 'postal_code': '12345', 'country': None}, 'assignee_id': None, 'company_id': 12030795, 'company_name': 'Indivisible CityA', 'contact_type_id': 501950, 'details': None, 'emails': [{'email': '*****@*****.**', 'category': 'work'}], 'phone_numbers': [ {'number': '(541) 555-9585', 'category': 'work'}, {'number': '555-555-9585', 'category': 'work'}], 'socials': [{'url': 'https://gravatar.com/gravatar', 'category': 'gravatar'}], 'tags': [], 'title': None, 'websites': [{'url': 'http://www.IndivisibleCityA.org', 'category': None}], 'custom_fields': [ {'custom_field_definition_id': 125880, 'value': None}, {'custom_field_definition_id': 107297, 'value': None}, {'custom_field_definition_id': 102127, 'value': None}, {'custom_field_definition_id': 135034, 'value': None}, {'custom_field_definition_id': 107298, 'value': None}, {'custom_field_definition_id': 108972, 'value': None}, {'custom_field_definition_id': 125881, 'value': None}], 'date_created': 1558169903, 'date_modified': 1558169910, 'date_last_contacted': 1558169891, 'interaction_count': 1, 'leads_converted_from': [], 'date_lead_created': None }, { 'id': 78477076, 'name': 'Person Two', 'prefix': None, 'first_name': 'Person', 'middle_name': None, 'last_name': 'Two', 'suffix': None, 'address': { 'street': None, 'city': None, 'state': None, 'postal_code': None, 'country': None}, 'assignee_id': 289533, 'company_id': 12096071, 'company_name': 'Indivisible StateII', 'contact_type_id': 501950, 'details': None, 'emails': [{'email': '*****@*****.**', 'category': 'work'}], 'phone_numbers': [{'number': '(908) 555-2941', 'category': 'work'}], 'socials': [], 'tags': ['treasurer'], 'title': 'Treasurer', 'websites': [], 'custom_fields': [ {'custom_field_definition_id': 125880, 'value': None}, {'custom_field_definition_id': 107297, 'value': None}, {'custom_field_definition_id': 102127, 'value': None}, {'custom_field_definition_id': 135034, 'value': None}, {'custom_field_definition_id': 107298, 'value': None}, {'custom_field_definition_id': 108972, 'value': None}, {'custom_field_definition_id': 125881, 'value': None}], 'date_created': 1557761054, 'date_modified': 1558218799, 'date_last_contacted': 1558196341, 'interaction_count': 14, 'leads_converted_from': [], 'date_lead_created': None }, { 'id': 78839154, 'name': 'Person Three', 'prefix': None, 'first_name': 'Person', 'middle_name': None, 'last_name': 'Three', 'suffix': None, 'address': { 'street': None, 'city': 'CityC', 'state': 'StateIII', 'postal_code': '54321', 'country': None}, 'assignee_id': None, 'company_id': 34966944, 'company_name': 'Flip StateIII', 'contact_type_id': 501950, 'details': None, 'emails': [{'email': '*****@*****.**', 'category': 'work'}], 'phone_numbers': [{'number': '(619) 555-7883', 'category': 'work'}], 'socials': [ {'url': 'https://twitter.com/ThreePerson', 'category': 'twitter'}, {'url': 'https://www.facebook.com/Person.n.Three', 'category': 'facebook'}, {'url': 'https://gravatar.com/PersonThree', 'category': 'gravatar'}], 'tags': [], 'title': None, 'websites': [], 'custom_fields': [ {'custom_field_definition_id': 125880, 'value': None}, {'custom_field_definition_id': 107297, 'value': None}, {'custom_field_definition_id': 102127, 'value': None}, {'custom_field_definition_id': 135034, 'value': None}, {'custom_field_definition_id': 107298, 'value': None}, {'custom_field_definition_id': 108972, 'value': None}, {'custom_field_definition_id': 125881, 'value': None}], 'date_created': 1558223367, 'date_modified': 1558223494, 'date_last_contacted': 1558223356, 'interaction_count': 2, 'leads_converted_from': [], 'date_lead_created': None } ] # Mock endpoints m.post( self.cp.uri + '/people/search', json=self.paginate_callback, headers={"filename": "people_search.txt"}) # self.assertTrue( assert_matching_tables( Table(self.blob), Table( self.cp.paginate_request( '/people/search', page_size=1, req_type='POST' ) ) )
def test_materialize_to_file(self): # Simple test that materializing doesn't change the table tbl_materialized = Table(self.lst_dicts) _ = tbl_materialized.materialize_to_file() assert_matching_tables(self.tbl, tbl_materialized)
def assert_file_matches_table(local_path, table): downloaded_tbl = Table.from_csv(local_path) assert_matching_tables(table, downloaded_tbl)
def test_to_from_temp_json_compressed(self): path = self.tbl.to_json(temp_file_compression='gzip') result_tbl = Table.from_json(path) assert_matching_tables(self.tbl, result_tbl)
def test_match_columns(self): raw = [ { 'first name': 'Mary', 'LASTNAME': 'Nichols', 'Middle__Name': 'D' }, { 'first name': 'Lucy', 'LASTNAME': 'Peterson', 'Middle__Name': 'S' }, ] tbl = Table(raw) desired_raw = [ { 'first_name': 'Mary', 'middle_name': 'D', 'last_name': 'Nichols' }, { 'first_name': 'Lucy', 'middle_name': 'S', 'last_name': 'Peterson' }, ] desired_tbl = Table(desired_raw) # Test with fuzzy matching tbl.match_columns(desired_tbl.columns) assert_matching_tables(desired_tbl, tbl) # Test disable fuzzy matching, and fail due due to the missing cols self.assertRaises(TypeError, Table(raw).match_columns, desired_tbl.columns, fuzzy_match=False, if_missing_columns='fail') # Test disable fuzzy matching, and fail due to the extra cols self.assertRaises(TypeError, Table(raw).match_columns, desired_tbl.columns, fuzzy_match=False, if_extra_columns='fail') # Test table that already has the right columns, shouldn't need fuzzy match tbl = Table(desired_raw) tbl.match_columns(desired_tbl.columns, fuzzy_match=False, if_missing_columns='fail', if_extra_columns='fail') assert_matching_tables(desired_tbl, tbl) # Test table with missing col, verify the missing col gets added by default tbl = Table([ { 'first name': 'Mary', 'LASTNAME': 'Nichols' }, { 'first name': 'Lucy', 'LASTNAME': 'Peterson' }, ]) tbl.match_columns(desired_tbl.columns) desired_tbl = ( Table(desired_raw).remove_column('middle_name').add_column( 'middle_name', index=1)) assert_matching_tables(desired_tbl, tbl) # Test table with extra col, verify the extra col gets removed by default tbl = Table([ { 'first name': 'Mary', 'LASTNAME': 'Nichols', 'Age': 32, 'Middle__Name': 'D' }, { 'first name': 'Lucy', 'LASTNAME': 'Peterson', 'Age': 26, 'Middle__Name': 'S' }, ]) desired_tbl = Table(desired_raw) tbl.match_columns(desired_tbl.columns) assert_matching_tables(desired_tbl, tbl) # Test table with two columns that normalize the same and aren't in desired cols, verify # they both get removed. tbl = Table([ { 'first name': 'Mary', 'LASTNAME': 'Nichols', 'Age': 32, 'Middle__Name': 'D', 'AGE': None }, { 'first name': 'Lucy', 'LASTNAME': 'Peterson', 'Age': 26, 'Middle__Name': 'S', 'AGE': None }, ]) tbl.match_columns(desired_tbl.columns) assert_matching_tables(desired_tbl, tbl) # Test table with two columns that match desired cols, verify only the first gets kept. tbl = Table([ { 'first name': 'Mary', 'LASTNAME': 'Nichols', 'First Name': None, 'Middle__Name': 'D' }, { 'first name': 'Lucy', 'LASTNAME': 'Peterson', 'First Name': None, 'Middle__Name': 'S' }, ]) tbl.match_columns(desired_tbl.columns) assert_matching_tables(desired_tbl, tbl)
def _assert_expected_csv(self, path, orig_tbl): result_tbl = Table.from_csv(path) assert_matching_tables(orig_tbl, result_tbl)
def test_to_from_temp_json(self): path = self.tbl.to_json() result_tbl = Table.from_json(path) assert_matching_tables(self.tbl, result_tbl)
def test_get_posts(self, m): m.get(self.ct.uri + '/posts', json=expected_posts) posts = self.ct.get_posts() exp_tbl = self.ct.unpack(Table(expected_posts['result']['posts'])) assert_matching_tables(posts, exp_tbl)
def test_get_links(self, m): m.get(self.ct.uri + '/links', json=expected_post) post = self.ct.get_links(link='https://nbcnews.to/34stfC2') exp_tbl = self.ct.unpack(Table(expected_post['result']['posts'])) assert_matching_tables(post, exp_tbl)
def test_distribute_task(self): global count global tableargs datatable = [ ('x', 'y'), (1, 2), (3, 4), (5, 6), (7, 8), (9, 0), (11, 12), (13, 14), (15, 16), (17, 18), (19, 10), ] count = 0 tableargs = None distribute_task( Table(datatable), fake_table_process, 'foo', # bucket group_count=5, storage='local', func_kwargs={ 'x': 1, 'y': [2, 3] }) self.assertEqual(count, 2) assert_matching_tables( tableargs[0], Table([('x', 'y'), (11, 12), (13, 14), (15, 16), (17, 18), (19, 10)])) count = 0 tableargs = None distribute_task( Table(datatable + [(0, 0)]), FakeRunner.foobar, 'foo', # bucket group_count=5, storage='local', func_class=FakeRunner, func_class_kwargs={'init1': 'initx'}, func_kwargs={ 'a': 1, 'x': 2, 'y': 3 }) self.assertEqual(count, 3) self.assertEqual(tableargs[1:], ('initx', 1, 2, 3)) self.assertEqual(tableargs[1:], ('initx', 1, 2, 3)) assert_matching_tables(tableargs[0], Table([('x', 'y'), (0, 0)])) # 3. catch=True (with throwing) count = 0 tableargs = None distribute_task( Table(datatable[:6]), FakeRunner.foobar, 'foo', # bucket group_count=5, storage='local', func_class=FakeRunner, func_class_kwargs={'init1': 'initx'}, catch=True, func_kwargs={ 'a': 'raise', 'x': 2, 'y': 3 })
def test_get_custom_field(self, m): m.get(self.van.connection.uri + 'customFields/157', json=custom_field) assert_matching_tables(custom_field, self.van.get_custom_field(157))
def test_get_custom_field_values(self, m): m.get(self.van.connection.uri + 'customFields', json=custom_field) assert_matching_tables(custom_field_values, self.van.get_custom_fields_values())
def test_get_opportunities(self, m): processed_opps = Table([{ "id": 14340759, "name": "Company1", "assignee_id": 659394, "close_date": None, "company_id": 29324143, "company_name": "Company1", "customer_source_id": None, "details": None, "loss_reason_id": None, "pipeline_id": 489028, "pipeline_stage_id": 2529569, "primary_contact_id": 67747998, "priority": "High", "status": "Open", "tags": ["opportunities import-1540158946352"], "interaction_count": 0, "monetary_unit": "USD", "monetary_value": 100000.0, "converted_unit": None, "converted_value": None, "win_probability": None, "date_stage_changed": 1548866182, "date_last_contacted": None, "leads_converted_from": [], "date_lead_created": None, "date_created": 1540159060, "date_modified": 1550858334 }, { "id": 14161592, "name": "Company2", "assignee_id": 659394, "close_date": "11/10/2018", "company_id": 28729196, "company_name": "Company2", "customer_source_id": None, "details": None, "loss_reason_id": None, "pipeline_id": 531482, "pipeline_stage_id": 2607171, "primary_contact_id": 67243374, "priority": "High", "status": "Open", "tags": [], "interaction_count": 36, "monetary_unit": "USD", "monetary_value": 77000.0, "converted_unit": None, "converted_value": None, "win_probability": None, "date_stage_changed": 1551191957, "date_last_contacted": 1552339800, "leads_converted_from": [], "date_lead_created": None, "date_created": 1539192375, "date_modified": 1552340016 }, { "id": 14286548, "name": "Company3", "assignee_id": 644608, "close_date": "11/18/2018", "company_id": 29492294, "company_name": "Company3", "customer_source_id": None, "details": None, "loss_reason_id": None, "pipeline_id": 531482, "pipeline_stage_id": 2482007, "primary_contact_id": 67637400, "priority": "None", "status": "Open", "tags": [], "interaction_count": 19, "monetary_unit": "USD", "monetary_value": 150000.0, "converted_unit": None, "converted_value": None, "win_probability": 0, "date_stage_changed": 1539870749, "date_last_contacted": 1555534313, "leads_converted_from": [], "date_lead_created": None, "date_created": 1539870749, "date_modified": 1555550658 }]) processed_opps_cf = Table([{ "id": 14340759, "custom_fields_custom_field_definition_id": 272931, "custom_fields_value": [] }, { "id": 14340759, "custom_fields_custom_field_definition_id": 272927, "custom_fields_value": None }, { "id": 14161592, "custom_fields_custom_field_definition_id": 272931, "custom_fields_value": [] }, { "id": 14161592, "custom_fields_custom_field_definition_id": 272927, "custom_fields_value": None }, { "id": 14286548, "custom_fields_custom_field_definition_id": 272931, "custom_fields_value": [] }, { "id": 14286548, "custom_fields_custom_field_definition_id": 272927, "custom_fields_value": None }]) m.post(self.cp.uri + '/opportunities/search', json=self.paginate_callback, headers={"filename": "opportunities_search.json"}) processed_blob = self.cp.get_opportunities() blob_opps = [ f for f in processed_blob if f['name'] == "opportunities" ][0]['tbl'] blob_opps_cf = [ f for f in processed_blob if f['name'] == "opportunities_custom_fields" ] blob_opps_cf = blob_opps_cf[0]['tbl'] assert_matching_tables(processed_opps, blob_opps) assert_matching_tables(processed_opps_cf, blob_opps_cf)
def test_get_companies(self, m): processed_companies = Table([{ 'id': 35015567, 'name': 'Company One', 'assignee_id': None, 'contact_type_id': 547508, 'details': None, 'email_domain': '*****@*****.**', 'tags': [], 'interaction_count': 1, 'date_created': 1558441519, 'date_modified': 1558441535, 'address_city': 'CityA', 'address_country': None, 'address_postal_code': '12345', 'address_state': 'New York', 'address_street': None }, { 'id': 35026533, 'name': 'Company Two', 'assignee_id': None, 'contact_type_id': 547508, 'details': None, 'email_domain': '*****@*****.**', 'tags': [], 'interaction_count': 1, 'date_created': 1558452953, 'date_modified': 1558452967, 'address_city': 'CityB', 'address_country': None, 'address_postal_code': '23451', 'address_state': 'New York', 'address_street': None }, { 'id': 35014973, 'name': 'Company Three', 'assignee_id': None, 'contact_type_id': 547508, 'details': None, 'email_domain': None, 'tags': [], 'interaction_count': 1, 'date_created': 1558434147, 'date_modified': 1558458137, 'address_city': None, 'address_country': None, 'address_postal_code': '34512', 'address_state': 'Alabama', 'address_street': None }, { 'id': 35029116, 'name': 'Company Four', 'assignee_id': None, 'contact_type_id': 547508, 'details': None, 'email_domain': '*****@*****.**', 'tags': [], 'interaction_count': 0, 'date_created': 1558461301, 'date_modified': 1558461301, 'address_city': 'CityD ', 'address_country': None, 'address_postal_code': '45123', 'address_state': 'California', 'address_street': None }, { 'id': 35082308, 'name': 'Company Five', 'assignee_id': None, 'contact_type_id': 547508, 'details': None, 'email_domain': '*****@*****.**', 'tags': [], 'interaction_count': 1, 'date_created': 1558639445, 'date_modified': 1558639459, 'address_city': 'CityE', 'address_country': None, 'address_postal_code': '51234', 'address_state': 'Arizona', 'address_street': None }]) processed_companies_phones = Table([{ 'id': 35082308, 'phone_numbers_category': 'work', 'phone_numbers_number': '123-555-9876' }]) m.post(self.cp.uri + '/companies/search', json=self.paginate_callback, headers={"filename": "companies_search.json"}) processed_blob = self.cp.get_companies() blob_companies = [ f for f in processed_blob if f['name'] == "companies" ][0]['tbl'] blob_companies_phones = [ f for f in processed_blob if f['name'] == "companies_phone_numbers" ][0]['tbl'] assert_matching_tables(processed_companies, blob_companies) assert_matching_tables(processed_companies_phones, blob_companies_phones)