def test_process_json(self):
# Stress-testing combination of unpack methods with contrived table from hell
fake_response = [
{
'id': 1, 'Simple List Col': ['one', 'two', 'three'],
'Mixed List Col': [None, 2, 'three'], 'Spotty List Col': [1, 2, 3],
'Multidim List Col': [[1, 2], [None, 'two'], []],
'Nested List Col': [
{'A': 1, 'B': 'one'},
{'A': 2, 'B': 'two'},
{'A': 3, 'B': 'three'}],
'Simple Dict Col': {'one': 1, 'two': 2, 'three': 3},
'Nested Dict Col': {'A': 1, 'B': ['two', 2], 'C': [None, 3, 'three']}
}, {
'id': 2, 'Simple List Col': ['four', 'five', 'six'],
'Mixed List Col': ['four', None, 6], 'Spotty List Col': [],
'Multidim List Col': [[3, None], [], ['three', 'four']],
'Nested List Col': [
{'A': 4, 'B': 'four'},
{'A': 5, 'B': 'five'},
{'A': 6, 'B': 'six'}],
'Simple Dict Col': {'one': 'I', 'two': 'II', 'three': 'III'},
'Nested Dict Col': {'A': ['one'], 'B': [], 'C': 3},
}, {
'id': 3, 'Simple List Col': ['seven', 'eight', 'nine'],
'Mixed List Col': [7, 'eight', None], 'Spotty List Col': None,
'Multidim List Col': [['five', 6], [None]],
'Nested List Col': [
{'A': 7, 'B': 'seven'},
{'A': 8, 'B': 'eight'},
{'A': 9, 'B': 'nine'}],
'Simple Dict Col': {'one': 'x', 'two': 'xx', 'three': 'xxx'},
'Nested Dict Col': {'A': None, 'B': 2, 'C': [None, 3, 'three']}
}
]
fake_response_tables = {}
table_names = ['fake_Nested List Col', 'fake']
fake_response_tables['fake_Nested List Col'] = Table([
{"id": 1, "Nested List Col_A": 1, "Nested List Col_B": "one"},
{"id": 1, "Nested List Col_A": 2, "Nested List Col_B": "two"},
{"id": 1, "Nested List Col_A": 3, "Nested List Col_B": "three"},
{"id": 2, "Nested List Col_A": 4, "Nested List Col_B": "four"},
{"id": 2, "Nested List Col_A": 5, "Nested List Col_B": "five"},
{"id": 2, "Nested List Col_A": 6, "Nested List Col_B": "six"},
{"id": 3, "Nested List Col_A": 7, "Nested List Col_B": "seven"},
{"id": 3, "Nested List Col_A": 8, "Nested List Col_B": "eight"},
{"id": 3, "Nested List Col_A": 9, "Nested List Col_B": "nine"}
])
fake_response_tables['fake'] = Table([
{"id": 1, "Simple List Col": ["one", "two", "three"],
"Mixed List Col": [None, 2, "three"], "Spotty List Col": [1, 2, 3],
"Multidim List Col": [[1, 2], [None, "two"], []],
"Simple Dict Col_one": 1, "Simple Dict Col_three": 3,
"Simple Dict Col_two": 2, "Nested Dict Col_A": 1,
"Nested Dict Col_B": ["two", 2], "Nested Dict Col_C": [None, 3, "three"]},
{"id": 2, "Simple List Col": ["four", "five", "six"],
"Mixed List Col": ["four", None, 6], "Spotty List Col": [],
"Multidim List Col": [[3, None], [], ["three", "four"]],
"Simple Dict Col_one": "I", "Simple Dict Col_three": "III",
"Simple Dict Col_two": "II", "Nested Dict Col_A": ["one"],
"Nested Dict Col_B": [], "Nested Dict Col_C": 3},
{"id": 3, "Simple List Col": ["seven", "eight", "nine"],
"Mixed List Col": [7, "eight", None], "Spotty List Col": [None],
"Multidim List Col": [["five", 6], [None]], "Simple Dict Col_one": "x",
"Simple Dict Col_three": "xxx", "Simple Dict Col_two": "xx",
"Nested Dict Col_A": None, "Nested Dict Col_B": 2,
"Nested Dict Col_C": [None, 3, "three"]}
])
fake_processed = self.cp.process_json(fake_response, 'fake')
self.assertTrue([f['name'] for f in fake_processed] == table_names)
for tbl in table_names:
assert_matching_tables(
[f['tbl'] for f in fake_processed if f['name'] == tbl][0],
fake_response_tables[tbl]
)
fake_tidy = self.cp.process_json(fake_response, 'fake', tidy=0)
self.assertTrue(len(fake_tidy) == len(fake_response[0])-1)
def test_get_table(live_sftp, simple_table): # noqa F811
file_util.create_temp_file()
tbl = live_sftp.get_table(CSV_PATH)
assert_matching_tables(tbl, simple_table)
def test_paginate_request(self, m):
# Anonymized real output with nested columns
self.blob = [
{
'id': 78757050,
'name': 'Person One',
'prefix': None,
'first_name': 'Person',
'middle_name': None,
'last_name': 'One',
'suffix': None,
'address': {
'street': None, 'city': 'CityA', 'state': 'StateI',
'postal_code': '12345', 'country': None},
'assignee_id': None,
'company_id': 12030795,
'company_name': 'Indivisible CityA',
'contact_type_id': 501950,
'details': None,
'emails': [{'email': '*****@*****.**', 'category': 'work'}],
'phone_numbers': [
{'number': '(541) 555-9585', 'category': 'work'},
{'number': '555-555-9585', 'category': 'work'}],
'socials': [{'url': 'https://gravatar.com/gravatar', 'category': 'gravatar'}],
'tags': [],
'title': None,
'websites': [{'url': 'http://www.IndivisibleCityA.org', 'category': None}],
'custom_fields': [
{'custom_field_definition_id': 125880, 'value': None},
{'custom_field_definition_id': 107297, 'value': None},
{'custom_field_definition_id': 102127, 'value': None},
{'custom_field_definition_id': 135034, 'value': None},
{'custom_field_definition_id': 107298, 'value': None},
{'custom_field_definition_id': 108972, 'value': None},
{'custom_field_definition_id': 125881, 'value': None}],
'date_created': 1558169903,
'date_modified': 1558169910,
'date_last_contacted': 1558169891,
'interaction_count': 1,
'leads_converted_from': [],
'date_lead_created': None
}, {
'id': 78477076,
'name': 'Person Two',
'prefix': None,
'first_name': 'Person',
'middle_name': None,
'last_name': 'Two',
'suffix': None,
'address': {
'street': None, 'city': None, 'state': None,
'postal_code': None, 'country': None},
'assignee_id': 289533,
'company_id': 12096071,
'company_name': 'Indivisible StateII',
'contact_type_id': 501950,
'details': None,
'emails': [{'email': '*****@*****.**', 'category': 'work'}],
'phone_numbers': [{'number': '(908) 555-2941', 'category': 'work'}],
'socials': [],
'tags': ['treasurer'],
'title': 'Treasurer',
'websites': [],
'custom_fields': [
{'custom_field_definition_id': 125880, 'value': None},
{'custom_field_definition_id': 107297, 'value': None},
{'custom_field_definition_id': 102127, 'value': None},
{'custom_field_definition_id': 135034, 'value': None},
{'custom_field_definition_id': 107298, 'value': None},
{'custom_field_definition_id': 108972, 'value': None},
{'custom_field_definition_id': 125881, 'value': None}],
'date_created': 1557761054,
'date_modified': 1558218799,
'date_last_contacted': 1558196341,
'interaction_count': 14,
'leads_converted_from': [],
'date_lead_created': None
}, {
'id': 78839154,
'name': 'Person Three',
'prefix': None,
'first_name': 'Person',
'middle_name': None,
'last_name': 'Three',
'suffix': None,
'address': {
'street': None, 'city': 'CityC', 'state': 'StateIII',
'postal_code': '54321', 'country': None},
'assignee_id': None,
'company_id': 34966944,
'company_name': 'Flip StateIII',
'contact_type_id': 501950,
'details': None,
'emails': [{'email': '*****@*****.**', 'category': 'work'}],
'phone_numbers': [{'number': '(619) 555-7883', 'category': 'work'}],
'socials': [
{'url': 'https://twitter.com/ThreePerson', 'category': 'twitter'},
{'url': 'https://www.facebook.com/Person.n.Three', 'category': 'facebook'},
{'url': 'https://gravatar.com/PersonThree', 'category': 'gravatar'}],
'tags': [],
'title': None,
'websites': [],
'custom_fields': [
{'custom_field_definition_id': 125880, 'value': None},
{'custom_field_definition_id': 107297, 'value': None},
{'custom_field_definition_id': 102127, 'value': None},
{'custom_field_definition_id': 135034, 'value': None},
{'custom_field_definition_id': 107298, 'value': None},
{'custom_field_definition_id': 108972, 'value': None},
{'custom_field_definition_id': 125881, 'value': None}],
'date_created': 1558223367,
'date_modified': 1558223494,
'date_last_contacted': 1558223356,
'interaction_count': 2,
'leads_converted_from': [],
'date_lead_created': None
}
]
# Mock endpoints
m.post(
self.cp.uri + '/people/search',
json=self.paginate_callback,
headers={"filename": "people_search.txt"})
# self.assertTrue(
assert_matching_tables(
Table(self.blob),
Table(
self.cp.paginate_request(
'/people/search',
page_size=1,
req_type='POST'
)
)
)
def test_materialize_to_file(self):
# Simple test that materializing doesn't change the table
tbl_materialized = Table(self.lst_dicts)
_ = tbl_materialized.materialize_to_file()
assert_matching_tables(self.tbl, tbl_materialized)
def assert_file_matches_table(local_path, table):
downloaded_tbl = Table.from_csv(local_path)
assert_matching_tables(table, downloaded_tbl)
def test_to_from_temp_json_compressed(self):
path = self.tbl.to_json(temp_file_compression='gzip')
result_tbl = Table.from_json(path)
assert_matching_tables(self.tbl, result_tbl)
def test_match_columns(self):
raw = [
{
'first name': 'Mary',
'LASTNAME': 'Nichols',
'Middle__Name': 'D'
},
{
'first name': 'Lucy',
'LASTNAME': 'Peterson',
'Middle__Name': 'S'
},
]
tbl = Table(raw)
desired_raw = [
{
'first_name': 'Mary',
'middle_name': 'D',
'last_name': 'Nichols'
},
{
'first_name': 'Lucy',
'middle_name': 'S',
'last_name': 'Peterson'
},
]
desired_tbl = Table(desired_raw)
# Test with fuzzy matching
tbl.match_columns(desired_tbl.columns)
assert_matching_tables(desired_tbl, tbl)
# Test disable fuzzy matching, and fail due due to the missing cols
self.assertRaises(TypeError,
Table(raw).match_columns,
desired_tbl.columns,
fuzzy_match=False,
if_missing_columns='fail')
# Test disable fuzzy matching, and fail due to the extra cols
self.assertRaises(TypeError,
Table(raw).match_columns,
desired_tbl.columns,
fuzzy_match=False,
if_extra_columns='fail')
# Test table that already has the right columns, shouldn't need fuzzy match
tbl = Table(desired_raw)
tbl.match_columns(desired_tbl.columns,
fuzzy_match=False,
if_missing_columns='fail',
if_extra_columns='fail')
assert_matching_tables(desired_tbl, tbl)
# Test table with missing col, verify the missing col gets added by default
tbl = Table([
{
'first name': 'Mary',
'LASTNAME': 'Nichols'
},
{
'first name': 'Lucy',
'LASTNAME': 'Peterson'
},
])
tbl.match_columns(desired_tbl.columns)
desired_tbl = (
Table(desired_raw).remove_column('middle_name').add_column(
'middle_name', index=1))
assert_matching_tables(desired_tbl, tbl)
# Test table with extra col, verify the extra col gets removed by default
tbl = Table([
{
'first name': 'Mary',
'LASTNAME': 'Nichols',
'Age': 32,
'Middle__Name': 'D'
},
{
'first name': 'Lucy',
'LASTNAME': 'Peterson',
'Age': 26,
'Middle__Name': 'S'
},
])
desired_tbl = Table(desired_raw)
tbl.match_columns(desired_tbl.columns)
assert_matching_tables(desired_tbl, tbl)
# Test table with two columns that normalize the same and aren't in desired cols, verify
# they both get removed.
tbl = Table([
{
'first name': 'Mary',
'LASTNAME': 'Nichols',
'Age': 32,
'Middle__Name': 'D',
'AGE': None
},
{
'first name': 'Lucy',
'LASTNAME': 'Peterson',
'Age': 26,
'Middle__Name': 'S',
'AGE': None
},
])
tbl.match_columns(desired_tbl.columns)
assert_matching_tables(desired_tbl, tbl)
# Test table with two columns that match desired cols, verify only the first gets kept.
tbl = Table([
{
'first name': 'Mary',
'LASTNAME': 'Nichols',
'First Name': None,
'Middle__Name': 'D'
},
{
'first name': 'Lucy',
'LASTNAME': 'Peterson',
'First Name': None,
'Middle__Name': 'S'
},
])
tbl.match_columns(desired_tbl.columns)
assert_matching_tables(desired_tbl, tbl)
def _assert_expected_csv(self, path, orig_tbl):
result_tbl = Table.from_csv(path)
assert_matching_tables(orig_tbl, result_tbl)
def test_to_from_temp_json(self):
path = self.tbl.to_json()
result_tbl = Table.from_json(path)
assert_matching_tables(self.tbl, result_tbl)
def test_get_posts(self, m):
m.get(self.ct.uri + '/posts', json=expected_posts)
posts = self.ct.get_posts()
exp_tbl = self.ct.unpack(Table(expected_posts['result']['posts']))
assert_matching_tables(posts, exp_tbl)
def test_get_links(self, m):
m.get(self.ct.uri + '/links', json=expected_post)
post = self.ct.get_links(link='https://nbcnews.to/34stfC2')
exp_tbl = self.ct.unpack(Table(expected_post['result']['posts']))
assert_matching_tables(post, exp_tbl)
def test_distribute_task(self):
global count
global tableargs
datatable = [
('x', 'y'),
(1, 2),
(3, 4),
(5, 6),
(7, 8),
(9, 0),
(11, 12),
(13, 14),
(15, 16),
(17, 18),
(19, 10),
]
count = 0
tableargs = None
distribute_task(
Table(datatable),
fake_table_process,
'foo', # bucket
group_count=5,
storage='local',
func_kwargs={
'x': 1,
'y': [2, 3]
})
self.assertEqual(count, 2)
assert_matching_tables(
tableargs[0],
Table([('x', 'y'), (11, 12), (13, 14), (15, 16), (17, 18),
(19, 10)]))
count = 0
tableargs = None
distribute_task(
Table(datatable + [(0, 0)]),
FakeRunner.foobar,
'foo', # bucket
group_count=5,
storage='local',
func_class=FakeRunner,
func_class_kwargs={'init1': 'initx'},
func_kwargs={
'a': 1,
'x': 2,
'y': 3
})
self.assertEqual(count, 3)
self.assertEqual(tableargs[1:], ('initx', 1, 2, 3))
self.assertEqual(tableargs[1:], ('initx', 1, 2, 3))
assert_matching_tables(tableargs[0], Table([('x', 'y'), (0, 0)]))
# 3. catch=True (with throwing)
count = 0
tableargs = None
distribute_task(
Table(datatable[:6]),
FakeRunner.foobar,
'foo', # bucket
group_count=5,
storage='local',
func_class=FakeRunner,
func_class_kwargs={'init1': 'initx'},
catch=True,
func_kwargs={
'a': 'raise',
'x': 2,
'y': 3
})
def test_get_custom_field(self, m):
m.get(self.van.connection.uri + 'customFields/157', json=custom_field)
assert_matching_tables(custom_field, self.van.get_custom_field(157))
def test_get_custom_field_values(self, m):
m.get(self.van.connection.uri + 'customFields', json=custom_field)
assert_matching_tables(custom_field_values, self.van.get_custom_fields_values())
def test_get_opportunities(self, m):
processed_opps = Table([{
"id": 14340759,
"name": "Company1",
"assignee_id": 659394,
"close_date": None,
"company_id": 29324143,
"company_name": "Company1",
"customer_source_id": None,
"details": None,
"loss_reason_id": None,
"pipeline_id": 489028,
"pipeline_stage_id": 2529569,
"primary_contact_id": 67747998,
"priority": "High",
"status": "Open",
"tags": ["opportunities import-1540158946352"],
"interaction_count": 0,
"monetary_unit": "USD",
"monetary_value": 100000.0,
"converted_unit": None,
"converted_value": None,
"win_probability": None,
"date_stage_changed": 1548866182,
"date_last_contacted": None,
"leads_converted_from": [],
"date_lead_created": None,
"date_created": 1540159060,
"date_modified": 1550858334
}, {
"id": 14161592,
"name": "Company2",
"assignee_id": 659394,
"close_date": "11/10/2018",
"company_id": 28729196,
"company_name": "Company2",
"customer_source_id": None,
"details": None,
"loss_reason_id": None,
"pipeline_id": 531482,
"pipeline_stage_id": 2607171,
"primary_contact_id": 67243374,
"priority": "High",
"status": "Open",
"tags": [],
"interaction_count": 36,
"monetary_unit": "USD",
"monetary_value": 77000.0,
"converted_unit": None,
"converted_value": None,
"win_probability": None,
"date_stage_changed": 1551191957,
"date_last_contacted": 1552339800,
"leads_converted_from": [],
"date_lead_created": None,
"date_created": 1539192375,
"date_modified": 1552340016
}, {
"id": 14286548,
"name": "Company3",
"assignee_id": 644608,
"close_date": "11/18/2018",
"company_id": 29492294,
"company_name": "Company3",
"customer_source_id": None,
"details": None,
"loss_reason_id": None,
"pipeline_id": 531482,
"pipeline_stage_id": 2482007,
"primary_contact_id": 67637400,
"priority": "None",
"status": "Open",
"tags": [],
"interaction_count": 19,
"monetary_unit": "USD",
"monetary_value": 150000.0,
"converted_unit": None,
"converted_value": None,
"win_probability": 0,
"date_stage_changed": 1539870749,
"date_last_contacted": 1555534313,
"leads_converted_from": [],
"date_lead_created": None,
"date_created": 1539870749,
"date_modified": 1555550658
}])
processed_opps_cf = Table([{
"id": 14340759,
"custom_fields_custom_field_definition_id": 272931,
"custom_fields_value": []
}, {
"id": 14340759,
"custom_fields_custom_field_definition_id": 272927,
"custom_fields_value": None
}, {
"id": 14161592,
"custom_fields_custom_field_definition_id": 272931,
"custom_fields_value": []
}, {
"id": 14161592,
"custom_fields_custom_field_definition_id": 272927,
"custom_fields_value": None
}, {
"id": 14286548,
"custom_fields_custom_field_definition_id": 272931,
"custom_fields_value": []
}, {
"id": 14286548,
"custom_fields_custom_field_definition_id": 272927,
"custom_fields_value": None
}])
m.post(self.cp.uri + '/opportunities/search',
json=self.paginate_callback,
headers={"filename": "opportunities_search.json"})
processed_blob = self.cp.get_opportunities()
blob_opps = [
f for f in processed_blob if f['name'] == "opportunities"
][0]['tbl']
blob_opps_cf = [
f for f in processed_blob
if f['name'] == "opportunities_custom_fields"
]
blob_opps_cf = blob_opps_cf[0]['tbl']
assert_matching_tables(processed_opps, blob_opps)
assert_matching_tables(processed_opps_cf, blob_opps_cf)
def test_get_companies(self, m):
processed_companies = Table([{
'id': 35015567,
'name': 'Company One',
'assignee_id': None,
'contact_type_id': 547508,
'details': None,
'email_domain': '*****@*****.**',
'tags': [],
'interaction_count': 1,
'date_created': 1558441519,
'date_modified': 1558441535,
'address_city': 'CityA',
'address_country': None,
'address_postal_code': '12345',
'address_state': 'New York',
'address_street': None
}, {
'id': 35026533,
'name': 'Company Two',
'assignee_id': None,
'contact_type_id': 547508,
'details': None,
'email_domain': '*****@*****.**',
'tags': [],
'interaction_count': 1,
'date_created': 1558452953,
'date_modified': 1558452967,
'address_city': 'CityB',
'address_country': None,
'address_postal_code': '23451',
'address_state': 'New York',
'address_street': None
}, {
'id': 35014973,
'name': 'Company Three',
'assignee_id': None,
'contact_type_id': 547508,
'details': None,
'email_domain': None,
'tags': [],
'interaction_count': 1,
'date_created': 1558434147,
'date_modified': 1558458137,
'address_city': None,
'address_country': None,
'address_postal_code': '34512',
'address_state': 'Alabama',
'address_street': None
}, {
'id': 35029116,
'name': 'Company Four',
'assignee_id': None,
'contact_type_id': 547508,
'details': None,
'email_domain': '*****@*****.**',
'tags': [],
'interaction_count': 0,
'date_created': 1558461301,
'date_modified': 1558461301,
'address_city': 'CityD ',
'address_country': None,
'address_postal_code': '45123',
'address_state': 'California',
'address_street': None
}, {
'id': 35082308,
'name': 'Company Five',
'assignee_id': None,
'contact_type_id': 547508,
'details': None,
'email_domain': '*****@*****.**',
'tags': [],
'interaction_count': 1,
'date_created': 1558639445,
'date_modified': 1558639459,
'address_city': 'CityE',
'address_country': None,
'address_postal_code': '51234',
'address_state': 'Arizona',
'address_street': None
}])
processed_companies_phones = Table([{
'id':
35082308,
'phone_numbers_category':
'work',
'phone_numbers_number':
'123-555-9876'
}])
m.post(self.cp.uri + '/companies/search',
json=self.paginate_callback,
headers={"filename": "companies_search.json"})
processed_blob = self.cp.get_companies()
blob_companies = [
f for f in processed_blob if f['name'] == "companies"
][0]['tbl']
blob_companies_phones = [
f for f in processed_blob if f['name'] == "companies_phone_numbers"
][0]['tbl']
assert_matching_tables(processed_companies, blob_companies)
assert_matching_tables(processed_companies_phones,
blob_companies_phones)
