def test_get_canvass_responses_input_types(self, m):
json = {"inputTypeId": 11, "name": "API"}
m.get(self.van.connection.uri + 'canvassResponses/inputTypes',
json=json)
assert_matching_tables(Table(json),
self.van.get_canvass_responses_input_types())
def setUp(self):
# Create Table object
self.lst = [{
'a': 1,
'b': 2,
'c': 3
}, {
'a': 4,
'b': 5,
'c': 6
}, {
'a': 7,
'b': 8,
'c': 9
}, {
'a': 10,
'b': 11,
'c': 12
}, {
'a': 13,
'b': 14,
'c': 15
}]
self.lst_dicts = [{'first': 'Bob', 'last': 'Smith'}]
self.tbl = Table(self.lst_dicts)
# Create a tmp dir
os.mkdir('tmp')
def get_attendances(self, organization_id=None, updated_since=None):
"""
Fetch all attendances which were either promoted by the organization or
were for events owned by the organization.
** API Key Required **
`Args:`
organization_id: list of int
Filter events by a single or multiple organization ids
updated_since: str
Filter to events updated since given date (ISO Date)
`Returns`
Parsons Table
See :ref:`parsons-table` for output options.
"""
url = self.uri + 'organizations/' + str(
organization_id) + '/attendances'
return Table(
self.request_paginate(
url,
args={'updated_since': iso_to_unix(updated_since)},
auth=True))
def select_rows(self, *filters):
"""
Select specific rows from a Parsons table based on the passed
filters.
Example filters:
.. code-block:: python
tbl = Table([['foo', 'bar', 'baz'],
['c', 4, 9.3],
['a', 2, 88.2],
['b', 1, 23.3],])
# You can structure the filter in multiple wayss
# Lambda Function
tbl2 = tbl.select_rows(lambda row: row.foo == 'a' and row.baz > 88.1)
tbl2
>>> {foo: 'a', 'bar': 2, 'baz': 88.1}
# Expression String
tbl3 = tbl.select_rows("{foo} == 'a' and {baz} > 88.1")
tbl3
>>> {foo: 'a', 'bar': 2, 'baz': 88.1}
`Args:`
\*filters: function or str
`Returns:`
A new parsons table containing the selected rows
""" # noqa: W605
from parsons.etl.table import Table
return Table(petl.select(self.table, *filters))
def get_events_deleted(self, organization_id=None, updated_since=None):
"""
Fetch deleted public events on the platform.
** Public end point **
`Args:`
organization_id: list or int
Filter events by a single or multiple organization ids
updated_since: str
Filter to events updated since given date (ISO Date)
`Returns`
Parsons Table
See :ref:`parsons-table` for output options.
"""
if isinstance(organization_id, (str, int)):
organization_id = [organization_id]
args = {
'organization_id': organization_id,
'updated_since': iso_to_unix(updated_since)
}
return Table(
self.request_paginate(self.uri + 'events/deleted', args=args))
def test_append_to_spreadsheet(self):
append_table = Table([
{
'first': 'Jim',
'last': 'Mitchell'
},
{
'first': 'Lucy',
'last': 'Simpson'
},
])
self.google_sheets.append_to_sheet(self.spreadsheet_id, append_table)
result_table = self.google_sheets.read_sheet(self.spreadsheet_id)
self.assertEqual(append_table.columns, result_table.columns)
# We should now have rows from both tables
self.assertEqual(self.test_table.num_rows + append_table.num_rows,
result_table.num_rows)
# First check that we didn't muck with the original data
for i in range(self.test_table.num_rows):
self.assertEqual(self.test_table.data[i], result_table.data[i])
orig_row_count = self.test_table.num_rows
# Then check that we appended the data properly
for i in range(append_table.num_rows):
self.assertEqual(append_table.data[i],
result_table.data[orig_row_count + i])
def test_sort(self):
# Test basic sort
unsorted_tbl = Table([['a', 'b'], [3, 1], [2, 2], [1, 3]])
sorted_tbl = unsorted_tbl.sort()
self.assertEqual(sorted_tbl[0], {'a': 1, 'b': 3})
# Test column sort
unsorted_tbl = Table([['a', 'b'], [3, 1], [2, 2], [1, 3]])
sorted_tbl = unsorted_tbl.sort('b')
self.assertEqual(sorted_tbl[0], {'a': 3, 'b': 1})
# Test reverse sort
unsorted_tbl = Table([['a', 'b'], [3, 1], [2, 2], [1, 3]])
sorted_tbl = unsorted_tbl.sort(reverse=True)
self.assertEqual(sorted_tbl[0], {'a': 3, 'b': 1})
def test_fillna_column(self):
# Test that None values in the column are filled
self.lst = [{
'a': 1,
'b': 2,
'c': 3
}, {
'a': 4,
'b': 5,
'c': None
}, {
'a': 7,
'b': 8,
'c': 9
}, {
'a': 10,
'b': 11,
'c': None
}, {
'a': 13,
'b': 14,
'c': 15
}]
# Fixed Value only
tbl = Table(self.lst)
tbl.fillna_column('c', 0)
self.assertEqual(list(tbl.table['c']), [3, 0, 9, 0, 15])
def test_unpack_list(self):
test_table = Table([{'a': 1, 'b': [1, 2, 3]}])
# Test that list at the top level
test_table.unpack_list('b', replace=True)
self.assertEqual(['a', 'b_0', 'b_1', 'b_2'], test_table.columns)
def test_empty_column(self):
# Test that returns True on an empty column and False on a populated one.
tbl = Table([['a', 'b'], ['1', None], ['2', None]])
self.assertTrue(tbl.empty_column('b'))
self.assertFalse(tbl.empty_column('a'))
def test_first(self):
# Test that the first value in the table is returned.
self.assertEqual(self.tbl.first, 'Bob')
# Test empty value returns None
empty_tbl = Table([[1], [], [3]])
self.assertIsNone(empty_tbl.first)
def test_remove_null_rows(self):
# Test that null rows are removed from a single column
null_table = Table([{'a': 1, 'b': 2}, {'a': 1, 'b': None}])
self.assertEqual(null_table.remove_null_rows('b').num_rows, 1)
# Teest that null rows are removed from multiple columns
null_table = Table([{
'a': 1,
'b': 2,
'c': 3
}, {
'a': 1,
'b': None,
'c': 3
}])
self.assertEqual(null_table.remove_null_rows(['b', 'c']).num_rows, 1)
def test_get_max_value(self):
date_tbl = Table([['id', 'date_modified'], [1, '2020-01-01'], [2, '1900-01-01']])
self.rs.copy(date_tbl, f'{self.temp_schema}.test_date')
# Test return string
self.assertEqual(self.rs.get_max_value(f'{self.temp_schema}.test_date', 'date_modified'),
'2020-01-01')
def setUp(self):
self.mysql = MySQL(username='******', password='******', host='test', db='test', port=123)
self.tbl = Table([['ID', 'Name', 'Score'],
[1, 'Jim', 1.9],
[2, 'John', -0.5],
[3, 'Sarah', .0004]])
def test_unpack_dict(self):
test_dict = [{'a': 1, 'b': {'nest1': 1, 'nest2': 2}}]
test_table = Table(test_dict)
# Test that dict at the top level
test_table.unpack_dict('b', prepend=False)
self.assertEqual(test_table.columns, ['a', 'nest1', 'nest2'])
def test_append_csv_compressed(self):
path = self.tbl.to_csv(temp_file_compression='gzip')
append_tbl = Table([{'first': 'Mary', 'last': 'Nichols'}])
append_tbl.append_csv(path)
result_tbl = Table.from_csv(path)
# Combine tables, so we can check the resulting file
self.tbl.concat(append_tbl)
assert_matching_tables(self.tbl, result_tbl)
def test_add_users_to_custom_audience_no_valid_columns(self):
# We don't yet support full names for matching, so this shouldn't work
tbl = Table([
{
"full name": "Bob Smith"
},
])
self.assertRaises(KeyError, self.fb_ads.add_users_to_custom_audience,
self.audience_id, tbl)
def test_get_column_max_with(self):
tbl = Table([['a', 'b'], ['wide_text', False], ['text', 2]])
# Basic test
self.assertEqual(tbl.get_column_max_width('a'), 9)
# Doesn't break for non-strings
self.assertEqual(tbl.get_column_max_width('b'), 5)
def test_get_canvass_responses_contact_types(self, m):
json = {"name": "Auto Dial",
"contactTypeId": 19,
"channelTypeName": "Phone"}
m.get(self.van.connection.uri + 'canvassResponses/contactTypes', json=json)
assert_matching_tables(Table(json), self.van.get_canvass_responses_contact_types())
def test_get_rows(self):
data = [['name', 'user_name', 'id'],
['me', 'myuser', '1'],
['you', 'hey', '2'],
['you', 'hey', '3']]
tbl = Table(data)
assert_matching_tables(self.tbl.get_rows(), tbl)
def test_get_match_schema_and_data(self):
match_table = Table([
{"FN": "Bob", "LN": "Smith"},
{"FN": "Sue", "LN": "Doe"},
])
(schema, data) = FacebookAds._get_match_schema_and_data(match_table)
self.assertEqual(["FN", "LN"], schema)
self.assertEqual(("Bob", "Smith"), data[0])
self.assertEqual(("Sue", "Doe"), data[1])
def setUp(self):
self.google_sheets = GoogleSheets()
self.spreadsheet_id = self.google_sheets.create_spreadsheet('Parsons Test')
self.test_table = Table([
{'first': 'Bob', 'last': 'Smith'},
{'first': 'Sue', 'last': 'Doe'},
])
self.google_sheets.overwrite_sheet(self.spreadsheet_id, self.test_table)
self.second_sheet_title = "2nd sheet"
self.google_sheets.add_sheet(self.spreadsheet_id, self.second_sheet_title)
self.second_test_table = Table([
{'city': 'San Francisco', 'state': 'SF'},
{'city': 'Chicago', 'state': 'IL'},
])
self.google_sheets.overwrite_sheet(self.spreadsheet_id, self.second_test_table, 1)
def get_match_table_for_users_table(users_table):
"""
Prepared an input table for matching into a FB custom audience, by identifying which
columns are supported for matching, renaming those columns to what FB expects, and
cutting away the other columns.
See ``FacebookAds.create_custom_audience`` for more details.
`Args`:
users_table: Table
The source table for matching
`Returns:`
Table
The prepared table
"""
# Copy the table to avoid messing up the source table
t = copy.deepcopy(users_table)
FacebookAds._preprocess_users_table(t)
# Map the FB keys to whatever source columns match.
matched_cols = []
fb_keys_to_orig_cols = collections.defaultdict(set)
for c in t.columns:
match_key = FacebookAds._get_match_key_for_column(c)
if match_key:
matched_cols.append(c)
fb_keys_to_orig_cols[match_key].add(c)
# Cut the table to just the columns that we can use for matching in FB
t = t.cut(matched_cols)
# For each of the FB match keys, create a new column from the source column.
# If there are more than one source cols for a given FB match key, we'll pick
# the first non-empty value for each row.
for fb_key, orig_cols in fb_keys_to_orig_cols.items():
value_fn = (
lambda bound_cols:
lambda row:
FacebookAds._get_first_non_empty_value_from_dict(row, bound_cols)
)(orig_cols)
# A little trickery here to handle the case where one of the "orig_cols" is already
# named like the "fb_key".
t.add_column(fb_key+"_fb_temp_col", value_fn)
t.remove_column(*orig_cols)
t.rename_column(fb_key+"_fb_temp_col", fb_key)
# Convert None values to empty strings. Otherwise the FB SDK chokes.
petl_table = t.to_petl()
t = Table(petl_table.replaceall(None, ''))
return t
def test_from_postgres(self):
tbl = Table([['id', 'name'], [1, 'Jim'], [2, 'John'], [3, 'Sarah']])
self.pg.copy(self.tbl,
f'{self.temp_schema}.test_copy',
if_exists='drop')
out_tbl = self.tbl.from_postgres(
f"SELECT * FROM {self.temp_schema}.test_copy")
assert_matching_tables(out_tbl, tbl)
def list_items_by_id(self,
folder_id=DEFAULT_FOLDER_ID,
item_type=None) -> Table:
url = 'https://api.box.com/2.0/folders/' + folder_id
json_response = self.client.make_request('GET', url)
items = Table(json_response.json()['item_collection']['entries'])
if item_type:
items = items.select_rows(lambda row: row.type == item_type)
return items
def test_create_statement(self):
# Assert that copy statement is expected
sql = self.rs.create_statement(self.tbl, 'tmc.test', distkey='ID')
exp_sql = """create table tmc.test (\n "id" int,\n "name" varchar(5)) \ndistkey(ID) ;""" # noqa: E501
self.assertEqual(sql, exp_sql)
# Assert that an error is raised by an empty table
empty_table = Table([['Col_1', 'Col_2']])
self.assertRaises(ValueError, self.rs.create_statement, empty_table, 'tmc.test')
def test_column_data(self):
# Test that that the data in the column is returned as a list
# Test a valid column
tbl = Table(self.lst)
lst = [1, 4, 7, 10, 13]
self.assertEqual(tbl.column_data('a'), lst)
# Test an invalid column
self.assertRaises(TypeError, tbl['c'])
def test_long_table_with_na(self):
# Create a long table that is 4 rows long
tbl = Table([{'id': 1, 'tag': [1, 2, 3, 4]}, {'id': 2, 'tag': None}])
self.assertEqual(tbl.long_table(['id'], 'tag').num_rows, 4)
# Assert that column has been dropped
self.assertEqual(tbl.columns, ['id'])
# Assert that column has been retained
tbl_keep = Table([{
'id': 1,
'tag': [1, 2, 3, 4]
}, {
'id': 2,
'tag': None
}])
tbl_keep.long_table(['id'], 'tag', retain_original=True)
self.assertEqual(tbl_keep.columns, ['id', 'tag'])
def request(self, url, args=None, raw=False):
r = requests.get(url, headers=self.headers, params=args)
# This allows me to deal with data that needs to be munged.
if raw:
return r.json()
return Table(r.json()['output'])
def test_from_list_of_lists(self):
list_of_lists = [
['a', 'b', 'c'],
[1, 2, 3],
[4, 5, 6],
]
tbl = Table(list_of_lists)
self.assertEqual(tbl[0], {'a': 1, 'b': 2, 'c': 3})
