def _table_convert(self, obj):
tbl = Table([x.__dict__['_properties'] for x in obj])
if 'subresource_uris' in tbl.columns and 'uri' in tbl.columns:
tbl.remove_column('subresource_uris', 'uri')
return tbl
def get_agents(self, email=None, mobile=None, phone=None, state=None):
"""
List agents.
`Args:`
email: str
Filter by email address.
mobile: str
Filter by mobile phone number
phone: str
Filter by phone number
state: str
Filter by state
`Returns:`
Parsons Table
See :ref:`parsons-table` for output options.
"""
params = {
'email': email,
'mobile': mobile,
'phone': phone,
'state': state
}
tbl = Table(self.get_request('agents', params=params))
logger.info(f'Found {tbl.num_rows} agents.')
tbl = self.transform_table(tbl)
tbl = tbl.unpack_dict('contact', prepend=False)
tbl.remove_column(
'signature') # Removing since raw HTML might cause issues.
return tbl
def get_surveys(self, page=None):
"""
Get a table of lists under the account.
`Args:`
page : int
Retrieve a specific page of responses. If not given,
then all pages are retrieved.
`Returns:`
Table Class
"""
r = self._client.api.survey.list(page)
data = r['data']
if not page:
while r['page'] < r['total_pages']:
r = self._client.api.survey.list(page=(r['page']+1))
data.extend(r['data'])
tbl = Table(data).remove_column('links')
tbl.unpack_dict('statistics', prepend=False)
logger.info(f"Found {tbl.num_rows} surveys.")
return tbl
def test_get_url(self):
file_name = 'delete_me.csv'
input_tbl = Table([['a'], ['1']])
self.cloud.upload_table(input_tbl, TEMP_BUCKET_NAME, file_name)
url = self.cloud.get_url(TEMP_BUCKET_NAME, file_name)
download_tbl = Table.from_csv(url)
assert_matching_tables(input_tbl, download_tbl)
def get_records(self, fields=None, max_records=None, view=None, formula=None, sort=None):
"""
`Args:`
fields: str or lst
Only return specified column or list of columns. The column name is
case sensitive
max_records: int
The maximum total number of records that will be returned.
view: str
If set, only the records in that view will be returned. The records will be
sorted according to the order of the view.
formula: str
The formula will be evaluated for each record, and if the result
is not 0, false, "", NaN, [], or #Error! the record will be included
in the response.
If combined with view, only records in that view which satisfy the
formula will be returned. For example, to only include records where
``COLUMN_A`` isn't empty, pass in: ``"NOT({COLUMN_A}='')"``
For more information see
`Airtable Docs on formulas. <https://airtable.com/api>`_
Usage - Text Column is not empty:
``airtable.get_all(formula="NOT({COLUMN_A}='')")``
Usage - Text Column contains:
``airtable.get_all(formula="FIND('SomeSubText', {COLUMN_STR})=1")``
sort: str or lst
Specifies how the records will be ordered. If you set the view parameter, the
returned records in that view will be sorted by these fields. If sorting by
multiple columns, column names can be passed as a list. Sorting Direction is
ascending by default, but can be reversed by prefixing the column name with a minus
sign -.
Example usage:
``airtable.get_records(sort=['ColumnA', '-ColumnB'])``
`Returns:`
Parsons Table
See :ref:`parsons-table` for output options.
"""
# Raises an error if sort is None type. Thus, only adding if populated.
kwargs = {'fields': fields, 'max_records': max_records, 'view': view, 'formula': formula}
if sort:
kwargs['sort'] = sort
tbl = Table(self.at.get_all(**kwargs))
# If the results are empty, then return an empty table.
if 'fields' not in tbl.columns:
return Table([[]])
return tbl.unpack_dict(column='fields', prepend=False)
def table_convert(self, obj):
# Internal method to create a Parsons table from a Twilio object.
tbl = Table([x.__dict__['_properties'] for x in obj])
if 'subresource_uris' in tbl.columns and 'uri' in tbl.columns:
tbl.remove_column('subresource_uris', 'uri')
return tbl
def test_create_leads(self, m):
m.post(HUSTLE_URI + 'groups/cMCH0hxwGt/leads', json=expected_json.leads_tbl_01)
tbl = Table([['phone_number', 'ln', 'first_name'],
['4435705355', 'Warren', 'Elizabeth'],
['5126993336', 'Obama', 'Barack']])
ids = self.hustle.create_leads(tbl, group_id='cMCH0hxwGt')
assert_matching_tables(ids, Table(expected_json.leads['items']))
def convert_to_table(self, data):
# Internal method to create a Parsons table from a data element.
table = None
if type(data) is list:
table = Table(data)
else:
table = Table([data])
return table
def test_get_leads(self, m):
# By Organization
m.get(HUSTLE_URI + 'organizations/cMCH0hxwGt/leads', json=expected_json.leads)
leads = self.hustle.get_leads(organization_id='cMCH0hxwGt')
assert_matching_tables(leads, Table(expected_json.leads['items']))
# By Group ID
m.get(HUSTLE_URI + 'groups/cMCH0hxwGt/leads', json=expected_json.leads)
leads = self.hustle.get_leads(group_id='cMCH0hxwGt')
assert_matching_tables(leads, Table(expected_json.leads['items']))
def test_get_poll_locations(self, m):
m.get(self.gc.uri + 'voterinfo', json=voterinfo_resp)
expected_tbl = Table(polling_data)
address_tbl = Table([['address'],
['900 N Washtenaw, Chicago, IL 60622'],
['900 N Washtenaw, Chicago, IL 60622']])
tbl = self.gc.get_polling_locations(2000, address_tbl)
assert_matching_tables(tbl, expected_tbl)
def get_campaign_emails(self,
campaign_id,
fields=None,
exclude_fields=None,
count=None,
offset=None,
since=None):
"""
Get a table of individual emails from a campaign based on query
parameters. Note that argument descriptions here are sourced from
Mailchimp's official API documentation.
`Args:`
campaign_id: string
The unique ID of the campaign to fetch emails from.
fields: list of strings
A comma-separated list of fields to return. Reference
parameters of sub-objects with dot notation.
exclude_fields: list of strings
A comma-separated list of fields to exclude. Reference
parameters of sub-objects with dot notation.
count: int
The number of records to return. Default value is 10. Maximum
value is 1000.
offset: int
The number of records from a collection to skip. Iterating over
large collections with this parameter can be slow. Default
value is 0.
since: string
Restrict results to email activity events that occur after a
specific time. We recommend ISO 8601 time format:
2015-10-21T15:41:36+00:00.
`Returns:`
Table Class
"""
params = {
'fields': fields,
'exclude_fields': exclude_fields,
'count': count,
'offset': offset,
'since': since
}
response = self.client.get_request(
f'reports/{campaign_id}/email-activity', params=params)
tbl = Table(response['emails'])
if tbl.num_rows > 0:
return tbl
else:
return Table()
def test_geocode_address_batch(self):
batch = [['id', 'street', 'city', 'state', 'zip'],
['1', '908 N Washtenaw', 'Chicago', 'IL', '60622'],
['2', '1405 Wilshire Blvd', 'Austin', 'TX', '78722'],
['3', '908 N Washtenaw', 'Chicago', 'IL', '60622'],
['4', '1405 Wilshire Blvd', 'Austin', 'TX', '78722'],
['5', '908 N Washtenaw', 'Chicago', 'IL', '60622']]
tbl = Table(batch)
self.cg.cg.addressbatch = mock.MagicMock(return_value=batch_resp)
geo = self.cg.geocode_address_batch(tbl)
assert_matching_tables(geo, Table(petl.fromdicts(batch_resp)))
def process_custom_fields(self, json_blob):
# Internal method to convert custom fields responses into a list of Parsons tables
# Original table & columns
custom_fields = Table(json_blob)
# Available On
available_on = custom_fields.long_table(['id'], 'available_on')
# Options
options = custom_fields.long_table(['id', 'name'], 'options')
return [{'name': 'custom_fields', 'tbl': custom_fields},
{'name': 'custom_fields_available', 'tbl': available_on},
{'name': 'custom_fields_options', 'tbl': options}]
def test_download_file(self) -> None:
box = Box()
table = Table([['phone_number', 'last_name', 'first_name'],
['4435705355', 'Warren', 'Elizabeth'],
['5126993336', 'Obama', 'Barack']])
uploaded_file = table.to_csv()
path_filename = f'{self.temp_folder_name}/my_path'
box.upload_table(table, path_filename)
downloaded_file = box.download_file(path_filename)
with open(uploaded_file) as uploaded, open(downloaded_file) as downloaded:
self.assertEqual(str(uploaded.read()), str(downloaded.read()))
def get_leads(self, organization_id=None, group_id=None):
"""
Get leads metadata. One of ``organization_id`` and ``group_id`` must be passed
as an argument. If both are passed, an error will be raised.
`Args:`
organization_id: str
The organization id.
group_id: str
The group id.
`Returns:`
Parsons Table
See :ref:`parsons-table` for output options.
"""
if organization_id is None and group_id is None:
raise ValueError('Either organization_id or group_id required.')
if organization_id is not None and group_id is not None:
raise ValueError(
'Only one of organization_id and group_id may be populated.')
if organization_id:
endpoint = f'organizations/{organization_id}/leads'
logger.info(f'Retrieving {organization_id} organization leads.')
if group_id:
endpoint = f'groups/{group_id}/leads'
logger.info(f'Retrieving {group_id} group leads.')
tbl = Table(self._request(endpoint))
logger.info(f'Got {tbl.num_rows} leads.')
return tbl
def get_contacts(self,
email=None,
mobile=None,
phone=None,
company_id=None,
state=None,
updated_since=None,
expand_custom_fields=None):
"""
Get contacts.
`Args:`
email: str
Filter by email address.
mobile: str
Filter by mobile phone number.
phone: str
Filter by phone number.
expand_custom_fields: boolean
Expand nested custom fields to their own columns.
"""
params = {
'email': email,
'mobile': mobile,
'phone': phone,
'company_id': company_id,
'state': state,
'_updated_since': updated_since
}
tbl = Table(self.get_request('contacts', params=params))
logger.info(f'Found {tbl.num_rows} contacts.')
return self.transform_table(tbl, expand_custom_fields)
def get_contributions(self, csv_type, date_range_start, date_range_end):
"""
Get specified contribution data from CSV API as Parsons table.
`Args:`
csv_type: str
Type of CSV you are requesting.
Options:
'paid_contributions': contains paid, non-refunded contributions to the entity
(campaign or organization) you created the credential for, during the specified
date range.
'refunded_contributions': contributions to your entity that were refunded,
during the specified date range.
'managed_form_contributions': contributions made through any form that is
managed by your entity, during the specified date range - including
contributions to other entities via that form if it is a tandem form.
date_range_start: str
Start of date range to withdraw contribution data (inclusive). Ex: '2020-01-01'
date_range_end: str
End of date range to withdraw contribution data (exclusive). Ex: '2020-02-01'
`Returns:`
Contents of the generated contribution CSV as a Parsons table.
"""
post_request_response = self.post_request(csv_type, date_range_start, date_range_end)
csv_id = post_request_response["id"]
download_url = self.poll_for_download_url(csv_id)
table = Table.from_csv(download_url)
logger.info('Completed conversion to Parsons Table.')
return table
def test_get_leaderboard(self, m):
m.get(self.ct.uri + '/leaderboard', json=expected_leaderboard)
leaderboard = self.ct.get_leaderboard()
exp_tbl = self.ct.unpack(
Table(expected_leaderboard['result']['accountStatistics']))
assert_matching_tables(leaderboard, exp_tbl)
def get_constituents(self,
page_number=1,
page_size=50,
order_by=None,
order_direction=None,
last_modified=None):
"""
`Args:`
page_number: int
Number of the page to fetch
page_size: int
Number of records per page (maximum allowed is 50)
order_by: str
Sorts by ``Id``, ``CreatedDate``, or ``LastModifiedDate`` (default ``Id``).
order_direction: str
Sorts the order_by in ``Asc`` or ``Desc`` order.
last_modified: str
Filters to constituents last modified after the specified date (ISO-8601 format).
`Returns:`
A Table of the entries.
"""
params = self._base_pagination_params(page_number, page_size)
params.update(self._base_ordering_params(order_by, order_direction))
if last_modified:
params["lastModified"] = last_modified
response = self._base_get('constituents', params=params)
return Table(response['Results'])
def test_get_elections(self, m):
m.get(self.gc.uri + 'elections', json=elections_resp)
expected_tbl = Table(elections_resp['elections'])
assert_matching_tables(self.gc.get_elections(), expected_tbl)
def test_upload_file(self) -> None:
# Count on environment variables being set
box = Box()
table = Table([['phone_number', 'last_name', 'first_name'],
['4435705355', 'Warren', 'Elizabeth'],
['5126993336', 'Obama', 'Barack']])
box_file = box.upload_table_to_folder_id(table, 'phone_numbers',
folder_id=self.temp_folder_id)
new_table = box.get_table_by_file_id(box_file.id)
# Check that what we saved is equal to what we got back
self.assertEqual(str(table), str(new_table))
# Check that things also work in JSON
box_file = box.upload_table_to_folder_id(table, 'phone_numbers_json',
folder_id=self.temp_folder_id,
format='json')
new_table = box.get_table_by_file_id(box_file.id, format='json')
# Check that what we saved is equal to what we got back
self.assertEqual(str(table), str(new_table))
# Now check the same thing with paths instead of file_id
path_filename = 'path_phone_numbers'
box_file = box.upload_table(table, f'{self.temp_folder_name}/{path_filename}')
new_table = box.get_table(path=f'{self.temp_folder_name}/{path_filename}')
# Check that we throw an exception with bad formats
with self.assertRaises(ValueError):
box.upload_table_to_folder_id(table, 'phone_numbers', format='illegal_format')
with self.assertRaises(ValueError):
box.get_table_by_file_id(box_file.id, format='illegal_format')
def get_leaderboard(self, start_date=None, end_date=None, list_ids=None, account_ids=None):
"""
Return advocates (person records).
`Args:`
start_date: str
Filter to the earliest date at which a post could be posted.
The time is formatted as UTC (e.g. ``yyyy-mm-ddThh:mm:ss``).
end_date: str
Filter to the latest date at which a post could be posted.
The time is formatted as UTC (e.g. ``yyyy-mm-ddThh:mm:ss``).
list_ids: list
Filter to the ids of lists or saved searches to retrieve.
account_ids: list
A list of CrowdTangle accountIds to retrieve leaderboard data for.
This and ``list_id`` are mutually exclusive; if both are sent, the
``account_ids`` value will be used.
`Returns:`
Parsons Table
See :ref:`parsons-table` for output options.
"""
args = {'startDate': start_date,
'endDate': end_date,
'listIds': self.list_to_string(list_ids),
'accountIds': self.list_to_string(account_ids)}
pt = Table(self.base_request('leaderboard', args=args))
logger.info(f'Retrieved {pt.num_rows} records from the leaderbooard.')
self.unpack(pt)
return pt
def test_list_files_by_id(self) -> None:
# Count on environment variables being set
box = Box()
subfolder = box.create_folder_by_id(
folder_name='id_subfolder', parent_folder_id=self.temp_folder_id)
# Create a couple of files in the temp folder
table = Table([['phone_number', 'last_name', 'first_name'],
['4435705355', 'Warren', 'Elizabeth'],
['5126993336', 'Obama', 'Barack']])
box.upload_table_to_folder_id(table, 'temp1', folder_id=subfolder)
box.upload_table_to_folder_id(table, 'temp2', folder_id=subfolder)
box.create_folder_by_id(folder_name='temp_folder1',
parent_folder_id=subfolder)
box.create_folder_by_id(folder_name='temp_folder2',
parent_folder_id=subfolder)
file_list = box.list_files_by_id(folder_id=subfolder)
self.assertEqual(['temp1', 'temp2'], file_list['name'])
# Check that if we delete a file, it's no longer there
for box_file in file_list:
if box_file['name'] == 'temp1':
box.delete_file_by_id(box_file['id'])
break
file_list = box.list_files_by_id(folder_id=subfolder)
self.assertEqual(['temp2'], file_list['name'])
folder_list = box.list_folders_by_id(folder_id=subfolder)['name']
self.assertEqual(['temp_folder1', 'temp_folder2'], folder_list)
def get_survey_responses(self, survey_id, page=None):
"""
Get the responses for a given survey.
`Args:`
survey_id: string
The id of survey for which to retrieve the responses.
page : int
Retrieve a specific page of responses. If not given,
then all pages are retrieved.
`Returns:`
Table Class
"""
r = self._client.api.surveyresponse.list(survey_id, page)
logger.info(f"{survey_id}: {r['total_count']} responses.")
data = r['data']
if not page:
while r['page'] < r['total_pages']:
r = self._client.api.surveyresponse.list(survey_id, page=(r['page']+1))
data.extend(r['data'])
tbl = Table(data).add_column('survey_id', survey_id, index=1)
logger.info(f"Found #{tbl.num_rows} responses.")
return tbl
def get_unsubscribes(self,
campaign_id,
fields=None,
exclude_fields=None,
count=None,
offset=None):
"""
Get a table of unsubscribes associated with a campaign based on query
parameters. Note that argument descriptions here are sourced from
Mailchimp's official API documentation.
`Args:`
campaign_id: string
The unique ID of the campaign to fetch unsubscribes from.
fields: list of strings
A comma-separated list of fields to return. Reference
parameters of sub-objects with dot notation.
exclude_fields: list of strings
A comma-separated list of fields to exclude. Reference
parameters of sub-objects with dot notation.
count: int
The number of records to return. Default value is 10. Maximum
value is 1000.
offset: int
The number of records from a collection to skip. Iterating over
large collections with this parameter can be slow. Default
value is 0.
`Returns:`
Table Class
"""
params = {
'fields': fields,
'exclude_fields': exclude_fields,
'count': count,
'offset': offset
}
response = self.client.get_request(
f'reports/{campaign_id}/unsubscribed', params=params)
tbl = Table(response['unsubscribes'])
logger.info(f'Found {tbl.num_rows} unsubscribes for {campaign_id}.')
if tbl.num_rows > 0:
return tbl
else:
return Table()
def test_get_companies(self, m):
processed_companies = Table([
{'id': 35015567, 'name': 'Company One', 'assignee_id': None, 'contact_type_id': 547508,
'details': None, 'email_domain': '*****@*****.**', 'tags': [],
'interaction_count': 1, 'date_created': 1558441519, 'date_modified': 1558441535,
'address_city': 'CityA', 'address_country': None, 'address_postal_code': '12345',
'address_state': 'New York', 'address_street': None},
{'id': 35026533, 'name': 'Company Two', 'assignee_id': None, 'contact_type_id': 547508,
'details': None, 'email_domain': '*****@*****.**', 'tags': [],
'interaction_count': 1, 'date_created': 1558452953, 'date_modified': 1558452967,
'address_city': 'CityB', 'address_country': None, 'address_postal_code': '23451',
'address_state': 'New York', 'address_street': None},
{'id': 35014973, 'name': 'Company Three', 'assignee_id': None, 'contact_type_id': 547508,
'details': None, 'email_domain': None, 'tags': [], 'interaction_count': 1,
'date_created': 1558434147, 'date_modified': 1558458137, 'address_city': None,
'address_country': None, 'address_postal_code': '34512', 'address_state': 'Alabama',
'address_street': None},
{'id': 35029116, 'name': 'Company Four', 'assignee_id': None, 'contact_type_id': 547508,
'details': None, 'email_domain': '*****@*****.**', 'tags': [],
'interaction_count': 0, 'date_created': 1558461301, 'date_modified': 1558461301,
'address_city': 'CityD ', 'address_country': None, 'address_postal_code': '45123',
'address_state': 'California', 'address_street': None},
{'id': 35082308, 'name': 'Company Five', 'assignee_id': None, 'contact_type_id': 547508,
'details': None, 'email_domain': '*****@*****.**', 'tags': [],
'interaction_count': 1, 'date_created': 1558639445, 'date_modified': 1558639459,
'address_city': 'CityE', 'address_country': None, 'address_postal_code': '51234',
'address_state': 'Arizona', 'address_street': None}
])
processed_companies_phones = Table([
{'id': 35082308, 'phone_numbers_category': 'work',
'phone_numbers_number': '123-555-9876'}
])
m.post(self.cp.uri + '/companies/search',
json=self.paginate_callback,
headers={"filename": "companies_search.json"})
processed_blob = self.cp.get_companies()
blob_companies = [f for f in processed_blob if f['name'] == "companies"][0]['tbl']
blob_companies_phones = [f for f in processed_blob if f['name'] == "companies_phone_numbers"][0]['tbl']
assert_matching_tables(processed_companies, blob_companies)
assert_matching_tables(processed_companies_phones, blob_companies_phones)
def test_create_leads(self, m):
m.post(HUSTLE_URI + 'groups/cMCH0hxwGt/leads',
json=expected_json.leads_tbl_01)
tbl = Table([['phone_number', 'ln', 'first_name', 'address'],
['4435705355', 'Johnson', 'Lyndon', '123 Main Street'],
['4435705354', 'Richards', 'Ann', '124 Main Street']])
self.hustle.create_leads(tbl, group_id='cMCH0hxwGt')
def test_get_poll_location(self, m):
m.get(self.gc.uri + 'voterinfo', json=voterinfo_resp)
expected_tbl = Table(voterinfo_resp['pollingLocations'])
tbl = self.gc.get_polling_location(2000, '900 N Washtenaw, Chicago, IL 60622')
assert_matching_tables(tbl, expected_tbl)
def test_insert_records(self, m):
m.post(self.base_uri, json=insert_responses)
tbl = Table([{'Name': 'Another row!'}, {'Name': 'Another!'}])
resp = self.at.insert_records(tbl)
# Assert that row count is expected
self.assertEqual(len(resp), 2)
def default_table(self):
return Table([
{
'num': 1,
'ltr': 'a'
},
{
'num': 2,
'ltr': 'b'
},
])
