def test_get_bulk_import_job_results(self, m):
results_tbl = Table([['BulkUploadDataID', 'ULFileID', 'PrimaryKey',
'PrimaryKeyType', 'MailingAddress_3581'],
['1', '1983', '101596008', 'VanID', 'Processed']])
bulk_import_job = {'id': 92,
'status': 'Completed',
'resourceType': 'Contacts',
'webhookUrl': None,
'resultFileSizeLimitKb': 5000,
'errors': [],
'resultFiles': [{
'url': Table.to_csv(results_tbl),
'dateExpired': '2020-09-04T22:07:04.0770295-04:00'
}]
}
m.get(self.van.connection.uri +
'bulkImportJobs/53407', json=bulk_import_job)
assert_matching_tables(
self.van.get_bulk_import_job_results(53407), results_tbl)
def test_table_sync_full_read_chunk(self):
dbsync = DBSync(self.fake_source,
self.fake_destination,
read_chunk_size=2)
source_data = Table([
{
'id': 1,
'value': 11
},
{
'id': 2,
'value': 121142
},
{
'id': 3,
'value': 111
},
{
'id': 4,
'value': 12211
},
{
'id': 5,
'value': 1231
},
])
self.fake_source.setup_table('source', source_data)
dbsync.table_sync_full('source', 'destination')
destination = self.fake_destination.table('destination')
# Make sure the data came through
assert_matching_tables(source_data, destination.data)
# Make sure copy was called the expected number of times
# read chunks of 2, 5 rows to write.. should be 3 copy calls
self.assertEqual(len(self.fake_destination.copy_call_args), 3,
self.fake_destination.copy_call_args)
def setUp(self):
self.temp_schema = TEMP_SCHEMA
self.pg = Postgres()
self.tbl = Table([['ID', 'Name'], [1, 'Jim'], [2, 'John'],
[3, 'Sarah']])
# Create a schema, create a table, create a view
setup_sql = f"""
drop schema if exists {self.temp_schema} cascade;
create schema {self.temp_schema};
"""
other_sql = f"""
create table {self.temp_schema}.test (id smallint,name varchar(5));
create view {self.temp_schema}.test_view as (select * from {self.temp_schema}.test);
""" # noqa: E501
self.pg.query(setup_sql)
self.pg.query(other_sql)
def test_get_standard_object(self, m):
processed_people_emails = Table([
{'id': 78757050, 'emails_category': 'work', 'emails_email': '*****@*****.**'},
{'id': 78477076, 'emails_category': 'work', 'emails_email': '*****@*****.**'},
{'id': 78839154, 'emails_category': 'work',
'emails_email': '*****@*****.**'}
])
m.post(
self.cp.uri + '/people/search',
json=self.paginate_callback,
headers={"filename": "people_search.txt"})
# Object-specific get_ functions are just wrappers for get_standard_object()
# So the following line is the only difference from test_get_people()
processed_blob = self.cp.get_standard_object("people")
blob_people = [f for f in processed_blob if f['name'] == "people"][0]['tbl']
blob_people_emails = [f for f in processed_blob if f['name'] == "people_emails"][0]['tbl']
assert_matching_tables(self.processed_people, blob_people)
assert_matching_tables(processed_people_emails, blob_people_emails)
def test_group_by_and_count():
input = [
{
"outreach_id": 10591961,
"created_date": "2021-04-22",
"target_names": "Jack Reed, Sheldon Whitehouse, David Cicilline",
"name": "Matthew Mellea",
"phone": ["(650) 946-7412"],
},
{
"outreach_id": 10592577,
"created_date": "2021-04-22",
"target_names": "Jack Reed, Sheldon Whitehouse, David Cicilline",
"name": "Matthew Mellea",
"phone": ["(650) 946-7412"],
},
{
"outreach_id": 10613970,
"created_date": "2021-04-23",
"target_names":
"Kirsten E. Gillibrand, Charles E. Schumer, Yvette D. Clarke",
"name": "Aracely Jimenez-Hudis",
"phone": ["(347) 204-7223"],
},
]
output = Table([
{
"created_date": "2021-04-22",
"num_calls": 2
},
{
"created_date": "2021-04-23",
"num_calls": 1
},
])
assert outreach_analytics.group_by_and_count(input,
"created_date") == output
def process_custom_fields(self, json_blob):
# Internal method to convert custom fields responses into a list of Parsons tables
# Original table & columns
custom_fields = Table(json_blob)
# Available On
available_on = custom_fields.long_table(['id'], 'available_on')
# Options
options = custom_fields.long_table(['id', 'name'], 'options')
return [{
'name': 'custom_fields',
'tbl': custom_fields
}, {
'name': 'custom_fields_available',
'tbl': available_on
}, {
'name': 'custom_fields_options',
'tbl': options
}]
def get_activity_types(self):
"""
Get activity types
`Args:`
`filters: dict`
Optional; pass additional parameters to filter the records returned.
See `Copper documentation <https://developer.copper.com/?version=latest#6bd339f1-f0de-48b4-8c34-5a5e245e036f>`_ for choices
`Returns:`
List of dicts of Parsons Tables:
* activitiy_types
""" # noqa: E501,E261
logger.info("Retrieving activity types.")
response = self.paginate_request('/activity_types/', req_type='GET')
orig_table = Table(response)
at_user = orig_table.long_table([], 'user', prepend=False)
at_sys = orig_table.long_table([], 'system', prepend=False)
Table.concat(at_sys, at_user)
return [{'name': 'activity_types', 'tbl': at_sys}]
def get_leaderboard(self,
start_date=None,
end_date=None,
list_ids=None,
account_ids=None):
"""
Return advocates (person records).
`Args:`
start_date: str
Filter to the earliest date at which a post could be posted.
The time is formatted as UTC (e.g. ``yyyy-mm-ddThh:mm:ss``).
end_date: str
Filter to the latest date at which a post could be posted.
The time is formatted as UTC (e.g. ``yyyy-mm-ddThh:mm:ss``).
list_ids: list
Filter to the ids of lists or saved searches to retrieve.
account_ids: list
A list of CrowdTangle accountIds to retrieve leaderboard data for.
This and ``list_id`` are mutually exclusive; if both are sent, the
``account_ids`` value will be used.
`Returns:`
Parsons Table
See :ref:`parsons-table` for output options.
"""
args = {
'startDate': start_date,
'endDate': end_date,
'listIds': self.list_to_string(list_ids),
'accountIds': self.list_to_string(account_ids)
}
pt = Table(self.base_request('leaderboard', args=args))
logger.info(f'Retrieved {pt.num_rows} records from the leaderbooard.')
self.unpack(pt)
return pt
def get_advocates(self, state=None, campaign_id=None, updated_since=None):
"""
Return advocates (person records).
`Args:`
state: str
Filter by US postal abbreviation for a state
or territory e.g., "CA" "NY" or "DC"
campaign_id: int
Filter to specific campaign
updated_since: str
Fetch all advocates updated since UTC date time provided
using (ex. '2014-01-05 23:59:43')
`Returns:`
A dict of parsons tables:
* emails
* phones
* memberships
* tags
* ids
* fields
* advocates
"""
url = self.uri + 'advocates'
args = {
'state': state,
'campaignid': campaign_id,
'updatedSince': updated_since
}
logger.info('Retrieving advocates...')
json = self._paginate_request(url, args=args)
return self._advocates_tables(Table(json))
def test_transaction_search(self, m):
m.post(
'https://api.braintreegateway.com:443'
'/merchants/abcd1234abcd1234/transactions/advanced_search_ids',
text="""
<search-results>
<page-size type="integer">50</page-size>
<ids type="array"><item>1234abcd</item> <item>0987asdf</item> </ids>
</search-results>
""")
table = self.braintree.get_transactions(
disbursement_start_date="2020-01-01",
disbursement_end_date="2020-01-02",
just_ids=True)
assert_matching_tables(table,
Table([['id'], ['1234abcd'], ['0987asdf']]))
m.post(
'https://api.braintreegateway.com:443'
'/merchants/abcd1234abcd1234/transactions/advanced_search',
text=open(f'{_dir}/test_data/transaction_example.xml').read())
full_table = self.braintree.get_transactions(
disbursement_start_date="2020-01-01",
disbursement_end_date="2020-01-02",
table_of_ids=table)
self.assertEqual(len(table.table), 3)
self.assertEqual(len(full_table.table), 3)
self.assertEqual(table[0]['id'], '1234abcd')
self.assertEqual(table[1]['id'], '0987asdf')
self.assertEqual(len(table[0].keys()), 1)
self.assertEqual(len(full_table[0].keys()), 64)
self.assertEqual(full_table[0]['disbursement_date'],
datetime.date(2019, 12, 30))
self.assertEqual(full_table[0]['credit_card_bin'], '789234')
self.assertEqual(full_table[0]['disbursement_success'], True)
self.assertEqual(full_table[0]['amount'], decimal.Decimal('150.00'))
def test_get_contact_types(self, m):
processed_ct = Table([
{'id': 501947, 'name': 'Potential Customer'},
{'id': 501948, 'name': 'Current Customer'},
{'id': 501949, 'name': 'Uncategorized'},
{'id': 501950, 'name': 'Group Leader'},
{'id': 540331, 'name': 'Partner'},
{'id': 540333, 'name': 'Funder'},
{'id': 540334, 'name': 'Potential Funder'},
{'id': 540335, 'name': 'Other'},
{'id': 547508, 'name': 'Local Group'},
{'id': 575833, 'name': 'Group Member'},
{'id': 744795, 'name': 'Hill Contact'},
{'id': 967249, 'name': 'State Leg Contact'}
])
m.get(
self.cp.uri + '/contact_types/',
json=self.paginate_callback,
headers={"filename": "contact_types_list.json"})
processed_blob = self.cp.get_contact_types()
assert_matching_tables(processed_ct, processed_blob)
def test_convert_columns_to_str(self):
# Test that all columns are string
mixed_raw = [{
'col1': 1,
'col2': 2,
'col3': 3
}, {
'col1': 'one',
'col2': 2,
'col3': [3, 'three', 3.0]
}, {
'col1': {
'one': 1,
"two": 2.0
},
'col2': None,
"col3": 'three'
}]
tbl = Table(mixed_raw)
tbl.convert_columns_to_str()
cols = tbl.get_columns_type_stats()
type_set = {i for x in cols for i in x['type']}
self.assertTrue('str' in type_set and len(type_set) == 1)
def test_table_split(self):
test1 = Table([('x', 'y', 'z'), ('a', 'b', ''), ('1', '', '3'),
('4', '', '6')])
tables = self.actionkit._split_tables_no_empties(test1, True, [])
self.assertEqual(len(tables), 2)
assert_matching_tables(tables[0], Table([('x', 'y'), ('a', 'b')]))
assert_matching_tables(tables[1],
Table([('x', 'z'), ('1', '3'), ('4', '6')]))
test2 = Table([('x', 'y', 'z'), ('a', 'b', 'c'), ('1', '2', '3'),
('4', '5', '6')])
tables2 = self.actionkit._split_tables_no_empties(test2, True, [])
self.assertEqual(len(tables2), 1)
assert_matching_tables(tables2[0], test2)
test3 = Table([('x', 'y', 'z'), ('a', 'b', ''), ('1', '2', '3'),
('4', '5', '6')])
tables3 = self.actionkit._split_tables_no_empties(test3, False, ['z'])
self.assertEqual(len(tables3), 2)
assert_matching_tables(tables3[0], Table([('x', 'y'), ('a', 'b')]))
assert_matching_tables(
tables3[1],
Table([('x', 'y', 'z'), ('1', '2', '3'), ('4', '5', '6')]))
def _request(self,
url,
req_type='GET',
post_data=None,
args=None,
limit=None):
# Make sure to have a current token before we make another request
now = datetime.now(timezone.utc)
if now > self.session_exp:
self._get_session_token()
# Based on PDI docs
# https://api.bluevote.com/docs/index
LIMIT_MAX = 2000
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {self.session_token}",
}
request_fn = {
"GET": requests.get,
"POST": requests.post,
"PUT": requests.put,
"DELETE": requests.delete,
}
if limit and limit <= LIMIT_MAX:
args = args or {}
args["limit"] = limit
args = self._clean_dict(args) if args else args
post_data = self._clean_dict(post_data) if post_data else post_data
res = request_fn[req_type](url,
headers=headers,
json=post_data,
params=args)
logger.debug(f"{res.url} - {res.status_code}")
logger.debug(res.request.body)
res.raise_for_status()
if not res.text:
return None
logger.debug(res.text)
try:
res_json = res.json()
except JSONDecodeError:
res_json = None
if "data" not in res_json:
return res_json
total_count = (0 if "totalCount" not in res_json else
res_json["totalCount"])
data = res_json["data"]
if not limit:
# We don't have a limit, so let's get everything
# Start a page 2 since we already go page 1
cursor = 2
while len(data) < total_count:
args = args or {}
args["cursor"] = cursor
args["limit"] = LIMIT_MAX
res = request_fn[req_type](url,
headers=headers,
json=post_data,
params=args)
data.extend(res.json()["data"])
cursor += 1
return Table(data)
else:
total_need = min(limit, total_count)
cursor = 2
while len(data) < total_need:
args = args or {}
args["cursor"] = cursor
args["limit"] = min(LIMIT_MAX, total_need - len(data))
res = request_fn[req_type](url,
headers=headers,
json=post_data,
params=args)
data.extend(res.json()["data"])
cursor += 1
return Table(data)
def get_posts(self,
start_date=None,
end_date=None,
language=None,
list_ids=None,
min_interations=None,
search_term=None,
types=None):
"""
Return advocates (person records).
`Args:`
start_date: str
Filter to the earliest date at which a post could be posted.
The time is formatted as UTC (e.g. ``yyyy-mm-ddThh:mm:ss``).
end_date: str
Filter to the latest date at which a post could be posted.
The time is formatted as UTC (e.g. ``yyyy-mm-ddThh:mm:ss``).
language: str
Filter to 2-character Locale code. Some languages require more
than two characters: Chinese (Simplified) is zh-CN and
Chinese (Traditional) is zh-TW.
list_ids: list
Filter to the ids of lists or saved searches to retrieve.
min_interactions: int
Filter to posts with total interactions above this threshold.
search_team: str
Returns only posts that match this search term. For multiple terms, separate
with commas for OR, use quotes for phrases.
types: list
Filter to post types including:
* ``episode``
* ``extra_clip``
* ``link``
* ``live_video``
* ``live_video_complete``
* ``live_video_scheduled``
* ``native_video``
* ``photo``
* ``status``
* ``trailer``
* ``tweet``
* ``vimeo``
* ``vine``
* ``youtube``
If you want all live videos (whether currently or formerly live),
pass include both ``live_video`` and ``live_video_complete``
parameters.
The ``video`` type does not mean all videos, it refers to videos
that are not ``native_video``, ``youtube`` or ``vine``.
`Returns:`
Parsons Table
See :ref:`parsons-table` for output options.
"""
args = {
'startDate': start_date,
'endDate': end_date,
'language': language,
'listIds': self.list_to_string(list_ids),
'minInteractions': min_interations,
'searchTerm': search_term,
'types': types
}
logger.info("Retrieving posts.")
pt = Table(self.base_request('posts', args=args))
logger.info(f'Retrieved {pt.num_rows} posts.')
self.unpack(pt)
return pt
def test_get_people(self, m):
m.get(f"{self.api_url}/people?page=1&per_page=25", text=json.dumps(self.fake_people_list_1))
m.get(f"{self.api_url}/people?page=2&per_page=25", text=json.dumps(self.fake_people_list_2))
m.get(f"{self.api_url}/people?page=3&per_page=25",
text=json.dumps({'_embedded': {"osdi:people": []}}))
assert_matching_tables(self.an.get_people(), Table(self.fake_people_list))
loaded = [['id', 'voterbase_id', 'date_updated']] # column names for log table
source_table = 'schema.table' # this is the table with the information I'm pushing to ActionKit
log_table = 'schema.table' # this is where we will log every user id that gets marked with a voterbase_id
logger.info("Running query to get matches...")
query = '''
select distinct id, voterbase_id
from {source_table}
left join {log_table} using (id, voterbase_id)
where voterbase_id is not null and date_updated is null
'''
source_data = rs.query(query)
if source_data.num_rows > 0:
logger.info(f"Will be updating voterbase_id for {source_data.num_rows}...")
for row in source_data:
user = ak.get_user(user_id=row['id'])
user_dict = {"fields": {"vb_voterbase_id": row['voterbase_id']}}
update_user = ak.update_user(user_id=row['id'], **user_dict)
user = ak.get_user(user_id=row['id'])
if user['fields']['vb_voterbase_id'] == row['voterbase_id']:
loaded.append([row['id'], row['voterbase_id'], timestamp])
logger.info("Done with loop! Loading into log table...")
Table(loaded).to_redshift(log_table, if_exists='append')
else:
logger.info(f"No one to update today...")
def test_get_users_by_email(self, m):
email = '*****@*****.**'
mock_users = [{'email': '*****@*****.**', 'id': 2}]
m.get(f'{self.auth0.base_url}/api/v2/users-by-email?email={email}', json=mock_users)
assert_matching_tables(self.auth0.get_users_by_email(email), Table(mock_users), True)
def test_get_organizations(self, m):
m.get(HUSTLE_URI + 'organizations', json=expected_json.organizations)
orgs = self.hustle.get_organizations()
assert_matching_tables(orgs,
Table(expected_json.organizations['items']))
def test_get_tags(self, m):
m.get(HUSTLE_URI + 'organizations/LePEoKzD3/tags',
json=expected_json.tags)
tags = self.hustle.get_tags(organization_id='LePEoKzD3')
assert_matching_tables(tags, Table(expected_json.tags['items']))
def test_get_tags(self, m):
m.get(f"{self.api_url}/tags?page=1&per_page=25", text=json.dumps(self.fake_tag_list))
m.get(f"{self.api_url}/tags?page=2&per_page=25",
text=json.dumps({'_embedded': {"osdi:tags": []}}))
assert_matching_tables(self.an.get_tags(),
Table(self.fake_tag_list['_embedded']['osdi:tags']))
def test_from_empty_list(self):
# Just ensure this doesn't throw an error
Table()
Table([])
Table([[]])
for contact_vanid in contact_vanids:
try:
person = ea.get_person(
contact_vanid,
id_type="vanid",
expand_fields=["reported_demographics", "custom_fields"]
)
transformed_person = transform_person_for_redshift(person)
extra_fields.append(transformed_person)
except HTTPError as e:
print(e)
error = {
"vanid": contact_vanid,
"error": str(e)[:999]
}
errors.append(error)
logger.info(f'Found {len(extra_fields)} new contacts to add to contacts_extra_fields')
logger.info(f'Identified {len(errors)} errors. Appending to errors table.')
# convert to Parsons table
tbl = Table(extra_fields)
errors_tbl = Table(errors)
tbl.to_csv('extra_fields_test.csv')
errors_tbl.to_csv('extra_fields_errors.csv')
# copy Table into Redshift, append new rows
rs.copy(tbl, 'sunrise.contacts_extra_fields' ,if_exists='append', distkey='vanid', sortkey = None, alter_table = True)
rs.copy(errors_tbl, 'sunrise.get_extra_fields_errors' ,if_exists='append', distkey='vanid', sortkey = None, alter_table = True)
def test_to_dicts(self):
self.assertEqual(self.lst, Table(self.lst).to_dicts())
self.assertEqual(self.lst_dicts, self.tbl.to_dicts())
def test_get_agents(self, m):
m.get(HUSTLE_URI + 'groups/Qqp6o90SiE/agents',
json=expected_json.agents)
agents = self.hustle.get_agents(group_id='Qqp6o90SiE')
assert_matching_tables(agents, Table(expected_json.agents['items']))
def test_materialize_to_file(self):
# Simple test that materializing doesn't change the table
tbl_materialized = Table(self.lst_dicts)
_ = tbl_materialized.materialize_to_file()
assert_matching_tables(self.tbl, tbl_materialized)
def test_get_groups(self, m):
m.get(HUSTLE_URI + 'organizations/LePEoKzD3/groups',
json=expected_json.groups)
groups = self.hustle.get_groups('LePEoKzD3')
assert_matching_tables(groups, Table(expected_json.groups['items']))
def test_bool(self):
empty = Table()
not_empty = Table([{'one': 1, 'two': 2}])
self.assertEqual(not empty, True)
self.assertEqual(not not_empty, False)
class TestTargets(unittest.TestCase):
mock_data = mock_data = (
'12827,Volunteer Recruitment Tiers,Tier,109957740\n'
'12827,Volunteer Recruitment Tiers,Tier,109957754')
mock_result = Table([
('12827', 'Volunteer Recruitment Tiers', 'Tier', '109957740'),
('12827', 'Volunteer Recruitment Tiers', 'Tier', '109957754')])
def setUp(self):
self.van = VAN(os.environ['VAN_API_KEY'], db="MyVoters", raise_for_status=False)
def tearDown(self):
pass
@requests_mock.Mocker()
def test_get_targets(self, m):
# Create response
json = {u'count': 2, u'items':
[{u'targetId': 12827,
u'type': u'TEST CODE',
u'name': u'TEST CODE',
u'description': None,
u'points': 20,
u'areSubgroupsSticky': False,
u'status': u'Active',
u'subgroups': None,
u'markedSubgroup': None}],
u'nextPageLink': None}
m.get(self.van.connection.uri + 'targets', json=json)
# Expected Structure
expected = ['targetId', 'type', 'name', 'description',
'points', 'areSubgroupsSticky', 'status', 'subgroups', 'markedSubgroup']
# Assert response is expected structure
self.assertTrue(validate_list(expected, self.van.get_targets()))
# To Do: Test what happens when it doesn't find any targets
@requests_mock.Mocker()
def test_get_target(self, m):
# Create response
json = {u'targetId': 15723,
u'name': u'Mail_VR_Chase',
u'type': u'Dynamic',
u'description': None,
u'points': 15,
u'areSubgroupsSticky': False,
u'status': u'Active',
u'subgroups':
[{u'targetId': 12827,
u'fullName': u'April_VR_Chase Calls',
u'name': u'April_Chase_20',
u'subgroupId': 46803,
u'isAssociatedWithBadges': True}],
u'markedSubgroup': None}
m.get(self.van.connection.uri + 'targets/15723', json=json)
self.assertEqual(json, self.van.get_target(15723))
@requests_mock.Mocker()
def test_create_target_export(self, m):
export_job_id = '{"exportJobId": "455961790"}'
target_id = 12827
m.post(self.van.connection.uri + 'targetExportJobs', json=export_job_id, status_code=204)
# Test that it doesn't throw and error
r = self.van.create_target_export(target_id, webhook_url=None)
self.assertEqual(r, export_job_id)
@requests_mock.Mocker()
def test_get_target_export(self, m):
export_job_id = 455961790
json = [{
"targetId": 12827,
"file": {
"downloadUrl": (
"https://ngpvan.blob.core.windows.net/"
"target-export-files/TargetExport_455961790.csv"),
"dateExpired": "null",
"recordCount": 1016883},
"webhookUrl": "null",
"exportJobId": 455961790,
"jobStatus": "Complete"}]
download_url = (
'https://ngpvan.blob.core.windows.net/target-export-files/TargetExport_455961790.csv')
m.post(self.van.connection.uri + 'targetExportJobs', json=export_job_id, status_code=204)
m.get(self.van.connection.uri + 'targetExportJobs/455961790', json=json)
m.get(download_url, text=self.mock_data)
assert_matching_tables(self.van.get_target_export(export_job_id),
self.mock_result)
import unittest
import os
from parsons import FacebookAds, Table
users_table = Table([
{
"first": "Bob",
"middle": "J",
"last": "Smith",
"phone": "1234567890",
"cell": None,
"vb_voterbase_dob": "19820413"
},
{
"first": "Sue",
"middle": "Lucy",
"last": "Doe",
"phone": None,
"cell": "2345678901",
"vb_voterbase_dob": None
},
])
@unittest.skipIf(not os.environ.get('LIVE_TEST'),
'Skipping because not running live test')
class TestFacebookAdsIntegration(unittest.TestCase):
def setUp(self):
self.fb_ads = FacebookAds()