def get(self, donor_type):
"""Simple endpoint to retrieve rows from table given a set of query terms and paginate if requested.
:param donor_type: Either caged or queued.
:return: donors
"""
# Grab the donor type and make sure it is one of the allowed values.
if donor_type not in ['caged', 'queued']:
raise TypeError
query_terms = build_filter_from_request_args(request.args)
# Sanitize incoming partial UUID to only hex characters.
if 'searchable_id' in query_terms:
if 'eq' in query_terms['searchable_id']:
is_hex_string(query_terms['searchable_id']['eq'])
elif 'in' in query_terms['searchable_id'] or 'nin' in query_terms[
'searchable_id']:
for in_nin in query_terms['searchable_id']:
for searchable_id in query_terms['searchable_id'][in_nin]:
is_hex_string(searchable_id)
page_information = {}
sort_information = []
if query_terms:
page_information = {}
if 'paginate' in query_terms and query_terms['paginate']:
page_information = {
'page_number': query_terms['paginate']['page_number'],
'rows_per_page': query_terms['paginate']['rows_per_page']
}
del query_terms['paginate']
if 'sort' in query_terms and query_terms['sort']:
sort_information = query_terms['sort']
del query_terms['sort']
donors = get_donors(donor_type,
query_terms,
page_information=page_information,
sort_information=sort_information)
if page_information:
transformed_data = transform_data(
'donate/donors/{}'.format(donor_type), page_information,
donors, CagedDonorSchema)
response = jsonify(transformed_data['page'])
response.headers['Link'] = transformed_data['link-header']
response.status_code = 200
return response
if donor_type == 'caged':
schema = CagedDonorSchema(many=True)
else:
schema = QueuedDonorSchema(many=True)
result = schema.dump(donors).data
return result, 200
def generate_caged_donor(row):
"""Generate caged donor."""
user_email_address = row['from_email_address']
user_first_name, user_last_name = process_name(row['name'])
caged_donor_payload = {
'user_email_address': user_email_address,
'user_first_name': user_first_name,
'user_last_name': user_last_name,
}
# The following columns/fields might be optional from CSV files:
# - address_line_1 --> user_address
# - state_province_region_county_territory_prefecture_republic --> user_state ( 2 )
# - town_city --> user_city
# - zip_postal_code( may have 5 or 9 ) --> user_zipcode ( 5 )
user_address = row.get('address_line_1')
user_state = row.get(
'state_province_region_county_territory_prefecture_republic')
user_city = row.get('town_city')
user_zipcode = row.get('zip_postal_code')
if user_address and user_address != 'NA':
caged_donor_payload['user_address'] = user_address
if user_state and user_state != 'NA':
caged_donor_payload['user_state'] = user_state[:2]
if user_city and user_city != 'NA':
caged_donor_payload['user_city'] = user_city
if user_zipcode and user_zipcode != 'NA':
caged_donor_payload['user_zipcode'] = user_zipcode[:5]
caged_donor_schema = from_json(CagedDonorSchema(), caged_donor_payload)
caged_donor_model = caged_donor_schema.data
return caged_donor_model
def test_categorize_donor(self, ultsys_user_function): # pylint: disable=unused-argument
"""The caging process categorizes donors as new, exists, cage, or caged. This function tests this process."""
with self.app.app_context():
# Use an existing user so category is exists.
category = categorize_donor(
flatten_user_dict(get_exists_donor_dict()))
self.assertEqual(category[0], 'exists')
# Use a caged donor so category is caged.
caged_donor_dict = get_caged_donor_dict(
{'gift_searchable_id': uuid.uuid4()})
caged_donor = from_json(CagedDonorSchema(), caged_donor_dict)
database.session.add(caged_donor.data)
database.session.commit()
category = categorize_donor(caged_donor_dict)
self.assertEqual(category[0], 'caged')
# Use the existing user, change the name so category is cage.
cage_donor = copy.deepcopy(get_exists_donor_dict())
cage_donor['user_address']['user_first_name'] = 'Sherry'
category = categorize_donor(flatten_user_dict(cage_donor))
self.assertEqual(category[0], 'cage')
# Get a new donor so category is new.
category = categorize_donor(flatten_user_dict(
get_new_donor_dict()))
self.assertEqual(category[0], 'new')
def test_caged_donor_model(self):
"""A test to ensure that caged donors are saved correctly to the database."""
with self.app.app_context():
caged_donor_dict = get_caged_donor_dict({
'gift_id':
1,
'gift_searchable_id':
uuid.uuid4(),
'customer_id':
'customer_id'
})
caged_donor_model = from_json(CagedDonorSchema(),
caged_donor_dict,
create=True)
database.session.add(caged_donor_model.data)
database.session.commit()
caged_donor_query = CagedDonorModel.query\
.filter_by( customer_id=caged_donor_dict[ 'customer_id' ] ).one()
caged_donor_session = database.session.query( CagedDonorModel )\
.filter_by( customer_id=caged_donor_dict[ 'customer_id' ] ).one()
kwargs = {
'self': self,
'model_dict': caged_donor_dict,
'model': CagedDonorModel,
'model_data': caged_donor_model.data,
'model_query': caged_donor_query,
'model_session': caged_donor_session
}
ensure_query_session_aligned(kwargs)
def resolve_user(gift_with_customer_id):
"""If the user is caged or queued then return the model.
:param gift_with_customer_id: The gift.
:return: The caged/queued donor models ( at least one will be None ).
"""
donor_model = None
try:
if gift_with_customer_id.user_id == -1:
caged_donor_model = CagedDonorModel.query.filter_by(
gift_id=gift_with_customer_id.id).one()
caged_donor_dict = to_json(CagedDonorSchema(), caged_donor_model)
donor_model = from_json(CagedDonorSchema(), caged_donor_dict.data)
if gift_with_customer_id.user_id == -2:
queued_donor_model = QueuedDonorModel.query.filter_by(
gift_id=gift_with_customer_id.id).one()
queued_donor_dict = to_json(QueuedDonorSchema(),
queued_donor_model)
donor_model = from_json(QueuedDonorSchema(),
queued_donor_dict.data)
return donor_model
except: # noqa: E722
raise BraintreeWebhooksIDPathError(
type_id=gift_with_customer_id.customer_id)
def test_caged_donors(self):
"""Caged donor API ( methods = [ GET ] )."""
with self.app.app_context():
url = '/donation/donors/caged'
# Ensure a GET with no saved caged_donors returns 0.
response = self.test_client.get(url, headers=self.headers)
self.assertEqual(len(json.loads(response.data.decode('utf-8'))), 0)
# Create some caged_donors to retrieve.
total_caged_donors = 5
caged_donor_models = create_model_list(CagedDonorSchema(),
get_caged_donor_dict(),
total_caged_donors)
database.session.bulk_save_objects(caged_donor_models)
database.session.commit()
# Ensure GET returns all caged_donors.
response = self.test_client.get(url, headers=self.headers)
self.assertEqual(len(json.loads(response.data.decode('utf-8'))),
total_caged_donors)
def create_database_tables():
"""Function to create the DONATE database tables, specifically the CagedDonorModel and QueuedDonorModel with UUID.
All that is said here for the CagedDonorModel also holds for the QueuedDonorModel. The CagedDonorModel is built
using Marshmallow schema CagedDonorSchema, which deserializes a dictionary to the model. The searchable_id in the
donor_json is:
donor_json[ 'searchable_id' ] = uuid.uuid4()
This gets passed to the CagedDonorSchema where:
searchable_id = fields.UUID()
And so the validation step is passed.
MySql does not have a UUID type though and there we have ( CagedDonorModel ):
searchable_id = database.Column( database.BINARY( 16 ), nullable=False, default=uuid.uuid4().bytes )
The helper model class BinaryUUID in binary_uuid.py handles the serialization in and out.
"""
with app.app_context():
drop_all_and_create()
caged_donors = []
queued_donors = []
# Create 100 caged donors.
for i in range(0, 100):
donor_json = get_caged_donor_dict(
{'gift_searchable_id': uuid.uuid4()})
donor_json['gift_id'] = i + 1
donor_json['customer_id'] = str((i + 1) + 1000)
del donor_json['id']
caged_donor = CagedDonorSchema().load(donor_json).data
queued_donor = QueuedDonorSchema().load(donor_json).data
caged_donors.append(caged_donor)
queued_donors.append(queued_donor)
# Create the agents.
agent_jsons = [{
'name': 'Donate API',
'user_id': None,
'staff_id': None,
'type': 'Automated'
}, {
'name': 'Braintree',
'user_id': None,
'staff_id': None,
'type': 'Organization'
}, {
'name': 'PayPal',
'user_id': None,
'staff_id': None,
'type': 'Organization'
}, {
'name': 'Credit Card Issuer',
'user_id': None,
'staf_id': None,
'type': 'Organization'
}, {
'name': 'Unspecified NumbersUSA Staff',
'user_id': None,
'staff_id': None,
'type': 'Staff Member'
}, {
'name': 'Dan Marsh',
'user_id': 1234,
'staff_id': 4321,
'type': 'Staff Member'
}, {
'name': 'Joshua Turcotte',
'user_id': 7041,
'staff_id': 1407,
'type': 'Staff Member'
}, {
'name': 'Donate API',
'user_id': None,
'staff_id': None,
'type': 'Automated'
}]
agents = []
for agent_json in agent_jsons:
agent_model = AgentSchema().load(agent_json).data
agents.append(agent_model)
database.session.bulk_save_objects(caged_donors)
database.session.bulk_save_objects(queued_donors)
database.session.bulk_save_objects(agents)
database.session.commit()
def ultsys_user_create( payload ):
"""Function to create an Ultsys user from a caged donor.
payload = {
"caged_donor_id": 1234,
"ultsys_user_id": Null,
"user_first_name": "Ralph",
"user_last_name": "Kramden",
"user_address": "328 Chauncey St",
"user_state": "NY",
"user_city": "Bensonhurst",
"user_zipcode": "11214",
"user_email_address": "*****@*****.**",
"user_phone_number": "9172307441"
}
:param dict payload: The required payload
:return: Ultsys user.
"""
# Find the gift ID from the caged donor.
caged_donor_model = CagedDonorModel.query.filter_by( id=payload[ 'caged_donor_id' ] ).one_or_none()
if not caged_donor_model:
raise ModelCagedDonorNotFoundError
caged_donor_json = to_json( CagedDonorSchema(), caged_donor_model ).data
caged_donor_json.pop( 'id' )
# Retrieve the completed gift from the transactions to get the gross_gift_amount.
gift = GiftModel.query.filter_by( id=caged_donor_model.gift_id ).one_or_none()
if not gift:
raise ModelGiftNotFoundError
# The gift will have at least one transaction and may also have multiple transactions.
# Get the most recent transaction which holds the current gross gift amount.
transaction = TransactionModel.query.filter_by( gift_id=caged_donor_model.gift_id )\
.order_by( TransactionModel.date_in_utc )\
.first()
if not transaction:
raise ModelTransactionNotFoundError
gross_gift_amount = transaction.gross_gift_amount
# The updating of the caged donor fields does not expect the 2 IDs in the dictionary.
# Build the payload for the Drupal user.
caged_donor = {
'action': 'create',
'firstname': caged_donor_json[ 'user_first_name' ],
'lastname': caged_donor_json[ 'user_last_name' ],
'zip': caged_donor_json[ 'user_zipcode' ],
'city': caged_donor_json[ 'user_city' ],
'state': caged_donor_json[ 'user_state' ],
'email': caged_donor_json[ 'user_email_address' ],
'phone': str( caged_donor_json[ 'user_phone_number' ] )
}
drupal_user_uid = create_user( caged_donor )
# Use the Drupal ID to retrieve the Ultsys user.
ultsys_user = find_ultsys_user( get_ultsys_user_query( { 'drupal_user_uid': drupal_user_uid } ) )
ultsys_user_id = ultsys_user[ 0 ][ 'ID' ]
# Update the gift with the new Ultsys user ID and the Ultsys user with the gross gift amount.
gift.user_id = ultsys_user_id
update_ultsys_user( { 'id': ultsys_user_id }, gross_gift_amount )
database.session.delete( caged_donor_model )
database.session.commit()
return ultsys_user[ 0 ]
def test_admin_caged_donor(self, ultsys_user_function,
mock_caging_function): # pylint: disable=unused-argument
"""Test the administrative donation of a donor who has been previously caged."""
with self.app.app_context():
agent_dict = get_agent_dict(
{'name': 'Unspecified NumbersUSA Staff'})
agent_model = from_json(AgentSchema(), agent_dict, create=True)
database.session.add(agent_model.data)
database.session.flush()
database.session.commit()
agent_dict = get_agent_dict({'name': 'Fidelity Bank'})
bank_agent_model = from_json(AgentSchema(),
agent_dict,
create=True)
database.session.add(bank_agent_model.data)
database.session.flush()
database.session.commit()
# Create the agent who will be referenced in the payload.
agent_user_id = '3255162'
agent_dict = get_agent_dict({
'name': 'Aaron Peters',
'user_id': agent_user_id,
'type': 'Staff Member'
})
user_agent_model = from_json(AgentSchema(),
agent_dict,
create=True)
database.session.add(user_agent_model.data)
database.session.flush()
database.session.commit()
# Create a caged donor, which will be caged again.
caged_donor_dict = get_caged_donor_dict({
'gift_searchable_id':
uuid.uuid4(),
'customer_id':
self.parameters['customer_id']
})
caged_donor_model = from_json(CagedDonorSchema(),
caged_donor_dict,
create=True)
database.session.add(caged_donor_model.data)
database.session.flush()
database.session.commit()
# Call function to tests.
payload = get_donate_dict({
'gift': {
'method_used': self.parameters['method_used'],
'date_of_method_used':
self.parameters['date_of_method_used']
},
'user': {
'user_address': {
'user_first_name':
caged_donor_model.data.user_first_name,
'user_last_name':
caged_donor_model.data.user_last_name,
'user_address': caged_donor_model.data.user_address
}
}
})
payload['sourced_from_agent_user_id'] = agent_user_id
payload['transaction']['reference_number'] = self.parameters[
'reference_number']
payload['transaction']['type'] = self.parameters[
'transaction_type']
self.assertEqual(
post_donation(payload)['job_id'], 'redis-queue-job-id')
# The function should create one transaction, a gift, and a user.
transactions = TransactionModel.query.all()
gift = GiftModel.query.filter_by(id=1).one_or_none()
self.assertEqual(transactions[0].gift_id, gift.id)
self.assertEqual(transactions[0].enacted_by_agent_id, 3)
self.assertEqual(transactions[0].type,
self.parameters['transaction_type'])
self.assertEqual(transactions[0].status,
self.parameters['transaction_status'])
self.assertEqual(transactions[0].reference_number,
self.parameters['reference_number'])
self.assertEqual(transactions[0].gross_gift_amount,
self.parameters['gross_gift_amount'])
self.assertEqual(transactions[1].gift_id, gift.id)
self.assertEqual(transactions[1].enacted_by_agent_id, 2)
self.assertEqual(transactions[1].type,
self.parameters['second_transaction_type'])
self.assertEqual(transactions[1].status,
self.parameters['transaction_status'])
self.assertEqual(transactions[1].reference_number,
self.parameters['bank_deposit_number'])
self.assertEqual(transactions[1].gross_gift_amount,
self.parameters['gross_gift_amount'])
caged_donor = CagedDonorModel.query.filter_by(id=2).one()
self.assertEqual(caged_donor.gift_searchable_id,
gift.searchable_id)
self.assertEqual(caged_donor.user_first_name,
caged_donor_dict['user_first_name'])
self.assertEqual(caged_donor.user_last_name,
caged_donor_dict['user_last_name'])
def redis_queue_caging(user, transactions, app_config_name):
"""A function for queueing a caging operation and updating models with caged donor or Ultsys user.
Here is what the user looks like:
user: {
"id": null,
"user_address": {
"user_first_name": "Aaron",
"user_last_name": "Peters",
"user_zipcode": "22202",
"user_address": "1400 Crystal City Dr",
"user_city": "Arlington",
"user_state": "VA",
"user_email_address": "*****@*****.**",
"user_phone_number": "7038168820"
},
"billing_address": {
"billing_first_name": "Aaron",
"billing_last_name": "Peters",
"billing_zipcode": "22202",
"billing_address": "1400 Crystal City Dr",
"billing_city": "Arlington",
"billing_state": "VA",
"billing_email_address": "*****@*****.**",
"billing_phone_number": "7038168820"
}
'payment_method_nonce': 'tokencc_bc_string',
'category': 'queued',
'customer_id': '476914249',
'gift_id': 3,
'searchable_id': UUID( 'd1aeac47-17ce-46ca-9d45-3f540f7a1d85' ),
'queued_donor_id': 3
}
:param user: The user dictionary
:param transactions: The list of transactions. If this is a Braintree sale, for example, there will be one
transaction in the list. On the other hand if this is an administrative sale where the method used is
a check or money order there will be 2 transactions.
:param app_config_name: The configuration ( PROD, DEV, TEST ) that the app is running.
:return:
"""
# This is getting pushed onto the queue outside an application context: create it here.
from application.app import create_app # pylint: disable=cyclic-import
app = create_app(app_config_name) # pylint: disable=C0103
with app.app_context():
# Categorize the user: new, cage, caged, exists.
# The variable category is a tuple:
# category[ 0 ]: the category of the donor.
# category[ 1 ]: if category is 'exists' this will hold an ID like [ 1234 ].
# If category[ 0 ] is 'cage' it might be donor matched 2 or more users: category[ 1 ] = [ 1234, 5678 ].
# If category[ 0 ] is 'exists' then len( category[ 1 ] ) == 1.
# This is a fix to a mismatch between what the back-end expects and what the front-end is passing.
# The fix is used at the donate controller to correct the mismatch. It is also used at the reprocess
# queued donor to create the user dictionary expected for caging. Finally, here we use it because
# A donor who never got into the queued donor table will be re-queued and can be reprocessed at this point.
user = validate_user_payload(user)
donor_dict = copy.deepcopy(user)
donor_dict = flatten_user_dict(donor_dict)
category = categorize_donor(donor_dict)
logging.debug('***** category: %s', category)
gross_gift_amount = str(transactions[0]['gross_gift_amount'])
if category[0] == 'exists':
ultsys_user_id = category[1][0]
user['id'] = ultsys_user_id
build_model_exists(user, gross_gift_amount)
gift_id = user['gift_id']
gift_model = GiftModel.query.filter_by(id=gift_id).one_or_none()
gift_model.user_id = ultsys_user_id
QueuedDonorModel.query.filter_by(
id=user['queued_donor_id']).delete()
elif category[0] == 'cage' or category[0] == 'caged':
gift_id = user['gift_id']
gift_model = GiftModel.query.filter_by(id=gift_id).one_or_none()
gift_model.user_id = -1
caged_donor_dict = user['user_address']
caged_donor_dict['gift_searchable_id'] = gift_model.searchable_id
caged_donor_dict['campaign_id'] = user['campaign_id']
caged_donor_dict['customer_id'] = user['customer_id']
caged_donor_model = from_json(CagedDonorSchema(),
caged_donor_dict,
create=True)
caged_donor_model.data.gift_id = gift_id
database.session.add(caged_donor_model.data)
QueuedDonorModel.query.filter_by(
id=user['queued_donor_id']).delete()
elif category[0] == 'new':
ultsys_user_id = build_model_new(user, gross_gift_amount)
user['id'] = ultsys_user_id
gift_id = user['gift_id']
gift_model = GiftModel.query.filter_by(id=gift_id).one_or_none()
gift_model.user_id = ultsys_user_id
QueuedDonorModel.query.filter_by(
id=user['queued_donor_id']).delete()
try:
database.session.commit()
except SQLAlchemyError as error:
database.session.rollback()
raise error
def test_donation_caged_donor(self, ultsys_user_function,
mock_caging_function): # pylint: disable=unused-argument
"""Test Braintree sale for a donor that has already been caged.
:param ultsys_user_function: Argument for mocked function.
:return:
"""
with self.app.app_context():
# Create agent ( Braintree ) for the online donation.
agent_dict = get_agent_dict()
agent_model = from_json(AgentSchema(), agent_dict, create=True)
database.session.add(agent_model.data)
# Flush to get agent ID and set database.
database.session.flush()
agent_id = agent_model.data.id
# Create a caged donor, which will be caged again.
caged_donor_dict = get_caged_donor_dict({
'gift_searchable_id':
uuid.uuid4(),
'customer_id':
self.parameters['customer_id']
})
caged_donor_model = from_json(CagedDonorSchema(), caged_donor_dict)
database.session.add(caged_donor_model.data)
database.session.commit()
# Call function to tests.
payload = get_donate_dict({
'user': {
'user_address': {
'user_first_name': 'Alice',
'user_email_address': '*****@*****.**'
},
'billing_address': {}
},
'recurring_subscription': False
})
result = post_donation(payload)
self.assertEqual(result['job_id'], 'redis-queue-job-id')
# The function should create one transaction, a gift, and an additional caged donor.
# Additional caged donor is the same as first, but attached to a separate gift.
transaction = TransactionModel.query.filter_by(id=1).one_or_none()
gift = GiftModel.query.filter_by(id=1).one_or_none()
caged_donor = CagedDonorModel.query.filter_by(id=2).one_or_none()
self.assertEqual(transaction.gift_id, gift.id)
self.assertEqual(transaction.enacted_by_agent_id, agent_id)
self.assertEqual(transaction.type,
self.parameters['transaction_type'])
self.assertEqual(transaction.status,
self.parameters['transaction_status'])
self.assertEqual(transaction.reference_number,
self.parameters['reference_number'])
self.assertEqual(transaction.gross_gift_amount,
self.parameters['gross_gift_amount'])
self.assertEqual(transaction.fee, self.parameters['fee'])
self.assertEqual(gift.user_id, -1)
self.assertEqual(gift.method_used_id, self.method_used_id)
self.assertEqual(gift.given_to, self.parameters['given_to'])
self.assertEqual(caged_donor.gift_searchable_id,
gift.searchable_id)
self.assertEqual(caged_donor.customer_id,
self.parameters['customer_id'])
self.assertEqual(caged_donor.user_first_name,
caged_donor_dict['user_first_name'])
self.assertEqual(caged_donor.user_last_name,
caged_donor_dict['user_last_name'])