def populate_db():
with login_database.login_mongodb_cloud() as client:
log.info('Step 1: We are going to use a database called donors')
log.info('But if it doesnt exist mongodb creates it')
db = client['donors']
log.info('And in that database use a collection called donor')
log.info('If it doesnt exist mongodb creates it')
donor = db['donor']
log.info('\nStep 2: Now we add data from the dictionary above')
donor_items = donor_data.get_donor_data()
donor.insert_many(donor_items)
def action(switch_dict):
"""Takes in a user input as a parameter, enters a donation, prints a report,
prints list, exit, prompts again if the input is bad
If the user types exit it'll go back to the main prompt"""
while True:
user_input = main_prompt()
try:
switch_dict.get(user_input)()
except (TypeError, ValueError):
print("Invalid input, {} please try again.".format(user_input))
# Python program to use main for function call
if __name__ == "__main__":
with login_database.login_mongodb_cloud() as client:
donor_db = donor_data.get_donor_data()
db = client['dev']
donor = db['donor']
donor.insert_many(donor_db)
switch_dict = {
'list': print_names,
'1': add_donation,
'2': delete_donor,
'3': update_donor,
'4': print_report,
'5': send_letters,
'0': clear_db
}
action(switch_dict)
class Main:
log.info('Initialize Class "Main".')
log = utilities.configure_logger('default', '../logs/nosql_dev.log')
log.info(
'\n\nRunning mongodb_mailroom_script for PART 1 assignment from learnnosql.py'
)
log.info("Initialize the Database.")
with login_database.login_mongodb_cloud() as client:
log.info('\n\nStep 1: We are going to use a database called donors')
log.info('\n\nBut if it doesnt exist mongodb creates it')
db = client['donors']
log.info('\n\nAnd in that database use a collection called Donor')
log.info('\n\nIf it doesnt exist mongodb creates it')
donor = db['donor']
log.info('Insert an item into the document')
new_donor = {
'name': {
'first_name': 'Ben',
'last_name': 'Master'
},
'donation': 1000.11
}
donor.insert_one(new_donor)
log.info('\n\nStep 2: Now we add data from the dictionary above')
donor_items = get_donor_data()
donor.insert_many(donor_items)
log.info("Show the collections in the DB.")
print(db.list_collection_names())
def __init__(self):
self.menu = {
1: 'Send a Thank You (and add/update donor)',
2: 'Create a Report',
3: 'Send letters to everyone',
4: 'Delete a Donor',
5: 'Quit'
}
def main_menu(self):
print('\n', 'Please select a number from the following choices:\n')
return {(print(str(k) + ':', v)) for k, v in self.menu.items()}
def selection(self):
while True:
input1 = input("Selection: ")
try:
if int(input1) in range(1, 5):
if int(input1) == 1:
print(
'\nType "list" to show names or hit the "Return/Enter" key to add names and/or donations.'
)
input2 = input('-> ')
if input2 == 'list':
donor_names = set()
for d in self.donor.find():
name = ' '.join([
d['name']['first_name'],
d['name']['last_name']
])
donor_names.add(name)
for d in donor_names:
print(d)
Main()
self.main_menu()
self.selection()
else:
first_name = input('First Name: ')
last_name = input('Last Name: ')
donation = float(input('Donation amount: '))
new_donor = {
'name': {
'first_name': first_name,
'last_name': last_name
},
'donation': donation
}
self.donor.insert_one(new_donor)
self.send_thanks(first_name, last_name, donation)
Main()
self.main_menu()
self.selection()
elif int(input1) == 2:
self.create_report()
elif int(input1) == 3:
self.send_letters_all()
elif int(input1) == 4:
self.delete_donor()
elif int(input1) == 5:
print("Exiting program...")
raise SystemExit()
except ValueError:
print("You must use a menu number between 1-4; try again!")
def send_thanks(self, first, last, amount):
letter = 'Thank you {} {} for your donation in the amount of ${}; it is very generous.'.format(
first, last, amount)
with open('Thank_You_{}, {}.txt'.format(last.lower(), first.lower()),
'w') as f:
f.write(letter)
print("Your thank you letter has been written to disk.")
def create_report(self):
print('\n{:<20} {:>20} {:>20} {:>20}'.format('Donor Name',
'| Total Given',
'| Num Gifts',
'| Average Gift'))
print('{}'.format('-' * 83))
result = self.donor.aggregate([{
'$group': {
'_id': {
'name': '$name'
},
'sum': {
'$sum': '$donation'
},
'count': {
'$sum': 1
},
'avg': {
'$avg': '$donation'
}
}
}, {
'$sort': {
'sum': -1
}
}])
for i in result:
i_name = ' '.join([
i['_id']['name']['first_name'], i['_id']['name']['last_name']
])
i_sum = i['sum']
i_count = i['count']
i_avg = i['avg']
print('{:<20} {:>20.02f} {:>20} {:>20.02f}'.format(
i_name, i_sum, i_count, i_avg))
Main()
self.main_menu()
self.selection()
def send_letters_all(self):
letters = 'Dear {},\n\n\tThank you for your total contributions in the amount of ${}.\n\n\tYou are making a difference in the lives of others.\n\n\t\tSincerely,\n\t\t"Working for America"'
result = self.donor.aggregate([{
'$group': {
'_id': {
'name': '$name'
},
'sum': {
'$sum': '$donation'
},
'count': {
'$sum': 1
},
'avg': {
'$avg': '$donation'
}
}
}, {
'$sort': {
'sum': -1
}
}])
for i in result:
i_name = ' '.join([
i['_id']['name']['first_name'], i['_id']['name']['last_name']
])
i_sum = i['sum']
i_count = i['count']
i_avg = i['avg']
with open(
'Thank_You_Letter_{}.txt'.format(i_name.title().replace(
' ', '_')), 'w') as f:
f.write(letters.format(i_name, i_sum))
print('\nYour letters have been printed to the current directory!')
Main()
self.main_menu()
self.selection()
def delete_donor(self):
print(
'\nType "list" to show names or hit the "Return/Enter" key to delete a name.'
)
input2 = input('-> ')
if input2 == 'list':
donor_names = set()
for d in self.donor.find():
name = ' '.join(
[d['name']['first_name'], d['name']['last_name']])
donor_names.add(name)
for d in donor_names:
print(d)
Main()
self.main_menu()
self.selection()
else:
first_name = input('First Name: ')
last_name = input('Last Name: ')
del_donor = {
'name': {
'first_name': first_name,
'last_name': last_name
}
}
check = self.donor.count_documents(del_donor) > 0
if check:
self.donor.delete_many(del_donor)
else:
print("That donor is not in our database; please try again.")
Main()
self.main_menu()
self.selection()
class Main:
log.info('Initialize Class "Main".')
log = utilities.configure_logger('default', '../logs/nosql_dev.log')
log.info('\n\nRunning mongodb_mailroom_script')
log.info("Initialize the Database.")
with login_database.login_mongodb_cloud() as client:
log.info('\n\nStep 1: We are going to use a database called donors')
log.info('\n\nBut if it doesnt exist mongodb creates it')
db = client['donors']
log.info('\n\nAnd in that database use a collection called Donor')
log.info('\n\nIf it doesnt exist mongodb creates it')
donor = db['donor']
log.info('Insert an item into the document')
new_donor = {
'name': {
'first_name': 'Test',
'last_name': 'Guy'
},
'donation': 1234.56
}
donor.insert_one(new_donor)
log.info('\n\nStep 2: Now we add data from the dictionary above')
donor_items = get_donor_data()
donor.insert_many(donor_items)
log.info("Show the collections in the DB.")
print(db.list_collection_names())
def __init__(self):
self.prompt = {
1: 'Send A Thank You To New Or Exsisting Donor',
2: 'Create a Report',
3: 'Send notes to everyone',
4: 'Delete a Donor',
5: 'Exit'
}
def menu_selection(self):
print("\nPick from the listed options.")
return {(print(str(k) + ':', v)) for k, v in self.prompt.items()}
def selection(self):
while True:
input1 = input("Selection: ")
try:
if int(input1) in range(1, 5):
if int(input1) == 1:
print('\n"Enter a name or list"')
input2 = input()
if input2 == 'list':
donor_names = set()
for i in self.donor.find():
name = ' '.join([
i['name']['first_name'],
i['name']['last_name']
])
donor_names.add(name)
for i in donor_names:
print(i)
Main()
self.menu_selection()
self.selection()
else:
first_name = input('First Name: ')
last_name = input('Last Name: ')
donation = float(input('Donation amount: '))
new_donor = {
'name': {
'first_name': first_name,
'last_name': last_name
},
'donation': donation
}
self.donor.insert_one(new_donor)
self.thank_you(first_name, last_name, donation)
Main()
self.menu_selection()
self.selection()
elif int(input1) == 2:
self.create_report()
elif int(input1) == 3:
self.thank_everyone()
elif int(input1) == 4:
self.delete_user()
elif int(input1) == 5:
raise SystemExit()
except ValueError:
print("\nPick from the listed options.")
def thank_you(self, first, amount):
letter = "Dear {},\nThank you for your generous donation in the amount \
of ${}; \nThe money will be put to good use.\n\nSincerely, \n -\
The Team".format(first, amount)
with open('{}.txt'.format(first.upper().replace(' ', '_')), 'w') as f:
f.write(letter)
def create_report(self):
print('\n{:<20} {:>20} {:>20} {:>20}'.format('Donor Name',
'| Total Given',
'| Num Gifts',
'| Average Gift'))
print('{}'.format('-' * 83))
result = self.donor.aggregate([{
'$group': {
'_id': {
'name': '$name'
},
'sum': {
'$sum': '$donation'
},
'count': {
'$sum': 1
},
'avg': {
'$avg': '$donation'
}
}
}, {
'$sort': {
'sum': -1
}
}])
for i in result:
i_name = ' '.join([
i['_id']['name']['first_name'], i['_id']['name']['last_name']
])
i_sum = i['sum']
i_count = i['count']
i_avg = i['avg']
print('{:<20} {:>20.02f} {:>20} {:>20.02f}'.format(
i_name, i_sum, i_count, i_avg))
Main()
self.menu_selection()
self.selection()
def thank_everyone(self):
notes = 'Dear {},\n\nThank you for your generous donations totaling \
${}. The money will be put to good use.\n\nSincerely,\n\t\t-The Team'
result = self.donor.aggregate([{
'$group': {
'_id': {
'name': '$name'
},
'sum': {
'$sum': '$donation'
},
'count': {
'$sum': 1
},
'avg': {
'$avg': '$donation'
}
}
}, {
'$sort': {
'sum': -1
}
}])
for i in result:
i_name = ' '.join([
i['_id']['name']['first_name'], i['_id']['name']['last_name']
])
i_sum = i['sum']
i_count = i['count']
i_avg = i['avg']
with open('{}.txt'.format(i_name.title().replace(' ', '_')),
'w') as f:
f.write(notes.format(i_name, i_sum))
Main()
self.menu_selection()
self.selection()
def delete_user(self):
print('\nPress ENTER or type list')
input2 = input()
if input2 == 'list':
donor_names = set()
for i in self.donor.find():
name = ' '.join(
[i['name']['first_name'], i['name']['last_name']])
donor_names.add(name)
for i in donor_names:
print(i)
Main()
self.menu_selection()
self.selection()
else:
first_name = input('First Name: ')
last_name = input('Last Name: ')
del_donor = {
'name': {
'first_name': first_name,
'last_name': last_name
}
}
check = self.donor.count_documents(del_donor) > 0
if check:
self.donor.delete_many(del_donor)
else:
print("That donor is not in our database; please try again.")
Main()
self.menu_selection()
self.selection()
def run_example():
with login_database.login_mongodb_cloud() as client:
log.info('Step 1: We are going to use a database called donors')
log.info('But if it doesnt exist mongodb creates it')
db = client['donors']
log.info('And in that database use a collection called donor')
log.info('If it doesnt exist mongodb creates it')
donor = db['donor']
log.info('\n\nStep 2: Now we add data from the dictionary above')
donor_items = donor_data.get_donor_data()
donor.insert_many(donor_items)
log.info('Step 3: List donor collection:')
records = donor.find()
for i in records:
print(i)
##########################################################################################
# Donations report Total, Average, Count
##########################################################################################
log.info('Step 4: List report: donor, sum, average of donation:')
try:
records = donor.aggregate(
[
{
'$unwind' : '$donations' # must $unwind array for aggregation
},
{
'$group' : {
'_id' : '$name',
'Total' : {
'$sum': '$donations'
},
'Average' : {
'$avg': '$donations'
},
# mind count implemented as sum of aggregated rows
'Count' : {
'$sum': 1
}
}
},
{
'$sort' : {'Total' : -1 }
}
]
)
except OperationFailure as e:
print(e)
for i in records:
print(i)
# log.info('\n\nStep 5: Prints number of donation per donor:')
# try:
# records = donor.aggregate([
# {
# '$unwind' : '$donations' # must $unwind array for aggregation
# },
# {
# "$group" : {
# '_id' :"$name",
# 'Number' :{'$sum':1}
# }
# }
# ])
# except OperationFailure as e:
# print(e)
# for i in records:
# print(i)
##########################################################################################
# List donors
##########################################################################################
log.info('Step 5: List donors:')
records = donor.find().sort('name.last_name')
for i in records:
print("{} {}".format(i['name']['first_name'],
i['name']['last_name']))
##########################################################################################
# Projection Challenge
##########################################################################################
log.info('Step 6: Projection factor, min=20, max=200:')
factor = 2
_min=20
_max=200
# try:
cursor1 = donor.aggregate([
{
'$unwind' : '$donations'
},
{
'$match' : { 'donations' : { '$gt' : _min, '$lt' : _max} # mind 'donations' without '$'
}
},
{
'$group' : {'_id' : '$name',
'Total' : {
'$sum': '$donations'
}
}
}
])
cursor2 = donor.aggregate([
{
'$unwind' : '$donations'
},
{
'$match' : {'$or': [
{'donations' : {'$lte' : _min}},
{'donations' : {'$gte' : _max}},
]
}
},
{
'$group' : {'_id' : '$name',
'Total' : {
'$sum': '$donations'
}
}
}
])
cursor3 = donor.aggregate([
{
'$unwind' : '$donations'
},
{
'$match' : { 'donations' : { '$gt' : _min, '$lt' : _max} # mind 'donations' without '$'
}
},
{
'$project' : { 'name' : 1, 'donations' : 1, 'projected' : {'$multiply' : ['$donations', factor]}
}
}
])
cursor4 = donor.aggregate([
# pipe1
{
'$unwind' : '$donations'
}, # pipe1 => pipe2
# pipe2
{
'$match' : { 'donations' : { '$gt' : _min, '$lt' : _max} # mind 'donations' without '$'
}
}, # pipe2 => pipe3
# pipe3
{
'$project' : { 'name' : 1, 'donations' : 1, 'projected' : {'$multiply' : ['$donations', factor]}
}
}, # pipe3 => to group aggregation - mind sum is done on created above column: 'projected'
{
'$group' : {'_id' : '$name',
'Total' : {
'$sum': '$projected'
}
}
}
])
# except OperationFailure as e:
# print(e)
print("Total of projected donations must be multiply by factor")
for i in cursor1:
print(i)
print("*"*80)
# print("Total of multiplied by 2 projected donations")
# for i in cursor4:
# print(i)
projection_dict = {}
for i in cursor4:
projection_dict[(i['_id']['first_name'], i['_id']['last_name'])] = i['Total']
# print("total of non projected donations")
# for i in cursor2:
# print(i)
for i in cursor2:
if (i['_id']['first_name'], i['_id']['last_name']) not in projection_dict.keys():
projection_dict[(i['_id']['first_name'], i['_id']['last_name'])] = i['Total']
else:
projection_dict[(i['_id']['first_name'], i['_id']['last_name'])] += i['Total']
print("*** Projected values:")
for k, v in sorted(projection_dict.items(), key=lambda item: item[1], reverse=True):
pp.pprint("{} {} Total: {}".format(k[0], k[1], v))
####
#### Looking for UNION ALL like operation to MERGE two cursors: cursor4 and cursor2, then to perform aggregation on it
####
##########################################################################################
# Projection Update
##########################################################################################
log.info('Step 7: Apply projected changes: factor, min=20, max=200:')
factor = 2
_min=20
_max=200
# option 1
# for update pipe processing doesn't apply, either $unwind for array elements
# result = donor.update(
# {
# '$unwind' : '$donations'
# }, # pipe1
# {
# '$match' : { 'donations' : { '$gt' : _min, '$lt' : _max} # mind 'donations' without '$'
# }
# }, # pipe2
# { '$mul': { '$donations': factor }
# },
# { '$upsert' : 'False' },
# { '$multi' : 'True' }
#
# )
# option 2
# select_upd = { 'donations' : {
# '$elemMatch' : { '$gt' : _min, '$lt' : _max}
# }
# }
# new_values = { '$mul': { 'donations.$[]': factor} # multiplies all elements in array
# }
#
# new_values = { '$mul': { 'donations.$': factor} # multiplies single elements in array
# }
### Looking for the way to update all array lements meeting the search criteria
# result = donor.update_many(select_upd, new_values)
print("+++ in database after update:")
records = donor.find()
for i in records:
print(i)
##########################################################################################
# Add new donor, donation
##########################################################################################
# looking for donor, if exist then add/append donation, if not create/insert new donot and donation
log.info('Step 8: Add/Update donor/donation:')
first = "Ivan"
last = "Smirnoff"
location = "Moscow" # update
# location = "Bothell" # insert
donation = 10
record = {
'name': {'first_name': first, 'last_name': last},
'location': location,
'donations' : [donation]
}
query = { '$and': [
{'name.first_name' : {'$eq' : first}},
{'name.last_name' : {'$eq' : last}},
{'location' : {'$eq' : location}}
]}
update = { '$push' : {'donations' : donation}}
result = donor.find(query)
for i in result:
print(i)
if result.count() > 0:
print("found, record to update:")
print(record)
result = donor.update_one(query, update)
else:
print("not found, record to insert:")
print(record)
result = donor.insert_one(record)
# print("insterted records: {}".format(result.inserted_count))
records = donor.find()
for i in records:
print(i)
log.info('Step 9: Delete the collection so we can start over')
db.drop_collection('donor')
import login_database
from donor_data import get_donor_data
with login_database.login_mongodb_cloud() as client:
db = client['donors']
db.drop_collection('donor')
donor = db['donor']
donor_items = get_donor_data()
donor.insert_many(donor_items)
r = login_database.login_redis_cloud()
r.flushdb()
donor_items = get_donor_data()
for donor in donor_items:
r.set(donor['name'], donor['donation'])
driver = login_database.login_neo4j_cloud()
with driver.session() as session:
session.run("MATCH (n) DETACH DELETE n")
with driver.session() as session:
for donor in donor_items:
cyph = "CREATE (n:Person {name:'%s'})" % (donor['name'])
session.run(cyph)