def run():
configure_logger()
args = get_commandline_args()
if args.license:
print(licence_message)
sys.exit()
output_manager_ref = LogOutputManagerActor.start()
start_conductor(args.db_path, output_manager_ref)
send_messages(args)
shutdown_conductor()
def redis_mailroom():
"""
uses non-presistent Redis only (as a cache).
seeding data one at a time
"""
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
log.info('Connect to Redis')
r = login_database.login_redis_cloud()
r.set('Donor_tjefferson', 'Thomas Jefferson')
r.set('Email_tjefferson', '*****@*****.**')
r.set('Donation_tjefferson','100')
r.set('Donor_brubble', 'Betty Rubble')
r.set('Email_brubble', '*****@*****.**')
r.set('Donation_brubble','500')
r.set('Donor_gjetson', 'George Jetson')
r.set('Email_gjetson', '*****@*****.**')
r.set('Donation_gjetson','60')
r.set('Donor_lsimpson', 'Lisa Simpson')
r.set('Email_lsimpson', '*****@*****.**')
r.set('Donation_lsimpson','100')
log.info('Retrieve some customer info')
donor_name = r.get('Donor_brubble')
log.info(f'Donor name is: {donor_name}')
donor_email = r.get('Email_lsimpson')
log.info(f'Lisa Simpson has email: {donor_email}')
donation = r.get('Donation_gjetson')
log.info(f'The initial donation by George Jetson: ${donation}')
except Exception as e:
print(f'Redis error: {e}')
def run_example_activity():
"""
uses non-presistent Redis only (as a cache)
"""
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
log.info('Step 1: connect to Redis')
r = login_database.login_redis_cloud()
log.info('Step 2: Have Redis store a customer name, telephone and zip for 6 or so customers')
r.rpush('andy', '111-111-1111', '11111')
r.rpush('bonnie', '222-222-2222', '22222')
r.rpush('camden', '333-333-3333', '33333')
r.rpush('duke', '444-444-4444', '44444')
r.rpush('ethridge', '555-555-5555', '55555')
r.rpush('foster', '777-777-7777', '77777')
log.info('Step 3: Then show how you can retrieve a zip code, and then a phone number, for a known customer')
log.info(f'Andy Phone: {r.lindex("andy", 0)}')
log.info(f'Camden Zip: {r.lindex("camden", 1)}')
except Exception as e:
print(f'Redis error: {e}')
def run_exercise():
"""
redis exercise
Assignment:
Add some customer data to the cache.
Have Redis store a customer name, telephone and zip for 6 or so customers.
Then show how you can retrieve a zip code, and then a phone number, for a known customer.
"""
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
log.info('Step 1: connect to Redis')
r = login_database.login_redis_cloud()
log.info('Step 2: cache customer data in Redis')
r.rpush('Mark', '808-555-1234', '96825')
r.rpush('Roger', '717-123-4567', '17331')
r.rpush('Tom', '425-222-1234', '98275')
r.rpush('Mimi', '206-999-9999', '98101')
r.rpush('Maureen', '212-867-5309', '10002')
r.rpush('Joanne', '617-542-3779', '02114')
log.info('Step 3: read customer info')
log.info(f"Roger's phone number is {r.lindex('Roger',2)}, and his zipcode is {r.lindex('Roger', 1)}.")
log.info(f"Mark's phone number is {r.lindex('Mark',2)}, and his zipcode is {r.lindex('Mark', 1)}.")
except Exception as e:
print(f'Redis error: {e}')
def showoff_databases():
"""
Here we illustrate basic interaction with nosql databases
"""
log = utilities.configure_logger('default', '../logs/nosql_dev.log')
log.info(
"\n\n\nMongodb example to use data from Furniture module, so get it.")
furniture = learn_data.get_furniture_data()
log.info("\n\n\nHere's the MongoDB script.")
mongodb_script.run_example(furniture)
log.info("Other databases use data embedded in the modules")
log.info("\n\n\nHere's the Redis script.")
redis_script.run_example()
log.info("\n\n\nHere's the Neo4J script.")
neo4j_script.run_example()
log.info(
"\n\n\nHere's the persistence/serialization (Pickle/Shelve) script.")
simple_script.run_example(furniture)
def showoff_databases():
"""
Here we illustrate basic interaction with nosql databases
"""
log = utilities.configure_logger('default', '../logs/nosql_dev.log')
log.info("Mongodb example to use data from Furniture module, so get it")
furniture = learn_data.get_furniture_data()
roygbiv = ["Red", "Orange", "Yellow", "Green", "Blue", "Indigo", "Violet"]
for item in furniture:
product_type = item['product']
color_name = 'N/A'
for color in roygbiv:
if color in product_type:
color_name = color
product_type = product_type.replace(color, '').lstrip(' ')
break
item['color'] = color_name
item['product type'] = product_type
mongodb_script.run_example(furniture)
log.info("Other databases use data embedded in the modules")
redis_script.run_example()
neo4j_script.run_example()
simple_script.run_example(furniture)
persistence_serialization.run_json()
def load_neo4j(driver):
"""Load our mailroom rdbms into a graph database. Each row in an entity
table is a node. Columns on those tables become a node property.
For each person, we will have a node with two properties, the person's
name and the person's donations. The node will have a label :Person.
Because donations are unique to the donor, it makes sense to make this a
node property along with the name of donor. If we were storing email,
address, and telephone, it would also make sense to store these as node
properties since they are unique to the donor.."""
log = utilities.configure_logger('default', '../logs/neo4j_LoadTables.log')
log.debug('Step 1: First, clear the entire database, so we can start over')
log.debug("Running clear_all")
# driver = login_database.login_neo4j_cloud()
with driver.session() as session:
session.run("MATCH (n) DETACH DELETE n")
log.debug("Step 2: Add a few people")
cyph = "CREATE (n:Person {donor:'Zach', donations:[10]})"
session.run(cyph)
cyph = "CREATE (n:Person {donor:'Shane', donations:[6, 5, 10]})"
session.run(cyph)
cyph = "CREATE (n:Person {donor:'Pete', donations:[7, 8]})"
session.run(cyph)
cyph = "CREATE (n:Person {donor:'Fitz', donations:[1]})"
session.run(cyph)
cyph = "CREATE (n:Person {donor:'Joe', donations:[5, 4, 3, 2, 1]})"
session.run(cyph)
def run_example():
"""
Refactored Redis example
"""
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
r = login_database.login_redis_cloud()
log.info(f'Add more customer data')
customers = [
['halpert', {'telephone': '111-111-1111', 'zip': '11111'}],
['michael', {'telephone': '222-222-2222', 'zip': '22222'}],
['kevin', {'telephone': '333-333-3333', 'zip': '33333'}],
['oscar', {'telephone': '444-444-4444', 'zip': '44444'}],
['ryan', {'telephone': '555-555-5555', 'zip': '55555'}],
['Phyllis', {'telephone': '666-666-6666', 'zip': '66666'}]
]
for key, data in customers:
r.hmset(key, data)
log.info(f'Retreive customer info')
key = "oscar"
result = r.hgetall(key)
print(f"Telephone for {key}: {result['telephone']}")
print(f"Zip for {key}: {result['zip']}")
except Exception as e:
print(f'Redis error: {e}')
def run_example():
"""
uses non-presistent Redis only (as a cache)
"""
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
log.info('Step 1: connect to Redis')
r = login_database.login_redis_cloud()
log.info('Step 2: Add customer data to the cache')
r.hmset('Aaron', {'phone': '505-554-3729', 'zip': 11561})
r.hmset('Blake', {'phone': '594-640-0741', 'zip': 17050})
r.hmset('Charles', {'phone': '711-652-1354', 'zip': 16614})
r.hmset('Denise', {'phone': '821-667-5095', 'zip': 30144})
r.hmset('Edward', {'phone': '808-558-9987', 'zip': 46530})
r.hmset('Frank', {'phone': '322-946-6274', 'zip': 28104})
log.info('Step 3: Retrieve Blake\'s phone number')
phone_blake = r.hmget('Blake', 'phone')
log.info(f'Blake\'s phone number: {phone_blake}')
log.info('Step 4: Retrieve Blake\'s zip code')
zip_blake = r.hmget('Blake', 'zip')
log.info(f'Blake\'s zip code: {zip_blake}')
except Exception as e:
print(f'Redis error: {e}')
def run_example():
"""
uses non-presistent Redis only (as a cache)
"""
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
log.info('Step 1: connect to Redis')
r = login_database.login_redis_cloud()
log.info('Step 2: cache some data in Redis')
r.set('andy', '*****@*****.**')
log.info('Step 2: now I can read it')
email = r.get('andy')
log.info('But I must know the key')
log.info(f'The results of r.get: {email}')
log.info('Step 3: cache more data in Redis')
r.set('pam', '*****@*****.**')
r.set('fred', '*****@*****.**')
log.info('Step 4: delete from cache')
r.delete('andy')
log.info(f'r.delete means andy is now: {email}')
log.info(
'Step 6: Redis can maintain a unique ID or count very efficiently')
r.set('user_count', 21)
r.incr('user_count')
r.incr('user_count')
r.decr('user_count')
result = r.get('user_count')
log.info('I could use this to generate unique ids')
log.info(f'Redis says 21+1+1-1={result}')
log.info('Step 7: richer data for a SKU')
r.rpush('186675', 'chair')
r.rpush('186675', 'red')
r.rpush('186675', 'leather')
r.rpush('186675', '5.99')
log.info('Step 8: pull some data from the structure')
cover_type = r.lindex('186675', 2)
log.info(f'Type of cover = {cover_type}')
log.info('Step 9: add users with phone # and zip code')
r.hmset('Bugs Bunny', {'Phone':'800-214-1234', 'Zip_Code':'98042'})
r.hmset('Porky Pig', {'Phone':'999-555-1212', 'Zip_Code':'98031'})
r.hmset('Foghorn Leghorn', {'Phone':'805-637-7032', 'Zip_Code':'93454'})
r.hmset('Tweety Bird', {'Phone':'425-222-333', 'Zip_Code':'90210'})
r.hmset('Elmer Fudd', {'Phone':'900-315-7654', 'Zip_Code':'99888'})
log.info('Step 10: Get phone and zip for user')
data = r.hgetall('Foghorn Leghorn')
log.info(f'Foghorn Leghorn info: Phone #: {data["Phone"]} Zip Code: {data["Zip_Code"]}')
except Exception as e:
print(f'Redis error: {e}')
def run_example():
"""
uses non-presistent Redis only (as a cache)
"""
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
log.info('Step 1: connect to Redis')
r = login_database.login_redis_cloud()
log.info('Step 2: Add donor data to the cache')
r.hmset('Aaron', {'phone': '505-554-3729', 'zip': 11561,
'donation': [20], 'email': '*****@*****.**'})
r.hmset('Blake', {'phone': '594-640-0741', 'zip': 17050,
'donation': [20, 50], 'email': '*****@*****.**'})
r.hmset('Charles', {'phone': '711-652-1354', 'zip': 16614,
'donation': [100], 'email': '*****@*****.**'})
r.hmset('Denise', {'phone': '821-667-5095', 'zip': 30144,
'donation': [5], 'email': '*****@*****.**'})
r.hmset('Edward', {'phone': '808-558-9987', 'zip': 46530,
'donation': [15], 'email': '*****@*****.**'})
except Exception as e:
print(f'Redis error: {e}')
def redis_assignment_create():
log = utilities.configure_logger('default', 'logs/redis_script.log')
try:
log.info('Step 1: connect to Redis')
r = login_database.login_redis_cloud()
phoneidx = 0
zipidx = 1
r.rpush('Jeroen', 15555222)
r.rpush('Jeroen', 2800)
r.rpush('Heinz', 2001589)
r.rpush('Heinz', 1000)
r.rpush('Knudde', 3658714)
r.rpush('Knudde', 6900)
r.rpush('Gaston', 22222221111)
r.rpush('Gaston', 98139)
r.rpush('Neintje', 121212129)
r.rpush('Neintje', 7000)
r.rpush('Bumbalu', 31346546)
r.rpush('Bumbalu', 9999)
phone = r.lindex('Jeroen', phoneidx)
log.info(f'Jeroen Phone Number = {phone}')
zipcode = r.lindex('Jeroen', zipidx)
log.info(f'Jeroen Zip Code = {zipcode}')
except Exception as e:
print(f'Redis error: {e}')
def run_example():
"""
uses non-presistent Redis only (as a cache)
"""
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
log.info('Step 1: connect to Redis')
r = login_database.login_redis_cloud()
log.info('Step 2: cache some data in Redis')
# r.set('andy', '*****@*****.**')
r.rpush('cust1', 'Albert', '206-555-1111', '98111')
r.rpush('cust2', 'Bill', '206-555-2222', '98222')
r.rpush('cust3', 'Charlie', '206-555-3333', '98333')
r.rpush('cust4', 'David', '206-555-4444', '98444')
r.rpush('cust5', 'Edward', '206-555-5555', '98555')
r.rpush('cust6', 'Frank', '206-555-6666', '98666')
print(f"Length of list for a customer: {r.llen('cust4')}")
print(f"DBSize: {r.dbsize()}")
print(f"Cust4 name: {r.lindex('cust4', 0)}")
print(f"Cust4 zip: {r.lindex('cust4', 2)}")
print(f"Cust4 phone: {r.lindex('cust4', 1)}")
r.flushdb()
except Exception as e:
print(f'Redis error: {e}')
def run_example():
"""
Redis
"""
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
log.info('Step 1: connect to Redis')
r = login_db.login_redis_cloud()
log.info('Step 2: cache data in db')
r.set('natalie', '*****@*****.**')
log.info('Step 2: Add some data to the cache')
r.hmset('Luke', {'phone': '319-706-8805', 'age': 29})
r.hmset('Virgil', {'phone': '913-366-2202', 'age': 15})
r.hmset('River', {'phone': '972-310-1058', 'age': 36})
r.hmset('Kibson', {'phone': '309-304-0112', 'age': 5})
log.info('Step 3: Return Virgil\'s phone number')
phone_virgil = r.hmget('Virgil', 'phone')
log.info(f'Virgil\'s phone number: {phone_virgil}')
log.info('Step 4: Get Virgil\'s')
age_virgil = r.hmget('Virgil', 'age')
log.info(f'Virgil\'s age: {age_Virgil}')
except Exception as e:
print(f'Redis error: {e}')
def run_example():
"""
uses non-persistent Redis only (as a cache)
"""
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
log.info('Step 1: connect to Redis')
r = login_database.login_redis_cloud()
log.info('Step 2: do the homework')
r.rpush('William Gates III', '*****@*****.**')
r.rpush('William Gates III', '4255551212')
r.rpush('William Gates III', '98101')
r.rpush('Jeff Bezos', '*****@*****.**')
r.rpush('Jeff Bezos', '2065551212')
r.rpush('Jeff Bezos', '98115')
r.rpush('Paul Allen', '*****@*****.**')
r.rpush('Paul Allen', '2065559000')
r.rpush('Paul Allen', '98104')
r.rpush('Mark Zuckerberg', '*****@*****.**')
r.rpush('Mark Zuckerberg', '4255552912')
r.rpush('Mark Zuckerberg', '98125')
except Exception as e:
print(f'Redis error: {e}')
def run_example():
"""
uses non-presistent Redis only (as a cache)
"""
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
log.info('Step 1: Connect to Redis')
r = login_database.login_redis_cloud()
log.info('Step 2: Adding 8 customers to the data cache')
r.hmset('Bill', {'phone': '2538-234-1433', 'zip': 98277})
r.hmset('Elon', {'phone': '125-290-2443', 'zip': 23904})
r.hmset('Jeff', {'phone': '654-490-2300', 'zip': 49039})
r.hmset('Nikola', {'phone': '453-872-9982', 'zip': 23922})
r.hmset('Steve', {'phone': '238-403-4923', 'zip': 34095})
r.hmset('John', {'phone': '299-343-4785', 'zip': 18492})
r.hmset('Joe', {'phone': '947-239-1114', 'zip': 24909})
r.hmset('Harry', {'phone': '808-333-9942', 'zip': 54848})
log.info('Step 3: Retrieve zip code')
ret_zip = r.hmget('Elon', 'zip')
log.info(f"Elon's zip is {ret_zip[0]}.")
log.info('Step 4: Retrieve phone number')
ret_phone = r.hmget('John', 'phone')
log.info(f"John's phone number is {ret_phone[0]}.")
except Exception as e:
print(f'Redis error: {e}')
def run_example():
"""
uses non-persistent Redis only (as a cache)
"""
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
log.info('Step 1: connect to Redis')
r = login_database.login_redis_cloud()
log.info('Step 2: cache some data in Redis')
r.set('test', '*****@*****.**')
log.info('Step 2: adding some data to cache')
r.hmset('Keith', {'phone': '425-123-4567', 'zip code': '91234'})
r.hmset('Allison', {'phone': '206-987-6543', 'zip code': '94321'})
r.hmset('Jesse', {'phone': '360-123-4567', 'zip code': '90101'})
r.hmset('Lauren', {'phone': '425-606-6060', 'zip code': '96060'})
r.hmset('John', {'phone': '206-206-2062', 'zip code': '92062'})
r.hmset('Merlin', {'phone': '800-900-1000', 'zip code': '87654'})
log.info('Step 3: return Merlin\'s zip code.')
merlin_zip = r.hmget('Merlin', 'zip code')
log.info(f'Merlin\'s zip code: {merlin_zip}')
log.info('Step 4: return Merlin\'s phone number.')
merlin_phone = r.hmget('Merlin', 'phone number')
log.info(f'Merlin\'s phone number: {merlin_phone}')
except Exception as e:
print(f'Redis error: {e}')
def run_example():
"""
uses non-presistent Redis only (as a cache)
"""
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
log.info('Step 1: connect to Redis')
r = login_database.login_redis_cloud()
log.info('Step 2: cache some data in Redis')
#r.set('andy', '*****@*****.**')
r.rpush('customer1', 'sirisha', '206-111-1111', '98111')
r.rpush('customer2', 'jessy', '206-222-2222', '98129')
r.rpush('customer3', 'david', '206-333-3333', '98127')
r.rpush('customer4', 'sam', '425-111-2222', '98052')
r.rpush('customer5', 'alex', '425-222-3333', '98007')
r.rpush('customer6', 'bill', '425-333-4444', '98006')
print(f"Length of list for customer1 is: {r.llen('customer1')}")
print(f"Database size: {r.dbsize()}")
print(f"customer1 name:{r.lindex('customer1', 0)}")
print(f"customer1 telephone: {r.lindex('zip',1)}")
print(f"customer1 zip: {r.lindex('customer1', 2)}")
except Exception as e:
print(f'Redis error: {e}')
def run_example():
"""
uses non-presistent Redis only (as a cache)
"""
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
log.info('Step 1: connect to Redis')
r = login_database.login_redis_cloud()
log.info('Step 2: cache some data in Redis')
r.rpush('0001', 'andy', '123-456-4444', '12345')
r.rpush('0002', 'brad', '123-456-5555', '23456')
r.rpush('0003', 'cory', '123-456-6666', '34567')
r.rpush('0004', 'diane', '123-456-7777', '45678')
r.rpush('0005', 'ellen', '123-456-8888', '56789')
r.rpush('0006', 'felicity', '123-456-9999'. '67891')
# log.info('Step 2: now I can read it')
# email = r.get('andy')
# log.info('But I must know the key')
# log.info(f'The results of r.get: {email}')
# log.info('Step 3: cache more data in Redis')
# r.set('pam', '*****@*****.**')
# r.set('fred', '*****@*****.**')
# log.info('Step 4: delete from cache')
# r.delete('andy')
# log.info(f'r.delete means andy is now: {email}')
# log.info(
# 'Step 6: Redis can maintain a unique ID or count very efficiently')
# r.set('user_count', 21)
# r.incr('user_count')
# r.incr('user_count')
# r.decr('user_count')
# result = r.get('user_count')
# log.info('I could use this to generate unique ids')
# log.info(f'Redis says 21+1+1-1={result}')
# log.info('Step 7: richer data for a SKU')
# r.rpush('186675', 'chair')
# r.rpush('186675', 'red')
# r.rpush('186675', 'leather')
# r.rpush('186675', '5.99')
# log.info('Step 8: pull some data from the structure')
# cover_type = r.lindex('186675', 2)
# log.info(f'Type of cover = {cover_type}')
log.info('Step 2: Retrieve data for cust 0006')
log.info(f'Zip code for cust 0006: {r.lindex('0006', 2)}')
log.info(f'Phone number for cust 0006: {r.lindex('0006', 1)}')
except Exception as e:
print(f'Redis error: {e}')
def persistence():
"""
Homework assignment for persistence.
"""
log = utilities.configure_logger('default', '../logs/nosql_assignment.log')
log.info('Running persistence assignment.')
persistence_assignment.run_json()
def add_customers():
"""
uses non-presistent Redis only (as a cache)
"""
PHONE = 0
ZIP = 1
customers = {
'joe': {
'phone': '9086543218',
'zip': '86902'
},
'bob': {
'phone': '8976543456',
'zip': '93675'
},
'fred': {
'phone': '7356497673',
'zip': '84720'
},
'sue': {
'phone': '1847264092',
'zip': '09164'
},
'mary': {
'phone': '8642876530',
'zip': '92810'
},
'james': {
'phone': '8503926593',
'zip': '02958'
}
}
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
log.info('Step 1: connect to Redis')
r = login_database.login_redis_cloud()
log.info('Step 2: cache customer data in Redis')
for customer, data in customers.items():
r.set(customer + '_phone', data['phone'])
r.set(customer + '_zip', data['zip'])
log.info("Step 3: Retrieve Joe's phone number.")
phone = r.get('joe_phone')
log.info(f"Joe's phone number is: {phone}.")
log.info("Step 4: Retrieve Joe's zip code.")
zipcode = r.get('joe_zip')
log.info(f"Joe's zip code is: {zipcode}.")
except Exception as e:
print(f'Redis error: {e}')
raise e
def run_example():
"""
uses non-presistent Redis only (as a cache)
"""
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
log.info('Step 1: connect to Redis')
r = login_database.login_redis_cloud()
log.info('Step 2: cache some data in Redis')
r.set('andy', '*****@*****.**')
log.info('Step 2: now I can read it')
email = r.get('andy')
log.info('But I must know the key')
log.info(f'The results of r.get: {email}')
log.info('Step 3: cache more data in Redis')
r.set('pam', '*****@*****.**')
r.set('fred', '*****@*****.**')
log.info('Step 4: delete from cache')
r.delete('andy')
log.info(f'r.delete means andy is now: {email}')
log.info(
'Step 6: Redis can maintain a unique ID or count very efficiently')
r.set('user_count', 21)
r.incr('user_count')
r.incr('user_count')
r.decr('user_count')
result = r.get('user_count')
log.info('I could use this to generate unique ids')
log.info(f'Redis says 21+1+1-1={result}')
log.info('Step 7: richer data for a SKU')
r.rpush('186675', 'chair')
r.rpush('186675', 'red')
r.rpush('186675', 'leather')
r.rpush('186675', '5.99')
log.info('Step 8: pull some data from the structure')
cover_type = r.lindex('186675', 2)
log.info(f'Type of cover = {cover_type}')
r.hset('nancy', 'location', 'Seattle')
r.hset('nancy', 'phone', '206 555 7755')
r.hset('nancy', 'zip', '90210')
print(
f"Nancy's phone #: {r.hget('nancy', 'phone')}, Nancy's Zip: {r.hget('nancy', 'zip')}"
)
except Exception as e:
print(f'Redis error: {e}')
def run_nosql_ex():
"""
redis data lesson 08
"""
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
r = login_database.login_redis_cloud()
r.flushall()
log.info('Data is being saved into redis')
r.hmset('andy', {
'email': '*****@*****.**',
'zip': '91700',
'phone': '555-666-8888'
})
r.hmset('pam', {
'email': '*****@*****.**',
'zip': '91755',
'phone': '322-666-8888'
})
r.hmset('fred', {
'email': '*****@*****.**',
'zip': '98534',
'phone': '555-666-2888'
})
r.hmset('tester_1', {
'email': '*****@*****.**',
'zip': '91755',
'phone': '555-666-8788'
})
r.hmset('tester_3', {
'email': '*****@*****.**',
'zip': '91755',
'phone': '555-666-8787'
})
r.hmset('tester_2', {
'email': '*****@*****.**',
'zip': '91785',
'phone': '555-666-8799'
})
log.info('Data is being read from redis')
test_result = r.keys()
log.info(f'{test_result}')
result = r.hgetall('tester_1')
email = result['email']
zip = result['zip']
phone = result['phone']
log.info('Here is are the information of one person')
log.info(f'tester_1 - email = {email}')
log.info(f'tester_1 - zip = {zip}')
log.info(f'tester_1 - phone= {phone}')
except Exception as e:
print(f'Redis error: {e}')
def run_example():
"""
uses non-presistent Redis only (as a cache)
"""
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
log.info('Step 1: connect to Redis')
r = login_database.login_redis_cloud()
log.info('Step 2: cache some data in Redis')
r.set('andy', '*****@*****.**')
log.info('Step 2: now I can read it')
email = r.get('andy')
log.info('But I must know the key')
log.info(f'The results of r.get: {email}')
log.info('Step 3: cache more data in Redis')
r.set('pam', '*****@*****.**')
r.set('fred', '*****@*****.**')
log.info('Step 4: delete from cache')
r.delete('andy')
log.info(f'r.delete means andy is now: {email}')
log.info('Step 7: richer data for a SKU')
r.rpush(1, 'Mark')
r.rpush(1, '27890')
r.rpush(1, '336-980-9878')
r.rpush(2, 'Tom')
r.rpush(2, '94920')
r.rpush(2, '414-215-6674')
r.rpush(3, 'Steve')
r.rpush(3, '27360')
r.rpush(3, '336-688-0961')
r.rpush(4, 'Che')
r.rpush(4, '32456')
r.rpush(4, '331-481-4509')
r.rpush(5, 'Ashley')
r.rpush(5, '91910')
r.rpush(5, '987-234-0978')
r.rpush(6, 'Kelly')
r.rpush(6, '45671')
r.rpush(6, '607-802-8222')
log.info('Step 8: pull some data from the structure')
for i in range(1, 7):
cust_name = r.lindex(i, 0)
cust_zip = r.lindex(i, 1)
cust_phone = r.lindex(i, 2)
log.info(f'{cust_name} zip = {cust_zip}; phone num = {cust_phone}')
except Exception as e:
print(f'Redis error: {e}')
def showoff_databases():
"""
Here we illustrate basic interaction with nosql databases
"""
log = utilities.configure_logger('default', '..//logs//nosql_dev.log')
log.info("Other databases use data embedded in the modules")
simplerick_script.run_example()
def showoff_databases():
"""
Here we illustrate basic interaction with nosql databases
"""
log = utilities.configure_logger('default', '../logs/nosql_dev.log')
log.info("Mongodb example to use data from Donor module, so get it")
donors = mailroom_data.get_donor_data()
mongodb_script.run_example(donors)
def starting_mailroom():
log = utilities.configure_logger('default', '../logs/nosql_dev.log')
mailroom = mailroom_data.get_mailroom_db_data()
log.info("Mailroom database with monogodb")
mongodb_script.run_mailroom(mailroom)
log.info("Mailroom database with redis")
redis_script.run_mailroom()
log.info("Mailroom database with neo4j")
neo4j_script.run_mailroom()
def set_donor_zip_code(donor_name, zip_code):
log = utilities.configure_logger('default', '../logs/redis_script.log')
try:
r = login_database.login_redis_cloud()
r.set(donor_name, zip_code)
email = r.get('andy')
except Exception as e:
print(f'Redis error: {e}')
def showoff_databases():
"""
Here we illustrate basic interaction with nosql databases
"""
log = utilities.configure_logger('default', '../logs/nosql_dev.log')
log.info("Mongodb example to use data from Furniture module, so get it")
furniture = learn_data.get_furniture_data()
mongodb_script.run_example(furniture)
log.info("Other databases use data embedded in the modules")
def get_donor_zip_code(donor_name):
log = utilities.configure_logger('default', '../logs/redis_script.log')
zip_code = ''
try:
r = login_database.login_redis_cloud()
zip_code = r.get(donor_name)
except Exception as e:
print(f'Redis error: {e}')
return zip_code
def database_updates():
"""
Homework assignment for database tasks.
"""
log = utilities.configure_logger('default', '../logs/nosql_assignment.log')
log.info('Running mongodb assignment.')
mongodb_assignment.add_furniture()
log.info('Running redis assignment.')
redis_assignment.add_customers()
log.info('Running neo4j assignment.')
neo4j_assignment.add_people()
""" Flask web app providing shipdb api.
"""
from api_1_0 import api as api_1_0_blueprint
import config
from flask import Flask, redirect, url_for, request, jsonify, render_template
from flask_cors import CORS
from output_manager import BaseOutputManagerActor
from utilities import configure_logger, get_logger
from conductor import start_conductor, shutdown_conductor
app = Flask(__name__)
CORS(app)
config.load(app.config)
configure_logger()
log = get_logger()
app.register_blueprint(api_1_0_blueprint, url_prefix='/api/1')
@app.route('/')
def index():
""" Show the index page.
"""
return render_template('index.html')
@app.route('/shutdown')
def shutdown():
shutdown_conductor()
def test_configure_logger(self, init_config, clean_up):
configure_logger()
assert get_logger() is not None
clean_up()
def test_configure_logger_config_no_init(self):
with pytest.raises(TypeError):
configure_logger()
except pymongo.errors.AutoReconnect as e:
logger.error(e.message)
except Exception as e:
logger.error(e.message)
def add_history(data):
try:
app.config['DB_COLL_HISTORY'].insert_one(data.__dict__)
logger.debug("User data: %s", data.__dict__)
except pymongo.errors.AutoReconnect as e:
logger.error(e.message)
except Exception as e:
logger.error(e.message)
if __name__ == '__main__':
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
logger = logging.getLogger(__name__)
utilities.configure_logger()
server = HTTPServer(WSGIContainer(app))
server.listen(app.config['PING_SERVICE_PORT'])
logger.info('Server listening at port: %s', app.config['PING_SERVICE_PORT'])
logger.info('Starting ping service...')
IOLoop.instance().start()
