def test_show_available_products(mongo_database):
d.import_data(mongo_database, "", "products.csv", "customers.csv", "rentals.csv")
result = d.show_available_products(mongo_database)
assert len(result) == 9999
assert "P000001" in result
assert "P010999" not in result
def test_import_data(self):
"""
Tests the import_data function
"""
#directory_path = r"C:/Users/Amir G/SP_Python220B_2019/students/amirg/" \
# r"lesson05/assignment/data"
directory_path = r"data"
parallel.drop_data()
list1 = parallel.import_data(directory_path, 'products.csv',
'customers.csv')
self.assertEqual(list1[0][0], 999)
self.assertEqual(list1[0][1], 0)
self.assertEqual(list1[0][2], 999)
self.assertTrue(list1[0][3] > 0)
self.assertEqual(list1[1][0], 999)
self.assertEqual(list1[1][1], 0)
self.assertEqual(list1[1][2], 999)
self.assertTrue(list1[1][3] > 0)
parallel.drop_data()
list2 = parallel.import_data(directory_path, 'products.csv',
'nothing.csv')
self.assertEqual(list2[0][0], 999)
self.assertEqual(list2[0][1], 0)
self.assertEqual(list2[0][2], 999)
self.assertTrue(list2[0][3] > 0)
self.assertEqual(list2[1], None)
def test_import_data(self):
'''test importing data files'''
dbs_cleanup()
LOGGER.info('Test import_data')
result_count, result_errors = import_data('csv_files',
'product_file.csv',
'customer_file.csv',
'rentals_file.csv')
LOGGER.info('test_import_data, result_count = %s, result_errors = %s',
str(result_count), str(result_errors))
self.assertEqual(result_count, (906, 906, 906))
self.assertEqual(result_errors, (0, 0, 0))
# err check filenotfound
result_count, result_errors = import_data('csv_files',
'product_file.csvXXX',
'customer_file.csvXXX',
'rentals_file.csvXXX')
LOGGER.info('test_import_data, result_count = %s, result_errors = %s',
str(result_count), str(result_errors))
self.assertEqual(result_count, (0, 0, 0))
self.assertEqual(result_errors, (1, 1, 1))
LOGGER.info('test_import_data completed')
def test_clear_data(mongo_database):
d.import_data(mongo_database, "", "products.csv", "customers.csv", "rentals.csv")
result = sorted(mongo_database.list_collection_names())
assert result == ["customers", "products", "rentals"]
d.clear_data(mongo_database)
result2 = mongo_database.list_collection_names()
assert result2 == []
def test_import_data_parallel(self):
""" Unit test for the import_data function """
parallel.drop_all_collections()
start_time = time.perf_counter()
parallel.import_data("../csv_files", "products.csv", "customers.csv")
LOGGER.info(
"It took %s second(s) to import and process data in parallel",
time.perf_counter() - start_time)
def test_show_available_products(_mongo_database, _show_available_products):
''' available products '''
p.import_data(_mongo_database, '../data', '../data/product.csv',
'../data/customer.csv', '../data/rental.csv')
students_response = p.show_available_products(_mongo_database)
assert len(students_response) == 9999
assert "P000001" in students_response
assert "P010999" not in students_response
def test_import_data(self):
with patch("builtins.input", side_effect=("yes", "yes")):
a = db.import_data("./Data/data_files_n=40", "products", "customers", "rentals")
self.assertEqual(a[0][0:4], (("products",40, 0, 40)))
self.assertEqual(a[1][0:4], (("customers",40, 0, 40)))
self.assertEqual(a[2][0:4], (("rentals",40, 0, 40)))
b = db.import_data("Willy", "wonka", "and", "the choccolate factory")
self.assertEqual(b[0][0:4], (("products",0, 0, 0)))
self.assertEqual(b[1][0:4], (("customers",0, 0, 0)))
self.assertEqual(b[2][0:4], (("rentals",0, 0, 0)))
def test_show_rentals(mongo_database):
d.import_data(mongo_database, "", "products.csv", "customers.csv", "rentals.csv")
result = d.show_rentals(mongo_database, "P000004")
expected = {'C000002': {'first_name': 'Blanca', 'last_name': 'Bashirian', 'address': '0193 Malvina Lake',
'phone_number': '(240)014-9496 x08349', 'email': '*****@*****.**',
'status': 'Active', 'credit_limit': '689'},
'C000004': {'first_name': 'Mittie', 'last_name': 'Turner', 'address': '996 Lorenza Points',
'phone_number': '1-324-023-8861 x025', 'email': '*****@*****.**',
'status': 'Active', 'credit_limit': '565'}}
assert len(result) == 2
assert list(result.keys()) == ["C000002", "C000004"]
assert result == expected
def setUp(self):
''' test setup '''
LOGGER.info('setup start')
dropped_status = dbs_cleanup()
LOGGER.info('test setup, dbs cleanup status = %s', dropped_status)
''' import data files '''
LOGGER.info('setup import data file to database')
import_data('csv_files',
'product_file.csv',
'customer_file.csv',
'rentals_file.csv')
LOGGER.info('test setup completed')
def test_import_data(self):
"""
Validates that import data returns the correct format.
"""
with patch("parallel.ingest_file"):
values = ParallelDB.import_data(".", ".", ".", ".")
self.assertEqual(type([]), type(values))
def _all_answers():
"""
underscore required on fixture to eliminate an
invalid redefinition warning
from pytest pylint
"""
answers_linear = linear.import_data('data', 'customers.csv',
'products.csv')
answers_parallel = parallel.import_data('data', 'customers.csv',
'products.csv')
return ({
"processed": answers_linear[0][0],
"count_prior": answers_linear[0][1],
"count_new": answers_linear[0][2],
"elapsed": answers_linear[0][3]
}, {
"processed": answers_linear[1][0],
"count_prior": answers_linear[1][1],
"count_new": answers_linear[1][2],
"elapsed": answers_linear[1][3]
}, {
"processed": answers_parallel[0][0],
"count_prior": answers_parallel[0][1],
"count_new": answers_parallel[0][2],
"elapsed": answers_parallel[0][3]
}, {
"processed": answers_parallel[1][0],
"count_prior": answers_parallel[1][1],
"count_new": answers_parallel[1][2],
"elapsed": answers_parallel[1][3]
})
def test_import_data(self):
"""Testing of the import data"""
parallel.drop_collections()
customer, product = parallel.import_data(DIRECTORY_NAME,
"customers.csv",
"products.csv", "rentals.csv")
actual_output = customer[0:3], product[0:3]
expected_output = ((0, 900, 900), (0, 984, 984))
assert actual_output == expected_output
def test_import_data(self):
parallel.drop_data()
good_import = parallel.import_data('data', 'products.csv',
'customers.csv', 'rentals.csv')
self.assertEqual(good_import[0][0], 9999)
self.assertEqual(good_import[0][1], 0)
self.assertEqual(good_import[0][2], 9999)
self.assertEqual(type(good_import[0][3]), type(float(1.02)))
self.assertEqual(good_import[0], good_import[1])
def test_import_data(self):
"""test the base import data function"""
build_test_csvs()
database = database_setup()
test = import_data(PATH, 'products.csv', 'customers.csv',
'rentals.csv')
self.assertEqual(test, ((1, 1, 1), (0, 0, 0)))
database.test.drop()
delete_test_csv()
def test_import_data(self):
"""Tests importing of data from csv files"""
current_dir = os.getcwd()
path = current_dir + r"\data_files"
clear_db()
import_test = import_data(path, "customer_data.csv",
"product_data.csv")
self.assertEqual((import_test[0][:3], import_test[1][:3]),
((1000, 0, 1000), (1000, 0, 1000)))
def test_show_available_products(self):
"""show user info that have rented"""
build_test_csvs()
database = database_setup()
import_data(PATH, 'products.csv', 'customers.csv', 'rentals.csv')
products = import_csv(PATH + 'products.csv')['data']
for row in products:
row['quantity_available'] = int(row['quantity_available'])
csv_results = [
next(prod for prod in products
if int(prod['quantity_available']) > 0)
]
self.assertEqual(
show_available_products(),
{product.pop('product_id'): product
for product in csv_results})
database.test.drop()
delete_test_csv()
def test_show_rentals(self):
"""show user info that have rented"""
build_test_csvs()
database = database_setup()
import_data(PATH, 'products.csv', 'customers.csv', 'rentals.csv')
customers = import_csv(PATH + 'customers.csv')['data']
rentals = import_csv(PATH + 'rentals.csv')['data']
for rental in rentals:
query_results = show_rentals(rental['product_id'])
csv_results = [
next(cust for cust in customers
if cust["user_id"] == rental['customer_id'])
]
self.assertEqual(query_results, {
customer.pop('user_id'): customer
for customer in csv_results
})
database.test.drop()
delete_test_csv()
def test_import_data(mongo_database):
result = d.import_data(mongo_database, "", "products.csv", "customers.csv", "rentals.csv")
assert result[0][0] == 9999
assert result[0][1] == 0
assert result[0][2] == 9999
assert isinstance(result[0][3], float)
assert result[1][0] == 9999
assert result[1][1] == 0
assert result[1][2] == 9999
assert isinstance(result[1][3], float)
def test_drop_data(self):
parallel.drop_data()
# Populate the database
parallel.import_data('data', 'products.csv', 'customers.csv',
'test.csv')
mongo = parallel.MongoDBConnection()
with mongo:
database = mongo.connection.media
record_count = database['rentals'].count_documents({})
self.assertEqual(record_count, 4)
# Make sure the database successfully clears
parallel.drop_data()
with mongo:
database = mongo.connection.media
record_count = database['rentals'].count_documents({})
self.assertEqual(record_count, 0)
def test_import_data(_mongo_database):
"""Import"""
result = p.import_data(_mongo_database, "", "../data/product.csv",
"../data/customer.csv", "../data/rental.csv")
assert result[0][0] == 9999
assert result[0][1] == 0
assert result[0][2] == 9999
assert isinstance(result[0][3], float)
assert result[1][0] == 9999
assert result[1][1] == 0
assert result[1][2] == 9999
assert isinstance(result[1][3], float)
def test_import_data(self):
"""
Test import_data function
"""
parallel.drop_db()
actual_1_raw = parallel.import_data('', 'data/products.csv',
'data/customers.csv',
'data/rentals.csv')
actual_1 = [actual_1_raw[0][:3], actual_1_raw[1][:3]]
expected_1 = [(1000, 0, 1000), (1000, 0, 1000)]
self.assertEqual(actual_1, expected_1)
def test_import_data(self):
clear_db()
'''tests csv files are imported correctly with
correct tuples being returned'''
test_import = import_data('csv_files', 'product_file.csv',
'customer_file.csv', 'rental_file.csv')
test_import_cust = test_import[0]
test_import_prod = test_import[1]
self.assertEqual(test_import_cust[0], 1000)
self.assertEqual(test_import_cust[1], 0)
self.assertEqual(test_import_cust[2], 1000)
self.assertEqual(test_import_prod[0], 1000)
self.assertEqual(test_import_prod[1], 0)
self.assertEqual(test_import_prod[2], 1000)
def test_import_data(set_up_connection):
"""
Tests main function which intakes a dict of names
and csv files, and creates a collection based on
value and adds documents from csv file to that collection.
This test tests that Main returns the three collections.
(products_db, rentals_db, customers_db)
"""
customers = 'customers.csv'
products = 'products.csv'
db = set_up_connection
result = import_data(db, customers, products)
assert result[0][:3] == (10, 0, 10)
assert result[1][:3] == (10, 0, 10)
def test_1_import(self):
"""
Test that the records are successfully imported.
"""
# Start fresh so we don't mess up all our tests...
linear.drop_data()
self.assertEqual(linear.show_available_products(), {})
# Check for proper error handling of import_data()
with self.assertRaises(FileNotFoundError):
result = linear.import_data('data2', 'p.csv', 'c.csv', 'r.csv')
# These tests are hard-coded to the expected values from the incluced CSV's.
# Your results may vary if the data sets change. :)
result = linear.import_data('data', 'products.csv', 'customers.csv', 'rentals.csv')
self.assertEqual(result[0][0], 1000)
self.assertEqual(result[0][1], 0)
self.assertEqual(result[0][2], 1000)
self.assertEqual(result[1][0], 1000)
self.assertEqual(result[1][1], 0)
self.assertEqual(result[1][2], 1000)
linear_available = linear.show_available_products()
# Run the parallel again -- we're going to grab the results of show_available_products()
# to consistency check later.
parallel.drop_data()
self.assertEqual(parallel.show_available_products(), {})
with self.assertRaises(FileNotFoundError):
result = linear.import_data('data2', 'p.csv', 'c.csv', 'r.csv')
result = parallel.import_data('data', 'products.csv', 'customers.csv', 'rentals.csv')
self.assertEqual(result[0][0], 1000)
self.assertEqual(result[0][1], 0)
self.assertEqual(result[0][2], 1000)
self.assertEqual(result[1][0], 1000)
self.assertEqual(result[1][1], 0)
self.assertEqual(result[1][2], 1000)
self.assertEqual(linear_available, parallel.show_available_products())
def test_show_available_products(self):
with patch("builtins.input", side_effect=("yes")):
db.import_data("./Data/data_files_n=40", "products", "customers", "rentals")
a = db.show_available_products()
self.assertEqual(a["prd1"][" Description"], ' 60-inch TV stand')
print("--- Running linear import ---")
start_time = datetime.datetime.now()
results = linear.import_data('data', 'products.csv', 'customers.csv', 'rentals.csv')
print("Products:")
print(" Records: {} processed, {} imported".format(results[0][0], results[0][2] - results[0][1]))
print(" Runtime: {} seconds".format(results[0][3]))
print()
print("Customers:")
print(" Records: {} processed, {} imported".format(results[1][0], results[1][2] - results[1][1]))
print(" Runtime: {} seconds".format(results[1][3]))
print()
print("Linear import complete in {} seconds.".format(
(datetime.datetime.now() - start_time).total_seconds()))
print()
parallel.drop_data()
print("--- Running parallel import ---")
start_time = datetime.datetime.now()
results = parallel.import_data('data', 'products.csv', 'customers.csv', 'rentals.csv')
print("Products:")
print(" Records: {} processed, {} imported".format(results[0][0], results[0][2] - results[0][1]))
print(" Runtime: {} seconds".format(results[0][3]))
print()
print("Customers:")
print(" Records: {} processed, {} imported".format(results[1][0], results[1][2] - results[1][1]))
print(" Runtime: {} seconds".format(results[1][3]))
print()
print("Parallel import complete in {} seconds.".format(
(datetime.datetime.now() - start_time).total_seconds()))
def test_show_rentals(_mongo_database, _show_rentals):
""" rentals """
p.import_data(_mongo_database, '../data', '../data/product.csv',
'../data/customer.csv', '../data/rental.csv')
students_response = p.show_rentals(_mongo_database, "P000005")
assert students_response == _show_rentals
