def test_import_data(self):
"""
Tests the import_data function
"""
#directory_path = r"C:/Users/Amir G/SP_Python220B_2019/students/amirg/" \
# r"lesson05/assignment/data"
directory_path = r"data"
linear.drop_data()
list1 = linear.import_data(directory_path, 'products.csv',
'customers.csv')
self.assertEqual(list1[0][0], 999)
self.assertEqual(list1[0][1], 0)
self.assertEqual(list1[0][2], 999)
self.assertTrue(list1[0][3] > 0)
self.assertEqual(list1[1][0], 999)
self.assertEqual(list1[1][1], 0)
self.assertEqual(list1[1][2], 999)
self.assertTrue(list1[1][3] > 0)
linear.drop_data()
list2 = linear.import_data(directory_path, 'products.csv',
'nothing.csv')
self.assertEqual(list2[0][0], 999)
self.assertEqual(list2[0][1], 0)
self.assertEqual(list2[0][2], 999)
self.assertTrue(list2[0][3] > 0)
self.assertEqual(len(list2), 1)
def test_import_data(self):
'''test importing data files'''
dbs_cleanup()
LOGGER.info('Test import_data')
result_count, result_errors = import_data('csv_files',
'product_file.csv',
'customer_file.csv',
'rentals_file.csv')
LOGGER.info('test_import_data, result_count = %s, result_errors = %s',
str(result_count), str(result_errors))
self.assertEqual(result_count, (906, 906, 906))
self.assertEqual(result_errors, (0, 0, 0))
# err check filenotfound
result_count, result_errors = import_data('csv_files',
'product_file.csvXXX',
'customer_file.csvXXX',
'rentals_file.csvXXX')
LOGGER.info('test_import_data, result_count = %s, result_errors = %s',
str(result_count), str(result_errors))
self.assertEqual(result_count, (0, 0, 0))
self.assertEqual(result_errors, (1, 1, 1))
LOGGER.info('test_import_data completed')
def test_import_data_rentals_file_not_found(self):
'''testing file not found error for rentals file'''
LOGGER.info('Testing import_data rentals_file_not_found')
import_data('csv_files', 'product_file.csv', 'customer_file.csv',
'rent_file.csv')
self.assertRaises(FileNotFoundError)
drop_dbs()
def test_show_available_products(mongo_database):
d.import_data(mongo_database, "", "products.csv", "customers.csv", "rentals.csv")
result = d.show_available_products(mongo_database)
assert len(result) == 9999
assert "P000001" in result
assert "P010999" not in result
def test_import_data_linear(self):
""" Unit test for the import_data function """
linear.drop_all_collections()
start_time = time.perf_counter()
linear.import_data("../csv_files", "products.csv", "customers.csv")
LOGGER.info("It took %s second(s) to import data in linear",
time.perf_counter() - start_time)
def test_clear_data(mongo_database):
d.import_data(mongo_database, "", "products.csv", "customers.csv", "rentals.csv")
result = sorted(mongo_database.list_collection_names())
assert result == ["customers", "products", "rentals"]
d.clear_data(mongo_database)
result2 = mongo_database.list_collection_names()
assert result2 == []
def test_import_data(self):
with patch("builtins.input", side_effect=("yes", "yes")):
a = ln.import_data("./Data/data_files_n=40", "products", "customers", "rentals")
self.assertEqual(a[0][0:4], (("products",40, 0, 40)))
self.assertEqual(a[1][0:4], (("customers",40, 0, 40)))
self.assertEqual(a[2][0:4], (("rentals",40, 0, 40)))
b = ln.import_data("Willy", "wonka", "and", "the choccolate factory")
self.assertEqual(b[0][0:4], (("products",0, 0, 0)))
self.assertEqual(b[1][0:4], (("customers",0, 0, 0)))
self.assertEqual(b[2][0:4], (("rentals",0, 0, 0)))
def setUp(self):
''' test setup '''
LOGGER.info('setup start')
dropped_status = dbs_cleanup()
LOGGER.info('test setup, dbs cleanup status = %s', dropped_status)
''' import data files '''
LOGGER.info('setup import data file to database')
import_data('csv_files', 'product_file.csv', 'customer_file.csv',
'rentals_file.csv')
LOGGER.info('test setup completed')
def test_show_rentals(mongo_database):
d.import_data(mongo_database, "", "products.csv", "customers.csv", "rentals.csv")
result = d.show_rentals(mongo_database, "P000004")
expected = {'C000002': {'first_name': 'Blanca', 'last_name': 'Bashirian', 'address': '0193 Malvina Lake',
'phone_number': '(240)014-9496 x08349', 'email': '*****@*****.**',
'status': 'Active', 'credit_limit': '689'},
'C000004': {'first_name': 'Mittie', 'last_name': 'Turner', 'address': '996 Lorenza Points',
'phone_number': '1-324-023-8861 x025', 'email': '*****@*****.**',
'status': 'Active', 'credit_limit': '565'}}
assert len(result) == 2
assert list(result.keys()) == ["C000002", "C000004"]
assert result == expected
def test_import_thousand_data_without_lock(self):
""" Test import_thousand_data() function parallel """
db_para.LOGGER.info('\n\n--- Start Test parallel without Lock ---')
# generate the existing records.
actual_res = db_linear.import_data(self.csv_file, 'products.csv',
'customers.csv', 'rentals.csv')
pr = cProfile.Profile()
pr.enable()
actual_res = db_para.import_thousand_data(self.csv_file,
self.product_file,
self.customer_file,
self.rental_file)
self.assertEqual(actual_res[0][0], 100000)
self.assertEqual(actual_res[0][1], 5)
self.assertEqual(actual_res[0][2], 100005)
self.assertEqual(actual_res[1][0], 100000)
self.assertEqual(actual_res[1][1], 5)
self.assertEqual(actual_res[1][2], 100005)
db_para.LOGGER.info('--- End Test parallel without Lock ---')
# set up the profiling
pr.disable()
with open('parallel_without_lock.res', 'w') as file:
ps = pstats.Stats(pr, stream=file)
#ps.strip_dirs().sort_stats('time').print_stats()
ps.strip_dirs().sort_stats('cumulative').print_stats(20)
def _all_answers():
"""
underscore required on fixture to eliminate an
invalid redefinition warning
from pytest pylint
"""
answers_linear = linear.import_data('data', 'customers.csv',
'products.csv')
answers_parallel = parallel.import_data('data', 'customers.csv',
'products.csv')
return ({
"processed": answers_linear[0][0],
"count_prior": answers_linear[0][1],
"count_new": answers_linear[0][2],
"elapsed": answers_linear[0][3]
}, {
"processed": answers_linear[1][0],
"count_prior": answers_linear[1][1],
"count_new": answers_linear[1][2],
"elapsed": answers_linear[1][3]
}, {
"processed": answers_parallel[0][0],
"count_prior": answers_parallel[0][1],
"count_new": answers_parallel[0][2],
"elapsed": answers_parallel[0][3]
}, {
"processed": answers_parallel[1][0],
"count_prior": answers_parallel[1][1],
"count_new": answers_parallel[1][2],
"elapsed": answers_parallel[1][3]
})
def test_import_data(self):
"""Testing of the import data"""
linear.drop_collections()
customer, product = linear.import_data(DIRECTORY_NAME, "customers.csv",
"products.csv", "rentals.csv")
actual_output = customer[0:3], product[0:3]
expected_output = ((0, 900, 900), (0, 984, 984))
assert actual_output == expected_output
def test_import_data(self):
linear.drop_data()
good_import = linear.import_data('data', 'products.csv',
'customers.csv', 'rentals.csv')
self.assertEqual(good_import[0][0], 9999)
self.assertEqual(good_import[0][1], 0)
self.assertEqual(good_import[0][2], 9999)
self.assertEqual(type(good_import[0][3]), type(float(1.02)))
self.assertEqual(good_import[0], good_import[1])
def test_import_data(self):
"""Tests importing of data from csv files"""
current_dir = os.getcwd()
path = current_dir + r"\data_files"
clear_db()
import_test = import_data(path, "customer_data.csv",
"product_data.csv")
self.assertEqual((import_test[0][:3], import_test[1][:3]),
((1000, 0, 1000), (1000, 0, 1000)))
def test_import_data(self):
"""
Validates that import data reports failures and successes accurately
"""
with patch("linear.ingest_file") as ingest_mock:
ingest_mock.return_value = ((1, 2, 3), (4, 5, 6))
values = Database.import_data(".", ".", ".", ".")
self.assertEqual(2, len(values[0]))
self.assertEqual(2, len(values[1]))
def test_drop_data(self):
linear.drop_data()
# Populate the database
linear.import_data('data', 'products.csv', 'customers.csv', 'test.csv')
mongo = linear.MongoDBConnection()
with mongo:
database = mongo.connection.media
record_count = database['rentals'].count_documents({})
self.assertEqual(record_count, 4)
# Make sure the database successfully clears
linear.drop_data()
with mongo:
database = mongo.connection.media
record_count = database['rentals'].count_documents({})
self.assertEqual(record_count, 0)
def test_1_import(self):
"""
Test that the records are successfully imported.
"""
# Start fresh so we don't mess up all our tests...
linear.drop_data()
self.assertEqual(linear.show_available_products(), {})
# Check for proper error handling of import_data()
with self.assertRaises(FileNotFoundError):
result = linear.import_data('data2', 'p.csv', 'c.csv', 'r.csv')
# These tests are hard-coded to the expected values from the incluced CSV's.
# Your results may vary if the data sets change. :)
result = linear.import_data('data', 'products.csv', 'customers.csv', 'rentals.csv')
self.assertEqual(result[0][0], 1000)
self.assertEqual(result[0][1], 0)
self.assertEqual(result[0][2], 1000)
self.assertEqual(result[1][0], 1000)
self.assertEqual(result[1][1], 0)
self.assertEqual(result[1][2], 1000)
linear_available = linear.show_available_products()
# Run the parallel again -- we're going to grab the results of show_available_products()
# to consistency check later.
parallel.drop_data()
self.assertEqual(parallel.show_available_products(), {})
with self.assertRaises(FileNotFoundError):
result = linear.import_data('data2', 'p.csv', 'c.csv', 'r.csv')
result = parallel.import_data('data', 'products.csv', 'customers.csv', 'rentals.csv')
self.assertEqual(result[0][0], 1000)
self.assertEqual(result[0][1], 0)
self.assertEqual(result[0][2], 1000)
self.assertEqual(result[1][0], 1000)
self.assertEqual(result[1][1], 0)
self.assertEqual(result[1][2], 1000)
self.assertEqual(linear_available, parallel.show_available_products())
def test_import_data(mongo_database):
result = d.import_data(mongo_database, "", "products.csv", "customers.csv", "rentals.csv")
assert result[0][0] == 9999
assert result[0][1] == 0
assert result[0][2] == 9999
assert isinstance(result[0][3], float)
assert result[1][0] == 9999
assert result[1][1] == 0
assert result[1][2] == 9999
assert isinstance(result[1][3], float)
def test_import_data(self):
'''testing import_data function'''
LOGGER.info('Testing import_data')
data = import_data('csv_files', 'product_file.csv',
'customer_file.csv', 'rentals_file.csv')
LOGGER.info('Testing import_data return')
self.assertEqual(data[0], (900, 900))
self.assertEqual(data[1], (0, 0))
self.assertEqual(data[2], (900, 900, 900))
LOGGER.info('test FileNotFoundError')
self.assertEqual(data[4], (2, 2, 1))
LOGGER.info('import_data returns successful')
drop_dbs()
def test_import_data(self):
"""
Test import_data function
"""
linear.drop_db()
actual_1_raw = linear.import_data('', 'data/products.csv',
'data/customers.csv',
'data/rentals.csv')
actual_1 = [actual_1_raw[0][:3], actual_1_raw[1][:3]]
expected_1 = [(1000, 0, 1000), (1000, 0, 1000)]
self.assertEqual(actual_1, expected_1)
def test_import_data(set_up_connection):
"""
Tests main function which intakes a dict of names
and csv files, and creates a collection based on
value and adds documents from csv file to that collection.
This test tests that Main returns the three collections.
(products_db, rentals_db, customers_db)
"""
customers = 'customers.csv'
products = 'products.csv'
db = set_up_connection
result = import_data(db, customers, products)
assert result[0][:3] == (10, 0, 10)
assert result[1][:3] == (10, 0, 10)
def test_import_data(self):
""" test import_data """
logger.info('Test Linear import...')
start = time.perf_counter()
result = import_data(self.data_folder, self.product_file,
self.customer_file, self.rental_file)
end = time.perf_counter()
logger.info(f'Linear import takes {end-start} seconds in total.')
logger.info(f"Customer tuple: {result[0]} ")
logger.info(f"Product tuple: {result[1]} ")
self.assertEqual(result[0][0], 1000)
self.assertEqual(result[0][1], 0)
self.assertEqual(result[0][2], 1000)
self.assertEqual(result[1][0], 1000)
self.assertEqual(result[1][1], 0)
self.assertEqual(result[1][2], 1000)
def test_import_data(self):
""" Test import_data() function """
#db_linear.LOGGER.info('--- Start Test import_old_data() ---')
db_linear.import_data(self.csv_file, self.product_file,
self.customer_file, self.rental_file)
def setUp(self):
'''importing csv files for tests'''
LOGGER.info('importing csv files for drop_dbs')
import_data('csv_files', 'product_file.csv', 'customer_file.csv',
'rentals_file.csv')
LOGGER.info('csv files successfully imported')
"""
Run linear and parallel imports and provide time data.
"""
import datetime
import linear
import parallel
linear.drop_data()
print("--- Running linear import ---")
start_time = datetime.datetime.now()
results = linear.import_data('data', 'products.csv', 'customers.csv', 'rentals.csv')
print("Products:")
print(" Records: {} processed, {} imported".format(results[0][0], results[0][2] - results[0][1]))
print(" Runtime: {} seconds".format(results[0][3]))
print()
print("Customers:")
print(" Records: {} processed, {} imported".format(results[1][0], results[1][2] - results[1][1]))
print(" Runtime: {} seconds".format(results[1][3]))
print()
print("Linear import complete in {} seconds.".format(
(datetime.datetime.now() - start_time).total_seconds()))
print()
parallel.drop_data()
print("--- Running parallel import ---")
start_time = datetime.datetime.now()
results = parallel.import_data('data', 'products.csv', 'customers.csv', 'rentals.csv')
print("Products:")
print(" Records: {} processed, {} imported".format(results[0][0], results[0][2] - results[0][1]))
def test_import_data_error(self):
"""Test error handling in import data function."""
linear.clear_database()
import_result = linear.import_data('wrong', 'place', 'to', 'look')
self.assertEqual(len(import_result), 3)
self.assertEqual(import_result[0][0], 0)
def test_import_data_exception(self):
""" test import_data function when exceptions occur"""
import_data('data', 'product.csv', 'customer.csv', 'rental.csv')
def test_show_available_products(self):
with patch("builtins.input", side_effect=("yes")):
ln.import_data("./Data/data_files_n=40", "products", "customers", "rentals")
a = ln.show_available_products()
self.assertEqual(a["prd1"][" Description"], ' 60-inch TV stand')
def test_import_data(self):
"""Test import data function."""
linear.clear_database()
import_result= linear.import_data(self.data_folder, *self.data_files)
self.assertEqual(len(import_result), 3)
self.assertEqual(import_result[0][0], 1000)
