def test_import_data(self): """ Tests the import_data function """ #directory_path = r"C:/Users/Amir G/SP_Python220B_2019/students/amirg/" \ # r"lesson05/assignment/data" directory_path = r"data" linear.drop_data() list1 = linear.import_data(directory_path, 'products.csv', 'customers.csv') self.assertEqual(list1[0][0], 999) self.assertEqual(list1[0][1], 0) self.assertEqual(list1[0][2], 999) self.assertTrue(list1[0][3] > 0) self.assertEqual(list1[1][0], 999) self.assertEqual(list1[1][1], 0) self.assertEqual(list1[1][2], 999) self.assertTrue(list1[1][3] > 0) linear.drop_data() list2 = linear.import_data(directory_path, 'products.csv', 'nothing.csv') self.assertEqual(list2[0][0], 999) self.assertEqual(list2[0][1], 0) self.assertEqual(list2[0][2], 999) self.assertTrue(list2[0][3] > 0) self.assertEqual(len(list2), 1)
def test_import_data(self): '''test importing data files''' dbs_cleanup() LOGGER.info('Test import_data') result_count, result_errors = import_data('csv_files', 'product_file.csv', 'customer_file.csv', 'rentals_file.csv') LOGGER.info('test_import_data, result_count = %s, result_errors = %s', str(result_count), str(result_errors)) self.assertEqual(result_count, (906, 906, 906)) self.assertEqual(result_errors, (0, 0, 0)) # err check filenotfound result_count, result_errors = import_data('csv_files', 'product_file.csvXXX', 'customer_file.csvXXX', 'rentals_file.csvXXX') LOGGER.info('test_import_data, result_count = %s, result_errors = %s', str(result_count), str(result_errors)) self.assertEqual(result_count, (0, 0, 0)) self.assertEqual(result_errors, (1, 1, 1)) LOGGER.info('test_import_data completed')
def test_import_data_rentals_file_not_found(self): '''testing file not found error for rentals file''' LOGGER.info('Testing import_data rentals_file_not_found') import_data('csv_files', 'product_file.csv', 'customer_file.csv', 'rent_file.csv') self.assertRaises(FileNotFoundError) drop_dbs()
def test_show_available_products(mongo_database): d.import_data(mongo_database, "", "products.csv", "customers.csv", "rentals.csv") result = d.show_available_products(mongo_database) assert len(result) == 9999 assert "P000001" in result assert "P010999" not in result
def test_import_data_linear(self): """ Unit test for the import_data function """ linear.drop_all_collections() start_time = time.perf_counter() linear.import_data("../csv_files", "products.csv", "customers.csv") LOGGER.info("It took %s second(s) to import data in linear", time.perf_counter() - start_time)
def test_clear_data(mongo_database): d.import_data(mongo_database, "", "products.csv", "customers.csv", "rentals.csv") result = sorted(mongo_database.list_collection_names()) assert result == ["customers", "products", "rentals"] d.clear_data(mongo_database) result2 = mongo_database.list_collection_names() assert result2 == []
def test_import_data(self): with patch("builtins.input", side_effect=("yes", "yes")): a = ln.import_data("./Data/data_files_n=40", "products", "customers", "rentals") self.assertEqual(a[0][0:4], (("products",40, 0, 40))) self.assertEqual(a[1][0:4], (("customers",40, 0, 40))) self.assertEqual(a[2][0:4], (("rentals",40, 0, 40))) b = ln.import_data("Willy", "wonka", "and", "the choccolate factory") self.assertEqual(b[0][0:4], (("products",0, 0, 0))) self.assertEqual(b[1][0:4], (("customers",0, 0, 0))) self.assertEqual(b[2][0:4], (("rentals",0, 0, 0)))
def setUp(self): ''' test setup ''' LOGGER.info('setup start') dropped_status = dbs_cleanup() LOGGER.info('test setup, dbs cleanup status = %s', dropped_status) ''' import data files ''' LOGGER.info('setup import data file to database') import_data('csv_files', 'product_file.csv', 'customer_file.csv', 'rentals_file.csv') LOGGER.info('test setup completed')
def test_show_rentals(mongo_database): d.import_data(mongo_database, "", "products.csv", "customers.csv", "rentals.csv") result = d.show_rentals(mongo_database, "P000004") expected = {'C000002': {'first_name': 'Blanca', 'last_name': 'Bashirian', 'address': '0193 Malvina Lake', 'phone_number': '(240)014-9496 x08349', 'email': '*****@*****.**', 'status': 'Active', 'credit_limit': '689'}, 'C000004': {'first_name': 'Mittie', 'last_name': 'Turner', 'address': '996 Lorenza Points', 'phone_number': '1-324-023-8861 x025', 'email': '*****@*****.**', 'status': 'Active', 'credit_limit': '565'}} assert len(result) == 2 assert list(result.keys()) == ["C000002", "C000004"] assert result == expected
def test_import_thousand_data_without_lock(self): """ Test import_thousand_data() function parallel """ db_para.LOGGER.info('\n\n--- Start Test parallel without Lock ---') # generate the existing records. actual_res = db_linear.import_data(self.csv_file, 'products.csv', 'customers.csv', 'rentals.csv') pr = cProfile.Profile() pr.enable() actual_res = db_para.import_thousand_data(self.csv_file, self.product_file, self.customer_file, self.rental_file) self.assertEqual(actual_res[0][0], 100000) self.assertEqual(actual_res[0][1], 5) self.assertEqual(actual_res[0][2], 100005) self.assertEqual(actual_res[1][0], 100000) self.assertEqual(actual_res[1][1], 5) self.assertEqual(actual_res[1][2], 100005) db_para.LOGGER.info('--- End Test parallel without Lock ---') # set up the profiling pr.disable() with open('parallel_without_lock.res', 'w') as file: ps = pstats.Stats(pr, stream=file) #ps.strip_dirs().sort_stats('time').print_stats() ps.strip_dirs().sort_stats('cumulative').print_stats(20)
def _all_answers(): """ underscore required on fixture to eliminate an invalid redefinition warning from pytest pylint """ answers_linear = linear.import_data('data', 'customers.csv', 'products.csv') answers_parallel = parallel.import_data('data', 'customers.csv', 'products.csv') return ({ "processed": answers_linear[0][0], "count_prior": answers_linear[0][1], "count_new": answers_linear[0][2], "elapsed": answers_linear[0][3] }, { "processed": answers_linear[1][0], "count_prior": answers_linear[1][1], "count_new": answers_linear[1][2], "elapsed": answers_linear[1][3] }, { "processed": answers_parallel[0][0], "count_prior": answers_parallel[0][1], "count_new": answers_parallel[0][2], "elapsed": answers_parallel[0][3] }, { "processed": answers_parallel[1][0], "count_prior": answers_parallel[1][1], "count_new": answers_parallel[1][2], "elapsed": answers_parallel[1][3] })
def test_import_data(self): """Testing of the import data""" linear.drop_collections() customer, product = linear.import_data(DIRECTORY_NAME, "customers.csv", "products.csv", "rentals.csv") actual_output = customer[0:3], product[0:3] expected_output = ((0, 900, 900), (0, 984, 984)) assert actual_output == expected_output
def test_import_data(self): linear.drop_data() good_import = linear.import_data('data', 'products.csv', 'customers.csv', 'rentals.csv') self.assertEqual(good_import[0][0], 9999) self.assertEqual(good_import[0][1], 0) self.assertEqual(good_import[0][2], 9999) self.assertEqual(type(good_import[0][3]), type(float(1.02))) self.assertEqual(good_import[0], good_import[1])
def test_import_data(self): """Tests importing of data from csv files""" current_dir = os.getcwd() path = current_dir + r"\data_files" clear_db() import_test = import_data(path, "customer_data.csv", "product_data.csv") self.assertEqual((import_test[0][:3], import_test[1][:3]), ((1000, 0, 1000), (1000, 0, 1000)))
def test_import_data(self): """ Validates that import data reports failures and successes accurately """ with patch("linear.ingest_file") as ingest_mock: ingest_mock.return_value = ((1, 2, 3), (4, 5, 6)) values = Database.import_data(".", ".", ".", ".") self.assertEqual(2, len(values[0])) self.assertEqual(2, len(values[1]))
def test_drop_data(self): linear.drop_data() # Populate the database linear.import_data('data', 'products.csv', 'customers.csv', 'test.csv') mongo = linear.MongoDBConnection() with mongo: database = mongo.connection.media record_count = database['rentals'].count_documents({}) self.assertEqual(record_count, 4) # Make sure the database successfully clears linear.drop_data() with mongo: database = mongo.connection.media record_count = database['rentals'].count_documents({}) self.assertEqual(record_count, 0)
def test_1_import(self): """ Test that the records are successfully imported. """ # Start fresh so we don't mess up all our tests... linear.drop_data() self.assertEqual(linear.show_available_products(), {}) # Check for proper error handling of import_data() with self.assertRaises(FileNotFoundError): result = linear.import_data('data2', 'p.csv', 'c.csv', 'r.csv') # These tests are hard-coded to the expected values from the incluced CSV's. # Your results may vary if the data sets change. :) result = linear.import_data('data', 'products.csv', 'customers.csv', 'rentals.csv') self.assertEqual(result[0][0], 1000) self.assertEqual(result[0][1], 0) self.assertEqual(result[0][2], 1000) self.assertEqual(result[1][0], 1000) self.assertEqual(result[1][1], 0) self.assertEqual(result[1][2], 1000) linear_available = linear.show_available_products() # Run the parallel again -- we're going to grab the results of show_available_products() # to consistency check later. parallel.drop_data() self.assertEqual(parallel.show_available_products(), {}) with self.assertRaises(FileNotFoundError): result = linear.import_data('data2', 'p.csv', 'c.csv', 'r.csv') result = parallel.import_data('data', 'products.csv', 'customers.csv', 'rentals.csv') self.assertEqual(result[0][0], 1000) self.assertEqual(result[0][1], 0) self.assertEqual(result[0][2], 1000) self.assertEqual(result[1][0], 1000) self.assertEqual(result[1][1], 0) self.assertEqual(result[1][2], 1000) self.assertEqual(linear_available, parallel.show_available_products())
def test_import_data(mongo_database): result = d.import_data(mongo_database, "", "products.csv", "customers.csv", "rentals.csv") assert result[0][0] == 9999 assert result[0][1] == 0 assert result[0][2] == 9999 assert isinstance(result[0][3], float) assert result[1][0] == 9999 assert result[1][1] == 0 assert result[1][2] == 9999 assert isinstance(result[1][3], float)
def test_import_data(self): '''testing import_data function''' LOGGER.info('Testing import_data') data = import_data('csv_files', 'product_file.csv', 'customer_file.csv', 'rentals_file.csv') LOGGER.info('Testing import_data return') self.assertEqual(data[0], (900, 900)) self.assertEqual(data[1], (0, 0)) self.assertEqual(data[2], (900, 900, 900)) LOGGER.info('test FileNotFoundError') self.assertEqual(data[4], (2, 2, 1)) LOGGER.info('import_data returns successful') drop_dbs()
def test_import_data(self): """ Test import_data function """ linear.drop_db() actual_1_raw = linear.import_data('', 'data/products.csv', 'data/customers.csv', 'data/rentals.csv') actual_1 = [actual_1_raw[0][:3], actual_1_raw[1][:3]] expected_1 = [(1000, 0, 1000), (1000, 0, 1000)] self.assertEqual(actual_1, expected_1)
def test_import_data(set_up_connection): """ Tests main function which intakes a dict of names and csv files, and creates a collection based on value and adds documents from csv file to that collection. This test tests that Main returns the three collections. (products_db, rentals_db, customers_db) """ customers = 'customers.csv' products = 'products.csv' db = set_up_connection result = import_data(db, customers, products) assert result[0][:3] == (10, 0, 10) assert result[1][:3] == (10, 0, 10)
def test_import_data(self): """ test import_data """ logger.info('Test Linear import...') start = time.perf_counter() result = import_data(self.data_folder, self.product_file, self.customer_file, self.rental_file) end = time.perf_counter() logger.info(f'Linear import takes {end-start} seconds in total.') logger.info(f"Customer tuple: {result[0]} ") logger.info(f"Product tuple: {result[1]} ") self.assertEqual(result[0][0], 1000) self.assertEqual(result[0][1], 0) self.assertEqual(result[0][2], 1000) self.assertEqual(result[1][0], 1000) self.assertEqual(result[1][1], 0) self.assertEqual(result[1][2], 1000)
def test_import_data(self): """ Test import_data() function """ #db_linear.LOGGER.info('--- Start Test import_old_data() ---') db_linear.import_data(self.csv_file, self.product_file, self.customer_file, self.rental_file)
def setUp(self): '''importing csv files for tests''' LOGGER.info('importing csv files for drop_dbs') import_data('csv_files', 'product_file.csv', 'customer_file.csv', 'rentals_file.csv') LOGGER.info('csv files successfully imported')
""" Run linear and parallel imports and provide time data. """ import datetime import linear import parallel linear.drop_data() print("--- Running linear import ---") start_time = datetime.datetime.now() results = linear.import_data('data', 'products.csv', 'customers.csv', 'rentals.csv') print("Products:") print(" Records: {} processed, {} imported".format(results[0][0], results[0][2] - results[0][1])) print(" Runtime: {} seconds".format(results[0][3])) print() print("Customers:") print(" Records: {} processed, {} imported".format(results[1][0], results[1][2] - results[1][1])) print(" Runtime: {} seconds".format(results[1][3])) print() print("Linear import complete in {} seconds.".format( (datetime.datetime.now() - start_time).total_seconds())) print() parallel.drop_data() print("--- Running parallel import ---") start_time = datetime.datetime.now() results = parallel.import_data('data', 'products.csv', 'customers.csv', 'rentals.csv') print("Products:") print(" Records: {} processed, {} imported".format(results[0][0], results[0][2] - results[0][1]))
def test_import_data_error(self): """Test error handling in import data function.""" linear.clear_database() import_result = linear.import_data('wrong', 'place', 'to', 'look') self.assertEqual(len(import_result), 3) self.assertEqual(import_result[0][0], 0)
def test_import_data_exception(self): """ test import_data function when exceptions occur""" import_data('data', 'product.csv', 'customer.csv', 'rental.csv')
def test_show_available_products(self): with patch("builtins.input", side_effect=("yes")): ln.import_data("./Data/data_files_n=40", "products", "customers", "rentals") a = ln.show_available_products() self.assertEqual(a["prd1"][" Description"], ' 60-inch TV stand')
def test_import_data(self): """Test import data function.""" linear.clear_database() import_result= linear.import_data(self.data_folder, *self.data_files) self.assertEqual(len(import_result), 3) self.assertEqual(import_result[0][0], 1000)