def test_generate_light_groupings_odd(self, mock_connect, mock_get_table_mapping, m_eval): """Tests the full workflow generation for grouping light tables odd into four tables and run concurrent ingestion""" m_eval.return_value = [['Col1', 'TIMESTAMP'], ['Col2', 'TIMESTAMP'], ['Col3', 'varchar']] mock_get_table_mapping.side_effect = [ fake_cens_tbl_prop, fake_ben_tbl_prop, light_3_prop, light_4_prop, light_5_prop, fake_cens_tbl_prop, fake_ben_tbl_prop, light_3_prop, light_4_prop, light_5_prop ] all_req = [ Request(light_req_1, self.cfg_mgr), Request(light_req_2, self.cfg_mgr), Request(light_req_3, self.cfg_mgr), Request(light_req_4, self.cfg_mgr), Request(light_req_5, self.cfg_mgr) ] available_tables, _, _ = self.req_inventory.get_available_requests( all_req) self.generator.generate(available_tables) bool_test = self.files_equal( os.path.join(self.cfg_mgr.files, 'test_workflow.xml'), os.path.join(BASE_DIR, 'expected_workflows/light_grouping_odd.xml')) self.assertTrue(bool_test)
def test_get_available_requests(self, mock_get_table_mapping): """Tests the retrieval of available and unavailable for workflow generation""" valid_req = {'db_username': '******', 'password_file': 'test', 'jdbcurl': 'test', 'domain': 'rx', 'source_database_name': 'fake_database', 'source_table_name': 'clm_fact', 'refresh_frequency': 'monthly'} unavailable_req = {'db_username': '******', 'password_file': 'test', 'jdbcurl': 'test', 'domain': 'rx', 'source_database_name': 'invalid_req', 'source_table_name': 'invalid_req', 'refresh_frequency': 'weekly'} onhold_req = {'db_username': '******', 'password_file': 'test', 'jdbcurl': 'test', 'domain': 'rx', 'source_database_name': 'onhold_req', 'source_table_name': 'onhold_req', 'refresh_frequency': 'weekly'} reqs = [Request(valid_req, self.cfg_mgr), Request(onhold_req, self.cfg_mgr), Request(unavailable_req, self.cfg_mgr)] mock_ret_val = { 'load': '000010', 'mappers': 10, 'domain': 'member', 'source_database_name': 'fake_database', 'target_dir': 'mdm/member/fake_database/clm_fact', 'connection_factories': 'com.quest.oraoop.OraOopManagerFactory', 'password_file': 'test', 'full_table_name': 'member.fake_database_clm_fact', 'source_table_name': 'clm_fact', 'db_username': '******', 'hold': 0, 'split_by': '', 'jdbcurl': 'jdbc:oracle:thin:@//fake.oracle:1521/' 'fake_servicename', 'fetch_size': 50000} mock_ret_val_2 = { 'load': '000010', 'mappers': 10, 'domain': 'member', 'source_database_name': 'invalid_db', 'target_dir': 'mdm/member/fake_database/clm_fact', 'connection_factories': 'com.quest.oraoop.OraOopManagerFactory', 'password_file': 'test', 'full_table_name': 'member.fake_database_clm_fact', 'source_table_name': 'invalid_table', 'db_username': '******', 'hold': 1, 'split_by': '', 'jdbcurl': 'jdbc:oracle:thin:@//fake.oracle:1521/' 'fake_servicename', 'fetch_size': 50000} mock_get_table_mapping.side_effect = [mock_ret_val, mock_ret_val_2, {}] available_tables, onhold_tables, unavailable_requests = \ self.inventory.get_available_requests(reqs) self.assertEqual((available_tables[0]).database, 'fake_database') self.assertEqual((onhold_tables[0]).database, 'invalid_db') self.assertEqual((unavailable_requests[0]).database, 'invalid_req')
def test_generate_one_req_custom_scripts( self, mock_connect, mock_get_table_mapping, mock_eval, m_get_col_types): """Tests the full workflow generation for one table request""" m_get_col_types.return_value = [('trans_time', 'TIMESTAMP')] full_ingest_tbl_custom_config['actions'] = \ 'custom_config_no_views_2.dsl' mock_get_table_mapping.side_effect = [full_ingest_tbl_custom_config, full_ingest_tbl_custom_config] all_req = [Request(full_ingest_tbl_custom_config_req, self.cfg_mgr)] # setup alternative requests dir self.cfg_mgr.requests_dir = os.path.join( self.cfg_mgr.files, 'requests') os.makedirs(self.cfg_mgr.requests_dir) os.makedirs(os.path.join(self.cfg_mgr.requests_dir, 'DEV')) self.generator.action_builder.dsl_parser.scripts_dir = \ self.cfg_mgr.requests_dir fixture_config_path = os.path.join( BASE_DIR, 'test_resources/custom_config_no_views_2.dsl') shutil.copy(fixture_config_path, self.cfg_mgr.requests_dir) fixture_hql_path = os.path.join( BASE_DIR, 'test_resources/hive_test.hql') shutil.copy(fixture_hql_path, os.path.join(self.cfg_mgr.requests_dir, 'DEV')) fixture_sh_path = os.path.join( BASE_DIR, 'test_resources/shell_test.sh') shutil.copy(fixture_sh_path, self.cfg_mgr.requests_dir) available_tables, _, _ = self.req_inventory.get_available_requests( all_req) self.generator.generate(available_tables) bool_test = self.files_equal( os.path.join(self.cfg_mgr.files, 'test_workflow.xml'), os.path.join(BASE_DIR, 'expected_workflows/one_req_custom_scripts.xml')) self.assertTrue(bool_test)
def test_get_available_requests_export(self, mock_get_table_mapping): """Tests the retrieval of available and unavailable for export workflow generation""" valid_req = {'db_username': '******', 'password_file': 'test', 'jdbcurl': 'test', 'domain': 'valid_member', 'db_env': 'dev', 'source_database_name': 'db_valid', 'source_table_name': 'QA_test_valid', 'refresh_frequency': 'weekly'} unavailable_req = {'db_username': '******', 'password_file': 'test', 'jdbcurl': 'test', 'domain': 'invalid_membe', 'source_database_name': 'db_invalid', 'source_table_name': 'QA_test_invalid', 'refresh_frequency': 'weekly'} reqs = [Request(valid_req, self.cfg_mgr), Request(unavailable_req, self.cfg_mgr)] mock_ret_val = { 'load': '33', 'mappers': 1, 'domain': 'valid_member', 'source_database_name': 'db_valid'} mock_get_table_mapping.side_effect = [mock_ret_val, {}] available_tables, unavailable_requests =\ self.inventory.get_available_requests_export(reqs) self.assertEqual((available_tables[0]).database, 'db_valid') self.assertEqual(unavailable_requests[0].database, 'db_invalid') """To test the condition without unavailable_requests""" table_req = {'db_username': '******', 'password_file': 'test', 'jdbcurl': 'test', 'domain': 'valid_member', 'db_env': 'dev', 'source_database_name': 'db_valid', 'source_table_name': 'QA_test_valid', 'refresh_frequency': 'weekly'} reqs = [Request(table_req, self.cfg_mgr)] mock_tab_ret_val = {'load': '44', 'mappers': 3, 'domain': 'valid_table', 'source_database_name': 'db_valid'} mock_get_table_mapping.side_effect = [mock_tab_ret_val] available_tables, unavailable_requests =\ self.inventory.get_available_requests_export(reqs) self.assertEqual((available_tables[0]).database, 'db_valid')
def test_generate_heavy_groupings(self, mock_connect, mock_get_table_mapping, m_eval): """Tests the full workflow generation for grouping heavy tables into two tables and run staggered ingestion""" m_eval.return_value = [['Col1', 'TIMESTAMP'], ['Col2', 'TIMESTAMP'], ['Col3', 'varchar']] # get_table_mapping is called twice mock_get_table_mapping.side_effect = [ fake_prof_tbl_prop, heavy_2_prop, heavy_3_prop, fake_prof_tbl_prop, heavy_2_prop, heavy_3_prop] all_req = [Request(heavy_req_1, self.cfg_mgr), Request(heavy_req_2, self.cfg_mgr), Request(heavy_req_3, self.cfg_mgr)] available_tables, _, _ = self.req_inventory.get_available_requests( all_req) self.generator.generate(available_tables) bool_test = self.files_equal( os.path.join(self.cfg_mgr.files, 'test_workflow.xml'), os.path.join(BASE_DIR, 'expected_workflows/heavy_grouping.xml')) self.assertTrue(bool_test)
def test_submit_it_file_insert(self, m1): """Test submit it file with a valid it table file.""" self.driver.req_inventory = MagicMock(spec=RequestInventory) self.driver.req_inventory.parse_file.return_value = \ ([Request(mock_table_mapping_val, self.cfg_mgr)], 'Parse File Success') self.driver.it_inventory.get_table_mapping = \ MagicMock(spec=ITInventory.get_table_mapping) self.driver.it_inventory.get_table_mapping.return_value = {} self.driver.it_inventory.insert = MagicMock(spec=ITInventory.insert) self.driver.it_inventory.insert.return_value = (True, 'Insert Success') result = self.driver.submit_it_file('test') self.assertEquals(result, 'Parse File Success\nInsert Success')
def test_optional_fields(self): """test optional""" request_dict = { 'db_username': '******', 'password_file': 'test_passwd', 'jdbcurl': 'test_jdbcurl', 'domain': 'test_domain', 'source_database_name': 'test_db', 'source_table_name': 'test_table' } req_obj = Request(request_dict, self.cfg_mgr) self.assertEqual(req_obj.views, False) self.assertEqual(req_obj.frequency_readable, False) self.assertEqual(req_obj.esp_group, False)
def test_generate_one_req(self, mock_connect, mock_get_table_mapping, m_eval): """Tests the full workflow generation for one table request""" m_eval.return_value = [['Col1', 'TIMESTAMP'], ['Col2', 'TIMESTAMP'], ['Col3', 'varchar']] mock_get_table_mapping.side_effect = [fake_cens_tbl_prop, fake_cens_tbl_prop] all_req = [Request(light_req_1, self.cfg_mgr)] available_tables, _, _ = self.req_inventory.get_available_requests( all_req) self.generator.generate(available_tables) bool_test = self.files_equal( os.path.join(self.cfg_mgr.files, 'test_workflow.xml'), os.path.join(BASE_DIR, 'expected_workflows/one_req.xml')) self.assertTrue(bool_test)
def test_fields(self): """test fields""" request_dict = { 'db_username': '******', 'password_file': 'test_passwd', 'jdbcurl': 'test_jdbcurl', 'domain': 'test_domain', 'source_database_name': 'test_DB', 'source_table_name': 'test_TABLE' } req_obj = Request(request_dict, self.cfg_mgr) self.assertEqual(req_obj.username, 'test_username') self.assertEqual(req_obj.password_file, 'test_passwd') self.assertEqual(req_obj.jdbcurl, 'test_jdbcurl') self.assertEqual(req_obj.database, 'test_db') self.assertEqual(req_obj.table_name, 'test_table') self.assertEqual(req_obj.db_table_name, 'test_db_test_table')
def test_submit_it_file_update(self, m1): """Test submit it file with an updated it table file.""" self.driver.req_inventory = MagicMock(spec=RequestInventory) self.driver.req_inventory.parse_file.return_value = \ ([Request(mock_table_mapping_val, self.cfg_mgr)], 'Parse File Success') self.driver.it_inventory.insert = MagicMock(spec=ITInventory.insert) self.driver.it_inventory.update = MagicMock(spec=ITInventory.update) self.driver.it_inventory.get_table_mapping = MagicMock( spec=ITInventory.get_table_mapping) self.driver.it_inventory.update.return_value = (True, 'Update Success') updated_table = copy.deepcopy(mock_table_mapping_val) updated_table['db_username'] = '******' self.driver.it_inventory.get_table_mapping.return_value = updated_table result = self.driver.submit_it_file('test') self.assertEquals(result, 'Parse File Success\nUpdate Success')