def test_generate_light_groupings_odd(self, mock_connect,
mock_get_table_mapping, m_eval):
"""Tests the full workflow generation for grouping light tables odd
into four tables and run concurrent ingestion"""
m_eval.return_value = [['Col1', 'TIMESTAMP'], ['Col2', 'TIMESTAMP'],
['Col3', 'varchar']]
mock_get_table_mapping.side_effect = [
fake_cens_tbl_prop, fake_ben_tbl_prop, light_3_prop, light_4_prop,
light_5_prop, fake_cens_tbl_prop, fake_ben_tbl_prop, light_3_prop,
light_4_prop, light_5_prop
]
all_req = [
Request(light_req_1, self.cfg_mgr),
Request(light_req_2, self.cfg_mgr),
Request(light_req_3, self.cfg_mgr),
Request(light_req_4, self.cfg_mgr),
Request(light_req_5, self.cfg_mgr)
]
available_tables, _, _ = self.req_inventory.get_available_requests(
all_req)
self.generator.generate(available_tables)
bool_test = self.files_equal(
os.path.join(self.cfg_mgr.files, 'test_workflow.xml'),
os.path.join(BASE_DIR,
'expected_workflows/light_grouping_odd.xml'))
self.assertTrue(bool_test)
def test_get_available_requests(self, mock_get_table_mapping):
"""Tests the retrieval of available and unavailable
for workflow generation"""
valid_req = {'db_username': '******', 'password_file': 'test',
'jdbcurl': 'test', 'domain': 'rx',
'source_database_name': 'fake_database',
'source_table_name': 'clm_fact',
'refresh_frequency': 'monthly'}
unavailable_req = {'db_username': '******', 'password_file': 'test',
'jdbcurl': 'test', 'domain': 'rx',
'source_database_name': 'invalid_req',
'source_table_name': 'invalid_req',
'refresh_frequency': 'weekly'}
onhold_req = {'db_username': '******', 'password_file': 'test',
'jdbcurl': 'test', 'domain': 'rx',
'source_database_name': 'onhold_req',
'source_table_name': 'onhold_req',
'refresh_frequency': 'weekly'}
reqs = [Request(valid_req, self.cfg_mgr),
Request(onhold_req, self.cfg_mgr),
Request(unavailable_req, self.cfg_mgr)]
mock_ret_val = {
'load': '000010', 'mappers': 10, 'domain': 'member',
'source_database_name': 'fake_database',
'target_dir': 'mdm/member/fake_database/clm_fact',
'connection_factories': 'com.quest.oraoop.OraOopManagerFactory',
'password_file': 'test',
'full_table_name': 'member.fake_database_clm_fact',
'source_table_name': 'clm_fact',
'db_username': '******', 'hold': 0, 'split_by': '',
'jdbcurl': 'jdbc:oracle:thin:@//fake.oracle:1521/'
'fake_servicename',
'fetch_size': 50000}
mock_ret_val_2 = {
'load': '000010', 'mappers': 10, 'domain': 'member',
'source_database_name': 'invalid_db',
'target_dir': 'mdm/member/fake_database/clm_fact',
'connection_factories': 'com.quest.oraoop.OraOopManagerFactory',
'password_file': 'test',
'full_table_name': 'member.fake_database_clm_fact',
'source_table_name': 'invalid_table',
'db_username': '******', 'hold': 1, 'split_by': '',
'jdbcurl': 'jdbc:oracle:thin:@//fake.oracle:1521/'
'fake_servicename',
'fetch_size': 50000}
mock_get_table_mapping.side_effect = [mock_ret_val, mock_ret_val_2, {}]
available_tables, onhold_tables, unavailable_requests = \
self.inventory.get_available_requests(reqs)
self.assertEqual((available_tables[0]).database, 'fake_database')
self.assertEqual((onhold_tables[0]).database, 'invalid_db')
self.assertEqual((unavailable_requests[0]).database, 'invalid_req')
def test_generate_one_req_custom_scripts(
self, mock_connect, mock_get_table_mapping, mock_eval,
m_get_col_types):
"""Tests the full workflow generation for one table request"""
m_get_col_types.return_value = [('trans_time', 'TIMESTAMP')]
full_ingest_tbl_custom_config['actions'] = \
'custom_config_no_views_2.dsl'
mock_get_table_mapping.side_effect = [full_ingest_tbl_custom_config,
full_ingest_tbl_custom_config]
all_req = [Request(full_ingest_tbl_custom_config_req, self.cfg_mgr)]
# setup alternative requests dir
self.cfg_mgr.requests_dir = os.path.join(
self.cfg_mgr.files, 'requests')
os.makedirs(self.cfg_mgr.requests_dir)
os.makedirs(os.path.join(self.cfg_mgr.requests_dir, 'DEV'))
self.generator.action_builder.dsl_parser.scripts_dir = \
self.cfg_mgr.requests_dir
fixture_config_path = os.path.join(
BASE_DIR, 'test_resources/custom_config_no_views_2.dsl')
shutil.copy(fixture_config_path, self.cfg_mgr.requests_dir)
fixture_hql_path = os.path.join(
BASE_DIR, 'test_resources/hive_test.hql')
shutil.copy(fixture_hql_path,
os.path.join(self.cfg_mgr.requests_dir, 'DEV'))
fixture_sh_path = os.path.join(
BASE_DIR, 'test_resources/shell_test.sh')
shutil.copy(fixture_sh_path, self.cfg_mgr.requests_dir)
available_tables, _, _ = self.req_inventory.get_available_requests(
all_req)
self.generator.generate(available_tables)
bool_test = self.files_equal(
os.path.join(self.cfg_mgr.files, 'test_workflow.xml'),
os.path.join(BASE_DIR,
'expected_workflows/one_req_custom_scripts.xml'))
self.assertTrue(bool_test)
def test_get_available_requests_export(self, mock_get_table_mapping):
"""Tests the retrieval of available and unavailable for
export workflow generation"""
valid_req = {'db_username': '******', 'password_file': 'test',
'jdbcurl': 'test', 'domain': 'valid_member',
'db_env': 'dev',
'source_database_name': 'db_valid',
'source_table_name': 'QA_test_valid',
'refresh_frequency': 'weekly'}
unavailable_req = {'db_username': '******',
'password_file': 'test',
'jdbcurl': 'test', 'domain': 'invalid_membe',
'source_database_name': 'db_invalid',
'source_table_name': 'QA_test_invalid',
'refresh_frequency': 'weekly'}
reqs = [Request(valid_req, self.cfg_mgr),
Request(unavailable_req, self.cfg_mgr)]
mock_ret_val = {
'load': '33', 'mappers': 1, 'domain': 'valid_member',
'source_database_name': 'db_valid'}
mock_get_table_mapping.side_effect = [mock_ret_val, {}]
available_tables, unavailable_requests =\
self.inventory.get_available_requests_export(reqs)
self.assertEqual((available_tables[0]).database, 'db_valid')
self.assertEqual(unavailable_requests[0].database, 'db_invalid')
"""To test the condition without unavailable_requests"""
table_req = {'db_username': '******', 'password_file': 'test',
'jdbcurl': 'test', 'domain': 'valid_member',
'db_env': 'dev',
'source_database_name': 'db_valid',
'source_table_name': 'QA_test_valid',
'refresh_frequency': 'weekly'}
reqs = [Request(table_req, self.cfg_mgr)]
mock_tab_ret_val = {'load': '44', 'mappers': 3,
'domain': 'valid_table',
'source_database_name': 'db_valid'}
mock_get_table_mapping.side_effect = [mock_tab_ret_val]
available_tables, unavailable_requests =\
self.inventory.get_available_requests_export(reqs)
self.assertEqual((available_tables[0]).database, 'db_valid')
def test_generate_heavy_groupings(self, mock_connect,
mock_get_table_mapping, m_eval):
"""Tests the full workflow generation for grouping heavy tables
into two tables and run staggered ingestion"""
m_eval.return_value = [['Col1', 'TIMESTAMP'], ['Col2', 'TIMESTAMP'],
['Col3', 'varchar']]
# get_table_mapping is called twice
mock_get_table_mapping.side_effect = [
fake_prof_tbl_prop, heavy_2_prop, heavy_3_prop, fake_prof_tbl_prop,
heavy_2_prop, heavy_3_prop]
all_req = [Request(heavy_req_1, self.cfg_mgr),
Request(heavy_req_2, self.cfg_mgr),
Request(heavy_req_3, self.cfg_mgr)]
available_tables, _, _ = self.req_inventory.get_available_requests(
all_req)
self.generator.generate(available_tables)
bool_test = self.files_equal(
os.path.join(self.cfg_mgr.files, 'test_workflow.xml'),
os.path.join(BASE_DIR, 'expected_workflows/heavy_grouping.xml'))
self.assertTrue(bool_test)
def test_submit_it_file_insert(self, m1):
"""Test submit it file with a valid it table file."""
self.driver.req_inventory = MagicMock(spec=RequestInventory)
self.driver.req_inventory.parse_file.return_value = \
([Request(mock_table_mapping_val, self.cfg_mgr)],
'Parse File Success')
self.driver.it_inventory.get_table_mapping = \
MagicMock(spec=ITInventory.get_table_mapping)
self.driver.it_inventory.get_table_mapping.return_value = {}
self.driver.it_inventory.insert = MagicMock(spec=ITInventory.insert)
self.driver.it_inventory.insert.return_value = (True, 'Insert Success')
result = self.driver.submit_it_file('test')
self.assertEquals(result, 'Parse File Success\nInsert Success')
def test_optional_fields(self):
"""test optional"""
request_dict = {
'db_username': '******',
'password_file': 'test_passwd',
'jdbcurl': 'test_jdbcurl',
'domain': 'test_domain',
'source_database_name': 'test_db',
'source_table_name': 'test_table'
}
req_obj = Request(request_dict, self.cfg_mgr)
self.assertEqual(req_obj.views, False)
self.assertEqual(req_obj.frequency_readable, False)
self.assertEqual(req_obj.esp_group, False)
def test_generate_one_req(self, mock_connect, mock_get_table_mapping,
m_eval):
"""Tests the full workflow generation for one table request"""
m_eval.return_value = [['Col1', 'TIMESTAMP'], ['Col2', 'TIMESTAMP'],
['Col3', 'varchar']]
mock_get_table_mapping.side_effect = [fake_cens_tbl_prop,
fake_cens_tbl_prop]
all_req = [Request(light_req_1, self.cfg_mgr)]
available_tables, _, _ = self.req_inventory.get_available_requests(
all_req)
self.generator.generate(available_tables)
bool_test = self.files_equal(
os.path.join(self.cfg_mgr.files, 'test_workflow.xml'),
os.path.join(BASE_DIR, 'expected_workflows/one_req.xml'))
self.assertTrue(bool_test)
def test_fields(self):
"""test fields"""
request_dict = {
'db_username': '******',
'password_file': 'test_passwd',
'jdbcurl': 'test_jdbcurl',
'domain': 'test_domain',
'source_database_name': 'test_DB',
'source_table_name': 'test_TABLE'
}
req_obj = Request(request_dict, self.cfg_mgr)
self.assertEqual(req_obj.username, 'test_username')
self.assertEqual(req_obj.password_file, 'test_passwd')
self.assertEqual(req_obj.jdbcurl, 'test_jdbcurl')
self.assertEqual(req_obj.database, 'test_db')
self.assertEqual(req_obj.table_name, 'test_table')
self.assertEqual(req_obj.db_table_name, 'test_db_test_table')
def test_submit_it_file_update(self, m1):
"""Test submit it file with an updated it table file."""
self.driver.req_inventory = MagicMock(spec=RequestInventory)
self.driver.req_inventory.parse_file.return_value = \
([Request(mock_table_mapping_val, self.cfg_mgr)],
'Parse File Success')
self.driver.it_inventory.insert = MagicMock(spec=ITInventory.insert)
self.driver.it_inventory.update = MagicMock(spec=ITInventory.update)
self.driver.it_inventory.get_table_mapping = MagicMock(
spec=ITInventory.get_table_mapping)
self.driver.it_inventory.update.return_value = (True, 'Update Success')
updated_table = copy.deepcopy(mock_table_mapping_val)
updated_table['db_username'] = '******'
self.driver.it_inventory.get_table_mapping.return_value = updated_table
result = self.driver.submit_it_file('test')
self.assertEquals(result, 'Parse File Success\nUpdate Success')
