class TestCli: """ Unit Tests for PipelineWise CLI executable """ def setup_method(self): """Create CLI arguments""" self.args = CliArgs(log='coverage.log') self.pipelinewise = PipelineWise(self.args, CONFIG_DIR, VIRTUALENVS_DIR) def teardown_method(self): """Delete test directories""" try: shutil.rmtree(TEST_PROJECT_DIR) shutil.rmtree(os.path.join(CONFIG_DIR, 'target_one/tap_one/log')) except Exception: pass def test_target_dir(self): """Singer target connector config path must be relative to the project config dir""" assert \ self.pipelinewise.get_target_dir('dummy-target') == \ '{}/dummy-target'.format(CONFIG_DIR) def test_tap_dir(self): """Singer tap connector config path must be relative to the target connector config path""" assert \ self.pipelinewise.get_tap_dir('dummy-target', 'dummy-tap') == \ '{}/dummy-target/dummy-tap'.format(CONFIG_DIR) def test_tap_log_dir(self): """Singer tap log path must be relative to the tap connector config path""" assert \ self.pipelinewise.get_tap_log_dir('dummy-target', 'dummy-tap') == \ '{}/dummy-target/dummy-tap/log'.format(CONFIG_DIR) def test_connector_bin(self): """Singer connector binary must be at a certain location under PIPELINEWISE_HOME .virtualenvs dir""" assert \ self.pipelinewise.get_connector_bin('dummy-type') == \ '{}/dummy-type/bin/dummy-type'.format(VIRTUALENVS_DIR) def test_connector_files(self): """Every singer connector must have a list of JSON files at certain locations""" # TODO: get_connector_files is duplicated in config.py and pipelinewise.py # Refactor to use only one assert \ self.pipelinewise.get_connector_files('/var/singer-connector') == \ { 'config': '/var/singer-connector/config.json', 'inheritable_config': '/var/singer-connector/inheritable_config.json', 'properties': '/var/singer-connector/properties.json', 'state': '/var/singer-connector/state.json', 'transformation': '/var/singer-connector/transformation.json', 'selection': '/var/singer-connector/selection.json', 'pidfile': '/var/singer-connector/pipelinewise.pid' } def test_not_existing_config_dir(self): """Test with not existing config dir""" # Create a new pipelinewise object pointing to a not existing config directory pipelinewise_with_no_config = PipelineWise(self.args, 'not-existing-config-dir', VIRTUALENVS_DIR) # It should return and empty config with empty list targets # TODO: Make this scenario to fail with error message of "config dir not exists" assert pipelinewise_with_no_config.config == {} assert pipelinewise_with_no_config.get_targets() == [] def test_get_targets(self): """Targets should be loaded from JSON as is""" assert self.pipelinewise.get_targets() == cli.utils.load_json('{}/config.json'.format(CONFIG_DIR)).get( 'targets', []) def test_get_target(self): """Selecting target by ID should append connector files""" # Get target definitions from JSON file targets = cli.utils.load_json('{}/config.json'.format(CONFIG_DIR)).get('targets', []) exp_target_one = next((item for item in targets if item['id'] == 'target_one'), False) exp_target_two = next((item for item in targets if item['id'] == 'target_two'), False) # Append the connector file paths to the expected targets exp_target_one['files'] = self.pipelinewise.get_connector_files('{}/target_one'.format(CONFIG_DIR)) exp_target_two['files'] = self.pipelinewise.get_connector_files('{}/target_two'.format(CONFIG_DIR)) # Getting target by ID should match to original JSON and should contains the connector files list assert self.pipelinewise.get_target('target_one') == exp_target_one assert self.pipelinewise.get_target('target_two') == exp_target_two def test_get_taps(self): """Selecting taps by target ID should append tap statuses""" # Get target definitions from JSON file targets = cli.utils.load_json('{}/config.json'.format(CONFIG_DIR)).get('targets', []) target_one = next((item for item in targets if item['id'] == 'target_one'), False) target_two = next((item for item in targets if item['id'] == 'target_two'), False) # Append the tap statuses to every tap in target_one exp_tap_one = target_one['taps'][0] exp_tap_two = target_one['taps'][1] exp_tap_one['status'] = self.pipelinewise.detect_tap_status('target_one', exp_tap_one['id']) exp_tap_two['status'] = self.pipelinewise.detect_tap_status('target_one', exp_tap_two['id']) # Append the tap statuses to every tap in target_one exp_tap_three = target_two['taps'][0] exp_tap_three['status'] = self.pipelinewise.detect_tap_status('target_two', exp_tap_three['id']) # Tap statuses should be appended to every tap assert self.pipelinewise.get_taps('target_one') == [exp_tap_one, exp_tap_two] assert self.pipelinewise.get_taps('target_two') == [exp_tap_three] def test_get_tap(self): """Getting tap by ID should return status, connector and target props as well""" # Get target definitions from JSON file targets = cli.utils.load_json('{}/config.json'.format(CONFIG_DIR)).get('targets', []) target_one = next((item for item in targets if item['id'] == 'target_one'), False) # Append the tap status, files and target keys to the tap exp_tap_one = target_one['taps'][0] exp_tap_one['status'] = self.pipelinewise.detect_tap_status('target_one', exp_tap_one['id']) exp_tap_one['files'] = self.pipelinewise.get_connector_files('{}/target_one/tap_one'.format(CONFIG_DIR)) exp_tap_one['target'] = self.pipelinewise.get_target('target_one') # Getting tap by ID should match to original JSON and should contain status, connector files and target props assert self.pipelinewise.get_tap('target_one', 'tap_one') == exp_tap_one def test_get_not_existing_target(self): """Test getting not existing target""" # Getting not existing from should raise exception with pytest.raises(Exception): assert self.pipelinewise.get_target('not-existing-target') == {} def test_get_taps_from_wrong_target(self): """Test getting taps from not existing target""" # Getting not existing from should raise exception with pytest.raises(Exception): assert self.pipelinewise.get_tap('not-existing-target', 'not-existing-tap') == {} def test_get_not_existing_tap(self): """Test getting not existing tap from existing target""" # Getting not existing from should raise exception with pytest.raises(Exception): assert self.pipelinewise.get_tap('target_one', 'not-existing-tap') == {} # pylint: disable=bad-continuation def test_create_filtered_tap_props(self): """Test creating fastsync and singer specific properties file""" ( tap_properties_fastsync, fastsync_stream_ids, tap_properties_singer, singer_stream_ids ) = self.pipelinewise.create_filtered_tap_properties( target_type='target-snowflake', tap_type='tap-mysql', tap_properties='{}/resources/sample_json_config/target_one/tap_one/properties.json'.format( os.path.dirname(__file__)), tap_state='{}/resources/sample_json_config/target_one/tap_one/state.json'.format( os.path.dirname(__file__)), filters={ 'selected': True, 'target_type': ['target-snowflake'], 'tap_type': ['tap-mysql', 'tap-postgres'], 'initial_sync_required': True }, create_fallback=True) # Fastsync and singer properties should be created assert os.path.isfile(tap_properties_fastsync) assert os.path.isfile(tap_properties_singer) # Delete generated properties file os.remove(tap_properties_fastsync) os.remove(tap_properties_singer) # Fastsync and singer properties should be created assert fastsync_stream_ids == ['db_test_mysql-table_one', 'db_test_mysql-table_two'] assert singer_stream_ids == ['db_test_mysql-table_one', 'db_test_mysql-table_two'] def test_merge_empty_catalog(self): """Merging two empty singer schemas should be another empty""" # TODO: Check if pipelinewise.merge_schemas is required at all or not assert self.pipelinewise.merge_schemas({}, {}) == {} def test_merge_empty_stream_catalog(self): """Merging empty schemas should be empty""" # TODO: Check if pipelinewise.merge_schemas is required at all or not assert self.pipelinewise.merge_schemas({'streams': []}, {'streams': []}) == {'streams': []} def test_merge_same_catalog(self): """Test merging not empty schemas""" # TODO: Check if pipelinewise.merge_schemas is required at all or not tap_one_catalog = cli.utils.load_json( '{}/resources/sample_json_config/target_one/tap_one/properties.json'.format(os.path.dirname(__file__))) assert self.pipelinewise.merge_schemas(tap_one_catalog, tap_one_catalog) == tap_one_catalog def test_merge_updated_catalog(self): """Test merging not empty schemas""" # TODO: Check if pipelinewise.merge_schemas is required at all or not tap_one_catalog = cli.utils.load_json( '{}/resources/sample_json_config/target_one/tap_one/properties.json'.format(os.path.dirname(__file__))) tap_one_updated_catalog = cli.utils.load_json( '{}/resources/sample_json_config/target_one/tap_one/properties_updated.json'.format( os.path.dirname(__file__))) assert self.pipelinewise.merge_schemas(tap_one_catalog, tap_one_updated_catalog) == tap_one_catalog def test_make_default_selection(self): """Test if streams selected correctly in catalog JSON""" tap_one_catalog = cli.utils.load_json( '{}/resources/sample_json_config/target_one/tap_one/properties.json'.format(os.path.dirname(__file__))) tap_one_selection_file = '{}/resources/sample_json_config/target_one/tap_one/selection.json'.format( os.path.dirname(__file__)) # Update catalog selection tap_one_with_selection = self.pipelinewise.make_default_selection(tap_one_catalog, tap_one_selection_file) # Table one has to be selected with LOG_BASED replication method assert tap_one_with_selection['streams'][0]['metadata'][0]['metadata']['selected'] is True assert tap_one_with_selection['streams'][0]['metadata'][0]['metadata']['replication-method'] == 'LOG_BASED' # Table two has to be selected with INCREMENTAL replication method assert tap_one_with_selection['streams'][1]['metadata'][0]['metadata']['selected'] is True assert tap_one_with_selection['streams'][1]['metadata'][0]['metadata']['replication-method'] == 'INCREMENTAL' assert tap_one_with_selection['streams'][1]['metadata'][0]['metadata']['replication-key'] == 'id' # Table three should not be selected assert tap_one_with_selection['streams'][2]['metadata'][0]['metadata']['selected'] is False def test_target_config(self): """Test merging target config.json and inheritable_config.json""" target_config = '{}/resources/target-config.json'.format(os.path.dirname(__file__)) tap_inheritable_config = '{}/resources/tap-inheritable-config.json'.format(os.path.dirname(__file__)) # The merged JSON written into a temp file temp_file = self.pipelinewise.create_consumable_target_config(target_config, tap_inheritable_config) cons_targ_config = cli.utils.load_json(temp_file) # The merged object needs assert cons_targ_config == { 'account': 'foo', 'aws_access_key_id': 'secret', 'aws_secret_access_key': 'secret/', 'client_side_encryption_master_key': 'secret=', 'dbname': 'my_db', 'file_format': 'my_file_format', 'password': '******', 's3_bucket': 'foo', 's3_key_prefix': 'foo/', 'stage': 'my_stage', 'user': '******', 'warehouse': 'MY_WAREHOUSE', 'batch_size_rows': 5000, 'data_flattening_max_level': 0, 'default_target_schema': 'jira_clear', 'default_target_schema_select_permissions': ['grp_power'], 'hard_delete': True, 'primary_key_required': True, 'schema_mapping': { 'jira': { 'target_schema': 'jira_clear', 'target_schema_select_permissions': ['grp_power'] } } } # Remove temp file with merged JSON os.remove(temp_file) def test_invalid_target_config(self): """Test merging invalid target config.json and inheritable_config.json""" target_config = '{}/resources/invalid.json'.format(os.path.dirname(__file__)) tap_inheritable_config = 'not-existing-json' # Merging invalid or not existing JSONs should raise exception with pytest.raises(Exception): self.pipelinewise.create_consumable_target_config(target_config, tap_inheritable_config) def test_command_encrypt_string(self, capsys): """Test vault encryption command output""" secret_path = '{}/resources/vault-secret.txt'.format(os.path.dirname(__file__)) args = CliArgs(string='plain text', secret=secret_path) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Encrypted string should be printed to stdout pipelinewise.encrypt_string() stdout, stderr = capsys.readouterr() assert not stderr.strip() assert stdout.startswith('!vault |') and '$ANSIBLE_VAULT;' in stdout def test_command_init(self): """Test init command""" args = CliArgs(name=TEST_PROJECT_NAME) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Init new project pipelinewise.init() # The test project should contain every sample YAML file for sample_yaml in os.listdir('{}/../../../pipelinewise/cli/samples'.format(os.path.dirname(__file__))): assert os.path.isfile(os.path.join(TEST_PROJECT_DIR, sample_yaml)) # Re-creating project should reaise exception of directory not empty with pytest.raises(SystemExit) as pytest_wrapped_e: pipelinewise.init() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 1 def test_command_status(self, capsys): """Test status command output""" # Status table should be printed to stdout self.pipelinewise.status() stdout, stderr = capsys.readouterr() assert not stderr.strip() # Exact output match # pylint: disable=line-too-long assert stdout == """Tap ID Tap Type Target ID Target Type Enabled Status Last Sync Last Sync Result --------- ------------ ----------- ---------------- --------- -------------- ----------- ------------------ tap_one tap-mysql target_one target-snowflake True ready unknown tap_two tap-postgres target_one target-snowflake True ready unknown tap_three tap-mysql target_two target-s3-csv True not-configured unknown 3 pipeline(s) """ def test_command_discover_tap(self): """Test discover tap command""" args = CliArgs(target='target_one', tap='tap_one') pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Running discovery mode should detect the tap type and path to the connector # Since the executable is not available in this test then it should fail result = pipelinewise.discover_tap() exp_err_pattern = '/tap-mysql/bin/tap-mysql: No such file or directory' assert exp_err_pattern in result def _test_command_run_tap(self): """Test run tap command""" args = CliArgs(target='target_one', tap='tap_one') pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Running run mode should detect the tap type and path to the connector # Since the executable is not available in this test then it should fail # TODO: sync discover_tap and run_tap behaviour. run_tap sys.exit but discover_tap does not. with pytest.raises(SystemExit) as pytest_wrapped_e: pipelinewise.run_tap() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 1 def test_command_stop_tap(self): """Test stop tap command""" args = CliArgs(target='target_one', tap='tap_one') pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Tap is not running, pid file not exist, should exit with error with pytest.raises(SystemExit) as pytest_wrapped_e: pipelinewise.stop_tap() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 1 def test_command_sync_tables(self): """Test run tap command""" args = CliArgs(target='target_one', tap='tap_one') pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Running sync_tables should detect the tap type and path to the connector # Since the executable is not available in this test then it should fail # TODO: sync discover_tap and run_tap behaviour. run_tap sys.exit but discover_tap does not. with pytest.raises(SystemExit) as pytest_wrapped_e: pipelinewise.sync_tables() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 1 # pylint: disable=protected-access def test_exit_gracefully(self): """Gracefully shoudl run tap command""" args = CliArgs(target='target_one', tap='tap_one') pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Create a test log file, simulating a running tap pipelinewise.tap_run_log_file = 'test-tap-run-dummy.log' Path('{}.running'.format(pipelinewise.tap_run_log_file)).touch() # Graceful exit should return 1 by default with pytest.raises(SystemExit) as pytest_wrapped_e: pipelinewise._exit_gracefully(signal.SIGINT, frame=None) assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 1 # Graceful exit should rename log file from running status to terminated assert os.path.isfile('{}.terminated'.format(pipelinewise.tap_run_log_file)) # Delete test log file os.remove('{}.terminated'.format(pipelinewise.tap_run_log_file)) def test_validate_command_1(self): """Test validate command should fail because of missing replication key for incremental""" test_validate_command_dir = \ f'{os.path.dirname(__file__)}/resources/test_validate_command/missing_replication_key_incremental' args = CliArgs(dir=test_validate_command_dir) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) with pytest.raises(SystemExit): pipelinewise.validate() def test_validate_command_2(self): """Test validate command should succeed""" test_validate_command_dir = \ f'{os.path.dirname(__file__)}/resources/test_validate_command/missing_replication_key' args = CliArgs(dir=test_validate_command_dir) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) pipelinewise.validate() def test_validate_command_3(self): """Test validate command should fail because of invalid target in tap config""" test_validate_command_dir = f'{os.path.dirname(__file__)}/resources/test_validate_command/invalid_target' args = CliArgs(dir=test_validate_command_dir) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) with pytest.raises(SystemExit): pipelinewise.validate() def test_validate_command_4(self): """Test validate command should fail because of duplicate targets""" test_validate_command_dir = \ f'{os.path.dirname(__file__)}/resources/test_validate_command/test_yaml_config_two_targets' args = CliArgs(dir=test_validate_command_dir) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) with pytest.raises(SystemExit): pipelinewise.validate() # pylint: disable=protected-access def test_post_import_checks(self): """Test post import checks""" args = CliArgs() pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) test_files_dir = '{}/resources/test_post_import_checks'.format(os.path.dirname(__file__)) tap_pk_required = cli.utils.load_json('{}/tap_config_pk_required.json'.format(test_files_dir)) tap_pk_not_required = cli.utils.load_json('{}/tap_config_pk_not_required.json'.format(test_files_dir)) tap_pk_not_defined = cli.utils.load_json('{}/tap_config_pk_not_defined.json'.format(test_files_dir)) tap_with_pk = cli.utils.load_json('{}//tap_properties_with_pk.json'.format(test_files_dir)) tap_with_no_pk_full_table = cli.utils.load_json( '{}//tap_properties_with_no_pk_full_table.json'.format(test_files_dir)) tap_with_no_pk_incremental = cli.utils.load_json( '{}//tap_properties_with_no_pk_incremental.json'.format(test_files_dir)) tap_with_no_pk_log_based = cli.utils.load_json( '{}//tap_properties_with_no_pk_log_based.json'.format(test_files_dir)) tap_with_no_pk_not_selected = cli.utils.load_json( '{}//tap_properties_with_no_pk_not_selected.json'.format(test_files_dir)) # Test scenarios when post import checks should pass assert pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_pk) == [] assert pipelinewise._run_post_import_tap_checks(tap_pk_not_required, tap_with_pk) == [] assert pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_full_table) == [] assert pipelinewise._run_post_import_tap_checks(tap_pk_not_required, tap_with_no_pk_incremental) == [] assert pipelinewise._run_post_import_tap_checks(tap_pk_not_required, tap_with_no_pk_log_based) == [] assert pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_not_selected) == [] assert pipelinewise._run_post_import_tap_checks(tap_pk_not_defined, tap_with_no_pk_full_table) == [] # Test scenarios when post import checks should fail due to primary keys not exists assert len(pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_incremental)) == 1 assert len(pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_log_based)) == 1 assert len(pipelinewise._run_post_import_tap_checks(tap_pk_not_defined, tap_with_no_pk_incremental)) == 1 assert len(pipelinewise._run_post_import_tap_checks(tap_pk_not_defined, tap_with_no_pk_log_based)) == 1
class TestCli(object): """ Unit Tests for PipelineWise CLI executable """ def setup_method(self): # Create CLI arguments self.args = CliArgs(log="coverage.log") self.pipelinewise = PipelineWise(self.args, CONFIG_DIR, VIRTUALENVS_DIR) def teardown_method(self): # Delete test directories try: shutil.rmtree(TEST_PROJECT_DIR) shutil.rmtree(os.path.join(CONFIG_DIR, "target_one/tap_one/log")) except: pass def test_target_dir(self): """Singer target connector config path must be relative to the project config dir""" assert self.pipelinewise.get_target_dir( "dummy-target") == "{}/dummy-target".format(CONFIG_DIR) def test_tap_dir(self): """Singer tap connector config path must be relative to the target connector config path""" assert self.pipelinewise.get_tap_dir( "dummy-target", "dummy-tap") == "{}/dummy-target/dummy-tap".format(CONFIG_DIR) def test_tap_log_dir(self): """Singer tap log path must be relative to the tap connector config path""" assert self.pipelinewise.get_tap_log_dir( "dummy-target", "dummy-tap") == "{}/dummy-target/dummy-tap/log".format(CONFIG_DIR) def test_connector_bin(self): """Singer connector binary must be at a certain location under PIPELINEWISE_HOME .virtualenvs dir""" assert self.pipelinewise.get_connector_bin( "dummy-type") == "{}/dummy-type/bin/dummy-type".format( VIRTUALENVS_DIR) def test_connector_files(self): """Every singer connector must have a list of JSON files at certain locations""" # TODO: get_connector_files is duplicated in config.py and pipelinewise.py # Refactor to use only one assert self.pipelinewise.get_connector_files( "/var/singer-connector") == { "config": "/var/singer-connector/config.json", "inheritable_config": "/var/singer-connector/inheritable_config.json", "properties": "/var/singer-connector/properties.json", "state": "/var/singer-connector/state.json", "transformation": "/var/singer-connector/transformation.json", "selection": "/var/singer-connector/selection.json", } def test_not_existing_config_dir(self): """Test with not existing config dir""" # Create a new pipelinewise object pointing to a not existing config directory pipelinewise_with_no_config = PipelineWise(self.args, "not-existing-config-dir", VIRTUALENVS_DIR) # It should return and empty config with empty list targets # TODO: Make this scenario to fail with error message of "config dir not exists" assert pipelinewise_with_no_config.config == {} assert pipelinewise_with_no_config.get_targets() == [] def test_get_targets(self): """Targets should be loaded from JSON as is""" assert self.pipelinewise.get_targets() == cli.utils.load_json( "{}/config.json".format(CONFIG_DIR)).get("targets", []) def test_get_target(self): """Selecting target by ID should append connector files""" # Get target definitions from JSON file targets = cli.utils.load_json("{}/config.json".format(CONFIG_DIR)).get( "targets", []) exp_target_one = next( (item for item in targets if item["id"] == "target_one"), False) exp_target_two = next( (item for item in targets if item["id"] == "target_two"), False) # Append the connector file paths to the expected targets exp_target_one["files"] = self.pipelinewise.get_connector_files( "{}/target_one".format(CONFIG_DIR)) exp_target_two["files"] = self.pipelinewise.get_connector_files( "{}/target_two".format(CONFIG_DIR)) # Getting target by ID should match to original JSON and should contains the connector files list assert self.pipelinewise.get_target("target_one") == exp_target_one assert self.pipelinewise.get_target("target_two") == exp_target_two def test_get_taps(self): """Selecting taps by target ID should append tap statuses""" # Get target definitions from JSON file targets = cli.utils.load_json("{}/config.json".format(CONFIG_DIR)).get( "targets", []) target_one = next( (item for item in targets if item["id"] == "target_one"), False) target_two = next( (item for item in targets if item["id"] == "target_two"), False) # Append the tap statuses to every tap in target_one exp_tap_one = target_one["taps"][0] exp_tap_two = target_one["taps"][1] exp_tap_one["status"] = self.pipelinewise.detect_tap_status( "target_one", exp_tap_one["id"]) exp_tap_two["status"] = self.pipelinewise.detect_tap_status( "target_one", exp_tap_two["id"]) # Append the tap statuses to every tap in target_one exp_tap_three = target_two["taps"][0] exp_tap_three["status"] = self.pipelinewise.detect_tap_status( "target_two", exp_tap_three["id"]) # Tap statuses should be appended to every tap assert self.pipelinewise.get_taps("target_one") == [ exp_tap_one, exp_tap_two ] assert self.pipelinewise.get_taps("target_two") == [exp_tap_three] def test_get_tap(self): """Getting tap by ID should return status, connector and target props as well""" # Get target definitions from JSON file targets = cli.utils.load_json("{}/config.json".format(CONFIG_DIR)).get( "targets", []) target_one = next( (item for item in targets if item["id"] == "target_one"), False) # Append the tap status, files and target keys to the tap exp_tap_one = target_one["taps"][0] exp_tap_one["status"] = self.pipelinewise.detect_tap_status( "target_one", exp_tap_one["id"]) exp_tap_one["files"] = self.pipelinewise.get_connector_files( "{}/target_one/tap_one".format(CONFIG_DIR)) exp_tap_one["target"] = self.pipelinewise.get_target("target_one") # Getting tap by ID should match to original JSON and should contain the status, connector files and target props assert self.pipelinewise.get_tap("target_one", "tap_one") == exp_tap_one def test_get_not_existing_target(self): """Test getting not existing target""" # Getting not existing from should raise exception with pytest.raises(Exception): assert self.pipelinewise.get_target("not-existing-target") == {} def test_get_taps_from_not_existing_target(self): """Test getting taps from not existing target""" # Getting not existing from should raise exception with pytest.raises(Exception): assert self.pipelinewise.get_tap("not-existing-target", "not-existing-tap") == {} def test_get_not_existing_tap(self): """Test getting not existing tap from existing target""" # Getting not existing from should raise exception with pytest.raises(Exception): assert self.pipelinewise.get_tap("target_one", "not-existing-tap") == {} def test_create_filtered_tap_properties(self): """Test creating fastsync and singer specific properties file""" # TODO: review this completely ( tap_properties_fastsync, fastsync_stream_ids, tap_properties_singer, singer_stream_ids, ) = self.pipelinewise.create_filtered_tap_properties( "target_snowflake", "tap_mysql", "{}/resources/sample_json_config/target_one/tap_one/properties.json" .format(os.path.dirname(__file__)), "{}/resources/sample_json_config/target_one/tap_one/state.json". format(os.path.dirname(__file__)), { "selected": True, "target_type": ["target-snowflake"], "tap_type": ["tap-mysql", "tap-postgres"], "initial_sync_required": True, }, create_fallback=True, ) # Fastsync and singer properties should be created assert os.path.isfile(tap_properties_fastsync) assert os.path.isfile(tap_properties_singer) # Delete generated properties file os.remove(tap_properties_fastsync) os.remove(tap_properties_singer) # Fastsync and singer properties should be created # assert fastsync_stream_ids == [] # assert singer_stream_ids == [] def test_merge_empty_catalog(self): """Merging two empty singer schemas should be another empty""" # TODO: Check if pipelinewise.merge_schemas is required at all or not assert self.pipelinewise.merge_schemas({}, {}) == {} def test_merge_empty_stream_catalog(self): """Merging empty schemas should be empty""" # TODO: Check if pipelinewise.merge_schemas is required at all or not assert self.pipelinewise.merge_schemas({"streams": []}, {"streams": []}) == { "streams": [] } def test_merge_same_catalog(self): """Test merging not empty schemas""" # TODO: Check if pipelinewise.merge_schemas is required at all or not tap_one_catalog = cli.utils.load_json( "{}/resources/sample_json_config/target_one/tap_one/properties.json" .format(os.path.dirname(__file__))) assert self.pipelinewise.merge_schemas( tap_one_catalog, tap_one_catalog) == tap_one_catalog def test_merge_updated_catalog(self): """Test merging not empty schemas""" # TODO: Check if pipelinewise.merge_schemas is required at all or not tap_one_catalog = cli.utils.load_json( "{}/resources/sample_json_config/target_one/tap_one/properties.json" .format(os.path.dirname(__file__))) tap_one_updated_catalog = cli.utils.load_json( "{}/resources/sample_json_config/target_one/tap_one/properties_updated.json" .format(os.path.dirname(__file__))) assert (self.pipelinewise.merge_schemas( tap_one_catalog, tap_one_updated_catalog) == tap_one_catalog) def test_make_default_selection(self): """Test if streams selected correctly in catalog JSON""" tap_one_catalog = cli.utils.load_json( "{}/resources/sample_json_config/target_one/tap_one/properties.json" .format(os.path.dirname(__file__))) tap_one_selection_file = "{}/resources/sample_json_config/target_one/tap_one/selection.json".format( os.path.dirname(__file__)) # Update catalog selection tap_one_with_selection = self.pipelinewise.make_default_selection( tap_one_catalog, tap_one_selection_file) # Table one has to be selected with LOG_BASED replication method assert tap_one_with_selection["streams"][0]["metadata"][0]["metadata"][ "selected"] == True assert (tap_one_with_selection["streams"][0]["metadata"][0]["metadata"] ["replication-method"] == "LOG_BASED") # Table two has to be selected with INCREMENTAL replication method assert tap_one_with_selection["streams"][1]["metadata"][0]["metadata"][ "selected"] == True assert (tap_one_with_selection["streams"][1]["metadata"][0]["metadata"] ["replication-method"] == "INCREMENTAL") assert (tap_one_with_selection["streams"][1]["metadata"][0]["metadata"] ["replication-key"] == "id") # Table three should not be selected assert tap_one_with_selection["streams"][2]["metadata"][0]["metadata"][ "selected"] == False def test_create_consumable_target_config(self): """Test merging target config.json and inheritable_config.json""" target_config = "{}/resources/target-config.json".format( os.path.dirname(__file__)) tap_inheritable_config = "{}/resources/tap-inheritable-config.json".format( os.path.dirname(__file__)) # The merged JSON written into a temp file temp_file = self.pipelinewise.create_consumable_target_config( target_config, tap_inheritable_config) cons_targ_config = cli.utils.load_json(temp_file) # The merged object needs assert cons_targ_config == { "account": "foo", "aws_access_key_id": "secret", "aws_secret_access_key": "secret/", "client_side_encryption_master_key": "secret=", "dbname": "my_db", "file_format": "my_file_format", "password": "******", "s3_bucket": "foo", "s3_key_prefix": "foo/", "stage": "my_stage", "user": "******", "warehouse": "MY_WAREHOUSE", "batch_size_rows": 5000, "data_flattening_max_level": 0, "default_target_schema": "jira_clear", "default_target_schema_select_permissions": ["grp_power"], "hard_delete": True, "primary_key_required": True, "schema_mapping": { "jira": { "target_schema": "jira_clear", "target_schema_select_permissions": ["grp_power"], } }, } # Remove temp file with merged JSON os.remove(temp_file) def test_invalid_create_consumable_target_config(self): """Test merging invalid target config.json and inheritable_config.json""" target_config = "{}/resources/invalid.json".format( os.path.dirname(__file__)) tap_inheritable_config = "not-existing-json" # Merging invalid or not existing JSONs should raise exception with pytest.raises(Exception): self.pipelinewise.create_consumable_target_config( target_config, tap_inheritable_config) def test_command_encrypt_string(self, capsys): """Test vault encryption command output""" secret_path = "{}/resources/vault-secret.txt".format( os.path.dirname(__file__)) args = CliArgs(string="plain text", secret=secret_path) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Encrypted string should be printed to stdout pipelinewise.encrypt_string() stdout, stderr = capsys.readouterr() assert not stderr.strip() assert stdout.startswith("!vault |") and "$ANSIBLE_VAULT;" in stdout def test_command_init(self): """Test init command""" args = CliArgs(name=TEST_PROJECT_NAME) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Init new project pipelinewise.init() # The test project should contain every sample YAML file for s in os.listdir("{}/../../../pipelinewise/cli/samples".format( os.path.dirname(__file__))): assert os.path.isfile(os.path.join(TEST_PROJECT_DIR, s)) # Re-creating project should reaise exception of directory not empty with pytest.raises(SystemExit) as pytest_wrapped_e: pipelinewise.init() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 1 def test_command_status(self, capsys): """Test status command output""" # Status table should be printed to stdout self.pipelinewise.status() stdout, stderr = capsys.readouterr() assert not stderr.strip() # Exact output match assert ( stdout == """Tap ID Tap Type Target ID Target Type Enabled Status Last Sync Last Sync Result --------- ------------ ----------- ---------------- --------- -------------- ----------- ------------------ tap_one tap-mysql target_one target-snowflake True ready unknown tap_two tap-postgres target_one target-snowflake True ready unknown tap_three tap-mysql target_two target-s3-csv True not-configured unknown 3 pipeline(s) """) def test_command_discover_tap(self, capsys): """Test discover tap command""" args = CliArgs(target="target_one", tap="tap_one") pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Running discovery mode should detect the tap type and path to the connector # Since the executable is not available in this test then it should fail pipelinewise.discover_tap() stdout, stderr = capsys.readouterr() exp_err_pattern = os.path.join( VIRTUALENVS_DIR, "/tap-mysql/bin/tap-mysql: No such file or directory") assert exp_err_pattern in stdout or exp_err_pattern in stderr def _test_command_run_tap(self, capsys): """Test run tap command""" args = CliArgs(target="target_one", tap="tap_one") pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Running run mode should detect the tap type and path to the connector # Since the executable is not available in this test then it should fail # TODO: sync discover_tap and run_tap behaviour. run_tap sys.exit but discover_tap does not. with pytest.raises(SystemExit) as pytest_wrapped_e: pipelinewise.run_tap() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 1 def test_command_sync_tables(self, capsys): """Test run tap command""" args = CliArgs(target="target_one", tap="tap_one") pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Running sync_tables should detect the tap type and path to the connector # Since the executable is not available in this test then it should fail # TODO: sync discover_tap and run_tap behaviour. run_tap sys.exit but discover_tap does not. with pytest.raises(SystemExit) as pytest_wrapped_e: pipelinewise.sync_tables() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 1
class TestCli: """ Unit Tests for PipelineWise CLI executable """ def setup_method(self): """Create CLI arguments""" self.args = CliArgs(log='coverage.log') self.pipelinewise = PipelineWise( self.args, CONFIG_DIR, VIRTUALENVS_DIR, PROFILING_DIR ) if os.path.exists('/tmp/pwtest'): shutil.rmtree('/tmp/pwtest') def teardown_method(self): """Delete test directories""" try: shutil.rmtree(TEST_PROJECT_DIR) shutil.rmtree(os.path.join(CONFIG_DIR, 'target_one/tap_one/log')) except Exception: pass @staticmethod def _init_for_sync_tables_states_cleanup(tables_arg: str = None) -> PipelineWise: temp_path = '/tmp/pwtest/' args = CliArgs(target='target_one', tap='tap_one', tables=tables_arg) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) pipelinewise.tap['files']['state'] = f'{temp_path}state.json' pipelinewise.venv_dir = temp_path # Making a test tap bin file os.makedirs(f'{temp_path}pipelinewise/bin') with open(f'{temp_path}pipelinewise/bin/mysql-to-snowflake', 'a', encoding='UTF-8'): pass return pipelinewise @staticmethod def _make_sample_state_file(test_state_file: str) -> None: sample_state_data = { 'currently_syncing': None, 'bookmarks': { 'table1': {'foo': 'bar'}, 'table2': {'foo': 'bar'}, 'table3': {'foo': 'bar'} } } with open(test_state_file, 'w', encoding='UTF-8') as state_file: json.dump(sample_state_data, state_file) @staticmethod def _assert_calling_sync_tables(pipelinewise: PipelineWise, side_effect_method: Optional[Callable] = None) -> None: with patch('pipelinewise.cli.pipelinewise.PipelineWise.run_tap_fastsync') as mocked_fastsync: if side_effect_method: mocked_fastsync.side_effect = side_effect_method pipelinewise.sync_tables() mocked_fastsync.assert_called_once() def test_target_dir(self): """Singer target connector config path must be relative to the project config dir""" assert self.pipelinewise.get_target_dir( 'dummy-target' ) == '{}/dummy-target'.format(CONFIG_DIR) def test_tap_dir(self): """Singer tap connector config path must be relative to the target connector config path""" assert self.pipelinewise.get_tap_dir( 'dummy-target', 'dummy-tap' ) == '{}/dummy-target/dummy-tap'.format(CONFIG_DIR) def test_tap_log_dir(self): """Singer tap log path must be relative to the tap connector config path""" assert self.pipelinewise.get_tap_log_dir( 'dummy-target', 'dummy-tap' ) == '{}/dummy-target/dummy-tap/log'.format(CONFIG_DIR) def test_connector_bin(self): """Singer connector binary must be at a certain location under PIPELINEWISE_HOME .virtualenvs dir""" assert self.pipelinewise.get_connector_bin( 'dummy-type' ) == '{}/dummy-type/bin/dummy-type'.format(VIRTUALENVS_DIR) def test_not_existing_config_dir(self): """Test with not existing config dir""" # Create a new pipelinewise object pointing to a not existing config directory pipelinewise_with_no_config = PipelineWise( self.args, 'not-existing-config-dir', VIRTUALENVS_DIR ) # It should return and empty config with empty list targets # TODO: Make this scenario to fail with error message of "config dir not exists" assert pipelinewise_with_no_config.config == {} assert pipelinewise_with_no_config.get_targets() == [] def test_get_targets(self): """Targets should be loaded from JSON as is""" assert self.pipelinewise.get_targets() == cli.utils.load_json( '{}/config.json'.format(CONFIG_DIR) ).get('targets', []) def test_get_target(self): """Selecting target by ID should append connector files""" # Get target definitions from JSON file targets = cli.utils.load_json('{}/config.json'.format(CONFIG_DIR)).get( 'targets', [] ) exp_target_one = next( (item for item in targets if item['id'] == 'target_one'), False ) exp_target_two = next( (item for item in targets if item['id'] == 'target_two'), False ) # Append the connector file paths to the expected targets exp_target_one['files'] = Config.get_connector_files( '{}/target_one'.format(CONFIG_DIR) ) exp_target_two['files'] = Config.get_connector_files( '{}/target_two'.format(CONFIG_DIR) ) # Getting target by ID should match to original JSON and should contains the connector files list assert self.pipelinewise.get_target('target_one') == exp_target_one assert self.pipelinewise.get_target('target_two') == exp_target_two def test_get_taps(self): """Selecting taps by target ID should append tap statuses""" # Get target definitions from JSON file targets = cli.utils.load_json('{}/config.json'.format(CONFIG_DIR)).get( 'targets', [] ) target_one = next( (item for item in targets if item['id'] == 'target_one'), False ) target_two = next( (item for item in targets if item['id'] == 'target_two'), False ) # Append the tap statuses to every tap in target_one exp_tap_one = target_one['taps'][0] exp_tap_two = target_one['taps'][1] exp_tap_one['status'] = self.pipelinewise.detect_tap_status( 'target_one', exp_tap_one['id'] ) exp_tap_two['status'] = self.pipelinewise.detect_tap_status( 'target_one', exp_tap_two['id'] ) # Append the tap statuses to every tap in target_one exp_tap_three = target_two['taps'][0] exp_tap_three['status'] = self.pipelinewise.detect_tap_status( 'target_two', exp_tap_three['id'] ) # Tap statuses should be appended to every tap assert self.pipelinewise.get_taps('target_one') == [exp_tap_one, exp_tap_two] assert self.pipelinewise.get_taps('target_two') == [exp_tap_three] def test_get_tap(self): """Getting tap by ID should return status, connector and target props as well""" # Get target definitions from JSON file targets = cli.utils.load_json('{}/config.json'.format(CONFIG_DIR)).get( 'targets', [] ) target_one = next( (item for item in targets if item['id'] == 'target_one'), False ) # Append the tap status, files and target keys to the tap exp_tap_one = target_one['taps'][0] exp_tap_one['status'] = self.pipelinewise.detect_tap_status( 'target_one', exp_tap_one['id'] ) exp_tap_one['files'] = Config.get_connector_files( '{}/target_one/tap_one'.format(CONFIG_DIR) ) exp_tap_one['target'] = self.pipelinewise.get_target('target_one') # Getting tap by ID should match to original JSON and should contain status, connector files and target props assert self.pipelinewise.get_tap('target_one', 'tap_one') == exp_tap_one def test_get_not_existing_target(self): """Test getting not existing target""" # Getting not existing from should raise exception with pytest.raises(Exception): assert self.pipelinewise.get_target('not-existing-target') == {} def test_get_taps_from_wrong_target(self): """Test getting taps from not existing target""" # Getting not existing from should raise exception with pytest.raises(Exception): assert ( self.pipelinewise.get_tap('not-existing-target', 'not-existing-tap') == {} ) def test_get_not_existing_tap(self): """Test getting not existing tap from existing target""" # Getting not existing from should raise exception with pytest.raises(Exception): assert self.pipelinewise.get_tap('target_one', 'not-existing-tap') == {} def test_create_filtered_tap_props(self): """Test creating fastsync and singer specific properties file""" ( tap_properties_fastsync, fastsync_stream_ids, tap_properties_singer, singer_stream_ids, ) = self.pipelinewise.create_filtered_tap_properties( target_type=ConnectorType('target-snowflake'), tap_type=ConnectorType('tap-mysql'), tap_properties='{}/resources/sample_json_config/target_one/tap_one/properties.json'.format( os.path.dirname(__file__) ), tap_state='{}/resources/sample_json_config/target_one/tap_one/state.json'.format( os.path.dirname(__file__) ), filters={ 'selected': True, 'tap_target_pairs': { ConnectorType.TAP_MYSQL: {ConnectorType.TARGET_SNOWFLAKE}, ConnectorType.TAP_POSTGRES: {ConnectorType.TARGET_SNOWFLAKE}, }, 'initial_sync_required': True, }, create_fallback=True, ) # Fastsync and singer properties should be created assert os.path.isfile(tap_properties_fastsync) assert os.path.isfile(tap_properties_singer) # Delete generated properties file os.remove(tap_properties_fastsync) os.remove(tap_properties_singer) # Fastsync and singer properties should be created assert fastsync_stream_ids == [ 'db_test_mysql-table_one', 'db_test_mysql-table_two', ] assert singer_stream_ids == [ 'db_test_mysql-table_one', 'db_test_mysql-table_two', ] def test_create_filtered_tap_props_no_fastsync(self): """Test creating only singer specific properties file""" ( tap_properties_fastsync, fastsync_stream_ids, tap_properties_singer, singer_stream_ids, ) = self.pipelinewise.create_filtered_tap_properties( target_type=ConnectorType('target-snowflake'), tap_type=ConnectorType('tap-mysql'), tap_properties='{}/resources/sample_json_config/target_one/tap_one/properties.json'.format( os.path.dirname(__file__) ), tap_state='{}/resources/sample_json_config/target_one/tap_one/state.json'.format( os.path.dirname(__file__) ), filters={ 'selected': True, 'tap_target_pairs': { ConnectorType.TAP_MYSQL: {ConnectorType.TARGET_REDSHIFT}, ConnectorType.TAP_POSTGRES: {ConnectorType.TARGET_SNOWFLAKE}, }, 'initial_sync_required': True, }, create_fallback=True, ) # fastsync and singer properties should be created assert os.path.isfile(tap_properties_fastsync) assert os.path.isfile(tap_properties_singer) # Delete generated properties file os.remove(tap_properties_fastsync) os.remove(tap_properties_singer) # only singer properties should be created assert fastsync_stream_ids == [] assert singer_stream_ids == [ 'db_test_mysql-table_one', 'db_test_mysql-table_two', ] def test_merge_empty_catalog(self): """Merging two empty singer schemas should be another empty""" # TODO: Check if pipelinewise.merge_schemas is required at all or not assert self.pipelinewise.merge_schemas({}, {}) == {} def test_merge_empty_stream_catalog(self): """Merging empty schemas should be empty""" # TODO: Check if pipelinewise.merge_schemas is required at all or not assert self.pipelinewise.merge_schemas({'streams': []}, {'streams': []}) == { 'streams': [] } def test_merge_same_catalog(self): """Test merging not empty schemas""" # TODO: Check if pipelinewise.merge_schemas is required at all or not tap_one_catalog = cli.utils.load_json( '{}/resources/sample_json_config/target_one/tap_one/properties.json'.format( os.path.dirname(__file__) ) ) assert ( self.pipelinewise.merge_schemas(tap_one_catalog, tap_one_catalog) == tap_one_catalog ) def test_merge_updated_catalog(self): """Test merging not empty schemas""" # TODO: Check if pipelinewise.merge_schemas is required at all or not tap_one_catalog = cli.utils.load_json( '{}/resources/sample_json_config/target_one/tap_one/properties.json'.format( os.path.dirname(__file__) ) ) tap_one_updated_catalog = cli.utils.load_json( '{}/resources/sample_json_config/target_one/tap_one/properties_updated.json'.format( os.path.dirname(__file__) ) ) assert ( self.pipelinewise.merge_schemas(tap_one_catalog, tap_one_updated_catalog) == tap_one_catalog ) def test_make_default_selection(self): """Test if streams selected correctly in catalog JSON""" tap_one_catalog = cli.utils.load_json( '{}/resources/sample_json_config/target_one/tap_one/properties.json'.format( os.path.dirname(__file__) ) ) tap_one_selection_file = ( '{}/resources/sample_json_config/target_one/tap_one/selection.json'.format( os.path.dirname(__file__) ) ) # Update catalog selection tap_one_with_selection = self.pipelinewise.make_default_selection( tap_one_catalog, tap_one_selection_file ) # Table one has to be selected with LOG_BASED replication method assert ( tap_one_with_selection['streams'][0]['metadata'][0]['metadata']['selected'] is True ) assert ( tap_one_with_selection['streams'][0]['metadata'][0]['metadata'][ 'replication-method' ] == 'LOG_BASED' ) # Table two has to be selected with INCREMENTAL replication method assert ( tap_one_with_selection['streams'][1]['metadata'][0]['metadata']['selected'] is True ) assert ( tap_one_with_selection['streams'][1]['metadata'][0]['metadata'][ 'replication-method' ] == 'INCREMENTAL' ) assert ( tap_one_with_selection['streams'][1]['metadata'][0]['metadata'][ 'replication-key' ] == 'id' ) # Table three should not be selected assert ( tap_one_with_selection['streams'][2]['metadata'][0]['metadata']['selected'] is False ) def test_target_config(self): """Test merging target config.json and inheritable_config.json""" target_config = '{}/resources/target-config.json'.format( os.path.dirname(__file__) ) tap_inheritable_config = '{}/resources/tap-inheritable-config.json'.format( os.path.dirname(__file__) ) # The merged JSON written into a temp file temp_file = self.pipelinewise.create_consumable_target_config( target_config, tap_inheritable_config ) cons_targ_config = cli.utils.load_json(temp_file) # The merged object needs assert cons_targ_config == { 'account': 'foo', 'aws_access_key_id': 'secret', 'aws_secret_access_key': 'secret/', 'client_side_encryption_master_key': 'secret=', 'dbname': 'my_db', 'file_format': 'my_file_format', 'password': '******', 's3_bucket': 'foo', 's3_key_prefix': 'foo/', 'stage': 'my_stage', 'user': '******', 'warehouse': 'MY_WAREHOUSE', 'batch_size_rows': 5000, 'data_flattening_max_level': 0, 'default_target_schema': 'jira_clear', 'default_target_schema_select_permissions': ['grp_power'], 'hard_delete': True, 'primary_key_required': True, 'schema_mapping': { 'jira': { 'target_schema': 'jira_clear', 'target_schema_select_permissions': ['grp_power'], } }, } # Remove temp file with merged JSON os.remove(temp_file) def test_invalid_target_config(self): """Test merging invalid target config.json and inheritable_config.json""" target_config = '{}/resources/invalid.json'.format(os.path.dirname(__file__)) tap_inheritable_config = 'not-existing-json' # Merging invalid or not existing JSONs should raise exception with pytest.raises(Exception): self.pipelinewise.create_consumable_target_config( target_config, tap_inheritable_config ) def test_send_alert(self): """Test if alert""" with patch( 'pipelinewise.cli.alert_sender.AlertSender.send_to_all_handlers' ) as alert_sender_mock: alert_sender_mock.return_value = {'sent': 1} # Should send alert and should return stats if alerting enabled on the tap self.pipelinewise.tap = self.pipelinewise.get_tap('target_one', 'tap_one') assert self.pipelinewise.send_alert('test-message') == {'sent': 1} # Should not send alert and should return none if alerting disabled on the tap self.pipelinewise.tap = self.pipelinewise.get_tap('target_one', 'tap_two') assert self.pipelinewise.send_alert('test-message') == {'sent': 0} def test_send_alert_to_tap_specific_slack_channel(self): """Test if sends alert to the tap specific channel""" config_dir = f'{RESOURCES_DIR}/sample_json_config_for_specific_slack_channel' pipelinewise = PipelineWise( self.args, config_dir, VIRTUALENVS_DIR, PROFILING_DIR ) pipelinewise.tap = pipelinewise.get_tap('target_one', 'tap_one') with patch.object(WebClient, 'chat_postMessage') as mocked_slack: pipelinewise.send_alert('test-message') # Assert if alert is sent to the the main channel and also to the tap channel mocked_slack.assert_has_calls( [ call( channel=pipelinewise.alert_sender.alert_handlers['slack']['channel'], text=None, attachments=[{'color': 'danger', 'title': 'test-message'}] ), call( channel=pipelinewise.tap['slack_alert_channel'], text=None, attachments=[{'color': 'danger', 'title': 'test-message'}] ) ] ) def test_command_encrypt_string(self, capsys): """Test vault encryption command output""" secret_path = '{}/resources/vault-secret.txt'.format(os.path.dirname(__file__)) args = CliArgs(string='plain text', secret=secret_path) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Encrypted string should be printed to stdout pipelinewise.encrypt_string() stdout, stderr = capsys.readouterr() assert not stderr.strip() assert stdout.startswith('!vault |') and '$ANSIBLE_VAULT;' in stdout def test_command_init(self): """Test init command""" args = CliArgs(name=TEST_PROJECT_NAME) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Init new project pipelinewise.init() # The test project should contain every sample YAML file for sample_yaml in os.listdir( '{}/../../../pipelinewise/cli/samples'.format(os.path.dirname(__file__)) ): assert os.path.isfile(os.path.join(TEST_PROJECT_DIR, sample_yaml)) # Re-creating project should raise exception of directory not empty with pytest.raises(SystemExit) as pytest_wrapped_e: pipelinewise.init() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 1 def test_command_status(self, capsys): """Test status command output""" # Status table should be printed to stdout self.pipelinewise.status() stdout, stderr = capsys.readouterr() assert not stderr.strip() # Exact output match # pylint: disable=line-too-long assert ( stdout == """Tap ID Tap Type Target ID Target Type Enabled Status Last Sync Last Sync Result --------- ------------ ----------- ---------------- --------- -------------- ----------- ------------------ tap_one tap-mysql target_one target-snowflake True ready unknown tap_two tap-postgres target_one target-snowflake True ready unknown tap_three tap-mysql target_two target-s3-csv True not-configured unknown 3 pipeline(s) """ ) def test_command_discover_tap(self): """Test discover tap command""" args = CliArgs(target='target_one', tap='tap_one') pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Running discovery mode should detect the tap type and path to the connector # Since the executable is not available in this test then it should fail result = pipelinewise.discover_tap() exp_err_pattern = '/tap-mysql/bin/tap-mysql: No such file or directory' assert exp_err_pattern in result def _test_command_run_tap(self): """Test run tap command""" args = CliArgs(target='target_one', tap='tap_one') pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Running run mode should detect the tap type and path to the connector # Since the executable is not available in this test then it should fail # TODO: sync discover_tap and run_tap behaviour. run_tap sys.exit but discover_tap does not. with pytest.raises(SystemExit) as pytest_wrapped_e: pipelinewise.run_tap() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 1 def test_command_stop_tap(self): """Test stop tap command""" args = CliArgs(target='target_one', tap='tap_one') pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) pipelinewise.tap_run_log_file = 'test-tap-run-dummy.log' Path('{}.running'.format(pipelinewise.tap_run_log_file)).touch() # Tap is not running, pid file not exist, should exit with error with pytest.raises(SystemExit) as pytest_wrapped_e: pipelinewise.stop_tap() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 1 # Stop tap command should stop all the child processes # 1. Start the pipelinewise mock executable that's running # linux piped dummy tap and target connectors with pidfile.PIDFile(pipelinewise.tap['files']['pidfile']): os.spawnl( os.P_NOWAIT, f'{RESOURCES_DIR}/test_stop_tap/scheduler-mock.sh', 'test_stop_tap/scheduler-mock.sh', ) # Wait 5 seconds making sure the dummy tap is running time.sleep(5) # Send the stop_tap command with pytest.raises(SystemExit): pipelinewise.stop_tap() # Should not have any remaining Pipelinewise related linux process for proc in psutil.process_iter(['cmdline']): full_command = ' '.join(proc.info['cmdline']) if proc.info['cmdline'] else '' assert re.match('scheduler|pipelinewise|tap|target', full_command) is None # Graceful exit should rename log file from running status to terminated assert os.path.isfile('{}.terminated'.format(pipelinewise.tap_run_log_file)) # Delete test log file os.remove('{}.terminated'.format(pipelinewise.tap_run_log_file)) def test_command_sync_tables(self): """Test run tap command""" args = CliArgs(target='target_one', tap='tap_one') pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Running sync_tables should detect the tap type and path to the connector # Since the executable is not available in this test then it should fail # TODO: sync discover_tap and run_tap behaviour. run_tap sys.exit but discover_tap does not. with pytest.raises(SystemExit) as pytest_wrapped_e: pipelinewise.sync_tables() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 1 def test_command_sync_tables_cleanup_state_if_file_not_exists_and_no_tables_argument(self): """Testing sync_tables cleanup state if file not exists and there is no tables argument""" pipelinewise = self._init_for_sync_tables_states_cleanup() self._assert_calling_sync_tables(pipelinewise) def test_command_sync_tables_cleanup_state_if_file_not_exists_and_tables_argument(self): """Testing sync_tables cleanup state if file not exists and there is tables argument""" pipelinewise = self._init_for_sync_tables_states_cleanup(tables_arg='table1,table3') self._assert_calling_sync_tables(pipelinewise) def test_command_sync_tables_cleanup_state_if_file_exists_and_no_table_argument(self): """Testing sync_tables cleanup state if file exists and there is no table argument""" def _assert_state_file_is_deleted(*args, **kwargs): # pylint: disable=unused-argument assert os.path.isfile(test_state_file) is False pipelinewise = self._init_for_sync_tables_states_cleanup() test_state_file = pipelinewise.tap['files']['state'] self._make_sample_state_file(test_state_file) self._assert_calling_sync_tables(pipelinewise, _assert_state_file_is_deleted) def test_command_sync_tables_cleanup_state_if_file_exists_and_table_argument(self): """Testing sync_tables cleanup state if file exists and there is table argument""" def _assert_state_file_is_cleaned(*args, **kwargs): # pylint: disable=unused-argument expected_state_data = { 'currently_syncing': None, 'bookmarks': { 'table2': {'foo': 'bar'}, } } with open(test_state_file, encoding='UTF-8') as state_file: state_data = json.load(state_file) assert state_data == expected_state_data pipelinewise = self._init_for_sync_tables_states_cleanup(tables_arg='table1,table3') test_state_file = pipelinewise.tap['files']['state'] self._make_sample_state_file(test_state_file) self._assert_calling_sync_tables(pipelinewise, _assert_state_file_is_cleaned) def test_command_sync_tables_cleanup_state_if_file_empty_and_table_argument(self): """Testing sync_tables cleanup state if file empty and there is table argument""" pipelinewise = self._init_for_sync_tables_states_cleanup(tables_arg='table1,table3') test_state_file = pipelinewise.tap['files']['state'] with open(test_state_file, 'a', encoding='UTF-8'): pass self._assert_calling_sync_tables(pipelinewise) def test_validate_command_1(self): """Test validate command should fail because of missing replication key for incremental""" test_validate_command_dir =\ f'{os.path.dirname(__file__)}/resources/test_validate_command/missing_replication_key_incremental' args = CliArgs(dir=test_validate_command_dir) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) with pytest.raises(InvalidConfigException): pipelinewise.validate() def test_validate_command_2(self): """Test validate command should succeed""" test_validate_command_dir =\ f'{os.path.dirname(__file__)}/resources/test_validate_command/missing_replication_key' args = CliArgs(dir=test_validate_command_dir) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) pipelinewise.validate() def test_validate_command_3(self): """Test validate command should fail because of invalid target in tap config""" test_validate_command_dir = f'{os.path.dirname(__file__)}/resources/test_validate_command/invalid_target' args = CliArgs(dir=test_validate_command_dir) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) with pytest.raises(InvalidConfigException): pipelinewise.validate() def test_validate_command_4(self): """Test validate command should fail because of duplicate targets""" test_validate_command_dir =\ f'{os.path.dirname(__file__)}/resources/test_validate_command/test_yaml_config_two_targets' args = CliArgs(dir=test_validate_command_dir) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) with pytest.raises(DuplicateConfigException): pipelinewise.validate() def test_validate_command_5(self): """ Test validate command should fail because of transformation on json properties for a tap-target combo that has Fastsync """ test_validate_command_dir = \ f'{os.path.dirname(__file__)}/resources/test_validate_command/json_transformation_in_fastsync' args = CliArgs(dir=test_validate_command_dir) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) with pytest.raises(InvalidTransformationException): pipelinewise.validate() # pylint: disable=protected-access def test_post_import_checks(self): """Test post import checks""" args = CliArgs() pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) test_files_dir = '{}/resources/test_post_import_checks'.format( os.path.dirname(__file__) ) tap_pk_required = cli.utils.load_json( '{}/tap_config_pk_required.json'.format(test_files_dir) ) tap_pk_not_required = cli.utils.load_json( '{}/tap_config_pk_not_required.json'.format(test_files_dir) ) tap_pk_not_defined = cli.utils.load_json( '{}/tap_config_pk_not_defined.json'.format(test_files_dir) ) tap_with_pk = cli.utils.load_json( '{}//tap_properties_with_pk.json'.format(test_files_dir) ) tap_with_no_pk_full_table = cli.utils.load_json( '{}//tap_properties_with_no_pk_full_table.json'.format(test_files_dir) ) tap_with_no_pk_incremental = cli.utils.load_json( '{}//tap_properties_with_no_pk_incremental.json'.format(test_files_dir) ) tap_with_no_pk_log_based = cli.utils.load_json( '{}//tap_properties_with_no_pk_log_based.json'.format(test_files_dir) ) tap_with_no_pk_not_selected = cli.utils.load_json( '{}//tap_properties_with_no_pk_not_selected.json'.format(test_files_dir) ) with patch( 'pipelinewise.cli.pipelinewise.commands.run_command' ) as run_command_mock: # Test scenarios when post import checks should pass assert ( pipelinewise._run_post_import_tap_checks( tap_pk_required, tap_with_pk, 'snowflake' ) == [] ) assert ( pipelinewise._run_post_import_tap_checks( tap_pk_not_required, tap_with_pk, 'snowflake' ) == [] ) assert ( pipelinewise._run_post_import_tap_checks( tap_pk_required, tap_with_no_pk_full_table, 'snowflake' ) == [] ) assert ( pipelinewise._run_post_import_tap_checks( tap_pk_not_required, tap_with_no_pk_incremental, 'snowflake' ) == [] ) assert ( pipelinewise._run_post_import_tap_checks( tap_pk_not_required, tap_with_no_pk_log_based, 'snowflake' ) == [] ) assert ( pipelinewise._run_post_import_tap_checks( tap_pk_required, tap_with_no_pk_not_selected, 'snowflake' ) == [] ) assert ( pipelinewise._run_post_import_tap_checks( tap_pk_not_defined, tap_with_no_pk_full_table, 'snowflake' ) == [] ) # Test scenarios when post import checks should fail due to primary keys not exists assert ( len( pipelinewise._run_post_import_tap_checks( tap_pk_required, tap_with_no_pk_incremental, 'snowflake' ) ) == 1 ) assert ( len( pipelinewise._run_post_import_tap_checks( tap_pk_required, tap_with_no_pk_log_based, 'snowflake' ) ) == 1 ) assert ( len( pipelinewise._run_post_import_tap_checks( tap_pk_not_defined, tap_with_no_pk_incremental, 'snowflake' ) ) == 1 ) assert ( len( pipelinewise._run_post_import_tap_checks( tap_pk_not_defined, tap_with_no_pk_log_based, 'snowflake' ) ) == 1 ) # Test scenarios when post import checks should fail due to transformations validation command fails tap_with_trans = cli.utils.load_json( '{}/tap_config_with_transformations.json'.format(test_files_dir) ) run_command_mock.return_value = ( 1, None, 'transformation `HASH` cannot be applied', ) assert ( len( pipelinewise._run_post_import_tap_checks( tap_with_trans, tap_with_no_pk_not_selected, 'snowflake' ) ) == 1 ) assert ( len( pipelinewise._run_post_import_tap_checks( tap_with_trans, tap_with_no_pk_incremental, 'snowflake' ) ) == 2 ) # mock successful transformation validation command run_command_mock.return_value = (0, None, None) assert ( len( pipelinewise._run_post_import_tap_checks( tap_with_trans, tap_with_no_pk_not_selected, 'snowflake' ) ) == 0 ) assert ( len( pipelinewise._run_post_import_tap_checks( tap_with_trans, tap_with_no_pk_incremental, 'snowflake' ) ) == 1 ) assert run_command_mock.call_count == 4