def test_post_import_checks(self): """Test post import checks""" args = CliArgs() pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) test_files_dir = '{}/resources/test_post_import_checks'.format(os.path.dirname(__file__)) tap_pk_required = cli.utils.load_json('{}/tap_config_pk_required.json'.format(test_files_dir)) tap_pk_not_required = cli.utils.load_json('{}/tap_config_pk_not_required.json'.format(test_files_dir)) tap_pk_not_defined = cli.utils.load_json('{}/tap_config_pk_not_defined.json'.format(test_files_dir)) tap_with_pk = cli.utils.load_json('{}//tap_properties_with_pk.json'.format(test_files_dir)) tap_with_no_pk_full_table = cli.utils.load_json( '{}//tap_properties_with_no_pk_full_table.json'.format(test_files_dir)) tap_with_no_pk_incremental = cli.utils.load_json( '{}//tap_properties_with_no_pk_incremental.json'.format(test_files_dir)) tap_with_no_pk_log_based = cli.utils.load_json( '{}//tap_properties_with_no_pk_log_based.json'.format(test_files_dir)) tap_with_no_pk_not_selected = cli.utils.load_json( '{}//tap_properties_with_no_pk_not_selected.json'.format(test_files_dir)) # Test scenarios when post import checks should pass assert pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_pk) == [] assert pipelinewise._run_post_import_tap_checks(tap_pk_not_required, tap_with_pk) == [] assert pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_full_table) == [] assert pipelinewise._run_post_import_tap_checks(tap_pk_not_required, tap_with_no_pk_incremental) == [] assert pipelinewise._run_post_import_tap_checks(tap_pk_not_required, tap_with_no_pk_log_based) == [] assert pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_not_selected) == [] assert pipelinewise._run_post_import_tap_checks(tap_pk_not_defined, tap_with_no_pk_full_table) == [] # Test scenarios when post import checks should fail due to primary keys not exists assert len(pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_incremental)) == 1 assert len(pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_log_based)) == 1 assert len(pipelinewise._run_post_import_tap_checks(tap_pk_not_defined, tap_with_no_pk_incremental)) == 1 assert len(pipelinewise._run_post_import_tap_checks(tap_pk_not_defined, tap_with_no_pk_log_based)) == 1
def test_command_stop_tap(self): """Test stop tap command""" args = CliArgs(target='target_one', tap='tap_one') pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Tap is not running, pid file not exist, should exit with error with pytest.raises(SystemExit) as pytest_wrapped_e: pipelinewise.stop_tap() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 1 # Stop tap command should stop all the child processes # 1. Start the pipelinewise mock executable that's running # linux piped dummy tap and target connectors with pidfile.PIDFile(pipelinewise.tap['files']['pidfile']): os.spawnl(os.P_NOWAIT, f'{RESOURCES_DIR}/test_stop_tap/scheduler-mock.sh', 'test_stop_tap/scheduler-mock.sh') # Wait 5 seconds making sure the dummy tap is running time.sleep(5) # Send the stop_tap command with pytest.raises(SystemExit): pipelinewise.stop_tap() # Should not have any remaining Pipelinewise related linux process for proc in psutil.process_iter(['cmdline']): full_command = ' '.join(proc.info['cmdline']) assert re.match('scheduler|pipelinewise|tap|target', full_command) is None
def setup_method(self): """Create CLI arguments""" self.args = CliArgs(log='coverage.log') self.pipelinewise = PipelineWise( self.args, CONFIG_DIR, VIRTUALENVS_DIR, PROFILING_DIR ) if os.path.exists('/tmp/pwtest'): shutil.rmtree('/tmp/pwtest')
def test_validate_command_2(self): """Test validate command should succeed""" test_validate_command_dir = \ f'{os.path.dirname(__file__)}/resources/test_validate_command/missing_replication_key' args = CliArgs(dir=test_validate_command_dir) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) pipelinewise.validate()
def test_validate_command_3(self): """Test validate command should fail because of invalid target in tap config""" test_validate_command_dir = f'{os.path.dirname(__file__)}/resources/test_validate_command/invalid_target' args = CliArgs(dir=test_validate_command_dir) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) with pytest.raises(SystemExit): pipelinewise.validate()
def test_command_stop_tap(self): """Test stop tap command""" args = CliArgs(target='target_one', tap='tap_one') pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Tap is not running, pid file not exist, should exit with error with pytest.raises(SystemExit) as pytest_wrapped_e: pipelinewise.stop_tap() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 1
def test_validate_command_1(self): """Test validate command should fail because of missing replication key for incremental""" test_validate_command_dir = \ f'{os.path.dirname(__file__)}/resources/test_validate_command/missing_replication_key_incremental' args = CliArgs(dir=test_validate_command_dir) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) with pytest.raises(SystemExit): pipelinewise.validate()
def test_validate_command_4(self): """Test validate command should fail because of duplicate targets""" test_validate_command_dir = \ f'{os.path.dirname(__file__)}/resources/test_validate_command/test_yaml_config_two_targets' args = CliArgs(dir=test_validate_command_dir) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) with pytest.raises(SystemExit): pipelinewise.validate()
def test_command_discover_tap(self): """Test discover tap command""" args = CliArgs(target='target_one', tap='tap_one') pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Running discovery mode should detect the tap type and path to the connector # Since the executable is not available in this test then it should fail result = pipelinewise.discover_tap() exp_err_pattern = '/tap-mysql/bin/tap-mysql: No such file or directory' assert exp_err_pattern in result
def test_command_discover_tap(self, capsys): """Test discover tap command""" args = CliArgs(target='target_one', tap='tap_one') pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Running discovery mode should detect the tap type and path to the connector # Since the executable is not available in this test then it should fail pipelinewise.discover_tap() stdout, stderr = capsys.readouterr() exp_err_pattern = os.path.join(VIRTUALENVS_DIR, '/tap-mysql/bin/tap-mysql: No such file or directory') assert exp_err_pattern in stdout or exp_err_pattern in stderr
def test_command_encrypt_string(self, capsys): """Test vault encryption command output""" secret_path = '{}/resources/vault-secret.txt'.format(os.path.dirname(__file__)) args = CliArgs(string='plain text', secret=secret_path) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Encrypted string should be printed to stdout pipelinewise.encrypt_string() stdout, stderr = capsys.readouterr() assert not stderr.strip() assert stdout.startswith('!vault |') and '$ANSIBLE_VAULT;' in stdout
def test_command_sync_tables(self): """Test run tap command""" args = CliArgs(target='target_one', tap='tap_one') pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Running sync_tables should detect the tap type and path to the connector # Since the executable is not available in this test then it should fail # TODO: sync discover_tap and run_tap behaviour. run_tap sys.exit but discover_tap does not. with pytest.raises(SystemExit) as pytest_wrapped_e: pipelinewise.sync_tables() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 1
def _init_for_sync_tables_states_cleanup(tables_arg: str = None) -> PipelineWise: temp_path = '/tmp/pwtest/' args = CliArgs(target='target_one', tap='tap_one', tables=tables_arg) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) pipelinewise.tap['files']['state'] = f'{temp_path}state.json' pipelinewise.venv_dir = temp_path # Making a test tap bin file os.makedirs(f'{temp_path}pipelinewise/bin') with open(f'{temp_path}pipelinewise/bin/mysql-to-snowflake', 'a', encoding='UTF-8'): pass return pipelinewise
def test_validate_command_5(self): """ Test validate command should fail because of transformation on json properties for a tap-target combo that has Fastsync """ test_validate_command_dir = \ f'{os.path.dirname(__file__)}/resources/test_validate_command/json_transformation_in_fastsync' args = CliArgs(dir=test_validate_command_dir) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) with pytest.raises(InvalidTransformationException): pipelinewise.validate()
def test_command_init(self): """Test init command""" args = CliArgs(name=TEST_PROJECT_NAME) pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Init new project pipelinewise.init() # The test project should contain every sample YAML file for sample_yaml in os.listdir('{}/../../../pipelinewise/cli/samples'.format(os.path.dirname(__file__))): assert os.path.isfile(os.path.join(TEST_PROJECT_DIR, sample_yaml)) # Re-creating project should reaise exception of directory not empty with pytest.raises(SystemExit) as pytest_wrapped_e: pipelinewise.init() assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 1
def test_exit_gracefully(self): """Gracefully shoudl run tap command""" args = CliArgs(target='target_one', tap='tap_one') pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) # Create a test log file, simulating a running tap pipelinewise.tap_run_log_file = 'test-tap-run-dummy.log' Path('{}.running'.format(pipelinewise.tap_run_log_file)).touch() # Graceful exit should return 1 by default with pytest.raises(SystemExit) as pytest_wrapped_e: pipelinewise._exit_gracefully(signal.SIGINT, frame=None) assert pytest_wrapped_e.type == SystemExit assert pytest_wrapped_e.value.code == 1 # Graceful exit should rename log file from running status to terminated assert os.path.isfile('{}.terminated'.format(pipelinewise.tap_run_log_file)) # Delete test log file os.remove('{}.terminated'.format(pipelinewise.tap_run_log_file))
def test_validate_command(self): """Test validate command""" test_validate_command_dir = f'{os.path.dirname(__file__)}/resources/test_validate_command' test_cases = [{ 'dir': 'missing_replication_key_incremental', 'exception': True }, { 'dir': 'missing_replication_key', 'exception': False }, { 'dir': 'invalid_target', 'exception': True }] for test_case in test_cases: args = CliArgs(dir=f'{test_validate_command_dir}/{test_case["dir"]}') pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) if test_case['exception']: with pytest.raises(SystemExit): pipelinewise.validate() else: pipelinewise.validate()
def test_logging_default(self): """Debug option should be disabled by default, LOGGING_CONF_FILE""" args = CliArgs() Logger(debug=args.debug) path = os.path.join(Path(__file__).parent, '..', '..', 'pipelinewise', 'logging.conf') assert os.environ['LOGGING_CONF_FILE'] == os.path.abspath(path)
def test_post_import_checks(self): """Test post import checks""" args = CliArgs() pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR) test_files_dir = '{}/resources/test_post_import_checks'.format( os.path.dirname(__file__)) tap_pk_required = cli.utils.load_json( '{}/tap_config_pk_required.json'.format(test_files_dir)) tap_pk_not_required = cli.utils.load_json( '{}/tap_config_pk_not_required.json'.format(test_files_dir)) tap_pk_not_defined = cli.utils.load_json( '{}/tap_config_pk_not_defined.json'.format(test_files_dir)) tap_with_pk = cli.utils.load_json( '{}//tap_properties_with_pk.json'.format(test_files_dir)) tap_with_no_pk_full_table = cli.utils.load_json( '{}//tap_properties_with_no_pk_full_table.json'.format( test_files_dir)) tap_with_no_pk_incremental = cli.utils.load_json( '{}//tap_properties_with_no_pk_incremental.json'.format( test_files_dir)) tap_with_no_pk_log_based = cli.utils.load_json( '{}//tap_properties_with_no_pk_log_based.json'.format( test_files_dir)) tap_with_no_pk_not_selected = cli.utils.load_json( '{}//tap_properties_with_no_pk_not_selected.json'.format( test_files_dir)) with patch('pipelinewise.cli.pipelinewise.commands.run_command' ) as run_command_mock: # Test scenarios when post import checks should pass assert pipelinewise._run_post_import_tap_checks( tap_pk_required, tap_with_pk, 'snowflake') == [] assert pipelinewise._run_post_import_tap_checks( tap_pk_not_required, tap_with_pk, 'snowflake') == [] assert pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_full_table, 'snowflake') \ == [] assert pipelinewise._run_post_import_tap_checks( tap_pk_not_required, tap_with_no_pk_incremental, 'snowflake') == [] assert pipelinewise._run_post_import_tap_checks( tap_pk_not_required, tap_with_no_pk_log_based, 'snowflake') == [] assert pipelinewise._run_post_import_tap_checks( tap_pk_required, tap_with_no_pk_not_selected, 'snowflake') == [] assert pipelinewise._run_post_import_tap_checks( tap_pk_not_defined, tap_with_no_pk_full_table, 'snowflake') == [] # Test scenarios when post import checks should fail due to primary keys not exists assert len( pipelinewise._run_post_import_tap_checks( tap_pk_required, tap_with_no_pk_incremental, 'snowflake')) == 1 assert len( pipelinewise._run_post_import_tap_checks( tap_pk_required, tap_with_no_pk_log_based, 'snowflake')) == 1 assert len( pipelinewise._run_post_import_tap_checks( tap_pk_not_defined, tap_with_no_pk_incremental, 'snowflake')) == 1 assert len( pipelinewise._run_post_import_tap_checks( tap_pk_not_defined, tap_with_no_pk_log_based, 'snowflake')) == 1 # Test scenarios when post import checks should fail due to transformations validation command fails tap_with_trans = cli.utils.\ load_json('{}/tap_config_with_transformations.json'.format(test_files_dir)) run_command_mock.return_value = ( 1, None, 'transformation `HASH` cannot be applied') assert len( pipelinewise._run_post_import_tap_checks( tap_with_trans, tap_with_no_pk_not_selected, 'snowflake')) == 1 assert len( pipelinewise._run_post_import_tap_checks( tap_with_trans, tap_with_no_pk_incremental, 'snowflake')) == 2 # mock successfull transformation validation command run_command_mock.return_value = (0, None, None) assert len( pipelinewise._run_post_import_tap_checks( tap_with_trans, tap_with_no_pk_not_selected, 'snowflake')) == 0 assert len( pipelinewise._run_post_import_tap_checks( tap_with_trans, tap_with_no_pk_incremental, 'snowflake')) == 1 assert run_command_mock.call_count == 4
def setup_method(self): """Create CLI arguments""" self.args = CliArgs(log='coverage.log') self.pipelinewise = PipelineWise(self.args, CONFIG_DIR, VIRTUALENVS_DIR)
def test_logging_debug(self): """Providing debug option should set LOGGING_CONF_FILE env var""" args = CliArgs(debug=True) Logger(debug=args.debug) path = os.path.join(Path(__file__).parent, '..', '..', 'pipelinewise', 'logging_debug.conf') assert os.environ['LOGGING_CONF_FILE'] == os.path.abspath(path)