def test_validate_command(self):
test_validate_command_dir = f'{os.path.dirname(__file__)}/resources/test_validate_command'
test_cases = [
{
'dir': 'missing_replication_key_incremental',
'exception': True
},
{
'dir': 'missing_replication_key',
'exception': False
},
{
'dir': 'invalid_target',
'exception': True
}
]
for test_case in test_cases:
args = CliArgs(dir=f'{test_validate_command_dir}/{test_case["dir"]}')
pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
if test_case['exception']:
with pytest.raises(SystemExit):
pipelinewise.validate()
else:
pipelinewise.validate()
def test_post_import_checks(self):
"""Test post import checks"""
args = CliArgs()
pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
test_files_dir = '{}/resources/test_post_import_checks'.format(os.path.dirname(__file__))
tap_pk_required = cli.utils.load_json('{}/tap_config_pk_required.json'.format(test_files_dir))
tap_pk_not_required = cli.utils.load_json('{}/tap_config_pk_not_required.json'.format(test_files_dir))
tap_pk_not_defined = cli.utils.load_json('{}/tap_config_pk_not_defined.json'.format(test_files_dir))
tap_with_pk = cli.utils.load_json('{}//tap_properties_with_pk.json'.format(test_files_dir))
tap_with_no_pk_full_table = cli.utils.load_json(
'{}//tap_properties_with_no_pk_full_table.json'.format(test_files_dir))
tap_with_no_pk_incremental = cli.utils.load_json(
'{}//tap_properties_with_no_pk_incremental.json'.format(test_files_dir))
tap_with_no_pk_log_based = cli.utils.load_json(
'{}//tap_properties_with_no_pk_log_based.json'.format(test_files_dir))
tap_with_no_pk_not_selected = cli.utils.load_json(
'{}//tap_properties_with_no_pk_not_selected.json'.format(test_files_dir))
# Test scenarios when post import checks should pass
assert pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_pk) == []
assert pipelinewise._run_post_import_tap_checks(tap_pk_not_required, tap_with_pk) == []
assert pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_full_table) == []
assert pipelinewise._run_post_import_tap_checks(tap_pk_not_required, tap_with_no_pk_incremental) == []
assert pipelinewise._run_post_import_tap_checks(tap_pk_not_required, tap_with_no_pk_log_based) == []
assert pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_not_selected) == []
assert pipelinewise._run_post_import_tap_checks(tap_pk_not_defined, tap_with_no_pk_full_table) == []
# Test scenarios when post import checks should fail due to primary keys not exists
assert len(pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_incremental)) == 1
assert len(pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_log_based)) == 1
assert len(pipelinewise._run_post_import_tap_checks(tap_pk_not_defined, tap_with_no_pk_incremental)) == 1
assert len(pipelinewise._run_post_import_tap_checks(tap_pk_not_defined, tap_with_no_pk_log_based)) == 1
def test_command_stop_tap(self):
"""Test stop tap command"""
args = CliArgs(target='target_one', tap='tap_one')
pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
# Tap is not running, pid file not exist, should exit with error
with pytest.raises(SystemExit) as pytest_wrapped_e:
pipelinewise.stop_tap()
assert pytest_wrapped_e.type == SystemExit
assert pytest_wrapped_e.value.code == 1
# Stop tap command should stop all the child processes
# 1. Start the pipelinewise mock executable that's running
# linux piped dummy tap and target connectors
with pidfile.PIDFile(pipelinewise.tap['files']['pidfile']):
os.spawnl(os.P_NOWAIT,
f'{RESOURCES_DIR}/test_stop_tap/scheduler-mock.sh',
'test_stop_tap/scheduler-mock.sh')
# Wait 5 seconds making sure the dummy tap is running
time.sleep(5)
# Send the stop_tap command
with pytest.raises(SystemExit):
pipelinewise.stop_tap()
# Should not have any remaining Pipelinewise related linux process
for proc in psutil.process_iter(['cmdline']):
full_command = ' '.join(proc.info['cmdline'])
assert re.match('scheduler|pipelinewise|tap|target',
full_command) is None
def test_send_alert_to_tap_specific_slack_channel(self):
"""Test if sends alert to the tap specific channel"""
config_dir = f'{RESOURCES_DIR}/sample_json_config_for_specific_slack_channel'
pipelinewise = PipelineWise(
self.args, config_dir, VIRTUALENVS_DIR, PROFILING_DIR
)
pipelinewise.tap = pipelinewise.get_tap('target_one', 'tap_one')
with patch.object(WebClient, 'chat_postMessage') as mocked_slack:
pipelinewise.send_alert('test-message')
# Assert if alert is sent to the the main channel and also to the tap channel
mocked_slack.assert_has_calls(
[
call(
channel=pipelinewise.alert_sender.alert_handlers['slack']['channel'],
text=None,
attachments=[{'color': 'danger', 'title': 'test-message'}]
),
call(
channel=pipelinewise.tap['slack_alert_channel'],
text=None,
attachments=[{'color': 'danger', 'title': 'test-message'}]
)
]
)
def setup_method(self):
"""Create CLI arguments"""
self.args = CliArgs(log='coverage.log')
self.pipelinewise = PipelineWise(
self.args, CONFIG_DIR, VIRTUALENVS_DIR, PROFILING_DIR
)
if os.path.exists('/tmp/pwtest'):
shutil.rmtree('/tmp/pwtest')
def test_validate_command_2(self):
"""Test validate command should succeed"""
test_validate_command_dir = \
f'{os.path.dirname(__file__)}/resources/test_validate_command/missing_replication_key'
args = CliArgs(dir=test_validate_command_dir)
pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
pipelinewise.validate()
def test_validate_command_3(self):
"""Test validate command should fail because of invalid target in tap config"""
test_validate_command_dir = f'{os.path.dirname(__file__)}/resources/test_validate_command/invalid_target'
args = CliArgs(dir=test_validate_command_dir)
pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
with pytest.raises(SystemExit):
pipelinewise.validate()
def test_not_existing_config_dir(self):
"""Test with not existing config dir"""
# Create a new pipelinewise object pointing to a not existing config directory
pipelinewise_with_no_config = PipelineWise(self.args, 'not-existing-config-dir', VIRTUALENVS_DIR)
# It should return and empty config with empty list targets
# TODO: Make this scenario to fail with error message of "config dir not exists"
assert pipelinewise_with_no_config.config == {}
assert pipelinewise_with_no_config.get_targets() == []
def test_command_stop_tap(self):
"""Test stop tap command"""
args = CliArgs(target='target_one', tap='tap_one')
pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
# Tap is not running, pid file not exist, should exit with error
with pytest.raises(SystemExit) as pytest_wrapped_e:
pipelinewise.stop_tap()
assert pytest_wrapped_e.type == SystemExit
assert pytest_wrapped_e.value.code == 1
def test_validate_command_1(self):
"""Test validate command should fail because of missing replication key for incremental"""
test_validate_command_dir = \
f'{os.path.dirname(__file__)}/resources/test_validate_command/missing_replication_key_incremental'
args = CliArgs(dir=test_validate_command_dir)
pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
with pytest.raises(SystemExit):
pipelinewise.validate()
def test_validate_command_4(self):
"""Test validate command should fail because of duplicate targets"""
test_validate_command_dir = \
f'{os.path.dirname(__file__)}/resources/test_validate_command/test_yaml_config_two_targets'
args = CliArgs(dir=test_validate_command_dir)
pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
with pytest.raises(SystemExit):
pipelinewise.validate()
def test_command_discover_tap(self):
"""Test discover tap command"""
args = CliArgs(target='target_one', tap='tap_one')
pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
# Running discovery mode should detect the tap type and path to the connector
# Since the executable is not available in this test then it should fail
result = pipelinewise.discover_tap()
exp_err_pattern = '/tap-mysql/bin/tap-mysql: No such file or directory'
assert exp_err_pattern in result
def test_command_sync_tables(self, capsys):
"""Test run tap command"""
args = CliArgs(target="target_one", tap="tap_one")
pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
# Running sync_tables should detect the tap type and path to the connector
# Since the executable is not available in this test then it should fail
# TODO: sync discover_tap and run_tap behaviour. run_tap sys.exit but discover_tap does not.
with pytest.raises(SystemExit) as pytest_wrapped_e:
pipelinewise.sync_tables()
assert pytest_wrapped_e.type == SystemExit
assert pytest_wrapped_e.value.code == 1
def test_command_discover_tap(self, capsys):
"""Test discover tap command"""
args = CliArgs(target='target_one', tap='tap_one')
pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
# Running discovery mode should detect the tap type and path to the connector
# Since the executable is not available in this test then it should fail
pipelinewise.discover_tap()
stdout, stderr = capsys.readouterr()
exp_err_pattern = os.path.join(VIRTUALENVS_DIR, '/tap-mysql/bin/tap-mysql: No such file or directory')
assert exp_err_pattern in stdout or exp_err_pattern in stderr
def test_command_encrypt_string(self, capsys):
"""Test vault encryption command output"""
secret_path = '{}/resources/vault-secret.txt'.format(os.path.dirname(__file__))
args = CliArgs(string='plain text', secret=secret_path)
pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
# Encrypted string should be printed to stdout
pipelinewise.encrypt_string()
stdout, stderr = capsys.readouterr()
assert not stderr.strip()
assert stdout.startswith('!vault |') and '$ANSIBLE_VAULT;' in stdout
def _init_for_sync_tables_states_cleanup(tables_arg: str = None) -> PipelineWise:
temp_path = '/tmp/pwtest/'
args = CliArgs(target='target_one', tap='tap_one', tables=tables_arg)
pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
pipelinewise.tap['files']['state'] = f'{temp_path}state.json'
pipelinewise.venv_dir = temp_path
# Making a test tap bin file
os.makedirs(f'{temp_path}pipelinewise/bin')
with open(f'{temp_path}pipelinewise/bin/mysql-to-snowflake', 'a', encoding='UTF-8'):
pass
return pipelinewise
def test_validate_command_5(self):
"""
Test validate command should fail because of transformation on json properties for a tap-target combo that
has Fastsync
"""
test_validate_command_dir = \
f'{os.path.dirname(__file__)}/resources/test_validate_command/json_transformation_in_fastsync'
args = CliArgs(dir=test_validate_command_dir)
pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
with pytest.raises(InvalidTransformationException):
pipelinewise.validate()
def test_command_init(self):
"""Test init command"""
args = CliArgs(name=TEST_PROJECT_NAME)
pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
# Init new project
pipelinewise.init()
# The test project should contain every sample YAML file
for sample_yaml in os.listdir('{}/../../../pipelinewise/cli/samples'.format(os.path.dirname(__file__))):
assert os.path.isfile(os.path.join(TEST_PROJECT_DIR, sample_yaml))
# Re-creating project should reaise exception of directory not empty
with pytest.raises(SystemExit) as pytest_wrapped_e:
pipelinewise.init()
assert pytest_wrapped_e.type == SystemExit
assert pytest_wrapped_e.value.code == 1
def test_exit_gracefully(self):
"""Gracefully shoudl run tap command"""
args = CliArgs(target='target_one', tap='tap_one')
pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
# Create a test log file, simulating a running tap
pipelinewise.tap_run_log_file = 'test-tap-run-dummy.log'
Path('{}.running'.format(pipelinewise.tap_run_log_file)).touch()
# Graceful exit should return 1 by default
with pytest.raises(SystemExit) as pytest_wrapped_e:
pipelinewise._exit_gracefully(signal.SIGINT, frame=None)
assert pytest_wrapped_e.type == SystemExit
assert pytest_wrapped_e.value.code == 1
# Graceful exit should rename log file from running status to terminated
assert os.path.isfile('{}.terminated'.format(pipelinewise.tap_run_log_file))
# Delete test log file
os.remove('{}.terminated'.format(pipelinewise.tap_run_log_file))
def setup_method(self):
# Create CLI arguments
self.args = CliArgs(log="coverage.log")
self.pipelinewise = PipelineWise(self.args, CONFIG_DIR,
VIRTUALENVS_DIR)
def test_post_import_checks(self):
"""Test post import checks"""
args = CliArgs()
pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
test_files_dir = '{}/resources/test_post_import_checks'.format(
os.path.dirname(__file__))
tap_pk_required = cli.utils.load_json(
'{}/tap_config_pk_required.json'.format(test_files_dir))
tap_pk_not_required = cli.utils.load_json(
'{}/tap_config_pk_not_required.json'.format(test_files_dir))
tap_pk_not_defined = cli.utils.load_json(
'{}/tap_config_pk_not_defined.json'.format(test_files_dir))
tap_with_pk = cli.utils.load_json(
'{}//tap_properties_with_pk.json'.format(test_files_dir))
tap_with_no_pk_full_table = cli.utils.load_json(
'{}//tap_properties_with_no_pk_full_table.json'.format(
test_files_dir))
tap_with_no_pk_incremental = cli.utils.load_json(
'{}//tap_properties_with_no_pk_incremental.json'.format(
test_files_dir))
tap_with_no_pk_log_based = cli.utils.load_json(
'{}//tap_properties_with_no_pk_log_based.json'.format(
test_files_dir))
tap_with_no_pk_not_selected = cli.utils.load_json(
'{}//tap_properties_with_no_pk_not_selected.json'.format(
test_files_dir))
with patch('pipelinewise.cli.pipelinewise.commands.run_command'
) as run_command_mock:
# Test scenarios when post import checks should pass
assert pipelinewise._run_post_import_tap_checks(
tap_pk_required, tap_with_pk, 'snowflake') == []
assert pipelinewise._run_post_import_tap_checks(
tap_pk_not_required, tap_with_pk, 'snowflake') == []
assert pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_full_table, 'snowflake') \
== []
assert pipelinewise._run_post_import_tap_checks(
tap_pk_not_required, tap_with_no_pk_incremental,
'snowflake') == []
assert pipelinewise._run_post_import_tap_checks(
tap_pk_not_required, tap_with_no_pk_log_based,
'snowflake') == []
assert pipelinewise._run_post_import_tap_checks(
tap_pk_required, tap_with_no_pk_not_selected,
'snowflake') == []
assert pipelinewise._run_post_import_tap_checks(
tap_pk_not_defined, tap_with_no_pk_full_table,
'snowflake') == []
# Test scenarios when post import checks should fail due to primary keys not exists
assert len(
pipelinewise._run_post_import_tap_checks(
tap_pk_required, tap_with_no_pk_incremental,
'snowflake')) == 1
assert len(
pipelinewise._run_post_import_tap_checks(
tap_pk_required, tap_with_no_pk_log_based,
'snowflake')) == 1
assert len(
pipelinewise._run_post_import_tap_checks(
tap_pk_not_defined, tap_with_no_pk_incremental,
'snowflake')) == 1
assert len(
pipelinewise._run_post_import_tap_checks(
tap_pk_not_defined, tap_with_no_pk_log_based,
'snowflake')) == 1
# Test scenarios when post import checks should fail due to transformations validation command fails
tap_with_trans = cli.utils.\
load_json('{}/tap_config_with_transformations.json'.format(test_files_dir))
run_command_mock.return_value = (
1, None, 'transformation `HASH` cannot be applied')
assert len(
pipelinewise._run_post_import_tap_checks(
tap_with_trans, tap_with_no_pk_not_selected,
'snowflake')) == 1
assert len(
pipelinewise._run_post_import_tap_checks(
tap_with_trans, tap_with_no_pk_incremental,
'snowflake')) == 2
# mock successfull transformation validation command
run_command_mock.return_value = (0, None, None)
assert len(
pipelinewise._run_post_import_tap_checks(
tap_with_trans, tap_with_no_pk_not_selected,
'snowflake')) == 0
assert len(
pipelinewise._run_post_import_tap_checks(
tap_with_trans, tap_with_no_pk_incremental,
'snowflake')) == 1
assert run_command_mock.call_count == 4
def setup_method(self):
"""Create CLI arguments"""
self.args = CliArgs(log='coverage.log')
self.pipelinewise = PipelineWise(self.args, CONFIG_DIR,
VIRTUALENVS_DIR, PROFILING_DIR)
