def test_build_target_command(self):
"""Tests the function that generates target executable command"""
# Should return a input piped command with an executable target command
target = commands.TargetParams(
id='my_target',
type='target-snowflake',
bin='/bin/target_postgres.py',
python_bin='/bin/python',
config='.ppw/config.json',
)
# profiling is not enabled
command = commands.build_target_command(target=target,
profiling_mode=False,
profiling_dir=None)
assert command == '/bin/target_postgres.py --config .ppw/config.json'
# profiling is enabled
command = commands.build_target_command(target=target,
profiling_mode=True,
profiling_dir='./profiling')
assert (
command ==
'/bin/python -m cProfile -o ./profiling/target_my_target.pstat /bin/target_postgres.py'
' --config .ppw/config.json')
def test_build_fastsync_command(self):
"""Tests the function that generates the fastsync command"""
transform_config = '{}/resources/transform-config.json'.format(
os.path.dirname(__file__))
state_mock = __file__
venv_dir = '.dummy_venv_dir'
temp_dir = 'dummy_temp_dir'
# Should generate a fastsync command with transformation
tap_params = commands.TapParams(type='tap-mysql',
bin='/bin/tap_mysql.py',
config='.ppw/tap_config.json',
properties='.ppw/properties.json',
state=state_mock)
target_params = commands.TargetParams(type='target-postgres',
bin='/bin/target_postgres.py',
config='.ppw/target_config.json')
transform_params = commands.TransformParams(
bin='/bin/transform_field.py', config=None)
command = commands.build_fastsync_command(tap_params, target_params,
transform_params, venv_dir,
temp_dir)
assert command == '.dummy_venv_dir/pipelinewise/bin/mysql-to-postgres' \
' --tap .ppw/tap_config.json' \
' --properties .ppw/properties.json' \
f' --state {state_mock}' \
' --target .ppw/target_config.json' \
' --temp_dir dummy_temp_dir'
# Should generate a fastsync command with transformation
transform_params = commands.TransformParams(
bin='/bin/transform_field.py', config=transform_config)
command = commands.build_fastsync_command(tap_params, target_params,
transform_params, venv_dir,
temp_dir)
assert command == '.dummy_venv_dir/pipelinewise/bin/mysql-to-postgres' \
' --tap .ppw/tap_config.json' \
' --properties .ppw/properties.json' \
f' --state {state_mock}' \
' --target .ppw/target_config.json' \
' --temp_dir dummy_temp_dir' \
f' --transform {transform_config}'
# Should generate a fastsync command with specific list of tables
command = commands.build_fastsync_command(
tap_params,
target_params,
transform_params,
venv_dir,
temp_dir,
tables='public.table_one,public.table_two')
assert command == '.dummy_venv_dir/pipelinewise/bin/mysql-to-postgres' \
' --tap .ppw/tap_config.json' \
' --properties .ppw/properties.json' \
f' --state {state_mock}' \
' --target .ppw/target_config.json' \
' --temp_dir dummy_temp_dir' \
f' --transform {transform_config}' \
' --tables public.table_one,public.table_two'
def _assert_target_config(config: str) -> None:
commands.TargetParams(
target_id='foo',
type='bar',
bin='foo_bin',
python_bin='foo/python',
config=config,
)
def test_build_singer_command(self):
"""Tests the function that generates the full singer singer command
that connects the required components with linux pipes"""
transform_config = '{}/resources/transform-config.json'.format(os.path.dirname(__file__))
transform_config_empty = '{}/resources/transform-config-empty.json'.format(os.path.dirname(__file__))
state_mock = __file__
# Should generate a command with tap state and transformation
tap_params = commands.TapParams(type='tap-mysql',
bin='/bin/tap_mysql.py',
config='.ppw/config.json',
properties='.ppw/properties.json',
state=state_mock)
target_params = commands.TargetParams(type='target-postgres',
bin='/bin/target_postgres.py',
config='.ppw/config.json')
transform_params = commands.TransformParams(bin='/bin/transform_field.py',
config=transform_config)
command = commands.build_singer_command(tap_params, target_params, transform_params)
assert command == f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json ' \
f'--state {state_mock}' \
f' | /bin/transform_field.py --config {transform_config}' \
' | /bin/target_postgres.py --config .ppw/config.json'
# Should generate a command with tap state and transformation and stream buffer
tap_params = commands.TapParams(type='tap-mysql',
bin='/bin/tap_mysql.py',
config='.ppw/config.json',
properties='.ppw/properties.json',
state=state_mock)
target_params = commands.TargetParams(type='target-postgres',
bin='/bin/target_postgres.py',
config='.ppw/config.json')
transform_params = commands.TransformParams(bin='/bin/transform_field.py',
config=transform_config)
command = commands.build_singer_command(tap_params, target_params, transform_params, stream_buffer_size=10)
assert command == f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json ' \
f'--state {state_mock}' \
f' | /bin/transform_field.py --config {transform_config}' \
' | mbuffer -m 10M' \
' | /bin/target_postgres.py --config .ppw/config.json'
# Should generate a command without state and with transformation
tap_params = commands.TapParams(type='tap-mysql',
bin='/bin/tap_mysql.py',
config='.ppw/config.json',
properties='.ppw/properties.json',
state=None)
target_params = commands.TargetParams(type='target-postgres',
bin='/bin/target_postgres.py',
config='.ppw/config.json')
transform_params = commands.TransformParams(bin='/bin/transform_field.py',
config=transform_config)
command = commands.build_singer_command(tap_params, target_params, transform_params)
assert command == f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json ' \
f' | /bin/transform_field.py --config {transform_config}' \
' | /bin/target_postgres.py --config .ppw/config.json'
# Should generate a command with state and without transformation
tap_params = commands.TapParams(type='tap-mysql',
bin='/bin/tap_mysql.py',
config='.ppw/config.json',
properties='.ppw/properties.json',
state=state_mock)
target_params = commands.TargetParams(type='target-postgres',
bin='/bin/target_postgres.py',
config='.ppw/config.json')
transform_params = commands.TransformParams(bin='/bin/transform_field.py',
config=transform_config_empty)
command = commands.build_singer_command(tap_params, target_params, transform_params)
assert command == f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json ' \
f'--state {state_mock}' \
' | /bin/target_postgres.py --config .ppw/config.json'
# Should generate a command without state and transformation
tap_params = commands.TapParams(type='tap-mysql',
bin='/bin/tap_mysql.py',
config='.ppw/config.json',
properties='.ppw/properties.json',
state='.ppw/state.json')
target_params = commands.TargetParams(type='target-postgres',
bin='/bin/target_postgres.py',
config='.ppw/config.json')
transform_params = commands.TransformParams(bin='/bin/transform_field.py',
config=transform_config_empty)
command = commands.build_singer_command(tap_params, target_params, transform_params)
assert command == '/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json ' \
' | /bin/target_postgres.py --config .ppw/config.json'
def test_build_fastsync_command(self):
"""Tests the function that generates the fastsync command"""
transform_config = '{}/resources/transform-config.json'.format(
os.path.dirname(__file__))
state_mock = __file__
venv_dir = '.dummy_venv_dir'
temp_dir = 'dummy_temp_dir'
# Should generate a fastsync command with transformation
tap_params = commands.TapParams(
id='my_tap',
type='tap-mysql',
bin='/bin/tap_mysql.py',
python_bin='/tap-mysql/bin/python',
config='.ppw/tap_config.json',
properties='.ppw/properties.json',
state=state_mock,
)
target_params = commands.TargetParams(
id='my_target',
type='target-postgres',
bin='/bin/target_postgres.py',
python_bin='/target-postgres/bin/python',
config='.ppw/target_config.json',
)
transform_params = commands.TransformParams(
bin='/bin/transform_field.py',
python_bin='transform/bin/python',
config=None,
tap_id='my_tap',
target_id='my_target',
)
# profiling disabled
command = commands.build_fastsync_command(tap_params, target_params,
transform_params, venv_dir,
temp_dir)
assert (command == '.dummy_venv_dir/pipelinewise/bin/mysql-to-postgres'
' --tap .ppw/tap_config.json'
' --properties .ppw/properties.json'
f' --state {state_mock}'
' --target .ppw/target_config.json'
' --temp_dir dummy_temp_dir')
# profiling enabled
command = commands.build_fastsync_command(
tap_params,
target_params,
transform_params,
venv_dir,
temp_dir,
profiling_mode=True,
profiling_dir='./profiling',
)
assert (
command == '.dummy_venv_dir/pipelinewise/bin/python -m cProfile '
'-o ./profiling/fastsync_my_tap_my_target.pstat'
' .dummy_venv_dir/pipelinewise/bin/mysql-to-postgres'
' --tap .ppw/tap_config.json'
' --properties .ppw/properties.json'
f' --state {state_mock}'
' --target .ppw/target_config.json'
' --temp_dir dummy_temp_dir')
# Should generate a fastsync command with transformation
transform_params = commands.TransformParams(
bin='/bin/transform_field.py',
python_bin='transform/bin/python',
config=transform_config,
tap_id='my_tap',
target_id='my_target',
)
# profiling disabled
command = commands.build_fastsync_command(tap_params, target_params,
transform_params, venv_dir,
temp_dir)
assert (command == '.dummy_venv_dir/pipelinewise/bin/mysql-to-postgres'
' --tap .ppw/tap_config.json'
' --properties .ppw/properties.json'
f' --state {state_mock}'
' --target .ppw/target_config.json'
' --temp_dir dummy_temp_dir'
f' --transform {transform_config}')
# profiling enabled
command = commands.build_fastsync_command(
tap_params,
target_params,
transform_params,
venv_dir,
temp_dir,
profiling_mode=True,
profiling_dir='./profiling',
)
assert (
command == '.dummy_venv_dir/pipelinewise/bin/python -m cProfile'
' -o ./profiling/fastsync_my_tap_my_target.pstat'
' .dummy_venv_dir/pipelinewise/bin/mysql-to-postgres'
' --tap .ppw/tap_config.json'
' --properties .ppw/properties.json'
f' --state {state_mock}'
' --target .ppw/target_config.json'
' --temp_dir dummy_temp_dir'
f' --transform {transform_config}')
# Should generate a fastsync command with specific list of tables
command = commands.build_fastsync_command(
tap_params,
target_params,
transform_params,
venv_dir,
temp_dir,
tables='public.table_one,public.table_two',
)
assert (command == '.dummy_venv_dir/pipelinewise/bin/mysql-to-postgres'
' --tap .ppw/tap_config.json'
' --properties .ppw/properties.json'
f' --state {state_mock}'
' --target .ppw/target_config.json'
' --temp_dir dummy_temp_dir'
f' --transform {transform_config}'
' --tables public.table_one,public.table_two')
def test_build_singer_command(self):
"""Tests the function that generates the full singer singer command
that connects the required components with linux pipes"""
transform_config = '{}/resources/transform-config.json'.format(
os.path.dirname(__file__))
transform_config_empty = '{}/resources/transform-config-empty.json'.format(
os.path.dirname(__file__))
state_mock = __file__
# Should generate a command with tap state and transformation
tap_params = commands.TapParams(
id='my_tap',
type='tap-mysql',
bin='/bin/tap_mysql.py',
python_bin='/bin/python',
config='.ppw/config.json',
properties='.ppw/properties.json',
state=state_mock,
)
target_params = commands.TargetParams(
id='my_target',
type='target-postgres',
bin='/bin/target_postgres.py',
python_bin='/bin/python',
config='.ppw/config.json',
)
transform_params = commands.TransformParams(
bin='/bin/transform_field.py',
python_bin='/bin/python',
config=transform_config,
tap_id='my_tap',
target_id='my_target',
)
# profiling disabled
command = commands.build_singer_command(tap_params, target_params,
transform_params)
assert (
command ==
f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json '
f'--state {state_mock}'
f' | /bin/transform_field.py --config {transform_config}'
' | /bin/target_postgres.py --config .ppw/config.json')
# profiling enabled
command = commands.build_singer_command(
tap_params,
target_params,
transform_params,
profiling_mode=True,
profiling_dir='./profiling',
)
assert (
command ==
f'/bin/python -m cProfile -o ./profiling/tap_my_tap.pstat '
f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json '
f'--state {state_mock}'
f' | /bin/python -m cProfile -o ./profiling/transformation_my_tap_my_target.pstat '
f'/bin/transform_field.py --config {transform_config}'
' | /bin/python -m cProfile -o ./profiling/target_my_target.pstat /bin/target_postgres.py '
'--config .ppw/config.json')
# Should generate a command with tap state and transformation and stream buffer
# profiling disabled
command = commands.build_singer_command(tap_params,
target_params,
transform_params,
stream_buffer_size=10)
assert (
command ==
f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json '
f'--state {state_mock}'
f' | /bin/transform_field.py --config {transform_config}'
' | mbuffer -m 10M'
' | /bin/target_postgres.py --config .ppw/config.json')
# profiling enabled
command = commands.build_singer_command(
tap_params,
target_params,
transform_params,
profiling_mode=True,
profiling_dir='./profiling',
stream_buffer_size=10,
)
assert (
command ==
f'/bin/python -m cProfile -o ./profiling/tap_my_tap.pstat /bin/tap_mysql.py '
f'--config .ppw/config.json --properties .ppw/properties.json --state {state_mock}'
f' | /bin/python -m cProfile -o ./profiling/transformation_my_tap_my_target.pstat '
f'/bin/transform_field.py --config {transform_config}'
' | mbuffer -m 10M'
' | /bin/python -m cProfile -o ./profiling/target_my_target.pstat /bin/target_postgres.py'
' --config .ppw/config.json')
# Should generate a command without state and with transformation
tap_params = commands.TapParams(
id='my_tap',
type='tap-mysql',
bin='/bin/tap_mysql.py',
python_bin='/bin/python',
config='.ppw/config.json',
properties='.ppw/properties.json',
state=None,
)
# profiling disabled
command = commands.build_singer_command(tap_params, target_params,
transform_params)
assert (
command ==
f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json '
f' | /bin/transform_field.py --config {transform_config}'
' | /bin/target_postgres.py --config .ppw/config.json')
# profiling enabled
command = commands.build_singer_command(
tap_params,
target_params,
transform_params,
profiling_mode=True,
profiling_dir='./profiling',
)
assert (
command ==
f'/bin/python -m cProfile -o ./profiling/tap_my_tap.pstat '
f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json '
f' | /bin/python -m cProfile -o ./profiling/transformation_my_tap_my_target.pstat '
f'/bin/transform_field.py --config {transform_config}'
' | /bin/python -m cProfile -o ./profiling/target_my_target.pstat '
'/bin/target_postgres.py --config .ppw/config.json')
# Should generate a command with state and without transformation
tap_params = commands.TapParams(
id='my_tap',
type='tap-mysql',
bin='/bin/tap_mysql.py',
python_bin='/bin/python',
config='.ppw/config.json',
properties='.ppw/properties.json',
state=state_mock,
)
transform_params = commands.TransformParams(
bin='/bin/transform_field.py',
python_bin='/bin/python',
config=transform_config_empty,
tap_id='my_tap',
target_id='my_target',
)
# profiling disabled
command = commands.build_singer_command(tap_params, target_params,
transform_params)
assert (
command ==
f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json '
f'--state {state_mock}'
' | /bin/target_postgres.py --config .ppw/config.json')
# profiling enabled
command = commands.build_singer_command(
tap_params,
target_params,
transform_params,
profiling_mode=True,
profiling_dir='./profiling',
)
assert (
command ==
f'/bin/python -m cProfile -o ./profiling/tap_my_tap.pstat /bin/tap_mysql.py '
f'--config .ppw/config.json --properties .ppw/properties.json '
f'--state {state_mock}'
' | /bin/python -m cProfile -o ./profiling/target_my_target.pstat /bin/target_postgres.py '
'--config .ppw/config.json')
# Should generate a command without state and transformation
tap_params = commands.TapParams(
id='my_tap',
type='tap-mysql',
bin='/bin/tap_mysql.py',
python_bin='/bin/python',
config='.ppw/config.json',
properties='.ppw/properties.json',
state='.ppw/state.json',
)
# profiling disabled
command = commands.build_singer_command(tap_params, target_params,
transform_params)
assert (
command ==
'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json '
' | /bin/target_postgres.py --config .ppw/config.json')
# profiling enabled
command = commands.build_singer_command(
tap_params,
target_params,
transform_params,
profiling_mode=True,
profiling_dir='./profiling',
)
assert (
command ==
'/bin/python -m cProfile -o ./profiling/tap_my_tap.pstat /bin/tap_mysql.py '
'--config .ppw/config.json --properties .ppw/properties.json '
' | /bin/python -m cProfile -o ./profiling/target_my_target.pstat /bin/target_postgres.py '
'--config .ppw/config.json')
