def integration_test__the_etl_state_management_should_go_through_all_state_sucessfully_and_persist(
self):
compose = DockerCompose(filepath=os.path.dirname(base.__file__))
with compose:
host = compose.get_service_host("cassandra", 9042)
port = int(compose.get_service_port("cassandra", 9042))
conn = connect(host, port)
conn.setup_connection("dev")
sync_etl_state_table()
sm = EtlSinkRecordStateManager(record_identifier="RandomRecordId",
etl_signature="SomeTestEtl")
print(sm.current_state())
print(sm.remote_state)
self.assertEqual(sm.current_state(), EtlStates.Ready)
sm.processing()
self.assertEqual(sm.current_state(), EtlStates.Processing)
sm.failed()
self.assertEqual(sm.current_state(), EtlStates.Failed)
sm.ready()
self.assertEqual(sm.current_state(), EtlStates.Ready)
sm.processing()
self.assertEqual(sm.current_state(), EtlStates.Processing)
sm.succeeded()
self.assertEqual(sm.current_state(), EtlStates.Succeeded)
def _stop_network(docker_client: DockerClient,
docker_compose_client: DockerCompose) -> None:
"""Bring down all images."""
print("Tearing down beer garden docker images. Please wait...")
docker_compose_client.stop()
if not _remove_volumes(docker_client):
raise LocalIntegrationTestError(
"Unable to remove volumes from this run")
def get_container():
COMPOSE_PATH = '{}/tests'.format(rootpath.detect('.', pattern='tests'))
compose = DockerCompose(COMPOSE_PATH)
compose.start()
bootstrap_server = 'localhost:9092'
consumer = KafkaConsumer(group_id='test',
bootstrap_servers=[bootstrap_server])
if not consumer.topics():
raise KafkaError('Unable to connect with kafka container!')
return compose
def __enter__(self):
self.compose = DockerCompose(self.testcontainers_path)
self.compose.start()
if self.kafka_input_config:
ensure_kafka_topic_exists(self.kafka_input_config)
if self.postgres_config:
ensure_postgres_database_exists(self.postgres_config)
return self
def test_full_integration_with_local_cassandra(self):
aws_conn = AwsConnectionSettings(region="us-east-1",
secrets_manager=None,
profile="default")
execfile("../../secrets.py")
compose = DockerCompose(filepath=os.path.dirname(base.__file__))
with compose:
host = compose.get_service_host("cassandra", 9042)
port = int(compose.get_service_port("cassandra", 9042))
cassandra_conn_setting = CassandraConnectionSettings(
cluster_ips=[host],
port=port,
load_balancing_policy=DCAwareRoundRobinPolicy(),
secrets_manager=CassandraSecretsManager(
source=DictKeyValueSource({
"CASSANDRA_USERNAME": "",
"CASSANDRA_PASSWORD": "",
})),
)
conn = verify_container_is_up(cassandra_conn_setting)
# conn.get_session('system').execute(""" DROP TABLE test.etl_sink_record_state""")
settings = AthenaToAdWordsOfflineConversionSettings(
source_database=os.getenv("dummy_athena_database"),
source_table=os.getenv("dummy_athena_table"),
source_connection_settings=aws_conn,
etl_identifier="test",
destination_batch_size=100,
etl_state_manager_connection=cassandra_conn_setting,
etl_state_manager_keyspace="test",
transformation_column_mapping={
'google_click_id': 'googleClickId',
'conversion_name': 'conversionName',
'conversion_time': 'conversionTime',
'conversion_value': 'conversionValue',
'conversion_currency_code': 'conversionCurrencyCode'
},
destination_connection_settings=GoogleAdWordsConnectionSettings(
client_id=os.getenv("adwords_client_id"),
user_agent="Tester",
client_customer_id=os.getenv("adwords_client_customer_id"),
secrets_manager=GoogleAdWordsSecretsManager()),
)
etl = AthenaToAdWordsOfflineConversion(settings)
files_actual = etl.list_source_files()
#
# self.assertListEqual(files_actual, [])
# etl.upload_all()
act = etl.upload_all()
self.assertListEqual(act, [])
def docker_compose(request, prepare, version):
# type: (FixtureRequest, Callable, str) -> None
module = request.module
cwd = dirname(inspect.getfile(module))
if version:
with open(os.path.join(cwd, 'requirements.txt'), mode='w') as req:
req.write(version)
compose = DockerCompose(filepath=cwd)
compose.start()
exception = None
for _ in range(0, 10):
try:
prepare()
exception = None
break
except Exception as e:
time.sleep(10)
exception = e
if exception:
compose.stop()
raise Exception("""Wait time exceeded {0} sec. Exception {1}""".format(
100, exception))
yield compose
compose.stop()
def test_can_throw_exception_if_no_port_exposed():
compose = DockerCompose(os.path.dirname(__file__))
compose.start()
with pytest.raises(NoSuchPortExposed):
compose.get_service_host("hub", 5555)
compose.stop()
def test_should_respect_key_limit():
with DockerCompose(
os.getcwd() + "/test",
compose_file_name="docker-compose.yml",
pull=True
) as compose:
(host, port) = assert_and_get_host_port(compose, REDIS_PORT)
cache = Cache(host, port)
namespace = "respect_key_limit"
@cache(namespace=namespace, limit = 1)
def test_func(test_input: int):
return test_input
assert cache.get_key_count() == 0
test_func(1)
# First call gives two keys. One for the base function and one for the inputs
assert cache.get_key_count() == 2
[test_func(i) for i in range(50)]
assert cache.get_key_count() == 2
def test_should_evict_least_accessed():
with DockerCompose(
os.getcwd() + "/test",
compose_file_name="docker-compose.yml",
pull=True
) as compose:
(host, port) = assert_and_get_host_port(compose, REDIS_PORT)
cache = Cache(host, port)
namespace = "evict-least-accessed"
@cache(namespace=namespace, limit = 3)
def test_func(test_input: int):
return test_input
func_input = range(3)
keys = []
# Create three keys
[test_func(i) for i in func_input]
[keys.append(get_cache_key(namespace, [i])) for i in func_input]
keys.append(f'rc:{namespace}:keys') # base function
assert set(cache.get_all_keys()) == set(keys)
# Access all except first input
[test_func(i) for i in func_input[1:]]
# Add new input
test_func(3)
keys.append(get_cache_key(namespace, [3]))
diff = set(keys) - set(cache.get_all_keys())
assert diff == set([keys[0]])
def test_integration(self):
with DockerCompose("./test") as compose:
time.sleep(5)
host = compose.get_service_host('rest-api', 8008)
port = compose.get_service_port('rest-api', 8008)
url = f'http://{host}:{port}'
print('\npublish production measurement:')
mea_response = self.publish_prod_measurement(url)
self.wait_for_commit(mea_response.json()['link'])
print('\npublish consumption measurement:')
mea_con_response = self.publish_con_measurement(url)
self.wait_for_commit(mea_con_response.json()['link'])
print('\nissue ggo:')
ggo_response = self.issue_ggo(url)
self.wait_for_commit(ggo_response.json()['link'])
print('\nsplit ggo:')
split_response = self.split_ggo(url)
self.wait_for_commit(split_response.json()['link'])
print('\ntransfer ggo:')
transfer_response = self.transfer_ggo(url)
self.wait_for_commit(transfer_response.json()['link'])
print('\nretire ggo:')
retire_response = self.retire_ggo(url)
self.wait_for_commit(retire_response.json()['link'])
def setUpClass(cls):
super().setUpClass()
cls.containers = DockerCompose('tests/fixtures/config')
cls.containers.start()
time.sleep(10)
def broker(request):
version = request.config.getoption("--publish-pact")
publish = True if version else False
# If the results are not going to be published to the broker, there is
# nothing further to do anyway
if not publish:
yield
return
run_broker = request.config.getoption("--run-broker")
if run_broker:
# Start up the broker using docker-compose
print("Starting broker")
with DockerCompose("../broker",
compose_file_name=["docker-compose.yml"],
pull=True) as compose:
stdout, stderr = compose.get_logs()
if stderr:
print("Errors\\n:{}".format(stderr))
print("{}".format(stdout))
print("Started broker")
yield
print("Stopping broker")
print("Broker stopped")
else:
# Assuming there is a broker available already, docker-compose has been
# used manually as the --run-broker option has not been provided
yield
return
def setUpClass(cls):
cls.compose = DockerCompose(filepath=dirname(inspect.getfile(cls)))
cls.compose.start()
cls.compose.wait_for(
cls.url(('consumer', '9090'),
'users?test=test1&test=test2&test2=test2'))
def broker(request):
version = request.config.getoption('--publish-pact')
publish = True if version else False
# yield
if not publish:
yield
return
run_broker = request.config.getoption('--run-broker')
if not run_broker:
yield
return
else:
print('Starting broker')
with DockerCompose("../broker",
compose_file_name=["docker-compose.yml"],
pull=True) as compose:
stdout, stderr = compose.get_logs()
if stderr:
print("Errors\\n:{}".format(stderr))
print(stdout)
yield
def test_can_pass_env_params_by_env_file():
with DockerCompose('tests',
compose_file_name='docker-compose-3.yml',
env_file='.env.test') as _:
check_env_is_set_cmd = 'docker exec tests_mysql_1 printenv | grep TEST_ASSERT_KEY'.split(
)
out = subprocess.run(check_env_is_set_cmd, stdout=subprocess.PIPE)
assert out.stdout.decode('utf-8').splitlines()[0], 'test_is_passed'
def test_can_spawn_service_via_compose():
compose = DockerCompose(os.path.dirname(__file__))
try:
compose.start()
host = compose.get_service_host("hub", 4444)
port = compose.get_service_port("hub", 4444)
assert host == "0.0.0.0"
assert port == "4444"
finally:
compose.stop()
def test_can_build_images_before_spawning_service_via_compose():
with patch.object(DockerCompose, "_call_command") as call_mock:
with DockerCompose(ROOT, build=True) as compose:
...
assert compose.build
docker_compose_cmd = call_mock.call_args_list[0][1]["cmd"]
assert "docker-compose" in docker_compose_cmd
assert "up" in docker_compose_cmd
assert "--build" in docker_compose_cmd
class CdlEnv:
def __init__(self, testcontainers_path, kafka_input_config = None, postgres_config = None):
self.testcontainers_path = testcontainers_path
self.kafka_input_config = kafka_input_config
self.postgres_config = postgres_config
def __enter__(self):
self.compose = DockerCompose(self.testcontainers_path)
self.compose.start()
if self.kafka_input_config:
ensure_kafka_topic_exists(self.kafka_input_config)
if self.postgres_config:
ensure_postgres_database_exists(self.postgres_config)
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.compose.stop()
def test_s3():
"""Ensure that HTTP service is up and responsive."""
compose = DockerCompose('tests')
with compose:
s3 = get_test_s3(compose)
s3.Bucket('test1').create()
s3.Bucket('test2').create()
s3.Object('test1','test1key').upload_fileobj(file_with_size(1024*1024*10))
s3.Object('test2', 'test2key').upload_fileobj(file_with_size(1024 * 1024))
s3.Object('test2', 'test2key2').upload_fileobj(file_with_size(1024 * 1024*2))
yield compose
def test_can_parse_multiple_compose_files():
with DockerCompose(
filepath="tests",
compose_file_name=["docker-compose.yml",
"docker-compose-2.yml"]) as compose:
host = compose.get_service_host("mysql", 3306)
port = compose.get_service_port("mysql", 3306)
assert host == "0.0.0.0"
assert port == "3306"
host = compose.get_service_host("hub", 4444)
port = compose.get_service_port("hub", 4444)
assert host == "0.0.0.0"
assert port == "4444"
def _get_composer(environment: Dict[str, Any],
env_file: Path) -> DockerCompose:
"""Create a DockerCompose object with the required .env file."""
env_file_path = str(env_file)
if env_file.exists():
env_file.unlink()
env_file.touch()
with env_file.open(mode="w") as file:
for key, val in environment.items():
file.write(f"{key}={val}\n")
return DockerCompose(
".",
compose_file_name="docker-compose-local-integration-tests.yml",
env_file=env_file_path,
)
def test_should_insert_key_after_call():
with DockerCompose(
os.getcwd() + "/test",
compose_file_name="docker-compose.yml",
pull=True
) as compose:
(host, port) = assert_and_get_host_port(compose, REDIS_PORT)
cache = Cache(host, port)
namespace = "insert-key-test"
@cache(namespace=namespace)
def test_func(first: str, second: str, third: int):
return f"first second {third}"
assert_key_exists(cache, namespace, list(TEST_INPUT.values()), exists = False)
test_func(TEST_INPUT["first"], TEST_INPUT["second"], TEST_INPUT["third"])
assert_key_exists(cache, namespace, list(TEST_INPUT.values()), exists = True)
def _start_network(docker_client: DockerClient,
docker_compose_client: DockerCompose) -> None:
"""Bring up and verify all images."""
_stop_network(docker_client, docker_compose_client)
label = ""
container_name = ""
try:
docker_compose_client.start()
for label, container_name in CONTAINERS.items():
container: Container = docker_client.containers.get(container_name)
assert (
container.status == "running"
), f"{label} not running after docker-compose ... up, cannot continue"
except AssertionError:
docker_compose_client.stop()
raise
except (HTTPError, NotFound):
docker_compose_client.stop()
raise LocalIntegrationTestError(
f"Docker API cannot access the {label} container, "
f"whose name is expected to be {container_name}")
def test_can_get_logs():
with DockerCompose("tests") as compose:
docker = DockerClient()
compose.wait_for("http://%s:4444/wd/hub" % docker.host())
stdout, stderr = compose.get_logs()
assert stdout, 'There should be something on stdout'
def test_compose_wait_for_container_ready():
with DockerCompose("tests") as compose:
docker = DockerClient()
compose.wait_for("http://%s:4444/wd/hub" % docker.host())
def test_can_throw_exception_if_no_port_exposed():
with DockerCompose("tests") as compose:
with pytest.raises(NoSuchPortExposed):
compose.get_service_host("hub", 5555)
def test_can_pull_images_before_spawning_service_via_compose():
with DockerCompose("tests", pull=True) as compose:
host = compose.get_service_host("hub", 4444)
port = compose.get_service_port("hub", 4444)
assert host == "0.0.0.0"
assert port == "4444"
def test_can_spawn_service_via_compose():
with DockerCompose('tests') as compose:
host = compose.get_service_host("hub", 4444)
port = compose.get_service_port("hub", 4444)
assert host == "0.0.0.0"
assert port == "4444"
def setUpClass(cls):
cls.compose = DockerCompose(filepath=dirname(abspath(__file__)))
cls.compose.start()
cls.compose.wait_for(cls.url(('producer', '9090'), 'users'))
def setUpClass(cls):
cls.compose = DockerCompose(filepath=dirname(abspath(__file__)))
cls.compose.start()
cls.compose.wait_for(cls.url(cls.collector_address()))
