def integration_test__the_etl_state_management_should_go_through_all_state_sucessfully_and_persist( self): compose = DockerCompose(filepath=os.path.dirname(base.__file__)) with compose: host = compose.get_service_host("cassandra", 9042) port = int(compose.get_service_port("cassandra", 9042)) conn = connect(host, port) conn.setup_connection("dev") sync_etl_state_table() sm = EtlSinkRecordStateManager(record_identifier="RandomRecordId", etl_signature="SomeTestEtl") print(sm.current_state()) print(sm.remote_state) self.assertEqual(sm.current_state(), EtlStates.Ready) sm.processing() self.assertEqual(sm.current_state(), EtlStates.Processing) sm.failed() self.assertEqual(sm.current_state(), EtlStates.Failed) sm.ready() self.assertEqual(sm.current_state(), EtlStates.Ready) sm.processing() self.assertEqual(sm.current_state(), EtlStates.Processing) sm.succeeded() self.assertEqual(sm.current_state(), EtlStates.Succeeded)
def _stop_network(docker_client: DockerClient, docker_compose_client: DockerCompose) -> None: """Bring down all images.""" print("Tearing down beer garden docker images. Please wait...") docker_compose_client.stop() if not _remove_volumes(docker_client): raise LocalIntegrationTestError( "Unable to remove volumes from this run")
def get_container(): COMPOSE_PATH = '{}/tests'.format(rootpath.detect('.', pattern='tests')) compose = DockerCompose(COMPOSE_PATH) compose.start() bootstrap_server = 'localhost:9092' consumer = KafkaConsumer(group_id='test', bootstrap_servers=[bootstrap_server]) if not consumer.topics(): raise KafkaError('Unable to connect with kafka container!') return compose
def __enter__(self): self.compose = DockerCompose(self.testcontainers_path) self.compose.start() if self.kafka_input_config: ensure_kafka_topic_exists(self.kafka_input_config) if self.postgres_config: ensure_postgres_database_exists(self.postgres_config) return self
def test_full_integration_with_local_cassandra(self): aws_conn = AwsConnectionSettings(region="us-east-1", secrets_manager=None, profile="default") execfile("../../secrets.py") compose = DockerCompose(filepath=os.path.dirname(base.__file__)) with compose: host = compose.get_service_host("cassandra", 9042) port = int(compose.get_service_port("cassandra", 9042)) cassandra_conn_setting = CassandraConnectionSettings( cluster_ips=[host], port=port, load_balancing_policy=DCAwareRoundRobinPolicy(), secrets_manager=CassandraSecretsManager( source=DictKeyValueSource({ "CASSANDRA_USERNAME": "", "CASSANDRA_PASSWORD": "", })), ) conn = verify_container_is_up(cassandra_conn_setting) # conn.get_session('system').execute(""" DROP TABLE test.etl_sink_record_state""") settings = AthenaToAdWordsOfflineConversionSettings( source_database=os.getenv("dummy_athena_database"), source_table=os.getenv("dummy_athena_table"), source_connection_settings=aws_conn, etl_identifier="test", destination_batch_size=100, etl_state_manager_connection=cassandra_conn_setting, etl_state_manager_keyspace="test", transformation_column_mapping={ 'google_click_id': 'googleClickId', 'conversion_name': 'conversionName', 'conversion_time': 'conversionTime', 'conversion_value': 'conversionValue', 'conversion_currency_code': 'conversionCurrencyCode' }, destination_connection_settings=GoogleAdWordsConnectionSettings( client_id=os.getenv("adwords_client_id"), user_agent="Tester", client_customer_id=os.getenv("adwords_client_customer_id"), secrets_manager=GoogleAdWordsSecretsManager()), ) etl = AthenaToAdWordsOfflineConversion(settings) files_actual = etl.list_source_files() # # self.assertListEqual(files_actual, []) # etl.upload_all() act = etl.upload_all() self.assertListEqual(act, [])
def docker_compose(request, prepare, version): # type: (FixtureRequest, Callable, str) -> None module = request.module cwd = dirname(inspect.getfile(module)) if version: with open(os.path.join(cwd, 'requirements.txt'), mode='w') as req: req.write(version) compose = DockerCompose(filepath=cwd) compose.start() exception = None for _ in range(0, 10): try: prepare() exception = None break except Exception as e: time.sleep(10) exception = e if exception: compose.stop() raise Exception("""Wait time exceeded {0} sec. Exception {1}""".format( 100, exception)) yield compose compose.stop()
def test_can_throw_exception_if_no_port_exposed(): compose = DockerCompose(os.path.dirname(__file__)) compose.start() with pytest.raises(NoSuchPortExposed): compose.get_service_host("hub", 5555) compose.stop()
def test_should_respect_key_limit(): with DockerCompose( os.getcwd() + "/test", compose_file_name="docker-compose.yml", pull=True ) as compose: (host, port) = assert_and_get_host_port(compose, REDIS_PORT) cache = Cache(host, port) namespace = "respect_key_limit" @cache(namespace=namespace, limit = 1) def test_func(test_input: int): return test_input assert cache.get_key_count() == 0 test_func(1) # First call gives two keys. One for the base function and one for the inputs assert cache.get_key_count() == 2 [test_func(i) for i in range(50)] assert cache.get_key_count() == 2
def test_should_evict_least_accessed(): with DockerCompose( os.getcwd() + "/test", compose_file_name="docker-compose.yml", pull=True ) as compose: (host, port) = assert_and_get_host_port(compose, REDIS_PORT) cache = Cache(host, port) namespace = "evict-least-accessed" @cache(namespace=namespace, limit = 3) def test_func(test_input: int): return test_input func_input = range(3) keys = [] # Create three keys [test_func(i) for i in func_input] [keys.append(get_cache_key(namespace, [i])) for i in func_input] keys.append(f'rc:{namespace}:keys') # base function assert set(cache.get_all_keys()) == set(keys) # Access all except first input [test_func(i) for i in func_input[1:]] # Add new input test_func(3) keys.append(get_cache_key(namespace, [3])) diff = set(keys) - set(cache.get_all_keys()) assert diff == set([keys[0]])
def test_integration(self): with DockerCompose("./test") as compose: time.sleep(5) host = compose.get_service_host('rest-api', 8008) port = compose.get_service_port('rest-api', 8008) url = f'http://{host}:{port}' print('\npublish production measurement:') mea_response = self.publish_prod_measurement(url) self.wait_for_commit(mea_response.json()['link']) print('\npublish consumption measurement:') mea_con_response = self.publish_con_measurement(url) self.wait_for_commit(mea_con_response.json()['link']) print('\nissue ggo:') ggo_response = self.issue_ggo(url) self.wait_for_commit(ggo_response.json()['link']) print('\nsplit ggo:') split_response = self.split_ggo(url) self.wait_for_commit(split_response.json()['link']) print('\ntransfer ggo:') transfer_response = self.transfer_ggo(url) self.wait_for_commit(transfer_response.json()['link']) print('\nretire ggo:') retire_response = self.retire_ggo(url) self.wait_for_commit(retire_response.json()['link'])
def setUpClass(cls): super().setUpClass() cls.containers = DockerCompose('tests/fixtures/config') cls.containers.start() time.sleep(10)
def broker(request): version = request.config.getoption("--publish-pact") publish = True if version else False # If the results are not going to be published to the broker, there is # nothing further to do anyway if not publish: yield return run_broker = request.config.getoption("--run-broker") if run_broker: # Start up the broker using docker-compose print("Starting broker") with DockerCompose("../broker", compose_file_name=["docker-compose.yml"], pull=True) as compose: stdout, stderr = compose.get_logs() if stderr: print("Errors\\n:{}".format(stderr)) print("{}".format(stdout)) print("Started broker") yield print("Stopping broker") print("Broker stopped") else: # Assuming there is a broker available already, docker-compose has been # used manually as the --run-broker option has not been provided yield return
def setUpClass(cls): cls.compose = DockerCompose(filepath=dirname(inspect.getfile(cls))) cls.compose.start() cls.compose.wait_for( cls.url(('consumer', '9090'), 'users?test=test1&test=test2&test2=test2'))
def broker(request): version = request.config.getoption('--publish-pact') publish = True if version else False # yield if not publish: yield return run_broker = request.config.getoption('--run-broker') if not run_broker: yield return else: print('Starting broker') with DockerCompose("../broker", compose_file_name=["docker-compose.yml"], pull=True) as compose: stdout, stderr = compose.get_logs() if stderr: print("Errors\\n:{}".format(stderr)) print(stdout) yield
def test_can_pass_env_params_by_env_file(): with DockerCompose('tests', compose_file_name='docker-compose-3.yml', env_file='.env.test') as _: check_env_is_set_cmd = 'docker exec tests_mysql_1 printenv | grep TEST_ASSERT_KEY'.split( ) out = subprocess.run(check_env_is_set_cmd, stdout=subprocess.PIPE) assert out.stdout.decode('utf-8').splitlines()[0], 'test_is_passed'
def test_can_spawn_service_via_compose(): compose = DockerCompose(os.path.dirname(__file__)) try: compose.start() host = compose.get_service_host("hub", 4444) port = compose.get_service_port("hub", 4444) assert host == "0.0.0.0" assert port == "4444" finally: compose.stop()
def test_can_build_images_before_spawning_service_via_compose(): with patch.object(DockerCompose, "_call_command") as call_mock: with DockerCompose(ROOT, build=True) as compose: ... assert compose.build docker_compose_cmd = call_mock.call_args_list[0][1]["cmd"] assert "docker-compose" in docker_compose_cmd assert "up" in docker_compose_cmd assert "--build" in docker_compose_cmd
class CdlEnv: def __init__(self, testcontainers_path, kafka_input_config = None, postgres_config = None): self.testcontainers_path = testcontainers_path self.kafka_input_config = kafka_input_config self.postgres_config = postgres_config def __enter__(self): self.compose = DockerCompose(self.testcontainers_path) self.compose.start() if self.kafka_input_config: ensure_kafka_topic_exists(self.kafka_input_config) if self.postgres_config: ensure_postgres_database_exists(self.postgres_config) return self def __exit__(self, exc_type, exc_val, exc_tb): self.compose.stop()
def test_s3(): """Ensure that HTTP service is up and responsive.""" compose = DockerCompose('tests') with compose: s3 = get_test_s3(compose) s3.Bucket('test1').create() s3.Bucket('test2').create() s3.Object('test1','test1key').upload_fileobj(file_with_size(1024*1024*10)) s3.Object('test2', 'test2key').upload_fileobj(file_with_size(1024 * 1024)) s3.Object('test2', 'test2key2').upload_fileobj(file_with_size(1024 * 1024*2)) yield compose
def test_can_parse_multiple_compose_files(): with DockerCompose( filepath="tests", compose_file_name=["docker-compose.yml", "docker-compose-2.yml"]) as compose: host = compose.get_service_host("mysql", 3306) port = compose.get_service_port("mysql", 3306) assert host == "0.0.0.0" assert port == "3306" host = compose.get_service_host("hub", 4444) port = compose.get_service_port("hub", 4444) assert host == "0.0.0.0" assert port == "4444"
def _get_composer(environment: Dict[str, Any], env_file: Path) -> DockerCompose: """Create a DockerCompose object with the required .env file.""" env_file_path = str(env_file) if env_file.exists(): env_file.unlink() env_file.touch() with env_file.open(mode="w") as file: for key, val in environment.items(): file.write(f"{key}={val}\n") return DockerCompose( ".", compose_file_name="docker-compose-local-integration-tests.yml", env_file=env_file_path, )
def test_should_insert_key_after_call(): with DockerCompose( os.getcwd() + "/test", compose_file_name="docker-compose.yml", pull=True ) as compose: (host, port) = assert_and_get_host_port(compose, REDIS_PORT) cache = Cache(host, port) namespace = "insert-key-test" @cache(namespace=namespace) def test_func(first: str, second: str, third: int): return f"first second {third}" assert_key_exists(cache, namespace, list(TEST_INPUT.values()), exists = False) test_func(TEST_INPUT["first"], TEST_INPUT["second"], TEST_INPUT["third"]) assert_key_exists(cache, namespace, list(TEST_INPUT.values()), exists = True)
def _start_network(docker_client: DockerClient, docker_compose_client: DockerCompose) -> None: """Bring up and verify all images.""" _stop_network(docker_client, docker_compose_client) label = "" container_name = "" try: docker_compose_client.start() for label, container_name in CONTAINERS.items(): container: Container = docker_client.containers.get(container_name) assert ( container.status == "running" ), f"{label} not running after docker-compose ... up, cannot continue" except AssertionError: docker_compose_client.stop() raise except (HTTPError, NotFound): docker_compose_client.stop() raise LocalIntegrationTestError( f"Docker API cannot access the {label} container, " f"whose name is expected to be {container_name}")
def test_can_get_logs(): with DockerCompose("tests") as compose: docker = DockerClient() compose.wait_for("http://%s:4444/wd/hub" % docker.host()) stdout, stderr = compose.get_logs() assert stdout, 'There should be something on stdout'
def test_compose_wait_for_container_ready(): with DockerCompose("tests") as compose: docker = DockerClient() compose.wait_for("http://%s:4444/wd/hub" % docker.host())
def test_can_throw_exception_if_no_port_exposed(): with DockerCompose("tests") as compose: with pytest.raises(NoSuchPortExposed): compose.get_service_host("hub", 5555)
def test_can_pull_images_before_spawning_service_via_compose(): with DockerCompose("tests", pull=True) as compose: host = compose.get_service_host("hub", 4444) port = compose.get_service_port("hub", 4444) assert host == "0.0.0.0" assert port == "4444"
def test_can_spawn_service_via_compose(): with DockerCompose('tests') as compose: host = compose.get_service_host("hub", 4444) port = compose.get_service_port("hub", 4444) assert host == "0.0.0.0" assert port == "4444"
def setUpClass(cls): cls.compose = DockerCompose(filepath=dirname(abspath(__file__))) cls.compose.start() cls.compose.wait_for(cls.url(('producer', '9090'), 'users'))
def setUpClass(cls): cls.compose = DockerCompose(filepath=dirname(abspath(__file__))) cls.compose.start() cls.compose.wait_for(cls.url(cls.collector_address()))