def test_docker_generic_db(): mongo_container = DockerContainer("mongo:latest") mongo_container.with_bind_ports(27017, 27017) with mongo_container: def connect(): return MongoClient("mongodb://{}:{}".format( mongo_container.get_container_host_ip(), mongo_container.get_exposed_port(27017))) db = wait_for(connect).primer result = db.restaurants.insert_one({ "address": { "street": "2 Avenue", "zipcode": "10075", "building": "1480", "coord": [-73.9557413, 40.7720266] }, "borough": "Manhattan", "cuisine": "Italian", "name": "Vella", "restaurant_id": "41704620" }) print(result.inserted_id) cursor = db.restaurants.find({"borough": "Manhattan"}) for document in cursor: print(document)
def wait_for_http_code(container: DockerContainer, status_code: int, port: int = 80, path: str = '/', scheme: str = 'http', timeout=None, interval=1, request_kwargs: dict = None): """ Wait for a specific http status code. Parameters ---------- container : DockerContainer Container which is queried for a specific status code. status_code : int Status code to wait for. port : int Port to query request on. path : str Path to use for request. Default is '/' scheme : str Scheme to use in request query. Default is 'http' timeout : float or None Number of seconds to wait for the port to be open. Defaults to wait indefinitely. interval : float Interval at which to poll the port. request_kwargs: dict kwargs to pass into the request, e.g.: {'verify': False} Returns ------- duration : float Number of seconds until the check passed. """ if request_kwargs is None: request_kwargs = {'timeout': 1.0} elif 'timeout' not in request_kwargs: request_kwargs['timeout'] = 1.0 start = time.time() # wait for port to open before continuing with http check wait_for_port(container, port, timeout, interval) while True: duration = time.time() - start dest = "%s://%s:%d%s" % (scheme, container.get_container_host_ip(), int(container.get_exposed_port(port)), path) res = None try: res = requests.get(dest, **request_kwargs) except RequestException: pass if res and res.status_code == status_code: return duration if timeout and duration > timeout: raise TimeoutError("container did not respond with %d listening on port %d in %.3f seconds" % (status_code, port, timeout))
def test_docker_kwargs(): code_dir = Path(__file__).parent container_first = DockerContainer("nginx:latest") container_first.with_volume_mapping(code_dir, '/code') container_second = DockerContainer("nginx:latest") with container_first: container_second.with_kwargs(volumes_from=[container_first._container.short_id]) with container_second: files_first = container_first.exec('ls /code').output.decode('utf-8').strip() files_second = container_second.exec('ls /code').output.decode('utf-8').strip() assert files_first == files_second
def test_containers_matched_by_label(self): """ There are two docker containers. Matching is by label org.riotkit.domain: {{ domain }} Checks: - Parsing of the labels syntax """ first = DockerContainer(image='nginx:1.19').with_kwargs( labels={ 'org.riotkit.domain': 'duckduckgo.com' }).start() second = DockerContainer(image='nginx:1.19')\ .with_kwargs(labels={'org.riotkit.domain': 'riseup.net,bing.com'}).start() try: wait_for_logs(first, 'ready for start up') wait_for_logs(second, 'ready for start up') os.environ['PATH'] = path + ':' + os.environ['PATH'] check = TlsDockerNetworkCheck(param_type='label', param_name='org.riotkit.domain', alert_days_before=1) \ .main() finally: first.stop() second.stop() self.assertIn('Domain duckduckgo.com is OK', check[0]) self.assertIn('Domain bing.com is OK', check[0]) self.assertIn('Domain riseup.net is OK', check[0]) self.assertTrue(check[1])
def test_multiple_containers_with_multiple_domains_per_container(self): """ There are two containers. Matching is by environment variable INFR_VIRTUAL_HOST={{ domain }} One with single domain: google.com Second one: duckduck.com and bing.com Checks: - There is a correct connection to docker, and proper parsing of the data from docker - "tls" check is correctly called """ first = DockerContainer(image='nginx:1.19').with_env( 'INFR_VIRTUAL_HOST', 'duckduckgo.com,bing.com').start() second = DockerContainer(image='nginx:1.19').with_env( 'INFR_VIRTUAL_HOST', 'google.com').start() try: wait_for_logs(first, 'ready for start up') wait_for_logs(second, 'ready for start up') os.environ['PATH'] = path + ':' + os.environ['PATH'] check = TlsDockerNetworkCheck(param_type='environment', param_name='INFR_VIRTUAL_HOST', alert_days_before=1)\ .main() finally: first.stop() second.stop() self.assertIn('Domain google.com is OK', check[0]) self.assertIn('Domain bing.com is OK', check[0]) self.assertIn('Domain duckduckgo.com is OK', check[0]) self.assertTrue(check[1])
def docker_builder_remote_registry(): with use_local_installation(), DockerContainer( 'registry:latest').with_bind_ports(REGISTRY_PORT, REGISTRY_PORT): yield DockerBuilder( ProviderMock(), DockerImage(IMAGE_NAME, registry=RemoteDockerRegistry(REGISTRY_HOST)))
class RedisOnlineStoreCreator(OnlineStoreCreator): def __init__(self, project_name: str): super().__init__(project_name) self.container = DockerContainer("redis").with_exposed_ports("6379") def create_online_store(self) -> Dict[str, str]: self.container.start() log_string_to_wait_for = "Ready to accept connections" wait_for_logs( container=self.container, predicate=log_string_to_wait_for, timeout=5 ) exposed_port = self.container.get_exposed_port("6379") return {"type": "redis", "connection_string": f"localhost:{exposed_port},db=0"} def teardown(self): self.container.stop()
def docker_builder_remote_registry(): with use_local_installation(), DockerContainer( 'registry:latest').with_exposed_ports(REGISTRY_PORT) as container: host = f'localhost:{container.get_exposed_port(REGISTRY_PORT)}' yield DockerBuilder( ProviderMock(), DockerImage(IMAGE_NAME, registry=RemoteDockerRegistry(host)))
def __init__(self, project_name: str, **kwargs): super().__init__(project_name) self.tables_created: List[str] = [] if "offline_container" not in kwargs or not kwargs.get( "offline_container", None): # If we don't get an offline container provided, we try to create it on the fly. # the problem here is that each test creates its own conatiner, which basically # browns out developer laptops. current_file = pathlib.Path(__file__).parent.resolve() catalog_dir = current_file.parent.joinpath("catalog") self.container = ( DockerContainer("trinodb/trino:376").with_volume_mapping( catalog_dir, "/etc/catalog/").with_exposed_ports("8080")) self.container.start() self.provided_container = False log_string_to_wait_for = "SERVER STARTED" wait_for_logs(container=self.container, predicate=log_string_to_wait_for, timeout=30) else: self.provided_container = True self.container = kwargs["offline_container"] self.exposed_port = self.container.get_exposed_port("8080") self.client = Trino( user="******", catalog="memory", host="localhost", port=self.exposed_port, )
def test_raise_timeout_invalid_path(): with pytest.raises(TimeoutError): with DockerContainer("nginx").with_exposed_ports(80) as container: wait_for_http_code(container, 200, path="/fail", port=80, timeout=2)
def __init__(self, project_id, instance_id, table, use_emulator): self.use_emulator = use_emulator self.table = table self.host = None if use_emulator: self.emulator = DockerContainer( 'gcr.io/cloud-spanner-emulator/emulator:latest').with_exposed_ports( 9010, 9020) retry(self.emulator.start, 3, 'Could not start spanner emulator.') time.sleep(3) self.host = f'{self.emulator.get_container_host_ip()}:' \ f'{self.emulator.get_exposed_port(9010)}' os.environ['SPANNER_EMULATOR_HOST'] = self.host self.client = spanner.Client(project_id) self.instance = self.client.instance(instance_id) if use_emulator: self.create_instance()
def s3server(): # `s3server` was renamed to `cloudserver` # image is described as being unmaintained but no `cloudserver` image is available for now with DockerContainer('scality/s3server:mem-latest') \ .with_env('SCALITY_ACCESS_KEY_ID', ACCESS_KEY) \ .with_env('SCALITY_SECRET_ACCESS_KEY', SECRET_KEY) \ .with_bind_ports(PORT, PORT): yield
def test_docker_custom_image(): container = DockerContainer("mysql:5.7.17") container.with_exposed_ports(3306) container.with_env("MYSQL_ROOT_PASSWORD", "root") with container: port = container.get_exposed_port(3306) assert int(port) > 0
def s3server(pytestconfig): if not has_docker() or 'not docker' in pytestconfig.getoption('markexpr'): pytest.skip('skipping docker tests') with DockerContainer('minio/minio:latest') \ .with_command('server /data') \ .with_env('MINIO_ACCESS_KEY', ACCESS_KEY) \ .with_env('MINIO_SECRET_KEY', SECRET_KEY) \ .with_exposed_ports(PORT) as container: sleep(5) # wait to ensure that S3 server has enough time to properly start yield container.get_exposed_port(PORT)
def postgres_server(pytestconfig): if not has_docker() or 'not docker' in pytestconfig.getoption('markexpr'): pytest.skip('skipping docker tests') with DockerContainer('postgres:alpine') \ .with_bind_ports(5432, PG_PORT) \ .with_env("POSTGRES_USER", PG_USER) \ .with_env("POSTGRES_PASSWORD", PG_PASS) \ .with_env("POSTGRES_DB", PG_DB): sleep(5) # wait to ensure that PostgreSQL server has enough time to properly start yield
def __init__(self, project_name: str): super().__init__(project_name) self.container = ( DockerContainer( "gcr.io/google.com/cloudsdktool/cloud-sdk:380.0.0-emulators" ) .with_command( "gcloud beta emulators datastore start --project test-project --host-port 0.0.0.0:8081" ) .with_exposed_ports("8081") )
def test_container_environments(): code_dir = Path(__file__).parent container = DockerContainer("nginx:latest") container.with_env('TEST', 'test') container.with_env('DOCKER', 'docker') with container: output = container.exec("bash -c 'echo $TEST $DOCKER'").output.decode( 'utf-8').strip() assert output == 'test docker'
class DynamoDBOnlineStoreCreator(OnlineStoreCreator): def __init__(self, project_name: str): super().__init__(project_name) self.container = DockerContainer( "amazon/dynamodb-local:latest").with_exposed_ports("8000") def create_online_store(self) -> Dict[str, str]: self.container.start() log_string_to_wait_for = ( "Initializing DynamoDB Local with the following configuration:") wait_for_logs(container=self.container, predicate=log_string_to_wait_for, timeout=5) exposed_port = self.container.get_exposed_port("8000") return { "type": "dynamodb", "endpoint_url": f"http://localhost:{exposed_port}", "region": "us-west-2", } def teardown(self): self.container.stop()
def set_localstack(self): self.localstack = DockerContainer('localstack/localstack:{}' .format(LOCALSTACK_VERSION))\ .with_env('SERVICES', 'kinesis')\ .with_env('KINESIS_PORT', '4568')\ .with_env('USE_SSL', 'true')\ .with_exposed_ports(4568)\ .with_volume_mapping('/var/run/docker.sock', '/var/run/docker.sock', 'rw') # Repeat if ReadTimeout is raised. for i in range(4): try: self.localstack.start() break except Exception as e: # pylint: disable=bare-except if i == 3: logging.error('Could not initialize localstack container') raise e self.aws_service_endpoint = 'https://{}:{}'.format( self.localstack.get_container_host_ip(), self.localstack.get_exposed_port('4568'), )
def get_singleton(cls): if not cls.is_running: cls.container = ( DockerContainer("trinodb/trino:376").with_volume_mapping( cls.catalog_dir, "/etc/catalog/").with_exposed_ports("8080")) cls.container.start() log_string_to_wait_for = "SERVER STARTED" wait_for_logs(container=cls.container, predicate=log_string_to_wait_for, timeout=30) cls.is_running = True return cls.container
def test_add_map_entry(): code_dir = Path(__file__).parent container = DockerContainer("nginx:latest") container._with_map_entry('something', 'is', 'arbitrary') container._with_map_entry('something', 'also', 'arbitrary') assert container._kwargs['something'] == { 'also': 'arbitrary', 'is': 'arbitrary' }
def test_update_authorized_keys(): with DockerContainer("quay.io/eliezio/sari-test-bh:v1.1.0" ).with_exposed_ports(SSH_PORT) as server: ssh_pub_keys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEfzjdkO1LKnS/it62jmw9tH4BznlnDCBrzaKguujJ15 " "*****@*****.**", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGD13Dbe1QoYrFZqCue1TzGkzDSra9ZHzv8gZy9+vb0Y " "*****@*****.**", ] errors = update_authorized_keys(server.get_container_host_ip(), "admin", "tests/data/admin_id_rsa", "", "acme", ssh_pub_keys, int(server.get_exposed_port(SSH_PORT))) assert not errors # noinspection PyProtectedMember _, stat = server._container.get_archive( "/home/acme/.ssh/authorized_keys2") assert stat["size"] == sum(map(len, ssh_pub_keys)) + (len(ssh_pub_keys) - 1)
def registry(tmpdir_factory): with DockerContainer('registry:latest').with_bind_ports( REGISTRY_PORT, REGISTRY_PORT): client = DockerClient() # migrate our image to custom Docker registry client.images.pull(IMAGE_NAME, 'latest').tag(TAG_NAME) client.images.push(TAG_NAME) tmpdir = str(tmpdir_factory.mktemp("image")) # create failing image: alpine is too small to have python inside with open(os.path.join(tmpdir, 'Dockerfile'), 'w') as f: f.write(""" FROM alpine:latest CMD python """) client.images.build(path=tmpdir, tag=BROKEN_TAG_NAME) client.images.push(BROKEN_TAG_NAME) yield RemoteDockerRegistry(REGISTRY_HOST)
def before_all(context): # Adding label to identify Docker container as testcontainer container (testcontainers-java does this # by default). # # Much of the testcontainers-python library is simply a wrapper around Docker SDK for Python. This is # a good example of how to make use of the parameters that the Docker SDK makes available, even if # testcontainers doesn't do so explicitly. When using testcontainers-python, it is therefore advisable # to be familiar with the Docker SDK (https://docker-py.readthedocs.io/en/stable/containers.html) # and the testcontainers-python source code (https://github.com/testcontainers/testcontainers-python). # context.rabbit = DockerContainer('rabbitmq:3.8.3-management', labels = {"org.testcontainers":"true"}) \ .with_exposed_ports(RABBIT_PORT) context.rabbit.start() wait_for_logs(context.rabbit, r"Server startup complete; \d+ plugins started", timeout=120) context.server = SimpleHTTPServer( HOST, 0, context.rabbit.get_exposed_port(RABBIT_PORT)) context.server.serve()
def test_waits_for_nginx_to_be_ready(self): container = DockerContainer( image='nginx:1.19-alpine').with_name('nginx') container.start() try: io = IO() signal = execute_app( WaitForOutputApp( container='ngin(.*)', command='', pattern='Configuration complete; ready for start up', timeout=15, io=io)) self.assertIn('Match found', signal.message) self.assertEqual(0, signal.exit_code) finally: container.stop()
def test_regexp_patterns(self): container = DockerContainer( image='nginx:1.19-alpine').with_name('nginx_0') container.start() try: io = IO() for pattern in ['ngin*', 'nginx.*', 'ng.*x']: signal = execute_app( WaitForOutputApp( container=pattern, command='', pattern='Configuration complete; ready for start up', timeout=15, io=io)) self.assertIn('Match found', signal.message) self.assertEqual(0, signal.exit_code) finally: container.stop()
def test_too_many_containers_found(self): first = DockerContainer(image='nginx:1.19-alpine').with_name('nginx_0') second = DockerContainer( image='nginx:1.19-alpine').with_name('nginx_1') first.start() second.start() io = IO() try: signal = execute_app( WaitForOutputApp( container='nginx_*', command='', pattern='Configuration complete; ready for start up', timeout=15, io=io)) self.assertEqual('Too many containers found', signal.message) self.assertEqual(1, signal.exit_code) finally: first.stop() second.stop()
def __init__(self, project_name: str): super().__init__(project_name) self.container = DockerContainer("redis").with_exposed_ports("6379")
class SpannerHelper(object): def __init__(self, project_id, instance_id, table, use_emulator): self.use_emulator = use_emulator self.table = table self.host = None if use_emulator: self.emulator = DockerContainer( 'gcr.io/cloud-spanner-emulator/emulator:latest').with_exposed_ports( 9010, 9020) retry(self.emulator.start, 3, 'Could not start spanner emulator.') time.sleep(3) self.host = f'{self.emulator.get_container_host_ip()}:' \ f'{self.emulator.get_exposed_port(9010)}' os.environ['SPANNER_EMULATOR_HOST'] = self.host self.client = spanner.Client(project_id) self.instance = self.client.instance(instance_id) if use_emulator: self.create_instance() def create_instance(self): self.instance.create().result(120) def create_database(self, database_id): database = self.instance.database( database_id, ddl_statements=[ f''' CREATE TABLE {self.table} ( f_string STRING(1024) NOT NULL, f_int64 INT64, f_boolean BOOL ) PRIMARY KEY (f_string)''' ]) database.create().result(120) def insert_values(self, database_id, values, columns=None): values = values or [] columns = columns or ('f_string', 'f_int64', 'f_boolean') with self.instance.database(database_id).batch() as batch: batch.insert( table=self.table, columns=columns, values=values, ) def get_emulator_host(self): return f'http://{self.host}' def read_data(self, database_id, prefix): database = self.instance.database(database_id) with database.snapshot() as snapshot: results = snapshot.execute_sql( f'''SELECT * FROM {self.table} WHERE f_string LIKE "{prefix}%" ORDER BY f_int64''') try: rows = list(results) if results else None except IndexError: raise ValueError(f"Spanner results not found for {prefix}.") return rows def drop_database(self, database_id): database = self.instance.database(database_id) database.drop() def shutdown(self): if self.use_emulator: try: self.emulator.stop() except: # pylint: disable=bare-except logging.error('Could not stop Spanner Cloud emulator.')
class CrossLanguageKinesisIOTest(unittest.TestCase): @unittest.skipUnless( TestPipeline().get_option('aws_kinesis_stream'), 'Cannot test on real aws without pipeline options provided') def test_kinesis_io_roundtrip(self): # TODO: enable this test for localstack once BEAM-10664 is resolved self.run_kinesis_write() self.run_kinesis_read() @unittest.skipIf( TestPipeline().get_option('aws_kinesis_stream'), 'Do not test on localstack when pipeline options were provided') def test_kinesis_write(self): # TODO: remove this test once BEAM-10664 is resolved self.run_kinesis_write() records = self.kinesis_helper.read_from_stream(self.aws_kinesis_stream) self.assertEqual( sorted(records), sorted([RECORD + str(i).encode() for i in range(NUM_RECORDS)])) def run_kinesis_write(self): with TestPipeline(options=PipelineOptions(self.pipeline_args)) as p: p.not_use_test_runner_api = True _ = ( p | 'Impulse' >> beam.Impulse() | 'Generate' >> beam.FlatMap(lambda x: range(NUM_RECORDS)) # pylint: disable=bad-option-value | 'Map to bytes' >> beam.Map(lambda x: RECORD + str(x).encode( )).with_output_types(bytes) | 'WriteToKinesis' >> WriteToKinesis( stream_name=self.aws_kinesis_stream, aws_access_key=self.aws_access_key, aws_secret_key=self.aws_secret_key, region=self.aws_region, service_endpoint=self.aws_service_endpoint, verify_certificate=(not self.use_localstack), partition_key='1', producer_properties=self.producer_properties, )) def run_kinesis_read(self): records = [RECORD + str(i).encode() for i in range(NUM_RECORDS)] with TestPipeline(options=PipelineOptions(self.pipeline_args)) as p: result = (p | 'ReadFromKinesis' >> ReadDataFromKinesis( stream_name=self.aws_kinesis_stream, aws_access_key=self.aws_access_key, aws_secret_key=self.aws_secret_key, region=self.aws_region, service_endpoint=self.aws_service_endpoint, verify_certificate=not self.use_localstack, max_num_records=NUM_RECORDS, max_read_time=MAX_READ_TIME, request_records_limit=REQUEST_RECORDS_LIMIT, watermark_policy=WatermarkPolicy.ARRIVAL_TIME, watermark_idle_duration_threshold=MAX_READ_TIME, initial_position_in_stream=InitialPositionInStream. AT_TIMESTAMP, initial_timestamp_in_stream=NOW_MILLIS, ).with_output_types(bytes)) assert_that(result, equal_to(records)) def set_localstack(self): self.localstack = DockerContainer('localstack/localstack:{}' .format(LOCALSTACK_VERSION))\ .with_env('SERVICES', 'kinesis')\ .with_env('KINESIS_PORT', '4568')\ .with_env('USE_SSL', 'true')\ .with_exposed_ports(4568)\ .with_volume_mapping('/var/run/docker.sock', '/var/run/docker.sock', 'rw') # Repeat if ReadTimeout is raised. for i in range(4): try: self.localstack.start() break except Exception as e: # pylint: disable=bare-except if i == 3: logging.error('Could not initialize localstack container') raise e self.aws_service_endpoint = 'https://{}:{}'.format( self.localstack.get_container_host_ip(), self.localstack.get_exposed_port('4568'), ) def setUp(self): parser = argparse.ArgumentParser() parser.add_argument( '--aws_kinesis_stream', default='beam_kinesis_xlang', help='Kinesis stream name', ) parser.add_argument( '--aws_access_key', default='accesskey', help=('Aws access key'), ) parser.add_argument( '--aws_secret_key', default='secretkey', help='Aws secret key', ) parser.add_argument( '--aws_region', default='us-east-1', help='Aws region', ) parser.add_argument( '--aws_service_endpoint', default=None, help='Url to external aws endpoint', ) parser.add_argument( '--use_real_aws', default=False, dest='use_real_aws', action='store_true', help='Flag whether to use real aws for the tests purpose', ) parser.add_argument( '--expansion_service', help='Url to externally launched expansion service.', ) pipeline = TestPipeline() argv = pipeline.get_full_options_as_args() known_args, self.pipeline_args = parser.parse_known_args(argv) self.aws_kinesis_stream = known_args.aws_kinesis_stream self.aws_access_key = known_args.aws_access_key self.aws_secret_key = known_args.aws_secret_key self.aws_region = known_args.aws_region self.aws_service_endpoint = known_args.aws_service_endpoint self.use_localstack = not known_args.use_real_aws self.expansion_service = known_args.expansion_service self.producer_properties = { 'CollectionMaxCount': str(NUM_RECORDS), 'ConnectTimeout': str(MAX_READ_TIME), } if self.use_localstack: self.set_localstack() self.kinesis_helper = KinesisHelper( self.aws_access_key, self.aws_secret_key, self.aws_region, self.aws_service_endpoint.replace('https', 'http') if self.aws_service_endpoint else None, ) if self.use_localstack: self.kinesis_helper.create_stream(self.aws_kinesis_stream) def tearDown(self): if self.use_localstack: self.kinesis_helper.delete_stream(self.aws_kinesis_stream) try: self.localstack.stop() except: # pylint: disable=bare-except logging.error('Could not stop the localstack container')