def test_docker_generic_db():
mongo_container = DockerContainer("mongo:latest")
mongo_container.with_bind_ports(27017, 27017)
with mongo_container:
def connect():
return MongoClient("mongodb://{}:{}".format(
mongo_container.get_container_host_ip(),
mongo_container.get_exposed_port(27017)))
db = wait_for(connect).primer
result = db.restaurants.insert_one({
"address": {
"street": "2 Avenue",
"zipcode": "10075",
"building": "1480",
"coord": [-73.9557413, 40.7720266]
},
"borough": "Manhattan",
"cuisine": "Italian",
"name": "Vella",
"restaurant_id": "41704620"
})
print(result.inserted_id)
cursor = db.restaurants.find({"borough": "Manhattan"})
for document in cursor:
print(document)
def wait_for_http_code(container: DockerContainer, status_code: int, port: int = 80, path: str = '/',
scheme: str = 'http', timeout=None, interval=1, request_kwargs: dict = None):
"""
Wait for a specific http status code.
Parameters
----------
container : DockerContainer
Container which is queried for a specific status code.
status_code : int
Status code to wait for.
port : int
Port to query request on.
path : str
Path to use for request. Default is '/'
scheme : str
Scheme to use in request query. Default is 'http'
timeout : float or None
Number of seconds to wait for the port to be open. Defaults to wait indefinitely.
interval : float
Interval at which to poll the port.
request_kwargs: dict
kwargs to pass into the request, e.g.: {'verify': False}
Returns
-------
duration : float
Number of seconds until the check passed.
"""
if request_kwargs is None:
request_kwargs = {'timeout': 1.0}
elif 'timeout' not in request_kwargs:
request_kwargs['timeout'] = 1.0
start = time.time()
# wait for port to open before continuing with http check
wait_for_port(container, port, timeout, interval)
while True:
duration = time.time() - start
dest = "%s://%s:%d%s" % (scheme,
container.get_container_host_ip(),
int(container.get_exposed_port(port)),
path)
res = None
try:
res = requests.get(dest, **request_kwargs)
except RequestException:
pass
if res and res.status_code == status_code:
return duration
if timeout and duration > timeout:
raise TimeoutError("container did not respond with %d listening on port %d in %.3f seconds"
% (status_code, port, timeout))
def test_docker_kwargs():
code_dir = Path(__file__).parent
container_first = DockerContainer("nginx:latest")
container_first.with_volume_mapping(code_dir, '/code')
container_second = DockerContainer("nginx:latest")
with container_first:
container_second.with_kwargs(volumes_from=[container_first._container.short_id])
with container_second:
files_first = container_first.exec('ls /code').output.decode('utf-8').strip()
files_second = container_second.exec('ls /code').output.decode('utf-8').strip()
assert files_first == files_second
def test_containers_matched_by_label(self):
"""
There are two docker containers. Matching is by label org.riotkit.domain: {{ domain }}
Checks:
- Parsing of the labels syntax
"""
first = DockerContainer(image='nginx:1.19').with_kwargs(
labels={
'org.riotkit.domain': 'duckduckgo.com'
}).start()
second = DockerContainer(image='nginx:1.19')\
.with_kwargs(labels={'org.riotkit.domain': 'riseup.net,bing.com'}).start()
try:
wait_for_logs(first, 'ready for start up')
wait_for_logs(second, 'ready for start up')
os.environ['PATH'] = path + ':' + os.environ['PATH']
check = TlsDockerNetworkCheck(param_type='label', param_name='org.riotkit.domain', alert_days_before=1) \
.main()
finally:
first.stop()
second.stop()
self.assertIn('Domain duckduckgo.com is OK', check[0])
self.assertIn('Domain bing.com is OK', check[0])
self.assertIn('Domain riseup.net is OK', check[0])
self.assertTrue(check[1])
def test_multiple_containers_with_multiple_domains_per_container(self):
"""
There are two containers. Matching is by environment variable INFR_VIRTUAL_HOST={{ domain }}
One with single domain: google.com
Second one: duckduck.com and bing.com
Checks:
- There is a correct connection to docker, and proper parsing of the data from docker
- "tls" check is correctly called
"""
first = DockerContainer(image='nginx:1.19').with_env(
'INFR_VIRTUAL_HOST', 'duckduckgo.com,bing.com').start()
second = DockerContainer(image='nginx:1.19').with_env(
'INFR_VIRTUAL_HOST', 'google.com').start()
try:
wait_for_logs(first, 'ready for start up')
wait_for_logs(second, 'ready for start up')
os.environ['PATH'] = path + ':' + os.environ['PATH']
check = TlsDockerNetworkCheck(param_type='environment', param_name='INFR_VIRTUAL_HOST', alert_days_before=1)\
.main()
finally:
first.stop()
second.stop()
self.assertIn('Domain google.com is OK', check[0])
self.assertIn('Domain bing.com is OK', check[0])
self.assertIn('Domain duckduckgo.com is OK', check[0])
self.assertTrue(check[1])
def docker_builder_remote_registry():
with use_local_installation(), DockerContainer(
'registry:latest').with_bind_ports(REGISTRY_PORT, REGISTRY_PORT):
yield DockerBuilder(
ProviderMock(),
DockerImage(IMAGE_NAME,
registry=RemoteDockerRegistry(REGISTRY_HOST)))
class RedisOnlineStoreCreator(OnlineStoreCreator):
def __init__(self, project_name: str):
super().__init__(project_name)
self.container = DockerContainer("redis").with_exposed_ports("6379")
def create_online_store(self) -> Dict[str, str]:
self.container.start()
log_string_to_wait_for = "Ready to accept connections"
wait_for_logs(
container=self.container, predicate=log_string_to_wait_for, timeout=5
)
exposed_port = self.container.get_exposed_port("6379")
return {"type": "redis", "connection_string": f"localhost:{exposed_port},db=0"}
def teardown(self):
self.container.stop()
def docker_builder_remote_registry():
with use_local_installation(), DockerContainer(
'registry:latest').with_exposed_ports(REGISTRY_PORT) as container:
host = f'localhost:{container.get_exposed_port(REGISTRY_PORT)}'
yield DockerBuilder(
ProviderMock(),
DockerImage(IMAGE_NAME, registry=RemoteDockerRegistry(host)))
def __init__(self, project_name: str, **kwargs):
super().__init__(project_name)
self.tables_created: List[str] = []
if "offline_container" not in kwargs or not kwargs.get(
"offline_container", None):
# If we don't get an offline container provided, we try to create it on the fly.
# the problem here is that each test creates its own conatiner, which basically
# browns out developer laptops.
current_file = pathlib.Path(__file__).parent.resolve()
catalog_dir = current_file.parent.joinpath("catalog")
self.container = (
DockerContainer("trinodb/trino:376").with_volume_mapping(
catalog_dir, "/etc/catalog/").with_exposed_ports("8080"))
self.container.start()
self.provided_container = False
log_string_to_wait_for = "SERVER STARTED"
wait_for_logs(container=self.container,
predicate=log_string_to_wait_for,
timeout=30)
else:
self.provided_container = True
self.container = kwargs["offline_container"]
self.exposed_port = self.container.get_exposed_port("8080")
self.client = Trino(
user="******",
catalog="memory",
host="localhost",
port=self.exposed_port,
)
def test_raise_timeout_invalid_path():
with pytest.raises(TimeoutError):
with DockerContainer("nginx").with_exposed_ports(80) as container:
wait_for_http_code(container,
200,
path="/fail",
port=80,
timeout=2)
def __init__(self, project_id, instance_id, table, use_emulator):
self.use_emulator = use_emulator
self.table = table
self.host = None
if use_emulator:
self.emulator = DockerContainer(
'gcr.io/cloud-spanner-emulator/emulator:latest').with_exposed_ports(
9010, 9020)
retry(self.emulator.start, 3, 'Could not start spanner emulator.')
time.sleep(3)
self.host = f'{self.emulator.get_container_host_ip()}:' \
f'{self.emulator.get_exposed_port(9010)}'
os.environ['SPANNER_EMULATOR_HOST'] = self.host
self.client = spanner.Client(project_id)
self.instance = self.client.instance(instance_id)
if use_emulator:
self.create_instance()
def s3server():
# `s3server` was renamed to `cloudserver`
# image is described as being unmaintained but no `cloudserver` image is available for now
with DockerContainer('scality/s3server:mem-latest') \
.with_env('SCALITY_ACCESS_KEY_ID', ACCESS_KEY) \
.with_env('SCALITY_SECRET_ACCESS_KEY', SECRET_KEY) \
.with_bind_ports(PORT, PORT):
yield
def test_docker_custom_image():
container = DockerContainer("mysql:5.7.17")
container.with_exposed_ports(3306)
container.with_env("MYSQL_ROOT_PASSWORD", "root")
with container:
port = container.get_exposed_port(3306)
assert int(port) > 0
def s3server(pytestconfig):
if not has_docker() or 'not docker' in pytestconfig.getoption('markexpr'):
pytest.skip('skipping docker tests')
with DockerContainer('minio/minio:latest') \
.with_command('server /data') \
.with_env('MINIO_ACCESS_KEY', ACCESS_KEY) \
.with_env('MINIO_SECRET_KEY', SECRET_KEY) \
.with_exposed_ports(PORT) as container:
sleep(5) # wait to ensure that S3 server has enough time to properly start
yield container.get_exposed_port(PORT)
def postgres_server(pytestconfig):
if not has_docker() or 'not docker' in pytestconfig.getoption('markexpr'):
pytest.skip('skipping docker tests')
with DockerContainer('postgres:alpine') \
.with_bind_ports(5432, PG_PORT) \
.with_env("POSTGRES_USER", PG_USER) \
.with_env("POSTGRES_PASSWORD", PG_PASS) \
.with_env("POSTGRES_DB", PG_DB):
sleep(5) # wait to ensure that PostgreSQL server has enough time to properly start
yield
def __init__(self, project_name: str):
super().__init__(project_name)
self.container = (
DockerContainer(
"gcr.io/google.com/cloudsdktool/cloud-sdk:380.0.0-emulators"
)
.with_command(
"gcloud beta emulators datastore start --project test-project --host-port 0.0.0.0:8081"
)
.with_exposed_ports("8081")
)
def test_container_environments():
code_dir = Path(__file__).parent
container = DockerContainer("nginx:latest")
container.with_env('TEST', 'test')
container.with_env('DOCKER', 'docker')
with container:
output = container.exec("bash -c 'echo $TEST $DOCKER'").output.decode(
'utf-8').strip()
assert output == 'test docker'
class DynamoDBOnlineStoreCreator(OnlineStoreCreator):
def __init__(self, project_name: str):
super().__init__(project_name)
self.container = DockerContainer(
"amazon/dynamodb-local:latest").with_exposed_ports("8000")
def create_online_store(self) -> Dict[str, str]:
self.container.start()
log_string_to_wait_for = (
"Initializing DynamoDB Local with the following configuration:")
wait_for_logs(container=self.container,
predicate=log_string_to_wait_for,
timeout=5)
exposed_port = self.container.get_exposed_port("8000")
return {
"type": "dynamodb",
"endpoint_url": f"http://localhost:{exposed_port}",
"region": "us-west-2",
}
def teardown(self):
self.container.stop()
def set_localstack(self):
self.localstack = DockerContainer('localstack/localstack:{}'
.format(LOCALSTACK_VERSION))\
.with_env('SERVICES', 'kinesis')\
.with_env('KINESIS_PORT', '4568')\
.with_env('USE_SSL', 'true')\
.with_exposed_ports(4568)\
.with_volume_mapping('/var/run/docker.sock', '/var/run/docker.sock', 'rw')
# Repeat if ReadTimeout is raised.
for i in range(4):
try:
self.localstack.start()
break
except Exception as e: # pylint: disable=bare-except
if i == 3:
logging.error('Could not initialize localstack container')
raise e
self.aws_service_endpoint = 'https://{}:{}'.format(
self.localstack.get_container_host_ip(),
self.localstack.get_exposed_port('4568'),
)
def get_singleton(cls):
if not cls.is_running:
cls.container = (
DockerContainer("trinodb/trino:376").with_volume_mapping(
cls.catalog_dir,
"/etc/catalog/").with_exposed_ports("8080"))
cls.container.start()
log_string_to_wait_for = "SERVER STARTED"
wait_for_logs(container=cls.container,
predicate=log_string_to_wait_for,
timeout=30)
cls.is_running = True
return cls.container
def test_add_map_entry():
code_dir = Path(__file__).parent
container = DockerContainer("nginx:latest")
container._with_map_entry('something', 'is', 'arbitrary')
container._with_map_entry('something', 'also', 'arbitrary')
assert container._kwargs['something'] == {
'also': 'arbitrary',
'is': 'arbitrary'
}
def test_update_authorized_keys():
with DockerContainer("quay.io/eliezio/sari-test-bh:v1.1.0"
).with_exposed_ports(SSH_PORT) as server:
ssh_pub_keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEfzjdkO1LKnS/it62jmw9tH4BznlnDCBrzaKguujJ15 "
"*****@*****.**",
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGD13Dbe1QoYrFZqCue1TzGkzDSra9ZHzv8gZy9+vb0Y "
"*****@*****.**",
]
errors = update_authorized_keys(server.get_container_host_ip(),
"admin", "tests/data/admin_id_rsa", "",
"acme", ssh_pub_keys,
int(server.get_exposed_port(SSH_PORT)))
assert not errors
# noinspection PyProtectedMember
_, stat = server._container.get_archive(
"/home/acme/.ssh/authorized_keys2")
assert stat["size"] == sum(map(len,
ssh_pub_keys)) + (len(ssh_pub_keys) - 1)
def registry(tmpdir_factory):
with DockerContainer('registry:latest').with_bind_ports(
REGISTRY_PORT, REGISTRY_PORT):
client = DockerClient()
# migrate our image to custom Docker registry
client.images.pull(IMAGE_NAME, 'latest').tag(TAG_NAME)
client.images.push(TAG_NAME)
tmpdir = str(tmpdir_factory.mktemp("image"))
# create failing image: alpine is too small to have python inside
with open(os.path.join(tmpdir, 'Dockerfile'), 'w') as f:
f.write("""
FROM alpine:latest
CMD python
""")
client.images.build(path=tmpdir, tag=BROKEN_TAG_NAME)
client.images.push(BROKEN_TAG_NAME)
yield RemoteDockerRegistry(REGISTRY_HOST)
def before_all(context):
# Adding label to identify Docker container as testcontainer container (testcontainers-java does this
# by default).
#
# Much of the testcontainers-python library is simply a wrapper around Docker SDK for Python. This is
# a good example of how to make use of the parameters that the Docker SDK makes available, even if
# testcontainers doesn't do so explicitly. When using testcontainers-python, it is therefore advisable
# to be familiar with the Docker SDK (https://docker-py.readthedocs.io/en/stable/containers.html)
# and the testcontainers-python source code (https://github.com/testcontainers/testcontainers-python).
#
context.rabbit = DockerContainer('rabbitmq:3.8.3-management', labels = {"org.testcontainers":"true"}) \
.with_exposed_ports(RABBIT_PORT)
context.rabbit.start()
wait_for_logs(context.rabbit,
r"Server startup complete; \d+ plugins started",
timeout=120)
context.server = SimpleHTTPServer(
HOST, 0, context.rabbit.get_exposed_port(RABBIT_PORT))
context.server.serve()
def test_waits_for_nginx_to_be_ready(self):
container = DockerContainer(
image='nginx:1.19-alpine').with_name('nginx')
container.start()
try:
io = IO()
signal = execute_app(
WaitForOutputApp(
container='ngin(.*)',
command='',
pattern='Configuration complete; ready for start up',
timeout=15,
io=io))
self.assertIn('Match found', signal.message)
self.assertEqual(0, signal.exit_code)
finally:
container.stop()
def test_regexp_patterns(self):
container = DockerContainer(
image='nginx:1.19-alpine').with_name('nginx_0')
container.start()
try:
io = IO()
for pattern in ['ngin*', 'nginx.*', 'ng.*x']:
signal = execute_app(
WaitForOutputApp(
container=pattern,
command='',
pattern='Configuration complete; ready for start up',
timeout=15,
io=io))
self.assertIn('Match found', signal.message)
self.assertEqual(0, signal.exit_code)
finally:
container.stop()
def test_too_many_containers_found(self):
first = DockerContainer(image='nginx:1.19-alpine').with_name('nginx_0')
second = DockerContainer(
image='nginx:1.19-alpine').with_name('nginx_1')
first.start()
second.start()
io = IO()
try:
signal = execute_app(
WaitForOutputApp(
container='nginx_*',
command='',
pattern='Configuration complete; ready for start up',
timeout=15,
io=io))
self.assertEqual('Too many containers found', signal.message)
self.assertEqual(1, signal.exit_code)
finally:
first.stop()
second.stop()
def __init__(self, project_name: str):
super().__init__(project_name)
self.container = DockerContainer("redis").with_exposed_ports("6379")
class SpannerHelper(object):
def __init__(self, project_id, instance_id, table, use_emulator):
self.use_emulator = use_emulator
self.table = table
self.host = None
if use_emulator:
self.emulator = DockerContainer(
'gcr.io/cloud-spanner-emulator/emulator:latest').with_exposed_ports(
9010, 9020)
retry(self.emulator.start, 3, 'Could not start spanner emulator.')
time.sleep(3)
self.host = f'{self.emulator.get_container_host_ip()}:' \
f'{self.emulator.get_exposed_port(9010)}'
os.environ['SPANNER_EMULATOR_HOST'] = self.host
self.client = spanner.Client(project_id)
self.instance = self.client.instance(instance_id)
if use_emulator:
self.create_instance()
def create_instance(self):
self.instance.create().result(120)
def create_database(self, database_id):
database = self.instance.database(
database_id,
ddl_statements=[
f'''
CREATE TABLE {self.table} (
f_string STRING(1024) NOT NULL,
f_int64 INT64,
f_boolean BOOL
) PRIMARY KEY (f_string)'''
])
database.create().result(120)
def insert_values(self, database_id, values, columns=None):
values = values or []
columns = columns or ('f_string', 'f_int64', 'f_boolean')
with self.instance.database(database_id).batch() as batch:
batch.insert(
table=self.table,
columns=columns,
values=values,
)
def get_emulator_host(self):
return f'http://{self.host}'
def read_data(self, database_id, prefix):
database = self.instance.database(database_id)
with database.snapshot() as snapshot:
results = snapshot.execute_sql(
f'''SELECT * FROM {self.table}
WHERE f_string LIKE "{prefix}%"
ORDER BY f_int64''')
try:
rows = list(results) if results else None
except IndexError:
raise ValueError(f"Spanner results not found for {prefix}.")
return rows
def drop_database(self, database_id):
database = self.instance.database(database_id)
database.drop()
def shutdown(self):
if self.use_emulator:
try:
self.emulator.stop()
except: # pylint: disable=bare-except
logging.error('Could not stop Spanner Cloud emulator.')
class CrossLanguageKinesisIOTest(unittest.TestCase):
@unittest.skipUnless(
TestPipeline().get_option('aws_kinesis_stream'),
'Cannot test on real aws without pipeline options provided')
def test_kinesis_io_roundtrip(self):
# TODO: enable this test for localstack once BEAM-10664 is resolved
self.run_kinesis_write()
self.run_kinesis_read()
@unittest.skipIf(
TestPipeline().get_option('aws_kinesis_stream'),
'Do not test on localstack when pipeline options were provided')
def test_kinesis_write(self):
# TODO: remove this test once BEAM-10664 is resolved
self.run_kinesis_write()
records = self.kinesis_helper.read_from_stream(self.aws_kinesis_stream)
self.assertEqual(
sorted(records),
sorted([RECORD + str(i).encode() for i in range(NUM_RECORDS)]))
def run_kinesis_write(self):
with TestPipeline(options=PipelineOptions(self.pipeline_args)) as p:
p.not_use_test_runner_api = True
_ = (
p
| 'Impulse' >> beam.Impulse()
| 'Generate' >> beam.FlatMap(lambda x: range(NUM_RECORDS)) # pylint: disable=bad-option-value
| 'Map to bytes' >> beam.Map(lambda x: RECORD + str(x).encode(
)).with_output_types(bytes)
| 'WriteToKinesis' >> WriteToKinesis(
stream_name=self.aws_kinesis_stream,
aws_access_key=self.aws_access_key,
aws_secret_key=self.aws_secret_key,
region=self.aws_region,
service_endpoint=self.aws_service_endpoint,
verify_certificate=(not self.use_localstack),
partition_key='1',
producer_properties=self.producer_properties,
))
def run_kinesis_read(self):
records = [RECORD + str(i).encode() for i in range(NUM_RECORDS)]
with TestPipeline(options=PipelineOptions(self.pipeline_args)) as p:
result = (p
| 'ReadFromKinesis' >> ReadDataFromKinesis(
stream_name=self.aws_kinesis_stream,
aws_access_key=self.aws_access_key,
aws_secret_key=self.aws_secret_key,
region=self.aws_region,
service_endpoint=self.aws_service_endpoint,
verify_certificate=not self.use_localstack,
max_num_records=NUM_RECORDS,
max_read_time=MAX_READ_TIME,
request_records_limit=REQUEST_RECORDS_LIMIT,
watermark_policy=WatermarkPolicy.ARRIVAL_TIME,
watermark_idle_duration_threshold=MAX_READ_TIME,
initial_position_in_stream=InitialPositionInStream.
AT_TIMESTAMP,
initial_timestamp_in_stream=NOW_MILLIS,
).with_output_types(bytes))
assert_that(result, equal_to(records))
def set_localstack(self):
self.localstack = DockerContainer('localstack/localstack:{}'
.format(LOCALSTACK_VERSION))\
.with_env('SERVICES', 'kinesis')\
.with_env('KINESIS_PORT', '4568')\
.with_env('USE_SSL', 'true')\
.with_exposed_ports(4568)\
.with_volume_mapping('/var/run/docker.sock', '/var/run/docker.sock', 'rw')
# Repeat if ReadTimeout is raised.
for i in range(4):
try:
self.localstack.start()
break
except Exception as e: # pylint: disable=bare-except
if i == 3:
logging.error('Could not initialize localstack container')
raise e
self.aws_service_endpoint = 'https://{}:{}'.format(
self.localstack.get_container_host_ip(),
self.localstack.get_exposed_port('4568'),
)
def setUp(self):
parser = argparse.ArgumentParser()
parser.add_argument(
'--aws_kinesis_stream',
default='beam_kinesis_xlang',
help='Kinesis stream name',
)
parser.add_argument(
'--aws_access_key',
default='accesskey',
help=('Aws access key'),
)
parser.add_argument(
'--aws_secret_key',
default='secretkey',
help='Aws secret key',
)
parser.add_argument(
'--aws_region',
default='us-east-1',
help='Aws region',
)
parser.add_argument(
'--aws_service_endpoint',
default=None,
help='Url to external aws endpoint',
)
parser.add_argument(
'--use_real_aws',
default=False,
dest='use_real_aws',
action='store_true',
help='Flag whether to use real aws for the tests purpose',
)
parser.add_argument(
'--expansion_service',
help='Url to externally launched expansion service.',
)
pipeline = TestPipeline()
argv = pipeline.get_full_options_as_args()
known_args, self.pipeline_args = parser.parse_known_args(argv)
self.aws_kinesis_stream = known_args.aws_kinesis_stream
self.aws_access_key = known_args.aws_access_key
self.aws_secret_key = known_args.aws_secret_key
self.aws_region = known_args.aws_region
self.aws_service_endpoint = known_args.aws_service_endpoint
self.use_localstack = not known_args.use_real_aws
self.expansion_service = known_args.expansion_service
self.producer_properties = {
'CollectionMaxCount': str(NUM_RECORDS),
'ConnectTimeout': str(MAX_READ_TIME),
}
if self.use_localstack:
self.set_localstack()
self.kinesis_helper = KinesisHelper(
self.aws_access_key,
self.aws_secret_key,
self.aws_region,
self.aws_service_endpoint.replace('https', 'http')
if self.aws_service_endpoint else None,
)
if self.use_localstack:
self.kinesis_helper.create_stream(self.aws_kinesis_stream)
def tearDown(self):
if self.use_localstack:
self.kinesis_helper.delete_stream(self.aws_kinesis_stream)
try:
self.localstack.stop()
except: # pylint: disable=bare-except
logging.error('Could not stop the localstack container')