def test_docker_custom_image():
container = DockerContainer("mysql:5.7.17")
container.with_exposed_ports(3306)
container.with_env("MYSQL_ROOT_PASSWORD", "root")
with container:
port = container.get_exposed_port(3306)
assert int(port) > 0
def wait_for_http_code(container: DockerContainer, status_code: int, port: int = 80, path: str = '/',
scheme: str = 'http', timeout=None, interval=1, request_kwargs: dict = None):
"""
Wait for a specific http status code.
Parameters
----------
container : DockerContainer
Container which is queried for a specific status code.
status_code : int
Status code to wait for.
port : int
Port to query request on.
path : str
Path to use for request. Default is '/'
scheme : str
Scheme to use in request query. Default is 'http'
timeout : float or None
Number of seconds to wait for the port to be open. Defaults to wait indefinitely.
interval : float
Interval at which to poll the port.
request_kwargs: dict
kwargs to pass into the request, e.g.: {'verify': False}
Returns
-------
duration : float
Number of seconds until the check passed.
"""
if request_kwargs is None:
request_kwargs = {'timeout': 1.0}
elif 'timeout' not in request_kwargs:
request_kwargs['timeout'] = 1.0
start = time.time()
# wait for port to open before continuing with http check
wait_for_port(container, port, timeout, interval)
while True:
duration = time.time() - start
dest = "%s://%s:%d%s" % (scheme,
container.get_container_host_ip(),
int(container.get_exposed_port(port)),
path)
res = None
try:
res = requests.get(dest, **request_kwargs)
except RequestException:
pass
if res and res.status_code == status_code:
return duration
if timeout and duration > timeout:
raise TimeoutError("container did not respond with %d listening on port %d in %.3f seconds"
% (status_code, port, timeout))
class RedisOnlineStoreCreator(OnlineStoreCreator):
def __init__(self, project_name: str):
super().__init__(project_name)
self.container = DockerContainer("redis").with_exposed_ports("6379")
def create_online_store(self) -> Dict[str, str]:
self.container.start()
log_string_to_wait_for = "Ready to accept connections"
wait_for_logs(
container=self.container, predicate=log_string_to_wait_for, timeout=5
)
exposed_port = self.container.get_exposed_port("6379")
return {"type": "redis", "connection_string": f"localhost:{exposed_port},db=0"}
def teardown(self):
self.container.stop()
class DynamoDBOnlineStoreCreator(OnlineStoreCreator):
def __init__(self, project_name: str):
super().__init__(project_name)
self.container = DockerContainer(
"amazon/dynamodb-local:latest").with_exposed_ports("8000")
def create_online_store(self) -> Dict[str, str]:
self.container.start()
log_string_to_wait_for = (
"Initializing DynamoDB Local with the following configuration:")
wait_for_logs(container=self.container,
predicate=log_string_to_wait_for,
timeout=5)
exposed_port = self.container.get_exposed_port("8000")
return {
"type": "dynamodb",
"endpoint_url": f"http://localhost:{exposed_port}",
"region": "us-west-2",
}
def teardown(self):
self.container.stop()
class CrossLanguageKinesisIOTest(unittest.TestCase):
@unittest.skipUnless(
TestPipeline().get_option('aws_kinesis_stream'),
'Cannot test on real aws without pipeline options provided')
def test_kinesis_io_roundtrip(self):
# TODO: enable this test for localstack once BEAM-10664 is resolved
self.run_kinesis_write()
self.run_kinesis_read()
@unittest.skipIf(
TestPipeline().get_option('aws_kinesis_stream'),
'Do not test on localstack when pipeline options were provided')
def test_kinesis_write(self):
# TODO: remove this test once BEAM-10664 is resolved
self.run_kinesis_write()
records = self.kinesis_helper.read_from_stream(self.aws_kinesis_stream)
self.assertEqual(
sorted(records),
sorted([RECORD + str(i).encode() for i in range(NUM_RECORDS)]))
def run_kinesis_write(self):
with TestPipeline(options=PipelineOptions(self.pipeline_args)) as p:
p.not_use_test_runner_api = True
_ = (
p
| 'Impulse' >> beam.Impulse()
| 'Generate' >> beam.FlatMap(lambda x: range(NUM_RECORDS)) # pylint: disable=bad-option-value
| 'Map to bytes' >> beam.Map(lambda x: RECORD + str(x).encode(
)).with_output_types(bytes)
| 'WriteToKinesis' >> WriteToKinesis(
stream_name=self.aws_kinesis_stream,
aws_access_key=self.aws_access_key,
aws_secret_key=self.aws_secret_key,
region=self.aws_region,
service_endpoint=self.aws_service_endpoint,
verify_certificate=(not self.use_localstack),
partition_key='1',
producer_properties=self.producer_properties,
))
def run_kinesis_read(self):
records = [RECORD + str(i).encode() for i in range(NUM_RECORDS)]
with TestPipeline(options=PipelineOptions(self.pipeline_args)) as p:
result = (p
| 'ReadFromKinesis' >> ReadDataFromKinesis(
stream_name=self.aws_kinesis_stream,
aws_access_key=self.aws_access_key,
aws_secret_key=self.aws_secret_key,
region=self.aws_region,
service_endpoint=self.aws_service_endpoint,
verify_certificate=not self.use_localstack,
max_num_records=NUM_RECORDS,
max_read_time=MAX_READ_TIME,
request_records_limit=REQUEST_RECORDS_LIMIT,
watermark_policy=WatermarkPolicy.ARRIVAL_TIME,
watermark_idle_duration_threshold=MAX_READ_TIME,
initial_position_in_stream=InitialPositionInStream.
AT_TIMESTAMP,
initial_timestamp_in_stream=NOW_MILLIS,
).with_output_types(bytes))
assert_that(result, equal_to(records))
def set_localstack(self):
self.localstack = DockerContainer('localstack/localstack:{}'
.format(LOCALSTACK_VERSION))\
.with_env('SERVICES', 'kinesis')\
.with_env('KINESIS_PORT', '4568')\
.with_env('USE_SSL', 'true')\
.with_exposed_ports(4568)\
.with_volume_mapping('/var/run/docker.sock', '/var/run/docker.sock', 'rw')
# Repeat if ReadTimeout is raised.
for i in range(4):
try:
self.localstack.start()
break
except Exception as e: # pylint: disable=bare-except
if i == 3:
logging.error('Could not initialize localstack container')
raise e
self.aws_service_endpoint = 'https://{}:{}'.format(
self.localstack.get_container_host_ip(),
self.localstack.get_exposed_port('4568'),
)
def setUp(self):
parser = argparse.ArgumentParser()
parser.add_argument(
'--aws_kinesis_stream',
default='beam_kinesis_xlang',
help='Kinesis stream name',
)
parser.add_argument(
'--aws_access_key',
default='accesskey',
help=('Aws access key'),
)
parser.add_argument(
'--aws_secret_key',
default='secretkey',
help='Aws secret key',
)
parser.add_argument(
'--aws_region',
default='us-east-1',
help='Aws region',
)
parser.add_argument(
'--aws_service_endpoint',
default=None,
help='Url to external aws endpoint',
)
parser.add_argument(
'--use_real_aws',
default=False,
dest='use_real_aws',
action='store_true',
help='Flag whether to use real aws for the tests purpose',
)
parser.add_argument(
'--expansion_service',
help='Url to externally launched expansion service.',
)
pipeline = TestPipeline()
argv = pipeline.get_full_options_as_args()
known_args, self.pipeline_args = parser.parse_known_args(argv)
self.aws_kinesis_stream = known_args.aws_kinesis_stream
self.aws_access_key = known_args.aws_access_key
self.aws_secret_key = known_args.aws_secret_key
self.aws_region = known_args.aws_region
self.aws_service_endpoint = known_args.aws_service_endpoint
self.use_localstack = not known_args.use_real_aws
self.expansion_service = known_args.expansion_service
self.producer_properties = {
'CollectionMaxCount': str(NUM_RECORDS),
'ConnectTimeout': str(MAX_READ_TIME),
}
if self.use_localstack:
self.set_localstack()
self.kinesis_helper = KinesisHelper(
self.aws_access_key,
self.aws_secret_key,
self.aws_region,
self.aws_service_endpoint.replace('https', 'http')
if self.aws_service_endpoint else None,
)
if self.use_localstack:
self.kinesis_helper.create_stream(self.aws_kinesis_stream)
def tearDown(self):
if self.use_localstack:
self.kinesis_helper.delete_stream(self.aws_kinesis_stream)
try:
self.localstack.stop()
except: # pylint: disable=bare-except
logging.error('Could not stop the localstack container')