def _get_warmup_conditions():
if MYSQL_REPLICATION == 'group':
return [
CheckDockerLogs(
'node1',
"X Plugin ready for connections. Bind-address: '::' port: 33060"
),
CheckDockerLogs(
'node2',
"X Plugin ready for connections. Bind-address: '::' port: 33060"
),
CheckDockerLogs(
'node3',
"X Plugin ready for connections. Bind-address: '::' port: 33060"
),
init_group_replication,
populate_database,
]
return [
WaitFor(init_master, wait=2),
WaitFor(init_slave, wait=2),
CheckDockerLogs(
'mysql-slave',
["ready for connections", "mariadb successfully initialized"]),
populate_database,
]
def dd_environment():
if pyodbc is None:
raise Exception("pyodbc is not installed!")
def sqlserver_can_connect():
conn = 'DRIVER={};Server={};Database=master;UID=sa;PWD=Password123;'.format(get_local_driver(), DOCKER_SERVER)
pyodbc.connect(conn, timeout=30)
compose_file = os.path.join(HERE, os.environ["COMPOSE_FOLDER"], 'docker-compose.yaml')
conditions = [
WaitFor(sqlserver_can_connect, wait=3, attempts=10),
]
if os.environ["COMPOSE_FOLDER"] == 'compose-ha':
conditions += [
CheckDockerLogs(
compose_file,
'Always On Availability Groups connection with primary database established for secondary database',
)
]
else:
conditions += [
CheckDockerLogs(
compose_file,
'setup.sql completed',
)
]
with docker_run(
compose_file=compose_file,
conditions=conditions,
mount_logs=True,
):
yield FULL_E2E_CONFIG
def dd_environment():
compose_file = os.path.join(HERE, 'compose', 'docker-compose.yaml')
with docker_run(
compose_file,
conditions=[
# Kafka Broker
CheckDockerLogs('broker',
'Created log for partition _confluent'),
# Kafka Schema Registry
CheckDockerLogs('schema-registry',
'Server started, listening for requests...',
attempts=90),
# Kafka Connect
CheckDockerLogs('connect',
'Kafka Connect started',
attempts=120),
# Create connectors
WaitFor(create_connectors),
CheckDockerLogs(
'connect',
'flushing 0 outstanding messages for offset commit'),
],
attempts=2,
):
yield CHECK_CONFIG, {'use_jmx': True}
def dd_environment():
"""
Start a cluster with one master, one replica and one unhealthy replica and
stop it after the tests are done.
If there's any problem executing `docker compose`, let the exception bubble
up.
"""
couch_version = os.environ["COUCH_VERSION"][0]
if couch_version == "1":
with docker_run(
compose_file=os.path.join(common.HERE, 'compose', 'compose_v1.yaml'),
env_vars={'COUCH_PORT': common.PORT},
conditions=[
CheckEndpoints([common.URL]),
CheckDockerLogs('couchdb-1', ['CouchDB has started', 'Application couch_index started']),
WaitFor(generate_data, args=(couch_version,)),
],
):
yield common.BASIC_CONFIG
else:
with docker_run(
compose_file=os.path.join(common.HERE, 'compose', 'compose_v2.yaml'),
env_vars={'COUCH_PORT': common.PORT, 'COUCH_USER': common.USER, 'COUCH_PASSWORD': common.PASSWORD},
conditions=[
CheckEndpoints([common.URL]),
CheckDockerLogs('couchdb-1', ['Started replicator db changes listener']),
WaitFor(enable_cluster),
WaitFor(generate_data, args=(couch_version,)),
WaitFor(check_node_stats),
WaitFor(send_replication),
WaitFor(get_replication),
],
):
yield common.BASIC_CONFIG_V2
def dd_environment():
instance = {
'server': '{}:{}'.format(HOST, PORT),
'user': USER,
'password': PASSWORD,
'service_name': 'InfraDB.us.oracle.com',
}
if CLIENT_LIB == 'jdbc':
e2e_metadata = E2E_METADATA_JDBC_CLIENT
instance[
'jdbc_driver_path'] = '/opt/oracle/instantclient_19_3/ojdbc8.jar'
else:
e2e_metadata = E2E_METADATA_ORACLE_CLIENT
with docker_run(
COMPOSE_FILE,
conditions=[
CheckDockerLogs(COMPOSE_FILE,
['The database is ready for use'],
wait=5,
attempts=120),
WaitFor(create_user),
],
env_vars={'ORACLE_DATABASE_VERSION': ORACLE_DATABASE_VERSION},
attempts=20,
attempts_wait=5,
):
yield instance, e2e_metadata
def dd_environment(config_e2e):
logs_path = _mysql_logs_path()
with TempDir('logs') as logs_host_path:
e2e_metadata = {
'docker_volumes': ['{}:{}'.format(logs_host_path, logs_path)]
}
with docker_run(
os.path.join(common.HERE, 'compose', COMPOSE_FILE),
env_vars={
'MYSQL_DOCKER_REPO': _mysql_docker_repo(),
'MYSQL_PORT': str(common.PORT),
'MYSQL_SLAVE_PORT': str(common.SLAVE_PORT),
'MYSQL_CONF_PATH': _mysql_conf_path(),
'MYSQL_LOGS_HOST_PATH': logs_host_path,
'MYSQL_LOGS_PATH': logs_path,
'WAIT_FOR_IT_SCRIPT_PATH': _wait_for_it_script(),
},
conditions=[
WaitFor(init_master, wait=2),
WaitFor(init_slave, wait=2),
CheckDockerLogs('mysql-slave', [
"ready for connections",
"mariadb successfully initialized"
]),
populate_database,
],
):
yield config_e2e, e2e_metadata
def dd_environment():
with kind_run(conditions=[setup_linkerd_cluster]) as kubeconfig:
compose_file = os.path.join(HERE, "compose", "docker-compose.yaml")
with docker_run(
compose_file=compose_file,
conditions=[
CheckDockerLogs(compose_file,
'LINKERD DEPLOY COMPLETE',
wait=5,
attempts=120)
],
):
with ExitStack() as stack:
ip, port = stack.enter_context(
port_forward(kubeconfig, 'linkerd', 4191, 'deployment',
'linkerd-controller'))
instance = {
'prometheus_url':
'http://{ip}:{port}/metrics'.format(ip=ip, port=port),
'metrics': [LINKERD_FIXTURE_METRICS],
'type_overrides':
LINKERD_FIXTURE_TYPES,
}
yield instance
def dd_environment(instance):
# type: (Instance) -> Iterator
compose_file = os.path.join(common.HERE, 'compose', 'docker-compose.yaml')
schema_file = os.path.join(common.HERE, 'compose', 'schema.sql')
with open(schema_file) as f:
schema = f.read()
conditions = [
CheckDockerLogs(compose_file,
patterns=['Server completed initialization']),
CreateSchema(compose_file, schema, container_name='voltdb0'),
EnsureExpectedMetricsShowUp(instance),
]
env_vars = {
'VOLTDB_IMAGE': common.VOLTDB_IMAGE,
'VOLTDB_DEPLOYMENT': common.VOLTDB_DEPLOYMENT,
'VOLTDB_CLIENT_PORT': str(common.VOLTDB_CLIENT_PORT),
}
if common.TLS_ENABLED:
# Must refer to a path within the Agent container.
instance = instance.copy()
instance['tls_cert'] = '/tmp/voltdb-certs/client.pem'
instance['tls_ca_cert'] = '/tmp/voltdb-certs/ca.pem'
e2e_metadata = {
'docker_volumes':
['{}:/tmp/voltdb-certs'.format(common.TLS_CERTS_DIR)]
}
else:
e2e_metadata = {}
with docker_run(compose_file, conditions=conditions, env_vars=env_vars):
yield instance, e2e_metadata
def dd_environment():
compose_file = os.path.join(common.HERE, 'docker', 'docker-compose.yaml')
with docker_run(
compose_file,
build=True,
mount_logs=True,
conditions=[
CheckDockerLogs(
compose_file,
[
# Management Center
'Hazelcast Management Center successfully started',
r'Members \[2',
# Members connected to each other
r'Members \{size:2',
],
matches='all',
attempts=120,
)
],
):
config = load_jmx_config()
config['instances'] = [
common.INSTANCE_MEMBER_JMX, common.INSTANCE_MC_JMX,
common.INSTANCE_MC_PYTHON
]
yield config, {'use_jmx': True}
def test_no_matches(self):
compose_file = os.path.join(DOCKER_DIR, 'test_default.yaml')
run_command(['docker-compose', '-f', compose_file, 'down'])
check_docker_logs = CheckDockerLogs(compose_file, 'Vault server started', attempts=1)
with pytest.raises(RetryError):
check_docker_logs()
def dd_environment():
"""
Spin up and initialize gitlab_runner
"""
# specify couchbase container name
env = {
'GITLAB_TEST_TOKEN': GITLAB_TEST_TOKEN,
'GITLAB_LOCAL_MASTER_PORT': str(GITLAB_LOCAL_MASTER_PORT),
'GITLAB_LOCAL_RUNNER_PORT': str(GITLAB_LOCAL_RUNNER_PORT),
}
compose_file = os.path.join(HERE, 'compose', 'docker-compose.yml')
with docker_run(
compose_file=compose_file,
env_vars=env,
conditions=[
CheckDockerLogs(compose_file, [
'Gitlab is up!', 'Configuration loaded',
'Metrics server listening'
],
wait=5),
CheckEndpoints(GITLAB_RUNNER_URL, attempts=180),
],
):
yield CONFIG, E2E_METADATA
def dd_environment():
if common.MQ_VERSION == 9:
log_pattern = "AMQ5026I: The listener 'DEV.LISTENER.TCP' has started. ProcessId"
elif common.MQ_VERSION == 8:
log_pattern = r".*QMNAME\({}\)\s*STATUS\(Running\).*".format(
common.QUEUE_MANAGER)
else:
raise RuntimeError('Invalid version: {}'.format(common.MQ_VERSION))
e2e_meta = copy.deepcopy(common.E2E_METADATA)
e2e_meta.setdefault('docker_volumes', [])
e2e_meta['docker_volumes'].append("{}:/opt/pki/keys".format(
os.path.join(common.HERE, 'keys')))
conditions = [CheckDockerLogs('ibm_mq1', log_pattern)]
if not ON_WINDOWS:
conditions.append(WaitFor(prepare_queue_manager))
with docker_run(compose_file=common.COMPOSE_FILE_PATH,
build=True,
conditions=conditions,
sleep=10,
attempts=2):
yield common.INSTANCE, e2e_meta
def dd_environment():
compose_file = os.path.join(HERE, 'compose', 'docker-compose.yaml')
# We need a custom condition to wait a bit longer
with docker_run(
compose_file=compose_file,
conditions=[
CheckDockerLogs(compose_file, 'spawning ceph --cluster ceph -w', wait=5),
CheckDockerLogs(compose_file, 'Running on http://0.0.0.0:5000/'),
],
):
# Clean the disk space warning
run_command(
['docker', 'exec', 'dd-test-ceph', 'ceph', 'tell', 'mon.*', 'injectargs', '--mon_data_avail_warn', '5']
)
# Wait a bit for the change to take effect
time.sleep(5)
yield BASIC_CONFIG, E2E_METADATA
def test_matches(self):
compose_file = os.path.join(DOCKER_DIR, 'test_default.yaml')
check_docker_logs = CheckDockerLogs(compose_file, 'Vault server started')
try:
run_command(['docker-compose', '-f', compose_file, 'up', '-d'], check=True)
check_docker_logs()
finally:
run_command(['docker-compose', '-f', compose_file, 'down'], capture=True)
def dd_environment(instance):
compose_file = get_docker_compose_file()
conditions = [
CheckDockerLogs(compose_file, "http server Running on", wait=3),
lambda: time.sleep(2),
CreateSimpleUser(),
]
with docker_run(compose_file, conditions=conditions):
yield instance
def dd_environment(instance):
compose_file = os.path.join(get_here(), 'docker', 'docker-compose.yaml')
with docker_run(compose_file,
conditions=[
WaitFor(make_query),
CheckDockerLogs(compose_file, 'SERVER STARTED')
]):
yield instance, {'use_jmx': True}
def dd_environment():
compose_file = os.path.join(HERE, 'compose', 'docker-compose.yaml')
with docker_run(
compose_file,
conditions=[
# Kafka Broker
CheckDockerLogs('broker', 'Monitored service is now ready'),
# Kafka Schema Registry
CheckDockerLogs('schema-registry',
'Server started, listening for requests...',
attempts=90),
# Kafka Connect
CheckDockerLogs('connect',
'Kafka Connect started',
attempts=120),
],
):
yield CHECK_CONFIG, {'use_jmx': True}
def dd_environment():
compose_file = os.path.join(HERE, 'docker', 'docker-compose.yml')
with docker_run(compose_file,
conditions=[
CheckDockerLogs(compose_file,
'Resin/4.0.62 started -server')
]):
instance = load_jmx_config()
yield instance, {'use_jmx': True}
def dd_environment():
compose_file = os.path.join(HERE, 'docker', 'docker-compose.yml')
log_patterns = 'Fullsync complete from'
with docker_run(
compose_file=compose_file,
conditions=[CheckEndpoints(URL, wait=5), CheckDockerLogs(compose_file, log_patterns, matches='all', wait=5)],
):
yield
def dd_environment():
compose_file = os.path.join(common.HERE, 'docker', 'docker-compose.yaml')
with docker_run(
compose_file,
build=True,
mount_logs=True,
conditions=[
CheckDockerLogs('hazelcast_management_center', ['Hazelcast Management Center successfully started']),
CheckDockerLogs('hazelcast_management_center', ['Started communication with member']),
CheckDockerLogs('hazelcast2', [r'Hazelcast JMX agent enabled']),
CheckDockerLogs('hazelcast2', [r'is STARTED']),
WaitFor(trigger_some_tcp_data),
],
attempts=5,
attempts_wait=5,
):
config = load_jmx_config()
config['instances'] = common.INSTANCE_MEMBERS + [common.INSTANCE_MC_JMX, common.INSTANCE_MC_PYTHON]
yield config, {'use_jmx': True}
def dd_environment():
compose_file = os.path.join(HERE, 'compose', 'docker-compose.yml')
with docker_run(compose_file,
conditions=[
CheckDockerLogs(compose_file,
[r'\[KafkaServer id=\d+\] started'],
matches="all")
]):
yield load_jmx_config(), {'use_jmx': True}
def dd_environment():
compose_file = os.path.join(HERE, 'docker', 'docker-compose.yaml')
with docker_run(
compose_file=compose_file,
conditions=[
CheckDockerLogs('spark-app-hudi',
'finished: show at script.scala:163')
],
):
yield CHECK_CONFIG, {'use_jmx': True}
def dd_environment(get_instance):
def condition_non_tls():
sys.stderr.write("Waiting for ZK to boot...\n")
booted = False
dummy_instance = {'host': HOST, 'port': PORT, 'timeout': 500}
for _ in range(10):
try:
out = ZookeeperCheck('zk', {},
[dummy_instance])._send_command('ruok')
out.seek(0)
if out.readline() != 'imok':
raise ZKConnectionFailure()
booted = True
break
except ZKConnectionFailure:
time.sleep(1)
if not booted:
raise RetryError("Zookeeper failed to boot!")
sys.stderr.write("ZK boot complete.\n")
is_tls = get_tls()
compose_file = os.path.join(HERE, 'compose', 'zk.yaml')
if [3, 5, 0] <= get_version() < [3, 6, 0]:
compose_file = os.path.join(HERE, 'compose', 'zk35.yaml')
if is_tls:
compose_file = os.path.join(HERE, 'compose', 'zk35_ssl.yaml')
elif get_version() >= [3, 6, 0]:
compose_file = os.path.join(HERE, 'compose', 'zk36plus.yaml')
if is_tls:
compose_file = os.path.join(HERE, 'compose', 'zk36plus_ssl.yaml')
private_key = os.path.join(HERE, 'compose/client', 'private_key.pem')
cert = os.path.join(HERE, 'compose/client', 'cert.pem')
ca_cert = os.path.join(HERE, 'compose/client', 'ca_cert.pem')
if is_tls:
condition = [
CheckDockerLogs(compose_file,
patterns=[
'Starting server', 'Started AdminServer',
'bound to port'
])
]
else:
condition = [condition_non_tls]
with docker_run(compose_file, conditions=condition):
yield get_instance, {
'docker_volumes': [
'{}:/conf/private_key.pem'.format(private_key),
'{}:/conf/cert.pem'.format(cert),
'{}:/conf/ca_cert.pem'.format(ca_cert),
]
}
def dd_environment():
compose_file = os.path.join(HERE, 'compose', 'docker-compose.yml')
with docker_run(
compose_file=compose_file,
sleep=120,
conditions=[
CheckDockerLogs(compose_file,
'php-fpm7.4 entered RUNNING state'),
],
):
yield CONFIG
def dd_environment():
compose_file = os.path.join(common.HERE, 'docker', 'docker-compose.yaml')
with docker_run(compose_file,
mount_logs=True,
conditions=[
CheckDockerLogs(compose_file, ['Started HiveMQ in'],
matches='all')
]):
config = load_jmx_config()
config['instances'] = [common.INSTANCE]
yield config, {'use_jmx': True}
def dd_environment(instance):
compose_file = os.path.join(get_here(), "docker", "docker-compose.yaml")
with docker_run(
compose_file,
conditions=[
WaitFor(make_query),
CheckDockerLogs(compose_file, "SERVER STARTED"),
],
):
yield instance, {"use_jmx": True}
def dd_environment():
compose_file = os.path.join(common.HERE, "compose", "docker-compose.yaml")
with docker_run(
compose_file=compose_file,
conditions=[
CheckDockerLogs("dd-test-riakcs",
"INFO success: riak-cs entered RUNNING state",
attempts=240)
],
):
yield common.generate_config_with_creds()
def dd_environment(integration_instance):
compose_file = os.path.join(HERE, 'compose', 'docker-compose.yaml')
# We need a custom condition to wait a bit longer
with docker_run(
compose_file=compose_file,
build=True,
conditions=[
CheckDockerLogs(compose_file, 'Running on ', wait=5),
],
):
yield integration_instance
def dd_environment():
compose_file = os.path.join(common.HERE, 'docker', 'docker-compose.yaml')
with docker_run(
compose_file,
conditions=[
CheckDockerLogs(compose_file, 'Server server1 open for e-business', attempts=120),
StartPerfServlet(),
CheckEndpoints(common.INSTANCE['servlet_url']),
],
):
yield common.INSTANCE
def dd_environment(e2e_instance):
compose_file = get_docker_compose_file()
expected_log = "http server Running on" if HARBOR_VERSION < [
1, 10, 0
] else "API server is serving at"
conditions = [
CheckDockerLogs(compose_file, expected_log, wait=3),
lambda: time.sleep(4),
CreateSimpleUser(),
]
with docker_run(compose_file, conditions=conditions, attempts=2):
yield e2e_instance
