def dd_environment():
"""
Start a kong cluster
"""
with docker_run(
compose_file=os.path.join(common.HERE, 'compose', 'docker-compose.yml'),
endpoints=common.STATUS_URL
):
yield common.instance_1
def dd_environment(instance):
compose_file = get_docker_compose_file()
conditions = [
CheckDockerLogs(compose_file, "http server Running on", wait=3),
lambda: time.sleep(2),
CreateSimpleUser(),
]
with docker_run(compose_file, conditions=conditions):
yield instance
def dd_environment():
cacert_path = os.path.join(HERE, 'fixtures', 'cacert.pem')
e2e_metadata = {
'docker_volumes': ['{}:/opt/cacert.pem'.format(cacert_path)]
}
with docker_run(os.path.join(HERE, 'compose', 'docker-compose.yml'),
build=True,
log_patterns=["starting server on port"]):
yield CONFIG_E2E, e2e_metadata
def dd_environment():
if os.environ['HAPROXY_LEGACY'] == 'true':
with legacy_environment() as e:
yield e
else:
with docker_run(compose_file=os.path.join(HERE, 'docker',
'haproxy.yaml'),
endpoints=[ENDPOINT_PROMETHEUS]):
yield INSTANCE
def dd_environment():
with docker_run(
common.COMPOSE_FILE,
endpoints=[
'http://{}:{}/metrics'.format(common.HOST, common.JMX_PORT),
'http://{}:{}/metrics'.format(common.HOST, common.NODE_PORT),
],
):
yield common.INSTANCE, common.E2E_METADATA
def dd_environment():
with docker_run(
os.path.join(HERE, 'compose', 'docker-compose.yaml'),
log_patterns=['ActiveMQ Jolokia REST API available'],
conditions=[
WaitForPortListening(HOST, TEST_PORT), populate_server
],
):
yield load_jmx_config(), {'use_jmx': True}
def dd_environment(e2e_instance):
"""
Start a standalone postgres server requiring authentication.
"""
with docker_run(
os.path.join(HERE, 'compose', 'docker-compose.yaml'),
conditions=[WaitFor(connect_to_pg)],
):
yield e2e_instance
def dd_environment(instance):
compose_file = os.path.join(common.HERE, 'compose', 'docker-compose.yml')
with docker_run(
compose_file,
service_name="mesos-master",
log_patterns=['A new leading master'],
):
yield instance
def dd_environment():
env = {'RIAK_CONFIG': os.path.join(common.HERE, 'config')}
with docker_run(
compose_file=os.path.join(common.HERE, 'compose', 'riak.yaml'),
env_vars=env,
conditions=[CheckEndpoints(['{}/riak/bucket'.format(common.BASE_URL)]), populate],
sleep=10, # some stats require a bit of time before the test will capture them
):
yield common.INSTANCE
def dd_environment(instance):
with docker_run(
os.path.join(HERE, 'compose', 'docker-compose.yaml'),
conditions=[
CheckEndpoints(COORDINATOR_URL + '/status', attempts=100),
CheckEndpoints(BROKER_URL + '/status', attempts=100),
],
):
yield instance
def dd_environment():
instance = INSTANCES['main']
wait_and_unseal = WaitAndUnsealVault('{}/sys/health'.format(
instance['api_url']))
with docker_run(os.path.join(DOCKER_DIR, 'docker-compose.yaml'),
conditions=[wait_and_unseal]):
instance['client_token'] = wait_and_unseal.root_token
yield instance
def dd_environment(instance):
compose_file = os.path.join(get_here(), 'docker', 'docker-compose.yaml')
with docker_run(compose_file,
conditions=[
WaitFor(make_query),
CheckDockerLogs(compose_file, 'SERVER STARTED')
]):
yield instance, {'use_jmx': True}
def dd_environment(instance):
compose_file = os.path.join(common.HERE, 'compose', 'docker-compose.yml')
with docker_run(
compose_file,
service_name="mesos-slave",
log_patterns=['Finished recovery'],
):
yield instance
def dd_environment(instance):
if V3_PREVIEW:
endpoints = '{}/metrics'.format(URL)
else:
endpoints = ('{}/v2/stats/self'.format(URL), '{}/v2/stats/store'.format(URL))
# Sleep a bit so all metrics are available
with docker_run(COMPOSE_FILE, conditions=[CheckEndpoints(endpoints), add_key], sleep=3):
yield instance
def dd_environment():
env = {'HOSTNAME': HOST}
with docker_run(
compose_file=os.path.join(HERE, "compose", "docker-compose.yaml"),
conditions=[WaitFor(setup_mapreduce, attempts=5, wait=5)],
env_vars=env,
):
# 'custom_hosts' in metadata provides native /etc/hosts mappings in the agent's docker container
yield INSTANCE_INTEGRATION, {'custom_hosts': get_custom_hosts()}
def legacy_environment():
env = {}
env['HAPROXY_CONFIG_DIR'] = os.path.join(HERE, 'compose')
env['HAPROXY_CONFIG_OPEN'] = os.path.join(HERE, 'compose', 'haproxy-open.cfg')
with docker_run(
compose_file=os.path.join(HERE, 'compose', 'haproxy.yaml'),
env_vars=env,
service_name="haproxy-open",
conditions=[WaitFor(wait_for_haproxy_open)],
):
if platform_supports_sockets:
with TempDir() as temp_dir:
host_socket_path = os.path.join(temp_dir, 'datadog-haproxy-stats.sock')
env['HAPROXY_CONFIG'] = os.path.join(HERE, 'compose', 'haproxy.cfg')
if HAPROXY_VERSION >= version.parse('1.6'):
env['HAPROXY_CONFIG'] = os.path.join(HERE, 'compose', 'haproxy-1_6.cfg')
env['HAPROXY_SOCKET_DIR'] = temp_dir
with docker_run(
compose_file=os.path.join(HERE, 'compose', 'haproxy.yaml'),
env_vars=env,
service_name="haproxy",
conditions=[WaitFor(wait_for_haproxy)],
):
try:
# on linux this needs access to the socket
# it won't work without access
chown_args = []
user = getpass.getuser()
if user != 'root':
chown_args += ['sudo']
chown_args += ["chown", user, host_socket_path]
subprocess.check_call(chown_args, env=env)
except subprocess.CalledProcessError:
# it's not always bad if this fails
pass
config = deepcopy(CHECK_CONFIG)
unixsocket_url = 'unix://{0}'.format(host_socket_path)
config['unixsocket_url'] = unixsocket_url
yield {'instances': [config, CONFIG_TCPSOCKET]}
else:
yield deepcopy(CHECK_CONFIG_OPEN)
def dd_environment():
with docker_run(
compose_file=os.path.join(HERE, "compose", "docker-compose.yaml"),
mount_logs=True,
conditions=[
CheckEndpoints(INSTANCE_INTEGRATION['resourcemanager_uri'],
attempts=240)
],
):
yield INSTANCE_INTEGRATION
def dd_environment(instance):
image_name = os.environ.get('ELASTIC_IMAGE')
compose_file = COMPOSE_FILES_MAP.get(image_name, 'docker-compose.yaml')
compose_file = os.path.join(HERE, 'compose', compose_file)
with docker_run(
compose_file=compose_file,
conditions=[WaitFor(ping_elastic)],
):
yield instance
def dd_environment(mock_local_http_dns):
cacert_path = os.path.join(HERE, 'fixtures', 'cacert.pem')
e2e_metadata = {
'docker_volumes': ['{}:/opt/cacert.pem'.format(cacert_path)],
'custom_hosts': [(host, '127.0.0.1') for host in MOCKED_HOSTS],
}
with docker_run(os.path.join(HERE, 'compose', 'docker-compose.yml'),
build=True,
log_patterns=["starting server on port"]):
yield CONFIG_E2E, e2e_metadata
def dd_environment():
with docker_run(
common.COMPOSE_FILE,
endpoints=[
'{}/rrd_updates'.format(common.E2E_INSTANCE[0]['url']),
'{}/rrd_updates'.format(common.E2E_INSTANCE[1]['url']),
'{}/rrd_updates'.format(common.E2E_INSTANCE[2]['url']),
],
):
yield common.E2E_INSTANCE
def dd_environment():
compose_file = os.path.join(HERE, 'docker', 'docker-compose.yaml')
with docker_run(
compose_file=compose_file,
conditions=[
CheckDockerLogs('spark-app-hudi',
'finished: show at script.scala:163')
],
):
yield CHECK_CONFIG, {'use_jmx': True}
def dd_environment(instance):
properties_dir = os.path.join(HERE, 'weblogic', 'properties')
compose_file = os.path.join(HERE, 'docker-compose.yml')
with docker_run(
compose_file=compose_file,
env_vars={'PROPERTIES_DIR': properties_dir},
sleep=60,
build=True,
):
yield instance, {'use_jmx': True}
def dd_environment(instance):
env_vars = {'PULSAR_VERSION': common.PULSAR_VERSION}
with docker_run(
os.path.join(common.HERE, 'docker', 'docker-compose.yaml'),
env_vars=env_vars,
endpoints=instance['openmetrics_endpoint'],
mount_logs=True,
sleep=10,
):
yield instance
def dd_environment():
"""
Start a cluster with one master, one replica and one unhealthy replica and
stop it after the tests are done.
If there's any problem executing docker-compose, let the exception bubble
up.
"""
couch_version = os.environ["COUCH_VERSION"][0]
if couch_version == "1":
with docker_run(
compose_file=os.path.join(common.HERE, 'compose',
'compose_v1.yaml'),
env_vars={'COUCH_PORT': common.PORT},
conditions=[
CheckEndpoints([common.URL]),
CheckDockerLogs('couchdb-1', ['CouchDB has started']),
WaitFor(generate_data, args=(couch_version, )),
],
):
yield common.BASIC_CONFIG
else:
with docker_run(
compose_file=os.path.join(common.HERE, 'compose',
'compose_v2.yaml'),
env_vars={
'COUCH_PORT': common.PORT,
'COUCH_USER': common.USER,
'COUCH_PASSWORD': common.PASSWORD
},
conditions=[
CheckEndpoints([common.URL]),
CheckDockerLogs(
'couchdb-1',
['Started replicator db changes listener']),
WaitFor(enable_cluster),
WaitFor(generate_data, args=(couch_version, )),
WaitFor(check_node_stats),
WaitFor(send_replication),
WaitFor(get_replication),
],
):
yield common.BASIC_CONFIG_V2
def dd_environment():
env = {
"ROOT_PASSWORD": common.INSTANCE_INTEGRATION.get("password", "1234"),
}
with docker_run(compose_file=os.path.join(get_here(), "compose",
"docker-compose.yml"),
env_vars=env,
log_patterns="Server listening on 0.0.0.0 port 22."):
yield common.INSTANCE_INTEGRATION
def dd_environment():
with docker_run(
common.COMPOSE_FILE,
endpoints=[
'http://{}:{}'.format(common.HOST, common.HTTP_START_PORT + i)
for i in range(6)
],
sleep=10,
):
yield common.CONFIG
def dd_environment():
with docker_run(
os.path.join(HERE, 'compose', 'docker-compose.yaml'),
log_patterns=[
r'datanode:\(\d+\) is available',
'Starting Hive Metastore Server', 'Starting HiveServer2'
],
sleep=2,
):
yield load_jmx_config(), {'use_jmx': True}
def dd_environment():
env = {
'COMPOSE_DIR': COMPOSE_DIR
}
with docker_run(
os.path.join(COMPOSE_DIR, 'docker-compose.yaml'),
env_vars=env,
log_patterns="Listening at"
):
yield generate_instance_config(SCALAR_OBJECTS + SCALAR_OBJECTS_WITH_TAGS + TABULAR_OBJECTS)
def dd_environment():
compose_file = os.path.join(HERE, 'compose', 'docker-compose.yml')
with docker_run(compose_file,
conditions=[
CheckDockerLogs(compose_file,
[r'\[KafkaServer id=\d+\] started'],
matches="all")
]):
yield load_jmx_config(), {'use_jmx': True}
def dd_environment(get_instance):
def condition_non_tls():
sys.stderr.write("Waiting for ZK to boot...\n")
booted = False
dummy_instance = {'host': HOST, 'port': PORT, 'timeout': 500}
for _ in range(10):
try:
out = ZookeeperCheck('zk', {},
[dummy_instance])._send_command('ruok')
out.seek(0)
if out.readline() != 'imok':
raise ZKConnectionFailure()
booted = True
break
except ZKConnectionFailure:
time.sleep(1)
if not booted:
raise RetryError("Zookeeper failed to boot!")
sys.stderr.write("ZK boot complete.\n")
is_tls = get_tls()
compose_file = os.path.join(HERE, 'compose', 'zk.yaml')
if [3, 5, 0] <= get_version() < [3, 6, 0]:
compose_file = os.path.join(HERE, 'compose', 'zk35.yaml')
if is_tls:
compose_file = os.path.join(HERE, 'compose', 'zk35_ssl.yaml')
elif get_version() >= [3, 6, 0]:
compose_file = os.path.join(HERE, 'compose', 'zk36plus.yaml')
if is_tls:
compose_file = os.path.join(HERE, 'compose', 'zk36plus_ssl.yaml')
private_key = os.path.join(HERE, 'compose/client', 'private_key.pem')
cert = os.path.join(HERE, 'compose/client', 'cert.pem')
ca_cert = os.path.join(HERE, 'compose/client', 'ca_cert.pem')
if is_tls:
condition = [
CheckDockerLogs(compose_file,
patterns=[
'Starting server', 'Started AdminServer',
'bound to port'
])
]
else:
condition = [condition_non_tls]
with docker_run(compose_file, conditions=condition):
yield get_instance, {
'docker_volumes': [
'{}:/conf/private_key.pem'.format(private_key),
'{}:/conf/cert.pem'.format(cert),
'{}:/conf/ca_cert.pem'.format(ca_cert),
]
}
