def _get_warmup_conditions(): if MYSQL_REPLICATION == 'group': return [ CheckDockerLogs( 'node1', "X Plugin ready for connections. Bind-address: '::' port: 33060" ), CheckDockerLogs( 'node2', "X Plugin ready for connections. Bind-address: '::' port: 33060" ), CheckDockerLogs( 'node3', "X Plugin ready for connections. Bind-address: '::' port: 33060" ), init_group_replication, populate_database, ] return [ WaitFor(init_master, wait=2), WaitFor(init_slave, wait=2), CheckDockerLogs( 'mysql-slave', ["ready for connections", "mariadb successfully initialized"]), populate_database, ]
def dd_environment(): if pyodbc is None: raise Exception("pyodbc is not installed!") def sqlserver_can_connect(): conn = 'DRIVER={};Server={};Database=master;UID=sa;PWD=Password123;'.format(get_local_driver(), DOCKER_SERVER) pyodbc.connect(conn, timeout=30) compose_file = os.path.join(HERE, os.environ["COMPOSE_FOLDER"], 'docker-compose.yaml') conditions = [ WaitFor(sqlserver_can_connect, wait=3, attempts=10), ] if os.environ["COMPOSE_FOLDER"] == 'compose-ha': conditions += [ CheckDockerLogs( compose_file, 'Always On Availability Groups connection with primary database established for secondary database', ) ] else: conditions += [ CheckDockerLogs( compose_file, 'setup.sql completed', ) ] with docker_run( compose_file=compose_file, conditions=conditions, mount_logs=True, ): yield FULL_E2E_CONFIG
def dd_environment(): compose_file = os.path.join(HERE, 'compose', 'docker-compose.yaml') with docker_run( compose_file, conditions=[ # Kafka Broker CheckDockerLogs('broker', 'Created log for partition _confluent'), # Kafka Schema Registry CheckDockerLogs('schema-registry', 'Server started, listening for requests...', attempts=90), # Kafka Connect CheckDockerLogs('connect', 'Kafka Connect started', attempts=120), # Create connectors WaitFor(create_connectors), CheckDockerLogs( 'connect', 'flushing 0 outstanding messages for offset commit'), ], attempts=2, ): yield CHECK_CONFIG, {'use_jmx': True}
def dd_environment(): """ Start a cluster with one master, one replica and one unhealthy replica and stop it after the tests are done. If there's any problem executing `docker compose`, let the exception bubble up. """ couch_version = os.environ["COUCH_VERSION"][0] if couch_version == "1": with docker_run( compose_file=os.path.join(common.HERE, 'compose', 'compose_v1.yaml'), env_vars={'COUCH_PORT': common.PORT}, conditions=[ CheckEndpoints([common.URL]), CheckDockerLogs('couchdb-1', ['CouchDB has started', 'Application couch_index started']), WaitFor(generate_data, args=(couch_version,)), ], ): yield common.BASIC_CONFIG else: with docker_run( compose_file=os.path.join(common.HERE, 'compose', 'compose_v2.yaml'), env_vars={'COUCH_PORT': common.PORT, 'COUCH_USER': common.USER, 'COUCH_PASSWORD': common.PASSWORD}, conditions=[ CheckEndpoints([common.URL]), CheckDockerLogs('couchdb-1', ['Started replicator db changes listener']), WaitFor(enable_cluster), WaitFor(generate_data, args=(couch_version,)), WaitFor(check_node_stats), WaitFor(send_replication), WaitFor(get_replication), ], ): yield common.BASIC_CONFIG_V2
def dd_environment(): instance = { 'server': '{}:{}'.format(HOST, PORT), 'user': USER, 'password': PASSWORD, 'service_name': 'InfraDB.us.oracle.com', } if CLIENT_LIB == 'jdbc': e2e_metadata = E2E_METADATA_JDBC_CLIENT instance[ 'jdbc_driver_path'] = '/opt/oracle/instantclient_19_3/ojdbc8.jar' else: e2e_metadata = E2E_METADATA_ORACLE_CLIENT with docker_run( COMPOSE_FILE, conditions=[ CheckDockerLogs(COMPOSE_FILE, ['The database is ready for use'], wait=5, attempts=120), WaitFor(create_user), ], env_vars={'ORACLE_DATABASE_VERSION': ORACLE_DATABASE_VERSION}, attempts=20, attempts_wait=5, ): yield instance, e2e_metadata
def dd_environment(config_e2e): logs_path = _mysql_logs_path() with TempDir('logs') as logs_host_path: e2e_metadata = { 'docker_volumes': ['{}:{}'.format(logs_host_path, logs_path)] } with docker_run( os.path.join(common.HERE, 'compose', COMPOSE_FILE), env_vars={ 'MYSQL_DOCKER_REPO': _mysql_docker_repo(), 'MYSQL_PORT': str(common.PORT), 'MYSQL_SLAVE_PORT': str(common.SLAVE_PORT), 'MYSQL_CONF_PATH': _mysql_conf_path(), 'MYSQL_LOGS_HOST_PATH': logs_host_path, 'MYSQL_LOGS_PATH': logs_path, 'WAIT_FOR_IT_SCRIPT_PATH': _wait_for_it_script(), }, conditions=[ WaitFor(init_master, wait=2), WaitFor(init_slave, wait=2), CheckDockerLogs('mysql-slave', [ "ready for connections", "mariadb successfully initialized" ]), populate_database, ], ): yield config_e2e, e2e_metadata
def dd_environment(): with kind_run(conditions=[setup_linkerd_cluster]) as kubeconfig: compose_file = os.path.join(HERE, "compose", "docker-compose.yaml") with docker_run( compose_file=compose_file, conditions=[ CheckDockerLogs(compose_file, 'LINKERD DEPLOY COMPLETE', wait=5, attempts=120) ], ): with ExitStack() as stack: ip, port = stack.enter_context( port_forward(kubeconfig, 'linkerd', 4191, 'deployment', 'linkerd-controller')) instance = { 'prometheus_url': 'http://{ip}:{port}/metrics'.format(ip=ip, port=port), 'metrics': [LINKERD_FIXTURE_METRICS], 'type_overrides': LINKERD_FIXTURE_TYPES, } yield instance
def dd_environment(instance): # type: (Instance) -> Iterator compose_file = os.path.join(common.HERE, 'compose', 'docker-compose.yaml') schema_file = os.path.join(common.HERE, 'compose', 'schema.sql') with open(schema_file) as f: schema = f.read() conditions = [ CheckDockerLogs(compose_file, patterns=['Server completed initialization']), CreateSchema(compose_file, schema, container_name='voltdb0'), EnsureExpectedMetricsShowUp(instance), ] env_vars = { 'VOLTDB_IMAGE': common.VOLTDB_IMAGE, 'VOLTDB_DEPLOYMENT': common.VOLTDB_DEPLOYMENT, 'VOLTDB_CLIENT_PORT': str(common.VOLTDB_CLIENT_PORT), } if common.TLS_ENABLED: # Must refer to a path within the Agent container. instance = instance.copy() instance['tls_cert'] = '/tmp/voltdb-certs/client.pem' instance['tls_ca_cert'] = '/tmp/voltdb-certs/ca.pem' e2e_metadata = { 'docker_volumes': ['{}:/tmp/voltdb-certs'.format(common.TLS_CERTS_DIR)] } else: e2e_metadata = {} with docker_run(compose_file, conditions=conditions, env_vars=env_vars): yield instance, e2e_metadata
def dd_environment(): compose_file = os.path.join(common.HERE, 'docker', 'docker-compose.yaml') with docker_run( compose_file, build=True, mount_logs=True, conditions=[ CheckDockerLogs( compose_file, [ # Management Center 'Hazelcast Management Center successfully started', r'Members \[2', # Members connected to each other r'Members \{size:2', ], matches='all', attempts=120, ) ], ): config = load_jmx_config() config['instances'] = [ common.INSTANCE_MEMBER_JMX, common.INSTANCE_MC_JMX, common.INSTANCE_MC_PYTHON ] yield config, {'use_jmx': True}
def test_no_matches(self): compose_file = os.path.join(DOCKER_DIR, 'test_default.yaml') run_command(['docker-compose', '-f', compose_file, 'down']) check_docker_logs = CheckDockerLogs(compose_file, 'Vault server started', attempts=1) with pytest.raises(RetryError): check_docker_logs()
def dd_environment(): """ Spin up and initialize gitlab_runner """ # specify couchbase container name env = { 'GITLAB_TEST_TOKEN': GITLAB_TEST_TOKEN, 'GITLAB_LOCAL_MASTER_PORT': str(GITLAB_LOCAL_MASTER_PORT), 'GITLAB_LOCAL_RUNNER_PORT': str(GITLAB_LOCAL_RUNNER_PORT), } compose_file = os.path.join(HERE, 'compose', 'docker-compose.yml') with docker_run( compose_file=compose_file, env_vars=env, conditions=[ CheckDockerLogs(compose_file, [ 'Gitlab is up!', 'Configuration loaded', 'Metrics server listening' ], wait=5), CheckEndpoints(GITLAB_RUNNER_URL, attempts=180), ], ): yield CONFIG, E2E_METADATA
def dd_environment(): if common.MQ_VERSION == 9: log_pattern = "AMQ5026I: The listener 'DEV.LISTENER.TCP' has started. ProcessId" elif common.MQ_VERSION == 8: log_pattern = r".*QMNAME\({}\)\s*STATUS\(Running\).*".format( common.QUEUE_MANAGER) else: raise RuntimeError('Invalid version: {}'.format(common.MQ_VERSION)) e2e_meta = copy.deepcopy(common.E2E_METADATA) e2e_meta.setdefault('docker_volumes', []) e2e_meta['docker_volumes'].append("{}:/opt/pki/keys".format( os.path.join(common.HERE, 'keys'))) conditions = [CheckDockerLogs('ibm_mq1', log_pattern)] if not ON_WINDOWS: conditions.append(WaitFor(prepare_queue_manager)) with docker_run(compose_file=common.COMPOSE_FILE_PATH, build=True, conditions=conditions, sleep=10, attempts=2): yield common.INSTANCE, e2e_meta
def dd_environment(): compose_file = os.path.join(HERE, 'compose', 'docker-compose.yaml') # We need a custom condition to wait a bit longer with docker_run( compose_file=compose_file, conditions=[ CheckDockerLogs(compose_file, 'spawning ceph --cluster ceph -w', wait=5), CheckDockerLogs(compose_file, 'Running on http://0.0.0.0:5000/'), ], ): # Clean the disk space warning run_command( ['docker', 'exec', 'dd-test-ceph', 'ceph', 'tell', 'mon.*', 'injectargs', '--mon_data_avail_warn', '5'] ) # Wait a bit for the change to take effect time.sleep(5) yield BASIC_CONFIG, E2E_METADATA
def test_matches(self): compose_file = os.path.join(DOCKER_DIR, 'test_default.yaml') check_docker_logs = CheckDockerLogs(compose_file, 'Vault server started') try: run_command(['docker-compose', '-f', compose_file, 'up', '-d'], check=True) check_docker_logs() finally: run_command(['docker-compose', '-f', compose_file, 'down'], capture=True)
def dd_environment(instance): compose_file = get_docker_compose_file() conditions = [ CheckDockerLogs(compose_file, "http server Running on", wait=3), lambda: time.sleep(2), CreateSimpleUser(), ] with docker_run(compose_file, conditions=conditions): yield instance
def dd_environment(instance): compose_file = os.path.join(get_here(), 'docker', 'docker-compose.yaml') with docker_run(compose_file, conditions=[ WaitFor(make_query), CheckDockerLogs(compose_file, 'SERVER STARTED') ]): yield instance, {'use_jmx': True}
def dd_environment(): compose_file = os.path.join(HERE, 'compose', 'docker-compose.yaml') with docker_run( compose_file, conditions=[ # Kafka Broker CheckDockerLogs('broker', 'Monitored service is now ready'), # Kafka Schema Registry CheckDockerLogs('schema-registry', 'Server started, listening for requests...', attempts=90), # Kafka Connect CheckDockerLogs('connect', 'Kafka Connect started', attempts=120), ], ): yield CHECK_CONFIG, {'use_jmx': True}
def dd_environment(): compose_file = os.path.join(HERE, 'docker', 'docker-compose.yml') with docker_run(compose_file, conditions=[ CheckDockerLogs(compose_file, 'Resin/4.0.62 started -server') ]): instance = load_jmx_config() yield instance, {'use_jmx': True}
def dd_environment(): compose_file = os.path.join(HERE, 'docker', 'docker-compose.yml') log_patterns = 'Fullsync complete from' with docker_run( compose_file=compose_file, conditions=[CheckEndpoints(URL, wait=5), CheckDockerLogs(compose_file, log_patterns, matches='all', wait=5)], ): yield
def dd_environment(): compose_file = os.path.join(common.HERE, 'docker', 'docker-compose.yaml') with docker_run( compose_file, build=True, mount_logs=True, conditions=[ CheckDockerLogs('hazelcast_management_center', ['Hazelcast Management Center successfully started']), CheckDockerLogs('hazelcast_management_center', ['Started communication with member']), CheckDockerLogs('hazelcast2', [r'Hazelcast JMX agent enabled']), CheckDockerLogs('hazelcast2', [r'is STARTED']), WaitFor(trigger_some_tcp_data), ], attempts=5, attempts_wait=5, ): config = load_jmx_config() config['instances'] = common.INSTANCE_MEMBERS + [common.INSTANCE_MC_JMX, common.INSTANCE_MC_PYTHON] yield config, {'use_jmx': True}
def dd_environment(): compose_file = os.path.join(HERE, 'compose', 'docker-compose.yml') with docker_run(compose_file, conditions=[ CheckDockerLogs(compose_file, [r'\[KafkaServer id=\d+\] started'], matches="all") ]): yield load_jmx_config(), {'use_jmx': True}
def dd_environment(): compose_file = os.path.join(HERE, 'docker', 'docker-compose.yaml') with docker_run( compose_file=compose_file, conditions=[ CheckDockerLogs('spark-app-hudi', 'finished: show at script.scala:163') ], ): yield CHECK_CONFIG, {'use_jmx': True}
def dd_environment(get_instance): def condition_non_tls(): sys.stderr.write("Waiting for ZK to boot...\n") booted = False dummy_instance = {'host': HOST, 'port': PORT, 'timeout': 500} for _ in range(10): try: out = ZookeeperCheck('zk', {}, [dummy_instance])._send_command('ruok') out.seek(0) if out.readline() != 'imok': raise ZKConnectionFailure() booted = True break except ZKConnectionFailure: time.sleep(1) if not booted: raise RetryError("Zookeeper failed to boot!") sys.stderr.write("ZK boot complete.\n") is_tls = get_tls() compose_file = os.path.join(HERE, 'compose', 'zk.yaml') if [3, 5, 0] <= get_version() < [3, 6, 0]: compose_file = os.path.join(HERE, 'compose', 'zk35.yaml') if is_tls: compose_file = os.path.join(HERE, 'compose', 'zk35_ssl.yaml') elif get_version() >= [3, 6, 0]: compose_file = os.path.join(HERE, 'compose', 'zk36plus.yaml') if is_tls: compose_file = os.path.join(HERE, 'compose', 'zk36plus_ssl.yaml') private_key = os.path.join(HERE, 'compose/client', 'private_key.pem') cert = os.path.join(HERE, 'compose/client', 'cert.pem') ca_cert = os.path.join(HERE, 'compose/client', 'ca_cert.pem') if is_tls: condition = [ CheckDockerLogs(compose_file, patterns=[ 'Starting server', 'Started AdminServer', 'bound to port' ]) ] else: condition = [condition_non_tls] with docker_run(compose_file, conditions=condition): yield get_instance, { 'docker_volumes': [ '{}:/conf/private_key.pem'.format(private_key), '{}:/conf/cert.pem'.format(cert), '{}:/conf/ca_cert.pem'.format(ca_cert), ] }
def dd_environment(): compose_file = os.path.join(HERE, 'compose', 'docker-compose.yml') with docker_run( compose_file=compose_file, sleep=120, conditions=[ CheckDockerLogs(compose_file, 'php-fpm7.4 entered RUNNING state'), ], ): yield CONFIG
def dd_environment(): compose_file = os.path.join(common.HERE, 'docker', 'docker-compose.yaml') with docker_run(compose_file, mount_logs=True, conditions=[ CheckDockerLogs(compose_file, ['Started HiveMQ in'], matches='all') ]): config = load_jmx_config() config['instances'] = [common.INSTANCE] yield config, {'use_jmx': True}
def dd_environment(instance): compose_file = os.path.join(get_here(), "docker", "docker-compose.yaml") with docker_run( compose_file, conditions=[ WaitFor(make_query), CheckDockerLogs(compose_file, "SERVER STARTED"), ], ): yield instance, {"use_jmx": True}
def dd_environment(): compose_file = os.path.join(common.HERE, "compose", "docker-compose.yaml") with docker_run( compose_file=compose_file, conditions=[ CheckDockerLogs("dd-test-riakcs", "INFO success: riak-cs entered RUNNING state", attempts=240) ], ): yield common.generate_config_with_creds()
def dd_environment(integration_instance): compose_file = os.path.join(HERE, 'compose', 'docker-compose.yaml') # We need a custom condition to wait a bit longer with docker_run( compose_file=compose_file, build=True, conditions=[ CheckDockerLogs(compose_file, 'Running on ', wait=5), ], ): yield integration_instance
def dd_environment(): compose_file = os.path.join(common.HERE, 'docker', 'docker-compose.yaml') with docker_run( compose_file, conditions=[ CheckDockerLogs(compose_file, 'Server server1 open for e-business', attempts=120), StartPerfServlet(), CheckEndpoints(common.INSTANCE['servlet_url']), ], ): yield common.INSTANCE
def dd_environment(e2e_instance): compose_file = get_docker_compose_file() expected_log = "http server Running on" if HARBOR_VERSION < [ 1, 10, 0 ] else "API server is serving at" conditions = [ CheckDockerLogs(compose_file, expected_log, wait=3), lambda: time.sleep(4), CreateSimpleUser(), ] with docker_run(compose_file, conditions=conditions, attempts=2): yield e2e_instance