def install_sync_gateway(self, cluster_config, sync_gateway_version, sync_gateway_config):
# Dirty hack -- these have to be put here in order to avoid circular imports
from libraries.provision.install_sync_gateway import install_sync_gateway
from libraries.provision.install_sync_gateway import SyncGatewayConfig
if version_is_binary(sync_gateway_version):
version, build = version_and_build(sync_gateway_version)
print("VERSION: {} BUILD: {}".format(version, build))
sg_config = SyncGatewayConfig(None, version, build, sync_gateway_config, "", False)
else:
sg_config = SyncGatewayConfig(sync_gateway_version, None, None, sync_gateway_config, "", False)
install_sync_gateway(cluster_config=cluster_config, sync_gateway_config=sg_config)
log_info("Verfying versions for cluster: {}".format(cluster_config))
with open("{}.json".format(cluster_config)) as f:
cluster_obj = json.loads(f.read())
# Verify sync_gateway versions
for sg in cluster_obj["sync_gateways"]:
verify_sync_gateway_version(sg["ip"], sync_gateway_version)
# Verify sg_accel versions, use the same expected version for sync_gateway for now
for ac in cluster_obj["sg_accels"]:
verify_sg_accel_version(ac["ip"], sync_gateway_version)
def provision_cluster(self, cluster_config, server_version, sync_gateway_version, sync_gateway_config, race_enabled=False, sg_ce=False, cbs_platform="centos7", sg_platform="centos", sa_platform="centos"):
if server_version is None or sync_gateway_version is None or sync_gateway_version is None:
raise ProvisioningError("Please make sure you have server_version, sync_gateway_version, and sync_gateway_config are set")
# Dirty hack -- these have to be put here in order to avoid circular imports
from libraries.provision.install_couchbase_server import CouchbaseServerConfig
from libraries.provision.provision_cluster import provision_cluster
from libraries.provision.install_sync_gateway import SyncGatewayConfig
cbs_config = CouchbaseServerConfig(server_version)
if version_is_binary(sync_gateway_version):
if race_enabled:
raise ProvisioningError("Race should only be enabled for source builds")
version, build = version_and_build(sync_gateway_version)
sg_config = SyncGatewayConfig(
commit=None,
version_number=version,
build_number=build,
config_path=sync_gateway_config,
build_flags="",
skip_bucketcreation=False
)
else:
build_flags = ""
if race_enabled:
build_flags = "-race"
sg_config = SyncGatewayConfig(
commit=sync_gateway_version,
version_number=None,
build_number=None,
config_path=sync_gateway_config,
build_flags=build_flags,
skip_bucketcreation=False
)
provision_cluster(
cluster_config=cluster_config,
couchbase_server_config=cbs_config,
sync_gateway_config=sg_config,
sg_ce=sg_ce,
cbs_platform=cbs_platform,
sg_platform=sg_platform,
sa_platform=sa_platform
)
# verify running services are the expected versions
self.verify_cluster_versions(cluster_config, server_version, sync_gateway_version)
def verify_sg_accel_version(host, expected_sg_accel_version):
running_ac_version = get_sg_accel_version(host)
log_info("Expected sg_accel Version: {}".format(expected_sg_accel_version))
log_info("Running sg_accel Version: {}".format(running_ac_version))
if version_is_binary(expected_sg_accel_version):
# Example, 1.2.1-4
if running_ac_version != expected_sg_accel_version:
raise ProvisioningError("Unexpected sync_gateway version!! Expected: {} Actual: {}".format(expected_sg_accel_version, running_ac_version))
else:
# Since sync_gateway does not return the full commit, verify the prefix
if running_ac_version != expected_sg_accel_version[:7]:
raise ProvisioningError("Unexpected sync_gateway version!! Expected: {} Actual: {}".format(expected_sg_accel_version, running_ac_version))
def verify_sync_gateway_version(host, expected_sync_gateway_version):
running_sg_version, running_sg_vendor_version = get_sync_gateway_version(host)
log_info("Expected sync_gateway Version: {}".format(expected_sync_gateway_version))
log_info("Running sync_gateway Version: {}".format(running_sg_version))
log_info("Running sync_gateway Vendor Version: {}".format(running_sg_vendor_version))
if version_is_binary(expected_sync_gateway_version):
# Example, 1.2.1-4
if running_sg_version != expected_sync_gateway_version:
raise ProvisioningError("Unexpected sync_gateway version!! Expected: {} Actual: {}".format(expected_sync_gateway_version, running_sg_version))
# Running vendor version: ex. '1.2', check that the expected version start with the vendor version
if not expected_sync_gateway_version.startswith(running_sg_vendor_version):
raise ProvisioningError("Unexpected sync_gateway vendor version!! Expected: {} Actual: {}".format(expected_sync_gateway_version, running_sg_vendor_version))
else:
# Since sync_gateway does not return the full commit, verify the prefix
if running_sg_version != expected_sync_gateway_version[:7]:
raise ProvisioningError("Unexpected sync_gateway version!! Expected: {} Actual: {}".format(expected_sync_gateway_version, running_sg_version))
def provision_cluster(self, cluster_config, server_version, sync_gateway_version, sync_gateway_config):
if server_version is None or sync_gateway_version is None or sync_gateway_version is None:
raise ProvisioningError("Please make sure you have server_version, sync_gateway_version, and sync_gateway_config are set")
# Dirty hack -- these have to be put here in order to avoid circular imports
from libraries.provision.install_couchbase_server import CouchbaseServerConfig
from libraries.provision.provision_cluster import provision_cluster
from libraries.provision.install_sync_gateway import SyncGatewayConfig
cbs_config = CouchbaseServerConfig(server_version)
if version_is_binary(sync_gateway_version):
version, build = version_and_build(sync_gateway_version)
sg_config = SyncGatewayConfig(None, version, build, sync_gateway_config, "", False)
else:
sg_config = SyncGatewayConfig(sync_gateway_version, None, None, sync_gateway_config, "", False)
provision_cluster(cluster_config, cbs_config, sg_config)
# verify running services are the expected versions
self.verify_cluster_versions(cluster_config, server_version, sync_gateway_version)
def setup_client_syncgateway_suite(request):
"""Suite setup fixture for client sync_gateway tests"""
log_info("Setting up client sync_gateway suite ...")
liteserv_platform = request.config.getoption("--liteserv-platform")
liteserv_version = request.config.getoption("--liteserv-version")
liteserv_host = request.config.getoption("--liteserv-host")
liteserv_port = request.config.getoption("--liteserv-port")
liteserv_storage_engine = request.config.getoption("--liteserv-storage-engine")
skip_provisioning = request.config.getoption("--skip-provisioning")
sync_gateway_version = request.config.getoption("--sync-gateway-version")
sync_gateway_mode = request.config.getoption("--sync-gateway-mode")
server_version = request.config.getoption("--server-version")
xattrs_enabled = request.config.getoption("--xattrs")
device_enabled = request.config.getoption("--device")
liteserv = LiteServFactory.create(platform=liteserv_platform,
version_build=liteserv_version,
host=liteserv_host,
port=liteserv_port,
storage_engine=liteserv_storage_engine)
if xattrs_enabled and version_is_binary(sync_gateway_version):
check_xattr_support(server_version, sync_gateway_version)
log_info("Downloading LiteServ ...")
# Download LiteServ
liteserv.download()
# Install LiteServ
if device_enabled and liteserv_platform == "ios":
liteserv.install_device()
else:
liteserv.install()
cluster_config = "{}/base_{}".format(CLUSTER_CONFIGS_DIR, sync_gateway_mode)
try:
server_version
except NameError:
log_info("Server version is not provided")
persist_cluster_config_environment_prop(cluster_config, 'server_version', "")
else:
log_info("Running test with server version {}".format(server_version))
persist_cluster_config_environment_prop(cluster_config, 'server_version', server_version)
try:
sync_gateway_version
except NameError:
log_info("Sync gateway version is not provided")
persist_cluster_config_environment_prop(cluster_config, 'sync_gateway_version', "")
else:
log_info("Running test with sync_gateway version {}".format(sync_gateway_version))
persist_cluster_config_environment_prop(cluster_config, 'sync_gateway_version', sync_gateway_version)
if xattrs_enabled:
log_info("Running test with xattrs for sync meta storage")
persist_cluster_config_environment_prop(cluster_config, 'xattrs_enabled', True)
else:
log_info("Using document storage for sync meta data")
persist_cluster_config_environment_prop(cluster_config, 'xattrs_enabled', False)
sg_config = sync_gateway_config_path_for_mode("listener_tests/listener_tests", sync_gateway_mode)
if not skip_provisioning:
log_info("Installing Sync Gateway + Couchbase Server + Accels ('di' only)")
cluster_utils = ClusterKeywords()
cluster_utils.provision_cluster(
cluster_config=cluster_config,
server_version=server_version,
sync_gateway_version=sync_gateway_version,
sync_gateway_config=sg_config
)
# Wait at the yeild until tests referencing this suite setup have run,
# Then execute the teardown
yield {
"liteserv": liteserv,
"cluster_config": cluster_config,
"sg_mode": sync_gateway_mode,
"xattrs_enabled": xattrs_enabled,
"device_enabled": device_enabled,
"liteserv_platform": liteserv_platform,
"liteserv_version": liteserv_version
}
log_info("Tearing down suite ...")
if not (device_enabled and liteserv_platform == "ios"):
liteserv.remove()
def params_from_base_suite_setup(request):
log_info("Setting up 'params_from_base_suite_setup' ...")
# pytest command line parameters
server_version = request.config.getoption("--server-version")
sync_gateway_version = request.config.getoption("--sync-gateway-version")
mode = request.config.getoption("--mode")
use_sequoia = request.config.getoption("--sequoia")
skip_provisioning = request.config.getoption("--skip-provisioning")
cbs_ssl = request.config.getoption("--server-ssl")
xattrs_enabled = request.config.getoption("--xattrs")
server_seed_docs = request.config.getoption("--server-seed-docs")
max_docs = request.config.getoption("--max-docs")
num_users = request.config.getoption("--num-users")
create_batch_size = request.config.getoption("--create-batch-size")
create_delay = request.config.getoption("--create-delay")
update_runtime_sec = request.config.getoption("--update-runtime-sec")
update_batch_size = request.config.getoption("--update-batch-size")
update_docs_percentage = request.config.getoption(
"--update-docs-percentage")
update_delay = request.config.getoption("--update-delay")
changes_delay = request.config.getoption("--changes-delay")
changes_limit = request.config.getoption("--changes-limit")
if xattrs_enabled and version_is_binary(sync_gateway_version):
check_xattr_support(server_version, sync_gateway_version)
log_info("server_version: {}".format(server_version))
log_info("sync_gateway_version: {}".format(sync_gateway_version))
log_info("mode: {}".format(mode))
log_info("skip_provisioning: {}".format(skip_provisioning))
log_info("cbs_ssl: {}".format(cbs_ssl))
log_info("xattrs_enabled: {}".format(xattrs_enabled))
# Make sure mode for sync_gateway is supported ('cc' or 'di')
validate_sync_gateway_mode(mode)
# use ci_lb_cc cluster config if mode is "cc" or ci_lb_di cluster config if more is "di"
log_info("Using 'ci_lb_{}' config!".format(mode))
cluster_config = "{}/ci_lb_{}".format(CLUSTER_CONFIGS_DIR, mode)
try:
server_version
except NameError:
log_info("Server version is not provided")
persist_cluster_config_environment_prop(cluster_config,
'server_version', "")
else:
log_info("Running test with server version {}".format(server_version))
persist_cluster_config_environment_prop(cluster_config,
'server_version',
server_version)
try:
sync_gateway_version
except NameError:
log_info("Sync gateway version is not provided")
persist_cluster_config_environment_prop(cluster_config,
'sync_gateway_version', "")
else:
log_info("Running test with sync_gateway version {}".format(
sync_gateway_version))
persist_cluster_config_environment_prop(cluster_config,
'sync_gateway_version',
sync_gateway_version)
# Only works with load balancer configs
persist_cluster_config_environment_prop(cluster_config, 'sg_lb_enabled',
True)
if cbs_ssl:
log_info("Running tests with cbs <-> sg ssl enabled")
# Enable ssl in cluster configs
persist_cluster_config_environment_prop(cluster_config,
'cbs_ssl_enabled', True)
else:
log_info("Running tests with cbs <-> sg ssl disabled")
# Disable ssl in cluster configs
persist_cluster_config_environment_prop(cluster_config,
'cbs_ssl_enabled', False)
if xattrs_enabled:
log_info("Running test with xattrs for sync meta storage")
persist_cluster_config_environment_prop(cluster_config,
'xattrs_enabled', True)
else:
log_info("Using document storage for sync meta data")
persist_cluster_config_environment_prop(cluster_config,
'xattrs_enabled', False)
sg_config = sync_gateway_config_path_for_mode(
"sync_gateway_default_functional_tests", mode)
# Skip provisioning if user specifies '--skip-provisoning' or '--sequoia'
should_provision = True
if skip_provisioning or use_sequoia:
should_provision = False
cluster_utils = ClusterKeywords()
if should_provision:
try:
cluster_utils.provision_cluster(
cluster_config=cluster_config,
server_version=server_version,
sync_gateway_version=sync_gateway_version,
sync_gateway_config=sg_config)
except ProvisioningError:
logging_helper = Logging()
logging_helper.fetch_and_analyze_logs(
cluster_config=cluster_config, test_name=request.node.name)
raise
# Hit this intalled running services to verify the correct versions are installed
cluster_utils.verify_cluster_versions(
cluster_config,
expected_server_version=server_version,
expected_sync_gateway_version=sync_gateway_version)
# Load topology as a dictionary
cluster_topology = cluster_utils.get_cluster_topology(cluster_config)
yield {
"cluster_config": cluster_config,
"cluster_topology": cluster_topology,
"mode": mode,
"xattrs_enabled": xattrs_enabled,
"server_seed_docs": server_seed_docs,
"max_docs": max_docs,
"num_users": num_users,
"create_batch_size": create_batch_size,
"create_delay": create_delay,
"update_runtime_sec": update_runtime_sec,
"update_batch_size": update_batch_size,
"update_docs_percentage": update_docs_percentage,
"update_delay": update_delay,
"changes_delay": changes_delay,
"changes_limit": changes_limit
}
log_info("Tearing down 'params_from_base_suite_setup' ...")
def params_from_base_suite_setup(request):
log_info("Setting up 'params_from_base_suite_setup' ...")
# pytest command line parameters
server_version = request.config.getoption("--server-version")
sync_gateway_version = request.config.getoption("--sync-gateway-version")
server_upgraded_version = request.config.getoption(
"--server-upgraded-version")
sync_gateway_upgraded_version = request.config.getoption(
"--sync-gateway-upgraded-version")
mode = request.config.getoption("--mode")
cluster_config = request.config.getoption("--cluster-config")
# use_sequoia = request.config.getoption("--sequoia")
skip_provisioning = request.config.getoption("--skip-provisioning")
cbs_ssl = request.config.getoption("--server-ssl")
xattrs_enabled = request.config.getoption("--xattrs")
liteserv_host = request.config.getoption("--liteserv-host")
liteserv_port = request.config.getoption("--liteserv-port")
liteserv_version = request.config.getoption("--liteserv-version")
liteserv_platform = request.config.getoption("--liteserv-platform")
liteserv_storage_engine = request.config.getoption(
"--liteserv-storage-engine")
num_docs = request.config.getoption("--num-docs")
cbs_platform = request.config.getoption("--cbs-platform")
cbs_toy_build = request.config.getoption("--cbs-upgrade-toybuild")
if xattrs_enabled and version_is_binary(sync_gateway_version):
check_xattr_support(server_upgraded_version,
sync_gateway_upgraded_version)
log_info("server_version: {}".format(server_version))
log_info("sync_gateway_version: {}".format(sync_gateway_version))
log_info("server_upgraded_version: {}".format(server_upgraded_version))
log_info("sync_gateway_upgraded_version: {}".format(
sync_gateway_upgraded_version))
log_info("mode: {}".format(mode))
log_info("skip_provisioning: {}".format(skip_provisioning))
log_info("cbs_ssl: {}".format(cbs_ssl))
log_info("xattrs_enabled: {}".format(xattrs_enabled))
log_info("liteserv_host: {}".format(liteserv_host))
log_info("liteserv_port: {}".format(liteserv_port))
log_info("liteserv_version: {}".format(liteserv_version))
log_info("liteserv_platform: {}".format(liteserv_platform))
log_info("liteserv_storage_engine: {}".format(liteserv_storage_engine))
log_info("num_docs: {}".format(num_docs))
log_info("cbs_platform: {}".format(cbs_platform))
log_info("cbs_toy_build: {}".format(cbs_toy_build))
# Make sure mode for sync_gateway is supported ('cc' or 'di')
validate_sync_gateway_mode(mode)
# use ci_lb_cc cluster config if mode is "cc" or ci_lb_di cluster config if more is "di"
# use base_(lb_)cc cluster config if mode is "cc" or base_(lb_)di cluster config if mode is "di"
cluster_config = "{}/{}_{}".format(CLUSTER_CONFIGS_DIR, cluster_config,
mode)
log_info("Using '{}' config!".format(cluster_config))
try:
server_version
except NameError:
log_info("Server version is not provided")
persist_cluster_config_environment_prop(cluster_config,
'server_version', "")
else:
log_info("Running test with server version {}".format(server_version))
persist_cluster_config_environment_prop(cluster_config,
'server_version',
server_version)
try:
sync_gateway_version
except NameError:
log_info("Sync gateway version is not provided")
persist_cluster_config_environment_prop(cluster_config,
'sync_gateway_version', "")
else:
log_info("Running test with sync_gateway version {}".format(
sync_gateway_version))
persist_cluster_config_environment_prop(cluster_config,
'sync_gateway_version',
sync_gateway_version)
# Only works with load balancer configs
persist_cluster_config_environment_prop(cluster_config, 'sg_lb_enabled',
True)
if cbs_ssl:
log_info("Running tests with cbs <-> sg ssl enabled")
# Enable ssl in cluster configs
persist_cluster_config_environment_prop(cluster_config,
'cbs_ssl_enabled', True)
else:
log_info("Running tests with cbs <-> sg ssl disabled")
# Disable ssl in cluster configs
persist_cluster_config_environment_prop(cluster_config,
'cbs_ssl_enabled', False)
sg_config = sync_gateway_config_path_for_mode(
"sync_gateway_default_functional_tests", mode)
liteserv = LiteServFactory.create(platform=liteserv_platform,
version_build=liteserv_version,
host=liteserv_host,
port=liteserv_port,
storage_engine=liteserv_storage_engine)
log_info("Downloading LiteServ ...")
# Download LiteServ
liteserv.download()
# Install LiteServ
liteserv.install()
# Skip provisioning if user specifies '--skip-provisoning' or '--sequoia'
should_provision = True
if skip_provisioning:
should_provision = False
cluster_utils = ClusterKeywords()
if should_provision:
try:
cluster_utils.provision_cluster(
cluster_config=cluster_config,
server_version=server_version,
sync_gateway_version=sync_gateway_version,
sync_gateway_config=sg_config,
cbs_platform=cbs_platform)
except ProvisioningError:
logging_helper = Logging()
logging_helper.fetch_and_analyze_logs(
cluster_config=cluster_config, test_name=request.node.name)
raise
# Hit this intalled running services to verify the correct versions are installed
cluster_utils.verify_cluster_versions(
cluster_config,
expected_server_version=server_version,
expected_sync_gateway_version=sync_gateway_version)
# Load topology as a dictionary
cluster_topology = cluster_utils.get_cluster_topology(cluster_config)
yield {
"cluster_config": cluster_config,
"cluster_topology": cluster_topology,
"mode": mode,
"xattrs_enabled": xattrs_enabled,
"server_version": server_version,
"sync_gateway_version": sync_gateway_version,
"server_upgraded_version": server_upgraded_version,
"sync_gateway_upgraded_version": sync_gateway_upgraded_version,
"liteserv_host": liteserv_host,
"liteserv_port": liteserv_port,
"liteserv_version": liteserv_version,
"liteserv_platform": liteserv_platform,
"liteserv_storage_engine": liteserv_storage_engine,
"liteserv": liteserv,
"num_docs": num_docs,
"cbs_platform": cbs_platform,
"cbs_toy_build": cbs_toy_build
}
log_info("Tearing down 'params_from_base_suite_setup' ...")