def verify_server_version(host, expected_server_version, cbs_ssl=False):
""" Verifies that the version of a running Couchbase Server is the 'expected_server_version' """
running_server_version = get_server_version(host, cbs_ssl=cbs_ssl)
expected_server_version_parts = expected_server_version.split("-")
# Check both version parts if expected version contains a build
if len(expected_server_version_parts) == 2:
# 4.1.1-5487
log_info("Expected Server Version: {}".format(expected_server_version))
log_info("Running Server Version: {}".format(running_server_version))
if running_server_version != expected_server_version:
raise ProvisioningError(
"Unexpected server version!! Expected: {} Actual: {}".format(
expected_server_version, running_server_version))
elif len(expected_server_version_parts) == 1:
# 4.1.1
running_server_version_parts = running_server_version.split("-")
log_info("Expected Server Version: {}".format(expected_server_version))
log_info("Running Server Version: {}".format(
running_server_version_parts[0]))
if expected_server_version != running_server_version_parts[0]:
raise ProvisioningError(
"Unexpected server version!! Expected: {} Actual: {}".format(
expected_server_version, running_server_version_parts[0]))
else:
raise ProvisioningError("Unsupported version format")
def is_valid(self):
if self._version_number is not None and self._build_number is not None:
if self.commit is not None:
raise ProvisioningError(
"Commit should be empty when provisioning with a binary")
elif self.commit is not None:
if self._version_number is not None:
raise ProvisioningError(
"Do not specify a version number when provisioning via a commit."
)
if self._build_number is not None:
raise ProvisioningError(
"Do not specify a build number when provisioning via a commit."
)
else:
log_info(
"You must provide a (version and build number) or (dev url and dev build number) or commit to build for sync_gateway"
)
return False
if not os.path.isfile(self.config_path):
log_info("Could not find sync_gateway config file: {}".format(
self.config_path))
log_info("Try to use an absolute path.")
return False
return True
def verify_sync_gateway_version(host, expected_sync_gateway_version):
running_sg_version, running_sg_vendor_version = get_sync_gateway_version(
host)
log_info("Expected sync_gateway Version: {}".format(
expected_sync_gateway_version))
log_info("Running sync_gateway Version: {}".format(running_sg_version))
log_info("Running sync_gateway Vendor Version: {}".format(
running_sg_vendor_version))
if version_is_binary(expected_sync_gateway_version):
# Example, 1.2.1-4
if running_sg_version != expected_sync_gateway_version:
raise ProvisioningError(
"Unexpected sync_gateway version!! Expected: {} Actual: {}".
format(expected_sync_gateway_version, running_sg_version))
# Running vendor version: ex. '1.2', check that the expected version start with the vendor version
if not expected_sync_gateway_version.startswith(
running_sg_vendor_version):
raise ProvisioningError(
"Unexpected sync_gateway vendor version!! Expected: {} Actual: {}"
.format(expected_sync_gateway_version,
running_sg_vendor_version))
else:
# Since sync_gateway does not return the full commit, verify the prefix
if running_sg_version != expected_sync_gateway_version[:7]:
raise ProvisioningError(
"Unexpected sync_gateway version!! Expected: {} Actual: {}".
format(expected_sync_gateway_version, running_sg_version))
def start_sync_gateways(self, cluster_config, url=None, config=None):
""" Start sync gateways in a cluster. If url is passed,
start the sync gateway at that url
"""
if config is None:
raise ProvisioningError(
"Starting a Sync Gateway requires a config")
ansible_runner = AnsibleRunner(cluster_config)
config_path = os.path.abspath(config)
couchbase_server_primary_node = add_cbs_to_sg_config_server_field(
cluster_config)
if is_cbs_ssl_enabled(cluster_config):
self.server_port = 18091
self.server_scheme = "https"
playbook_vars = {
"sync_gateway_config_filepath": config_path,
"server_port": self.server_port,
"server_scheme": self.server_scheme,
"autoimport": "",
"xattrs": "",
"no_conflicts": "",
"revs_limit": "",
"couchbase_server_primary_node": couchbase_server_primary_node
}
if is_xattrs_enabled(cluster_config):
playbook_vars["autoimport"] = '"import_docs": "continuous",'
playbook_vars["xattrs"] = '"enable_shared_bucket_access": true,'
if no_conflicts_enabled(cluster_config):
playbook_vars["no_conflicts"] = '"allow_conflicts": false,'
try:
revs_limit = get_revs_limit(cluster_config)
playbook_vars["revs_limit"] = '"revs_limit": {},'.format(
revs_limit)
except KeyError as ex:
log_info("Keyerror in getting revs_limit{}".format(ex.message))
if url is not None:
target = hostname_for_url(cluster_config, url)
log_info("Starting {} sync_gateway.".format(target))
status = ansible_runner.run_ansible_playbook(
"start-sync-gateway.yml",
extra_vars=playbook_vars,
subset=target)
else:
log_info("Starting all sync_gateways.")
status = ansible_runner.run_ansible_playbook(
"start-sync-gateway.yml", extra_vars=playbook_vars)
if status != 0:
raise ProvisioningError("Could not start sync_gateway")
def provision_cluster(self, cluster_config, server_version, sync_gateway_version, sync_gateway_config, race_enabled=False, sg_ce=False, cbs_platform="centos7", sg_platform="centos", sa_platform="centos"):
if server_version is None or sync_gateway_version is None or sync_gateway_version is None:
raise ProvisioningError("Please make sure you have server_version, sync_gateway_version, and sync_gateway_config are set")
# Dirty hack -- these have to be put here in order to avoid circular imports
from libraries.provision.install_couchbase_server import CouchbaseServerConfig
from libraries.provision.provision_cluster import provision_cluster
from libraries.provision.install_sync_gateway import SyncGatewayConfig
cbs_config = CouchbaseServerConfig(server_version)
if version_is_binary(sync_gateway_version):
if race_enabled:
raise ProvisioningError("Race should only be enabled for source builds")
version, build = version_and_build(sync_gateway_version)
sg_config = SyncGatewayConfig(
commit=None,
version_number=version,
build_number=build,
config_path=sync_gateway_config,
build_flags="",
skip_bucketcreation=False
)
else:
build_flags = ""
if race_enabled:
build_flags = "-race"
sg_config = SyncGatewayConfig(
commit=sync_gateway_version,
version_number=None,
build_number=None,
config_path=sync_gateway_config,
build_flags=build_flags,
skip_bucketcreation=False
)
provision_cluster(
cluster_config=cluster_config,
couchbase_server_config=cbs_config,
sync_gateway_config=sg_config,
sg_ce=sg_ce,
cbs_platform=cbs_platform,
sg_platform=sg_platform,
sa_platform=sa_platform
)
# verify running services are the expected versions
self.verify_cluster_versions(cluster_config, server_version, sync_gateway_version)
def install_deps(cluster_config):
log_info("Installing dependencies for cluster_config: {}".format(
cluster_config))
ansible_runner = AnsibleRunner(config=cluster_config)
status = ansible_runner.run_ansible_playbook("os-level-modifications.yml")
if status != 0:
raise ProvisioningError("Failed to make os modifications")
status = ansible_runner.run_ansible_playbook("install-common-tools.yml")
if status != 0:
raise ProvisioningError("Failed to install dependencies")
def clean_cluster(cluster_config):
log_info("Cleaning cluster: {}".format(cluster_config))
ansible_runner = AnsibleRunner(config=cluster_config)
status = ansible_runner.run_ansible_playbook(
"remove-previous-installs.yml")
if status != 0:
raise ProvisioningError("Failed to removed previous installs")
# Clear firewall rules
status = ansible_runner.run_ansible_playbook("flush-firewall.yml")
if status != 0:
raise ProvisioningError("Failed to flush firewall")
def sync_gateway_base_url_and_package(self,
sg_ce=False,
sg_platform="centos",
sa_platform="centos"):
platform_extension = {
"centos": "rpm",
"ubuntu": "deb",
"windows": "exe"
}
if self._version_number == "1.1.0" or self._build_number == "1.1.1":
log_info("Version unsupported in provisioning.")
raise ProvisioningError("Unsupport version of sync_gateway")
# http://latestbuilds.service.couchbase.com/couchbase-sync-gateway/release/1.1.1/1.1.1-10/couchbase-sync-gateway-enterprise_1.1.1-10_x86_64.rpm
# base_url = "http://latestbuilds.service.couchbase.com/couchbase-sync-gateway/release/{0}/{1}-{2}".format(version, version, build)
# sg_package_name = "couchbase-sync-gateway-enterprise_{0}-{1}_x86_64.rpm".format(version, build)
else:
# http://latestbuilds.service.couchbase.com/builds/latestbuilds/sync_gateway/1.3.1.5/2/couchbase-sync-gateway-enterprise_1.2.0-6_x86_64.rpm
base_url = "http://latestbuilds.service.couchbase.com/builds/latestbuilds/sync_gateway/{0}/{1}".format(
self._version_number, self._build_number)
sg_type = "enterprise"
if sg_ce:
sg_type = "community"
sg_package_name = "couchbase-sync-gateway-{0}_{1}-{2}_x86_64.{3}".format(
sg_type, self._version_number, self._build_number,
platform_extension[sg_platform])
accel_package_name = "couchbase-sg-accel-enterprise_{0}-{1}_x86_64.{2}".format(
self._version_number, self._build_number,
platform_extension[sa_platform])
return base_url, sg_package_name, accel_package_name
def install_couchbase_server(cluster_config,
couchbase_server_config,
cbs_platform="centos7"):
log_info(cluster_config)
log_info(couchbase_server_config)
ansible_runner = AnsibleRunner(cluster_config)
cluster_keywords = ClusterKeywords()
cluster_topology = cluster_keywords.get_cluster_topology(cluster_config)
server_url = cluster_topology["couchbase_servers"][0]
cb_server = CouchbaseServer(server_url)
log_info(">>> Installing Couchbase Server")
# Install Server
server_baseurl, server_package_name = couchbase_server_config.get_baseurl_package(
cb_server, cbs_platform)
status = ansible_runner.run_ansible_playbook(
"install-couchbase-server-package.yml",
extra_vars={
"couchbase_server_package_base_url": server_baseurl,
"couchbase_server_package_name": server_package_name
})
if status != 0:
raise ProvisioningError("Failed to install Couchbase Server")
# Wait for server to be in 'healthy state'
print(">>> Waiting for server to be in 'healthy' state")
cb_server.wait_for_ready_state()
def set_cluster_config(self, name):
"""Sets CLUSTER_CONFIG environment variable for provisioning
Checks if CLUSTER_CONFIG is set, will fail if it is.
Checks if cluster configuration file exists, will fail if it does not
"""
if "CLUSTER_CONFIG" in os.environ:
raise ProvisioningError("CLUSTER_CONFIG will be set by suite setup. Make sure it is unset.")
path = "{}/{}".format(CLUSTER_CONFIGS_DIR, name)
if not os.path.isfile(path):
raise ProvisioningError("{} not found. Make sure you have generated your cluster configurations.")
log_info("Setting CLUSTER_CONFIG: {}".format(path))
os.environ["CLUSTER_CONFIG"] = path
def build_sgload(ansible_runner):
ansible_status = ansible_runner.run_ansible_playbook(
"build-sgload.yml",
extra_vars={},
)
if ansible_status != 0:
raise ProvisioningError("Failed to build sgload")
def verify_sg_accel_version(host, expected_sg_accel_version):
running_ac_version = get_sg_accel_version(host)
log_info("Expected sg_accel Version: {}".format(expected_sg_accel_version))
log_info("Running sg_accel Version: {}".format(running_ac_version))
if version_is_binary(expected_sg_accel_version):
# Example, 1.2.1-4
if running_ac_version != expected_sg_accel_version:
raise ProvisioningError(
"Unexpected sync_gateway version!! Expected: {} Actual: {}".
format(expected_sg_accel_version, running_ac_version))
else:
# Since sync_gateway does not return the full commit, verify the prefix
if running_ac_version != expected_sg_accel_version[:7]:
raise ProvisioningError(
"Unexpected sync_gateway version!! Expected: {} Actual: {}".
format(expected_sg_accel_version, running_ac_version))
def upgrade_server(self,
cluster_config,
server_version_build,
cbs_platform,
target=None,
toy_build=None):
ansible_runner = AnsibleRunner(cluster_config)
log_info(">>> Upgrading Couchbase Server")
# Install Server
if toy_build:
# http://server.jenkins.couchbase.com/view/All/job/watson-toy/1770/artifact/couchbase-server-enterprise-5.0.0-9900-centos7.x86_64.rpm
toy_build_url_parts = toy_build.split('/')
toy_build_url_len = len(toy_build_url_parts)
server_package_name = toy_build_url_parts[-1]
server_baseurl = "/".join(
toy_build_url_parts[0:(toy_build_url_len - 1)])
else:
version_build = server_version_build.split("-")
server_verion = version_build[0]
if len(version_build) == 2:
# Build number is included
server_build = version_build[1]
else:
server_build = None
if server_build is None:
server_baseurl, server_package_name = self.resolve_cb_mobile_url(
server_verion, cbs_platform)
else:
server_baseurl, server_package_name = self.resolve_cb_nas_url(
server_verion, server_build, cbs_platform)
if target is not None:
target = hostname_for_url(cluster_config, target)
log_info("Upgrading Couchbase server on {} ...".format(target))
status = ansible_runner.run_ansible_playbook(
"upgrade-couchbase-server-package.yml",
subset=target,
extra_vars={
"couchbase_server_package_base_url": server_baseurl,
"couchbase_server_package_name": server_package_name
})
else:
log_info("Upgrading Couchbase server on all nodes")
status = ansible_runner.run_ansible_playbook(
"upgrade-couchbase-server-package.yml",
extra_vars={
"couchbase_server_package_base_url": server_baseurl,
"couchbase_server_package_name": server_package_name
})
if status != 0:
raise ProvisioningError("Failed to install Couchbase Server")
self.wait_for_ready_state()
def generate_cluster_configs_from_vagrant(private_network, public_network,
public_network_ethernet):
"""
1. Gets the status for a running vagrant vm set.
2. Uses the host name to look up the ip allocated to each vagrant vm instance
3. Uses this IP list to build a pool.json file and generate the cluster configurations
"""
check_network_options(private_network, public_network,
public_network_ethernet)
cwd = os.getcwd()
# Change directory to where the appropriate Vagrantfile lives
if private_network:
os.chdir("vagrant/private_network")
elif public_network:
os.chdir("vagrant/public_network")
else:
os.chdir("vagrant/public_network_ethernet")
v = vagrant.Vagrant()
status = v.status()
# Get vagrant ips
vagrant_ips = []
print("Getting ip addresses from running vagrant vms ...")
for stat in status:
name = stat.name
# Expected output: '10.0.2.15 192.168.0.61 2605:e000:9092:200:a00:27ff:fe7b:9bbf \r\n'
# where second ip is the publicly routable ip
output = subprocess.check_output(
'vagrant ssh {} -c "hostname -I"'.format(name), shell=True)
cleaned_output = output.strip()
ip_addresses = cleaned_output.split()
if len(ip_addresses) < 2:
raise ProvisioningError(
"Expected at least 2 ip addresses hostname -I result: {}".
format(ip_addresses))
public_ip = ip_addresses[1]
print("host: {} ip: {}".format(name, public_ip))
vagrant_ips.append(public_ip)
# Restore previous directory
os.chdir(cwd)
# Write pool.json
pool_file = "resources/pool.json"
pool_def = {"ips": vagrant_ips}
with open(pool_file, "w") as f:
print("Writing 'resources/pool.json' ...")
f.write(json.dumps(pool_def, indent=4))
# Generate cluster configs
print("Generating cluster_configs ...")
generate_clusters_from_pool(pool_file)
def unset_cluster_config(self):
"""Will unset the CLUSTER_CONFIG environment variable if it is set.
Will fail if CLUSTER_CONFIG is not set
"""
if "CLUSTER_CONFIG" not in os.environ:
raise ProvisioningError("Trying to unset CLUSTER_CONFIG but it is not defined")
log_info("Unsetting CLUSTER_CONFIG")
del os.environ["CLUSTER_CONFIG"]
def check_network_options(private_network, public_network,
public_network_ethernet):
# Check if only one of the options is set, private_network or public_network or public_network_ethernet
if private_network and (public_network or public_network_ethernet):
raise ProvisioningError(
"Invalid private_network and public_network/public_network_ethernet flags"
)
elif public_network and (private_network or public_network_ethernet):
raise ProvisioningError(
"Invalid public_network and private_network/public_network_ethernet flags"
)
elif public_network_ethernet and (public_network or private_network):
raise ProvisioningError(
"Invalid public_network_ethernet and public_network/private_network flags"
)
# Check if none of the options are set
if not private_network and not public_network and not public_network_ethernet:
raise ProvisioningError(
"Invalid private_network, public_network and public_network_ethernet flags"
)
def get_load_balancer_ip(cluster_config):
""" Loads cluster config to fetch load balancer ip """
cluster = load_cluster_config_json(cluster_config)
num_lbs = len(cluster["load_balancers"])
if num_lbs != 1:
raise ProvisioningError(
"Expecting exactly 1 load balancer IP in {}".format(
cluster_config))
lb_ip = cluster["load_balancers"][0]["ip"]
return lb_ip
def verify_sync_gateway_product_info(host):
""" Get the product information from host and verify for Sync Gateway:
- vendor name in GET / request
- Server header in response
"""
resp = requests.get("http://{}:4984".format(host))
log_r(resp)
resp.raise_for_status()
resp_obj = resp.json()
server_header = resp.headers["server"]
log_info("'server' header: {}".format(server_header))
if not server_header.startswith("Couchbase Sync Gateway"):
raise ProvisioningError(
"Wrong product info. Expected 'Couchbase Sync Gateway'")
vendor_name = resp_obj["vendor"]["name"]
log_info("vendor name: {}".format(vendor_name))
if vendor_name != "Couchbase Sync Gateway":
raise ProvisioningError(
"Wrong vendor name. Expected 'Couchbase Sync Gateway'")
def collect_packet_capture(self, cluster_config, test_name):
ansible_runner = AnsibleRunner(config=cluster_config)
status = ansible_runner.run_ansible_playbook("collect-ngrep.yml")
if status != 0:
raise ProvisioningError("Failed to collect packet capture")
# zip logs and timestamp
if os.path.isdir("/tmp/sys-logs"):
date_time = time.strftime("%Y-%m-%d-%H-%M-%S")
name = "/tmp/ngrep-{}-{}-output".format(test_name, date_time)
shutil.make_archive(name, "zip", "/tmp/sys-logs")
shutil.rmtree("/tmp/sys-logs")
print("ngrep logs copied here {}.zip\n".format(name))
def install_aws_credentials(cluster_config, aws_access_key_id, aws_secret_access_key):
log_info("Installing aws credentials for cluster_config: {}".format(cluster_config))
ansible_runner = AnsibleRunner(config=cluster_config)
status = ansible_runner.run_ansible_playbook(
"install-aws-credentials.yml",
extra_vars={
"aws_access_key_id": aws_access_key_id,
"aws_secret_access_key": aws_secret_access_key,
},
)
if status != 0:
raise ProvisioningError("Failed to aws credentials")
def restart_sync_gateways(self, cluster_config, url=None):
""" Restart sync gateways in a cluster. If url is passed, restart
the sync gateway at that url
"""
ansible_runner = AnsibleRunner(cluster_config)
if url is not None:
target = hostname_for_url(cluster_config, url)
log_info("Restarting sync_gateway on {} ...".format(target))
status = ansible_runner.run_ansible_playbook(
"restart-sync-gateway.yml", subset=target)
else:
log_info("Restarting all sync_gateways")
status = ansible_runner.run_ansible_playbook(
"restart-sync-gateway.yml", )
if status != 0:
raise ProvisioningError("Could not restart sync_gateway")
def persist_cluster_config_environment_prop(cluster_config,
property_name,
value,
property_name_check=True):
""" Loads the cluster_config and sets
[environment]
property_name=value
for cluster_config and
"property_name": value
for cluster_config.json
"""
if property_name_check is True:
valid_props = [
"cbs_ssl_enabled", "xattrs_enabled", "sg_lb_enabled",
"sync_gateway_version", "server_version", "no_conflicts_enabled"
]
if property_name not in valid_props:
raise ProvisioningError(
"Make sure the property you are trying to change is one of: {}"
.format(valid_props))
# Write property = value in the cluster_config.json
cluster_config_json = "{}.json".format(cluster_config)
with open(cluster_config_json) as f:
cluster = json.loads(f.read())
cluster["environment"][property_name] = value
with open(cluster_config_json, "w") as f:
json.dump(cluster, f, indent=4)
# Write [section] property = value in the cluster_config
config = CustomConfigParser()
config.read(cluster_config)
config.set('environment', property_name, str(value))
with open(cluster_config, 'w') as f:
config.write(f)
def _get_mem_total_lowest(self, server_info):
# Workaround for https://github.com/couchbaselabs/mobile-testkit/issues/709
# Later updated for https://github.com/couchbaselabs/mobile-testkit/issues/1038
# where some node report mem_total = 0. Loop over all the nodes and find the smallest non-zero val
mem_total_lowest = None
for node in server_info["nodes"]:
mem_total = node["systemStats"]["mem_total"]
if mem_total == 0:
# ignore nodes that report mem_total = 0
continue
if mem_total_lowest is None:
# no previous value for mem_total_lowest, use non-zero value we got back from node
mem_total_lowest = mem_total
elif mem_total < mem_total_lowest:
# only use it if it's lower than previous low
mem_total_lowest = mem_total
if mem_total_lowest is None:
raise ProvisioningError("All nodes reported 0MB of RAM available")
return mem_total_lowest
def run_sgload_perf_test(cluster_config, sgload_arg_list_main,
skip_build_sgload):
print(
"Running sgload perf test against cluster: {}".format(cluster_config))
main_ansible_runner = AnsibleRunner(cluster_config)
print(">>> Starting profile collection scripts")
status = main_ansible_runner.run_ansible_playbook(
"start-profile-collection.yml",
extra_vars={
"stats_run_time": 3600,
"delay_profiling_secs": 60
},
)
assert status == 0, "Could not start profiling collection scripts"
# Install + configure telegraf
status = main_ansible_runner.run_ansible_playbook("install-telegraf.yml")
if status != 0:
raise ProvisioningError("Failed to install telegraf")
# build_sgload (ansible)
if not skip_build_sgload:
build_sgload(main_ansible_runner)
# get load generator and sg hostnames
lg_hosts_main = get_load_generators_hosts(cluster_config)
sg_hosts_main = get_sync_gateways_hosts(cluster_config)
# Get the first SG host from the list of SG hosts
sg_host_main = sg_hosts_main[0]
run_sgload_on_single_loadgenerator(lg_hosts_main, sgload_arg_list_main,
sg_host_main)
log_info("Finished")
def reset(self, sg_config_path):
ansible_runner = AnsibleRunner(self._cluster_config)
log_info(">>> Reseting cluster ...")
log_info(">>> CBS SSL enabled: {}".format(self.cbs_ssl))
log_info(">>> Using xattrs: {}".format(self.xattrs))
# Stop sync_gateways
log_info(">>> Stopping sync_gateway")
status = ansible_runner.run_ansible_playbook("stop-sync-gateway.yml")
assert status == 0, "Failed to stop sync gateway"
# Stop sync_gateway accels
log_info(">>> Stopping sg_accel")
status = ansible_runner.run_ansible_playbook("stop-sg-accel.yml")
assert status == 0, "Failed to stop sg_accel"
# Deleting sync_gateway artifacts
log_info(">>> Deleting sync_gateway artifacts")
status = ansible_runner.run_ansible_playbook(
"delete-sync-gateway-artifacts.yml")
assert status == 0, "Failed to delete sync_gateway artifacts"
# Deleting sg_accel artifacts
log_info(">>> Deleting sg_accel artifacts")
status = ansible_runner.run_ansible_playbook(
"delete-sg-accel-artifacts.yml")
assert status == 0, "Failed to delete sg_accel artifacts"
# Delete buckets
log_info(">>> Deleting buckets on: {}".format(self.servers[0].url))
self.servers[0].delete_buckets()
# Parse config and grab bucket names
config_path_full = os.path.abspath(sg_config_path)
config = Config(config_path_full)
mode = config.get_mode()
bucket_name_set = config.get_bucket_name_set()
self.sync_gateway_config = config
is_valid, reason = validate_cluster(self.sync_gateways, self.sg_accels,
config)
if not is_valid:
raise ProvisioningError(reason)
log_info(">>> Creating buckets on: {}".format(self.servers[0].url))
log_info(">>> Creating buckets {}".format(bucket_name_set))
self.servers[0].create_buckets(bucket_name_set)
# Wait for server to be in a warmup state to work around
# https://github.com/couchbase/sync_gateway/issues/1745
log_info(">>> Waiting for Server: {} to be in a healthy state".format(
self.servers[0].url))
self.servers[0].wait_for_ready_state()
log_info(">>> Starting sync_gateway with configuration: {}".format(
config_path_full))
server_port = 8091
server_scheme = "http"
couchbase_server_primary_node = add_cbs_to_sg_config_server_field(
self._cluster_config)
if self.cbs_ssl:
server_port = 18091
server_scheme = "https"
# Start sync-gateway
playbook_vars = {
"sync_gateway_config_filepath": config_path_full,
"server_port": server_port,
"server_scheme": server_scheme,
"autoimport": "",
"xattrs": "",
"no_conflicts": "",
"revs_limit": "",
"couchbase_server_primary_node": couchbase_server_primary_node
}
# Add configuration to run with xattrs
if self.xattrs:
playbook_vars["autoimport"] = '"import_docs": "continuous",'
playbook_vars["xattrs"] = '"enable_shared_bucket_access": true,'
if no_conflicts_enabled(self._cluster_config):
playbook_vars["no_conflicts"] = '"allow_conflicts": false,'
try:
revs_limit = get_revs_limit(self._cluster_config)
playbook_vars["revs_limit"] = '"revs_limit": {},'.format(
revs_limit)
except KeyError as ex:
log_info("Keyerror in getting revs_limit{}".format(ex.message))
status = ansible_runner.run_ansible_playbook("start-sync-gateway.yml",
extra_vars=playbook_vars)
assert status == 0, "Failed to start to Sync Gateway"
# HACK - only enable sg_accel for distributed index tests
# revise this with https://github.com/couchbaselabs/sync-gateway-testcluster/issues/222
if mode == "di":
# Start sg-accel
status = ansible_runner.run_ansible_playbook(
"start-sg-accel.yml", extra_vars=playbook_vars)
assert status == 0, "Failed to start sg_accel"
# Validate CBGT
if mode == "di":
if not self.validate_cbgt_pindex_distribution_retry(
len(self.sg_accels)):
self.save_cbgt_diagnostics()
raise Exception("Failed to validate CBGT Pindex distribution")
log_info(">>> Detected valid CBGT Pindex distribution")
else:
log_info(">>> Running in channel cache")
return mode
def provision_cluster(cluster_config, couchbase_server_config, sync_gateway_config, sg_ce=False, cbs_platform="centos7", sg_platform="centos", sa_platform="centos"):
log_info("\n>>> Cluster info:\n")
server_version = "{}-{}".format(couchbase_server_config.version, couchbase_server_config.build)
sg_version = "{}-{}".format(sync_gateway_config._version_number, sync_gateway_config._build_number)
try:
server_version
except NameError:
log_info("Server version is not provided")
persist_cluster_config_environment_prop(cluster_config, 'server_version', "")
else:
log_info("Running test with server version {}".format(server_version))
persist_cluster_config_environment_prop(cluster_config, 'server_version', server_version)
try:
sg_version
except NameError:
log_info("Sync gateway version is not provided")
persist_cluster_config_environment_prop(cluster_config, 'sync_gateway_version', "")
else:
log_info("Running test with sync_gateway version {}".format(sg_version))
persist_cluster_config_environment_prop(cluster_config, 'sync_gateway_version', sg_version)
with open(cluster_config, "r") as ansible_hosts:
log_info(ansible_hosts.read())
log_info(couchbase_server_config)
log_info(sync_gateway_config)
if not sync_gateway_config.is_valid():
log_info("Invalid sync_gateway provisioning configuration. Exiting ...")
sys.exit(1)
cluster = Cluster(config=cluster_config)
config_path_full = os.path.abspath(sync_gateway_config.config_path)
config = Config(config_path_full)
is_valid, reason = validate_cluster(
cluster.sync_gateways,
cluster.sg_accels,
config,
)
if not is_valid:
raise ProvisioningError(reason)
log_info(">>> Provisioning cluster...")
# Get server base url and package name
cluster_keywords = ClusterKeywords()
cluster_topology = cluster_keywords.get_cluster_topology(cluster_config)
server_url = cluster_topology["couchbase_servers"][0]
cb_server = CouchbaseServer(server_url)
server_baseurl, server_package_name = couchbase_server_config.get_baseurl_package(cb_server, cbs_platform)
log_info(">>> Server package: {0}/{1}".format(server_baseurl, server_package_name))
log_info(">>> Using sync_gateway config: {}".format(sync_gateway_config.config_path))
# Reset previous installs
clean_cluster(cluster_config)
# Install server package
log_info("Installing Couchbase Server")
install_couchbase_server.install_couchbase_server(
cluster_config=cluster_config,
couchbase_server_config=couchbase_server_config,
cbs_platform=cbs_platform
)
# Install sync_gateway
log_info("Installing Sync Gateway")
install_sync_gateway.install_sync_gateway(
cluster_config=cluster_config,
sync_gateway_config=sync_gateway_config,
sg_platform=sg_platform,
sa_platform=sa_platform,
sg_ce=sg_ce
)
# Install nginx
install_nginx(cluster_config)
log_info(">>> Done provisioning cluster...")
action="store", dest="delay_expvar_collect_secs",
default=None,
help="The delay time between expvar collection")
arg_parameters = sys.argv[1:]
(opts, args) = parser.parse_args(arg_parameters)
if opts.test_id is None:
print("You must provide a test identifier to run the test")
sys.exit(1)
# Validate feed_type
valid_feed_types = ["continuous", "longpoll"]
if opts.feed_type not in valid_feed_types:
raise ProvisioningError("Make sure you provide a valid feed type!")
# Wrap gateload params into a named tuple
gateload_params_from_args = GateloadParams(
number_pullers=opts.number_pullers,
number_pushers=opts.number_pushers,
doc_size=opts.doc_size,
runtime_ms=opts.runtime_ms,
rampup_interval_ms=opts.rampup_interval_ms,
feed_type=opts.feed_type,
sleep_time_ms=opts.sleep_time_ms,
channel_active_users=opts.channel_active_users,
channel_concurrent_users=opts.channel_concurrent_users
)
# Start load generator
def run_gateload_perf_test(gen_gateload_config, test_id, gateload_params, delay_profiling_secs, delay_expvar_collect_secs):
try:
cluster_config = os.environ["CLUSTER_CONFIG"]
except KeyError:
print ("Make sure CLUSTER_CONFIG is defined and pointing to the configuration you would like to provision")
sys.exit(1)
print("Running perf test against cluster: {}".format(cluster_config))
ansible_runner = AnsibleRunner(cluster_config)
# Install + configure telegraf
status = ansible_runner.run_ansible_playbook("install-telegraf.yml")
if status != 0:
raise ProvisioningError("Failed to install telegraf")
test_run_id = "{}_{}".format(test_id, time.strftime("%Y-%m-%d-%H-%M-%S"))
# Create test results directory
os.makedirs("testsuites/syncgateway/performance/results/{}".format(test_run_id))
# Copy provisioning_config to performance_results/ folder
shutil.copy("{}".format(cluster_config), "testsuites/syncgateway/performance/results/{}".format(test_run_id))
if int(gateload_params.number_pullers) > 0 and not gen_gateload_config:
raise Exception("You specified --num-pullers but did not set --gen-gateload-config")
# Build gateload
print(">>> Building gateload")
status = ansible_runner.run_ansible_playbook(
"build-gateload.yml",
extra_vars={},
)
assert status == 0, "Could not build gateload"
# Generate gateload config
print(">>> Generate gateload configs")
if gen_gateload_config:
generate_gateload_configs.main(
cluster_config=cluster_config,
test_id=test_run_id,
gateload_params=gateload_params
)
print(">>> Starting profile collection scripts")
runtime_s = int(gateload_params.runtime_ms) // 1000
status = ansible_runner.run_ansible_playbook(
"start-profile-collection.yml",
extra_vars={
"stats_run_time": runtime_s,
"delay_profiling_secs": int(delay_profiling_secs)
},
)
assert status == 0, "Could not start profiling collection scripts"
# Start gateload
print(">>> Starting gateload with {0} pullers and {1} pushers".format(
gateload_params.number_pullers, gateload_params.number_pushers
))
status = ansible_runner.run_ansible_playbook(
"start-gateload.yml",
extra_vars={
"delay_expvar_collect_secs": int(delay_expvar_collect_secs)
},
)
assert status == 0, "Could not start gateload"
# write expvars to file, will exit when gateload scenario is done
print(">>> Logging expvars")
gateload_finished_successfully = log_expvars(cluster_config, test_run_id)
print(">>> Fetch Sync Gateway profile")
fetch_sync_gateway_profile(cluster_config, test_run_id)
print(">>> Shutdown gateload")
kill_gateload()
if not gateload_finished_successfully:
raise RuntimeError("It appears that gateload did not finish successfully. Check logs for details")
def stop_packet_capture(self, cluster_config):
ansible_runner = AnsibleRunner(config=cluster_config)
status = ansible_runner.run_ansible_playbook("stop-ngrep.yml")
if status != 0:
raise ProvisioningError("Failed to stop packet capture")
def install_sync_gateway(cluster_config,
sync_gateway_config,
sg_ce=False,
sg_platform="centos",
sa_platform="centos"):
log_info(sync_gateway_config)
if sync_gateway_config.build_flags != "":
log_warn(
"\n\n!!! WARNING: You are building with flags: {} !!!\n\n".format(
sync_gateway_config.build_flags))
ansible_runner = AnsibleRunner(cluster_config)
config_path = os.path.abspath(sync_gateway_config.config_path)
couchbase_server_primary_node = add_cbs_to_sg_config_server_field(
cluster_config)
# Create buckets unless the user explicitly asked to skip this step
if not sync_gateway_config.skip_bucketcreation:
create_server_buckets(cluster_config, sync_gateway_config)
server_port = 8091
server_scheme = "http"
if is_cbs_ssl_enabled(cluster_config):
server_port = 18091
server_scheme = "https"
# Shared vars
playbook_vars = {
"sync_gateway_config_filepath": config_path,
"server_port": server_port,
"server_scheme": server_scheme,
"autoimport": "",
"xattrs": "",
"no_conflicts": "",
"couchbase_server_primary_node": couchbase_server_primary_node
}
if is_xattrs_enabled(cluster_config):
playbook_vars["autoimport"] = '"import_docs": "continuous",'
playbook_vars["xattrs"] = '"enable_shared_bucket_access": true,'
if no_conflicts_enabled(cluster_config):
playbook_vars["no_conflicts"] = '"allow_conflicts": false,'
try:
revs_limit = get_revs_limit(cluster_config)
playbook_vars["revs_limit"] = '"revs_limit": {},'.format(revs_limit)
except KeyError as ex:
log_info("Keyerror in getting revs_limit{}".format(ex.message))
# Install Sync Gateway via Source or Package
if sync_gateway_config.commit is not None:
# Install from source
playbook_vars["commit"] = sync_gateway_config.commit
playbook_vars["build_flags"] = sync_gateway_config.build_flags
status = ansible_runner.run_ansible_playbook(
"install-sync-gateway-source.yml", extra_vars=playbook_vars)
if status != 0:
raise ProvisioningError("Failed to install sync_gateway source")
else:
# Install from Package
sync_gateway_base_url, sync_gateway_package_name, sg_accel_package_name = sync_gateway_config.sync_gateway_base_url_and_package(
sg_ce=sg_ce, sg_platform=sg_platform, sa_platform=sa_platform)
playbook_vars[
"couchbase_sync_gateway_package_base_url"] = sync_gateway_base_url
playbook_vars[
"couchbase_sync_gateway_package"] = sync_gateway_package_name
playbook_vars["couchbase_sg_accel_package"] = sg_accel_package_name
if sg_platform == "windows":
status = ansible_runner.run_ansible_playbook(
"install-sync-gateway-package-windows.yml",
extra_vars=playbook_vars)
else:
status = ansible_runner.run_ansible_playbook(
"install-sync-gateway-package.yml", extra_vars=playbook_vars)
if status != 0:
raise ProvisioningError("Failed to install sync_gateway package")
if sa_platform == "windows":
status = ansible_runner.run_ansible_playbook(
"install-sg-accel-package-windows.yml",
extra_vars=playbook_vars)
else:
status = ansible_runner.run_ansible_playbook(
"install-sg-accel-package.yml", extra_vars=playbook_vars)
if status != 0:
raise ProvisioningError("Failed to install sg_accel package")
# Configure aws cloudwatch logs forwarder
status = ansible_runner.run_ansible_playbook(
"configure-sync-gateway-awslogs-forwarder.yml", extra_vars={})
if status != 0:
raise ProvisioningError(
"Failed to configure sync_gateway awslogs forwarder")