def test_config_init(sync_gateway_config, buckets, mode):
cwd = os.getcwd()
conf_path = cwd + "/resources/sync_gateway_configs/" + sync_gateway_config
config = Config(conf_path)
assert config
assert config.mode == mode
assert config.bucket_name_set == buckets
def get_buckets_from_sync_gateway_config(sync_gateway_config_path):
# Remove the sync function before trying to extract the bucket names
with open(sync_gateway_config_path) as fp:
conf_data = fp.read()
fp.close()
temp_config_path = ""
temp_config = ""
# Check if a sync function id defined between ` `
if re.search('`', conf_data):
log_info("Ignoring the sync function to extract bucket names")
conf = re.split('`', conf_data)
split_len = len(conf)
# Replace the sync function with a string "function"
for i in range(0, split_len, 2):
if i == split_len - 1:
temp_config += conf[i]
else:
temp_config += conf[i] + " \"function\" "
temp_config_path = "/".join(
sync_gateway_config_path.split('/')[:-2]) + '/temp_conf.json'
with open(temp_config_path, 'w') as fp:
fp.write(temp_config)
config_path_full = os.path.abspath(temp_config_path)
else:
config_path_full = os.path.abspath(sync_gateway_config_path)
config = Config(config_path_full)
bucket_name_set = config.get_bucket_name_set()
if os.path.exists(temp_config_path):
os.remove(temp_config_path)
return bucket_name_set
def reset(self, sg_config_path):
self.validate_cluster()
ansible_runner = AnsibleRunner(self._cluster_config)
# Stop sync_gateways
log_info(">>> Stopping sync_gateway")
status = ansible_runner.run_ansible_playbook("stop-sync-gateway.yml")
assert status == 0, "Failed to stop sync gateway"
# Stop sync_gateways
log_info(">>> Stopping sg_accel")
status = ansible_runner.run_ansible_playbook("stop-sg-accel.yml")
assert status == 0, "Failed to stop sg_accel"
# Deleting sync_gateway artifacts
log_info(">>> Deleting sync_gateway artifacts")
status = ansible_runner.run_ansible_playbook("delete-sync-gateway-artifacts.yml")
assert status == 0, "Failed to delete sync_gateway artifacts"
# Deleting sg_accel artifacts
log_info(">>> Deleting sg_accel artifacts")
status = ansible_runner.run_ansible_playbook("delete-sg-accel-artifacts.yml")
assert status == 0, "Failed to delete sg_accel artifacts"
# Delete buckets
log_info(">>> Deleting buckets on: {}".format(self.cb_server.url))
self.cb_server.delete_buckets()
# Parse config and grab bucket names
config_path_full = os.path.abspath(sg_config_path)
config = Config(config_path_full)
mode = config.get_mode()
bucket_name_set = config.get_bucket_name_set()
self.sync_gateway_config = config
log_info(">>> Creating buckets on: {}".format(self.cb_server.url))
log_info(">>> Creating buckets {}".format(bucket_name_set))
self.cb_server.create_buckets(bucket_name_set)
# Wait for server to be in a warmup state to work around
# https://github.com/couchbase/sync_gateway/issues/1745
log_info(">>> Waiting for Server: {} to be in a healthy state".format(self.cb_server.url))
self.cb_server.wait_for_ready_state()
log_info(">>> Starting sync_gateway with configuration: {}".format(config_path_full))
utils.dump_file_contents_to_logs(config_path_full)
# Start sync-gateway
status = ansible_runner.run_ansible_playbook(
"start-sync-gateway.yml",
extra_vars={
"sync_gateway_config_filepath": config_path_full
}
)
assert status == 0, "Failed to start to Sync Gateway"
# HACK - only enable sg_accel for distributed index tests
# revise this with https://github.com/couchbaselabs/sync-gateway-testcluster/issues/222
if mode == "di":
# Start sg-accel
status = ansible_runner.run_ansible_playbook(
"start-sg-accel.yml",
extra_vars={
"sync_gateway_config_filepath": config_path_full
}
)
assert status == 0, "Failed to start sg_accel"
# Validate CBGT
if mode == "di":
if not self.validate_cbgt_pindex_distribution_retry(len(self.sg_accels)):
self.save_cbgt_diagnostics()
raise Exception("Failed to validate CBGT Pindex distribution")
log_info(">>> Detected valid CBGT Pindex distribution")
else:
log_info(">>> Running in channel cache")
return mode
def provision_cluster(cluster_config, couchbase_server_config, sync_gateway_config, sg_ce=False, cbs_platform="centos7", sg_platform="centos", sa_platform="centos"):
log_info("\n>>> Cluster info:\n")
server_version = "{}-{}".format(couchbase_server_config.version, couchbase_server_config.build)
sg_version = "{}-{}".format(sync_gateway_config._version_number, sync_gateway_config._build_number)
try:
server_version
except NameError:
log_info("Server version is not provided")
persist_cluster_config_environment_prop(cluster_config, 'server_version', "")
else:
log_info("Running test with server version {}".format(server_version))
persist_cluster_config_environment_prop(cluster_config, 'server_version', server_version)
try:
sg_version
except NameError:
log_info("Sync gateway version is not provided")
persist_cluster_config_environment_prop(cluster_config, 'sync_gateway_version', "")
else:
log_info("Running test with sync_gateway version {}".format(sg_version))
persist_cluster_config_environment_prop(cluster_config, 'sync_gateway_version', sg_version)
with open(cluster_config, "r") as ansible_hosts:
log_info(ansible_hosts.read())
log_info(couchbase_server_config)
log_info(sync_gateway_config)
if not sync_gateway_config.is_valid():
log_info("Invalid sync_gateway provisioning configuration. Exiting ...")
sys.exit(1)
cluster = Cluster(config=cluster_config)
config_path_full = os.path.abspath(sync_gateway_config.config_path)
config = Config(config_path_full)
is_valid, reason = validate_cluster(
cluster.sync_gateways,
cluster.sg_accels,
config,
)
if not is_valid:
raise ProvisioningError(reason)
log_info(">>> Provisioning cluster...")
# Get server base url and package name
cluster_keywords = ClusterKeywords()
cluster_topology = cluster_keywords.get_cluster_topology(cluster_config)
server_url = cluster_topology["couchbase_servers"][0]
cb_server = CouchbaseServer(server_url)
server_baseurl, server_package_name = couchbase_server_config.get_baseurl_package(cb_server, cbs_platform)
log_info(">>> Server package: {0}/{1}".format(server_baseurl, server_package_name))
log_info(">>> Using sync_gateway config: {}".format(sync_gateway_config.config_path))
# Reset previous installs
clean_cluster(cluster_config)
# Install server package
log_info("Installing Couchbase Server")
install_couchbase_server.install_couchbase_server(
cluster_config=cluster_config,
couchbase_server_config=couchbase_server_config,
cbs_platform=cbs_platform
)
# Install sync_gateway
log_info("Installing Sync Gateway")
install_sync_gateway.install_sync_gateway(
cluster_config=cluster_config,
sync_gateway_config=sync_gateway_config,
sg_platform=sg_platform,
sa_platform=sa_platform,
sg_ce=sg_ce
)
# Install nginx
install_nginx(cluster_config)
log_info(">>> Done provisioning cluster...")
def get_buckets_from_sync_gateway_config(sync_gateway_config_path):
config_path_full = os.path.abspath(sync_gateway_config_path)
config = Config(config_path_full)
bucket_name_set = config.get_bucket_name_set()
return bucket_name_set
def reset(self, sg_config_path):
ansible_runner = AnsibleRunner(self._cluster_config)
log_info(">>> Reseting cluster ...")
log_info(">>> CBS SSL enabled: {}".format(self.cbs_ssl))
log_info(">>> Using xattrs: {}".format(self.xattrs))
# Stop sync_gateways
log_info(">>> Stopping sync_gateway")
status = ansible_runner.run_ansible_playbook("stop-sync-gateway.yml")
assert status == 0, "Failed to stop sync gateway"
# Stop sync_gateway accels
log_info(">>> Stopping sg_accel")
status = ansible_runner.run_ansible_playbook("stop-sg-accel.yml")
assert status == 0, "Failed to stop sg_accel"
# Deleting sync_gateway artifacts
log_info(">>> Deleting sync_gateway artifacts")
status = ansible_runner.run_ansible_playbook(
"delete-sync-gateway-artifacts.yml")
assert status == 0, "Failed to delete sync_gateway artifacts"
# Deleting sg_accel artifacts
log_info(">>> Deleting sg_accel artifacts")
status = ansible_runner.run_ansible_playbook(
"delete-sg-accel-artifacts.yml")
assert status == 0, "Failed to delete sg_accel artifacts"
# Delete buckets
log_info(">>> Deleting buckets on: {}".format(self.servers[0].url))
self.servers[0].delete_buckets()
# Parse config and grab bucket names
config_path_full = os.path.abspath(sg_config_path)
config = Config(config_path_full)
mode = config.get_mode()
bucket_name_set = config.get_bucket_name_set()
self.sync_gateway_config = config
is_valid, reason = validate_cluster(self.sync_gateways, self.sg_accels,
config)
if not is_valid:
raise ProvisioningError(reason)
log_info(">>> Creating buckets on: {}".format(self.servers[0].url))
log_info(">>> Creating buckets {}".format(bucket_name_set))
self.servers[0].create_buckets(bucket_name_set)
# Wait for server to be in a warmup state to work around
# https://github.com/couchbase/sync_gateway/issues/1745
log_info(">>> Waiting for Server: {} to be in a healthy state".format(
self.servers[0].url))
self.servers[0].wait_for_ready_state()
log_info(">>> Starting sync_gateway with configuration: {}".format(
config_path_full))
server_port = 8091
server_scheme = "http"
couchbase_server_primary_node = add_cbs_to_sg_config_server_field(
self._cluster_config)
if self.cbs_ssl:
server_port = 18091
server_scheme = "https"
# Start sync-gateway
playbook_vars = {
"sync_gateway_config_filepath": config_path_full,
"server_port": server_port,
"server_scheme": server_scheme,
"autoimport": "",
"xattrs": "",
"no_conflicts": "",
"revs_limit": "",
"couchbase_server_primary_node": couchbase_server_primary_node
}
# Add configuration to run with xattrs
if self.xattrs:
playbook_vars["autoimport"] = '"import_docs": "continuous",'
playbook_vars["xattrs"] = '"enable_shared_bucket_access": true,'
if no_conflicts_enabled(self._cluster_config):
playbook_vars["no_conflicts"] = '"allow_conflicts": false,'
try:
revs_limit = get_revs_limit(self._cluster_config)
playbook_vars["revs_limit"] = '"revs_limit": {},'.format(
revs_limit)
except KeyError as ex:
log_info("Keyerror in getting revs_limit{}".format(ex.message))
status = ansible_runner.run_ansible_playbook("start-sync-gateway.yml",
extra_vars=playbook_vars)
assert status == 0, "Failed to start to Sync Gateway"
# HACK - only enable sg_accel for distributed index tests
# revise this with https://github.com/couchbaselabs/sync-gateway-testcluster/issues/222
if mode == "di":
# Start sg-accel
status = ansible_runner.run_ansible_playbook(
"start-sg-accel.yml", extra_vars=playbook_vars)
assert status == 0, "Failed to start sg_accel"
# Validate CBGT
if mode == "di":
if not self.validate_cbgt_pindex_distribution_retry(
len(self.sg_accels)):
self.save_cbgt_diagnostics()
raise Exception("Failed to validate CBGT Pindex distribution")
log_info(">>> Detected valid CBGT Pindex distribution")
else:
log_info(">>> Running in channel cache")
return mode