def test_config_init(sync_gateway_config, buckets, mode): cwd = os.getcwd() conf_path = cwd + "/resources/sync_gateway_configs/" + sync_gateway_config config = Config(conf_path) assert config assert config.mode == mode assert config.bucket_name_set == buckets
def get_buckets_from_sync_gateway_config(sync_gateway_config_path): # Remove the sync function before trying to extract the bucket names with open(sync_gateway_config_path) as fp: conf_data = fp.read() fp.close() temp_config_path = "" temp_config = "" # Check if a sync function id defined between ` ` if re.search('`', conf_data): log_info("Ignoring the sync function to extract bucket names") conf = re.split('`', conf_data) split_len = len(conf) # Replace the sync function with a string "function" for i in range(0, split_len, 2): if i == split_len - 1: temp_config += conf[i] else: temp_config += conf[i] + " \"function\" " temp_config_path = "/".join( sync_gateway_config_path.split('/')[:-2]) + '/temp_conf.json' with open(temp_config_path, 'w') as fp: fp.write(temp_config) config_path_full = os.path.abspath(temp_config_path) else: config_path_full = os.path.abspath(sync_gateway_config_path) config = Config(config_path_full) bucket_name_set = config.get_bucket_name_set() if os.path.exists(temp_config_path): os.remove(temp_config_path) return bucket_name_set
def reset(self, sg_config_path): self.validate_cluster() ansible_runner = AnsibleRunner(self._cluster_config) # Stop sync_gateways log_info(">>> Stopping sync_gateway") status = ansible_runner.run_ansible_playbook("stop-sync-gateway.yml") assert status == 0, "Failed to stop sync gateway" # Stop sync_gateways log_info(">>> Stopping sg_accel") status = ansible_runner.run_ansible_playbook("stop-sg-accel.yml") assert status == 0, "Failed to stop sg_accel" # Deleting sync_gateway artifacts log_info(">>> Deleting sync_gateway artifacts") status = ansible_runner.run_ansible_playbook("delete-sync-gateway-artifacts.yml") assert status == 0, "Failed to delete sync_gateway artifacts" # Deleting sg_accel artifacts log_info(">>> Deleting sg_accel artifacts") status = ansible_runner.run_ansible_playbook("delete-sg-accel-artifacts.yml") assert status == 0, "Failed to delete sg_accel artifacts" # Delete buckets log_info(">>> Deleting buckets on: {}".format(self.cb_server.url)) self.cb_server.delete_buckets() # Parse config and grab bucket names config_path_full = os.path.abspath(sg_config_path) config = Config(config_path_full) mode = config.get_mode() bucket_name_set = config.get_bucket_name_set() self.sync_gateway_config = config log_info(">>> Creating buckets on: {}".format(self.cb_server.url)) log_info(">>> Creating buckets {}".format(bucket_name_set)) self.cb_server.create_buckets(bucket_name_set) # Wait for server to be in a warmup state to work around # https://github.com/couchbase/sync_gateway/issues/1745 log_info(">>> Waiting for Server: {} to be in a healthy state".format(self.cb_server.url)) self.cb_server.wait_for_ready_state() log_info(">>> Starting sync_gateway with configuration: {}".format(config_path_full)) utils.dump_file_contents_to_logs(config_path_full) # Start sync-gateway status = ansible_runner.run_ansible_playbook( "start-sync-gateway.yml", extra_vars={ "sync_gateway_config_filepath": config_path_full } ) assert status == 0, "Failed to start to Sync Gateway" # HACK - only enable sg_accel for distributed index tests # revise this with https://github.com/couchbaselabs/sync-gateway-testcluster/issues/222 if mode == "di": # Start sg-accel status = ansible_runner.run_ansible_playbook( "start-sg-accel.yml", extra_vars={ "sync_gateway_config_filepath": config_path_full } ) assert status == 0, "Failed to start sg_accel" # Validate CBGT if mode == "di": if not self.validate_cbgt_pindex_distribution_retry(len(self.sg_accels)): self.save_cbgt_diagnostics() raise Exception("Failed to validate CBGT Pindex distribution") log_info(">>> Detected valid CBGT Pindex distribution") else: log_info(">>> Running in channel cache") return mode
def provision_cluster(cluster_config, couchbase_server_config, sync_gateway_config, sg_ce=False, cbs_platform="centos7", sg_platform="centos", sa_platform="centos"): log_info("\n>>> Cluster info:\n") server_version = "{}-{}".format(couchbase_server_config.version, couchbase_server_config.build) sg_version = "{}-{}".format(sync_gateway_config._version_number, sync_gateway_config._build_number) try: server_version except NameError: log_info("Server version is not provided") persist_cluster_config_environment_prop(cluster_config, 'server_version', "") else: log_info("Running test with server version {}".format(server_version)) persist_cluster_config_environment_prop(cluster_config, 'server_version', server_version) try: sg_version except NameError: log_info("Sync gateway version is not provided") persist_cluster_config_environment_prop(cluster_config, 'sync_gateway_version', "") else: log_info("Running test with sync_gateway version {}".format(sg_version)) persist_cluster_config_environment_prop(cluster_config, 'sync_gateway_version', sg_version) with open(cluster_config, "r") as ansible_hosts: log_info(ansible_hosts.read()) log_info(couchbase_server_config) log_info(sync_gateway_config) if not sync_gateway_config.is_valid(): log_info("Invalid sync_gateway provisioning configuration. Exiting ...") sys.exit(1) cluster = Cluster(config=cluster_config) config_path_full = os.path.abspath(sync_gateway_config.config_path) config = Config(config_path_full) is_valid, reason = validate_cluster( cluster.sync_gateways, cluster.sg_accels, config, ) if not is_valid: raise ProvisioningError(reason) log_info(">>> Provisioning cluster...") # Get server base url and package name cluster_keywords = ClusterKeywords() cluster_topology = cluster_keywords.get_cluster_topology(cluster_config) server_url = cluster_topology["couchbase_servers"][0] cb_server = CouchbaseServer(server_url) server_baseurl, server_package_name = couchbase_server_config.get_baseurl_package(cb_server, cbs_platform) log_info(">>> Server package: {0}/{1}".format(server_baseurl, server_package_name)) log_info(">>> Using sync_gateway config: {}".format(sync_gateway_config.config_path)) # Reset previous installs clean_cluster(cluster_config) # Install server package log_info("Installing Couchbase Server") install_couchbase_server.install_couchbase_server( cluster_config=cluster_config, couchbase_server_config=couchbase_server_config, cbs_platform=cbs_platform ) # Install sync_gateway log_info("Installing Sync Gateway") install_sync_gateway.install_sync_gateway( cluster_config=cluster_config, sync_gateway_config=sync_gateway_config, sg_platform=sg_platform, sa_platform=sa_platform, sg_ce=sg_ce ) # Install nginx install_nginx(cluster_config) log_info(">>> Done provisioning cluster...")
def get_buckets_from_sync_gateway_config(sync_gateway_config_path): config_path_full = os.path.abspath(sync_gateway_config_path) config = Config(config_path_full) bucket_name_set = config.get_bucket_name_set() return bucket_name_set
def reset(self, sg_config_path): ansible_runner = AnsibleRunner(self._cluster_config) log_info(">>> Reseting cluster ...") log_info(">>> CBS SSL enabled: {}".format(self.cbs_ssl)) log_info(">>> Using xattrs: {}".format(self.xattrs)) # Stop sync_gateways log_info(">>> Stopping sync_gateway") status = ansible_runner.run_ansible_playbook("stop-sync-gateway.yml") assert status == 0, "Failed to stop sync gateway" # Stop sync_gateway accels log_info(">>> Stopping sg_accel") status = ansible_runner.run_ansible_playbook("stop-sg-accel.yml") assert status == 0, "Failed to stop sg_accel" # Deleting sync_gateway artifacts log_info(">>> Deleting sync_gateway artifacts") status = ansible_runner.run_ansible_playbook( "delete-sync-gateway-artifacts.yml") assert status == 0, "Failed to delete sync_gateway artifacts" # Deleting sg_accel artifacts log_info(">>> Deleting sg_accel artifacts") status = ansible_runner.run_ansible_playbook( "delete-sg-accel-artifacts.yml") assert status == 0, "Failed to delete sg_accel artifacts" # Delete buckets log_info(">>> Deleting buckets on: {}".format(self.servers[0].url)) self.servers[0].delete_buckets() # Parse config and grab bucket names config_path_full = os.path.abspath(sg_config_path) config = Config(config_path_full) mode = config.get_mode() bucket_name_set = config.get_bucket_name_set() self.sync_gateway_config = config is_valid, reason = validate_cluster(self.sync_gateways, self.sg_accels, config) if not is_valid: raise ProvisioningError(reason) log_info(">>> Creating buckets on: {}".format(self.servers[0].url)) log_info(">>> Creating buckets {}".format(bucket_name_set)) self.servers[0].create_buckets(bucket_name_set) # Wait for server to be in a warmup state to work around # https://github.com/couchbase/sync_gateway/issues/1745 log_info(">>> Waiting for Server: {} to be in a healthy state".format( self.servers[0].url)) self.servers[0].wait_for_ready_state() log_info(">>> Starting sync_gateway with configuration: {}".format( config_path_full)) server_port = 8091 server_scheme = "http" couchbase_server_primary_node = add_cbs_to_sg_config_server_field( self._cluster_config) if self.cbs_ssl: server_port = 18091 server_scheme = "https" # Start sync-gateway playbook_vars = { "sync_gateway_config_filepath": config_path_full, "server_port": server_port, "server_scheme": server_scheme, "autoimport": "", "xattrs": "", "no_conflicts": "", "revs_limit": "", "couchbase_server_primary_node": couchbase_server_primary_node } # Add configuration to run with xattrs if self.xattrs: playbook_vars["autoimport"] = '"import_docs": "continuous",' playbook_vars["xattrs"] = '"enable_shared_bucket_access": true,' if no_conflicts_enabled(self._cluster_config): playbook_vars["no_conflicts"] = '"allow_conflicts": false,' try: revs_limit = get_revs_limit(self._cluster_config) playbook_vars["revs_limit"] = '"revs_limit": {},'.format( revs_limit) except KeyError as ex: log_info("Keyerror in getting revs_limit{}".format(ex.message)) status = ansible_runner.run_ansible_playbook("start-sync-gateway.yml", extra_vars=playbook_vars) assert status == 0, "Failed to start to Sync Gateway" # HACK - only enable sg_accel for distributed index tests # revise this with https://github.com/couchbaselabs/sync-gateway-testcluster/issues/222 if mode == "di": # Start sg-accel status = ansible_runner.run_ansible_playbook( "start-sg-accel.yml", extra_vars=playbook_vars) assert status == 0, "Failed to start sg_accel" # Validate CBGT if mode == "di": if not self.validate_cbgt_pindex_distribution_retry( len(self.sg_accels)): self.save_cbgt_diagnostics() raise Exception("Failed to validate CBGT Pindex distribution") log_info(">>> Detected valid CBGT Pindex distribution") else: log_info(">>> Running in channel cache") return mode