def install_sync_gateway(self, cluster_config, sync_gateway_version, sync_gateway_config): # Dirty hack -- these have to be put here in order to avoid circular imports from libraries.provision.install_sync_gateway import install_sync_gateway from libraries.provision.install_sync_gateway import SyncGatewayConfig if version_is_binary(sync_gateway_version): version, build = version_and_build(sync_gateway_version) print("VERSION: {} BUILD: {}".format(version, build)) sg_config = SyncGatewayConfig(None, version, build, sync_gateway_config, "", False) else: sg_config = SyncGatewayConfig(sync_gateway_version, None, None, sync_gateway_config, "", False) install_sync_gateway(cluster_config=cluster_config, sync_gateway_config=sg_config) log_info("Verfying versions for cluster: {}".format(cluster_config)) with open("{}.json".format(cluster_config)) as f: cluster_obj = json.loads(f.read()) # Verify sync_gateway versions for sg in cluster_obj["sync_gateways"]: verify_sync_gateway_version(sg["ip"], sync_gateway_version) # Verify sg_accel versions, use the same expected version for sync_gateway for now for ac in cluster_obj["sg_accels"]: verify_sg_accel_version(ac["ip"], sync_gateway_version)
def provision_or_reset_cluster(provision_or_reset, sg_deploy_type, couchbase_server_version, sync_gateway_version, sync_gateway_commit, sync_gateway_config_file, cluster_config): server_config = CouchbaseServerConfig(version=couchbase_server_version) version_number, build_number = version_and_build(sync_gateway_version) sync_gateway_conf = SyncGatewayConfig(version_number=version_number, build_number=build_number, commit=sync_gateway_commit, build_flags="", config_path=sync_gateway_config_file, skip_bucketcreation=False) # Don't specify version number on a source build if sg_deploy_type == "Source": sync_gateway_conf.version_number = None else: # Likewise, don't specify commmit on a package build sync_gateway_conf.commit = None if provision_or_reset == "Provision": print("provisioning cluster") provision_cluster(cluster_config=cluster_config, couchbase_server_config=server_config, sync_gateway_config=sync_gateway_conf) elif provision_or_reset == "Reset": print("resetting cluster") cluster = Cluster(config=cluster_config) cluster.reset(sync_gateway_config_file)
def download(self): """ 1. Check to see if .apk is downloaded already. If so, return 2. Download the LiteServ .apk from latest builds to 'deps/binaries' """ version, build = version_and_build(self.version_build) if version == "1.2.1": package_name = "couchbase-lite-android-liteserv-SQLite-{}-debug.apk".format(self.version_build) else: if self.storage_engine == "SQLite": package_name = "couchbase-lite-android-liteserv-SQLite-{}-debug.apk".format(self.version_build) else: package_name = "couchbase-lite-android-liteserv-SQLCipher-ForestDB-Encryption-{}-debug.apk".format(self.version_build) expected_binary_path = "{}/{}".format(BINARY_DIR, package_name) if os.path.isfile(expected_binary_path): log_info("Package is already downloaded. Skipping.") return # Package not downloaded, proceed to download from latest builds if version == "1.2.1": url = "{}/couchbase-lite-android/release/{}/{}/{}".format(LATEST_BUILDS, version, self.version_build, package_name) else: url = "{}/couchbase-lite-android/{}/{}/{}".format(LATEST_BUILDS, version, self.version_build, package_name) log_info("Downloading {} -> {}/{}".format(url, BINARY_DIR, package_name)) resp = requests.get(url) resp.raise_for_status() with open("{}/{}".format(BINARY_DIR, package_name), "wb") as f: f.write(resp.content)
def download(self): """ 1. Check to see if package is downloaded already. If so, return 2. Download the LiteServ package from latest builds to 'deps/binaries' 3. Unzip the packages and make the binary executable """ # Skip download if packages is already downloaded expected_binary = "{}/couchbase-lite-net-mono-{}-liteserv/LiteServ.exe".format(BINARY_DIR, self.version_build) if os.path.isfile(expected_binary): log_info("Package already downloaded: {}".format(expected_binary)) return version, build = version_and_build(self.version_build) download_url = "{}/couchbase-lite-net/{}/{}/LiteServ.zip".format(LATEST_BUILDS, version, build) downloaded_package_zip_name = "couchbase-lite-net-mono-{}-liteserv.zip".format(self.version_build) log_info("Downloading {} -> {}/{}".format(download_url, BINARY_DIR, downloaded_package_zip_name)) resp = requests.get(download_url) resp.raise_for_status() with open("{}/{}".format(BINARY_DIR, downloaded_package_zip_name), "wb") as f: f.write(resp.content) extracted_directory_name = downloaded_package_zip_name.replace(".zip", "") with ZipFile("{}/{}".format(BINARY_DIR, downloaded_package_zip_name)) as zip_f: zip_f.extractall("{}/{}".format(BINARY_DIR, extracted_directory_name)) # Remove .zip os.remove("{}/{}".format(BINARY_DIR, downloaded_package_zip_name)) # HACK - To get around https://github.com/couchbase/couchbase-lite-net/issues/672 # This is fixed 1.4+ but need to keep it around to allow running against older versions of LiteServ if version.startswith("1.2") or version.startswith("1.3"): shutil.rmtree("{}/{}/x64".format(BINARY_DIR, extracted_directory_name)) shutil.rmtree("{}/{}/x86".format(BINARY_DIR, extracted_directory_name))
def provision_cluster(self, cluster_config, server_version, sync_gateway_version, sync_gateway_config, race_enabled=False, sg_ce=False, cbs_platform="centos7", sg_platform="centos", sa_platform="centos"): if server_version is None or sync_gateway_version is None or sync_gateway_version is None: raise ProvisioningError("Please make sure you have server_version, sync_gateway_version, and sync_gateway_config are set") # Dirty hack -- these have to be put here in order to avoid circular imports from libraries.provision.install_couchbase_server import CouchbaseServerConfig from libraries.provision.provision_cluster import provision_cluster from libraries.provision.install_sync_gateway import SyncGatewayConfig cbs_config = CouchbaseServerConfig(server_version) if version_is_binary(sync_gateway_version): if race_enabled: raise ProvisioningError("Race should only be enabled for source builds") version, build = version_and_build(sync_gateway_version) sg_config = SyncGatewayConfig( commit=None, version_number=version, build_number=build, config_path=sync_gateway_config, build_flags="", skip_bucketcreation=False ) else: build_flags = "" if race_enabled: build_flags = "-race" sg_config = SyncGatewayConfig( commit=sync_gateway_version, version_number=None, build_number=None, config_path=sync_gateway_config, build_flags=build_flags, skip_bucketcreation=False ) provision_cluster( cluster_config=cluster_config, couchbase_server_config=cbs_config, sync_gateway_config=sg_config, sg_ce=sg_ce, cbs_platform=cbs_platform, sg_platform=sg_platform, sa_platform=sa_platform ) # verify running services are the expected versions self.verify_cluster_versions(cluster_config, server_version, sync_gateway_version)
def upgrade_sync_gateways(self, cluster_config, sg_conf, sync_gateway_version, url=None): """ Upgrade sync gateways in a cluster. If url is passed, upgrade the sync gateway at that url """ ansible_runner = AnsibleRunner(cluster_config) from libraries.provision.install_sync_gateway import SyncGatewayConfig version, build = version_and_build(sync_gateway_version) sg_config = SyncGatewayConfig(commit=None, version_number=version, build_number=build, config_path=sg_conf, build_flags="", skip_bucketcreation=False) sg_conf = os.path.abspath(sg_config.config_path) # Shared vars playbook_vars = {} sync_gateway_base_url, sync_gateway_package_name, sg_accel_package_name = sg_config.sync_gateway_base_url_and_package( ) playbook_vars[ "couchbase_sync_gateway_package_base_url"] = sync_gateway_base_url playbook_vars[ "couchbase_sync_gateway_package"] = sync_gateway_package_name playbook_vars["couchbase_sg_accel_package"] = sg_accel_package_name if url is not None: target = hostname_for_url(cluster_config, url) log_info( "Upgrading sync_gateway/sg_accel on {} ...".format(target)) status = ansible_runner.run_ansible_playbook( "upgrade-sg-sgaccel-package.yml", subset=target, extra_vars=playbook_vars) log_info("Completed upgrading {}".format(url)) else: log_info("Upgrading all sync_gateways/sg_accels") status = ansible_runner.run_ansible_playbook( "upgrade-sg-sgaccel-package.yml", extra_vars=playbook_vars) log_info("Completed upgrading all sync_gateways/sg_accels") if status != 0: raise Exception("Could not upgrade sync_gateway/sg_accel")
def test_ios_logging(request, setup_liteserv_ios_logging): liteserv = setup_liteserv_ios_logging test_name = request.node.name logfile = "{}/logs/{}-{}-{}.txt".format(RESULTS_DIR, type(liteserv).__name__, test_name, datetime.datetime.now()) liteserv.start(logfile) liteserv.stop() version, build = version_and_build(liteserv.version_build) with open(logfile, "r") as f: contents = f.read() assert "LiteServ {} (build {}) is listening at".format(version, build) in contents
def test_macosx_logging(request, liteserv_with_storage_engine_from_fixture): macosx_version = request.config.getoption("--macosx-version") liteserv = liteserv_with_storage_engine_from_fixture test_name = request.node.name logfile = "{}/logs/{}-{}-{}.txt".format(RESULTS_DIR, type(liteserv).__name__, test_name, datetime.datetime.now()) liteserv.start(logfile) liteserv.stop() version, build = version_and_build(macosx_version) with open(logfile, "r") as f: contents = f.read() assert "LiteServ {} (build {}) is listening at".format(version, build) in contents
def _verify_launched(self): """ Poll on expected http://<host>:<port> until it is reachable Assert that the response contains the expected version information """ resp_obj = self._wait_until_reachable() log_info(resp_obj) if resp_obj["vendor"]["name"] != "Couchbase Lite (Objective-C)": raise LiteServError("Unexpected LiteServ platform running!") version, build = version_and_build(self.version_build) expected_version = "{} (build {})".format(version, build) running_version = resp_obj["vendor"]["version"] if expected_version != running_version: raise LiteServError("Expected version: {} does not match running version: {}".format(expected_version, running_version))
def download(self): """ 1. Downloads the LiteServ.zip package from latestbuild to the remote Windows host to Desktop\LiteServ\ 2. Extracts the package and removes the zip """ version, build = version_and_build(self.version_build) download_url = "{}/couchbase-lite-net/{}/{}/LiteServ.zip".format(LATEST_BUILDS, version, build) package_name = "couchbase-lite-net-msft-{}-liteserv".format(self.version_build) # Download LiteServ via Ansible on remote machine status = self.ansible_runner.run_ansible_playbook("download-liteserv-msft.yml", extra_vars={ "download_url": download_url, "package_name": package_name }) if status != 0: raise LiteServError("Failed to download LiteServ package on remote machine")
def test_macosx_logging(request, liteserv_with_storage_engine_from_fixture): macosx_version = request.config.getoption("--macosx-version") liteserv = liteserv_with_storage_engine_from_fixture test_name = request.node.name logfile = "{}/logs/{}-{}-{}.txt".format(RESULTS_DIR, type(liteserv).__name__, test_name, datetime.datetime.now()) liteserv.start(logfile) liteserv.stop() version, build = version_and_build(macosx_version) with open(logfile, "r") as f: contents = f.read() assert "LiteServ {} (build {}) is listening at".format( version, build) in contents
def test_ios_logging(request, setup_liteserv_ios_logging): liteserv = setup_liteserv_ios_logging test_name = request.node.name logfile = "{}/logs/{}-{}-{}.txt".format(RESULTS_DIR, type(liteserv).__name__, test_name, datetime.datetime.now()) liteserv.start(logfile) liteserv.stop() version, build = version_and_build(liteserv.version_build) with open(logfile, "r") as f: contents = f.read() assert "LiteServ {} (build {}) is listening at".format( version, build) in contents
def download(self, version_build=None): """ 1. Check to see if package is downloaded already. If so, return 2. Download the LiteServ package from latest builds to 'deps/binaries' 3. Unzip the packages and make the binary executable """ if(version_build is not None): self.version_build = version_build package_name = "couchbase-lite-macosx-enterprise_{}.zip".format(self.version_build) # Skip download if packages is already downloaded expected_binary = "{}/couchbase-lite-macosx-enterprise_{}/LiteServ".format(BINARY_DIR, self.version_build) if os.path.isfile(expected_binary): log_info("Package already downloaded: {}".format(expected_binary)) return version, build = version_and_build(self.version_build) if version == "1.2.0": package_url = "{}/couchbase-lite-ios/release/{}/macosx/{}/{}".format(LATEST_BUILDS, version, self.version_build, package_name) else: package_url = "{}/couchbase-lite-ios/{}/macosx/{}/{}".format(LATEST_BUILDS, version, build, package_name) # Download package to deps/binaries log_info("Downloading: {}".format(package_url)) resp = requests.get(package_url) resp.raise_for_status() with open("{}/{}".format(BINARY_DIR, package_name), "wb") as f: f.write(resp.content) # Unzip package directory_name = package_name.replace(".zip", "") with ZipFile("{}/{}".format(BINARY_DIR, package_name)) as zip_f: zip_f.extractall("{}/{}".format(BINARY_DIR, directory_name)) # Remove .zip os.remove("{}/{}".format(BINARY_DIR, package_name)) # Make binary executable binary_path = "{}/{}/LiteServ".format(BINARY_DIR, directory_name) os.chmod(binary_path, 0755) log_info("LiteServ: {}".format(binary_path))
def download(self): """ 1. Check to see if package is downloaded already. If so, return 2. Download the LiteServ package from latest builds to 'deps/binaries' 3. Unzip the packages and make the binary executable """ package_name = "couchbase-lite-macosx-enterprise_{}.zip".format(self.version_build) # Skip download if packages is already downloaded expected_binary = "{}/couchbase-lite-macosx-enterprise_{}/LiteServ".format(BINARY_DIR, self.version_build) if os.path.isfile(expected_binary): log_info("Package already downloaded: {}".format(expected_binary)) return version, build = version_and_build(self.version_build) if version == "1.2.0": package_url = "{}/couchbase-lite-ios/release/{}/macosx/{}/{}".format(LATEST_BUILDS, version, self.version_build, package_name) else: package_url = "{}/couchbase-lite-ios/{}/macosx/{}/{}".format(LATEST_BUILDS, version, self.version_build, package_name) # Download package to deps/binaries log_info("Downloading: {}".format(package_url)) resp = requests.get(package_url) resp.raise_for_status() with open("{}/{}".format(BINARY_DIR, package_name), "wb") as f: f.write(resp.content) # Unzip package directory_name = package_name.replace(".zip", "") with ZipFile("{}/{}".format(BINARY_DIR, package_name)) as zip_f: zip_f.extractall("{}/{}".format(BINARY_DIR, directory_name)) # Remove .zip os.remove("{}/{}".format(BINARY_DIR, package_name)) # Make binary executable binary_path = "{}/{}/LiteServ".format(BINARY_DIR, directory_name) os.chmod(binary_path, 0755) log_info("LiteServ: {}".format(binary_path))
def test_ce_ee_package(sg_ce, sg_type, sg_platform, sa_platform, platform_ext): sync_gateway_version = "1.5.0-477" cwd = os.getcwd() sync_gateway_config = cwd + "/resources/sync_gateway_configs/sync_gateway_default_cc.json" version, build = version_and_build(sync_gateway_version) sg_config = SyncGatewayConfig(commit=None, version_number=version, build_number=build, config_path=sync_gateway_config, build_flags="", skip_bucketcreation=False) sync_gateway_base_url, sync_gateway_package_name, sg_accel_package_name = sg_config.sync_gateway_base_url_and_package( sg_ce, sg_platform, sa_platform) assert sync_gateway_package_name == "couchbase-sync-gateway-{}_1.5.0-477_x86_64.{}".format( sg_type, platform_ext) assert sg_accel_package_name == "couchbase-sg-accel-enterprise_1.5.0-477_x86_64.{}".format( platform_ext) assert sync_gateway_base_url == "http://latestbuilds.service.couchbase.com/builds/latestbuilds/sync_gateway/1.5.0/477"
def download(self, version_build=None): """ 1. Check to see if package is downloaded already. If so, return 2. Download the LiteServ package from latest builds to 'deps/binaries' 3. Unzip the packages and make the binary executable """ if version_build is not None: self.version_build = version_build # Skip download if packages is already downloaded if has_dot_net4_dot_5(self.version_build): expected_binary = "{}/couchbase-lite-net-mono-{}-liteserv/net45/LiteServ.exe".format(BINARY_DIR, self.version_build) else: expected_binary = "{}/couchbase-lite-net-mono-{}-liteserv/LiteServ.exe".format(BINARY_DIR, self.version_build) if os.path.isfile(expected_binary): log_info("Package already downloaded: {}".format(expected_binary)) return version, build = version_and_build(self.version_build) download_url = "{}/couchbase-lite-net/{}/{}/LiteServ.zip".format(LATEST_BUILDS, version, build) downloaded_package_zip_name = "couchbase-lite-net-mono-{}-liteserv.zip".format(self.version_build) log_info("Downloading {} -> {}/{}".format(download_url, BINARY_DIR, downloaded_package_zip_name)) resp = requests.get(download_url) resp.raise_for_status() with open("{}/{}".format(BINARY_DIR, downloaded_package_zip_name), "wb") as f: f.write(resp.content) extracted_directory_name = downloaded_package_zip_name.replace(".zip", "") with ZipFile("{}/{}".format(BINARY_DIR, downloaded_package_zip_name)) as zip_f: zip_f.extractall("{}/{}".format(BINARY_DIR, extracted_directory_name)) # Remove .zip os.remove("{}/{}".format(BINARY_DIR, downloaded_package_zip_name)) # HACK - To get around https://github.com/couchbase/couchbase-lite-net/issues/672 # This is fixed 1.4+ but need to keep it around to allow running against older versions of LiteServ if version.startswith("1.2") or version.startswith("1.3"): shutil.rmtree("{}/{}/x64".format(BINARY_DIR, extracted_directory_name)) shutil.rmtree("{}/{}/x86".format(BINARY_DIR, extracted_directory_name))
def download(self, version_build=None): """ 1. Check to see if package is downloaded already. If so, return 2. Download the LiteServ package from latest builds to 'deps/binaries' 3. Unzip the packages and make the binary executable """ if version_build is not None: self.version_build = version_build version, build = version_and_build(self.version_build) package_name = "LiteServ-iOS.zip" app_name = "LiteServ-iOS.app" app_dir = "LiteServ-iOS" if self.storage_engine == "SQLCipher": package_name = "LiteServ-iOS-SQLCipher.zip" app_name = "LiteServ-iOS-SQLCipher.app" app_dir = "LiteServ-iOS-SQLCipher" expected_binary_path = "{}/{}/{}".format(BINARY_DIR, app_dir, app_name) if os.path.isfile(expected_binary_path): log_info("Package is already downloaded. Skipping.") return # Package not downloaded, proceed to download from latest builds downloaded_package_zip_name = "{}/{}".format(BINARY_DIR, package_name) url = "{}/couchbase-lite-ios/{}/ios/{}/{}".format(LATEST_BUILDS, version, build, package_name) log_info("Downloading {} -> {}/{}".format(url, BINARY_DIR, package_name)) resp = requests.get(url) resp.raise_for_status() with open("{}/{}".format(BINARY_DIR, package_name), "wb") as f: f.write(resp.content) extracted_directory_name = downloaded_package_zip_name.replace(".zip", "") with ZipFile("{}".format(downloaded_package_zip_name)) as zip_f: zip_f.extractall("{}".format(extracted_directory_name)) # Remove .zip os.remove("{}".format(downloaded_package_zip_name))
def download(self, version_build=None): """ 1. Check to see if .apk is downloaded already. If so, return 2. Download the LiteServ .apk from latest builds to 'deps/binaries' """ if (version_build is not None): self.version_build = version_build version, build = version_and_build(self.version_build) if version == "1.2.1": package_name = "couchbase-lite-android-liteserv-SQLite-{}-debug.apk".format( self.version_build) else: if self.storage_engine == "SQLite": package_name = "couchbase-lite-android-liteserv-SQLite-{}-debug.apk".format( self.version_build) else: package_name = "couchbase-lite-android-liteserv-SQLCipher-ForestDB-Encryption-{}-debug.apk".format( self.version_build) expected_binary_path = "{}/{}".format(BINARY_DIR, package_name) if os.path.isfile(expected_binary_path): log_info("Package is already downloaded. Skipping.") return # Package not downloaded, proceed to download from latest builds if version == "1.2.1": url = "{}/couchbase-lite-android/release/{}/{}/{}".format( LATEST_BUILDS, version, self.version_build, package_name) else: url = "{}/couchbase-lite-android/{}/{}/{}".format( LATEST_BUILDS, version, build, package_name) log_info("Downloading {} -> {}/{}".format(url, BINARY_DIR, package_name)) resp = requests.get(url) resp.raise_for_status() with open("{}/{}".format(BINARY_DIR, package_name), "wb") as f: f.write(resp.content)
def provision_cluster(self, cluster_config, server_version, sync_gateway_version, sync_gateway_config): if server_version is None or sync_gateway_version is None or sync_gateway_version is None: raise ProvisioningError("Please make sure you have server_version, sync_gateway_version, and sync_gateway_config are set") # Dirty hack -- these have to be put here in order to avoid circular imports from libraries.provision.install_couchbase_server import CouchbaseServerConfig from libraries.provision.provision_cluster import provision_cluster from libraries.provision.install_sync_gateway import SyncGatewayConfig cbs_config = CouchbaseServerConfig(server_version) if version_is_binary(sync_gateway_version): version, build = version_and_build(sync_gateway_version) sg_config = SyncGatewayConfig(None, version, build, sync_gateway_config, "", False) else: sg_config = SyncGatewayConfig(sync_gateway_version, None, None, sync_gateway_config, "", False) provision_cluster(cluster_config, cbs_config, sg_config) # verify running services are the expected versions self.verify_cluster_versions(cluster_config, server_version, sync_gateway_version)
def download(self, version_build=None): """ 1. Downloads the LiteServ.zip package from latestbuild to the remote Windows host to Desktop\LiteServ\ 2. Extracts the package and removes the zip """ if version_build is not None: self.version_build = version_build version, build = version_and_build(self.version_build) download_url = "{}/couchbase-lite-net/{}/{}/LiteServ.zip".format( LATEST_BUILDS, version, build) package_name = "couchbase-lite-net-msft-{}-liteserv".format( self.version_build) # Download LiteServ via Ansible on remote machine status = self.ansible_runner.run_ansible_playbook( "download-liteserv-msft.yml", extra_vars={ "download_url": download_url, "package_name": package_name }) if status != 0: raise LiteServError( "Failed to download LiteServ package on remote machine")
try: cluster_conf = os.environ["CLUSTER_CONFIG"] except KeyError as ke: print ("Make sure CLUSTER_CONFIG is defined and pointing to the configuration you would like to provision") raise KeyError("CLUSTER_CONFIG not defined. Unable to provision cluster.") server_config = CouchbaseServerConfig( version=opts.server_version ) sync_gateway_version = None sync_gateway_build = None if opts.sync_gateway_version is not None: sync_gateway_version, sync_gateway_build = version_and_build(opts.sync_gateway_version) if opts.install_deps_flag: install_deps(cluster_conf) sync_gateway_conf = SyncGatewayConfig( version_number=sync_gateway_version, build_number=sync_gateway_build, commit=opts.source_commit, build_flags=opts.build_flags, config_path=opts.sync_gateway_config_file, skip_bucketcreation=False ) provision_cluster( cluster_config=cluster_conf,