def sg_doc_updates(sg_client, sg_url, sg_db, sg_docs, number_updates, auth, channels):
for doc in sg_docs:
try:
sg_client.update_doc(sg_url, sg_db, doc['id'], number_updates, auth=auth, channels=channels)
except HTTPError as e:
if e.response.status_code == 409 and e.message.startswith('409 Client Error: Conflict for url:'):
log_info("Got conflict with sdk update, skip it")
def stop(self):
"""
Stops a .NET listener on a remote windows machine via ansible and pulls logs.
"""
# The package structure for LiteServ is different pre 1.4. Handle for this case
if has_dot_net4_dot_5(self.version_build):
binary_path = "couchbase-lite-net-msft-{}-liteserv/net45/LiteServ.exe".format(
self.version_build)
else:
binary_path = "couchbase-lite-net-msft-{}-liteserv/LiteServ.exe".format(
self.version_build)
log_full_path = "{}/{}".format(os.getcwd(), self.logfile)
log_info("Stoping {} on windows maching ...".format(binary_path))
log_info("Pulling logs to {} ...".format(log_full_path))
status = self.ansible_runner.run_ansible_playbook(
"stop-liteserv-windows.yml",
extra_vars={
"binary_path": binary_path,
"log_full_path": log_full_path
})
if status != 0:
raise LiteServError("Could not start Liteserv")
def delete_buckets(self):
count = 0
while count < 3:
resp = self._session.get("{}/pools/default/buckets".format(self.url))
log_r(resp)
resp.raise_for_status()
obj = json.loads(resp.text)
existing_bucket_names = []
for entry in obj:
existing_bucket_names.append(entry["name"])
log_info("Existing buckets: {}".format(existing_bucket_names))
log_info("Deleting buckets: {}".format(existing_bucket_names))
# HACK around Couchbase Server issue where issuing a bucket delete via REST occasionally returns 500 error
delete_num = 0
# Delete existing buckets
for bucket_name in existing_bucket_names:
resp = self._session.delete("{0}/pools/default/buckets/{1}".format(self.url, bucket_name))
log_r(resp)
if resp.status_code == 200:
delete_num += 1
if delete_num == len(existing_bucket_names):
break
else:
# A 500 error may have occured, query for buckets and try to delete them again
time.sleep(5)
count += 1
# Check that max retries did not occur
if count == 3:
raise CBServerError("Max retries for bucket creation hit. Could not delete buckets!")
def install_sync_gateway(self, cluster_config, sync_gateway_version,
sync_gateway_config):
# Dirty hack -- these have to be put here in order to avoid circular imports
from libraries.provision.install_sync_gateway import install_sync_gateway
from libraries.provision.install_sync_gateway import SyncGatewayConfig
if version_is_binary(sync_gateway_version):
version, build = version_and_build(sync_gateway_version)
print("VERSION: {} BUILD: {}".format(version, build))
sg_config = SyncGatewayConfig(None, version, build,
sync_gateway_config, "", False)
else:
sg_config = SyncGatewayConfig(sync_gateway_version, None, None,
sync_gateway_config, "", False)
install_sync_gateway(cluster_config=cluster_config,
sync_gateway_config=sg_config)
log_info("Verfying versions for cluster: {}".format(cluster_config))
with open("{}.json".format(cluster_config)) as f:
cluster_obj = json.loads(f.read())
# Verify sync_gateway versions
for sg in cluster_obj["sync_gateways"]:
verify_sync_gateway_version(sg["ip"], sync_gateway_version)
# Verify sg_accel versions, use the same expected version for sync_gateway for now
for ac in cluster_obj["sg_accels"]:
verify_sg_accel_version(ac["ip"], sync_gateway_version)
def test_openidconnect_invalid_scope(params_from_base_test_setup, sg_conf_name):
"""Try to discover the authenticate endpoint URL with a test provider that has an
invalid scope, and expect an error"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_invalid_scope'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
try:
discover_authenticate_endpoint(sg_url, sg_db, "testinvalidscope")
except HTTPError:
log_info("got expected HTTPError trying to get the authenticate endpoint")
# ok we got an exception, which is expected since we are using an invalid scope
return
raise Exception("Expected HTTPError since we are using invalid scope")
def params_from_base_test_setup(request, params_from_base_suite_setup):
# pytest command line parameters
collect_logs = request.config.getoption("--collect-logs")
test_name = request.node.name
log_info("Setting up test '{}'".format(test_name))
cluster_config = params_from_base_suite_setup["cluster_config"]
mode = params_from_base_suite_setup["mode"]
yield {"cluster_config": cluster_config, "mode": mode}
log_info("Tearing down test '{}'".format(test_name))
# Capture testkit socket usage
network_utils = NetworkUtils()
network_utils.list_connections()
# Verify all sync_gateways and sg_accels are reachable
c = cluster.Cluster(cluster_config)
errors = c.verify_alive(mode)
# if the test failed pull logs
if collect_logs or request.node.rep_call.failed or len(errors) != 0:
logging_helper = Logging()
logging_helper.fetch_and_analyze_logs(cluster_config=cluster_config,
test_name=test_name)
assert len(errors) == 0
def test_openidconnect_oidc_challenge_invalid_provider_name(params_from_base_test_setup, sg_conf_name):
"""
If oidc_challenge is called with an invalid provider name, it should not return
an Www-Authenticate header
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_oidc_challenge_invalid_provider_name'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
# make a request to the _oidc_challenge endpoint
oidc_challenge_url = "{}/{}/_oidc_challenge?provider={}".format(sg_url, sg_db, "bogusprovider")
response = requests.get(oidc_challenge_url)
log_info("response.headers: {}".format(response.headers))
assert "Www-Authenticate" not in response.headers
assert response.status_code == 400
def _verify_launched(self):
""" Poll on expected http://<host>:<port> until it is reachable
Assert that the response contains the expected version information
"""
resp_obj = self._wait_until_reachable()
log_info(resp_obj)
# .NET OS X 10.12/x86_64 1.3.1-build0013/5d1553d
running_version = resp_obj["vendor"]["version"]
if not (running_version.startswith(".NET OS X")):
raise LiteServError("Invalid platform running: {}!".format(running_version))
# [u'.NET', u'OS', u'X', u'10.12', u'x86_64', u'1.3.1', u'build0013', u'5d1553d']
running_version_parts = re.split("[ /-]", running_version)
running_version = running_version_parts[5]
running_build = int(running_version_parts[6].strip("build"))
running_version_composed = "{}-{}".format(running_version, running_build)
if self.version_build != running_version_composed:
raise LiteServError("Expected version does not match actual version: Expected={} Actual={}".format(
self.version_build,
running_version_composed)
)
def params_from_base_test_setup(request, params_from_base_suite_setup):
# Code before the yeild will execute before each test starts
cluster_config = params_from_base_suite_setup["cluster_config"]
mode = params_from_base_suite_setup["mode"]
test_name = request.node.name
log_info("Setting up test '{}'".format(test_name))
network_utils = NetworkUtils()
network_utils.start_packet_capture(cluster_config)
# This dictionary is passed to each test
yield {"cluster_config": cluster_config, "mode": mode}
# Code after the yeild will execute when each test finishes
log_info("Tearing down test '{}'".format(test_name))
network_utils.list_connections()
network_utils.stop_packet_capture(cluster_config)
network_utils.collect_packet_capture(cluster_config=cluster_config, test_name=test_name)
# Verify all sync_gateways and sg_accels are reachable
c = cluster.Cluster(cluster_config)
errors = c.verify_alive(mode)
assert len(errors) == 0
# if the test failed pull logs
if request.node.rep_call.failed:
logging_helper = Logging()
logging_helper.fetch_and_analyze_logs(cluster_config=cluster_config, test_name=test_name)
def fetch_sync_gateway_logs(cluster_config, prefix):
ansible_runner = AnsibleRunner(cluster_config)
log_info("Pulling sync_gateway / sg_accel logs")
# fetch logs from sync_gateway instances
status = ansible_runner.run_ansible_playbook("fetch-sync-gateway-logs.yml")
if status != 0:
raise CollectionError("Could not pull logs")
# zip logs and timestamp
if os.path.isdir("/tmp/sg_logs"):
date_time = time.strftime("%Y-%m-%d-%H-%M-%S")
temp_log_path = "/tmp/{}-{}-sglogs".format(prefix, date_time)
shutil.make_archive(temp_log_path, "zip", "/tmp/sg_logs")
shutil.rmtree("/tmp/sg_logs")
# Copy logs to results dir
zip_file_path = "{}.zip".format(temp_log_path)
log_results_location = "{}/logs".format(RESULTS_DIR)
shutil.copy(zip_file_path, log_results_location)
zip_name = "{}-{}-sglogs.zip".format(prefix, date_time)
result_zip = "{}/{}".format(log_results_location, zip_name)
log_info("sync_gateway logs copied to {}".format(result_zip))
return result_zip
else:
raise CollectionError("Error finding pulled logs at /tmp/sg_logs")
def remove(self):
"""
Remove the iOS app from the simulator
"""
bundle_id = "com.couchbase.LiteServ-iOS"
if self.storage_engine == "SQLCipher":
bundle_id = "com.couchbase.LiteServ-iOS-SQLCipher"
log_info("Removing LiteServ")
self.stop()
# Stop the simulator
log_info("device_id: {}".format(self.device_id))
output = subprocess.check_output([
"killall", "Simulator"
])
# Erase the simulator
output = subprocess.check_output([
"xcrun", "simctl", "erase", self.device_id
])
if bundle_id in output:
raise LiteServError("{} is still present after uninstall".format(bundle_id))
def test_raw_attachment(setup_client_syncgateway_test):
"""
1. Add Text attachment to sync_gateway
2. Try to get the raw attachment
Pass: It is possible to get the raw attachment
"""
log_info("Running 'test_raw_attachment'")
ls_url = setup_client_syncgateway_test["ls_url"]
log_info("ls_url: {}".format(ls_url))
client = MobileRestClient()
ls_db = client.create_database(ls_url, name="ls_db")
ls_user_channels = ["NBC"]
doc_with_att = document.create_doc(
doc_id="att_doc",
content={"sample_key": "sample_val"},
attachment_name="sample_text.txt",
channels=ls_user_channels,
)
doc = client.add_doc(url=ls_url, db=ls_db, doc=doc_with_att)
att = client.get_attachment(url=ls_url, db=ls_db, doc_id=doc["id"], attachment_name="sample_text.txt")
expected_text = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\nUt enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\nDuis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.\nExcepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."
assert expected_text == att
def fetch_sync_gateway_logs(cluster_config, prefix):
ansible_runner = AnsibleRunner(cluster_config)
log_info("Pulling sync_gateway / sg_accel logs")
# fetch logs from sync_gateway instances
status = ansible_runner.run_ansible_playbook("fetch-sync-gateway-logs.yml")
if status != 0:
raise CollectionError("Could not pull logs")
# zip logs and timestamp
if os.path.isdir("/tmp/sg_logs"):
date_time = time.strftime("%Y-%m-%d-%H-%M-%S")
temp_log_path = "/tmp/{}-{}-sglogs".format(prefix, date_time)
shutil.make_archive(temp_log_path, "zip", "/tmp/sg_logs")
shutil.rmtree("/tmp/sg_logs")
# Copy logs to results dir
zip_file_path = "{}.zip".format(temp_log_path)
log_results_location = "{}/logs".format(RESULTS_DIR)
shutil.copy(zip_file_path, log_results_location)
zip_name = "{}-{}-sglogs.zip".format(prefix, date_time)
result_zip = "{}/{}".format(log_results_location, zip_name)
log_info("sync_gateway logs copied to {}".format(result_zip))
return result_zip
else:
raise CollectionError("Error finding pulled logs at /tmp/sg_logs")
def process_changes(self, results):
"""
Add each doc from longpoll changes results to the processed changes list in the following format:
{ "doc_id": [ {"rev": "rev1"}, {"rev", "rev2"}, ...] }
"""
log_info("[Changes Tracker] New changes: {}".format(len(results)))
for doc in results:
if len(doc["changes"]) > 0:
if doc["id"] in self.processed_changes:
# doc has already been seen in the changes feed,
# append new changes to revs accociated with that id
revs_list = self.processed_changes[doc["id"]]
# If the document is already in 'processed_changes', make sure
# that the revision doesn't already exist. If we see one, raise an exception
# because we are seeing the same revision being sent twice
# Checking against this scenario - https://github.com/couchbase/sync_gateway/issues/2186
changes_revs = [change["rev"] for change in doc["changes"]]
revs_list_revs = [rev["rev"] for rev in revs_list]
for change in changes_revs:
if change in revs_list_revs:
raise keywords.exceptions.ChangesError(
"Duplicates in changes feed!")
revs_list.extend(doc["changes"])
self.processed_changes[doc["id"]] = revs_list
else:
# Stored the doc with the list of rev changes
self.processed_changes[doc["id"]] = doc["changes"]
log_info("[Changes Tracker] Total processed changes: {}".format(
len(self.processed_changes)))
def test_auto_prune_listener_sanity(setup_client_syncgateway_test):
"""Sanity test for the autoprune feature
1. Create a db and put a doc
2. Update the docs past the default revs_limit (20)
3. Assert the the docs only retain 20 revs
"""
ls_url = setup_client_syncgateway_test["ls_url"]
client = MobileRestClient()
log_info("Running 'test_auto_prune_listener_sanity' ...")
log_info("ls_url: {}".format(ls_url))
num_docs = 1
num_revs = 100
ls_db = client.create_database(url=ls_url, name="ls_db")
docs = client.add_docs(url=ls_url,
db=ls_db,
number=num_docs,
id_prefix="ls_db_doc")
assert len(docs) == num_docs
client.update_docs(url=ls_url,
db=ls_db,
docs=docs,
number_updates=num_revs)
client.verify_max_revs_num_for_docs(url=ls_url,
db=ls_db,
docs=docs,
expected_max_number_revs_per_doc=20)
def start(self, logfile_name):
"""
1. Starts a LiteServ with logging to provided logfile file object.
The running LiteServ process will be stored in the self.process property.
2. The method will poll on the endpoint to make sure LiteServ is available.
3. The expected version will be compared with the version reported by http://<host>:<port>
4. Return the url of the running LiteServ
"""
if self.storage_engine != "SQLite":
raise NotImplementedError("Need to make sure to support other storage types")
self._verify_not_running()
if self.port == 59850:
raise LiteServError("On iOS, port 59850 is reserved for the admin port")
liteserv_admin_url = "http://{}:59850".format(self.host)
log_info("Starting LiteServ: {}".format(liteserv_admin_url))
data = {
"port": int(self.port)
}
resp = self.session.put("{}/start".format(liteserv_admin_url), data=json.dumps(data))
log_r(resp)
resp.raise_for_status()
self._verify_launched()
return "http://{}:{}".format(self.host, self.port)
def rebalance_out(self, cluster_servers, server_to_remove):
"""
Issues a call to the admin_serve to remove a server from a pool.
Then wait for rebalance to complete.
"""
if not isinstance(server_to_remove, CouchbaseServer):
raise TypeError("'server_to_remove' must be a 'CouchbaseServer'")
# Add all servers except server_to_add to known nodes
known_nodes = "knownNodes="
for server in cluster_servers:
server = server.replace("http://", "")
server = server.replace(":8091", "")
known_nodes += "ns_1@{},".format(server)
# Add server_to_add to known nodes
ejected_node = "ejectedNodes=ns_1@{}".format(server_to_remove.host)
data = "{}&{}".format(ejected_node, known_nodes)
log_info("Starting rebalance out: {} with nodes {}".format(server_to_remove.host, data))
# Override session headers for this one off request
resp = self._session.post(
"{}/controller/rebalance".format(self.url),
headers={"Content-Type": "application/x-www-form-urlencoded"},
data=data
)
log_r(resp)
resp.raise_for_status()
self._wait_for_rebalance_complete()
return True
def download(self):
"""
1. Check to see if package is downloaded already. If so, return
2. Download the LiteServ package from latest builds to 'deps/binaries'
3. Unzip the packages and make the binary executable
"""
# Skip download if packages is already downloaded
expected_binary = "{}/couchbase-lite-net-mono-{}-liteserv/LiteServ.exe".format(BINARY_DIR, self.version_build)
if os.path.isfile(expected_binary):
log_info("Package already downloaded: {}".format(expected_binary))
return
version, build = version_and_build(self.version_build)
download_url = "{}/couchbase-lite-net/{}/{}/LiteServ.zip".format(LATEST_BUILDS, version, build)
downloaded_package_zip_name = "couchbase-lite-net-mono-{}-liteserv.zip".format(self.version_build)
log_info("Downloading {} -> {}/{}".format(download_url, BINARY_DIR, downloaded_package_zip_name))
resp = requests.get(download_url)
resp.raise_for_status()
with open("{}/{}".format(BINARY_DIR, downloaded_package_zip_name), "wb") as f:
f.write(resp.content)
extracted_directory_name = downloaded_package_zip_name.replace(".zip", "")
with ZipFile("{}/{}".format(BINARY_DIR, downloaded_package_zip_name)) as zip_f:
zip_f.extractall("{}/{}".format(BINARY_DIR, extracted_directory_name))
# Remove .zip
os.remove("{}/{}".format(BINARY_DIR, downloaded_package_zip_name))
# HACK - To get around https://github.com/couchbase/couchbase-lite-net/issues/672
# This is fixed 1.4+ but need to keep it around to allow running against older versions of LiteServ
if version.startswith("1.2") or version.startswith("1.3"):
shutil.rmtree("{}/{}/x64".format(BINARY_DIR, extracted_directory_name))
shutil.rmtree("{}/{}/x86".format(BINARY_DIR, extracted_directory_name))
def _verify_launched(self):
"""Poll on expected http://<host>:<port> until it is reachable
Assert that the response contains the expected version information
"""
resp_obj = self._wait_until_reachable()
log_info(resp_obj)
# .NET Microsoft Windows 10.12/x86_64 1.3.1-build0013/5d1553d
running_version = resp_obj["vendor"]["version"]
if not running_version.startswith(".NET Microsoft Windows"):
raise LiteServError("Invalid platform running!")
# ['.NET', 'Microsoft', 'Windows', '10', 'Enterprise', 'x64', '1.4.0', 'build0043', '5cfe25b']
running_version_parts = re.split("[ /-]", running_version)
running_version = running_version_parts[6]
running_build = int(running_version_parts[7].strip("build"))
running_version_composed = "{}-{}".format(running_version, running_build)
if self.version_build != running_version_composed:
raise LiteServError("Expected version does not match actual version: Expected={} Actual={}".format(
self.version_build,
running_version_composed)
)
def params_from_base_test_setup(request, params_from_base_suite_setup):
test_name = request.node.name
log_info("Setting up test '{}'".format(test_name))
cluster_config = params_from_base_suite_setup["cluster_config"]
mode = params_from_base_suite_setup["mode"]
yield {
"cluster_config": cluster_config,
"mode": mode
}
log_info("Tearing down test '{}'".format(test_name))
# Capture testkit socket usage
network_utils = NetworkUtils()
network_utils.list_connections()
# Verify all sync_gateways and sg_accels are reachable
c = cluster.Cluster(cluster_config)
errors = c.verify_alive(mode)
assert len(errors) == 0
# if the test failed pull logs
if request.node.rep_call.failed:
logging_helper = Logging()
logging_helper.fetch_and_analyze_logs(cluster_config=cluster_config, test_name=test_name)
def setup_p2p_test(request, setup_p2p_suite):
"""Test setup fixture for p2p client tests"""
log_info("Setting up P2P test ...")
liteserv_one = setup_p2p_suite["liteserv_one"]
liteserv_two = setup_p2p_suite["liteserv_two"]
test_name = request.node.name
print("Starting LiteServ One ...")
ls_logging_one = "{}/logs/{}-ls1-{}-{}.txt".format(RESULTS_DIR, type(liteserv_one).__name__, test_name, datetime.datetime.now())
ls_url_one = liteserv_one.start(ls_logging_one)
print("Starting LiteServ Two ...")
ls_logging_two = "{}/logs/{}-ls2-{}-{}.txt".format(RESULTS_DIR, type(liteserv_two).__name__, test_name, datetime.datetime.now())
ls_url_two = liteserv_two.start(ls_logging_two)
# Yield values to test case via fixture argument
yield {"ls_url_one": ls_url_one, "ls_url_two": ls_url_two}
log_info("Tearing down test")
# Teardown test
client = MobileRestClient()
client.delete_databases(ls_url_one)
client.delete_databases(ls_url_two)
liteserv_one.stop()
liteserv_two.stop()
def create_server_buckets(cluster_config, sync_gateway_config):
# get the couchbase server url
cluster_helper = ClusterKeywords()
cluster_topology = cluster_helper.get_cluster_topology(cluster_config)
# Handle the case of resources/cluster_configs/1sg, where we are targeting a
# sync_gateway without a backing server
if len(cluster_topology["couchbase_servers"]) == 0:
log_info(
"The cluster_config: {} does not have a couchbase server. Skipping bucket creation!!"
.format(cluster_config))
return
couchbase_server_url = cluster_topology["couchbase_servers"][0]
# delete existing buckets
cb_server = CouchbaseServer(couchbase_server_url)
cb_server.delete_buckets()
# find bucket names from sg config
bucket_names = get_buckets_from_sync_gateway_config(
sync_gateway_config.config_path)
# create couchbase server buckets
cb_server.create_buckets(bucket_names)
def get_cluster_topology(self, cluster_config):
"""
Returns a dictionary of cluster endpoints that will be consumable
${sg1} = cluster["sync_gateways"][0]["public"]
${sg1_admin} = cluster["sync_gateways"][0]["admin"]
${ac1} = cluster["sg_accels"][0]
${cbs} = cluster["couchbase_servers"][0]
"""
with open("{}.json".format(cluster_config)) as f:
cluster = json.loads(f.read())
sg_urls = []
for sg in cluster["sync_gateways"]:
public = "http://{}:4984".format(sg["ip"])
admin = "http://{}:4985".format(sg["ip"])
sg_urls.append({"public": public, "admin": admin})
ac_urls = ["http://{}:4985".format(sga["ip"]) for sga in cluster["sg_accels"]]
cbs_urls = ["http://{}:8091".format(cb["ip"]) for cb in cluster["couchbase_servers"]]
lbs_urls = ["http://{}".format(lb["ip"]) for lb in cluster["load_balancers"]]
# Format into urls that robot keywords can consume easily
formatted_cluster = {
"sync_gateways": sg_urls,
"sg_accels": ac_urls,
"couchbase_servers": cbs_urls,
"load_balancers": lbs_urls
}
log_info(cluster)
return formatted_cluster
def sync_gateway_base_url_and_package(self,
sg_ce=False,
sg_platform="centos",
sa_platform="centos"):
platform_extension = {
"centos": "rpm",
"ubuntu": "deb",
"windows": "exe"
}
if self._version_number == "1.1.0" or self._build_number == "1.1.1":
log_info("Version unsupported in provisioning.")
raise ProvisioningError("Unsupport version of sync_gateway")
# http://latestbuilds.service.couchbase.com/couchbase-sync-gateway/release/1.1.1/1.1.1-10/couchbase-sync-gateway-enterprise_1.1.1-10_x86_64.rpm
# base_url = "http://latestbuilds.service.couchbase.com/couchbase-sync-gateway/release/{0}/{1}-{2}".format(version, version, build)
# sg_package_name = "couchbase-sync-gateway-enterprise_{0}-{1}_x86_64.rpm".format(version, build)
else:
# http://latestbuilds.service.couchbase.com/builds/latestbuilds/sync_gateway/1.3.1.5/2/couchbase-sync-gateway-enterprise_1.2.0-6_x86_64.rpm
base_url = "http://latestbuilds.service.couchbase.com/builds/latestbuilds/sync_gateway/{0}/{1}".format(
self._version_number, self._build_number)
sg_type = "enterprise"
if sg_ce:
sg_type = "community"
sg_package_name = "couchbase-sync-gateway-{0}_{1}-{2}_x86_64.{3}".format(
sg_type, self._version_number, self._build_number,
platform_extension[sg_platform])
accel_package_name = "couchbase-sg-accel-enterprise_{0}-{1}_x86_64.{2}".format(
self._version_number, self._build_number,
platform_extension[sa_platform])
return base_url, sg_package_name, accel_package_name
def verify_cluster_versions(self, cluster_config, expected_server_version, expected_sync_gateway_version):
log_info("Verfying versions for cluster: {}".format(cluster_config))
with open("{}.json".format(cluster_config)) as f:
cluster_obj = json.loads(f.read())
cbs_ssl = False
if cluster_obj["environment"]["cbs_ssl_enabled"]:
cbs_ssl = True
# Verify Server version
for server in cluster_obj["couchbase_servers"]:
couchbaseserver.verify_server_version(server["ip"], expected_server_version, cbs_ssl=cbs_ssl)
# Verify sync_gateway versions
for sg in cluster_obj["sync_gateways"]:
verify_sync_gateway_product_info(sg["ip"])
verify_sync_gateway_version(sg["ip"], expected_sync_gateway_version)
# Verify sg_accel versions, use the same expected version for sync_gateway for now
for ac in cluster_obj["sg_accels"]:
if compare_versions(expected_sync_gateway_version, "1.5.0") >= 0:
# Only verify the correct product naming after 1.5 since it was fixed in 1.5
verify_sg_accel_product_info(ac["ip"])
verify_sg_accel_version(ac["ip"], expected_sync_gateway_version)
def scan_for_errors(error_strings, log_file_path):
"""
Scans a log file line by line for a provided array of words.
We use this to look for errors, so we expect that no words will be found
If any of the words are found, we raise an exception.
'error_strings' should be a list. Example ['panic', 'error']
"""
if type(error_strings) != list:
raise ValueError("'error_strings must be a list'")
log_info("Looking for {} in {} ...".format(error_strings, log_file_path))
# Scan each line in the log file for the words to search for
with open(log_file_path) as f:
for line in f:
for word in error_strings:
# convert the word to lowercase and the line to all lower case
# which handles the case where 'warning' will catch 'WARNING' and 'Warning', etc
if word.lower() in line.lower():
log_error(line)
raise AssertionError("{} found!! Please review: {} ".format(word, log_file_path))
# No errors found
log_info("Scan complete. Did not find any error strings.")
def load_sync_gateway_config(sync_gateway_config, mode, server_url,
xattrs_enabled, cluster_config):
""" Loads a syncgateway configuration for modification"""
server_scheme, server_ip, server_port = server_url.split(":")
server_ip = server_ip.replace("//", "")
with open(sync_gateway_config) as default_conf:
template = Template(default_conf.read())
if xattrs_enabled:
autoimport_prop = '"import_docs": "continuous",'
xattrs_prop = '"enable_shared_bucket_access": true,'
else:
autoimport_prop = ""
xattrs_prop = ""
couchbase_server_primary_node = add_cbs_to_sg_config_server_field(
cluster_config)
temp = template.render(
couchbase_server_primary_node=couchbase_server_primary_node,
is_index_writer="false",
server_scheme=server_scheme,
server_port=server_port,
autoimport=autoimport_prop,
xattrs=xattrs_prop)
data = json.loads(temp)
log_info("Loaded sync_gateway config: {}".format(data))
return data
def process_changes(self, results):
"""
Add each doc from longpoll changes results to the processed changes list in the following format:
{ "doc_id": [ {"rev": "rev1"}, {"rev", "rev2"}, ...] }
"""
log_info("[Changes Tracker] New changes: {}".format(len(results)))
for doc in results:
if len(doc["changes"]) > 0:
if doc["id"] in self.processed_changes:
# doc has already been seen in the changes feed,
# append new changes to revs accociated with that id
revs_list = self.processed_changes[doc["id"]]
# If the document is already in 'processed_changes', make sure
# that the revision doesn't already exist. If we see one, raise an exception
# because we are seeing the same revision being sent twice
# Checking against this scenario - https://github.com/couchbase/sync_gateway/issues/2186
changes_revs = [change["rev"] for change in doc["changes"]]
revs_list_revs = [rev["rev"] for rev in revs_list]
for change in changes_revs:
if change in revs_list_revs:
raise keywords.exceptions.ChangesError("Duplicates in changes feed!")
revs_list.extend(doc["changes"])
self.processed_changes[doc["id"]] = revs_list
else:
# Stored the doc with the list of rev changes
self.processed_changes[doc["id"]] = doc["changes"]
log_info("[Changes Tracker] Total processed changes: {}".format(len(self.processed_changes)))
def install_sync_gateway(self, cluster_config, sync_gateway_version, sync_gateway_config):
# Dirty hack -- these have to be put here in order to avoid circular imports
from libraries.provision.install_sync_gateway import install_sync_gateway
from libraries.provision.install_sync_gateway import SyncGatewayConfig
if version_is_binary(sync_gateway_version):
version, build = version_and_build(sync_gateway_version)
print("VERSION: {} BUILD: {}".format(version, build))
sg_config = SyncGatewayConfig(None, version, build, sync_gateway_config, "", False)
else:
sg_config = SyncGatewayConfig(sync_gateway_version, None, None, sync_gateway_config, "", False)
install_sync_gateway(cluster_config=cluster_config, sync_gateway_config=sg_config)
log_info("Verfying versions for cluster: {}".format(cluster_config))
with open("{}.json".format(cluster_config)) as f:
cluster_obj = json.loads(f.read())
# Verify sync_gateway versions
for sg in cluster_obj["sync_gateways"]:
verify_sync_gateway_version(sg["ip"], sync_gateway_version)
# Verify sg_accel versions, use the same expected version for sync_gateway for now
for ac in cluster_obj["sg_accels"]:
verify_sg_accel_version(ac["ip"], sync_gateway_version)
def _create_internal_rbac_bucket_user(self, bucketname):
# Create user with username=bucketname and assign role
# bucket_admin and cluster_admin
roles = "ro_admin,bucket_full_access[{}]".format(bucketname)
password = '******'
data_user_params = {
"name": bucketname,
"roles": roles,
"password": password
}
log_info("Creating RBAC user {} with password {} and roles {}".format(
bucketname, password, roles))
rbac_url = "{}/settings/rbac/users/local/{}".format(
self.url, bucketname)
resp = None
try:
resp = self._session.put(rbac_url,
data=data_user_params,
auth=('Administrator', 'password'))
log_r(resp)
resp.raise_for_status()
except HTTPError as h:
log_info("resp code: {}; error: {}".format(resp, h))
raise RBACUserCreationError(h)
def test_raw_attachment(setup_client_syncgateway_test):
"""
1. Add Text attachment to sync_gateway
2. Try to get the raw attachment
Pass: It is possible to get the raw attachment
"""
log_info("Running 'test_raw_attachment'")
ls_url = setup_client_syncgateway_test["ls_url"]
log_info("ls_url: {}".format(ls_url))
client = MobileRestClient()
ls_db = client.create_database(ls_url, name="ls_db")
ls_user_channels = ["NBC"]
atts = attachment.load_from_data_dir(["sample_text.txt"])
doc_with_att = document.create_doc(doc_id="att_doc",
content={"sample_key": "sample_val"},
attachments=atts,
channels=ls_user_channels)
doc = client.add_doc(url=ls_url, db=ls_db, doc=doc_with_att)
att = client.get_attachment(url=ls_url,
db=ls_db,
doc_id=doc["id"],
attachment_name="sample_text.txt")
expected_text = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\nUt enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\nDuis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.\nExcepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."
assert expected_text == att
def test_openidconnect_invalid_scope(params_from_base_test_setup, sg_conf_name):
"""Try to discover the authenticate endpoint URL with a test provider that has an
invalid scope, and expect an error"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_invalid_scope'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
try:
discover_authenticate_endpoint(sg_url, sg_db, "testinvalidscope")
except HTTPError:
log_info("got expected HTTPError trying to get the authenticate endpoint")
# ok we got an exception, which is expected since we are using an invalid scope
return
raise Exception("Expected HTTPError since we are using invalid scope")
def setup_p2p_test(request, setup_p2p_suite):
"""Test setup fixture for p2p client tests"""
log_info("Setting up P2P test ...")
liteserv_one = setup_p2p_suite["liteserv_one"]
liteserv_two = setup_p2p_suite["liteserv_two"]
test_name = request.node.name
print("Starting LiteServ One ...")
ls_logging_one = "{}/logs/{}-ls1-{}-{}.txt".format(RESULTS_DIR, type(liteserv_one).__name__, test_name, datetime.datetime.now())
ls_url_one = liteserv_one.start(ls_logging_one)
print("Starting LiteServ Two ...")
ls_logging_two = "{}/logs/{}-ls2-{}-{}.txt".format(RESULTS_DIR, type(liteserv_two).__name__, test_name, datetime.datetime.now())
ls_url_two = liteserv_two.start(ls_logging_two)
# Yield values to test case via fixture argument
yield {"ls_url_one": ls_url_one, "ls_url_two": ls_url_two}
log_info("Tearing down test")
# Teardown test
client = MobileRestClient()
client.delete_databases(ls_url_one)
client.delete_databases(ls_url_two)
liteserv_one.stop()
liteserv_two.stop()
def __init__(self, config):
self._cluster_config = config
if not os.path.isfile(self._cluster_config):
log_info("Cluster config not found in 'resources/cluster_configs/'")
raise IOError("Cluster config not found in 'resources/cluster_configs/'")
log_info(self._cluster_config)
# Load resources/cluster_configs/<cluster_config>.json
with open("{}.json".format(config)) as f:
cluster = json.loads(f.read())
cbs = [{"name": cbs["name"], "ip": cbs["ip"]} for cbs in cluster["couchbase_servers"]]
sgs = [{"name": sg["name"], "ip": sg["ip"]} for sg in cluster["sync_gateways"]]
acs = [{"name": ac["name"], "ip": ac["ip"]} for ac in cluster["sg_accels"]]
log_info("cbs: {}".format(cbs))
log_info("sgs: {}".format(sgs))
log_info("acs: {}".format(acs))
self.sync_gateways = [SyncGateway(cluster_config=self._cluster_config, target=sg) for sg in sgs]
self.sg_accels = [SgAccel(cluster_config=self._cluster_config, target=ac) for ac in acs]
self.servers = [Server(cluster_config=self._cluster_config, target=cb) for cb in cbs]
self.sync_gateway_config = None # will be set to Config object when reset() called
# for integrating keywords
self.cb_server = keywords.CouchbaseServer.CouchbaseServer(self.servers[0].url)
def start(self, config):
conf_path = os.path.abspath(config)
log.info(">>> Starting sync_gateway with configuration: {}".format(
conf_path))
playbook_vars = {
"sync_gateway_config_filepath": conf_path,
"server_port": self.server_port,
"server_scheme": self.server_scheme,
"autoimport": "",
"xattrs": "",
"no_conflicts": "",
"couchbase_server_primary_node": self.couchbase_server_primary_node
}
if is_xattrs_enabled(self.cluster_config):
playbook_vars["autoimport"] = '"import_docs": "continuous",'
playbook_vars["xattrs"] = '"enable_shared_bucket_access": true,'
if no_conflicts_enabled(self.cluster_config):
playbook_vars["no_conflicts"] = '"allow_conflicts": false,'
try:
revs_limit = get_revs_limit(self.cluster_config)
playbook_vars["revs_limit"] = '"revs_limit": {},'.format(
revs_limit)
except KeyError as ex:
log_info("Keyerror in getting revs_limit{}".format(ex.message))
status = self.ansible_runner.run_ansible_playbook(
"start-sync-gateway.yml",
extra_vars=playbook_vars,
subset=self.hostname)
return status
def test_openidconnect_oidc_challenge_invalid_provider_name(params_from_base_test_setup, sg_conf_name):
"""
If oidc_challenge is called with an invalid provider name, it should not return
an Www-Authenticate header
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_oidc_challenge_invalid_provider_name'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
# make a request to the _oidc_challenge endpoint
oidc_challenge_url = "{}/{}/_oidc_challenge?provider={}".format(sg_url, sg_db, "bogusprovider")
response = requests.get(oidc_challenge_url)
log_info("response.headers: {}".format(response.headers))
assert "Www-Authenticate" not in response.headers
assert response.status_code == 400
def get_bucket_names(self):
""" Returns list of the bucket names for a given Couchbase Server."""
bucket_names = []
error_count = 0
max_retries = 5
# Retry to avoid intermittent Connection issues when getting buckets
while True:
if error_count == max_retries:
raise CBServerError(
"Error! Could not get buckets after retries.")
try:
resp = self._session.get("{}/pools/default/buckets".format(
self.url))
log_r(resp)
resp.raise_for_status()
break
except ConnectionError:
log_info(
"Hit a ConnectionError while trying to get buckets. Retrying ..."
)
error_count += 1
time.sleep(1)
obj = json.loads(resp.text)
for entry in obj:
bucket_names.append(entry["name"])
log_info("Found buckets: {}".format(bucket_names))
return bucket_names
def params_from_base_test_setup(request, params_from_base_suite_setup):
# Code before the yeild will execute before each test starts
# pytest command line parameters
collect_logs = request.config.getoption("--collect-logs")
cluster_config = params_from_base_suite_setup["cluster_config"]
mode = params_from_base_suite_setup["mode"]
xattrs_enabled = params_from_base_suite_setup["xattrs_enabled"]
test_name = request.node.name
log_info("Setting up test '{}'".format(test_name))
# This dictionary is passed to each test
yield {
"cluster_config": cluster_config,
"mode": mode,
"xattrs_enabled": xattrs_enabled
}
# Code after the yeild will execute when each test finishes
log_info("Tearing down test '{}'".format(test_name))
# Verify all sync_gateways and sg_accels are reachable
c = cluster.Cluster(cluster_config)
errors = c.verify_alive(mode)
# if the test failed pull logs
if collect_logs or request.node.rep_call.failed or len(errors) != 0:
logging_helper = Logging()
logging_helper.fetch_and_analyze_logs(cluster_config=cluster_config,
test_name=test_name)
assert len(errors) == 0
def download(self):
"""
1. Check to see if .apk is downloaded already. If so, return
2. Download the LiteServ .apk from latest builds to 'deps/binaries'
"""
version, build = version_and_build(self.version_build)
if version == "1.2.1":
package_name = "couchbase-lite-android-liteserv-SQLite-{}-debug.apk".format(self.version_build)
else:
if self.storage_engine == "SQLite":
package_name = "couchbase-lite-android-liteserv-SQLite-{}-debug.apk".format(self.version_build)
else:
package_name = "couchbase-lite-android-liteserv-SQLCipher-ForestDB-Encryption-{}-debug.apk".format(self.version_build)
expected_binary_path = "{}/{}".format(BINARY_DIR, package_name)
if os.path.isfile(expected_binary_path):
log_info("Package is already downloaded. Skipping.")
return
# Package not downloaded, proceed to download from latest builds
if version == "1.2.1":
url = "{}/couchbase-lite-android/release/{}/{}/{}".format(LATEST_BUILDS, version, self.version_build, package_name)
else:
url = "{}/couchbase-lite-android/{}/{}/{}".format(LATEST_BUILDS, version, self.version_build, package_name)
log_info("Downloading {} -> {}/{}".format(url, BINARY_DIR, package_name))
resp = requests.get(url)
resp.raise_for_status()
with open("{}/{}".format(BINARY_DIR, package_name), "wb") as f:
f.write(resp.content)
def __init__(self, version_build, host, port, storage_engine):
# Initialize baseclass properies
super(LiteServNetMsft, self).__init__(version_build, host, port, storage_engine)
if "LITESERV_MSFT_HOST_USER" not in os.environ:
raise LiteServError("Make sure you define 'LITESERV_MSFT_HOST_USER' as the windows user for the host you are targeting")
if "LITESERV_MSFT_HOST_PASSWORD" not in os.environ:
raise LiteServError("Make sure you define 'LITESERV_MSFT_HOST_PASSWORD' as the windows user for the host you are targeting")
# Create config for LiteServ Windows host
ansible_liteserv_mfst_target_lines = [
"[windows]",
"win1 ansible_host={}".format(host),
"[windows:vars]",
"ansible_user={}".format(os.environ["LITESERV_MSFT_HOST_USER"]),
"ansible_password={}".format(os.environ["LITESERV_MSFT_HOST_PASSWORD"]),
"ansible_port=5986",
"ansible_connection=winrm",
"# The following is necessary for Python 2.7.9+ when using default WinRM self-signed certificates:",
"ansible_winrm_server_cert_validation=ignore",
]
ansible_liteserv_mfst_target_string = "\n".join(ansible_liteserv_mfst_target_lines)
log_info("Writing: {}".format(ansible_liteserv_mfst_target_string))
config_location = "resources/liteserv_configs/net-msft"
with open(config_location, "w") as f:
f.write(ansible_liteserv_mfst_target_string)
self.ansible_runner = AnsibleRunner(config=config_location)
def do_GET(self):
log_info('Received GET request')
self.send_response(200)
self.send_header('Last-Modified', self.date_time_string(time.time()))
self.end_headers()
self.wfile.write('Response body\n')
return
def _verify_launched(self):
"""Poll on expected http://<host>:<port> until it is reachable
Assert that the response contains the expected version information
"""
resp_obj = self._wait_until_reachable()
log_info(resp_obj)
# .NET Microsoft Windows 10.12/x86_64 1.3.1-build0013/5d1553d
running_version = resp_obj["vendor"]["version"]
if not running_version.startswith(".NET Microsoft Windows"):
raise LiteServError("Invalid platform running!")
# ['.NET', 'Microsoft', 'Windows', '10', 'Enterprise', 'x64', '1.4.0', 'build0043', '5cfe25b']
running_version_parts = re.split("[ /-]", running_version)
running_version = running_version_parts[6]
running_build = int(running_version_parts[7].strip("build"))
running_version_composed = "{}-{}".format(running_version,
running_build)
if self.version_build != running_version_composed:
raise LiteServError(
"Expected version does not match actual version: Expected={} Actual={}"
.format(self.version_build, running_version_composed))
def test_bucket_shadow_low_revs_limit_repeated_deletes(params_from_base_test_setup):
"""
Validate that Sync Gateway doesn't panic (and instead creates a conflict branch
and prints a warning) after doing the following steps:
- Set revs_limit to 5
- Create a doc via SG
- Issue a delete operation for that doc via SG
- Repeat step 3 5x. (each additional delete will create a new revision in SG, but the delete on the source bucket will fail with the 'not found' error, which also means that upstream_rev won't get incremented
- Recreate the doc in the source bucket
See https://github.com/couchbaselabs/sync-gateway-testcluster/issues/291#issuecomment-191521993
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
default_config_path_shadower_low_revs = sync_gateway_config_path_for_mode("sync_gateway_bucketshadow_low_revs", mode)
default_config_path_non_shadower_low_revs = sync_gateway_config_path_for_mode("sync_gateway_default_low_revs", mode)
log_info("Running 'test_bucket_shadow_low_revs_limit_repeated_deletes'")
log_info("Using cluster_config: {}".format(cluster_config))
cluster = Cluster(config=cluster_config)
sc = init_shadow_cluster(cluster,
default_config_path_shadower_low_revs,
default_config_path_non_shadower_low_revs)
# Write doc into shadower SG
doc_id = sc.alice_shadower.add_doc()
# Wait until it gets to source bucket
get_doc_from_source_bucket_retry(doc_id, sc.source_bucket)
# Wait until upstream-rev in _sync metadata is non empty
# Otherwise, this will not reproduce a panic
while True:
doc = sc.data_bucket.get(doc_id)
if doc.success:
if "upstream_rev" in doc.value["_sync"]:
break
time.sleep(1)
# Repeatedly issue a delete operation for that doc via SG
# Keep adding tombstone revs to the one and only branch
rev_id_to_delete = None
for i in xrange(100):
resp = sc.alice_shadower.delete_doc(doc_id, rev_id_to_delete)
rev_id_to_delete = resp["rev"]
# Recreate doc with that ID in the source bucket
sc.source_bucket.upsert(doc_id, json.loads('{"foo":"bar"}'))
# Check if SG's are up
errors = cluster.verify_alive(sc.mode)
assert len(errors) == 0
# Restart Shadow SG
sc.shadower_sg.stop()
sc.shadower_sg.start(default_config_path_shadower_low_revs)
def verify_sdk_deletes(sdk_client, expected_deleted_ids):
for doc_id in expected_deleted_ids:
nfe = None
with pytest.raises(NotFoundError) as nfe:
sdk_client.get(doc_id)
assert nfe is not None
log_info(nfe.value.message)
assert 'The key does not exist on the server' in str(nfe)
def verify_sg_purges(sg_client, sg_url, sg_db, expected_deleted_ids, sg_auth):
for doc_id in expected_deleted_ids:
he = None
with pytest.raises(HTTPError) as he:
sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id, auth=sg_auth)
assert he is not None
log_info(he.value.message)
assert he.value.message.startswith('404 Client Error: Not Found for url:')
def _verify_launched(self):
""" Poll on expected http://<host>:<port> until it is reachable
Assert that the response contains the expected version information
"""
resp_obj = self._wait_until_reachable()
log_info(resp_obj)
if resp_obj["version"] != self.version_build:
raise LiteServError("Expected version: {} does not match running version: {}".format(self.version_build, resp_obj["version"]))
def start(self, timeout=1000, heartbeat=None, request_timeout=None):
"""
Start a longpoll changes feed and and store the results in self.processed changes
"""
# convert to seconds for use with requests lib api
if request_timeout is not None:
request_timeout /= 1000
auth_type = get_auth_type(self.auth)
current_seq_num = 0
log_info("[Changes Tracker] Changes Tracker Starting ...")
while not self.cancel:
data = {
"feed": "longpoll",
"style": "all_docs",
"since": current_seq_num
}
if timeout is not None:
data["timeout"] = timeout
if heartbeat is not None:
data["heartbeat"] = heartbeat
if auth_type == AuthType.session:
try:
resp = requests.post("{}/_changes".format(self.endpoint), data=json.dumps(data), cookies=dict(SyncGatewaySession=self.auth[1]), timeout=request_timeout)
except Timeout as to:
log_info("Request timed out. Exiting longpoll loop ...")
logging.debug(to)
break
elif auth_type == AuthType.http_basic:
try:
resp = requests.post("{}/_changes".format(self.endpoint), data=json.dumps(data), auth=self.auth, timeout=request_timeout)
except Timeout as to:
log_info("Request timed out. Exiting longpoll loop ...")
logging.debug(to)
break
else:
try:
resp = requests.post("{}/_changes".format(self.endpoint), data=json.dumps(data), timeout=request_timeout)
except Timeout as to:
log_info("Request timed out. Exiting longpoll loop ...")
logging.debug(to)
break
log_r(resp)
resp.raise_for_status()
resp_obj = resp.json()
self.process_changes(resp_obj["results"])
current_seq_num = resp_obj["last_seq"]
log_info("[Changes Tracker] End of longpoll changes loop")
def test_view_query_performance(setup_client_syncgateway_test):
"""
@summary
Run this test when new iOS version arrives to make sure CBL query performance is not diminishing
1. Add 100000 docs to the client with content
2. Create design doc version 1 to fetch doc._id, doc._rev for docs with content
3. Update docs 3 times which gets revision number 4-
4. Run a query and check for 100000 expected docs with design doc version 1
3. Verify view query finished less than 5 seconds
"""
log_info("Running 'test_design_doc_update'")
ls_url = setup_client_syncgateway_test["ls_url"]
log_info("ls_url: {}".format(ls_url))
client = MobileRestClient()
ls_url = setup_client_syncgateway_test["ls_url"]
num_content_docs_per_db = 100000
d_doc_name = "dd"
ls_db = client.create_database(ls_url, name="ls_db")
# Add 100000 docs to the client with content
bulk_docs_content = create_docs("doc_content_",
num_content_docs_per_db,
content={"hi": "I should be in the view"})
ls_db_docs1 = client.add_bulk_docs(url=ls_url,
db=ls_db,
docs=bulk_docs_content)
assert len(ls_db_docs1) == num_content_docs_per_db
client.update_docs(url=ls_url,
db=ls_db,
docs=ls_db_docs1,
number_updates=3,
delay=0.1)
# Design doc to to fetch doc._id, doc._rev for docs with content
view = """{
"language" : "javascript",
"views" : {
"content_view" : {
"map" : "function(doc, meta) { if (doc.content) { emit(doc._id, doc._rev); } }"
}
}
}"""
client.add_design_doc(url=ls_url, db=ls_db, name=d_doc_name, doc=view)
start = time.time()
content_view_rows_1 = client.get_view(url=ls_url,
db=ls_db,
design_doc_name=d_doc_name,
view_name="content_view")
finish = time.time()
assert finish - start < 5
client.verify_view_row_num(view_response=content_view_rows_1,
expected_num_rows=10000)
def send_changes_termination_doc(sg_url, sg_db, users, terminator_doc_id,
terminator_channel):
sg_client = MobileRestClient()
random_user_id = random.choice(users.keys())
random_user = users[random_user_id]
log_info('Sending changes termination doc for all users')
doc = {'_id': terminator_doc_id, 'channels': [terminator_channel]}
sg_client.add_doc(url=sg_url, db=sg_db, doc=doc, auth=random_user['auth'])
def start(self):
log_info('Starting webserver on port :8080 ...')
thread = threading.Thread(target=self.server.serve_forever)
thread.daemon = True
try:
thread.start()
except Exception as e:
raise ValueError(
"Caught exception could not launch webserver thread", e)
def unzip_log_files(directory):
""" Scan directory recursively for .zip files and unzip them to a folder with the same name """
zip_files = get_file_paths_with_extension(directory, '.zip')
for zip_file in zip_files:
zip_file_extract_dir = zip_file.replace('.zip', '')
log_info('Unzipping: {}'.format(zip_file))
with zipfile.ZipFile(zip_file) as zf:
zf.extractall(zip_file_extract_dir)
def start(self, logfile_name):
"""
1. Starts a LiteServ with logging to provided logfile file object.
The running LiteServ process will be stored in the self.process property.
2. The method will poll on the endpoint to make sure LiteServ is available.
3. The expected version will be compared with the version reported by http://<host>:<port>
4. eturn the url of the running LiteServ
"""
self._verify_not_running()
self.logfile = logfile_name
process_args = [
"--port", str(self.port),
"--dir", "."
]
if self.storage_engine == "ForestDB" or self.storage_engine == "ForestDB+Encryption":
process_args.append("--storage")
process_args.append("ForestDB")
else:
process_args.append("--storage")
process_args.append("SQLite")
if self.storage_engine == "SQLCipher" or self.storage_engine == "ForestDB+Encryption":
log_info("Using Encryption ...")
db_flags = []
for db_name in REGISTERED_CLIENT_DBS:
db_flags.append("--dbpassword")
db_flags.append("{}=pass".format(db_name))
process_args.extend(db_flags)
# The package structure for LiteServ is different pre 1.4. Handle for this case
if self.version_build.startswith("1.2") or self.version_build.startswith("1.3"):
binary_path = "couchbase-lite-net-msft-{}-liteserv/LiteServ.exe".format(self.version_build)
else:
binary_path = "couchbase-lite-net-msft-{}-liteserv/net45/LiteServ.exe".format(self.version_build)
joined_args = " ".join(process_args)
log_info("Starting LiteServ {} with: {}".format(binary_path, joined_args))
# Start LiteServ via Ansible on remote machine
status = self.ansible_runner.run_ansible_playbook(
"start-liteserv-msft.yml",
extra_vars={
"binary_path": binary_path,
"launch_args": joined_args,
}
)
if status != 0:
raise LiteServError("Could not start Liteserv")
self._verify_launched()
return "http://{}:{}".format(self.host, self.port)
def parallel_process(objects, method, *args):
with concurrent.futures.ProcessPoolExecutor(max_workers=settings.MAX_REQUEST_WORKERS) as executor:
futures = {executor.submit(getattr(obj, method), *args): obj for obj in objects}
for future in concurrent.futures.as_completed(futures):
if concurrent.futures.as_completed(futures):
obj = futures[future]
try:
log_debug("Object {} method {} output {}".format(obj, method, future.result()))
except Exception as exception:
log_info('Generated an exception : {} : {}'.format(obj, exception))
def dump_results(test_folder, gateload_results, sync_gateway_results):
filename = "testsuites/syncgateway/performance/results/{}/gateload_expvars.json".format(test_folder)
log_info("Writing gateload_expvars to: {}".format(filename))
with open(filename, "w") as f:
f.write(json.dumps(gateload_results))
filename = "testsuites/syncgateway/performance/results/{}/sync_gateway_expvars.json".format(test_folder)
log_info("Writing sync_gateway_expvars to: {}".format(filename))
with open(filename, "w") as f:
f.write(json.dumps(sync_gateway_results))
def stop_sync_gateway(self, cluster_config, url):
target = hostname_for_url(cluster_config, url)
log_info("Shutting down sync_gateway on {} ...".format(target))
ansible_runner = AnsibleRunner(cluster_config)
status = ansible_runner.run_ansible_playbook(
"stop-sync-gateway.yml",
subset=target
)
if status != 0:
raise ProvisioningError("Could not stop sync_gateway")
def test_openidconnect_large_scope(params_from_base_test_setup, sg_conf_name):
"""Authenticate against a test provider config that only has a larger scope than the default,
and make sure things like the nickname are returned in the jwt token returned back"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_large_scope'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
# multipart/form data content
formdata = {
'username': ('', 'testuser'),
'authenticated': ('', 'Return a valid authorization code for this user')
}
# get the authenticate endpoint and query params, should look something like:
# authenticate?client_id=sync_gateway&redirect_uri= ...
authenticate_endpoint = discover_authenticate_endpoint(sg_url, sg_db, "testlargescope")
# build the full url
url = "{}/{}/_oidc_testing/{}".format(
sg_url,
sg_db,
authenticate_endpoint
)
# Make the request to _oidc_testing
response = requests.post(url, files=formdata)
log_r(response)
# extract the token from the response
response_json = response.json()
id_token = response_json["id_token"]
# {u'iss': u'http://localhost:4984/db/_oidc_testing', u'iat': 1466050188, u'aud': u'sync_gateway', u'exp': 1466053788, u'sub': u'testuser'}
decoded_id_token = jwt.decode(id_token, verify=False)
log_info("decoded_id_token: {}".format(decoded_id_token))
assert "nickname" in decoded_id_token.keys()
def test_sg_replicate_push_async(params_from_base_test_setup, num_docs):
assert num_docs > 0
# if the async stuff works, we should be able to kick off a large
# push replication and get a missing doc before the replication has
# a chance to finish. And then we should later see that doc.
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
log_info("Running 'test_sg_replicate_push_async'")
log_info("Using cluster_config: {}".format(cluster_config))
config = sync_gateway_config_path_for_mode("sync_gateway_sg_replicate", mode)
sg1, sg2 = create_sync_gateways(
cluster_config=cluster_config,
sg_config_path=config
)
admin = Admin(sg1)
admin.admin_url = sg1.url
sg1_user, sg2_user = create_sg_users(sg1, sg2, DB1, DB2)
# Add docs to sg1
doc_ids_added = []
last_doc_id_added = None
for i in xrange(num_docs):
doc_id = sg1_user.add_doc()
doc_ids_added.append(doc_id)
last_doc_id_added = doc_id
# Wait until doc shows up on sg1's changes feed
wait_until_doc_in_changes_feed(sg1, DB1, last_doc_id_added)
# try to get the last doc added from the target -- assert that we get an exception
assert_does_not_have_doc(sg2_user, last_doc_id_added)
# kick off a one-off push replication with async=true
sg1.start_push_replication(
sg2.admin.admin_url,
DB1,
DB2,
continuous=False,
use_remote_source=True,
async=True,
use_admin_url=True
)
# wait until that doc shows up on the target
wait_until_doc_sync(sg2_user, last_doc_id_added)
# At this point, the active tasks should be empty
wait_until_active_tasks_empty(sg1)
def test_net_msft_storage_engine(request, liteserv_with_storage_engine_from_fixture):
liteserv = liteserv_with_storage_engine_from_fixture
test_name = request.node.name
logfile = "{}/logs/{}-{}-{}.txt".format(RESULTS_DIR, type(liteserv).__name__, test_name, datetime.datetime.now())
ls_url = liteserv.start(logfile)
client = MobileRestClient()
client.create_database(ls_url, "ls_db")
liteserv.stop()
storage_engine = liteserv.storage_engine
log_info("Testing storage_engine: {}".format(storage_engine))
with open(logfile, "r") as f:
contents = f.read()
if storage_engine == "SQLite":
# Note: SQLite mode uses SQLCipher by default
assert (
"Using Couchbase.Lite.Storage.SQLCipher.SqliteCouchStore for db at C:\Users\user\Desktop\LiteServ\ls_db.cblite2"
in contents
)
assert "encryption key given" not in contents
elif storage_engine == "SQLCipher":
assert (
"Using Couchbase.Lite.Storage.SQLCipher.SqliteCouchStore for db at C:\Users\user\Desktop\LiteServ\ls_db.cblite2"
in contents
)
assert "Open C:\Users\user\Desktop\LiteServ\ls_db.cblite2\db.sqlite3" in contents
assert "encryption key given"
elif storage_engine == "ForestDB":
assert (
"Using Couchbase.Lite.Storage.ForestDB.ForestDBCouchStore for db at C:\Users\user\Desktop\LiteServ\ls_db.cblite2"
in contents
)
assert "Database is encrypted; setting CBForest encryption key" not in contents
elif storage_engine == "ForestDB+Encryption":
assert (
"Using Couchbase.Lite.Storage.ForestDB.ForestDBCouchStore for db at C:\Users\user\Desktop\LiteServ\ls_db.cblite2"
in contents
)
assert "Database is encrypted; setting CBForest encryption key" in contents
else:
pytest.xfail("Invalid Storage Engine")
def unset_cluster_config(self):
"""Will unset the CLUSTER_CONFIG environment variable if it is set.
Will fail if CLUSTER_CONFIG is not set
"""
if "CLUSTER_CONFIG" not in os.environ:
raise ProvisioningError("Trying to unset CLUSTER_CONFIG but it is not defined")
log_info("Unsetting CLUSTER_CONFIG")
del os.environ["CLUSTER_CONFIG"]
