def test_log_rotation_negative(params_from_base_test_setup, sg_conf_name):
"""Test log rotation with negative values for:
"maxsize": -1,
"maxage": -30,
"maxbackups": -2
SG shouldn't start
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup["xattrs_enabled"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_helper = ClusterKeywords()
cluster_hosts = cluster_helper.get_cluster_topology(cluster_conf)
sg_one_url = cluster_hosts["sync_gateways"][0]["public"]
# read sample sg_conf
data = load_sync_gateway_config(sg_conf, mode,
cluster_hosts["couchbase_servers"][0],
xattrs_enabled, cluster_conf)
# set negative values for rotation section
data['logging']["default"]["rotation"] = {
"maxsize": -1,
"maxage": -30,
"maxbackups": -2,
"localtime": True
}
# create temp config file in the same folder as sg_conf
temp_conf = "/".join(sg_conf.split('/')[:-2]) + '/temp_conf.json'
with open(temp_conf, 'w') as fp:
json.dump(data, fp)
# Stop sync_gateways
log_info(">>> Stopping sync_gateway")
sg_helper = SyncGateway()
sg_helper.stop_sync_gateways(cluster_config=cluster_conf, url=sg_one_url)
try:
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=temp_conf)
except ProvisioningError:
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=sg_conf)
# Remove generated conf file
os.remove(temp_conf)
return
# Remove generated conf file
os.remove(temp_conf)
pytest.fail("SG shouldn't be started!!!!")
def test_log_rotation_invalid_path(params_from_base_test_setup, sg_conf_name):
"""Test to check that SG is not started with invalid logFilePath.
OS specific case. SG should check if path correct on startup
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup["xattrs_enabled"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_helper = ClusterKeywords()
cluster_hosts = cluster_helper.get_cluster_topology(cluster_conf)
sg_one_url = cluster_hosts["sync_gateways"][0]["public"]
# read sample sg_conf
data = load_sync_gateway_config(sg_conf, mode,
cluster_hosts["couchbase_servers"][0],
xattrs_enabled, cluster_conf)
# set non existing logFilePath
data['logging']["default"]["logFilePath"] = "/12345/1231/131231.log"
# create temp config file in the same folder as sg_conf
temp_conf = "/".join(sg_conf.split('/')[:-2]) + '/temp_conf.json'
with open(temp_conf, 'w') as fp:
json.dump(data, fp)
# Stop sync_gateways
log_info(">>> Stopping sync_gateway")
sg_helper = SyncGateway()
sg_helper.stop_sync_gateways(cluster_config=cluster_conf, url=sg_one_url)
try:
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=temp_conf)
except ProvisioningError:
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=sg_conf)
# Remove generated conf file
os.remove(temp_conf)
return
# Remove generated conf file
os.remove(temp_conf)
pytest.fail("SG shouldn't be started!!!!")
def test_log_logLevel_invalid(params_from_base_test_setup, sg_conf_name):
"""Run SG with non existing logLevel value
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup["xattrs_enabled"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_helper = ClusterKeywords()
cluster_hosts = cluster_helper.get_cluster_topology(cluster_conf)
sg_one_url = cluster_hosts["sync_gateways"][0]["public"]
# read sample sg_conf
data = load_sync_gateway_config(sg_conf, mode,
cluster_hosts["couchbase_servers"][0],
xattrs_enabled, cluster_conf)
# 'debugFake' invalid value for logLevel
data['logging']["default"]["logLevel"] = "debugFake"
temp_conf = "/".join(sg_conf.split('/')[:-2]) + '/temp_conf.json'
# create temp config file in the same folder as sg_conf
with open(temp_conf, 'w') as fp:
json.dump(data, fp)
# Stop sync_gateways
log_info(">>> Stopping sync_gateway")
sg_helper = SyncGateway()
sg_helper.stop_sync_gateways(cluster_config=cluster_conf, url=sg_one_url)
try:
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=temp_conf)
except ProvisioningError:
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=sg_conf)
# Remove generated conf file
os.remove(temp_conf)
return
# Remove generated conf file
os.remove(temp_conf)
pytest.fail("SG shouldn't be started!!!!")
def test_log_logKeys_string(params_from_base_test_setup, sg_conf_name):
"""Negative test to verify that we are not able start SG when
logKeys is string
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup["xattrs_enabled"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
# read sample sg_conf
cluster_helper = ClusterKeywords()
cluster_hosts = cluster_helper.get_cluster_topology(cluster_conf)
sg_one_url = cluster_hosts["sync_gateways"][0]["public"]
data = load_sync_gateway_config(sg_conf, mode,
cluster_hosts["couchbase_servers"][0],
xattrs_enabled, cluster_conf)
# set logKeys as string in config file
data['logging']["default"]["logKeys"] = "http"
# create temp config file in the same folder as sg_conf
temp_conf = "/".join(sg_conf.split('/')[:-2]) + '/temp_conf.json'
with open(temp_conf, 'w') as fp:
json.dump(data, fp)
# Stop sync_gateways
log_info(">>> Stopping sync_gateway")
sg_helper = SyncGateway()
sg_helper.stop_sync_gateways(cluster_config=cluster_conf, url=sg_one_url)
try:
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=temp_conf)
except ProvisioningError:
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=sg_conf)
# Remove generated conf file
os.remove(temp_conf)
return
# Remove generated conf file
os.remove(temp_conf)
pytest.fail("SG shouldn't be started!!!!")
def test_log_nondefault_logKeys_set(params_from_base_test_setup, sg_conf_name):
"""Test to verify non default logKeys with any invalid area.
SG should work even with non existing logging area
(positive case)
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup["xattrs_enabled"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
# read sample sg_conf
cluster_helper = ClusterKeywords()
cluster_hosts = cluster_helper.get_cluster_topology(cluster_conf)
sg_one_url = cluster_hosts["sync_gateways"][0]["public"]
data = load_sync_gateway_config(sg_conf, mode,
cluster_hosts["couchbase_servers"][0],
xattrs_enabled, cluster_conf)
# "FAKE" not valid area in logging
data['logging']["default"]["logKeys"] = ["HTTP", "FAKE"]
# create temp config file in the same folder as sg_conf
temp_conf = "/".join(sg_conf.split('/')[:-2]) + '/temp_conf.json'
with open(temp_conf, 'w') as fp:
json.dump(data, fp)
# Stop sync_gateways
log_info(">>> Stopping sync_gateway")
sg_helper = SyncGateway()
sg_helper.stop_sync_gateways(cluster_config=cluster_conf, url=sg_one_url)
# Start sync_gateways
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=temp_conf)
# Remove generated conf file
os.remove(temp_conf)
def test_detect_stale_channel_index(params_from_base_test_setup, sg_conf):
"""
1. Bring up single Sync Gateway node, backed by Couchbase Server with 3 accels indexing
2. Configure such that the primary bucket and the channel index bucket are different (which is the norm)
3. Add 1000 documents
4. Shutdown Sync Gateway
5. Delete / create the primary bucket ('data-bucket'), but do not touch the channel index bucket
6. Start Sync Gateway
7. Assert that sync_gateway fails to start due to stale channel index
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
log_info("Running 'test_detect_stale_channel_index'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_util = ClusterKeywords()
topology = cluster_util.get_cluster_topology(cluster_conf)
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
cb_server_url = topology["couchbase_servers"][0]
sg_db = "db"
num_docs = 1000
cb_server = couchbaseserver.CouchbaseServer(url=cb_server_url)
client = MobileRestClient()
# Create doc pusher user
doc_pusher_user_info = userinfo.UserInfo(name="doc_pusher",
password="******",
channels=["NASA"],
roles=[])
doc_pusher_auth = client.create_user(
url=sg_admin_url,
db=sg_db,
name=doc_pusher_user_info.name,
password=doc_pusher_user_info.password,
channels=doc_pusher_user_info.channels)
# Add some docs to Sync Gateway to cause indexing
docs = document.create_docs(None,
number=num_docs,
channels=doc_pusher_user_info.channels)
pushed_docs = client.add_bulk_docs(url=sg_url,
db=sg_db,
docs=docs,
auth=doc_pusher_auth)
assert len(pushed_docs) == num_docs
# Shut down sync_gateway
sg_util = SyncGateway()
sg_util.stop_sync_gateways(cluster_config=cluster_conf, url=sg_url)
# Delete server bucket
cb_server.delete_bucket(name="data-bucket")
# Create server bucket
ram_per_bucket_mb = cb_server.get_ram_per_bucket(num_buckets=2)
cb_server.create_bucket(name="data-bucket", ram_quota_mb=ram_per_bucket_mb)
# Start sync_gateway and assert that a Provisioning error is raised due to detecting stale index
with pytest.raises(exceptions.ProvisioningError):
sg_util.start_sync_gateways(cluster_config=cluster_conf,
url=sg_url,
config=sg_conf)
# TODO: To make this check even more accurate, could
# run remote ssh command "systemctl status sync_gateway.service" and look for
# regex pattern: Main PID: 7185 (code=exited, status=2)
# Delete index bucket and recreate it
cb_server.delete_bucket(name="index-bucket")
cb_server.create_bucket(name="index-bucket",
ram_quota_mb=ram_per_bucket_mb)
# Start sync gateway, should succeed now
sg_util.start_sync_gateways(cluster_config=cluster_conf,
url=sg_url,
config=sg_conf)
def test_view_backfill_for_deletes(params_from_base_test_setup, sg_conf_name,
validate_changes_before_restart):
"""
Scenario:
1. Write a bunch of docs
2. Delete 1/2
3. Restart Sync Gateway
4. Issue _changes, assert view backfills docs and delete notifications
"""
num_docs = 1000
sg_db = 'db'
cluster_conf = params_from_base_test_setup['cluster_config']
cluster_topology = params_from_base_test_setup['cluster_topology']
mode = params_from_base_test_setup['mode']
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
sg_admin_url = cluster_topology['sync_gateways'][0]['admin']
sg_url = cluster_topology['sync_gateways'][0]['public']
cbs_url = cluster_topology['couchbase_servers'][0]
log_info('sg_conf: {}'.format(sg_conf))
log_info('sg_admin_url: {}'.format(sg_admin_url))
log_info('sg_url: {}'.format(sg_url))
log_info('cbs_url: {}'.format(cbs_url))
log_info('validate_changes_before_restart: {}'.format(
validate_changes_before_restart))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
# Create clients
sg_client = MobileRestClient()
# Create user / session
seth_user_info = UserInfo(name='seth',
password='******',
channels=['NASA', 'NATGEO'],
roles=[])
sg_client.create_user(url=sg_admin_url,
db=sg_db,
name=seth_user_info.name,
password=seth_user_info.password,
channels=seth_user_info.channels)
seth_auth = sg_client.create_session(url=sg_admin_url,
db=sg_db,
name=seth_user_info.name,
password=seth_user_info.password)
# Add 'num_docs' to Sync Gateway
doc_bodies = document.create_docs('test_doc',
number=num_docs,
channels=seth_user_info.channels)
bulk_resp = sg_client.add_bulk_docs(url=sg_url,
db=sg_db,
docs=doc_bodies,
auth=seth_auth)
assert len(bulk_resp) == num_docs
# Delete half of the docs randomly
deleted_docs = []
for _ in range(num_docs / 2):
random_doc = random.choice(bulk_resp)
deleted_doc = sg_client.delete_doc(url=sg_url,
db=sg_db,
doc_id=random_doc['id'],
rev=random_doc['rev'],
auth=seth_auth)
deleted_docs.append(deleted_doc)
bulk_resp.remove(random_doc)
log_info('Number of docs deleted: {}'.format(len(deleted_docs)))
all_docs = bulk_resp + deleted_docs
log_info('Number of docs to look for in changes: {}'.format(len(all_docs)))
# This test will check changes before and after SG restart if
# validate_changes_before_restart == True
# If it is not set to True, only build the changes after restart
if validate_changes_before_restart:
# Verify deletions and inital docs show up in changes feed
sg_client.verify_docs_in_changes(url=sg_url,
db=sg_db,
expected_docs=all_docs,
auth=seth_auth)
changes = sg_client.get_changes(url=sg_url,
db=sg_db,
since=0,
auth=seth_auth)
# All docs should show up + _user doc
assert len(changes['results']) == num_docs + 1
deleted_doc_ids = [doc['id'] for doc in deleted_docs]
assert len(deleted_doc_ids) == num_docs / 2
deleted_docs_in_changes = [
change['id'] for change in changes['results']
if 'deleted' in change and change['deleted']
]
assert len(deleted_docs_in_changes) == num_docs / 2
# All deleted docs should show up in the changes feed
for doc_id in deleted_docs_in_changes:
assert doc_id in deleted_doc_ids
deleted_doc_ids.remove(doc_id)
assert len(deleted_doc_ids) == 0
# Restart Sync Gateway
sg_controller = SyncGateway()
sg_controller.stop_sync_gateways(url=sg_url, cluster_config=cluster_conf)
sg_controller.start_sync_gateways(url=sg_url,
cluster_config=cluster_conf,
config=sg_conf)
# Verify deletions and inital docs show up in changes feed
sg_client.verify_docs_in_changes(url=sg_url,
db=sg_db,
expected_docs=all_docs,
auth=seth_auth)
changes = sg_client.get_changes(url=sg_url,
db=sg_db,
since=0,
auth=seth_auth)
# All docs should show up + _user doc
assert len(changes['results']) == num_docs + 1
deleted_doc_ids = [doc['id'] for doc in deleted_docs]
assert len(deleted_doc_ids) == num_docs / 2
deleted_docs_in_changes = [
change['id'] for change in changes['results']
if 'deleted' in change and change['deleted']
]
assert len(deleted_docs_in_changes) == num_docs / 2
# All deleted docs should show up in th changes feed
for doc_id in deleted_docs_in_changes:
assert doc_id in deleted_doc_ids
deleted_doc_ids.remove(doc_id)
assert len(deleted_doc_ids) == 0
def test_system_test(params_from_base_test_setup):
cluster_config = params_from_base_test_setup['cluster_config']
mode = params_from_base_test_setup['mode']
# Scenario parameters
server_seed_docs = int(params_from_base_test_setup['server_seed_docs'])
max_docs = int(params_from_base_test_setup['max_docs'])
num_users = int(params_from_base_test_setup['num_users'])
# Create paramters
create_batch_size = int(params_from_base_test_setup['create_batch_size'])
create_delay = float(params_from_base_test_setup['create_delay'])
# Update parameters
update_runtime_sec = int(params_from_base_test_setup['update_runtime_sec'])
update_batch_size = int(params_from_base_test_setup['update_batch_size'])
update_docs_percentage = float(
params_from_base_test_setup['update_docs_percentage'])
update_delay = float(params_from_base_test_setup['update_delay'])
# Changes parameters
changes_delay = float(params_from_base_test_setup['changes_delay'])
changes_limit = int(params_from_base_test_setup['changes_limit'])
changes_terminator_doc_id = 'terminator'
docs_per_user = max_docs / num_users
docs_per_user_per_update = int(update_docs_percentage * docs_per_user)
log_info('Running System Test #1')
log_info('> server_seed_docs = {}'.format(server_seed_docs))
log_info('> max_docs = {}'.format(max_docs))
log_info('> num_users = {}'.format(num_users))
log_info('> docs_per_user = {}'.format(docs_per_user))
log_info('> create_batch_size = {}'.format(create_batch_size))
log_info('> create_delay = {}'.format(create_delay))
log_info('> update_batch_size = {}'.format(update_batch_size))
log_info('> update_docs_percentage = {}'.format(update_docs_percentage))
log_info(
'> docs_per_user_per_update = {}'.format(docs_per_user_per_update))
log_info('> update_delay = {}'.format(update_delay))
log_info('> update_runtime_sec = {}'.format(update_runtime_sec))
log_info('> changes_delay = {}'.format(changes_delay))
log_info('> changes_limit = {}'.format(changes_limit))
log_info(
'> changes_terminator_doc_id = {}'.format(changes_terminator_doc_id))
# Validate
# Server docs should be a multiple of 1000 for batching purposes
if server_seed_docs % 1000 != 0:
raise ValueError('server_seed_docs must be divisible by 1000')
# Number of docs should be equally divisible by number of users
if max_docs % num_users != 0:
raise ValueError('max_docs must be divisible by number_of_users')
# Number of docs per user (max_docs / num_users) should be equally
# divisible by the batch size for easier computation
if docs_per_user % create_batch_size != 0:
raise ValueError(
'docs_per_user ({}) must be devisible by create_batch_size ({})'.
format(docs_per_user, create_batch_size))
# We want an even distributed of users per type
if num_users % len(USER_TYPES) != 0:
raise ValueError("'num_users' should be a multiple of 4")
# Make sure that the 'update_batch_size' is complatible with
# then number of users per type
num_users_per_type = num_users / len(USER_TYPES)
if update_batch_size > num_users_per_type:
raise ValueError(
"'batch_size' cannot be larger than number of users per type")
if num_users_per_type % update_batch_size != 0:
raise ValueError(
"'update_batch_size' ({}) should be a multiple of number_users_per_type ({})"
.format(update_batch_size, num_users_per_type))
sg_conf_name = 'sync_gateway_default'
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
# Reset cluster state
c = Cluster(config=cluster_config)
c.reset(sg_config_path=sg_conf)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cbs_url = topology['couchbase_servers'][0]
cbs_admin_url = cbs_url.replace('8091', '8092')
cb_server = couchbaseserver.CouchbaseServer(cbs_url)
bucket_name = "data-bucket"
cbs_ip = cb_server.host
headers = {'Content-Type': 'application/json'}
cbs_session = Session()
cbs_session.headers = headers
cbs_session.auth = ('Administrator', 'password')
log_info('Seeding {} with {} docs'.format(cbs_ip, server_seed_docs))
sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name),
password='******',
timeout=300)
# Stop SG before loading the server
lb_url = topology['sync_gateways'][0]['public']
sg_admin_url = topology['sync_gateways'][0]['admin']
sg_db = 'db'
sg_helper = SyncGateway()
sg_helper.stop_sync_gateways(cluster_config=cluster_config)
# Scenario Actions
delete_views(cbs_session, cbs_admin_url, bucket_name)
load_bucket(sdk_client, server_seed_docs)
sg_helper.start_sync_gateways(cluster_config, config=sg_conf)
wait_for_view_creation(cbs_session, cbs_admin_url, bucket_name)
# Start concurrent creation of docs (max docs / num users)
# Each user will add batch_size number of docs via bulk docs and sleep for 'create_delay'
# Once a user has added number of expected docs 'docs_per_user', it will terminate.
log_info('------------------------------------------')
log_info('START concurrent user / doc creation')
log_info('------------------------------------------')
users = create_docs(sg_admin_url=sg_admin_url,
sg_url=lb_url,
sg_db=sg_db,
num_users=num_users,
number_docs_per_user=docs_per_user,
create_batch_size=create_batch_size,
create_delay=create_delay)
assert len(users) == num_users
log_info('------------------------------------------')
log_info('END concurrent user / doc creation')
log_info('------------------------------------------')
# Start changes processing
with ProcessPoolExecutor(max_workers=((len(users) * 3) +
update_batch_size + 3)) as pex:
# Start changes feeds in background process
changes_workers_task = pex.submit(start_changes_processing, lb_url,
sg_db, users, changes_delay,
changes_limit,
changes_terminator_doc_id)
log_info('------------------------------------------')
log_info('START concurrent updates')
log_info('------------------------------------------')
# Start concurrent updates of update
# Update batch size is the number of users that will concurrently update all of their docs
users = update_docs(sg_url=lb_url,
sg_db=sg_db,
users=users,
update_runtime_sec=update_runtime_sec,
batch_size=update_batch_size,
docs_per_user_per_update=docs_per_user_per_update,
update_delay=update_delay)
all_user_channels = []
for k, v in users.items():
log_info('User ({}) updated docs {} times!'.format(
k, v['updates']))
all_user_channels.append(k)
log_info('------------------------------------------')
log_info('END concurrent updates')
log_info('------------------------------------------')
# Broadcast termination doc to all users
terminator_channel = 'terminator'
send_changes_termination_doc(lb_url, sg_db, users,
changes_terminator_doc_id,
terminator_channel)
# Overwrite each users channels with 'terminator' so their changes feed will backfill with the termination doc
grant_users_access(users, [terminator_channel], sg_admin_url, sg_db)
# Block on changes completion
users = changes_workers_task.result()
# Print the summary of the system test
print_summary(users)
def test_attachment_revpos_when_ancestor_unavailable(
params_from_base_test_setup, sg_conf_name):
"""
Creates a document with an attachment, then updates that document so that
the body of the revision that originally pushed the document is no
longer available. Add a new revision that's not a child of the
active revision, and validate that it's uploaded successfully.
Example:
1. Document is created with attachment at rev-1
2. Document is updated (strip digests and length, only put revpos & stub) multiple times on the server, goes to rev-10
3. Client attempts to add a new (conflicting) revision 2, with parent rev-1.
4. If the body of rev-1 is no longer available on the server (temporary backup of revision has expired, and is no longer stored
in the in-memory rev cache), we were throwing an error to client
because we couldn't verify based on the _attachments property in rev-1.
5. In this scenario, before returning error, we are now checking if the active revision has a common ancestor with the incoming revision.
If so, we can validate any revpos values equal to or earlier than the common ancestor against the active revision
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
no_conflicts_enabled = params_from_base_test_setup["no_conflicts_enabled"]
if no_conflicts_enabled:
pytest.skip('--no-conflicts is not enabled, so skipping the test')
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(cluster_config, sg_conf)
topology = cluster_helper.get_cluster_topology(cluster_config)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
sg_db = "db"
bucket = "data-bucket"
log_info("Running 'test_attachment_revpos_when_ancestor_unavailable'")
log_info("Using cbs_url: {}".format(cbs_url))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_url_admin: {}".format(sg_url_admin))
log_info("Using sg_db: {}".format(sg_db))
log_info("Using bucket: {}".format(bucket))
channels_list = ["ABC"]
client = MobileRestClient()
sg_util = SyncGateway()
cb_server = couchbaseserver.CouchbaseServer(cbs_url)
user1 = client.create_user(url=sg_url_admin,
db=sg_db,
name="user1",
password="******",
channels=channels_list)
atts = attachment.load_from_data_dir(["sample_text.txt"])
doc_with_att = document.create_doc(doc_id="att_doc",
content={"sample_key": "sample_val"},
attachments=atts,
channels=channels_list)
doc_gen_1 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_with_att,
auth=user1)
client.update_doc(url=sg_url,
db=sg_db,
doc_id=doc_gen_1["id"],
number_updates=10,
auth=user1)
# Clear cached rev doc bodys from server and cycle sync_gateway
sg_util.stop_sync_gateways(cluster_config=cluster_config, url=sg_url)
cb_server.delete_couchbase_server_cached_rev_bodies(bucket=bucket)
sg_util.start_sync_gateways(cluster_config=cluster_config,
url=sg_url,
config=sg_conf)
client.add_conflict(url=sg_url,
db=sg_db,
doc_id=doc_gen_1["id"],
parent_revisions=doc_gen_1["rev"],
new_revision="2-foo",
auth=user1)
def test_log_maxbackups_0(params_from_base_test_setup, sg_conf_name):
"""Test with maxbackups=0 that means do not limit the number of backups
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup["xattrs_enabled"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_helper = ClusterKeywords()
cluster_hosts = cluster_helper.get_cluster_topology(cluster_conf)
sg_one_url = cluster_hosts["sync_gateways"][0]["public"]
remote_executor = RemoteExecutor(cluster.sync_gateways[0].ip)
# Stop sync_gateways
log_info(">>> Stopping sync_gateway")
sg_helper = SyncGateway()
sg_helper.stop_sync_gateways(cluster_config=cluster_conf, url=sg_one_url)
remote_executor.execute("mkdir -p /tmp/sg_logs")
remote_executor.execute("sudo rm -rf /tmp/sg_logs/sg_log_rotation*")
# generate log file with almost 1MB
remote_executor.execute(
"sudo dd if=/dev/zero of=/tmp/sg_logs/sg_log_rotation.log bs=1030000 count=1"
)
remote_executor.execute("sudo chmod 777 -R /tmp/sg_logs")
# read sample sg_conf
data = load_sync_gateway_config(sg_conf, mode,
cluster_hosts["couchbase_servers"][0],
xattrs_enabled, cluster_conf)
# set maxbackups=0 in config file
data['logging']["default"]["rotation"]["maxbackups"] = 0
# create temp config file in the same folder as sg_conf
temp_conf = "/".join(sg_conf.split('/')[:-2]) + '/temp_conf.json'
with open(temp_conf, 'w') as fp:
json.dump(data, fp)
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=temp_conf)
# ~1M MB will be added to log file after requests
remote_executor.execute(
"for ((i=1;i <= 1000;i += 1)); do curl -s http://localhost:4984/ > /dev/null; done"
)
status, stdout, stderr = remote_executor.execute(
"ls /tmp/sg_logs/ | grep sg_log_rotation | wc -l")
assert stdout[0].rstrip() == '2'
# Remove generated conf file
os.remove(temp_conf)
def test_log_rotation_default_values(params_from_base_test_setup,
sg_conf_name):
"""Test to verify default values for rotation section:
maxsize = 100 MB
MaxAge = 0(do not limit the number of MaxAge)
MaxBackups = 0(do not limit the number of backups)
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup["xattrs_enabled"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
remote_executor = RemoteExecutor(cluster.sync_gateways[0].ip)
# Stop sync_gateways
log_info(">>> Stopping sync_gateway")
sg_helper = SyncGateway()
cluster_helper = ClusterKeywords()
cluster_hosts = cluster_helper.get_cluster_topology(cluster_conf)
sg_one_url = cluster_hosts["sync_gateways"][0]["public"]
sg_helper.stop_sync_gateways(cluster_config=cluster_conf, url=sg_one_url)
# read sample sg_conf
data = load_sync_gateway_config(sg_conf, mode,
cluster_hosts["couchbase_servers"][0],
xattrs_enabled, cluster_conf)
# delete rotation from sample config
del data['logging']["default"]["rotation"]
# create temp config file in the same folder as sg_conf
temp_conf = "/".join(sg_conf.split('/')[:-2]) + '/temp_conf.json'
log_info("TEMP_CONF: {}".format(temp_conf))
with open(temp_conf, 'w') as fp:
json.dump(data, fp)
remote_executor.execute("mkdir -p /tmp/sg_logs")
remote_executor.execute("sudo rm -rf /tmp/sg_logs/sg_log_rotation*")
# generate log file with size ~94MB to check that backup file not created while 100MB not reached
remote_executor.execute(
"sudo dd if=/dev/zero of=/tmp/sg_logs/sg_log_rotation.log bs=94850000 count=1"
)
remote_executor.execute("sudo chmod 777 -R /tmp/sg_logs")
# iterate 5th times to verify that every time we get new backup file with ~100MB
for i in xrange(5):
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=temp_conf)
# ~1M MB will be added to log file after requests
remote_executor.execute(
"for ((i=1;i <= 1000;i += 1)); do curl -s http://localhost:4984/ > /dev/null; done"
)
_, stdout, _ = remote_executor.execute(
"ls /tmp/sg_logs/ | grep sg_log_rotation | wc -l")
# verify num of log files
assert stdout[0].rstrip() == str(i + 1)
sg_helper.stop_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url)
# generate log file with size ~99MB
remote_executor.execute(
"sudo dd if=/dev/zero of=/tmp/sg_logs/sg_log_rotation.log bs=104850000 count=1"
)
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=sg_conf)
# Remove generated conf file
os.remove(temp_conf)
def test_log_number_backups(params_from_base_test_setup, sg_conf_name):
"""Test to check general behaviour for number of backups.
In test the following params have been used:
"maxsize": 1,
"maxage": 30,
"maxbackups": 2
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
remote_executor = RemoteExecutor(cluster.sync_gateways[0].ip)
# Stop sync_gateways
log_info(">>> Stopping sync_gateway")
sg_helper = SyncGateway()
cluster_helper = ClusterKeywords()
cluster_hosts = cluster_helper.get_cluster_topology(cluster_conf)
sg_one_url = cluster_hosts["sync_gateways"][0]["public"]
sg_helper.stop_sync_gateways(cluster_config=cluster_conf, url=sg_one_url)
remote_executor.execute("mkdir -p /tmp/sg_logs")
remote_executor.execute("sudo rm -rf /tmp/sg_logs/sg_log_rotation*")
# generate log file with almost 1MB
remote_executor.execute(
"sudo dd if=/dev/zero of=/tmp/sg_logs/sg_log_rotation.log bs=1030000 count=1"
)
remote_executor.execute("sudo chmod 777 -R /tmp/sg_logs")
# iterate 5 times
for i in xrange(5):
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=sg_conf)
# ~1M MB will be added to log file after requests
remote_executor.execute(
"for ((i=1;i <= 1000;i += 1)); do curl -s http://localhost:4984/ > /dev/null; done"
)
_, stdout, _ = remote_executor.execute(
"ls /tmp/sg_logs/ | grep sg_log_rotation | wc -l")
# max 3 files: 2 backups + 1 log file
assert stdout[0].rstrip() == str(min(3, i + 2))
sg_helper.stop_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url)
# generate log file with almost 1MB
remote_executor.execute(
"sudo dd if=/dev/zero of=/tmp/sg_logs/sg_log_rotation.log bs=1030000 count=1"
)
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=sg_conf)
def test_log_maxage_10_timestamp_ignored(params_from_base_test_setup,
sg_conf_name):
"""Test to verify SG continues to wrile logs in the same file even when
timestamp for the log file has been changed
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup["xattrs_enabled"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
remote_executor = RemoteExecutor(cluster.sync_gateways[0].ip)
# Stop sync_gateways
log_info(">>> Stopping sync_gateway")
sg_helper = SyncGateway()
cluster_helper = ClusterKeywords()
cluster_hosts = cluster_helper.get_cluster_topology(cluster_conf)
sg_one_url = cluster_hosts["sync_gateways"][0]["public"]
sg_helper.stop_sync_gateways(cluster_config=cluster_conf, url=sg_one_url)
remote_executor.execute("mkdir -p /tmp/sg_logs")
remote_executor.execute("sudo rm -rf /tmp/sg_logs/sg_log_rotation*")
# generate log file with almost 1MB
remote_executor.execute(
"sudo dd if=/dev/zero of=/tmp/sg_logs/sg_log_rotation.log bs=1030000 count=1"
)
remote_executor.execute("sudo chmod 777 -R /tmp/sg_logs")
# read sample sg_conf
data = load_sync_gateway_config(sg_conf, mode,
cluster_hosts["couchbase_servers"][0],
xattrs_enabled, cluster_conf)
# set maxage = 10 days
data['logging']["default"]["rotation"]["maxage"] = 10
# create temp config file in the same folder as sg_conf
temp_conf = "/".join(sg_conf.split('/')[:-2]) + '/temp_conf.json'
with open(temp_conf, 'w') as fp:
json.dump(data, fp)
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=temp_conf)
# ~1M MB will be added to log file after requests
remote_executor.execute(
"for ((i=1;i <= 1000;i += 1)); do curl -s http://localhost:4984/ > /dev/null; done"
)
sg_helper.stop_sync_gateways(cluster_config=cluster_conf, url=sg_one_url)
# change timestamp for log when SG stopped( we don't change file naming)
remote_executor.execute(
"sudo touch -d \"10 days ago\" /tmp/sg_logs/sg_log_rotation*")
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=temp_conf)
_, stdout, _ = remote_executor.execute(
"ls /tmp/sg_logs/ | grep sg_log_rotation | wc -l")
# verify that new log file was not created
assert stdout[0].rstrip() == '2'
# Remove generated conf file
os.remove(temp_conf)
