def test_basic_xdcr_with_cert_regenerate(self):
cluster1 = self.servers[0:2]
cluster2 = self.servers[2:4]
remote_cluster_name = 'sslcluster'
restCluster1 = RestConnection(cluster1[0])
restCluster2 = RestConnection(cluster2[0])
try:
#Setup cluster1
x509main(cluster1[0]).setup_master()
x509main(cluster1[1])._setup_node_certificates(reload_cert=False)
restCluster1.add_node('Administrator','password',cluster1[1].ip)
known_nodes = ['ns_1@'+cluster1[0].ip,'ns_1@' + cluster1[1].ip]
restCluster1.rebalance(known_nodes)
self.assertTrue(self.check_rebalance_complete(restCluster1),"Issue with rebalance")
restCluster1.create_bucket(bucket='default', ramQuotaMB=100)
restCluster1.remove_all_replications()
restCluster1.remove_all_remote_clusters()
#Setup cluster2
x509main(cluster2[0]).setup_master()
x509main(cluster2[1])._setup_node_certificates(reload_cert=False)
restCluster2.add_node('Administrator','password',cluster2[1].ip)
known_nodes = ['ns_1@'+cluster2[0].ip,'ns_1@' + cluster2[1].ip]
restCluster2.rebalance(known_nodes)
self.assertTrue(self.check_rebalance_complete(restCluster2),"Issue with rebalance")
restCluster2.create_bucket(bucket='default', ramQuotaMB=100)
test = x509main.CACERTFILEPATH + x509main.CACERTFILE
data = open(test, 'rb').read()
restCluster1.add_remote_cluster(cluster2[0].ip,cluster2[0].port,'Administrator','password',remote_cluster_name,certificate=data)
replication_id = restCluster1.start_replication('continuous','default',remote_cluster_name)
#restCluster1.set_xdcr_param('default','default','pauseRequested',True)
x509main(self.master)._delete_inbox_folder()
x509main(self.master)._generate_cert(self.servers,root_cn="CB\ Authority")
self.log.info ("Setting up the first cluster for new certificate")
x509main(cluster1[0]).setup_master()
x509main(cluster1[1])._setup_node_certificates(reload_cert=False)
self.log.info ("Setting up the second cluster for new certificate")
x509main(cluster2[0]).setup_master()
x509main(cluster2[1])._setup_node_certificates(reload_cert=False)
status = restCluster1.is_replication_paused('default','default')
if not status:
restCluster1.set_xdcr_param('default','default','pauseRequested',False)
restCluster1.set_xdcr_param('default','default','pauseRequested',True)
status = restCluster1.is_replication_paused('default','default')
self.assertTrue(status,"Replication has not started after certificate upgrade")
finally:
known_nodes = ['ns_1@'+cluster2[0].ip,'ns_1@'+cluster2[1].ip]
restCluster2.rebalance(known_nodes,['ns_1@' + cluster2[1].ip])
self.assertTrue(self.check_rebalance_complete(restCluster2),"Issue with rebalance")
restCluster2.delete_bucket()
def test_basic_xdcr_with_cert_regenerate(self):
cluster1 = self.servers[0:2]
cluster2 = self.servers[2:4]
remote_cluster_name = 'sslcluster'
restCluster1 = RestConnection(cluster1[0])
restCluster2 = RestConnection(cluster2[0])
try:
#Setup cluster1
x509main(cluster1[0]).setup_master()
x509main(cluster1[1])._setup_node_certificates(reload_cert=False)
restCluster1.add_node('Administrator','password',cluster1[1].ip)
known_nodes = ['ns_1@'+cluster1[0].ip,'ns_1@' + cluster1[1].ip]
restCluster1.rebalance(known_nodes)
self.assertTrue(self.check_rebalance_complete(restCluster1),"Issue with rebalance")
restCluster1.create_bucket(bucket='default', ramQuotaMB=100)
restCluster1.remove_all_replications()
restCluster1.remove_all_remote_clusters()
#Setup cluster2
x509main(cluster2[0]).setup_master()
x509main(cluster2[1])._setup_node_certificates(reload_cert=False)
restCluster2.add_node('Administrator','password',cluster2[1].ip)
known_nodes = ['ns_1@'+cluster2[0].ip,'ns_1@' + cluster2[1].ip]
restCluster2.rebalance(known_nodes)
self.assertTrue(self.check_rebalance_complete(restCluster2),"Issue with rebalance")
restCluster2.create_bucket(bucket='default', ramQuotaMB=100)
test = x509main.CACERTFILEPATH + x509main.CACERTFILE
data = open(test, 'rb').read()
restCluster1.add_remote_cluster(cluster2[0].ip,cluster2[0].port,'Administrator','password',remote_cluster_name,certificate=data)
replication_id = restCluster1.start_replication('continuous','default',remote_cluster_name)
#restCluster1.set_xdcr_param('default','default','pauseRequested',True)
x509main(self.master)._delete_inbox_folder()
x509main(self.master)._generate_cert(self.servers,root_cn="CB\ Authority")
self.log.info ("Setting up the first cluster for new certificate")
x509main(cluster1[0]).setup_master()
x509main(cluster1[1])._setup_node_certificates(reload_cert=False)
self.log.info ("Setting up the second cluster for new certificate")
x509main(cluster2[0]).setup_master()
x509main(cluster2[1])._setup_node_certificates(reload_cert=False)
status = restCluster1.is_replication_paused('default','default')
if not status:
restCluster1.set_xdcr_param('default','default','pauseRequested',False)
restCluster1.set_xdcr_param('default','default','pauseRequested',True)
status = restCluster1.is_replication_paused('default','default')
self.assertTrue(status,"Replication has not started after certificate upgrade")
finally:
known_nodes = ['ns_1@'+cluster2[0].ip,'ns_1@'+cluster2[1].ip]
restCluster2.rebalance(known_nodes,['ns_1@' + cluster2[1].ip])
self.assertTrue(self.check_rebalance_complete(restCluster2),"Issue with rebalance")
restCluster2.delete_bucket()
def cluster_xdcr_remote_clusters_read(self,username,password,host,port=8091, servers=None,cluster=None,httpCode=None,user_role=None):
remote_cluster_name = 'rbac_cluster'
rest = RestConnection(servers[0])
remote_server01 = servers[1]
remote_server02 = servers[2]
rest_remote01 = RestConnection(remote_server01)
rest_remote01.delete_bucket()
rest_remote01.create_bucket(bucket='default', ramQuotaMB=100)
rest_remote02 = RestConnection(remote_server02)
#------ First Test the Get Requests for XDCR --------------#
#Remove all remote cluster references
rest.remove_all_replications()
rest.remove_all_remote_clusters()
#Add remote cluster reference and replications
rest.add_remote_cluster(remote_server01.ip,8091,'Administrator','password',remote_cluster_name)
time.sleep(20)
replication_id = rest.start_replication('continuous','default',remote_cluster_name)
_cluster_xdcr_remote_clusters_read ={
"remove_cluser_read":"/pools/default/remoteClusters;GET",
}
result = self._return_http_code(_cluster_xdcr_remote_clusters_read,username,password,host=host,port=8091, httpCode=httpCode, user_role=user_role)
def start_replication(self, master, slave, replication_type='continuous',
buckets=None, bidir=False, suffix='A'):
"""Add remote cluster and start replication"""
master_rest_conn = RestConnection(master)
remote_reference = 'remote_cluster_' + suffix
master_rest_conn.add_remote_cluster(slave.ip, slave.port,
slave.rest_username,
slave.rest_password,
remote_reference)
if not buckets:
buckets = self.get_buckets()
else:
buckets = self.get_buckets(reversed=True)
for bucket in buckets:
master_rest_conn.start_replication(replication_type, bucket,
remote_reference)
if self.parami('num_buckets', 1) > 1 and suffix == 'A':
self.start_replication(slave, master, replication_type, buckets,
suffix='B')
if bidir:
self.start_replication(slave, master, replication_type, buckets,
suffix='B')
def _link_clusters(self, src_cluster_name, src_master, dest_cluster_name,
dest_master):
rest_conn_src = RestConnection(src_master)
rest_conn_src.add_remote_cluster(dest_master.ip, dest_master.port,
dest_master.rest_username,
dest_master.rest_password,
dest_cluster_name)
def _link_create_replications(self, master_1, master_2, cluster_name):
rest = RestConnection(master_1)
rest.add_remote_cluster(master_2.ip, master_2.port, master_1.rest_username,
master_1.rest_password, cluster_name)
time.sleep(30)
if len(self._buckets) == 0:
self._buckets = rest.get_buckets()
for bucket in set(self._buckets):
rep_id = rest.start_replication("continuous", bucket, cluster_name)
def _link_create_replications(self, master_1, master_2, cluster_name):
rest = RestConnection(master_1)
rest.add_remote_cluster(master_2.ip, master_2.port, master_1.rest_username,
master_1.rest_password, cluster_name)
time.sleep(30)
if len(self._buckets) == 0:
self._buckets = rest.get_buckets()
for bucket in set(self._buckets):
rep_database, rep_id = rest.start_replication("continuous", bucket, cluster_name)
def test_continuous_unidirectional_deletes_2(self):
cluster_ref_a = "cluster_ref_a"
master_a = self._input.clusters.get(0)[0]
rest_conn_a = RestConnection(master_a)
cluster_ref_b = "cluster_ref_b"
master_b = self._input.clusters.get(1)[0]
rest_conn_b = RestConnection(master_b)
# Load some data on cluster a. Do it a few times so that the seqnos are
# bumped up and then delete it.
kvstore = ClientKeyValueStore()
self._params["ops"] = "set"
load_thread_list = []
for i in [1, 2, 3]:
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore)
load_thread_list.append(load_thread)
for lt in load_thread_list:
lt.start()
for lt in load_thread_list:
lt.join()
time.sleep(10)
self._params["ops"] = "delete"
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore)
load_thread.start()
load_thread.join()
# Start replication to replicate the deletes from cluster a
# to cluster b where the keys never existed.
replication_type = "continuous"
rest_conn_a.add_remote_cluster(master_b.ip, master_b.port,
master_b.rest_username,
master_b.rest_password, cluster_ref_b)
(rep_database, rep_id) = rest_conn_a.start_replication(replication_type,
self._buckets[0],
cluster_ref_b)
self._state.append((rest_conn_a, cluster_ref_b, rep_database, rep_id))
time.sleep(15)
# Verify replicated data#
self.assertTrue(XDCRBaseTest.verify_del_items(rest_conn_a,
rest_conn_b,
self._buckets[0],
kvstore.keys(),
self._poll_sleep,
self._poll_timeout),
"Changes feed verification failed")
def cluster_bucket_xdcr_write(self,username,password,host,port=8091,servers=None,cluster=None,httpCode=None,user_role=None):
_cluster_xdcr_settings_read = {
"create_replication":"controller/createReplication;POST",
"cancel_XDCR":"controller/cancelXDCR/<xid>;POST",
"delete_XDCR":"controller/cancelXDCR/<xid>;DELETE"
}
rest = RestConnection(servers[0])
rest.remove_all_replications()
rest.remove_all_remote_clusters()
remote_cluster_name = 'rbac_cluster'
rest = RestConnection(servers[0])
remote_server01 = servers[1]
remote_server02 = servers[2]
rest_remote01 = RestConnection(remote_server01)
rest_remote01.delete_bucket()
rest_remote01.create_bucket(bucket='default1', ramQuotaMB=100,proxyPort=11252)
rest_remote02 = RestConnection(remote_server02)
remote_id = rest.add_remote_cluster(remote_server01.ip,8091,'Administrator','password',remote_cluster_name)
time.sleep(10)
#replication_id = rest.start_replication('continuous','default',remote_cluster_name)
param_map = {'replicationType': 'continuous','toBucket': 'default1','fromBucket': 'default','toCluster': remote_cluster_name,
'type': 'capi'}
create_replication = {"create_replication":"controller/createReplication;POST;"+str(param_map)}
result = self._return_http_code(create_replication,username,password,host=host,port=port, httpCode=httpCode, user_role=user_role)
rest.remove_all_replications()
rest.remove_all_remote_clusters()
remote_id = rest.add_remote_cluster(remote_server01.ip,8091,'Administrator','password',remote_cluster_name)
time.sleep(20)
replication_id = rest.start_replication('continuous',fromBucket='default',toCluster=remote_cluster_name,toBucket='default1')
replication_id = replication_id.replace("/","%2F")
cancel_replication = {"cancel_XDCR":"controller/cancelXDCR/" + replication_id + ";POST"}
result = self._return_http_code(cancel_replication,username,password,host=host,port=port, httpCode=httpCode, user_role=user_role)
rest.remove_all_replications()
rest.remove_all_remote_clusters()
remote_id = rest.add_remote_cluster(remote_server01.ip,8091,'Administrator','password',remote_cluster_name)
time.sleep(20)
replication_id = rest.start_replication('continuous',fromBucket='default',toCluster=remote_cluster_name,toBucket='default1')
replication_id = replication_id.replace("/","%2F")
cancel_replication = {"cancel_XDCR":"controller/cancelXDCR/" + replication_id + ";DELETE"}
result = self._return_http_code(cancel_replication,username,password,host=host,port=port, httpCode=httpCode, user_role=user_role)
rest.remove_all_replications()
rest.remove_all_remote_clusters()
rest_remote01.delete_bucket('default1')
def test_continuous_unidirectional_sets_deletes(self):
cluster_ref_b = "cluster_ref_a"
master_a = self._input.clusters.get(0)[0]
rest_conn_a = RestConnection(master_a)
cluster_ref_b = "cluster_ref_b"
master_b = self._input.clusters.get(1)[0]
rest_conn_b = RestConnection(master_b)
# Start replication
replication_type = "continuous"
rest_conn_a.add_remote_cluster(master_b.ip, master_b.port,
master_b.rest_username,
master_b.rest_password, cluster_ref_b)
(rep_database, rep_id) = rest_conn_a.start_replication(replication_type,
self._buckets[0],
cluster_ref_b)
self._state.append((rest_conn_a, cluster_ref_b, rep_database, rep_id))
# Start load
kvstore = ClientKeyValueStore()
self._params["ops"] = "set"
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore)
load_thread.start()
load_thread.join()
# Do some deletes
self._params["ops"] = "delete"
self._params["count"] = self._num_items/5
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore)
load_thread.start()
load_thread.join()
# Verify replication
self.assertTrue(XDCRBaseTest.verify_replicated_data(rest_conn_b,
self._buckets[0],
kvstore,
self._poll_sleep,
self._poll_timeout),
"Verification of replicated data failed")
self.assertTrue(XDCRBaseTest.verify_replicated_revs(rest_conn_a,
rest_conn_b,
self._buckets[0],
self._poll_sleep,
self._poll_timeout),
"Verification of replicated revisions failed")
def test_basic_xdcr_with_cert(self):
cluster1 = self.servers[0:2]
cluster2 = self.servers[2:4]
remote_cluster_name = 'sslcluster'
restCluster1 = RestConnection(cluster1[0])
restCluster2 = RestConnection(cluster2[0])
try:
#Setup cluster1
x509main(cluster1[0]).setup_master()
x509main(cluster1[1])._setup_node_certificates(reload_cert=False)
restCluster1.add_node('Administrator', 'password', cluster1[1].ip)
known_nodes = ['ns_1@' + cluster1[0].ip, 'ns_1@' + cluster1[1].ip]
restCluster1.rebalance(known_nodes)
self.assertTrue(self.check_rebalance_complete(restCluster1),
"Issue with rebalance")
restCluster1.create_bucket(bucket='default', ramQuotaMB=100)
restCluster1.remove_all_replications()
restCluster1.remove_all_remote_clusters()
#Setup cluster2
x509main(cluster2[0]).setup_master()
x509main(cluster2[1])._setup_node_certificates(reload_cert=False)
restCluster2.add_node('Administrator', 'password', cluster2[1].ip)
known_nodes = ['ns_1@' + cluster2[0].ip, 'ns_1@' + cluster2[1].ip]
restCluster2.rebalance(known_nodes)
self.assertTrue(self.check_rebalance_complete(restCluster2),
"Issue with rebalance")
restCluster2.create_bucket(bucket='default', ramQuotaMB=100)
test = x509main.CACERTFILEPATH + x509main.CACERTFILE
data = open(test, 'rb').read()
restCluster1.add_remote_cluster(cluster2[0].ip,
cluster2[0].port,
'Administrator',
'password',
remote_cluster_name,
certificate=data)
replication_id = restCluster1.start_replication(
'continuous', 'default', remote_cluster_name)
if replication_id is not None:
self.assertTrue(True,
"Replication was not created successfully")
finally:
known_nodes = ['ns_1@' + cluster2[0].ip, 'ns_1@' + cluster2[1].ip]
restCluster2.rebalance(known_nodes, ['ns_1@' + cluster2[1].ip])
self.assertTrue(self.check_rebalance_complete(restCluster2),
"Issue with rebalance")
restCluster2.delete_bucket()
def test_XDCR_with_ldap_setup_half_encryption(self):
rest2 = RestConnection(self.servers[1])
rest1 = RestConnection(self.servers[0])
rest2.remove_all_replications()
rest2.remove_all_remote_clusters()
rest2.create_bucket("default", ramQuotaMB=100)
rest1.create_bucket("default", ramQuotaMB=100)
remote_cluster2 = 'C2'
remote_server01 = self.servers[0]
remote_id = rest2.add_remote_cluster(remote_server01.ip,
8091,
'cbadminbucket',
'password',
remote_cluster2,
demandEncryption="on",
encryptionType="half")
replication_id = rest2.start_replication('continuous', 'default',
remote_cluster2)
if replication_id is not None:
self.assertTrue(True, "Replication was not created successfully")
rest2.remove_all_replications()
rest2.remove_all_remote_clusters()
def cluster_xdcr_remote_clusters_write(self,username,password,host,port=8091, servers=None,cluster=None,httpCode=None,user_role=None):
rest = RestConnection(servers[0])
rest.remove_all_replications()
rest.remove_all_remote_clusters()
_cluster_xdcr_remove_cluster_write = {
"remoteClusters":"pools/default/remoteClusters;POST",
"remote_cluster_id":"pools/default/remoteClusters/<id>;PUT",
"delete_remote":"pools/default/remoteClusters/<id>;DELETE"
}
params = {'hostname': "{0}:{1}".format(servers[1].ip, servers[1].port),'username': '******','password': '******','name':'rbac_remote01'}
add_node = {"remoteClusters":"pools/default/remoteClusters;POST;" + str(params)}
result = self._return_http_code(add_node,username,password,host=host,port=port, httpCode=httpCode, user_role=user_role)
rest.remove_all_replications()
rest.remove_all_remote_clusters()
remote_cluster_name = 'rbac_cluster'
rest = RestConnection(servers[0])
remote_server01 = servers[1]
remote_server02 = servers[2]
rest_remote01 = RestConnection(remote_server01)
rest_remote01.delete_bucket()
rest_remote01.create_bucket(bucket='default', ramQuotaMB=100)
rest_remote02 = RestConnection(remote_server02)
rest.remove_all_replications()
rest.remove_all_remote_clusters()
remote_id = rest.add_remote_cluster(remote_server01.ip,8091,'Administrator','password',remote_cluster_name)
time.sleep(20)
delete_remote = {"delete_remote":"pools/default/remoteClusters/" + str(remote_cluster_name) + ";DELETE"}
result = self._return_http_code(delete_remote,username,password,host=host,port=port, httpCode=httpCode, user_role=user_role)
def cluster_xdcr_settings_write(self,username,password,host,port=8091,servers=None,cluster=None,httpCode=None,user_role=None):
_cluster_xdcr_settings_read = {
"replication_settings":"settings/replications;POST;{'httpConnections': 20}"
}
rest = RestConnection(servers[0])
rest.remove_all_replications()
rest.remove_all_remote_clusters()
remote_cluster_name = 'rbac_cluster'
rest = RestConnection(servers[0])
remote_server01 = servers[1]
remote_server02 = servers[2]
rest_remote01 = RestConnection(remote_server01)
rest_remote01.delete_bucket()
rest_remote01.create_bucket(bucket='default', ramQuotaMB=100)
rest_remote02 = RestConnection(remote_server02)
rest_remote02.delete_bucket()
remote_id = rest.add_remote_cluster(remote_server01.ip,8091,'Administrator','password',remote_cluster_name)
time.sleep(20)
replication_id = rest.start_replication('continuous','default',remote_cluster_name)
result = self._return_http_code(_cluster_xdcr_settings_read,username,password,host=host,port=port, httpCode=httpCode, user_role=user_role)
rest.remove_all_replications()
rest.remove_all_remote_clusters()
rest_remote01.delete_bucket()
def cluster_bucket_xdcr_read(self,username,password,host,port=8091,servers=None,cluster=None,httpCode=None,user_role=None):
_cluster_bucket_xdcr_read = {
"replication_settings":"settings/replications/<id>;GET"
}
rest = RestConnection(servers[0])
rest.remove_all_replications()
rest.remove_all_remote_clusters()
remote_cluster_name = 'rbac_cluster'
rest = RestConnection(servers[0])
remote_server01 = servers[1]
remote_server02 = servers[2]
rest_remote01 = RestConnection(remote_server01)
rest_remote01.delete_bucket()
rest_remote01.create_bucket(bucket='default', ramQuotaMB=100)
rest_remote02 = RestConnection(remote_server02)
remote_id = rest.add_remote_cluster(remote_server01.ip,8091,'Administrator','password',remote_cluster_name)
replication_id = rest.start_replication('continuous','default',remote_cluster_name)
replication_id = replication_id.replace("/","%2F")
bucket_xdcr_read = {"replication_settings":"settings/replications/" + replication_id + ";GET"}
result = self._return_http_code(bucket_xdcr_read,username,password,host=host,port=port, httpCode=httpCode, user_role=user_role)
rest.remove_all_replications()
rest.remove_all_remote_clusters()
rest_remote01.delete_bucket()
def _start_es_replication(self, bucket='default', xdcr_params={}):
rest_conn = RestConnection(self.src_cluster.get_master_node())
if bucket == 'default':
self.log.info("Creating default bucket")
rest_conn.create_bucket(bucket='default', ramQuotaMB=100, authType='none', saslPassword='', replicaNumber=1,
proxyPort=11211, bucketType='membase', replica_index=1, threadsNumber=3,
flushEnabled=1, lww=False)
self.src_cluster.add_bucket(ramQuotaMB=100, bucket='default', authType='none',
saslPassword='', replicaNumber=1, proxyPort=11211, bucketType='membase',
evictionPolicy='valueOnly')
elif bucket == 'sasl':
self.log.info("Creating sasl bucket")
rest_conn.create_bucket(bucket='sasl', ramQuotaMB=100, authType='sasl', saslPassword='******', replicaNumber=1,
proxyPort=11211, bucketType='membase', replica_index=1, threadsNumber=3,
flushEnabled=1, lww=False)
self.src_cluster.add_bucket(ramQuotaMB=100, bucket='sasl', authType='sasl',
saslPassword='******', replicaNumber=1, proxyPort=11211, bucketType='membase',
evictionPolicy='valueOnly')
elif bucket == 'standard':
self.log.info("Creating standard bucket")
rest_conn.create_bucket(bucket='standard', ramQuotaMB=100, authType='none', saslPassword='', replicaNumber=1,
proxyPort=STANDARD_BUCKET_PORT, bucketType='membase', replica_index=1, threadsNumber=3,
flushEnabled=1, lww=False)
self.src_cluster.add_bucket(ramQuotaMB=100, bucket='standard', authType='none',
saslPassword='', replicaNumber=1, proxyPort=STANDARD_BUCKET_PORT, bucketType='membase',
evictionPolicy='valueOnly')
elif bucket== 'lww':
self.log.info("Creating lww bucket")
rest_conn.create_bucket(bucket='lww', ramQuotaMB=100, authType='none', saslPassword='', replicaNumber=1,
proxyPort=11211, bucketType='membase', replica_index=1, threadsNumber=3,
flushEnabled=1, lww=True)
self.src_cluster.add_bucket(ramQuotaMB=100, bucket='lww', authType='none',
saslPassword='', replicaNumber=1, proxyPort=11211, bucketType='membase',
evictionPolicy='valueOnly')
esrest_conn = EsRestConnection(self.dest_cluster.get_master_node())
esrest_conn.create_index(bucket)
rest_conn.add_remote_cluster(remoteIp=self.dest_master.ip, remotePort=9091, username='******',
password='******', name='es')
self.src_cluster.get_remote_clusters().append(XDCRRemoteClusterRef(self.src_cluster, self.dest_cluster,
Utility.get_rc_name(self.src_cluster.get_name(),
self.dest_cluster.get_name())))
repl_id = rest_conn.start_replication(replicationType='continuous', fromBucket=bucket, toCluster='es',
rep_type='capi', toBucket=bucket, xdcr_params=xdcr_params)
return repl_id
def test_basic_xdcr_with_cert(self):
cluster1 = self.servers[0:2]
cluster2 = self.servers[2:4]
remote_cluster_name = 'sslcluster'
restCluster1 = RestConnection(cluster1[0])
restCluster2 = RestConnection(cluster2[0])
try:
#Setup cluster1
x509main(cluster1[0]).setup_master()
x509main(cluster1[1])._setup_node_certificates(reload_cert=False)
restCluster1.add_node('Administrator','password',cluster1[1].ip)
known_nodes = ['ns_1@'+cluster1[0].ip,'ns_1@' + cluster1[1].ip]
restCluster1.rebalance(known_nodes)
self.assertTrue(self.check_rebalance_complete(restCluster1),"Issue with rebalance")
restCluster1.create_bucket(bucket='default', ramQuotaMB=100)
restCluster1.remove_all_replications()
restCluster1.remove_all_remote_clusters()
#Setup cluster2
x509main(cluster2[0]).setup_master()
x509main(cluster2[1])._setup_node_certificates(reload_cert=False)
restCluster2.add_node('Administrator','password',cluster2[1].ip)
known_nodes = ['ns_1@'+cluster2[0].ip,'ns_1@' + cluster2[1].ip]
restCluster2.rebalance(known_nodes)
self.assertTrue(self.check_rebalance_complete(restCluster2),"Issue with rebalance")
restCluster2.create_bucket(bucket='default', ramQuotaMB=100)
test = x509main.CACERTFILEPATH + x509main.CACERTFILE
data = open(test, 'rb').read()
restCluster1.add_remote_cluster(cluster2[0].ip,cluster2[0].port,'Administrator','password',remote_cluster_name,certificate=data)
replication_id = restCluster1.start_replication('continuous','default',remote_cluster_name)
if replication_id is not None:
self.assertTrue(True,"Replication was not created successfully")
finally:
known_nodes = ['ns_1@'+cluster2[0].ip,'ns_1@'+cluster2[1].ip]
restCluster2.rebalance(known_nodes,['ns_1@' + cluster2[1].ip])
self.assertTrue(self.check_rebalance_complete(restCluster2),"Issue with rebalance")
restCluster2.delete_bucket()
def _XDCR_role_test(self):
params = {}
remote_cluster_name = 'rbac_cluster'
remote_server01 = self.servers[1]
remote_server02 = self.servers[2]
read_role = '_replication_admin_read'
write_role = '_replication_admin_write'
rest_remote01 = RestConnection(remote_server01)
rest_remote01.create_bucket(bucket='default', ramQuotaMB=100)
rest_remote02 = RestConnection(remote_server02)
rest_remote02.create_bucket(bucket='default', ramQuotaMB=100)
#------ First Test the Get Requests for XDCR --------------#
#Remove all remote cluster references
self.rest.remove_all_replications()
self.rest.remove_all_remote_clusters()
#Add remote cluster reference and replications
self.rest.add_remote_cluster(remote_server01.ip,8091,'Administrator','password',remote_cluster_name)
replication_id = self.rest.start_replication('continuous','default',remote_cluster_name)
masDict,tc_status = self.rbac._iterate_role_mapping(read_role,"Administrator","password",{'replication_id':replication_id})
self.rest.remove_all_replications()
self.rest.remove_all_remote_clusters()
rest_remote01.remove_all_replications()
rest_remote01.remove_all_remote_clusters()
rest_remote02.remove_all_replications()
rest_remote02.remove_all_remote_clusters()
# ----------- Second Test for POST requests for XDCR ---------------#
self.rest.remove_all_replications()
self.rest.remove_all_remote_clusters()
rest_remote01.remove_all_replications()
rest_remote01.remove_all_remote_clusters()
rest_remote02.remove_all_replications()
rest_remote02.remove_all_remote_clusters()
self.rest.add_remote_cluster(remote_server01.ip,8091,'Administrator','password',"onetotwo")
#self.rest.add_remote_cluster(remote_server02.ip,8091,'Administrator','password','onetothree')
#rest_remote01.add_remote_cluster(remote_server02.ip,8091,'Administrator','password',"twotothree")
rest_remote01.add_remote_cluster(self.master.ip,8091,'Administrator','password','twotoone')
rest_remote02.add_remote_cluster(remote_server01.ip,8091,'Administrator','password',"threetotwo")
rest_remote02.add_remote_cluster(self.master.ip,8091,'Administrator','password','threetoone')
params['remote_cluster_name']='onetotwo'
params['remoteCluster01'] = {'username': '******', 'password': '******', 'hostname': '192.168.46.103:8091', 'name': 'onetothree'}
params['create_replication'] = {'replicationType': 'continuous','toBucket': 'default','fromBucket': 'default','toCluster': 'twotoone','type': 'xmem'}
params['replication_id'] = rest_remote01.start_replication('continuous','default','twotoone')
masDict,tc_status = self.rbac._iterate_role_mapping('_replication_admin_write01',"Administrator","password",params)
masDict,tc_status = self.rbac._iterate_role_mapping('_replication_admin_write02',"Administrator","password",params,self.servers[1])
'''
def start_replication(self,
master,
slave,
replication_type='continuous',
buckets=None,
bidir=False,
suffix='A'):
"""Add remote cluster and start replication"""
master_rest_conn = RestConnection(master)
remote_reference = 'remote_cluster_' + suffix
master_rest_conn.add_remote_cluster(slave.ip, slave.port,
slave.rest_username,
slave.rest_password,
remote_reference)
if not buckets:
buckets = self.get_buckets()
else:
buckets = self.get_buckets(reversed=True)
for bucket in buckets:
master_rest_conn.start_replication(replication_type, bucket,
remote_reference)
if self.parami('num_buckets', 1) > 1 and suffix == 'A':
self.start_replication(slave,
master,
replication_type,
buckets,
suffix='B')
if bidir:
self.start_replication(slave,
master,
replication_type,
buckets,
suffix='B')
def add(self):
"""create cluster reference- add remote cluster
"""
rest_conn_src = RestConnection(self.__src_cluster.master)
certificate = ""
dest_master = self.__dest_cluster.master
if self.__encryption:
rest_conn_dest = RestConnection(dest_master)
certificate = rest_conn_dest.get_cluster_ceritificate()
if self.__replicator_target_role:
self.dest_user = "******"
self.dest_pass = "******"
else:
self.dest_user = dest_master.rest_username
self.dest_pass = dest_master.rest_password
if not self.__use_scramsha:
self.__rest_info = rest_conn_src.add_remote_cluster(
dest_master.ip,
dest_master.port,
self.dest_user,
self.dest_pass,
self.name,
demandEncryption=self.__encryption,
certificate=certificate)
else:
print("Using scram-sha authentication")
self.__rest_info = rest_conn_src.add_remote_cluster(
dest_master.ip,
dest_master.port,
self.dest_user,
self.dest_pass,
self.name,
demandEncryption=self.__encryption,
encryptionType="half")
self.__validate_create_event()
class rbacTest(ldaptest):
def setUp(self):
super(rbacTest, self).setUp()
self.rest = RestConnection(self.master)
self.userList = "bjons:password"
self.userList = self.returnUserList(self.userList)
#self._removeLdapUserRemote(self.userList)
#self._createLDAPUser(self.userList)
self.roleName = self.input.param("roleName")
self.rbac = rbacmain(self.master,'default')
#rest.ldapUserRestOperation(True, ROadminUser=self.userList, exclude=None)
def tearDown(self):
super(rbacTest, self).tearDown()
def _XDCR_role_test(self):
params = {}
remote_cluster_name = 'rbac_cluster'
remote_server01 = self.servers[1]
remote_server02 = self.servers[2]
read_role = '_replication_admin_read'
write_role = '_replication_admin_write'
rest_remote01 = RestConnection(remote_server01)
rest_remote01.create_bucket(bucket='default', ramQuotaMB=100)
rest_remote02 = RestConnection(remote_server02)
rest_remote02.create_bucket(bucket='default', ramQuotaMB=100)
#------ First Test the Get Requests for XDCR --------------#
#Remove all remote cluster references
self.rest.remove_all_replications()
self.rest.remove_all_remote_clusters()
#Add remote cluster reference and replications
self.rest.add_remote_cluster(remote_server01.ip,8091,'Administrator','password',remote_cluster_name)
replication_id = self.rest.start_replication('continuous','default',remote_cluster_name)
masDict,tc_status = self.rbac._iterate_role_mapping(read_role,"Administrator","password",{'replication_id':replication_id})
self.rest.remove_all_replications()
self.rest.remove_all_remote_clusters()
rest_remote01.remove_all_replications()
rest_remote01.remove_all_remote_clusters()
rest_remote02.remove_all_replications()
rest_remote02.remove_all_remote_clusters()
# ----------- Second Test for POST requests for XDCR ---------------#
self.rest.remove_all_replications()
self.rest.remove_all_remote_clusters()
rest_remote01.remove_all_replications()
rest_remote01.remove_all_remote_clusters()
rest_remote02.remove_all_replications()
rest_remote02.remove_all_remote_clusters()
self.rest.add_remote_cluster(remote_server01.ip,8091,'Administrator','password',"onetotwo")
#self.rest.add_remote_cluster(remote_server02.ip,8091,'Administrator','password','onetothree')
#rest_remote01.add_remote_cluster(remote_server02.ip,8091,'Administrator','password',"twotothree")
rest_remote01.add_remote_cluster(self.master.ip,8091,'Administrator','password','twotoone')
rest_remote02.add_remote_cluster(remote_server01.ip,8091,'Administrator','password',"threetotwo")
rest_remote02.add_remote_cluster(self.master.ip,8091,'Administrator','password','threetoone')
params['remote_cluster_name']='onetotwo'
params['remoteCluster01'] = {'username': '******', 'password': '******', 'hostname': '192.168.46.103:8091', 'name': 'onetothree'}
params['create_replication'] = {'replicationType': 'continuous','toBucket': 'default','fromBucket': 'default','toCluster': 'twotoone','type': 'xmem'}
params['replication_id'] = rest_remote01.start_replication('continuous','default','twotoone')
masDict,tc_status = self.rbac._iterate_role_mapping('_replication_admin_write01',"Administrator","password",params)
masDict,tc_status = self.rbac._iterate_role_mapping('_replication_admin_write02',"Administrator","password",params,self.servers[1])
'''
self.rest.remove_all_replications()
self.rest.remove_all_remote_clusters()
rest_remote01.remove_all_replications()
rest_remote01.remove_all_remote_clusters()
rest_remote02.remove_all_replications()
rest_remote02.remove_all_remote_clusters()
'''
def _raw_urllib(self):
api = "http://192.168.46.101:8092/" + "default/_design/test1"
header = {'Content-Type': 'application/json'}
params = '{"views":{"test1":{"map":"function (doc, meta) {emit(meta.id, null);}"}}}'
http = httplib2.Http()
status, content = http.request(api, 'PUT', headers=header,body=params)
print status
print content
def _view_admin_role_test(self):
#----------------Get ddocs -----------#
view = ['abcd']
default_design_doc_name = "Doc1"
default_map_func = 'function (doc) { emit(doc.age, doc.first_name);}'
params = {}
view = View('abcd', default_map_func, None)
self.cluster.create_view(
self.master, default_design_doc_name, view,
'default',180,with_query=False)
masDict,tc_status = self.rbac._iterate_role_mapping('_view_admin_read',"Administrator","password")
doc_id = "_design/dev_Doc1"
params['doc_id'] = doc_id
test_param = {"views" : {"byloc" : {"map" : "function (doc, meta) {\n if (meta.type == \"json\") {\n emit(doc.city, doc.sales);\n } else {\n emit([\"blob\"]);\n }\n}"}}}
masDict,tc_status = self.rbac._iterate_role_mapping('_view_admin_write01',"Administrator","password",params)
# ------------------------------------#
def checkRole(self):
masDict = {}
tc_status = None
#---Setup for XDCR ------#
#self._XDCR_role_test()
#self._view_admin_role_test()
self._raw_urllib()
def _link_es_cluster(self):
rest_conn_src = RestConnection(self.master)
rest_conn_src.add_remote_cluster(self.es_host, self.es_port,
"Administrator",
"password", "ElasticSearch")
self.es_cluster_name = "ElasticSearch"
def _link_clusters(self, src_cluster_name, src_master, dest_cluster_name, dest_master):
rest_conn_src = RestConnection(src_master)
rest_conn_src.add_remote_cluster(dest_master.ip, dest_master.port,
dest_master.rest_username,
dest_master.rest_password, dest_cluster_name)
def test_continuous_unidirectional_recreates(self):
cluster_ref_a = "cluster_ref_a"
master_a = self._input.clusters.get(0)[0]
rest_conn_a = RestConnection(master_a)
cluster_ref_b = "cluster_ref_b"
master_b = self._input.clusters.get(1)[0]
rest_conn_b = RestConnection(master_b)
# Disable compaction, we should remove these
# once the bug in comapctor get fixed
#rest_conn_a.set_auto_compaction("false", 100, 100)
#rest_conn_b.set_auto_compaction("false", 100, 100)
# Start load
kvstore = ClientKeyValueStore()
self._params["ops"] = "set"
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore)
load_thread.start()
load_thread.join()
# Start replication
replication_type = "continuous"
rest_conn_a.add_remote_cluster(master_b.ip, master_b.port,
master_b.rest_username,
master_b.rest_password, cluster_ref_b)
(rep_database, rep_id) = rest_conn_a.start_replication(replication_type,
self._buckets[0],
cluster_ref_b)
self._state.append((rest_conn_a, cluster_ref_b, rep_database, rep_id))
# Verify replicated data
self.assertTrue(XDCRBaseTest.verify_replicated_data(rest_conn_b,
self._buckets[0],
kvstore,
self._poll_sleep,
self._poll_timeout),
"Replication verification failed")
# Delete all keys
self._params["ops"] = "delete"
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore)
load_thread.start()
load_thread.join()
# Verify replicated data
self.assertTrue(XDCRBaseTest.verify_replicated_data(rest_conn_b,
self._buckets[0],
kvstore,
self._poll_sleep,
self._poll_timeout),
"Replication verification failed")
# Recreate the keys with different values
kvstore = ClientKeyValueStore()
self._params["ops"] = "set"
self._params["padding"] = "recreated"
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore)
load_thread.start()
load_thread.join()
# Verify replicated data
self.assertTrue(XDCRBaseTest.verify_replicated_data(rest_conn_b,
self._buckets[0],
kvstore,
self._poll_sleep,
self._poll_timeout),
"Replication verification failed")
def test_failover_source_sets(self):
replication_type = "continuous"
self.log.info("Force initial rebalance.")
# This test starts with a 2-2 unidirectional replication from cluster a
# to cluster b; during the replication, we trigger failover of one node
# on source cluster , resulting a 1-2 replication.
# After all loading finish, verify data and rev on both clusters.
replication_type = "continuous"
self.log.info("Force initial rebalance.")
cluster_ref_a = "cluster_ref_a"
master_a = self._input.clusters.get(0)[0]
rest_conn_a = RestConnection(master_a)
cluster_ref_b = "cluster_ref_b"
master_b = self._input.clusters.get(1)[0]
rest_conn_b = RestConnection(master_b)
self.log.info("START XDC replication...")
# Start replication
rest_conn_a.add_remote_cluster(master_b.ip, master_b.port,
master_b.rest_username,
master_b.rest_password, cluster_ref_b)
(rep_database, rep_id) = rest_conn_a.start_replication(replication_type,
self._buckets[0],
cluster_ref_b)
self._state.append((rest_conn_a, cluster_ref_b, rep_database, rep_id))
# Start load
self.log.info("START loading data...")
load_thread_list = []
kvstore = ClientKeyValueStore()
self._params["ops"] = "set"
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore)
load_thread.start()
# sleep a while to allow more data loaded
time.sleep(5)
self.log.info("current nodes on source cluster: {0}".format(RebalanceHelper.getOtpNodeIds(master_a)))
# Trigger failover, we fail over one node each time until there is only one node remaining
self.log.info("DURING replication, start failover...")
self.log.info("FAILOVER nodes on Cluster A ...")
nodes_a = rest_conn_a.node_statuses()
while len(nodes_a) > 1:
toBeFailedOverNode = RebalanceHelper.pick_node(master_a)
self.log.info("failover node {0}".format(toBeFailedOverNode.id))
rest_conn_a.fail_over(toBeFailedOverNode)
self.log.info("rebalance after failover")
rest_conn_a.rebalance(otpNodes=[node.id for node in rest_conn_a.node_statuses()], \
ejectedNodes=[toBeFailedOverNode.id])
self.assertTrue(rest_conn_a.monitorRebalance(),
msg="rebalance operation failed after removing node {0}".format(toBeFailedOverNode.id))
nodes_a = rest_conn_a.node_statuses()
self.log.info("ALL failed over done...")
# Wait for loading threads to finish
for lt in load_thread_list:
lt.join()
self.log.info("All loading threads finished")
# Verify replication
self.log.info("START data verification at cluster A...")
self.assertTrue(XDCRBaseTest.verify_replicated_data(rest_conn_a,
self._buckets[0],
kvstore,
self._poll_sleep,
self._poll_timeout),
"Verification of replicated data failed")
self.log.info("START data verification at cluster B...")
self.assertTrue(XDCRBaseTest.verify_replicated_data(rest_conn_b,
self._buckets[0],
kvstore,
self._poll_sleep,
self._poll_timeout),
"Verification of replicated data failed")
self.log.info("START revision verification on both clusters...")
self.assertTrue(XDCRBaseTest.verify_replicated_revs(rest_conn_a,
rest_conn_b,
self._buckets[0],
self._poll_sleep,
self._poll_timeout),
"Verification of replicated revisions failed")
def test_rebalance_in_dest_sets(self):
# This test starts with a 1-1 unidirectional replication from cluster a
# to cluster b; during the replication, we trigger rebalace-in on dest
# cluster b, to create a 1-2 replication. After all loading finish,
# verify data and rev on both clusters.
replication_type = "continuous"
cluster_ref_a = "cluster_ref_a"
master_a = self._input.clusters.get(0)[0]
rest_conn_a = RestConnection(master_a)
cluster_ref_b = "cluster_ref_b"
master_b = self._input.clusters.get(1)[0]
rest_conn_b = RestConnection(master_b)
self.log.info("START XDC replication...")
# Start replication
rest_conn_a.add_remote_cluster(master_b.ip, master_b.port,
master_b.rest_username,
master_b.rest_password, cluster_ref_b)
(rep_database, rep_id) = rest_conn_a.start_replication(replication_type,
self._buckets[0],
cluster_ref_b)
self._state.append((rest_conn_a, cluster_ref_b, rep_database, rep_id))
# Start load
self.log.info("START loading data...")
load_thread_list = []
kvstore = ClientKeyValueStore()
self._params["ops"] = "set"
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore)
load_thread.start()
# Trigger rebalance
self.log.info("DURING replication, start rebalancing...")
servers_a = self._input.clusters.get(0)
self.log.info("REBALANCING IN Cluster B ...")
RebalanceHelper.rebalance_in(servers_b, len(servers_b)-1, monitor=False)
self.assertTrue(rest_conn_b.monitorRebalance(),
msg="rebalance operation on cluster {0}".format(servers_b))
self.log.info("ALL rebalancing done...")
# Wait for loading to finish
load_thread.join()
self.log.info("All deleting threads finished")
# Verify replication
self.log.info("START data verification at cluster A...")
self.assertTrue(XDCRBaseTest.verify_replicated_data(rest_conn_a,
self._buckets[0],
kvstore,
self._poll_sleep,
self._poll_timeout),
"Verification of replicated data failed")
self.log.info("START data verification at cluster B...")
self.assertTrue(XDCRBaseTest.verify_replicated_data(rest_conn_b,
self._buckets[0],
kvstore,
self._poll_sleep,
self._poll_timeout),
"Verification of replicated data failed")
self.log.info("START revision verification on both clusters...")
self.assertTrue(XDCRBaseTest.verify_replicated_revs(rest_conn_a,
rest_conn_b,
self._buckets[0],
self._poll_sleep,
self._poll_timeout),
"Verification of replicated revisions failed")
class XDCRUpgradeCollectionsTests(NewUpgradeBaseTest):
def setUp(self):
super(XDCRUpgradeCollectionsTests, self).setUp()
self.nodes_init = self.input.param('nodes_init', 2)
self.queue = queue.Queue()
self.rate_limit = self.input.param("rate_limit", 100000)
self.batch_size = self.input.param("batch_size", 1000)
self.doc_size = self.input.param("doc_size", 100)
self.loader = self.input.param("loader", "high_doc_ops")
self.instances = self.input.param("instances", 4)
self.threads = self.input.param("threads", 5)
self.use_replica_to = self.input.param("use_replica_to", False)
self.index_name_prefix = None
self.rest_src = RestConnection(self.servers[0])
def tearDown(self):
super(XDCRUpgradeCollectionsTests, self).tearDown()
def enable_migration_mode(self, src_bucket, dest_bucket):
setting_val_map = {"collectionsMigrationMode": "true",
"colMappingRules": '{"REGEXP_CONTAINS(META().id,\'0$\')":"scope1.mycollection_scope1"}'
}
self.rest_src.set_xdcr_params(src_bucket, dest_bucket, setting_val_map)
def verify_doc_counts(self):
des_master = self.servers[self.nodes_init]
src_cbver = RestConnection(self.master).get_nodes_version()
des_cbver = RestConnection(des_master).get_nodes_version()
src_items = RestConnection(self.master).get_buckets_itemCount()
des_items = RestConnection(des_master).get_buckets_itemCount()
if src_cbver[:3] < "7.0" and des_cbver[:3] >= "7.0":
des_items = self.get_col_item_count(des_master, "default", "_default",
"_default", self.des_stat_col)
if src_items["default"] != des_items:
self.fail("items do not match. src: {0} != des: {1}"
.format(src_items["default"], des_items))
elif src_cbver[:3] >= "7.0" and des_cbver[:3] < "7.0":
src_items = self.get_col_item_count(self.master, "default", "_default",
"_default", self.stat_col)
if src_items != des_items["default"]:
self.fail("items do not match. src: {0} != des: {1}"
.format(src_items, des_items["default"]))
elif src_cbver[:3] >= "7.0" and des_cbver[:3] >= "7.0":
if src_items["default"] != des_items["default"]:
self.fail("items do not match. src: {0} != des: {1}"
.format(src_items["default"], des_items["default"]))
def test_xdcr_upgrade_with_services(self):
after_upgrade_services_in = self.input.param("after_upgrade_services_in", False)
after_upgrade_buckets_in = self.input.param("after_upgrade_buckets_in", False)
after_upgrade_buckets_out = self.input.param("after_upgrade_buckets_out", False)
after_upgrade_buckets_flush = self.input.param("after_upgrade_buckets_flush", False)
# Install initial version on the specified nodes
self._install(self.servers[:self.nodes_init])
# Configure the nodes with services on cluster1
self.operations(self.servers[:self.nodes_init], services="kv,kv")
# get the n1ql node which will be used in pre,during and post upgrade for running n1ql commands
self.n1ql_server = self.get_nodes_from_services_map(service_type="n1ql")
# Run the pre upgrade operations, typically creating index
self.pre_upgrade(self.servers[:self.nodes_init])
if self.input.param("ddocs_num", 0) > 0:
self.create_ddocs_and_views()
self._install(self.servers[self.nodes_init:self.num_servers])
self.master = self.servers[self.nodes_init]
# Configure the nodes with services on the other cluster2
try:
self.operations(self.servers[self.nodes_init:self.num_servers], services="kv,kv")
self.sleep(timeout=10)
except Exception as ex:
if ex:
print("error: ", str(ex))
self.log.info("bucket is created")
# create a xdcr relationship between cluster1 and cluster2
self.rest_src.add_remote_cluster(self.servers[self.nodes_init].ip,
self.servers[self.nodes_init].port,
'Administrator', 'password', "C2")
repl_id = self.rest_src.start_replication('continuous', 'default', "C2")
if repl_id is not None:
self.log.info("Replication created successfully")
# Run the post_upgrade operations
self._create_ephemeral_buckets()
self.post_upgrade(self.servers[:self.nodes_init])
# Add new services after the upgrade
for upgrade_version in self.upgrade_versions:
src_nodes = self.servers[:self.nodes_init]
for server in src_nodes:
remote = RemoteMachineShellConnection(server)
remote.stop_server()
src_upgrade_threads = self._async_update(upgrade_version, src_nodes)
for upgrade_thread in src_upgrade_threads:
upgrade_thread.join()
src_success_upgrade = True
while not self.queue.empty():
src_success_upgrade &= self.queue.get()
if not src_success_upgrade:
self.fail("Upgrade failed in source cluster. See logs above!")
else:
self.log.info("Upgrade source cluster success")
des_nodes = self.servers[self.nodes_init:self.num_servers]
self.master = self.servers[self.nodes_init]
for server in des_nodes:
remote = RemoteMachineShellConnection(server)
remote.stop_server()
des_upgrade_threads = self._async_update(upgrade_version, des_nodes)
for upgrade_thread in des_upgrade_threads:
upgrade_thread.join()
des_success_upgrade = True
while not self.queue.empty():
des_success_upgrade &= self.queue.get()
if not des_success_upgrade:
self.fail("Upgrade failed in des cluster. See logs above!")
else:
self.log.info("Upgrade des cluster success")
self.master = self.servers[0]
self.rest = RestConnection(self.master)
self.rest_col = CollectionsRest(self.master)
self.cli_col = CollectionsCLI(self.master)
self.stat_col = CollectionsStats(self.master)
self.log.info("Create scope collection at src cluster")
#self.rest_col.create_scope_collection_count()
self._create_scope_collection(self.rest_col, self.cli_col, self.buckets[0].name)
self.sleep(10)
self.des_rest = RestConnection(self.servers[self.nodes_init])
self.des_rest_col = CollectionsRest(self.servers[self.nodes_init])
self.des_cli_col = CollectionsCLI(self.servers[self.nodes_init])
self.des_stat_col = CollectionsStats(self.servers[self.nodes_init])
self.log.info("Create scope collection at des cluster")
self.buckets = RestConnection(self.servers[self.nodes_init]).get_buckets()
self._create_scope_collection(self.des_rest_col, self.des_cli_col, self.buckets[0].name)
self.load_to_collections_bucket()
self.enable_migration = self.input.param("enable_migration", False)
if self.enable_migration:
self.enable_migration_mode(self.buckets[0].name, self.buckets[0].name)
self.verify_doc_counts()
if after_upgrade_buckets_in is not False:
self.bucket_size = 100
self._create_sasl_buckets(self.master, 1)
self._create_standard_buckets(self.master, 1)
if self.input.param("ddocs_num", 0) > 0:
self.create_ddocs_and_views()
gen_load = BlobGenerator('upgrade', 'upgrade-', self.value_size,
end=self.num_items)
self._load_all_buckets(self.master, gen_load, "create", self.expire_time,
flag=self.item_flag)
# deleting buckets after upgrade
if after_upgrade_buckets_out is not False:
self._all_buckets_delete(self.master)
# flushing buckets after upgrade
if after_upgrade_buckets_flush is not False:
self._all_buckets_flush()
def run_view_queries(self):
view_query_thread = Thread(target=self.view_queries, name="run_queries",
args=(self.run_view_query_iterations,))
return view_query_thread
def view_queries(self, iterations):
query = {"connectionTimeout": 60000}
for count in range(iterations):
for i in range(self.view_num):
self.cluster.query_view(self.master, self.ddocs[0].name,
self.default_view_name + str(i), query,
expected_rows=None, bucket="default", retry_time=2)
def create_user(self, node):
self.log.info("inside create_user")
testuser = [{'id': 'cbadminbucket', 'name': 'cbadminbucket',
'password': '******'}]
rolelist = [{'id': 'cbadminbucket', 'name': 'cbadminbucket',
'roles': 'admin'}]
self.log.info("before create_user_source")
RbacBase().create_user_source(testuser, 'builtin', node)
self.log.info("before add_user_role")
RbacBase().add_user_role(rolelist, RestConnection(node), 'builtin')
def test_continuous_unidirectional_deletes_1(self):
cluster_ref_a = "cluster_ref_a"
master_a = self._input.clusters.get(0)[0]
rest_conn_a = RestConnection(master_a)
cluster_ref_b = "cluster_ref_b"
master_b = self._input.clusters.get(1)[0]
rest_conn_b = RestConnection(master_b)
# Load some data on cluster a. Do it a few times so that the seqnos are
# bumped up and then delete it.
kvstore = ClientKeyValueStore()
self._params["ops"] = "set"
load_thread_list = []
for i in [1, 2, 3]:
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore)
load_thread_list.append(load_thread)
for lt in load_thread_list:
lt.start()
for lt in load_thread_list:
lt.join()
time.sleep(10)
self._params["ops"] = "delete"
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore)
load_thread.start()
load_thread.join()
# Load the same data on cluster b but only once. This will cause the
# seqno values to be lower than those on cluster a allowing the latter
# to win during conflict resolution later. Then delete this data, too.
kvstore = ClientKeyValueStore()
self._params["ops"] = "set"
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_b,
self._buckets[0],
task_def, kvstore)
load_thread.start()
load_thread.join()
time.sleep(10)
self._params["ops"] = "delete"
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_b,
self._buckets[0],
task_def, kvstore)
load_thread.start()
load_thread.join()
# Start replication to replicate the deletes from cluster a (having
# higher seqnos) to cluster b.
replication_type = "continuous"
rest_conn_a.add_remote_cluster(master_b.ip, master_b.port,
master_b.rest_username,
master_b.rest_password, cluster_ref_b)
(rep_database, rep_id) = rest_conn_a.start_replication(replication_type,
self._buckets[0],
cluster_ref_b)
self._state.append((rest_conn_a, cluster_ref_b, rep_database, rep_id))
time.sleep(15)
# Verify replicated data
self.assertTrue(XDCRBaseTest.verify_changes_feed(rest_conn_a,
rest_conn_b,
self._buckets[0],
self._poll_sleep,
self._poll_timeout),
"Changes feed verification failed")
def test_continuous_bidirectional_sets(self):
cluster_ref_a = "cluster_ref_a"
master_a = self._input.clusters.get(0)[0]
rest_conn_a = RestConnection(master_a)
cluster_ref_b = "cluster_ref_b"
master_b = self._input.clusters.get(1)[0]
rest_conn_b = RestConnection(master_b)
load_thread_list = []
# Load cluster a with keys that will be exclusively owned by it
kvstore_a0 = ClientKeyValueStore()
self._params["ops"] = "set"
self._params["seed"] = "cluster-a"
self._params["count"] = self._num_items/4
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore_a0)
load_thread_list.append(load_thread)
# Load cluster a with keys that it will share with cluster b and which
# will win during conflict resolution
kvstore_a1 = ClientKeyValueStore()
self._params["ops"] = "set"
self._params["seed"] = "cluster-a-wins"
self._params["padding"] = "cluster-a-wins"
self._params["count"] = self._num_items/4
# Mutating these keys several times will increase their seqnos and allow
# them to win during conflict resolution
for i in [1, 2, 3]:
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def,
kvstore_a1)
load_thread_list.append(load_thread)
# Load cluster a with keys that it will share with cluster b but which
# will lose during conflict resolution
kvstore_a2 = ClientKeyValueStore()
self._params["ops"] = "set"
self._params["seed"] = "cluster-b-wins"
self._params["padding"] = "cluster-a-loses"
self._params["count"] = self._num_items/4
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore_a2)
load_thread_list.append(load_thread)
# Load cluster b with keys that will be exclusively owned by it
kvstore_b0 = ClientKeyValueStore()
self._params["ops"] = "set"
self._params["seed"] = "cluster-b"
self._params["count"] = self._num_items/4
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_b,
self._buckets[0],
task_def, kvstore_b0)
load_thread_list.append(load_thread)
# Load cluster b with keys that it will share with cluster a and which
# will win during conflict resolution
kvstore_b1 = ClientKeyValueStore()
self._params["ops"] = "set"
self._params["seed"] = "cluster-b-wins"
self._params["padding"] = "cluster-b-wins"
self._params["count"] = self._num_items/4
# Mutating these keys several times will increase their seqnos and allow
# them to win during conflict resolution
for i in [1, 2, 3]:
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_b,
self._buckets[0],
task_def,
kvstore_b1)
load_thread_list.append(load_thread)
# Load cluster b with keys that it will share with cluster a but which
# will lose during conflict resolution
kvstore_b2 = ClientKeyValueStore()
self._params["ops"] = "set"
self._params["seed"] = "cluster-a-wins"
self._params["padding"] = "cluster-b-loses"
self._params["count"] = self._num_items/4
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_b,
self._buckets[0],
task_def, kvstore_b2)
load_thread_list.append(load_thread)
# Setup bidirectional replication between cluster a and cluster b
replication_type = "continuous"
rest_conn_a.add_remote_cluster(master_b.ip, master_b.port,
master_b.rest_username,
master_b.rest_password, cluster_ref_b)
rest_conn_b.add_remote_cluster(master_a.ip, master_a.port,
master_a.rest_username,
master_a.rest_password, cluster_ref_a)
(rep_database_a, rep_id_a) = rest_conn_a.start_replication(
replication_type, self._buckets[0],
cluster_ref_b)
(rep_database_b, rep_id_b) = rest_conn_b.start_replication(
replication_type, self._buckets[0],
cluster_ref_a)
self._state.append((rest_conn_a, cluster_ref_b, rep_database_a, rep_id_a))
self._state.append((rest_conn_b, cluster_ref_a, rep_database_b, rep_id_b))
# Start all loads concurrently and wait for them to end
for lt in load_thread_list:
lt.start()
for lt in load_thread_list:
lt.join()
self.log.info("All loading threads finished")
# Verify replicated data
for rest_conn in [rest_conn_a, rest_conn_b]:
for kvstore in [kvstore_a0, kvstore_a1, kvstore_b0, kvstore_b1]:
self.assertTrue(
XDCRBaseTest.verify_replicated_data(rest_conn,
self._buckets[0],
kvstore,
self._poll_sleep,
self._poll_timeout),
"Verification of replicated data failed")
self.assertTrue(XDCRBaseTest.verify_replicated_revs(rest_conn_a,
rest_conn_b,
self._buckets[0],
self._poll_sleep,
self._poll_timeout),
"Verification of replicated revisions failed")
def test_incremental_rebalance_in_continuous_bidirectional_sets_deletes(self):
cluster_ref_a = "cluster_ref_a"
master_a = self._input.clusters.get(0)[0]
rest_conn_a = RestConnection(master_a)
cluster_ref_b = "cluster_ref_b"
master_b = self._input.clusters.get(1)[0]
rest_conn_b = RestConnection(master_b)
# Setup bi-directional continuous replication
replication_type = "continuous"
rest_conn_a.add_remote_cluster(master_b.ip, master_b.port,
master_b.rest_username,
master_b.rest_password, cluster_ref_b)
rest_conn_b.add_remote_cluster(master_a.ip, master_a.port,
master_a.rest_username,
master_a.rest_password, cluster_ref_a)
(rep_database_a, rep_id_a) = rest_conn_a.start_replication(
replication_type, self._buckets[0],
cluster_ref_b)
(rep_database_b, rep_id_b) = rest_conn_b.start_replication(
replication_type, self._buckets[0],
cluster_ref_a)
self._state.append((rest_conn_a, cluster_ref_b, rep_database_a, rep_id_a))
self._state.append((rest_conn_b, cluster_ref_a, rep_database_b, rep_id_b))
load_thread_list = []
# Start load
kvstore = ClientKeyValueStore()
self._params["ops"] = "set"
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore)
load_thread.start()
load_thread.join()
# Do some deletes
self._params["ops"] = "delete"
self._params["count"] = self._num_items/5
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore)
load_thread_list.append(load_thread)
# Start all loads concurrently
for lt in load_thread_list:
lt.start()
# Trigger rebalance on both source and destination clusters
servers_a = self._input.clusters.get(0)
servers_b = self._input.clusters.get(1)
nodes_a = rest_conn_a.node_statuses()
nodes_b = rest_conn_b.node_statuses()
rebalanced_servers_a = [master_a]
rebalanced_servers_b = [master_b]
which_servers_a = []
which_servers_b = []
# Incremental rebalance in one node in cluster_a, then cluster_b
while len(nodes_a) < len(servers_a):
self.log.info("current nodes : {0}".format([node.id for node in rest_conn_a.node_statuses()]))
rebalanced_in, which_servers_a = RebalanceHelper.rebalance_in(servers_a, 1, monitor=False)
self.assertTrue(rebalanced_in, msg="unable to add and rebalance more nodes")
self.assertTrue(rest_conn_a.monitorRebalance(),
msg="rebalance operation on cluster {0}".format(nodes_a))
while len(nodes_b) < len(servers_b):
self.log.info("current nodes : {0}".format([node.id for node in rest_conn_b.node_statuses()]))
rebalanced_in, which_servers_b = RebalanceHelper.rebalance_in(servers_b, 1, monitor=False)
self.assertTrue(rebalanced_in, msg="unable to add and rebalance more nodes")
break
self.assertTrue(rest_conn_b.monitorRebalance(),
msg="rebalance operation on cluster {0}".format(nodes_b))
rebalanced_servers_b.extend(which_servers_b)
nodes_b = rest_conn_b.node_statuses()
rebalanced_servers_a.extend(which_servers_a)
nodes_a = rest_conn_a.node_statuses()
# Wait for loading threads to finish
for lt in load_thread_list:
lt.join()
self.log.info("All loading threads finished")
# Verify replication
self.assertTrue(XDCRBaseTest.verify_replicated_data(rest_conn_b,
self._buckets[0],
kvstore,
self._poll_sleep,
self._poll_timeout),
"Verification of replicated data failed")
self.assertTrue(XDCRBaseTest.verify_replicated_revs(rest_conn_a,
rest_conn_b,
self._buckets[0],
self._poll_sleep,
self._poll_timeout),
"Verification of replicated revisions failed")
def test_incremental_rebalance_out_continuous_bidirectional_sets_deletes(self):
cluster_ref_a = "cluster_ref_a"
master_a = self._input.clusters.get(0)[0]
rest_conn_a = RestConnection(master_a)
cluster_ref_b = "cluster_ref_b"
master_b = self._input.clusters.get(1)[0]
rest_conn_b = RestConnection(master_b)
# Setup bi-directional continuous replication
replication_type = "continuous"
rest_conn_a.add_remote_cluster(master_b.ip, master_b.port,
master_b.rest_username,
master_b.rest_password, cluster_ref_b)
rest_conn_b.add_remote_cluster(master_a.ip, master_a.port,
master_a.rest_username,
master_a.rest_password, cluster_ref_a)
(rep_database_a, rep_id_a) = rest_conn_a.start_replication(
replication_type, self._buckets[0],
cluster_ref_b)
(rep_database_b, rep_id_b) = rest_conn_b.start_replication(
replication_type, self._buckets[0],
cluster_ref_a)
self._state.append((rest_conn_a, cluster_ref_b, rep_database_a, rep_id_a))
self._state.append((rest_conn_b, cluster_ref_a, rep_database_b, rep_id_b))
load_thread_list = []
# Start load
kvstore = ClientKeyValueStore()
self._params["ops"] = "set"
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore)
load_thread.start()
load_thread.join()
# Do some deletes
self._params["ops"] = "delete"
self._params["count"] = self._num_items/5
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore)
load_thread_list.append(load_thread)
# Start all loads concurrently
for lt in load_thread_list:
lt.start()
# Trigger rebalance on both source and destination clusters
servers_a = self._input.clusters.get(0)
servers_b = self._input.clusters.get(1)
rebalanced_servers_a = []
rebalanced_servers_b = []
which_servers_a = []
which_servers_b = []
# Rebalance all the nodes together
RebalanceHelper.rebalance_in(servers_a, len(servers_a)-1)
RebalanceHelper.rebalance_in(servers_b, len(servers_b)-1)
rebalanced_servers_a.extend(servers_a)
rebalanced_servers_b.extend(servers_b)
nodes_a = rest_conn_a.node_statuses()
nodes_b = rest_conn_b.node_statuses()
# Incremental rebalance out one node in cluster_a, then cluster_b
while len(nodes_a) > 1:
toBeEjectedNode = RebalanceHelper.pick_node(master_a)
self.log.info("current nodes : {0}".format(RebalanceHelper.getOtpNodeIds(master_a)))
self.log.info("removing node {0} and rebalance afterwards".format(toBeEjectedNode.id))
rest_conn_a.rebalance(otpNodes=[node.id for node in rest_conn_a.node_statuses()], \
ejectedNodes=[toBeEjectedNode.id])
self.assertTrue(rest_conn_a.monitorRebalance(),
msg="rebalance operation failed after adding node {0}".format(toBeEjectedNode.id))
while len(nodes_b) > 1:
toBeEjectedNode = RebalanceHelper.pick_node(master_b)
self.log.info("current nodes : {0}".format(RebalanceHelper.getOtpNodeIds(master_b)))
self.log.info("removing node {0} and rebalance afterwards".format(toBeEjectedNode.id))
rest_conn_b.rebalance(otpNodes=[node.id for node in rest_conn_b.node_statuses()],\
ejectedNodes=[toBeEjectedNode.id])
self.assertTrue(rest_conn_b.monitorRebalance(),
msg="rebalance operation failed after adding node {0}".format(toBeEjectedNode.id))
break
for node in nodes_b:
for rebalanced_server in rebalanced_servers_b:
if rebalanced_server.ip.find(node.ip) != -1:
rebalanced_servers_b.remove(rebalanced_server)
break
nodes_b = rest_conn_a.node_statuses()
for node in nodes_a:
for rebalanced_server in rebalanced_servers_a:
if rebalanced_server.ip.find(node.ip) != -1:
rebalanced_servers_a.remove(rebalanced_server)
break
nodes_a = rest_conn_a.node_statuses()
for node in nodes_a:
for rebalanced_server in rebalanced_servers_a:
if rebalanced_server.ip.find(node.ip) != -1:
rebalanced_servers_a.remove(rebalanced_server)
break
nodes_a= rest_conn_a.node_statuses()
# Wait for loading threads to finish
for lt in load_thread_list:
lt.join()
self.log.info("All loading threads finished")
# Verify replication
self.assertTrue(XDCRBaseTest.verify_replicated_data(rest_conn_b,
self._buckets[0],
kvstore,
self._poll_sleep,
self._poll_timeout),
"Verification of replicated data failed")
self.assertTrue(XDCRBaseTest.verify_replicated_revs(rest_conn_a,
rest_conn_b,
self._buckets[0],
self._poll_sleep,
self._poll_timeout),
"Verification of replicated revisions failed")
def test_failover_continuous_bidirectional_sets_deletes(self):
cluster_ref_a = "cluster_ref_a"
master_a = self._input.clusters.get(0)[0]
rest_conn_a = RestConnection(master_a)
cluster_ref_b = "cluster_ref_b"
master_b = self._input.clusters.get(1)[0]
rest_conn_b = RestConnection(master_b)
# Rebalance all the nodes together
servers_a = self._input.clusters.get(0)
servers_b = self._input.clusters.get(1)
rebalanced_servers_a = []
rebalanced_servers_b = []
RebalanceHelper.rebalance_in(servers_a, len(servers_a)-1)
RebalanceHelper.rebalance_in(servers_b, len(servers_b)-1)
rebalanced_servers_a.extend(servers_a)
rebalanced_servers_b.extend(servers_b)
# Setup bi-directional continuous replication
replication_type = "continuous"
rest_conn_a.add_remote_cluster(master_b.ip, master_b.port,
master_b.rest_username,
master_b.rest_password, cluster_ref_b)
rest_conn_b.add_remote_cluster(master_a.ip, master_a.port,
master_a.rest_username,
master_a.rest_password, cluster_ref_a)
(rep_database_a, rep_id_a) = rest_conn_a.start_replication(
replication_type, self._buckets[0],
cluster_ref_b)
(rep_database_b, rep_id_b) = rest_conn_b.start_replication(
replication_type, self._buckets[0],
cluster_ref_a)
load_thread_list = []
# Start load
kvstore = ClientKeyValueStore()
self._params["ops"] = "set"
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore)
load_thread.start()
load_thread.join()
RebalanceHelper.wait_for_persistence(master_a, self._buckets[0])
# Do some deletes
self._params["ops"] = "delete"
self._params["count"] = self._num_items/5
task_def = RebalanceDataGenerator.create_loading_tasks(self._params)
load_thread = RebalanceDataGenerator.start_load(rest_conn_a,
self._buckets[0],
task_def, kvstore)
load_thread_list.append(load_thread)
# Start all loads concurrently
for lt in load_thread_list:
lt.start()
# Do the failover of nodes on both clusters
self.log.info("Failing over nodes")
self.log.info("current nodes on cluster 1: {0}".format(RebalanceHelper.getOtpNodeIds(master_a)))
self.log.info("current nodes on cluster 2: {0}".format(RebalanceHelper.getOtpNodeIds(master_b)))
# Find nodes to be failed_over
toBeEjectedNodes = RebalanceHelper.pick_nodes(master_a, howmany=self._failover_factor)
optNodesIds_a = [node.id for node in toBeEjectedNodes]
if self._fail_orchestrator_a:
status, content = ClusterOperationHelper.find_orchestrator(master_a)
self.assertTrue(status, msg="Unable to find orchestrator: {0}:{1}".\
format(status, content))
optNodesIds_a[0] = content
master_a = self._input.clusters.get(0)[-1]
rest_conn_a = RestConnection(master_a)
#Failover selected nodes
for node in optNodesIds_a:
self.log.info("failover node {0} and rebalance afterwards".format(node))
rest_conn_a.fail_over(node)
toBeEjectedNodes = RebalanceHelper.pick_nodes(master_b, howmany=self._failover_factor)
optNodesIds_b = [node.id for node in toBeEjectedNodes]
if self._fail_orchestrator_b:
status, content = ClusterOperationHelper.find_orchestrator(master_b)
self.assertTrue(status, msg="Unable to find orchestrator: {0}:{1}".\
format(status, content))
optNodesIds_b[0] = content
master_b = self._input.clusters.get(1)[-1]
rest_conn_b = RestConnection(master_b)
self._state.append((rest_conn_a, cluster_ref_b, rep_database_a, rep_id_a))
self._state.append((rest_conn_b, cluster_ref_a, rep_database_b, rep_id_b))
#Failover selected nodes
for node in optNodesIds_b:
self.log.info("failover node {0} and rebalance afterwards".format(node))
rest_conn_b.fail_over(node)
rest_conn_a.rebalance(otpNodes=[node.id for node in rest_conn_a.node_statuses()],\
ejectedNodes=optNodesIds_a)
rest_conn_b.rebalance(otpNodes=[node.id for node in rest_conn_b.node_statuses()],\
ejectedNodes=optNodesIds_b)
self.assertTrue(rest_conn_a.monitorRebalance(),
msg="rebalance operation failed after adding node on cluster 1")
self.assertTrue(rest_conn_b.monitorRebalance(),
msg="rebalance operation failed after adding node on cluster 2")
# Wait for loading threads to finish
for lt in load_thread_list:
lt.join()
self.log.info("All loading threads finished")
# Verify replication
self.assertTrue(XDCRBaseTest.verify_replicated_data(rest_conn_b,
self._buckets[0],
kvstore,
self._poll_sleep,
self._poll_timeout),
"Verification of replicated data failed")
self.assertTrue(XDCRBaseTest.verify_replicated_revs(rest_conn_a,
rest_conn_b,
self._buckets[0],
self._poll_sleep,
self._poll_timeout),
"Verification of replicated revisions failed")
class DoctorXDCR():
def __init__(self, src_cluster, dest_cluster):
self.src_cluster = src_cluster
self.dest_cluster = dest_cluster
self.src_master = self.src_cluster.master
self.dest_master = self.dest_cluster.master
self.src_rest = RestConnection(self.src_master)
self.dest_rest = RestConnection(self.dest_master)
# List of XDCReplication objects for src_cluster
self.replications = []
def create_remote_ref(self, name="remote"):
'''
Create remote cluster reference from src_cluster to remote_cluster
:return: None
'''
self.src_rest.add_remote_cluster(self.dest_master.ip,
self.dest_master.port,
self.dest_master.rest_username,
self.dest_master.rest_password, name)
def create_replication(self, remote_ref, src_bkt, dest_bkt):
'''
Create replication in the src cluster to replicate data to dest cluster
'''
repl_id = self.src_rest.start_replication("continuous", src_bkt,
remote_ref, "xmem", dest_bkt)
if repl_id is not None:
self.replications.append(
XDCReplication(remote_ref, self.src_cluster, self.dest_cluster,
src_bkt, dest_bkt, repl_id))
def set_xdcr_param(self, src_bkt, dest_bkt, param, value):
"""Set a replication setting to a value
"""
self.src_rest.set_xdcr_param(src_bkt, dest_bkt, param, value)
def drop_replication_by_id(self, repl_id):
'''
Drop replication in the src cluster
'''
self.src_rest.stop_replication(repl_id)
for repl in self.replications:
if repl_id == repl.repl_id:
self.replications.remove(repl)
def drop_all_replications(self):
self.src_rest.remove_all_replications()
self.replications = []
def delete_remote_ref(self, name):
self.src_rest.remove_remote_cluster(name)
def monitor_replications(self, duration=0, print_duration=600):
st_time = time.time()
update_time = time.time()
if duration == 0:
while not self.stop_run:
if st_time + print_duration < time.time():
for current_repl in self.src_rest.get_replications():
for repl in self.replications:
if current_repl['id'] == repl.repl_id:
print(
"Source bucket {0} doc count = {1}\nDest bucket {2} doc count = {3}"
.format(
repl.src_bkt,
self.bucket_util.
get_bucket_current_item_count(
repl.src_cluster,
repl.src_bkt), repl.dest_bkt,
self.bucket_util.
get_bucket_current_item_count(
repl.dest_cluster, repl.src_bkt)))
else:
while st_time + duration > time.time():
if update_time + print_duration < time.time():
update_time = time.time()