def test_load_regexp(self): template = '{{ "mutated" : 0, "age": {0}, "first_name": "{1}" }}' gen_load = DocumentGenerator('load_by_id_test', template, list(range(5)), ['james', 'john'], start=0, end=self.num_items) gen_load2 = DocumentGenerator('cbtransfer', template, list(range(5)), ['james', 'john'], start=0, end=self.num_items) verify_gen = copy.deepcopy(gen_load2) for bucket in self.buckets: bucket.kvs[2] = KVStore() self.cluster.load_gen_docs(self.server_origin, bucket.name, gen_load, self.buckets[0].kvs[2], "create", exp=0, flag=0, only_store_hash=True, batch_size=1000, compression=self.sdk_compression) self.cluster.load_gen_docs(self.server_origin, bucket.name, gen_load2, self.buckets[0].kvs[1], "create", exp=0, flag=0, only_store_hash=True, batch_size=1000, compression=self.sdk_compression) transfer_source = 'http://%s:%s' % (self.server_origin.ip, self.server_origin.port) transfer_destination = 'http://%s:%s' % (self.server_recovery.ip, self.server_recovery.port) self._run_cbtransfer_all_buckets(transfer_source, transfer_destination, "-k cbtransfer-[0-9]+ -u {0} -p {1}"\ .format(self.server_recovery.rest_username, self.server_recovery.rest_password)) self._wait_curr_items_all_buckets() self._verify_data_all_buckets(verify_gen)
def __init__(self, new_params={}): self.name = new_params.get(Bucket.name, "default") self.bucketType = new_params.get(Bucket.bucketType, Bucket.bucket_type.MEMBASE) self.replicaNumber = new_params.get(Bucket.replicaNumber, 0) self.replicaServers = new_params.get(Bucket.replicaServers, []) self.ramQuotaMB = new_params.get(Bucket.ramQuotaMB, 100) self.kvs = {1: KVStore()} if self.bucketType == Bucket.bucket_type.EPHEMERAL: self.evictionPolicy = new_params.get( Bucket.evictionPolicy, Bucket.bucket_eviction_policy.NO_EVICTION) else: self.evictionPolicy = new_params.get( Bucket.evictionPolicy, Bucket.bucket_eviction_policy.VALUE_ONLY) self.replicaIndex = new_params.get(Bucket.replicaIndex, 0) self.priority = new_params.get(Bucket.priority, None) self.threadsNumber = new_params.get(Bucket.threadsNumber, 3) self.uuid = None self.lww = new_params.get(Bucket.lww, False) self.maxTTL = new_params.get(Bucket.maxTTL, 0) self.flushEnabled = new_params.get(Bucket.flushEnabled, 1) self.compressionMode = new_params.get( Bucket.compressionMode, Bucket.bucket_compression_mode.PASSIVE) self.nodes = None self.stats = None self.servers = [] self.vbuckets = [] self.forward_map = []
def incremental_rebalance_out_with_mutation_and_deletion(self): gen_2 = BlobGenerator('rebalance-del', 'rebalance-del-', self.value_size, start=self.num_items // 2 + 2000, end=self.num_items) batch_size = 1000 for i in reversed(list(range(self.num_servers))[1:]): # don't use batch for rebalance out 2-1 nodes for bucket in self.buckets: bucket.kvs[2] = KVStore() tasks = [ self.cluster.async_rebalance(self.servers[:i], [], [self.servers[i]]) ] tasks += self._async_load_all_buckets(self.master, self.gen_update, "update", 0, kv_store=1, batch_size=batch_size, timeout_secs=60) tasks += self._async_load_all_buckets(self.master, gen_2, "delete", 0, kv_store=2, batch_size=batch_size, timeout_secs=60) for task in tasks: task.result() self.sleep(5) self._load_all_buckets(self.master, gen_2, "create", 0) self.verify_cluster_stats(self.servers[:i]) self.verify_unacked_bytes_all_buckets()
def online_upgrade_rebalance_in_out(self): self._install(self.servers[:self.nodes_init]) self.operations(self.servers[:self.nodes_init]) seqno_expected = 1 seqno_comparator = '>=' if not self.initial_version.startswith("1.") and self.input.param( 'check_seqno', True): self.check_seqno(seqno_expected) seqno_comparator = '==' if self.ddocs_num: self.create_ddocs_and_views() self.sleep(self.sleep_time, "Pre-setup of old version is done. Wait for upgrade") self.initial_version = self.upgrade_versions[0] self.product = 'couchbase-server' self._install(self.servers[self.nodes_init:self.num_servers]) self.sleep(self.sleep_time, "Installation of new version is done. Wait for rebalance") if self.during_ops: for opn in self.during_ops: getattr(self, opn)() if self.wait_expire: self.sleep(self.expire_time) for bucket in self.buckets: bucket.kvs[1] = KVStore() self.online_upgrade() self.sleep(self.sleep_time) if self.input.param('reboot_cluster', False): self.warm_up_node(self.servers[self.nodes_init:self.num_servers]) self.verification(self.servers[self.nodes_init:self.num_servers]) if self.input.param('check_seqno', True): self.check_seqno(seqno_expected, comparator=seqno_comparator)
def restoreAndVerify(self, bucket_names, kvs_before, expected_error=None): for bucket in self.buckets: bucket.kvs[1] = kvs_before[bucket.name] del kvs_before gc.collect() errors, outputs = self.shell.restore_backupFile( self.couchbase_login_info, self.backup_location, bucket_names) errors.extend(outputs) error_found = False if expected_error: for line in errors: if line.find(expected_error) != -1: error_found = True break self.assertTrue(error_found, "Expected error not found: %s" % expected_error) self._wait_for_stats_all_buckets(self.servers[:self.num_servers]) if expected_error: for bucket in self.buckets: bucket.kvs[1] = KVStore() self.verify_results(self.master) self._verify_stats_all_buckets(self.servers[:self.num_servers])
def __init__(self, new_params={}): self.name = new_params.get(Bucket.name, "default") self.type = new_params.get(Bucket.type, Bucket.bucket_type.MEMBASE) self.replicas = new_params.get(Bucket.replicas, 0) self.size = new_params.get(Bucket.size, 100) self.kvs = {1:KVStore()} self.eviction_policy = new_params.get(Bucket.eviction_policy, Bucket.bucket_eviction_policy.VALUE_ONLY) self.replicaIndex = new_params.get(Bucket.replicaIndex, 0) self.priority = new_params.get(Bucket.priority, None) self.uuid = None self.lww = new_params.get(Bucket.lww, False) self.maxTTL = new_params.get(Bucket.maxTTL, None) self.flush_enabled = new_params.get(Bucket.flush_enabled, 1) self.compressionMode = new_params.get(Bucket.compressionMode, Bucket.bucket_compression_mode.PASSIVE) self.nodes = None self.stats = None self.servers = [] self.vbuckets = [] self.forward_map = []
def setUp(self): self.times_teardown_called = 1 super(BackupBaseTest, self).setUp() self.shell = RemoteMachineShellConnection(self.master) info = self.shell.extract_remote_info() self.os = info.type.lower() self.value_size = self.input.param("value_size", 256) self.expire_time = self.input.param("expire_time", 60) self.number_of_backups = self.input.param("number_of_backups", 1) self.backup_type = self.input.param("backup_type", None) self.item_flag = self.input.param("item_flag", 0) self.couchbase_login_info = "%s:%s" % ( self.input.membase_settings.rest_username, self.input.membase_settings.rest_password) self.backup_location = self.input.param("backup_location", "/tmp/backup") self.command_options = self.input.param("command_options", '') if self.command_options is not '': self.command_options = self.command_options.split(";") self.doc_ops = self.input.param("doc_ops", None) if self.doc_ops is not None: self.doc_ops = self.doc_ops.split(";") self.backup_x_options = self.input.param("backup_x_options", None) if self.backup_x_options is not None: temp = self.backup_x_options.split(";") temp_x_options = {} for element in temp: temp_array = element.split() temp_x_options[temp_array[0]] = temp_array[1] self.backup_x_options = temp_x_options self.restore_x_options = self.input.param("restore_x_options", None) if self.restore_x_options is not None: temp = self.restore_x_options.split(";") temp_x_options = {} for element in temp: temp_array = element.split() temp_x_options[temp_array[0]] = temp_array[1] self.restore_x_options = temp_x_options servers_in = [self.servers[i + 1] for i in range(self.num_servers - 1)] for bucket in self.buckets: bucket.kvs[2] = KVStore() self.cluster.rebalance(self.servers[:1], servers_in, [])
def setUp(self): self.times_teardown_called = 1 super(BackupBaseTest, self).setUp() self.shell = RemoteMachineShellConnection(self.master) self.value_size = self.input.param("value_size", 256) self.expire_time = self.input.param("expire_time", 60) self.item_flag = self.input.param("item_flag", 0) self.couchbase_login_info = "%s:%s" % (self.input.membase_settings.rest_username, self.input.membase_settings.rest_password) self.backup_location = self.input.param("backup_location", "/tmp/backup") self.command_options = self.input.param("command_options", None) if self.command_options is not None: self.command_options = self.command_options.split(";") self.doc_ops = self.input.param("doc_ops", None) if self.doc_ops is not None: self.doc_ops = self.doc_ops.split(";") servers_in = [self.servers[i+1] for i in range(self.num_servers-1)] for bucket in self.buckets: bucket.kvs[2]= KVStore() self.cluster.rebalance(self.servers[:1], servers_in, [])
def __init__(self, bucket_size='', name="", authType="sasl", saslPassword="", num_replicas=0, port=11211, master_id=None, type='', eviction_policy="valueOnly", bucket_priority=None, uuid="", lww=False, storageBackend="couchstore"): self.name = name self.port = port self.type = type self.nodes = None self.stats = None self.servers = [] self.vbuckets = [] self.forward_map = [] self.numReplicas = num_replicas self.saslPassword = saslPassword self.authType = "" self.bucket_size = bucket_size self.kvs = {1:KVStore()} self.authType = authType self.master_id = master_id self.eviction_policy = eviction_policy self.bucket_priority = bucket_priority self.uuid = uuid self.lww = lww self.storageBackend = storageBackend