def test_Transfer(self): shell = RemoteMachineShellConnection(self.master) gen_update = BlobGenerator('testdata', 'testdata-', self.value_size, end=100) self._load_all_buckets(self.master, gen_update, "create", 0, 1, 0, True, batch_size=20000, pause_secs=5, timeout_secs=180) self._wait_for_stats_all_buckets(self.servers[:self.num_servers]) source = "http://" + self.master.ip + ":8091" info = shell.extract_remote_info() path = '/tmp/backup' #if info.type.lower() == "windows": # path = '/cygdrive/c' + path shell.delete_files(path) create_dir = "mkdir " + path shell.execute_command(create_dir) options = "-b default " + " -u " + self.master.rest_username + " -p " + self.master.rest_password shell.execute_cbtransfer(source, path, options) expectedResults = { "peername": self.master.ip, "sockname": self.master.ip + ":11210", "source": "memcached", "user": "******", 'bucket': 'default' } self.checkConfig(self.eventID, self.master, expectedResults)
def test_Transfer(self): shell = RemoteMachineShellConnection(self.master) gen_update = BlobGenerator('testdata', 'testdata-', self.value_size, end=100) self._load_all_buckets(self.master, gen_update, "create", 0, 1, 0, True, batch_size=20000, pause_secs=5, timeout_secs=180) self._wait_for_stats_all_buckets(self.servers[:self.num_servers]) source = "http://" + self.master.ip + ":8091" info = shell.extract_remote_info() path = '/tmp/backup' #if info.type.lower() == "windows": # path = '/cygdrive/c' + path shell.delete_files(path) create_dir = "mkdir " + path shell.execute_command(create_dir) options = "-b default " + " -u " + self.master.rest_username + " -p " + self.master.rest_password shell.execute_cbtransfer(source, path, options) expectedResults = {"peername":self.master.ip, "sockname":self.master.ip + ":11210", "source":"memcached", "user":"******", 'bucket':'default'} self.checkConfig(self.eventID, self.master, expectedResults)
class RecoveryUseTransferTests(TransferBaseTest): def setUp(self): self.times_teardown_called = 1 super(RecoveryUseTransferTests, self).setUp() self.server_origin = self.servers[0] self.server_recovery = self.servers[1] self.shell = RemoteMachineShellConnection(self.server_origin) info = self.shell.extract_remote_info() self.os = info.type.lower() def tearDown(self): if not self.input.param("skip_cleanup", True): if self.times_teardown_called > 1 : if self.os == 'windows': self.shell.delete_files("/cygdrive/c%s" % (self.backup_location)) else: self.shell.delete_files(self.backup_location) self.shell.disconnect() del self.buckets if self.input.param("skip_cleanup", True): if self.case_number > 1 or self.times_teardown_called > 1: if self.os == 'windows': self.shell.delete_files("/cygdrive/c%s" % (self.backup_location)) else: self.shell.delete_files(self.backup_location) self.shell.disconnect() del self.buckets self.times_teardown_called += 1 super(RecoveryUseTransferTests, self).tearDown() def recover_to_cbserver(self): """Recover data with 2.0 couchstore files to a 2.0 online server We load a number of items to one node first and then do some mutation on these items. Later we use cbtranfer to transfer the couchstore files we have on this node to a new node. We verify the data by comparison between the items in KVStore and items in the new node.""" self.load_data() kvs_before = {} bucket_names = [] for bucket in self.buckets: kvs_before[bucket.name] = bucket.kvs[1] bucket_names.append(bucket.name) if self.default_bucket: self.cluster.create_default_bucket(self.server_recovery, self.bucket_size, self.num_replicas) self.buckets.append(Bucket(name="default", authType="sasl", saslPassword="", num_replicas=self.num_replicas, bucket_size=self.bucket_size)) self._create_sasl_buckets(self.server_recovery, self.sasl_buckets) self._create_standard_buckets(self.server_recovery, self.standard_buckets) for bucket in self.buckets: bucket.kvs[1] = kvs_before[bucket.name] transfer_source = "couchstore-files://%s" % (COUCHBASE_DATA_PATH) transfer_destination = "http://%s@%s:%s -b %s -B %s -v -v -v" % (self.couchbase_login_info, self.server_recovery.ip, self.server_recovery.port, bucket.name, bucket.name) self.shell.execute_cbtransfer(transfer_source, transfer_destination) del kvs_before time.sleep(self.expire_time + 1) shell_server_recovery = RemoteMachineShellConnection(self.server_recovery) for bucket in self.buckets: shell_server_recovery.execute_cbepctl(bucket, "", "set flush_param", "exp_pager_stime", 5) shell_server_recovery.disconnect() time.sleep(30) self._wait_for_stats_all_buckets([self.server_recovery]) self._verify_all_buckets(self.server_recovery, 1, self.wait_timeout * 50, self.max_verify, True, 1) self._verify_stats_all_buckets([self.server_recovery]) def recover_to_backupdir(self): """Recover data with 2.0 couchstore files to a 2.0 backup diretory We load a number of items to a node first and then do some mutataion on these items. Later we use cbtransfer to transfer the couchstore files we have on this node to a backup directory. We use cbrestore to restore these backup files to the same node for verification.""" self.load_data() kvs_before = {} bucket_names = [] self.shell.delete_files(self.backup_location) self.shell.create_directory(self.backup_location) for bucket in self.buckets: kvs_before[bucket.name] = bucket.kvs[1] bucket_names.append(bucket.name) transfer_source = "-v -v -v couchstore-files://%s" % (COUCHBASE_DATA_PATH) transfer_destination = self.backup_location self.shell.execute_cbtransfer(transfer_source, transfer_destination) self._all_buckets_delete(self.server_origin) if self.default_bucket: self.cluster.create_default_bucket(self.server_origin, self.bucket_size, self.num_replicas) self.buckets.append(Bucket(name="default", authType="sasl", saslPassword="", num_replicas=self.num_replicas, bucket_size=self.bucket_size)) self._create_sasl_buckets(self.server_origin, self.sasl_buckets) self._create_standard_buckets(self.server_origin, self.standard_buckets) for bucket in self.buckets: bucket.kvs[1] = kvs_before[bucket.name] del kvs_before self.shell.restore_backupFile(self.couchbase_login_info, self.backup_location, bucket_names) time.sleep(self.expire_time + 1) for bucket in self.buckets: self.shell.execute_cbepctl(bucket, "", "set flush_param", "exp_pager_stime", 5) time.sleep(30) self._wait_for_stats_all_buckets([self.server_origin]) self._verify_all_buckets(self.server_origin, 1, self.wait_timeout * 50, self.max_verify, True, 1) self._verify_stats_all_buckets([self.server_origin]) def load_data(self): gen_load = BlobGenerator('nosql', 'nosql-', self.value_size, end=self.num_items) gen_update = BlobGenerator('nosql', 'nosql-', self.value_size, end=(self.num_items / 2 - 1)) gen_expire = BlobGenerator('nosql', 'nosql-', self.value_size, start=self.num_items / 2, end=(self.num_items * 3 / 4 - 1)) gen_delete = BlobGenerator('nosql', 'nosql-', self.value_size, start=self.num_items * 3 / 4, end=self.num_items) self._load_all_buckets(self.server_origin, gen_load, "create", 0, 1, self.item_flag, True, batch_size=20000, pause_secs=5, timeout_secs=180) if(self.doc_ops is not None): if("update" in self.doc_ops): self._load_all_buckets(self.server_origin, gen_update, "update", 0, 1, self.item_flag, True, batch_size=20000, pause_secs=5, timeout_secs=180) if("delete" in self.doc_ops): self._load_all_buckets(self.server_origin, gen_delete, "delete", 0, 1, self.item_flag, True, batch_size=20000, pause_secs=5, timeout_secs=180) if("expire" in self.doc_ops): self._load_all_buckets(self.server_origin, gen_expire, "update", self.expire_time, 1, self.item_flag, True, batch_size=20000, pause_secs=5, timeout_secs=180) self._wait_for_stats_all_buckets([self.server_origin]) time.sleep(30)
class RecoveryUseTransferTests(TransferBaseTest): def setUp(self): self.times_teardown_called = 1 super(RecoveryUseTransferTests, self).setUp() self.server_origin = self.servers[0] self.server_recovery = self.servers[1] self.shell = RemoteMachineShellConnection(self.server_origin) info = self.shell.extract_remote_info() self.os = info.type.lower() def tearDown(self): super(RecoveryUseTransferTests, self).tearDown() if not self.input.param("skip_cleanup", True): if times_tear_down_called > 1: if self.os == 'windows': self.shell.delete_files("/cygdrive/c%s" % (self.backup_location)) else: self.shell.delete_files(self.backup_location) self.shell.disconnect() del self.buckets if self.input.param("skip_cleanup", True): if self.case_number > 1 or self.times_teardown_called > 1: if self.os == 'windows': self.shell.delete_files("/cygdrive/c%s" % (self.backup_location)) else: self.shell.delete_files(self.backup_location) self.shell.disconnect() del self.buckets self.times_teardown_called += 1 def recover_to_cbserver(self): """Recover data with 2.0 couchstore files to a 2.0 online server We load a number of items to one node first and then do some mutation on these items. Later we use cbtranfer to transfer the couchstore files we have on this node to a new node. We verify the data by comparison between the items in KVStore and items in the new node.""" self.load_data() kvs_before = {} bucket_names = [] for bucket in self.buckets: kvs_before[bucket.name] = bucket.kvs[1] bucket_names.append(bucket.name) if self.default_bucket: self.cluster.create_default_bucket(self.server_recovery, self.bucket_size, self.num_replicas) self.buckets.append( Bucket(name="default", authType="sasl", saslPassword="", num_replicas=self.num_replicas, bucket_size=self.bucket_size)) self._create_sasl_buckets(self.server_recovery, self.sasl_buckets) self._create_standard_buckets(self.server_recovery, self.standard_buckets) for bucket in self.buckets: bucket.kvs[1] = kvs_before[bucket.name] transfer_source = "couchstore-files://%s" % (COUCHBASE_DATA_PATH) transfer_destination = "http://%s@%s:%s -b %s -B %s -v -v -v" % ( self.couchbase_login_info, self.server_recovery.ip, self.server_recovery.port, bucket.name, bucket.name) self.shell.execute_cbtransfer(transfer_source, transfer_destination) del kvs_before time.sleep(self.expire_time + 1) shell_server_recovery = RemoteMachineShellConnection( self.server_recovery) for bucket in self.buckets: shell_server_recovery.execute_cbepctl(bucket, "", "set flush_param", "exp_pager_stime", 5) shell_server_recovery.disconnect() time.sleep(30) self._wait_for_stats_all_buckets([self.server_recovery]) self._verify_all_buckets(self.server_recovery, 1, self.wait_timeout * 50, None, True) self._verify_stats_all_buckets([self.server_recovery]) def recover_to_backupdir(self): """Recover data with 2.0 couchstore files to a 2.0 backup diretory We load a number of items to a node first and then do some mutataion on these items. Later we use cbtransfer to transfer the couchstore files we have on this node to a backup directory. We use cbrestore to restore these backup files to the same node for verification.""" self.load_data() kvs_before = {} bucket_names = [] self.shell.delete_files(self.backup_location) self.shell.create_directory(self.backup_location) for bucket in self.buckets: kvs_before[bucket.name] = bucket.kvs[1] bucket_names.append(bucket.name) transfer_source = "-v -v -v couchstore-files://%s" % ( COUCHBASE_DATA_PATH) transfer_destination = self.backup_location self.shell.execute_cbtransfer(transfer_source, transfer_destination) self._all_buckets_delete(self.server_origin) if self.default_bucket: self.cluster.create_default_bucket(self.server_origin, self.bucket_size, self.num_replicas) self.buckets.append( Bucket(name="default", authType="sasl", saslPassword="", num_replicas=self.num_replicas, bucket_size=self.bucket_size)) self._create_sasl_buckets(self.server_origin, self.sasl_buckets) self._create_standard_buckets(self.server_origin, self.standard_buckets) for bucket in self.buckets: bucket.kvs[1] = kvs_before[bucket.name] del kvs_before self.shell.restore_backupFile(self.couchbase_login_info, self.backup_location, bucket_names) time.sleep(self.expire_time + 1) for bucket in self.buckets: self.shell.execute_cbepctl(bucket, "", "set flush_param", "exp_pager_stime", 5) time.sleep(30) self._wait_for_stats_all_buckets([self.server_origin]) self._verify_all_buckets(self.server_origin, 1, self.wait_timeout * 50, None, True) self._verify_stats_all_buckets([self.server_origin]) def load_data(self): gen_load = BlobGenerator('nosql', 'nosql-', self.value_size, end=self.num_items) gen_update = BlobGenerator('nosql', 'nosql-', self.value_size, end=(self.num_items / 2 - 1)) gen_expire = BlobGenerator('nosql', 'nosql-', self.value_size, start=self.num_items / 2, end=(self.num_items * 3 / 4 - 1)) gen_delete = BlobGenerator('nosql', 'nosql-', self.value_size, start=self.num_items * 3 / 4, end=self.num_items) self._load_all_buckets(self.server_origin, gen_load, "create", 0, 1, self.item_flag, True, batch_size=20000, pause_secs=5, timeout_secs=180) if (self.doc_ops is not None): if ("update" in self.doc_ops): self._load_all_buckets(self.server_origin, gen_update, "update", 0, 1, self.item_flag, True, batch_size=20000, pause_secs=5, timeout_secs=180) if ("delete" in self.doc_ops): self._load_all_buckets(self.server_origin, gen_delete, "delete", 0, 1, self.item_flag, True, batch_size=20000, pause_secs=5, timeout_secs=180) if ("expire" in self.doc_ops): self._load_all_buckets(self.server_origin, gen_expire, "update", self.expire_time, 1, self.item_flag, True, batch_size=20000, pause_secs=5, timeout_secs=180) self._wait_for_stats_all_buckets([self.server_origin]) time.sleep(30)