def store_keys(self, servers, buckets, backup_num, backup_validation_path, mode="memory", backup_type="backup"): """ Collects the latest keys from a cluster after a restore :param servers: servers in the cluster :param buckets: buckets in the cluster :param backup_num: current backup number :param backup_validation_path: where to store the data :param get_replica: bool to decide if replica data to be copied :param mode: where to get the items from - can be "disk" or "memory" :param backup_type: backup or restore """ data_collector = DataCollector() info, complete_map = data_collector.collect_data( servers, buckets, userId=servers[0].rest_username, password=servers[0].rest_password, perNode=False, mode=mode) for bucket in list(complete_map.keys()): file_name = "{0}-{1}-{2}.json".format(bucket, "keys", backup_num) file_path = os.path.join(backup_validation_path, file_name) keys = list(complete_map[bucket].keys()) with open(file_path, 'w') as f: json.dump({"keys": keys}, f)
def store_latest(self, servers, buckets, backup_num, backup_validation_path, get_replica=False, mode="memory", backup_type="backup"): """ Collects the latest key value pairs from a cluster after a restore :param servers: servers in the cluster :param buckets: buckets in the cluster :param backup_num: current backup number :param backup_validation_path: where to store the data :param get_replica: bool to decide if replica data to be copied :param mode: where to get the items from - can be "disk" or "memory" :param backup_type: backup or restore """ data_collector = DataCollector() info, complete_map = data_collector.collect_data(servers, buckets, userId=servers[0].rest_username, password=servers[0].rest_password, perNode=False, getReplica=get_replica, mode=mode) for bucket in complete_map.keys(): file_name = "{0}-{1}-{2}-{3}.json".format(bucket, "key_value", backup_type, backup_num) file_path = os.path.join(backup_validation_path, file_name) data = complete_map[bucket] for key in data: value = data[key] value = ",".join(value.split(',')[4:5]) data[key] = value with open(file_path, 'w') as f: json.dump(data, f)
def validate_restore(self, backup_number, backup_vbucket_seqno, restored_vbucket_seqno, compare_uuid=False, compare="==", get_replica=False, mode="memory"): """ Validates ep-engine stats and restored data :param backup_number: backup end number :param backup_vbucket_seqno: vbucket map of all backups :param restored_vbucket_seqno: vbucket map of restored cluster :param compare_uuid: bool to decide whether to compare uuid or not :param compare: comparator function :param get_replica: bool to decide whether to compare replicas or not :param mode: where to get the items from - can be "disk" or "memory" :return: status and message """ self.log.info("backup start: " + str(self.backupset.start)) self.log.info("backup end: " + str(self.backupset.end)) success_msg = "" if not self.backupset.force_updates: status, msg = self.compare_vbucket_stats(backup_vbucket_seqno[backup_number - 1], restored_vbucket_seqno, compare_uuid=compare_uuid, seqno_compare=compare) if not status: return status, msg success_msg = "{0}\n".format(msg) for bucket in self.buckets: kv_file_name = "{0}-{1}-{2}-{3}.json".format(bucket.name, "key_value", "backup", backup_number) kv_file_path = os.path.join(self.backup_validation_path, kv_file_name) backedup_kv = {} if os.path.exists(kv_file_path): try: with open(kv_file_path, 'r') as f: backedup_kv = json.load(f) except Exception, e: raise e data_collector = DataCollector() info, restored_data = data_collector.collect_data(self.restore_cluster, [bucket], userId=self.restore_cluster[0].rest_username, password=self.restore_cluster[0].rest_password, perNode=False, getReplica=get_replica, mode=mode) data = restored_data[bucket.name] for key in data: value = data[key] value = ",".join(value.split(',')[4:5]) data[key] = value is_equal, not_equal, extra, not_present = \ self.compare_dictionary(backedup_kv, data) status, msg = self.compare_dictionary_result_analyser(is_equal, not_equal, extra, not_present, "{0} Data".format(bucket.name)) if not status: return status, msg success_msg += "{0}\n".format(msg)
def setUp(self): self.input = TestInputSingleton.input self.parse_params() self.nodes_init = self.input.param("nodes_init", 3) self.log = logger.Logger.get_logger() if self.log_level: self.log.setLevel(level=self.log_level) self.use_https = True CbServer.use_https = True if not hasattr(self, 'cluster'): self.cluster = Cluster() self.buckets = [] self.collection_name = {} CbServer.rest_username = self.input.membase_settings.rest_username CbServer.rest_password = self.input.membase_settings.rest_password self.data_collector = DataCollector() self.data_analyzer = DataAnalyzer() self.result_analyzer = DataAnalysisResultAnalyzer() self.set_testrunner_client() self.pod = pod(self.input.capella["pod"]) self.tenant = tenant( self.input.capella["tenant_id"], self.input.capella["capella_user"], self.input.capella["capella_pwd"], ) self.project_id = self.input.capella["project_id"] self.tenant.project_id = self.project_id cluster_details = self.create_capella_config() cluster_id, _, _ = CapellaAPI.create_cluster(self.pod, self.tenant, cluster_details) self.cluster_id = cluster_id CbServer.cluster_id = cluster_id CapellaAPI.add_allowed_ip(self.pod, self.tenant, self.cluster_id) CapellaAPI.create_db_user(self.pod, self.tenant, self.cluster_id, self.input.membase_settings.rest_username, self.input.membase_settings.rest_password) CbServer.pod = self.pod CbServer.tenant = self.tenant CbServer.cluster_id = self.cluster_id servers = CapellaAPI.get_nodes_formatted( self.pod, self.tenant, self.cluster_id, self.input.membase_settings.rest_username, self.input.membase_settings.rest_password) self.cluster_config = ClusterConfig(self.input.membase_settings.rest_username, self.input.membase_settings.rest_password, self.create_input_servers()) self.cluster_config.update_servers(servers) if not self.skip_buckets_handle: BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self) self.bucket_base_params = {'membase': {}} shared_params = self._create_bucket_params(server=self.master, size=self.bucket_size, replicas=self.num_replicas, enable_replica_index=self.enable_replica_index, eviction_policy=self.eviction_policy, bucket_priority=None, lww=self.lww, maxttl=self.maxttl, compression_mode=self.compression_mode) membase_params = copy.deepcopy(shared_params) membase_params['bucket_type'] = 'membase' self.bucket_base_params['membase']['non_ephemeral'] = membase_params membase_ephemeral_params = copy.deepcopy(shared_params) membase_ephemeral_params['bucket_type'] = 'ephemeral' self.bucket_base_params['membase']['ephemeral'] = membase_ephemeral_params self.bucket_base_params['membase']['non_ephemeral']['size'] = self.bucket_size self.bucket_base_params['membase']['ephemeral']['size'] = self.bucket_size self._bucket_creation()
def validate_restore(self, backup_number, backup_vbucket_seqno, restored_vbucket_seqno, compare_uuid=False, compare="==", get_replica=False, mode="memory", backups_on_disk=None): """ Validates ep-engine stats and restored data :param backup_number: backup end number :param backup_vbucket_seqno: vbucket map of all backups :param restored_vbucket_seqno: vbucket map of restored cluster :param compare_uuid: bool to decide whether to compare uuid or not :param compare: comparator function :param get_replica: bool to decide whether to compare replicas or not :param mode: where to get the items from - can be "disk" or "memory" :return: status and message """ self.log.info("backup start: " + str(self.backupset.start)) self.log.info("backup end: " + str(self.backupset.end)) success_msg = "" if not self.backupset.force_updates: status, msg = self.compare_vbucket_stats( backup_vbucket_seqno[backup_number - 1], restored_vbucket_seqno, compare_uuid=compare_uuid, seqno_compare=compare, mapBucket=self.backupset.map_buckets) if not status: return status, msg success_msg = "{0}\n".format(msg) for bucket in self.buckets: kv_file_name = "{0}-{1}-{2}-{3}.json".format( bucket.name, "key_value", "backup", backup_number) kv_file_path = os.path.join(self.backup_validation_path, kv_file_name) if backup_number == self.backupset.end and backup_number == backups_on_disk: backup_files = [ f"{self.backup_validation_path}/" + f for f in listdir(self.backup_validation_path) if f"{bucket.name}-key_value-backup" in f ] backup_files.sort(key=os.path.getmtime) try: kv_file_path = backup_files[-1] except IndexError as e: if self.backupset.deleted_buckets: return True, "Deleted bucket, will not check" else: raise e backedup_kv = {} if os.path.exists(kv_file_path): try: with open(kv_file_path, 'r') as f: backedup_kv = json.load(f) except Exception as e: raise e else: raise Exception(f"{kv_file_path} is missing!") data_collector = DataCollector() info, restored_data = data_collector.collect_data( self.restore_cluster, [bucket], userId=self.restore_cluster[0].rest_username, password=self.restore_cluster[0].rest_password, perNode=False, getReplica=get_replica, mode=mode) data = restored_data[bucket.name] for key in data: value = data[key] if '"b\'' in value: value = ",".join(value.split(',')[4:8]) else: value = ",".join(value.split(',')[4:5]) value = value.replace('""', '"') if value.startswith('"b\''): value = value[3:-2] elif value.startswith("b"): value = value.split(',')[0] data[key] = value self.log.info("Compare backup data in bucket %s " % bucket.name) is_equal, not_equal, extra, not_present = \ self.compare_dictionary(backedup_kv, data) status, msg = self.compare_dictionary_result_analyser( is_equal, not_equal, extra, not_present, "{0} Data".format(bucket.name)) if not status: return status, msg success_msg += "{0}\n".format(msg) success_msg += "Data validation success" return True, success_msg