def monitor_bucket(self, bucket_name, staging_UUID): # To monitor the replication logger.debug( "Monitoring the replication for bucket {} ".format(bucket_name)) kwargs = { ENV_VAR_KEY: { 'password': self.staged_source.parameters.xdcr_admin_password } } command = CommandFactory.monitor_replication( source_username=self.staged_source.parameters.xdcr_admin, source_hostname=self.source_config.couchbase_src_host, source_port=self.source_config.couchbase_src_port, bucket_name=bucket_name, uuid=staging_UUID) stdout, stderr, exit_code = utilities.execute_bash( self.connection, command, **kwargs) logger.debug("stdout: {}".format(stdout)) content = json.loads(stdout) pending_docs = self._get_last_value_of_node_stats( content["nodeStats"].values()[0]) while pending_docs != 0: logger.debug( "Documents pending for replication: {}".format(pending_docs)) helper_lib.sleepForSecond(30) stdout, stderr, exit_code = utilities.execute_bash( self.connection, command, **kwargs) content = json.loads(stdout) pending_docs = self._get_last_value_of_node_stats( content["nodeStats"].values()[0]) else: logger.debug( "Replication for bucket {} completed".format(bucket_name))
def _do_provision(provision_process, snapshot): bucket_list_and_size = snapshot.bucket_list if not bucket_list_and_size: raise FailedToReadBucketDataFromSnapshot("Snapshot Data is empty.") else: logger.debug( "snapshot bucket data is: {}".format(bucket_list_and_size)) for item in bucket_list_and_size.split(':'): logger.debug("Creating bucket is: {}".format(item)) # try: bucket_name = item.split(',')[0] bkt_size_mb = int(item.split(',')[1].strip()) // 1024 // 1024 provision_process.bucket_create(bucket_name, bkt_size_mb) helper_lib.sleepForSecond(2) # getting config directory path directory = provision_process.get_config_directory() # making directory and changing permission to 755. provision_process.make_directory(directory) # This file path is being used to store the bucket information coming in snapshot config_file_path = provision_process.get_config_file_path() content = "BUCKET_LIST=" + _find_bucket_name_from_snapshot(snapshot) # Adding bucket list in config file path .config file, inside .delphix folder helper_lib.write_file(provision_process.connection, content, config_file_path)
def bucket_remove(self, bucket_name): logger.debug("Removing bucket: {} ".format(bucket_name)) self.bucket_edit(bucket_name, flush_value=1) self.bucket_flush(bucket_name) self.bucket_edit(bucket_name, flush_value=0) self.bucket_delete(bucket_name) helper_lib.sleepForSecond(2)
def check_index_build(self): # env = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}} # cmd = CommandFactory.check_index_build(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path),self.connection.environment.host.name, self.parameters.couchbase_port, self.parameters.couchbase_admin) # logger.debug("check_index_build cmd: {}".format(cmd)) # set timeout to 12 hours end_time = time.time() + 3660 * 12 tobuild = 1 #break the loop either end_time is exceeding from 1 minute or server is successfully started while time.time() < end_time and tobuild <> 0: command_output, std_err, exit_code = self.run_couchbase_command( couchbase_command='check_index_build', base_path=helper_lib.get_base_directory_of_given_path( self.repository.cb_shell_path)) #command_output, std_err, exit_code = utilities.execute_bash(self.connection, command_name=cmd, **env) logger.debug("command_output is {}".format(command_output)) logger.debug("std_err is {}".format(std_err)) logger.debug("exit_code is {}".format(exit_code)) try: command_output_dict = json.loads(command_output) logger.debug("dict {}".format(command_output_dict)) tobuild = command_output_dict['results'][0]['unbuilt'] logger.debug("to_build is {}".format(tobuild)) helper_lib.sleepForSecond(30) # waiting for 1 second except Exception as e: logger.debug(str(e))
def stop_couchbase(self): """ stop the couchbase service""" try: logger.debug("Stopping couchbase services") self.run_couchbase_command('stop_couchbase') end_time = time.time() + 60 server_status = Status.ACTIVE while time.time() < end_time and server_status == Status.ACTIVE: helper_lib.sleepForSecond(1) # waiting for 1 second server_status = self.status() # fetching status logger.debug("Leaving stop loop") if server_status == Status.ACTIVE: logger.debug("Have failed to stop couchbase server") raise CouchbaseServicesError( "Have failed to stop couchbase server") except CouchbaseServicesError as err: logger.debug("Error: {}".format(err)) raise err except Exception as err: logger.debug("Exception Error: {}".format(err)) if self.status() == Status.INACTIVE: logger.debug( "Seems like couchbase service is not running. {}".format( err.message)) else: raise CouchbaseServicesError(err.message)
def start_couchbase(self, provision=False, no_wait=False): """ start the couchbase service""" logger.debug("Starting couchbase services") self.run_couchbase_command('start_couchbase') server_status = Status.INACTIVE helper_lib.sleepForSecond(10) if no_wait: logger.debug("no wait - leaving start procedure") return #Waiting for one minute to start the server # for prox to investigate end_time = time.time() + 3660 #break the loop either end_time is exceeding from 1 minute or server is successfully started while time.time() < end_time and server_status == Status.INACTIVE: helper_lib.sleepForSecond(1) # waiting for 1 second server_status = self.status(provision) # fetching status logger.debug("server status {}".format(server_status)) # if the server is not running even in 60 seconds, then stop the further execution if server_status == Status.INACTIVE: raise CouchbaseServicesError( "Have failed to start couchbase server")
def _bucket_common_task(provision_process, bucket_list): for bkt in bucket_list: bkt = bkt.strip() logger.debug("Deletion of bucket {} started".format(bkt)) provision_process.bucket_edit(bkt, flush_value=1) provision_process.bucket_flush(bkt) provision_process.bucket_edit(bkt, flush_value=0) provision_process.bucket_delete(bkt) helper_lib.sleepForSecond(2)
def _do_provision(provision_process, snapshot): bucket_list_and_size = snapshot.bucket_list if not bucket_list_and_size: raise FailedToReadBucketDataFromSnapshot("Snapshot Data is empty.") else: logger.debug( "snapshot bucket data is: {}".format(bucket_list_and_size)) bucket_list_and_size = json.loads(bucket_list_and_size) try: bucket_list = provision_process.bucket_list() bucket_list = helper_lib.filter_bucket_name_from_output(bucket_list) logger.debug(bucket_list) except Exception as err: logger.debug("Failed to get bucket list. Error is " + err.message) renamed_folders = [] for item in bucket_list_and_size: logger.debug("Checking bucket: {}".format(item)) # try: bucket_name = item['name'] bkt_size = item['ram'] bkt_type = item['bucketType'] bkt_compression = item['compressionMode'] bkt_size_mb = helper_lib.get_bucket_size_in_MB(0, bkt_size) if bucket_name not in bucket_list: # a new bucket needs to be created logger.debug("Creating bucket: {}".format(bucket_name)) provision_process.bucket_create(bucket_name, bkt_size_mb, bkt_type, bkt_compression) helper_lib.sleepForSecond(2) else: logger.debug( "Bucket {} exist - no need to rename directory".format( bucket_name)) provision_process.stop_couchbase() for item in helper_lib.filter_bucket_name_from_output( bucket_list_and_size): logger.debug("Checking bucket: {}".format(item)) bucket_name = item.split(',')[0] logger.debug("restoring folders") provision_process.move_bucket(bucket_name, 'restore') provision_process.start_couchbase() # getting config directory path directory = provision_process.get_config_directory() # making directory and changing permission to 755. provision_process.make_directory(directory) # This file path is being used to store the bucket information coming in snapshot config_file_path = provision_process.get_config_file_path()
def _bucket_modify_task(provision_process, bucket_list, snapshot_bucket_list_and_size): for bkt in bucket_list: bkt = bkt.strip() logger.debug("Modification of bucket {} started".format(bkt)) ramquotasize = _find_bucket_size_byname(bkt, snapshot_bucket_list_and_size) logger.debug("Update bucket {} with ramsize {}MB".format( bkt, ramquotasize)) provision_process.bucket_edit_ramquota(bkt, _ramsize=ramquotasize) helper_lib.sleepForSecond(2)
def start_node_bootstrap(self): logger.debug("start start_node_bootstrap") self.start_couchbase(no_wait=True) end_time = time.time() + 3660 server_status = Status.INACTIVE #break the loop either end_time is exceeding from 1 minute or server is successfully started while time.time() < end_time and server_status <> Status.ACTIVE: helper_lib.sleepForSecond(1) # waiting for 1 second server_status = self.staging_bootstrap_status() # fetching status logger.debug("server status {}".format(server_status))
def xdcr_setup(self): logger.debug("Started XDCR set up ...") kwargs = { ENV_VAR_KEY: { 'source_password': self.parameters.xdcr_admin_password, 'password': self.parameters.couchbase_admin_password } } env = _XDCrMixin.generate_environment_map(self) cmd = CommandFactory.xdcr_setup( cluster_name=self.parameters.stg_cluster_name, **env) stdout, stderr, exit_code = utilities.execute_bash( self.connection, cmd, **kwargs) helper_lib.sleepForSecond(3)
def cluster_init(self): # Cluster initialization logger.debug("Cluster Initialization started") fts_service = self.parameters.fts_service #analytics_service = self.parameters.analytics_service eventing_service = self.parameters.eventing_service cluster_name = self._get_cluster_name() kwargs = { ENV_VAR_KEY: { 'password': self.parameters.couchbase_admin_password } } additional_service = "query" if fts_service == True: additional_service = additional_service + ",fts" # if analytics_service: # additional_service = additional_service + ",analytics" if eventing_service == True: additional_service = additional_service + ",eventing" logger.debug("additional services : {}".format(additional_service)) lambda_expr = lambda output: bool( re.search(ALREADY_CLUSTER_INIT, output)) env = _ClusterMixin.generate_environment_map(self) env['additional_services'] = additional_service cmd = CommandFactory.cluster_init(cluster_name=cluster_name, **env) logger.debug("Cluster init: {}".format(cmd)) stdout, stderr, exit_code = utilities.execute_bash( self.connection, command_name=cmd, callback_func=lambda_expr, **kwargs) if re.search(r"ERROR", str(stdout)): if re.search(r"ERROR: Cluster is already initialized", stdout): logger.debug( "Performing cluster setting as cluster is already initialized" ) self.cluster_setting() else: logger.error("Cluster init failed. Throwing exception") raise Exception(stdout) else: logger.debug("Cluster init succeeded") # here we should wait for indexer to start sleepForSecond(10) return [stdout, stderr, exit_code]
def start_couchbase(self): """ start the couchbase service""" logger.debug("Starting couchbase services") command = CommandFactory.start_couchbase(self.repository.cb_install_path) utilities.execute_bash(self.connection, command) server_status = Status.INACTIVE #Waiting for one minute to start the server end_time = time.time() + 60 #break the loop either end_time is exceeding from 1 minute or server is successfully started while time.time() < end_time and server_status == Status.INACTIVE: helper_lib.sleepForSecond(1) # waiting for 1 second server_status = self.status() # fetching status # if the server is not running even in 60 seconds, then stop the further execution if server_status == Status.INACTIVE: raise CouchbaseServicesError("Have failed to start couchbase server")
def stop_couchbase(self): """ stop the couchbase service""" try: logger.debug("Stopping couchbase services") command = CommandFactory.stop_couchbase(self.repository.cb_install_path) utilities.execute_bash(self.connection, command) end_time = time.time() + 60 server_status = Status.ACTIVE while time.time() < end_time and server_status == Status.ACTIVE: helper_lib.sleepForSecond(1) # waiting for 1 second server_status = self.status() # fetching status if server_status == Status.ACTIVE: raise CouchbaseServicesError("Have failed to stop couchbase server") except CouchbaseServicesError as err: raise err except Exception as err: if self.status() == Status.INACTIVE: logger.debug("Seems like couchbase service is not running. {}".format(err.message)) else: raise CouchbaseServicesError(err.message)
def bucket_create(self, bucket_name, ram_size, bucket_type, bucket_compression): logger.debug("Creating bucket: {} ".format(bucket_name)) # To create the bucket with given ram size self.__validate_bucket_name(bucket_name) if ram_size is None: logger.debug( "Needed ramsize for bucket_create. Currently it is: {}".format( ram_size)) return if bucket_type == 'membase': # API return different type bucket_type = 'couchbase' if bucket_compression is not None: bucket_compression = '--compression-mode {}'.format( bucket_compression) else: bucket_compression = '' policy = self.parameters.bucket_eviction_policy env = _BucketMixin.generate_environment_map(self) command = CommandFactory.bucket_create( bucket_name=bucket_name, ramsize=ram_size, evictionpolicy=policy, bucket_type=bucket_type, bucket_compression=bucket_compression, **env) kwargs = { ENV_VAR_KEY: { 'password': self.parameters.couchbase_admin_password } } logger.debug("create bucket {}".format(command)) output, error, exit_code = utilities.execute_bash( self.connection, command, **kwargs) logger.debug("create bucket output: {} {} {}".format( output, error, exit_code)) helper_lib.sleepForSecond(2)
def configure_cluster(couchbase_obj): # configure Couchbase cluster logger.debug("Checking cluster config") if couchbase_obj.check_config(): logger.debug("cluster config found - restoring") couchbase_obj.stop_couchbase() couchbase_obj.restore_config() couchbase_obj.start_couchbase() else: logger.debug("cluster config not found - preparing node") # no config in delphix directory # initial cluster setup couchbase_obj.stop_couchbase() # we can't use normal monitor as server is not configured yet couchbase_obj.start_couchbase(no_wait=True) end_time = time.time() + 3660 server_status = Status.INACTIVE #break the loop either end_time is exceeding from 1 hour or server is successfully started while time.time() < end_time and server_status<>Status.ACTIVE: helper_lib.sleepForSecond(1) # waiting for 1 second server_status = couchbase_obj.staging_bootstrap_status() # fetching status logger.debug("server status {}".format(server_status)) # check if cluster not configured and raise an issue if couchbase_obj.check_cluster_notconfigured(): logger.debug("Node not configured - creating a new cluster") couchbase_obj.node_init() couchbase_obj.cluster_init() logger.debug("Cluster configured") else: logger.debug("Node configured but no configuration in Delphix - ???????") if couchbase_obj.check_cluster_configured(): logger.debug("Configured with staging user/password and alive so not a problem - continue") else: logger.debug("Cluster configured but not with user/password given in Delphix potentially another cluster") raise UserError("Cluster configured but not with user/password given in Delphix potentially another cluster")
def bucket_create(self, bucket_name, ram_size=0): logger.debug("Creating bucket: {} ".format(bucket_name)) # To create the bucket with given ram size self.__validate_bucket_name(bucket_name) if ram_size is None: logger.debug( "Needed ramsize for bucket_create. Currently it is: {}".format( ram_size)) return policy = self.parameters.bucket_eviction_policy env = _BucketMixin.generate_environment_map(self) command = CommandFactory.bucket_create(bucket_name=bucket_name, ramsize=ram_size, evictionpolicy=policy, **env) kwargs = { ENV_VAR_KEY: { 'password': self.parameters.couchbase_admin_password } } output, error, exit_code = utilities.execute_bash( self.connection, command, **kwargs) helper_lib.sleepForSecond(2)
def xdcr_replicate(self, src, tgt): try: logger.debug("Started XDCR replication for bucket {}".format(src)) kwargs = { ENV_VAR_KEY: { 'source_password': self.parameters.xdcr_admin_password } } env = _XDCrMixin.generate_environment_map(self) cmd = CommandFactory.xdcr_replicate( source_bucket_name=src, target_bucket_name=tgt, cluster_name=self.parameters.stg_cluster_name, **env) stdout, stderr, exit_code = utilities.execute_bash( self.connection, cmd, **kwargs) if exit_code != 0: logger.debug("XDCR replication create failed") raise Exception(stdout) logger.debug("{} : XDCR replication create succeeded".format(tgt)) helper_lib.sleepForSecond(2) except Exception as e: logger.debug("XDCR error {}".format(e.message))
def resync_cbbkpmgr(staged_source, repository, source_config, input_parameters): dsource_type = input_parameters.d_source_type dsource_name = source_config.pretty_name couchbase_host = input_parameters.couchbase_host bucket_size = staged_source.parameters.bucket_size rx_connection = staged_source.staged_connection resync_process = CouchbaseOperation( Resource.ObjectBuilder.set_staged_source(staged_source).set_repository( repository).set_source_config(source_config).build()) linking.check_for_concurrent(resync_process, dsource_type, dsource_name, couchbase_host) # validate if this works as well for backup linking.configure_cluster(resync_process) logger.debug("Finding source and staging bucket list") bucket_details_source = resync_process.source_bucket_list_offline() bucket_details_staged = helper_lib.filter_bucket_name_from_output( resync_process.bucket_list()) buckets_toprocess = linking.buckets_precreation(resync_process, bucket_details_source, bucket_details_staged) csv_bucket_list = ",".join(buckets_toprocess) logger.debug("Started CB backup manager") helper_lib.sleepForSecond(30) resync_process.cb_backup_full(csv_bucket_list) helper_lib.sleepForSecond(30) linking.build_indexes(resync_process) logger.info("Stopping Couchbase") resync_process.stop_couchbase() resync_process.save_config('parent')
def vdb_reconfigure(virtual_source, repository, source_config, snapshot): # delete all buckets # calll configure logger.debug("In vdb_reconfigure...") provision_process = CouchbaseOperation( Resource.ObjectBuilder.set_virtual_source(virtual_source). set_repository(repository).set_source_config(source_config).build()) provision_process.stop_couchbase() if provision_process.parameters.node_list is not None and len( provision_process.parameters.node_list) > 0: multinode = True server_count = len(provision_process.parameters.node_list) + 1 else: multinode = False nodeno = 1 provision_process.restore_config(what='current', nodeno=nodeno) provision_process.start_couchbase(no_wait=multinode) if provision_process.parameters.node_list is not None and len( provision_process.parameters.node_list) > 0: for node in provision_process.parameters.node_list: nodeno = nodeno + 1 logger.debug("+++++++++++++++++++++++++++") logger.debug(node) logger.debug(nodeno) logger.debug("+++++++++++++++++++++++++++") addnode = CouchbaseOperation( Resource.ObjectBuilder.set_virtual_source( virtual_source).set_repository( repository).set_source_config(source_config).build(), make_nonprimary_connection(provision_process.connection, node['environment'], node['environmentUser'])) addnode.stop_couchbase() addnode.restore_config(what='current', nodeno=nodeno) addnode.start_couchbase(no_wait=multinode) logger.debug("reconfigure for multinode: {}".format(multinode)) if multinode == True: active_servers = {} logger.debug("wait for nodes") logger.debug("server count: {} active servers: {}".format( server_count, sum(active_servers.values()))) end_time = time.time() + 3660 #break the loop either end_time is exceeding from 1 minute or server is successfully started while time.time() < end_time and sum( active_servers.values()) <> server_count: logger.debug("server count 2: {} active servers: {}".format( server_count, sum(active_servers.values()))) nodeno = 1 helper_lib.sleepForSecond(1) # waiting for 1 second server_status = provision_process.status() # fetching status logger.debug("server status {}".format(server_status)) if server_status == Status.ACTIVE: active_servers[nodeno] = 1 for node in provision_process.parameters.node_list: nodeno = nodeno + 1 logger.debug("+++++++++++++++++++++++++++") logger.debug(node) logger.debug(nodeno) logger.debug("+++++++++++++++++++++++++++") addnode = CouchbaseOperation( Resource.ObjectBuilder.set_virtual_source(virtual_source). set_repository(repository).set_source_config( source_config).build(), make_nonprimary_connection(provision_process.connection, node['environment'], node['environmentUser'])) server_status = addnode.status() # fetching status logger.debug("server status {}".format(server_status)) if server_status == Status.ACTIVE: active_servers[nodeno] = 1 return _source_config(virtual_source, repository, source_config, snapshot)
def addnode(self, nodeno, node_def): logger.debug("start addnode") self.delete_config() self.start_node_bootstrap() self.node_init(nodeno) helper_lib.sleepForSecond(10) services = ['data', 'index', 'query'] if "fts_service" in node_def and node_def["fts_service"] == True: services.append('fts') if "eventing_service" in node_def and node_def[ "eventing_service"] == True: services.append('eventing') if "analytics_service" in node_def and node_def[ "analytics_service"] == True: services.append('analytics') logger.debug("services to add: {}".format(services)) # hostip_command = CommandFactory.get_ip_of_hostname() # logger.debug("host ip command: {}".format(hostip_command)) # host_ip_output, std_err, exit_code = utilities.execute_bash(self.connection, hostip_command) # logger.debug("host ip Output {} ".format(host_ip_output)) logger.debug("node host name / IP: {}".format(node_def["node_addr"])) resolve_name_command = CommandFactory.resolve_name( hostname=node_def["node_addr"]) logger.debug( "resolve_name_command command: {}".format(resolve_name_command)) resolve_name_output, std_err, exit_code = utilities.execute_bash( self.connection, resolve_name_command) logger.debug( "resolve_name_command Output {} ".format(resolve_name_output)) command_output, std_err, exit_code = self.run_couchbase_command( couchbase_command='server_add', hostname=self.connection.environment.host.name, newhost=resolve_name_output, services=','.join(services)) logger.debug("Add node Output {} stderr: {} exit_code: {} ".format( command_output, std_err, exit_code)) if exit_code != 0: logger.debug("Adding node error") raise UserError( "Problem with adding node", "Check an output and fix problem before retrying to provision a VDB", "stdout: {} stderr:{}".format(command_output, std_err)) command_output, std_err, exit_code = self.run_couchbase_command( couchbase_command='rebalance', hostname=self.connection.environment.host.name) logger.debug("Rebalance Output {} stderr: {} exit_code: {} ".format( command_output, std_err, exit_code)) if exit_code != 0: logger.debug("Rebalancing error") raise UserError( "Problem with rebalancing cluster", "Check an output and fix problem before retrying to provision a VDB", "stdout: {} stderr:{}".format(command_output, std_err))