def check_server_is_used(connection, path):
ret = Status.INACTIVE
output, stderr, exit_code = utilities.execute_bash(connection,
CommandFactory.mount())
if exit_code != 0:
logger.error("mount retured error")
logger.error("stdout: {} stderr: {} exit_code: {}".format(
output.encode('utf-8'), stderr.encode('utf-8'), exit_code))
raise UserError("Problem with reading mounted file systems",
"Ask OS admin to check mount", stderr)
else:
# parse a mount output to find another Delphix mount points
fs_re = re.compile(r'(\S*)\son\s(\S*)\stype\s(\S*)')
for i in output.split("\n"):
match = re.search(fs_re, i)
if match is not None:
groups = match.groups()
if groups[2] == 'nfs':
if path == groups[1]:
# this is our mount point - skip it
ret = Status.ACTIVE
continue
if "domain0" in groups[0] and "timeflow" in groups[0]:
# this is a delphix mount point but it's not ours
# raise an exception
raise UserError(
"Another database (VDB or staging) is using this server.",
"Disable another one to provision or enable this one",
"{} {}".format(groups[0], groups[1]))
return ret
def start_mysql(installPath, baseDir, mountPath, port, serverId, connection):
#This function will stop a running MySQL Database.
logger.debug("Commence > start_mysql()")
port_stat = get_port_status(port, connection)
logger.debug("Port Status > " + port_stat.name)
environment_vars = {}
if (port_stat == Status.INACTIVE):
logger.debug("DB is not running. Starting the MySQL DB")
start_cmd = get_start_cmd(installPath, baseDir, mountPath, port,
serverId)
logger.debug("Startup Command: {}".format(start_cmd))
result = libs.run_bash(connection,
start_cmd,
environment_vars,
check=True)
output = result.stdout.strip()
error = result.stderr.strip()
exit_code = result.exit_code
if exit_code != 0:
logger.debug("There was an error trying to start the DB : " +
error)
raise UserError(constants.ERR_START_MSG,
constants.ERR_START_ACTION,
"ExitCode:{} \n {}".format(exit_code, error))
else:
logger.debug("Output: " + output)
time.sleep(30)
if (Status.ACTIVE == get_port_status(port, connection)):
logger.debug("DB Started Successfully")
else:
logger.debug("There was an issue starting the DB")
else:
logger.debug(" DB is already Running.")
def create_backup_options(logsync, dbs, logger):
"""
Creates the backup options string to be used in restore.sh script for source backup
Args:
logsync (string) : Are we enabling replication?
dbs (string): Which databases are we backing up?
Returns:
backup_options (string): Complete backup options string
"""
backup_options="--skip-lock-tables --single-transaction --flush-logs --hex-blob --no-tablespaces"
try:
if logsync=="true":
backup_options+=" --master-data=2"
if dbs is None or dbs=="ALL":
backup_options+=" -A"
else:
backup_options+=" --databases {}".format(dbs)
logger.debug("Returning backup_options:"+backup_options)
return backup_options
except Exception as err:
user_error= UserError(
"An exception occurred in create_backup_options() while creating DBOptions string for backup",
"Please verify the databases list provided",
"ExitCode:{} \n {}".format("11","Input values are / logsync:{} and dbs:{}".format(logsync,dbs))
)
def clean_stale_mountpoint(connection, path):
umount_std, umount_stderr, umount_exit_code = utilities.execute_bash(
connection,
CommandFactory.unmount_file_system(mount_path=path, options='-lf'))
if umount_exit_code != 0:
logger.error("Problem with cleaning mount path")
logger.error("stderr {}".format(umount_stderr))
raise UserError("Problem with cleaning mount path",
"Ask OS admin to check mount points", umount_stderr)
def process_exit_codes(exit_code,operation,std_err=None):
"""
Processes exit code and returns a UserError
Args:
exit_code: Exit code from run_bash
operation: The operation that was performed.
Returns:
UserError
"""
err_out=const.ERR_GENERAL_OUT
if std_err:
err_out= remove_nonascii(std_err)
if exit_code == 3: # Unable to start MySQL
err_msg=const.ERR_START_MSG
err_action=const.ERR_START_ACTION
elif exit_code == 4: # Could not find my.cnf
err_msg=const.ERR_MYCNF_MSG
err_action=const.ERR_MYCNF_ACTION
elif exit_code == 5: # Unable to change password
err_msg=const.ERR_PWD_MSG
err_action=const.ERR_PWD_ACTION
elif exit_code == 6: # Unable restore backup
err_msg=const.ERR_RESTORE_MSG
err_action=const.ERR_RESTORE_ACTION
elif exit_code == 7: # Unable to connect after backup
err_msg=const.ERR_CONNECT_MSG
err_action=const.ERR_CONNECT_ACTION
elif exit_code == 8: # Delphix Backup failed.
err_msg=const.ERR_BACKUP_MSG
err_action=const.ERR_BACKUP_ACTION
elif exit_code == 9: # Delphix Backup failed.
err_msg=const.ERR_CUST_BACKUP_MSG
err_action=const.ERR_CUST_BACKUP_ACTION
elif exit_code == 10: # Invalid Binary Path
err_msg=const.ERR_INVALID_BINARY_MSG
err_action=const.ERR_INVALID_BINARY_ACTION
elif exit_code == 11: # Connection Test Failure / Backup INgestion
err_msg=const.ERR_BI_CONNECT_MSG
err_action=const.ERR_BI_CONNECT_ACTION
elif exit_code == 12: # User creation issue/ Backup Ingestion
err_msg=const.ERR_BI_USERCREATE_MSG
err_action=const.ERR_BI_USERCREATE_ACTION
elif exit_code == 2:
err_msg=const.ERR_GENERAL_MSG
err_action=const.ERR_GENERAL_ACTION
user_error= UserError(
err_msg,
err_action,
"ExitCode:{} \n {}".format(exit_code,err_out)
)
return user_error
def resync_xdcr(staged_source, repository, source_config, input_parameters):
logger.debug("START resync_xdcr")
dsource_type = input_parameters.d_source_type
dsource_name = source_config.pretty_name
bucket_size = staged_source.parameters.bucket_size
rx_connection = staged_source.staged_connection
resync_process = CouchbaseOperation(
Resource.ObjectBuilder.set_staged_source(staged_source).set_repository(
repository).set_source_config(source_config).build())
couchbase_host = input_parameters.couchbase_host
linking.check_for_concurrent(resync_process, dsource_type, dsource_name,
couchbase_host)
linking.configure_cluster(resync_process)
# common steps for both XDCR & CB back up
bucket_details_source = resync_process.source_bucket_list()
bucket_details_staged = resync_process.bucket_list()
buckets_toprocess = linking.buckets_precreation(resync_process,
bucket_details_source,
bucket_details_staged)
# run this for all buckets
resync_process.setup_replication()
logger.debug("Finding staging_uuid & cluster_name on staging")
staging_uuid = resync_process.get_replication_uuid()
if staging_uuid is None:
logger.debug("Can't find a replication UUID after setting it up")
raise UserError("Can't find a replication UUID after setting it up")
# bucket_details_staged = resync_process.bucket_list()
# logger.debug("Filtering bucket name from output")
# filter_bucket_list = helper_lib.filter_bucket_name_from_output(bucket_details_staged)
for bkt in buckets_toprocess:
resync_process.monitor_bucket(bkt, staging_uuid)
linking.build_indexes(resync_process)
logger.info("Stopping Couchbase")
resync_process.stop_couchbase()
resync_process.save_config('parent')
def cb_backup_full(self, csv_bucket):
logger.debug("Starting Restore via Backup file...")
logger.debug("csv_bucket_list: {}".format(csv_bucket))
skip = '--disable-analytics'
if self.parameters.fts_service != True:
skip = skip + ' {} {} '.format('--disable-ft-indexes',
'--disable-ft-alias')
if self.parameters.eventing_service != True:
skip = skip + ' {} '.format('--disable-eventing')
logger.debug("skip backup is set to: {}".format(skip))
# kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}}
# env = _CBBackupMixin.generate_environment_map(self)
# cmd = CommandFactory.cb_backup_full(backup_location=self.parameters.couchbase_bak_loc,
# csv_bucket_list=csv_bucket,
# backup_repo=self.parameters.couchbase_bak_repo,
# need_sudo=self.need_sudo, uid=self.uid,
# skip=skip,
# **env)
# logger.debug("Backup restore: {}".format(cmd))
# utilities.execute_bash(self.connection, cmd, **kwargs)
stdout, stderr, exit_code = self.run_couchbase_command(
couchbase_command='cb_backup_full',
backup_location=self.parameters.couchbase_bak_loc,
csv_bucket_list=csv_bucket,
backup_repo=self.parameters.couchbase_bak_repo,
skip=skip,
base_path=helper_lib.get_base_directory_of_given_path(
self.repository.cb_shell_path))
if exit_code != 0:
raise UserError(
"Problem with restoring backup using cbbackupmgr",
"Check if repo and all privileges are correct",
"stdout: {}, stderr: {}, exit_code: {}".format(
stdout, stderr, exit_code))
def configure_cluster(couchbase_obj):
# configure Couchbase cluster
logger.debug("Checking cluster config")
if couchbase_obj.check_config():
logger.debug("cluster config found - restoring")
couchbase_obj.stop_couchbase()
couchbase_obj.restore_config()
couchbase_obj.start_couchbase()
else:
logger.debug("cluster config not found - preparing node")
# no config in delphix directory
# initial cluster setup
couchbase_obj.stop_couchbase()
# we can't use normal monitor as server is not configured yet
couchbase_obj.start_couchbase(no_wait=True)
end_time = time.time() + 3660
server_status = Status.INACTIVE
#break the loop either end_time is exceeding from 1 hour or server is successfully started
while time.time() < end_time and server_status<>Status.ACTIVE:
helper_lib.sleepForSecond(1) # waiting for 1 second
server_status = couchbase_obj.staging_bootstrap_status() # fetching status
logger.debug("server status {}".format(server_status))
# check if cluster not configured and raise an issue
if couchbase_obj.check_cluster_notconfigured():
logger.debug("Node not configured - creating a new cluster")
couchbase_obj.node_init()
couchbase_obj.cluster_init()
logger.debug("Cluster configured")
else:
logger.debug("Node configured but no configuration in Delphix - ???????")
if couchbase_obj.check_cluster_configured():
logger.debug("Configured with staging user/password and alive so not a problem - continue")
else:
logger.debug("Cluster configured but not with user/password given in Delphix potentially another cluster")
raise UserError("Cluster configured but not with user/password given in Delphix potentially another cluster")
def get_replication_uuid(self):
# False for string
logger.debug("Finding the replication uuid through host name")
is_ip_or_string = False
kwargs = {ENV_VAR_KEY: {}}
cluster_name = self.parameters.stg_cluster_name
stdout, stderr, exit_code = self.run_couchbase_command(
'get_replication_uuid',
source_hostname=self.source_config.couchbase_src_host,
source_port=self.source_config.couchbase_src_port,
source_username=self.parameters.xdcr_admin,
source_password=self.parameters.xdcr_admin_password)
if exit_code != 0 or stdout is None or stdout == "":
logger.debug("No Replication ID identified")
return None
try:
logger.debug("xdcr remote references : {}".format(stdout))
stg_hostname = self.connection.environment.host.name
logger.debug("Environment hostname {}".format(stg_hostname))
# it can have more than single IP address
host_ips = self.get_ip()
# conver output into variables
clusters = {}
l = stdout.split("\n")
while l:
line = l.pop(0)
g = re.match(r"\s*cluster name:\s(\S*)", line)
if g:
xdrc_cluster_name = g.group(1)
uuid = re.match(r"\s*uuid:\s(\S*)", l.pop(0)).group(1)
hostname = re.match(r"\s*host name:\s(\S*):(\d*)",
l.pop(0)).group(1)
user_name = l.pop(0)
uri = l.pop(0)
clusters[xdrc_cluster_name.lower()] = {
"hostname": hostname,
"uuid": uuid
}
# check if a cluster name is really connected to staging - just in case
if cluster_name.lower() in clusters:
logger.debug("Cluster {} found in xdrc-setup output".format(
cluster_name))
# check if hostname returned from source match hostname or IP's of staging server
logger.debug(stg_hostname)
logger.debug(clusters[cluster_name.lower()]["hostname"])
if stg_hostname == clusters[cluster_name.lower()]["hostname"]:
# hostname matched
logger.debug(
"Cluster {} hostname {} is matching staging server hostname"
.format(cluster_name, stg_hostname))
uuid = clusters[cluster_name.lower()]["uuid"]
else:
# check for IP's
logger.debug("Checking for IP match")
logger.debug(clusters[cluster_name.lower()])
if clusters[cluster_name.lower()]["hostname"] in host_ips:
# ip matched
logger.debug(
"Cluster {} IP {} is matching staging server IPs {}"
.format(cluster_name,
clusters[cluster_name.lower()]["hostname"],
host_ips))
uuid = clusters[cluster_name.lower()]["uuid"]
else:
logger.debug(
"Can't confirm that xdrc-setup is matching staging"
)
raise UserError(
"XDRC Remote cluster {} on the source server is not pointed to staging server"
.format(cluster_name),
"Please check and delete remote cluster definition",
clusters[cluster_name.lower()])
else:
logger.debug(
"Cluster {} configuration not found in XDCR of source".
format(cluster_name))
return None
logger.debug("uuid for {} cluster : {}".format(uuid, cluster_name))
return uuid
except UserError:
raise
except Exception as err:
logger.warn("Error identified: {} ".format(err.message))
logger.warn("UUID is None. Not able to find any cluster")
return None
def save_config(self, what, nodeno=1):
# TODO
# Error handling
logger.debug("start save_config")
targetdir = self.get_config_directory()
target_config_filename = os.path.join(targetdir,
"config.dat_{}".format(nodeno))
target_local_filename = os.path.join(targetdir,
"local.ini_{}".format(nodeno))
target_encryption_filename = os.path.join(
targetdir, "encrypted_data_keys_{}".format(nodeno))
if nodeno == 1:
ip_file = "{}/../var/lib/couchbase/ip".format(
helper_lib.get_base_directory_of_given_path(
self.repository.cb_shell_path))
target_ip_filename = os.path.join(targetdir,
"ip_{}".format(nodeno))
else:
ip_file = "{}/../var/lib/couchbase/ip_start".format(
helper_lib.get_base_directory_of_given_path(
self.repository.cb_shell_path))
target_ip_filename = os.path.join(targetdir,
"ip_start_{}".format(nodeno))
filename = "{}/../var/lib/couchbase/config/config.dat".format(
helper_lib.get_base_directory_of_given_path(
self.repository.cb_shell_path))
command_output, std_err, exit_code = self.run_os_command(
os_command='os_cp',
srcname=filename,
trgname=target_config_filename)
logger.debug(
"save config.dat cp - exit_code: {} stdout: {} std_err: {}".format(
exit_code, command_output, std_err))
if exit_code != 0:
raise UserError(
"Error saving configuration file: config.dat",
"Check sudo or user privileges to read Couchbase config.dat file",
std_err)
# encryption data keys may not exist on Community edition
filename = "{}/../var/lib/couchbase/config/encrypted_data_keys".format(
helper_lib.get_base_directory_of_given_path(
self.repository.cb_shell_path))
check_encrypted_data_keys, check_ip_file_err, exit_code = self.run_os_command(
os_command='check_file', file_path=filename)
if exit_code == 0 and "Found" in check_encrypted_data_keys:
# cmd = CommandFactory.os_cp(filename, target_encryption_filename, self.need_sudo, self.uid)
# logger.debug("save encrypted_data_keys cp: {}".format(cmd))
# command_output, std_err, exit_code = utilities.execute_bash(self.connection, command_name=cmd)
command_output, std_err, exit_code = self.run_os_command(
os_command='os_cp',
srcname=filename,
trgname=target_encryption_filename)
logger.debug(
"save encrypted_data_keys.dat cp - exit_code: {} stdout: {} std_err: {}"
.format(exit_code, command_output, std_err))
if exit_code != 0:
raise UserError(
"Error saving configuration file: encrypted_data_keys",
"Check sudo or user privileges to read Couchbase encrypted_data_keys file",
std_err)
command_output, std_err, exit_code = self.run_os_command(
os_command='os_cp', srcname=ip_file, trgname=target_ip_filename)
logger.debug("save {} - exit_code: {} stdout: {} std_err: {}".format(
ip_file, exit_code, command_output, std_err))
if exit_code != 0:
raise UserError(
"Error saving configuration file: {}".format(ip_file),
"Check sudo or user privileges to read Couchbase {} file".
format(ip_file), std_err)
filename = "{}/../etc/couchdb/local.ini".format(
helper_lib.get_base_directory_of_given_path(
self.repository.cb_shell_path))
command_output, std_err, exit_code = self.run_os_command(
os_command='os_cp',
srcname=filename,
trgname=target_local_filename)
logger.debug(
"save local.ini cp - exit_code: {} stdout: {} std_err: {}".format(
exit_code, command_output, std_err))
if exit_code != 0:
raise UserError(
"Error saving configuration file: local.ini",
"Check sudo or user privileges to read Couchbase local.ini file",
std_err)
def addnode(self, nodeno, node_def):
logger.debug("start addnode")
self.delete_config()
self.start_node_bootstrap()
self.node_init(nodeno)
helper_lib.sleepForSecond(10)
services = ['data', 'index', 'query']
if "fts_service" in node_def and node_def["fts_service"] == True:
services.append('fts')
if "eventing_service" in node_def and node_def[
"eventing_service"] == True:
services.append('eventing')
if "analytics_service" in node_def and node_def[
"analytics_service"] == True:
services.append('analytics')
logger.debug("services to add: {}".format(services))
# hostip_command = CommandFactory.get_ip_of_hostname()
# logger.debug("host ip command: {}".format(hostip_command))
# host_ip_output, std_err, exit_code = utilities.execute_bash(self.connection, hostip_command)
# logger.debug("host ip Output {} ".format(host_ip_output))
logger.debug("node host name / IP: {}".format(node_def["node_addr"]))
resolve_name_command = CommandFactory.resolve_name(
hostname=node_def["node_addr"])
logger.debug(
"resolve_name_command command: {}".format(resolve_name_command))
resolve_name_output, std_err, exit_code = utilities.execute_bash(
self.connection, resolve_name_command)
logger.debug(
"resolve_name_command Output {} ".format(resolve_name_output))
command_output, std_err, exit_code = self.run_couchbase_command(
couchbase_command='server_add',
hostname=self.connection.environment.host.name,
newhost=resolve_name_output,
services=','.join(services))
logger.debug("Add node Output {} stderr: {} exit_code: {} ".format(
command_output, std_err, exit_code))
if exit_code != 0:
logger.debug("Adding node error")
raise UserError(
"Problem with adding node",
"Check an output and fix problem before retrying to provision a VDB",
"stdout: {} stderr:{}".format(command_output, std_err))
command_output, std_err, exit_code = self.run_couchbase_command(
couchbase_command='rebalance',
hostname=self.connection.environment.host.name)
logger.debug("Rebalance Output {} stderr: {} exit_code: {} ".format(
command_output, std_err, exit_code))
if exit_code != 0:
logger.debug("Rebalancing error")
raise UserError(
"Problem with rebalancing cluster",
"Check an output and fix problem before retrying to provision a VDB",
"stdout: {} stderr:{}".format(command_output, std_err))
def __init__(self, message, action, error_string):
super(UserConvertibleException, self).__init__(message)
# Create the UserError now in case someone asks for it later
self.user_error = UserError(message, action, error_string)
def staged_post_snapshot(repository, source_config, staged_source,
optional_snapshot_parameters):
common.add_debug_heading_block("Start Staged Post Snapshot")
helpers._record_hook("staging post snapshot",
staged_source.staged_connection)
helpers._set_running(staged_source.staged_connection, staged_source.guid)
staged_source.mongo_install_path = repository.mongo_install_path
staged_source.mongo_shell_path = repository.mongo_shell_path
if staged_source.parameters.d_source_type == "extendedcluster":
staged_source.parameters.mongo_db_user = staged_source.parameters.src_db_user
staged_source.parameters.mongo_db_password = staged_source.parameters.src_db_password
snapshot_possible_file_path = "{}/{}".format(
staged_source.parameters.mount_path,
".delphix/snapshot_not_possible.txt")
cmd = "test -f {} && cat {} || echo 'file does not exist.'".format(
snapshot_possible_file_path, snapshot_possible_file_path)
res = common.execute_bash_cmd(staged_source.staged_connection, cmd, {})
if res != "file does not exist.":
cmd = "rm {}".format(snapshot_possible_file_path)
res_rm = common.execute_bash_cmd(staged_source.staged_connection,
cmd, {})
errorMsg = "Cannot perform Snapshot as the host {} is in state {}".format(
res.split(" ")[1].split(" : ")[1],
res.split(" ")[2].split(" : ")[1])
logger.info(errorMsg)
raise UserError(errorMsg)
elif staged_source.parameters.d_source_type == "stagingpush":
staged_source.parameters.mongo_db_user = staged_source.parameters.src_db_user
staged_source.parameters.mongo_db_password = staged_source.parameters.src_db_password
if staged_source.parameters.d_source_type == "nonshardedsource":
staged_source.parameters.mongos_port = staged_source.parameters.start_portpool
logger.info("In Post snapshot...")
logger.debug("len shard_backupfiles: {}".format(
len(staged_source.parameters.shard_backupfiles)))
script_content = 'echo "$(uname):$(uname -p):$(cat /etc/*-release)"'
res = common.execute_bash_cmd(staged_source.staged_connection,
script_content, {})
output = res.strip().split(":")
logger.debug("output = {}".format(output))
if staged_source.parameters.d_source_type in [
"shardedsource", "offlinemongodump", "nonshardedsource"
]:
cmd = "cat {}".format(staged_source.parameters.backup_metadata_file)
lastbackup_datetime = common.execute_bash_cmd(
staged_source.staged_connection, cmd, {})
dateTimeObj = datetime.strptime(lastbackup_datetime, "%m%d%Y_%H%M%S")
else:
dateTimeObj = datetime.now()
timestampStr = dateTimeObj.strftime("%m%d%Y-%H%M%S.%f")
snapshot = SnapshotDefinition(validate=False)
snapshot.toolkit_version = _version.Version
snapshot.timestamp = timestampStr
snapshot.architecture = output[1]
snapshot.os_type = output[0]
snapshot.os_version = re.search('.*"VERSION="([\d\.]+).*',
output[2]).group(1)
snapshot.mongo_version = repository.version
snapshot.delphix_mount = staged_source.parameters.mount_path
snapshot.storage_engine = staged_source.parameters.storage_engine
snapshot.user_auth_mode = staged_source.parameters.user_auth_mode
snapshot.keyfile_path = staged_source.parameters.keyfile_path
snapshot.replica_set = "N/A"
snapshot.journal_interval = staged_source.parameters.journal_interval
snapshot.oplog_size = staged_source.parameters.oplog_size
snapshot.d_source_type = staged_source.parameters.d_source_type
snapshot.append_db_path = "N/A"
snapshot.mongo_db_user = staged_source.parameters.mongo_db_user
snapshot.mongo_db_password = staged_source.parameters.mongo_db_password
snapshot.source_sharded = staged_source.parameters.source_sharded
snapshot.shard_count = (len(staged_source.parameters.shard_backupfiles))
snapshot.source_encrypted = staged_source.parameters.source_encrypted
snapshot.cluster_auth_mode = staged_source.parameters.cluster_auth_mode
snapshot.encryption_method = staged_source.parameters.encryption_method
snapshot.encryption_keyfile = ".delphix/.dlpx_enckeyfile"
snapshot.kmip_params = staged_source.parameters.kmip_params
#logger.debug("Staging Post Snapshot - Unfreeze IO")
#common.fsync_unlock_sharded_mongo(staged_source, 'Staging')
#logger.debug("Staging Post Snapshot - Unfreeze IO - done")
mask_snap = copy.deepcopy(snapshot)
mask_snap.mongo_db_password = '******'
logger.debug("snapshot schema: {}".format(mask_snap))
common.add_debug_heading_block("End Staged Post Snapshot")
# ADD start Balancer
return snapshot
def staged_pre_snapshot(repository, source_config, staged_source,
optional_snapshot_parameters):
common.add_debug_heading_block("Start Staged Pre Snapshot")
helpers._record_hook("staging pre snapshot",
staged_source.staged_connection)
staged_source.mongo_install_path = repository.mongo_install_path
staged_source.mongo_shell_path = repository.mongo_shell_path
logger.info(
"optional_snapshot_parameters={}".format(optional_snapshot_parameters))
if optional_snapshot_parameters is not None and optional_snapshot_parameters.resync:
common.add_debug_heading_block("Start Staged Pre Snapshot Resync")
if staged_source.parameters.d_source_type == "stagingpush":
cmd = "(ls {}/.delphix/MongoOps_Automation_DSOURCE_RESYNC.cfg >> /dev/null 2>&1 && echo yes) || echo no".format(
staged_source.parameters.mount_path)
result = common.execute_bash_cmd(staged_source.staged_connection,
cmd, {})
if result == "yes":
logger.error(
"dSource Re-Syncronization is not a valid option for {} ingestion type on host: {}"
.format(staged_source.parameters.d_source_type,
staged_source.parameters.mongo_host))
raise UserError(
"dSource Re-Syncronization is not a valid option for {} ingestion type on host: {}"
.format(staged_source.parameters.d_source_type,
staged_source.parameters.mongo_host),
'\nPlease remove / cleanup the dSource manually in case of re-syncronizing dSource'
)
cmd = "(ls {}/.delphix/DSOURCE_RESYNC.cfg >> /dev/null 2>&1 && echo yes) || echo no".format(
staged_source.parameters.mount_path)
res = common.execute_bash_cmd(staged_source.staged_connection, cmd, {})
if res == "yes":
logger.info(
"Its resync operation on dSource as File {}/.delphix/DSOURCE_RESYNC.cfg exists."
.format(staged_source.parameters.mount_path))
linked.stg_cleanup_pre_snapsync(staged_source, repository, None)
else:
logger.info(
"Its new dSource as File {}/.delphix/DSOURCE_RESYNC.cfg does not exists."
.format(staged_source.parameters.mount_path))
if staged_source.parameters.d_source_type == "shardedsource":
common.setup_dataset(staged_source, 'Staging', None,
"shardedsource")
elif staged_source.parameters.d_source_type == "nonshardedsource":
staged_source.parameters.mongos_port = staged_source.parameters.start_portpool
common.setup_dataset(staged_source, 'Staging', None,
"nonshardedsource")
elif staged_source.parameters.d_source_type == "offlinemongodump":
staged_source.parameters.mongos_port = staged_source.parameters.start_portpool
linked.setup_dataset_mongodump_offline(staged_source, 'Staging',
None, "offlinemongodump")
# Write backup information
linked.write_first_backup_timestamp(staged_source)
elif staged_source.parameters.d_source_type == "onlinemongodump":
staged_source.parameters.mongos_port = staged_source.parameters.start_portpool
linked.setup_dataset_mongodump_online(staged_source, 'Staging',
None, "onlinemongodump")
elif staged_source.parameters.d_source_type == "seed":
staged_source.parameters.mongos_port = staged_source.parameters.start_portpool
linked.setup_dataset_seed(staged_source, 'Staging', None, "seed")
elif staged_source.parameters.d_source_type == "extendedcluster":
staged_source.parameters.mongo_db_user = staged_source.parameters.src_db_user
staged_source.parameters.mongo_db_password = staged_source.parameters.src_db_password
staged_source.parameters.mongos_port = staged_source.parameters.start_portpool
linked.setup_replicaset_dsource(staged_source, 'Staging',
"extendedcluster")
elif staged_source.parameters.d_source_type == "stagingpush":
staged_source.parameters.mongo_db_user = staged_source.parameters.src_db_user
staged_source.parameters.mongo_db_password = staged_source.parameters.src_db_password
staged_source.parameters.mongos_port = staged_source.parameters.start_portpool
linked.initiate_emptyfs_for_dsource(staged_source, 'Staging',
"stagingpush")
cmd = "echo \"DO NOT DELETE THIS FILE. It is used to check if its resync or new dsource\" >> {}/.delphix/MongoOps_Automation_DSOURCE_RESYNC.cfg".format(
staged_source.parameters.mount_path)
status = common.execute_bash_cmd(staged_source.staged_connection,
cmd, {})
cmd = "echo \"DO NOT DELETE THIS FILE. It is used to check if its resync or new dsource\" >> {}/.delphix/DSOURCE_RESYNC.cfg".format(
staged_source.parameters.mount_path)
res = common.execute_bash_cmd(staged_source.staged_connection, cmd, {})
common.add_debug_heading_block("End Staged Pre Snapshot Resync")
# Pre-Snapshot
common.add_debug_space()
common.add_debug_heading_block("Pre-Snapshot")
if staged_source.parameters.d_source_type == "extendedcluster":
linked.check_pre_snapshot_possible(staged_source,
optional_snapshot_parameters)
staged_source.parameters.mongo_db_user = staged_source.parameters.src_db_user
staged_source.parameters.mongo_db_password = staged_source.parameters.src_db_password
if staged_source.parameters.d_source_type not in [
"onlinemongodump", "extendedcluster", "stagingpush", "seed"
]:
ret = linked.stg_pre_snapsync(staged_source)
else:
ret = 0
if ret == 0:
if staged_source.parameters.d_source_type == "shardedsource":
common.setup_dataset(staged_source, 'Staging', None,
"shardedsource")
elif staged_source.parameters.d_source_type == "nonshardedsource":
staged_source.parameters.mongos_port = staged_source.parameters.start_portpool
common.setup_dataset(staged_source, 'Staging', None,
"nonshardedsource")
elif staged_source.parameters.d_source_type == "offlinemongodump":
staged_source.parameters.mongos_port = staged_source.parameters.start_portpool
linked.setup_dataset_mongodump_offline(staged_source, 'Staging',
None, "offlinemongodump")
elif staged_source.parameters.d_source_type == "onlinemongodump":
staged_source.parameters.mongos_port = staged_source.parameters.start_portpool
linked.presync_mongodump_online(staged_source, 'Staging', None,
"onlinemongodump")
elif staged_source.parameters.d_source_type in ["seed", "stagingpush"]:
staged_source.parameters.mongos_port = staged_source.parameters.start_portpool
if staged_source.parameters.d_source_type not in [
"onlinemongodump", "extendedcluster", "stagingpush", "seed"
]:
# Write backup information
cmd = "cat {}".format(
staged_source.parameters.backup_metadata_file)
src_lastbackup_datetime = common.execute_bash_cmd(
staged_source.staged_connection, cmd, {})
cmd = "echo {} > {}/.delphix/.stg_lastbackup_datetime.txt".format(
src_lastbackup_datetime, staged_source.parameters.mount_path)
res = common.execute_bash_cmd(staged_source.staged_connection, cmd,
{})
#logger.debug("Staging Pre Snapshot - Freeze IO")
#common.fsync_lock_sharded_mongo(staged_source, 'Staging')
#logger.debug("Staging Pre Snapshot - Freeze IO - done")
logger.debug("End of pre snapshot")
common.add_debug_heading_block("End Staged Pre Snapshot")
logger.debug(" ")
def __init__(self, message, action, error_string):
super(UserConvertibleException, self).__init__(message)
self.user_error = UserError(message, action, error_string)