def impl(context, filetype, dir):
if dir == 'master_data_directory':
dir = master_data_dir
if filetype == 'report':
filename = '%s/gp_restore_%s.rpt' % (dir, context.backup_timestamp)
if not os.path.isfile(filename):
raise Exception('Report file %s is not present in master data directory' % filename)
elif filetype == 'status':
gparray = GpArray.initFromCatalog(dbconn.DbURL())
if dir == 'segment_data_directory':
primary_segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary()]
for seg in primary_segs:
host = seg.getSegmentHostName()
seg_data_dir = seg.getSegmentDataDirectory()
cmd = Command('check status file', "ls %s/gp_restore_status_*_%s" % (seg_data_dir, context.backup_timestamp), ctxt=REMOTE, remoteHost=host)
cmd.run(validateAfter=True)
results = cmd.get_results()
if not results.stdout.strip():
raise Exception('Status file ending with timestamp %s is not present in segment %s data directory' % (context.backup_timestamp, host))
else:
count = 0
primary_segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary()]
for seg in primary_segs:
host = seg.getSegmentHostName()
cmd = Command('check status file', "ls %s/gp_restore_status_*_%s" % (dir, context.backup_timestamp), ctxt=REMOTE, remoteHost=host)
cmd.run(validateAfter=True)
results = cmd.get_results()
if results.stdout.strip():
count += 1
else:
raise Exception('Status file not found in segment: %s' % host)
segs = len(primary_segs)
if count != segs:
raise Exception('Expected %d status file but found %d' % (segs, count))
def checkForPortAndDirectoryConflicts(self, gpArray):
"""
Check gpArray for internal consistency -- no duplicate ports or directories on the same host, for example
A detected problem causes an Exception to be raised
"""
for hostName, segmentArr in GpArray.getSegmentsByHostName(gpArray.getDbList()).iteritems():
usedPorts = {}
usedDataDirectories = {}
for segment in segmentArr:
# check for port conflict
port = segment.getSegmentPort()
dbid = segment.getSegmentDbId()
if port in usedPorts:
raise Exception(
"Segment dbid's %s and %s on host %s cannot have the same port %s." %
(dbid, usedPorts.get(port), hostName, port))
usedPorts[port] = dbid
# check for directory conflict; could improve this by reporting nicer the conflicts
path = segment.getSegmentDataDirectory()
if path in usedDataDirectories:
raise Exception(
"Segment dbid's %s and %s on host %s cannot have the same data directory '%s'." %
(dbid, usedDataDirectories.get(path), hostName, path))
usedDataDirectories[path] = dbid
def execute(self):
ValidateGpToolkit(database = self.dump_database,
master_port = self.master_port).run()
operations = []
gparray = GpArray.initFromCatalog(dbconn.DbURL(port = self.master_port), utility=True)
segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary(current_role=True)]
for seg in segs:
operations.append(RemoteOperation(ValidateSegDiskSpace(free_space_percent = self.free_space_percent,
compress = self.compress,
dump_database = self.dump_database,
include_dump_tables = self.include_dump_tables,
datadir = seg.getSegmentDataDirectory(),
segport = seg.getSegmentPort()),
seg.getSegmentHostName()))
ParallelOperation(operations, self.batch_default).run()
success = 0
for remote in operations:
host = remote.host
try:
remote.get_ret()
except NotEnoughDiskSpace, e:
logger.error("%s has insufficient disk space. [Need: %dK, Free %dK]" % (host, e.needed_space, e.free_space))
else:
success += 1
def __runStartCommand(self, segments, startMethod, numContentsInCluster, resultOut, gpArray, era):
"""
Putt results into the resultOut object
"""
if len(segments) == 0:
return
if startMethod == START_AS_PRIMARY_OR_MIRROR:
logger.info("Commencing parallel primary and mirror segment instance startup, please wait...")
else:
logger.info("Commencing parallel segment instance startup, please wait...")
dbIdToPeerMap = gpArray.getDbIdToPeerMap()
mirroringModePreTransition = MIRROR_MODE_MIRRORLESS if startMethod == START_AS_MIRRORLESS else MIRROR_MODE_QUIESCENT
# launch the start
for hostName, segments in GpArray.getSegmentsByHostName(segments).iteritems():
logger.debug("Dispatching command to start segments on host: %s, " \
"with %s contents in cluster" % (hostName, numContentsInCluster))
pickledTransitionData = None
if startMethod == START_AS_PRIMARY_OR_MIRROR:
mirroringModePerSegment = []
for seg in segments:
modeThisSegment = MIRROR_MODE_PRIMARY if seg.isSegmentPrimary(True) else MIRROR_MODE_MIRROR
mirroringModePerSegment.append(modeThisSegment)
pickledTransitionData = self.__createPickledTransitionParameters(segments, mirroringModePerSegment, None, dbIdToPeerMap)
#
# This will call sbin/gpsegstart.py
#
cmd = gp.GpSegStartCmd("remote segment starts on host '%s'" % hostName,
self.__gpHome, segments,
self.__gpVersion,
mirroringModePreTransition,
numContentsInCluster,
era,
self.master_checksum_value,
self.__timeout,
verbose=logging_is_verbose(),
ctxt=base.REMOTE,
remoteHost=segments[0].getSegmentAddress(),
pickledTransitionData=pickledTransitionData,
specialMode=self.__specialMode,
wrapper=self.__wrapper,
wrapper_args=self.__wrapper_args,
parallel=self.__parallel,
logfileDirectory=self.logfileDirectory)
self.__workerPool.addCommand(cmd)
if self.__quiet:
self.__workerPool.join()
else:
base.join_and_indicate_progress(self.__workerPool)
# process results
self.__processStartOrConvertCommands(resultOut)
self.__workerPool.empty_completed_items()
def execute(self):
gparray = GpArray.initFromCatalog(dbconn.DbURL(port = self.master_port), utility=True)
failed_segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary(current_role=True) and seg.isSegmentDown()]
if len(failed_segs) != 0:
logger.warn("Failed primary segment instances detected")
failed_dbids = [seg.getSegmentDbid() for seg in failed_segs]
raise ExceptionNoStackTraceNeeded("Detected failed segment(s) with dbid=%s" % ",".join(failed_dbids))
def test_filespace(self):
"""
pg_basebackup should work with user filespace.
@tags sanity
"""
# Add standby entry in catalog before regisering filespace.
fsbase = os.path.join(self.fsprefix, 'fs')
shutil.rmtree(fsbase, True)
os.makedirs(fsbase)
shutil.rmtree(self.standby_datadir, True)
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
if gparray.standbyMaster:
self.standby.remove_catalog_standby(dburl)
self.standby.add_catalog_standby(dburl, gparray)
#self.preprocess_file(local_path('filespace.sql.in'))
sql_file = local_path('filespace.sql')
from mpp.lib.gpfilespace import Gpfilespace
gpfile = Gpfilespace()
gpfile.create_filespace('fs_walrepl_a')
result = PSQL.run_sql_file(sql_file, dbname=self.db_name)
self.assertTrue(result)
subprocess.check_call(['pg_basebackup', '-D', self.standby_datadir])
#fsdir = os.path.join(self.fsprefix, 'fs', 'gpdb1')
fsdir = os.path.join(os.path.split(self.standby_datadir)[0], 'fs_walrepl_a','mirror','pg_system')
self.assertTrue(os.path.isdir(fsdir),
'{0} does not dir'.format(fsdir))
def impl(context, filetype, dir):
if filetype == "report":
if dir == "master_data_directory":
dir = master_data_dir
filenames = os.listdir(dir)
for filename in filenames:
if filename.startswith("gp_restore") and filename.endswith(".rpt"):
filename = "%s/%s" % (dir, filename)
os.remove(filename)
if filetype == "status":
gparray = GpArray.initFromCatalog(dbconn.DbURL())
if dir == "segment_data_directory":
primary_segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary()]
for seg in primary_segs:
host = seg.getSegmentHostName()
seg_data_dir = seg.getSegmentDataDirectory()
cmd = Command(
"remove status file", "rm -f %s/gp_restore_status_*" % (seg_data_dir), ctxt=REMOTE, remoteHost=host
)
cmd.run(validateAfter=True)
else:
primary_segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary()]
for seg in primary_segs:
host = seg.getSegmentHostName()
cmd = Command("remove status file", "rm -f %s/gp_restore_status_*" % dir, ctxt=REMOTE, remoteHost=host)
cmd.run(validateAfter=True)
def execute(self):
gparray = GpArray.initFromCatalog(dbconn.DbURL(port=self.master_port), utility=True)
from_host, from_path = self.host, self.path
logger.info("Commencing remote database dump file recovery process, please wait...")
segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary(current_role=True) or seg.isSegmentMaster()]
pool = WorkerPool(numWorkers = min(len(segs), self.batch_default))
for seg in segs:
if seg.isSegmentMaster():
file = '%s%s' % (MASTER_DBDUMP_PREFIX, self.restore_timestamp)
else:
file = '%s0_%d_%s' % (DBDUMP_PREFIX, seg.getSegmentDbId(), self.restore_timestamp)
if self.compress:
file += '.gz'
to_host = seg.getSegmentHostName()
to_path = os.path.join(seg.getSegmentDataDirectory(), DUMP_DIR, self.restore_timestamp[0:8])
if not CheckRemoteDir(to_path, to_host).run():
logger.info('Creating directory %s on %s' % (to_path, to_host))
try:
MakeRemoteDir(to_path, to_host).run()
except OSError, e:
raise ExceptionNoStackTraceNeeded("Failed to create directory %s on %s" % (to_path, to_host))
logger.info("Commencing remote copy from %s to %s:%s" % (from_host, to_host, to_path))
pool.addCommand(Scp('Copying dump for seg %d' % seg.getSegmentDbId(),
srcFile=os.path.join(from_path, file),
dstFile=os.path.join(to_path, file),
srcHost=from_host,
dstHost=to_host))
def __ensureStopped(self, gpEnv, directives):
"""
@param directives a list of the GpStopSegmentDirectoryDirective values indicating which segments to stop
"""
if len(directives) == 0:
return
logger.info("Ensuring %d failed segment(s) are stopped" % (len(directives)))
segments = [d.getSegment() for d in directives]
segments = self._get_running_postgres_segments(segments)
segmentByHost = GpArray.getSegmentsByHostName(segments)
cmds = []
for hostName, segments in segmentByHost.iteritems():
cmd=gp.GpSegStopCmd("remote segment stop on host '%s'" % hostName,
gpEnv.getGpHome(), gpEnv.getGpVersion(),
mode='fast', dbs=segments, verbose=logging_is_verbose(),
ctxt=base.REMOTE, remoteHost=hostName)
cmds.append( cmd)
# we suppress checking for the error. This is because gpsegstop will actually error
# in many cases where the stop is actually done (that is, for example, the segment is
# running but slow to shutdown so gpsegstop errors after whacking it with a kill)
#
# Perhaps we should make it so that it so that is checks if the seg is running and only attempt stop
# if it's running? In that case, we could propagate the error
#
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(cmds, "stopping segments", suppressErrorCheck=True)
def get_seginfo_for_primaries(self):
seglist = []
gparray = GpArray.initFromCatalog(dbconn.DbURL())
for seg in gparray.getDbList():
if seg.isSegmentPrimary() and not seg.isSegmentMaster():
seglist.append(seg)
return seglist
def get_host_list():
"""
Returns a tuple which consists of the standby
and segment hosts comprising the cluster.
@return: tuple containing the standby and segment hosts
tuple[0] contains standby
tuple[1] contains segment hosts
"""
gparr = GpArray.initFromCatalog(dbconn.DbURL(port = MASTER_PORT), utility = True)
segs = gparr.getDbList()
master = None
standby_host = None
segment_host_list = []
for seg in segs:
if seg.isSegmentStandby(current_role=True):
standby_host = seg.getSegmentHostName()
elif not seg.isSegmentMaster(current_role=True):
segment_host_list.append(seg.getSegmentHostName())
elif seg.isSegmentMaster(current_role=True):
master = seg.getSegmentHostName()
#Deduplicate the hosts so that we
#dont install multiple times on the same host
segment_host_list = list(set(segment_host_list))
if master in segment_host_list:
segment_host_list.remove(master)
return (standby_host, segment_host_list)
def impl(context, seg):
if seg == "mirror":
gparray = GpArray.initFromCatalog(dbconn.DbURL())
mirror_segs = [seg for seg in gparray.getDbList() if seg.isSegmentMirror() and seg.getSegmentHostName() != platform.node()]
context.remote_mirror_segdbId = mirror_segs[0].getSegmentDbId()
context.remote_mirror_segdbname = mirror_segs[0].getSegmentHostName()
context.remote_mirror_datadir = mirror_segs[0].getSegmentDataDirectory()
def create(self):
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
if gparray.standbyMaster:
self.remove_catalog_standby(dburl)
self.add_catalog_standby(dburl, gparray)
shutil.rmtree(self.datadir, True)
rc = subprocess.call(
['pg_basebackup', '-x', '-R', '-D', self.datadir])
# this is ugly, but required by gpstart, who determines
# port number from postgresql.conf and removes UNIX socket file.
postgresql_conf = os.path.join(self.datadir, 'postgresql.conf')
conf_tmp = open(postgresql_conf + '.tmp', 'w')
with open(postgresql_conf) as conf:
for line in conf.readlines():
if "port=" in line:
line = "port={0} #{1}".format(self.port, line)
conf_tmp.write(line)
conf_tmp.close()
os.rename(postgresql_conf + '.tmp', postgresql_conf)
self.update_primary_pg_hba()
return rc
def execute(self):
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl)
my_fault_strategy = gparray.getFaultStrategy()
if my_fault_strategy != FAULT_STRATEGY_FILE_REPLICATION:
raise NoMirroringError(
'Fault strategy %s does not support mirror verification.' %
FAULT_STRATEGY_LABELS[my_fault_strategy])
if self.content is not None:
contents = set(
[seg.getSegmentContentId() for seg in gparray.getDbList()])
if self.content not in contents:
raise InvalidContentIDError(self.content)
logger.info('Validating target contents...')
to_verify = [x for x in gparray.getDbList() if x.isSegmentQE()]
if self.content is not None:
to_verify = [
x for x in to_verify if x.getSegmentContentId() == self.content
]
if self.primaries_only:
to_verify = [
x for x in to_verify if x.isSegmentPrimary(current_role=True)
]
return to_verify
def execute(self):
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
config_backup_file = "gp_master_config_files_%s.tar" % timestamp
if self.backup_dir is not None:
path = os.path.join(self.backup_dir, DUMP_DIR, DUMP_DATE, config_backup_file)
else:
path = os.path.join(self.master_datadir, DUMP_DIR, DUMP_DATE, config_backup_file)
logger.info("Dumping master config files")
Command("Dumping master configuration files",
"tar cf %s %s/*.conf" % (path, self.master_datadir)).run(validateAfter=True)
logger.info("Dumping segment config files")
gparray = GpArray.initFromCatalog(dbconn.DbURL(port = self.master_port), utility=True)
primaries = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary(current_role=True)]
for seg in primaries:
config_backup_file = "gp_segment_config_files_0_%d_%s.tar" % (seg.getSegmentDbId(), timestamp)
if self.backup_dir is not None:
path = os.path.join(self.backup_dir, DUMP_DIR, DUMP_DATE, config_backup_file)
else:
path = os.path.join(seg.getSegmentDataDirectory(), DUMP_DIR, DUMP_DATE, config_backup_file)
host = seg.getSegmentHostName()
Command("Dumping segment config files",
"tar cf %s %s/*.conf" % (path, seg.getSegmentDataDirectory()),
ctxt=REMOTE,
remoteHost=host).run(validateAfter=True)
def impl(context):
if hasattr(context, 'backup_timestamp'):
ts = context.backup_timestamp
if hasattr(context, 'netbackup_service_host'):
netbackup_service_host = context.netbackup_service_host
if not hasattr(context, "dump_prefix"):
context.dump_prefix = ''
master_config_filename = os.path.join(master_data_dir, 'db_dumps', context.backup_timestamp[0:8],
'%sgp_master_config_files_%s.tar' % (context.dump_prefix, ts))
command_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (netbackup_service_host, master_config_filename)
cmd = Command('Validate master config file', command_str)
cmd.run(validateAfter=True)
results = cmd.get_results().stdout.strip()
if results != master_config_filename:
raise Exception('Expected Master Config file: %s and found: %s. Master Config file was not backup up to NetBackup server' % (master_config_filename, results))
gparray = GpArray.initFromCatalog(dbconn.DbURL())
primary_segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary(current_role=True)]
for seg in primary_segs:
segment_config_filename = os.path.join(seg.getSegmentDataDirectory(), 'db_dumps', context.backup_timestamp[0:8],
'%sgp_segment_config_files_0_%s_%s.tar' % (context.dump_prefix, seg.getSegmentDbId(), context.backup_timestamp))
command_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (netbackup_service_host, segment_config_filename)
cmd = Command('Validate segment config file', command_str, ctxt=REMOTE, remoteHost = seg.getSegmentHostName())
cmd.run(validateAfter=True)
results = cmd.get_results().stdout.strip()
if results != segment_config_filename:
raise Exception('Expected Segment Config file: %s and found: %s. Segment Config file was not backup up to NetBackup server' % (segment_config_filename, results))
def are_segments_running():
gparray = GpArray.initFromCatalog(dbconn.DbURL())
segments = gparray.getDbList()
for seg in segments:
if seg.status != 'u':
return False
return True
def are_segments_synchronized():
gparray = GpArray.initFromCatalog(dbconn.DbURL())
segments = gparray.getDbList()
for seg in segments:
if seg.mode != MODE_SYNCHRONIZED and not seg.isSegmentMaster(True):
return False
return True
def preprocess_file(self, in_path):
"""
Proprocess .in files, to match enviroment. Currently CREATE FILESPACE
will be substituted.
e.g.
CREATE FILESPACE fs();
will be completed with dbid/directory path.
"""
dburl = gppylib.db.dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
assert(in_path.endswith('.in'))
out_path = in_path[:-3]
pat = re.compile(r'CREATE\s+FILESPACE\s+(\w+)', re.I)
with open(out_path, 'w') as out_file:
for line in open(in_path):
m = pat.match(line)
if m:
# The file paths will look like
# <fsprefix>/<fsname>/gpdb<dbid>
fsname = m.group(1)
basepath = os.path.join(self.fsprefix, fsname)
if not os.path.exists(basepath):
os.makedirs(basepath)
buf = list()
for db in gparray.getDbList():
fselocation = os.path.join(basepath,
'gpdb' + str(db.dbid))
buf.append("{0}: '{1}'".format(
db.dbid, fselocation))
line = "CREATE FILESPACE {0}({1});\n".format(
fsname, ', '.join(buf))
out_file.write(line)
def test_init_standby(self):
gparr = GpArray.initFromCatalog(dbconn.DbURL(dbname=DEFAULT_DATABASE))
if not gparr.standbyMaster:
standby_host = self.create_standby_master(gparr)
else:
self.remove_standby_master(gparr)
standby_host = self.create_standby_master(gparr)
self.verify_standby_ips(gparr, standby_host)
def impl(context):
gparray = GpArray.initFromCatalog(dbconn.DbURL())
for seg in gparray.getDbList():
if seg.isSegmentPrimary() and seg.getSegmentContentId() == context.remote_mirror_segcid:
context.remote_pair_primary_segdbId = seg.getSegmentDbId()
context.remote_pair_primary_datadir = seg.getSegmentDataDirectory()
context.remote_pair_primary_port = seg.getSegmentPort()
context.remote_pair_primary_host = seg.getSegmentHostName()
def impl(context, output):
gparray = GpArray.initFromCatalog(dbconn.DbURL())
segments = gparray.getDbList()
for segment in segments:
if segment.isSegmentMirror():
expected = r'\(dbid {}\): {}'.format(segment.dbid, output)
check_stdout_msg(context, expected)
def checkForPortAndDirectoryConflicts(self, gpArray):
"""
Check gpArray for internal consistency -- no duplicate ports or directories on the same host, for example
A detected problem causes an Exception to be raised
"""
for hostName, segmentArr in GpArray.getSegmentsByHostName(gpArray.getDbList()).iteritems():
usedPorts = {}
usedDataDirectories = {}
for segment in segmentArr:
# check for port conflict
replicationPort = segment.getSegmentReplicationPort()
port = segment.getSegmentPort()
dbid = segment.getSegmentDbId()
if port in usedPorts:
raise Exception(
"On host %s, a port for segment with dbid %s conflicts with a port for segment dbid %s"
% (hostName, dbid, usedPorts.get(port))
)
if segment.isSegmentQE():
if replicationPort is not None:
raise Exception(
"On host %s, the replication port is set for segment with dbid %s" % (hostName, dbid)
)
if replicationPort in usedPorts:
raise Exception(
"On host %s, a port for segment with dbid %s conflicts with a port for segment dbid %s"
% (hostName, dbid, usedPorts.get(replicationPort))
)
if port == replicationPort:
raise Exception(
"On host %s, segment with dbid %s has equal port and replication port" % (hostName, dbid)
)
usedPorts[port] = dbid
if replicationPort is not None:
usedPorts[replicationPort] = dbid
# check for directory conflict; could improve this by reporting nicer the conflicts
paths = [
path for oid, path in segment.getSegmentFilespaces().items() if oid != gparray.SYSTEM_FILESPACE
]
paths.append(segment.getSegmentDataDirectory())
for path in paths:
if path in usedDataDirectories and 0:
raise Exception(
"On host %s, directory (base or filespace) for segment with dbid %s conflicts with a "
"directory (base or filespace) for segment dbid %s; directory: %s"
% (hostName, dbid, usedDataDirectories.get(path), path)
)
usedDataDirectories[path] = dbid
def __ensureMarkedDown(self, gpEnv, toEnsureMarkedDown):
"""Waits for FTS prober to mark segments as down"""
wait_time = 60 * 30 # Wait up to 30 minutes to handle very large, busy
# clusters that may have faults. In most cases the
# actual time to wait will be small and this operation
# is only needed when moving mirrors that are up and
# needed to be stopped, an uncommon operation.
dburl = dbconn.DbURL(port=gpEnv.getMasterPort(), dbname="template1")
time_elapsed = 0
seg_up_count = 0
initial_seg_up_count = len(toEnsureMarkedDown)
last_seg_up_count = initial_seg_up_count
if initial_seg_up_count == 0:
# Nothing to wait on
return
logger.info("Waiting for segments to be marked down.")
logger.info("This may take up to %d seconds on large clusters." % wait_time)
# wait for all needed segments to be marked down by the prober. We'll wait
# a max time of double the interval
while wait_time > time_elapsed:
seg_up_count = 0
current_gparray = GpArray.initFromCatalog(dburl, True)
seg_db_map = current_gparray.getSegDbMap()
# go through and get the status of each segment we need to be marked down
for segdb in toEnsureMarkedDown:
if segdb.getSegmentDbId() in seg_db_map and seg_db_map[segdb.getSegmentDbId()].isSegmentUp() == True:
seg_up_count += 1
if seg_up_count == 0:
break
else:
if last_seg_up_count != seg_up_count:
print "\n",
logger.info(
"%d of %d segments have been marked down."
% (initial_seg_up_count - seg_up_count, initial_seg_up_count)
)
last_seg_up_count = seg_up_count
for _i in range(1, 5):
time.sleep(1)
sys.stdout.write(".")
sys.stdout.flush()
time_elapsed += 5
if seg_up_count == 0:
print "\n",
logger.info("%d of %d segments have been marked down." % (initial_seg_up_count, initial_seg_up_count))
else:
raise Exception("%d segments were not marked down by FTS" % seg_up_count)
def _gather_log_from_gp_log_filter(start_time, end_time=None, out_file=_DEFAULT_OUT_FILE, host='localhost',
port=_DEFAULT_PORT, user=_DEFAULT_USER, dbname=_DEFAULT_USER, errors_only=False, master_only=False):
"""
This retrieves log messages from all segments that happened within the last
'duration' seconds. The format of start_time and end_time is YYYY-MM-DD [hh:mm[:ss]]
The tuples returned are (dbid, hostname, datadir, logdata). sorted by dbid.
Returns True/False based on whether matching log entries were found.
"""
format_start_time = time.strftime("%Y-%m-%dT%H:%M:%S",time.localtime(start_time))
if end_time:
format_end_time = time.strftime("%Y-%m-%dT%H:%M:%S",time.localtime(end_time))
else:
format_end_time = time.strftime("%Y-%m-%dT%H:%M:%S",time.localtime())
tinctest.logger.info("Collecting log from %s to %s into the file -%s" % (format_start_time,format_end_time, out_file))
array = GpArray.initFromCatalog(DbURL(hostname=host, port=port, username=user, dbname=dbname), True)
log_chunks = []
for seg in array.getDbList():
tinctest.logger.info("Collecting log for segment - %s : %s" %(seg.getSegmentHostName(), seg.getSegmentContentId()))
if master_only and seg.getSegmentContentId() != -1:
continue
cmd = GpLogFilter('collect log chunk',
'\\`ls -rt %s | tail -1\\`' % os.path.join(seg.getSegmentDataDirectory(), 'pg_log', '*.csv'),
start=format_start_time, end=format_end_time,
trouble=errors_only,
ctxt=REMOTE,
remoteHost=seg.getSegmentHostName())
cmd.run()
rc = cmd.get_results().rc
if rc:
tinctest.logger.warning("Failed command execution %s : %s" %(cmd, cmd.get_results().stderr))
continue
log_data = cmd.get_results().stdout
if not log_data:
tinctest.logger.warning("No log data returned for the given time frame.")
else:
log_chunks.append((seg.getSegmentContentId(),
seg.getSegmentHostName(),
seg.getSegmentDataDirectory(),
log_data))
if log_chunks:
tinctest.logger.info("Writing log data to file - %s" %(out_file))
with open(out_file, 'w') as f:
for part in log_chunks:
f.write("-"*70)
f.write("\n DBID %s (%s:%s)\n" % (part[0], part[1], part[2]))
f.write("-"*70)
f.write("\n%s" % part[3])
f.write("\n\n")
def setUp(self):
super(XidlimitsTests, self).setUp()
Command('re-build regress.so',
'make -C %s xidhelper.so' % local_path('.')).run(validateAfter=True)
SQLTemplateTestCase.perform_transformation_on_sqlfile(local_path('load_xidhelper.sql'),
local_path('load_xidhelper.sql.t'),
{'@source@' : local_path('xidhelper.so')})
PSQL.run_sql_file(sql_file=local_path('load_xidhelper.sql.t'),
out_file=local_path('load_xidhelper.out.t'))
self.gparray = GpArray.initFromCatalog(dbconn.DbURL(), utility=True)
def verify_timestamps_on_segments_with_nbu(timestamp):
gparray = GpArray.initFromCatalog(dbconn.DbURL())
primary_segs = [segdb for segdb in gparray.getDbList() if segdb.isSegmentPrimary()]
for seg in primary_segs:
db_dumps_dir = os.path.join(seg.getSegmentDataDirectory(), "db_dumps", timestamp[:8])
list_cmd = "ls -l %s/*%s* | wc -l" % (db_dumps_dir, timestamp)
cmd = Command("get list of dump files", list_cmd, ctxt=REMOTE, remoteHost=seg.getSegmentHostName())
cmd.run(validateAfter=True)
verify_num_files_with_nbu(cmd.get_results(), "1", timestamp)
def __init__(self, methodName):
self.gpact = GpactivateStandby()
self.host = socket.gethostname()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.pgport = os.environ.get('PGPORT')
self.port = self.pgport
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
self.numcontent = gparray.getNumSegmentContents()
self.orig_master = gparray.master
self.standby_pid = ''
super(GpactivateStandbyTestCase,self).__init__(methodName)
def __cleanUpSegmentDirectories(self, directives):
if len(directives) == 0:
return
logger.info("Cleaning files from %d segment(s)" % (len(directives)))
segments = [d.getSegment() for d in directives]
segmentByHost = GpArray.getSegmentsByHostName(segments)
cmds = []
for hostName, segments in segmentByHost.iteritems():
cmds.append( gp.GpCleanSegmentDirectories("clean segment directories on %s" % hostName, \
segments, gp.REMOTE, hostName))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(cmds, "cleaning existing directories")
def __init__(self):
self.gpinit = GpinitStandby()
self.pgutil = GpUtility()
self.runmixin = StandbyRunMixin()
self.runmixin.createdb(dbname='walrepl')
self.gphome = os.environ.get('GPHOME')
self.pgport = os.environ.get('PGPORT')
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.config = GPDBConfig()
self.host = socket.gethostname()
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
self.numcontent = gparray.getNumSegmentContents()
self.orig_master = gparray.master
def impl(context):
make_temp_dir(context, path.join('/tmp', 'gpconfig'))
temp_base_dir = context.temp_base_dir
context.gpconfig_context.working_directory = temp_base_dir
gparray = GpArray.initFromCatalog(dbconn.DbURL())
segments = gparray.getDbList()
restore_commands = []
for segment in segments:
segment_tmp_directory = path.join(temp_base_dir, str(segment.dbid))
os.mkdir(segment_tmp_directory)
backup_path = path.join(segment_tmp_directory, 'postgresql.conf')
original_path = path.join(segment.datadir, 'postgresql.conf')
copy_command = (
'scp %s:%s %s' %
(segment.hostname, original_path, backup_path)).split(' ')
restore_command = (
'scp %s %s:%s' %
(backup_path, segment.hostname, original_path)).split(' ')
restore_commands.append(restore_command)
subprocess.check_call(copy_command)
if segment.content == -1:
if segment.role == 'p':
context.gpconfig_context.master_postgres_file = original_path
else:
context.gpconfig_context.standby_postgres_file = original_path
def delete_temp_directory():
if 'temp_base_dir' in context:
shutil.rmtree(context.temp_base_dir)
def restore_conf_files():
for cmd in restore_commands:
subprocess.check_call(cmd)
context.add_cleanup(delete_temp_directory)
context.add_cleanup(restore_conf_files)
def __ensureStopped(self, gpEnv, directives):
"""
@param directives a list of the GpStopSegmentDirectoryDirective values indicating which segments to stop
"""
if len(directives) == 0:
return
self.__logger.info("Ensuring %d failed segment(s) are stopped" %
(len(directives)))
segments = [d.getSegment() for d in directives]
segments = self._get_running_postgres_segments(segments)
segmentByHost = GpArray.getSegmentsByHostName(segments)
cmds = []
for hostName, segments in segmentByHost.iteritems():
cmd = gp.GpSegStopCmd("remote segment stop on host '%s'" %
hostName,
gpEnv.getGpHome(),
gpEnv.getGpVersion(),
mode='fast',
dbs=segments,
verbose=gplog.logging_is_verbose(),
ctxt=base.REMOTE,
remoteHost=hostName)
cmds.append(cmd)
# we suppress checking for the error. This is because gpsegstop will actually error
# in many cases where the stop is actually done (that is, for example, the segment is
# running but slow to shutdown so gpsegstop errors after whacking it with a kill)
#
# Perhaps we should make it so that it so that is checks if the seg is running and only attempt stop
# if it's running? In that case, we could propagate the error
#
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds, "stopping segments", suppressErrorCheck=True)
def __init__(self, name):
self.name = name
self.path = tempfile.mkdtemp()
self.dbname = 'tablespace_db_%s' % name
self.table_counter = 0
self.initial_data = None
gparray = GpArray.initFromCatalog(dbconn.DbURL())
for host in gparray.getHostList():
run_cmd('ssh %s mkdir -p %s' % (pipes.quote(host), pipes.quote(self.path)))
conn = dbconn.connect(dbconn.DbURL(), unsetSearchPath=False)
dbconn.execSQL(conn, "CREATE TABLESPACE %s LOCATION '%s'" % (self.name, self.path))
dbconn.execSQL(conn, "CREATE DATABASE %s TABLESPACE %s" % (self.dbname, self.name))
conn.close()
conn = dbconn.connect(dbconn.DbURL(dbname=self.dbname), unsetSearchPath=False)
dbconn.execSQL(conn, "CREATE TABLE tbl (i int) DISTRIBUTED RANDOMLY")
dbconn.execSQL(conn, "INSERT INTO tbl VALUES (GENERATE_SERIES(0, 25))")
# save the distributed data for later verification
self.initial_data = dbconn.query(conn, "SELECT gp_segment_id, i FROM tbl").fetchall()
conn.close()
def __updateGpIdFile(self, gpEnv, gpArray, segments):
segmentByHost = GpArray.getSegmentsByHostName(segments)
newSegmentInfo = gp.ConfigureNewSegment.buildSegmentInfoForNewSegment(
segments)
cmds = []
for hostName in list(segmentByHost.keys()):
segmentInfo = newSegmentInfo[hostName]
checkNotNone("segmentInfo for %s" % hostName, segmentInfo)
cmd = gp.ConfigureNewSegment("update gpid file",
segmentInfo,
gplog.get_logger_dir(),
newSegments=False,
verbose=gplog.logging_is_verbose(),
batchSize=self.__parallelDegree,
ctxt=gp.REMOTE,
remoteHost=hostName,
validationOnly=False,
writeGpIdFileOnly=True)
cmds.append(cmd)
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds, "writing updated gpid files")
def loadSystemConfig(self, useUtilityMode, verbose=True):
"""
Load all segment information from the configuration source.
Returns a new GpArray object
"""
# ensure initializeProvider() was called
checkNotNone("coordinatorDbUrl", self.__coordinatorDbUrl)
if verbose:
logger.info("Obtaining Segment details from coordinator...")
array = GpArray.initFromCatalog(self.__coordinatorDbUrl,
useUtilityMode)
if get_local_db_mode(
array.coordinator.getSegmentDataDirectory()) != 'UTILITY':
logger.debug("Validating configuration...")
if not array.is_array_valid():
raise InvalidSegmentConfiguration(array)
return array
def test_prepare_segment_start_returns_up_and_down_segments(self):
# Boilerplate: create a gpstart object
parser = self.subject.GpStart.createParser()
options, args = parser.parse_args([])
gpstart = self.subject.GpStart.createProgram(options, args)
# Up segments
coordinator = Segment.initFromString("1|-1|p|p|n|u|cdw|cdw|5432|/data/coordinator")
primary1 = Segment.initFromString("3|1|p|p|n|u|sdw2|sdw2|40001|/data/primary1")
mirror0 = Segment.initFromString("4|0|m|m|n|u|sdw2|sdw2|50000|/data/mirror0")
# Down segments
primary0 = Segment.initFromString("2|0|p|p|n|d|sdw1|sdw1|40000|/data/primary0")
mirror1 = Segment.initFromString("5|1|m|m|n|d|sdw1|sdw1|50001|/data/mirror1")
standby = Segment.initFromString("6|-1|m|m|n|d|sdw3|sdw3|5433|/data/standby")
gpstart.gparray = GpArray([coordinator, primary0, primary1, mirror0, mirror1, standby])
up, down = gpstart._prepare_segment_start()
# The coordinator and standby should not be accounted for in these lists.
self.assertCountEqual(up, [primary1, mirror0])
self.assertCountEqual(down, [primary0, mirror1])
def get_host_list():
"""
Returns a tuple which consists of the standby
and segment hosts
"""
gparr = GpArray.initFromCatalog(dbconn.DbURL(port=MASTER_PORT),
utility=True)
segs = gparr.getDbList()
standby_host = None
segment_host_list = []
for seg in segs:
if seg.isSegmentStandby(current_role=True):
standby_host = seg.getSegmentHostName()
elif not seg.isSegmentMaster(current_role=True):
segment_host_list.append(seg.getSegmentHostName())
# Deduplicate the hosts so that we
# dont install multiple times on the same host
segment_host_list = list(set(segment_host_list))
return (standby_host, segment_host_list)
def getTriplets(self):
def _check_new_hosts():
if len(self.newHosts) > len(failedSegments):
self.interfaceHostnameWarnings.append(
"The following recovery hosts were not needed:")
for h in self.newHosts[len(failedSegments):]:
self.interfaceHostnameWarnings.append("\t%s" % h)
if len(self.newHosts) < len(failedSegments):
raise Exception(
'Not enough new recovery hosts given for recovery.')
unreachable_hosts = get_unreachable_segment_hosts(
self.newHosts[:len(failedSegments)], len(failedSegments))
if unreachable_hosts:
raise ExceptionNoStackTraceNeeded(
"Cannot recover. The following recovery target hosts are "
"unreachable: %s" % unreachable_hosts)
failedSegments = GpArray.getSegmentsByHostName([
seg for seg in self.gpArray.getSegDbList() if seg.isSegmentDown()
])
_check_new_hosts()
requests = []
for failedHost, failoverHost in zip(sorted(failedSegments.keys()),
self.newHosts):
for failed in failedSegments[failedHost]:
failoverPort = self.portAssigner.findAndReservePort(
failoverHost, failoverHost)
req = RecoveryTripletRequest(failed, failoverHost,
failoverPort,
failed.getSegmentDataDirectory(),
True)
requests.append(req)
return self._convert_requests_to_triplets(requests)
def __init__(self, values=None):
if values:
self.defaults.update(values.__dict__) # Ensure that context has default values for all unset variables
super(self.__class__, self).__init__(vars(Values(self.defaults)))
if self.masterDataDirectory:
self.master_datadir = self.masterDataDirectory
else:
self.master_datadir = gp.get_masterdatadir()
self.master_port = self.get_master_port()
if self.local_dump_prefix:
self.dump_prefix = self.local_dump_prefix + "_"
else:
self.dump_prefix = ""
if not self.include_dump_tables: self.include_dump_tables = []
if not self.exclude_dump_tables: self.exclude_dump_tables = []
if not self.output_options: self.output_options = []
if not self.dump_schema: self.dump_schema = []
if not self.exclude_dump_schema: self.exclude_dump_schema = []
self.gparray = GpArray.initFromCatalog(dbconn.DbURL(dbname="template1", port=self.master_port), utility=True)
self.use_old_filename_format = False # Use new filename format by default
self.content_map = self.setup_content_map()
def test_host_option_if_standby_running_on_the_host_fails(self):
sys.argv = ["gpstop", "-a", "--host", "sdw1"]
parser = self.subject.GpStop.createParser()
options, args = parser.parse_args()
self.master = Segment.initFromString(
"1|-1|p|p|s|u|mdw|mdw|5432|/data/master")
self.standby = Segment.initFromString(
"2|-1|m|m|s|u|sdw1|sdw1|25432|/data/master")
self.primary0 = Segment.initFromString(
"3|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0")
self.mirror0 = Segment.initFromString(
"4|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0")
self.mock_gparray.return_value = GpArray(
[self.master, self.standby, self.primary0, self.mirror0])
gpstop = self.subject.GpStop.createProgram(options, args)
with self.assertRaisesRegexp(
Exception,
"Specified host '%s' has the master or standby master on it. This node can only be stopped as part of a full-cluster gpstop, without '--host'."
% self.standby.getSegmentHostName()):
gpstop.run()
self.assertEquals(0, self.mock_GpSegStopCmdInit.call_count)
def test_host_option_if_no_mirrors_fails(self):
sys.argv = ["gpstop", "-a", "--host", "sdw2"]
parser = self.subject.GpStop.createParser()
options, args = parser.parse_args()
self.master = Segment.initFromString(
"1|-1|p|p|s|u|mdw|mdw|5432|/data/master")
self.standby = Segment.initFromString(
"2|-1|m|m|s|u|sdw1|sdw1|25432|/data/master")
self.primary0 = Segment.initFromString(
"3|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0")
self.primary1 = Segment.initFromString(
"4|0|p|p|s|u|sdw2|sdw2|40001|/data/primary1")
self.mock_gparray.return_value = GpArray(
[self.master, self.standby, self.primary0, self.primary1])
gpstop = self.subject.GpStop.createProgram(options, args)
with self.assertRaisesRegexp(
Exception,
"Cannot perform host-specific gpstop on a cluster without segment mirroring."
):
gpstop.run()
self.assertEquals(0, self.mock_GpSegStopCmdInit.call_count)
def execute(self):
fake_timestamp = PickDumpTimestamp(
restore_timestamp=self.restore_timestamp,
compress=self.compress,
master_datadir=self.master_datadir).run()
gparray = GpArray.initFromCatalog(dbconn.DbURL(port=self.master_port),
utility=True)
primaries = [
seg for seg in gparray.getDbList()
if seg.isSegmentPrimary(current_role=True)
]
operations = []
for seg in primaries:
real_filename = os.path.join(
seg.getSegmentDataDirectory(), DUMP_DIR,
self.restore_timestamp[0:8], "%s0_%d_%s" %
(DBDUMP_PREFIX, seg.getSegmentDbId(), self.restore_timestamp))
fake_filename = os.path.join(
seg.getSegmentDataDirectory(), DUMP_DIR, fake_timestamp[0:8],
"%s0_%d_%s" %
(DBDUMP_PREFIX, seg.getSegmentDbId(), fake_timestamp))
operations.append(
BuildRemoteTableDump(self.restore_tables, real_filename,
fake_filename, self.compress,
seg.getSegmentHostName()))
ParallelOperation(operations, self.batch_default).run()
for operation in operations:
try:
operation.get_ret()
except Exception, e:
logger.exception('Parallel table dump file build failed.')
raise ExceptionNoStackTraceNeeded(
'Parallel table dump file build failed, review log file for details'
)
def test_host_option_segment_down_is_skipped_succeeds(self):
sys.argv = ["gpstop", "-a", "--host", "sdw1"]
parser = self.subject.GpStop.createParser()
options, args = parser.parse_args()
self.primary0 = Segment.initFromString(
"2|0|m|p|s|d|sdw1|sdw1|40000|/data/primary0")
self.mirror0 = Segment.initFromString(
"6|0|p|m|c|u|sdw2|sdw2|50000|/data/mirror0")
self.mock_gparray.return_value = GpArray([self.master, self.primary0,
self.primary1, self.primary2,
self.primary3, self.mirror0,
self.mirror1, self.mirror2,
self.mirror3])
gpstop = self.subject.GpStop.createProgram(options, args)
gpstop.run()
log_messages = self.get_info_messages()
self.assertEquals(2, self.mock_GpSegStopCmdInit.call_count)
self.assertIn("Targeting dbid %s for shutdown" % [self.primary1.getSegmentDbId(),
self.mirror2.getSegmentDbId(),
self.mirror3.getSegmentDbId()], log_messages)
self.assertIn("Successfully shutdown 3 of 8 segment instances ", log_messages)
def mirrorlayout_test(self, hostlist, interface_list, primary_list, primary_portbase, mirror_type,
mirror_list, mirror_portbase, dir_prefix, primary_replication_portbase, mirror_replication_portbase):
master = GpDB(content = -1,
preferred_role = 'p',
dbid = 0,
role = 'p',
mode = 's',
status = 'u',
hostname = 'masterhost',
address = 'masterhost-1',
port = 5432,
datadir = '/masterdir',
replicationPort = 5433)
allrows = []
allrows.append(master)
rows = createSegmentRows(hostlist, interface_list, primary_list, primary_portbase, mirror_type,
mirror_list, mirror_portbase, dir_prefix, primary_replication_portbase, mirror_replication_portbase)
for row in rows:
newrow = GpDB(content = row.content,
preferred_role = 'p' if convert_bool(row.isprimary) else 'm',
dbid = row.dbid,
role = 'p' if convert_bool(row.isprimary) else 'm',
mode = 's',
status = 'u',
hostname = row.host,
address = row.address,
port = row.port,
datadir = row.fulldir,
replicationPort = row.prPort)
allrows.append(newrow)
gparray = GpArray(allrows)
self._validate_array(gparray)
def impl(context, filetype, dir):
if filetype == 'report':
if dir == 'master_data_directory':
dir = master_data_dir
filenames = os.listdir(dir)
for filename in filenames:
if filename.startswith('gp_restore') and filename.endswith('.rpt'):
filename = '%s/%s' % (dir, filename)
os.remove(filename)
if filetype == 'status':
gparray = GpArray.initFromCatalog(dbconn.DbURL())
if dir == 'segment_data_directory':
primary_segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary()]
for seg in primary_segs:
host = seg.getSegmentHostName()
seg_data_dir = seg.getSegmentDataDirectory()
cmd = Command('remove status file', "rm -f %s/gp_restore_status_*" % (seg_data_dir), ctxt=REMOTE, remoteHost=host)
cmd.run(validateAfter=True)
else:
primary_segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary()]
for seg in primary_segs:
host = seg.getSegmentHostName()
cmd = Command('remove status file', "rm -f %s/gp_restore_status_*" % dir, ctxt=REMOTE, remoteHost=host)
cmd.run(validateAfter=True)
def test_host_option_segment_on_same_host_with_mirror_fails(self):
sys.argv = ["gpstop", "-a", "--host", "sdw1"]
parser = self.subject.GpStop.createParser()
options, args = parser.parse_args()
self.master = Segment.initFromString(
"1|-1|p|p|s|u|mdw|mdw|5432|/data/master")
self.primary0 = Segment.initFromString(
"2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0")
self.mirror0 = Segment.initFromString(
"3|0|m|m|s|u|sdw1|sdw1|50000|/data/mirror0")
self.mock_gparray.return_value = GpArray(
[self.master, self.primary0, self.mirror0])
gpstop = self.subject.GpStop.createProgram(options, args)
with self.assertRaisesRegexp(
Exception,
"Segment host '%s' has both of corresponding primary '%s' and mirror '%s'. Aborting."
%
(self.primary0.getSegmentHostName(), self.primary0, self.mirror0)):
gpstop.run()
self.assertEquals(0, self.mock_GpSegStopCmdInit.call_count)
def execute(self):
ValidateGpToolkit(database=self.dump_database,
master_port=self.master_port).run()
operations = []
gparray = GpArray.initFromCatalog(dbconn.DbURL(port=self.master_port),
utility=True)
segs = [
seg for seg in gparray.getDbList()
if seg.isSegmentPrimary(current_role=True)
]
for seg in segs:
operations.append(
RemoteOperation(
ValidateSegDiskSpace(
free_space_percent=self.free_space_percent,
compress=self.compress,
dump_database=self.dump_database,
include_dump_tables=self.include_dump_tables,
datadir=seg.getSegmentDataDirectory(),
segport=seg.getSegmentPort()),
seg.getSegmentHostName()))
ParallelOperation(operations, self.batch_default).run()
success = 0
for remote in operations:
host = remote.host
try:
remote.get_ret()
except NotEnoughDiskSpace, e:
logger.error(
"%s has insufficient disk space. [Need: %dK, Free %dK]" %
(host, e.needed_space, e.free_space))
else:
success += 1
def rebalance(self):
self.logger.info("Determining primary and mirror segment pairs to rebalance")
# The current implementation of rebalance calls "gprecoverseg -a" below.
# Thus, if another balanced pair is not synchronized, or has a down mirror
# that pair will be recovered as a side-effect of rebalancing.
unbalanced_primary_segs = []
for segmentPair in self.gpArray.segmentPairs:
if segmentPair.balanced():
continue
if segmentPair.up() and segmentPair.reachable() and segmentPair.synchronized():
unbalanced_primary_segs.append(segmentPair.primaryDB)
else:
self.logger.warning(
"Not rebalancing primary segment dbid %d with its mirror dbid %d because one is either down, unreachable, or not synchronized" \
% (segmentPair.primaryDB.dbid, segmentPair.mirrorDB.dbid))
if not len(unbalanced_primary_segs):
self.logger.info("No segments to rebalance")
return True
unbalanced_primary_segs = GpArray.getSegmentsByHostName(unbalanced_primary_segs)
pool = base.WorkerPool(min(len(unbalanced_primary_segs), self.batch_size))
try:
# Disable ctrl-c
signal.signal(signal.SIGINT, signal.SIG_IGN)
self.logger.info("Stopping unbalanced primary segments...")
for hostname in list(unbalanced_primary_segs.keys()):
cmd = GpSegStopCmd("stop unbalanced primary segs",
self.gpEnv.getGpHome(),
self.gpEnv.getGpVersion(),
'fast',
unbalanced_primary_segs[hostname],
ctxt=base.REMOTE,
remoteHost=hostname,
timeout=600,
segment_batch_size=self.segment_batch_size)
pool.addCommand(cmd)
base.join_and_indicate_progress(pool)
failed_count = 0
completed = pool.getCompletedItems()
for res in completed:
if not res.get_results().wasSuccessful():
failed_count += 1
allSegmentsStopped = (failed_count == 0)
if not allSegmentsStopped:
self.logger.warn("%d segments failed to stop. A full rebalance of the" % failed_count)
self.logger.warn("system is not possible at this time. Please check the")
self.logger.warn("log files, correct the problem, and run gprecoverseg -r")
self.logger.warn("again.")
self.logger.info("gprecoverseg will continue with a partial rebalance.")
pool.empty_completed_items()
segment_reconfigurer = SegmentReconfigurer(logger=self.logger,
worker_pool=pool, timeout=MIRROR_PROMOTION_TIMEOUT)
segment_reconfigurer.reconfigure()
# Final step is to issue a recoverseg operation to resync segments
self.logger.info("Starting segment synchronization")
original_sys_args = sys.argv[:]
self.logger.info("=============================START ANOTHER RECOVER=========================================")
# import here because GpRecoverSegmentProgram and GpSegmentRebalanceOperation have a circular dependency
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
cmd_args = ['gprecoverseg', '-a', '-B', str(self.batch_size), '-b', str(self.segment_batch_size)]
sys.argv = cmd_args[:]
local_parser = GpRecoverSegmentProgram.createParser()
local_options, args = local_parser.parse_args()
recover_cmd = GpRecoverSegmentProgram.createProgram(local_options, args)
try:
recover_cmd.run()
except SystemExit as e:
if e.code != 0:
self.logger.error("Failed to start the synchronization step of the segment rebalance.")
self.logger.error("Check the gprecoverseg log file, correct any problems, and re-run")
self.logger.error(' '.join(cmd_args))
raise Exception("Error synchronizing.\nError: %s" % str(e))
finally:
if recover_cmd:
recover_cmd.cleanup()
sys.argv = original_sys_args
self.logger.info("==============================END ANOTHER RECOVER==========================================")
except Exception as ex:
raise ex
finally:
pool.join()
pool.haltWork()
pool.joinWorkers()
signal.signal(signal.SIGINT, signal.default_int_handler)
return allSegmentsStopped # if all segments stopped, then a full rebalance was done
def test_master_panic_after_phase1(self):
"""PANIC master after recording distributed commit.
Trigger PANIC in master after completing phase 1 of 2PC,
right after recording distributed commit in xlog but before
broadcasting COMMIT PREPARED to segments. Master's recovery
cycle should correctly broadcast COMMIT PREPARED because
master should find distributed commit record in its xlog
during recovery. Verify that the transaction is committed
after recovery.
JIRA: MPP-19044
"""
tinctest.logger.info("running test: test_crash_master_after_phase1")
gparray = GpArray.initFromCatalog(dbconn.DbURL(), utility=True)
assert len(gparray.getHostList()) == 1, "cannot run on multi-node"
host = gparray.getHostList()[0]
# Must have at least one in-sync and up segment.
primaries = [
p for p in gparray.get_list_of_primary_segments_on_host(host)
if p.getSegmentMode() == "s" and p.getSegmentStatus() == "u"
]
assert len(primaries) > 0, "in-sync and up primary not found"
primary = primaries[0]
tinctest.logger.info("chose primary: %s" % primary.datadir)
# Inject suspend fault after recording distributed commit on master.
cmd = Command("Suspend master post distributed commit",
self.faultcmd % "suspend")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
# Trigger the fault.
cmd = Command("run DDL",
"psql -f %s" % local_path('sql/ao_create.sql'))
self.proc = cmd.runNoWait()
tinctest.logger.info("runNoWait: %s, pid: %d" %
(cmd.cmdStr, self.proc.pid))
commitBlocked = self.filereputil.check_fault_status(
fault_name='dtm_xlog_distributed_commit',
status="triggered",
seg_id='1',
num_times_hit=1)
# Shutdown of primary (and mirror) should happen only after
# the commit is blocked due to suspend fault.
assert commitBlocked, "timeout waiting for commit to be blocked"
tinctest.logger.info("commit is blocked due to suspend fault")
# At this point, segments have already recorded the
# transaction as prepared by writing PREPARE record in xlog.
# Crash one primary (and its mirror).
mirror = None
mirrors = [
m for m in gparray.get_list_of_mirror_segments_on_host(host)
if m.getSegmentMode() == "s" and m.getSegmentStatus() == "u"
and primary.getSegmentContentId() == m.getSegmentContentId()
]
if len(mirrors) > 0:
mirror = mirrors[0]
tinctest.logger.info("chose mirror: %s" % mirror.datadir)
# Pause FTS probes to avoid a failover while we bring down
# segments. Note that we bring down both primary and its
# mirror, thereby causing double failure. This prevents
# FTS from making changes to segment configuration, even
# if FTS probes are unpaused. It is necessary to unpause
# FTS probes to prevent gang creation from being blocked.
PSQL.run_sql_command_utility_mode("SET gp_fts_probe_pause = on")
tinctest.logger.info("FTS probes paused")
cmdstr = 'pg_ctl -D %s stop -m immediate' % mirror.datadir
tinctest.logger.info("bringing down primary: %s" % cmdstr)
cmd = Command("Shutdown a primary segment", cmdstr)
cmd.run(validateAfter=True)
cmdstr = 'pg_ctl -D %s stop -m immediate' % primary.datadir
tinctest.logger.info("bringing down primary: %s" % cmdstr)
cmd = Command("Shutdown a primary segment", cmdstr)
cmd.run(validateAfter=True)
if mirror is not None:
PSQL.run_sql_command_utility_mode("SET gp_fts_probe_pause = off")
tinctest.logger.info("FTS probes unpaused")
# Resume master. Master should PANIC and go through crash recovery.
cmd = Command("resume master", self.faultcmd % "resume")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
(rc, out, err) = self.proc.communicate2()
self.proc = None
tinctest.logger.info("runNoWait rc: %d, output: %s, err: %s" %
(rc, out, err))
# Fail if QD did not PANIC.
assert (out.find("commit succeeded") == -1
and err.find("commit succeeded") == -1
and err.find("PANIC") != -1)
# Wait for recovery to complete, timeout after ~ 5 mins.
attempts = 1
recoveryComplete = False
while attempts < 600 and not recoveryComplete:
recoveryComplete = "aaa150" in PSQL.run_sql_command_utility_mode(
"select 'aaa' || (100+50)")
time.sleep(0.5)
attempts = attempts + 1
assert recoveryComplete, "timeout waiting for master to recover"
cmdstr = "gpstop -ar"
cmd = Command("restart", cmdstr)
tinctest.logger.info("restarting the cluster with '%s'" % cmdstr)
cmd.run(validateAfter=True)
tinctest.logger.info("restart complete")
# Verify table got created (commit was successful).
assert PSQL.run_sql_file(local_path('sql/ao_select.sql'))
gpverify = GpdbVerify()
(errorCode, hasError, gpcheckcat_output,
repairScript) = gpverify.gpcheckcat()
assert errorCode == 0, ("gpcheckcat failed: %s" % gpcheckcat_output[0])
# No need to restart GPDB again in tearDown()
self.skipRestart = True
def __runStartCommand(self, segments, startMethod, numContentsInCluster,
resultOut, gpArray, era):
"""
Putt results into the resultOut object
"""
if len(segments) == 0:
return
if startMethod == START_AS_PRIMARY_OR_MIRROR:
logger.info(
"Commencing parallel primary and mirror segment instance startup, please wait..."
)
else:
logger.info(
"Commencing parallel segment instance startup, please wait...")
dbIdToPeerMap = gpArray.getDbIdToPeerMap()
mirroringModePreTransition = MIRROR_MODE_MIRRORLESS if startMethod == START_AS_MIRRORLESS else MIRROR_MODE_QUIESCENT
# launch the start
for hostName, segments in GpArray.getSegmentsByHostName(
segments).items():
logger.debug("Dispatching command to start segments on host: %s, " \
"with %s contents in cluster" % (hostName, numContentsInCluster))
pickledTransitionData = None
if startMethod == START_AS_PRIMARY_OR_MIRROR:
mirroringModePerSegment = []
for seg in segments:
modeThisSegment = MIRROR_MODE_PRIMARY if seg.isSegmentPrimary(
True) else MIRROR_MODE_MIRROR
mirroringModePerSegment.append(modeThisSegment)
pickledTransitionData = self.__createPickledTransitionParameters(
segments, mirroringModePerSegment, None, dbIdToPeerMap)
#
# This will call sbin/gpsegstart.py
#
cmd = gp.GpSegStartCmd("remote segment starts on host '%s'" %
hostName,
self.__gpHome,
segments,
self.__gpVersion,
mirroringModePreTransition,
numContentsInCluster,
era,
self.coordinator_checksum_value,
self.__timeout,
verbose=logging_is_verbose(),
ctxt=base.REMOTE,
remoteHost=segments[0].getSegmentAddress(),
pickledTransitionData=pickledTransitionData,
specialMode=self.__specialMode,
wrapper=self.__wrapper,
wrapper_args=self.__wrapper_args,
parallel=self.__parallel,
logfileDirectory=self.logfileDirectory)
self.__workerPool.addCommand(cmd)
if self.__quiet:
self.__workerPool.join()
else:
base.join_and_indicate_progress(self.__workerPool)
# process results
self.__processStartOrConvertCommands(resultOut)
self.__workerPool.empty_completed_items()
def __copySegmentDirectories(self, gpEnv, gpArray, directives):
"""
directives should be composed of GpCopySegmentDirectoryDirective values
"""
if len(directives) == 0:
return
srcSegments = [d.getSrcSegment() for d in directives]
destSegments = [d.getDestSegment() for d in directives]
isTargetReusedLocation = [
d.isTargetReusedLocation() for d in directives
]
destSegmentByHost = GpArray.getSegmentsByHostName(destSegments)
newSegmentInfo = gp.ConfigureNewSegment.buildSegmentInfoForNewSegment(
destSegments, isTargetReusedLocation)
logger.info('Building template directory')
# In GPSQL, we need to create a template and copy it to all of failed segments.
if gpArray.getFaultStrategy() == gparray.FAULT_STRATEGY_NONE:
tempDir = '/tmp/GPSQL'
templateDir = tempDir + '/gpsql_template' + time.strftime(
"%Y%m%d_%H%M%S")
unix.MakeDirectory("create blank directory for segment",
templateDir).run(validateAfter=True)
unix.Chmod.local('set permissions on template dir', templateDir,
'0700') # set perms so postgres can start
logger.info('Creating template')
srcSegments[0].createTemplate(templateDir)
# Don't need log files and gpperfmon files in template.
rmCmd = unix.RemoveFiles(
'gprecoverseg remove gppermfon data from template',
templateDir + '/gpperfmon/data')
rmCmd.run(validateAfter=True)
rmCmd = unix.RemoveFiles('gprecoverseg remove logs from template',
templateDir + '/pg_log/*')
rmCmd.run(validateAfter=True)
#other files not needed
rmCmd = unix.RemoveFiles(
'gprecoverseg remove postmaster.opt from template',
templateDir + '/postmaster.opts')
rmCmd.run(validateAfter=True)
rmCmd = unix.RemoveFiles(
'gprecoverseg remove postmaster.pid from template',
templateDir + '/postmaster.pid')
rmCmd.run(validateAfter=True)
# template the temporary directories file
template_temporary_directories(templateDir, srcSegments[0].content)
tarFileName = "gpsqlSegmentTemplate.tar"
blankTarFile = tempDir + "/" + tarFileName
cmd = gp.CreateTar('gpbuildingmirrorsegment tar segment template',
templateDir, blankTarFile)
cmd.run(validateAfter=True)
def createConfigureNewSegmentCommand(hostName, cmdLabel,
validationOnly):
segmentInfo = newSegmentInfo[hostName]
checkNotNone("segmentInfo for %s" % hostName, segmentInfo)
return gp.ConfigureNewSegment(cmdLabel,
segmentInfo,
tarFile=tarFileName,
newSegments=True,
verbose=gplog.logging_is_verbose(),
batchSize=self.__parallelDegree,
ctxt=gp.REMOTE,
remoteHost=hostName,
validationOnly=validationOnly)
#
# validate directories for target segments
#
logger.info('Validating remote directories')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(
createConfigureNewSegmentCommand(hostName,
'validate blank segments',
True))
for cmd in cmds:
self.__pool.addCommand(cmd)
self.__pool.wait_and_printdots(len(cmds), self.__quiet)
validationErrors = []
for item in self.__pool.getCompletedItems():
results = item.get_results()
if not results.wasSuccessful():
if results.rc == 1:
# stdoutFromFailure = results.stdout.replace("\n", " ").strip()
lines = results.stderr.split("\n")
for line in lines:
if len(line.strip()) > 0:
validationErrors.append(
"Validation failure on host %s %s" %
(item.remoteHost, line))
else:
validationErrors.append(str(item))
self.__pool.empty_completed_items()
if validationErrors:
raise ExceptionNoStackTraceNeeded("\n" +
("\n".join(validationErrors)))
#
# copy tar from master to target hosts
#
logger.info('Copying template directory file')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(
gp.RemoteCopy("copy segment tar", blankTarFile, hostName,
tarFileName))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds, "building and transferring basic segment directory")
#
# unpack and configure new segments
#
logger.info('Configuring new segments')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(
createConfigureNewSegmentCommand(hostName,
'configure blank segments',
False))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds, "unpacking basic segment directory")
#
# Clean up copied tar from each remote host
#
logger.info('Cleaning files')
cmds = []
for hostName, segments in destSegmentByHost.iteritems():
cmds.append(
unix.RemoveFiles('remove tar file',
tarFileName,
ctxt=gp.REMOTE,
remoteHost=hostName))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds, "cleaning up tar file on segment hosts")
#
# clean up the local temp directory
#
unix.RemoveFiles.local('remove temp directory', tempDir)
def __copySegmentDirectories(self, gpEnv, gpArray, directives):
"""
directives should be composed of GpCopySegmentDirectoryDirective values
"""
if len(directives) == 0:
return
srcSegments = []
destSegments = []
isTargetReusedLocation = []
timeStamp = datetime.datetime.today().strftime('%Y%m%d_%H%M%S')
for directive in directives:
srcSegment = directive.getSrcSegment()
destSegment = directive.getDestSegment()
destSegment.primaryHostname = srcSegment.getSegmentHostName()
destSegment.primarySegmentPort = srcSegment.getSegmentPort()
destSegment.progressFile = '%s/pg_basebackup.%s.dbid%s.out' % (
gplog.get_logger_dir(), timeStamp,
destSegment.getSegmentDbId())
srcSegments.append(srcSegment)
destSegments.append(destSegment)
isTargetReusedLocation.append(directive.isTargetReusedLocation())
destSegmentByHost = GpArray.getSegmentsByHostName(destSegments)
newSegmentInfo = gp.ConfigureNewSegment.buildSegmentInfoForNewSegment(
destSegments, isTargetReusedLocation)
def createConfigureNewSegmentCommand(hostName, cmdLabel,
validationOnly):
segmentInfo = newSegmentInfo[hostName]
checkNotNone("segmentInfo for %s" % hostName, segmentInfo)
return gp.ConfigureNewSegment(cmdLabel,
segmentInfo,
gplog.get_logger_dir(),
newSegments=True,
verbose=gplog.logging_is_verbose(),
batchSize=self.__parallelDegree,
ctxt=gp.REMOTE,
remoteHost=hostName,
validationOnly=validationOnly,
forceoverwrite=self.__forceoverwrite)
#
# validate directories for target segments
#
self.__logger.info('Validating remote directories')
cmds = []
for hostName in list(destSegmentByHost.keys()):
cmds.append(
createConfigureNewSegmentCommand(hostName,
'validate blank segments',
True))
for cmd in cmds:
self.__pool.addCommand(cmd)
if self.__quiet:
self.__pool.join()
else:
base.join_and_indicate_progress(self.__pool)
validationErrors = []
for item in self.__pool.getCompletedItems():
results = item.get_results()
if not results.wasSuccessful():
if results.rc == 1:
# stdoutFromFailure = results.stdout.replace("\n", " ").strip()
lines = results.stderr.split("\n")
for line in lines:
if len(line.strip()) > 0:
validationErrors.append(
"Validation failure on host %s %s" %
(item.remoteHost, line))
else:
validationErrors.append(str(item))
self.__pool.empty_completed_items()
if validationErrors:
raise ExceptionNoStackTraceNeeded("\n" +
("\n".join(validationErrors)))
# Configure a new segment
#
# Recover segments using gpconfigurenewsegment, which
# uses pg_basebackup. gprecoverseg generates a log filename which is
# passed to gpconfigurenewsegment as a confinfo parameter. gprecoverseg
# tails this file to show recovery progress to the user, and removes the
# file when one done. A new file is generated for each run of
# gprecoverseg based on a timestamp.
self.__logger.info('Configuring new segments')
cmds = []
progressCmds = []
removeCmds = []
for hostName in list(destSegmentByHost.keys()):
for segment in destSegmentByHost[hostName]:
progressCmd, removeCmd = self.__getProgressAndRemoveCmds(
segment.progressFile, segment.getSegmentDbId(), hostName)
removeCmds.append(removeCmd)
if progressCmd:
progressCmds.append(progressCmd)
cmds.append(
createConfigureNewSegmentCommand(hostName,
'configure blank segments',
False))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds,
"unpacking basic segment directory",
suppressErrorCheck=False,
progressCmds=progressCmds)
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
removeCmds,
"removing pg_basebackup progress logfiles",
suppressErrorCheck=False)
#
# copy dump files from old segment to new segment
#
for srcSeg in srcSegments:
for destSeg in destSegments:
if srcSeg.content == destSeg.content:
src_dump_dir = os.path.join(
srcSeg.getSegmentDataDirectory(), 'db_dumps')
cmd = base.Command('check existence of db_dumps directory',
'ls %s' % (src_dump_dir),
ctxt=base.REMOTE,
remoteHost=destSeg.getSegmentAddress())
cmd.run()
if cmd.results.rc == 0: # Only try to copy directory if it exists
cmd = Scp(
'copy db_dumps from old segment to new segment',
os.path.join(srcSeg.getSegmentDataDirectory(),
'db_dumps*', '*'),
os.path.join(destSeg.getSegmentDataDirectory(),
'db_dumps'),
srcSeg.getSegmentAddress(),
destSeg.getSegmentAddress(),
recursive=True)
cmd.run(validateAfter=True)
break
def setUp(self):
# because gpstart does not have a .py extension,
# we have to use imp to import it
# if we had a gpstart.py, this is equivalent to:
# import gpstart
# self.subject = gpstart
gpstart_file = os.path.abspath(os.path.dirname(__file__) + "/../../../gpstart")
self.subject = imp.load_source('gpstart', gpstart_file)
self.subject.logger = Mock(
spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal', 'warning_to_file_only'])
self.os_environ = dict(COORDINATOR_DATA_DIRECTORY='/tmp/cdd', GPHOME='/tmp/gphome', GP_MGMT_PROCESS_COUNT=1,
LANGUAGE=None)
self.gparray = self._createGpArrayWith2Primary2Mirrors()
self.segments_by_content_id = GpArray.getSegmentsByContentId(self.gparray.getSegDbList())
start_result = StartSegmentsResult()
start_result.addSuccess(self.primary0)
self.apply_patches([
patch('os.getenv', side_effect=self._get_env),
patch('gpstart.os.path.exists'),
patch('gpstart.gp'),
patch('gpstart.pgconf'),
patch('gpstart.unix'),
patch('gpstart.dbconn.DbURL'),
patch('gpstart.dbconn.connect'),
patch('gpstart.GpArray.initFromCatalog', return_value=self.gparray),
patch('gpstart.GpArray.getSegmentsByContentId', return_value=self.segments_by_content_id),
patch('gpstart.GpArray.getSegmentsGroupedByValue',
side_effect=[{2: self.primary0, 3: self.primary1}, [], []]),
patch('gpstart.GpEraFile'),
patch('gpstart.userinput'),
patch('gpstart.HeapChecksum'),
patch('gpstart.log_to_file_only'),
patch("gpstart.StartSegmentsOperation"),
patch("gpstart.base.WorkerPool"),
patch("gpstart.gp.CoordinatorStart.local"),
patch("gpstart.pg.DbStatus.local"),
patch("gpstart.TableLogger"),
])
self.mock_start_result = self.get_mock_from_apply_patch('StartSegmentsOperation')
self.mock_start_result.return_value.startSegments.return_value.getSuccessfulSegments.return_value = start_result.getSuccessfulSegments()
self.mock_os_path_exists = self.get_mock_from_apply_patch('exists')
self.mock_gp = self.get_mock_from_apply_patch('gp')
self.mock_pgconf = self.get_mock_from_apply_patch('pgconf')
self.mock_userinput = self.get_mock_from_apply_patch('userinput')
self.mock_heap_checksum = self.get_mock_from_apply_patch('HeapChecksum')
self.mock_heap_checksum.return_value.get_segments_checksum_settings.return_value = ([1], [1])
self.mock_heap_checksum.return_value.are_segments_consistent.return_value = True
self.mock_heap_checksum.return_value.check_segment_consistency.return_value = ([], [], None)
self.mock_pgconf.readfile.return_value = Mock()
self.mock_gplog_log_to_file_only = self.get_mock_from_apply_patch("log_to_file_only")
self.mock_gp.get_coordinatordatadir.return_value = 'masterdatadir'
self.mock_gp.GpCatVersion.local.return_value = 1
self.mock_gp.GpCatVersionDirectory.local.return_value = 1
self.mock_gp.DEFAULT_GPSTART_NUM_WORKERS = gp.DEFAULT_GPSTART_NUM_WORKERS
sys.argv = ["gpstart"] # reset to relatively empty args list
def rebalance(self):
# Get the unbalanced primary segments grouped by hostname
# These segments are what we will shutdown.
logger.info("Getting unbalanced segments")
unbalanced_primary_segs = GpArray.getSegmentsByHostName(
self.gpArray.get_unbalanced_primary_segdbs())
pool = WorkerPool()
count = 0
try:
# Disable ctrl-c
signal.signal(signal.SIGINT, signal.SIG_IGN)
logger.info("Stopping unbalanced primary segments...")
for hostname in unbalanced_primary_segs.keys():
cmd = GpSegStopCmd("stop unbalanced primary segs",
self.gpEnv.getGpHome(),
self.gpEnv.getGpVersion(),
'fast',
unbalanced_primary_segs[hostname],
ctxt=REMOTE,
remoteHost=hostname,
timeout=600)
pool.addCommand(cmd)
count += 1
pool.wait_and_printdots(count, False)
failed_count = 0
completed = pool.getCompletedItems()
for res in completed:
if not res.get_results().wasSuccessful():
failed_count += 1
if failed_count > 0:
logger.warn(
"%d segments failed to stop. A full rebalance of the")
logger.warn(
"system is not possible at this time. Please check the")
logger.warn(
"log files, correct the problem, and run gprecoverseg -r")
logger.warn("again.")
logger.info(
"gprecoverseg will continue with a partial rebalance.")
pool.empty_completed_items()
# issue a distributed query to make sure we pick up the fault
# that we just caused by shutting down segments
conn = None
try:
logger.info("Triggering segment reconfiguration")
dburl = dbconn.DbURL()
conn = dbconn.connect(dburl)
cmd = ReconfigDetectionSQLQueryCommand(conn)
pool.addCommand(cmd)
pool.wait_and_printdots(1, False)
except Exception:
# This exception is expected
pass
finally:
if conn:
conn.close()
# Final step is to issue a recoverseg operation to resync segments
logger.info("Starting segment synchronization")
cmd = GpRecoverseg("rebalance recoverseg")
pool.addCommand(cmd)
pool.wait_and_printdots(1, False)
except Exception, ex:
raise ex
def _gather_log_from_gp_log_filter(start_time,
end_time=None,
out_file=_DEFAULT_OUT_FILE,
host='localhost',
port=_DEFAULT_PORT,
user=_DEFAULT_USER,
dbname=_DEFAULT_USER,
errors_only=False,
master_only=False):
"""
This retrieves log messages from all segments that happened within the last
'duration' seconds. The format of start_time and end_time is YYYY-MM-DD [hh:mm[:ss]]
The tuples returned are (dbid, hostname, datadir, logdata). sorted by dbid.
Returns True/False based on whether matching log entries were found.
"""
format_start_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(start_time))
if end_time:
format_end_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(end_time))
else:
format_end_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime())
tinctest.logger.info("Collecting log from %s to %s into the file -%s" %
(format_start_time, format_end_time, out_file))
array = GpArray.initFromCatalog(
DbURL(hostname=host, port=port, username=user, dbname=dbname),
True)
log_chunks = []
for seg in array.getDbList():
tinctest.logger.info(
"Collecting log for segment - %s : %s" %
(seg.getSegmentHostName(), seg.getSegmentContentId()))
if master_only and seg.getSegmentContentId() != -1:
continue
cmd = GpLogFilter(
'collect log chunk',
'\\`ls -rt %s | tail -1\\`' %
os.path.join(seg.getSegmentDataDirectory(), 'pg_log', '*.csv'),
start=format_start_time,
end=format_end_time,
trouble=errors_only,
ctxt=REMOTE,
remoteHost=seg.getSegmentHostName())
cmd.run()
rc = cmd.get_results().rc
if rc:
tinctest.logger.warning("Failed command execution %s : %s" %
(cmd, cmd.get_results().stderr))
continue
log_data = cmd.get_results().stdout
if not log_data:
tinctest.logger.warning(
"No log data returned for the given time frame.")
else:
log_chunks.append(
(seg.getSegmentContentId(), seg.getSegmentHostName(),
seg.getSegmentDataDirectory(), log_data))
if log_chunks:
tinctest.logger.info("Writing log data to file - %s" % (out_file))
with open(out_file, 'w') as f:
for part in log_chunks:
f.write("-" * 70)
f.write("\n DBID %s (%s:%s)\n" %
(part[0], part[1], part[2]))
f.write("-" * 70)
f.write("\n%s" % part[3])
f.write("\n\n")
def __ensureMarkedDown(self, gpEnv, toEnsureMarkedDown):
"""Waits for FTS prober to mark segments as down"""
wait_time = 60 * 30 # Wait up to 30 minutes to handle very large, busy
# clusters that may have faults. In most cases the
# actual time to wait will be small and this operation
# is only needed when moving mirrors that are up and
# needed to be stopped, an uncommon operation.
dburl = dbconn.DbURL(port=gpEnv.getCoordinatorPort(),
dbname='template1')
time_elapsed = 0
seg_up_count = 0
initial_seg_up_count = len(toEnsureMarkedDown)
last_seg_up_count = initial_seg_up_count
if initial_seg_up_count == 0:
# Nothing to wait on
return
self.__logger.info("Waiting for segments to be marked down.")
self.__logger.info(
"This may take up to %d seconds on large clusters." % wait_time)
# wait for all needed segments to be marked down by the prober. We'll wait
# a max time of double the interval
while wait_time > time_elapsed:
seg_up_count = 0
current_gparray = GpArray.initFromCatalog(dburl, True)
seg_db_map = current_gparray.getSegDbMap()
# go through and get the status of each segment we need to be marked down
for segdb in toEnsureMarkedDown:
if segdb.getSegmentDbId() in seg_db_map and seg_db_map[
segdb.getSegmentDbId()].isSegmentUp():
seg_up_count += 1
if seg_up_count == 0:
break
else:
if last_seg_up_count != seg_up_count:
print("\n", end=' ')
self.__logger.info(
"%d of %d segments have been marked down." %
(initial_seg_up_count - seg_up_count,
initial_seg_up_count))
last_seg_up_count = seg_up_count
for _i in range(1, 5):
time.sleep(1)
sys.stdout.write(".")
sys.stdout.flush()
time_elapsed += 5
if seg_up_count == 0:
print("\n", end=' ')
self.__logger.info("%d of %d segments have been marked down." %
(initial_seg_up_count, initial_seg_up_count))
else:
raise Exception("%d segments were not marked down by FTS" %
seg_up_count)
def impl(context, filetype, prefix, subdir):
if hasattr(context, 'netbackup_service_host'):
netbackup_service_host = context.netbackup_service_host
if hasattr(context, 'backup_timestamp'):
backup_timestamp = context.backup_timestamp
subdir = subdir.strip()
if len(subdir) > 0:
dump_dir = os.path.join(subdir, 'db_dumps',
'%s' % (backup_timestamp[0:8]))
else:
dump_dir = os.path.join(master_data_dir, 'db_dumps',
'%s' % (backup_timestamp[0:8]))
prefix = prefix.strip()
if len(prefix) > 0:
prefix = prefix + '_'
if filetype == 'report':
#use_dir = get_backup_directory(master_data_dir, subdir, 'db_dumps', backup_timestamp)
filename = "%s/%sgp_dump_%s.rpt" % (dump_dir, prefix, backup_timestamp)
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (
netbackup_service_host, filename)
cmd = Command("Querying NetBackup server for report file", cmd_str)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != filename:
raise Exception(
'Report file %s was not backup up to NetBackup server %s successfully'
% (filename, netbackup_service_host))
elif filetype == 'global':
filename = os.path.join(
dump_dir, "%sgp_global_1_1_%s" % (prefix, backup_timestamp))
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (
netbackup_service_host, filename)
cmd = Command("Querying NetBackup server for global file", cmd_str)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != filename:
raise Exception(
'Global file %s was not backup up to NetBackup server %s successfully'
% (filename, netbackup_service_host))
elif filetype == 'config':
use_dir = get_backup_directory(master_data_dir, subdir, 'db_dumps',
backup_timestamp)
master_config_filename = os.path.join(
dump_dir,
"%sgp_master_config_files_%s.tar" % (prefix, backup_timestamp))
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (
netbackup_service_host, master_config_filename)
cmd = Command("Querying NetBackup server for master config file",
cmd_str)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != master_config_filename:
raise Exception(
'Master config file %s was not backup up to NetBackup server %s successfully'
% (master_config_filename, netbackup_service_host))
master_port = os.environ.get('PGPORT')
gparray = GpArray.initFromCatalog(dbconn.DbURL(port=master_port),
utility=True)
segs = [
seg for seg in gparray.getDbList()
if seg.isSegmentPrimary(current_role=True)
]
for seg in segs:
use_dir = get_backup_directory(seg.getSegmentDataDirectory(),
subdir, 'db_dumps',
backup_timestamp)
seg_config_filename = os.path.join(
use_dir, "%sgp_segment_config_files_0_%d_%s.tar" %
(prefix, seg.getSegmentDbId(), backup_timestamp))
seg_host = seg.getSegmentHostName()
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (
netbackup_service_host, seg_config_filename)
cmd = Command("Querying NetBackup server for segment config file",
cmd_str,
ctxt=REMOTE,
remoteHost=seg_host)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != seg_config_filename:
raise Exception(
'Segment config file %s was not backup up to NetBackup server %s successfully'
% (seg_config_filename, netbackup_service_host))
elif filetype == 'state':
filename = "%s/%sgp_dump_%s_ao_state_file" % (dump_dir, prefix,
backup_timestamp)
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (
netbackup_service_host, filename)
cmd = Command("Querying NetBackup server for AO state file", cmd_str)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != filename:
raise Exception(
'AO state file %s was not backup up to NetBackup server %s successfully'
% (filename, netbackup_service_host))
filename = "%s/%sgp_dump_%s_co_state_file" % (dump_dir, prefix,
backup_timestamp)
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (
netbackup_service_host, filename)
cmd = Command("Querying NetBackup server for CO state file", cmd_str)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != filename:
raise Exception(
'CO state file %s was not backup up to NetBackup server %s successfully'
% (filename, netbackup_service_host))
filename = "%s/%sgp_dump_%s_last_operation" % (dump_dir, prefix,
backup_timestamp)
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (
netbackup_service_host, filename)
cmd = Command(
"Querying NetBackup server for last operation state file", cmd_str)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != filename:
raise Exception(
'Last operation state file %s was not backup up to NetBackup server %s successfully'
% (filename, netbackup_service_host))
elif filetype == 'cdatabase':
filename = "%s/%sgp_cdatabase_1_1_%s" % (dump_dir, prefix,
backup_timestamp)
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (
netbackup_service_host, filename)
cmd = Command("Querying NetBackup server for cdatabase file", cmd_str)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != filename:
raise Exception(
'Cdatabase file %s was not backup up to NetBackup server %s successfully'
% (filename, netbackup_service_host))
def test_fail_back(self):
"""
This test verifies that the fail-back mode is not allowed.
Fail-back means original master acting as the new standby.
"""
# Verify if the database is up. Run some sql.
PSQL.run_sql_command('DROP table if exists foo')
Command('remove standby', 'gpinitstandby -ra').run()
self.assertEqual(self.standby.create(), 0)
res = self.standby.start()
self.assertTrue(res.wasSuccessful())
# Wait for the walreceiver to start
num_walsender = self.wait_for_walsender()
self.assertEqual(num_walsender, 1)
logger.info('Activated WAL Receiver...')
# Promote the standby & shutdown the old Master
# Generate a recovery.conf file for the old Master so
# to make him the new standby that connects to the new
# master (originally standby)
logger.info('Promoting the standby...')
self.standby.promote()
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
numcontent = gparray.getNumSegmentContents()
orig_master = gparray.master
self.standby.remove_catalog_standby(dburl)
if (os.path.exists(os.path.join(orig_master.datadir, 'wal_rcv.pid'))):
os.remove(os.path.join(orig_master.datadir, 'wal_rcv.pid'))
logger.info('Stop the original master...')
cmd = Command("gpstop", "gpstop -aim")
cmd.run()
self.assertEqual(cmd.get_results().rc, 0, str(cmd))
logger.info(
'Generate recovery.conf for original master to make a new standby...'
)
master_recv_conf = open(
os.path.join(orig_master.datadir, 'recovery.conf'), 'w')
standby_recv_done = open(
os.path.join(self.standby.datadir, 'recovery.done'))
for line in standby_recv_done:
master_recv_conf.write(
line.replace("port=" + str(os.environ.get('PGPORT')),
"port=" + str(self.standby.port)))
master_recv_conf.close()
standby_recv_done.close()
logger.info(
'Start the old master again (to act as the new standby)...')
master = gp.MasterStart("Starting orig Master in standby mode",
orig_master.datadir, orig_master.port,
orig_master.dbid, 0, numcontent, None, None,
None)
# -w option would wait forever.
master.cmdStr = master.cmdStr.replace(' -w', '')
master.run(validateAfter=True)
self.assertTrue((master.get_results()).wasSuccessful())
# Have to do this to give the new standby some time to be active
subprocess.check_call("psql -c 'create database foo' -p " +
str(self.standby.port),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
subprocess.check_call("psql -c 'drop database foo' -p " +
str(self.standby.port),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
time.sleep(3)
# The new standby can re-start but should not be able to connect to the new
# master (originally standby). Thats the test
self.assertTrue(
os.path.exists(os.path.join(orig_master.datadir, 'wal_rcv.pid')))
logger.info(
'The WAL receiver pid file exists which means the new standby started\n'
'but still could not connect to the new Master (originally standby) and hence the\n'
'pid file was not cleared')
# Remove the recovery.conf file from the new standby directory
# as its no more needed
os.remove(os.path.join(orig_master.datadir, 'recovery.conf'))
logger.info('Stop the original master again...')
rc = subprocess.Popen('pg_ctl stop -D ' + orig_master.datadir +
' -m immediate',
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
# Perform gpstart to get the original master (& cluster) back again
cmd = Command("gpstart", "gpstart -a")
cmd.run()
self.assertTrue(cmd.get_results().rc in (0, 1), str(cmd))
logger.info('Pass')
def __copySegmentDirectories(self, gpEnv, gpArray, directives):
"""
directives should be composed of GpCopySegmentDirectoryDirective values
"""
if len(directives) == 0:
return
srcSegments = [d.getSrcSegment() for d in directives]
destSegments = [d.getDestSegment() for d in directives]
isTargetReusedLocation = [
d.isTargetReusedLocation() for d in directives
]
destSegmentByHost = GpArray.getSegmentsByHostName(destSegments)
newSegmentInfo = gp.ConfigureNewSegment.buildSegmentInfoForNewSegment(
destSegments, isTargetReusedLocation)
logger.info('Building template directory')
(tempDir, blankTarFile,
tarFileName) = self.__buildTarFileForTransfer(gpEnv, gpArray.master,
srcSegments[0],
destSegments)
def createConfigureNewSegmentCommand(hostName, cmdLabel,
validationOnly):
segmentInfo = newSegmentInfo[hostName]
checkNotNone("segmentInfo for %s" % hostName, segmentInfo)
return gp.ConfigureNewSegment(cmdLabel,
segmentInfo,
tarFile=tarFileName,
newSegments=True,
verbose=gplog.logging_is_verbose(),
batchSize=self.__parallelDegree,
ctxt=gp.REMOTE,
remoteHost=hostName,
validationOnly=validationOnly)
#
# validate directories for target segments
#
logger.info('Validating remote directories')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(
createConfigureNewSegmentCommand(hostName,
'validate blank segments',
True))
for cmd in cmds:
self.__pool.addCommand(cmd)
self.__pool.wait_and_printdots(len(cmds), self.__quiet)
validationErrors = []
for item in self.__pool.getCompletedItems():
results = item.get_results()
if not results.wasSuccessful():
if results.rc == 1:
# stdoutFromFailure = results.stdout.replace("\n", " ").strip()
lines = results.stderr.split("\n")
for line in lines:
if len(line.strip()) > 0:
validationErrors.append(
"Validation failure on host %s %s" %
(item.remoteHost, line))
else:
validationErrors.append(str(item))
self.__pool.empty_completed_items()
if validationErrors:
raise ExceptionNoStackTraceNeeded("\n" +
("\n".join(validationErrors)))
#
# copy tar from master to target hosts
#
logger.info('Copying template directory file')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(
gp.RemoteCopy("copy segment tar", blankTarFile, hostName,
tarFileName))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds, "building and transferring basic segment directory")
#
# unpack and configure new segments
#
logger.info('Configuring new segments')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(
createConfigureNewSegmentCommand(hostName,
'configure blank segments',
False))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds, "unpacking basic segment directory")
#
# copy dump files from old segment to new segment
#
for srcSeg in srcSegments:
for destSeg in destSegments:
if srcSeg.content == destSeg.content:
cmd = Scp('copy db_dumps from old segment to new segment',
os.path.join(srcSeg.getSegmentDataDirectory(),
'db_dumps*', '*'),
os.path.join(destSeg.getSegmentDataDirectory(),
'db_dumps'),
srcSeg.getSegmentAddress(),
destSeg.getSegmentAddress(),
recursive=True)
cmd.run(validateAfter=True)
break
#
# Clean up copied tar from each remote host
#
logger.info('Cleaning files')
cmds = []
for hostName, segments in destSegmentByHost.iteritems():
cmds.append(
unix.RemoveFiles('remove tar file',
tarFileName,
ctxt=gp.REMOTE,
remoteHost=hostName))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds, "cleaning up tar file on segment hosts")
#
# clean up the local temp directory
#
unix.RemoveFiles.local('remove temp directory', tempDir)
except NoStatusFile, e:
logger.warn('Status file %s not found on master' % status_file)
return {'exit_status': 1, 'timestamp': timestamp}
except StatusFileError, e:
logger.warn('Status file %s on master indicates errors' %
status_file)
return {'exit_status': 1, 'timestamp': timestamp}
except NoDumpFile, e:
logger.warn('Dump file %s not found on master' % dump_file)
return {'exit_status': 1, 'timestamp': timestamp}
else:
logger.info('Checked master status file and master dump file.')
# Perform similar checks for primary segments
operations = []
gparray = GpArray.initFromCatalog(dbconn.DbURL(port=self.master_port),
utility=True)
segs = [
seg for seg in gparray.getDbList()
if seg.isSegmentPrimary(current_role=True)
]
for seg in segs:
path = self.backup_dir if self.backup_dir is not None else seg.getSegmentDataDirectory(
)
path = os.path.join(path, DUMP_DIR, DUMP_DATE)
status_file = os.path.join(
path, "%s%d_%s" %
(SEG_STATUS_PREFIX, seg.getSegmentDbId(), timestamp))
dump_file = os.path.join(
path, "%s%d_%s" %
(SEG_DBDUMP_PREFIX, seg.getSegmentDbId(), timestamp))
if self.compress: dump_file += ".gz"
