def execute(self):
ValidateGpToolkit(database = self.dump_database,
master_port = self.master_port).run()
operations = []
gparray = GpArray.initFromCatalog(dbconn.DbURL(port = self.master_port), utility=True)
segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary(current_role=True)]
for seg in segs:
operations.append(RemoteOperation(ValidateSegDiskSpace(free_space_percent = self.free_space_percent,
compress = self.compress,
dump_database = self.dump_database,
include_dump_tables = self.include_dump_tables,
datadir = seg.getSegmentDataDirectory(),
segport = seg.getSegmentPort()),
seg.getSegmentHostName()))
ParallelOperation(operations, self.batch_default).run()
success = 0
for remote in operations:
host = remote.host
try:
remote.get_ret()
except NotEnoughDiskSpace, e:
logger.error("%s has insufficient disk space. [Need: %dK, Free %dK]" % (host, e.needed_space, e.free_space))
else:
success += 1
def create(self):
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
if gparray.standbyMaster:
self.remove_catalog_standby(dburl)
self.add_catalog_standby(dburl, gparray)
shutil.rmtree(self.datadir, True)
rc = subprocess.call(
['pg_basebackup', '-x', '-R', '-D', self.datadir])
# this is ugly, but required by gpstart, who determines
# port number from postgresql.conf and removes UNIX socket file.
postgresql_conf = os.path.join(self.datadir, 'postgresql.conf')
conf_tmp = open(postgresql_conf + '.tmp', 'w')
with open(postgresql_conf) as conf:
for line in conf.readlines():
if "port=" in line:
line = "port={0} #{1}".format(self.port, line)
conf_tmp.write(line)
conf_tmp.close()
os.rename(postgresql_conf + '.tmp', postgresql_conf)
self.update_primary_pg_hba()
return rc
def get_seginfo_for_primaries(self):
seglist = []
gparray = GpArray.initFromCatalog(dbconn.DbURL())
for seg in gparray.getDbList():
if seg.isSegmentPrimary() and not seg.isSegmentMaster():
seglist.append(seg)
return seglist
def test_filespace(self):
"""
pg_basebackup should work with user filespace.
@tags sanity
"""
# Add standby entry in catalog before regisering filespace.
fsbase = os.path.join(self.fsprefix, 'fs')
shutil.rmtree(fsbase, True)
os.makedirs(fsbase)
shutil.rmtree(self.standby_datadir, True)
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
if gparray.standbyMaster:
self.standby.remove_catalog_standby(dburl)
self.standby.add_catalog_standby(dburl, gparray)
#self.preprocess_file(local_path('filespace.sql.in'))
sql_file = local_path('filespace.sql')
from mpp.lib.gpfilespace import Gpfilespace
gpfile = Gpfilespace()
gpfile.create_filespace('fs_walrepl_a')
result = PSQL.run_sql_file(sql_file, dbname=self.db_name)
self.assertTrue(result)
subprocess.check_call(['pg_basebackup', '-D', self.standby_datadir])
#fsdir = os.path.join(self.fsprefix, 'fs', 'gpdb1')
fsdir = os.path.join(os.path.split(self.standby_datadir)[0], 'fs_walrepl_a','mirror','pg_system')
self.assertTrue(os.path.isdir(fsdir),
'{0} does not dir'.format(fsdir))
def execute(self):
gparray = GpArray.initFromCatalog(dbconn.DbURL(port = self.master_port), utility=True)
failed_segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary(current_role=True) and seg.isSegmentDown()]
if len(failed_segs) != 0:
logger.warn("Failed primary segment instances detected")
failed_dbids = [seg.getSegmentDbid() for seg in failed_segs]
raise ExceptionNoStackTraceNeeded("Detected failed segment(s) with dbid=%s" % ",".join(failed_dbids))
def are_segments_running():
gparray = GpArray.initFromCatalog(dbconn.DbURL())
segments = gparray.getDbList()
for seg in segments:
if seg.status != 'u':
return False
return True
def execute(self):
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl)
my_fault_strategy = gparray.getFaultStrategy()
if my_fault_strategy != FAULT_STRATEGY_FILE_REPLICATION:
raise NoMirroringError(
'Fault strategy %s does not support mirror verification.' %
FAULT_STRATEGY_LABELS[my_fault_strategy])
if self.content is not None:
contents = set(
[seg.getSegmentContentId() for seg in gparray.getDbList()])
if self.content not in contents:
raise InvalidContentIDError(self.content)
logger.info('Validating target contents...')
to_verify = [x for x in gparray.getDbList() if x.isSegmentQE()]
if self.content is not None:
to_verify = [
x for x in to_verify if x.getSegmentContentId() == self.content
]
if self.primaries_only:
to_verify = [
x for x in to_verify if x.isSegmentPrimary(current_role=True)
]
return to_verify
def preprocess_file(self, in_path):
"""
Proprocess .in files, to match enviroment. Currently CREATE FILESPACE
will be substituted.
e.g.
CREATE FILESPACE fs();
will be completed with dbid/directory path.
"""
dburl = gppylib.db.dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
assert(in_path.endswith('.in'))
out_path = in_path[:-3]
pat = re.compile(r'CREATE\s+FILESPACE\s+(\w+)', re.I)
with open(out_path, 'w') as out_file:
for line in open(in_path):
m = pat.match(line)
if m:
# The file paths will look like
# <fsprefix>/<fsname>/gpdb<dbid>
fsname = m.group(1)
basepath = os.path.join(self.fsprefix, fsname)
if not os.path.exists(basepath):
os.makedirs(basepath)
buf = list()
for db in gparray.getDbList():
fselocation = os.path.join(basepath,
'gpdb' + str(db.dbid))
buf.append("{0}: '{1}'".format(
db.dbid, fselocation))
line = "CREATE FILESPACE {0}({1});\n".format(
fsname, ', '.join(buf))
out_file.write(line)
def are_segments_synchronized():
gparray = GpArray.initFromCatalog(dbconn.DbURL())
segments = gparray.getDbList()
for seg in segments:
if seg.mode != MODE_SYNCHRONIZED and not seg.isSegmentMaster(True):
return False
return True
def impl(context):
if hasattr(context, 'backup_timestamp'):
ts = context.backup_timestamp
if hasattr(context, 'netbackup_service_host'):
netbackup_service_host = context.netbackup_service_host
if not hasattr(context, "dump_prefix"):
context.dump_prefix = ''
master_config_filename = os.path.join(master_data_dir, 'db_dumps', context.backup_timestamp[0:8],
'%sgp_master_config_files_%s.tar' % (context.dump_prefix, ts))
command_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (netbackup_service_host, master_config_filename)
cmd = Command('Validate master config file', command_str)
cmd.run(validateAfter=True)
results = cmd.get_results().stdout.strip()
if results != master_config_filename:
raise Exception('Expected Master Config file: %s and found: %s. Master Config file was not backup up to NetBackup server' % (master_config_filename, results))
gparray = GpArray.initFromCatalog(dbconn.DbURL())
primary_segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary(current_role=True)]
for seg in primary_segs:
segment_config_filename = os.path.join(seg.getSegmentDataDirectory(), 'db_dumps', context.backup_timestamp[0:8],
'%sgp_segment_config_files_0_%s_%s.tar' % (context.dump_prefix, seg.getSegmentDbId(), context.backup_timestamp))
command_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (netbackup_service_host, segment_config_filename)
cmd = Command('Validate segment config file', command_str, ctxt=REMOTE, remoteHost = seg.getSegmentHostName())
cmd.run(validateAfter=True)
results = cmd.get_results().stdout.strip()
if results != segment_config_filename:
raise Exception('Expected Segment Config file: %s and found: %s. Segment Config file was not backup up to NetBackup server' % (segment_config_filename, results))
def impl(context, filetype, dir):
if filetype == "report":
if dir == "master_data_directory":
dir = master_data_dir
filenames = os.listdir(dir)
for filename in filenames:
if filename.startswith("gp_restore") and filename.endswith(".rpt"):
filename = "%s/%s" % (dir, filename)
os.remove(filename)
if filetype == "status":
gparray = GpArray.initFromCatalog(dbconn.DbURL())
if dir == "segment_data_directory":
primary_segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary()]
for seg in primary_segs:
host = seg.getSegmentHostName()
seg_data_dir = seg.getSegmentDataDirectory()
cmd = Command(
"remove status file", "rm -f %s/gp_restore_status_*" % (seg_data_dir), ctxt=REMOTE, remoteHost=host
)
cmd.run(validateAfter=True)
else:
primary_segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary()]
for seg in primary_segs:
host = seg.getSegmentHostName()
cmd = Command("remove status file", "rm -f %s/gp_restore_status_*" % dir, ctxt=REMOTE, remoteHost=host)
cmd.run(validateAfter=True)
def impl(context, seg):
if seg == "mirror":
gparray = GpArray.initFromCatalog(dbconn.DbURL())
mirror_segs = [seg for seg in gparray.getDbList() if seg.isSegmentMirror() and seg.getSegmentHostName() != platform.node()]
context.remote_mirror_segdbId = mirror_segs[0].getSegmentDbId()
context.remote_mirror_segdbname = mirror_segs[0].getSegmentHostName()
context.remote_mirror_datadir = mirror_segs[0].getSegmentDataDirectory()
def get_host_list():
"""
Returns a tuple which consists of the standby
and segment hosts comprising the cluster.
@return: tuple containing the standby and segment hosts
tuple[0] contains standby
tuple[1] contains segment hosts
"""
gparr = GpArray.initFromCatalog(dbconn.DbURL(port = MASTER_PORT), utility = True)
segs = gparr.getDbList()
master = None
standby_host = None
segment_host_list = []
for seg in segs:
if seg.isSegmentStandby(current_role=True):
standby_host = seg.getSegmentHostName()
elif not seg.isSegmentMaster(current_role=True):
segment_host_list.append(seg.getSegmentHostName())
elif seg.isSegmentMaster(current_role=True):
master = seg.getSegmentHostName()
#Deduplicate the hosts so that we
#dont install multiple times on the same host
segment_host_list = list(set(segment_host_list))
if master in segment_host_list:
segment_host_list.remove(master)
return (standby_host, segment_host_list)
def impl(context, filetype, dir):
if dir == 'master_data_directory':
dir = master_data_dir
if filetype == 'report':
filename = '%s/gp_restore_%s.rpt' % (dir, context.backup_timestamp)
if not os.path.isfile(filename):
raise Exception('Report file %s is not present in master data directory' % filename)
elif filetype == 'status':
gparray = GpArray.initFromCatalog(dbconn.DbURL())
if dir == 'segment_data_directory':
primary_segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary()]
for seg in primary_segs:
host = seg.getSegmentHostName()
seg_data_dir = seg.getSegmentDataDirectory()
cmd = Command('check status file', "ls %s/gp_restore_status_*_%s" % (seg_data_dir, context.backup_timestamp), ctxt=REMOTE, remoteHost=host)
cmd.run(validateAfter=True)
results = cmd.get_results()
if not results.stdout.strip():
raise Exception('Status file ending with timestamp %s is not present in segment %s data directory' % (context.backup_timestamp, host))
else:
count = 0
primary_segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary()]
for seg in primary_segs:
host = seg.getSegmentHostName()
cmd = Command('check status file', "ls %s/gp_restore_status_*_%s" % (dir, context.backup_timestamp), ctxt=REMOTE, remoteHost=host)
cmd.run(validateAfter=True)
results = cmd.get_results()
if results.stdout.strip():
count += 1
else:
raise Exception('Status file not found in segment: %s' % host)
segs = len(primary_segs)
if count != segs:
raise Exception('Expected %d status file but found %d' % (segs, count))
def execute(self):
gparray = GpArray.initFromCatalog(dbconn.DbURL(port=self.master_port), utility=True)
from_host, from_path = self.host, self.path
logger.info("Commencing remote database dump file recovery process, please wait...")
segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary(current_role=True) or seg.isSegmentMaster()]
pool = WorkerPool(numWorkers = min(len(segs), self.batch_default))
for seg in segs:
if seg.isSegmentMaster():
file = '%s%s' % (MASTER_DBDUMP_PREFIX, self.restore_timestamp)
else:
file = '%s0_%d_%s' % (DBDUMP_PREFIX, seg.getSegmentDbId(), self.restore_timestamp)
if self.compress:
file += '.gz'
to_host = seg.getSegmentHostName()
to_path = os.path.join(seg.getSegmentDataDirectory(), DUMP_DIR, self.restore_timestamp[0:8])
if not CheckRemoteDir(to_path, to_host).run():
logger.info('Creating directory %s on %s' % (to_path, to_host))
try:
MakeRemoteDir(to_path, to_host).run()
except OSError, e:
raise ExceptionNoStackTraceNeeded("Failed to create directory %s on %s" % (to_path, to_host))
logger.info("Commencing remote copy from %s to %s:%s" % (from_host, to_host, to_path))
pool.addCommand(Scp('Copying dump for seg %d' % seg.getSegmentDbId(),
srcFile=os.path.join(from_path, file),
dstFile=os.path.join(to_path, file),
srcHost=from_host,
dstHost=to_host))
def execute(self):
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
config_backup_file = "gp_master_config_files_%s.tar" % timestamp
if self.backup_dir is not None:
path = os.path.join(self.backup_dir, DUMP_DIR, DUMP_DATE, config_backup_file)
else:
path = os.path.join(self.master_datadir, DUMP_DIR, DUMP_DATE, config_backup_file)
logger.info("Dumping master config files")
Command("Dumping master configuration files",
"tar cf %s %s/*.conf" % (path, self.master_datadir)).run(validateAfter=True)
logger.info("Dumping segment config files")
gparray = GpArray.initFromCatalog(dbconn.DbURL(port = self.master_port), utility=True)
primaries = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary(current_role=True)]
for seg in primaries:
config_backup_file = "gp_segment_config_files_0_%d_%s.tar" % (seg.getSegmentDbId(), timestamp)
if self.backup_dir is not None:
path = os.path.join(self.backup_dir, DUMP_DIR, DUMP_DATE, config_backup_file)
else:
path = os.path.join(seg.getSegmentDataDirectory(), DUMP_DIR, DUMP_DATE, config_backup_file)
host = seg.getSegmentHostName()
Command("Dumping segment config files",
"tar cf %s %s/*.conf" % (path, seg.getSegmentDataDirectory()),
ctxt=REMOTE,
remoteHost=host).run(validateAfter=True)
def impl(context):
gparray = GpArray.initFromCatalog(dbconn.DbURL())
for seg in gparray.getDbList():
if seg.isSegmentPrimary() and seg.getSegmentContentId() == context.remote_mirror_segcid:
context.remote_pair_primary_segdbId = seg.getSegmentDbId()
context.remote_pair_primary_datadir = seg.getSegmentDataDirectory()
context.remote_pair_primary_port = seg.getSegmentPort()
context.remote_pair_primary_host = seg.getSegmentHostName()
def test_init_standby(self):
gparr = GpArray.initFromCatalog(dbconn.DbURL(dbname=DEFAULT_DATABASE))
if not gparr.standbyMaster:
standby_host = self.create_standby_master(gparr)
else:
self.remove_standby_master(gparr)
standby_host = self.create_standby_master(gparr)
self.verify_standby_ips(gparr, standby_host)
def impl(context, output):
gparray = GpArray.initFromCatalog(dbconn.DbURL())
segments = gparray.getDbList()
for segment in segments:
if segment.isSegmentMirror():
expected = r'\(dbid {}\): {}'.format(segment.dbid, output)
check_stdout_msg(context, expected)
def __ensureMarkedDown(self, gpEnv, toEnsureMarkedDown):
"""Waits for FTS prober to mark segments as down"""
wait_time = 60 * 30 # Wait up to 30 minutes to handle very large, busy
# clusters that may have faults. In most cases the
# actual time to wait will be small and this operation
# is only needed when moving mirrors that are up and
# needed to be stopped, an uncommon operation.
dburl = dbconn.DbURL(port=gpEnv.getMasterPort(), dbname="template1")
time_elapsed = 0
seg_up_count = 0
initial_seg_up_count = len(toEnsureMarkedDown)
last_seg_up_count = initial_seg_up_count
if initial_seg_up_count == 0:
# Nothing to wait on
return
logger.info("Waiting for segments to be marked down.")
logger.info("This may take up to %d seconds on large clusters." % wait_time)
# wait for all needed segments to be marked down by the prober. We'll wait
# a max time of double the interval
while wait_time > time_elapsed:
seg_up_count = 0
current_gparray = GpArray.initFromCatalog(dburl, True)
seg_db_map = current_gparray.getSegDbMap()
# go through and get the status of each segment we need to be marked down
for segdb in toEnsureMarkedDown:
if segdb.getSegmentDbId() in seg_db_map and seg_db_map[segdb.getSegmentDbId()].isSegmentUp() == True:
seg_up_count += 1
if seg_up_count == 0:
break
else:
if last_seg_up_count != seg_up_count:
print "\n",
logger.info(
"%d of %d segments have been marked down."
% (initial_seg_up_count - seg_up_count, initial_seg_up_count)
)
last_seg_up_count = seg_up_count
for _i in range(1, 5):
time.sleep(1)
sys.stdout.write(".")
sys.stdout.flush()
time_elapsed += 5
if seg_up_count == 0:
print "\n",
logger.info("%d of %d segments have been marked down." % (initial_seg_up_count, initial_seg_up_count))
else:
raise Exception("%d segments were not marked down by FTS" % seg_up_count)
def _gather_log_from_gp_log_filter(start_time, end_time=None, out_file=_DEFAULT_OUT_FILE, host='localhost',
port=_DEFAULT_PORT, user=_DEFAULT_USER, dbname=_DEFAULT_USER, errors_only=False, master_only=False):
"""
This retrieves log messages from all segments that happened within the last
'duration' seconds. The format of start_time and end_time is YYYY-MM-DD [hh:mm[:ss]]
The tuples returned are (dbid, hostname, datadir, logdata). sorted by dbid.
Returns True/False based on whether matching log entries were found.
"""
format_start_time = time.strftime("%Y-%m-%dT%H:%M:%S",time.localtime(start_time))
if end_time:
format_end_time = time.strftime("%Y-%m-%dT%H:%M:%S",time.localtime(end_time))
else:
format_end_time = time.strftime("%Y-%m-%dT%H:%M:%S",time.localtime())
tinctest.logger.info("Collecting log from %s to %s into the file -%s" % (format_start_time,format_end_time, out_file))
array = GpArray.initFromCatalog(DbURL(hostname=host, port=port, username=user, dbname=dbname), True)
log_chunks = []
for seg in array.getDbList():
tinctest.logger.info("Collecting log for segment - %s : %s" %(seg.getSegmentHostName(), seg.getSegmentContentId()))
if master_only and seg.getSegmentContentId() != -1:
continue
cmd = GpLogFilter('collect log chunk',
'\\`ls -rt %s | tail -1\\`' % os.path.join(seg.getSegmentDataDirectory(), 'pg_log', '*.csv'),
start=format_start_time, end=format_end_time,
trouble=errors_only,
ctxt=REMOTE,
remoteHost=seg.getSegmentHostName())
cmd.run()
rc = cmd.get_results().rc
if rc:
tinctest.logger.warning("Failed command execution %s : %s" %(cmd, cmd.get_results().stderr))
continue
log_data = cmd.get_results().stdout
if not log_data:
tinctest.logger.warning("No log data returned for the given time frame.")
else:
log_chunks.append((seg.getSegmentContentId(),
seg.getSegmentHostName(),
seg.getSegmentDataDirectory(),
log_data))
if log_chunks:
tinctest.logger.info("Writing log data to file - %s" %(out_file))
with open(out_file, 'w') as f:
for part in log_chunks:
f.write("-"*70)
f.write("\n DBID %s (%s:%s)\n" % (part[0], part[1], part[2]))
f.write("-"*70)
f.write("\n%s" % part[3])
f.write("\n\n")
def verify_timestamps_on_segments_with_nbu(timestamp):
gparray = GpArray.initFromCatalog(dbconn.DbURL())
primary_segs = [segdb for segdb in gparray.getDbList() if segdb.isSegmentPrimary()]
for seg in primary_segs:
db_dumps_dir = os.path.join(seg.getSegmentDataDirectory(), "db_dumps", timestamp[:8])
list_cmd = "ls -l %s/*%s* | wc -l" % (db_dumps_dir, timestamp)
cmd = Command("get list of dump files", list_cmd, ctxt=REMOTE, remoteHost=seg.getSegmentHostName())
cmd.run(validateAfter=True)
verify_num_files_with_nbu(cmd.get_results(), "1", timestamp)
def setUp(self):
super(XidlimitsTests, self).setUp()
Command('re-build regress.so',
'make -C %s xidhelper.so' % local_path('.')).run(validateAfter=True)
SQLTemplateTestCase.perform_transformation_on_sqlfile(local_path('load_xidhelper.sql'),
local_path('load_xidhelper.sql.t'),
{'@source@' : local_path('xidhelper.so')})
PSQL.run_sql_file(sql_file=local_path('load_xidhelper.sql.t'),
out_file=local_path('load_xidhelper.out.t'))
self.gparray = GpArray.initFromCatalog(dbconn.DbURL(), utility=True)
def __init__(self, methodName):
self.gpact = GpactivateStandby()
self.host = socket.gethostname()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.pgport = os.environ.get('PGPORT')
self.port = self.pgport
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
self.numcontent = gparray.getNumSegmentContents()
self.orig_master = gparray.master
self.standby_pid = ''
super(GpactivateStandbyTestCase,self).__init__(methodName)
def __init__(self):
self.gpinit = GpinitStandby()
self.pgutil = GpUtility()
self.runmixin = StandbyRunMixin()
self.runmixin.createdb(dbname='walrepl')
self.gphome = os.environ.get('GPHOME')
self.pgport = os.environ.get('PGPORT')
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.config = GPDBConfig()
self.host = socket.gethostname()
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
self.numcontent = gparray.getNumSegmentContents()
self.orig_master = gparray.master
def cleanup(self):
with dbconn.connect(dbconn.DbURL(dbname="postgres")) as conn:
db = pg.DB(conn)
db.query("DROP DATABASE IF EXISTS %s" % self.dbname)
db.query("DROP TABLESPACE IF EXISTS %s" % self.name)
# Without synchronous_commit = 'remote_apply' introduced in 9.6, there
# is no guarantee that the mirrors have removed their tablespace
# directories by the time the DROP TABLESPACE command returns.
# We need those directories to no longer be in use by the mirrors
# before removing them below.
_checkpoint_and_wait_for_replication_replay(db)
gparray = GpArray.initFromCatalog(dbconn.DbURL())
for host in gparray.getHostList():
run_cmd('ssh %s rm -rf %s' % (pipes.quote(host), pipes.quote(self.path)))
def test_from_not_running(self):
"""
Test gpactivatestandby -f while standby is not running.
"""
dburl = dbconn.DbURL()
array = GpArray.initFromCatalog(dburl, utility=True)
self.standby_host = array.master.getSegmentHostName()
# create standby locally
shutil.rmtree(self.basepath, True)
try:
os.makedirs(self.basepath)
except OSError, e:
if e.errno != 17:
raise
pass
def cleanupFilespaces(dbname):
"""
Deletes filespaces except system filespace along with tablespace
associated with the filespaces and tables, then delete the local
directories for them. The dbname parameter is used to drop tables
under non-system filespace, and this works only if the tables belong
to this database. Needs improvement.
"""
dburl = gppylib.db.dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
with DbConn(dbname=dbname) as conn:
cleanupTablespaces(conn)
for fs in gparray.getFilespaces(includeSystemFilespace=False):
conn.execute("DROP FILESPACE {0}".format(fs.getName()))
def execute(self):
""" TODO: Improve with grouping by host and ParallelOperation dispatch. """
gparray = GpArray.initFromCatalog(dbconn.DbURL(port=self.master_port), utility=True)
primaries = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary(current_role=True)]
dump_count = 0
for seg in primaries:
if seg.isSegmentDown():
""" Why must every Segment function have the word Segment in it ?! """
raise ExceptionNoStackTraceNeeded("Host %s dir %s dbid %d marked as invalid" % (seg.getSegmentHostName(), seg.getSegmentDataDirectory(), seg.getSegmentDbId()))
path = os.path.join(seg.getSegmentDataDirectory(), DUMP_DIR, self.restore_timestamp[0:8])
host = seg.getSegmentHostName()
path = os.path.join(path, "%s0_%d_%s" % (DBDUMP_PREFIX, seg.getSegmentDbId(), self.restore_timestamp))
if self.compress:
path += ".gz"
exists = CheckRemoteFile(path, host).run()
if not exists:
raise ExceptionNoStackTraceNeeded("No dump file on %s at %s" % (seg.getSegmentHostName(), path))
def start(self):
"""
Start the standby postmaster. The options to pg_ctl needs to be
determined by gppylib logic.
"""
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
numcontent = gparray.getNumSegmentContents()
standby = gparray.standbyMaster
master = gp.MasterStart("Starting Master Standby",
self.datadir, self.port, standby.dbid,
0, numcontent, None, None, None)
# -w option would wait forever.
master.cmdStr = master.cmdStr.replace(' -w', '')
master.run(validateAfter=True)
return master.get_results()
def _get_gpdb_host_list(self):
"""
TODO: Perhaps the host list should be produced by gparray instead of here.
This method gets the host names
of all hosts in the gpdb array.
It sets the following variables
GpPkgProgram.coordinator_host to coordinator
GpPkgProgram.standby_host to standby
GpPkgProgram.segment_host_list to segment hosts
"""
logger.debug('_get_gpdb_host_list')
gparr = GpArray.initFromCatalog(dbconn.DbURL(port = self.coordinator_port), utility = True)
coordinator_host = None
standby_host = None
segment_host_list = []
segs = gparr.getDbList()
for seg in segs:
if seg.isSegmentCoordinator(current_role = True):
coordinator_host = seg.getSegmentHostName()
elif seg.isSegmentStandby(current_role = True):
standby_host = seg.getSegmentHostName()
else:
segment_host_list.append(seg.getSegmentHostName())
# Deduplicate the hosts so that we
# dont install multiple times on the same host
segment_host_list = list(set(segment_host_list))
# Segments might exist on the coordinator host. Since we store the
# coordinator host separately in self.coordinator_host, storing the coordinator_host
# in the segment_host_list is redundant.
for host in segment_host_list:
if host == coordinator_host or host == standby_host:
segment_host_list.remove(host)
self.coordinator_host = coordinator_host
self.standby_host = standby_host
self.segment_host_list = segment_host_list
def impl(context, seg):
gparray = GpArray.initFromCatalog(dbconn.DbURL())
if seg == "primary":
primary_segs = [
seg for seg in gparray.getDbList() if seg.isSegmentPrimary()
]
context.pseg = primary_segs[0]
context.pseg_data_dir = context.pseg.getSegmentDataDirectory()
context.pseg_hostname = context.pseg.getSegmentHostName()
context.pseg_dbid = context.pseg.getSegmentDbId()
elif seg == "mirror":
mirror_segs = [
seg for seg in gparray.getDbList() if seg.isSegmentMirror()
]
context.mseg = mirror_segs[0]
context.mseg_hostname = context.mseg.getSegmentHostName()
context.mseg_dbid = context.mseg.getSegmentDbId()
context.mseg_data_dir = context.mseg.getSegmentDataDirectory()
def loadSystemConfig( self, useUtilityMode ) :
"""
Load all segment information from the configuration source.
Returns a new GpArray object
"""
# ensure initializeProvider() was called
checkNotNone("masterDbUrl", self.__masterDbUrl)
logger.info("Obtaining Segment details from master...")
array = GpArray.initFromCatalog(self.__masterDbUrl, useUtilityMode)
if get_local_db_mode(array.master.getSegmentDataDirectory()) != 'UTILITY':
logger.debug("Validating configuration...")
if not array.is_array_valid():
raise InvalidSegmentConfiguration(array)
return array
def testReadPostmasterTempFile(self):
logger.info("testReadPostmasterTempFile")
url = dbconn.DbURL()
gpdb = GpArray.initFromCatalog(url)
logger.info("Search for valid master port: %s" % gpdb.master.port)
cmd = pg.ReadPostmasterTempFile.local('test pg tempfile read',gpdb.master.port)
(exists,PID,datadir)=cmd.getResults()
logger.info("exists:=%s PID=%d datadir='%s'" % (exists,PID,datadir))
self.assertTrue(exists)
self.assertTrue(PID > 0)
self.assertEquals(datadir,gpdb.master.datadir)
gpdb.master.port=4000
logger.info("Search for bogus master port: %s" % gpdb.master.port)
cmd = pg.ReadPostmasterTempFile.local('test pg tempfile read',gpdb.master.port)
(exists,PID,datadir)=cmd.getResults()
logger.info("exists:=%s PID=%d datadir='%s'" % (exists,PID,datadir))
self.assertFalse(exists)
def execute(self):
fake_timestamp = PickDumpTimestamp(restore_timestamp = self.restore_timestamp,
compress = self.compress,
master_datadir = self.master_datadir).run()
gparray = GpArray.initFromCatalog(dbconn.DbURL(port=self.master_port), utility=True)
primaries = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary(current_role=True)]
operations = []
for seg in primaries:
real_filename = os.path.join(seg.getSegmentDataDirectory(), DUMP_DIR, self.restore_timestamp[0:8], "%s0_%d_%s" % (DBDUMP_PREFIX, seg.getSegmentDbId(), self.restore_timestamp))
fake_filename = os.path.join(seg.getSegmentDataDirectory(), DUMP_DIR, fake_timestamp[0:8], "%s0_%d_%s" % (DBDUMP_PREFIX, seg.getSegmentDbId(), fake_timestamp))
operations.append( BuildRemoteTableDump(self.restore_tables, real_filename, fake_filename, self.compress, seg.getSegmentHostName()) )
ParallelOperation(operations, self.batch_default).run()
for operation in operations:
try:
operation.get_ret()
except Exception, e:
logger.exception('Parallel table dump file build failed.')
raise ExceptionNoStackTraceNeeded('Parallel table dump file build failed, review log file for details')
def impl(context, recovery_types, contents):
recovery_types = recovery_types.split(',')
contents = [int(c) for c in contents.split(',')]
contents = set(contents)
all_segments = GpArray.initFromCatalog(dbconn.DbURL()).getDbList()
segments_to_display = []
segments_to_not_display = []
for seg in all_segments:
if seg.getSegmentContentId() in contents and seg.getSegmentRole() == ROLE_MIRROR:
segments_to_display.append(seg)
else:
segments_to_not_display.append(seg)
for index, seg_to_display in enumerate(segments_to_display):
hostname = seg_to_display.getSegmentHostName()
port = seg_to_display.getSegmentPort()
expected_msg = "{}[ \t]+{}[ \t]+{}[ \t]+[0-9]+[ \t]+[0-9]+[ \t]+[0-9]+\%".format(hostname, port,
recovery_types[index])
check_stdout_msg(context, expected_msg)
def execute(self):
gparray = GpArray.initFromCatalog(dbconn.DbURL(port=self.master_port),
utility=True)
from_host, from_path = self.host, self.path
logger.info(
"Commencing remote database dump file recovery process, please wait..."
)
segs = [
seg for seg in gparray.getDbList() if seg.isSegmentPrimary(
current_role=True) or seg.isSegmentMaster()
]
pool = WorkerPool(numWorkers=min(len(segs), self.batch_default))
for seg in segs:
if seg.isSegmentMaster():
file = '%s%s' % (MASTER_DBDUMP_PREFIX, self.restore_timestamp)
else:
file = '%s0_%d_%s' % (DBDUMP_PREFIX, seg.getSegmentDbId(),
self.restore_timestamp)
if self.compress:
file += '.gz'
to_host = seg.getSegmentHostName()
to_path = os.path.join(seg.getSegmentDataDirectory(), DUMP_DIR,
self.restore_timestamp[0:8])
if not CheckRemoteDir(to_path, to_host).run():
logger.info('Creating directory %s on %s' % (to_path, to_host))
try:
MakeRemoteDir(to_path, to_host).run()
except OSError, e:
raise ExceptionNoStackTraceNeeded(
"Failed to create directory %s on %s" %
(to_path, to_host))
logger.info("Commencing remote copy from %s to %s:%s" %
(from_host, to_host, to_path))
pool.addCommand(
Scp('Copying dump for seg %d' % seg.getSegmentDbId(),
srcFile=os.path.join(from_path, file),
dstFile=os.path.join(to_path, file),
srcHost=from_host,
dstHost=to_host))
def impl(context, filetype):
backup_utils = Context()
if hasattr(context, 'netbackup_service_host'):
backup_utils.netbackup_service_host = context.netbackup_service_host
if hasattr(context, 'backup_timestamp'):
backup_utils.timestamp = context.backup_timestamp
if hasattr(context, 'backup_dir'):
backup_utils.backup_dir = context.backup_dir
else:
backup_dir = None
if filetype not in ['config', 'state']:
filename = backup_utils.generate_filename(filetype)
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (backup_utils.netbackup_service_host, filename)
cmd = Command("Querying NetBackup server for %s file" % filetype, cmd_str)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != filename:
raise Exception('File %s was not backup up to NetBackup server %s successfully' % (filename, backup_utils.netbackup_service_host))
if filetype == 'config':
master_port = os.environ.get('PGPORT')
gparray = GpArray.initFromCatalog(dbconn.DbURL(port = master_port), utility=True)
segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary(current_role=True)]
for seg in segs:
backup_utils.master_datadir = seg.getSegmentDataDirectory()
seg_config_filename = backup_utils.generate_filename('segment_config', dbid=seg.getSegmentDbId())
seg_host = seg.getSegmentHostName()
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (backup_utils.netbackup_service_host, seg_config_filename)
cmd = Command("Querying NetBackup server for segment config file", cmd_str, ctxt=REMOTE, remoteHost=seg_host)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != seg_config_filename:
raise Exception('Segment config file %s was not backup up to NetBackup server %s successfully' % (seg_config_filename, netbackup_service_host))
elif filetype == 'state':
for type in ['ao', 'co', 'last_operation']:
filename = backup_utils.generate_filename(type)
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (backup_utils.netbackup_service_host, filename)
cmd = Command("Querying NetBackup server for %s file" % type, cmd_str)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != filename:
raise Exception('The %s file %s was not backup up to NetBackup server %s successfully' % (type, filename, netbackup_service_host))
def validate(self):
pool = base.WorkerPool()
gp_array = GpArray.initFromCatalog(dbconn.DbURL(), utility=True)
host_list = list(set(gp_array.get_hostlist(True)))
msg = None
for h in host_list:
cmd = Command(h, "gpcheckresgroupimpl", REMOTE, h)
pool.addCommand(cmd)
pool.join()
items = pool.getCompletedItems()
failed = []
for i in items:
if not i.was_successful():
failed.append("[%s:%s]"%(i.remoteHost, i.get_stderr().rstrip()))
pool.haltWork()
pool.joinWorkers()
if failed:
msg = ",".join(failed)
return msg
def __init__(self, name):
self.name = name
self.path = tempfile.mkdtemp()
self.dbname = 'tablespace_db_%s' % name
self.table_counter = 0
self.initial_data = None
gparray = GpArray.initFromCatalog(dbconn.DbURL())
for host in gparray.getHostList():
run_cmd('ssh %s mkdir -p %s' % (pipes.quote(host), pipes.quote(self.path)))
with dbconn.connect(dbconn.DbURL(), unsetSearchPath=False) as conn:
db = pg.DB(conn)
db.query("CREATE TABLESPACE %s LOCATION '%s'" % (self.name, self.path))
db.query("CREATE DATABASE %s TABLESPACE %s" % (self.dbname, self.name))
with dbconn.connect(dbconn.DbURL(dbname=self.dbname), unsetSearchPath=False) as conn:
db = pg.DB(conn)
db.query("CREATE TABLE tbl (i int) DISTRIBUTED RANDOMLY")
db.query("INSERT INTO tbl VALUES (GENERATE_SERIES(0, 25))")
# save the distributed data for later verification
self.initial_data = db.query("SELECT gp_segment_id, i FROM tbl").getresult()
def get_host_list():
"""
Returns a tuple which consists of the standby
and segment hosts
"""
gparr = GpArray.initFromCatalog(dbconn.DbURL(port=COORDINATOR_PORT), utility=True)
segs = gparr.getDbList()
standby_host = None
segment_host_list = []
for seg in segs:
if seg.isSegmentStandby(current_role=True):
standby_host = seg.getSegmentHostName()
elif not seg.isSegmentCoordinator(current_role=True):
segment_host_list.append(seg.getSegmentHostName())
# Deduplicate the hosts so that we
# dont install multiple times on the same host
segment_host_list = list(set(segment_host_list))
return (standby_host, segment_host_list)
def impl(context):
make_temp_dir(context, path.join('/tmp', 'gpconfig'))
temp_base_dir = context.temp_base_dir
context.gpconfig_context.working_directory = temp_base_dir
gparray = GpArray.initFromCatalog(dbconn.DbURL())
segments = gparray.getDbList()
restore_commands = []
for segment in segments:
segment_tmp_directory = path.join(temp_base_dir, str(segment.dbid))
os.mkdir(segment_tmp_directory)
backup_path = path.join(segment_tmp_directory, 'postgresql.conf')
original_path = path.join(segment.datadir, 'postgresql.conf')
copy_command = (
'scp %s:%s %s' %
(segment.hostname, original_path, backup_path)).split(' ')
restore_command = (
'scp %s %s:%s' %
(backup_path, segment.hostname, original_path)).split(' ')
restore_commands.append(restore_command)
subprocess.check_call(copy_command)
if segment.content == -1:
if segment.role == 'p':
context.gpconfig_context.master_postgres_file = original_path
else:
context.gpconfig_context.standby_postgres_file = original_path
def delete_temp_directory():
if 'temp_base_dir' in context:
shutil.rmtree(context.temp_base_dir)
def restore_conf_files():
for cmd in restore_commands:
subprocess.check_call(cmd)
context.add_cleanup(delete_temp_directory)
context.add_cleanup(restore_conf_files)
def __init__(self, values=None):
if values:
self.defaults.update(values.__dict__) # Ensure that context has default values for all unset variables
super(self.__class__, self).__init__(vars(Values(self.defaults)))
if self.masterDataDirectory:
self.master_datadir = self.masterDataDirectory
else:
self.master_datadir = gp.get_masterdatadir()
self.master_port = self.get_master_port()
if self.local_dump_prefix:
self.dump_prefix = self.local_dump_prefix + "_"
else:
self.dump_prefix = ""
if not self.include_dump_tables: self.include_dump_tables = []
if not self.exclude_dump_tables: self.exclude_dump_tables = []
if not self.output_options: self.output_options = []
if not self.dump_schema: self.dump_schema = []
if not self.exclude_dump_schema: self.exclude_dump_schema = []
self.gparray = GpArray.initFromCatalog(dbconn.DbURL(dbname="template1", port=self.master_port), utility=True)
self.use_old_filename_format = False # Use new filename format by default
self.content_map = self.setup_content_map()
def impl(context, filetype, dir):
if filetype == 'report':
if dir == 'master_data_directory':
dir = master_data_dir
filenames = os.listdir(dir)
for filename in filenames:
if filename.startswith('gp_restore') and filename.endswith('.rpt'):
filename = '%s/%s' % (dir, filename)
os.remove(filename)
if filetype == 'status':
gparray = GpArray.initFromCatalog(dbconn.DbURL())
if dir == 'segment_data_directory':
primary_segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary()]
for seg in primary_segs:
host = seg.getSegmentHostName()
seg_data_dir = seg.getSegmentDataDirectory()
cmd = Command('remove status file', "rm -f %s/gp_restore_status_*" % (seg_data_dir), ctxt=REMOTE, remoteHost=host)
cmd.run(validateAfter=True)
else:
primary_segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary()]
for seg in primary_segs:
host = seg.getSegmentHostName()
cmd = Command('remove status file', "rm -f %s/gp_restore_status_*" % dir, ctxt=REMOTE, remoteHost=host)
cmd.run(validateAfter=True)
def execute(self):
ValidateGpToolkit(database=self.dump_database,
master_port=self.master_port).run()
operations = []
gparray = GpArray.initFromCatalog(dbconn.DbURL(port=self.master_port),
utility=True)
segs = [
seg for seg in gparray.getDbList()
if seg.isSegmentPrimary(current_role=True)
]
for seg in segs:
operations.append(
RemoteOperation(
ValidateSegDiskSpace(
free_space_percent=self.free_space_percent,
compress=self.compress,
dump_database=self.dump_database,
include_dump_tables=self.include_dump_tables,
datadir=seg.getSegmentDataDirectory(),
segport=seg.getSegmentPort()),
seg.getSegmentHostName()))
ParallelOperation(operations, self.batch_default).run()
success = 0
for remote in operations:
host = remote.host
try:
remote.get_ret()
except NotEnoughDiskSpace, e:
logger.error(
"%s has insufficient disk space. [Need: %dK, Free %dK]" %
(host, e.needed_space, e.free_space))
else:
success += 1
def impl(context):
gparray = GpArray.initFromCatalog(dbconn.DbURL())
primary_segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary()]
for seg in primary_segs:
subprocess.check_call(['pg_isready', '-h', seg.getSegmentHostName(), '-p', str(seg.getSegmentPort())])
def _gather_log_from_gp_log_filter(start_time,
end_time=None,
out_file=_DEFAULT_OUT_FILE,
host='localhost',
port=_DEFAULT_PORT,
user=_DEFAULT_USER,
dbname=_DEFAULT_USER,
errors_only=False,
master_only=False):
"""
This retrieves log messages from all segments that happened within the last
'duration' seconds. The format of start_time and end_time is YYYY-MM-DD [hh:mm[:ss]]
The tuples returned are (dbid, hostname, datadir, logdata). sorted by dbid.
Returns True/False based on whether matching log entries were found.
"""
format_start_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(start_time))
if end_time:
format_end_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(end_time))
else:
format_end_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime())
tinctest.logger.info("Collecting log from %s to %s into the file -%s" %
(format_start_time, format_end_time, out_file))
array = GpArray.initFromCatalog(
DbURL(hostname=host, port=port, username=user, dbname=dbname),
True)
log_chunks = []
for seg in array.getDbList():
tinctest.logger.info(
"Collecting log for segment - %s : %s" %
(seg.getSegmentHostName(), seg.getSegmentContentId()))
if master_only and seg.getSegmentContentId() != -1:
continue
cmd = GpLogFilter(
'collect log chunk',
'\\`ls -rt %s | tail -1\\`' %
os.path.join(seg.getSegmentDataDirectory(), 'pg_log', '*.csv'),
start=format_start_time,
end=format_end_time,
trouble=errors_only,
ctxt=REMOTE,
remoteHost=seg.getSegmentHostName())
cmd.run()
rc = cmd.get_results().rc
if rc:
tinctest.logger.warning("Failed command execution %s : %s" %
(cmd, cmd.get_results().stderr))
continue
log_data = cmd.get_results().stdout
if not log_data:
tinctest.logger.warning(
"No log data returned for the given time frame.")
else:
log_chunks.append(
(seg.getSegmentContentId(), seg.getSegmentHostName(),
seg.getSegmentDataDirectory(), log_data))
if log_chunks:
tinctest.logger.info("Writing log data to file - %s" % (out_file))
with open(out_file, 'w') as f:
for part in log_chunks:
f.write("-" * 70)
f.write("\n DBID %s (%s:%s)\n" %
(part[0], part[1], part[2]))
f.write("-" * 70)
f.write("\n%s" % part[3])
f.write("\n\n")
def execute(self):
try:
# Disable Ctrl+C
signal.signal(signal.SIGINT, signal.SIG_IGN)
# StopGPDB - Check for GPDB connections first and fail if connections exist
if not self.check_database_stopped():
self.stop_database()
# StartGPDB in Master only mode
self.start_master_only()
try:
if not CheckSuperUser(
dbconn.DbURL(username=self.user,
password=self.pswd)).run():
raise MoveFilespaceError(
'gpfilespace requires database superuser privileges.')
gparray = GpArray.initFromCatalog(dbconn.DbURL(), utility=True)
# CheckFilespace - Check if new filespace exists
if not CheckFilespace(self.new_filespace_name).run():
raise MoveFilespaceError('Filespace %s does not exist' %
self.new_filespace_name)
# CheckFilespaceIsSame - Check if filespace is different from the old one
if CheckFilespaceIsSame(gparray, self.new_filespace_name,
self.file_type).run():
raise MoveFilespaceError(
'Filespace %s is same as the current filespace' %
self.new_filespace_name)
except Exception, e:
# Bring GPDB Offline
self.stop_master_only()
# Restart the database
self.start_database()
raise
# Bring GPDB Offline
if 'UTILITY' == get_local_db_mode(MASTER_DATA_DIR):
self.stop_master_only()
else:
raise MoveFilespaceError('Database state is invalid.')
if not CheckConnectivity(gparray).run():
raise MoveFilespaceError('Failed connectivity test')
logger.info('Obtaining current filespace information')
# Find the filespace directory used for each segment
# query using pg_filespace, pg_filespace_entry, gp_segment_configuration
# If gp_transaction/temporary flat files exist and is not empty, then we know
# the filespace being used. Otherwise, we assume that it is the pg_system
# filespace by default
cur_filespace_entries = GetFilespaceEntriesDict(
GetCurrentFilespaceEntries(gparray,
self.file_type).run()).run()
new_filespace_entries = GetFilespaceEntriesDict(
GetFilespaceEntries(gparray,
self.new_filespace_name).run()).run()
pg_system_filespace_entries = GetFilespaceEntriesDict(
GetFilespaceEntries(gparray, PG_SYSTEM_FILESPACE).run()).run()
cur_filespace_name = gparray.getFileSpaceName(
int(cur_filespace_entries[1][0]))
logger.info('Obtaining segment information ...')
segments = gparray.getDbList()
# MoveTemp/Trans files
try:
operations = GetMoveOperationList(
segments, self.file_type, self.new_filespace_name,
new_filespace_entries, cur_filespace_entries,
pg_system_filespace_entries).run()
except Exception, e:
raise MoveFilespaceError(
'Failed to create operations list. %s' % str(e))
def test_fail_back(self):
"""
This test verifies that the fail-back mode is not allowed.
Fail-back means original master acting as the new standby.
"""
# Verify if the database is up. Run some sql.
PSQL.run_sql_command('DROP table if exists foo')
Command('remove standby', 'gpinitstandby -ra').run()
self.assertEqual(self.standby.create(), 0)
res = self.standby.start()
self.assertTrue(res.wasSuccessful())
# Wait for the walreceiver to start
num_walsender = self.wait_for_walsender()
self.assertEqual(num_walsender, 1)
logger.info('Activated WAL Receiver...')
# Promote the standby & shutdown the old Master
# Generate a recovery.conf file for the old Master so
# to make him the new standby that connects to the new
# master (originally standby)
logger.info('Promoting the standby...')
self.standby.promote()
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
numcontent = gparray.getNumSegmentContents()
orig_master = gparray.master
self.standby.remove_catalog_standby(dburl)
if (os.path.exists(os.path.join(orig_master.datadir, 'wal_rcv.pid'))):
os.remove(os.path.join(orig_master.datadir, 'wal_rcv.pid'))
logger.info('Stop the original master...')
cmd = Command("gpstop", "gpstop -aim")
cmd.run()
self.assertEqual(cmd.get_results().rc, 0, str(cmd))
logger.info(
'Generate recovery.conf for original master to make a new standby...'
)
master_recv_conf = open(
os.path.join(orig_master.datadir, 'recovery.conf'), 'w')
standby_recv_done = open(
os.path.join(self.standby.datadir, 'recovery.done'))
for line in standby_recv_done:
master_recv_conf.write(
line.replace("port=" + str(os.environ.get('PGPORT')),
"port=" + str(self.standby.port)))
master_recv_conf.close()
standby_recv_done.close()
logger.info(
'Start the old master again (to act as the new standby)...')
master = gp.MasterStart("Starting orig Master in standby mode",
orig_master.datadir, orig_master.port,
orig_master.dbid, 0, numcontent, None, None,
None)
# -w option would wait forever.
master.cmdStr = master.cmdStr.replace(' -w', '')
master.run(validateAfter=True)
self.assertTrue((master.get_results()).wasSuccessful())
# Have to do this to give the new standby some time to be active
subprocess.check_call("psql -c 'create database foo' -p " +
str(self.standby.port),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
subprocess.check_call("psql -c 'drop database foo' -p " +
str(self.standby.port),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
time.sleep(3)
# The new standby can re-start but should not be able to connect to the new
# master (originally standby). Thats the test
self.assertTrue(
os.path.exists(os.path.join(orig_master.datadir, 'wal_rcv.pid')))
logger.info(
'The WAL receiver pid file exists which means the new standby started\n'
'but still could not connect to the new Master (originally standby) and hence the\n'
'pid file was not cleared')
# Remove the recovery.conf file from the new standby directory
# as its no more needed
os.remove(os.path.join(orig_master.datadir, 'recovery.conf'))
logger.info('Stop the original master again...')
rc = subprocess.Popen('pg_ctl stop -D ' + orig_master.datadir +
' -m immediate',
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
# Perform gpstart to get the original master (& cluster) back again
cmd = Command("gpstart", "gpstart -a")
cmd.run()
self.assertTrue(cmd.get_results().rc in (0, 1), str(cmd))
logger.info('Pass')
def test_master_panic_after_phase1(self):
"""PANIC master after recording distributed commit.
Trigger PANIC in master after completing phase 1 of 2PC,
right after recording distributed commit in xlog but before
broadcasting COMMIT PREPARED to segments. Master's recovery
cycle should correctly broadcast COMMIT PREPARED because
master should find distributed commit record in its xlog
during recovery. Verify that the transaction is committed
after recovery.
JIRA: MPP-19044
"""
tinctest.logger.info("running test: test_crash_master_after_phase1")
gparray = GpArray.initFromCatalog(dbconn.DbURL(), utility=True)
assert len(gparray.getHostList()) == 1, "cannot run on multi-node"
host = gparray.getHostList()[0]
# Must have at least one in-sync and up segment.
primaries = [
p for p in gparray.get_list_of_primary_segments_on_host(host)
if p.getSegmentMode() == "s" and p.getSegmentStatus() == "u"
]
assert len(primaries) > 0, "in-sync and up primary not found"
primary = primaries[0]
tinctest.logger.info("chose primary: %s" % primary.datadir)
# Inject suspend fault after recording distributed commit on master.
cmd = Command("Suspend master post distributed commit",
self.faultcmd % "suspend")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
# Trigger the fault.
cmd = Command("run DDL",
"psql -f %s" % local_path('sql/ao_create.sql'))
self.proc = cmd.runNoWait()
tinctest.logger.info("runNoWait: %s, pid: %d" %
(cmd.cmdStr, self.proc.pid))
commitBlocked = self.filereputil.check_fault_status(
fault_name='dtm_xlog_distributed_commit',
status="triggered",
seg_id='1',
num_times_hit=1)
# Shutdown of primary (and mirror) should happen only after
# the commit is blocked due to suspend fault.
assert commitBlocked, "timeout waiting for commit to be blocked"
tinctest.logger.info("commit is blocked due to suspend fault")
# At this point, segments have already recorded the
# transaction as prepared by writing PREPARE record in xlog.
# Crash one primary (and its mirror).
mirror = None
mirrors = [
m for m in gparray.get_list_of_mirror_segments_on_host(host)
if m.getSegmentMode() == "s" and m.getSegmentStatus() == "u"
and primary.getSegmentContentId() == m.getSegmentContentId()
]
if len(mirrors) > 0:
mirror = mirrors[0]
tinctest.logger.info("chose mirror: %s" % mirror.datadir)
# Pause FTS probes to avoid a failover while we bring down
# segments. Note that we bring down both primary and its
# mirror, thereby causing double failure. This prevents
# FTS from making changes to segment configuration, even
# if FTS probes are unpaused. It is necessary to unpause
# FTS probes to prevent gang creation from being blocked.
PSQL.run_sql_command_utility_mode("SET gp_fts_probe_pause = on")
tinctest.logger.info("FTS probes paused")
cmdstr = 'pg_ctl -D %s stop -m immediate' % mirror.datadir
tinctest.logger.info("bringing down primary: %s" % cmdstr)
cmd = Command("Shutdown a primary segment", cmdstr)
cmd.run(validateAfter=True)
cmdstr = 'pg_ctl -D %s stop -m immediate' % primary.datadir
tinctest.logger.info("bringing down primary: %s" % cmdstr)
cmd = Command("Shutdown a primary segment", cmdstr)
cmd.run(validateAfter=True)
if mirror is not None:
PSQL.run_sql_command_utility_mode("SET gp_fts_probe_pause = off")
tinctest.logger.info("FTS probes unpaused")
# Resume master. Master should PANIC and go through crash recovery.
cmd = Command("resume master", self.faultcmd % "resume")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
(rc, out, err) = self.proc.communicate2()
self.proc = None
tinctest.logger.info("runNoWait rc: %d, output: %s, err: %s" %
(rc, out, err))
# Fail if QD did not PANIC.
assert (out.find("commit succeeded") == -1
and err.find("commit succeeded") == -1
and err.find("PANIC") != -1)
# Wait for recovery to complete, timeout after ~ 5 mins.
attempts = 1
recoveryComplete = False
while attempts < 600 and not recoveryComplete:
recoveryComplete = "aaa150" in PSQL.run_sql_command_utility_mode(
"select 'aaa' || (100+50)")
time.sleep(0.5)
attempts = attempts + 1
assert recoveryComplete, "timeout waiting for master to recover"
cmdstr = "gpstop -ar"
cmd = Command("restart", cmdstr)
tinctest.logger.info("restarting the cluster with '%s'" % cmdstr)
cmd.run(validateAfter=True)
tinctest.logger.info("restart complete")
# Verify table got created (commit was successful).
assert PSQL.run_sql_file(local_path('sql/ao_select.sql'))
gpverify = GpdbVerify()
(errorCode, hasError, gpcheckcat_output,
repairScript) = gpverify.gpcheckcat()
assert errorCode == 0, ("gpcheckcat failed: %s" % gpcheckcat_output[0])
# No need to restart GPDB again in tearDown()
self.skipRestart = True
def setUp(self):
super(AuxOidsTest, self).setUp()
self.gparray = GpArray.initFromCatalog(dbconn.DbURL(), utility=True)
except NoStatusFile, e:
logger.warn('Status file %s not found on master' % status_file)
return {'exit_status': 1, 'timestamp': timestamp}
except StatusFileError, e:
logger.warn('Status file %s on master indicates errors' %
status_file)
return {'exit_status': 1, 'timestamp': timestamp}
except NoDumpFile, e:
logger.warn('Dump file %s not found on master' % dump_file)
return {'exit_status': 1, 'timestamp': timestamp}
else:
logger.info('Checked master status file and master dump file.')
# Perform similar checks for primary segments
operations = []
gparray = GpArray.initFromCatalog(dbconn.DbURL(port=self.master_port),
utility=True)
segs = [
seg for seg in gparray.getDbList()
if seg.isSegmentPrimary(current_role=True)
]
for seg in segs:
path = self.backup_dir if self.backup_dir is not None else seg.getSegmentDataDirectory(
)
path = os.path.join(path, DUMP_DIR, DUMP_DATE)
status_file = os.path.join(
path, "%s%d_%s" %
(SEG_STATUS_PREFIX, seg.getSegmentDbId(), timestamp))
dump_file = os.path.join(
path, "%s%d_%s" %
(SEG_DBDUMP_PREFIX, seg.getSegmentDbId(), timestamp))
if self.compress: dump_file += ".gz"
def impl(context, filetype, prefix, subdir):
if hasattr(context, 'netbackup_service_host'):
netbackup_service_host = context.netbackup_service_host
if hasattr(context, 'backup_timestamp'):
backup_timestamp = context.backup_timestamp
subdir = subdir.strip()
if len(subdir) > 0:
dump_dir = os.path.join(subdir, 'db_dumps',
'%s' % (backup_timestamp[0:8]))
else:
dump_dir = os.path.join(master_data_dir, 'db_dumps',
'%s' % (backup_timestamp[0:8]))
prefix = prefix.strip()
if len(prefix) > 0:
prefix = prefix + '_'
if filetype == 'report':
#use_dir = get_backup_directory(master_data_dir, subdir, 'db_dumps', backup_timestamp)
filename = "%s/%sgp_dump_%s.rpt" % (dump_dir, prefix, backup_timestamp)
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (
netbackup_service_host, filename)
cmd = Command("Querying NetBackup server for report file", cmd_str)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != filename:
raise Exception(
'Report file %s was not backup up to NetBackup server %s successfully'
% (filename, netbackup_service_host))
elif filetype == 'global':
filename = os.path.join(
dump_dir, "%sgp_global_1_1_%s" % (prefix, backup_timestamp))
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (
netbackup_service_host, filename)
cmd = Command("Querying NetBackup server for global file", cmd_str)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != filename:
raise Exception(
'Global file %s was not backup up to NetBackup server %s successfully'
% (filename, netbackup_service_host))
elif filetype == 'config':
use_dir = get_backup_directory(master_data_dir, subdir, 'db_dumps',
backup_timestamp)
master_config_filename = os.path.join(
dump_dir,
"%sgp_master_config_files_%s.tar" % (prefix, backup_timestamp))
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (
netbackup_service_host, master_config_filename)
cmd = Command("Querying NetBackup server for master config file",
cmd_str)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != master_config_filename:
raise Exception(
'Master config file %s was not backup up to NetBackup server %s successfully'
% (master_config_filename, netbackup_service_host))
master_port = os.environ.get('PGPORT')
gparray = GpArray.initFromCatalog(dbconn.DbURL(port=master_port),
utility=True)
segs = [
seg for seg in gparray.getDbList()
if seg.isSegmentPrimary(current_role=True)
]
for seg in segs:
use_dir = get_backup_directory(seg.getSegmentDataDirectory(),
subdir, 'db_dumps',
backup_timestamp)
seg_config_filename = os.path.join(
use_dir, "%sgp_segment_config_files_0_%d_%s.tar" %
(prefix, seg.getSegmentDbId(), backup_timestamp))
seg_host = seg.getSegmentHostName()
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (
netbackup_service_host, seg_config_filename)
cmd = Command("Querying NetBackup server for segment config file",
cmd_str,
ctxt=REMOTE,
remoteHost=seg_host)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != seg_config_filename:
raise Exception(
'Segment config file %s was not backup up to NetBackup server %s successfully'
% (seg_config_filename, netbackup_service_host))
elif filetype == 'state':
filename = "%s/%sgp_dump_%s_ao_state_file" % (dump_dir, prefix,
backup_timestamp)
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (
netbackup_service_host, filename)
cmd = Command("Querying NetBackup server for AO state file", cmd_str)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != filename:
raise Exception(
'AO state file %s was not backup up to NetBackup server %s successfully'
% (filename, netbackup_service_host))
filename = "%s/%sgp_dump_%s_co_state_file" % (dump_dir, prefix,
backup_timestamp)
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (
netbackup_service_host, filename)
cmd = Command("Querying NetBackup server for CO state file", cmd_str)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != filename:
raise Exception(
'CO state file %s was not backup up to NetBackup server %s successfully'
% (filename, netbackup_service_host))
filename = "%s/%sgp_dump_%s_last_operation" % (dump_dir, prefix,
backup_timestamp)
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (
netbackup_service_host, filename)
cmd = Command(
"Querying NetBackup server for last operation state file", cmd_str)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != filename:
raise Exception(
'Last operation state file %s was not backup up to NetBackup server %s successfully'
% (filename, netbackup_service_host))
elif filetype == 'cdatabase':
filename = "%s/%sgp_cdatabase_1_1_%s" % (dump_dir, prefix,
backup_timestamp)
cmd_str = "gp_bsa_query_agent --netbackup-service-host %s --netbackup-filename %s" % (
netbackup_service_host, filename)
cmd = Command("Querying NetBackup server for cdatabase file", cmd_str)
cmd.run(validateAfter=True)
if cmd.get_results().stdout.strip() != filename:
raise Exception(
'Cdatabase file %s was not backup up to NetBackup server %s successfully'
% (filename, netbackup_service_host))
def __ensureMarkedDown(self, gpEnv, toEnsureMarkedDown):
"""Waits for FTS prober to mark segments as down"""
wait_time = 60 * 30 # Wait up to 30 minutes to handle very large, busy
# clusters that may have faults. In most cases the
# actual time to wait will be small and this operation
# is only needed when moving mirrors that are up and
# needed to be stopped, an uncommon operation.
dburl = dbconn.DbURL(port=gpEnv.getCoordinatorPort(),
dbname='template1')
time_elapsed = 0
seg_up_count = 0
initial_seg_up_count = len(toEnsureMarkedDown)
last_seg_up_count = initial_seg_up_count
if initial_seg_up_count == 0:
# Nothing to wait on
return
self.__logger.info("Waiting for segments to be marked down.")
self.__logger.info(
"This may take up to %d seconds on large clusters." % wait_time)
# wait for all needed segments to be marked down by the prober. We'll wait
# a max time of double the interval
while wait_time > time_elapsed:
seg_up_count = 0
current_gparray = GpArray.initFromCatalog(dburl, True)
seg_db_map = current_gparray.getSegDbMap()
# go through and get the status of each segment we need to be marked down
for segdb in toEnsureMarkedDown:
if segdb.getSegmentDbId() in seg_db_map and seg_db_map[
segdb.getSegmentDbId()].isSegmentUp():
seg_up_count += 1
if seg_up_count == 0:
break
else:
if last_seg_up_count != seg_up_count:
print("\n", end=' ')
self.__logger.info(
"%d of %d segments have been marked down." %
(initial_seg_up_count - seg_up_count,
initial_seg_up_count))
last_seg_up_count = seg_up_count
for _i in range(1, 5):
time.sleep(1)
sys.stdout.write(".")
sys.stdout.flush()
time_elapsed += 5
if seg_up_count == 0:
print("\n", end=' ')
self.__logger.info("%d of %d segments have been marked down." %
(initial_seg_up_count, initial_seg_up_count))
else:
raise Exception("%d segments were not marked down by FTS" %
seg_up_count)
def get_seginfo(self, preferred_role='p', content=0):
gparray = GpArray.initFromCatalog(dbconn.DbURL())
for seg in gparray.getDbList():
if seg.getSegmentContentId() == content and seg.getSegmentPreferredRole() == preferred_role:
return seg
def test_initFromCatalog_mismatched_versions(self, mock_connect, mock_execSQL):
with self.assertRaisesRegexp(Exception, 'Cannot connect to GPDB version 5 from installed version 6'):
GpArray.initFromCatalog(None)
def get_all_segment_addresses(master_port):
gparray = GpArray.initFromCatalog(dbconn.DbURL(port=master_port), utility=True)
addresses = [seg.getSegmentAddress() for seg in gparray.getDbList() if seg.isSegmentPrimary(current_role=True)]
return list(set(addresses))
