def getPortCoordinatorOnly(host='localhost',
coordinator_value=None,
user=os.environ.get('USER'),
gphome=os.environ['GPHOME'],
cdd=get_coordinatordatadir(),
port=os.environ['PGPORT']):
coordinator_pattern = r"Context:\s*-1\s*Value:\s*\d+"
command = "gpconfig -s %s" % ("port")
cmd = "source %s/greenplum_path.sh; export COORDINATOR_DATA_DIRECTORY=%s; export PGPORT=%s; %s" \
% (gphome, cdd, port, command)
(ok, out) = run(cmd)
if not ok:
cmd = "python %s/bin/gpconfig -s port" % (gphome)
(ok, out) = run(cmd)
if not ok:
raise Exception(
"Unable to connect to segment server %s as user %s" %
(host, user))
for line in out:
out = line.decode().split('\n')
for line in out:
if re.search(coordinator_pattern, line):
coordinator_value = int(line.split()[3].strip())
if coordinator_value is None:
error_msg = "".join(out)
raise Exception(error_msg)
return str(coordinator_value)
def make_data_directory_called(data_directory_name):
cdd_parent_parent = os.path.realpath(get_coordinatordatadir() +
"../../../")
mirror_data_dir = os.path.join(cdd_parent_parent, data_directory_name)
if not os.path.exists(mirror_data_dir):
os.mkdir(mirror_data_dir)
return mirror_data_dir
def impl(context):
context.mirror_config = _generate_input_config()
cdd = get_coordinatordatadir()
del os.environ['COORDINATOR_DATA_DIRECTORY']
try:
cmd = Command('gpaddmirrors ', 'gpaddmirrors -a -i %s -d %s' % (context.mirror_config, cdd))
cmd.run(validateAfter=True)
finally:
os.environ['COORDINATOR_DATA_DIRECTORY'] = cdd
def get_port_from_conf():
file = get_coordinatordatadir() + '/postgresql.conf'
if os.path.isfile(file):
with open(file) as f:
for line in f.xreadlines():
match = re.search(r'port=\d+', line)
if match:
match1 = re.search(r'\d+', match.group())
if match1:
return match1.group()
def check_database_is_running(context):
if not 'PGPORT' in os.environ:
raise Exception('PGPORT should be set')
pgport = int(os.environ['PGPORT'])
running_status = chk_local_db_running(get_coordinatordatadir(), pgport)
gpdb_running = running_status[0] and running_status[1] and running_status[2] and running_status[3]
return gpdb_running
def get_port():
file = get_coordinatordatadir() + '/postgresql.conf'
if os.path.isfile(file):
f = open(file)
for line in f:
match = re.search('port=\d+', line)
if match:
match1 = re.search('\d+', match.group())
if match1:
return match1.group()
f.close()
raise Exception('Could not get port from ' + file)
else:
raise Exception(file + ' does not exist. Cannot get port.')
def __init__(self,
coordinatorDataDir,
readFromCoordinatorCatalog,
timeout=None,
retries=None,
verbose=True):
"""
coordinatorDataDir: if None then we try to find it from the system environment
readFromCoordinatorCatalog: if True then we will connect to the coordinator in utility mode and fetch some more
data from there (like collation settings)
"""
if coordinatorDataDir is None:
self.__coordinatorDataDir = gp.get_coordinatordatadir()
else:
self.__coordinatorDataDir = coordinatorDataDir
logger.debug(
"Obtaining coordinator's port from coordinator data directory")
pgconf_dict = pgconf.readfile(self.__coordinatorDataDir +
"/postgresql.conf")
self.__coordinatorPort = pgconf_dict.int('port')
logger.debug("Read from postgresql.conf port=%s" %
self.__coordinatorPort)
self.__coordinatorMaxConnections = pgconf_dict.int('max_connections')
logger.debug("Read from postgresql.conf max_connections=%s" %
self.__coordinatorMaxConnections)
self.__gpHome = gp.get_gphome()
self.__gpVersion = gp.GpVersion.local(
'local GP software version check', self.__gpHome)
if verbose:
logger.info("local Greenplum Version: '%s'" % self.__gpVersion)
# read collation settings from coordinator
if readFromCoordinatorCatalog:
dbUrl = dbconn.DbURL(port=self.__coordinatorPort,
dbname='template1',
timeout=timeout,
retries=retries)
conn = dbconn.connect(dbUrl, utility=True)
# MPP-13807, read/show the coordinator's database version too
self.__pgVersion = dbconn.queryRow(conn, "select version();")[0]
logger.info("coordinator Greenplum Version: '%s'" %
self.__pgVersion)
conn.close()
else:
self.__pgVersion = None
def __init__(self, mainOptions):
self.pidlockpath = mainOptions.get('pidlockpath', None) # the directory we're using for locking
self.parentpidvar = mainOptions.get('parentpidvar', None) # environment variable holding parent pid
self.parentpid = None # parent pid which already has the lock
self.ppath = None # complete path to the lock file
self.pidlockfile = None # PIDLockFile object
self.pidfilepid = None # pid of the process which has the lock
self.locktorelease = None # PIDLockFile object we should release when done
if self.parentpidvar is not None and self.parentpidvar in os.environ:
self.parentpid = int(os.environ[self.parentpidvar])
if self.pidlockpath is not None:
self.ppath = os.path.join(gp.get_coordinatordatadir(), self.pidlockpath)
self.pidlockfile = PIDLockFile(self.ppath)
def run(self):
if self.build:
if self.filename:
BuildGppkg(self.build, self.filename).run()
else:
BuildGppkg(self.build, None).run()
return
if linux_distribution_id() == 'ubuntu':
try:
cmd = Command(name='Check for dpkg', cmdStr='dpkg --version')
cmd.run(validateAfter=True)
cmd = Command(name='Check for fakeroot',
cmdStr='fakeroot --version')
cmd.run(validateAfter=True)
except Exception as ex:
raise ExceptionNoStackTraceNeeded(
'fakeroot and dpkg are both required by gppkg')
else:
try:
cmd = Command(name='Check for rpm', cmdStr='rpm --version')
cmd.run(validateAfter=True)
results = cmd.get_results().stdout.strip()
rpm_version_string = results.split(' ')[-1]
if not rpm_version_string.startswith('4.'):
raise ExceptionNoStackTraceNeeded(
'gppkg requires rpm version 4.x')
except ExecutionError as ex:
results = ex.cmd.get_results().stderr.strip()
if len(results) != 0 and 'not found' in results:
raise ExceptionNoStackTraceNeeded(
'gppkg requires RPM to be available in PATH')
if self.coordinator_datadir is None:
self.coordinator_datadir = gp.get_coordinatordatadir()
self.coordinator_port = self._get_coordinator_port(
self.coordinator_datadir)
self._get_gpdb_host_list()
if self.migrate:
MigratePackages(from_gphome=self.migrate[0],
to_gphome=self.migrate[1],
standby_host=self.standby_host,
segment_host_list=self.segment_host_list).run()
return
if self.install:
pkg = Gppkg.from_package_path(self.install)
InstallPackage(pkg, self.coordinator_host, self.standby_host,
self.segment_host_list).run()
elif self.query:
query_type, package_path = self.query
QueryPackage(query_type, package_path).run()
elif self.remove:
# Check for exact match first, then use wildcard for what will be removed.
pkg_file_list = ListFilesByPattern(
GPPKG_ARCHIVE_PATH, self.remove + GPPKG_EXTENSION).run()
if len(pkg_file_list) == 0:
# now try wildcard
pkg_file_list = ListFilesByPattern(
GPPKG_ARCHIVE_PATH,
self.remove + '*' + GPPKG_EXTENSION).run()
if len(pkg_file_list) == 0:
raise ExceptionNoStackTraceNeeded(
'Package %s has not been installed.' % self.remove)
# refuse to remove at all if the match is too broad, i.e., > 1
if len(pkg_file_list) > 1:
err_msg = "Remove request '%s' too broad. " \
"Multiple packages match remove request: ( %s )." % (self.remove, ", ".join(pkg_file_list))
raise ExceptionNoStackTraceNeeded(err_msg)
pkg_file = pkg_file_list[0]
pkg = Gppkg.from_package_path(
os.path.join(GPPKG_ARCHIVE_PATH, pkg_file))
UninstallPackage(pkg, self.coordinator_host, self.standby_host,
self.segment_host_list).run()
elif self.update:
logger.warning(
'WARNING: The process of updating a package includes removing all'
)
logger.warning(
'previous versions of the system objects related to the package. For'
)
logger.warning(
'example, previous versions of shared libraries are removed.')
logger.warning(
'After the update process, a database function will fail when it is'
)
logger.warning(
'called if the function references a package file that has been removed.'
)
if self.interactive:
if not ask_yesno(None, 'Do you still want to continue ?', 'N'):
logger.info('Skipping update of gppkg based on user input')
return
pkg = Gppkg.from_package_path(self.update)
UpdatePackage(pkg, self.coordinator_host, self.standby_host,
self.segment_host_list).run()
elif self.clean:
CleanGppkg(self.standby_host, self.segment_host_list).run()
def impl(context, segment):
if segment not in ('standby', 'mirrors'):
raise Exception("invalid segment type")
context.standby_hostname = 'mdw-2'
context.execute_steps("""
Given the segments are synchronized
Then replication connections can be made from the acting {segment}
Given a tablespace is created with data
""".format(segment=segment))
# For the 'standby' case, we set PGHOST back to its original value instead
# of 'mdw-1'. When the function impl() is called, PGHOST is initially unset
# by the test framework, and we want to respect that.
orig_PGHOST = os.environ.get('PGHOST')
# Fail over to standby/mirrors.
if segment == 'standby':
coordinator_data_dir = get_coordinatordatadir()
context.standby_port = os.environ.get('PGPORT')
context.standby_data_dir = coordinator_data_dir
context.new_standby_data_dir = '%s_1' % coordinator_data_dir
context.execute_steps("""
When the coordinator goes down
And the user runs command "gpactivatestandby -a" from standby coordinator
Then gpactivatestandby should return a return code of 0
""")
os.environ['PGHOST'] = 'mdw-2'
else: # mirrors
context.execute_steps("""
Given user stops all primary processes
And user can start transactions
When the user runs "gprecoverseg -a"
Then gprecoverseg should return a return code of 0
""")
context.execute_steps("""
Then the segments are synchronized
And the tablespace is valid
And replication connections can be made from the acting {segment}
Given another tablespace is created with data
""".format(segment=segment))
# Fail over (rebalance) to original coordinator/primaries.
if segment == 'standby':
# Re-initialize the standby with a new directory, since
# the previous coordinator cannot assume the role of standby
# because it does not have the required recover.conf file.
context.execute_steps("""
When the user runs command "gpinitstandby -a -s mdw-1 -S {datadir}" from standby coordinator
Then gpinitstandby should return a return code of 0
""".format(datadir=context.new_standby_data_dir))
os.environ['COORDINATOR_DATA_DIRECTORY'] = context.new_standby_data_dir
# NOTE: this must be set before gpactivatestandby is called
if orig_PGHOST is None:
del os.environ['PGHOST']
else:
os.environ['PGHOST'] = orig_PGHOST
context.standby_hostname = 'mdw-1'
context.execute_steps("""
When the coordinator goes down on "mdw-2"
And the user runs "gpactivatestandby -a"
Then gpactivatestandby should return a return code of 0
""")
context.execute_steps("""
When the user runs "gpinitstandby -a -s mdw-2 -S {datadir}"
Then gpinitstandby should return a return code of 0
""".format(datadir=context.new_standby_data_dir))
else: # mirrors
context.execute_steps("""
When the user runs "gprecoverseg -ra"
Then gprecoverseg should return a return code of 0
""")
context.execute_steps("""
Then the segments are synchronized
And the tablespace is valid
And the other tablespace is valid
And replication connections can be made from the acting {segment}
And all tablespaces are dropped
""".format(segment=segment))
class Context(object):
filename = os.path.join(gp.get_coordinatordatadir(), 'gpexpand.status')
dbname = os.getenv('PGDATABASE', 'postgres')
dburl = dbconn.DbURL(dbname=dbname)
conn = dbconn.connect(dburl)
day = 0
import difflib
import pg
from contextlib import closing
from datetime import datetime
from gppylib.commands.base import Command, ExecutionError, REMOTE
from gppylib.commands.gp import chk_local_db_running, get_coordinatordatadir
from gppylib.db import dbconn
from gppylib.gparray import GpArray, MODE_SYNCHRONIZED
PARTITION_START_DATE = '2010-01-01'
PARTITION_END_DATE = '2013-01-01'
coordinator_data_dir = get_coordinatordatadir()
if coordinator_data_dir is None:
raise Exception('COORDINATOR_DATA_DIRECTORY is not set')
# query_sql returns a cursor object, so the caller is responsible for closing
# the dbconn connection.
def query_sql(dbname, sql):
result = None
with dbconn.connect(dbconn.DbURL(dbname=dbname), unsetSearchPath=False) as conn:
result = dbconn.query(conn, sql)
return result
def execute_sql(dbname, sql):
result = None
def __init__(self, cdd=None):
if not cdd:
cdd = get_coordinatordatadir()
cmd_str = "export COORDINATOR_DATA_DIRECTORY=%s; echo -e \"y\\ny\\n\" | gpdeletesystem -d %s" % (
cdd, cdd)
Command.__init__(self, 'run gpdeletesystem', cmd_str)
def get_analyze_dir(dbname):
coordinator_data_dir = get_coordinatordatadir()
analyze_dir = os.path.join(coordinator_data_dir, 'db_analyze', dbname)
return analyze_dir
def impl(context):
gprecoverseg_lock_dir = os.path.join(get_coordinatordatadir() + '/gprecoverseg.lock')
os.mkdir(gprecoverseg_lock_dir)
