def create_parser():
parser = OptParser(option_class=OptChecker,
description='update the pg_hba.conf on all segments')
addStandardLoggingAndHelpOptions(parser, includeNonInteractiveOption=True)
parser.add_option('-p',
'--pg-hba-info',
dest='pg_hba_info',
metavar='<pg_hba entries>',
help='Entries that get added to pg_hba.conf file')
parser.add_option(
'-d',
'--data-dirs',
dest='data_dirs',
metavar='<list of data dirs>',
help='A list of all data directories present on this host')
parser.add_option('-b',
'--backup',
action='store_true',
help='Backup the pg_hba.conf file')
parser.add_option('-r',
'--restore',
action='store_true',
help='Restore the pg_hba.conf file')
parser.add_option('-D',
'--delete',
action='store_true',
help='Cleanup the pg_hba.conf backup file')
return parser
def parseargs(self):
parser = OptParser(option_class=OptChecker,
description=' '.join(self.description.split()),
version='%prog version $Revision: $')
parser.set_usage(
'%prog is a utility script used by gprecoverseg, and gpaddmirrors and is not intended to be run separately.'
)
parser.remove_option('-h')
parser.add_option('-v',
'--verbose',
action='store_true',
help='debug output.',
default=False)
parser.add_option('-c', '--confinfo', type='string')
parser.add_option('-b',
'--batch-size',
type='int',
default=DEFAULT_SEGHOST_NUM_WORKERS,
metavar='<batch_size>')
parser.add_option('-f',
'--force-overwrite',
dest='forceoverwrite',
action='store_true',
default=False)
parser.add_option('-l',
'--log-dir',
dest="logfileDirectory",
type="string")
# Parse the command line arguments
options, _ = parser.parse_args()
return options
def parseargs():
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('-t', '--timestamp', type='string')
parser.add_option('-b', '--backupdir', type='string')
parser.add_option('-d', '--dbname', type='string')
parser.add_option('-p', '--password', type='string')
parser.add_option('-n', '--nthreads', type='int')
(options, args) = parser.parse_args()
if options.help or (not options.dbname and not options.filename):
print """Script performs serial restore of the backup files in case
of the cluster topology change.
Usage:
./serial_restore.py -n thread_number -t backup_timestamp -b backup_directory -d dbname [-p gpadmin_password]
Parameters:
thread_number - number of parallel threads to run
backup_timestamp - timestamp of the backup to be restored
backup_directory - folder with the complete backup set visible from the current server
dbname - name of the database to restore to
gpadmin_password - password of the gpadmin user"""
sys.exit(0)
if not options.timestamp:
logger.error('Failed to start utility. Please, specify backup timestamp with "-t" key')
sys.exit(1)
if not options.backupdir:
logger.error('Failed to start utility. Please, specify backup directory with "-b" key')
sys.exit(1)
if not options.dbname:
logger.error('Failed to start utility. Please, specify database name with "-d" key')
sys.exit(1)
if not options.nthreads:
options.nthreads = 1
return options
def createParser():
parser = OptParser(option_class=OptChecker,
description=' '.join(description.split()),
version='%prog version $Revision: #12 $')
parser.setHelp([])
addStandardLoggingAndHelpOptions(parser,
includeNonInteractiveOption=False)
parser.add_option("-D",
"--db",
dest="dblist",
action="append",
type="string")
parser.add_option("-V",
"--gp-version",
dest="gpversion",
metavar="GP_VERSION",
help="expected software version")
parser.add_option(
"-m",
"--mode",
dest="mode",
metavar="<MODE>",
help="how to shutdown. modes are smart,fast, or immediate")
parser.add_option("-t",
"--timeout",
dest="timeout",
type="int",
default=SEGMENT_STOP_TIMEOUT_DEFAULT,
help="seconds to wait")
return parser
def parseargs():
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('-d', '--dbname', type='string')
parser.add_option('-p', '--password', type='string')
parser.add_option('-n', '--nthreads', type='int')
(options, args) = parser.parse_args()
if options.help or (not options.dbname and not options.filename):
print """Script performs parallel analyze of all tables
Usage:
./parallel_analyze.py -n thread_number -d dbname [-p gpadmin_password]
Parameters:
thread_number - number of parallel threads to run
dbname - name of the database
gpadmin_password - password of the gpadmin user"""
sys.exit(0)
if not options.dbname:
logger.error(
'Failed to start utility. Please, specify database name with "-d" key'
)
sys.exit(1)
if not options.nthreads:
logger.error(
'Failed to start utility. Please, specify number of threads with "-n" key'
)
sys.exit(1)
return options
def parseargs():
parser = OptParser(option_class=OptChecker)
parser.remove_option("-h")
parser.add_option("-h", "-?", "--help", action="store_true")
parser.add_option("-d", "--dbname", type="string")
parser.add_option("-p", "--password", type="string")
parser.add_option("-n", "--nthreads", type="int")
(options, args) = parser.parse_args()
if options.help or (not options.dbname and not options.filename):
print """Script performs serial restore of the backup files in case
of the cluster topology change.
Usage:
./parallel_analyze.py -n thread_number -d dbname [-p gpadmin_password]
Parameters:
thread_number - number of parallel threads to run
dbname - name of the database
gpadmin_password - password of the gpadmin user"""
sys.exit(0)
if not options.dbname:
logger.error('Failed to start utility. Please, specify database name with "-d" key')
sys.exit(1)
if not options.nthreads:
logger.error('Failed to start utility. Please, specify number of threads with "-n" key')
sys.exit(1)
return options
def parseargs():
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('-d', '--database', type='string')
parser.add_option('-u', '--username', type='string')
parser.add_option('-p', '--password', type='string')
parser.add_option('-l', '--logfile', type='string')
parser.add_option('-n', '--nrows', type='int')
(options, args) = parser.parse_args()
if options.help:
print """
Script executes baseline performance test on the database
Usage:
python performance_baseline.py -d database_name
[-u username -p password]
[-l logfile]
[-n number_of_rows]
-d | --database - name of the database to run the test
-u | --username - name of the user to be used for testing (default is $PGUSER)
-p | --password - password of the user used for testing (default is $PGPASSWORD)
-l | --logfile - performance test output file (default is stdout)
-n | --nrows - number of rows generated in test table (default is 5000)
"""
sys.exit(0)
if not options.nrows:
options.nrows = 5000
if options.nrows < 5000:
raise_err('Number of rows should be 5000 or more')
if not options.database:
raise_err('You must specify database name (-d)')
if (options.password and not options.username) or (not options.password and options.username):
raise_err('You should either specify both username and password or not specify them both')
return options
def create_parser():
parser = OptParser(option_class=OptChecker,
description='update the pg_hba.conf on all segments')
addStandardLoggingAndHelpOptions(parser, includeNonInteractiveOption=True)
parser.add_option('-p', '--pg-hba-info', dest='pg_hba_info', metavar='<pg_hba entries>',
help='Entries that get added to pg_hba.conf file')
parser.add_option('-d', '--data-dirs', dest='data_dirs', metavar='<list of data dirs>',
help='A list of all data directories present on this host')
parser.add_option('-b', '--backup', action='store_true',
help='Backup the pg_hba.conf file')
parser.add_option('-r', '--restore', action='store_true',
help='Restore the pg_hba.conf file')
parser.add_option('-D', '--delete', action='store_true',
help='Cleanup the pg_hba.conf backup file')
return parser
def parseargs():
parser = OptParser(option_class=OptChecker,
description=' '.join(description.split()))
parser.setHelp([])
parser.add_option('-d',
'--data-dir',
dest='datadir',
metavar='<data dir of the segment>',
help='Data dir of the segment to update pg_hba.conf')
parser.add_option("-e",
"--entries",
dest="entries",
metavar="<entries to be added>",
help="entries to be added to pg_hba.conf")
options, args = parser.parse_args()
return validate_args(options)
def createParser():
"""
Constructs and returns an option parser.
Called by simple_main()
"""
parser = OptParser(option_class=OptChecker,
description=' '.join(description.split()),
version='%prog version $Revision: #1 $')
parser.setHelp([])
addStandardLoggingAndHelpOptions(parser,
includeNonInteractiveOption=False)
parser.add_option(
"-C",
"--collation",
type="string",
help="values for lc_collate, lc_monetary, lc_numeric separated by :"
)
parser.add_option("-D",
"--datadir",
dest="dblist",
action="append",
type="string")
parser.add_option("-p",
"--pickledTransitionData",
dest="pickledTransitionData",
type="string")
parser.add_option("-M",
"--mirroringMode",
dest="mirroringMode",
type="string")
parser.add_option("-V",
"--gp-version",
dest="gpversion",
metavar="GP_VERSION",
help="expected software version")
parser.set_defaults(verbose=False, filters=[], slice=(None, None))
return parser
def parseargs():
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('-f', '--fromthreads', type='int')
parser.add_option('-t', '--tothreads', type='int')
parser.add_option('-d', '--database', type='string')
(options, args) = parser.parse_args()
if options.help:
print """
Script generates a big CPU workload on the cluster within configured diapason of
thread number to check the elasicity of cluster CPU resources
Usage:
python cpu_stresstest.py -f fromthreads -t tothreads -d database
-t | --fromthreads - Lower bound of thread number to start
-t | --tothreads - Upper bound of thread number to start
-d | --database - Database to run the test on
"""
sys.exit(0)
if not options.fromthreads:
raise_err('You must specify the lower bound of thread number with -f parameter')
if not options.tothreads:
raise_err('You must specify the upper bound of thread number with -t parameter')
if not options.database:
raise_err('You must specify the database name with -d parameter')
return options
def parseargs():
parser = OptParser(option_class=OptChecker)
parser.setHelp(_help)
parser.remove_option('-h')
parser.add_option('-h',
'-?',
'--help',
action='help',
help='show this help message and exit')
parser.add_option('--entry', type='string')
parser.add_option('--value', type='string')
parser.add_option('--removeonly', action='store_true')
parser.set_defaults(removeonly=False)
# Parse the command line arguments
(options, args) = parser.parse_args()
# sanity check
if not options.entry:
print "--entry is required"
sys.exit(1)
if (not options.value) and (not options.removeonly):
print "Select either --value or --removeonly"
sys.exit(1)
return options
def parseargs():
parser = OptParser(option_class=OptChecker)
parser.setHelp(_help)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='help', help='show this help message and exit')
parser.add_option('--entry', type='string')
parser.add_option('--value', type='string')
parser.add_option('--removeonly', action='store_true')
parser.set_defaults(removeonly=False)
# Parse the command line arguments
(options, args) = parser.parse_args()
# sanity check
if not options.entry:
print "--entry is required"
sys.exit(1)
if (not options.value) and (not options.removeonly):
print "Select either --value or --removeonly"
sys.exit(1)
return options
def parseargs():
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('-i', '--index', type='string')
(options, args) = parser.parse_args()
if options.help:
print """
Script is calling Solr commit with the expungeDeletes flag to cause
all the index segments with deletes in them to clean up their segments
from the deleted records
Usage:
python gptext_expunge_deletes.py -i index_name
-i | --index - Name of the index to expunge
Examples:
python gptext_expunge_deletes.py -i test.public.test_table
"""
sys.exit(0)
if not options.index:
logger.error('You must specify index name with -i or --index key')
sys.exit(3)
return options
def createParser():
parser = OptParser(option_class=OptChecker,
description=' '.join(description.split()),
version='%prog version $Revision: #12 $')
parser.setHelp([])
addStandardLoggingAndHelpOptions(parser, includeNonInteractiveOption=False)
parser.add_option("-D","--db",dest="dblist", action="append", type="string")
parser.add_option("-V", "--gp-version", dest="gpversion",metavar="GP_VERSION",
help="expected software version")
parser.add_option("-m", "--mode", dest="mode",metavar="<MODE>",
help="how to shutdown. modes are smart,fast, or immediate")
parser.add_option("-t", "--timeout", dest="timeout", type="int", default=SEGMENT_STOP_TIMEOUT_DEFAULT,
help="seconds to wait")
return parser
def createParser():
"""
Constructs and returns an option parser.
Called by simple_main()
"""
parser = OptParser(option_class=OptChecker,
description=' '.join(description.split()),
version='%prog version $Revision: #1 $')
parser.setHelp([])
addStandardLoggingAndHelpOptions(parser, includeNonInteractiveOption=False)
parser.add_option("-C", "--collation", type="string",
help="values for lc_collate, lc_monetary, lc_numeric separated by :")
parser.add_option("-D","--datadir",dest="dblist", action="append", type="string")
parser.add_option("-p","--pickledTransitionData",dest="pickledTransitionData", type="string")
parser.add_option("-M","--mirroringMode",dest="mirroringMode", type="string")
parser.add_option("-V", "--gp-version", dest="gpversion",metavar="GP_VERSION",
help="expected software version")
parser.set_defaults(verbose=False, filters=[], slice=(None, None))
return parser
def __init__(self) :
parser = OptParser(option_class=OptChecker)
parser.remove_option("-h")
parser.add_option("-h", "-?", "--help", action="store_true")
parser.add_option("-f", "--file", type="string")
parser.add_option("-t", "--table", type="string")
parser.add_option("-c", "--config", type="string")
parser.add_option("-d", "--drop", action="store_true")
(parser_result, args) = parser.parse_args()
helpstr = "Usage:" + os.linesep + "./ddl_migration.py -f path"
if parser_result.help :
print helpstr
sys.exit(0)
if not parser_result.file and not parser_result.table:
print "Failed to start utility. Please, specify path to input file with -f key or table with -t key"
sys.exit(1)
configpath = parser_result.config or "./ddl_migration.conf"
try:
configfile = open(configpath, "rb")
self.connection_options = json.load(configfile)
configfile.close()
except Exception, e :
raise Exception("Could not read config file " + configpath + ". " +str(e))
def parseargs():
global allocation_rate, memory_split, cpu_split
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('-f', '--force', action='store_true')
parser.add_option('-s', '--memsplit', type='float')
parser.add_option('-a', '--allocrate', type='float')
parser.add_option('-c', '--cpusplit', type='float')
(options, args) = parser.parse_args()
if options.help:
print """
Script configures memory and CPU allocation for GPText and GPDB. GPDB should be running when
the script is started. The script should work on master server. Local GPDB connection under
gpadmin to template1 database should be passwordless.
Usage:
python gptext_tune_settings.py [-s memory_split] [-a allocation_rate] [-c cpu_split] [-f | --force]
memory_split - [0.1 .. 0.9] - split of the memory between GPText and GPDB. Greater value - more memory for GPText
allocation_rate - [0.1 .. 0.9] - part of the system memory available to GPText and GPDB
cpu_split - [0.3 .. 2.0] - part of the CPU dedicated to GPDB. Over utilization is allowed
force - do not ask for confirmation of changing the memory settings
"""
sys.exit(0)
if options.allocrate:
if not (options.allocrate >= 0.1 and options.allocrate <= 0.9):
logger.error('Correct values for --allocrate are [0.1 .. 0.9]')
sys.exit(3)
allocation_rate = options.allocrate
if options.memsplit:
if not (options.memsplit >= 0.1 and options.memsplit <= 0.9):
logger.error('Correct values for --memsplit are [0.1 .. 0.9]')
sys.exit(3)
memory_split = options.memsplit
if options.cpusplit:
if not (options.cpusplit >= 0.3 and options.cpusplit <= 2.0):
logger.error('Correct values for --cpusplit are [0.3 .. 2.0]')
sys.exit(3)
cpu_split = options.cpusplit
return options
def parseargs():
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('-d', '--dbname', type='string')
parser.add_option('-p', '--password', type='string')
parser.add_option('-n', '--nthreads', type='int')
(options, args) = parser.parse_args()
if options.help or (not options.dbname and not options.filename):
print """Script performs parallel analyze of all tables
Usage:
./parallel_analyze.py -n thread_number -d dbname [-p gpadmin_password]
Parameters:
thread_number - number of parallel threads to run
dbname - name of the database
gpadmin_password - password of the gpadmin user"""
sys.exit(0)
if not options.dbname:
logger.error('Failed to start utility. Please, specify database name with "-d" key')
sys.exit(1)
if not options.nthreads:
logger.error('Failed to start utility. Please, specify number of threads with "-n" key')
sys.exit(1)
return options
of the schema or table name
"""
if not os.path.exists(change_schema_file):
raise Exception('change schema file path %s does not exist' %
change_schema_file)
change_schema_name = None
with open(change_schema_file) as fr:
line = fr.read()
change_schema_name = line.strip('\n')
return change_schema_name
if __name__ == "__main__":
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('-t', '--tablefile', type='string', default=None)
parser.add_option('-m', '--master_only', action='store_true')
parser.add_option('-c',
'--change-schema-file',
type='string',
default=None)
parser.add_option('-s', '--schema-level-file', type='string', default=None)
(options, args) = parser.parse_args()
if not (options.tablefile or options.schema_level_file):
raise Exception(
'-t table file name or -s schema level file name must be specified'
)
elif options.schema_level_file and options.change_schema_file:
raise Exception(
'-s schema level file option can not be specified with -c change schema file option'
def parseargs():
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('-d', '--dbname', type='string')
parser.add_option('-p', '--password', type='string')
parser.add_option('-n', '--nthreads', type='int')
parser.add_option('-u', '--user', type='string')
parser.add_option('-l', '--location', type='string')
parser.add_option('-m', '--stat_mem', type='string')
parser.add_option('-f', '--filename', type='string')
(options, args) = parser.parse_args()
if options.help:
print """Script performs serial restore of the backup files in case
of the cluster itopology change.
Usage:
./parallel_analyze.py -n thread_number -d dbname [-p gpadmin_password]
Parameters:
thread_number - number of parallel threads to run
dbname - name of the database
gpadmin_password - password of the gpadmin user"""
sys.exit(0)
if not options.dbname:
logger.error('Failed to start utility. Please, specify database name with "-d" key')
sys.exit(1)
if not options.nthreads:
logger.error('Failed to start utility. Please, specify number of threads with "-n" key')
sys.exit(1)
if not options.stat_mem:
logger.error('Failed to start utility. Please, specify statement_mem parameter with "-m" key')
sys.exit(1)
if not options.filename:
logger.error('Failed to start utility. Please, specify filename parameter (e.g. initial, compare) with "-f" key')
sys.exit(1)
if not options.location:
logger.error('Failed to start utility. Please, specify result folder parameter with "-l" key')
sys.exit(1)
return options
def parseargs():
parser = OptParser(option_class=OptChecker
, description=' '.join(DESCRIPTION.split())
, version='%prog version $Revision: #12 $'
)
parser.setHelp(_help)
parser.set_usage('%prog ' + _usage)
parser.remove_option('-h')
parser.add_option('-f', '--file', default='',
help='the name of a file containing the re-sync file list.')
parser.add_option('-v', '--verbose', action='store_true',
help='debug output.', default=False)
parser.add_option('-h', '-?', '--help', action='help',
help='show this help message and exit.', default=False)
parser.add_option('--usage', action="briefhelp")
parser.add_option('-d', '--master_data_directory', type='string',
dest="masterDataDirectory",
metavar="<master data directory>",
help="Optional. The master host data directory. If not specified, the value set for $MASTER_DATA_DIRECTORY will be used.",
default=get_masterdatadir()
)
parser.add_option('-a', help='don\'t ask to confirm repairs',
dest='confirm', default=True, action='store_false')
"""
Parse the command line arguments
"""
(options, args) = parser.parse_args()
if len(args) > 0:
logger.error('Unknown argument %s' % args[0])
parser.exit()
return options, args
def parseargs(self):
parser = OptParser(option_class=OptChecker,
description=' '.join(self.description.split()),
version='%prog version $Revision: $')
parser.set_usage(
'%prog is a utility script used by gprecoverseg, and gpaddmirrors and is not intended to be run separately.'
)
parser.remove_option('-h')
#TODO we may not need the verbose flag
parser.add_option('-v',
'--verbose',
action='store_true',
help='debug output.',
default=False)
parser.add_option('-c', '--confinfo', type='string')
parser.add_option('-b',
'--batch-size',
type='int',
default=DEFAULT_SEGHOST_NUM_WORKERS,
metavar='<batch_size>')
parser.add_option('-f',
'--force-overwrite',
dest='forceoverwrite',
action='store_true',
default=False)
parser.add_option('-l',
'--log-dir',
dest="logfileDirectory",
type="string")
parser.add_option(
'',
'--era',
dest="era",
help="coordinator era",
)
# Parse the command line arguments
self.options, _ = parser.parse_args()
if not self.options.confinfo:
raise Exception('Missing --confinfo argument.')
if not self.options.logfileDirectory:
raise Exception('Missing --log-dir argument.')
self.logger = gplog.setup_tool_logging(
os.path.split(self.file_name)[-1],
unix.getLocalHostname(),
unix.getUserName(),
logdir=self.options.logfileDirectory)
if self.options.batch_size <= 0:
self.logger.warn('batch_size was less than zero. Setting to 1.')
self.options.batch_size = 1
if self.options.verbose:
gplog.enable_verbose_logging()
self.seg_recovery_info_list = recoveryinfo.deserialize_list(
self.options.confinfo)
if len(self.seg_recovery_info_list) == 0:
raise Exception(
'No segment configuration values found in --confinfo argument')
def parseargs():
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('-d', '--dbname', type='string')
parser.add_option('-u', '--user', type='string')
parser.add_option('-p', '--password', type='string')
parser.add_option('-n', '--nthreads', type='int')
parser.add_option('-s', '--stat_mem', type='string')
parser.add_option('-f', '--tablefile', type='string')
parser.add_option('-t', '--distkeyfile', type='string')
parser.add_option('-m', '--metadatatablesuffix', type='string')
(options, args) = parser.parse_args()
if options.help:
print """Script performs analysis of table row number and number of
unique values of table distribution key
Usage:
./data_consistency_check.py -d dbname [-n thread_number] [-u user_name] [-p password]
[-s statement_mem] [-f tablefile] [-t distkeyfile]
[-m metadatatablesuffix]
Parameters:
-d | --dbname - name of the database to process
-n | --nthreads - number of parallel threads to run
-u | --user - user to connect to the database
-p | --password - password to connect to the database
-s | --statement_mem - the value of statement_mem to use
-f | --tablefile - file with the list of tables to process
-t | --distkeyfile - file with the tables which should be analyzed with
counting distinct values of distribution key
-m | --metadatatablesuffix
- suffix for the table to store script metadata in
Metadata objects created are:
public.__zz_pivotal_{suffix} - view with the final information on row counts
public.__zz_pivotal_{suffix}_l - list of tables to process
public.__zz_pivotal_{suffix}_p - current progress of table row count calculation
After the run has finished for the second time, join two metadata tables by
the "tablename" field like this:
select m1.tablename as table_first,
m2.tablename as table_second,
m1.rowcount as rows_before,
m2.rowcount as rows_after,
m1.distkeycount as dist_keys_before,
m2.distkeycount as dist_keys_after
from {metadatatable1} as m1
full outer join {metadatatable2} as m2
on m1.tablename = m2.tablename
where m1.tablename is null
or m2.tablename is null
or m1.rowcount is distinct from m2.rowcount
or m1.distkeycount is distinct from m2.distkeycount
"""
sys.exit(0)
if not options.dbname:
logger.error('Failed to start utility. Please, specify database name with "-d" key')
sys.exit(1)
if not options.nthreads:
logger.info('Number of threads is not specified. Using 1 by default')
options.nthreads = 1
if not options.stat_mem:
logger.info('Statement memory is not specified. Using 125MB by default')
options.stat_mem = '125MB'
if not options.metadatatablesuffix:
logger.info('Metadata table suffix is not specified. Using "table_list" by default')
options.metadatatablesuffix = 'table_list'
else:
if not re.match('^[0-9a-z_]*$', options.metadatatablesuffix):
logger.error ('Metadata suffix must contain only lowercase letters, numbers 0-9 and underscore sign')
sys.exit(1)
if not options.tablefile:
logger.info('No tablefile specified. Will process all the tables in database by default')
if not options.distkeyfile:
logger.info('No distribution key table file specified. Will omit distribution key analysis')
return options
parts = t.split(".")
if len(parts) != 2:
raise Exception("Bad table in filter list")
schema = parts[0].strip()
table = parts[1].strip()
dump_tables.add((schema, table))
dump_schemas.add(schema)
return (dump_schemas, dump_tables)
def extract_schema(line):
temp = line[len_search_path_expr:]
idx = temp.find(",")
if idx == -1:
return None
schema = temp[:idx]
return schema.strip('"')
if __name__ == "__main__":
parser = OptParser(option_class=OptChecker)
parser.remove_option("-h")
parser.add_option("-h", "-?", "--help", action="store_true")
parser.add_option("-t", "--tablefile", type="string", default=None)
(options, args) = parser.parse_args()
if not options.tablefile:
raise Exception("-t table file name has to be specified")
(schemas, tables) = get_table_schema_set(options.tablefile)
process_schema(schemas, tables, sys.stdin, sys.stdout)
def parse_command_line():
parser = OptParser(option_class=OptChecker,
description=' '.join(_description.split()))
parser.setHelp(_help)
parser.set_usage('%prog ' + _usage)
parser.remove_option('-h')
parser.add_option('--start', action='store_true',
help='Start the Greenplum Performance Monitor web server.')
parser.add_option('--stop', action='store_true',
help='Stop the Greenplum Performance Monitor web server.')
parser.add_option('--restart', action='store_true',
help='Restart the Greenplum Performance Monitor web server.')
parser.add_option('--status', action='store_true',
help='Display the status of the Gerrnplum Performance Monitor web server.')
parser.add_option('--setup', action='store_true',
help='Setup the Greenplum Performance Monitor web server.')
parser.add_option('--version', action='store_true',
help='Display version information')
parser.add_option('--upgrade', action='store_true',
help='Upgrade a previous installation of the Greenplum Performance Monitors web UI')
parser.set_defaults(verbose=False,filters=[], slice=(None, None))
# Parse the command line arguments
(options, args) = parser.parse_args()
if options.version:
version()
sys.exit(0)
# check for too many options
opt_count = 0
if options.start:
opt_count+=1
if options.stop:
opt_count+=1
if options.setup:
opt_count+=1
if options.upgrade:
opt_count+=1
if options.status:
opt_count+=1
if opt_count > 1:
parser.print_help()
parser.exit()
return options, args
def parseargs():
global allocation_rate, memory_split, cpu_split
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('-s', '--seghosts', type='int')
parser.add_option('-d', '--segdbs', type='int')
parser.add_option('-p', '--primarydirs', type='string')
parser.add_option('-m', '--mirrordirs', type='string')
parser.add_option('-o', '--outfile', type='string')
parser.add_option('-i', '--initsystem', action='store_true')
parser.add_option('-e', '--expansion', action='store_true')
parser.add_option('-a', '--added', type='int')
parser.add_option('-C', '--maxcontent', type='int')
parser.add_option('-D', '--maxdbid', type='int')
(options, args) = parser.parse_args()
if options.help:
print """
Script generates the segment placement map for Greenplum initialization.
By default the primary segment directories are /data1/primary and /data2/primary
and the mirror directories are /data1/mirror and /data2/mirror. Output file is
by default located in the working directory and is called gpinitsystem_map_<timestamp>
Usage:
python generate_segment_map.py -s number_of_segment_hosts -d number_of_dbs_per_host
[-o outfile]
[-p directories_for_primaries -m directories_for_mirrors]
[-i | --initsystem]
[-e -a number_of_hosts_added --maxcontent max_content
--maxdbid max_dbid]
-s | --seghosts - Number of segment hosts in the system
-d | --segdbs - Number of segment databases per host
-o | --outfile - Output file
-p | --primarydirs - Colon-separated list of primary segment directories
-m | --mirrordirs - Colon-separated list of mirror segment directories
-i | --initsystem - Generate map file for system initialization
-e | --expansion - Generate map file for system expansion
-a | --added - Number of segment hosts added during expansion
-C | --maxcontent - Maximal number of content in existing GPDB
-D | --maxdbid - Maximal number of dbid in existing GPDB
Examples:
1. Initialize system with 16 segment servers and 4 segments per host:
python generate_segment_map.py -s 16 -d 4 -i > gpinitsystem_map
2. Prepare expansion map to add 8 segment servers to existing system with
16 segment servers and 4 segment databases per host:
python generate_segment_map.py -s 16 -d 4 -e -a 8 --maxcontent 100 --maxdbid 100 > gpexpand_map
"""
sys.exit(0)
if not options.seghosts or not options.segdbs:
raise_err('You must specify both number of segment hosts (-s) and number of segment databases on each host (-d)')
if (options.primarydirs and not options.mirrordirs) or (not options.primarydirs and options.mirrordirs):
raise_err('You must either specify both folders for primaries and mirrors or use defaults for both')
if (not options.initsystem and not options.expansion) or (options.initsystem and options.expansion):
raise_err('You should either specify init system mode ( -i ) or expansion mode ( -e )')
if options.expansion and not options.added:
raise_err('In expansion mode you must specify number of segment servers added')
return options
def parseargs():
parser = OptParser(option_class=OptChecker,
description=' '.join(DESCRIPTION.split()),
version='%prog version $Revision: #12 $')
parser.setHelp(_help)
parser.set_usage('%prog ' + _usage)
parser.remove_option('-h')
parser.add_option(
'-f',
'--file',
default='',
help='the name of a file containing the re-sync file list.')
parser.add_option('-v',
'--verbose',
action='store_true',
help='debug output.',
default=False)
parser.add_option('-h',
'-?',
'--help',
action='help',
help='show this help message and exit.',
default=False)
parser.add_option('--usage', action="briefhelp")
parser.add_option(
'-d',
'--master_data_directory',
type='string',
dest="masterDataDirectory",
metavar="<master data directory>",
help=
"Optional. The master host data directory. If not specified, the value set for $MASTER_DATA_DIRECTORY will be used.",
default=get_masterdatadir())
parser.add_option('-a',
help='don\'t ask to confirm repairs',
dest='confirm',
default=True,
action='store_false')
"""
Parse the command line arguments
"""
(options, args) = parser.parse_args()
if len(args) > 0:
logger.error('Unknown argument %s' % args[0])
parser.exit()
return options, args
"""
Note: white space in schema and table name is supported now, don't do strip on them
"""
if not os.path.exists(schema_level_restore_file):
raise Exception("schema level restore file path %s does not exist" % schema_level_restore_file)
schema_level_restore_list = []
with open(schema_level_restore_file) as fr:
schema_entries = fr.read()
schema_level_restore_list = schema_entries.splitlines()
return schema_level_restore_list
if __name__ == "__main__":
parser = OptParser(option_class=OptChecker)
parser.remove_option("-h")
parser.add_option("-h", "-?", "--help", action="store_true")
parser.add_option("-t", "--tablefile", type="string", default=None)
parser.add_option("-c", "--change-schema-file", type="string", default=None)
parser.add_option("-s", "--schema-level-file", type="string", default=None)
(options, args) = parser.parse_args()
if not (options.tablefile or options.schema_level_file):
raise Exception("-t table file name or -s schema level file name must be specified")
elif options.schema_level_file and options.change_schema_file:
raise Exception("-s schema level file option can not be specified with -c change schema file option")
schemas, tables = None, None
if options.tablefile:
(schemas, tables) = get_table_schema_set(options.tablefile)
change_schema_name = None
if options.change_schema_file:
"""
Only strip the '\n' as it is one of the non-supported chars to be part
of the schema or table name
"""
if not os.path.exists(change_schema_file):
raise Exception('change schema file path %s does not exist' % change_schema_file)
change_schema_name = None
with open(change_schema_file) as fr:
line = fr.read()
change_schema_name = line.strip('\n')
return change_schema_name
if __name__ == "__main__":
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('-t', '--tablefile', type='string', default=None)
parser.add_option('-m', '--master_only', action='store_true')
parser.add_option('-c', '--change-schema-file', type='string', default=None)
parser.add_option('-s', '--schema-level-file', type='string', default=None)
(options, args) = parser.parse_args()
if not (options.tablefile or options.schema_level_file):
raise Exception('-t table file name or -s schema level file name must be specified')
elif options.schema_level_file and options.change_schema_file:
raise Exception('-s schema level file option can not be specified with -c change schema file option')
schemas, tables = None, None
if options.tablefile:
(schemas, tables) = get_table_schema_set(options.tablefile)
change_schema_name = None
if retCode:
return 1
return 0
###### main()
if __name__ == '__main__':
gphome = os.environ.get('GPHOME')
if not gphome:
sys.exit(2)
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('-s', '--spaceusage', action='store_true')
parser.add_option('-u', '--user', type='string')
parser.add_option('-d', '--dblist', type='string')
(options, args) = parser.parse_args()
if options.help:
print __doc__
sys.exit(1)
cmd=""
# Get defaults from healthmon config
if options.spaceusage:
dblist=options.dblist.split(',')
if len(dblist) == 0 or not options.user:
def parseargs():
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('-d', '--dbname', type='string')
parser.add_option('-p', '--password', type='string')
parser.add_option('-n', '--nthreads', type='int')
parser.add_option('-u', '--user', type='string')
parser.add_option('-l', '--location', type='string')
parser.add_option('-m', '--stat_mem', type='string')
parser.add_option('-f', '--filename', type='string')
(options, args) = parser.parse_args()
if options.help:
print """Script performs serial restore of the backup files in case
of the cluster itopology change.
Usage:
./parallel_analyze.py -n thread_number -d dbname [-p gpadmin_password]
Parameters:
thread_number - number of parallel threads to run
dbname - name of the database
gpadmin_password - password of the gpadmin user"""
sys.exit(0)
if not options.dbname:
logger.error(
'Failed to start utility. Please, specify database name with "-d" key'
)
sys.exit(1)
if not options.nthreads:
logger.error(
'Failed to start utility. Please, specify number of threads with "-n" key'
)
sys.exit(1)
if not options.stat_mem:
logger.error(
'Failed to start utility. Please, specify statement_mem parameter with "-m" key'
)
sys.exit(1)
if not options.filename:
logger.error(
'Failed to start utility. Please, specify filename parameter (e.g. initial, compare) with "-f" key'
)
sys.exit(1)
if not options.location:
logger.error(
'Failed to start utility. Please, specify result folder parameter with "-l" key'
)
sys.exit(1)
return options
def createParser():
"""
Create parser expected by simple_main
"""
parser = OptParser(option_class=OptChecker,
description=' '.join(DESCRIPTION.split()),
version='%prog version main build dev')
parser.setHelp(HELP)
#
# Note that this mirroringmode parameter should only be either mirrorless or quiescent.
# If quiescent then it is implied that there is pickled transition data that will be
# provided (using -p) to immediately convert to a primary or a mirror.
#
addStandardLoggingAndHelpOptions(parser,
includeNonInteractiveOption=False)
parser.add_option(
"-C",
"--collation",
type="string",
help="values for lc_collate, lc_monetary, lc_numeric separated by :"
)
parser.add_option("-D",
"--dblist",
dest="dblist",
action="append",
type="string")
parser.add_option("-M",
"--mirroringmode",
dest="mirroringMode",
type="string")
parser.add_option("-p",
"--pickledTransitionData",
dest="pickledTransitionData",
type="string")
parser.add_option("-V",
"--gp-version",
dest="gpversion",
metavar="GP_VERSION",
help="expected software version")
parser.add_option("-n",
"--numsegments",
dest="num_cids",
help="number of distinct content ids in cluster")
parser.add_option("", "--era", dest="era", help="master era")
parser.add_option("-t",
"--timeout",
dest="timeout",
type="int",
default=gp.SEGMENT_TIMEOUT_DEFAULT,
help="seconds to wait")
parser.add_option(
'-U',
'--specialMode',
type='choice',
choices=['upgrade', 'maintenance'],
metavar='upgrade|maintenance',
action='store',
default=None,
help='start the instance in upgrade or maintenance mode')
parser.add_option('',
'--wrapper',
dest="wrapper",
default=None,
type='string')
parser.add_option('',
'--wrapper-args',
dest="wrapper_args",
default=None,
type='string')
return parser
def parseargs(args):
global logger
pguser = os.environ.get("PGUSER") or unix.getUserName()
pghost = os.environ.get("PGHOST") or unix.getLocalHostname()
pgport = os.environ.get("PGPORT") or 5432
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-?', '--help', '-h', action='store_true', default=False)
parser.add_option('-n', '--host', default=pghost)
parser.add_option('-p', '--port', default=pgport)
parser.add_option('-u', '--username', default=pguser)
parser.add_option('-w', '--password', default=False, action='store_true')
parser.add_option('-v', '--verbose', default=False, action='store_true')
parser.add_option('-q', '--quiet', default=True, action='store_true')
(options, args) = parser.parse_args()
if options.help:
print __doc__
sys.exit(1)
try:
options.port = int(options.port)
except:
logger.error("Invalid PORT: '%s'" % options.port)
sys.exit(1)
if options.verbose:
gplog.enable_verbose_logging()
elif options.quiet:
gplog.quiet_stdout_logging()
return options
def createParser():
"""
Create parser expected by simple_main
"""
parser = OptParser(option_class=OptChecker,
description=' '.join(DESCRIPTION.split()),
version='%prog version main build dev')
parser.setHelp(HELP)
#
# Note that this mirroringmode parameter should only be either mirrorless or quiescent.
# If quiescent then it is implied that there is pickled transition data that will be
# provided (using -p) to immediately convert to a primary or a mirror.
#
addStandardLoggingAndHelpOptions(parser, includeNonInteractiveOption=False)
parser.add_option("-D", "--dblist", dest="dblist", action="append", type="string")
parser.add_option("-M", "--mirroringmode", dest="mirroringMode", type="string")
parser.add_option("-p", "--pickledTransitionData", dest="pickledTransitionData", type="string")
parser.add_option("-V", "--gp-version", dest="gpversion", metavar="GP_VERSION", help="expected software version")
parser.add_option("-n", "--numsegments", dest="num_cids", help="number of distinct content ids in cluster")
parser.add_option("", "--era", dest="era", help="master era")
parser.add_option("-t", "--timeout", dest="timeout", type="int", default=gp.SEGMENT_TIMEOUT_DEFAULT,
help="seconds to wait")
parser.add_option('-U', '--specialMode', type='choice', choices=['upgrade', 'maintenance'],
metavar='upgrade|maintenance', action='store', default=None,
help='start the instance in upgrade or maintenance mode')
parser.add_option('', '--wrapper', dest="wrapper", default=None, type='string')
parser.add_option('', '--wrapper-args', dest="wrapper_args", default=None, type='string')
parser.add_option('', '--master-checksum-version', dest="master_checksum_version", default=None, type='string', action="store")
parser.add_option('-B', '--parallel', type="int", dest="parallel", default=gp.DEFAULT_GPSTART_NUM_WORKERS, help='maximum size of a threadpool to start segments')
return parser
def parseargs():
parser = OptParser(option_class=OptChecker)
parser.setHelp(_help)
parser.remove_option('-h')
parser.add_option('-h',
'-?',
'--help',
action='help',
help='show this help message and exit')
parser.add_option('--file',
type='string',
help='Required: The absolute path of postgresql.conf')
parser.add_option(
'--add-parameter',
type='string',
help='The configuration parameter to add. --value is required.')
parser.add_option(
'--value',
type='string',
help='The configuration value to add when using --add-parameter.')
parser.add_option('--get-parameter',
type='string',
help='The configuration parameter value to return.')
parser.add_option('--remove-parameter',
type='string',
help='The configuration parameter value to disable.')
(options, args) = parser.parse_args()
return validate_args(options)
def parseargs(args):
global logger
pguser = os.environ.get("PGUSER") or unix.getUserName()
pghost = os.environ.get("PGHOST") or unix.getLocalHostname()
pgport = os.environ.get("PGPORT") or 5432
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-?', '--help', '-h', action='store_true', default=False)
parser.add_option('-n', '--host', default=pghost)
parser.add_option('-p', '--port', default=pgport)
parser.add_option('-u', '--username', default=pguser)
parser.add_option('-w', '--password', default=False, action='store_true')
parser.add_option('-v', '--verbose', default=False, action='store_true')
parser.add_option('-q', '--quiet', default=True, action='store_true')
(options, args) = parser.parse_args()
if options.help:
print __doc__
sys.exit(1)
try:
options.port = int(options.port)
except:
logger.error("Invalid PORT: '%s'" % options.port)
sys.exit(1)
if options.verbose:
gplog.enable_verbose_logging()
elif options.quiet:
gplog.quiet_stdout_logging()
return options
elif (line[0] == copy_start) and line.startswith(copy_expr) and line.endswith(copy_expr_end):
table = extract_table(line)
table = table.strip('"')
if table and (schema, table) in dump_tables or (schema, '*') in dump_tables:
output = True
elif output and (line[0] == copy_end_start) and line.startswith(copy_end_expr):
table = None
output = False
fdout.write(line)
if output:
fdout.write(line)
if __name__ == "__main__":
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('-t', '--tablefile', type='string', default=None)
parser.add_option('-m', '--master_only', action='store_true')
(options, args) = parser.parse_args()
if not options.tablefile:
raise Exception('-t table file name has to be specified')
(schemas, tables) = get_table_schema_set(options.tablefile)
if options.master_only:
process_schema(schemas, tables, sys.stdin, sys.stdout)
else:
process_data(schemas, tables, sys.stdin, sys.stdout)
def parseargs():
evs = ((g_gphome, 'GPHOME'), (g_gpperfmonhome, 'GPPERFMONHOME'), (g_master_data_directory, 'MASTER_DATA_DIRECTORY'))
for (var, desc) in evs:
if not var:
print >> sys.stderr, "$%s must be set" % desc
sys.exit(1)
global g_options
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('-c', '--commandline', type='string')
parser.add_option('-d', '--directory', type='string')
parser.add_option('-g', '--group', type='string')
parser.add_option('-a', '--application', type='string')
parser.add_option('-t', '--token', type='string')
parser.add_option('-r', '--remotehost', type='string')
parser.add_option('--nodaemon', action='store_true')
parser.add_option('--nostreaming', action='store_true')
(g_options, args) = parser.parse_args()
if g_options.help:
print __doc__
sys.exit(0)
mustHaves = ['commandline', 'directory', 'application', 'token']
for clo in mustHaves:
if not getattr(g_options, clo):
print >> sys.stderr, "Missing required command line attribute: --%s" % clo
sys.exit(1)
def parseargs():
parser = OptParser(option_class=OptChecker)
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('-d', '--dbname', type='string')
parser.add_option('-u', '--user', type='string')
parser.add_option('-p', '--password', type='string')
parser.add_option('-n', '--nthreads', type='int')
parser.add_option('-s', '--stat_mem', type='string')
parser.add_option('-f', '--tablefile', type='string')
parser.add_option('-t', '--distkeyfile', type='string')
parser.add_option('-m', '--metadatatable', type='string')
(options, args) = parser.parse_args()
if options.help:
print """Script performs analysis of table row number and number of
unique values of table distribution key
Usage:
./data_consistency_check.py -d dbname [-n thread_number] [-u user_name] [-p password]
[-s statement_mem] [-f tablefile] [-t distkeyfile]
[-m metadatatable]
Parameters:
-d | --dbname - name of the database to process
-n | --nthreads - number of parallel threads to run
-u | --user - user to connect to the database
-p | --password - password to connect to the database
-s | --statement_mem - the value of statement_mem to use
-f | --tablefile - file with the list of tables to process
-t | --distkeyfile - file with the tables which should be analyzed with
counting distinct values of distribution key
-m | --metadatatable - name of the table to store the metadata in
Metadata objects created are:
{metadatatable} - view with the final information on row counts
{metadatatable}_l - list of tables to process
{metadatatable}_p - current progress of table row count calculation
After the run has finished for the second time, join two metadata tables by
the "tablename" field like this:
select m1.tablename as table_first,
m2.tablename as table_second,
m1.rowcount as rows_before,
m2.rowcount as rows_after,
m1.distkeycount as dist_keys_before,
m2.distkeycount as dist_keys_after
from {metadatatable1} as m1
full outer join {metadatatable2} as m2
on m1.tablename = m2.tablename
where m1.tablename is null
or m2.tablename is null
or m1.rowcount is distinct from m2.rowcount
or m1.distkeycount is distinct from m2.distkeycount
"""
sys.exit(0)
if not options.dbname:
logger.error('Failed to start utility. Please, specify database name with "-d" key')
sys.exit(1)
if not options.nthreads:
logger.info('Number of threads is not specified. Using 1 by default')
options.nthreads = 1
if not options.stat_mem:
logger.info('Statement memory is not specified. Using 125MB by default')
options.stat_mem = '125MB'
if not options.metadatatable:
logger.info('Metadata table name is not specified. Using public.__aag_table_list by default')
options.metadatatable = 'public.__aag_table_list'
if not options.tablefile:
logger.info('No tablefile specified. Will process all the tables in database by default')
if not options.distkeyfile:
logger.info('No distribution key table file specified. Will omit distribution key analysis')
return options
