def additional_args_process(self):
if self.args_handler.args.create_dst_table:
self.src_to_dst_table_ddl(self.args_handler.args.src_table,
self.args_handler.args.dst_table,
self.src_conn, self.dst_conn,
self.args_handler)
print('-- created destination table: %s' %
self.args_handler.args.dst_table)
#thread use may have severe impact on the YB cluster, so I'm limiting it to super users
if (self.args_handler.args.threads > 1
and not self.src_conn.ybdb['is_super_user']
and not self.dst_conn.ybdb['is_super_user']):
Common.error(
Text.color(
"The '--threads' option is only supported for YBDB super users.",
'yellow'))
if (self.args_handler.args.chunk_rows
and self.src_conn.ybdb['version_major'] < 4):
Common.error(
Text.color(
"The '--chunk_rows' option is only supported on YBDB version 4 or higher."
" The source db is running YBDB %s..." %
self.src_conn.ybdb['version'], 'yellow'))
self.log_file_name_template = (
'{}{}{}_{}_{{{{CofC}}}}{{{{TofT}}}}_{{log_type}}.log'.format(
('%s/' % self.args_handler.args.log_dir
if self.args_handler.args.log_dir else ''),
('%s_' % self.args_handler.args.log_prefix
if self.args_handler.args.log_prefix else ''),
datetime.now().strftime("%Y%m%d_%H%M%S"),
"%04d" % random.randint(0, 9999)))
def main():
acs = analyze_columns()
sys.stdout.write('-- Running column analysis.\n')
acs.execute()
if acs.cmd_results.stdout != '':
if acs.args_handler.args.output_format == 1:
rows = []
headers = True
for line in acs.cmd_results.stdout.split('\n'):
if line == '':
continue
row = line.split('|')
if headers:
row = [col.replace('_', '\n') for col in row]
headers = False
rows.append(row)
print(tabulate(rows, headers="firstrow"))
else:
sys.stdout.write(acs.cmd_results.stdout)
if acs.cmd_results.stderr != '':
Common.error(acs.cmd_results.stderr, no_exit=True)
else:
sys.stdout.write('-- Completed column analysis.\n')
exit(acs.cmd_results.exit_code)
def execute(self):
dbs = self.get_dbs()
self.db_filter_args.schema_set_all_if_none()
db_ct = 0
broken_view_ct = 0
sys.stdout.write('-- Running broken view check.\n')
for db in dbs:
db_ct += 1
cmd_results = self.db_conn.call_stored_proc_as_anonymous_block(
'yb_check_db_views_p',
args={'a_filter': self.db_filter_sql()},
pre_sql=('\c %s\n' % db))
if cmd_results.exit_code == 0:
sys.stdout.write(Common.quote_object_paths(cmd_results.stdout))
elif cmd_results.stderr.find('permission denied') == -1:
Common.error(cmd_results.stderr)
exit(cmd_results.exit_code)
db_broken_view_ct = len(cmd_results.stdout.split())
broken_view_ct += db_broken_view_ct
sys.stdout.write('-- %d broken view/s in "%s".\n' %
(db_broken_view_ct, db))
sys.stdout.write(
'-- Completed check, found %d broken view/s in %d db/s.\n' %
(broken_view_ct, db_ct))
def main():
ts = table_skew()
if not ts.db_conn.ybdb['is_super_user']:
Common.error('must be run by a database super user...')
print(ts.execute())
exit(0)
def init_args(self):
"""Initialize the args class.
This initialization performs argument parsing.
It also provides access to functions such as logging and command
execution.
:return: An instance of the `args` class
"""
cnfg = Util.config_default.copy()
cnfg['description'] = 'Run unit test cases on utility.'
cnfg['positional_args_usage'] = None
args_handler = ArgsHandler(cnfg, init_default=False)
args_handler.args_process_init()
args_handler.args_add_positional_args()
args_handler.args_add_optional()
args_test_required_grp = args_handler.args_parser.add_argument_group(
'test required arguments')
args_test_required_grp.add_argument(
"--test_name",
"--tn",
"-t",
dest="name",
help="the test case name to run, like 'yb_get_table_name'"
" for the test case file 'test_cases__yb_get_table_name.py'")
args_test_required_grp.add_argument(
"--all",
action="store_true",
help="run test cases for all the test case files")
args_test_optional_grp = args_handler.args_parser.add_argument_group(
'test optional arguments')
args_test_optional_grp.add_argument(
"--host",
"-h",
"-H",
dest="host",
help="database server hostname,"
" overrides YBHOST env variable, the host where the tests are run")
args_test_optional_grp.add_argument(
"--case",
type=int,
default=None,
help="unit test case number to execute")
args_test_optional_grp.add_argument(
"--print_test",
"--pt",
action="store_true",
help="instead of the test command display what the test ran")
args_test_optional_grp.add_argument("--print_output",
"--po",
action="store_true",
help="print the test output")
args_test_optional_grp.add_argument(
"--print_diff",
"--pd",
action="store_true",
help="if the test fails, print the diff of the expected"
" verse actual result")
args_test_optional_grp.add_argument(
"--python_exe",
default=None,
help="python executable to run tests with, this allows testing"
" with different python versions, defaults to 'python3'")
args = args_handler.args_process()
if args.python_exe:
if os.access(args.python_exe, os.X_OK):
cmd_results = Cmd('%s --version' % args.python_exe)
self.test_py_version = (int(
cmd_results.stderr.split(' ')[1].split('.')[0]))
else:
Common.error("'%s' is not found or not executable..." %
args.python_exe)
else:
self.test_py_version = 3
if not args.host and os.environ.get("YBHOST"):
args.host = os.environ.get("YBHOST")
elif not args.host and len(self.config.hosts) == 1:
args.host = self.config.hosts[0]
if (args.host and not (args.host in self.config.hosts)):
Common.error(
"the '%s' host is not configured for testing,"
" run 'test_create_host_objects.py' to create host db objects for testing"
% args.host,
color='white')
elif len(self.config.hosts) == 0:
Common.error(
"currently there are no hosts configures for testing,"
" run 'test_create_host_objects.py' to create host db objects for testing",
color='white')
elif len(self.config.hosts) > 1 and not args.host:
Common.error(
"currently there is more than 1 host(%s) configures for testing,"
" use the --host option or YBHOST environment variable to select a host"
% self.config.hosts,
color='white')
if bool(args.name) == args.all: # exclusive or
Common.error(
"either the option --test_name or --all must be specified not both"
)
self.test_case_files = []
if args.name:
test_case_file_path = '%s/test_cases__%s.py' % (path, args.name)
if os.access(test_case_file_path, os.R_OK):
self.test_case_files.append(test_case_file_path)
else:
Common.error("test case '%s' has no test case file '%s'..." %
(args.name, test_case_file_path))
else:
for test_case_file_path in glob.glob("%s/test_cases__*.py" % path):
if os.access(test_case_file_path, os.R_OK):
self.test_case_files.append(test_case_file_path)
self.test_case_files.sort()
self.args = args
return args
def __init__(self):
args_handler = self.init_args()
args_handler.args.conn_db = 'yellowbrick'
db_conn = DBConnect(args_handler)
if not (db_conn.ybdb['has_create_user']
and db_conn.ybdb['has_create_db']):
Common.error(
'You must login as a user with create database/'
'user permission to drop the test database/user objects...')
configFilePath = '%s/%s' % (os.path.expanduser('~'), '.YbEasyCli')
config = configparser.ConfigParser()
config.read(configFilePath)
section = '%s_%s' % ('test', db_conn.env['host'])
if config.has_section(section):
test_user = config.get(section, 'user')
print(
"\nDropping the '%s' test environment, including the '%s' and '%s' databases.\n"
% (Text.color(db_conn.env['host'], 'cyan'),
Text.color(config.get(section, 'db1'), 'cyan'),
Text.color(config.get(section, 'db2'), 'cyan')))
answered_yes = input(
" Enter(yes) to continue: ").lower() == 'yes'
if answered_yes:
cmd_results = db_conn.ybsql_query(
"DROP DATABASE IF EXISTS {db1}; DROP DATABASE IF EXISTS {db2};"
.format(db1=config.get(section, 'db1'),
db2=config.get(section, 'db2')))
cmd_results.write()
if cmd_results.stderr != '':
exit(cmd_results.exit_code)
print("\nDropped databases '%s' and '%s', if they existed..." %
(Text.color(config.get(section, 'db1'), 'cyan'),
Text.color(config.get(section, 'db2'), 'cyan')))
config.remove_section(section)
config_fp = open(configFilePath, 'w')
config.write(config_fp)
config_fp.close()
os.chmod(configFilePath, stat.S_IREAD | stat.S_IWRITE)
answered_yes = input(
"\n Enter(yes) to drop user '%s': " %
Text.color(test_user, 'cyan')).lower() == 'yes'
if answered_yes:
cmd_results = db_conn.ybsql_query(
"DROP USER IF EXISTS %s" % test_user)
cmd_results.write()
if cmd_results.stderr != '':
exit(cmd_results.exit_code)
print("\nDropped user '%s', if existed..." %
(Text.color(test_user, 'cyan')))
else:
print(Text.color('\nExiting without clean up...', 'yellow'))
else:
Common.error(
"There is no test environment setup for '%s' in your"
" '~/.YbEasyCli file', run 'test_create_host_objects.py'"
" to set up this host" % db_conn.env['host'],
color='yellow')
def __init__(self):
args_handler = self.init_args()
args_handler.args.conn_db = 'yellowbrick'
DBConn = DBConnect(args_handler)
if (not (DBConn.ybdb['is_super_user'])
and not (DBConn.ybdb['has_create_user']
and DBConn.ybdb['has_create_db'])):
Common.error(
'You must login as a user with create database/'
'user permission to create all the required test database objects...'
)
configFilePath = '%s/%s' % (os.path.expanduser('~'), '.YbEasyCli')
config = configparser.ConfigParser()
config.read(configFilePath)
section = '%s_%s' % ('test', DBConn.env['host'])
if config.has_section(section):
Common.error(
"A test environment has already been set up for '%s',"
" first run test_drop_host_objects.py to clean up the old host objects or cleanup '%s'..."
% (DBConn.env['host'], configFilePath),
color='yellow')
print(
'\nThe util testing framework requires a test user and 2 test databases.\n'
"The test user may be an existing user, if the user doesn't exist it will be created.\n"
"If you continue the test user password will be stored in the ~/.YbEasyCli file.\n"
"Additionally, 2 test databases will be created, these databases must not already exist.\n"
)
config.add_section(section)
while True:
test_user = input(" Supply the DB test user name: ")
if test_user != '':
config.set(section, 'user', test_user)
break
while True:
test_pwd = getpass.getpass(" Supply the password for '%s': " %
(Text.color(test_user, 'cyan')))
if test_pwd != '':
config.set(section, 'password', test_pwd)
break
cmd_results = DBConn.ybsql_query(
"""SELECT TRUE FROM sys.user WHERE name = '%s'""" % (test_user))
if cmd_results.stdout.strip() == 't':
# exits on failed connection
self.get_DBConn(test_user, test_pwd, DBConn.env['conn_db'],
DBConn.env['host'])
#test_user_DBConn = self.get_DBConn(test_user, test_pwd
# , DBConn.env['conn_db'], DBConn.env['host'], on_fail_exit=False)
else:
cmd_results = DBConn.ybsql_query(
"""CREATE USER %s PASSWORD '%s'""" % (test_user, test_pwd))
cmd_results.on_error_exit()
print("\nCreated database user '%s'..." %
Text.color(test_user, 'cyan'))
self.test_user_login(test_user, test_pwd, DBConn.env['conn_db'],
DBConn.env['host'])
while True:
test_db_prefix = input(
"\n Supply the database prefix for the 2 test dbs: ")
if test_db_prefix != '':
test_db1 = '%s_db1' % test_db_prefix
test_db2 = '%s_db2' % test_db_prefix
config.set(section, 'db1', test_db1)
config.set(section, 'db2', test_db2)
break
cmd_results = DBConn.ybsql_query(
"CREATE DATABASE {db1} ENCODING=LATIN9; CREATE DATABASE {db2} ENCODING=UTF8;"
" ALTER DATABASE {db1} OWNER TO {user};"
" ALTER DATABASE {db2} OWNER TO {user};"
" GRANT CONNECT ON DATABASE {db1} TO {user};"
" GRANT CONNECT ON DATABASE {db2} TO {user};".format(
db1=test_db1, db2=test_db2, user=test_user))
cmd_results.on_error_exit()
print("\nCreated '%s' as LATIN9 DB..." % Text.color(test_db1, 'cyan'))
print("Created '%s' as UTF8 DB..." % Text.color(test_db2, 'cyan'))
config_fp = open(configFilePath, 'w')
config.write(config_fp)
config_fp.close()
os.chmod(configFilePath, stat.S_IREAD | stat.S_IWRITE)
self.config = config
self.host = DBConn.env['host']
self.section = section
self.create_db_objects()