def test_execute_queries_from_file(self): args = '-f %s/test_file_comments.sql --quiet -B' % QUERY_FILE_PATH result = run_impala_shell_cmd(args) output = result.stdout args = '-f %s/test_file_no_comments.sql --quiet -B' % QUERY_FILE_PATH result = run_impala_shell_cmd(args) assert output == result.stdout, "Queries with comments not parsed correctly"
def test_var_substitution(self):
args = '--var=foo=123 --var=BAR=456 --delimited --output_delimiter=" " -c -f %s' \
% (os.path.join(QUERY_FILE_PATH, 'test_var_substitution.sql'))
result = run_impala_shell_cmd(args, expect_success=True)
assert_var_substitution(result)
args = '--var=foo'
result = run_impala_shell_cmd(args, expect_success=False)
assert ("Error: Could not parse key-value \"foo\". It must follow the pattern "
"\"KEY=VALUE\".") in result.stderr
# IMPALA-7673: Test that variable substitution in command line can accept values
# from other variables just like the one in interactive shell.
result = run_impala_shell_cmd('--var="msg1=1" --var="msg2=${var:msg1}2" '
'--var="msg3=${var:msg1}${var:msg2}" '
'--query="select ${var:msg3}"')
self._validate_shell_messages(result.stderr, ['112', 'Fetched 1 row(s)'],
should_exist=True)
# Test with an escaped variable.
result = run_impala_shell_cmd('--var="msg1=1" --var="msg2=${var:msg1}2" '
'--var="msg3=\${var:msg1}${var:msg2}" '
'--query="select \'${var:msg3}\'"')
self._validate_shell_messages(result.stderr, ['${var:msg1}12', 'Fetched 1 row(s)'],
should_exist=True)
# Referencing a non-existent variable will result in an error.
result = run_impala_shell_cmd('--var="msg1=1" --var="msg2=${var:doesnotexist}2" '
'--var="msg3=\${var:msg1}${var:msg2}" '
'--query="select \'${var:msg3}\'"',
expect_success=False)
self._validate_shell_messages(result.stderr,
['Error: Unknown variable DOESNOTEXIST',
'Could not execute command: select \'${var:msg3}\''],
should_exist=True)
def test_large_sql(self, unique_database):
# In this test, we are only interested in the performance of Impala shell and not
# the performance of Impala in general. So, this test will execute a large query
# from a non-existent table since this will make the query execution time negligible.
sql_file, sql_path = tempfile.mkstemp()
num_cols = 10000
os.write(sql_file, "select \n")
for i in xrange(num_cols):
if i < num_cols:
os.write(sql_file, "col_{0} as a{1},\n".format(i, i))
os.write(sql_file, "col_{0} as b{1},\n".format(i, i))
os.write(sql_file, "col_{0} as c{1}{2}\n".format(
i, i, "," if i < num_cols - 1 else ""))
os.write(sql_file, "from non_existence_large_table;")
os.close(sql_file)
try:
args = "-q -f {0} -d {1}".format(sql_path, unique_database)
start_time = time()
run_impala_shell_cmd(args, False)
end_time = time()
time_limit_s = 10
actual_time_s = end_time - start_time
assert actual_time_s <= time_limit_s, (
"It took {0} seconds to execute the query. Time limit is {1} seconds.".format(
actual_time_s, time_limit_s))
finally:
os.remove(sql_path)
def test_kudu_dml_reporting(self, vector, unique_database):
db = unique_database
run_impala_shell_cmd(vector, [
'--query=create table %s.dml_test (id int primary key, '
'age int null) partition by hash(id) partitions 2 stored as kudu' % db])
self._validate_dml_stmt(
vector, "insert into %s.dml_test (id) values (7), (7)" % db, 1, 1)
self._validate_dml_stmt(vector, "insert into %s.dml_test (id) values (7)" % db, 0, 1)
self._validate_dml_stmt(
vector, "upsert into %s.dml_test (id) values (7), (7)" % db, 2, 0)
self._validate_dml_stmt(
vector, "update %s.dml_test set age = 1 where id = 7" % db, 1, 0)
self._validate_dml_stmt(vector, "delete from %s.dml_test where id = 7" % db, 1, 0)
# UPDATE/DELETE where there are no matching rows; there are no errors because the
# scan produced no rows.
self._validate_dml_stmt(
vector, "update %s.dml_test set age = 1 where id = 8" % db, 0, 0)
self._validate_dml_stmt(vector, "delete from %s.dml_test where id = 7" % db, 0, 0)
# WITH clauses, only apply to INSERT and UPSERT
self._validate_dml_stmt(vector,
"with y as (values(7)) insert into %s.dml_test (id) select * from y" % db, 1, 0)
self._validate_dml_stmt(vector,
"with y as (values(7)) insert into %s.dml_test (id) select * from y" % db, 0, 1)
self._validate_dml_stmt(vector,
"with y as (values(7)) upsert into %s.dml_test (id) select * from y" % db, 1, 0)
def test_kerberos_option(self):
args = "-k"
# If you have a valid kerberos ticket in your cache, this test fails - so
# here we set a bogus KRB5CCNAME in the environment so that klist (and other
# kerberos commands) won't find the normal ticket cache.
# KERBEROS TODO: add kerberized cluster test case
os.environ["KRB5CCNAME"] = "/tmp/this/file/hopefully/does/not/exist"
# The command will fail because we're trying to connect to a kerberized impalad.
results = run_impala_shell_cmd(args, expect_success=False)
# Check that impala is using the right service name.
assert "Using service name 'impala'" in results.stderr
assert "Starting Impala Shell using Kerberos authentication" in results.stderr
# Check that Impala warns the user if klist does not exist on the system, or if
# no kerberos tickets are initialized.
try:
call(["klist"])
expected_error_msg = ("-k requires a valid kerberos ticket but no valid kerberos "
"ticket found.")
assert expected_error_msg in results.stderr
except OSError:
assert 'klist not found on the system' in results.stderr
# Make sure we don't try to re-connect
assert "retrying the connection with a secure transport" not in results.stderr
# Change the service name
args += " -s foobar"
results = run_impala_shell_cmd(args, expect_success=False)
assert "Using service name 'foobar'" in results.stderr
def test_execute_queries_from_file(self, vector):
args = ['-f', '{0}/test_file_comments.sql'.format(QUERY_FILE_PATH), '--quiet', '-B']
result = run_impala_shell_cmd(vector, args)
output = result.stdout
args = ['-f', '{0}/test_file_no_comments.sql'.format(QUERY_FILE_PATH), '--quiet',
'-B']
result = run_impala_shell_cmd(vector, args)
assert output == result.stdout, "Queries with comments not parsed correctly"
def test_malformed_query(self): """Test that malformed queries are handled by the commandline shell""" args = " -q \"with foo as (select bar from temp where temp.a='\"" result = run_impala_shell_cmd(args, expect_success=False) assert "Encountered: EOF" in result.stderr args = "-q \"with v as (select 1) \;\"" result = run_impala_shell_cmd(args, expect_success=False) assert "Encountered: Unexpected character" in result.stderr
def test_ldap_password_from_shell(self):
args = "-l --auth_creds_ok_in_clear --ldap_password_cmd='%s'"
result = run_impala_shell_cmd(args % 'cmddoesntexist', expect_success=False)
assert ("Error retrieving LDAP password (command was: 'cmddoesntexist', exception "
"was: '[Errno 2] No such file or directory')") in result.stderr
result = run_impala_shell_cmd(args % 'cat filedoesntexist', expect_success=False)
assert ("Error retrieving LDAP password (command was 'cat filedoesntexist', error "
"was: 'cat: filedoesntexist: No such file or directory')") in result.stderr
def test_query_time_and_link_message(self, unique_database):
shell_messages = ["Query submitted at: ", "(Coordinator: ",
"Query progress can be monitored at: "]
# CREATE statements should not print query time and webserver address.
results = run_impala_shell_cmd('--query="create table %s.shell_msg_test (id int)"' %
unique_database)
self._validate_shell_messages(results.stderr, shell_messages, should_exist=False)
# SELECT, INSERT and CTAS queries should print the query time message and webserver
# address.
results = run_impala_shell_cmd('--query="insert into %s.shell_msg_test values (1)"' %
unique_database)
self._validate_shell_messages(results.stderr, shell_messages, should_exist=True)
results = run_impala_shell_cmd('--query="select * from %s.shell_msg_test"' %
unique_database)
self._validate_shell_messages(results.stderr, shell_messages, should_exist=True)
results = run_impala_shell_cmd('--query="create table %s.shell_msg_ctas_test as \
select * from %s.shell_msg_test"' % (unique_database, unique_database))
self._validate_shell_messages(results.stderr, shell_messages, should_exist=True)
# DROP statements should not print query time and webserver address.
results = run_impala_shell_cmd('--query="drop table %s.shell_msg_test"' %
unique_database)
self._validate_shell_messages(results.stderr, shell_messages, should_exist=False)
run_impala_shell_cmd('--query="drop table %s.shell_msg_ctas_test"' %
unique_database)
# Simple queries should not print query time and webserver address.
results = run_impala_shell_cmd('--query="use default"')
self._validate_shell_messages(results.stderr, shell_messages, should_exist=False)
results = run_impala_shell_cmd('--query="show tables"')
self._validate_shell_messages(results.stderr, shell_messages, should_exist=False)
def test_config_file(self):
"""Test the optional configuration file."""
# Positive tests
args = '--config_file=%s/good_impalarc' % QUERY_FILE_PATH
result = run_impala_shell_cmd(args)
assert 'WARNING:' not in result.stderr
assert 'Query: select 1' in result.stderr
# override option in config file through command line
args = '--config_file=%s/good_impalarc --query="select 2"' % QUERY_FILE_PATH
result = run_impala_shell_cmd(args)
assert 'Query: select 2' in result.stderr
# IMPALA-8317: Add support for list-type, i.e. action=append in config file.
args = '--config_file=%s/good_impalarc ' \
'--query="select \'${VAR:msg1}\'; select \'${VAR:msg2}\'; ' \
'select \'${VAR:msg3}\'; select \'${VAR:msg4}\'; set"' % \
QUERY_FILE_PATH
result = run_impala_shell_cmd(args)
assert 'Query: select \'hello\'' in result.stderr
assert 'Query: select \'world\'' in result.stderr
assert 'Query: select \'foo\'' in result.stderr
assert 'Query: select \'bar\'' in result.stderr
assert 'DEFAULT_FILE_FORMAT: parquet' in result.stdout
# Override the variables in the config file with the ones passed via --var.
args = '--config_file=%s/good_impalarc --var=msg1=foo --var=msg2=bar ' \
'--query="select \'${VAR:msg1}\'; select \'${VAR:msg2}\'"' % \
QUERY_FILE_PATH
result = run_impala_shell_cmd(args)
assert 'Query: select \'foo\'' in result.stderr
assert 'Query: select \'bar\'' in result.stderr
# Negative Tests
# specified config file does not exist
args = '--config_file=%s/does_not_exist' % QUERY_FILE_PATH
run_impala_shell_cmd(args, expect_success=False)
# bad formatting of config file
args = '--config_file=%s/bad_impalarc' % QUERY_FILE_PATH
run_impala_shell_cmd(args, expect_success=False)
# Testing config file related warning and error messages
args = '--config_file=%s/impalarc_with_warnings' % QUERY_FILE_PATH
result = run_impala_shell_cmd(args, expect_success=True, wait_until_connected=False)
assert "WARNING: Option 'config_file' can be only set from shell." in result.stderr
err_msg = ("WARNING: Unable to read configuration file correctly. "
"Ignoring unrecognized config option: 'invalid_option'\n")
assert err_msg in result.stderr
args = '--config_file=%s/impalarc_with_error' % QUERY_FILE_PATH
result = run_impala_shell_cmd(args, expect_success=False)
err_msg = ("Unexpected value in configuration file. "
"'maybe' is not a valid value for a boolean option.")
assert err_msg in result.stderr
def test_print_header(self, populated_table):
args = '--print_header -B --output_delim="," -q "select * from %s"' % populated_table
result = run_impala_shell_cmd(args)
result_rows = result.stdout.strip().split('\n')
assert len(result_rows) == 4
assert result_rows[0].split(',') == ['i', 's']
args = '-B --output_delim="," -q "select * from %s"' % populated_table
result = run_impala_shell_cmd(args)
result_rows = result.stdout.strip().split('\n')
assert len(result_rows) == 3
def test_output_format(self):
expected_output = ['1'] * 3
args = '-q "select 1,1,1" -B --quiet'
result = run_impala_shell_cmd(args)
actual_output = [r.strip() for r in result.stdout.split('\t')]
assert actual_output == expected_output
result = run_impala_shell_cmd(args + ' --output_delim="|"')
actual_output = [r.strip() for r in result.stdout.split('|')]
assert actual_output == expected_output
result = run_impala_shell_cmd(args + ' --output_delim="||"',
expect_success=False)
assert "Illegal delimiter" in result.stderr
def test_output_format(self, vector):
expected_output = ['1'] * 3
args = ['-q', 'select 1,1,1', '-B', '--quiet']
result = run_impala_shell_cmd(vector, args)
actual_output = [r.strip() for r in result.stdout.split('\t')]
assert actual_output == expected_output
result = run_impala_shell_cmd(vector, args + ['--output_delim=|'])
actual_output = [r.strip() for r in result.stdout.split('|')]
assert actual_output == expected_output
result = run_impala_shell_cmd(vector, args + ['--output_delim=||'],
expect_success=False)
assert "Illegal delimiter" in result.stderr
def test_print_header(self, vector, populated_table):
args = ['--print_header', '-B', '--output_delim=,', '-q',
'select * from {0}'.format(populated_table)]
result = run_impala_shell_cmd(vector, args)
result_rows = result.stdout.strip().split('\n')
assert len(result_rows) == 4
assert result_rows[0].split(',') == ['i', 's']
args = ['-B', '--output_delim=,', '-q', 'select * from {0}'.format(populated_table)]
result = run_impala_shell_cmd(vector, args)
result_rows = result.stdout.strip().split('\n')
assert len(result_rows) == 3
def test_impala_shell_timeout(self):
"""Tests that impala shell times out during connect.
This creates a random listening socket and we try to connect to this
socket through the impala-shell. The impala-shell should timeout and not hang
indefinitely while connecting
"""
with closing(socket.socket()) as s:
s.bind(("", 0))
# maximum number of queued connections on this socket is 1.
s.listen(1)
test_port = s.getsockname()[1]
args = '-q "select foo; select bar;" --ssl -t 2000 -i localhost:%d' % (test_port)
run_impala_shell_cmd(args, expect_success=False)
def test_completed_query_errors_2(self):
args = ('-q "set abort_on_error=true;'
' select id, cnt from functional_parquet.bad_column_metadata t,'
' (select 1 cnt) u"')
result = run_impala_shell_cmd(args, expect_success=False)
assert 'ERROR: Column metadata states there are 11 values, ' in result.stderr
assert 'but read 10 values from column id.' in result.stderr
def test_international_characters_prettyprint(self, vector):
"""IMPALA-2717: ensure we can handle international characters in pretty-printed
output"""
args = ['-q', "select '{0}'".format(RUSSIAN_CHARS.encode('utf-8'))]
result = run_impala_shell_cmd(vector, args)
assert 'UnicodeDecodeError' not in result.stderr
assert RUSSIAN_CHARS.encode('utf-8') in result.stdout
def test_allow_creds_in_clear(self):
args = '-l'
result = run_impala_shell_cmd(args, expect_success=False)
err_msg = ("LDAP credentials may not be sent over insecure connections. "
"Enable SSL or set --auth_creds_ok_in_clear")
assert err_msg in result.stderr
def test_international_characters_prettyprint(self):
"""IMPALA-2717: ensure we can handle international characters in pretty-printed
output"""
args = """-q "select '%s'" """ % RUSSIAN_CHARS
result = run_impala_shell_cmd(args.encode('utf-8'))
assert 'UnicodeDecodeError' not in result.stderr
assert RUSSIAN_CHARS.encode('utf-8') in result.stdout
def test_international_characters_prettyprint_tabs(self, vector):
"""IMPALA-2717: ensure we can handle international characters in pretty-printed
output when pretty-printing falls back to delimited output."""
args = ['-q', "select '{0}\\t'".format(RUSSIAN_CHARS.encode('utf-8'))]
result = run_impala_shell_cmd(vector, args)
assert 'Reverting to tab delimited text' in result.stderr
assert 'UnicodeDecodeError' not in result.stderr
assert RUSSIAN_CHARS.encode('utf-8') in result.stdout
def test_international_characters_prettyprint_tabs(self):
"""IMPALA-2717: ensure we can handle international characters in pretty-printed
output when pretty-printing falls back to delimited output."""
args = """-q "select '%s\\t'" """ % RUSSIAN_CHARS
result = run_impala_shell_cmd(args.encode('utf-8'))
assert 'Reverting to tab delimited text' in result.stderr
assert 'UnicodeDecodeError' not in result.stderr
assert RUSSIAN_CHARS.encode('utf-8') in result.stdout
def test_summary(self):
args = "-q 'select count(*) from functional.alltypes; summary;'"
result_set = run_impala_shell_cmd(args)
assert "03:AGGREGATE" in result_set.stdout
args = "-q 'summary;'"
result_set = run_impala_shell_cmd(args, expect_success=False)
assert "Could not retrieve summary for query" in result_set.stderr
args = "-q 'show tables; summary;'"
result_set = run_impala_shell_cmd(args)
assert "Summary not available" in result_set.stderr
# Test queries without an exchange
args = "-q 'select 1; summary;'"
result_set = run_impala_shell_cmd(args)
assert "00:UNION" in result_set.stdout
def test_international_characters(self):
"""Sanity test to ensure that the shell can read international characters."""
russian_chars = (u"А, Б, В, Г, Д, Е, Ё, Ж, З, И, Й, К, Л, М, Н, О, П, Р,"
u"С, Т, У, Ф, Х, Ц,Ч, Ш, Щ, Ъ, Ы, Ь, Э, Ю, Я")
args = """-B -q "select '%s'" """ % russian_chars
result = run_impala_shell_cmd(args.encode('utf-8'))
assert 'UnicodeDecodeError' not in result.stderr
assert russian_chars.encode('utf-8') in result.stdout
def test_completed_query_errors(self):
args = ('-q "set abort_on_error=false;'
' select count(*) from functional_seq_snap.bad_seq_snap"')
result = run_impala_shell_cmd(args)
assert 'WARNINGS:' in result.stderr
assert 'Bad synchronization marker' in result.stderr
assert 'Expected: ' in result.stderr
assert 'Actual: ' in result.stderr
assert 'Problem parsing file' in result.stderr
def test_runtime_profile(self):
# test summary is in both the profile printed by the
# -p option and the one printed by the profile command
args = "-p -q 'select 1; profile;'"
result_set = run_impala_shell_cmd(args)
# This regex helps us uniquely identify a profile.
regex = re.compile("Operator\s+#Hosts\s+Avg\s+Time")
# We expect two query profiles.
assert len(re.findall(regex, result_set.stdout)) == 2, \
"Could not detect two profiles, stdout: %s" % result_set.stdout
def test_null_values(self, vector):
"""Test that null values are displayed correctly."""
query = """select id, int_col, double_col
from functional_parquet.alltypesagg where id < 5 order by id"""
result = run_impala_shell_cmd(vector, ['-q', query, '-B'])
assert "0\tNULL\tNULL" in result.stdout, result.stdout
assert "0\tNULL\tNULL" in result.stdout, result.stdout
assert "1\t1\t10.1" in result.stdout, result.stdout
assert "2\t2\t20.2" in result.stdout, result.stdout
assert "3\t3\t30.3" in result.stdout, result.stdout
assert "4\t4\t40.4" in result.stdout, result.stdout
def test_query_time_and_link_message(self, vector, unique_database):
shell_messages = ["Query submitted at: ", "(Coordinator: ",
"Query progress can be monitored at: "]
# CREATE statements should not print query time and webserver address.
results = run_impala_shell_cmd(
vector, ['--query=create table %s.shell_msg_test (id int)' % unique_database])
self._validate_shell_messages(results.stderr, shell_messages, should_exist=False)
# SELECT, INSERT, CTAS, CTE queries should print the query time message and webserver
# address.
results = run_impala_shell_cmd(
vector, ['--query=insert into %s.shell_msg_test values (1)' % unique_database])
self._validate_shell_messages(results.stderr, shell_messages, should_exist=True)
results = run_impala_shell_cmd(
vector, ['--query=select * from %s.shell_msg_test' % unique_database])
self._validate_shell_messages(results.stderr, shell_messages, should_exist=True)
results = run_impala_shell_cmd(vector, [
'--query=create table %s.shell_msg_ctas_test as \
select * from %s.shell_msg_test' % (unique_database, unique_database)])
self._validate_shell_messages(results.stderr, shell_messages, should_exist=True)
results = run_impala_shell_cmd(vector, [
'--query=create table {db}.shell_msg_cte_test(i int); '
'with abc as (select 1) '
'insert overwrite {db}.shell_msg_cte_test '
'select * from {db}.shell_msg_cte_test;'.format(db=unique_database)])
self._validate_shell_messages(results.stderr, shell_messages, should_exist=True)
# DROP statements should not print query time and webserver address.
results = run_impala_shell_cmd(
vector, ['--query=drop table %s.shell_msg_test' % unique_database])
self._validate_shell_messages(results.stderr, shell_messages, should_exist=False)
run_impala_shell_cmd(
vector, ['--query=drop table %s.shell_msg_ctas_test' % unique_database])
# Simple queries should not print query time and webserver address.
results = run_impala_shell_cmd(vector, ['--query=use default'])
self._validate_shell_messages(results.stderr, shell_messages, should_exist=False)
results = run_impala_shell_cmd(vector, ['--query=show tables'])
self._validate_shell_messages(results.stderr, shell_messages, should_exist=False)
def test_default_timezone(self):
"""Test that the default TIMEZONE query option is a valid timezone.
It would be nice to check that the default timezone is the system's timezone,
but doing this reliably on different Linux distributions is quite hard.
We skip this test on non-local clusters because the result_set from the
cluster is platform specific, but there's no guarantee the local machine
is the same OS, and the assert fails if there's a mismatch.
"""
result_set = run_impala_shell_cmd('-q "set;"')
tzname = find_query_option("TIMEZONE", result_set.stdout)
assert os.path.isfile("/usr/share/zoneinfo/" + tzname)
def test_float_formatting(self, vector):
"""Test that rounding of float types is as expected. Includes some floating point
values that can be printed in multiple valid ways. Includes additional expressions
to confirm that the output of ROUND() is a DOUBLE and that a plain FLOAT is
formatted correctly."""
query = """select typeof(round(cast(8.072 as double), 3)), cast(0.5 as float),
round(cast(8.072 as double), 3), round(cast(8 as double), 3),
round(cast(8.0719999999 as double),3),
round(cast(8.072 as double), 0), round(8.072, 4)"""
result = run_impala_shell_cmd(vector, ['-q', query, '-B'])
# Note that Beeswax formats results differently (i.e. the "fix" for IMPALA-266 stopped
# working for Beeswax at some point.
assert ("DOUBLE\t0.5\t8.071999999999999\t8\t8.071999999999999\t8\t8.072"
in result.stdout)
def test_kudu_dml_reporting(self, unique_database):
db = unique_database
run_impala_shell_cmd('--query="create table %s.dml_test (id int primary key, '\
'age int null) partition by hash(id) partitions 2 stored as kudu"' % db)
self._validate_dml_stmt("insert into %s.dml_test (id) values (7), (7)" % db, 1, 1)
self._validate_dml_stmt("insert into %s.dml_test (id) values (7)" % db, 0, 1)
self._validate_dml_stmt("upsert into %s.dml_test (id) values (7), (7)" % db, 2, 0)
self._validate_dml_stmt("update %s.dml_test set age = 1 where id = 7" % db, 1, 0)
self._validate_dml_stmt("delete from %s.dml_test where id = 7" % db, 1, 0)
# UPDATE/DELETE where there are no matching rows; there are no errors because the
# scan produced no rows.
self._validate_dml_stmt("update %s.dml_test set age = 1 where id = 8" % db, 0, 0)
self._validate_dml_stmt("delete from %s.dml_test where id = 7" % db, 0, 0)
# WITH clauses, only apply to INSERT and UPSERT
self._validate_dml_stmt(\
"with y as (values(7)) insert into %s.dml_test (id) select * from y" % db, 1, 0)
self._validate_dml_stmt(\
"with y as (values(7)) insert into %s.dml_test (id) select * from y" % db, 0, 1)
self._validate_dml_stmt(\
"with y as (values(7)) upsert into %s.dml_test (id) select * from y" % db, 1, 0)
def test_international_characters_prettyprint_tabs(self, vector):
"""IMPALA-2717: ensure we can handle international characters in pretty-printed
output when pretty-printing falls back to delimited output."""
args = ['-q', "select '{0}\\t'".format(RUSSIAN_CHARS.encode('utf-8'))]
result = run_impala_shell_cmd(vector, args)
protocol = vector.get_value('protocol')
if protocol == 'beeswax':
assert 'Reverting to tab delimited text' in result.stderr
else:
# HS2 does not need to fall back, but should behave appropriately.
assert protocol in ('hs2', 'hs2-http'), protocol
assert 'Reverting to tab delimited text' not in result.stderr
assert 'UnicodeDecodeError' not in result.stderr
assert RUSSIAN_CHARS.encode('utf-8') in result.stdout
def test_float_formatting(self, vector):
"""Test that rounding of float types is as expected. Includes some floating point
values that can be printed in multiple valid ways. Includes additional expressions
to confirm that the output of ROUND() is a DOUBLE and that a plain FLOAT is
formatted correctly."""
query = """select typeof(round(cast(8.072 as double), 3)), cast(0.5 as float),
round(cast(8.072 as double), 3), round(cast(8 as double), 3),
round(cast(8.0719999999 as double),3),
round(cast(8.072 as double), 0), round(8.072, 4)"""
result = run_impala_shell_cmd(vector, ['-q', query, '-B'])
# Note that Beeswax formats results differently (i.e. the "fix" for IMPALA-266 stopped
# working for Beeswax at some point.
assert (
"DOUBLE\t0.5\t8.071999999999999\t8\t8.071999999999999\t8\t8.072"
in result.stdout)
def test_type_formatting(self, vector):
"""Test formatting of data types that should be identical between HS2 and beeswax,
i.e. excluding floating point values which are tested in test_float_formatting."""
# Query covers bool, all int types, string, timestamp, date, various decimal types,
# CHAR (including a value with padding) and VARCHAR.
query = """select bool_col, tinyint_col, smallint_col, int_col, bigint_col,
string_col, timestamp_col, date_col, dt.c1, dt.c2, dt.c3,
ct.cs, ct.vc
from functional.alltypestiny att, functional.date_tbl dtbl,
functional.decimal_tiny dt, functional.chars_tiny ct
where att.id = 1 and dtbl.id_col = 1 and dt.c1 = 2.2220 and ct.vc = '3ccc'
"""
result = run_impala_shell_cmd(vector, ['-q', query, '-B'])
assert("false\t1\t1\t1\t10\t1\t2009-01-01 00:01:00\t0001-12-31\t2.2220\t124.44440"
"\t0.0\t3aaa \t3ccc" in result.stdout), result.stdout
def test_type_formatting(self, vector):
"""Test formatting of data types that should be identical between HS2 and beeswax,
i.e. excluding floating point values which are tested in test_float_formatting."""
# Query covers bool, all int types, string, timestamp, date, various decimal types,
# CHAR (including a value with padding) and VARCHAR.
query = """select bool_col, tinyint_col, smallint_col, int_col, bigint_col,
string_col, timestamp_col, date_col, dt.c1, dt.c2, dt.c3,
ct.cs, ct.vc
from functional.alltypestiny att, functional.date_tbl dtbl,
functional.decimal_tiny dt, functional.chars_tiny ct
where att.id = 1 and dtbl.id_col = 1 and dt.c1 = 2.2220 and ct.vc = '3ccc'
"""
result = run_impala_shell_cmd(vector, ['-q', query, '-B'])
assert("false\t1\t1\t1\t10\t1\t2009-01-01 00:01:00\t0001-12-31\t2.2220\t124.44440"
"\t0.0\t3aaa \t3ccc" in result.stdout), result.stdout
def test_result_spooling_timeout(self, vector):
"""Regression test for IMPALA-9953. Validates that if a fetch timeout occurs in the
middle of reading rows from Impala that all rows are still printed by the Impala
shell."""
# This query was stolen from __test_fetch_timeout in test_fetch_timeout.py. The query
# has a large delay between RowBatch production. So a fetch timeout will occur while
# fetching rows.
query_options = "set num_nodes=1; \
set fetch_rows_timeout_ms=1; \
set batch_size=1; \
set spool_query_results=true;"
query = "select bool_col, avg(id) from functional.alltypes group by bool_col"
result = run_impala_shell_cmd(vector, ['-q', query_options + query, '-B'])
result_rows = result.stdout.strip().split('\n')
assert len(result_rows) == 2
def test_default_db(self, empty_table):
db_name, table_name = empty_table.split('.')
args = '-d %s -q "describe %s" --quiet' % (db_name, table_name)
run_impala_shell_cmd(args)
args = '-q "describe %s"' % table_name
run_impala_shell_cmd(args, expect_success=False)
# test keyword parquet is interpreted as an identifier
# when passed as an argument to -d
args = '-d parquet'
result = run_impala_shell_cmd(args)
assert "Query: use `parquet`" in result.stderr, result.stderr
# test if backticking is idempotent
args = "-d '```parquet```'"
result = run_impala_shell_cmd(args)
assert "Query: use `parquet`" in result.stderr, result.stderr
def test_default_db(self, vector, empty_table):
db_name, table_name = empty_table.split('.')
args = ['-d', db_name, '-q', 'describe {0}'.format(table_name), '--quiet']
run_impala_shell_cmd(vector, args)
args = ['-q', 'describe {0}'.format(table_name)]
run_impala_shell_cmd(vector, args, expect_success=False)
# test keyword parquet is interpreted as an identifier
# when passed as an argument to -d
args = ['-d', 'parquet']
result = run_impala_shell_cmd(vector, args)
assert "Query: use `parquet`" in result.stderr, result.stderr
# test if backticking is idempotent
args = ['-d', '```parquet```']
result = run_impala_shell_cmd(vector, args)
assert "Query: use `parquet`" in result.stderr, result.stderr
def test_null_values(self, vector):
"""Test that null values are displayed correctly."""
query = """select id, int_col, double_col
from functional_parquet.alltypesagg where id < 5 order by id"""
result = run_impala_shell_cmd(vector, ['-q', query, '-B'])
assert "0\tNULL\tNULL" in result.stdout, result.stdout
assert "0\tNULL\tNULL" in result.stdout, result.stdout
assert "1\t1\t10.1" in result.stdout, result.stdout
assert "2\t2\t20.2" in result.stdout, result.stdout
if (vector.get_value("protocol") in ('hs2', 'hs2-http')) and not SHELL_IS_PYTHON_2:
# The HS2 client returns binary values for float/double types, and these must
# be converted to strings for display. However, due to differences between the
# way that python2 and python3 represent floating point values, the output
# from the shell will differ with regard to which version of python the
# shell is running under.
assert "3\t3\t30.299999999999997" in result.stdout, result.stdout
else:
# python 2, or python 3 with beeswax protocol
assert "3\t3\t30.3" in result.stdout, result.stdout
assert "4\t4\t40.4" in result.stdout, result.stdout
def test_get_log_once(self, vector, empty_table):
"""Test that get_log() is always called exactly once."""
# Query with fetch
args = ['-q', 'select * from functional.alltypeserror']
result = run_impala_shell_cmd(vector, args)
assert result.stderr.count('WARNINGS') == 1
# Insert query (doesn't fetch)
drop_args = ['-q', 'drop table if exists {0}'.format(empty_table)]
run_impala_shell_cmd(vector, drop_args)
args = ['-q', 'create table {0} like functional.alltypeserror'.format(empty_table)]
run_impala_shell_cmd(vector, args)
args = ['-q', 'insert overwrite {0} partition(year, month)'
'select * from functional.alltypeserror'.format(empty_table)]
result = run_impala_shell_cmd(vector, args)
assert result.stderr.count('WARNINGS') == 1
def test_get_log_once(self, empty_table):
"""Test that get_log() is always called exactly once."""
# Query with fetch
args = '-q "select * from functional.alltypeserror"'
result = run_impala_shell_cmd(args)
assert result.stderr.count('WARNINGS') == 1
# Insert query (doesn't fetch)
drop_args = '-q "drop table if exists %s"' % empty_table
run_impala_shell_cmd(drop_args)
args = '-q "create table %s like functional.alltypeserror"' % empty_table
run_impala_shell_cmd(args)
args = '-q "insert overwrite %s partition(year, month)' \
'select * from functional.alltypeserror"' % empty_table
result = run_impala_shell_cmd(args)
assert result.stderr.count('WARNINGS') == 1
def test_config_file(self):
"""Test the optional configuration file."""
# Positive tests
args = '--config_file=%s/good_impalarc' % QUERY_FILE_PATH
result = run_impala_shell_cmd(args)
assert 'Query: select 1' in result.stderr
assert 'Invalidating Metadata' in result.stderr
# override option in config file through command line
args = '--config_file=%s/good_impalarc --query="select 2"' % QUERY_FILE_PATH
result = run_impala_shell_cmd(args)
assert 'Query: select 2' in result.stderr
# Negative Tests
# specified config file does not exist
args = '--config_file=%s/does_not_exist' % QUERY_FILE_PATH
run_impala_shell_cmd(args, expect_success=False)
# bad formatting of config file
args = '--config_file=%s/bad_impalarc' % QUERY_FILE_PATH
run_impala_shell_cmd(args, expect_success=False)
def test_global_config_file(self, vector):
"""Test global and user configuration files."""
args = []
# shell uses shell options in global config
env = {
ImpalaShellClass.GLOBAL_CONFIG_FILE:
'{0}/good_impalarc2'.format(QUERY_FILE_PATH)
}
result = run_impala_shell_cmd(vector, args, env=env)
assert 'WARNING:' not in result.stderr, \
"A valid config file should not trigger any warning: {0}".format(result.stderr)
assert 'Query: select 2' in result.stderr
# shell uses query options in global config
args = ['-q', 'set;']
result = run_impala_shell_cmd(vector, args, env=env)
assert 'DEFAULT_FILE_FORMAT: avro' in result.stdout
# shell options and query options in global config get overriden
# by options in user config
args = [
'--config_file={0}/good_impalarc'.format(QUERY_FILE_PATH),
"""--query=select '${VAR:msg1}'; set"""
]
result = run_impala_shell_cmd(vector, args, env=env)
assert 'Query: select \'hello\'' in result.stderr
assert 'DEFAULT_FILE_FORMAT: parquet' in result.stdout
# command line options override options in global config
args = [
'--query_option=DEFAULT_FILE_FORMAT=text',
"""--query=select '${VAR:msg1}'; set"""
]
result = run_impala_shell_cmd(vector, args, env=env)
assert 'Query: select \'test\'' in result.stderr
assert 'DEFAULT_FILE_FORMAT: text' in result.stdout
# specified global config file does not exist
env = {ImpalaShellClass.GLOBAL_CONFIG_FILE: '/does_not_exist'}
run_impala_shell_cmd(vector, args, env=env, expect_success=False)
class TestImpalaShell(ImpalaTestSuite):
"""A set of sanity tests for the Impala shell commandline parameters.
The tests need to maintain Python 2.4 compatibility as a sub-goal of having
shell tests is to ensure that it's not broken in systems running Python 2.4.
The tests need a running impalad instance in order to execute queries.
TODO:
* Test individual modules.
* Add a test for a kerberized impala.
"""
@classmethod
def get_workload(self):
return 'functional-query'
@classmethod
def add_test_dimensions(cls):
# Run with both beeswax and HS2 to ensure that behaviour is the same.
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
def test_no_args(self, vector):
args = ['-q', DEFAULT_QUERY]
run_impala_shell_cmd(vector, args)
def test_multiple_queries(self, vector):
queries = ';'.join([DEFAULT_QUERY] * 3)
args = ['-q', queries, '-B']
run_impala_shell_cmd(vector, args)
def test_multiple_queries_with_escaped_backslash(self, vector):
"""Regression test for string containing an escaped backslash.
"""
run_impala_shell_cmd(vector, ['-q', r'''select '\\'; select '\'';''', '-B'])
def test_default_db(self, vector, empty_table):
db_name, table_name = empty_table.split('.')
args = ['-d', db_name, '-q', 'describe {0}'.format(table_name), '--quiet']
run_impala_shell_cmd(vector, args)
args = ['-q', 'describe {0}'.format(table_name)]
run_impala_shell_cmd(vector, args, expect_success=False)
# test keyword parquet is interpreted as an identifier
# when passed as an argument to -d
args = ['-d', 'parquet']
result = run_impala_shell_cmd(vector, args)
assert "Query: use `parquet`" in result.stderr, result.stderr
# test if backticking is idempotent
args = ['-d', '```parquet```']
result = run_impala_shell_cmd(vector, args)
assert "Query: use `parquet`" in result.stderr, result.stderr
def test_unsecure_message(self, vector):
results = run_impala_shell_cmd(vector, [], wait_until_connected=False)
assert "with no authentication" in results.stderr
def test_print_header(self, vector, populated_table):
args = ['--print_header', '-B', '--output_delim=,', '-q',
'select * from {0}'.format(populated_table)]
result = run_impala_shell_cmd(vector, args)
result_rows = result.stdout.strip().split('\n')
assert len(result_rows) == 4
assert result_rows[0].split(',') == ['i', 's']
args = ['-B', '--output_delim=,', '-q', 'select * from {0}'.format(populated_table)]
result = run_impala_shell_cmd(vector, args)
result_rows = result.stdout.strip().split('\n')
assert len(result_rows) == 3
@pytest.mark.execute_serially
def test_kerberos_option(self, vector):
args = ['-k']
# If you have a valid kerberos ticket in your cache, this test fails - so
# here we set a bogus KRB5CCNAME in the environment so that klist (and other
# kerberos commands) won't find the normal ticket cache.
# KERBEROS TODO: add kerberized cluster test case
os.environ["KRB5CCNAME"] = "/tmp/this/file/hopefully/does/not/exist"
# The command will fail because we're trying to connect to a kerberized impalad.
results = run_impala_shell_cmd(vector, args, expect_success=False)
# Check that impala is using the right service name.
assert "Using service name 'impala'" in results.stderr
assert "with Kerberos authentication" in results.stderr
# Check that Impala warns the user if klist does not exist on the system, or if
# no kerberos tickets are initialized.
try:
call(["klist"])
expected_error_msg = ("-k requires a valid kerberos ticket but no valid kerberos "
"ticket found.")
assert expected_error_msg in results.stderr
except OSError:
assert 'klist not found on the system' in results.stderr
# Make sure we don't try to re-connect
assert "retrying the connection with a secure transport" not in results.stderr
# Change the service name
args += ['-s', 'foobar']
results = run_impala_shell_cmd(vector, args, expect_success=False)
assert "Using service name 'foobar'" in results.stderr
def test_continue_on_error(self, vector):
args = ['-c', '-q', 'select foo; select bar;']
run_impala_shell_cmd(vector, args)
# Should fail
args = ['-q', 'select foo; select bar;']
run_impala_shell_cmd(vector, args, expect_success=False)
def test_execute_queries_from_file(self, vector):
args = ['-f', '{0}/test_file_comments.sql'.format(QUERY_FILE_PATH), '--quiet', '-B']
result = run_impala_shell_cmd(vector, args)
output = result.stdout
args = ['-f', '{0}/test_file_no_comments.sql'.format(QUERY_FILE_PATH), '--quiet',
'-B']
result = run_impala_shell_cmd(vector, args)
assert output == result.stdout, "Queries with comments not parsed correctly"
def test_completed_query_errors(self, vector):
args = ['-q', 'set abort_on_error=false;'
'select count(*) from functional_seq_snap.bad_seq_snap']
result = run_impala_shell_cmd(vector, args)
assert 'WARNINGS:' in result.stderr
assert 'Bad synchronization marker' in result.stderr
assert 'Expected: ' in result.stderr
assert 'Actual: ' in result.stderr
assert 'Problem parsing file' in result.stderr
def test_completed_query_errors_1(self, vector):
args = ['-q', 'set abort_on_error=true;'
'select id from functional_parquet.bad_column_metadata t']
result = run_impala_shell_cmd(vector, args, expect_success=False)
assert 'ERROR: Column metadata states there are 11 values, ' in result.stderr
assert 'but read 10 values from column id.' in result.stderr
def test_completed_query_errors_2(self, vector):
args = ['-q', 'set abort_on_error=true; '
'select id, cnt from functional_parquet.bad_column_metadata t, '
'(select 1 cnt) u']
result = run_impala_shell_cmd(vector, args, expect_success=False)
assert 'ERROR: Column metadata states there are 11 values, ' in result.stderr,\
result.stderr
assert 'but read 10 values from column id.' in result.stderr, result.stderr
def test_no_warnings_in_log_with_quiet_mode(self, vector):
"""Regression test for IMPALA-4222."""
args = ['--quiet', '-q', 'set abort_on_error=false;'
'select count(*) from functional_seq_snap.bad_seq_snap']
result = run_impala_shell_cmd(vector, args)
assert 'WARNINGS:' not in result.stderr, result.stderr
def test_removed_query_option(self, vector):
"""Test that removed query options produce warning."""
result = run_impala_shell_cmd(vector, ['-q', 'set disable_cached_reads=true'],
expect_success=True)
assert "Ignoring removed query option: 'disable_cached_reads'" in result.stderr,\
result.stderr
def test_output_format(self, vector):
expected_output = ['1'] * 3
args = ['-q', 'select 1,1,1', '-B', '--quiet']
result = run_impala_shell_cmd(vector, args)
actual_output = [r.strip() for r in result.stdout.split('\t')]
assert actual_output == expected_output
result = run_impala_shell_cmd(vector, args + ['--output_delim=|'])
actual_output = [r.strip() for r in result.stdout.split('|')]
assert actual_output == expected_output
result = run_impala_shell_cmd(vector, args + ['--output_delim=||'],
expect_success=False)
assert "Illegal delimiter" in result.stderr
def test_do_methods(self, vector, empty_table):
"""Ensure that the do_ methods in the shell work.
Some of the do_ methods are implicitly tested in other tests, and as part of the
test setup.
"""
# explain
args = ['-q', 'explain select 1']
run_impala_shell_cmd(vector, args)
# show
args = ['-q', 'show tables']
run_impala_shell_cmd(vector, args)
# with
args = ['-q', 'with t1 as (select 1) select * from t1']
run_impala_shell_cmd(vector, args)
# set
# spaces around the = sign
args = ['-q', 'set batch_size = 10']
run_impala_shell_cmd(vector, args)
# no spaces around the = sign
args = ['-q', 'set batch_size=10']
run_impala_shell_cmd(vector, args)
# test query options displayed
args = ['-q', 'set']
result_set = run_impala_shell_cmd(vector, args)
assert 'MEM_LIMIT: [0]' in result_set.stdout
# test to check that explain_level is STANDARD
assert 'EXPLAIN_LEVEL: [STANDARD]' in result_set.stdout
# test to check that configs without defaults show up as []
assert 'COMPRESSION_CODEC: []' in result_set.stdout
# test values displayed after setting value
args = ['-q', 'set mem_limit=1g;set']
result_set = run_impala_shell_cmd(vector, args)
# single list means one instance of mem_limit in displayed output
assert 'MEM_LIMIT: 1g' in result_set.stdout
assert 'MEM_LIMIT: [0]' not in result_set.stdout
# Negative tests for set
# use : instead of =
args = ['-q', 'set batch_size:10']
run_impala_shell_cmd(vector, args, expect_success=False)
# use 2 = signs
args = ['-q', 'set batch_size=10=50']
run_impala_shell_cmd(vector, args, expect_success=False)
# describe and desc should return the same result.
args = ['-q', 'describe {0}'.format(empty_table), '-B']
result_describe = run_impala_shell_cmd(vector, args)
args = ['-q', 'desc {0}'.format(empty_table), '-B']
result_desc = run_impala_shell_cmd(vector, args)
assert result_describe.stdout == result_desc.stdout
def test_runtime_profile(self, vector):
# test summary is in both the profile printed by the
# -p option and the one printed by the profile command
args = ['-p', '-q', 'select 1; profile;']
result_set = run_impala_shell_cmd(vector, args)
# This regex helps us uniquely identify a profile.
regex = re.compile("Operator\s+#Hosts\s+#Inst\s+Avg\s+Time")
# We expect two query profiles.
assert len(re.findall(regex, result_set.stdout)) == 2, \
"Could not detect two profiles, stdout: %s" % result_set.stdout
def test_summary(self, vector):
args = ['-q', 'select count(*) from functional.alltypes; summary;']
result_set = run_impala_shell_cmd(vector, args)
assert "03:AGGREGATE" in result_set.stdout
args = ['-q', 'summary;']
result_set = run_impala_shell_cmd(vector, args, expect_success=False)
assert "Could not retrieve summary: no previous query" in result_set.stderr
args = ['-q', 'show tables; summary;']
result_set = run_impala_shell_cmd(vector, args)
assert "Summary not available" in result_set.stderr
# Test queries without an exchange
args = ['-q', 'select 1; summary;']
result_set = run_impala_shell_cmd(vector, args)
assert "00:UNION" in result_set.stdout
@pytest.mark.execute_serially
def test_queries_closed(self, vector):
"""Regression test for IMPALA-897."""
args = ['-f', '{0}/test_close_queries.sql'.format(QUERY_FILE_PATH), '--quiet', '-B']
# Execute the shell command async
p = ImpalaShell(vector, args)
impalad_service = ImpaladService(get_impalad_host_port(vector).split(':')[0])
# The last query in the test SQL script will sleep for 10 seconds, so sleep
# here for 5 seconds and verify the number of in-flight queries is 1.
sleep(5)
assert 1 == impalad_service.get_num_in_flight_queries()
assert p.get_result().rc == 0
assert 0 == impalad_service.get_num_in_flight_queries()
def test_cancellation(self, vector):
"""Test cancellation (Ctrl+C event). Run a query that sleeps 10ms per row so will run
for 110s if not cancelled, but will detect cancellation quickly because of the small
batch size."""
query = "set num_nodes=1; set mt_dop=1; set batch_size=1; \
select sleep(10) from functional_parquet.alltypesagg"
p = ImpalaShell(vector, ['-q', query])
p.wait_for_query_start()
os.kill(p.pid(), signal.SIGINT)
result = p.get_result()
assert "Cancelling Query" in result.stderr, result.stderr
@pytest.mark.execute_serially
def test_query_cancellation_during_fetch(self, vector):
"""IMPALA-1144: Test cancellation (CTRL+C) while results are being
fetched"""
pytest.skip("""Skipping as it occasionally gets stuck in Jenkins builds
resulting the build to timeout.""")
# A select query where fetch takes several seconds
stmt = "with v as (values (1 as x), (2), (3), (4)) " + \
"select * from v, v v2, v v3, v v4, v v5, v v6, v v7, v v8, " + \
"v v9, v v10, v v11"
# Kill happens when the results are being fetched
self.run_and_verify_query_cancellation_test(vector, stmt, "FINISHED")
@pytest.mark.execute_serially
def test_query_cancellation_during_wait_to_finish(self, vector):
"""IMPALA-1144: Test cancellation (CTRL+C) while the query is in the
wait_to_finish state"""
# A select where wait_to_finish takes several seconds
stmt = "select * from tpch.customer c1, tpch.customer c2, " + \
"tpch.customer c3 order by c1.c_name"
# Kill happens in wait_to_finish state
self.run_and_verify_query_cancellation_test(vector, stmt, "RUNNING")
def wait_for_query_state(self, vector, stmt, state, max_retry=15):
"""Checks the in flight queries on Impala debug page. Polls the state of
the query statement from parameter every second until the query gets to
a state given via parameter or a maximum retry count is reached.
Restriction: Only works if there is only one in flight query."""
impalad_service = ImpaladService(get_impalad_host_port(vector).split(':')[0])
if not impalad_service.wait_for_num_in_flight_queries(1):
raise Exception("No in flight query found")
retry_count = 0
while retry_count <= max_retry:
query_info = impalad_service.get_in_flight_queries()[0]
if query_info['stmt'] != stmt:
exc_text = "The found in flight query is not the one under test: " + \
query_info['stmt']
raise Exception(exc_text)
if query_info['state'] == state:
return
retry_count += 1
sleep(1.0)
raise Exception("Query didn't reach desired state: " + state)
def run_and_verify_query_cancellation_test(self, vector, stmt, cancel_at_state):
"""Starts the execution of the received query, waits until the query
execution in fact starts and then cancels it. Expects the query
cancellation to succeed."""
p = ImpalaShell(vector, ['-q', stmt])
self.wait_for_query_state(vector, stmt, cancel_at_state)
os.kill(p.pid(), signal.SIGINT)
result = p.get_result()
assert "Cancelling Query" in result.stderr
assert "Invalid or unknown query handle" not in result.stderr
def test_get_log_once(self, vector, empty_table):
"""Test that get_log() is always called exactly once."""
# Query with fetch
args = ['-q', 'select * from functional.alltypeserror']
result = run_impala_shell_cmd(vector, args)
assert result.stderr.count('WARNINGS') == 1
# Insert query (doesn't fetch)
drop_args = ['-q', 'drop table if exists {0}'.format(empty_table)]
run_impala_shell_cmd(vector, drop_args)
args = ['-q', 'create table {0} like functional.alltypeserror'.format(empty_table)]
run_impala_shell_cmd(vector, args)
args = ['-q', 'insert overwrite {0} partition(year, month)'
'select * from functional.alltypeserror'.format(empty_table)]
result = run_impala_shell_cmd(vector, args)
assert result.stderr.count('WARNINGS') == 1
def test_international_characters(self, vector):
"""Sanity test to ensure that the shell can read international characters."""
args = ['-B', '-q', "select '{0}'".format(RUSSIAN_CHARS.encode('utf-8'))]
result = run_impala_shell_cmd(vector, args)
assert 'UnicodeDecodeError' not in result.stderr
assert RUSSIAN_CHARS.encode('utf-8') in result.stdout
def test_international_characters_prettyprint(self, vector):
"""IMPALA-2717: ensure we can handle international characters in pretty-printed
output"""
args = ['-q', "select '{0}'".format(RUSSIAN_CHARS.encode('utf-8'))]
result = run_impala_shell_cmd(vector, args)
assert 'UnicodeDecodeError' not in result.stderr
assert RUSSIAN_CHARS.encode('utf-8') in result.stdout
def test_international_characters_prettyprint_tabs(self, vector):
"""IMPALA-2717: ensure we can handle international characters in pretty-printed
output when pretty-printing falls back to delimited output."""
args = ['-q', "select '{0}\\t'".format(RUSSIAN_CHARS.encode('utf-8'))]
result = run_impala_shell_cmd(vector, args)
protocol = vector.get_value('protocol')
if protocol == 'beeswax':
assert 'Reverting to tab delimited text' in result.stderr
else:
# HS2 does not need to fall back, but should behave appropriately.
assert protocol in ('hs2', 'hs2-http'), protocol
assert 'Reverting to tab delimited text' not in result.stderr
assert 'UnicodeDecodeError' not in result.stderr
assert RUSSIAN_CHARS.encode('utf-8') in result.stdout
def test_global_config_file(self, vector):
"""Test global and user configuration files."""
args = []
# shell uses shell options in global config
env = dict(os.environ)
env[ImpalaShellClass.GLOBAL_CONFIG_FILE] = '{0}/good_impalarc2'.format(
QUERY_FILE_PATH)
result = run_impala_shell_cmd(vector, args, env=env)
assert 'WARNING:' not in result.stderr, \
"A valid config file should not trigger any warning: {0}".format(result.stderr)
assert 'Query: select 2' in result.stderr
# shell uses query options in global config
args = ['-q', 'set;']
result = run_impala_shell_cmd(vector, args, env=env)
assert 'DEFAULT_FILE_FORMAT: avro' in result.stdout
# shell options and query options in global config get overriden
# by options in user config
args = ['--config_file={0}/good_impalarc'.format(QUERY_FILE_PATH),
"""--query=select '${VAR:msg1}'; set"""]
result = run_impala_shell_cmd(vector, args, env=env)
assert 'Query: select \'hello\'' in result.stderr
assert 'DEFAULT_FILE_FORMAT: parquet' in result.stdout
# command line options override options in global config
args = ['--query_option=DEFAULT_FILE_FORMAT=text',
"""--query=select '${VAR:msg1}'; set"""]
result = run_impala_shell_cmd(vector, args, env=env)
assert 'Query: select \'test\'' in result.stderr
assert 'DEFAULT_FILE_FORMAT: text' in result.stdout
# specified global config file does not exist
env = {ImpalaShellClass.GLOBAL_CONFIG_FILE: '/does_not_exist'}
run_impala_shell_cmd(vector, args, env=env, expect_success=False)
def test_config_file(self, vector):
"""Test the optional configuration file."""
# Positive tests
args = ['--config_file=%s/good_impalarc' % QUERY_FILE_PATH]
result = run_impala_shell_cmd(vector, args)
assert 'WARNING:' not in result.stderr, \
"A valid config file should not trigger any warning: {0}".format(result.stderr)
assert 'Query: select 1' in result.stderr
# override option in config file through command line
args = ['--config_file={0}/good_impalarc'.format(QUERY_FILE_PATH), '--query=select 2']
result = run_impala_shell_cmd(vector, args)
assert 'Query: select 2' in result.stderr
# IMPALA-8317: Add support for list-type, i.e. action=append in config file.
args = ['--config_file={0}/good_impalarc'.format(QUERY_FILE_PATH),
"""--query=select '${VAR:msg1}'; select '${VAR:msg2}';
select '${VAR:msg3}'; select '${VAR:msg4}'; set"""]
result = run_impala_shell_cmd(vector, args)
assert 'Query: select \'hello\'' in result.stderr
assert 'Query: select \'world\'' in result.stderr
assert 'Query: select \'foo\'' in result.stderr
assert 'Query: select \'bar\'' in result.stderr
assert 'DEFAULT_FILE_FORMAT: parquet' in result.stdout
# Override the variables in the config file with the ones passed via --var.
args = ['--config_file={0}/good_impalarc'.format(QUERY_FILE_PATH), '--var=msg1=foo',
'--var=msg2=bar', """--query=select '${VAR:msg1}'; select '${VAR:msg2}'"""]
result = run_impala_shell_cmd(vector, args)
assert 'Query: select \'foo\'' in result.stderr
assert 'Query: select \'bar\'' in result.stderr
# Negative Tests
# specified config file does not exist
args = ['--config_file=%s/does_not_exist' % QUERY_FILE_PATH]
run_impala_shell_cmd(vector, args, expect_success=False)
# bad formatting of config file
args = ['--config_file=%s/bad_impalarc' % QUERY_FILE_PATH]
run_impala_shell_cmd(vector, args, expect_success=False)
# Testing config file related warning and error messages
args = ['--config_file=%s/impalarc_with_warnings' % QUERY_FILE_PATH]
result = run_impala_shell_cmd(
vector, args, expect_success=True, wait_until_connected=False)
assert "WARNING: Option 'config_file' can be only set from shell." in result.stderr
err_msg = ("WARNING: Unable to read configuration file correctly. "
"Ignoring unrecognized config option: 'invalid_option'\n")
assert err_msg in result.stderr
args = ['--config_file=%s/impalarc_with_error' % QUERY_FILE_PATH]
result = run_impala_shell_cmd(vector, args, expect_success=False)
err_msg = ("Unexpected value in configuration file. "
"'maybe' is not a valid value for a boolean option.")
assert err_msg in result.stderr
# Test the optional configuration file with live_progress and live_summary
# Positive test
args = ['--config_file=%s/good_impalarc3' % QUERY_FILE_PATH]
result = run_impala_shell_cmd(vector, args)
assert 'WARNING:' not in result.stderr, \
"A valid config file should not trigger any warning: {0}".format(result.stderr)
# Negative Tests
# specified config file with live_summary enabled for non interactive mode
args = ['--config_file=%s/good_impalarc3' % QUERY_FILE_PATH, '--query=select 3']
result = run_impala_shell_cmd(vector, args, expect_success=False)
assert 'live_summary is available for interactive mode only' in result.stderr
# testing config file related warning messages
args = ['--config_file=%s/impalarc_with_warnings2' % QUERY_FILE_PATH]
result = run_impala_shell_cmd(
vector, args, expect_success=True, wait_until_connected=False)
err_msg = ("WARNING: Unable to read configuration file correctly. "
"Ignoring unrecognized config option: 'Live_Progress'\n")
assert err_msg in result.stderr
# bad formatting of config file with invalid value
args = ['--config_file={0}/impalarc_with_error2'.format(QUERY_FILE_PATH)]
result = run_impala_shell_cmd(vector, args, expect_success=False)
err_msg = ("Unexpected value in configuration file. "
"'maybe' is not a valid value for a boolean option.")
assert err_msg in result.stderr
def test_execute_queries_from_stdin(self, vector):
"""Test that queries get executed correctly when STDIN is given as the sql file."""
args = ['-f', '-', '--quiet', '-B']
query_file = "{0}/test_file_comments.sql".format(QUERY_FILE_PATH)
query_file_handle = None
try:
query_file_handle = open(query_file, 'r')
query = query_file_handle.read()
query_file_handle.close()
except Exception, e:
assert query_file_handle is not None, "Exception %s: Could not find query file" % e
result = run_impala_shell_cmd(vector, args, expect_success=True, stdin_input=query)
output = result.stdout
args = ['-f', '{0}/test_file_no_comments.sql'.format(QUERY_FILE_PATH), '--quiet',
'-B']
result = run_impala_shell_cmd(vector, args)
assert output == result.stdout, "Queries from STDIN not parsed correctly."
def test_client_identifier(self, vector): """Confirms that a version string is passed along.""" args = ['--query', 'select 0; profile'] result = run_impala_shell_cmd(vector, args) assert 'client_identifier=impala shell' in result.stdout.lower()
def test_default_port(self, vector):
"""Ensure that we can run when -i does not specify a port."""
args = ['-q', DEFAULT_QUERY, '-i', IMPALAD_HS2_HOST_PORT.split(":")[0]]
run_impala_shell_cmd(vector, args)
def _validate_dml_stmt(self, vector, stmt, expected_rows_modified, expected_row_errors):
results = run_impala_shell_cmd(vector, ['--query=%s' % stmt])
expected_output = "Modified %d row(s), %d row error(s)" %\
(expected_rows_modified, expected_row_errors)
assert expected_output in results.stderr, results.stderr
def test_missing_query_file(self, vector): result = run_impala_shell_cmd(vector, ['-f', 'nonexistent.sql'], expect_success=False) assert "Could not open file 'nonexistent.sql'" in result.stderr
def test_unsecure_message(self, vector): results = run_impala_shell_cmd(vector, [], wait_until_connected=False) assert "with no authentication" in results.stderr
def test_insert_status(self, vector, unique_database):
run_impala_shell_cmd(
vector, ['--query=create table %s.insert_test (id int)' % unique_database])
results = run_impala_shell_cmd(
vector, ['--query=insert into %s.insert_test values (1)' % unique_database])
assert "Modified 1 row(s)" in results.stderr
def test_no_args(self, vector): args = ['-q', DEFAULT_QUERY] run_impala_shell_cmd(vector, args)
def test_international_characters(self, vector):
"""Sanity test to ensure that the shell can read international characters."""
args = ['-B', '-q', "select '{0}'".format(RUSSIAN_CHARS.encode('utf-8'))]
result = run_impala_shell_cmd(vector, args)
assert 'UnicodeDecodeError' not in result.stderr
assert RUSSIAN_CHARS.encode('utf-8') in result.stdout
def test_removed_query_option(self, vector):
"""Test that removed query options produce warning."""
result = run_impala_shell_cmd(vector, ['-q', 'set disable_cached_reads=true'],
expect_success=True)
assert "Ignoring removed query option: 'disable_cached_reads'" in result.stderr,\
result.stderr
def test_do_methods(self, vector, empty_table):
"""Ensure that the do_ methods in the shell work.
Some of the do_ methods are implicitly tested in other tests, and as part of the
test setup.
"""
# explain
args = ['-q', 'explain select 1']
run_impala_shell_cmd(vector, args)
# show
args = ['-q', 'show tables']
run_impala_shell_cmd(vector, args)
# with
args = ['-q', 'with t1 as (select 1) select * from t1']
run_impala_shell_cmd(vector, args)
# set
# spaces around the = sign
args = ['-q', 'set batch_size = 10']
run_impala_shell_cmd(vector, args)
# no spaces around the = sign
args = ['-q', 'set batch_size=10']
run_impala_shell_cmd(vector, args)
# test query options displayed
args = ['-q', 'set']
result_set = run_impala_shell_cmd(vector, args)
assert 'MEM_LIMIT: [0]' in result_set.stdout
# test to check that explain_level is STANDARD
assert 'EXPLAIN_LEVEL: [STANDARD]' in result_set.stdout
# test to check that configs without defaults show up as []
assert 'COMPRESSION_CODEC: []' in result_set.stdout
# test values displayed after setting value
args = ['-q', 'set mem_limit=1g;set']
result_set = run_impala_shell_cmd(vector, args)
# single list means one instance of mem_limit in displayed output
assert 'MEM_LIMIT: 1g' in result_set.stdout
assert 'MEM_LIMIT: [0]' not in result_set.stdout
# Negative tests for set
# use : instead of =
args = ['-q', 'set batch_size:10']
run_impala_shell_cmd(vector, args, expect_success=False)
# use 2 = signs
args = ['-q', 'set batch_size=10=50']
run_impala_shell_cmd(vector, args, expect_success=False)
# describe and desc should return the same result.
args = ['-q', 'describe {0}'.format(empty_table), '-B']
result_describe = run_impala_shell_cmd(vector, args)
args = ['-q', 'desc {0}'.format(empty_table), '-B']
result_desc = run_impala_shell_cmd(vector, args)
assert result_describe.stdout == result_desc.stdout
def test_no_warnings_in_log_with_quiet_mode(self, vector):
"""Regression test for IMPALA-4222."""
args = ['--quiet', '-q', 'set abort_on_error=false;'
'select count(*) from functional_seq_snap.bad_seq_snap']
result = run_impala_shell_cmd(vector, args)
assert 'WARNINGS:' not in result.stderr, result.stderr
def test_completed_query_errors_1(self, vector):
args = ['-q', 'set abort_on_error=true;'
'select id from functional_parquet.bad_column_metadata t']
result = run_impala_shell_cmd(vector, args, expect_success=False)
assert 'ERROR: Column metadata states there are 11 values, ' in result.stderr
assert 'but read 10 values from column id.' in result.stderr
def test_continue_on_error(self, vector): args = ['-c', '-q', 'select foo; select bar;'] run_impala_shell_cmd(vector, args) # Should fail args = ['-q', 'select foo; select bar;'] run_impala_shell_cmd(vector, args, expect_success=False)
def test_bool_display(self, vector): """Test that boolean values are displayed correctly.""" query = "select true a, false b" result = run_impala_shell_cmd(vector, ['-q', query]) assert "| a | b |" in result.stdout, result.stdout assert "| true | false |" in result.stdout, result.stdout
def test_config_file(self, vector):
"""Test the optional configuration file."""
# Positive tests
args = ['--config_file=%s/good_impalarc' % QUERY_FILE_PATH]
result = run_impala_shell_cmd(vector, args)
assert 'WARNING:' not in result.stderr, \
"A valid config file should not trigger any warning: {0}".format(result.stderr)
assert 'Query: select 1' in result.stderr
# override option in config file through command line
args = ['--config_file={0}/good_impalarc'.format(QUERY_FILE_PATH), '--query=select 2']
result = run_impala_shell_cmd(vector, args)
assert 'Query: select 2' in result.stderr
# IMPALA-8317: Add support for list-type, i.e. action=append in config file.
args = ['--config_file={0}/good_impalarc'.format(QUERY_FILE_PATH),
"""--query=select '${VAR:msg1}'; select '${VAR:msg2}';
select '${VAR:msg3}'; select '${VAR:msg4}'; set"""]
result = run_impala_shell_cmd(vector, args)
assert 'Query: select \'hello\'' in result.stderr
assert 'Query: select \'world\'' in result.stderr
assert 'Query: select \'foo\'' in result.stderr
assert 'Query: select \'bar\'' in result.stderr
assert 'DEFAULT_FILE_FORMAT: parquet' in result.stdout
# Override the variables in the config file with the ones passed via --var.
args = ['--config_file={0}/good_impalarc'.format(QUERY_FILE_PATH), '--var=msg1=foo',
'--var=msg2=bar', """--query=select '${VAR:msg1}'; select '${VAR:msg2}'"""]
result = run_impala_shell_cmd(vector, args)
assert 'Query: select \'foo\'' in result.stderr
assert 'Query: select \'bar\'' in result.stderr
# Negative Tests
# specified config file does not exist
args = ['--config_file=%s/does_not_exist' % QUERY_FILE_PATH]
run_impala_shell_cmd(vector, args, expect_success=False)
# bad formatting of config file
args = ['--config_file=%s/bad_impalarc' % QUERY_FILE_PATH]
run_impala_shell_cmd(vector, args, expect_success=False)
# Testing config file related warning and error messages
args = ['--config_file=%s/impalarc_with_warnings' % QUERY_FILE_PATH]
result = run_impala_shell_cmd(
vector, args, expect_success=True, wait_until_connected=False)
assert "WARNING: Option 'config_file' can be only set from shell." in result.stderr
err_msg = ("WARNING: Unable to read configuration file correctly. "
"Ignoring unrecognized config option: 'invalid_option'\n")
assert err_msg in result.stderr
args = ['--config_file=%s/impalarc_with_error' % QUERY_FILE_PATH]
result = run_impala_shell_cmd(vector, args, expect_success=False)
err_msg = ("Unexpected value in configuration file. "
"'maybe' is not a valid value for a boolean option.")
assert err_msg in result.stderr
# Test the optional configuration file with live_progress and live_summary
# Positive test
args = ['--config_file=%s/good_impalarc3' % QUERY_FILE_PATH]
result = run_impala_shell_cmd(vector, args)
assert 'WARNING:' not in result.stderr, \
"A valid config file should not trigger any warning: {0}".format(result.stderr)
# Negative Tests
# specified config file with live_summary enabled for non interactive mode
args = ['--config_file=%s/good_impalarc3' % QUERY_FILE_PATH, '--query=select 3']
result = run_impala_shell_cmd(vector, args, expect_success=False)
assert 'live_summary is available for interactive mode only' in result.stderr
# testing config file related warning messages
args = ['--config_file=%s/impalarc_with_warnings2' % QUERY_FILE_PATH]
result = run_impala_shell_cmd(
vector, args, expect_success=True, wait_until_connected=False)
err_msg = ("WARNING: Unable to read configuration file correctly. "
"Ignoring unrecognized config option: 'Live_Progress'\n")
assert err_msg in result.stderr
# bad formatting of config file with invalid value
args = ['--config_file={0}/impalarc_with_error2'.format(QUERY_FILE_PATH)]
result = run_impala_shell_cmd(vector, args, expect_success=False)
err_msg = ("Unexpected value in configuration file. "
"'maybe' is not a valid value for a boolean option.")
assert err_msg in result.stderr
def test_multiple_queries_with_escaped_backslash(self, vector): """Regression test for string containing an escaped backslash. """ run_impala_shell_cmd(vector, ['-q', r'''select '\\'; select '\'';''', '-B'])
def test_multiple_queries(self, vector): queries = ';'.join([DEFAULT_QUERY] * 3) args = ['-q', queries, '-B'] run_impala_shell_cmd(vector, args)
