def test_admission_status(self):
"""Test whether the admission status gets printed if a query gets queued when
either live_summary or live_progress is set to true"""
expected_admission_status = "Query queued. Latest queuing reason: " \
"number of running queries 1 is at or over limit 1"
# Start a long running query so that the next one gets queued.
sleep_query_handle = self.client.execute_async("select sleep(10000)")
self.client.wait_for_admission_control(sleep_query_handle)
# Iterate over test vector within test function to avoid restarting cluster.
for vector in\
[ImpalaTestVector([value]) for value in create_client_protocol_dimension()]:
cmd = get_shell_cmd(vector)
proc = pexpect.spawn(cmd[0], cmd[1:])
# Check with only live_summary set to true.
proc.expect("{0}] default>".format(get_impalad_port(vector)))
proc.sendline("set live_summary=true;")
proc.sendline("select 1;")
proc.expect(expected_admission_status)
proc.sendcontrol('c')
proc.expect("Cancelling Query")
# Check with only live_progress set to true.
proc.sendline("set live_summary=false;")
proc.sendline("set live_progress=true;")
proc.sendline("select 1;")
proc.expect(expected_admission_status)
def test_saml2_browser_profile_no_group_filter(self, vector):
# Iterate over test vector within test function to avoid restarting cluster.
for vector in\
[ImpalaTestVector([value]) for value in create_client_protocol_dimension()]:
protocol = vector.get_value("protocol")
if protocol != "hs2-http":
# SAML2 should not affect non http protocols.
args = ["--protocol=%s" % protocol, "-q", "select 1 + 2"]
run_impala_shell_cmd(vector, args, expect_success=True)
continue
# hs2-http connections without further arguments should be rejected.
args = ["--protocol=hs2-http", "-q", "select 1 + 2"]
run_impala_shell_cmd(vector, args, expect_success=False)
# test the SAML worflow with different attributes
self._test_saml2_browser_workflow("", True)
attributes_xml = TestClientSaml.ATTRIBUTE_STATEMENT.format(
group_name="group1")
self._test_saml2_browser_workflow(attributes_xml, True)
attributes_xml = TestClientSaml.ATTRIBUTE_STATEMENT.format(
group_name="bad_group")
self._test_saml2_browser_workflow(attributes_xml, True)
def add_test_dimensions(cls):
super(TestNestedTypesInSelectListWithBeeswax, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_constraint(lambda v:
v.get_value('protocol') == 'beeswax')
cls.ImpalaTestMatrix.add_dimension(create_orc_dimension(cls.get_workload()))
cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension(
disable_codegen_options=[True]))
def add_test_dimensions(cls):
super(TestNestedTArraysInSelectList, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_constraint(lambda v: v.get_value(
'table_format').file_format in ['parquet', 'orc'])
cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension('mt_dop', 0, 2))
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension_from_dict(
{'disable_codegen': ['False', 'True']}))
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
def add_test_dimensions(cls):
super(TestShellClient, cls).add_test_dimensions()
# Limit to uncompressed text with default exec options
cls.ImpalaTestMatrix.add_dimension(
create_uncompressed_text_dimension(cls.get_workload()))
cls.ImpalaTestMatrix.add_dimension(
create_single_exec_option_dimension())
# Run with beeswax and HS2
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_dimension(
create_client_protocol_no_strict_dimension())
def add_test_dimensions(cls):
super(TestStringQueries, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension(
disable_codegen_options=[False, True]))
cls.ImpalaTestMatrix.add_constraint(lambda v:\
v.get_value('table_format').file_format in ['text'] and
v.get_value('table_format').compression_codec in ['none'])
# Run these queries through both beeswax and HS2 to get coverage of CHAR/VARCHAR
# returned via both protocols.
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_constraint(hs2_text_constraint)
def test_manual_reconnect(self):
# Iterate over test vector within test function to avoid restarting cluster.
for vector in\
[ImpalaTestVector([value]) for value in create_client_protocol_dimension()]:
p = ImpalaShell(vector)
p.send_cmd("USE functional")
# Connect without arguments works because the custom cluster will have the default
# HS2 and Beeswax ports.
p.send_cmd("CONNECT")
p.send_cmd("SHOW TABLES")
result = p.get_result()
assert "alltypesaggmultifilesnopart" in result.stdout, result.stdout
def add_test_dimensions(cls):
super(TestAsyncLoadData, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(
create_uncompressed_text_dimension(cls.get_workload()))
# Test all clients: hs2, hs2-http and beeswax
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
# Test two exec modes per client
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension('enable_async_load_data_execution', True,
False))
# Disable codegen = false
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension(disable_codegen_options=[False]))
def add_test_dimensions(cls):
super(TestQueries, cls).add_test_dimensions()
if cls.exploration_strategy() == 'core':
cls.ImpalaTestMatrix.add_constraint(lambda v:\
v.get_value('table_format').file_format == 'parquet')
# Run these queries through both beeswax and HS2 to get coverage of both protocols.
# Don't run all combinations of table format and protocol - the dimensions should
# be orthogonal.
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_constraint(hs2_parquet_constraint)
# Adding a test dimension here to test the small query opt in exhaustive.
if cls.exploration_strategy() == 'exhaustive':
extend_exec_option_dimension(cls,
"exec_single_node_rows_threshold",
"100")
def add_test_dimensions(cls):
super(TestDateQueries, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension_from_dict({
'batch_size': [0, 1],
'disable_codegen': ['false', 'true'],
'disable_codegen_rows_threshold': [0]
}))
# DATE type is only supported for text, parquet and avro fileformat on HDFS and HBASE.
cls.ImpalaTestMatrix.add_constraint(lambda v: v.get_value(
'table_format').file_format in ('text', 'hbase', 'parquet') or (
v.get_value('table_format').file_format == 'avro' and v.
get_value('table_format').compression_codec == 'snap'))
# Run these queries through both beeswax and HS2 to get coverage of date returned
# via both protocols.
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_constraint(hs2_parquet_constraint)
def add_test_dimensions(cls):
super(TestDecimalQueries, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension_from_dict({
'decimal_v2' : ['false', 'true'],
'batch_size' : [0, 1],
'disable_codegen' : ['false', 'true'],
'disable_codegen_rows_threshold' : [0]}))
# Hive < 0.11 does not support decimal so we can't run these tests against the other
# file formats.
# TODO: Enable them on Hive >= 0.11.
cls.ImpalaTestMatrix.add_constraint(lambda v:\
(v.get_value('table_format').file_format == 'text' and
v.get_value('table_format').compression_codec == 'none') or
v.get_value('table_format').file_format in ['parquet', 'orc', 'kudu'])
# Run these queries through both beeswax and HS2 to get coverage of decimals returned
# via both protocols.
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_constraint(hs2_parquet_constraint)
def test_auto_reconnect(self):
impalad = ImpaladService(socket.getfqdn())
# Iterate over test vector within test function to avoid restarting cluster.
for vector in\
[ImpalaTestVector([value]) for value in create_client_protocol_dimension()]:
p = ImpalaShell(vector)
# ImpalaShell startup may issue query to get server info - get num queries after
# starting shell.
start_num_queries = impalad.get_metric_value(NUM_QUERIES)
p.send_cmd("USE functional")
# wait for the USE command to finish
impalad.wait_for_metric_value(NUM_QUERIES, start_num_queries + 1)
assert impalad.wait_for_num_in_flight_queries(0)
self._start_impala_cluster([])
p.send_cmd("SHOW TABLES")
result = p.get_result()
assert "alltypesaggmultifilesnopart" in result.stdout, result.stdout
def test_auto_reconnect_after_impalad_died(self):
"""Test reconnect after restarting the remote impalad without using connect;"""
# Use pexpect instead of ImpalaShell() since after using get_result() in ImpalaShell()
# to check Disconnect, send_cmd() will no longer have any effect so we can not check
# reconnect.
impalad = ImpaladService(socket.getfqdn())
# Iterate over test vector within test function to avoid restarting cluster.
for vector in\
[ImpalaTestVector([value]) for value in create_client_protocol_dimension()]:
cmd = get_shell_cmd(vector)
proc = pexpect.spawn(cmd[0], cmd[1:])
proc.expect("{0}] default>".format(get_impalad_port(vector)))
# ImpalaShell startup may issue query to get server info - get num queries after
# starting shell.
start_num_queries = impalad.get_metric_value(NUM_QUERIES)
proc.sendline("use tpch;")
# wait for the USE command to finish
impalad.wait_for_metric_value(NUM_QUERIES, start_num_queries + 1)
assert impalad.wait_for_num_in_flight_queries(0)
# Disconnect
self.cluster.impalads[0].kill()
proc.sendline("show tables;")
# Search from [1:] since the square brackets "[]" are special characters in regex
proc.expect(ImpalaShellClass.DISCONNECTED_PROMPT[1:])
# Restarting Impalad
self.cluster.impalads[0].start()
# Check reconnect
proc.sendline("show tables;")
proc.expect("nation")
proc.expect("{0}] tpch>".format(get_impalad_port(vector)))
proc.sendeof()
proc.wait()
# Ensure no sessions or queries are left dangling.
verifier = MetricVerifier(self.impalad_test_service)
verifier.verify_metrics_are_zero()
def add_test_dimensions(cls):
super(TestClientSsl, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_dimension(
create_impala_shell_executable_dimension())
def add_test_dimensions(cls):
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
def add_test_dimensions(cls): # Run with both beeswax and HS2 to ensure that behaviour is the same. cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
def add_test_dimensions(cls):
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_constraint(
lambda v: v.get_value('protocol') == 'beeswax')
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension(disable_codegen_options=[True]))