def add_test_dimensions(cls):
super(TestInsertQueries, cls).add_test_dimensions()
# Fix the exec_option vector to have a single value. This is needed should we decide
# to run the insert tests in parallel (otherwise there will be two tests inserting
# into the same table at the same time for the same file format).
# TODO: When we do decide to run these tests in parallel we could create unique temp
# tables for each test case to resolve the concurrency problems.
if cls.exploration_strategy() == 'core':
cls.TestMatrix.add_dimension(create_exec_option_dimension(
cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0],
sync_ddl=[0]))
cls.TestMatrix.add_dimension(create_uncompressed_text_dimension(cls.get_workload()))
else:
cls.TestMatrix.add_dimension(create_exec_option_dimension(
cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0],
sync_ddl=[0, 1]))
cls.TestMatrix.add_dimension(TestDimension("compression_codec", *PARQUET_CODECS));
# Insert is currently only supported for text and parquet
# For parquet, we want to iterate through all the compression codecs
# TODO: each column in parquet can have a different codec. We could
# test all the codecs in one table/file with some additional flags.
cls.TestMatrix.add_constraint(lambda v:\
v.get_value('table_format').file_format == 'parquet' or \
(v.get_value('table_format').file_format == 'text' and \
v.get_value('compression_codec') == 'none'))
cls.TestMatrix.add_constraint(lambda v:\
v.get_value('table_format').compression_codec == 'none')
def add_test_dimensions(cls):
super(TestStringQueries, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension(disable_codegen_options=[False, True]))
cls.ImpalaTestMatrix.add_constraint(lambda v:\
v.get_value('table_format').file_format in ['text'] and
v.get_value('table_format').compression_codec in ['none'])
def add_test_dimensions(cls):
super(TestFailpoints, cls).add_test_dimensions()
# Executing an explain on the the test query will fail in an enviornment where hbase
# tables don't exist (s3). Since this happens before the tests are run, the skipif
# marker won't catch it. If 's3' is detected as a file system, return immedietely.
if os.getenv("TARGET_FILESYSTEM") in ["s3", "isilon", "local"]: return
node_id_map = TestFailpoints.parse_plan_nodes_from_explain_output(QUERY, "functional")
assert node_id_map
cls.TestMatrix.add_dimension(TestDimension('location', *FAILPOINT_LOCATION))
cls.TestMatrix.add_dimension(TestDimension('target_node', *(node_id_map.items())))
cls.TestMatrix.add_dimension(TestDimension('action', *FAILPOINT_ACTION))
cls.TestMatrix.add_dimension(TestDimension('query_type', *QUERY_TYPE))
cls.TestMatrix.add_dimension(create_exec_option_dimension([0], [False], [0]))
# These are invalid test cases.
# For more info see IMPALA-55 and IMPALA-56.
cls.TestMatrix.add_constraint(lambda v: not (
v.get_value('action') == 'FAIL' and
v.get_value('location') in ['CLOSE'] and
v.get_value('target_node')[0] in ['AGGREGATE', 'HASH JOIN']) and
not (v.get_value('location') in ['PREPARE'] and
v.get_value('action') == 'CANCEL'))
# Don't create CLOSE:WAIT debug actions to avoid leaking plan fragments (there's no
# way to cancel a plan fragment once Close() has been called)
cls.TestMatrix.add_constraint(
lambda v: not (v.get_value('action') == 'CANCEL'
and v.get_value('location') == 'CLOSE'))
# No need to test error in scanner preparation for non-scan nodes.
cls.TestMatrix.add_constraint(
lambda v: (v.get_value('location') != 'PREPARE_SCANNER' or
v.get_value('target_node')[0] == 'SCAN HDFS'))
def add_test_dimensions(cls):
super(TestAggregationQueries, cls).add_test_dimensions()
cls.TestMatrix.add_dimension(
create_exec_option_dimension(disable_codegen_options=[False, True]))
if cls.exploration_strategy() == 'core':
cls.TestMatrix.add_dimension(create_uncompressed_text_dimension(cls.get_workload()))
def add_test_dimensions(cls):
super(TestLocalTzConversion, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension(
cluster_sizes=[0], disable_codegen_options=[False, True], batch_sizes=[0]))
# Test with and without expr rewrites to cover regular expr evaluations
# as well as constant folding, in particular, timestamp literals.
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension('enable_expr_rewrites', *[0,1]))
def add_test_dimensions(cls):
super(TestWideAggregationQueries, cls).add_test_dimensions()
cls.TestMatrix.add_dimension(
create_exec_option_dimension(disable_codegen_options=[False, True]))
# File format doesn't matter for this test.
cls.TestMatrix.add_constraint(
lambda v: v.get_value('table_format').file_format == 'parquet')
def add_test_dimensions(cls):
super(TestCharFormats, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension(disable_codegen_options=[False, True]))
cls.ImpalaTestMatrix.add_constraint(lambda v:
(v.get_value('table_format').file_format in ['avro'] and
v.get_value('table_format').compression_codec in ['snap']) or
v.get_value('table_format').file_format in ['parquet'] or
(v.get_value('table_format').file_format in ['text'] and
v.get_value('table_format').compression_codec in ['none']))
def add_test_dimensions(cls):
super(TestStringQueries, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension(disable_codegen_options=[False, True]))
cls.ImpalaTestMatrix.add_constraint(lambda v:\
v.get_value('table_format').file_format in ['text'] and
v.get_value('table_format').compression_codec in ['none'])
# Run these queries through both beeswax and HS2 to get coverage of CHAR/VARCHAR
# returned via both protocols.
cls.ImpalaTestMatrix.add_dimension(create_beeswax_hs2_dimension())
cls.ImpalaTestMatrix.add_constraint(hs2_text_constraint)
def add_test_dimensions(cls):
super(TestInsertParquetVerifySize, cls).add_test_dimensions()
# Fix the exec_option vector to have a single value.
cls.TestMatrix.add_dimension(create_exec_option_dimension(
cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0],
sync_ddl=[1]))
cls.TestMatrix.add_constraint(lambda v:\
v.get_value('table_format').file_format == 'parquet')
cls.TestMatrix.add_constraint(lambda v:\
v.get_value('table_format').compression_codec == 'none')
cls.TestMatrix.add_dimension(TestDimension("compression_codec", *PARQUET_CODECS));
def add_test_dimensions(cls):
super(TestInsertParquetInvalidCodec, cls).add_test_dimensions()
# Fix the exec_option vector to have a single value.
cls.TestMatrix.add_dimension(
create_exec_option_dimension(
cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0], sync_ddl=[1]
)
)
cls.TestMatrix.add_dimension(TestDimension("compression_codec", "bzip2"))
cls.TestMatrix.add_constraint(lambda v: v.get_value("table_format").file_format == "parquet")
cls.TestMatrix.add_constraint(lambda v: v.get_value("table_format").compression_codec == "none")
def add_test_dimensions(cls):
super(TestExprLimits, cls).add_test_dimensions()
if cls.exploration_strategy() != 'exhaustive':
# Ensure the test runs with codegen enabled and disabled, even when the
# exploration strategy is not exhaustive.
cls.TestMatrix.clear_dimension('exec_option')
cls.TestMatrix.add_dimension(create_exec_option_dimension(
cluster_sizes=[0], disable_codegen_options=[False, True], batch_sizes=[0]))
# There is no reason to run these tests using all dimensions.
cls.TestMatrix.add_dimension(create_uncompressed_text_dimension(cls.get_workload()))
def add_test_dimensions(cls):
super(TestInsertPartKey, cls).add_test_dimensions()
# Only run for a single table type
cls.TestMatrix.add_dimension(create_exec_option_dimension(
cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0],
sync_ddl=[1]))
cls.TestMatrix.add_constraint(lambda v:
(v.get_value('table_format').file_format == 'text'))
cls.TestMatrix.add_constraint(lambda v:\
v.get_value('table_format').compression_codec == 'none')
def __create_exec_option_dimension(cls):
cluster_sizes = ALL_CLUSTER_SIZES
disable_codegen_options = ALL_DISABLE_CODEGEN_OPTIONS
batch_sizes = ALL_BATCH_SIZES
exec_single_node_option = [0]
if cls.exploration_strategy() == 'core':
disable_codegen_options = [False]
cluster_sizes = ALL_NODES_ONLY
return create_exec_option_dimension(cluster_sizes, disable_codegen_options,
batch_sizes,
exec_single_node_option=exec_single_node_option)
def add_test_dimensions(cls):
super(TestExprLimits, cls).add_test_dimensions()
cls.TestMatrix.clear_dimension("exec_option")
# Run with codegen enabled and disabled.
cls.TestMatrix.add_dimension(
create_exec_option_dimension(cluster_sizes=[0], disable_codegen_options=[False, True], batch_sizes=[0])
)
# There is no reason to run these tests using all dimensions.
cls.TestMatrix.add_constraint(
lambda v: v.get_value("table_format").file_format == "text"
and v.get_value("table_format").compression_codec == "none"
)
def add_test_dimensions(cls):
super(TestInsertQueriesWithPermutation, cls).add_test_dimensions()
# Fix the exec_option vector to have a single value. This is needed should we decide
# to run the insert tests in parallel (otherwise there will be two tests inserting
# into the same table at the same time for the same file format).
# TODO: When we do decide to run these tests in parallel we could create unique temp
# tables for each test case to resolve the concurrency problems.
# TODO: do we need to run with multiple file formats? This seems to be really
# targeting FE behavior.
cls.TestMatrix.add_dimension(create_exec_option_dimension(
cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0]))
cls.TestMatrix.add_dimension(create_uncompressed_text_dimension(cls.get_workload()))
def add_test_dimensions(cls):
super(TestMetadataQueryStatements, cls).add_test_dimensions()
sync_ddl_opts = [0, 1]
if cls.exploration_strategy() != 'exhaustive':
# Cut down on test runtime by only running with SYNC_DDL=0
sync_ddl_opts = [0]
cls.TestMatrix.add_dimension(create_exec_option_dimension(
cluster_sizes=ALL_NODES_ONLY,
disable_codegen_options=[False],
batch_sizes=[0],
sync_ddl=sync_ddl_opts))
cls.TestMatrix.add_dimension(create_uncompressed_text_dimension(cls.get_workload()))
def add_test_dimensions(cls):
super(TestInsertQueriesWithPermutation, cls).add_test_dimensions()
# Fix the exec_option vector to have a single value. This is needed should we decide
# to run the insert tests in parallel (otherwise there will be two tests inserting
# into the same table at the same time for the same file format).
# TODO: When we do decide to run these tests in parallel we could create unique temp
# tables for each test case to resolve the concurrency problems.
cls.TestMatrix.add_dimension(
create_exec_option_dimension(cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0])
)
# Insert is currently only supported for text and parquet
cls.TestMatrix.add_constraint(lambda v: v.get_value("table_format").file_format in ["text", "parquet"])
cls.TestMatrix.add_constraint(lambda v: v.get_value("table_format").compression_codec == "none")
def add_test_dimensions(cls):
super(TestDdlBase, cls).add_test_dimensions()
sync_ddl_opts = [0, 1]
if cls.exploration_strategy() != 'exhaustive':
# Only run with sync_ddl on exhaustive since it increases test runtime.
sync_ddl_opts = [0]
cls.TestMatrix.add_dimension(create_exec_option_dimension(
cluster_sizes=ALL_NODES_ONLY,
disable_codegen_options=[False],
batch_sizes=[0],
sync_ddl=sync_ddl_opts))
# There is no reason to run these tests using all dimensions.
cls.TestMatrix.add_dimension(create_uncompressed_text_dimension(cls.get_workload()))
def add_test_dimensions(cls):
super(TestInsertNullQueries, cls).add_test_dimensions()
# Fix the exec_option vector to have a single value. This is needed should we decide
# to run the insert tests in parallel (otherwise there will be two tests inserting
# into the same table at the same time for the same file format).
# TODO: When we do decide to run these tests in parallel we could create unique temp
# tables for each test case to resolve the concurrency problems.
cls.TestMatrix.add_dimension(create_exec_option_dimension(
cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0]))
# These tests only make sense for inserting into a text table with special
# logic to handle all the possible ways NULL needs to be written as ascii
cls.TestMatrix.add_constraint(lambda v:\
(v.get_value('table_format').file_format == 'text' and \
v.get_value('table_format').compression_codec == 'none'))
def add_test_dimensions(cls):
super(TestInsertParquetQueries, cls).add_test_dimensions()
# Fix the exec_option vector to have a single value. This is needed should we decide
# to run the insert tests in parallel (otherwise there will be two tests inserting
# into the same table at the same time for the same file format).
# TODO: When we do decide to run these tests in parallel we could create unique temp
# tables for each test case to resolve the concurrency problems.
cls.TestMatrix.add_dimension(create_exec_option_dimension(
cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0],
sync_ddl=[1]))
cls.TestMatrix.add_dimension(TestDimension("compression_codec", *PARQUET_CODECS));
cls.TestMatrix.add_dimension(TestDimension("file_size", *PARQUET_FILE_SIZES));
cls.TestMatrix.add_constraint(lambda v:\
v.get_value('table_format').file_format == 'parquet')
cls.TestMatrix.add_constraint(lambda v:\
v.get_value('table_format').compression_codec == 'none')
def add_test_dimensions(cls):
super(TestInsertWideTable, cls).add_test_dimensions()
# Only vary codegen
cls.TestMatrix.add_dimension(create_exec_option_dimension(
cluster_sizes=[0], disable_codegen_options=[True, False], batch_sizes=[0]))
# Inserts only supported on text and parquet
cls.TestMatrix.add_constraint(lambda v:\
v.get_value('table_format').file_format == 'parquet' or \
v.get_value('table_format').file_format == 'text')
cls.TestMatrix.add_constraint(lambda v:\
v.get_value('table_format').compression_codec == 'none')
# Don't run on core. This test is very slow (IMPALA-864) and we are unlikely to
# regress here.
if cls.exploration_strategy() == 'core':
cls.TestMatrix.add_constraint(lambda v: False);
def add_test_dimensions(cls):
super(TestFailpoints, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension('query', *QUERIES))
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension('action', *FAILPOINT_ACTIONS))
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension('location', *FAILPOINT_LOCATIONS))
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension('mt_dop', *MT_DOP_VALUES))
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension([0], [False], [0]))
# Don't create PREPARE:WAIT debug actions because cancellation is not checked until
# after the prepare phase once execution is started.
cls.ImpalaTestMatrix.add_constraint(
lambda v: not (v.get_value('action') == 'CANCEL'
and v.get_value('location') == 'PREPARE'))
# Don't create CLOSE:WAIT debug actions to avoid leaking plan fragments (there's no
# way to cancel a plan fragment once Close() has been called)
cls.ImpalaTestMatrix.add_constraint(
lambda v: not (v.get_value('action') == 'CANCEL'
and v.get_value('location') == 'CLOSE'))
def add_test_dimensions(cls):
super(TestFailpoints, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension('query', *QUERIES))
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension('action', *FAILPOINT_ACTIONS))
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension('location', *FAILPOINT_LOCATIONS))
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension('mt_dop', *MT_DOP_VALUES))
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension([0], [False], [0]))
# Don't create PREPARE:WAIT debug actions because cancellation is not checked until
# after the prepare phase once execution is started.
cls.ImpalaTestMatrix.add_constraint(
lambda v: not (v.get_value('action') == 'CANCEL'
and v.get_value('location') == 'PREPARE'))
# Don't create CLOSE:WAIT debug actions to avoid leaking plan fragments (there's no
# way to cancel a plan fragment once Close() has been called)
cls.ImpalaTestMatrix.add_constraint(
lambda v: not (v.get_value('action') == 'CANCEL'
and v.get_value('location') == 'CLOSE'))
def add_test_dimensions(cls):
super(TestInsertQueries, cls).add_test_dimensions()
# Fix the exec_option vector to have a single value. This is needed should we decide
# to run the insert tests in parallel (otherwise there will be two tests inserting
# into the same table at the same time for the same file format).
# TODO: When we do decide to run these tests in parallel we could create unique temp
# tables for each test case to resolve the concurrency problems.
cls.TestMatrix.add_dimension(create_exec_option_dimension(
cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0],
sync_ddl=[0, 1]))
cls.TestMatrix.add_dimension(TestDimension("compression_codec", *PARQUET_CODECS));
# Insert is currently only supported for text and parquet
# For parquet, we want to iterate through all the compression codecs
# TODO: each column in parquet can have a different codec. We could
# test all the codecs in one table/file with some additional flags.
cls.TestMatrix.add_constraint(lambda v:\
v.get_value('table_format').file_format == 'parquet' or \
(v.get_value('table_format').file_format == 'text' and \
v.get_value('compression_codec') == 'none'))
cls.TestMatrix.add_constraint(lambda v:\
v.get_value('table_format').compression_codec == 'none')
def add_test_dimensions(cls):
super(TestFailpoints, cls).add_test_dimensions()
# Executing an explain on the the test query will fail in an enviornment where hbase
# tables don't exist (s3). Since this happens before the tests are run, the skipif
# marker won't catch it. If 's3' is detected as a file system, return immedietely.
if os.getenv("TARGET_FILESYSTEM") in ["s3", "isilon", "local"]: return
node_id_map = TestFailpoints.parse_plan_nodes_from_explain_output(
QUERY, "functional")
assert node_id_map
cls.TestMatrix.add_dimension(
TestDimension('location', *FAILPOINT_LOCATION))
cls.TestMatrix.add_dimension(
TestDimension('target_node', *(node_id_map.items())))
cls.TestMatrix.add_dimension(TestDimension('action',
*FAILPOINT_ACTION))
cls.TestMatrix.add_dimension(TestDimension('query_type', *QUERY_TYPE))
cls.TestMatrix.add_dimension(
create_exec_option_dimension([0], [False], [0]))
# These are invalid test cases.
# For more info see IMPALA-55 and IMPALA-56.
cls.TestMatrix.add_constraint(
lambda v: not (v.get_value('action') == 'FAIL' and v.get_value(
'location') in ['CLOSE'] and v.get_value('target_node')[0] in [
'AGGREGATE', 'HASH JOIN'
]) and not (v.get_value('location') in ['PREPARE'] and v.
get_value('action') == 'CANCEL'))
# Don't create CLOSE:WAIT debug actions to avoid leaking plan fragments (there's no
# way to cancel a plan fragment once Close() has been called)
cls.TestMatrix.add_constraint(lambda v: not (v.get_value(
'action') == 'CANCEL' and v.get_value('location') == 'CLOSE'))
# No need to test error in scanner preparation for non-scan nodes.
cls.TestMatrix.add_constraint(
lambda v: (v.get_value('location') != 'PREPARE_SCANNER' or v.
get_value('target_node')[0] == 'SCAN HDFS'))
def add_test_dimensions(cls):
super(TestFailpoints, cls).add_test_dimensions()
node_id_map = TestFailpoints.parse_plan_nodes_from_explain_output(QUERY, "functional")
assert node_id_map
cls.TestMatrix.add_dimension(TestDimension('location', *FAILPOINT_LOCATION))
cls.TestMatrix.add_dimension(TestDimension('target_node', *(node_id_map.items())))
cls.TestMatrix.add_dimension(TestDimension('action', *FAILPOINT_ACTION))
cls.TestMatrix.add_dimension(TestDimension('query_type', *QUERY_TYPE))
cls.TestMatrix.add_dimension(create_exec_option_dimension([0], [False], [0]))
# These are invalid test cases.
# For more info see IMPALA-55 and IMPALA-56.
cls.TestMatrix.add_constraint(lambda v: not (\
v.get_value('action') == 'FAIL' and\
v.get_value('location') in ['CLOSE'] and\
v.get_value('target_node')[0] in ['AGGREGATE', 'HASH JOIN']) and\
not (v.get_value('location') in ['PREPARE'] and \
v.get_value('action') == 'CANCEL'))
# Don't create CLOSE:WAIT debug actions to avoid leaking plan fragments (there's no
# way to cancel a plan fragment once Close() has been called)
cls.TestMatrix.add_constraint(
lambda v: not (v.get_value('action') == 'CANCEL'
and v.get_value('location') == 'CLOSE'))
def add_test_dimensions(cls):
super(TestCorruptTableStats, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension(
disable_codegen_options=[False], exec_single_node_option=[100]))
cls.ImpalaTestMatrix.add_dimension(
create_uncompressed_text_dimension(cls.get_workload()))
def add_test_dimensions(cls):
super(TestDataErrors, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension(batch_sizes=cls.BATCH_SIZES))
def add_test_dimensions(cls):
super(TestDataErrors, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension(batch_sizes=cls.BATCH_SIZES))
def add_test_dimensions(cls):
super(TestErrorLogs, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_constraint(lambda v:\
v.get_value('table_format').file_format == 'text')
cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension(
cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0]))
def add_test_dimensions(cls):
super(TestAggregationQueries, cls).add_test_dimensions()
cls.TestMatrix.add_dimension(
create_exec_option_dimension(disable_codegen_options=[False, True]))
def add_test_dimensions(cls):
super(TestParquet, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension(debug_action_options=DEBUG_ACTION_DIMS))
cls.ImpalaTestMatrix.add_constraint(
lambda v: v.get_value('table_format').file_format == 'parquet')
def add_test_dimensions(cls):
super(TestCorruptTableStats, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension(
disable_codegen_options=[False], exec_single_node_option=[100]))
cls.ImpalaTestMatrix.add_dimension(
create_uncompressed_text_dimension(cls.get_workload()))
def add_test_dimensions(cls):
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_constraint(
lambda v: v.get_value('protocol') == 'beeswax')
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension(disable_codegen_options=[True]))