def add_test_dimensions(cls): super(TestInsertQueries, cls).add_test_dimensions() # Fix the exec_option vector to have a single value. This is needed should we decide # to run the insert tests in parallel (otherwise there will be two tests inserting # into the same table at the same time for the same file format). # TODO: When we do decide to run these tests in parallel we could create unique temp # tables for each test case to resolve the concurrency problems. if cls.exploration_strategy() == 'core': cls.TestMatrix.add_dimension(create_exec_option_dimension( cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0], sync_ddl=[0])) cls.TestMatrix.add_dimension(create_uncompressed_text_dimension(cls.get_workload())) else: cls.TestMatrix.add_dimension(create_exec_option_dimension( cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0], sync_ddl=[0, 1])) cls.TestMatrix.add_dimension(TestDimension("compression_codec", *PARQUET_CODECS)); # Insert is currently only supported for text and parquet # For parquet, we want to iterate through all the compression codecs # TODO: each column in parquet can have a different codec. We could # test all the codecs in one table/file with some additional flags. cls.TestMatrix.add_constraint(lambda v:\ v.get_value('table_format').file_format == 'parquet' or \ (v.get_value('table_format').file_format == 'text' and \ v.get_value('compression_codec') == 'none')) cls.TestMatrix.add_constraint(lambda v:\ v.get_value('table_format').compression_codec == 'none')
def add_test_dimensions(cls): super(TestStringQueries, cls).add_test_dimensions() cls.ImpalaTestMatrix.add_dimension( create_exec_option_dimension(disable_codegen_options=[False, True])) cls.ImpalaTestMatrix.add_constraint(lambda v:\ v.get_value('table_format').file_format in ['text'] and v.get_value('table_format').compression_codec in ['none'])
def add_test_dimensions(cls): super(TestFailpoints, cls).add_test_dimensions() # Executing an explain on the the test query will fail in an enviornment where hbase # tables don't exist (s3). Since this happens before the tests are run, the skipif # marker won't catch it. If 's3' is detected as a file system, return immedietely. if os.getenv("TARGET_FILESYSTEM") in ["s3", "isilon", "local"]: return node_id_map = TestFailpoints.parse_plan_nodes_from_explain_output(QUERY, "functional") assert node_id_map cls.TestMatrix.add_dimension(TestDimension('location', *FAILPOINT_LOCATION)) cls.TestMatrix.add_dimension(TestDimension('target_node', *(node_id_map.items()))) cls.TestMatrix.add_dimension(TestDimension('action', *FAILPOINT_ACTION)) cls.TestMatrix.add_dimension(TestDimension('query_type', *QUERY_TYPE)) cls.TestMatrix.add_dimension(create_exec_option_dimension([0], [False], [0])) # These are invalid test cases. # For more info see IMPALA-55 and IMPALA-56. cls.TestMatrix.add_constraint(lambda v: not ( v.get_value('action') == 'FAIL' and v.get_value('location') in ['CLOSE'] and v.get_value('target_node')[0] in ['AGGREGATE', 'HASH JOIN']) and not (v.get_value('location') in ['PREPARE'] and v.get_value('action') == 'CANCEL')) # Don't create CLOSE:WAIT debug actions to avoid leaking plan fragments (there's no # way to cancel a plan fragment once Close() has been called) cls.TestMatrix.add_constraint( lambda v: not (v.get_value('action') == 'CANCEL' and v.get_value('location') == 'CLOSE')) # No need to test error in scanner preparation for non-scan nodes. cls.TestMatrix.add_constraint( lambda v: (v.get_value('location') != 'PREPARE_SCANNER' or v.get_value('target_node')[0] == 'SCAN HDFS'))
def add_test_dimensions(cls): super(TestAggregationQueries, cls).add_test_dimensions() cls.TestMatrix.add_dimension( create_exec_option_dimension(disable_codegen_options=[False, True])) if cls.exploration_strategy() == 'core': cls.TestMatrix.add_dimension(create_uncompressed_text_dimension(cls.get_workload()))
def add_test_dimensions(cls): super(TestLocalTzConversion, cls).add_test_dimensions() cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension( cluster_sizes=[0], disable_codegen_options=[False, True], batch_sizes=[0])) # Test with and without expr rewrites to cover regular expr evaluations # as well as constant folding, in particular, timestamp literals. cls.ImpalaTestMatrix.add_dimension( ImpalaTestDimension('enable_expr_rewrites', *[0,1]))
def add_test_dimensions(cls): super(TestWideAggregationQueries, cls).add_test_dimensions() cls.TestMatrix.add_dimension( create_exec_option_dimension(disable_codegen_options=[False, True])) # File format doesn't matter for this test. cls.TestMatrix.add_constraint( lambda v: v.get_value('table_format').file_format == 'parquet')
def add_test_dimensions(cls): super(TestCharFormats, cls).add_test_dimensions() cls.ImpalaTestMatrix.add_dimension( create_exec_option_dimension(disable_codegen_options=[False, True])) cls.ImpalaTestMatrix.add_constraint(lambda v: (v.get_value('table_format').file_format in ['avro'] and v.get_value('table_format').compression_codec in ['snap']) or v.get_value('table_format').file_format in ['parquet'] or (v.get_value('table_format').file_format in ['text'] and v.get_value('table_format').compression_codec in ['none']))
def add_test_dimensions(cls): super(TestStringQueries, cls).add_test_dimensions() cls.ImpalaTestMatrix.add_dimension( create_exec_option_dimension(disable_codegen_options=[False, True])) cls.ImpalaTestMatrix.add_constraint(lambda v:\ v.get_value('table_format').file_format in ['text'] and v.get_value('table_format').compression_codec in ['none']) # Run these queries through both beeswax and HS2 to get coverage of CHAR/VARCHAR # returned via both protocols. cls.ImpalaTestMatrix.add_dimension(create_beeswax_hs2_dimension()) cls.ImpalaTestMatrix.add_constraint(hs2_text_constraint)
def add_test_dimensions(cls): super(TestInsertParquetVerifySize, cls).add_test_dimensions() # Fix the exec_option vector to have a single value. cls.TestMatrix.add_dimension(create_exec_option_dimension( cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0], sync_ddl=[1])) cls.TestMatrix.add_constraint(lambda v:\ v.get_value('table_format').file_format == 'parquet') cls.TestMatrix.add_constraint(lambda v:\ v.get_value('table_format').compression_codec == 'none') cls.TestMatrix.add_dimension(TestDimension("compression_codec", *PARQUET_CODECS));
def add_test_dimensions(cls): super(TestInsertParquetInvalidCodec, cls).add_test_dimensions() # Fix the exec_option vector to have a single value. cls.TestMatrix.add_dimension( create_exec_option_dimension( cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0], sync_ddl=[1] ) ) cls.TestMatrix.add_dimension(TestDimension("compression_codec", "bzip2")) cls.TestMatrix.add_constraint(lambda v: v.get_value("table_format").file_format == "parquet") cls.TestMatrix.add_constraint(lambda v: v.get_value("table_format").compression_codec == "none")
def add_test_dimensions(cls): super(TestExprLimits, cls).add_test_dimensions() if cls.exploration_strategy() != 'exhaustive': # Ensure the test runs with codegen enabled and disabled, even when the # exploration strategy is not exhaustive. cls.TestMatrix.clear_dimension('exec_option') cls.TestMatrix.add_dimension(create_exec_option_dimension( cluster_sizes=[0], disable_codegen_options=[False, True], batch_sizes=[0])) # There is no reason to run these tests using all dimensions. cls.TestMatrix.add_dimension(create_uncompressed_text_dimension(cls.get_workload()))
def add_test_dimensions(cls): super(TestInsertPartKey, cls).add_test_dimensions() # Only run for a single table type cls.TestMatrix.add_dimension(create_exec_option_dimension( cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0], sync_ddl=[1])) cls.TestMatrix.add_constraint(lambda v: (v.get_value('table_format').file_format == 'text')) cls.TestMatrix.add_constraint(lambda v:\ v.get_value('table_format').compression_codec == 'none')
def __create_exec_option_dimension(cls): cluster_sizes = ALL_CLUSTER_SIZES disable_codegen_options = ALL_DISABLE_CODEGEN_OPTIONS batch_sizes = ALL_BATCH_SIZES exec_single_node_option = [0] if cls.exploration_strategy() == 'core': disable_codegen_options = [False] cluster_sizes = ALL_NODES_ONLY return create_exec_option_dimension(cluster_sizes, disable_codegen_options, batch_sizes, exec_single_node_option=exec_single_node_option)
def add_test_dimensions(cls): super(TestExprLimits, cls).add_test_dimensions() cls.TestMatrix.clear_dimension("exec_option") # Run with codegen enabled and disabled. cls.TestMatrix.add_dimension( create_exec_option_dimension(cluster_sizes=[0], disable_codegen_options=[False, True], batch_sizes=[0]) ) # There is no reason to run these tests using all dimensions. cls.TestMatrix.add_constraint( lambda v: v.get_value("table_format").file_format == "text" and v.get_value("table_format").compression_codec == "none" )
def add_test_dimensions(cls): super(TestInsertQueriesWithPermutation, cls).add_test_dimensions() # Fix the exec_option vector to have a single value. This is needed should we decide # to run the insert tests in parallel (otherwise there will be two tests inserting # into the same table at the same time for the same file format). # TODO: When we do decide to run these tests in parallel we could create unique temp # tables for each test case to resolve the concurrency problems. # TODO: do we need to run with multiple file formats? This seems to be really # targeting FE behavior. cls.TestMatrix.add_dimension(create_exec_option_dimension( cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0])) cls.TestMatrix.add_dimension(create_uncompressed_text_dimension(cls.get_workload()))
def add_test_dimensions(cls): super(TestMetadataQueryStatements, cls).add_test_dimensions() sync_ddl_opts = [0, 1] if cls.exploration_strategy() != 'exhaustive': # Cut down on test runtime by only running with SYNC_DDL=0 sync_ddl_opts = [0] cls.TestMatrix.add_dimension(create_exec_option_dimension( cluster_sizes=ALL_NODES_ONLY, disable_codegen_options=[False], batch_sizes=[0], sync_ddl=sync_ddl_opts)) cls.TestMatrix.add_dimension(create_uncompressed_text_dimension(cls.get_workload()))
def add_test_dimensions(cls): super(TestInsertQueriesWithPermutation, cls).add_test_dimensions() # Fix the exec_option vector to have a single value. This is needed should we decide # to run the insert tests in parallel (otherwise there will be two tests inserting # into the same table at the same time for the same file format). # TODO: When we do decide to run these tests in parallel we could create unique temp # tables for each test case to resolve the concurrency problems. cls.TestMatrix.add_dimension( create_exec_option_dimension(cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0]) ) # Insert is currently only supported for text and parquet cls.TestMatrix.add_constraint(lambda v: v.get_value("table_format").file_format in ["text", "parquet"]) cls.TestMatrix.add_constraint(lambda v: v.get_value("table_format").compression_codec == "none")
def add_test_dimensions(cls): super(TestDdlBase, cls).add_test_dimensions() sync_ddl_opts = [0, 1] if cls.exploration_strategy() != 'exhaustive': # Only run with sync_ddl on exhaustive since it increases test runtime. sync_ddl_opts = [0] cls.TestMatrix.add_dimension(create_exec_option_dimension( cluster_sizes=ALL_NODES_ONLY, disable_codegen_options=[False], batch_sizes=[0], sync_ddl=sync_ddl_opts)) # There is no reason to run these tests using all dimensions. cls.TestMatrix.add_dimension(create_uncompressed_text_dimension(cls.get_workload()))
def add_test_dimensions(cls): super(TestInsertNullQueries, cls).add_test_dimensions() # Fix the exec_option vector to have a single value. This is needed should we decide # to run the insert tests in parallel (otherwise there will be two tests inserting # into the same table at the same time for the same file format). # TODO: When we do decide to run these tests in parallel we could create unique temp # tables for each test case to resolve the concurrency problems. cls.TestMatrix.add_dimension(create_exec_option_dimension( cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0])) # These tests only make sense for inserting into a text table with special # logic to handle all the possible ways NULL needs to be written as ascii cls.TestMatrix.add_constraint(lambda v:\ (v.get_value('table_format').file_format == 'text' and \ v.get_value('table_format').compression_codec == 'none'))
def add_test_dimensions(cls): super(TestInsertParquetQueries, cls).add_test_dimensions() # Fix the exec_option vector to have a single value. This is needed should we decide # to run the insert tests in parallel (otherwise there will be two tests inserting # into the same table at the same time for the same file format). # TODO: When we do decide to run these tests in parallel we could create unique temp # tables for each test case to resolve the concurrency problems. cls.TestMatrix.add_dimension(create_exec_option_dimension( cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0], sync_ddl=[1])) cls.TestMatrix.add_dimension(TestDimension("compression_codec", *PARQUET_CODECS)); cls.TestMatrix.add_dimension(TestDimension("file_size", *PARQUET_FILE_SIZES)); cls.TestMatrix.add_constraint(lambda v:\ v.get_value('table_format').file_format == 'parquet') cls.TestMatrix.add_constraint(lambda v:\ v.get_value('table_format').compression_codec == 'none')
def add_test_dimensions(cls): super(TestInsertWideTable, cls).add_test_dimensions() # Only vary codegen cls.TestMatrix.add_dimension(create_exec_option_dimension( cluster_sizes=[0], disable_codegen_options=[True, False], batch_sizes=[0])) # Inserts only supported on text and parquet cls.TestMatrix.add_constraint(lambda v:\ v.get_value('table_format').file_format == 'parquet' or \ v.get_value('table_format').file_format == 'text') cls.TestMatrix.add_constraint(lambda v:\ v.get_value('table_format').compression_codec == 'none') # Don't run on core. This test is very slow (IMPALA-864) and we are unlikely to # regress here. if cls.exploration_strategy() == 'core': cls.TestMatrix.add_constraint(lambda v: False);
def add_test_dimensions(cls): super(TestFailpoints, cls).add_test_dimensions() cls.ImpalaTestMatrix.add_dimension( ImpalaTestDimension('query', *QUERIES)) cls.ImpalaTestMatrix.add_dimension( ImpalaTestDimension('action', *FAILPOINT_ACTIONS)) cls.ImpalaTestMatrix.add_dimension( ImpalaTestDimension('location', *FAILPOINT_LOCATIONS)) cls.ImpalaTestMatrix.add_dimension( ImpalaTestDimension('mt_dop', *MT_DOP_VALUES)) cls.ImpalaTestMatrix.add_dimension( create_exec_option_dimension([0], [False], [0])) # Don't create PREPARE:WAIT debug actions because cancellation is not checked until # after the prepare phase once execution is started. cls.ImpalaTestMatrix.add_constraint( lambda v: not (v.get_value('action') == 'CANCEL' and v.get_value('location') == 'PREPARE')) # Don't create CLOSE:WAIT debug actions to avoid leaking plan fragments (there's no # way to cancel a plan fragment once Close() has been called) cls.ImpalaTestMatrix.add_constraint( lambda v: not (v.get_value('action') == 'CANCEL' and v.get_value('location') == 'CLOSE'))
def add_test_dimensions(cls): super(TestInsertQueries, cls).add_test_dimensions() # Fix the exec_option vector to have a single value. This is needed should we decide # to run the insert tests in parallel (otherwise there will be two tests inserting # into the same table at the same time for the same file format). # TODO: When we do decide to run these tests in parallel we could create unique temp # tables for each test case to resolve the concurrency problems. cls.TestMatrix.add_dimension(create_exec_option_dimension( cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0], sync_ddl=[0, 1])) cls.TestMatrix.add_dimension(TestDimension("compression_codec", *PARQUET_CODECS)); # Insert is currently only supported for text and parquet # For parquet, we want to iterate through all the compression codecs # TODO: each column in parquet can have a different codec. We could # test all the codecs in one table/file with some additional flags. cls.TestMatrix.add_constraint(lambda v:\ v.get_value('table_format').file_format == 'parquet' or \ (v.get_value('table_format').file_format == 'text' and \ v.get_value('compression_codec') == 'none')) cls.TestMatrix.add_constraint(lambda v:\ v.get_value('table_format').compression_codec == 'none')
def add_test_dimensions(cls): super(TestFailpoints, cls).add_test_dimensions() # Executing an explain on the the test query will fail in an enviornment where hbase # tables don't exist (s3). Since this happens before the tests are run, the skipif # marker won't catch it. If 's3' is detected as a file system, return immedietely. if os.getenv("TARGET_FILESYSTEM") in ["s3", "isilon", "local"]: return node_id_map = TestFailpoints.parse_plan_nodes_from_explain_output( QUERY, "functional") assert node_id_map cls.TestMatrix.add_dimension( TestDimension('location', *FAILPOINT_LOCATION)) cls.TestMatrix.add_dimension( TestDimension('target_node', *(node_id_map.items()))) cls.TestMatrix.add_dimension(TestDimension('action', *FAILPOINT_ACTION)) cls.TestMatrix.add_dimension(TestDimension('query_type', *QUERY_TYPE)) cls.TestMatrix.add_dimension( create_exec_option_dimension([0], [False], [0])) # These are invalid test cases. # For more info see IMPALA-55 and IMPALA-56. cls.TestMatrix.add_constraint( lambda v: not (v.get_value('action') == 'FAIL' and v.get_value( 'location') in ['CLOSE'] and v.get_value('target_node')[0] in [ 'AGGREGATE', 'HASH JOIN' ]) and not (v.get_value('location') in ['PREPARE'] and v. get_value('action') == 'CANCEL')) # Don't create CLOSE:WAIT debug actions to avoid leaking plan fragments (there's no # way to cancel a plan fragment once Close() has been called) cls.TestMatrix.add_constraint(lambda v: not (v.get_value( 'action') == 'CANCEL' and v.get_value('location') == 'CLOSE')) # No need to test error in scanner preparation for non-scan nodes. cls.TestMatrix.add_constraint( lambda v: (v.get_value('location') != 'PREPARE_SCANNER' or v. get_value('target_node')[0] == 'SCAN HDFS'))
def add_test_dimensions(cls): super(TestFailpoints, cls).add_test_dimensions() node_id_map = TestFailpoints.parse_plan_nodes_from_explain_output(QUERY, "functional") assert node_id_map cls.TestMatrix.add_dimension(TestDimension('location', *FAILPOINT_LOCATION)) cls.TestMatrix.add_dimension(TestDimension('target_node', *(node_id_map.items()))) cls.TestMatrix.add_dimension(TestDimension('action', *FAILPOINT_ACTION)) cls.TestMatrix.add_dimension(TestDimension('query_type', *QUERY_TYPE)) cls.TestMatrix.add_dimension(create_exec_option_dimension([0], [False], [0])) # These are invalid test cases. # For more info see IMPALA-55 and IMPALA-56. cls.TestMatrix.add_constraint(lambda v: not (\ v.get_value('action') == 'FAIL' and\ v.get_value('location') in ['CLOSE'] and\ v.get_value('target_node')[0] in ['AGGREGATE', 'HASH JOIN']) and\ not (v.get_value('location') in ['PREPARE'] and \ v.get_value('action') == 'CANCEL')) # Don't create CLOSE:WAIT debug actions to avoid leaking plan fragments (there's no # way to cancel a plan fragment once Close() has been called) cls.TestMatrix.add_constraint( lambda v: not (v.get_value('action') == 'CANCEL' and v.get_value('location') == 'CLOSE'))
def add_test_dimensions(cls): super(TestCorruptTableStats, cls).add_test_dimensions() cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension( disable_codegen_options=[False], exec_single_node_option=[100])) cls.ImpalaTestMatrix.add_dimension( create_uncompressed_text_dimension(cls.get_workload()))
def add_test_dimensions(cls): super(TestDataErrors, cls).add_test_dimensions() cls.ImpalaTestMatrix.add_dimension( create_exec_option_dimension(batch_sizes=cls.BATCH_SIZES))
def add_test_dimensions(cls): super(TestErrorLogs, cls).add_test_dimensions() cls.ImpalaTestMatrix.add_constraint(lambda v:\ v.get_value('table_format').file_format == 'text') cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension( cluster_sizes=[0], disable_codegen_options=[False], batch_sizes=[0]))
def add_test_dimensions(cls): super(TestAggregationQueries, cls).add_test_dimensions() cls.TestMatrix.add_dimension( create_exec_option_dimension(disable_codegen_options=[False, True]))
def add_test_dimensions(cls): super(TestParquet, cls).add_test_dimensions() cls.ImpalaTestMatrix.add_dimension( create_exec_option_dimension(debug_action_options=DEBUG_ACTION_DIMS)) cls.ImpalaTestMatrix.add_constraint( lambda v: v.get_value('table_format').file_format == 'parquet')
def add_test_dimensions(cls): cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension()) cls.ImpalaTestMatrix.add_constraint( lambda v: v.get_value('protocol') == 'beeswax') cls.ImpalaTestMatrix.add_dimension( create_exec_option_dimension(disable_codegen_options=[True]))