def create_test_database(self, unique_database): create_stmt = 'create table {db}.{tbl} ( '\ ' int8_col TINYINT, '\ ' int16_col SMALLINT, '\ ' int32_col INT, '\ ' int64_col BIGINT, '\ ' float_col FLOAT, '\ ' double_col DOUBLE, '\ ' string_col STRING, '\ ' char_col VARCHAR(3) '\ ') '\ 'stored as parquet ' create_table_and_copy_files( self.client, create_stmt, unique_database, 'parquet_bloom_filter', ['testdata/data/parquet-bloom-filtering.parquet'])
def test_deprecated_stats(self, vector, unique_database): """Test that reading parquet files with statistics with deprecated 'min'/'max' fields works correctly. The statistics will be used for known-good types (boolean, integral, float) and will be ignored for all other types (string, decimal, timestamp).""" # We use CTAS instead of "create table like" to convert the partition columns into # normal table columns. create_table_and_copy_files(self.client, 'create table {db}.{tbl} stored as parquet ' 'as select * from functional.alltypessmall ' 'limit 0', unique_database, 'deprecated_stats', ['testdata/data/deprecated_statistics.parquet']) # The test makes assumptions about the number of row groups that are processed and # skipped inside a fragment, so we ensure that the tests run in a single fragment. vector.get_value('exec_option')['num_nodes'] = 1 self.run_test_case('QueryTest/parquet-deprecated-stats', vector, unique_database)
def test_deprecated_stats(self, vector, unique_database): """Test that reading parquet files with statistics with deprecated 'min'/'max' fields works correctly. The statistics will be used for known-good types (boolean, integral, float) and will be ignored for all other types (string, decimal, timestamp).""" # We use CTAS instead of "create table like" to convert the partition columns into # normal table columns. create_table_and_copy_files(self.client, 'create table {db}.{tbl} stored as parquet ' 'as select * from functional.alltypessmall ' 'limit 0', unique_database, 'deprecated_stats', ['testdata/data/deprecated_statistics.parquet']) # The test makes assumptions about the number of row groups that are processed and # skipped inside a fragment, so we ensure that the tests run in a single fragment. vector.get_value('exec_option')['num_nodes'] = 1 self.run_test_case('QueryTest/parquet-deprecated-stats', vector, unique_database)
def test_fileformat_support(self, vector, unique_database): """ Test that scanning and writing DATE is supported for text tables only.""" # This test specifies databases and locations explicitly. No need to execute it for # anything other than text fileformat on HDFS. if vector.get_value('table_format').file_format != 'text': pytest.skip() # Parquet table with date column TABLE_NAME = "parquet_date_tbl" CREATE_SQL = "CREATE TABLE {0}.{1} (date_col DATE) STORED AS PARQUET".format( unique_database, TABLE_NAME) create_table_and_copy_files(self.client, CREATE_SQL, unique_database, TABLE_NAME, ["/testdata/data/date_tbl.parquet"]) # Avro table with date column TABLE_NAME = "avro_date_tbl" CREATE_SQL = "CREATE TABLE {0}.{1} (date_col DATE) STORED AS AVRO".format( unique_database, TABLE_NAME) create_table_and_copy_files(self.client, CREATE_SQL, unique_database, TABLE_NAME, ["/testdata/data/date_tbl.avro"]) # Partitioned table with parquet and avro partitions TABLE_NAME = "date_tbl" CREATE_SQL = """CREATE TABLE {0}.{1} (date_col DATE) PARTITIONED BY (date_part DATE)""".format(unique_database, TABLE_NAME) self.client.execute(CREATE_SQL) # Add partitions ADD_PART_SQL = """ALTER TABLE {0}.{1} ADD PARTITION (date_part='1899-12-31') LOCATION '/test-warehouse/{0}.db/parquet_date_tbl' PARTITION (date_part='1999-12-31') LOCATION '/test-warehouse/{0}.db/avro_date_tbl' """.format(unique_database, TABLE_NAME) self.client.execute(ADD_PART_SQL) # Parquet fileformat SET_PART_FF_SQL = """ALTER TABLE {0}.{1} PARTITION (date_part='1899-12-31') SET FILEFORMAT PARQUET""".format(unique_database, TABLE_NAME) self.client.execute(SET_PART_FF_SQL) # Avro fileformat SET_PART_FF_SQL = """ALTER TABLE {0}.{1} PARTITION (date_part='1999-12-31') SET FILEFORMAT AVRO""".format(unique_database, TABLE_NAME) self.client.execute(SET_PART_FF_SQL) # Test scanning/writing tables with different fileformats. self.run_test_case('QueryTest/date-fileformat-support', vector, use_db=unique_database)
def _create_test_table_from_file(self, db_name, filename): create_table_and_copy_files(self.client, self.create_stmt, db_name, 'parquet_bloom_filter', [filename])