def _create_random_table(self, table_name, min_col_count, max_col_count, allowed_storage_formats): '''Create and return a Table with a random number of cols.''' col_count = randint(min_col_count, max_col_count) storage_format = choice(allowed_storage_formats) table = Table(table_name) table.storage_format = storage_format allowed_types = list(TYPES) # Avro doesn't support timestamps yet. if table.storage_format == 'AVRO': allowed_types.remove(Timestamp) # TODO: 'table.cols' returns a copy of all scalar cols, so 'table.cols.append()' # doesn't actually modify the table's columns. 'table.cols' should be changed # to allow access to the real columns. cols = table.cols for col_idx in xrange(col_count): col_type = choice(allowed_types) col_type = choice( filter(lambda type_: issubclass(type_, col_type), EXACT_TYPES)) if issubclass(col_type, VarChar) and not issubclass(col_type, String): col_type = get_varchar_class(randint(1, VarChar.MAX)) elif issubclass(col_type, Char) and not issubclass(col_type, String): col_type = get_char_class(randint(1, Char.MAX)) elif issubclass(col_type, Decimal): max_digits = randint(1, Decimal.MAX_DIGITS) col_type = get_decimal_class(max_digits, randint(1, max_digits)) col = Column( table, '%s_col_%s' % (col_type.__name__.lower(), col_idx + 1), col_type) cols.append(col) table.cols = cols return table
def create_random_table(self, table_name, min_number_of_cols, max_number_of_cols, allowed_storage_formats): '''Create and return a Table with a random number of cols.''' col_count = randint(min_number_of_cols, max_number_of_cols) storage_format = choice(allowed_storage_formats) table = Table(table_name) table.storage_format = storage_format for col_idx in xrange(col_count): col_type = choice(TYPES) col_type = choice(filter(lambda type_: issubclass(type_, col_type), EXACT_TYPES)) if issubclass(col_type, VarChar) and not issubclass(col_type, String): col_type = get_varchar_class(randint(1, VarChar.MAX)) elif issubclass(col_type, Char) and not issubclass(col_type, String): col_type = get_char_class(randint(1, Char.MAX)) elif issubclass(col_type, Decimal): max_digits = randint(1, Decimal.MAX_DIGITS) col_type = get_decimal_class(max_digits, randint(1, max_digits)) col = Column( table, '%s_col_%s' % (col_type.__name__.lower(), col_idx + 1), col_type) table.cols.append(col) return table
def create_random_table(self, table_name, min_number_of_cols, max_number_of_cols, allowed_storage_formats): '''Create and return a Table with a random number of cols.''' col_count = randint(min_number_of_cols, max_number_of_cols) storage_format = choice(allowed_storage_formats) table = Table(table_name) table.storage_format = storage_format for col_idx in xrange(col_count): col_type = choice(TYPES) col_type = choice( filter(lambda type_: issubclass(type_, col_type), EXACT_TYPES)) if issubclass(col_type, VarChar) and not issubclass(col_type, String): col_type = get_varchar_class(randint(1, VarChar.MAX)) elif issubclass(col_type, Char) and not issubclass(col_type, String): col_type = get_char_class(randint(1, Char.MAX)) elif issubclass(col_type, Decimal): max_digits = randint(1, Decimal.MAX_DIGITS) col_type = get_decimal_class(max_digits, randint(1, max_digits)) col = Column( table, '%s_col_%s' % (col_type.__name__.lower(), col_idx + 1), col_type) table.cols.append(col) return table
def _create_random_table(self, table_name, min_col_count, max_col_count, allowed_storage_formats): '''Create and return a Table with a random number of cols.''' col_count = randint(min_col_count, max_col_count) storage_format = choice(allowed_storage_formats) table = Table(table_name) table.storage_format = storage_format allowed_types = list(TYPES) # Avro doesn't support timestamps yet. if table.storage_format == 'AVRO': allowed_types.remove(Timestamp) # TODO: 'table.cols' returns a copy of all scalar cols, so 'table.cols.append()' # doesn't actually modify the table's columns. 'table.cols' should be changed # to allow access to the real columns. cols = table.cols for col_idx in xrange(col_count): col_type = choice(allowed_types) col_type = choice(filter(lambda type_: issubclass(type_, col_type), EXACT_TYPES)) if issubclass(col_type, VarChar) and not issubclass(col_type, String): col_type = get_varchar_class(randint(1, VarChar.MAX)) elif issubclass(col_type, Char) and not issubclass(col_type, String): col_type = get_char_class(randint(1, Char.MAX)) elif issubclass(col_type, Decimal): max_digits = randint(1, Decimal.MAX_DIGITS) col_type = get_decimal_class(max_digits, randint(1, max_digits)) col = Column( table, '%s_col_%s' % (col_type.__name__.lower(), col_idx + 1), col_type) cols.append(col) table.cols = cols return table
def FakeTable(name, fake_columns, storage_format='TEXTFILE'): """ Return a Table consisting of one or more FakeColumns. Because Columns are added via method, we support nesting here instead. """ table = Table(name) if not fake_columns: raise Exception('You must supply at least one FakeColumn argument') for fake_column in fake_columns: table.add_col(fake_column) table.storage_format = storage_format return table
def test_table_model(self, cursor, hive_cursor): table = Table("some_test_table") cursor.drop_table(table.name, if_exists=True) table.storage_format = 'textfile' table.add_col(Column(table, "bigint_col", BigInt)) table.add_col(Column(table, "string_col", String)) cursor.create_table(table) try: other = hive_cursor.describe_table(table.name) assert other.name == table.name assert other.cols == table.cols finally: cursor.drop_table(table.name)
def test_table_model(self, cursor, hive_cursor): table = Table("some_test_table") cursor.drop_table(table.name, if_exists=True) table.storage_format = 'textfile' table.add_col(Column(table, "bigint_col", BigInt)) table.add_col(Column(table, "string_col", String)) cursor.create_table(table) try: other = hive_cursor.describe_table(table.name) assert other.name == table.name assert other.cols == table.cols finally: cursor.drop_table(table.name)