def main():
from metacomm.combinatorics.all_pairs2 import all_pairs2 as all_pairs
params = {'AddressMode': ['SLAAC', 'DHCP6 stateless', 'DHCP6 statefull'],
'Switch': ['InternalL2', 'ExternalL2'],
'Router': ['InternalL3', 'ExternalL3'],
'Security': ['SecNo', 'SecVPN', 'SecFirewall'],
'Net': ['Only6', '64']}
def is_valid_combination(vector):
if 'SLAAC' in vector and 'ExternalL3' in vector:
return False
else:
return True
for i, kv in enumerate(params.iteritems(), start=1):
print 'Test dimension {0}: {1}\t Possible values: {2}'.format(i, kv[0], kv[1])
pairwise = all_pairs(params.values(), filter_func=is_valid_combination)
for i, v in enumerate(pairwise, start=1):
print 'Test {0}:\t{1}'.format(i, v)
def main():
from metacomm.combinatorics.all_pairs2 import all_pairs2 as all_pairs
params = {
'AddressMode': ['SLAAC', 'DHCP6 stateless', 'DHCP6 statefull'],
'Switch': ['InternalL2', 'ExternalL2'],
'Router': ['InternalL3', 'ExternalL3'],
'Security': ['SecNo', 'SecVPN', 'SecFirewall'],
'Net': ['Only6', '64']
}
def is_valid_combination(vector):
if 'SLAAC' in vector and 'ExternalL3' in vector:
return False
else:
return True
for i, kv in enumerate(params.iteritems(), start=1):
print 'Test dimension {0}: {1}\t Possible values: {2}'.format(
i, kv[0], kv[1])
pairwise = all_pairs(params.values(), filter_func=is_valid_combination)
for i, v in enumerate(pairwise, start=1):
print 'Test {0}:\t{1}'.format(i, v)
def __init__(self, number):
parameters = [[0, 1]] * number
self.pairs = all_pairs(parameters)
class TestDecimalCasting(ImpalaTestSuite):
"""Test Suite to verify that casting to Decimal works.
Specifically, this test suite ensures that:
- overflows and underflows and handled correctly.
- casts from decimal/string to their exact decimal types are correct.
- max/min/NULL/0 can be expressed with their respective decimal types.
- TODO: Add cases for cast from float/double to decimal types.
"""
DECIMAL_TYPES_MAP = {
# All possible decimal types.
# (0 < precision <= 38 && 0 <= scale <= 38 && scale <= precision)
'exhaustive':
[(p, s) for p in xrange(1, 39) for s in xrange(0, p + 1)],
# Core only deals with precision 6,16,26 (different integer types)
'core': [(p, s) for p in [6, 16, 26] for s in xrange(0, p + 1)],
# mimics test_vectors.py and takes a subset of all decimal types
'pairwise':
all_pairs([(p, s) for p in xrange(1, 39) for s in xrange(0, p + 1)])
}
# We can cast for numerics or string types.
CAST_FROM = ['string', 'number']
# Set the default precision to 38 to operate on decimal values.
getcontext().prec = 38
# Represents a 0 in decimal
DECIMAL_ZERO = Decimal('0')
@classmethod
def get_workload(cls):
return 'functional-query'
@classmethod
def add_test_dimensions(cls):
cls.ImpalaTestMatrix = ImpalaTestMatrix()
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension(
'decimal_type', *TestDecimalCasting.DECIMAL_TYPES_MAP[
cls.exploration_strategy()]))
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension('cast_from', *TestDecimalCasting.CAST_FROM))
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension_from_dict(
{'decimal_v2': ['false', 'true']}))
cls.iterations = 1
def _gen_decimal_val(self, precision, scale):
"""Generates a Decimal object with the exact number of digits as the precision."""
# Generates numeric string which has as many digits as the precision.
num = str(randint(10**(precision - 1), int('9' * precision)))
# Incorporate scale into the string.
if scale != 0:
num = "{0}.{1}".format(num[:-scale], num[precision - scale:])
# Convert the generated decimal string into a Decimal object and return a -ive/+ive
# version of it with equal probability.
return Decimal(num) if randint(0, 1) else Decimal("-{0}".format(num))
def _assert_decimal_result(self, cast, actual, expected):
assert expected == actual, "Cast: {0}, Expected: {1}, Actual: {2}".format(cast,\
expected, actual)
def _normalize_cast_expr(self, decimal_val, precision, cast_from):
if cast_from == 'string':
return "select cast('{0}' as Decimal({1},{2}))"
else:
return "select cast({0} as Decimal({1},{2}))"
def test_min_max_zero_null(self, vector):
"""Sanity test at limits.
Verify that:
- We can read decimal values at their +ive and -ive limits.
- 0 is expressible in all decimal types.
- NULL is expressible in all decimal types
"""
precision, scale = vector.get_value('decimal_type')
dec_max = Decimal('{0}.{1}'.format('9' * (precision - scale),
'9' * scale))
# Multiplying large values eith -1 can produce an overflow.
dec_min = Decimal('-{0}'.format(str(dec_max)))
cast = self._normalize_cast_expr(dec_max, precision,
vector.get_value('cast_from'))
# Test max
res = Decimal(
self.execute_scalar(cast.format(dec_max, precision, scale)))
self._assert_decimal_result(cast, res, dec_max)
# Test Min
res = Decimal(
self.execute_scalar(cast.format(dec_min, precision, scale)))
self._assert_decimal_result(cast, res, dec_min)
# Test zero
res = Decimal(self.execute_scalar(\
cast.format(TestDecimalCasting.DECIMAL_ZERO, precision, scale)))
self._assert_decimal_result(cast, res, TestDecimalCasting.DECIMAL_ZERO)
# Test NULL
null_cast = "select cast(NULL as Decimal({0}, {1}))".format(
precision, scale)
res = self.execute_scalar(null_cast)
self._assert_decimal_result(null_cast, res, 'NULL')
def test_exact(self, vector):
"""Test to verify that an exact representation of the desired Decimal type is
maintained."""
precision, scale = vector.get_value('decimal_type')
if vector.get_value('cast_from') == 'decimal':
pytest.skip(
"Casting between the same decimal type isn't interesting")
for i in xrange(self.iterations):
val = self._gen_decimal_val(precision, scale)
cast = self._normalize_cast_expr(val, precision, vector.get_value('cast_from'))\
.format(val, precision, scale)
res = Decimal(self.execute_scalar(cast))
self._assert_decimal_result(cast, res, val)
def test_overflow(self, vector):
"""Test to verify that we always return NULL when trying to cast a number with greater
precision that its intended decimal type"""
precision, scale = vector.get_value('decimal_type')
for i in xrange(self.iterations):
# Generate a decimal with a larger precision than the one we're casting to.
from_precision = randint(precision + 1, 39)
val = self._gen_decimal_val(from_precision, scale)
cast = self._normalize_cast_expr(val, from_precision,\
vector.get_value('cast_from')).format(val, precision, scale)
if vector.get_value('cast_from') == "string":
# TODO: This should be an error in both cases (IMPALA-6405).
res = self.execute_scalar(cast)
self._assert_decimal_result(cast, res, 'NULL')
else:
res = self.execute_query_expect_failure(self.client, cast)
def test_underflow(self, vector):
"""Test to verify that we truncate when the scale of the number being cast is higher
than the target decimal type (with no change in precision).
"""
precision, scale = vector.get_value('decimal_type')
is_decimal_v2 = vector.get_value('exec_option')['decimal_v2'] == 'true'
cast_from = vector.get_value('cast_from')
if precision == scale:
pytest.skip(
"Cannot underflow scale when precision and scale are equal")
for i in xrange(self.iterations):
from_scale = randint(scale + 1, precision)
val = self._gen_decimal_val(precision, from_scale)
cast = self._normalize_cast_expr(val, precision, cast_from)\
.format(val, precision, scale)
res = Decimal(
self.execute_scalar(cast, vector.get_value('exec_option')))
# TODO: Remove check for cast_from once string to decimal is supported in decimal_v2.
if is_decimal_v2:
expected_val = val.quantize(Decimal('0e-%s' % scale),
rounding=ROUND_HALF_UP)
else:
expected_val = val.quantize(Decimal('0e-%s' % scale),
rounding=ROUND_DOWN)
self._assert_decimal_result(cast, res, expected_val)
class TestDecimalCasting(ImpalaTestSuite):
"""Test Suite to verify that casting to Decimal works.
Specifically, this test suite ensures that:
- overflows and underflows and handled correctly.
- casts from floats/string to their exact decimal types are correct.
- max/min/NULL/0 can be expressed with their respective decimal types.
"""
DECIMAL_TYPES_MAP = {
# All possible decimal types.
# (0 < precision <= 38 && 0 <= scale <= 38 && scale <= precision)
'exhaustive' : [(p, s) for p in xrange(1, 39) for s in xrange(0, p + 1)],
# Core only deals with precision 6,16,26 (different integer types)
'core' : [(p, s) for p in [6,16,26] for s in xrange(0, p + 1)],
# mimics test_vectors.py and takes a subset of all decimal types
'pairwise' : all_pairs([(p, s) for p in xrange(1, 39) for s in xrange(0, p + 1)])
}
# We can cast for numerrics or string types.
CAST_FROM = ['string', 'number']
# Set the default precisin to 38 to operate on decimal values.
getcontext().prec = 38
# Represents a 0 in decimal
DECIMAL_ZERO = Decimal('0')
@classmethod
def get_workload(cls):
return 'functional-query'
@classmethod
def add_test_dimensions(cls):
cls.TestMatrix = TestMatrix()
cls.TestMatrix.add_dimension(TestDimension('decimal_type',
*TestDecimalCasting.DECIMAL_TYPES_MAP[cls.exploration_strategy()]))
cls.TestMatrix.add_dimension(
TestDimension('cast_from', *TestDecimalCasting.CAST_FROM))
cls.iterations = 1
def setup_method(self, method):
self.max_bigint = int(self.execute_scalar("select max_bigint()"))
def _gen_decimal_val(self, precision, scale):
"""Generates a Decimal object with the exact number of digits as the precision."""
# Generates numeric string which has as many digits as the precision.
num = str(randint(10**(precision - 1), int('9' * precision)))
# Incorporate scale into the string.
if scale != 0: num = "{0}.{1}".format(num[:-scale], num[precision - scale:])
# Convert the generated decimal string into a Decimal object and return a -ive/+ive
# version of it with equal probability.
return Decimal(num) if randint(0,1) else Decimal("-{0}".format(num))
def _assert_decimal_result(self, cast, actual, expected):
assert actual == expected, "Cast: {0}, Expected: {1}, Actual: {2}".format(cast,\
expected, actual)
def _normalize_cast_expr(self, decimal_val, scale, from_string=False):
"""Convert the decimal value to a string litetal to avoid overflow.
If an integer literal is greater than the max bigint supported by Impala, it
overflows. This methods replaces it with a string literal.
"""
if (scale == 0 and abs(decimal_val) > self.max_bigint) or from_string:
return "select cast('{0}' as Decimal({1}, {2}))"
return "select cast({0} as Decimal({1}, {2}))"
def test_min_max_zero_null(self, vector):
"""Sanity test at limits.
Verify that:
- We can read decimal values at their +ive and -ive limits.
- 0 is expressible in all decimal types.
- NULL is expressible in all decimal types
"""
precision, scale = vector.get_value('decimal_type')
from_string = vector.get_value('cast_from') == 'string'
dec_max = Decimal('{0}.{1}'.format('9' * (precision - scale), '9' * scale))
# Multiplying large values eith -1 can produce an overflow.
dec_min = Decimal('-{0}'.format(str(dec_max)))
cast = self._normalize_cast_expr(dec_max, scale, from_string=from_string)
# Test max
res = Decimal(self.execute_scalar(cast.format(dec_max, precision, scale)))
self._assert_decimal_result(cast, res, dec_max)
# Test Min
res = Decimal(self.execute_scalar(cast.format(dec_min, precision, scale)))
self._assert_decimal_result(cast, res, dec_min)
# Test zero
res = Decimal(self.execute_scalar(cast.format(TestDecimalCasting.DECIMAL_ZERO,
precision, scale)))
self._assert_decimal_result(cast, res, TestDecimalCasting.DECIMAL_ZERO)
# Test NULL
null_cast = "select cast(NULL as Decimal({0}, {1}))".format(precision, scale)
res = self.execute_scalar(null_cast)
self._assert_decimal_result(null_cast, res, 'NULL')
def test_exact(self, vector):
"""Test to verify that an exact representation of the desired Decimal type is
maintained."""
precision, scale = vector.get_value('decimal_type')
from_string = vector.get_value('cast_from') == 'string'
for i in xrange(self.iterations):
val = self._gen_decimal_val(precision, scale)
cast = self._normalize_cast_expr(val, scale, from_string=from_string)\
.format(val, precision, scale)
res = Decimal(self.execute_scalar(cast))
self._assert_decimal_result(cast, res, val)
def test_overflow(self, vector):
"""Test to verify that we always return NULL when trying to cast a number with greater
precision that its intended decimal type"""
precision, scale = vector.get_value('decimal_type')
from_string = vector.get_value('cast_from') == 'string'
for i in xrange(self.iterations):
# Generate a decimal with a larger precision than the one we're casting to.
val = self._gen_decimal_val(randint(precision + 1, 39), scale)
cast = self._normalize_cast_expr(val, scale, from_string=from_string)\
.format(val, precision, scale)
res = self.execute_scalar(cast)
self._assert_decimal_result(cast, res, 'NULL')
def test_underflow(self, vector):
"""Test to verify that we truncate when the scale of the number being cast is higher
than the target decimal type (with no change in precision).
"""
precision, scale = vector.get_value('decimal_type')
from_string = vector.get_value('cast_from') == 'string'
if precision == scale:
pytest.skip("Cannot underflow scale when precision and scale are equal")
for i in xrange(self.iterations):
new_scale = randint(scale + 1, precision)
val = self._gen_decimal_val(precision, randint(new_scale, precision))
# We don't need to normalize the cast expr because scale will never be zero
cast = self._normalize_cast_expr(val, scale, from_string=from_string)\
.format(val, precision, scale)
res = Decimal(self.execute_scalar(cast))
# Truncate the decimal value to its target scale with quantize.
self._assert_decimal_result(cast, res, val.quantize(Decimal('0e-%s' % scale),
rounding=ROUND_DOWN))