def test_literal_geospatial():
# point
point_0 = (0, 0)
point = ibis.literal(point_0, type='point')
assert ibis.mapd.compile(point) == "SELECT 'POINT(0 0)' AS tmp"
# line
line_0 = [point_0, point_0]
line = ibis.literal(line_0, type='linestring')
assert ibis.mapd.compile(line) == "SELECT 'LINESTRING(0 0, 0 0)' AS tmp"
# polygon
polygon_0 = [tuple(line_0), tuple(line_0)]
polygon = ibis.literal(polygon_0, type='polygon')
assert ibis.mapd.compile(polygon) == (
"SELECT 'POLYGON((0 0, 0 0), (0 0, 0 0))' AS tmp"
)
# multipolygon
mpolygon_0 = [tuple(polygon_0), tuple(polygon_0)]
mpolygon = ibis.literal(mpolygon_0, type='multipolygon')
assert ibis.mapd.compile(mpolygon) == (
"SELECT 'MULTIPOLYGON(((0 0, 0 0), (0 0, 0 0)), "
"((0 0, 0 0), (0 0, 0 0)))' AS tmp"
)
def test_custom_type_binary_operations():
class Foo(ir.ValueExpr):
def __add__(self, other):
op = self.op()
return type(op)(op.value + other).to_expr()
__radd__ = __add__
class FooNode(ops.ValueOp):
value = Arg(rlz.integer)
def output_type(self):
return functools.partial(Foo, dtype=dt.int64)
left = ibis.literal(2)
right = FooNode(3).to_expr()
result = left + right
assert isinstance(result, Foo)
assert isinstance(result.op(), FooNode)
left = FooNode(3).to_expr()
right = ibis.literal(2)
result = left + right
assert isinstance(result, Foo)
assert isinstance(result.op(), FooNode)
def test_log_variants(self):
opnames = ["ln", "log", "log2", "log10"]
columns = ["a", "b", "c", "d", "e", "f"]
for opname in opnames:
def f(x):
return getattr(x, opname)()
for c in columns:
result = f(self.table[c])
assert isinstance(result, api.DoubleArray)
# is this what we want?
# assert result.get_name() == c
assert isinstance(f(ibis.literal(5)), api.DoubleScalar)
assert isinstance(f(ibis.literal(5.5)), api.DoubleScalar)
klass = getattr(ops, opname.capitalize())
with self.assertRaises(IbisTypeError):
if opname == "log":
klass(self.table["g"], None).to_expr()
else:
klass(self.table["g"]).to_expr()
# boolean not implemented for these
with self.assertRaises(IbisTypeError):
f(self.table["h"])
def test_between(table):
result = table.f.between(0, 1)
assert isinstance(result, ir.BooleanColumn)
assert isinstance(result.op(), ops.Between)
# it works!
result = table.g.between('a', 'f')
assert isinstance(result, ir.BooleanColumn)
result = ibis.literal(1).between(table.a, table.c)
assert isinstance(result, ir.BooleanColumn)
result = ibis.literal(7).between(5, 10)
assert isinstance(result, ir.BooleanScalar)
# Cases where between should immediately fail, e.g. incomparables
with pytest.raises(TypeError):
table.f.between('0', '1')
with pytest.raises(TypeError):
table.f.between(0, '1')
with pytest.raises(TypeError):
table.f.between('0', 1)
def test_log_variants(self):
opnames = ['ln', 'log', 'log2', 'log10']
columns = ['a', 'b', 'c', 'd', 'e', 'f']
for opname in opnames:
def f(x):
return getattr(x, opname)()
for c in columns:
result = f(self.table[c])
assert isinstance(result, api.DoubleArray)
# is this what we want?
# assert result.get_name() == c
assert isinstance(f(ibis.literal(5)), api.DoubleScalar)
assert isinstance(f(ibis.literal(5.5)), api.DoubleScalar)
klass = getattr(ops, opname.capitalize())
with self.assertRaises(IbisTypeError):
if opname == 'log':
klass(self.table['g'], None).to_expr()
else:
klass(self.table['g']).to_expr()
# boolean not implemented for these
with self.assertRaises(IbisTypeError):
f(self.table['h'])
def test_boolean(self):
val = True
expr = ibis.literal(val)
self._check_literal(expr, api.BooleanScalar, val)
val = False
expr = ibis.literal(val)
self._check_literal(expr, api.BooleanScalar, val)
def test_array_concat_scalar(client, op):
raw_left = [1, 2, 3]
raw_right = [3, 4]
left = ibis.literal(raw_left)
right = ibis.literal(raw_right)
expr = op(left, right)
result = client.execute(expr)
assert result == op(raw_left, raw_right)
def test_join(table):
dash = literal('-')
expr = dash.join([table.f.cast('string'), table.g])
assert isinstance(expr, ir.StringColumn)
expr = dash.join([literal('ab'), literal('cd')])
assert isinstance(expr, ir.StringScalar)
def test_isnan_isinf_scalar(value):
expr = ibis.literal(value).isnan()
assert isinstance(expr, ir.BooleanScalar)
assert isinstance(expr.op(), ops.IsNan)
expr = ibis.literal(value).isinf()
assert isinstance(expr, ir.BooleanScalar)
assert isinstance(expr.op(), ops.IsInf)
def test_literal_in_list(self):
cases = [
(ibis.literal(2).isin([self.table.a, self.table.b, self.table.c]),
'2 IN (a, b, c)'),
(ibis.literal(2).notin([self.table.a, self.table.b, self.table.c]),
'2 NOT IN (a, b, c)')
]
self._check_expr_cases(cases)
def test_expr_list_no_table_refs(self):
exlist = ibis.api.expr_list([ibis.literal(1).name('a'),
ibis.now().name('b'),
ibis.literal(2).log().name('c')])
result = to_sql(exlist)
expected = """\
SELECT 1 AS `a`, now() AS `b`, ln(2) AS `c`"""
assert result == expected
def test_join(self):
dash = literal('-')
expr = dash.join([self.table.f.cast('string'),
self.table.g])
assert isinstance(expr, ir.StringArray)
expr = dash.join([literal('ab'), literal('cd')])
assert isinstance(expr, ir.StringScalar)
def test_timestamp_literals(self):
tv1 = '2015-01-01 12:34:56'
ex1 = ("'2015-01-01 12:34:56'")
cases = [
(ibis.literal(pd.Timestamp(tv1)), ex1),
(ibis.literal(pd.Timestamp(tv1).to_pydatetime()), ex1),
(ibis.timestamp(tv1), ex1)
]
self._check_expr_cases(cases)
def test_execute_exprs_no_table_ref(self):
cases = [(ibis.literal(1) + ibis.literal(2), 3)]
for expr, expected in cases:
result = self.con.execute(expr)
assert result == expected
# ExprList
exlist = ibis.api.expr_list([ibis.literal(1).name("a"), ibis.now().name("b"), ibis.literal(2).log().name("c")])
self.con.execute(exlist)
def test_concat():
exprs = [ibis.literal(1).name('a'), ibis.literal(2).name('b')]
exprs2 = [ibis.literal(3).name('c'), ibis.literal(4).name('d')]
list1 = ibis.expr_list(exprs)
list2 = ibis.expr_list(exprs2)
result = list1.concat(list2)
expected = ibis.expr_list(exprs + exprs2)
assert_equal(result, expected)
def test_concat(self):
exprs = [ibis.literal(1).name("a"), ibis.literal(2).name("b")]
exprs2 = [ibis.literal(3).name("c"), ibis.literal(4).name("d")]
list1 = ibis.expr_list(exprs)
list2 = ibis.expr_list(exprs2)
result = list1.concat(list2)
expected = ibis.expr_list(exprs + exprs2)
assert_equal(result, expected)
def test_expr_list_no_table_refs():
exlist = ibis.api.expr_list(
[
ibis.literal(1).name('a'),
ibis.now().name('b'),
ibis.literal(2).log().name('c'),
]
)
result = ibis.clickhouse.compile(exlist)
expected = """\
SELECT 1 AS `a`, now() AS `b`, log(2) AS `c`"""
assert result == expected
def test_group_concat(functional_alltypes):
col = functional_alltypes.string_col
expr = col.group_concat()
assert isinstance(expr.op(), ops.GroupConcat)
arg, sep, where = expr.op().args
assert sep.equals(ibis.literal(','))
assert where is None
expr = col.group_concat('|')
arg, sep, where = expr.op().args
assert sep.equals(ibis.literal('|'))
assert where is None
def test_non_inferrable_literal():
expected_msg = (
'The datatype of value .* cannot be inferred, try '
'passing it explicitly with the `type` keyword.'
)
value = tuple(pointA)
with pytest.raises(TypeError, match=expected_msg):
ibis.literal(value)
point = ibis.literal(value, type='point')
assert point.type() == dt.point
def test_decimal_builtins(self):
d = ibis.literal(5.245)
general_cases = [
(ibis.literal(-5).abs(), 5),
(d.cast("int32"), 5),
(d.ceil(), 6),
(d.isnull(), False),
(d.floor(), 5),
(d.notnull(), True),
(d.round(), 5),
(d.round(2), Decimal("5.25")),
(d.sign(), 1),
]
self.assert_cases_equality(general_cases)
def test_int_builtins(self):
i8 = ibis.literal(50)
i32 = ibis.literal(50000)
mod_cases = [(i8 % 5, 0), (i32 % 10, 0), (250 % i8, 0)]
nullif_cases = [
(5 / i8.nullif(0), 0.1),
(5 / i8.nullif(i32), 0.1),
(5 / i32.nullif(0), 0.0001),
(i32.zeroifnull(), 50000),
]
self.assert_cases_equality(mod_cases + nullif_cases)
def test_string_to_timestamp(client):
timestamp = pd.Timestamp(
datetime.datetime(year=2017, month=2, day=6), tz=pytz.timezone('UTC')
)
expr = ibis.literal('2017-02-06').to_timestamp('%F')
result = client.execute(expr)
assert result == timestamp
timestamp_tz = pd.Timestamp(
datetime.datetime(year=2017, month=2, day=6, hour=5),
tz=pytz.timezone('UTC'),
)
expr_tz = ibis.literal('2017-02-06').to_timestamp('%F', 'America/New_York')
result_tz = client.execute(expr_tz)
assert result_tz == timestamp_tz
def test_boolean_logical_ops(table, operation):
expr = table.a > 0
result = operation(expr, table.h)
assert isinstance(result, ir.BooleanColumn)
result = operation(expr, True)
refl_result = operation(True, expr)
assert isinstance(result, ir.BooleanColumn)
assert isinstance(refl_result, ir.BooleanColumn)
true = ibis.literal(True)
false = ibis.literal(False)
result = operation(true, false)
assert isinstance(result, ir.BooleanScalar)
def test_substr(self):
lit = literal('FoO')
result = self.table.g.substr(2, 4)
lit_result = lit.substr(0, 2)
assert isinstance(result, ir.StringArray)
assert isinstance(lit_result, ir.StringScalar)
op = result.op()
assert isinstance(op, ops.Substring)
start, length = op.args[1:]
assert start.equals(literal(2))
assert length.equals(literal(4))
def test_decimal_casts(self):
cases = [
(ibis.literal('9.9999999').cast('decimal(38,5)'),
"CAST('9.9999999' AS decimal(38,5))"),
(self.table.f.cast('decimal(12,2)'), "CAST(f AS decimal(12,2))")
]
self._check_expr_cases(cases)
def test_literal_complex_types(value, expected_type, expected_class):
expr = ibis.literal(value)
expr_type = expr.type()
assert expr_type.equals(dt.validate_type(expected_type))
assert isinstance(expr, expected_class)
assert isinstance(expr.op(), ops.Literal)
assert expr.op().value is value
def test_integer_to_interval_date_failure(backend, con, alltypes, df, unit):
interval = alltypes.int_col.to_interval(unit=unit)
array = alltypes.date_string_col.split('/')
month, day, year = array[0], array[1], array[2]
date_col = ibis.literal('-').join(['20' + year, month, day]).cast('date')
with pytest.raises(TypeError):
date_col + interval
def test_null(self):
expr = ibis.literal(None)
assert isinstance(expr, ir.NullScalar)
assert isinstance(expr.op(), ir.NullLiteral)
expr2 = ibis.null()
assert_equal(expr, expr2)
def test_day_of_week_scalar(backend, con, date, expected_index, expected_day):
expr = ibis.literal(date).cast(dt.date)
result_index = con.execute(expr.day_of_week.index())
assert result_index == expected_index
result_day = con.execute(expr.day_of_week.full_name())
assert result_day.lower() == expected_day.lower()
def test_decimal_builtins_2(self):
d = ibis.literal("5.245")
dc = d.cast("decimal(12,5)")
cases = [
(dc % 5, Decimal("0.245")),
(dc.fillna(0), Decimal("5.245")),
(dc.exp(), 189.6158),
(dc.log(), 1.65728),
(dc.log2(), 2.39094),
(dc.log10(), 0.71975),
(dc.sqrt(), 2.29019),
(dc.zeroifnull(), Decimal("5.245")),
(-dc, Decimal("-5.245")),
]
def approx_equal(a, b, eps):
assert abs(a - b) < eps
for expr, expected in cases:
result = self.con.execute(expr)
if isinstance(expected, Decimal):
tol = Decimal("0.0001")
else:
tol = 0.0001
approx_equal(result, expected, tol)
def test_literal(client):
lit = ibis.literal(1)
result = client.execute(lit)
assert result == 1
def test_special_strings(con, alltypes, data, data_type):
lit = ibis.literal(data, type=data_type).name('tmp')
expr = alltypes[[alltypes.id, lit]].head(1)
df = expr.execute()
assert df['tmp'].iloc[0] == uuid.UUID(data)
def test_implicit_typecasting(self):
col = self.alltypes.tinyint_col
literal = ibis.literal(1000)
self._identity_func_testing('int32', literal, col)
def test_literal(client):
assert client.execute(ibis.literal(1)) == 1
def test_float_format(client):
expr = ibis.literal(1.0, type='float')
assert client.compile(expr) == 'SELECT CAST(1.0 AS FLOAT) AS `tmp`'
def test_double_wrapping(self):
col = self.alltypes.double_col
literal = ibis.literal(3.14)
self._identity_func_testing('double', literal, col)
param(
lambda t: t.date_string_col[:],
lambda t: t.date_string_col,
id='expr_empty_slice',
),
param(
lambda t: t.date_string_col[t.date_string_col.length() - 2:t.
date_string_col.length() - 1],
lambda t: t.date_string_col.str[-2:-1],
id='expr_slice_begin_end',
),
param(
lambda t: t.date_string_col.split('/'),
lambda t: t.date_string_col.str.split('/'),
id='split',
),
param(
lambda t: ibis.literal('-').join(['a', t.string_col, 'c']),
lambda t: 'a-' + t.string_col + '-c',
id='join',
),
],
)
@pytest.mark.xfail_unsupported
def test_string(backend, alltypes, df, result_func, expected_func):
expr = result_func(alltypes)
result = expr.execute()
expected = backend.default_series_rename(expected_func(df))
backend.assert_series_equal(result, expected)
""" Tests for geo spatial data types"""
import numpy as np
import pytest
from numpy import testing
from pytest import param
import ibis
from ibis.tests.backends import OmniSciDB, PostgreSQL
geopandas = pytest.importorskip('geopandas')
shapely = pytest.importorskip('shapely')
shapely_wkt = pytest.importorskip('shapely.wkt')
# geo literals declaration
point_0 = ibis.literal((0, 0), type='point').name('tmp')
point_0_4326 = ibis.literal((0, 0), type='point;4326').name('tmp')
point_geom_0 = ibis.literal((0, 0), type='point;4326:geometry').name('p')
point_geom_1 = ibis.literal((1, 1), type='point;4326:geometry').name('p')
point_geom_0_srid0 = ibis.literal((0, 0), type='point;0:geometry').name('p')
point_geom_1_srid0 = ibis.literal((1, 1), type='point;0:geometry').name('p')
point_geom_2 = ibis.literal((2, 2), type='point;4326:geometry').name('p')
point_geog_0 = ibis.literal((0, 0), type='point;4326:geography').name('p')
point_geog_1 = ibis.literal((1, 1), type='point;4326:geography').name('p')
point_geog_2 = ibis.literal((2, 2), type='point;4326:geography').name('p')
polygon_0 = ibis.literal(
(
((1, 0), (0, 1), (-1, 0), (0, -1), (1, 0)),
((0.1, 0), (0, 0.1), (-0.1, 0), (0, -0.1), (0.1, 0)),
),
type='polygon;4326:geometry',
@pytest.fixture
def sch_decimal():
return ibis.schema([('index', 'int64'), ('n1', 'decimal'),
('n2', 'decimal')])
@pytest.mark.parametrize(
('operand_fn', 'expected_operand_fn'),
[
param(lambda t: t.float_col, lambda t: t.float_col, id='float-column'),
param(lambda t: t.double_col,
lambda t: t.double_col,
id='double-column'),
param(lambda t: ibis.literal(1.3), lambda t: 1.3, id='float-literal'),
param(lambda t: ibis.literal(np.nan),
lambda t: np.nan,
id='nan-literal'),
param(lambda t: ibis.literal(np.inf),
lambda t: np.inf,
id='inf-literal'),
param(
lambda t: ibis.literal(-np.inf),
lambda t: -np.inf,
id='-inf-literal',
),
],
)
@pytest.mark.parametrize(
('expr_fn', 'expected_expr_fn'),
def test_boolean_wrapping(self):
col = self.alltypes.bool_col
literal = ibis.literal(True)
self._identity_func_testing('boolean', literal, col)
def test_datetime64_infer(client, unit):
value = np.datetime64('2018-01-02', unit)
expr = ibis.literal(value, type='timestamp')
result = client.execute(expr)
assert result == value
def test_timestamp_precedence():
ts = ibis.literal(datetime.now())
highest_type = rlz.highest_precedence_dtype([ibis.NA, ts])
assert highest_type == dt.timestamp
def test_float_wrapping(self):
col = self.alltypes.float_col
literal = ibis.literal(3.14)
self._identity_func_testing('float', literal, col)
def test_literal_multi_geospatial_inferred(backend, con, shp, expected):
result = str(con.compile(ibis.literal(shp)))
result_expected = "SELECT {} AS tmp".format(expected[backend.name])
assert result in result_expected
),
(
lambda t: t.int_col.cast(dt.timestamp),
lambda at: sa.func.datetime(at.c.int_col, 'unixepoch'),
),
],
)
def test_timestamp_cast_noop(alltypes, func, translate, alltypes_sqla,
expected_func, sqla_compile):
# See GH #592
result = func(alltypes)
expected = expected_func(alltypes_sqla)
assert translate(result) == sqla_compile(expected)
TIMESTAMP_CONSTANT = ibis.literal('2015-09-01 14:48:05.359').cast('timestamp')
@pytest.mark.parametrize(
('expr', 'expected'),
[
(TIMESTAMP_CONSTANT.strftime('%Y%m%d'), '20150901'),
(TIMESTAMP_CONSTANT.year(), 2015),
(TIMESTAMP_CONSTANT.month(), 9),
(TIMESTAMP_CONSTANT.day(), 1),
(TIMESTAMP_CONSTANT.hour(), 14),
(TIMESTAMP_CONSTANT.minute(), 48),
(TIMESTAMP_CONSTANT.second(), 5),
(TIMESTAMP_CONSTANT.millisecond(), 359),
],
)
def test_int_wrapping(self):
col = self.alltypes.int_col
literal = ibis.literal(1000)
self._identity_func_testing('int32', literal, col)
def test_bigint_wrapping(self):
col = self.alltypes.bigint_col
literal = ibis.literal(1000).cast('int64')
self._identity_func_testing('int64', literal, col)
def test_simple_case_scalar(client):
x = ibis.literal(2)
expr = x.case().when(2, x - 1).when(3, x + 1).when(4, x + 2).end()
result = client.execute(expr)
expected = np.int8(1)
assert result == expected
def test_async(client):
expr = ibis.literal(1)
result = client.execute(expr, async=True)
assert result.get_result() == 1
def test_string_wrapping(self):
col = self.alltypes.string_col
literal = ibis.literal('ibis')
self._identity_func_testing('string', literal, col)
result_index,
self.dtype,
self.max_lookback,
*args,
**kwargs,
)
return result
@pytest.fixture(scope='session')
def sort_kind():
return 'mergesort'
default = pytest.mark.parametrize('default', [ibis.NA, ibis.literal('a')])
row_offset = pytest.mark.parametrize('row_offset',
list(map(ibis.literal, [-1, 1, 0])))
range_offset = pytest.mark.parametrize(
'range_offset',
[
ibis.interval(days=1),
2 * ibis.interval(days=1),
-2 * ibis.interval(days=1),
],
)
@pytest.fixture
def row_window():
return ibis.window(following=0, order_by='plain_int64')
def test_tinyint_wrapping(self):
col = self.alltypes.tinyint_col
literal = ibis.literal(5)
self._identity_func_testing('int8', literal, col)
assert result == expected
@pytest.mark.parametrize('column', ['float64_with_zeros', 'int64_with_zeros'])
def test_null_if_zero(t, df, column):
expr = t[column].nullifzero()
result = expr.execute()
expected = df[column].replace(0, np.nan)
tm.assert_series_equal(result, expected)
@pytest.mark.parametrize(
('left', 'right', 'expected', 'compare'),
[
pytest.param(
lambda t: ibis.literal(1),
lambda t: ibis.literal(1),
lambda df: np.nan,
np.testing.assert_array_equal, # treats NaNs as equal
id='literal_literal_equal',
),
pytest.param(
lambda t: ibis.literal(1),
lambda t: ibis.literal(2),
lambda df: 1,
np.testing.assert_equal,
id='literal_literal_not_equal',
),
pytest.param(
lambda t: t.dup_strings,
lambda t: ibis.literal('a'),
@pytest.mark.parametrize(
('klass', 'value', 'expected'),
[
(ir.TableExpr, object, IbisTypeError),
(ir.IntegerValue, 4, IbisTypeError),
],
)
def test_invalid_instance_of(klass, value, expected):
with pytest.raises(expected):
assert rlz.instance_of(klass, value)
@pytest.mark.parametrize(
('dtype', 'value', 'expected'),
[
pytest.param(dt.int8, 26, ibis.literal(26)),
pytest.param(dt.int16, 26, ibis.literal(26)),
pytest.param(dt.int32, 26, ibis.literal(26)),
pytest.param(dt.int64, 26, ibis.literal(26)),
pytest.param(dt.uint8, 26, ibis.literal(26)),
pytest.param(dt.uint16, 26, ibis.literal(26)),
pytest.param(dt.uint32, 26, ibis.literal(26)),
pytest.param(dt.uint64, 26, ibis.literal(26)),
pytest.param(dt.float32, 26, ibis.literal(26)),
pytest.param(dt.float64, 26.4, ibis.literal(26.4)),
pytest.param(dt.double, 26.3, ibis.literal(26.3)),
pytest.param(dt.string, 'bar', ibis.literal('bar')),
pytest.param(dt.Array(dt.float), [3.4, 5.6], ibis.literal([3.4, 5.6])),
pytest.param(
dt.Map(dt.string, dt.Array(dt.boolean)),
{'a': [True, False], 'b': [True]},
def test_map_value_for_key_literal_broadcast(t):
lookup_table = ibis.literal({'a': 1, 'b': 2, 'c': 3, 'd': 4})
expr = lookup_table.get(t.dup_strings)
result = expr.execute()
expected = pd.Series([4, 1, 4], name='dup_strings')
tm.assert_series_equal(result, expected)
def test_literal_execute(client):
expected = '1234'
expr = ibis.literal(expected)
result = client.execute(expr)
assert result == expected
def test_map_values_scalar(client, t):
expr = ibis.literal({'a': 10, 'b': 50, 'c': 20, 'd': 40})
expr = expr.values()
result = client.execute(expr)
expected = np.array([10, 50, 20, 40])
tm.assert_numpy_array_equal(result, expected)
def test_double_format(client):
expr = ibis.literal(1.0)
assert client.compile(expr) == 'SELECT 1.0d AS `tmp`'
def _cross_join(translator, expr):
args = expr.op().args
left, right = args[:2]
return translator.translate(left.join(right, ibis.literal(True)))
def test_endswith(table):
assert isinstance(table.g.endswith('foo'), ir.BooleanColumn)
assert isinstance(literal('bar').endswith('foo'), ir.BooleanScalar)