def test_select_basics(t):
name = 'testing123456'
expr = t.limit(10)
ast = build_ast(expr, SparkDialect.make_context())
select = ast.queries[0]
stmt = ddl.InsertSelect(name, select, database='foo')
result = stmt.compile()
expected = """\
INSERT INTO foo.`testing123456`
SELECT *
FROM functional_alltypes
LIMIT 10"""
assert result == expected
stmt = ddl.InsertSelect(name, select, database='foo', overwrite=True)
result = stmt.compile()
expected = """\
INSERT OVERWRITE TABLE foo.`testing123456`
SELECT *
FROM functional_alltypes
LIMIT 10"""
assert result == expected
def _create_table(
table_name, expr, database=None, can_exist=False, format='parquet'
):
ast = build_ast(expr, SparkDialect.make_context())
select = ast.queries[0]
statement = ddl.CTAS(
table_name,
select,
database=database,
format=format,
can_exist=can_exist,
)
return statement
def insert(
self,
obj=None,
overwrite=False,
values=None,
validate=True,
):
"""
Insert into Spark table.
Parameters
----------
obj : TableExpr or pandas DataFrame
overwrite : boolean, default False
If True, will replace existing contents of table
validate : boolean, default True
If True, do more rigorous validation that schema of table being
inserted is compatible with the existing table
Examples
--------
>>> t.insert(table_expr) # doctest: +SKIP
# Completely overwrite contents
>>> t.insert(table_expr, overwrite=True) # doctest: +SKIP
"""
if isinstance(obj, pd.DataFrame):
spark_df = self._session.createDataFrame(obj)
spark_df.insertInto(self.name, overwrite=overwrite)
return
expr = obj
if values is not None:
raise NotImplementedError
if validate:
existing_schema = self.schema()
insert_schema = expr.schema()
if not insert_schema.equals(existing_schema):
_validate_compatible(insert_schema, existing_schema)
ast = build_ast(expr, SparkDialect.make_context())
select = ast.queries[0]
statement = ddl.InsertSelect(
self._qualified_name,
select,
overwrite=overwrite,
)
return self._execute(statement.compile())
def _build_ast(self, expr, context):
result = comp.build_ast(expr, context)
return result
