def schema(self):
if self._schema is None:
table_ref = InlineView(self._query_ast.to_sql(),
_random_id('inline_', 4))
self._schema = _get_table_schema_hack(self._ic._cursor,
table_ref.to_sql())
return self._schema
def from_sql_table(ic, table):
"""Create a BDF from a table name usable in Impala"""
table_name = _to_TableName(table)
table_ref = BaseTableRef(table_name)
schema = _get_table_schema_hack(ic._cursor, table_ref.to_sql())
select_list = tuple([SelectItem(expr=Literal(col)) for (col, ty) in schema])
return BigDataFrame(ic, SelectStmt(select_list, table_ref))
def from_sql_query(ic, query, alias=None):
"""Create a BDF from a SQL query executed by Impala"""
query_alias = alias if alias else _random_id('inline_', 4)
table_ref = InlineView(query, query_alias)
schema = _get_table_schema_hack(ic._cursor, table_ref.to_sql())
select_list = tuple([SelectItem(expr=Literal(col)) for (col, ty) in schema])
return BigDataFrame(ic, SelectStmt(select_list, table_ref))
def schema(self):
if self._schema is None:
table_ref = InlineView(
self._query_ast.to_sql(), _random_id('inline_', 4))
self._schema = _get_table_schema_hack(
self._ic._cursor, table_ref.to_sql())
return self._schema
def from_sql_table(ic, table):
"""Create a BDF from a table name usable in Impala"""
table_name = _to_TableName(table)
table_ref = BaseTableRef(table_name)
schema = _get_table_schema_hack(ic._cursor, table_ref.to_sql())
select_list = tuple(
[SelectItem(expr=Literal(col)) for (col, ty) in schema])
return BigDataFrame(ic, SelectStmt(select_list, table_ref))
def from_sql_query(ic, query, alias=None):
"""Create a BDF from a SQL query executed by Impala"""
query_alias = alias if alias else _random_id('inline_', 4)
table_ref = InlineView(query, query_alias)
schema = _get_table_schema_hack(ic._cursor, table_ref.to_sql())
select_list = tuple(
[SelectItem(expr=Literal(col)) for (col, ty) in schema])
return BigDataFrame(ic, SelectStmt(select_list, table_ref))
def _validate_schema(self):
schema = _get_table_schema_hack(self._ic._cursor, self.name)
if len(schema) != 2:
raise ValueError("schema of blob store must have two cols")
if schema[0][0] != 'key' or schema[0][1] != 'STRING':
raise ValueError("first col of blob store must be 'key STRING'")
if schema[1][0] != 'value' or schema[1][1] != 'STRING':
raise ValueError("second col of blob store must be 'value STRING'")