def compile(self):
from ibis.expr.api import schema
buf = StringIO()
buf.write(self._create_line())
def _push_schema(x):
formatted = format_schema(x)
buf.write('{0}'.format(formatted))
if self.partition is not None:
modified_schema = []
partition_schema = []
for name, dtype in zip(self.schema.names, self.schema.types):
if name in self.partition:
partition_schema.append((name, dtype))
else:
modified_schema.append((name, dtype))
buf.write('\n')
_push_schema(schema(modified_schema))
buf.write('\nPARTITIONED BY ')
_push_schema(schema(partition_schema))
else:
buf.write('\n')
_push_schema(self.schema)
format_ddl = self.table_format.to_ddl()
if format_ddl:
buf.write(format_ddl)
buf.write(self._location())
return buf.getvalue()
def compile(self):
buf = StringIO()
buf.write(self._create_line())
select_query = self.select.compile()
buf.write('\nAS\n{0}'.format(select_query))
return buf.getvalue()
def compile(self):
from ibis.expr.api import schema
buf = StringIO()
buf.write(self._create_line())
def _push_schema(x):
formatted = format_schema(x)
buf.write('{0}'.format(formatted))
if self.partition is not None:
modified_schema = []
partition_schema = []
for name, dtype in zip(self.schema.names, self.schema.types):
if name in self.partition:
partition_schema.append((name, dtype))
else:
modified_schema.append((name, dtype))
buf.write('\n')
_push_schema(schema(modified_schema))
buf.write('\nPARTITIONED BY ')
_push_schema(schema(partition_schema))
else:
buf.write('\n')
_push_schema(self.schema)
format_ddl = self.table_format.to_ddl()
if format_ddl:
buf.write(format_ddl)
buf.write(self._location())
return buf.getvalue()
def to_ddl(self):
buf = StringIO()
buf.write("\nROW FORMAT DELIMITED")
if self.delimiter is not None:
buf.write("\nFIELDS TERMINATED BY '{0}'".format(self.delimiter))
if self.escapechar is not None:
buf.write("\nESCAPED BY '{0}'".format(self.escapechar))
if self.lineterminator is not None:
buf.write("\nLINES TERMINATED BY '{0}'"
.format(self.lineterminator))
buf.write("\nLOCATION '{0}'".format(self.path))
if self.na_rep is not None:
props = {
'serialization.null.format': self.na_rep
}
buf.write('\n')
buf.write(format_tblproperties(props))
return buf.getvalue()
def compile(self):
buf = StringIO()
buf.write(self._create_line())
select_query = self.select.compile()
buf.write('\nAS\n{0}'.format(select_query))
return buf.getvalue()
def compile(self):
buf = StringIO()
buf.write(self._create_line())
format_ddl = self.table_format.to_ddl()
buf.write(format_ddl)
return buf.getvalue()
def compile(self):
buf = StringIO()
buf.write(self._create_line())
format_ddl = self.table_format.to_ddl()
buf.write(format_ddl)
return buf.getvalue()
def to_ddl(self):
import json
buf = StringIO()
buf.write('\nSTORED AS AVRO')
buf.write("\nLOCATION '{0}'".format(self.path))
schema = json.dumps(self.avro_schema, indent=2, sort_keys=True)
schema = '\n'.join([x.rstrip() for x in schema.split('\n')])
buf.write("\nTBLPROPERTIES ('avro.schema.literal'='{0}')"
.format(schema))
return buf.getvalue()
def to_ddl(self):
import json
buf = StringIO()
buf.write('\nSTORED AS AVRO')
buf.write("\nLOCATION '{0}'".format(self.path))
schema = json.dumps(self.avro_schema, indent=2, sort_keys=True)
schema = '\n'.join([x.rstrip() for x in schema.split('\n')])
buf.write(
"\nTBLPROPERTIES ('avro.schema.literal'='{0}')".format(schema))
return buf.getvalue()
def to_ddl(self):
import json
buf = StringIO()
buf.write('\nSTORED AS AVRO')
buf.write("\nLOCATION '{0}'".format(self.path))
schema = json.dumps(self.avro_schema, indent=2, sort_keys=True)
schema = '\n'.join([x.rstrip() for x in schema.split('\n')])
props = {'avro.schema.literal': schema}
buf.write('\n')
buf.write(format_tblproperties(props))
return buf.getvalue()
def to_ddl(self):
import json
buf = StringIO()
buf.write('\nSTORED AS AVRO')
buf.write("\nLOCATION '{0}'".format(self.path))
schema = json.dumps(self.avro_schema, indent=2, sort_keys=True)
schema = '\n'.join([x.rstrip() for x in schema.split('\n')])
props = {'avro.schema.literal': schema}
buf.write('\n')
buf.write(format_tblproperties(props))
return buf.getvalue()
def compile(self):
buf = StringIO()
buf.write(self._create_line())
if self.example_file is not None:
buf.write("\nLIKE PARQUET '{0}'".format(self.example_file))
elif self.example_table is not None:
buf.write("\nLIKE {0}".format(self.example_table))
elif self.schema is not None:
schema = format_schema(self.schema)
buf.write('\n{0}'.format(schema))
else:
raise NotImplementedError
buf.write(self._storage())
buf.write(self._location())
return buf.getvalue()
def compile(self):
buf = StringIO()
buf.write(self._create_line())
if self.example_file is not None:
buf.write("\nLIKE PARQUET '{0}'".format(self.example_file))
elif self.example_table is not None:
buf.write("\nLIKE {0}".format(self.example_table))
elif self.schema is not None:
schema = format_schema(self.schema)
buf.write('\n{0}'.format(schema))
else:
raise NotImplementedError
buf.write(self._storage())
buf.write(self._location())
return buf.getvalue()
def to_ddl(self):
buf = StringIO()
buf.write("\nROW FORMAT DELIMITED")
if self.delimiter is not None:
buf.write("\nFIELDS TERMINATED BY '{0}'".format(self.delimiter))
if self.escapechar is not None:
buf.write("\nESCAPED BY '{0}'".format(self.escapechar))
if self.lineterminator is not None:
buf.write("\nLINES TERMINATED BY '{0}'"
.format(self.lineterminator))
buf.write("\nLOCATION '{0}'".format(self.path))
return buf.getvalue()
def compile(self):
from ibis.expr.api import Schema
buf = StringIO()
buf.write(self._create_line())
def _push_schema(x):
formatted = format_schema(x)
buf.write('{0}'.format(formatted))
if self.partition is not None:
main_schema = self.schema
part_schema = self.partition
if not isinstance(part_schema, Schema):
part_schema = Schema(
part_schema,
[self.schema[name] for name in part_schema])
to_delete = []
for name in self.partition:
if name in self.schema:
to_delete.append(name)
if len(to_delete):
main_schema = main_schema.delete(to_delete)
buf.write('\n')
_push_schema(main_schema)
buf.write('\nPARTITIONED BY ')
_push_schema(part_schema)
else:
buf.write('\n')
_push_schema(self.schema)
if self.table_format is not None:
buf.write(self.table_format.to_ddl())
else:
buf.write(self._storage())
buf.write(self._location())
return buf.getvalue()
def compile(self):
from ibis.expr.api import Schema
buf = StringIO()
buf.write(self._create_line())
def _push_schema(x):
formatted = format_schema(x)
buf.write('{0}'.format(formatted))
if self.partition is not None:
main_schema = self.schema
part_schema = self.partition
if not isinstance(part_schema, Schema):
part_schema = Schema(
part_schema,
[self.schema[name] for name in part_schema])
to_delete = []
for name in self.partition:
if name in self.schema:
to_delete.append(name)
if len(to_delete):
main_schema = main_schema.delete(to_delete)
buf.write('\n')
_push_schema(main_schema)
buf.write('\nPARTITIONED BY ')
_push_schema(part_schema)
else:
buf.write('\n')
_push_schema(self.schema)
format_ddl = self.table_format.to_ddl()
if format_ddl:
buf.write(format_ddl)
buf.write(self._location())
return buf.getvalue()
def to_ddl(self):
buf = StringIO()
buf.write("\nROW FORMAT DELIMITED")
if self.delimiter is not None:
buf.write("\nFIELDS TERMINATED BY '{0}'".format(self.delimiter))
if self.escapechar is not None:
buf.write("\nESCAPED BY '{0}'".format(self.escapechar))
if self.lineterminator is not None:
buf.write("\nLINES TERMINATED BY '{0}'"
.format(self.lineterminator))
buf.write("\nLOCATION '{0}'".format(self.path))
if self.na_rep is not None:
buf.write("\nTBLPROPERTIES('serialization.null.format'='{0}')"
.format(self.na_rep))
return buf.getvalue()
def to_ddl(self):
buf = StringIO()
buf.write("\nROW FORMAT DELIMITED")
if self.delimiter is not None:
buf.write("\nFIELDS TERMINATED BY '{0}'".format(self.delimiter))
if self.escapechar is not None:
buf.write("\nESCAPED BY '{0}'".format(self.escapechar))
if self.lineterminator is not None:
buf.write("\nLINES TERMINATED BY '{0}'"
.format(self.lineterminator))
buf.write("\nLOCATION '{0}'".format(self.path))
if self.na_rep is not None:
buf.write("\nTBLPROPERTIES('serialization.null.format'='{0}')"
.format(self.na_rep))
return buf.getvalue()
def to_ddl(self):
buf = StringIO()
buf.write("\nROW FORMAT DELIMITED")
if self.delimiter is not None:
buf.write("\nFIELDS TERMINATED BY '{0}'".format(self.delimiter))
if self.escapechar is not None:
buf.write("\nESCAPED BY '{0}'".format(self.escapechar))
if self.lineterminator is not None:
buf.write("\nLINES TERMINATED BY '{0}'".format(
self.lineterminator))
buf.write("\nLOCATION '{0}'".format(self.path))
if self.na_rep is not None:
props = {'serialization.null.format': self.na_rep}
buf.write('\n')
buf.write(format_tblproperties(props))
return buf.getvalue()
def test_table_info(self):
t = self.con.table('functional_alltypes')
buf = StringIO()
t.info(buf=buf)
assert buf.getvalue() is not None
def to_ddl(self):
buf = StringIO()
buf.write('\nSTORED AS PARQUET')
buf.write("\nLOCATION '{0}'".format(self.path))
return buf.getvalue()
def to_ddl(self):
buf = StringIO()
buf.write('\nSTORED AS PARQUET')
buf.write("\nLOCATION '{0}'".format(self.path))
return buf.getvalue()
def test_table_info(self):
t = self.con.table('functional_alltypes')
buf = StringIO()
t.info(buf=buf)
assert buf.getvalue() is not None
def test_table_info(alltypes):
buf = StringIO()
alltypes.info(buf=buf)
assert buf.getvalue() is not None