def create_table_using_sql(self, ddl, provider):
sparkSession = SnappySession(self.sc)
schema = StructType().add("col1", IntegerType()).add("col2", IntegerType()).add("col3", IntegerType())
input = SnappyContextTests.testdata
dataDF = sparkSession.createDataFrame(input, schema)
sparkSession.sql("DROP TABLE IF EXISTS " + SnappyContextTests.tablename)
sparkSession.sql(ddl)
dataDF.write.insertInto(SnappyContextTests.tablename)
def create_table_using_datasource(self, provider, schemaddl=False):
sparkSession = SnappySession(self.sc)
schema = StructType().add("col1", IntegerType()).add("col2", IntegerType()).add("col3", IntegerType())
input = SnappyContextTests.testdata
df = sparkSession.createDataFrame(input, schema)
if schemaddl is False:
sparkSession.createTable(SnappyContextTests.tablename, provider, schema)
else:
sparkSession.createTable(SnappyContextTests.tablename, provider, "(COL1 INT , COL2 INT , COL3 INT)")
df.write.format("row").mode("append").saveAsTable(SnappyContextTests.tablename)
class SchemaDStream(DStream):
"""
A SQL based DStream with support for schema/Product
This class offers the ability to manipulate SQL query on DStreams
It is similar to SchemaRDD, which offers the similar functions
Internally, RDD of each batch duration is treated as a small
table and CQs are evaluated on those small tables
Some of the abstraction and code is borrowed from the project:
https://github.com/Intel-bigdata/spark-streamingsql
@param snsc
@param queryExecution
"""
def __init__(self, jdstream, ssc, jrdd_deserializer, schema):
DStream.__init__(self, jdstream, ssc, jrdd_deserializer)
self._schema = schema
self._snappySession = SnappySession(self._sc)
def foreachDataFrame(self, func):
def createDataFrame(_, rdd):
df = self._snappySession.createDataFrame(rdd, self._schema)
func(df)
self.foreachRDD(createDataFrame)
