def test_delete(self):
self.drop_table(True)
self.create_table_using_datasource("row")
sqlcontext = SnappyContext.getOrCreate(self.sc)
self.assertTrue(
sqlcontext.delete(SnappyContextTests.tablename, "col1=1"), 2)
self.drop_table()
def insert_table(self):
sqlcontext = SnappyContext.getOrCreate(self.sc)
newrow = [1L, 2L, 3L], [2L, 3L, 4L]
sqlcontext.insert(SnappyContextTests.tablename, newrow)
self.verify_table_rows(7)
newrow = [1L, 2L, 3L]
sqlcontext.insert(SnappyContextTests.tablename , newrow)
self.verify_table_rows(8)
def create_table_using_datasource(self, provider, schemaddl=False):
sqlcontext = SnappyContext.getOrCreate(self.sc)
df = sqlcontext._sc.parallelize(SnappyContextTests.testdata, 5).toDF(["COL1", "COL2", "COL3"])
if schemaddl is False:
sqlcontext.createTable(SnappyContextTests.tablename, provider, df.schema)
else:
sqlcontext.createTable(SnappyContextTests.tablename, provider, "(COL1 INT , COL2 INT , COL3 INT)")
df.write.format("row").mode("append").saveAsTable(SnappyContextTests.tablename)
def insert_table(self):
sqlcontext = SnappyContext.getOrCreate(self.sc)
newrow = [1L, 2L, 3L], [2L, 3L, 4L]
sqlcontext.insert(SnappyContextTests.tablename, newrow)
self.verify_table_rows(7)
newrow = [1L, 2L, 3L]
sqlcontext.insert(SnappyContextTests.tablename, newrow)
self.verify_table_rows(8)
def create_table_using_sql(self, ddl, provider):
sqlcontext = SnappyContext.getOrCreate(self.sc)
dataDF = sqlcontext._sc.parallelize(SnappyContextTests.testdata,
5).toDF()
sqlcontext.sql("DROP TABLE IF EXISTS " + SnappyContextTests.tablename)
sqlcontext.sql(ddl)
dataDF.write.format(provider).mode("append").saveAsTable(
SnappyContextTests.tablename)
def test_new_session(self):
sqlcontext1 = SnappyContext.getOrCreate(self.sc)
sqlcontext1.setConf("test_key", "a")
sqlcontext2 = sqlcontext1.newSession()
sqlcontext2.setConf("test_key", "b")
self.assertEqual(sqlcontext1.getConf("test_key", ""), "a")
self.assertEqual(sqlcontext2.getConf("test_key", ""), "b")
def test_new_session(self):
sqlcontext1 = SnappyContext.getOrCreate(self.sc)
sqlcontext1.setConf("test_key", "a")
sqlcontext2 = sqlcontext1.newSession()
sqlcontext2.setConf("test_key", "b")
self.assertEqual(sqlcontext1.getConf("test_key", ""), "a")
self.assertEqual(sqlcontext2.getConf("test_key", ""), "b")
def create_table_using_datasource(self, provider, schemaddl=False):
sqlcontext = SnappyContext.getOrCreate(self.sc)
df = sqlcontext._sc.parallelize(SnappyContextTests.testdata,
5).toDF(["COL1", "COL2", "COL3"])
if schemaddl is False:
sqlcontext.createTable(SnappyContextTests.tablename, provider,
df.schema)
else:
sqlcontext.createTable(SnappyContextTests.tablename, provider,
"(COL1 INT , COL2 INT , COL3 INT)")
df.write.format("row").mode("append").saveAsTable(
SnappyContextTests.tablename)
def update_table(self):
sqlcontext = SnappyContext.getOrCreate(self.sc)
modifiedrows = sqlcontext.update(SnappyContextTests.tablename, "COL2 =2", [7L], ["COL1"])
self.assertTrue(modifiedrows == 3)
def test_get_or_create(self):
sqlcontext = SnappyContext.getOrCreate(self.sc)
self.assertTrue(SnappyContext.getOrCreate(self.sc) is sqlcontext)
def update_table(self):
sqlcontext = SnappyContext.getOrCreate(self.sc)
modifiedrows = sqlcontext.update(SnappyContextTests.tablename,
"COL2 =2", [7L], ["COL1"])
self.assertTrue(modifiedrows == 3)
def create_table_using_sql(self, ddl, provider):
sqlcontext = SnappyContext.getOrCreate(self.sc)
dataDF = sqlcontext._sc.parallelize(SnappyContextTests.testdata, 5).toDF()
sqlcontext.sql("DROP TABLE IF EXISTS " + SnappyContextTests.tablename)
sqlcontext.sql(ddl)
dataDF.write.format(provider).mode("append").saveAsTable(SnappyContextTests.tablename)
def drop_table(self, ifexists=False):
sqlcontext = SnappyContext.getOrCreate(self.sc)
sqlcontext.dropTable(SnappyContextTests.tablename, ifexists)
def verify_table_rows(self, rowcount):
sqlcontext = SnappyContext.getOrCreate(self.sc)
result = sqlcontext.sql("SELECT COUNT(*) FROM " + SnappyContextTests.tablename).collect()
self.assertTrue(result[0]._c0 == rowcount)
def truncate_table(self):
sqlcontext = SnappyContext.getOrCreate(self.sc)
sqlcontext.truncateTable(SnappyContextTests.tablename)
def update_table(self):
sqlcontext = SnappyContext.getOrCreate(self.sc)
newColumnvalues = Row(col1=7L)
modifiedrows = sqlcontext.update(SnappyContextTests.tablename, "COL2 =2", newColumnvalues, "COL1")
self.assertTrue(modifiedrows == 3)
def truncate_table(self):
sqlcontext = SnappyContext.getOrCreate(self.sc)
sqlcontext.truncateTable(SnappyContextTests.tablename)
def test_delete(self):
self.drop_table(True)
self.create_table_using_datasource("row")
sqlcontext = SnappyContext.getOrCreate(self.sc)
self.assertTrue(sqlcontext.delete(SnappyContextTests.tablename, "col1=1"), 2)
self.drop_table()
def verify_table_rows(self, rowcount):
sqlcontext = SnappyContext.getOrCreate(self.sc)
result = sqlcontext.sql("SELECT COUNT(*) FROM " +
SnappyContextTests.tablename).collect()
self.assertTrue(result[0]._c0 == rowcount)
def test_get_or_create(self):
sqlcontext = SnappyContext.getOrCreate(self.sc)
self.assertTrue(SnappyContext.getOrCreate(self.sc) is sqlcontext)
def drop_table(self, ifexists=False):
sqlcontext = SnappyContext.getOrCreate(self.sc)
sqlcontext.dropTable(SnappyContextTests.tablename, ifexists)
