def test_scala_livy_sql_options(): query = "abc" sqlquery = SQLQuery(query, samplemethod="take", maxrows=100) assert_equals( sqlquery._scala_command("sqlContext"), Command('sqlContext.sql("""{}""").toJSON.take(100).foreach(println)'.format(query)), ) sqlquery = SQLQuery(query, samplemethod="take", maxrows=-1) assert_equals( sqlquery._scala_command("sqlContext"), Command('sqlContext.sql("""{}""").toJSON.collect.foreach(println)'.format(query)), ) sqlquery = SQLQuery(query, samplemethod="sample", samplefraction=0.25, maxrows=-1) assert_equals( sqlquery._scala_command("sqlContext"), Command('sqlContext.sql("""{}""").toJSON.sample(false, 0.25).collect.foreach(println)'.format(query)), ) sqlquery = SQLQuery(query, samplemethod="sample", samplefraction=0.33, maxrows=3234) assert_equals( sqlquery._scala_command("sqlContext"), Command('sqlContext.sql("""{}""").toJSON.sample(false, 0.33).take(3234).foreach(println)'.format(query)), )
def test_scala_livy_sql_options(): query = "abc" sqlquery = SQLQuery(query, samplemethod='take', maxrows=100) assert_equals( sqlquery._scala_command(), Command('sqlContext.sql("""{}""").toJSON.take(100).foreach(println)'. format(query))) sqlquery = SQLQuery(query, samplemethod='take', maxrows=-1) assert_equals( sqlquery._scala_command(), Command( 'sqlContext.sql("""{}""").toJSON.collect.foreach(println)'.format( query))) sqlquery = SQLQuery(query, samplemethod='sample', samplefraction=0.25, maxrows=-1) assert_equals( sqlquery._scala_command(), Command( 'sqlContext.sql("""{}""").toJSON.sample(false, 0.25).collect.foreach(println)' .format(query))) sqlquery = SQLQuery(query, samplemethod='sample', samplefraction=0.33, maxrows=3234) assert_equals( sqlquery._scala_command(), Command( 'sqlContext.sql("""{}""").toJSON.sample(false, 0.33).take(3234).foreach(println)' .format(query)))
def test_unicode_sql(): query = u"SELECT 'è'" sqlquery = SQLQuery(query, samplemethod='take', maxrows=120) assert_equals(sqlquery._pyspark_command("spark"), Command(u'for {} in spark.sql(u"""{} """).toJSON().take(120): print({}.encode("{}"))'\ .format(LONG_RANDOM_VARIABLE_NAME, query, LONG_RANDOM_VARIABLE_NAME, conf.pyspark_dataframe_encoding()))) assert_equals(sqlquery._scala_command("spark"), Command(u'spark.sql("""{}""").toJSON.take(120).foreach(println)'.format(query))) assert_equals(sqlquery._r_command("spark"), Command(u'for ({} in (jsonlite:::toJSON(take(sql("{}"),120)))) {{cat({})}}'.format(LONG_RANDOM_VARIABLE_NAME, query, LONG_RANDOM_VARIABLE_NAME)))
def test_unicode_sql(): query = u"SELECT 'è'" sqlquery = SQLQuery(query, samplemethod='take', maxrows=120) assert_equals(sqlquery._pyspark_command(), Command(u'for {} in sqlContext.sql(u"""{} """).toJSON().take(120): print({}.encode("{}"))'\ .format(LONG_RANDOM_VARIABLE_NAME, query, LONG_RANDOM_VARIABLE_NAME, conf.pyspark_sql_encoding()))) assert_equals(sqlquery._scala_command(), Command(u'sqlContext.sql("""{}""").toJSON.take(120).foreach(println)'.format(query))) try: sqlquery._r_command() assert False except NotImplementedError: pass
def test_unicode_sql(): query = u"SELECT 'è'" sqlquery = SQLQuery(query, samplemethod='take', maxrows=120) assert_equals(sqlquery._pyspark_command(), Command(u'for {} in sqlContext.sql(u"""{} """).toJSON().take(120): print({}.encode("{}"))'\ .format(LONG_RANDOM_VARIABLE_NAME, query, LONG_RANDOM_VARIABLE_NAME, conf.pyspark_sql_encoding()))) assert_equals( sqlquery._scala_command(), Command(u'sqlContext.sql("""{}""").toJSON.take(120).foreach(println)'. format(query))) try: sqlquery._r_command() assert False except NotImplementedError: pass
def test_unicode_sql(): query = u"SELECT 'è'" longvar = LONG_RANDOM_VARIABLE_NAME sqlquery = SQLQuery(query, samplemethod='take', maxrows=120) assert_equals(sqlquery._pyspark_command("spark"), Command(u'import sys\nfor {} in spark.sql(u"""{} """).toJSON(use_unicode=(sys.version_info.major > 2)).take(120): print({})'\ .format(longvar, query, longvar))) assert_equals( sqlquery._scala_command("spark"), Command( u'spark.sql("""{}""").toJSON.take(120).foreach(println)'.format( query))) assert_equals( sqlquery._r_command("spark"), Command( u'for ({} in (jsonlite:::toJSON(take(sql("{}"),120)))) {{cat({})}}' .format(longvar, query, longvar)))
def test_unicode_sql(): query = u"SELECT 'è'" sqlquery = SQLQuery(query, samplemethod="take", maxrows=120) assert_equals( sqlquery._pyspark_command("spark"), Command( u'for {} in spark.sql(u"""{} """).toJSON().take(120): print({}.encode("{}"))'.format( LONG_RANDOM_VARIABLE_NAME, query, LONG_RANDOM_VARIABLE_NAME, conf.pyspark_sql_encoding() ) ), ) assert_equals( sqlquery._scala_command("spark"), Command(u'spark.sql("""{}""").toJSON.take(120).foreach(println)'.format(query)), ) assert_equals( sqlquery._r_command(), Command( u'for ({} in (jsonlite:::toJSON(take(sql("{}"),120)))) {{cat({})}}'.format( LONG_RANDOM_VARIABLE_NAME, query, LONG_RANDOM_VARIABLE_NAME ) ), )