def test_scala_livy_sql_options():
    query = "abc"

    sqlquery = SQLQuery(query, samplemethod="take", maxrows=100)
    assert_equals(
        sqlquery._scala_command("sqlContext"),
        Command('sqlContext.sql("""{}""").toJSON.take(100).foreach(println)'.format(query)),
    )

    sqlquery = SQLQuery(query, samplemethod="take", maxrows=-1)
    assert_equals(
        sqlquery._scala_command("sqlContext"),
        Command('sqlContext.sql("""{}""").toJSON.collect.foreach(println)'.format(query)),
    )

    sqlquery = SQLQuery(query, samplemethod="sample", samplefraction=0.25, maxrows=-1)
    assert_equals(
        sqlquery._scala_command("sqlContext"),
        Command('sqlContext.sql("""{}""").toJSON.sample(false, 0.25).collect.foreach(println)'.format(query)),
    )

    sqlquery = SQLQuery(query, samplemethod="sample", samplefraction=0.33, maxrows=3234)
    assert_equals(
        sqlquery._scala_command("sqlContext"),
        Command('sqlContext.sql("""{}""").toJSON.sample(false, 0.33).take(3234).foreach(println)'.format(query)),
    )
Exemple #2
0
def test_scala_livy_sql_options():
    query = "abc"

    sqlquery = SQLQuery(query, samplemethod='take', maxrows=100)
    assert_equals(
        sqlquery._scala_command(),
        Command('sqlContext.sql("""{}""").toJSON.take(100).foreach(println)'.
                format(query)))

    sqlquery = SQLQuery(query, samplemethod='take', maxrows=-1)
    assert_equals(
        sqlquery._scala_command(),
        Command(
            'sqlContext.sql("""{}""").toJSON.collect.foreach(println)'.format(
                query)))

    sqlquery = SQLQuery(query,
                        samplemethod='sample',
                        samplefraction=0.25,
                        maxrows=-1)
    assert_equals(
        sqlquery._scala_command(),
        Command(
            'sqlContext.sql("""{}""").toJSON.sample(false, 0.25).collect.foreach(println)'
            .format(query)))

    sqlquery = SQLQuery(query,
                        samplemethod='sample',
                        samplefraction=0.33,
                        maxrows=3234)
    assert_equals(
        sqlquery._scala_command(),
        Command(
            'sqlContext.sql("""{}""").toJSON.sample(false, 0.33).take(3234).foreach(println)'
            .format(query)))
Exemple #3
0
def test_unicode_sql():
    query = u"SELECT 'è'"

    sqlquery = SQLQuery(query, samplemethod='take', maxrows=120)
    assert_equals(sqlquery._pyspark_command("spark"),
                  Command(u'for {} in spark.sql(u"""{} """).toJSON().take(120): print({}.encode("{}"))'\
                          .format(LONG_RANDOM_VARIABLE_NAME, query,
                                  LONG_RANDOM_VARIABLE_NAME, conf.pyspark_dataframe_encoding())))
    assert_equals(sqlquery._scala_command("spark"),
                  Command(u'spark.sql("""{}""").toJSON.take(120).foreach(println)'.format(query)))
    assert_equals(sqlquery._r_command("spark"),
                  Command(u'for ({} in (jsonlite:::toJSON(take(sql("{}"),120)))) {{cat({})}}'.format(LONG_RANDOM_VARIABLE_NAME, query, LONG_RANDOM_VARIABLE_NAME)))
def test_unicode_sql():
    query = u"SELECT 'è'"

    sqlquery = SQLQuery(query, samplemethod='take', maxrows=120)
    assert_equals(sqlquery._pyspark_command(),
                  Command(u'for {} in sqlContext.sql(u"""{} """).toJSON().take(120): print({}.encode("{}"))'\
                          .format(LONG_RANDOM_VARIABLE_NAME, query,
                                  LONG_RANDOM_VARIABLE_NAME, conf.pyspark_sql_encoding())))
    assert_equals(sqlquery._scala_command(),
                  Command(u'sqlContext.sql("""{}""").toJSON.take(120).foreach(println)'.format(query)))

    try:
        sqlquery._r_command()
        assert False
    except NotImplementedError:
        pass
Exemple #5
0
def test_unicode_sql():
    query = u"SELECT 'è'"

    sqlquery = SQLQuery(query, samplemethod='take', maxrows=120)
    assert_equals(sqlquery._pyspark_command(),
                  Command(u'for {} in sqlContext.sql(u"""{} """).toJSON().take(120): print({}.encode("{}"))'\
                          .format(LONG_RANDOM_VARIABLE_NAME, query,
                                  LONG_RANDOM_VARIABLE_NAME, conf.pyspark_sql_encoding())))
    assert_equals(
        sqlquery._scala_command(),
        Command(u'sqlContext.sql("""{}""").toJSON.take(120).foreach(println)'.
                format(query)))

    try:
        sqlquery._r_command()
        assert False
    except NotImplementedError:
        pass
Exemple #6
0
def test_unicode_sql():
    query = u"SELECT 'è'"
    longvar = LONG_RANDOM_VARIABLE_NAME

    sqlquery = SQLQuery(query, samplemethod='take', maxrows=120)
    assert_equals(sqlquery._pyspark_command("spark"),
                  Command(u'import sys\nfor {} in spark.sql(u"""{} """).toJSON(use_unicode=(sys.version_info.major > 2)).take(120): print({})'\
                          .format(longvar, query,
                                  longvar)))
    assert_equals(
        sqlquery._scala_command("spark"),
        Command(
            u'spark.sql("""{}""").toJSON.take(120).foreach(println)'.format(
                query)))
    assert_equals(
        sqlquery._r_command("spark"),
        Command(
            u'for ({} in (jsonlite:::toJSON(take(sql("{}"),120)))) {{cat({})}}'
            .format(longvar, query, longvar)))
def test_unicode_sql():
    query = u"SELECT 'è'"

    sqlquery = SQLQuery(query, samplemethod="take", maxrows=120)
    assert_equals(
        sqlquery._pyspark_command("spark"),
        Command(
            u'for {} in spark.sql(u"""{} """).toJSON().take(120): print({}.encode("{}"))'.format(
                LONG_RANDOM_VARIABLE_NAME, query, LONG_RANDOM_VARIABLE_NAME, conf.pyspark_sql_encoding()
            )
        ),
    )
    assert_equals(
        sqlquery._scala_command("spark"),
        Command(u'spark.sql("""{}""").toJSON.take(120).foreach(println)'.format(query)),
    )
    assert_equals(
        sqlquery._r_command(),
        Command(
            u'for ({} in (jsonlite:::toJSON(take(sql("{}"),120)))) {{cat({})}}'.format(
                LONG_RANDOM_VARIABLE_NAME, query, LONG_RANDOM_VARIABLE_NAME
            )
        ),
    )