def percentile_approx(col, percentage, accuracy=10000):
"""
Returns the approximate percentile value of numeric column col at the given percentage.
The value of percentage must be between 0.0 and 1.0.
The accuracy parameter (default: 10000)
is a positive numeric literal which controls approximation accuracy at the cost of memory.
Higher value of accuracy yields better accuracy, 1.0/accuracy is the relative error
of the approximation.
When percentage is an array, each value of the percentage array must be between 0.0 and 1.0.
In this case, returns the approximate percentile array of column col
at the given percentage array.
Ported from Spark 3.1.
"""
sc = SparkContext._active_spark_context
if isinstance(percentage, (list, tuple)):
# A local list
percentage = sc._jvm.functions.array(
_to_seq(sc, [_create_column_from_literal(x) for x in percentage]))
elif isinstance(percentage, Column):
# Already a Column
percentage = _to_java_column(percentage)
else:
# Probably scalar
percentage = _create_column_from_literal(percentage)
accuracy = (_to_java_column(accuracy) if isinstance(accuracy, Column) else
_create_column_from_literal(accuracy))
return _call_udf(sc, "percentile_approx", _to_java_column(col), percentage,
accuracy)
def repeat(col: Column, n: Union[int, Column]) -> Column:
"""
Repeats a string column n times, and returns it as a new string column.
"""
sc = SparkContext._active_spark_context
n = _to_java_column(n) if isinstance(n, Column) else _create_column_from_literal(n)
return _call_udf(sc, "repeat", _to_java_column(col), n)
def date_part(field: Union[str, Column], source: Column) -> Column:
"""
Extracts a part of the date/timestamp or interval source.
"""
sc = SparkContext._active_spark_context
field = (_to_java_column(field) if isinstance(field, Column) else
_create_column_from_literal(field))
return _call_udf(sc, "date_part", field, _to_java_column(source))
def hlike(col, regexps):
"""Hyperscan regex like. Returns true if col matches one of regexps
:param col: Column
:param regexps: list of patterns to match
:return: boolean column with match result
"""
sc = SparkContext._active_spark_context
patterns = sc._jvm.functions.array(_to_seq(sc, [
_create_column_from_literal(x) for x in regexps
]))
return Column(sc._jvm.ru.napalabs.spark.hscan.functions.hlike(_to_java_column(col), patterns))
def _(col, other):
# convert other to a Row if necessary
jcol = col._jc
sc = SparkContext._active_spark_context
loader = sc._jvm.Thread.currentThread().getContextClassLoader()
wclass = loader.loadClass(name)
expr_class = sc._jvm.java.lang.Object
expr_array = sc._gateway.new_array(expr_class, 2)
expr_array[0] = jcol.expr()
expr_array[1] = _create_column_from_literal(other)
w = wclass.getConstructors()[0].newInstance(expr_array)
wcol = sc._jvm.org.apache.spark.sql.Column(w)
return Column(wcol)
def _(col, other):
# convert other to a Row if necessary
jcol = col._jc
sc = SparkContext._active_spark_context
loader = sc._jvm.Thread.currentThread().getContextClassLoader()
wclass = loader.loadClass(name)
expr_class = sc._jvm.java.lang.Object
expr_array = sc._gateway.new_array(expr_class, 2)
expr_array[0] = jcol.expr()
expr_array[1] = _create_column_from_literal(other)
w = wclass.getConstructors()[0].newInstance(expr_array)
wcol = sc._jvm.org.apache.spark.sql.Column(w)
return Column(wcol)