def simple_example_2():
spark = SparkSession.builder.appName(
'simple-tensorframes-example-2').getOrCreate()
spark.sparkContext.setLogLevel('WARN')
rdd = [Row(y=[float(y), float(-y)]) for y in range(10)]
df = spark.createDataFrame(rdd)
df.show()
tfs.print_schema(df)
# Analyze first to find the dimensions of the vectors.
df2 = tfs.analyze(df)
tfs.print_schema(df2)
# Make a copy of the 'y' column: An inexpensive operation in Spark 2.0+.
df3 = df2.select(df2.y, df2.y.alias('z'))
# Execute the tensor graph.
with tf.Graph().as_default() as graph:
y_input = tfs.block(df3, 'y', tf_name='y_input')
z_input = tfs.block(df3, 'z', tf_name='z_input')
# Perform elementwise sum and minimum.
y = tf.reduce_sum(y_input, [0], name='y')
z = tf.reduce_min(z_input, [0], name='z')
(data_sum, data_min) = tfs.reduce_blocks([y, z], df3)
print('Elementwise sum: %s and minimum: %s' % (data_sum, data_min))
def run_one_step(dataframe, start_centers):
"""
Performs one iteration of K-Means.
This function takes a dataframe with dense feature vectors, a set of centroids, and returns
a new set of centroids along with the total distance of points to centroids.
This function calculates for each point the closest centroid and then aggregates the newly
formed clusters to find the new centroids.
This function uses Spark to distribute the aggregation amongst the node.
:param dataframe: a dataframe containing a column of features (an array of doubles)
:param start_centers: a k x m matrix with k the number of centroids and m the number of features
:return: a k x m matrix, and a positive double
"""
# The dimensions in the problem
(num_centroids, num_features) = np.shape(start_centers)
# For each feature vector, compute the nearest centroid and the distance to that centroid.
# The index of the nearest centroid is stored in the 'indexes' column.
# We also add a column of 1's that will be reduced later to count the number of elements in
# each cluster.
with tf.Graph().as_default() as g:
# The placeholder for the input: we use the block format
points = tf.placeholder(tf.double,
shape=[None, num_features],
name='features')
# The shape of the block is extracted as a TF variable.
num_points = tf.stack([tf.shape(points)[0]], name="num_points")
distances = tf_compute_distances(points, start_centers)
# The outputs of the program.
# The closest centroids are extracted.
indexes = tf.argmin(distances, 1, name='indexes')
# This could be done based on the indexes as well.
min_distances = tf.reduce_min(distances, 1, name='min_distances')
counts = tf.tile(tf.constant([1]), num_points, name='count')
df2 = tfs.map_blocks([indexes, counts, min_distances], dataframe)
# Perform the reduction: we regroup the points by their centroid indexes.
gb = df2.groupBy("indexes")
with tf.Graph().as_default() as g:
# Look at the documentation of tfs.aggregate for the naming conventions of the placeholders.
x_input = tfs.block(df2, "features", tf_name="features_input")
count_input = tfs.block(df2, "count", tf_name="count_input")
md_input = tfs.block(df2,
"min_distances",
tf_name="min_distances_input")
# Each operation is just the sum.
x = tf.reduce_sum(x_input, [0], name='features')
count = tf.reduce_sum(count_input, [0], name='count')
min_distances = tf.reduce_sum(md_input, [0], name='min_distances')
df3 = tfs.aggregate([x, count, min_distances], gb)
# Get the new centroids
df3_c = df3.collect()
# The new centroids.
new_centers = np.array(
[np.array(row.features) / row['count'] for row in df3_c])
total_distances = np.sum([row['min_distances'] for row in df3_c])
return (new_centers, total_distances)
def run_one_step(dataframe, start_centers):
"""
Performs one iteration of K-Means.
This function takes a dataframe with dense feature vectors, a set of centroids, and returns
a new set of centroids along with the total distance of points to centroids.
This function calculates for each point the closest centroid and then aggregates the newly
formed clusters to find the new centroids.
This function uses Spark to distribute the aggregation amongst the node.
:param dataframe: a dataframe containing a column of features (an array of doubles)
:param start_centers: a k x m matrix with k the number of centroids and m the number of features
:return: a k x m matrix, and a positive double
"""
# The dimensions in the problem
(num_centroids, num_features) = np.shape(start_centers)
# For each feature vector, compute the nearest centroid and the distance to that centroid.
# The index of the nearest centroid is stored in the 'indexes' column.
# We also add a column of 1's that will be reduced later to count the number of elements in
# each cluster.
with tf.Graph().as_default() as g:
# The placeholder for the input: we use the block format
points = tf.placeholder(tf.double, shape=[None, num_features], name='features')
# The shape of the block is extracted as a TF variable.
num_points = tf.stack([tf.shape(points)[0]], name="num_points")
distances = tf_compute_distances(points, start_centers)
# The outputs of the program.
# The closest centroids are extracted.
indexes = tf.argmin(distances, 1, name='indexes')
# This could be done based on the indexes as well.
min_distances = tf.reduce_min(distances, 1, name='min_distances')
counts = tf.tile(tf.constant([1]), num_points, name='count')
df2 = tfs.map_blocks([indexes, counts, min_distances], dataframe)
# Perform the reduction: we regroup the points by their centroid indexes.
gb = df2.groupBy("indexes")
with tf.Graph().as_default() as g:
# Look at the documentation of tfs.aggregate for the naming conventions of the placeholders.
x_input = tfs.block(df2, "features", tf_name="features_input")
count_input = tfs.block(df2, "count", tf_name="count_input")
md_input = tfs.block(df2, "min_distances", tf_name="min_distances_input")
# Each operation is just the sum.
x = tf.reduce_sum(x_input, [0], name='features')
count = tf.reduce_sum(count_input, [0], name='count')
min_distances = tf.reduce_sum(md_input, [0], name='min_distances')
df3 = tfs.aggregate([x, count, min_distances], gb)
# Get the new centroids
df3_c = df3.collect()
# The new centroids.
new_centers = np.array([np.array(row.features) / row['count'] for row in df3_c])
total_distances = np.sum([row['min_distances'] for row in df3_c])
return (new_centers, total_distances)
def test_groupby_1(self):
data = [Row(x=float(x), key=str(x % 2)) for x in range(4)]
df = self.sql.createDataFrame(data)
gb = df.groupBy("key")
with tf.Graph().as_default() as g:
x_input = tfs.block(df, "x", tf_name="x_input")
x = tf.reduce_sum(x_input, [0], name='x')
df2 = tfs.aggregate(x, gb)
data2 = df2.collect()
assert data2 == [Row(key='0', x=2.0), Row(key='1', x=4.0)], data2
def simple_example_1():
spark = SparkSession.builder.appName(
'simple-tensorframes-example-1').getOrCreate()
spark.sparkContext.setLogLevel('WARN')
rdd = [Row(x=float(x)) for x in range(10)]
df = spark.createDataFrame(rdd)
df.show()
# Execute the tensor graph.
with tf.Graph().as_default() as graph:
# A block placeholder.
x = tfs.block(df, 'x')
z = tf.add(x, 3, name='z')
# Tensor -> dataframe.
df2 = tfs.map_blocks(z, df)
print('z =', z)
df2.show()
t1 = tf.tile(t1a, t1b)
t2a = tf.expand_dims(squares, 1)
t2b = tf.pack([1, k])
t2 = tf.tile(t2a, t2b)
distances = t1 + t2 - 2 * prods
# TODO cast
indexes = tf.argmin(distances, 1, name='indexes')
min_distances = tf.reduce_min(distances, 1, name='min_distances')
counts = tf.tile(tf.constant([1]), tf.pack([num_points]), name='count')
df2 = tfs.map_blocks([indexes, counts, min_distances], df0)
# Perform the reduction
gb = df2.groupBy("indexes")
with tf.Graph().as_default() as g:
# Look at the documentation of tfs.aggregate for the naming conventions of the placeholders.
x_input = tfs.block(df2, "features", tf_name="features_input")
count_input = tfs.block(df2, "count", tf_name="count_input")
md_input = tfs.block(df2, "min_distances", tf_name="min_distances_input")
x = tf.reduce_sum(x_input, [0], name='features')
count = tf.reduce_sum(count_input, [0], name='count')
min_distances = tf.reduce_sum(md_input, [0], name='min_distances')
df3 = tfs.aggregate([x, count, min_distances], gb)
# Get the new centroids
df3_c = df3.collect()
new_centers = np.array([np.array(row.features) / row['count'] for row in df3_c])
total_distances = np.sum([row['min_distances'] for row in df3_c])
def run_one_step(dataframe, start_centers):
"""
data = [Row(y=[float(y), float(-y)]) for y in range(10)]
df = sqlContext.createDataFrame(data)
# Because the dataframe contains vectors, we need to analyze it first to find the
# dimensions of the vectors.
df2 = tfs.analyze(df)
# The information gathered by TF can be printed to check the content:
tfs.print_schema(df2)
# TF has inferred that y contains vectors of size 2
# root
# |-- y: array (nullable = false) DoubleType[?,2]
# Let's use the analyzed dataframe to compute the sum and the elementwise minimum
# of all the vectors:
# First, let's make a copy of the 'y' column. This will be very cheap in Spark 2.0+
df3 = df2.select(df2.y, df2.y.alias("z"))
with tf.Graph().as_default() as g:
# The placeholders. Note the special name that end with '_input':
y_input = tfs.block(df3, 'y', tf_name="y_input")
z_input = tfs.block(df3, 'z', tf_name="z_input")
y = tf.reduce_sum(y_input, [0], name='y')
z = tf.reduce_min(z_input, [0], name='z')
# The resulting dataframe
(data_sum, data_min) = tfs.reduce_blocks([y, z], df3)
# The final results are numpy arrays:
print data_sum
# [45.0, -45.0]
print data_min
# [0.0, -9.0]
import tensorframes as tfs
import tensorflow as tf
from pyspark.sql import Row
from pyspark.sql.functions import *
from pyspark.sql.types import DoubleType, IntegerType, LongType, FloatType
from tensorframes.core import _java_api
japi = _java_api()
_java_api().initialize_logging()
data = [Row(x=float(x), key=str(x / 3)) for x in range(1, 6)]
df = sqlContext.createDataFrame(data)
tfs.block(df, "x")
data = [Row(x=float(x), key=str(x / 3)) for x in range(1, 6)]
df = sqlContext.createDataFrame(data)
gb = df.groupBy("key")
with tf.Graph().as_default() as g:
x_input = tfs.block(df, "x", tf_name="x_input")
x = tf.reduce_sum(x_input, [0], name='x')
df2 = tfs.aggregate(x, gb)
data = [Row(x=float(x)) for x in range(5)]
df = sqlContext.createDataFrame(data)
with tf.Graph().as_default() as g:
# The placeholder that corresponds to column 'x'
x = tf.placeholder(tf.double, shape=[None], name="x")
# The output that adds 3 to x
z = tf.add(x, 3, name='z')
# The resulting dataframe
# The input data
data = [Row(x=[float(x), float(2 * x)], key=str(x % 2)) for x in range(1, 6)]
df = sqlContext.createDataFrame(data)
df = tfs.analyze(sqlContext.createDataFrame(data))
# The geometric mean:
# TODO(tjh) make a test out of this, it found some bugs
# - non numeric columns (string)
# - unused columns
# - output that has a child
col_name = "x"
col_key = "key"
with tf.Graph().as_default() as g:
x = tfs.block(df, col_name)
invs = tf.inv(tf.to_double(x), name="invs")
df2 = tfs.map_blocks([invs, tf.ones_like(invs, name="count")], df)
# The geometric mean
gb = df2.select(col_key, "invs", "count").groupBy("key")
with tf.Graph().as_default() as g:
x_input = tfs.block(df2, "invs", tf_name="invs_input")
count_input = tfs.block(df2, "invs", tf_name="count_input")
x = tf.reduce_sum(x_input, [0], name='invs')
count = tf.reduce_sum(count_input, [0], name='count')
df3 = tfs.aggregate([x, count], gb)
with tf.Graph().as_default() as g:
invs = tfs.block(df2, "invs")
_java_api().initialize_logging()
# The input data
data = [Row(x=[float(x), float(2 * x)], key=str(x % 2)) for x in range(1, 6)]
df = sqlContext.createDataFrame(data)
df = tfs.analyze(sqlContext.createDataFrame(data))
# The geometric mean:
# TODO(tjh) make a test out of this, it found some bugs
# - non numeric columns (string)
# - unused columns
# - output that has a child
col_name = "x"
col_key = "key"
with tf.Graph().as_default() as g:
x = tfs.block(df, col_name)
invs = tf.inv(tf.to_double(x), name="invs")
df2 = tfs.map_blocks([invs, tf.ones_like(invs, name="count")], df)
# The geometric mean
gb = df2.select(col_key, "invs", "count").groupBy("key")
with tf.Graph().as_default() as g:
x_input = tfs.block(df2, "invs", tf_name="invs_input")
count_input = tfs.block(df2, "invs", tf_name="count_input")
x = tf.reduce_sum(x_input, [0], name='invs')
count = tf.reduce_sum(count_input, [0], name='count')
df3 = tfs.aggregate([x, count], gb)
with tf.Graph().as_default() as g:
invs = tfs.block(df2, "invs")
count = tfs.block(df2, "count")
t1 = tf.tile(t1a, t1b)
t2a = tf.expand_dims(squares, 1)
t2b = tf.pack([1, k])
t2 = tf.tile(t2a, t2b)
distances = t1 + t2 - 2 * prods
# TODO cast
indexes = tf.argmin(distances, 1, name='indexes')
min_distances = tf.reduce_min(distances, 1, name='min_distances')
counts = tf.tile(tf.constant([1]), tf.pack([num_points]), name='count')
df2 = tfs.map_blocks([indexes, counts, min_distances], df0)
# Perform the reduction
gb = df2.groupBy("indexes")
with tf.Graph().as_default() as g:
# Look at the documentation of tfs.aggregate for the naming conventions of the placeholders.
x_input = tfs.block(df2, "features", tf_name="features_input")
count_input = tfs.block(df2, "count", tf_name="count_input")
md_input = tfs.block(df2, "min_distances", tf_name="min_distances_input")
x = tf.reduce_sum(x_input, [0], name='features')
count = tf.reduce_sum(count_input, [0], name='count')
min_distances = tf.reduce_sum(md_input, [0], name='min_distances')
df3 = tfs.aggregate([x, count, min_distances], gb)
# Get the new centroids
df3_c = df3.collect()
new_centers = np.array(
[np.array(row.features) / row['count'] for row in df3_c])
total_distances = np.sum([row['min_distances'] for row in df3_c])
def run_one_step(dataframe, start_centers):
data = [Row(y=[float(y), float(-y)]) for y in range(10)]
df = sqlContext.createDataFrame(data)
# Because the dataframe contains vectors, we need to analyze it first to find the
# dimensions of the vectors.
df2 = tfs.analyze(df)
# The information gathered by TF can be printed to check the content:
tfs.print_schema(df2)
# TF has inferred that y contains vectors of size 2
# root
# |-- y: array (nullable = false) DoubleType[?,2]
# Let's use the analyzed dataframe to compute the sum and the elementwise minimum
# of all the vectors:
# First, let's make a copy of the 'y' column. This will be very cheap in Spark 2.0+
df3 = df2.select(df2.y, df2.y.alias("z"))
with tf.Graph().as_default() as g:
# The placeholders. Note the special name that end with '_input':
y_input = tfs.block(df3, 'y', tf_name="y_input")
z_input = tfs.block(df3, 'z', tf_name="z_input")
y = tf.reduce_sum(y_input, [0], name='y')
z = tf.reduce_min(z_input, [0], name='z')
# The resulting dataframe
(data_sum, data_min) = tfs.reduce_blocks([y, z], df3)
# The final results are numpy arrays:
print data_sum
# [45.0, -45.0]
print data_min
# [0.0, -9.0]
import tensorflow as tf
import tensorframes as tfs
from pyspark.sql import Row
data = [Row(x=float(x)) for x in range(10)]
df = sqlContext.createDataFrame(data)
with tf.Graph().as_default() as g:
# The TensorFlow placeholder that corresponds to column 'x'.
# The shape of the placeholder is automatically inferred from the DataFrame.
x = tfs.block(df, "x")
# The output that adds 3 to x
z = tf.add(x, 3, name='z')
# The resulting dataframe
df2 = tfs.map_blocks(z, df)
# The transform is lazy as for most DataFrame operations. This will trigger it:
df2.collect()
def run_one_step(dataframe, start_centers):
"""
Performs one iteration of K-Means.
This function takes a dataframe with dense feature vectors, a set of centroids, and returns
a new set of centroids along with the total distance of points to centroids.
This function calculates for each point the closest centroid and then aggregates the newly
formed clusters to find the new centroids.
:param dataframe: a dataframe containing a column of features (an array of doubles)
:param start_centers: a k x m matrix with k the number of centroids and m the number of features
:return: a k x m matrix, and a positive double
"""
# The dimensions in the problem
(num_centroids, num_features) = np.shape(start_centers)
# For each feature vector, compute the nearest centroid and the distance to that centroid.
# The index of the nearest centroid is stored in the 'indexes' column.
# We also add a column of 1's that will be reduced later to count the number of elements in
# each cluster.
with tf.Graph().as_default() as g:
# The placeholder for the input: we use the block format
points = tf.placeholder(tf.double, shape=[None, num_features], name='features')
# The shape of the block is extracted as a TF variable.
num_points = tf.shape(points)[0]
# The centers are embedded in the TF program.
centers = tf.constant(start_centers)
# Computation of the minimum distance. This is a standard implementation that follows
# what MLlib does.
squares = tf.reduce_sum(tf.square(points), reduction_indices=1)
center_squares = tf.reduce_sum(tf.square(centers), reduction_indices=1)
prods = tf.matmul(points, centers, transpose_b = True)
# This code simply expresses two outer products: center_squares * ones(num_points)
# and ones(num_centroids) * squares
t1a = tf.expand_dims(center_squares, 0)
t1b = tf.pack([num_points, 1])
t1 = tf.tile(t1a, t1b)
t2a = tf.expand_dims(squares, 1)
t2b = tf.pack([1, num_centroids])
t2 = tf.tile(t2a, t2b)
distances = t1 + t2 - 2 * prods
# The outputs of the program.
# The closest centroids are extracted.
indexes = tf.argmin(distances, 1, name='indexes')
# This could be done based on the indexes as well.
min_distances = tf.reduce_min(distances, 1, name='min_distances')
counts = tf.tile(tf.constant([1]), tf.pack([num_points]), name='count')
df2 = tfs.map_blocks([indexes, counts, min_distances], dataframe)
# Perform the reduction: we regroup the point by their centroid indexes.
gb = df2.groupBy("indexes")
with tf.Graph().as_default() as g:
# Look at the documentation of tfs.aggregate for the naming conventions of the placeholders.
x_input = tfs.block(df2, "features", tf_name="features_input")
count_input = tfs.block(df2, "count", tf_name="count_input")
md_input = tfs.block(df2, "min_distances", tf_name="min_distances_input")
# Each operation is just the sum.
x = tf.reduce_sum(x_input, [0], name='features')
count = tf.reduce_sum(count_input, [0], name='count')
min_distances = tf.reduce_sum(md_input, [0], name='min_distances')
df3 = tfs.aggregate([x, count, min_distances], gb)
# Get the new centroids
df3_c = df3.collect()
# The new centroids.
new_centers = np.array([np.array(row.features) / row['count'] for row in df3_c])
total_distances = np.sum([row['min_distances'] for row in df3_c])
return (new_centers, total_distances)
import tensorframes as tfs
import tensorflow as tf
from pyspark.sql import Row
from pyspark.sql.functions import *
from pyspark.sql.types import DoubleType, IntegerType, LongType, FloatType
from tensorframes.core import _java_api
japi = _java_api()
_java_api().initialize_logging()
data = [Row(x=float(x), key=str(x / 3)) for x in range(1, 6)]
df = sqlContext.createDataFrame(data)
tfs.block(df, "x")
data = [Row(x=float(x), key=str(x / 3)) for x in range(1, 6)]
df = sqlContext.createDataFrame(data)
gb = df.groupBy("key")
with tf.Graph().as_default() as g:
x_input = tfs.block(df, "x", tf_name="x_input")
x = tf.reduce_sum(x_input, [0], name='x')
df2 = tfs.aggregate(x, gb)
data = [Row(x=float(x)) for x in range(5)]
df = sqlContext.createDataFrame(data)
with tf.Graph().as_default() as g:
# The placeholder that corresponds to column 'x'
x = tf.placeholder(tf.double, shape=[None], name="x")
# The output that adds 3 to x
z = tf.add(x, 3, name='z')
# The resulting dataframe
df2 = tfs.map_blocks(z, df)