def get_image_features_for_layer(model_name,
layer_num_from_top,
starting_layer_df,
starting_layer,
joined=True):
"""
Staged CNN inference.
:param model_name: CNN model name (AlexNet, VGG16, ResNet50)
:param layer_num_from_top: Layer index from the top most layer of the CNN
:param starting_layer_df: Input DataFrame for the staged inference
:param starting_layer: Starting layer index. Zero means raw images
:param joined: Boolean. Whether the input DataFrame is already joined with structured features.
:return: DataFrame
"""
g = tf.Graph()
with g.as_default():
input_buffer = tf.placeholder(tf.string, [], 'input_layer')
input = tf.decode_raw(input_buffer, tf.float32)
if model_name == 'alexnet':
input_layer_name = AlexNet.get_transfer_learning_layer_names(
)[starting_layer]
model = AlexNet(input,
input_layer_name=input_layer_name,
model_name='alexnet')
elif model_name == 'resnet50':
input_layer_name = ResNet50.get_transfer_learning_layer_names(
)[starting_layer]
model = ResNet50(input,
input_layer_name=input_layer_name,
model_name='resnet50')
elif model_name == 'vgg16':
input_layer_name = VGG16.get_transfer_learning_layer_names(
)[starting_layer]
model = VGG16(input,
input_layer_name=input_layer_name,
model_name='vgg16')
output = tf.reshape(
model.transfer_layers[layer_num_from_top],
[-1, model.transfer_layer_flattened_sizes[layer_num_from_top]],
name='image_features')
if joined:
image_features_df = tfs.map_rows(output, starting_layer_df).select(
col('id'), col('image_features'), col('features'),
col('label'))
else:
image_features_df = tfs.map_rows(output, starting_layer_df).select(
col('id'), col('image_features'))
return image_features_df, model.transfer_layers_shapes[layer_num_from_top]
def test_map_rows_3(self):
data = [dict(x=float(x)) for x in range(5)]
df = pd.DataFrame(data)
with tf.Graph().as_default() as g:
# The placeholder that corresponds to column 'x'
x = tf.placeholder(tf.double, shape=[None], name="x")
# The output that adds 3 to x
z = tf.add(x, 3, name='z')
# The resulting dataframe
df2 = tfs.map_rows(z, df)
data2 = df2
assert data2.z[0] == 3.0, data2
def test_map_rows_2(self):
data = [Row(y=float(y)) for y in range(5)]
df = self.sql.createDataFrame(data)
with tf.Graph().as_default() as g:
# The placeholder that corresponds to column 'x'
x = tf.placeholder(tf.double, shape=[], name="x")
# The output that adds 3 to x
z = tf.add(x, 3, name='z')
# The resulting dataframe
df2 = tfs.map_rows(z, df, feed_dict={'x': 'y'})
data2 = df2.collect()
assert data2[0].z == 3.0, data2
def test_map_rows_1(self):
data = [Row(x=float(x)) for x in range(5)]
df = self.sql.createDataFrame(data)
with tf.Graph().as_default() as g:
# The placeholder that corresponds to column 'x'
x = tf.placeholder(tf.double, shape=[], name="x")
# The output that adds 3 to x
z = tf.add(x, 3, name='z')
# The resulting dataframe
df2 = tfs.map_rows(z, df)
data2 = df2.collect()
assert data2[0].z == 3.0, data2
def get_all_image_features(model_name,
joined_df,
num_layers_to_explore,
cnn_input_layer_index=0):
"""
Bulk cnn inference
:param model_name: CNN model name (AlexNet, VGG16, ResNet50)
:param joined_df: Input DataFrame containing structured features and raw images
:param num_layers_to_explore: Number of layer from the top of the CNN to be explored
:param cnn_input_layer_index: Starting layer index. Zero means raw images
:return: DataFrame
"""
g = tf.Graph()
with g.as_default():
image_buffer = tf.placeholder(tf.string, [], 'input_layer')
image = tf.decode_raw(image_buffer, tf.float32)
if model_name == 'alexnet':
model = AlexNet(
image,
input_layer_name=AlexNet.get_transfer_learning_layer_names()
[cnn_input_layer_index],
model_name='alexnet')
elif model_name == 'resnet50':
model = ResNet50(
image,
input_layer_name=ResNet50.get_transfer_learning_layer_names()
[cnn_input_layer_index],
model_name='resnet50')
elif model_name == 'vgg16':
model = VGG16(
image,
input_layer_name=VGG16.get_transfer_learning_layer_names()
[cnn_input_layer_index],
model_name='vgg16')
concat_layers = [
tf.reshape(model.transfer_layers[-1 * i],
[-1, model.transfer_layer_flattened_sizes[-1 * i]])
for i in range(1, num_layers_to_explore + 1)
]
output = tf.concat(concat_layers, 1, name='image_features')
cumulative_sizes = [0]
for i in range(1, num_layers_to_explore + 1):
cumulative_sizes.append(cumulative_sizes[i - 1] +
model.transfer_layer_flattened_sizes[-1 *
i])
image_features_df = tfs.map_rows(output, joined_df)
return image_features_df, cumulative_sizes, model.transfer_layers_shapes[
-1 * num_layers_to_explore:]
def _transform(self, dataset):
graph = self.getGraph()
composed_graph = self._addReshapeLayers(graph,
self._getImageDtype(dataset))
final_graph = self._stripGraph(composed_graph)
with final_graph.as_default():
image = dataset[self.getInputCol()]
image_df_exploded = (dataset.withColumn(
"__sdl_image_height", image.height).withColumn(
"__sdl_image_width", image.width).withColumn(
"__sdl_image_nchannels",
image.nChannels).withColumn("__sdl_image_data",
image.data))
final_output_name = self._getFinalOutputTensorName()
output_tensor = final_graph.get_tensor_by_name(final_output_name)
final_df = (tfs.map_rows(
[output_tensor],
image_df_exploded,
feed_dict={
"height": "__sdl_image_height",
"width": "__sdl_image_width",
"num_channels": "__sdl_image_nchannels",
"image_buffer": "__sdl_image_data"
}).drop("__sdl_image_height", "__sdl_image_width",
"__sdl_image_nchannels", "__sdl_image_data"))
tfs_output_name = tfx.op_name(output_tensor, final_graph)
original_output_name = self._getOriginalOutputTensorName()
output_shape = final_graph.get_tensor_by_name(
original_output_name).shape
output_mode = self.getOrDefault(self.outputMode)
# TODO: support non-1d tensors (return np.array).
if output_mode == "image":
return self._convertOutputToImage(final_df, tfs_output_name,
output_shape)
else:
assert output_mode == "vector", "Unknown output mode: %s" % output_mode
return self._convertOutputToVector(final_df, tfs_output_name)
def _transform(self, dataset):
graph = self.getGraph()
composed_graph = self._addReshapeLayers(graph, self._getImageDtype(dataset))
final_graph = self._stripGraph(composed_graph)
with final_graph.as_default(): # pylint: disable=not-context-manager
image = dataset[self.getInputCol()]
image_df_exploded = (dataset
.withColumn("__sdl_image_height", image.height)
.withColumn("__sdl_image_width", image.width)
.withColumn("__sdl_image_nchannels", image.nChannels)
.withColumn("__sdl_image_data", image.data)
) # yapf: disable
final_output_name = self._getFinalOutputTensorName()
output_tensor = final_graph.get_tensor_by_name(final_output_name)
final_df = (
tfs.map_rows([output_tensor], image_df_exploded,
feed_dict={
"height": "__sdl_image_height",
"width": "__sdl_image_width",
"num_channels": "__sdl_image_nchannels",
"image_buffer": "__sdl_image_data"})
.drop("__sdl_image_height", "__sdl_image_width", "__sdl_image_nchannels",
"__sdl_image_data")
) # yapf: disable
tfs_output_name = tfx.op_name(output_tensor, final_graph)
original_output_name = self._getOriginalOutputTensorName()
output_shape = final_graph.get_tensor_by_name(original_output_name).shape
output_mode = self.getOrDefault(self.outputMode)
# TODO: support non-1d tensors (return np.array).
if output_mode == "image":
return self._convertOutputToImage(final_df, tfs_output_name, output_shape)
else:
assert output_mode == "vector", "Unknown output mode: %s" % output_mode
return self._convertOutputToVector(final_df, tfs_output_name)
sc.setLogLevel('INFO')
with g2.as_default():
index_output = tf.identity(g2.get_tensor_by_name('top_predictions:1'), name="index")
value_output = tf.identity(g2.get_tensor_by_name('top_predictions:0'), name="value")
raw_images_miscast = sc.binaryFiles("file:"+image_path) # file:
raw_images = raw_images_miscast.map(lambda x: (x[0], bytearray(x[1])))
df = spark.createDataFrame(raw_images).toDF('image_uri', 'image_data')
df
with g2.as_default():
pred_df = tfs.map_rows([index_output, value_output], df, feed_dict={'DecodeJpeg/contents':'image_data'})
pred_df.select('index', 'value').head()
## Tim
#
# raw_images_miscast = sc.binaryFiles("file:/media/sf_tensorflow_mount/101_ObjectCategories/ant/")
# raw_images = raw_images_miscast.map(lambda x: (x[0], bytearray(x[1])))
#
# df = spark.createDataFrame(raw_images).toDF('image_uri', 'image_data')
# df
# with g2.as_default():
# pred_df = tfs.map_rows([index_output, value_output], df, feed_dict={'DecodeJpeg/contents':'image_data'})
#
#refs : https://www.youtube.com/watch?v=08mrnJxcIWw
# https://github.com/databricks/tensorframes
#Spark version 2.1.1
#bin/pyspark --master spark://vishnu-macbook-pro:7077 --packages databricks:tensorframes:0.2.8-s_2.11
import tensorflow as tf
import tensorframes as tfs
df = spark.createDataFrame(zip(range(0, 10), range(1, 11))).toDF("x", "y")
df.show(10)
x = tfs.row(df, "x")
y = tfs.row(df, "y")
output = tf.add(x, y, name="out")
df2 = tfs.map_rows(output, df)
df2.show()
with g2.as_default():
index_output = tf.identity(g2.get_tensor_by_name('top_predictions:1'),
name="index")
value_output = tf.identity(g2.get_tensor_by_name('top_predictions:0'),
name="value")
raw_images_miscast = sc.binaryFiles("file:" + image_path) # file:
raw_images = raw_images_miscast.map(lambda x: (x[0], bytearray(x[1])))
df = spark.createDataFrame(raw_images).toDF('image_uri', 'image_data')
df
with g2.as_default():
pred_df = tfs.map_rows([index_output, value_output],
df,
feed_dict={'DecodeJpeg/contents': 'image_data'})
pred_df.select('index', 'value').head()
## Tim
#
# raw_images_miscast = sc.binaryFiles("file:/media/sf_tensorflow_mount/101_ObjectCategories/ant/")
# raw_images = raw_images_miscast.map(lambda x: (x[0], bytearray(x[1])))
#
# df = spark.createDataFrame(raw_images).toDF('image_uri', 'image_data')
# df
# with g2.as_default():
# pred_df = tfs.map_rows([index_output, value_output], df, feed_dict={'DecodeJpeg/contents':'image_data'})
#
# pred_df.select('index', 'value').show()
