def getSampleImageDF():
return imageIO.readImagesWithCustomFn(path=_getSampleJPEGDir(),
decode_f=imageIO.PIL_decode)
# conf = spark.sparkContext._conf.setAll([('spark.executor.memory', '4g'), ('spark.app.name', 'Spark Updated Conf'), (
# 'spark.executor.cores', '4'), ('spark.cores.max', '4'), ('spark.driver.memory', '8g')])
imageDir = "T://courses//BigData//data//flower_photos"
# Load images
# image_df = ImageSchema.readImages(imageDir, recursive = True).withColumn("label", lit(1))
# image_df.printSchema()
# image_df.show(5)
# train_df, test_df, _=image_df.randomSplit([0.1, 0.05, 0.85])
# read images using two methods
tulips_df = ImageSchema.readImages(imageDir + "/tulips").withColumn(
"label", lit(1))
daisy_df = imageIO.readImagesWithCustomFn(
imageDir + "/daisy",
decode_f=imageIO.PIL_decode).withColumn("label", lit(0))
# use larger training sets (e.g. [0.6, 0.4] for getting more images)
tulips_train, tulips_test, _ = tulips_df.randomSplit([0.1, 0.05, 0.85])
# use larger training sets (e.g. [0.6, 0.4] for getting more images)
daisy_train, daisy_test, _ = daisy_df.randomSplit([0.1, 0.05, 0.85])
train_df = tulips_train.unionAll(daisy_train)
test_df = tulips_test.unionAll(daisy_test)
# Under the hood, each of the partitions is fully loaded in memory, which may be expensive.
# This ensure that each of the paritions has a small size.
train_df = train_df.repartition(100)
test_df = test_df.repartition(100)
# df_row = spark.createDataFrame(rdd)
def assert_spark_df(df):
assert isinstance(df, pyspark.sql.dataframe.DataFrame), "Not a Spark DF"
def assert_spark_model(model):
assert isinstance(model, pyspark.ml.PipelineModel), "Not a model"
tulips_df = ImageSchema.readImages("tests/testtulips/").withColumn(
"label", lit(1))
daisy_df = imageIO.readImagesWithCustomFn(
"tests/testdaisy/",
decode_f=imageIO.PIL_decode).withColumn("label", lit(0))
train_df = tulips_df.unionAll(daisy_df)
def test_image_classifier_lr():
model, df_preds = op.dl.image_classifier_lr(train_df)
assert_spark_model(model)
assert_spark_df(df_preds)
def test_evaluate_img_lr():
model, df_preds = op.dl.image_classifier_lr(train_df)
result = op.dl.evaluate_image_classifier(train_df, model)
from pyspark.sql.functions import lit
from sparkdl.image import imageIO
from pyspark.ml.classification import LogisticRegression
from pyspark.ml import Pipeline
from sparkdl import DeepImageFeaturizer
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
img_dir = "/home/lshang/Downloads/flower_photos"
tulips_df = imageIO.readImagesWithCustomFn(
img_dir + "/tulips",
decode_f=imageIO.PIL_decode).withColumn("label", lit(1))
daisy_df = imageIO.readImagesWithCustomFn(
img_dir + "/daisy",
decode_f=imageIO.PIL_decode).withColumn("label", lit(0))
tulips_train, tulips_test = tulips_df.randomSplit([0.6, 0.4])
daisy_train, daisy_test = daisy_df.randomSplit([0.6, 0.4])
#dataframe for training a classification model
train_df = tulips_train.unionAll(daisy_train)
#dataframe for testing the classification model
test_df = tulips_test.unionAll(daisy_test)
featurizer = DeepImageFeaturizer(inputCol="image",
outputCol="features",
modelName="InceptionV3")
lr = LogisticRegression(maxIter=20,
regParam=0.05,
elasticNetParam=0.3,
labelCol="label")
from pyspark.sql.functions import lit
from sparkdl.image import imageIO
from pyspark.ml.classification import LogisticRegression
from pyspark.ml import Pipeline
from sparkdl import DeepImageFeaturizer
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
img_dir = "/home/lshang/Downloads/personalities"
jobs_df = imageIO.readImagesWithCustomFn(
img_dir + "/jobs",
decode_f=imageIO.PIL_decode).withColumn("label", lit(1))
zuckerberg_df = imageIO.readImagesWithCustomFn(
img_dir + "/zuckerberg",
decode_f=imageIO.PIL_decode).withColumn("label", lit(0))
jobs_train, jobs_test = jobs_df.randomSplit([0.6, 0.4])
zuckerberg_train, zuckerberg_test = zuckerberg_df.randomSplit([0.6, 0.4])
#dataframe for training a classification model
train_df = jobs_train.unionAll(zuckerberg_train)
#dataframe for testing the classification model
test_df = jobs_test.unionAll(zuckerberg_test)
featurizer = DeepImageFeaturizer(inputCol="image",
outputCol="features",
modelName="InceptionV3")
lr = LogisticRegression(maxIter=20,
regParam=0.05,
elasticNetParam=0.3,
labelCol="label")
def getSampleImageDF():
return imageIO.readImagesWithCustomFn(path=_getSampleJPEGDir(), decode_f=imageIO.PIL_decode)