def transform(self, image_feature, bigdl_type="float"):
"""
transform ImageFeature
"""
callBigDlFunc(bigdl_type, "transformImageFeature", self.value,
image_feature)
return image_feature
def set_criterion(self, criterion):
"""
set new criterion, for optimizer reuse
:param criterion: new criterion
:return:
"""
callBigDlFunc(self.bigdl_type, "setCriterion", self.value, criterion)
def set_gradclip_l2norm(self, clip_norm):
"""
Configure L2 norm clipping settings.
:param clip_norm: gradient L2-Norm threshold
"""
callBigDlFunc(self.bigdl_type, "setL2NormClip", self.value, clip_norm)
def set_traindata(self, training_rdd, batch_size):
"""
Set new training dataset, for optimizer reuse
:param training_rdd: the training dataset
:param batch_size: training batch size
:return:
"""
callBigDlFunc(self.bigdl_type, "setTrainData", self.value,
training_rdd, batch_size)
def set_gradclip_const(self, min_value, max_value):
"""
Configure constant clipping settings.
:param min_value: the minimum value to clip by
:param max_value: the maxmimum value to clip by
"""
callBigDlFunc(self.bigdl_type, "setConstantClip", self.value,
min_value, max_value)
def set_train_summary(self, summary):
"""
Set train summary. A TrainSummary object contains information
necessary for the optimizer to know how often the logs are recorded,
where to store the logs and how to retrieve them, etc. For details,
refer to the docs of TrainSummary.
:param summary: a TrainSummary object
"""
callBigDlFunc(self.bigdl_type, "setTrainSummary", self.value, summary)
return self
def __call__(self, image_frame, bigdl_type="float"):
"""
transform ImageFrame
"""
jframe = callBigDlFunc(bigdl_type, "transformImageFrame", self.value,
image_frame)
return ImageFrame(jvalue=jframe)
def set_checkpoint(self,
checkpoint_trigger,
checkpoint_path,
isOverWrite=True):
"""
Configure checkpoint settings.
:param checkpoint_trigger: the interval to write snapshots
:param checkpoint_path: the path to write snapshots into
:param isOverWrite: whether to overwrite existing snapshots in path.default is True
"""
if not os.path.exists(checkpoint_path):
mkpath(checkpoint_path)
callBigDlFunc(self.bigdl_type, "setCheckPoint", self.value,
checkpoint_trigger, checkpoint_path, isOverWrite)
def get_label(self):
"""
get label list from ImageFrame
"""
labels = callBigDlFunc(self.bigdl_type, "localImageFrameToLabelTensor",
self.value)
return map(lambda tensor: tensor.to_ndarray(), labels)
def transform(self, transformer, bigdl_type="float"):
"""
transformImageFrame
"""
self.value = callBigDlFunc(bigdl_type, "transformImageFrame",
transformer, self.value)
return self
def get_label(self):
"""
get label as ndarray from ImageFeature
"""
label = callBigDlFunc(self.bigdl_type, "imageFeatureToLabelTensor",
self.value)
return label.to_ndarray()
def get_image(self, float_key="floats", to_chw=True):
"""
get image as ndarray from ImageFeature
"""
tensor = callBigDlFunc(self.bigdl_type, "imageFeatureToImageTensor",
self.value, float_key, to_chw)
return tensor.to_ndarray()
def set_validation(self, batch_size, val_rdd, trigger, val_method=None):
"""
Configure validation settings.
:param batch_size: validation batch size
:param val_rdd: validation dataset
:param trigger: validation interval
:param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss"
"""
if val_method is None:
val_method = [Top1Accuracy()]
func_name = "setValidation"
if isinstance(val_rdd, DataSet):
func_name = "setValidationFromDataSet"
callBigDlFunc(self.bigdl_type, func_name, self.value, batch_size,
trigger, val_rdd, to_list(val_method))
def get_image(self, float_key="floats", to_chw=True):
"""
get image list from ImageFrame
"""
tensors = callBigDlFunc(self.bigdl_type,
"localImageFrameToImageTensor", self.value,
float_key, to_chw)
return map(lambda tensor: tensor.to_ndarray(), tensors)
def get_label(self):
"""
get label rdd from ImageFrame
"""
tensor_rdd = callBigDlFunc(self.bigdl_type,
"distributedImageFrameToLabelTensorRdd",
self.value)
return tensor_rdd.map(lambda tensor: tensor.to_ndarray())
def get_image(self, float_key="floats", to_chw=True):
"""
get image rdd from ImageFrame
"""
tensor_rdd = callBigDlFunc(self.bigdl_type,
"distributedImageFrameToImageTensorRdd",
self.value, float_key, to_chw)
return tensor_rdd.map(lambda tensor: tensor.to_ndarray())
def add(self, scheduler, max_iteration, bigdl_type="float"):
"""
Add a learning rate scheduler to the contained `schedules`
:param scheduler: learning rate scheduler to be add
:param max_iteration: iteration numbers this scheduler will run
"""
return callBigDlFunc(bigdl_type, "addScheduler", self.value, scheduler,
max_iteration)
def __init__(self, jvalue, bigdl_type, *args):
if (jvalue):
assert (type(jvalue) == JavaObject)
self.value = jvalue
else:
self.value = callBigDlFunc(bigdl_type,
JavaValue.jvm_class_constructor(self),
*args)
self.bigdl_type = bigdl_type
def save(self, path, overWrite):
"""
save OptimMethod
:param path path
:param overWrite whether to overwrite
"""
method = self.value
return callBigDlFunc(self.bigdl_type, "saveOptimMethod", method, path,
overWrite)
def __init__(self, image=None, label=None, path=None, bigdl_type="float"):
image_tensor = JTensor.from_ndarray(
image) if image is not None else None
label_tensor = JTensor.from_ndarray(
label) if label is not None else None
self.bigdl_type = bigdl_type
self.value = callBigDlFunc(bigdl_type,
JavaValue.jvm_class_constructor(self),
image_tensor, label_tensor, path)
def get_predict(self, key="predict"):
"""
get prediction rdd from ImageFrame
"""
predicts = callBigDlFunc(self.bigdl_type,
"distributedImageFrameToPredict", self.value,
key)
return predicts.map(lambda predict: (predict[0], predict[1].to_ndarray(
)) if predict[1] else (predict[0], None))
def save_variable_bigdl(tensors, target_path, bigdl_type="float"):
"""
Save a variable dictionary to a Java object file, so it can be read by BigDL
:param tensors: tensor dictionary
:param target_path: where is the Java object file store
:param bigdl_type: model variable numeric type
:return: nothing
"""
import numpy as np
jtensors = {}
for tn in tensors.keys():
if not isinstance(tensors[tn], np.ndarray):
value = np.array(tensors[tn])
else:
value = tensors[tn]
jtensors[tn] = JTensor.from_ndarray(value)
callBigDlFunc(bigdl_type, "saveTensorDictionary", jtensors, target_path)
def set_summary_trigger(self, name, trigger):
"""
Set the interval of recording for each indicator.
:param tag: tag name. Supported tag names are "LearningRate", "Loss","Throughput", "Parameters". "Parameters" is an umbrella tag thatincludes weight, bias, gradWeight, gradBias, and some running status(eg. runningMean and runningVar in BatchNormalization). If youdidn't set any triggers, we will by default record Loss and Throughputin each iteration, while *NOT* recording LearningRate and Parameters,as recording parameters may introduce substantial overhead when themodel is very big, LearningRate is not a public attribute for allOptimMethod.
:param trigger: trigger
"""
return callBigDlFunc(self.bigdl_type, "summarySetTrigger", self.value,
name, trigger)
def read_scalar(self, tag):
"""
Retrieve train logs by type. Return an array of records in the format
(step,value,wallClockTime). - "Step" is the iteration count by default.
:param tag: the type of the logs, Supported tags are: "LearningRate","Loss", "Throughput"
"""
return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value,
tag)
def read_scalar(self, tag):
"""
Retrieve validation logs by type. Return an array of records in the
format (step,value,wallClockTime). - "Step" is the iteration count
by default.
:param tag: the type of the logs. The tag should match the name ofthe ValidationMethod set into the optimizer. e.g."Top1AccuracyLoss","Top1Accuracy" or "Top5Accuracy".
"""
return callBigDlFunc(self.bigdl_type, "summaryReadScalar", self.value,
tag)
def write_parquet(cls,
path,
output,
sc,
partition_num=1,
bigdl_type="float"):
"""
write ImageFrame as parquet file
"""
return callBigDlFunc(bigdl_type, "writeParquet", path, output, sc,
partition_num)
def set_validation(self,
batch_size,
X_val,
Y_val,
trigger,
val_method=None):
"""
Configure validation settings.
:param batch_size: validation batch size
:param X_val: features of validation dataset
:param Y_val: label of validation dataset
:param trigger: validation interval
:param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss"
"""
if val_method is None:
val_method = [Top1Accuracy()]
callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size,
trigger,
[JTensor.from_ndarray(X) for X in to_list(X_val)],
JTensor.from_ndarray(Y_val), to_list(val_method))
def read(cls, path, sc=None, min_partitions=1, bigdl_type="float"):
"""
Read images as Image Frame
if sc is defined, Read image as DistributedImageFrame from local file system or HDFS
if sc is null, Read image as LocalImageFrame from local file system
:param path path to read images
if sc is defined, path can be local or HDFS. Wildcard character are supported.
if sc is null, path is local directory/image file/image file with wildcard character
:param sc SparkContext
:param min_partitions A suggestion value of the minimal splitting number for input data.
:return ImageFrame
"""
return ImageFrame(
jvalue=callBigDlFunc(bigdl_type, "read", path, sc, min_partitions))
def backward(self, input, target):
"""
NB: It's for debug only, please use optimizer.optimize() in production.
Performs a back-propagation step through the criterion, with respect to the given input.
:param input: ndarray or list of ndarray
:param target: ndarray or list of ndarray
:return: ndarray
"""
jinput, input_is_table = Layer.check_input(input)
jtarget, target_is_table = Layer.check_input(target)
output = callBigDlFunc(self.bigdl_type, "criterionBackward",
self.value, jinput, input_is_table, jtarget,
target_is_table)
return Layer.convert_output(output)
def forward(self, input, target):
"""
NB: It's for debug only, please use optimizer.optimize() in production.
Takes an input object, and computes the corresponding loss of the criterion,
compared with `target`
:param input: ndarray or list of ndarray
:param target: ndarray or list of ndarray
:return: value of loss
"""
jinput, input_is_table = Layer.check_input(input)
jtarget, target_is_table = Layer.check_input(target)
output = callBigDlFunc(self.bigdl_type, "criterionForward", self.value,
jinput, input_is_table, jtarget,
target_is_table)
return output
