def __init__(self,
X,
Y,
model,
criterion,
end_trigger,
batch_size,
optim_method=None,
cores=None,
bigdl_type="float"):
if not optim_method:
optim_methods = {model.name(): SGD()}
elif isinstance(optim_method, OptimMethod):
optim_methods = {model.name(): optim_method}
elif isinstance(optim_method, JavaObject):
optim_methods = {model.name(): OptimMethod(optim_method, bigdl_type)}
else:
optim_methods = optim_method
if cores is None:
cores = multiprocessing.cpu_count()
JavaValue.__init__(self, None, bigdl_type,
[JTensor.from_ndarray(X) for X in to_list(X)],
JTensor.from_ndarray(Y),
model.value,
criterion,
optim_methods, end_trigger, batch_size, cores)
def __init__(self,
X,
Y,
model,
criterion,
end_trigger,
batch_size,
optim_method=None,
cores=None,
bigdl_type="float"):
if not optim_method:
optim_methods = {model.name(): SGD()}
elif isinstance(optim_method, OptimMethod):
optim_methods = {model.name(): optim_method}
elif isinstance(optim_method, JavaObject):
optim_methods = {
model.name(): OptimMethod(optim_method, bigdl_type)
}
else:
optim_methods = optim_method
if cores is None:
cores = multiprocessing.cpu_count()
JavaValue.__init__(self, None, bigdl_type,
[JTensor.from_ndarray(X) for X in to_list(X)],
JTensor.from_ndarray(Y), model.value, criterion,
optim_methods, end_trigger, batch_size, cores)
def __init__(self,
matrix_b,
matrix_c,
alpha=float(1.0),
beta=float(1.0),
trans_a=0,
trans_b=0,
bigdl_type="float"):
super(Gemm, self).__init__(None, bigdl_type, alpha, beta, trans_a,
trans_b, JTensor.from_ndarray(matrix_b),
JTensor.from_ndarray(matrix_c))
def set_validation(self, batch_size, X_val, Y_val, trigger, val_method=None):
"""
Configure validation settings.
:param batch_size: validation batch size
:param X_val: features of validation dataset
:param Y_val: label of validation dataset
:param trigger: validation interval
:param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss"
"""
if val_method is None:
val_method = [Top1Accuracy()]
callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size,
trigger, [JTensor.from_ndarray(X) for X in to_list(X_val)],
JTensor.from_ndarray(Y_val), to_list(val_method))
def __init__(self,
learningrate=1e-3,
learningrate_decay=0.0,
weightdecay=0.0,
momentum=0.0,
dampening=DOUBLEMAX,
nesterov=False,
leaningrate_schedule=None,
learningrates=None,
weightdecays=None,
bigdl_type="float"):
super(SGD, self).__init__(None, bigdl_type, learningrate, learningrate_decay, weightdecay,
momentum, dampening, nesterov,
leaningrate_schedule if (leaningrate_schedule) else Default(),
JTensor.from_ndarray(learningrates), JTensor.from_ndarray(weightdecays))
def set_validation(self, batch_size, X_val, Y_val, trigger, val_method=None):
"""
Configure validation settings.
:param batch_size: validation batch size
:param X_val: features of validation dataset
:param Y_val: label of validation dataset
:param trigger: validation interval
:param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss"
"""
if val_method is None:
val_method = [Top1Accuracy()]
callBigDlFunc(self.bigdl_type, "setValidation", self.value, batch_size,
trigger, [JTensor.from_ndarray(X) for X in to_list(X_val)],
JTensor.from_ndarray(Y_val), to_list(val_method))
def __init__(self,
learningrate=1e-3,
learningrate_decay=0.0,
weightdecay=0.0,
momentum=0.0,
dampening=DOUBLEMAX,
nesterov=False,
leaningrate_schedule=None,
learningrates=None,
weightdecays=None,
bigdl_type="float"):
JavaValue.__init__(self, None, bigdl_type, learningrate, learningrate_decay, weightdecay,
momentum, dampening, nesterov,
leaningrate_schedule if (leaningrate_schedule) else Default(),
JTensor.from_ndarray(learningrates), JTensor.from_ndarray(weightdecays))
def __init__(self,
weights=None,
size_average=True,
bigdl_type="float"):
super(MultiLabelSoftMarginCriterion, self).__init__(None, bigdl_type,
JTensor.from_ndarray(weights),
size_average)
def get_wide_tensor(row, column_info):
"""
convert a row to tensor given column feature information of a WideAndDeep model
:param row: Row of userId, itemId, features and label
:param column_info: ColumnFeatureInfo specify information of different features
:return: an array of tensors as input for wide part of a WideAndDeep model
"""
wide_columns = column_info.wide_base_cols + column_info.wide_cross_cols
wide_dims = column_info.wide_base_dims + column_info.wide_cross_dims
wide_length = len(wide_columns)
acc = 0
indices = []
for i in range(0, wide_length):
index = row[wide_columns[i]]
if i == 0:
res = index
else:
acc += wide_dims[i - 1]
res = acc + index
indices.append(res)
values = np.array([i + 1 for i in indices])
shape = np.array([sum(wide_dims)])
return JTensor.sparse(values, np.array(indices), shape)
def __init__(self,
text1_length,
text2_length,
embedding_file,
word_index=None,
train_embed=True,
kernel_num=21,
sigma=0.1,
exact_sigma=0.001,
target_mode="ranking",
bigdl_type="float"):
embed_weights = prepare_embedding(embedding_file,
word_index,
randomize_unknown=True,
normalize=True)
vocab_size, embed_size = embed_weights.shape
super(KNRM, self).__init__(text1_length, vocab_size, embed_size,
embed_weights, train_embed, target_mode,
bigdl_type)
self.text2_length = text2_length
self.kernel_num = kernel_num
self.sigma = float(sigma)
self.exact_sigma = float(exact_sigma)
self.model = self.build_model()
super(TextMatcher,
self).__init__(None, self.bigdl_type, self.text1_length,
self.text2_length, self.vocab_size,
self.embed_size,
JTensor.from_ndarray(embed_weights),
self.train_embed, self.kernel_num, self.sigma,
self.exact_sigma, self.target_mode, self.model)
def __init__(self,
text1_length,
text2_length,
vocab_size,
embed_size=300,
embed_weights=None,
train_embed=True,
kernel_num=21,
sigma=0.1,
exact_sigma=0.001,
bigdl_type="float"):
super(KNRM, self).__init__(text1_length, vocab_size, embed_size,
embed_weights, train_embed, bigdl_type)
self.text2_length = text2_length
self.kernel_num = kernel_num
self.sigma = float(sigma)
self.exact_sigma = float(exact_sigma)
self.model = self.build_model()
super(TextMatcher,
self).__init__(None, self.bigdl_type, self.text1_length,
self.text2_length, self.vocab_size,
self.embed_size,
JTensor.from_ndarray(embed_weights),
self.train_embed, self.kernel_num, self.sigma,
self.exact_sigma, self.model)
def __init__(self,
X,
y,
model,
criterion,
end_trigger,
batch_size,
optim_method=None,
cores=None,
bigdl_type="float"):
if cores is None:
cores = multiprocessing.cpu_count()
JavaValue.__init__(self, None, bigdl_type,
[JTensor.from_ndarray(X) for X in to_list(X)],
JTensor.from_ndarray(y), model.value, criterion,
optim_method if optim_method else SGD(),
end_trigger, batch_size, cores)
def __init__(self,
weights=None,
size_average=True,
logProbAsInput=True,
bigdl_type="float"):
super(ClassNLLCriterion, self).__init__(None, bigdl_type,
JTensor.from_ndarray(weights),
size_average, logProbAsInput)
def __init__(self, log_prob_as_input=False, zero_based_label=True,
weights=None, size_average=True, padding_value=-1, bigdl_type="float"):
super(SparseCategoricalCrossEntropy, self).__init__(None, bigdl_type,
log_prob_as_input,
zero_based_label,
JTensor.from_ndarray(weights),
size_average,
padding_value)
def __init__(self,
weights=None,
size_average=True,
logProbAsInput=True,
bigdl_type="float"):
super(ClassNLLCriterion, self).__init__(None, bigdl_type,
JTensor.from_ndarray(weights),
size_average, logProbAsInput)
def set_running_mean(self, running_mean):
"""
Set the running mean of the BatchNormalization layer.
:param running_mean: a Numpy array.
"""
callBigDlFunc(self.bigdl_type, "setRunningMean",
self.value, JTensor.from_ndarray(running_mean))
return self
def set_running_std(self, running_std):
"""
Set the running variance of the BatchNormalization layer.
:param running_std: a Numpy array.
"""
callBigDlFunc(self.bigdl_type, "setRunningStd",
self.value, JTensor.from_ndarray(running_std))
return self
def set_running_std(self, running_std):
"""
Set the running variance of the BatchNormalization layer.
:param running_std: a Numpy array.
"""
callBigDlFunc(self.bigdl_type, "setRunningStd", self.value,
JTensor.from_ndarray(running_std))
return self
def __init__(self,
weights=None,
size_average=True,
bigdl_type="float"):
super(CrossEntropyCriterion, self).__init__(None, bigdl_type,
JTensor.from_ndarray(
weights),
size_average)
def __init__(self,
affine_mat,
translation=JTensor.from_ndarray(np.zeros(3)),
clamp_mode="clamp",
pad_val=0.0,
bigdl_type="float"):
super(AffineTransform, self).__init__(bigdl_type, affine_mat,
translation, clamp_mode, pad_val)
def set_running_mean(self, running_mean):
"""
Set the running mean of the BatchNormalization layer.
:param running_mean: a Numpy array.
"""
callBigDlFunc(self.bigdl_type, "setRunningMean", self.value,
JTensor.from_ndarray(running_mean))
return self
def __init__(self,
p=1,
weights=None,
margin=1.0,
size_average=True,
bigdl_type="float"):
super(MultiMarginCriterion,
self).__init__(None, bigdl_type, p,
JTensor.from_ndarray(weights), margin,
size_average)
def __init__(self,
p=1,
weights=None,
margin=1.0,
size_average=True,
bigdl_type="float"):
super(MultiMarginCriterion, self).__init__(None, bigdl_type,
p,
JTensor.from_ndarray(weights),
margin,
size_average)
def __init__(self, input_dim, output_dim, init="uniform", weights=None, trainable=True,
input_length=None, W_regularizer=None, input_shape=None, **kwargs):
if input_length:
input_shape = (input_length, )
super(Embedding, self).__init__(None,
input_dim,
output_dim,
init,
JTensor.from_ndarray(weights),
trainable,
W_regularizer,
list(input_shape) if input_shape else None,
**kwargs)
def row_to_sample(row, column_info, model_type="wide_n_deep"):
wide_tensor = get_wide_tensor(row, column_info)
deep_tensor = JTensor.from_ndarray(get_deep_tensor(row, column_info))
label = row[column_info.label]
model_type = model_type.lower()
if model_type == "wide_n_deep":
feature = [wide_tensor, deep_tensor]
elif model_type == "wide":
feature = wide_tensor
elif model_type == "deep":
feature = deep_tensor
else:
raise TypeError("Unsupported model_type: %s" % model_type)
return Sample.from_jtensor(feature, label)
def row_to_sample(row, column_info, model_type="wide_n_deep"):
wide_tensor = get_wide_tensor(row, column_info)
deep_tensor = JTensor.from_ndarray(get_deep_tensor(row, column_info))
label = row[column_info.label]
model_type = model_type.lower()
if model_type == "wide_n_deep":
feature = [wide_tensor, deep_tensor]
elif model_type == "wide":
feature = wide_tensor
elif model_type == "deep":
feature = deep_tensor
else:
raise TypeError("Unsupported model_type: %s" % model_type)
return Sample.from_jtensor(feature, label)
def save_variable_bigdl(tensors, target_path, bigdl_type="float"):
"""
Save a variable dictionary to a Java object file, so it can be read by BigDL
:param tensors: tensor dictionary
:param target_path: where is the Java object file store
:param bigdl_type: model variable numeric type
:return: nothing
"""
jtensors = {}
for tn in tensors.keys():
jtensors[tn] = JTensor.from_ndarray(tensors[tn])
callBigDlFunc(bigdl_type, "saveTensorDictionary", jtensors, target_path)
def get_wide_tensor(row, column_info):
wide_columns = column_info.wide_base_cols + column_info.wide_cross_cols
wide_dims = column_info.wide_base_dims + column_info.wide_cross_dims
wide_length = len(wide_columns)
acc = 0
indices = []
for i in range(0, wide_length):
index = row[wide_columns[i]]
if i == 0:
res = index
else:
acc += wide_dims[i-1]
res = acc + index
indices.append(res)
values = np.array([i + 1 for i in indices])
shape = np.array([sum(wide_dims)])
return JTensor.sparse(values, np.array(indices), shape)
def get_wide_tensor(row, column_info):
wide_columns = column_info.wide_base_cols + column_info.wide_cross_cols
wide_dims = column_info.wide_base_dims + column_info.wide_cross_dims
wide_length = len(wide_columns)
acc = 0
indices = []
for i in range(0, wide_length):
index = row[wide_columns[i]]
if i == 0:
res = index
else:
acc += wide_dims[i-1]
res = acc + index
indices.append(res)
values = np.array([i + 1 for i in indices])
shape = np.array([sum(wide_dims)])
return JTensor.sparse(values, np.array(indices), shape)
def save_variable_bigdl(tensors, target_path, bigdl_type="float"):
"""
Save a variable dictionary to a Java object file, so it can be read by BigDL
:param tensors: tensor dictionary
:param target_path: where is the Java object file store
:param bigdl_type: model variable numeric type
:return: nothing
"""
import numpy as np
jtensors = {}
for tn in tensors.keys():
if not isinstance(tensors[tn], np.ndarray):
value = np.array(tensors[tn])
else:
value = tensors[tn]
jtensors[tn] = JTensor.from_ndarray(value)
callBigDlFunc(bigdl_type, "saveTensorDictionary", jtensors, target_path)
def from_pytorch(model):
"""
Create a TorchModel directly from PyTorch model, e.g. model in torchvision.models.
:param model: a PyTorch model, or a function to create PyTorch model
"""
weights = []
import types
if isinstance(model, types.FunctionType) or isinstance(
model, types.ClassType):
for param in trainable_param(model()):
weights.append(param.view(-1))
else:
for param in trainable_param(model):
weights.append(param.view(-1))
flatten_weight = torch.nn.utils.parameters_to_vector(
weights).data.numpy()
bys = io.BytesIO()
torch.save(model, bys, pickle_module=zoo_pickle_module)
weights = JTensor.from_ndarray(flatten_weight)
jvalue = callZooFunc("float", "createTorchModel", bys.getvalue(),
weights)
net = TorchModel(jvalue, bys.getvalue())
return net
def row_to_sample(row, column_info, model_type="wide_n_deep"):
"""
convert a row to sample given column feature information of a WideAndDeep model
:param row: Row of userId, itemId, features and label
:param column_info: ColumnFeatureInfo specify information of different features
:return: TensorSample as input for WideAndDeep model
"""
wide_tensor = get_wide_tensor(row, column_info)
deep_tensor = get_deep_tensors(row, column_info)
deep_tensors = [JTensor.from_ndarray(ele) for ele in deep_tensor]
label = row[column_info.label]
model_type = model_type.lower()
if model_type == "wide_n_deep":
feature = [wide_tensor] + deep_tensors
elif model_type == "wide":
feature = wide_tensor
elif model_type == "deep":
feature = deep_tensors
else:
raise TypeError("Unsupported model_type: %s" % model_type)
return Sample.from_jtensor(feature, label)
def __init__(self, weights=None, size_average=True, bigdl_type="float"):
super(MultiLabelSoftMarginCriterion,
self).__init__(None, bigdl_type, JTensor.from_ndarray(weights),
size_average)
def __init__(self, weights=None, size_average=True, bigdl_type="float"):
super(CrossEntropyCriterion,
self).__init__(None, bigdl_type, JTensor.from_ndarray(weights),
size_average)
def __init__(self, value, bigdl_type="float"):
super(Constant, self).__init__(None, bigdl_type,
JTensor.from_ndarray(value))
def __init__(self, data, name=None, bigdl_type="float"):
self.data = data
super(Constant, self).__init__(None, bigdl_type, JTensor.from_ndarray(data), name)
def __init__(self, module_bytes, weights, bigdl_type="float"):
weights = JTensor.from_ndarray(weights)
self.module_bytes = module_bytes
self.value = callZooFunc(bigdl_type, self.jvm_class_constructor(),
module_bytes, weights)
self.bigdl_type = bigdl_type
