def run(self):
param = self.params
from tfos.choices import BOOLEAN
from tfos.utils import convert_bool
from tfos.nets.facenet import TFOS_MTCNN
# param = json.loads('<#zzjzParam#>')
cluster_size = param.get('cluster_size', 3)
input_dir = param.get('input_dir')
output_dir = param.get('output_dir')
image_size = param.get('image_size', 160)
margin = param.get('margin', 30)
random_order = param.get('random_order', BOOLEAN[0])
gpu_memory_fraction = param.get('gpu_memory_fraction', 1.0)
detect_multiple_faces = param.get('detect_multiple_faces', BOOLEAN[1])
cluster_size = int(cluster_size)
kwargs = dict(input_dir=input_dir, output_dir=output_dir)
if image_size:
kwargs['image_size'] = int(image_size)
if margin:
kwargs['margin'] = int(margin)
if random_order:
kwargs['random_order'] = convert_bool(random_order)
if gpu_memory_fraction:
kwargs['gpu_memory_fraction'] = float(gpu_memory_fraction)
if detect_multiple_faces:
kwargs['detect_multiple_faces'] = convert_bool(
detect_multiple_faces)
output_df = TFOS_MTCNN(sc, cluster_size=cluster_size).run(**kwargs)
outputRDD('<#zzjzRddName#>_data', output_df)
def run(self):
param = self.params
from tfos.k.layers import BatchNormalizationLayer
from tfos.utils import convert_bool
from tfos.choices import BOOLEAN
# param = json.loads('<#zzjzParam#>')
input_prev_layers = param.get("input_prev_layers")
axis = param.get("axis", '-1')
momentum = param.get("momentum", '0.99')
epsilon = param.get("epsilon", '0.001')
center = param.get("center", BOOLEAN[0])
scale = param.get("scale", BOOLEAN[0])
beta_initializer = param.get("beta_initializer", 'zeros')
gamma_initializer = param.get("gamma_initializer", 'ones')
moving_mean_initializer = param.get("moving_mean_initializer", 'zeros')
moving_variance_initializer = param.get("moving_variance_initializer",
'ones')
beta_regularizer = param.get("beta_regularizer", '')
gamma_regularizer = param.get("gamma_regularizer", '')
beta_constraint = param.get("beta_constraint", '')
gamma_constraint = param.get("gamma_constraint", '')
kwargs = {}
if axis:
kwargs['axis'] = int(axis)
if momentum:
kwargs['momentum'] = float(momentum)
if epsilon:
kwargs['epsilon'] = float(epsilon)
if center:
kwargs['center'] = convert_bool(center)
if scale:
kwargs['scale'] = convert_bool(scale)
if beta_initializer:
kwargs['beta_initializer'] = beta_initializer
if gamma_initializer:
kwargs['gamma_initializer'] = gamma_initializer
if moving_mean_initializer:
kwargs['moving_mean_initializer'] = moving_mean_initializer
if moving_variance_initializer:
kwargs['moving_variance_initializer'] = moving_variance_initializer
if beta_regularizer:
kwargs['beta_regularizer'] = beta_regularizer
if gamma_regularizer:
kwargs['gamma_regularizer'] = gamma_regularizer
if beta_constraint:
kwargs['beta_constraint'] = beta_constraint
if gamma_constraint:
kwargs['gamma_constraint'] = gamma_constraint
model_rdd = inputRDD(input_prev_layers)
output_df = BatchNormalizationLayer(model_rdd, sqlc=sqlc).add(**kwargs)
outputRDD('<#zzjzRddName#>_BatchNormalization', output_df)
def run(self):
param = self.params
from tfos import TFOS
from tfos.choices import BOOLEAN
from tfos.utils import convert_bool
# param = json.loads('<#zzjzParam#>')
input_rdd_name = param.get('input_rdd_name')
cluster_size = param.get('cluster_size', 3)
num_ps = param.get('num_ps', 1)
steps = param.get('steps', 0)
model_dir = param.get('model_dir')[0]['path']
output_prob = param.get('output_prob', BOOLEAN[0])
# param check
cluster_size = int(cluster_size)
num_ps = int(num_ps)
steps = int(steps)
output_prob = convert_bool(output_prob)
# load data
assert input_rdd_name is not None, "parameter input_rdd_name cannot empty!"
input_rdd = inputRDD(input_rdd_name)
assert input_rdd, "cannot get rdd data from previous input layer!"
output_df = TFOS(sc, sqlc, cluster_size,
num_ps).predict(input_rdd, steps, model_dir,
output_prob)
if output_df:
output_df.show()
outputRDD('<#zzjzRddName#>_predict_result', output_df)
def run(self):
param = self.params
from tfos.k.layers import EmbeddingLayer
from tfos.utils import convert_bool
from tfos.choices import BOOLEAN
# param = json.loads('<#zzjzParam#>')
input_dim = param.get("input_dim")
output_dim = param.get("output_dim")
embeddings_initializer = param.get("embeddings_initializer", '')
embeddings_regularizer = param.get("embeddings_regularizer", '')
embeddings_constraint = param.get("embeddings_constraint", '')
mask_zero = param.get("mask_zero", BOOLEAN[1])
input_length = param.get("input_length", '')
kwargs = dict(input_dim=int(input_dim), output_dim=int(output_dim))
if embeddings_initializer:
kwargs['embeddings_initializer'] = embeddings_initializer
if embeddings_regularizer:
kwargs['embeddings_regularizer'] = embeddings_regularizer
if embeddings_constraint:
kwargs['embeddings_constraint'] = embeddings_constraint
if mask_zero:
kwargs['mask_zero'] = convert_bool(mask_zero)
if input_length:
kwargs['input_length'] = int(input_length)
output_df = EmbeddingLayer(sqlc=sqlc).add(**kwargs)
outputRDD('<#zzjzRddName#>_Embedding', output_df)
def run(self):
param = self.params
from tfos.choices import BOOLEAN
from tfos.utils import convert_bool
from tfos.nets.facenet.tfos import TFOSFaceNetSoftMax
# param = json.loads('<#zzjzParam#>')
cluster_size = param.get('cluster_size', 3)
num_ps = param.get('num_ps', 3)
batch_size = param.get('batch_size', 3)
epochs = param.get('epochs', 3)
model_dir = param.get('model_dir', 3)
go_on = param.get('go_on', BOOLEAN[0])
cluster_size = int(cluster_size)
num_ps = int(num_ps)
kwargs = dict(batch_size=int(batch_size),
epochs=int(epochs),
model_dir=model_dir)
if go_on:
kwargs['go_on'] = convert_bool(go_on)
output_df = TFOSFaceNetSoftMax(sc, sqlc, cluster_size,
num_ps).train(**kwargs)
outputRDD('<#zzjzRddName#>_Masking', output_df)
def run(self):
param = self.params
from tfos.nets.yolov3.tfos import YoloTFOS
from tfos.choices import BOOLEAN
from tfos.utils import convert_bool
# param = json.loads('<#zzjzParam#>')
input_prev_layers = param.get('input_prev_layers')
cluster_size = param.get('cluster_size', 3)
num_ps = param.get('num_ps', 1)
batch_size = param.get('batch_size', 32)
epochs = param.get('epochs', 1)
image_size = param.get('image_size')
model_dir = param.get('model_dir')
freeze_body = param.get('freeze_body', '2')
go_on = param.get('go_on', BOOLEAN[1])
model_rdd = inputRDD(input_prev_layers)
cluster_size = int(cluster_size)
num_ps = int(num_ps)
kwargs = dict()
kwargs['model_rdd'] = model_rdd
kwargs['batch_size'] = int(batch_size)
kwargs['epochs'] = int(epochs)
kwargs['model_dir'] = model_dir
if freeze_body:
kwargs['freeze_body'] = int(freeze_body)
kwargs['go_on'] = convert_bool(go_on)
output_df = YoloTFOS(sc, sqlc, cluster_size,
num_ps).yolov3_tiny_train(**kwargs)
output_df.show()
outputRDD('<#zzjzRddName#>_yolov3_tiny', output_df)
def run(self):
param = self.params
from tfos.k.layers import SGDLayer
from tfos.choices import BOOLEAN
from tfos.utils import convert_bool
# param = json.loads('<#zzjzParam#>')
lr = param.get("lr", '0.01')
momentum = param.get("momentum", '0.')
decay = param.get("decay", '0.')
nesterov = param.get("nesterov", BOOLEAN[1])
kwargs = {}
if lr:
lr = float(lr)
assert lr >= 0, "学习率必须大于等于零"
kwargs['lr'] = lr
if momentum:
momentum = float(momentum)
assert momentum >= 0, "加速度必须大于等于0"
kwargs['momentum'] = momentum
if decay:
decay = float(decay)
assert decay >= 0, "习率衰减值必须大于等于0"
kwargs['decay'] = decay
if nesterov:
kwargs['nesterov'] = convert_bool(nesterov)
output_df = SGDLayer(sqlc=sqlc).add(**kwargs)
outputRDD('<#zzjzRddName#>_SGD', output_df)
def run(self):
param = self.params
from tfos.utils import convert_bool
from tfos.choices import BOOLEAN
from tfos.nets.facenet.class_image import ClassImage
# param = json.loads('<#zzjzParam#>')
data_dir = param.get('data_dir')
image_size = param.get('image_size')
valid_ratio = param.get('valid_ratio', '0.0')
min_images_per_class = param.get('min_images_per_class', '0')
mode = param.get('mode')
random_rotate = param.get('random_rotate', BOOLEAN[0])
random_crop = param.get('random_crop', BOOLEAN[0])
random_flip = param.get('random_flip', BOOLEAN[0])
use_fixed_image_standardization = param.get(
'use_fixed_image_standardization', BOOLEAN[0])
kwargs = dict(sc=sc, data_dir=data_dir, image_size=int(image_size))
if valid_ratio:
kwargs['valid_ratio'] = float(valid_ratio)
if min_images_per_class:
kwargs['min_images_per_class'] = int(min_images_per_class)
if mode:
kwargs['mode'] = mode
if random_rotate:
kwargs['random_rotate'] = convert_bool(random_rotate)
if random_crop:
kwargs['random_crop'] = convert_bool(random_crop)
if random_flip:
kwargs['random_flip'] = convert_bool(random_flip)
if use_fixed_image_standardization:
kwargs['use_fixed_image_standardization'] = convert_bool(
use_fixed_image_standardization)
train_df, val_df = ClassImage(**kwargs).process_data()
train_df.persist()
val_df.persist()
train_df.show()
val_df.show()
outputRDD('<#zzjzRddName#>_train_data', train_df)
outputRDD('<#zzjzRddName#>_val_data', val_df)
def run(self):
param = self.params
from tfos.data.cifar import Cifar100
from tfos.choices import DATA_MODE, BOOLEAN
from tfos.utils import convert_bool
# param = json.loads('<#zzjzParam#>')
data_dir = param.get('data_dir')[0]['path']
mode = param.get('mode', DATA_MODE[0])
one_hot = param.get('one_hot', BOOLEAN[1])
flat = param.get('flat', BOOLEAN[1])
cifar = Cifar100(sc=sc,
path=data_dir,
one_hot=convert_bool(one_hot),
flat=convert_bool(flat))
output_df = cifar.train_df if mode == 'train' else cifar.test_df
output_df.persist()
output_df.show()
outputRDD('<#zzjzRddName#>_cifar100_{}'.format(mode), output_df)
def run(self):
param = self.params
from tfos.data.mnist import Mnist
from tfos.utils import convert_bool
from tfos.choices import BOOLEAN, MNIST_FORMAT, DATA_MODE
# param = json.loads('<#zzjzParam#>')
mnist_dir = param.get('mnist_dir')[0]['path']
mode = param.get('mode', DATA_MODE[0])
data_format = param.get('data_format', MNIST_FORMAT[0])
one_hot = param.get('one_hot', BOOLEAN[0])
is_conv = param.get('is_conv', BOOLEAN[1])
one_hot = convert_bool(one_hot)
is_conv = convert_bool(is_conv)
mnist = Mnist(sc=sc, path=mnist_dir, data_format=data_format, one_hot=one_hot, flat=not is_conv)
output_df = mnist.train_df if mode == 'train' else mnist.test_df
output_df.persist()
output_df.show()
outputRDD('<#zzjzRddName#>_mnist_{}'.format(mode), output_df)
def run(self):
param = self.params
from tfos import TFOS
from tfos.choices import BOOLEAN
from tfos.utils import convert_bool
# param = json.loads('<#zzjzParam#>')
input_rdd_name = param.get('input_rdd_name')
input_prev_layers = param.get('input_prev_layers')
cluster_size = param.get('cluster_size', 3)
num_ps = param.get('num_ps', 1)
batch_size = param.get('batch_size', 32)
epochs = param.get('epochs', 1)
model_dir = param.get('model_dir')[0]['path']
go_on = param.get('go_on', BOOLEAN[1])
# param check
cluster_size = int(cluster_size)
num_ps = int(num_ps)
batch_size = int(batch_size)
epochs = int(epochs)
go_on = convert_bool(go_on)
# load data
assert input_rdd_name is not None, "parameter input_rdd_name cannot empty!"
input_rdd = inputRDD(input_rdd_name)
assert input_rdd, "cannot get rdd data from previous input layer!"
# load model
assert input_prev_layers is not None, "parameter input_model_config cannot empty!"
model_rdd = inputRDD(input_prev_layers)
assert model_rdd, "cannot get model config rdd from previous model layer!"
output_df = TFOS(sc, sqlc, cluster_size,
num_ps).train(data_rdd=input_rdd,
model_rdd=model_rdd,
batch_size=batch_size,
epochs=epochs,
model_dir=model_dir,
go_on=go_on)
if output_df:
output_df.show()
outputRDD('<#zzjzRddName#>_train_result', output_df)
def run(self):
param = self.params
from tfos.k.layers import AdamLayer
from tfos.utils import convert_bool
from tfos.choices import BOOLEAN
# param = json.loads('<#zzjzParam#>')
lr = param.get("lr", '0.001')
beta_1 = param.get("beta_1", '0.9')
beta_2 = param.get("beta_2", '0.999')
epsilon = param.get("epsilon", '')
decay = param.get("decay", '0.')
amsgrad = param.get("amsgrad", BOOLEAN[1])
kwargs = {}
if lr:
lr = float(lr)
assert lr >= 0, "学习率必须大于等于零"
kwargs['lr'] = lr
if beta_1:
beta_1 = float(beta_1)
assert 0 < beta_1 < 1, "beta_1必须大于0小于1"
kwargs['beta_1'] = beta_1
if beta_2:
beta_2 = float(beta_2)
assert 0 < beta_2 < 1, "beta_2必须大于0小于1"
kwargs['beta_2'] = beta_2
if epsilon:
epsilon = float(epsilon)
assert epsilon >= 0, "模糊因子必须大于等于0"
kwargs['epsilon'] = epsilon
if decay:
decay = float(decay)
assert decay >= 0, "习率衰减值必须大于等于0"
kwargs['decay'] = decay
if amsgrad:
kwargs['amsgrad'] = convert_bool(amsgrad)
output_df = AdamLayer(sqlc=sqlc).add(**kwargs)
outputRDD('<#zzjzRddName#>_Adam', output_df)
def run(self):
param = self.params
from tfos.k.layers import LSTMLayer
from tfos.choices import BOOLEAN, ACTIVATIONS
from tfos.utils import convert_bool
# param = json.loads('<#zzjzParam#>')
input_prev_layers = param.get("input_prev_layers", '')
units = param.get("units")
activation = param.get("activation", ACTIVATIONS[2])
recurrent_activation = param.get("recurrent_activation",
ACTIVATIONS[9])
use_bias = param.get("use_bias", BOOLEAN[0])
kernel_initializer = param.get("kernel_initializer", 'glorot_uniform')
recurrent_initializer = param.get("recurrent_initializer",
'orthogonal')
bias_initializer = param.get("bias_initializer", 'zeros')
unit_forget_bias = param.get("unit_forget_bias", BOOLEAN[0])
kernel_regularizer = param.get("kernel_regularizer", '')
recurrent_regularizer = param.get("recurrent_regularizer", '')
bias_regularizer = param.get("bias_regularizer", '')
activity_regularizer = param.get("activity_regularizer", '')
kernel_constraint = param.get("kernel_constraint", '')
recurrent_constraint = param.get("recurrent_constraint", '')
bias_constraint = param.get("bias_constraint", '')
dropout = param.get("dropout", '0')
recurrent_dropout = param.get("recurrent_dropout", '0')
implementation = param.get("implementation", '1')
return_sequences = param.get("return_sequences", BOOLEAN[1])
return_state = param.get("return_state", BOOLEAN[1])
go_backwards = param.get("go_backwards", BOOLEAN[1])
stateful = param.get("stateful", BOOLEAN[1])
unroll = param.get("unroll", BOOLEAN[1])
input_shape = param.get("input_shape", '')
# 必填参数
kwargs = dict(units=int(units))
# 选填参数
if activation:
kwargs['activation'] = activation
if recurrent_activation:
kwargs['recurrent_activation'] = recurrent_activation
kwargs['use_bias'] = use_bias
if kernel_initializer:
kwargs['kernel_initializer'] = kernel_initializer
if recurrent_initializer:
kwargs['recurrent_initializer'] = recurrent_initializer
if bias_initializer:
kwargs['bias_initializer'] = bias_initializer
if kernel_regularizer:
kwargs['kernel_regularizer'] = kernel_regularizer
if recurrent_regularizer:
kwargs['recurrent_regularizer'] = recurrent_regularizer
if bias_regularizer:
kwargs['bias_regularizer'] = bias_regularizer
kwargs['unit_forget_bias'] = convert_bool(unit_forget_bias)
if activity_regularizer:
kwargs['activity_regularizer'] = activity_regularizer
if kernel_constraint:
kwargs['kernel_constraint'] = kernel_constraint
if recurrent_constraint:
kwargs['recurrent_constraint'] = recurrent_constraint
if bias_constraint:
kwargs['bias_constraint'] = bias_constraint
if dropout:
kwargs['dropout'] = float(dropout)
if recurrent_dropout:
kwargs['recurrent_dropout'] = float(recurrent_dropout)
if implementation:
kwargs['implementation'] = int(implementation)
kwargs['return_sequences'] = convert_bool(return_sequences)
kwargs['return_state'] = convert_bool(return_state)
kwargs['go_backwards'] = convert_bool(go_backwards)
kwargs['stateful'] = convert_bool(stateful)
kwargs['unroll'] = convert_bool(unroll)
if input_shape:
kwargs['input_shape'] = tuple(
[int(i) for i in input_shape.split(',') if i])
model_rdd = inputRDD(input_prev_layers)
output_df = LSTMLayer(model_rdd, sqlc=sqlc).add(**kwargs)
outputRDD('<#zzjzRddName#>_Masking', output_df)
