def run(self): param = self.params from tfos.choices import BOOLEAN from tfos.utils import convert_bool from tfos.nets.facenet import TFOS_MTCNN # param = json.loads('<#zzjzParam#>') cluster_size = param.get('cluster_size', 3) input_dir = param.get('input_dir') output_dir = param.get('output_dir') image_size = param.get('image_size', 160) margin = param.get('margin', 30) random_order = param.get('random_order', BOOLEAN[0]) gpu_memory_fraction = param.get('gpu_memory_fraction', 1.0) detect_multiple_faces = param.get('detect_multiple_faces', BOOLEAN[1]) cluster_size = int(cluster_size) kwargs = dict(input_dir=input_dir, output_dir=output_dir) if image_size: kwargs['image_size'] = int(image_size) if margin: kwargs['margin'] = int(margin) if random_order: kwargs['random_order'] = convert_bool(random_order) if gpu_memory_fraction: kwargs['gpu_memory_fraction'] = float(gpu_memory_fraction) if detect_multiple_faces: kwargs['detect_multiple_faces'] = convert_bool( detect_multiple_faces) output_df = TFOS_MTCNN(sc, cluster_size=cluster_size).run(**kwargs) outputRDD('<#zzjzRddName#>_data', output_df)
def run(self): param = self.params from tfos.k.layers import BatchNormalizationLayer from tfos.utils import convert_bool from tfos.choices import BOOLEAN # param = json.loads('<#zzjzParam#>') input_prev_layers = param.get("input_prev_layers") axis = param.get("axis", '-1') momentum = param.get("momentum", '0.99') epsilon = param.get("epsilon", '0.001') center = param.get("center", BOOLEAN[0]) scale = param.get("scale", BOOLEAN[0]) beta_initializer = param.get("beta_initializer", 'zeros') gamma_initializer = param.get("gamma_initializer", 'ones') moving_mean_initializer = param.get("moving_mean_initializer", 'zeros') moving_variance_initializer = param.get("moving_variance_initializer", 'ones') beta_regularizer = param.get("beta_regularizer", '') gamma_regularizer = param.get("gamma_regularizer", '') beta_constraint = param.get("beta_constraint", '') gamma_constraint = param.get("gamma_constraint", '') kwargs = {} if axis: kwargs['axis'] = int(axis) if momentum: kwargs['momentum'] = float(momentum) if epsilon: kwargs['epsilon'] = float(epsilon) if center: kwargs['center'] = convert_bool(center) if scale: kwargs['scale'] = convert_bool(scale) if beta_initializer: kwargs['beta_initializer'] = beta_initializer if gamma_initializer: kwargs['gamma_initializer'] = gamma_initializer if moving_mean_initializer: kwargs['moving_mean_initializer'] = moving_mean_initializer if moving_variance_initializer: kwargs['moving_variance_initializer'] = moving_variance_initializer if beta_regularizer: kwargs['beta_regularizer'] = beta_regularizer if gamma_regularizer: kwargs['gamma_regularizer'] = gamma_regularizer if beta_constraint: kwargs['beta_constraint'] = beta_constraint if gamma_constraint: kwargs['gamma_constraint'] = gamma_constraint model_rdd = inputRDD(input_prev_layers) output_df = BatchNormalizationLayer(model_rdd, sqlc=sqlc).add(**kwargs) outputRDD('<#zzjzRddName#>_BatchNormalization', output_df)
def run(self): param = self.params from tfos import TFOS from tfos.choices import BOOLEAN from tfos.utils import convert_bool # param = json.loads('<#zzjzParam#>') input_rdd_name = param.get('input_rdd_name') cluster_size = param.get('cluster_size', 3) num_ps = param.get('num_ps', 1) steps = param.get('steps', 0) model_dir = param.get('model_dir')[0]['path'] output_prob = param.get('output_prob', BOOLEAN[0]) # param check cluster_size = int(cluster_size) num_ps = int(num_ps) steps = int(steps) output_prob = convert_bool(output_prob) # load data assert input_rdd_name is not None, "parameter input_rdd_name cannot empty!" input_rdd = inputRDD(input_rdd_name) assert input_rdd, "cannot get rdd data from previous input layer!" output_df = TFOS(sc, sqlc, cluster_size, num_ps).predict(input_rdd, steps, model_dir, output_prob) if output_df: output_df.show() outputRDD('<#zzjzRddName#>_predict_result', output_df)
def run(self): param = self.params from tfos.k.layers import EmbeddingLayer from tfos.utils import convert_bool from tfos.choices import BOOLEAN # param = json.loads('<#zzjzParam#>') input_dim = param.get("input_dim") output_dim = param.get("output_dim") embeddings_initializer = param.get("embeddings_initializer", '') embeddings_regularizer = param.get("embeddings_regularizer", '') embeddings_constraint = param.get("embeddings_constraint", '') mask_zero = param.get("mask_zero", BOOLEAN[1]) input_length = param.get("input_length", '') kwargs = dict(input_dim=int(input_dim), output_dim=int(output_dim)) if embeddings_initializer: kwargs['embeddings_initializer'] = embeddings_initializer if embeddings_regularizer: kwargs['embeddings_regularizer'] = embeddings_regularizer if embeddings_constraint: kwargs['embeddings_constraint'] = embeddings_constraint if mask_zero: kwargs['mask_zero'] = convert_bool(mask_zero) if input_length: kwargs['input_length'] = int(input_length) output_df = EmbeddingLayer(sqlc=sqlc).add(**kwargs) outputRDD('<#zzjzRddName#>_Embedding', output_df)
def run(self): param = self.params from tfos.choices import BOOLEAN from tfos.utils import convert_bool from tfos.nets.facenet.tfos import TFOSFaceNetSoftMax # param = json.loads('<#zzjzParam#>') cluster_size = param.get('cluster_size', 3) num_ps = param.get('num_ps', 3) batch_size = param.get('batch_size', 3) epochs = param.get('epochs', 3) model_dir = param.get('model_dir', 3) go_on = param.get('go_on', BOOLEAN[0]) cluster_size = int(cluster_size) num_ps = int(num_ps) kwargs = dict(batch_size=int(batch_size), epochs=int(epochs), model_dir=model_dir) if go_on: kwargs['go_on'] = convert_bool(go_on) output_df = TFOSFaceNetSoftMax(sc, sqlc, cluster_size, num_ps).train(**kwargs) outputRDD('<#zzjzRddName#>_Masking', output_df)
def run(self): param = self.params from tfos.nets.yolov3.tfos import YoloTFOS from tfos.choices import BOOLEAN from tfos.utils import convert_bool # param = json.loads('<#zzjzParam#>') input_prev_layers = param.get('input_prev_layers') cluster_size = param.get('cluster_size', 3) num_ps = param.get('num_ps', 1) batch_size = param.get('batch_size', 32) epochs = param.get('epochs', 1) image_size = param.get('image_size') model_dir = param.get('model_dir') freeze_body = param.get('freeze_body', '2') go_on = param.get('go_on', BOOLEAN[1]) model_rdd = inputRDD(input_prev_layers) cluster_size = int(cluster_size) num_ps = int(num_ps) kwargs = dict() kwargs['model_rdd'] = model_rdd kwargs['batch_size'] = int(batch_size) kwargs['epochs'] = int(epochs) kwargs['model_dir'] = model_dir if freeze_body: kwargs['freeze_body'] = int(freeze_body) kwargs['go_on'] = convert_bool(go_on) output_df = YoloTFOS(sc, sqlc, cluster_size, num_ps).yolov3_tiny_train(**kwargs) output_df.show() outputRDD('<#zzjzRddName#>_yolov3_tiny', output_df)
def run(self): param = self.params from tfos.k.layers import SGDLayer from tfos.choices import BOOLEAN from tfos.utils import convert_bool # param = json.loads('<#zzjzParam#>') lr = param.get("lr", '0.01') momentum = param.get("momentum", '0.') decay = param.get("decay", '0.') nesterov = param.get("nesterov", BOOLEAN[1]) kwargs = {} if lr: lr = float(lr) assert lr >= 0, "学习率必须大于等于零" kwargs['lr'] = lr if momentum: momentum = float(momentum) assert momentum >= 0, "加速度必须大于等于0" kwargs['momentum'] = momentum if decay: decay = float(decay) assert decay >= 0, "习率衰减值必须大于等于0" kwargs['decay'] = decay if nesterov: kwargs['nesterov'] = convert_bool(nesterov) output_df = SGDLayer(sqlc=sqlc).add(**kwargs) outputRDD('<#zzjzRddName#>_SGD', output_df)
def run(self): param = self.params from tfos.utils import convert_bool from tfos.choices import BOOLEAN from tfos.nets.facenet.class_image import ClassImage # param = json.loads('<#zzjzParam#>') data_dir = param.get('data_dir') image_size = param.get('image_size') valid_ratio = param.get('valid_ratio', '0.0') min_images_per_class = param.get('min_images_per_class', '0') mode = param.get('mode') random_rotate = param.get('random_rotate', BOOLEAN[0]) random_crop = param.get('random_crop', BOOLEAN[0]) random_flip = param.get('random_flip', BOOLEAN[0]) use_fixed_image_standardization = param.get( 'use_fixed_image_standardization', BOOLEAN[0]) kwargs = dict(sc=sc, data_dir=data_dir, image_size=int(image_size)) if valid_ratio: kwargs['valid_ratio'] = float(valid_ratio) if min_images_per_class: kwargs['min_images_per_class'] = int(min_images_per_class) if mode: kwargs['mode'] = mode if random_rotate: kwargs['random_rotate'] = convert_bool(random_rotate) if random_crop: kwargs['random_crop'] = convert_bool(random_crop) if random_flip: kwargs['random_flip'] = convert_bool(random_flip) if use_fixed_image_standardization: kwargs['use_fixed_image_standardization'] = convert_bool( use_fixed_image_standardization) train_df, val_df = ClassImage(**kwargs).process_data() train_df.persist() val_df.persist() train_df.show() val_df.show() outputRDD('<#zzjzRddName#>_train_data', train_df) outputRDD('<#zzjzRddName#>_val_data', val_df)
def run(self): param = self.params from tfos.data.cifar import Cifar100 from tfos.choices import DATA_MODE, BOOLEAN from tfos.utils import convert_bool # param = json.loads('<#zzjzParam#>') data_dir = param.get('data_dir')[0]['path'] mode = param.get('mode', DATA_MODE[0]) one_hot = param.get('one_hot', BOOLEAN[1]) flat = param.get('flat', BOOLEAN[1]) cifar = Cifar100(sc=sc, path=data_dir, one_hot=convert_bool(one_hot), flat=convert_bool(flat)) output_df = cifar.train_df if mode == 'train' else cifar.test_df output_df.persist() output_df.show() outputRDD('<#zzjzRddName#>_cifar100_{}'.format(mode), output_df)
def run(self): param = self.params from tfos.data.mnist import Mnist from tfos.utils import convert_bool from tfos.choices import BOOLEAN, MNIST_FORMAT, DATA_MODE # param = json.loads('<#zzjzParam#>') mnist_dir = param.get('mnist_dir')[0]['path'] mode = param.get('mode', DATA_MODE[0]) data_format = param.get('data_format', MNIST_FORMAT[0]) one_hot = param.get('one_hot', BOOLEAN[0]) is_conv = param.get('is_conv', BOOLEAN[1]) one_hot = convert_bool(one_hot) is_conv = convert_bool(is_conv) mnist = Mnist(sc=sc, path=mnist_dir, data_format=data_format, one_hot=one_hot, flat=not is_conv) output_df = mnist.train_df if mode == 'train' else mnist.test_df output_df.persist() output_df.show() outputRDD('<#zzjzRddName#>_mnist_{}'.format(mode), output_df)
def run(self): param = self.params from tfos import TFOS from tfos.choices import BOOLEAN from tfos.utils import convert_bool # param = json.loads('<#zzjzParam#>') input_rdd_name = param.get('input_rdd_name') input_prev_layers = param.get('input_prev_layers') cluster_size = param.get('cluster_size', 3) num_ps = param.get('num_ps', 1) batch_size = param.get('batch_size', 32) epochs = param.get('epochs', 1) model_dir = param.get('model_dir')[0]['path'] go_on = param.get('go_on', BOOLEAN[1]) # param check cluster_size = int(cluster_size) num_ps = int(num_ps) batch_size = int(batch_size) epochs = int(epochs) go_on = convert_bool(go_on) # load data assert input_rdd_name is not None, "parameter input_rdd_name cannot empty!" input_rdd = inputRDD(input_rdd_name) assert input_rdd, "cannot get rdd data from previous input layer!" # load model assert input_prev_layers is not None, "parameter input_model_config cannot empty!" model_rdd = inputRDD(input_prev_layers) assert model_rdd, "cannot get model config rdd from previous model layer!" output_df = TFOS(sc, sqlc, cluster_size, num_ps).train(data_rdd=input_rdd, model_rdd=model_rdd, batch_size=batch_size, epochs=epochs, model_dir=model_dir, go_on=go_on) if output_df: output_df.show() outputRDD('<#zzjzRddName#>_train_result', output_df)
def run(self): param = self.params from tfos.k.layers import AdamLayer from tfos.utils import convert_bool from tfos.choices import BOOLEAN # param = json.loads('<#zzjzParam#>') lr = param.get("lr", '0.001') beta_1 = param.get("beta_1", '0.9') beta_2 = param.get("beta_2", '0.999') epsilon = param.get("epsilon", '') decay = param.get("decay", '0.') amsgrad = param.get("amsgrad", BOOLEAN[1]) kwargs = {} if lr: lr = float(lr) assert lr >= 0, "学习率必须大于等于零" kwargs['lr'] = lr if beta_1: beta_1 = float(beta_1) assert 0 < beta_1 < 1, "beta_1必须大于0小于1" kwargs['beta_1'] = beta_1 if beta_2: beta_2 = float(beta_2) assert 0 < beta_2 < 1, "beta_2必须大于0小于1" kwargs['beta_2'] = beta_2 if epsilon: epsilon = float(epsilon) assert epsilon >= 0, "模糊因子必须大于等于0" kwargs['epsilon'] = epsilon if decay: decay = float(decay) assert decay >= 0, "习率衰减值必须大于等于0" kwargs['decay'] = decay if amsgrad: kwargs['amsgrad'] = convert_bool(amsgrad) output_df = AdamLayer(sqlc=sqlc).add(**kwargs) outputRDD('<#zzjzRddName#>_Adam', output_df)
def run(self): param = self.params from tfos.k.layers import LSTMLayer from tfos.choices import BOOLEAN, ACTIVATIONS from tfos.utils import convert_bool # param = json.loads('<#zzjzParam#>') input_prev_layers = param.get("input_prev_layers", '') units = param.get("units") activation = param.get("activation", ACTIVATIONS[2]) recurrent_activation = param.get("recurrent_activation", ACTIVATIONS[9]) use_bias = param.get("use_bias", BOOLEAN[0]) kernel_initializer = param.get("kernel_initializer", 'glorot_uniform') recurrent_initializer = param.get("recurrent_initializer", 'orthogonal') bias_initializer = param.get("bias_initializer", 'zeros') unit_forget_bias = param.get("unit_forget_bias", BOOLEAN[0]) kernel_regularizer = param.get("kernel_regularizer", '') recurrent_regularizer = param.get("recurrent_regularizer", '') bias_regularizer = param.get("bias_regularizer", '') activity_regularizer = param.get("activity_regularizer", '') kernel_constraint = param.get("kernel_constraint", '') recurrent_constraint = param.get("recurrent_constraint", '') bias_constraint = param.get("bias_constraint", '') dropout = param.get("dropout", '0') recurrent_dropout = param.get("recurrent_dropout", '0') implementation = param.get("implementation", '1') return_sequences = param.get("return_sequences", BOOLEAN[1]) return_state = param.get("return_state", BOOLEAN[1]) go_backwards = param.get("go_backwards", BOOLEAN[1]) stateful = param.get("stateful", BOOLEAN[1]) unroll = param.get("unroll", BOOLEAN[1]) input_shape = param.get("input_shape", '') # 必填参数 kwargs = dict(units=int(units)) # 选填参数 if activation: kwargs['activation'] = activation if recurrent_activation: kwargs['recurrent_activation'] = recurrent_activation kwargs['use_bias'] = use_bias if kernel_initializer: kwargs['kernel_initializer'] = kernel_initializer if recurrent_initializer: kwargs['recurrent_initializer'] = recurrent_initializer if bias_initializer: kwargs['bias_initializer'] = bias_initializer if kernel_regularizer: kwargs['kernel_regularizer'] = kernel_regularizer if recurrent_regularizer: kwargs['recurrent_regularizer'] = recurrent_regularizer if bias_regularizer: kwargs['bias_regularizer'] = bias_regularizer kwargs['unit_forget_bias'] = convert_bool(unit_forget_bias) if activity_regularizer: kwargs['activity_regularizer'] = activity_regularizer if kernel_constraint: kwargs['kernel_constraint'] = kernel_constraint if recurrent_constraint: kwargs['recurrent_constraint'] = recurrent_constraint if bias_constraint: kwargs['bias_constraint'] = bias_constraint if dropout: kwargs['dropout'] = float(dropout) if recurrent_dropout: kwargs['recurrent_dropout'] = float(recurrent_dropout) if implementation: kwargs['implementation'] = int(implementation) kwargs['return_sequences'] = convert_bool(return_sequences) kwargs['return_state'] = convert_bool(return_state) kwargs['go_backwards'] = convert_bool(go_backwards) kwargs['stateful'] = convert_bool(stateful) kwargs['unroll'] = convert_bool(unroll) if input_shape: kwargs['input_shape'] = tuple( [int(i) for i in input_shape.split(',') if i]) model_rdd = inputRDD(input_prev_layers) output_df = LSTMLayer(model_rdd, sqlc=sqlc).add(**kwargs) outputRDD('<#zzjzRddName#>_Masking', output_df)