def TestMultiInputJob(): with flow.scope.placement("gpu", "0:0"): x1 = flow.get_variable( "x1", shape=shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) x2 = flow.get_variable( "x2", shape=shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) loss = TestMultiInput(x1, x2) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0 ).minimize(loss) flow.watch(x1, test_global_storage.Setter("x1")) flow.watch_diff(x1, test_global_storage.Setter("x1_diff")) flow.watch(x2, test_global_storage.Setter("x2")) flow.watch_diff(x2, test_global_storage.Setter("x2_diff")) return loss
def _hybrid_embedding(name, ids, embedding_size, vocab_size, hf_vocab_size): b, s = ids.shape ids = flow.flatten(ids) unique_ids, unique_ids_idx, _, _ = flow.experimental.unique_with_counts(ids) hf_vocab_size_constant = flow.constant(hf_vocab_size, dtype=flow.int32) hf_indices = flow.argwhere(flow.math.less(unique_ids, hf_vocab_size_constant)) lf_indices = flow.argwhere(flow.math.greater_equal(unique_ids, hf_vocab_size_constant)) hf_ids = flow.gather_nd(params=unique_ids, indices=hf_indices) lf_ids = flow.gather_nd(params=unique_ids, indices=lf_indices) hf_embedding_table = flow.get_variable( name=f'hf_{name}', shape=(hf_vocab_size, embedding_size), dtype=flow.float, initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05), ) hf_embedding = flow.gather(params=hf_embedding_table, indices=hf_ids) lf_ids = lf_ids - hf_vocab_size_constant with flow.scope.placement('cpu', '0:0'): lf_embedding_table = flow.get_variable( name=f'lf_{name}', shape=(vocab_size - hf_vocab_size, embedding_size), dtype=flow.float, initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05), ) lf_embedding = flow.gather(params=lf_embedding_table, indices=lf_ids) unique_embedding = flow.reshape(flow.zeros_like(unique_ids, dtype=flow.float), (-1, 1)) * flow.constant(0.0, dtype=flow.float, shape=(1,embedding_size)) unique_embedding = flow.tensor_scatter_nd_update(params=unique_embedding, updates=hf_embedding, indices=hf_indices) unique_embedding = flow.tensor_scatter_nd_update(params=unique_embedding, updates=lf_embedding, indices=lf_indices) unique_embedding = flow.gather(params=unique_embedding, indices=unique_ids_idx) unique_embedding = flow.cast_to_static_shape(unique_embedding) unique_embedding = flow.reshape(unique_embedding, shape=(b, s*embedding_size)) return unique_embedding
def _conv2d_layer( name, input, filters, kernel_size=3, strides=1, padding="SAME", data_format="NCHW", dilation_rate=1, activation=flow.math.relu, use_bias=False, weight_initializer=flow.random_uniform_initializer(), bias_initializer=flow.random_uniform_initializer(), ): weight_shape = (filters, input.shape[1], kernel_size, kernel_size) weight = flow.get_variable( name + "-weight", shape=weight_shape, dtype=input.dtype, initializer=weight_initializer, ) output = flow.nn.conv2d( input, weight, strides, padding, None, data_format, dilation_rate, name=name ) if use_bias: bias = flow.get_variable( name + "-bias", shape=(filters,), dtype=input.dtype, initializer=bias_initializer, ) output = flow.nn.bias_add(output, bias, data_format) if callable(activation): output = activation(output) return output
def ConcatJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=x_shape, dtype=type_name_to_flow_type[dtype], initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) y = flow.get_variable( "y", shape=y_shape, dtype=type_name_to_flow_type[dtype], initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) x = flow.cast_to_current_logical_view(x) y = flow.cast_to_current_logical_view(y) loss = flow.concat([x, y], axis) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0 ).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(y, test_global_storage.Setter("y")) flow.watch_diff(y, test_global_storage.Setter("y_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def MatmulJob(): with flow.scope.placement(device_type, "0:0"): a = flow.get_variable( "a", shape=a_shape, dtype=dtype, initializer=flow.random_uniform_initializer(minval=0, maxval=1), trainable=True, ) b = flow.get_variable( "b", shape=b_shape, dtype=dtype, initializer=flow.random_uniform_initializer(minval=0, maxval=1), trainable=True, ) if data_type == "float16": out = flow.matmul( flow.cast(a, dtype=flow.float16), flow.cast(b, dtype=flow.float16), transpose_a, transpose_b, alpha, ) c = flow.get_variable( "c", shape=out.shape, dtype=dtype, initializer=flow.random_uniform_initializer(minval=-1, maxval=1), trainable=True, ) loss = flow.cast(out + flow.cast(c, dtype=flow.float16), dtype=flow.float) else: out = flow.matmul(a, b, transpose_a, transpose_b, alpha) c = flow.get_variable( "c", shape=out.shape, dtype=dtype, initializer=flow.random_uniform_initializer(minval=-1, maxval=1), trainable=True, ) loss = out + c flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(a, test_global_storage.Setter("a")) flow.watch_diff(a, test_global_storage.Setter("a_diff")) flow.watch(b, test_global_storage.Setter("b")) flow.watch_diff(b, test_global_storage.Setter("b_diff")) flow.watch(c, test_global_storage.Setter("c")) flow.watch_diff(c, test_global_storage.Setter("c_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def _model(dense_fields, wide_sparse_fields, deep_sparse_fields): wide_sparse_fields = flow.parallel_cast( wide_sparse_fields, distribute=flow.distribute.broadcast()) wide_embedding_table = flow.get_variable( name='wide_embedding', shape=(FLAGS.wide_vocab_size, 1), initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05), distribute=flow.distribute.split(0), ) wide_embedding = flow.gather(params=wide_embedding_table, indices=wide_sparse_fields) wide_embedding = flow.reshape(wide_embedding, shape=(-1, wide_embedding.shape[-1] * wide_embedding.shape[-2])) wide_scores = flow.math.reduce_sum(wide_embedding, axis=[1], keepdims=True) wide_scores = flow.parallel_cast( wide_scores, distribute=flow.distribute.split(0), gradient_distribute=flow.distribute.broadcast()) deep_sparse_fields = flow.parallel_cast( deep_sparse_fields, distribute=flow.distribute.broadcast()) deep_embedding_table = flow.get_variable( name='deep_embedding', shape=(FLAGS.deep_vocab_size, FLAGS.deep_embedding_vec_size), initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05), distribute=flow.distribute.split(1), ) deep_embedding = flow.gather(params=deep_embedding_table, indices=deep_sparse_fields) deep_embedding = flow.parallel_cast( deep_embedding, distribute=flow.distribute.split(0), gradient_distribute=flow.distribute.split(2)) deep_embedding = flow.reshape(deep_embedding, shape=(-1, deep_embedding.shape[-1] * deep_embedding.shape[-2])) deep_features = flow.concat([deep_embedding, dense_fields], axis=1) for idx, units in enumerate(DEEP_HIDDEN_UNITS): deep_features = flow.layers.dense( deep_features, units=units, kernel_initializer=flow.glorot_uniform_initializer(), bias_initializer=flow.constant_initializer(0.0), activation=flow.math.relu, name='fc' + str(idx + 1)) deep_features = flow.nn.dropout(deep_features, rate=FLAGS.deep_dropout_rate) deep_scores = flow.layers.dense( deep_features, units=1, kernel_initializer=flow.glorot_uniform_initializer(), bias_initializer=flow.constant_initializer(0.0), name='fc' + str(len(DEEP_HIDDEN_UNITS) + 1)) scores = wide_scores + deep_scores return scores
def _data_loader_synthetic(batch_size): def _blob_random(shape, dtype=flow.int32, initializer=flow.zeros_initializer(flow.int32)): return flow.data.decode_random(shape=shape, dtype=dtype, batch_size=batch_size, initializer=initializer) labels = _blob_random((1,), initializer=flow.random_uniform_initializer(dtype=flow.int32)) dense_fields = _blob_random((FLAGS.num_dense_fields,), dtype=flow.float, initializer=flow.random_uniform_initializer()) wide_sparse_fields = _blob_random((FLAGS.num_wide_sparse_fields,)) deep_sparse_fields = _blob_random((FLAGS.num_deep_sparse_fields,)) print('use synthetic data') return [labels, dense_fields, wide_sparse_fields, deep_sparse_fields]
def TestMultiOptimizerJob(): with flow.scope.placement(device_type, "0:0-0"): var1 = flow.get_variable( name="var1", shape=var1_shape, dtype=flow.float32, initializer=flow.random_uniform_initializer(minval=0, maxval=100), trainable=True, ) var2 = flow.get_variable( name="var2", shape=var2_shape, dtype=flow.float32, initializer=flow.random_uniform_initializer(minval=0, maxval=100), trainable=True, ) var3 = flow.get_variable( name="var3", shape=var3_shape, dtype=flow.float32, initializer=flow.random_uniform_initializer(minval=0, maxval=100), trainable=True, ) loss = flow.math.reduce_sum(var1 + var2 + var3) sgd_opt = flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler( [], [sgd_opt_args["lr"]]), momentum=sgd_opt_args["momentum"], variables=["var1"], ) rmsprop_opt = flow.optimizer.RMSProp( flow.optimizer.PiecewiseConstantScheduler( [], [rmsprop_opt_args["lr"]]), decay_rate=rmsprop_opt_args["decay_rate"], epsilon=0, centered=rmsprop_opt_args["centered"], variables=["var2"], ) adam_opt = flow.optimizer.Adam( flow.optimizer.PiecewiseConstantScheduler( [], [adam_opt_args["lr"]]), beta1=adam_opt_args["beta1"], beta2=adam_opt_args["beta2"], epsilon=adam_opt_args["epsilon"], do_bias_correction=True, variables=["var3"], ) flow.optimizer.CombinedOptimizer([sgd_opt, rmsprop_opt, adam_opt]).minimize(loss) return (var1, var2, var3)
def RunConvBias(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=x_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=0, maxval=100), trainable=True, ) if data_format == "NCHW": weight_shape = (filters, x.shape[1] // groups, kernel_size, kernel_size) else: weight_shape = (filters, kernel_size, kernel_size, x.shape[3] // groups) weight = flow.get_variable( "conv-weight", shape=weight_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=0, maxval=100), ) bias = flow.get_variable( "conv-bias", shape=(filters, ), dtype=flow.float, initializer=flow.random_uniform_initializer(minval=0, maxval=100), ) loss = flow.nn.conv2d( x, weight, bias=bias, strides=[stride, stride], padding=padding, dilations=[1, 1], groups=groups, name="conv", ) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(weight, test_global_storage.Setter("weight")) flow.watch_diff(weight, test_global_storage.Setter("weight_diff")) flow.watch(bias, test_global_storage.Setter("bias")) flow.watch_diff(bias, test_global_storage.Setter("bias_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def add() -> tp.Numpy: with flow.scope.placement("gpu", "0:0-1"): x = flow.get_variable( name="x", shape=(2, 3), initializer=flow.random_uniform_initializer(), ) y = flow.get_variable( name="y", shape=(2, 3), initializer=flow.random_uniform_initializer(), ) return flow.math.add_n([x, y])
def alexnet(images, labels): conv1 = _conv2d_layer("conv1", images, filters=64, kernel_size=11, strides=4, padding="VALID") pool1 = flow.nn.avg_pool2d(conv1, 3, 2, "VALID", "NCHW", name="pool1") conv2 = _conv2d_layer("conv2", pool1, filters=192, kernel_size=5) pool2 = flow.nn.avg_pool2d(conv2, 3, 2, "VALID", "NCHW", name="pool2") conv3 = _conv2d_layer("conv3", pool2, filters=384) conv4 = _conv2d_layer("conv4", conv3, filters=384) conv5 = _conv2d_layer("conv5", conv4, filters=256) pool5 = flow.nn.avg_pool2d(conv5, 3, 2, "VALID", "NCHW", name="pool5") if len(pool5.shape) > 2: pool5 = flow.reshape(pool5, shape=(pool5.shape[0], -1)) fc1 = flow.layers.dense( inputs=pool5, units=4096, activation=flow.math.relu, use_bias=False, kernel_initializer=flow.random_uniform_initializer(), bias_initializer=False, trainable=True, name="fc1", ) dropout1 = flow.nn.dropout(fc1, rate=0.5) fc2 = flow.layers.dense( inputs=dropout1, units=4096, activation=flow.math.relu, use_bias=False, kernel_initializer=flow.random_uniform_initializer(), bias_initializer=False, trainable=True, name="fc2", ) dropout2 = flow.nn.dropout(fc2, rate=0.5) fc3 = flow.layers.dense( inputs=dropout2, units=1001, activation=None, use_bias=False, kernel_initializer=flow.random_uniform_initializer(), bias_initializer=False, trainable=True, name="fc3", ) loss = flow.nn.sparse_softmax_cross_entropy_with_logits( labels, fc3, name="softmax_loss") return loss
def ConvJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=x_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=0, maxval=100), trainable=True, ) if data_format == "NCDHW": weight_shape = ( filters, x.shape[1] // groups, kernel_size, kernel_size, kernel_size, ) else: weight_shape = ( filters, kernel_size, kernel_size, kernel_size, x.shape[4] // groups, ) weight = flow.get_variable( "conv-weight", shape=weight_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=0, maxval=100), ) loss = flow.nn.conv3d( x, weight, strides=[stride_d, stride_h, stride_w], padding=of_padding, data_format=data_format, dilations=[dilation_d, dilation_h, dilation_w], groups=groups, ) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0 ).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(weight, test_global_storage.Setter("weight")) flow.watch_diff(weight, test_global_storage.Setter("weight_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def ConvJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=x_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=0, maxval=100), trainable=True, ) loss = flow.layers.conv3d( x, filters, kernel_size=kernel_size, strides=1, padding="valid", data_format="NCDHW", dilation_rate=1, groups=groups, use_bias=False, kernel_initializer=flow.random_uniform_initializer(minval=0, maxval=100), weight_name="conv3d_weight", ) weight_shape = ( filters, x.shape[1] // groups, kernel_size, kernel_size, kernel_size, ) weight = flow.get_variable( name="conv3d_weight", shape=weight_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=0, maxval=100), ) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(weight, test_global_storage.Setter("weight")) flow.watch_diff(weight, test_global_storage.Setter("weight_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def _conv2d_layer( args, name, input, filters, kernel_size=3, strides=1, padding="SAME", data_format="NCHW", dilation_rate=1, activation=op_conf_util.kRelu, use_bias=False, weight_initializer=flow.random_uniform_initializer(), bias_initializer=flow.random_uniform_initializer(), ): weight_shape = (filters, input.shape[1], kernel_size, kernel_size) weight = flow.get_variable( name + "-weight", shape=weight_shape, dtype=input.dtype, initializer=weight_initializer, ) weight = flow.identity(weight) weight = flow.repeat(weight, args.num_piece_in_batch) output = flow.nn.conv2d(input, weight, strides, padding, None, data_format, dilation_rate, name=name) if use_bias: bias = flow.get_variable( name + "-bias", shape=(filters, ), dtype=input.dtype, initializer=bias_initializer, ) bias = flow.identity(bias) bias = flow.repeat(bias, args.num_piece_in_batch) output = flow.nn.bias_add(output, bias, data_format) if activation is not None: if activation == op_conf_util.kRelu: output = flow.math.relu(output) else: raise NotImplementedError return output
def PartialFcJob(labels: oft.Numpy.Placeholder( (batch_size, ), dtype=type_name_to_flow_type[label_type])): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x-weight", shape=(num_classes, 128), dtype=flow.float, initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) with flow.scope.placement(device_type, "0:0-3"): lebels_distribute = flow.distribute.broadcast() weight_distribute = flow.distribute.split(0) ( maped_label, sampled_label, sampled_weight, ) = flow.distributed_partial_fc_sample( weight=x.with_distribute(weight_distribute), label=labels.with_distribute(lebels_distribute), num_sample=num_sample, ) with flow.scope.placement(device_type, "0:0"): sampled_weight = flow.identity(sampled_weight) loss = flow.math.square(sampled_weight) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch_diff(sampled_weight, test_global_storage.Setter("sampled_weight_diff")) return (x, maped_label, sampled_label, sampled_weight)
def SoftmaxJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=x_shape, dtype=dtype, initializer=flow.random_uniform_initializer(minval=-1.0, maxval=1.0), trainable=True, ) x1 = x x = flow.identity(x) if data_type == "float16": loss = flow.cast( flow.nn.softmax(flow.cast(x, dtype=flow.float16), axis=axis), dtype=flow.float, ) else: loss = flow.nn.softmax(x, axis=axis) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) total_loss = loss * x1 flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(total_loss) return loss
def DropoutJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=x_shape, dtype=dtype, initializer=flow.random_uniform_initializer(minval=-1, maxval=1), trainable=True, ) if data_type == "float16": x = flow.cast(flow.cast(x, flow.float16), dtype) of_out = flow.cast( flow.nn.dropout(flow.cast(x, flow.float16), rate=rate, name="dropout"), dtype, ) else: of_out = flow.nn.dropout(x, rate=rate, name="dropout") loss = flow.math.square(of_out) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(of_out, test_global_storage.Setter("out")) flow.watch_diff(of_out, test_global_storage.Setter("out_diff")) return loss
def get_var(var_name): return flow.get_variable( name=var_name, shape=(2, 256, 14, 14), dtype=flow.float32, initializer=flow.random_uniform_initializer(), )
def SparseSoftmaxCrossEntropyWithLogitsJob(labels: oft.Numpy.Placeholder( (batch_size, ), dtype=type_name_to_flow_type[label_type])): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=(batch_size, num_classes), dtype=type_name_to_flow_type[data_type], initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) with flow.scope.placement(device_type, "0:0-3"): labels = flow.parallel_cast(labels, distribute=flow.distribute.broadcast()) logits = flow.parallel_cast( x, distribute=flow.distribute.split(len(x.shape) - 1)) loss = flow.nn.distributed_sparse_softmax_cross_entropy_with_logits( labels, logits) loss = flow.math.square(loss) with flow.scope.placement(device_type, "0:0"): loss = flow.identity(loss) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def decode_random( shape: Sequence[int], dtype: flow.dtype, batch_size: int = 1, initializer: Optional[initializer_conf_util.InitializerConf] = None, tick: Optional[oneflow._oneflow_internal.BlobDesc] = None, name: Optional[str] = None, ) -> oneflow._oneflow_internal.BlobDesc: op_conf = op_conf_util.OperatorConf() if name is None: name = id_util.UniqueStr("DecodeRandom_") assert isinstance(name, str) op_conf.name = name assert isinstance(shape, (list, tuple)) op_conf.decode_random_conf.shape.dim.extend(shape) assert dtype is not None setattr( op_conf.decode_random_conf, "data_type", oneflow._oneflow_internal.deprecated.GetProtoDtype4OfDtype(dtype), ) op_conf.decode_random_conf.batch_size = batch_size if initializer is not None: op_conf.decode_random_conf.data_initializer.CopyFrom(initializer) else: op_conf.decode_random_conf.data_initializer.CopyFrom( flow.random_uniform_initializer()) if tick: op_conf.decode_random_conf.tick = tick.unique_name op_conf.decode_random_conf.out = "out" lbi = logical_blob_id_util.LogicalBlobId() lbi.op_name = op_conf.name lbi.blob_name = "out" interpret_util.ConsistentForward(op_conf) return remote_blob_util.RemoteBlob(lbi)
def hybrid_concat_job( input_0_def: oft.ListNumpy.Placeholder(shape=static_shape, dtype=flow.float), input_1_def: oft.ListNumpy.Placeholder(shape=static_shape, dtype=flow.float), ): var = flow.get_variable( "var", shape=static_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(), trainable=True, ) constant = flow.constant(1.0, dtype=flow.float, shape=rand_sub_shape) inputs = [ flow.cast_to_current_logical_view(input) for input in [var, input_0_def, input_1_def, constant] ] concated = flow.concat(inputs, axis=axis, max_dim_size=max_dim_size) if verbose: print("concated static shape:", concated.shape) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.001]), momentum=0 ).minimize(concated) flow.watch_diff(var, compare_var_diff) if max_dim_size is None: test_case.assertTrue( concated.shape[axis] == static_shape[axis] * 3 + rand_sub_shape[axis] ) else: test_case.assertTrue(concated.shape[axis] == max_dim_size) return (var, concated)
def UpsampleJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "input", shape=input_shape, dtype=type_name_to_flow_type[dtype], initializer=flow.random_uniform_initializer(minval=2, maxval=5), trainable=True, ) loss = flow.layers.upsample_2d( x, size=size, data_format=data_format, interpolation=interpolation, align_corners=align_corners, ) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def get_v(): return flow.get_variable( name="var", shape=(5, 2), dtype=flow.float32, initializer=flow.random_uniform_initializer(), )
def deconv(input, out_channel, name_prefix, kernel_size=4, strides=[2, 2], trainable=True): weight = flow.get_variable( name_prefix + "_weight", shape=(input.shape[1], out_channel, kernel_size, kernel_size), dtype=flow.float, initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) return flow.nn.conv2d_transpose( input, weight, strides=strides, padding="SAME", output_shape=( input.shape[0], out_channel, input.shape[2] * strides[0], input.shape[3] * strides[1], ), )
def get_var() -> tp.Numpy: return flow.get_variable( name="var", shape=shape, dtype=dtype, initializer=flow.random_uniform_initializer(), reuse=True, )
def get_var(name, shape=(2, 5), dtype=flow.float, trainable=False): return flow.get_variable( name=name, shape=shape, dtype=dtype, trainable=trainable, initializer=flow.random_uniform_initializer(), )
def _embedding(name, ids, embedding_size, vocab_size, split_axis=0): ids = flow.parallel_cast(ids, distribute=flow.distribute.broadcast()) params = flow.get_variable( name=name, shape=(vocab_size, embedding_size), initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05), distribute=flow.distribute.split(split_axis), ) embedding = flow.gather(params=params, indices=ids) embedding = flow.reshape(embedding, shape=(-1, embedding.shape[-1] * embedding.shape[-2])) return embedding
def gen_var(x: tp.Numpy.Placeholder(shape=shape, dtype=dtype)) -> tp.Numpy: var = flow.get_variable( name="var", shape=shape, dtype=dtype, initializer=flow.random_uniform_initializer(), ) y = var + x flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler([], [lr]), momentum=0).minimize(y) return var
def FuseCastScaleJob(x: oft.Numpy.Placeholder( shape, dtype=in_type)) -> Tuple[oft.Numpy, oft.Numpy]: with flow.scope.placement(device, "0:0-0"): scale = flow.get_variable( "scale", shape=(1, ), dtype=out_type, initializer=flow.random_uniform_initializer(), trainable=False, ) loss = flow.cast(x, dtype=out_type) * scale return (loss, scale)
def matmul_job() -> typing.Tuple[flow.typing.Numpy, flow.typing.Numpy, flow.typing.Numpy, flow.typing.Numpy]: a_var = flow.get_variable( "a", shape=a_shape, dtype=flow.float32, initializer=flow.random_uniform_initializer(minval=0, maxval=1), trainable=True, ) b_var = flow.get_variable( "b", shape=b_shape, dtype=flow.float32, initializer=flow.random_uniform_initializer(minval=0, maxval=1), trainable=True, ) flow.watch_diff(a_var, test_global_storage.Setter("a_diff")) flow.watch_diff(b_var, test_global_storage.Setter("b_diff")) if dtype is flow.float16: a = flow.amp_white_identity(a_var) b = flow.amp_white_identity(b_var) else: a = a_var b = b_var c = flow.matmul(a, b, trans_a, trans_b, alpha) add_to = flow.get_variable( "c", shape=c.shape, dtype=flow.float32, initializer=flow.random_uniform_initializer(minval=-1, maxval=1), trainable=True, ) if test_add_to_output: flow.watch_diff(add_to, test_global_storage.Setter("add_to_diff")) if dtype is flow.float16: add_to = flow.amp_white_identity(add_to) c = c + add_to flow.watch_diff(c, test_global_storage.Setter("c_diff")) get_optimizer().minimize(c) return (a_var, b_var, add_to, c)