def test_fused_scale_tril_fw_bw_job( x: oft.Numpy.Placeholder(shape, dtype=flow_type) ): with flow.scope.placement(device, "0:0"): x_var = flow.get_variable( name="xv", shape=(1,), dtype=flow.float, initializer=flow.zeros_initializer(), ) x += flow.cast(x_var, dtype=flow_type) if type_name == "float16": out = flow.cast( flow.math.fused_scale_tril( flow.cast(x, flow.float16), diagonal, scale=scale ), flow.float, ) else: out = flow.math.fused_scale_tril(x, diagonal, scale=scale) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0 ).minimize(out) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(out, test_global_storage.Setter("out")) flow.watch_diff(out, test_global_storage.Setter("out_diff")) return out
def SoftmaxJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=x_shape, dtype=dtype, initializer=flow.random_uniform_initializer(minval=-1.0, maxval=1.0), trainable=True, ) x1 = x x = flow.identity(x) if data_type == "float16": loss = flow.cast( flow.nn.softmax(flow.cast(x, dtype=flow.float16), axis=axis), dtype=flow.float, ) else: loss = flow.nn.softmax(x, axis=axis) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) total_loss = loss * x1 flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(total_loss) return loss
def SparseSoftmaxCrossEntropyWithLogitsJob(labels: oft.Numpy.Placeholder( (batch_size, ), dtype=type_name_to_flow_type[label_type])): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=(batch_size, num_classes), dtype=type_name_to_flow_type[data_type], initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) with flow.scope.placement(device_type, "0:0-3"): labels = flow.parallel_cast(labels, distribute=flow.distribute.broadcast()) logits = flow.parallel_cast( x, distribute=flow.distribute.split(len(x.shape) - 1)) loss = flow.nn.distributed_sparse_softmax_cross_entropy_with_logits( labels, logits) loss = flow.math.square(loss) with flow.scope.placement(device_type, "0:0"): loss = flow.identity(loss) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def test_masked_fill_fw_bw_job( x: oft.Numpy.Placeholder(x_shape, dtype=flow_type), mask: oft.Numpy.Placeholder(mask_shape, dtype=flow_type), ): with flow.scope.placement(device, "0:0"): y = flow.get_variable( name="vx", shape=(1, ), dtype=flow.float, initializer=flow.zeros_initializer(), ) x += flow.cast(y, flow_type) mask = flow.cast(mask, dtype=flow.int8) if type_name == "float16": out = flow.cast( flow.masked_fill(flow.cast(x, flow.float16), mask, value), flow.float, ) else: out = flow.masked_fill(x, mask, value) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(out) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(out, test_global_storage.Setter("out")) flow.watch_diff(out, test_global_storage.Setter("out_diff")) return out
def TestMultiInputJob(): with flow.scope.placement("gpu", "0:0"): x1 = flow.get_variable( "x1", shape=shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) x2 = flow.get_variable( "x2", shape=shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) loss = TestMultiInput(x1, x2) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0 ).minimize(loss) flow.watch(x1, test_global_storage.Setter("x1")) flow.watch_diff(x1, test_global_storage.Setter("x1_diff")) flow.watch(x2, test_global_storage.Setter("x2")) flow.watch_diff(x2, test_global_storage.Setter("x2_diff")) return loss
def UpsampleJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "input", shape=input_shape, dtype=type_name_to_flow_type[dtype], initializer=flow.random_uniform_initializer(minval=2, maxval=5), trainable=True, ) loss = flow.layers.upsample_2d( x, size=size, data_format=data_format, interpolation=interpolation, align_corners=align_corners, ) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def DropoutJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=x_shape, dtype=dtype, initializer=flow.random_uniform_initializer(minval=-1, maxval=1), trainable=True, ) if data_type == "float16": x = flow.cast(flow.cast(x, flow.float16), dtype) of_out = flow.cast( flow.nn.dropout(flow.cast(x, flow.float16), rate=rate, name="dropout"), dtype, ) else: of_out = flow.nn.dropout(x, rate=rate, name="dropout") loss = flow.math.square(of_out) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(of_out, test_global_storage.Setter("out")) flow.watch_diff(of_out, test_global_storage.Setter("out_diff")) return loss
def PartialFcJob(labels: oft.Numpy.Placeholder( (batch_size, ), dtype=type_name_to_flow_type[label_type])): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x-weight", shape=(num_classes, 128), dtype=flow.float, initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) with flow.scope.placement(device_type, "0:0-3"): lebels_distribute = flow.distribute.broadcast() weight_distribute = flow.distribute.split(0) ( maped_label, sampled_label, sampled_weight, ) = flow.distributed_partial_fc_sample( weight=x.with_distribute(weight_distribute), label=labels.with_distribute(lebels_distribute), num_sample=num_sample, ) with flow.scope.placement(device_type, "0:0"): sampled_weight = flow.identity(sampled_weight) loss = flow.math.square(sampled_weight) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch_diff(sampled_weight, test_global_storage.Setter("sampled_weight_diff")) return (x, maped_label, sampled_label, sampled_weight)
def test_job(x: oft.Numpy.Placeholder(input_shape, dtype=flow.float32)): v = flow.get_variable( name="v", shape=(1, ), dtype=flow.float32, initializer=flow.zeros_initializer(), ) x = x + v x1 = flow.identity(x) x2 = flow.identity(x) flow.watch_diff(x1, test_global_storage.Setter("x1_diff")) flow.watch_diff(x2, test_global_storage.Setter("x2_diff")) x1 = flow.cast(x1, data_type) x2 = flow.cast(x2, data_type) y1 = flow.layers.batch_normalization_relu(x1, axis=axis, name="BN1") y2 = flow.math.relu( flow.layers.batch_normalization(x2, axis=axis, name="BN2")) y1 = flow.cast(y1, flow.float32) y2 = flow.cast(y2, flow.float32) flow.watch(y1, test_global_storage.Setter("y1")) flow.watch(y2, test_global_storage.Setter("y2")) y1 = flow.where(flow.math.greater(y2, v), y1, v) y2 = flow.where(flow.math.greater(y1, v), y2, v) loss = y1 + y2 flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler([], [0.001]), momentum=0).minimize(flow.math.reduce_sum(loss)) return loss
def Matmul( x: tp.Numpy.Placeholder((4, 4), dtype=flow.float32), y: tp.Numpy.Placeholder((4, 4), dtype=flow.float32), ) -> tp.Numpy: s = flow.matmul(x, y) flow.watch(s, Watch) z = flow.matmul(s, x) return z
def FlowJob( value: oft.Numpy.Placeholder(x_shape), bias: oft.Numpy.Placeholder(bias_shape) ): with flow.scope.placement(device_type, "0:0"): value += flow.get_variable( name="v1", shape=(1,), dtype=flow.float, initializer=flow.zeros_initializer(), ) bias += flow.get_variable( name="v2", shape=(1,), dtype=flow.float, initializer=flow.zeros_initializer(), ) x1 = flow.identity(value) x2 = flow.identity(value) bias1 = flow.identity(bias) bias2 = flow.identity(bias) flow.watch_diff(x1, test_global_storage.Setter("x1_diff")) flow.watch_diff(x2, test_global_storage.Setter("x2_diff")) flow.watch_diff(bias1, test_global_storage.Setter("bias1_diff")) flow.watch_diff(bias2, test_global_storage.Setter("bias2_diff")) if data_type == "float16": y1 = flow.cast( flow.math.gelu( flow.nn.bias_add( flow.cast(x1, dtype=flow.float16), flow.cast(bias1, dtype=flow.float16), data_format=data_format, ) ), dtype=flow.float, ) y2 = flow.cast( flow.nn.fused_bias_add_gelu( flow.cast(x2, dtype=flow.float16), flow.cast(bias2, dtype=flow.float16), data_format=data_format, ), dtype=flow.float, ) else: y1 = flow.math.gelu( flow.nn.bias_add(x1, bias1, data_format=data_format) ) y2 = flow.nn.fused_bias_add_gelu(x2, bias2, data_format=data_format) flow.watch(y1, test_global_storage.Setter("y1")) flow.watch(y2, test_global_storage.Setter("y2")) flow.watch_diff(y1, test_global_storage.Setter("y1_diff")) flow.watch_diff(y2, test_global_storage.Setter("y2_diff")) loss = y1 + y2 flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.001]), momentum=0 ).minimize(flow.math.reduce_sum(loss)) return loss
def slice_fn(): with flow.scope.placement(device_tag, "0:0-{}".format(device_num - 1)): var = flow.get_variable( name="var", shape=var_shape, dtype=flow_dtype, initializer=flow.random_uniform_initializer(-10, 10, dtype=flow_dtype), distribute=flow.distribute.split(split_axis), ) flow.watch(var, test_global_storage.Setter("var")) ret = flow.experimental.logical_slice(var, slice_tuples) return ret
def MatmulJob(): with flow.scope.placement(device_type, "0:0"): a = flow.get_variable( "a", shape=a_shape, dtype=dtype, initializer=flow.random_uniform_initializer(minval=0, maxval=1), trainable=True, ) b = flow.get_variable( "b", shape=b_shape, dtype=dtype, initializer=flow.random_uniform_initializer(minval=0, maxval=1), trainable=True, ) if data_type == "float16": out = flow.matmul( flow.cast(a, dtype=flow.float16), flow.cast(b, dtype=flow.float16), transpose_a, transpose_b, alpha, ) c = flow.get_variable( "c", shape=out.shape, dtype=dtype, initializer=flow.random_uniform_initializer(minval=-1, maxval=1), trainable=True, ) loss = flow.cast(out + flow.cast(c, dtype=flow.float16), dtype=flow.float) else: out = flow.matmul(a, b, transpose_a, transpose_b, alpha) c = flow.get_variable( "c", shape=out.shape, dtype=dtype, initializer=flow.random_uniform_initializer(minval=-1, maxval=1), trainable=True, ) loss = out + c flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(a, test_global_storage.Setter("a")) flow.watch_diff(a, test_global_storage.Setter("a_diff")) flow.watch(b, test_global_storage.Setter("b")) flow.watch_diff(b, test_global_storage.Setter("b_diff")) flow.watch(c, test_global_storage.Setter("c")) flow.watch_diff(c, test_global_storage.Setter("c_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def FlattenJob() -> flow.typing.Numpy: with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "in", shape=input_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=2, maxval=5), trainable=True, ) loss = flow.flatten(x, start_dim=start_dim, end_dim=end_dim) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) return loss
def ReduceMinJob(x: oft.Numpy.Placeholder(input_shape, dtype=flow.float)): with flow.scope.placement(device_type, "0:0"): x += flow.get_variable( name="v1", shape=input_shape, dtype=flow.float, initializer=flow.zeros_initializer(), ) loss = flow.math.reduce_min(x, axis=axis, keepdims=keepdims) loss = flow.identity(loss) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def CastJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "in", shape=input_shape, dtype=type_name_to_flow_type[dtype], initializer=flow.random_uniform_initializer(), trainable=True, ) loss = flow.cast(x, dtype=flow.float) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def SquareJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=x_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) loss = flow.math.square(x) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def ActivationJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=shape, dtype=data_type, initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) loss = of_activation_map[activation_type](x) lr_scheduler = flow.optimizer.PiecewiseConstantScheduler([], [0.0001]) flow.optimizer.SGD(lr_scheduler, momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def ReduceSumJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "in", shape=input_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=2, maxval=5), trainable=True, ) loss = flow.math.reduce_sum(x, axis=axis, keepdims=keepdims) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def LeakyReluJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=x_shape, dtype=type_name_to_flow_type[data_type], initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) loss = flow.nn.leaky_relu(x, alpha=alpha) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def RunConvBias(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=x_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=0, maxval=100), trainable=True, ) if data_format == "NCHW": weight_shape = (filters, x.shape[1] // groups, kernel_size, kernel_size) else: weight_shape = (filters, kernel_size, kernel_size, x.shape[3] // groups) weight = flow.get_variable( "conv-weight", shape=weight_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=0, maxval=100), ) bias = flow.get_variable( "conv-bias", shape=(filters, ), dtype=flow.float, initializer=flow.random_uniform_initializer(minval=0, maxval=100), ) loss = flow.nn.conv2d( x, weight, bias=bias, strides=[stride, stride], padding=padding, dilations=[1, 1], groups=groups, name="conv", ) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(weight, test_global_storage.Setter("weight")) flow.watch_diff(weight, test_global_storage.Setter("weight_diff")) flow.watch(bias, test_global_storage.Setter("bias")) flow.watch_diff(bias, test_global_storage.Setter("bias_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def UpsampleJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "input", shape=input_shape, dtype=type_name_to_flow_type[dtype], initializer=flow.random_uniform_initializer(minval=2, maxval=5), trainable=False, ) loss = flow.layers.upsample_2d( x, size=size, data_format=data_format, interpolation=interpolation, align_corners=align_corners, ) flow.watch(x, test_global_storage.Setter("x1")) flow.watch(loss, test_global_storage.Setter("loss1")) return loss
def SigmoidCrossEntropyWithLogitsJob(labels: oft.Numpy.Placeholder( shape, dtype)): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=shape, dtype=type_name_to_flow_type[data_type], initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) loss = flow.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=x) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def two_stage_reduce_job(x: oft.Numpy.Placeholder((4, 20, 20, 20))): with flow.scope.placement(device_type, "0:0"): x += flow.get_variable( name="v1", shape=(1,), dtype=flow.float, initializer=flow.zeros_initializer(), ) with flow.scope.placement(device_type, "0:0-3"): loss = flow_func( x.with_distribute(flow.distribute.split(split_axis)), axis=axis, keepdims=True, ) loss = flow.identity(loss) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0 ).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) return loss
def test_job( x: oft.Numpy.Placeholder(input_shape, dtype=flow.float32), labels: oft.Numpy.Placeholder(label_shape, dtype=flow.int32), ): with flow.scope.placement(device_type, "0:0"): v = flow.get_variable( name="v", shape=(1,), dtype=flow.float32, initializer=flow.zeros_initializer(), ) x = x + v x1 = flow.identity(x) x2 = flow.identity(x) flow.watch_diff(x1, test_global_storage.Setter("x1_diff")) flow.watch_diff(x2, test_global_storage.Setter("x2_diff")) x1 = flow.cast(x1, data_type) x2 = flow.cast(x2, data_type) with flow.scope.placement(device_type, "0:0-3"): y1 = ( flow.combined_margin_loss( x1.with_distribute(flow.distribute.split(1)), labels.with_distribute(flow.distribute.broadcast()), m1, m2, m3, ) * s ) y2 = margin_loss(m1, m2, m3, s, x2, labels) with flow.scope.placement(device_type, "0:0"): y1 = flow.cast(y1, flow.float) y2 = flow.cast(y2, flow.float) flow.watch(y1, test_global_storage.Setter("y1")) flow.watch(y2, test_global_storage.Setter("y2")) loss = y1 + y2 flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.001]), momentum=0 ).minimize(flow.math.reduce_sum(loss)) return loss
def ConcatJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=x_shape, dtype=type_name_to_flow_type[dtype], initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) y = flow.get_variable( "y", shape=y_shape, dtype=type_name_to_flow_type[dtype], initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) x = flow.cast_to_current_logical_view(x) y = flow.cast_to_current_logical_view(y) loss = flow.concat([x, y], axis) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0 ).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(y, test_global_storage.Setter("y")) flow.watch_diff(y, test_global_storage.Setter("y_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def test_element_wise_mul_job( x: oft.Numpy.Placeholder(shape, dtype=flow.float), y: oft.Numpy.Placeholder(shape, dtype=flow.float), ): with flow.scope.placement(device, "0:0"): x += flow.get_variable( name="vx", shape=(1, ), dtype=flow.float, initializer=flow.zeros_initializer(), ) y += flow.get_variable( name="vy", shape=(1, ), dtype=flow.float, initializer=flow.zeros_initializer(), ) x = flow.cast(x, dtype=flow_type) y = flow.cast(y, dtype=flow_type) out = flow.math.multiply(x, y) out = flow.cast(out, dtype=flow.float) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [0.0001]), momentum=0).minimize(out) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(y, test_global_storage.Setter("y")) flow.watch_diff(y, test_global_storage.Setter("y_diff")) flow.watch(out, test_global_storage.Setter("out")) flow.watch_diff(out, test_global_storage.Setter("out_diff")) return out
def SplitLikeJob(x: oft.Numpy.Placeholder(x_shape, dtype=flow.float)): v = flow.get_variable( "x", shape=x_shape, dtype=flow.float, initializer=flow.constant_initializer(0), trainable=True, ) x += v like0 = flow.constant(0, dtype=flow.float, shape=like0_shape) like1 = flow.constant(0, dtype=flow.float, shape=like1_shape) with flow.scope.placement("gpu", "0:0"): (y0, y1) = split_like(x, [like0, like1], "split_like") loss = y0 flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0 ).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return (y0, y1)
def SparseSoftmaxCrossEntropyWithLogitsJob( labels: oft.Numpy.Placeholder( (batch_size,), dtype=type_name_to_flow_type[label_type] ) ): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=(batch_size, num_classes), dtype=type_name_to_flow_type[data_type], initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) prediction = flow.nn.softmax(logits=x) loss = flow.nn.sparse_cross_entropy(labels=labels, prediction=prediction) loss = flow.math.square(loss) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0 ).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def ConvJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=x_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=0, maxval=100), trainable=True, ) if data_format == "NCDHW": weight_shape = ( filters, x.shape[1] // groups, kernel_size, kernel_size, kernel_size, ) else: weight_shape = ( filters, kernel_size, kernel_size, kernel_size, x.shape[4] // groups, ) weight = flow.get_variable( "conv-weight", shape=weight_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=0, maxval=100), ) loss = flow.nn.conv3d( x, weight, strides=[stride_d, stride_h, stride_w], padding=of_padding, data_format=data_format, dilations=[dilation_d, dilation_h, dilation_w], groups=groups, ) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0 ).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(weight, test_global_storage.Setter("weight")) flow.watch_diff(weight, test_global_storage.Setter("weight_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss