def avg_pooling_2d_k2s2_same_nhwc(x=flow.FixedTensorDef((2, 3, 5, 4))): x += flow.get_variable( name="v1", shape=(1, 1), dtype=flow.float, initializer=flow.zeros_initializer(), ) return flow.nn.avg_pool2d( x, ksize=2, strides=2, padding="SAME", data_format="NHWC" )
def max_pooling_2d_k3s1_valid_nchw(x: tp.Numpy.Placeholder((2, 3, 5, 4))): x += flow.get_variable( name="v1", shape=(1, 1), dtype=flow.float, initializer=flow.zeros_initializer(), ) return flow.nn.max_pool2d( x, ksize=3, strides=1, padding="VALID", data_format="NCHW" )
def avg_pooling_2d_k3s1_valid_nchw(x=flow.FixedTensorDef((2, 3, 5, 4))): x += flow.get_variable( name="v1", shape=(1, 1), dtype=flow.float, initializer=flow.zeros_initializer(), ) return flow.nn.avg_pool2d( x, ksize=3, strides=1, padding="VALID", data_format="NCHW" )
def max_pooling_2d_k2s2_same_nhwc(x: tp.Numpy.Placeholder((2, 3, 5, 4))): x += flow.get_variable( name="v1", shape=(1, 1), dtype=flow.float, initializer=flow.zeros_initializer(), ) return flow.nn.max_pool2d( x, ksize=2, strides=2, padding="SAME", data_format="NHWC" )
def test_job( x: oft.Numpy.Placeholder(input_shape, dtype=flow.float32), addend: oft.Numpy.Placeholder(input_shape, dtype=flow.float32), ): v = flow.get_variable( name="v", shape=(1, ), dtype=flow.float32, initializer=flow.zeros_initializer(), ) x = x + v addend = addend + v x1 = flow.identity(x) x2 = flow.identity(x) addend1 = flow.identity(addend) addend2 = flow.identity(addend) flow.watch_diff(x1, test_global_storage.Setter("x1_diff")) flow.watch_diff(x2, test_global_storage.Setter("x2_diff")) flow.watch_diff(addend1, test_global_storage.Setter("addend1_diff")) flow.watch_diff(addend2, test_global_storage.Setter("addend2_diff")) x1 = flow.cast(x1, data_type) x2 = flow.cast(x2, data_type) addend1 = flow.cast(addend1, data_type) addend2 = flow.cast(addend2, data_type) y1 = flow.layers.batch_normalization_add_relu(x1, addend=addend1, axis=axis, name="BN1") y2 = flow.math.relu( flow.layers.batch_normalization(x2, axis=axis, name="BN2") + addend2) y1 = flow.cast(y1, flow.float32) y2 = flow.cast(y2, flow.float32) flow.watch(y1, test_global_storage.Setter("y1")) flow.watch(y2, test_global_storage.Setter("y2")) y1 = flow.where(flow.math.greater(y2, v), y1, v) y2 = flow.where(flow.math.greater(y1, v), y2, v) loss = y1 + y2 flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler([], [0.001]), momentum=0).minimize(flow.math.reduce_sum(loss)) return loss
def conv2d_layer( name, input, filters, weight_initializer, kernel_size=3, strides=1, padding="SAME", data_format="NCHW", dilation_rate=1, activation="Relu", use_bias=True, bias_initializer=flow.zeros_initializer(), weight_regularizer=_get_regularizer(), # weight_decay bias_regularizer=_get_regularizer(), bn=True, ): weight_shape = (filters, input.shape[1], kernel_size, kernel_size) if data_format == "NCHW" else (filters, kernel_size, kernel_size, input.shape[3]) weight = flow.get_variable( name + "_weight", shape=weight_shape, dtype=input.dtype, initializer=weight_initializer, ) output = flow.nn.conv2d(input, weight, strides, padding, data_format, dilation_rate, name=name) if use_bias: bias = flow.get_variable( name + "_bias", shape=(filters, ), dtype=input.dtype, initializer=bias_initializer, ) output = flow.nn.bias_add(output, bias, data_format) if activation is not None: if activation == "Relu": if bn: output = _batch_norm(output, name + "_bn", True, data_format) output = flow.nn.relu(output) else: output = flow.nn.relu(output) else: raise NotImplementedError return output
def conv2d_layer(name, input, filters, kernel_size=3, strides=1, padding="SAME", data_format="NCHW", dilation_rate=1, activation="Relu", use_bias=True, weight_initializer=flow.variance_scaling_initializer( 2, 'fan_out', 'random_normal', data_format="NCHW"), bias_initializer=flow.zeros_initializer(), bn=True, reuse=False, trainable=True): name_ = name if reuse == False else name + "_reuse" weight_shape = (filters, input.shape[1], kernel_size, kernel_size) weight = flow.get_variable(name + "_weight", shape=weight_shape, dtype=input.dtype, initializer=weight_initializer, trainable=trainable) output = flow.nn.conv2d(input, weight, strides, padding, data_format, dilation_rate, name=name_) if use_bias: bias = flow.get_variable(name + "_bias", shape=(filters, ), dtype=input.dtype, initializer=bias_initializer, trainable=trainable) output = flow.nn.bias_add(output, bias, data_format) if activation is not None: if activation == "Relu": if bn: # use of_layers(layers) batch_norm output = layers.batch_norm(output, name + "_bn", reuse=reuse) output = flow.nn.relu(output) else: output = flow.nn.relu(output) else: raise NotImplementedError return output
def oneflow_Xmum( of_input_1: tp.Numpy.Placeholder(shape=input_1.shape, dtype=value_type["of_type"]), of_input_2: tp.Numpy.Placeholder(shape=input_2.shape, dtype=value_type["of_type"]), ) -> tp.Numpy: with flow.scope.placement(device_type, "0:0"): v1 = flow.get_variable( shape=input_1.shape, dtype=value_type["of_type"], initializer=flow.zeros_initializer(), name="x1_var", ) x1_var = of_input_1 + v1 if not dx_only: v2 = flow.get_variable( shape=input_2.shape, dtype=value_type["of_type"], initializer=flow.zeros_initializer(), name="x2_var", ) x2_var = of_input_2 + v2 else: x2_var = flow.constant(value=1.5, shape=of_input_2.shape, dtype=value_type["of_type"]) flow.watch_diff(x1_var, assert_prediction_grad) # Only Compare input1 Grad if compare_type == "maximum": of_Xmum_out = flow.math.maximum(x1_var, x2_var) elif compare_type == "minimum": of_Xmum_out = flow.math.minimum(x1_var, x2_var) with flow.scope.placement(device_type, "0:0"): flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [1e-3]), momentum=0).minimize(of_Xmum_out) return of_Xmum_out
def oneflow_bce_with_logits_loss( of_input: tp.Numpy.Placeholder(shape=input.shape), of_target: tp.Numpy.Placeholder(shape=target.shape), of_weight: tp.Numpy.Placeholder(shape=weight.shape), of_pos_weight: tp.Numpy.Placeholder(shape=pos_weight.shape), ) -> Dict[str, tp.Numpy]: with flow.scope.placement(device_type, "0:0"): v = flow.get_variable( shape=input.shape, dtype=flow.float32, initializer=flow.zeros_initializer(), name="v", ) x_var = of_input + v flow.watch_diff(x_var, assert_prediction_grad) bceloss = flow.nn.BCEWithLogitsLoss( x_var, of_target, of_weight, of_pos_weight, reduction="none", name="of_mseloss", ) bceloss_mean = flow.nn.BCEWithLogitsLoss( x_var, of_target, of_weight, of_pos_weight, reduction="mean", name="of_mseloss_reduce_mean", ) bceloss_sum = flow.nn.BCEWithLogitsLoss( x_var, of_target, of_weight, of_pos_weight, reduction="sum", name="of_mseloss_reduce_sum", ) # Because our gradient is use "mean" mode to compute with flow.scope.placement(device_type, "0:0"): flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [1e-3]), momentum=0).minimize(bceloss_mean) return { "of_bce_with_logits_loss": bceloss, "of_bce_with_logits_loss_mean": bceloss_mean, "of_bce_with_logits_loss_sum": bceloss_sum, }
def CompareJob(x: tp.Numpy.Placeholder((5, ), dtype=flow.float32)) -> tp.Numpy: x += flow.get_variable( name="v1", shape=(5, ), dtype=flow.float, initializer=flow.zeros_initializer(), ) loss = 3 * x * x flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler([], [1e-4]), momentum=0).minimize(loss) flow.watch_diff(x, global_storage_setter("x2_diff")) return loss
def load_synthetic(image_size, batch_size): label = flow.data.decode_random( shape=(), dtype=flow.int32, batch_size=batch_size, initializer=flow.zeros_initializer(flow.int32), ) image = flow.data.decode_random(shape=(3, image_size, image_size), dtype=flow.float, batch_size=batch_size) return label, image
def do_gather_nd(x, index): x_var = flow.get_variable( "params", shape=(1,), dtype=x_dtype, initializer=flow.zeros_initializer(), ) x = x + flow.cast_to_current_logical_view(x_var) y = flow.gather_nd(x, index) if need_grad: flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [1e-3]), momentum=0 ).minimize(y) if callable(comp_diff_fn): flow.watch_diff(x, comp_diff_fn) return y
def conv2d_layer( name, input, filters, kernel_size=1, strides=1, padding="VALID", data_format="NCHW", dilation_rate=1, activation="Relu", use_bias=True, weight_initializer=flow.variance_scaling_initializer( 2, 'fan_out', 'random_normal', data_format="NCHW"), bias_initializer=flow.zeros_initializer(), bn=True, ): weight_shape = (filters, input.shape[1], kernel_size, kernel_size) weight = flow.get_variable( name + "_weight", shape=weight_shape, dtype=input.dtype, initializer=weight_initializer, ) output = flow.nn.conv2d(input, weight, strides, padding, data_format, dilation_rate, name=name) if use_bias: bias = flow.get_variable( name + "_bias", shape=(filters, ), dtype=input.dtype, initializer=bias_initializer, ) output = flow.nn.bias_add(output, bias, data_format) if activation is not None: if activation == "Relu": if bn: output = _batch_norm(output, name + "_bn") # flow.watch(output) output = flow.nn.relu(output) else: output = flow.nn.relu(output) else: raise NotImplementedError return output
def loss_con(self, one_hot_labels, feature): branch = feature fc_part1 = flow.layers.dense( flow.reshape(branch, (branch.shape[0], -1)), units=8, # 车辆颜色类别8 use_bias=True, kernel_initializer=flow.variance_scaling_initializer( 2, 'fan_in', 'random_normal'), bias_initializer=flow.zeros_initializer(), name="fc1", ) loss_con = flow.nn.softmax_cross_entropy_with_logits( one_hot_labels, fc_part1, name="softmax_loss1") return loss_con
def load_synthetic(config): batch_size = config.train_batch_size image_size = 112 label = flow.data.decode_random( shape=(), dtype=flow.int32, batch_size=batch_size, initializer=flow.zeros_initializer(flow.int32), ) image = flow.data.decode_random( shape=(image_size, image_size, 3), dtype=flow.float, batch_size=batch_size, ) return label, image
def FlowJob( value: oft.Numpy.Placeholder(value.shape), bias: oft.Numpy.Placeholder(bias.shape), ): with flow.scope.placement(device_type, "0:0"): value += flow.get_variable( name="v1", shape=(1, ), dtype=flow.float, initializer=flow.zeros_initializer(), ) bias += flow.get_variable( name="v2", shape=(1, ), dtype=flow.float, initializer=flow.zeros_initializer(), ) loss = flow.nn.bias_add(value, bias, *flow_args) flow.losses.add_loss(loss) flow.watch_diff(value, test_global_storage.Setter("value_diff")) flow.watch_diff(bias, test_global_storage.Setter("bias_diff")) return loss
def load_synthetic(args): total_device_num = args.num_nodes * args.gpu_num_per_node batch_size = total_device_num * args.batch_size_per_device label = flow.data.decode_random( shape=(), dtype=flow.int32, batch_size=batch_size, initializer=flow.zeros_initializer(flow.int32), ) image = flow.data.decode_random( shape=(args.image_size, args.image_size, 3), dtype=flow.float, batch_size=batch_size ) return label, image
def instance_norm(input, name_prefix, trainable=True): (mean, variance) = flow.nn.moments(input, [2, 3], keepdims=True) gamma = flow.get_variable(name_prefix + "_gamma", shape=(1, input.shape[1], 1, 1), dtype=input.dtype, initializer=flow.ones_initializer(), trainable=trainable) beta = flow.get_variable(name_prefix + "_beta", shape=(1, input.shape[1], 1, 1), dtype=input.dtype, initializer=flow.zeros_initializer(), trainable=trainable) epsilon = 1e-3 normalized = (input - mean) / flow.math.sqrt(variance + epsilon) return gamma * normalized + beta
def oneflow_Xmum( of_input_1: tp.ListNumpy.Placeholder(shape=data_shape), of_input_2: tp.ListNumpy.Placeholder(shape=data_shape), ) -> tp.ListNumpy: with flow.scope.placement(device_type, "0:0"): v1 = flow.get_variable( shape=(1, ), dtype=flow.float32, initializer=flow.zeros_initializer(), name="x1_var", ) v1 = flow.cast_to_current_logical_view(v1) x1_var = of_input_1 + v1 v2 = flow.get_variable( shape=(1, ), dtype=flow.float32, initializer=flow.zeros_initializer(), name="x2_var", ) v2 = flow.cast_to_current_logical_view(v2) x2_var = of_input_2 + v2 flow.watch_diff(x1_var, assert_prediction_grad) # Only Compare input1 Grad if compare_type == "maximum": of_Xmum_out = flow.math.maximum(x1_var, x2_var) elif compare_type == "minimum": of_Xmum_out = flow.math.minimum(x1_var, x2_var) with flow.scope.placement(device_type, "0:0"): flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [1e-3]), momentum=0).minimize(of_Xmum_out) return of_Xmum_out
def op_function(x: tp.Numpy.Placeholder(input.shape, dtype=value_type)): with flow.scope.placement(device_type, "0:0"): x += flow.get_variable( name="input", shape=input.shape, dtype=value_type, initializer=flow.zeros_initializer(), ) out = flow.reflection_pad2d(x, padding) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0]), momentum=0 ).minimize(out) flow.watch_diff(x, _compare_diff) return out
def FlowJob(x: oft.Numpy.Placeholder(x.shape)): with flow.scope.placement(device_type, "0:0"): x += flow.get_variable( name="v1", shape=(1, ), dtype=flow.float, initializer=flow.zeros_initializer(), ) loss = flow_op(x, *flow_args) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler([], [0]), momentum=0).minimize(loss) flow.watch_diff(x, test_global_storage.Setter("x_diff")) return loss
def op_function(x: tp.Numpy.Placeholder(input.shape, dtype=flow.float32)): with flow.scope.placement(device_type, "0:0"): x += flow.get_variable( name="input", shape=input.shape, dtype=flow.float32, initializer=flow.zeros_initializer(), ) y_int32 = flow.replication_pad2d(x, padding) y_fp32 = flow.cast(y_int32, dtype=flow.float32) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0]), momentum=0 ).minimize(y_fp32) flow.watch_diff(x, _compare_diff) return y_fp32
def oneflow_kldivloss( of_input: tp.Numpy.Placeholder(shape=input.shape), of_target: tp.Numpy.Placeholder(shape=target.shape), ) -> Dict[str, tp.Numpy]: with flow.scope.placement(device_type, "0:0"): v = flow.get_variable( shape=input.shape, dtype=flow.float32, initializer=flow.zeros_initializer(), name="x_var", ) of_input = of_input + v flow.watch_diff(of_input, assert_prediction_grad) of_kldivloss = flow.nn.KLDivLoss( of_input, of_target, log_target=log_target, reduction="none", name="kldivloss", ) of_kldivloss_mean = flow.nn.KLDivLoss( of_input, of_target, log_target=log_target, reduction="mean", name="kldivloss_mean", ) of_kldivloss_sum = flow.nn.KLDivLoss( of_input, of_target, log_target=log_target, reduction="sum", name="kldivloss_sum", ) with flow.scope.placement(device_type, "0:0"): flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [1e-3]), momentum=0).minimize(of_kldivloss_mean) return { "of_kldivloss": of_kldivloss, "of_kldivloss_mean": of_kldivloss_mean, "of_kldivloss_sum": of_kldivloss_sum, }
def ReduceMaxJob(x: oft.Numpy.Placeholder(input_shape, dtype=flow.float)): with flow.scope.placement(device_type, "0:0"): x += flow.get_variable( name="v1", shape=input_shape, dtype=flow.float, initializer=flow.zeros_initializer(), ) loss = flow.math.reduce_max(x, axis=axis, keepdims=keepdims) loss = flow.identity(loss) flow.losses.add_loss(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def PolyValJob(x: tp.Numpy.Placeholder(shape=in_shape)): with flow.scope.placement(device_type, "0:0"): x += flow.get_variable( name="x", shape=in_shape, dtype=flow_data_type, initializer=flow.zeros_initializer(), trainable=True, ) flow.watch_diff(x, assert_prediction_grad) out = flow.math.polyval(coeffs, x) with flow.scope.placement(device_type, "0:0"): flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [1e-4]), momentum=0).minimize(out) return out
def _build_dense(self, x, unit, name="dense_"): """ Build Dense Layer :param x: :return: """ self.init_range = 0.2 self.init = flow.truncated_normal_initializer(self.init_range) self.reg = flow.regularizers.l2(0.01) return flow.layers.dense(x, units=unit, kernel_initializer=self.init, kernel_regularizer=self.reg, bias_initializer=flow.zeros_initializer(), bias_regularizer=self.reg, name=name+"w")
def cast_to_static_shape_fn(x: flow.typing.ListNumpy.Placeholder( shape=shape, dtype=dtype)) -> flow.typing.ListNumpy: x_var = flow.get_variable( name="x_var", shape=(1, ), dtype=flow.float32, initializer=flow.zeros_initializer(), ) x = x + flow.cast(x_var, dtype=dtype) y = flow.cast_to_static_shape(x) test_case.assertFalse(y.is_dynamic) if require_grad: flow.watch_diff(x, compare_diff_fn) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [1e-3]), momentum=0).minimize(y) return y
def DynamicConvJob(x: oft.ListNumpy.Placeholder((10, 3, 100, 100))): with flow.scope.placement(device_type, "0:0"): x_var = flow.get_variable( name="v1", shape=(1, ), dtype=flow.float, initializer=flow.zeros_initializer(), ) x_var = flow.cast_to_current_logical_view(x_var) x += x_var if data_format == "NCHW": weight_shape = (filters, x_shape[1] // groups, kernel_size, kernel_size) else: weight_shape = (filters, kernel_size, kernel_size, x_shape[3] // groups) weight = flow.get_variable( "conv-weight", shape=weight_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(minval=0, maxval=100), ) weight = flow.cast_to_current_logical_view(weight) loss = flow.nn.conv2d( x, weight, strides=[stride, stride], padding=of_padding, data_format=data_format, dilations=[1, 1], groups=groups, ) flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [1e-4]), momentum=0).minimize(loss) flow.watch(x, global_storage_setter("x")) flow.watch_diff(x, global_storage_setter("x_diff")) flow.watch(weight, global_storage_setter("weight")) flow.watch_diff(weight, global_storage_setter("weight_diff")) flow.watch(loss, global_storage_setter("loss")) flow.watch_diff(loss, global_storage_setter("loss_diff")) return loss
def _batch_norm(inputs, momentum, epsilon, name, training=True): return flow.layers.batch_normalization( inputs=inputs, axis=-1, momentum=momentum, epsilon=epsilon, center=True, scale=True, # beta_initializer=flow.zeros_initializer(), # gamma_initializer=flow.ones_initializer(), # beta_regularizer=flow.zeros_initializer(), # gamma_regularizer=flow.ones_initializer(), moving_mean_initializer=flow.zeros_initializer(), moving_variance_initializer=flow.ones_initializer(), trainable=True, training=training, name=name)
def oneflow_ones() -> tp.Numpy: with flow.scope.placement(device_type, "0:0"): v = flow.get_variable( shape=np_out_ones.shape, dtype=flow.float32, initializer=flow.zeros_initializer(), name="x_var", ) of_ones = flow.ones(shape=input_shape, dtype=flow.float32) of_out = of_ones + v with flow.scope.placement(device_type, "0:0"): flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [1e-3]), momentum=0).minimize(of_out) return of_ones