def ReduceElements(self, cntk_op, inputs): """ Returns a reduction operation (max, min, mean, sum, prod) or a calculation which matches CNTK's LogSum reduction (`reduce_log_sum_exp` function). Arguments: cntk_op: CNTK operation to be imported. inputs: List of inputs to this node. Returns: A ngraph Op. """ assert len(inputs) == 1 reduction_op_name = cntk_op.attributes.get('reductionOpName') # CNTK API defines a reductionKeepDimensions flag, but we currently don't use it # keep_dimensions = cntk_op.attributes.get('reductionKeepDimensions', False) cntk_op_attribute_axes = [] if cntk_op.attributes.get('axisVec'): cntk_op_attribute_axes.extend(cntk_op.attributes.get('axisVec')) elif cntk_op.attributes.get('axis'): cntk_op_attribute_axes.append(cntk_op.attributes.get('axis')) # CNTK axes are numbered in reverse order: the last axis is labeled 0, the previous 1, etc. reduction_axes_indexes = [len(inputs[0].axes) - 1 - i for (_, _, i) in cntk_op_attribute_axes] reduction_ng_axes_list = [axis for (i, axis) in enumerate(inputs[0].axes) if i in reduction_axes_indexes] reduction_ng_axes = ng.Axes(axes=reduction_ng_axes_list) if reduction_op_name == 'Max': return ng.max(inputs[0], reduction_axes=reduction_ng_axes).named(cntk_op.uid) if reduction_op_name == 'Min': return ng.min(inputs[0], reduction_axes=reduction_ng_axes).named(cntk_op.uid) if reduction_op_name == 'Mean': return ng.mean(inputs[0], reduction_axes=reduction_ng_axes).named(cntk_op.uid) if reduction_op_name == 'Sum': return ng.sum(inputs[0], reduction_axes=reduction_ng_axes).named(cntk_op.uid) if reduction_op_name == 'Prod': return ng.prod(inputs[0], reduction_axes=reduction_ng_axes).named(cntk_op.uid) if reduction_op_name == 'LogSum': return ng.log(ng.sum(ng.exp(inputs[0]), reduction_axes=reduction_ng_axes))\ .named(cntk_op.uid) raise NotImplementedError('CNTKImporter: ReduceElements does not support operation %s', reduction_op_name)
def test_prod_deriv( prod_deriv_arrays): # Argon Transformer error - TODO triage """ Test reduce product's gradient """ def power_set(lst): """ power_set([0, 1, 2]) is: [[], [0], [1], [0, 1], [2], [0, 2], [1, 2], [0, 1, 2]] """ result = [[]] for x in lst: result.extend([subset + [x] for subset in result]) return result def get_all_reduction_axes(axes): """ Get all possible reduction axes """ ndim = len(axes.lengths) if ndim == 0: return axes else: results = [] all_indices = power_set(range(ndim)) for indices in all_indices: if not indices: results.append(ng.make_axes([])) else: results.append( ng.make_axes([axes[index] for index in indices])) return results def shape_to_axes(shape): """ Convert shape to axes """ if not shape: return ng.make_axes() axes = ng.make_axes([ng.make_axis(length=s) for s in shape]) return axes x_val = prod_deriv_arrays axes = shape_to_axes(x_val.shape) all_reduction_axes = get_all_reduction_axes(axes) for reduction_axes in all_reduction_axes: x = ng.placeholder(axes=axes) x_prod = ng.prod(x, reduction_axes) check_derivative(x_prod, x, 0.001, x_val, atol=1e-3, rtol=1e-3)
def test_prod_constant(transformer_factory): """ Test reduce product of constants """ A0 = ng.make_axis(length=2) A1 = ng.make_axis(length=3) A2 = ng.make_axis(length=4) # ngrpah ops const_3d = ng.broadcast(ng.constant(2., axes=[]), axes=[A0, A1, A2]) prod_0 = ng.prod(const_3d, reduction_axes=[A0]) prod_1 = ng.prod(const_3d, reduction_axes=[A1]) prod_2 = ng.prod(const_3d, reduction_axes=[A2]) prod_0_1 = ng.prod(const_3d, reduction_axes=[A0, A1]) prod_0_2 = ng.prod(const_3d, reduction_axes=[A0, A2]) prod_1_2 = ng.prod(const_3d, reduction_axes=[A1, A2]) prod_0_1_2 = ng.prod(const_3d, reduction_axes=[A0, A1, A2]) # numpy results np_const_3d = np.ones((2, 3, 4)) * 2. res_0_np = np.prod(np_const_3d, axis=(0)) res_1_np = np.prod(np_const_3d, axis=(1)) res_2_np = np.prod(np_const_3d, axis=(2)) res_0_1_np = np.prod(np_const_3d, axis=(0, 1)) res_0_2_np = np.prod(np_const_3d, axis=(0, 2)) res_1_2_np = np.prod(np_const_3d, axis=(1, 2)) res_0_1_2_np = np.prod(np_const_3d, axis=(0, 1, 2)) # define comp with ExecutorFactory() as ex: comps = ex.executor( [prod_0, prod_1, prod_2, prod_0_1, prod_0_2, prod_1_2, prod_0_1_2]) res_0_ng, res_1_ng, res_2_ng, res_0_1_ng, res_0_2_ng, res_1_2_ng, res_0_1_2_ng = comps( ) np.testing.assert_allclose(res_0_np, res_0_ng) np.testing.assert_allclose(res_1_np, res_1_ng) np.testing.assert_allclose(res_2_np, res_2_ng) np.testing.assert_allclose(res_0_1_np, res_0_1_ng) np.testing.assert_allclose(res_0_2_np, res_0_2_ng) np.testing.assert_allclose(res_1_2_np, res_1_2_ng) np.testing.assert_allclose(res_0_1_2_np, res_0_1_2_ng)
def test_prod_constant(prod_constant): """ Test reduce product of constants """ np_axis, ng_axis, axes_values = prod_constant # ngrpah op const_3d = ng.broadcast(ng.constant(2., axes=[]), axes=axes_values) prod = ng.prod(const_3d, reduction_axes=ng_axis) # numpy results np_const_3d = np.ones((2, 3, 4)) * 2. res_np = np.prod(np_const_3d, axis=np_axis) # define comp with ExecutorFactory() as ex: comps = ex.executor(prod) res_ng = comps() np.testing.assert_allclose(res_np, res_ng)
def test_prod_deriv(transformer_factory): """ Test reduce product's gradient """ def power_set(lst): """ power_set([0, 1, 2]) is: [[], [0], [1], [0, 1], [2], [0, 2], [1, 2], [0, 1, 2]] """ result = [[]] for x in lst: result.extend([subset + [x] for subset in result]) return result def get_all_reduction_axes(axes): """ Get all possible reduction axes """ ndim = len(axes.lengths) if ndim == 0: return axes else: results = [] all_indices = power_set(range(ndim)) for indices in all_indices: if not indices: results.append(ng.make_axes([])) else: results.append( ng.make_axes([axes[index] for index in indices])) return results def shape_to_axes(shape): """ Convert shape to axes """ if not shape: return ng.make_axes() axes = ng.make_axes([ng.make_axis(length=s) for s in shape]) return axes # test cases test_cases = [ np.array([[[1., 2., 3.], [4., 5., 0.], [0., 6., 0.]], [[1., 2., 3.], [4., 5., 6.], [7., 8., 0.]]]), np.array([[1., 2., 3.], [4., 5., 0.], [0., 6., 0.]]), np.array([1., 2., 3.]), np.array([0., 2., 3.]), np.array([0., 0., 3.]), np.array([0., 0., 0.]), np.array([0.]), np.array([2.]), np.array(0.), np.array(2.), ] for x_val in test_cases: axes = shape_to_axes(x_val.shape) all_reduction_axes = get_all_reduction_axes(axes) for reduction_axes in all_reduction_axes: x = ng.placeholder(axes=axes) x_prod = ng.prod(x, reduction_axes) check_derivative(x_prod, x, 0.001, x_val, atol=1e-3, rtol=1e-3)
def __call__(self, iteration): masked_gamma = (iteration >= self.schedule) * self.gamma masked_holes = (iteration < self.schedule) return self.base_lr * ng.prod(masked_gamma + masked_holes, out_axes=())