def common_check_2(self, name, fct, weight_name=None, verbose=0, classification=False, rnd=True, second_name='Y_grad', **kwargs): onx = function_onnx_graph(name, target_opset=get_max_opset(), dtype=numpy.float32, weight_name=weight_name, **kwargs) if verbose > 0: with open(name + ".onnx", "wb") as f: f.write(onx.SerializeToString()) if classification: N = 10 p = numpy.random.randn(N, 1).astype(numpy.float32) p[0, :] = 0 p[1, :] = 100 p[2, :] = -100 p[3, :] = 1 p[4, :] = -1 y = (numpy.random.randn(N, 1).astype(numpy.float32) > 0).astype( numpy.int64) x2 = p x1 = y else: if rnd: x1 = numpy.random.randn(10, 1).astype(numpy.float32) x2 = numpy.random.randn(10, 1).astype(numpy.float32) else: x1 = numpy.zeros((10, 1), dtype=numpy.float32) x2 = numpy.zeros((10, 1), dtype=numpy.float32) + 1 if rnd: w = numpy.random.rand(10).astype(numpy.float32) else: w = numpy.zeros(10, dtype=numpy.float32) + 0.2 if weight_name is None: exp_loss, exp_grad = fct(x1, x2) else: exp_loss, exp_grad = fct(x1, x2, w.reshape((-1, 1))) oinf = OnnxInference(onx) run_params = dict(verbose=verbose, fLOG=print) if verbose > 0 else {} if verbose > 0: print(f"\n+++++ name(1)={name!r}") if weight_name is None: got = oinf.run({'X1': x1, 'X2': x2}, **run_params) else: got = oinf.run({'X1': x1, 'X2': x2, 'weight': w}, **run_params) self.assertEqual(len(exp_grad.shape), 2) self.assertEqual(exp_grad.shape[-1], 1) self.assertEqualArray(exp_grad, got[second_name], decimal=5) self.assertEqualArray(exp_loss, got['Y'], decimal=5) providers = device_to_providers('cpu') so = SessionOptions() so.log_severity_level = 0 if verbose > 0 else 4 so.log_verbosity_level = 0 if verbose > 0 else 4 sess = InferenceSession(onx.SerializeToString(), so, providers=providers) if verbose > 0: print("+++ run") if weight_name is None: got = sess.run(None, {'X1': x1, 'X2': x2}) else: got = sess.run(None, {'X1': x1, 'X2': x2, 'weight': w}) self.assertEqualArray(exp_loss, got[0], decimal=5) self.assertEqualArray(exp_grad, got[1], decimal=5) if weight_name is not None: if verbose > 0: print("+++ run*") got = sess.run(None, {'X1': x1, 'X2': x2}) exp_loss2, exp_grad2 = fct(x1, x2, numpy.array([1], dtype=x1.dtype)) self.assertEqualArray(exp_loss2, got[0], decimal=5) self.assertEqualArray(exp_grad2, got[1], decimal=5) if 'grad' in name: rew = unreduced_onnx_loss(onx) if 'ReduceSum' in str(rew): raise AssertionError(f"Isse with:\n{rew!r}") if verbose > 0: with open(name + ".unreduced.onnx", "wb") as f: f.write(rew.SerializeToString()) if verbose > 0: print(f"\n+++++ name(2)={name!r}") oinf = OnnxInference(rew) if weight_name is None: got = oinf.run({'X1': x1, 'X2': x2}, **run_params) else: got = oinf.run({'X1': x1, 'X2': x2, 'weight': w}, **run_params) score = got['score'] self.assertEqual(len(score.shape), 2) self.assertEqual(score.shape[0], 10) self.assertEqual(score.shape[1], 1) self.assertEqualFloat(exp_loss, score.sum()) sess = InferenceSession(rew.SerializeToString(), so, providers=providers) if verbose > 0: print("+++ run") if weight_name is None: got = sess.run(None, {'X1': x1, 'X2': x2}) else: got = sess.run(None, {'X1': x1, 'X2': x2, 'weight': w}) score = got[0] self.assertEqual(len(score.shape), 2) self.assertEqual(score.shape[0], 10) self.assertEqual(score.shape[1], 1) self.assertEqualFloat(exp_loss, score.sum())
def _create_training_session(self, training_onnx, weights_to_train, loss_output_name='loss', training_optimizer_name='SGDOptimizer', device='cpu'): """ Creates an instance of :epkg:`TrainingSession`. :param training_onnx: an ONNX graph with a loss function :param weights_to_train: list of initializer names to optimize :param loss_output_name: output name for the loss :param training_optimizer_name: optimizer name :param device: one :epkg:`C_OrtDevice` or a string :return: an instance of :epkg:`TrainingSession` """ if training_optimizer_name != 'SGDOptimizer': raise NotImplementedError( "Only the SGDOptimizer is implemented not %r." "" % training_optimizer_name) ort_parameters = TrainingParameters() ort_parameters.loss_output_name = loss_output_name ort_parameters.use_mixed_precision = False # ort_parameters.world_rank = -1 # ort_parameters.world_size = 1 # ort_parameters.gradient_accumulation_steps = 1 # ort_parameters.allreduce_post_accumulation = False # ort_parameters.deepspeed_zero_stage = 0 # ort_parameters.enable_grad_norm_clip = False # ort_parameters.set_gradients_as_graph_outputs = False # ort_parameters.use_memory_efficient_gradient = False # ort_parameters.enable_adasum = False if self.saved_gradient is not None: name = self.saved_gradient name2 = name + ".training.onnx" ort_parameters.model_with_gradient_graph_path = name ort_parameters.model_with_training_graph_path = name2 output_types = {} for output in training_onnx.graph.output: output_types[output.name] = output.type.tensor_type ort_parameters.weights_to_train = set(weights_to_train) ort_parameters.training_optimizer_name = training_optimizer_name # ort_parameters.lr_params_feed_name = lr_params_feed_name ort_parameters.optimizer_attributes_map = { name: {} for name in weights_to_train } ort_parameters.optimizer_int_attributes_map = { name: {} for name in weights_to_train } session_options = SessionOptions() session_options.log_severity_level = 4 session_options.log_verbosity_level = 4 # session_options.use_deterministic_compute = True providers = device_to_providers(self.device) session = TrainingSession(training_onnx.SerializeToString(), ort_parameters, session_options, providers=providers) return session