def __init__(self, base_encoder, dim=128, queue_size=65536, momentum=0.999, scale=50, margin=0.3): super(DCQ, self).__init__() self.queue_size = queue_size self.momentum = momentum self.scale = scale self.margin = margin # create the encoders # num_classes is the output fc dimension self.encoder_q = base_encoder(num_classes=dim, name_prefix='q') self.encoder_k = base_encoder(num_classes=dim, name_prefix='k') for param_q, param_k in zip( self.encoder_q.parameters(include_sublayers=True), self.encoder_k.parameters(include_sublayers=True)): param_k.stop_gradient = True param_q.set_value(param_k) self.register_buffer("weight_queue", paddle.randn([dim, queue_size])) self.weight_queue = normalize(self.weight_queue, axis=0) self.register_buffer("label_queue", paddle.randn([1, queue_size])) self.register_buffer("queue_ptr", paddle.zeros([ 1, ], dtype='int64'))
def setUp(self): self.in_num = 16 self.out_num = 16 self.x_spec = paddle.static.InputSpec([-1, 16], name='x') self.y_spec = paddle.static.InputSpec([16], name='y') self.x = paddle.randn([4, 16]) self.y = paddle.randn([16])
def style_mixing(generator, mean_style, n_source, n_target): source_code = paddle.randn([n_source, generator.style_dim]) target_code = paddle.randn([n_target, generator.style_dim]) resolution = 2**((generator.n_latent + 2) // 2) images = [paddle.ones([1, 3, resolution, resolution]) * -1] source_image = generator([source_code], truncation_latent=mean_style, truncation=0.7)[0] target_image = generator([target_code], truncation_latent=mean_style, truncation=0.7)[0] images.append(source_image) for i in range(n_target): image = generator( [target_code[i].unsqueeze(0).tile([n_source, 1]), source_code], truncation_latent=mean_style, truncation=0.7, )[0] images.append(target_image[i].unsqueeze(0)) images.append(image) images = paddle.concat(images, 0) return images
def test_save_load_finetune_load(self): model_path = "test_jit_save_load_finetune_load/model" IMAGE_SIZE = 224 inps0 = paddle.randn([1, IMAGE_SIZE]) inps1 = paddle.randn([2, IMAGE_SIZE]) # Use new namespace with unique_name.guard(): layer_save = LayerSaved(IMAGE_SIZE, IMAGE_SIZE) layer_save(inps0) #save paddle.jit.save(layer_save, model_path) #load with unique_name.guard(): layer_load = LayerLoadFinetune(IMAGE_SIZE, IMAGE_SIZE, model_path) #train train(layer_load, input_size=IMAGE_SIZE) result_00 = layer_load(inps0) result_01 = layer_load(inps1) #save paddle.jit.save(layer_load, model_path) #load layer_finetune = paddle.jit.load(model_path) result_10 = layer_finetune(inps0) result_11 = layer_finetune(inps1) self.assertTrue(float((result_00 - result_10).abs().max()) < 1e-5) self.assertTrue(float(((result_01 - result_11)).abs().max()) < 1e-5)
def test_forward_reshape(): @paddle.jit.to_static def reshape1(inputs, x): new_shape = paddle.shape(x) return paddle.reshape(inputs, new_shape) @paddle.jit.to_static def reshape2(inputs): return inputs.reshape([-1]) @paddle.jit.to_static def reshape3(inputs): data_shape = inputs.shape return inputs.reshape([data_shape[0] * data_shape[1], data_shape[2]]) @paddle.jit.to_static def reshape4(inputs, x): new_shape = paddle.shape(x) return paddle.reshape(inputs, [new_shape[2], 2, -1]) input_shape = [2, 1, 10, 1, 10] input_data = paddle.rand(input_shape, dtype="float32") input_data2 = paddle.randn([2, 1, 10, 10]) verify_model(reshape1, input_data=[input_data, input_data2]) verify_model(reshape2, input_data=input_data) verify_model(reshape3, input_data=paddle.randn((2, 3, 4))) verify_model(reshape4, input_data=[input_data, input_data2])
def train(print_result=False): # 1. initialize parallel environment dist.init_parallel_env() # 2. create data parallel layer & optimizer layer = LinearNet() dp_layer = paddle.DataParallel(layer) loss_fn = nn.MSELoss() adam = opt.Adam(learning_rate=0.001, parameters=dp_layer.parameters()) # 3. run layer inputs = paddle.randn([10, 10], 'float32') outputs = dp_layer(inputs) labels = paddle.randn([10, 1], 'float32') loss = loss_fn(outputs, labels) if print_result is True: print("Rank:", int(os.getenv("PADDLE_TRAINER_ID"))) loss.backward() adam.step() adam.clear_grad() return int(os.getenv("PADDLE_TRAINER_ID"))
def test_save_in_eval(self, with_training=True): paddle.jit.ProgramTranslator().enable(True) net = Net(12, 2) x = paddle.randn((2, 10, 12)) if with_training: x.stop_gradient = False dygraph_out = net(x) loss = paddle.mean(dygraph_out) sgd = paddle.optimizer.SGD(learning_rate=0.001, parameters=net.parameters()) loss.backward() sgd.step() # switch eval mode firstly net.eval() x = paddle.randn((2, 10, 12)) net = paddle.jit.to_static( net, input_spec=[paddle.static.InputSpec(shape=[-1, 10, 12])]) model_path = os.path.join(self.temp_dir.name, 'simple_lstm') paddle.jit.save(net, model_path) dygraph_out = net(x) # load saved model load_net = paddle.jit.load(model_path) static_out = load_net(x) self.assertTrue(np.allclose(dygraph_out.numpy(), static_out.numpy()), msg='dygraph_out is {}\n static_out is \n{}'.format( dygraph_out, static_out)) # switch back into train mode. net.train() train_out = net(x) self.assertTrue(np.allclose(dygraph_out.numpy(), train_out.numpy()), msg='dygraph_out is {}\n static_out is \n{}'.format( dygraph_out, train_out))
def test_save_load_finetune_load(self): model_path = "test_jit_save_load_save_without_running/model" IMAGE_SIZE = 224 inps0 = paddle.randn([1, IMAGE_SIZE]) inps1 = paddle.randn([2, IMAGE_SIZE]) # Use new namespace with unique_name.guard(): layer_save = LayerSaved(IMAGE_SIZE, IMAGE_SIZE) #save paddle.jit.save(layer_save, model_path, input_spec=[ paddle.static.InputSpec(shape=[None, IMAGE_SIZE], dtype='float32') ]) result_00 = layer_save(inps0) result_01 = layer_save(inps1) #load and save without running with unique_name.guard(): layer_load = paddle.jit.load(model_path) paddle.jit.save(layer_load, model_path, input_spec=[ paddle.static.InputSpec( shape=[None, IMAGE_SIZE], dtype='float32') ]) #reload layer_reload = paddle.jit.load(model_path) result_10 = layer_reload(inps0) result_11 = layer_reload(inps1) self.assertTrue(float((result_00 - result_10).abs().max()) < 1e-5) self.assertTrue(float((result_01 - result_11).abs().max()) < 1e-5)
def train(print_result=False): # 1. enable dynamic mode paddle.disable_static() # 2. initialize parallel environment dist.init_parallel_env() # 3. create data parallel layer & optimizer layer = LinearNet() dp_layer = paddle.DataParallel(layer) loss_fn = nn.MSELoss() adam = opt.Adam( learning_rate=0.001, parameters=dp_layer.parameters()) # 4. run layer inputs = paddle.randn([10, 10], 'float32') outputs = dp_layer(inputs) labels = paddle.randn([10, 1], 'float32') loss = loss_fn(outputs, labels) if print_result is True: print("loss:", loss.numpy()) loss.backward() adam.step() adam.clear_grad()
def test_save_in_eval(self): paddle.jit.ProgramTranslator().enable(True) net = LinearNet() x = paddle.randn((2, 10)) x.stop_gradient = False dygraph_out = net(x) loss = paddle.mean(dygraph_out) sgd = paddle.optimizer.SGD(learning_rate=0.001, parameters=net.parameters()) loss.backward() sgd.step() # switch eval mode firstly net.eval() # save directly net = paddle.jit.to_static( net, input_spec=[paddle.static.InputSpec(shape=[-1, 10])]) model_path = os.path.join(self.temp_dir.name, 'linear_net') paddle.jit.save(net, model_path) # load saved model load_net = paddle.jit.load(model_path) x = paddle.randn((2, 10)) eval_out = net(x) infer_out = load_net(x) self.assertTrue(np.allclose(eval_out.numpy(), infer_out.numpy()), msg='eval_out is {}\n infer_out is \n{}'.format( eval_out, infer_out))
def test_api(self): shape = [1000, 784] train_program = Program() startup_program = Program() with program_guard(train_program, startup_program): x1 = paddle.randn(shape, 'float32') x2 = paddle.randn(shape, 'float64') dim_1 = paddle.fluid.layers.fill_constant([1], "int64", 20) dim_2 = paddle.fluid.layers.fill_constant([1], "int32", 50) x3 = paddle.randn([dim_1, dim_2, 784]) var_shape = paddle.static.data('X', [2], 'int32') x4 = paddle.randn(var_shape) place = paddle.CUDAPlace( 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() exe = paddle.static.Executor(place) res = exe.run(train_program, feed={'X': np.array(shape, dtype='int32')}, fetch_list=[x1, x2, x3, x4]) for out in res: self.assertAlmostEqual(np.mean(out), .0, delta=0.1) self.assertAlmostEqual(np.std(out), 1., delta=0.1)
def train(print_result=True): """train""" # 1. initialize parallel environment train_data_list1 = [] train_data_list2 = [] dist.init_parallel_env() # 2. create data parallel layer & optimizer layer = LinearNet() dp_layer = paddle.DataParallel(layer) loss_fn = nn.MSELoss() adam = opt.Adam(learning_rate=0.001, parameters=dp_layer.parameters()) # 3. run layer inputs = paddle.randn([10, 10], 'float32') outputs = dp_layer(inputs) labels = paddle.randn([10, 1], 'float32') loss = loss_fn(outputs, labels) assert len(loss) == 1 if print_result is True: train_data_list1.append(loss.numpy()) assert len(train_data_list1) loss.backward() adam.step() adam.clear_grad()
def train(): # 1. enable dynamic mode paddle.disable_static() # 2. initialize parallel environment dist.init_parallel_env() # 3. create data parallel layer & optimizer layer = LinearNet() dp_layer = paddle.DataParallel(layer) loss_fn = nn.MSELoss() adam = opt.Adam(learning_rate=0.001, parameters=dp_layer.parameters()) # 4. run layer inputs = paddle.randn([10, 10], 'float32') outputs = dp_layer(inputs) labels = paddle.randn([10, 1], 'float32') loss = loss_fn(outputs, labels) loss = dp_layer.scale_loss(loss) loss.backward() dp_layer.apply_collective_grads() adam.step() adam.clear_grad()
def make_noise(self): noises = [paddle.randn((1, 1, 2**2, 2**2))] for i in range(3, self.log_size + 1): for _ in range(2): noises.append(paddle.randn((1, 1, 2**i, 2**i))) return noises
def make_noise(self, batch, num_noise): if num_noise == 1: noises = paddle.randn([batch, self.num_style_feat]) else: noises = [] for _ in range(num_noise): noises.append(paddle.randn([batch, self.num_style_feat])) return noises
def sample_prior(self, prob_decode=False): """ Sample a molecule from prior distribution. Args: prob_decode(bool): using bernoulli distribution in graph decode if prob_decode=true. Returns: smiles. """ z_tree = paddle.randn([1, self.latent_size]) z_mol = paddle.randn([1, self.latent_size]) return self.decode(z_tree, z_mol, prob_decode)
def test_GRU_base(): """ api: paddle.nn.GRU op version: 9, 10, 11, 12 """ op = Net() op.eval() # net, name, ver_list, delta=1e-10, rtol=1e-11 obj = APIOnnx(op, 'nn_GRU', [9, 10, 11, 12]) obj.set_input_data("input_data", paddle.randn((4, 23, 16)), paddle.randn((2, 4, 32))) obj.run()
def train(): """bergin train""" arr1 = [] arr2 = [] dist.init_parallel_env() set_seed(2021) layer = LinearNet() if dist.get_world_size() > 1: dp_layer = paddle.DataParallel(layer) else: dp_layer = layer layer2 = LinearNet() if dist.get_world_size() > 1: dp_layer2 = paddle.DataParallel(layer2) else: dp_layer2 = layer2 dp_layer2.set_state_dict(dp_layer.state_dict()) loss_fn = nn.MSELoss() adam = opt.Adam( learning_rate=0.001, parameters=dp_layer.parameters()) adam2 = opt.Adam( learning_rate=0.001, parameters=dp_layer2.parameters()) for i in range(2): batch_size = 10 shard = int(batch_size / dist.get_world_size()) start_no = shard * dist.get_rank() end_no = start_no + shard inputs = paddle.randn([10, 10], 'float32')[start_no:end_no] outputs = dp_layer(inputs) labels = paddle.randn([10, 1], 'float32')[start_no:end_no] loss = loss_fn(outputs, labels) if dist.get_rank() == 0: arr1.append(loss.numpy()[0]) loss.backward() adam.step() adam.clear_grad() outputs = dp_layer2(inputs) loss = loss_fn(outputs, labels) loss.backward() if dist.get_rank() == 0: arr2.append(loss.numpy()[0]) adam2.step() adam2.clear_grad() check_data(arr1, arr2)
def train(): dist.init_parallel_env() # 1. initialize parallel environment set_seed(2021) # 2. create data parallel layer & optimizer layer = LinearNet() if dist.get_world_size() > 1: dp_layer = paddle.DataParallel(layer) else: dp_layer = layer layer2 = LinearNet() if dist.get_world_size() > 1: dp_layer2 = paddle.DataParallel(layer2) else: dp_layer2 = layer2 dp_layer2.set_state_dict(dp_layer.state_dict()) loss_fn = nn.MSELoss() adam = opt.Adam(learning_rate=0.001, parameters=dp_layer.parameters()) adam2 = opt.Adam(learning_rate=0.001, parameters=dp_layer2.parameters()) # 3. run layer print("Start") for i in range(10): batch_size = 10 shard = int(batch_size / dist.get_world_size()) start_no = shard * dist.get_rank() end_no = start_no + shard inputs = paddle.randn([10, 10], 'float32')[start_no:end_no] outputs = dp_layer(inputs) labels = paddle.randn([10, 1], 'float32')[start_no:end_no] loss = loss_fn(outputs, labels) if dist.get_rank() == 0: print("Loss1", loss.numpy()[0]) print(dp_layer.parameters()) loss.backward() adam.step() adam.clear_grad() outputs = dp_layer2(inputs) loss = loss_fn(outputs, labels) loss.backward() if dist.get_rank() == 0: print("Loss2", loss.numpy()[0]) print(dp_layer2.parameters()) adam2.step() adam2.clear_grad()
def test_dygraph(self): with paddle.fluid.dygraph.base.guard(): x = paddle.randn([10, 10], dtype='float32') y = paddle.poisson(x) self.assertTrue(np.min(y.numpy()) >= 0) with _test_eager_guard(): x = paddle.randn([10, 10], dtype='float32') x.stop_gradient = False y = paddle.poisson(x) y.backward() self.assertTrue(np.min(y.numpy()) >= 0) self.assertTrue(np.array_equal(np.zeros_like(x), x.gradient()))
def setUp(self): model = ModelInputDict() sp_net_config = supernet(expand_ratio=[0.5, 1.0]) self.model = Convert(sp_net_config).convert(model) self.images = paddle.randn(shape=[2, 3, 32, 32], dtype='float32') self.images2 = { 'data': paddle.randn(shape=[2, 12, 32, 32], dtype='float32') } default_run_config = {'skip_layers': ['conv1.0', 'conv2.0']} self.run_config = RunConfig(**default_run_config) self.ofa_model = OFA(self.model, run_config=self.run_config) self.ofa_model._clear_search_space(self.images, data=self.images2)
def setup_input(self, input): """Unpack input data from the dataloader and perform necessary pre-processing steps. Args: input (dict): include the data itself and its metadata information. The option 'direction' can be used to swap images in domain A and domain B. """ pass self.input = input self.input['z_trg'] = paddle.randn( (input['src'].shape[0], self.latent_dim)) self.input['z_trg2'] = paddle.randn( (input['src'].shape[0], self.latent_dim))
def test_cuda_rng_tracker(self): seed_1 = 2021 seed_2 = 1024 size = [20, 15] paddle.seed(seed_1) target_11 = paddle.randn(size, "float32") target_12 = paddle.randn(size, "float32") paddle.seed(seed_2) target_21 = paddle.randn(size, "float32") target_22 = paddle.randn(size, "float32") paddle.seed(seed_1) fleet.meta_parallel.get_rng_state_tracker().add("test", seed_2) result_11 = paddle.randn(size, "float32") with fleet.meta_parallel.get_rng_state_tracker().rng_state("test"): result_21 = paddle.randn(size, "float32") result_12 = paddle.randn(size, "float32") with fleet.meta_parallel.get_rng_state_tracker().rng_state("test"): result_22 = paddle.randn(size, "float32") np.testing.assert_allclose(result_11.numpy(), target_11.numpy()) np.testing.assert_allclose(result_12.numpy(), target_12.numpy()) np.testing.assert_allclose(result_21.numpy(), target_21.numpy()) np.testing.assert_allclose(result_22.numpy(), target_22.numpy())
def test_api(self): input = paddle.randn([6, 4, 2, 2]) out = paddle.fluid.layers.temporal_shift(x=input, seg_num=2, shift_ratio=0.2) out_from_function = paddle.nn.functional.temporal_shift( x=input, seg_num=2, shift_ratio=0.2) # dygraph with paddle.fluid.dygraph.guard(): input = paddle.randn([6, 4, 2, 2]) out = paddle.nn.functional.temporal_shift(x=input, seg_num=2, shift_ratio=0.2)
def __init__(self, num_classes=5013, feat_dim=2048): super(CenterLoss, self).__init__() self.num_classes = num_classes self.feat_dim = feat_dim self.centers = paddle.randn( shape=[self.num_classes, self.feat_dim]).astype( "float64") #random center
def ddpm_steps(x, seq, model, b, **kwargs): with paddle.no_grad(): n = x.shape[0] seq_next = [-1] + list(seq[:-1]) xs = [x] x0_preds = [] betas = b for i, j in zip(reversed(seq), reversed(seq_next)): t = (paddle.ones([n]) * i) next_t = (paddle.ones([n]) * j) at = compute_alpha(betas, t.astype('int64')) atm1 = compute_alpha(betas, next_t.astype('int64')) beta_t = 1 - at / atm1 x = xs[-1] output = model(x, t.astype('float32')) e = output x0_from_e = (1.0 / at).sqrt() * x - (1.0 / at - 1).sqrt() * e x0_from_e = paddle.clip(x0_from_e, -1, 1) x0_preds.append(x0_from_e) mean_eps = ((atm1.sqrt() * beta_t) * x0_from_e + ((1 - beta_t).sqrt() * (1 - atm1)) * x) / (1.0 - at) mean = mean_eps noise = paddle.randn(x.shape) mask = 1 - (t == 0).astype('float32') mask = mask.reshape([-1, 1, 1, 1]) logvar = beta_t.log() sample = mean + mask * paddle.exp(0.5 * logvar) * noise xs.append(sample) return xs, x0_preds
def __init__(self, dim, in_dim, head_cnt=1, kernel_ratio=0.5, dp1=0.1, dp2=0.1): super().__init__() self.emb = in_dim * head_cnt # we use 1, so it is no need here self.kqv = nn.Linear(dim, 3 * self.emb) self.dp = nn.Dropout(dp1) self.proj = nn.Linear(self.emb, self.emb) self.head_cnt = head_cnt self.norm1 = nn.LayerNorm(dim) self.norm2 = nn.LayerNorm(self.emb) self.epsilon = 1e-8 # for stable in division self.mlp = nn.Sequential( nn.Linear(self.emb, 1 * self.emb), nn.GELU(), nn.Linear(1 * self.emb, self.emb), nn.Dropout(dp2), ) self.m = int(self.emb * kernel_ratio) self.w = paddle.randn((self.m, self.emb)) self.w = add_parameter(self, orthogonal_(self.w) * math.sqrt(self.m))
def build_program(): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): with paddle.static.device_guard('cpu'): data = paddle.ones([4, 64], dtype='float32', name='data') # data -> [memcpy_h2d] -> data' -> [matmul] -> out ->[add] -> add_out with paddle.static.device_guard('gpu'): weight = paddle.randn([64, 64], name='weight') # gpu matmul_out = paddle.matmul(data, weight, name='matmul_out') # gpus bias = paddle.ones([4, 64], dtype='float32', name='bias') add_out = paddle.add(matmul_out, bias, name='add_out') # add_out -> [memcpy_d2h] -> add_out' -> [sub] -> sub_out -> [tanh] -> tanh_out with paddle.static.device_guard('cpu'): sub_out = paddle.subtract(add_out, data, name='sub_out') tanh_out = paddle.tanh(sub_out, name='tanh_out') with paddle.static.device_guard('gpu'): bias_1 = paddle.add(bias, sub_out, name='bias_1') out_before = paddle.tanh(bias_1, name='out_before') out_last = paddle.subtract(tanh_out, data, name='out_last') out = paddle.add(out_before, out_last, name='out') mean = paddle.mean(out, name='mean_out') return main_program, startup_program, [mean]
def test_error(self): x = paddle.randn([2, 3, 4, 5]) # src must have the same number with dst with self.assertRaises(AssertionError): paddle.moveaxis(x, [1, 0], [2]) # each element of src must be unique with self.assertRaises(ValueError): paddle.moveaxis(x, [1, 1], [0, 2]) # each element of dst must be unique with self.assertRaises(ValueError): paddle.moveaxis(x, [0, 1], [2, 2]) # each element of src must be integer with self.assertRaises(AssertionError): paddle.moveaxis(x, [0.5], [1]) # each element of dst must be integer with self.assertRaises(AssertionError): paddle.moveaxis(x, [0], [1.5]) # each element of src must be in the range of [-4, 3) with self.assertRaises(AssertionError): paddle.moveaxis(x, [-10, 1], [2, 3]) # each element of dst must be in the range of [-4, 3) with self.assertRaises(AssertionError): paddle.moveaxis(x, [2, 1], [10, 3])
def setUp(self): self.in_num = 16 self.out_num = 16 self.x = paddle.randn([4, 16]) self.spec = [ paddle.static.InputSpec(shape=[None, 16], dtype='float32') ]