def torch_kspace_location(shape_y, shape_x): """Construct matrix with k-psace normalized location as tensor.""" y = torch.cast(torch.range(shape_y), torch.float32) y = y / torch.cast(shape_y, torch.float32) - 0.5 x = torch.cast(torch.range(shape_x), torch.float32) x = x / torch.cast(shape_x, torch.float32) - 0.5 [yg, xg] = torch.meshgrid(y, x) yg = torch.transpose(yg, [1, 0]) xg = torch.transpose(xg, [1, 0]) out = torch.stack((yg, xg)) return out
def ifftc(im, name="ifftc", do_orthonorm=True): """Centered iFFT on second to last dimension.""" im_out = im if do_orthonorm: fftscale = torch.sqrt(1.0 * im_out.get_shape().as_list()[-2]) else: fftscale = 1.0 fftscale = torch.cast(fftscale, dtype=torch.complex64) if len(im.get_shape()) == 4: im_out = torch.transpose(im_out, [0, 3, 1, 2]) im_out = fftshift(im_out, axis=3) else: im_out = torch.transpose(im_out, [2, 0, 1]) im_out = fftshift(im_out, axis=2) with torch.device("/gpu:0"): # FFT is only supported on the GPU im_out = torch.ifft(im_out) * fftscale if len(im.get_shape()) == 4: im_out = fftshift(im_out, axis=3) im_out = torch.transpose(im_out, [0, 2, 3, 1]) else: im_out = fftshift(im_out, axis=2) im_out = torch.transpose(im_out, [1, 2, 0]) return im_out
def predict(): """Predict unseen images""" """Step 0: load data and trained model""" mnist = input_data.read_data_sets("./data/", one_hot=True) checkpoint_dir = sys.argv[1] """Step 1: build the rnn model""" x = tf.placeholder("float", [None, n_steps, n_input]) y = tf.placeholder("float", [None, n_classes]) weights = tf.Variable(tf.random_normal([n_hidden, n_classes]), name='weights') biases = tf.Variable(tf.random_normal([n_classes]), name='biases') pred = rnn_model(x, weights, biases) correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) """Step 2: predict new images with the trained model""" with tf.Session() as sess: sess.run(tf.initialize_all_variables()) """Step 2.0: load the trained model""" checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir + 'checkpoints') print('Loaded the trained model: {}'.format(checkpoint_file)) saver = tf.train.Saver() saver.restore(sess, checkpoint_file) """Step 2.1: predict new data""" test_len = 500 test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input)) test_label = mnist.test.labels[:test_len] print("Testing Accuracy:", sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
def kspace_threshhold(image_orig, threshhold=1e-8, name="kspace_threshhold"): """Find k-space mask based on threshhold. Anything less the specified threshhold is set to 0. Anything above the specified threshhold is set to 1. """ mask_x = torch.greater(torch.abs(image_orig), threshhold) mask_x = torch.cast(mask_x, dtype=torch.float32) return mask_x
def gpi(self, observation, cumulant_weights): q_values = self.__call__(th.expand_dims(observation, axis=0))[0] q_w = th.tensordot(q_values, cumulant_weights, axes=[1, 0]) # [P,a] q_w_actions = th.reduce_max(q_w, axis=0) action = th.cast(th.argmax(q_w_actions), th.int32) return action
def forward(self, x): word_embedding = self.word_embedding(x) # 保证x转化到 d ^ (1/2) 正态分布,且要比position encoding大 word_embedding *= torch.sqrt(torch.cast(self.d_model, torch.float32)) # (batch, seq_len) pos_seq = torch.arange(0, x.size(1)).repeat(x.size(0), 1) if torch.cuda.is_available(): pos_seq = pos_seq.cuda() embedding = word_embedding + self.pos_embedding(pos_seq) embedding = self.embedding_dropout(embedding) return embedding, pos_seq
def forward(self, q, k, v, mask=None): """ q, k, v: (batch*n_head, t_len, hidden_size) mask: (batch*n_head, q_len, q_len) """ attention = torch.bmm(q, k.transpose(1, 2)) attention = attention / torch.sqrt(torch.cast(q.shape[-1], torch.float32)) if mask is not None: attention = attention.masked_fill(mask, -np.inf) attention = self.softmax(attention) # (batch*n_head, t_len, t_len) output = torch.bmm(attention, v) # (batch*n_head, t_len, hidden_size) return output, attention
def positional_encoding(sentence_length, d_model): angle_rads = get_angles( np.arange(sentence_length)[:, np.newaxis], np.arange(d_model)[np.newaxis, :], d_model) # 将 sin 应用于数组中的偶数索引(indices);2i angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2]) # 将 cos 应用于数组中的奇数索引;2i+1 angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2]) pos_encoding = angle_rads[np.newaxis, ...] pos_encoding = torch.cast(pos_encoding, dtype=torch.float32) pos_encoding = pos_encoding.permute(1, 0, 2).contiguous() # (len, 1, d_model) return pos_encoding
def test_cast_variable(self): inputs = [ torch.ByteTensor(1), torch.CharTensor(1), torch.DoubleTensor(1), torch.FloatTensor(1), torch.IntTensor(1), torch.LongTensor(1), torch.ShortTensor(1), ] for inp in inputs: assert type(inp) == type(torch.cast(Variable(inp), type(inp)).data)
def to_scores(self, qk, mask, v): b = 0 if mask is not None: b = torch.logical_not(mask) b = torch.cast(b, torch.floatx()) * qu.big_neg() if self.proxim_b is not None: b += self.proxim_b b = b[:, None, :, None] y = torch.softmax(qk * self.scale + b) cfg = self.cfg y = self.drop(y, cfg.drop_attn or cfg.drop) y = torch.einsum("bnij,bnjv->bniv", y, v) return y
def loss(self, y_true, y_pred, from_logits=False, label_smoothing=0): """ Calculate the loss (The test process will use this function) TODO you should provide this function no matter you use it in training or not; because the test process would call this function :return: loss (float) """ y_true = torch.view(-1, y_true) # calculate the padding mask mask = torch.cast(torch.math.not_equal(y_true, 0), y_pred.dtype) # calculate the loss loss_ = self.compile_params['loss'](y_true, y_pred) # remove the padding part's loss by timing the mask loss_ *= mask # calculate the mean loss return tf.reduce_mean(loss_)
def cast(value, dtype=None, lib=None, hint=None): if pylist(value): value = stack(value, axis=-1, lib=lib) lib = as_lib(lib, hint=value) if dtype is None: if hint is None: hint = value dtypes = pyflat(as_dtype(hint, lib=lib, deep=True)) if not functools.reduce(operator.eq, dtypes): raise ValueError("dtypes not equal for {}".format(value)) dtype = dtypes[0] dtypes = pyflat(as_dtype(dtype, lib=lib, deep=True)) if not functools.reduce(operator.eq, dtypes): raise ValueError("dtypes not equal for {}".format(value)) dtype = dtypes[0] if lib == np: return np.array(value, dtype=dtype) elif lib == tensorflow: return tensorflow.cast(value, dtype=dtype) else: assert lib == torch return torch.cast(value, dtype=dtype)
def kspace_mask(image_orig, name="kspace_mask", dtype=None): """Find k-space mask.""" mask_x = torch.not_equal(image_orig, 0) if dtype is not None: mask_x = torch.cast(mask_x, dtype=dtype) return mask_x
def forward(self, x, mask=None): y = self.pos_b if mask is not None: y *= torch.cast(mask, self.pos_b.dtype) return x + y
def _build_likelihood(self): L = th.cumsum(self.E_log_p_Y(self.X, self.Y))[:] KL = th.cumsum([layer.KL() for layer in self.layers])[:] scale = th.cumsum(self.num_data, float_type) scale /= th.cast(th.shape(self.X)[0], float_type) # minibatch size return L * scale - KL
def predict_density(self, Xnew, Ynew, num_samples): Fmean, Fvar = self._build_predict(Xnew, full_cov=False, S=num_samples) l = self.likelihood.predict_density(Fmean, Fvar, Ynew) log_num_samples = th.log(th.cast(num_samples, float_type)) return th.reduce_logsumexp(l - log_num_samples, axis=0)
def KL(self): return torch.cast(0., dtype=settings.float_type)
def penalty(self, n): n = torch.cast(n, torch.floatx()) y = torch.pow(((5.0 + n) / 6.0), self.cfg.beam_alpha) return y
def top_out(self, x, lp, i): cfg = self.cfg score = lp / self.penalty(i + 1) flag = torch.equal(x[:, :, -1], cfg.END) score += (1.0 - torch.cast(flag, torch.floatx())) * utils.big_neg return self.top_beams([x, score, flag], score)
def top_tgt(self, x, lp): cfg = self.cfg fs = torch.equal(x[:, :, -1], cfg.END) lp += torch.cast(fs, torch.floatx()) * utils.big_neg return self.top_beams([x, lp], lp)
def call(self, inputs): """Implements call() for the layer.""" inp_k = inputs['inp_k'] seg_id = inputs['seg_id'] input_mask = inputs['input_mask'] mems = inputs['mems'] perm_mask = inputs['perm_mask'] target_mapping = inputs['target_mapping'] inp_q = inputs['inp_q'] new_mems = [] bsz = torch.shape(inp_k)[1] qlen = inp_k.shape.as_list()[0] mlen = mems[0].shape.as_list()[0] if mems is not None else 0 klen = mlen + qlen ##### Attention mask # causal attention mask if self.attn_type == 'uni': attn_mask = _create_mask(qlen, mlen, self.tf_float, self.same_length) # pylint: enable=protected-access attn_mask = attn_mask[:, :, None, None] elif self.attn_type == 'bi': attn_mask = None else: raise ValueError('Unsupported attention type: {}'.format( self.attn_type)) # data mask: input mask & perm mask if input_mask is not None and perm_mask is not None: data_mask = input_mask[None] + perm_mask elif input_mask is not None and perm_mask is None: data_mask = input_mask[None] elif input_mask is None and perm_mask is not None: data_mask = perm_mask else: data_mask = None if data_mask is not None: # all mems can be attended to mems_mask = torch.zeros([tf.shape(data_mask)[0], mlen, bsz], dtype=self.tf_float) data_mask = torch.cat([mems_mask, data_mask], 1) if attn_mask is None: attn_mask = data_mask[:, :, :, None] else: attn_mask += data_mask[:, :, :, None] if attn_mask is not None: attn_mask = torch.cast(attn_mask > 0, dtype=self.tf_float) if attn_mask is not None: non_tgt_mask = -torch.eye(qlen, dtype=self.tf_float) non_tgt_mask = torch.cat( [tf.zeros([qlen, mlen], dtype=self.tf_float), non_tgt_mask], axis=-1) non_tgt_mask = torch.cast( (attn_mask + non_tgt_mask[:, :, None, None]) > 0, dtype=self.tf_float) else: non_tgt_mask = None word_emb_k = self.embedding_lookup(inp_k) if inp_q is not None: if target_mapping is not None: word_emb_q = torch.tile(self.mask_emb, [tf.shape(target_mapping)[0], bsz, 1]) else: inp_q_ext = inp_q[:, :, None] word_emb_q = inp_q_ext * self.mask_emb + ( 1 - inp_q_ext) * word_emb_k output_h = self.h_dropout(word_emb_k) output_g = None if inp_q is not None: output_g = self.g_dropout(word_emb_q) ##### Segment embedding if seg_id is not None: # Convert `seg_id` to one-hot `seg_mat` mem_pad = torch.zeros([mlen, bsz], dtype=tf.int32) cat_id = torch.concat([mem_pad, seg_id], 0) if self.use_cls_mask: # `1` indicates not in the same segment [qlen x klen x bsz] # seg_id: [qlen x bsz] & cat_id: [klen x bsz] cls_mat = torch.logical_or( torch.equal(seg_id, tf.constant([data_utils.SEG_ID_CLS]))[:, None], torch.equal(cat_id, tf.constant([data_utils.SEG_ID_CLS]))[None, :]) seg_mat = torch.equal(seg_id[:, None], cat_id[None, :]) seg_mat = torch.logical_or(cls_mat, seg_mat) else: seg_mat = tf.logical_not( tf.equal(seg_id[:, None], cat_id[None, :])) else: seg_mat = None dtype = self.tf_float freq_seq = tf.range(0, self.d_model, 2.0) if dtype is not None and dtype != tf.float32: freq_seq = tf.cast(freq_seq, dtype=self.dtype) if self.attn_type == 'bi': beg, end = klen, -qlen elif self.attn_type == 'uni': beg, end = klen, -1 else: raise ValueError('Unknown `attn_type` {}.'.format(self.attn_type)) if self.bi_data: fwd_pos_seq = torch.range(beg, end, -1.0) bwd_pos_seq = torch.range(-beg, -end, 1.0) if dtype is not None and dtype != tf.float32: fwd_pos_seq = torch.cast(fwd_pos_seq, dtype=dtype) bwd_pos_seq = torxh.cast(bwd_pos_seq, dtype=dtype) if self.clamp_len > 0: fwd_pos_seq = torch.clip_by_value(fwd_pos_seq, -self.clamp_len, self.clamp_len) bwd_pos_seq = torch.clip_by_value(bwd_pos_seq, -self.clamp_len, self.clamp_len) if bsz is not None: fwd_pos_emb = self.fwd_position_embedding( fwd_pos_seq, bsz // 2) bwd_pos_emb = self.bwd_position_embedding( bwd_pos_seq, bsz // 2) else: fwd_pos_emb = self.fwd_position_embedding(fwd_pos_seq, None) bwd_pos_emb = self.bwd_position_embedding(bwd_pos_seq, None) pos_emb = tf.concat([fwd_pos_emb, bwd_pos_emb], axis=1) else: fwd_pos_seq = tf.range(beg, end, -1.0) if dtype is not None and dtype != tf.float32: fwd_pos_seq = tf.cast(fwd_pos_seq, dtype=dtype) if self.clamp_len > 0: fwd_pos_seq = tf.clip_by_value(fwd_pos_seq, -self.clamp_len, self.lamp_len) pos_emb = self.fwd_position_embedding(fwd_pos_seq, bsz) pos_emb = self.emb_dropout(pos_emb) if mems is None: mems = [None] * self.n_layer for i in range(self.n_layer): # cache new mems new_mems.append( _cache_mem(output_h, mems[i], self.mem_len, self.reuse_len)) # pylint: enable=protected-access # segment bias if seg_id is None: r_s_bias_i = None seg_embed_i = None else: r_s_bias_i = self.r_s_bias if not self.untie_r else self.r_s_bias[ i] seg_embed_i = self.seg_embed[i] ffn_layer = self.h_positionwise_ffn_layers[i] attention_layer = self.rel_multihead_layers[i] output_h, output_g = attention_layer( h=output_h, g=output_g, r=pos_emb, r_w_bias=self.r_w_bias if not self.untie_r else self.r_w_bias[i], r_r_bias=self.r_r_bias if not self.untie_r else self.r_r_bias[i], seg_mat=seg_mat, r_s_bias=r_s_bias_i, seg_embed=seg_embed_i, attn_mask_h=non_tgt_mask, attn_mask_g=attn_mask, mems=mems[i], target_mapping=target_mapping) output_h = ffn_layer(output_h) if output_g is not None: output_g = ffn_layer(output_g) if inp_q is not None: output = output_g else: output = output_h return output