def torch_kspace_location(shape_y, shape_x):
"""Construct matrix with k-psace normalized location as tensor."""
y = torch.cast(torch.range(shape_y), torch.float32)
y = y / torch.cast(shape_y, torch.float32) - 0.5
x = torch.cast(torch.range(shape_x), torch.float32)
x = x / torch.cast(shape_x, torch.float32) - 0.5
[yg, xg] = torch.meshgrid(y, x)
yg = torch.transpose(yg, [1, 0])
xg = torch.transpose(xg, [1, 0])
out = torch.stack((yg, xg))
return out
def ifftc(im, name="ifftc", do_orthonorm=True):
"""Centered iFFT on second to last dimension."""
im_out = im
if do_orthonorm:
fftscale = torch.sqrt(1.0 * im_out.get_shape().as_list()[-2])
else:
fftscale = 1.0
fftscale = torch.cast(fftscale, dtype=torch.complex64)
if len(im.get_shape()) == 4:
im_out = torch.transpose(im_out, [0, 3, 1, 2])
im_out = fftshift(im_out, axis=3)
else:
im_out = torch.transpose(im_out, [2, 0, 1])
im_out = fftshift(im_out, axis=2)
with torch.device("/gpu:0"):
# FFT is only supported on the GPU
im_out = torch.ifft(im_out) * fftscale
if len(im.get_shape()) == 4:
im_out = fftshift(im_out, axis=3)
im_out = torch.transpose(im_out, [0, 2, 3, 1])
else:
im_out = fftshift(im_out, axis=2)
im_out = torch.transpose(im_out, [1, 2, 0])
return im_out
def predict():
"""Predict unseen images"""
"""Step 0: load data and trained model"""
mnist = input_data.read_data_sets("./data/", one_hot=True)
checkpoint_dir = sys.argv[1]
"""Step 1: build the rnn model"""
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
weights = tf.Variable(tf.random_normal([n_hidden, n_classes]), name='weights')
biases = tf.Variable(tf.random_normal([n_classes]), name='biases')
pred = rnn_model(x, weights, biases)
correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
"""Step 2: predict new images with the trained model"""
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
"""Step 2.0: load the trained model"""
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir + 'checkpoints')
print('Loaded the trained model: {}'.format(checkpoint_file))
saver = tf.train.Saver()
saver.restore(sess, checkpoint_file)
"""Step 2.1: predict new data"""
test_len = 500
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print("Testing Accuracy:", sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
def kspace_threshhold(image_orig, threshhold=1e-8, name="kspace_threshhold"):
"""Find k-space mask based on threshhold.
Anything less the specified threshhold is set to 0.
Anything above the specified threshhold is set to 1.
"""
mask_x = torch.greater(torch.abs(image_orig), threshhold)
mask_x = torch.cast(mask_x, dtype=torch.float32)
return mask_x
def gpi(self, observation, cumulant_weights):
q_values = self.__call__(th.expand_dims(observation, axis=0))[0]
q_w = th.tensordot(q_values, cumulant_weights, axes=[1, 0]) # [P,a]
q_w_actions = th.reduce_max(q_w, axis=0)
action = th.cast(th.argmax(q_w_actions), th.int32)
return action
def forward(self, x):
word_embedding = self.word_embedding(x)
# 保证x转化到 d ^ (1/2) 正态分布，且要比position encoding大
word_embedding *= torch.sqrt(torch.cast(self.d_model, torch.float32))
# (batch, seq_len)
pos_seq = torch.arange(0, x.size(1)).repeat(x.size(0), 1)
if torch.cuda.is_available():
pos_seq = pos_seq.cuda()
embedding = word_embedding + self.pos_embedding(pos_seq)
embedding = self.embedding_dropout(embedding)
return embedding, pos_seq
def forward(self, q, k, v, mask=None):
"""
q, k, v: (batch*n_head, t_len, hidden_size)
mask: (batch*n_head, q_len, q_len)
"""
attention = torch.bmm(q, k.transpose(1, 2))
attention = attention / torch.sqrt(torch.cast(q.shape[-1], torch.float32))
if mask is not None:
attention = attention.masked_fill(mask, -np.inf)
attention = self.softmax(attention) # (batch*n_head, t_len, t_len)
output = torch.bmm(attention, v) # (batch*n_head, t_len, hidden_size)
return output, attention
def positional_encoding(sentence_length, d_model):
angle_rads = get_angles(
np.arange(sentence_length)[:, np.newaxis],
np.arange(d_model)[np.newaxis, :], d_model)
# 将 sin 应用于数组中的偶数索引（indices）；2i
angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])
# 将 cos 应用于数组中的奇数索引；2i+1
angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])
pos_encoding = angle_rads[np.newaxis, ...]
pos_encoding = torch.cast(pos_encoding, dtype=torch.float32)
pos_encoding = pos_encoding.permute(1, 0,
2).contiguous() # (len, 1, d_model)
return pos_encoding
def test_cast_variable(self):
inputs = [
torch.ByteTensor(1),
torch.CharTensor(1),
torch.DoubleTensor(1),
torch.FloatTensor(1),
torch.IntTensor(1),
torch.LongTensor(1),
torch.ShortTensor(1),
]
for inp in inputs:
assert type(inp) == type(torch.cast(Variable(inp), type(inp)).data)
def to_scores(self, qk, mask, v):
b = 0
if mask is not None:
b = torch.logical_not(mask)
b = torch.cast(b, torch.floatx()) * qu.big_neg()
if self.proxim_b is not None:
b += self.proxim_b
b = b[:, None, :, None]
y = torch.softmax(qk * self.scale + b)
cfg = self.cfg
y = self.drop(y, cfg.drop_attn or cfg.drop)
y = torch.einsum("bnij,bnjv->bniv", y, v)
return y
def loss(self, y_true, y_pred, from_logits=False, label_smoothing=0):
"""
Calculate the loss
(The test process will use this function)
TODO
you should provide this function no matter you use it in training or not;
because the test process would call this function
:return: loss (float)
"""
y_true = torch.view(-1, y_true)
# calculate the padding mask
mask = torch.cast(torch.math.not_equal(y_true, 0), y_pred.dtype)
# calculate the loss
loss_ = self.compile_params['loss'](y_true, y_pred)
# remove the padding part's loss by timing the mask
loss_ *= mask
# calculate the mean loss
return tf.reduce_mean(loss_)
def cast(value, dtype=None, lib=None, hint=None):
if pylist(value):
value = stack(value, axis=-1, lib=lib)
lib = as_lib(lib, hint=value)
if dtype is None:
if hint is None:
hint = value
dtypes = pyflat(as_dtype(hint, lib=lib, deep=True))
if not functools.reduce(operator.eq, dtypes):
raise ValueError("dtypes not equal for {}".format(value))
dtype = dtypes[0]
dtypes = pyflat(as_dtype(dtype, lib=lib, deep=True))
if not functools.reduce(operator.eq, dtypes):
raise ValueError("dtypes not equal for {}".format(value))
dtype = dtypes[0]
if lib == np:
return np.array(value, dtype=dtype)
elif lib == tensorflow:
return tensorflow.cast(value, dtype=dtype)
else:
assert lib == torch
return torch.cast(value, dtype=dtype)
def kspace_mask(image_orig, name="kspace_mask", dtype=None):
"""Find k-space mask."""
mask_x = torch.not_equal(image_orig, 0)
if dtype is not None:
mask_x = torch.cast(mask_x, dtype=dtype)
return mask_x
def forward(self, x, mask=None):
y = self.pos_b
if mask is not None:
y *= torch.cast(mask, self.pos_b.dtype)
return x + y
def _build_likelihood(self):
L = th.cumsum(self.E_log_p_Y(self.X, self.Y))[:]
KL = th.cumsum([layer.KL() for layer in self.layers])[:]
scale = th.cumsum(self.num_data, float_type)
scale /= th.cast(th.shape(self.X)[0], float_type) # minibatch size
return L * scale - KL
def predict_density(self, Xnew, Ynew, num_samples):
Fmean, Fvar = self._build_predict(Xnew, full_cov=False, S=num_samples)
l = self.likelihood.predict_density(Fmean, Fvar, Ynew)
log_num_samples = th.log(th.cast(num_samples, float_type))
return th.reduce_logsumexp(l - log_num_samples, axis=0)
def KL(self):
return torch.cast(0., dtype=settings.float_type)
def penalty(self, n):
n = torch.cast(n, torch.floatx())
y = torch.pow(((5.0 + n) / 6.0), self.cfg.beam_alpha)
return y
def top_out(self, x, lp, i):
cfg = self.cfg
score = lp / self.penalty(i + 1)
flag = torch.equal(x[:, :, -1], cfg.END)
score += (1.0 - torch.cast(flag, torch.floatx())) * utils.big_neg
return self.top_beams([x, score, flag], score)
def top_tgt(self, x, lp):
cfg = self.cfg
fs = torch.equal(x[:, :, -1], cfg.END)
lp += torch.cast(fs, torch.floatx()) * utils.big_neg
return self.top_beams([x, lp], lp)
def call(self, inputs):
"""Implements call() for the layer."""
inp_k = inputs['inp_k']
seg_id = inputs['seg_id']
input_mask = inputs['input_mask']
mems = inputs['mems']
perm_mask = inputs['perm_mask']
target_mapping = inputs['target_mapping']
inp_q = inputs['inp_q']
new_mems = []
bsz = torch.shape(inp_k)[1]
qlen = inp_k.shape.as_list()[0]
mlen = mems[0].shape.as_list()[0] if mems is not None else 0
klen = mlen + qlen
##### Attention mask
# causal attention mask
if self.attn_type == 'uni':
attn_mask = _create_mask(qlen, mlen, self.tf_float,
self.same_length)
# pylint: enable=protected-access
attn_mask = attn_mask[:, :, None, None]
elif self.attn_type == 'bi':
attn_mask = None
else:
raise ValueError('Unsupported attention type: {}'.format(
self.attn_type))
# data mask: input mask & perm mask
if input_mask is not None and perm_mask is not None:
data_mask = input_mask[None] + perm_mask
elif input_mask is not None and perm_mask is None:
data_mask = input_mask[None]
elif input_mask is None and perm_mask is not None:
data_mask = perm_mask
else:
data_mask = None
if data_mask is not None:
# all mems can be attended to
mems_mask = torch.zeros([tf.shape(data_mask)[0], mlen, bsz],
dtype=self.tf_float)
data_mask = torch.cat([mems_mask, data_mask], 1)
if attn_mask is None:
attn_mask = data_mask[:, :, :, None]
else:
attn_mask += data_mask[:, :, :, None]
if attn_mask is not None:
attn_mask = torch.cast(attn_mask > 0, dtype=self.tf_float)
if attn_mask is not None:
non_tgt_mask = -torch.eye(qlen, dtype=self.tf_float)
non_tgt_mask = torch.cat(
[tf.zeros([qlen, mlen], dtype=self.tf_float), non_tgt_mask],
axis=-1)
non_tgt_mask = torch.cast(
(attn_mask + non_tgt_mask[:, :, None, None]) > 0,
dtype=self.tf_float)
else:
non_tgt_mask = None
word_emb_k = self.embedding_lookup(inp_k)
if inp_q is not None:
if target_mapping is not None:
word_emb_q = torch.tile(self.mask_emb,
[tf.shape(target_mapping)[0], bsz, 1])
else:
inp_q_ext = inp_q[:, :, None]
word_emb_q = inp_q_ext * self.mask_emb + (
1 - inp_q_ext) * word_emb_k
output_h = self.h_dropout(word_emb_k)
output_g = None
if inp_q is not None:
output_g = self.g_dropout(word_emb_q)
##### Segment embedding
if seg_id is not None:
# Convert `seg_id` to one-hot `seg_mat`
mem_pad = torch.zeros([mlen, bsz], dtype=tf.int32)
cat_id = torch.concat([mem_pad, seg_id], 0)
if self.use_cls_mask:
# `1` indicates not in the same segment [qlen x klen x bsz]
# seg_id: [qlen x bsz] & cat_id: [klen x bsz]
cls_mat = torch.logical_or(
torch.equal(seg_id,
tf.constant([data_utils.SEG_ID_CLS]))[:, None],
torch.equal(cat_id,
tf.constant([data_utils.SEG_ID_CLS]))[None, :])
seg_mat = torch.equal(seg_id[:, None], cat_id[None, :])
seg_mat = torch.logical_or(cls_mat, seg_mat)
else:
seg_mat = tf.logical_not(
tf.equal(seg_id[:, None], cat_id[None, :]))
else:
seg_mat = None
dtype = self.tf_float
freq_seq = tf.range(0, self.d_model, 2.0)
if dtype is not None and dtype != tf.float32:
freq_seq = tf.cast(freq_seq, dtype=self.dtype)
if self.attn_type == 'bi':
beg, end = klen, -qlen
elif self.attn_type == 'uni':
beg, end = klen, -1
else:
raise ValueError('Unknown `attn_type` {}.'.format(self.attn_type))
if self.bi_data:
fwd_pos_seq = torch.range(beg, end, -1.0)
bwd_pos_seq = torch.range(-beg, -end, 1.0)
if dtype is not None and dtype != tf.float32:
fwd_pos_seq = torch.cast(fwd_pos_seq, dtype=dtype)
bwd_pos_seq = torxh.cast(bwd_pos_seq, dtype=dtype)
if self.clamp_len > 0:
fwd_pos_seq = torch.clip_by_value(fwd_pos_seq, -self.clamp_len,
self.clamp_len)
bwd_pos_seq = torch.clip_by_value(bwd_pos_seq, -self.clamp_len,
self.clamp_len)
if bsz is not None:
fwd_pos_emb = self.fwd_position_embedding(
fwd_pos_seq, bsz // 2)
bwd_pos_emb = self.bwd_position_embedding(
bwd_pos_seq, bsz // 2)
else:
fwd_pos_emb = self.fwd_position_embedding(fwd_pos_seq, None)
bwd_pos_emb = self.bwd_position_embedding(bwd_pos_seq, None)
pos_emb = tf.concat([fwd_pos_emb, bwd_pos_emb], axis=1)
else:
fwd_pos_seq = tf.range(beg, end, -1.0)
if dtype is not None and dtype != tf.float32:
fwd_pos_seq = tf.cast(fwd_pos_seq, dtype=dtype)
if self.clamp_len > 0:
fwd_pos_seq = tf.clip_by_value(fwd_pos_seq, -self.clamp_len,
self.lamp_len)
pos_emb = self.fwd_position_embedding(fwd_pos_seq, bsz)
pos_emb = self.emb_dropout(pos_emb)
if mems is None:
mems = [None] * self.n_layer
for i in range(self.n_layer):
# cache new mems
new_mems.append(
_cache_mem(output_h, mems[i], self.mem_len, self.reuse_len))
# pylint: enable=protected-access
# segment bias
if seg_id is None:
r_s_bias_i = None
seg_embed_i = None
else:
r_s_bias_i = self.r_s_bias if not self.untie_r else self.r_s_bias[
i]
seg_embed_i = self.seg_embed[i]
ffn_layer = self.h_positionwise_ffn_layers[i]
attention_layer = self.rel_multihead_layers[i]
output_h, output_g = attention_layer(
h=output_h,
g=output_g,
r=pos_emb,
r_w_bias=self.r_w_bias
if not self.untie_r else self.r_w_bias[i],
r_r_bias=self.r_r_bias
if not self.untie_r else self.r_r_bias[i],
seg_mat=seg_mat,
r_s_bias=r_s_bias_i,
seg_embed=seg_embed_i,
attn_mask_h=non_tgt_mask,
attn_mask_g=attn_mask,
mems=mems[i],
target_mapping=target_mapping)
output_h = ffn_layer(output_h)
if output_g is not None:
output_g = ffn_layer(output_g)
if inp_q is not None:
output = output_g
else:
output = output_h
return output