def _my_get_params_init(shape, init, lookup):
# shape is a tuple of dims
assert init in [
"default", "random", "glorot", "ortho", "gaussian", "zeros"
], "Unknown init method %s" % init
poss_scale = COMMON_CONFIG.init_scale_l if lookup else COMMON_CONFIG.init_scale_nl
if len(shape) == 1: # set bias to 0
return np.zeros((shape[0], ))
else:
# get defaults
if init == "default":
init = COMMON_CONFIG.init_def_l if lookup else COMMON_CONFIG.init_def_nl
# specifics
if init == "glorot":
if lookup: # special for lookups
shape_g = (shape[-1], ) # fan-out for lookup
else:
shape_g = shape
w0 = Random.random_sample(shape, "winit") # [0,1)
w0 = (w0 - 0.5) * (2 * (np.sqrt(3.0 * len(shape_g) /
(sum(shape_g)))))
return w0 * poss_scale
elif init == "random":
w0 = Random.random_sample(shape, "winit") # [0,1)
w0 = (w0 - 0.5) * 2
return w0 * poss_scale
elif init == "gaussian":
w0 = Random.randn_clip(shape, "winit")
return w0 * poss_scale
elif init == "ortho":
# todo(note): always assume init square matrices
assert len(shape) == 2 and (shape[0] % shape[1] == 0
or shape[1] % shape[0] == 0
), f"Bad shape {shape} for ortho_init!"
orig_num = shape[0] // shape[1]
if orig_num == 0:
num = shape[1] // shape[0]
else:
num = orig_num
if num == 1:
w0 = Random.ortho_weight(shape[1], "winit")
else:
w0 = np.concatenate([
Random.ortho_weight(shape[1], "winit") for _ in range(num)
])
if orig_num == 0: # reverse it!
w0 = np.transpose(w0)
return w0 * poss_scale
elif init == "zeros":
return np.zeros(shape)
def _input_f_obtain(self, edrop):
if edrop <= 0.:
return lambda x: x
else:
# todo(warn): replaced sample, maybe an efficient but approx. impl & only works for 2d
edrop_rands = Random.random_sample(
(int(self.dropout_wordceil * edrop), )) # [0,1)
edrop_idxes = [int(self.dropout_wordceil * z) for z in edrop_rands]
edrop_set = set(edrop_idxes)
return lambda x: [0 if one in edrop_set else one
for one in x] # drop to 0 if fall in the set
def prepare(self, insts: List[ParseInstance], training):
conf = self.conf
word_idxes = [z.words.idxes for z in insts]
word_arr, input_mask = self.padder.pad(word_idxes) # [bsize, slen]
# prepare for the masks
input_word_mask = (Random.random_sample(word_arr.shape) <
conf.mask_rate) & (input_mask > 0.)
input_word_mask &= (word_arr >= conf.min_mask_rank)
input_word_mask[:, 0] = False # no masking for special ROOT
output_pred_mask = (input_word_mask & (word_arr <= conf.max_pred_rank))
return input_word_mask.astype(np.float32), output_pred_mask.astype(
np.float32), word_arr
def filter_pembed(self,
wv: WordVectors,
init_nohit=0.,
scale=1.0,
assert_all_hit=True,
set_init=True):
if init_nohit <= 0.:
get_nohit = lambda s: np.zeros((s, ), dtype=np.float32)
else:
get_nohit = lambda s: (Random.random_sample(
(s, )).astype(np.float32) - 0.5) * (2 * init_nohit)
#
ret = [np.zeros((wv.embed_size, ),
dtype=np.float32)] # init NIL is zero
record = defaultdict(int)
for ws in self.pools_hint_lexicon[1:]:
res = np.zeros((wv.embed_size, ), dtype=np.float32)
for w in ws:
hit, norm_name, norm_w = wv.norm_until_hit(w)
if hit:
value = np.asarray(wv.get_vec(norm_w, norm=False),
dtype=np.float32)
record[norm_name] += 1
else:
value = get_nohit(wv.embed_size)
record["no-hit"] += 1
res += value
ret.append(res)
#
assert not assert_all_hit or record[
"no-hit"] == 0, f"Filter-embed error: assert all-hit but get no-hit of {record['no-hit']}"
zlog(
f"Filter pre-trained Pembed: {record}, no-hit is inited with {init_nohit}."
)
ret = np.asarray(ret, dtype=np.float32) * scale
if set_init:
self.set_pool_init(ret)
return ret
def filter_embed(self, wv: 'WordVectors', init_nohit=0., scale=1.0, assert_all_hit=False):
if init_nohit <= 0.:
get_nohit = lambda s: np.zeros((s,), dtype=np.float32)
else:
get_nohit = lambda s: (Random.random_sample((s,)).astype(np.float32)-0.5) * (2*init_nohit)
#
ret = []
res = defaultdict(int)
for w in self.final_words:
hit, norm_name, norm_w = wv.norm_until_hit(w)
if hit:
value = np.asarray(wv.get_vec(norm_w, norm=False), dtype=np.float32)
res[norm_name] += 1
else:
value = get_nohit(wv.embed_size)
# value = np.zeros((wv.embed_size,), dtype=np.float32)
res["no-hit"] += 1
ret.append(value)
#
if assert_all_hit:
zcheck(res["no-hit"]==0, f"Filter-embed error: assert all-hit but get no-hit of {res['no-hit']}")
printing("Filter pre-trained embed: %s, no-hit is inited with %s." % (res, init_nohit))
return np.asarray(ret, dtype=np.float32) * scale