def readvec(self, vecfile):
"""
read pre-trained word vectors from binary file
:param vecfile: file path
:return: self
"""
with io.open(vecfile) as fin:
header = fin.readline()
vocab_size, vector_size = map(int, header.split()) # throws for invalid file format
binary_len = np.dtype(np.float32).itemsize * vector_size
# vocab_size = 5000
self.W = np.zeros((vocab_size, vector_size), dtype=np.float32)
for index in xrange(vocab_size):
# mixed text and binary: read text first, then binary
word = []
while True:
ch = fin.read(1)
if ch == b' ':
break
if ch != b'\n': # ignore newlines in front of words (some binary files have)
word.append(ch)
word = unicode(b''.join(word), encoding='utf-8')
weights = multiarray.fromstring(fin.read(binary_len), dtype=np.float32)
self.vocub.append(word)
self.word_idx_map[word] = index
self.W[index] = weights
self.last_word_index = index
return self
def create_random_block(bitlist, piece):
assert len(piece) % len(bitlist) == 0
blocksize = len(piece) / len(bitlist)
random_block = ma.zeros(blocksize, numpy.uint8)
for i in xrange(len(bitlist)):
if bitlist[i] == 1:
block = ma.fromstring(
piece[i * blocksize:i * blocksize + blocksize], numpy.uint8)
random_block ^= block
return random_block
def _raptor_precode(self, piece):
assert len(piece) % self.blocksize == 0
es = EquationSet(self.L, self.blocksize)
for i in xrange(0, self.K, 1):
block = ma.fromstring(piece[i*self.blocksize:i*self.blocksize+self.blocksize],numpy.uint8)
e = Equation(LTlist(i,self.K,self.L,self.L1),block)
es.puteq(e)
block = ma.zeros(self.blocksize,numpy.uint8)
for i in xrange(0, self.S, 1):
e = Equation(self.raptorSmtx.getrow(i)[:self.L],block)
es.puteq(e)
for i in xrange(0, self.H, 1):
e = Equation(self.raptorHmtx.getrow(i)[:self.L],block)
es.puteq(e)
assert es.done()
return es.getdata()
def _raptor_precode(self, piece):
assert len(piece) % self.blocksize == 0
es = EquationSet(self.L, self.blocksize)
for i in xrange(0, self.K, 1):
block = ma.fromstring(
piece[i * self.blocksize:i * self.blocksize + self.blocksize],
numpy.uint8)
e = Equation(LTlist(i, self.K, self.L, self.L1), block)
es.puteq(e)
block = ma.zeros(self.blocksize, numpy.uint8)
for i in xrange(0, self.S, 1):
e = Equation(self.raptorSmtx.getrow(i)[:self.L], block)
es.puteq(e)
for i in xrange(0, self.H, 1):
e = Equation(self.raptorHmtx.getrow(i)[:self.L], block)
es.puteq(e)
assert es.done()
return es.getdata()
def fromstring(string, typecode='l', count=-1, dtype=None):
dtype = convtypecode(typecode, dtype)
return mu.fromstring(string, dtype, count=count)
def decode(self, seqnum, block):
block = ma.fromstring(block,numpy.uint8)
return self._use_raptor_block(seqnum, block)
def use_random_block(self, seed, rblock):
rblock = ma.fromstring(rblock, numpy.uint8)
e = Equation(rand50list(seed, self.L), rblock)
return self.es.puteq(e)
def decode(self, seqnum, block):
block = ma.fromstring(block, numpy.uint8)
return self._use_raptor_block(seqnum, block)