def write(self, todir):
"""
Write GMM parameters to files in a directory.
@param todir: Directory to read parameters from. The files
'means', 'variances', and 'mixture_weights' will be created in
this directory.
@ptype todir: string
"""
s3gau.open(os.path.join(todir, "means"),'wb').writeall([[self.means]])
s3gau.open(os.path.join(todir, "variances"),'wb').writeall([[self.variances]])
s3mixw.open(os.path.join(todir, "mixture_weights"),'wb').writeall \
(self.mixw[numpy.newaxis,numpy.newaxis,:])
def write(self, todir):
"""
Write GMM parameters to files in a directory.
@param todir: Directory to read parameters from. The files
'means', 'variances', and 'mixture_weights' will be created in
this directory.
@ptype todir: string
"""
s3gau.open(os.path.join(todir, "means"), 'wb').writeall([[self.means]])
s3gau.open(os.path.join(todir, "variances"),
'wb').writeall([[self.variances]])
s3mixw.open(os.path.join(todir, "mixture_weights"),'wb').writeall \
(self.mixw[numpy.newaxis,numpy.newaxis,:])
def read(self, path):
self.mdef = s3mdef.open(os.path.join(path, "mdef"))
self.mean = s3gau.open(os.path.join(path, "means"))
self.var = s3gau.open(os.path.join(path, "variances"))
self.mixw = s3mixw.open(os.path.join(path, "mixture_weights"))
self.tmat = s3tmat.open(os.path.join(path, "transition_matrices"))
# Normalize transition matrices and mixture weights
for t in range(0, len(self.tmat)):
self.tmat[t] = (self.tmat[t].T / self.tmat[t].sum(1)).T
for t in range(0, len(self.mixw)):
self.mixw[t] = (self.mixw[t].T / self.mixw[t].sum(1)).T.clip(self.mwfloor, 1.0)
# Floor variances and precompute normalizing and inverse variance terms
self.norm = numpy.empty((len(self.var),
len(self.var[0]),
len(self.var[0][0])),'d')
for m,mgau in enumerate(self.var):
for f,feat in enumerate(mgau):
fvar = feat.clip(self.varfloor, numpy.inf)
# log of 1/sqrt((2*pi)^N * det(var))
det = numpy.log(fvar).sum(1)
lrd = -0.5 * (det + numpy.log(2 * numpy.pi) * feat.shape[1])
self.norm[m,f] = lrd
# "Invert" variances
feat[:] = (1 / (fvar * 2))
# Construct senone to codebook mapping
if os.access(os.path.join(path, "senmgau"), os.F_OK):
self.senmgau = s3file.S3File(os.path.join(path, "senmgau")).read1d()
elif len(self.mean) == 1:
self.senmgau = numpy.ones(len(self.mixw))
else:
self.senmgau = numpy.arange(0, len(self.mixw))
self.senscr = numpy.ones(len(self.mixw)) * WORSTSCORE
def read(self, path):
self.mdef = s3mdef.open(os.path.join(path, "mdef"))
self.mean = s3gau.open(os.path.join(path, "means"))
self.var = s3gau.open(os.path.join(path, "variances"))
self.mixw = s3mixw.open(os.path.join(path, "mixture_weights"))
self.tmat = s3tmat.open(os.path.join(path, "transition_matrices"))
# Normalize transition matrices and mixture weights
for t in range(0, len(self.tmat)):
self.tmat[t] = (self.tmat[t].T / self.tmat[t].sum(1)).T
for t in range(0, len(self.mixw)):
self.mixw[t] = (self.mixw[t].T / self.mixw[t].sum(1)).T.clip(self.mwfloor, 1.0)
# Floor variances and precompute normalizing and inverse variance terms
self.norm = numpy.empty((len(self.var),
len(self.var[0]),
len(self.var[0][0])),'d')
for m,mgau in enumerate(self.var):
for f,feat in enumerate(mgau):
fvar = feat.clip(self.varfloor, numpy.inf)
# log of 1/sqrt((2*pi)^N * det(var))
det = numpy.log(fvar).sum(1)
lrd = -0.5 * (det + numpy.log(2 * numpy.pi) * feat.shape[1])
self.norm[m,f] = lrd
# "Invert" variances
feat[:] = (1 / (fvar * 2))
# Construct senone to codebook mapping
if os.access(os.path.join(path, "senmgau"), os.F_OK):
self.senmgau = s3file.S3File(os.path.join(path, "senmgau")).read1d()
elif len(self.mean) == 1:
self.senmgau = numpy.ones(len(self.mixw))
else:
self.senmgau = numpy.arange(0, len(self.mixw))
self.senscr = numpy.ones(len(self.mixw)) * WORSTSCORE
def read(self, fromdir):
"""
Read GMM parameters from files in a directory.
@param fromdir: Directory to read parameters from. The files
'means', 'variances', and 'mixture_weights' will be read from
this directory.
@ptype fromdir: string
"""
self.means = s3gau.open(os.path.join(fromdir, "means"))
self.variances = s3gau.open(os.path.join(fromdir, "variances"))
self.mixw = s3mixw.open(os.path.join(fromdir, "mixture_weights"))
self.featlen = self.means.veclen[0]
self.ndensity = self.means.density
def read(self, fromdir):
"""
Read GMM parameters from files in a directory.
@param fromdir: Directory to read parameters from. The files
'means', 'variances', and 'mixture_weights' will be read from
this directory.
@ptype fromdir: string
"""
self.means = s3gau.open(os.path.join(fromdir, "means"))
self.variances = s3gau.open(os.path.join(fromdir, "variances"))
self.mixw = s3mixw.open(os.path.join(fromdir, "mixture_weights"))
self.featlen = self.means.veclen[0]
self.ndensity = self.means.density
return None
else:
return fh.read(nbytes)
if len(sys.argv) < 3:
usage()
sys.exit(2)
sendump = open(sys.argv[1])
title = readstr(sendump)
while True:
header = readstr(sendump)
if header == None:
break
# Number of codewords and pdfs
r, c = struct.unpack("II", sendump.read(8))
print "rows: %d, columns: %d" % (r, c)
# Now read the stuff
opdf_8b = numpy.empty((c, 4, r))
for i in range(0, 4):
for j in range(0, r):
# Read bytes, expand to ints, shift them up
mixw = numpy.fromfile(sendump, "B", c).astype("i") << 10
# Negate, exponentiate, and untranspose
opdf_8b[:, i, j] = numpy.power(1.0001, -mixw)
s3mixw.open(sys.argv[2], "wb").writeall(opdf_8b)
if header.startswith("cluster_bits"):
cluster_bits = (int)(header.split()[1])
if header.startswith("feature_count"):
feature_count = (int)(header.split()[1])
if header.startswith("cluster_count"):
cluster_count = (int)(header.split()[1])
if cluster_count == 0:
# Number of codewords and pdfs
model_count = self.unpack_endian(sendump.read(4))
mixture_count = self.unpack_endian(sendump.read(4))
print "rows (model_count): %d, columns (mixture_count): %d, features (feature_count): %d" % (model_count, mixture_count, feature_count)
# Now read the stuff
self.opdf = numpy.empty((mixture_count, feature_count, model_count))
for i in range(0,feature_count):
for j in range(0,model_count):
# Read bytes, expand to ints, shift them up
mixw = numpy.fromfile(sendump, 'B', mixture_count).astype('i') << mixw_shift
# Negate, exponentiate, and untranspose
self.opdf[:,i,j] = numpy.power(1.0001, -mixw)
def usage():
print "Usage: %s IN_SENDUMP OUT_MIXW" % sys.argv[0]
if __name__ == '__main__':
if len(sys.argv) < 3:
usage()
sys.exit(2)
s3mixw.open(sys.argv[2], 'wb').writeall(Sendump(sys.argv[1]).mixw())
return None
else:
return fh.read(nbytes)
if len(sys.argv) < 3:
usage()
sys.exit(2)
sendump = open(sys.argv[1])
title = readstr(sendump)
while True:
header = readstr(sendump)
if header == None:
break
# Number of codewords and pdfs
r, c = struct.unpack('II', sendump.read(8))
print "rows: %d, columns: %d" % (r, c)
# Now read the stuff
opdf_8b = numpy.empty((c, 4, r))
for i in range(0, 4):
for j in range(0, r):
# Read bytes, expand to ints, shift them up
mixw = numpy.fromfile(sendump, 'B', c).astype('i') << 10
# Negate, exponentiate, and untranspose
opdf_8b[:, i, j] = numpy.power(1.0001, -mixw)
s3mixw.open(sys.argv[2], 'wb').writeall(opdf_8b)
break
if header.startswith("mixture_count"):
mixture_count = (int)(header.split()[1])
if header.startswith("model_count"):
model_count = (int)(header.split()[1])
if header.startswith("mixw_shift"):
mixw_shift = (int)(header.split()[1])
if header.startswith("cluster_bits"):
cluster_bits = (int)(header.split()[1])
if header.startswith("feature_count"):
feature_count = (int)(header.split()[1])
if header.startswith("cluster_count"):
cluster_count = (int)(header.split()[1])
if cluster_count == 0:
# Number of codewords and pdfs
model_count = unpack_endian(sendump.read(4))
mixture_count = unpack_endian(sendump.read(4))
print "rows (model_count): %d, columns (mixture_count): %d" % (model_count, mixture_count)
# Now read the stuff
opdf_8b = numpy.empty((mixture_count, feature_count, model_count))
for i in range(0,feature_count):
for j in range(0,model_count):
# Read bytes, expand to ints, shift them up
mixw = numpy.fromfile(sendump, 'B', mixture_count).astype('i') << mixw_shift
# Negate, exponentiate, and untranspose
opdf_8b[:,i,j] = numpy.power(1.0001, -mixw)
s3mixw.open(sys.argv[2], 'wb').writeall(opdf_8b)
def write_senmgau(outfile, tree, mixw, nclust):
"""
Create and write a senone to codebook mapping based on a senone
tree, with a maximum of nclust clusters (splitting the largest
cluster first at each level).
"""
clusters = []
clusters.append(tree)
while len(clusters) < nclust:
clusters.sort(key=lambda x: len(leaves(x)), reverse=True)
big = clusters[0]
del clusters[0:1]
clusters.extend((big[0][0], big[0][1]))
print "cluster sizes:", [len(leaves(x)) for x in clusters]
mixwmap = numpy.zeros(len(mixw), 'int32')
for i, c in enumerate(clusters):
for mixwid in leaves(c):
mixwmap[mixwid] = i
print "writing %d senone mappings" % len(mixwmap)
s3senmgau.open(outfile, "wb").write_mapping(mixwmap)
if __name__ == '__main__':
mixw, outfile = sys.argv[1:]
mixw = norm_floor_mixw(s3mixw.open(mixw).getall())
# Build merged mixture weight tree
tree = cluster_merged(mixw)
writetree_merged(outfile, tree, mixw)
# Number of codewords and pdfs
r, c = struct.unpack('II', sendump.read(8))
print r,c
# Now read the stuff
opdf_8b = numpy.empty((670,n_cb,r))
for i in range(0,n_cb):
for j in range(0,r):
# Read bytes, expand to ints, shift them up
mixw = numpy.fromfile(sendump, dtype='int8', count=670).astype('i') << 10
sendump.read(2)
# Negate, exponentiate, and untranspose
opdf_8b[:,i,j] = numpy.power(1.0001, -mixw)
return opdf_8b
def norm_floor_mixw(mixw, floor=1e-7):
return (mixw.T / mixw.T.sum(0)).T.clip(floor, 1.0)
if __name__ == '__main__':
ifn, ofn = sys.argv[1:]
if os.path.basename(ifn).startswith('sendump'):
mixw = read_sendump(ifn)
else:
mixw = norm_floor_mixw(s3mixw.open(ifn).getall(), 1e-7)
cb = quantize_mixw_kmeans(mixw, 15, 1e-7)
print cb
mwmap = map_mixw_cb(mixw, cb, 1e-7)
write_sendump_hb(mwmap, cb, ofn)
def norm_floor_mixw(mixw, floor=1e-7):
return (mixw.T / mixw.T.sum(0)).T.clip(floor, 1.0)
def write_senmgau(outfile, tree, mixw, nclust):
"""
Create and write a senone to codebook mapping based on a senone
tree, with a maximum of nclust clusters (splitting the largest
cluster first at each level).
"""
clusters = []
clusters.append(tree)
while len(clusters) < nclust:
clusters.sort(key=lambda x:len(leaves(x)), reverse=True)
big = clusters[0]
del clusters[0:1]
clusters.extend((big[0][0], big[0][1]))
print "cluster sizes:", [len(leaves(x)) for x in clusters]
mixwmap = numpy.zeros(len(mixw), 'int32')
for i, c in enumerate(clusters):
for mixwid in leaves(c):
mixwmap[mixwid] = i
print "writing %d senone mappings" % len(mixwmap)
s3senmgau.open(outfile, "wb").write_mapping(mixwmap)
if __name__ == '__main__':
mixw, outfile = sys.argv[1:]
mixw = norm_floor_mixw(s3mixw.open(mixw).getall())
# Build merged mixture weight tree
tree = cluster_merged(mixw)
writetree_merged(outfile, tree, mixw)
if header.startswith("cluster_count"):
cluster_count = (int)(header.split()[1])
if cluster_count == 0:
# Number of codewords and pdfs
model_count = self.unpack_endian(sendump.read(4))
mixture_count = self.unpack_endian(sendump.read(4))
print "rows (model_count): %d, columns (mixture_count): %d, features (feature_count): %d" % (
model_count, mixture_count, feature_count)
# Now read the stuff
self.opdf = numpy.empty((mixture_count, feature_count, model_count))
for i in range(0, feature_count):
for j in range(0, model_count):
# Read bytes, expand to ints, shift them up
mixw = numpy.fromfile(sendump, 'B',
mixture_count).astype('i') << mixw_shift
# Negate, exponentiate, and untranspose
self.opdf[:, i, j] = numpy.power(1.0001, -mixw)
def usage():
print "Usage: %s IN_SENDUMP OUT_MIXW" % sys.argv[0]
if __name__ == '__main__':
if len(sys.argv) < 3:
usage()
sys.exit(2)
s3mixw.open(sys.argv[2], 'wb').writeall(Sendump(sys.argv[1]).mixw())
print r, c
# Now read the stuff
opdf_8b = numpy.empty((670, n_cb, r))
for i in range(0, n_cb):
for j in range(0, r):
# Read bytes, expand to ints, shift them up
mixw = numpy.fromfile(sendump, dtype='int8',
count=670).astype('i') << 10
sendump.read(2)
# Negate, exponentiate, and untranspose
opdf_8b[:, i, j] = numpy.power(1.0001, -mixw)
return opdf_8b
def norm_floor_mixw(mixw, floor=1e-7):
return (mixw.T / mixw.T.sum(0)).T.clip(floor, 1.0)
if __name__ == '__main__':
ifn, ofn = sys.argv[1:]
if os.path.basename(ifn).startswith('sendump'):
mixw = read_sendump(ifn)
else:
mixw = norm_floor_mixw(s3mixw.open(ifn).getall(), 1e-7)
cb = quantize_mixw_kmeans(mixw, 15, 1e-7)
print cb
mwmap = map_mixw_cb(mixw, cb, 1e-7)
write_sendump_hb(mwmap, cb, ofn)