def Open(self, command, binary, ignore_stderr=True):
"""Open a KaldiOutputStream for the given pipe command.
Args:
command: The given command for pipe output.
binary: The opening mode, actually does not matter here.
Returns:
A boolean variable indicating if the operation is successful.
"""
if self.stream:
LogError('Called on already open file.')
if not command or command[0] != '|':
LogError('Invalid command \"%s\"' % command)
self.command = command[1:].split('|')
# TODO(cfyeh): make sure the pipes are concatenated correctly
stdin = sys.stdin
stderr = DEVNULL if ignore_stderr else sys.stderr
for i in xrange(len(self.command)):
self.process.append(
subprocess.Popen(self.command[i].split(),
stdin=stdin,
stdout=subprocess.PIPE,
stderr=stderr))
stdin = self.process[-1].stdout
self.stream = KaldiOutputStream(self.process[0])
return True
def AddData(self, data):
if self.data_begin > 0:
if self.data_begin > self.data_end: # Sanity check.
LogError('Sanity check failed: self.data_begin \"%d\" > '
'self.data_end \"%d\"' %
(self.data_begin, self.data_end))
left_over = self.data_end - self.data_begin
if left_over >= self.data_begin: # No overlap.
LogError('Unexpected data overlap: left_over \"%d\" >= '
'self.data_begin \"%d\"' %
(left_over, self.data_begin))
if left_over > 0:
self.data[0:left_over] = \
self.data[self.data_begin:self.data_begin+left_over]
self.data_begin = 0
self.data_end = left_over
if len(self.data) < self.data_end + len(data):
self.data_aux = numpy.empty_like(self.data)
self.data_aux[:] = self.data
size = self.data_end + data.shape[
0] + 1000 # Add extra 1000 elements, so we
# don't reallocate soon.
self.data = numpy.empty(size, self.data.dtype)
self.data[0:self.data_aux.shape[0]] = self.data_aux
self.data[self.data_end:self.data_end + data.shape[0]] = data
self.data_end += data.shape[0]
def Read(self, stream, binary):
if not binary:
LogError('We do not supoort text mode Read(), as it could be '
'quite slow using python. If you would really like to '
'use text format, please use \'nnet-copy\' in kaldi '
'to convert it to binary.')
# Read all the '<Tokens>' in arbitrary order,
# TODO(cfyeh): figure out whether to store these values.
while stream.Peek(1) == '<':
if stream.Peek(2) == '<L':
ExpectToken(stream, binary, '<LearnRateCoef>')
learn_rate_coef = ReadFloat(stream, binary)
elif stream.Peek(2) == '<B':
ExpectToken(stream, binary, '<BiasLearnRateCoef>')
bias_learn_rate_coef = ReadFloat(stream, binary)
elif stream.Peek(2) == '<M':
ExpectToken(stream, binary, '<MaxNorm>')
max_norm = ReadFloat(stream, binary)
else:
token = ReadToken(stream, binary)
LogError('Unknown token \"%s\"' % token)
matrix = FloatMatrix()
matrix.Read(stream, binary)
self.linearity = matrix.Value()
vector = FloatVector()
vector.Read(stream, binary)
self.bias = vector.Value()
return True
def ReadNextObject(self):
if self.state != RandomAccessTableReaderStateType.kNoObject:
LogError('Called from the wrong state \"%s\"' % self.state)
if self.input.Stream().Eof():
self.state = RandomAccessTableReaderStateType.kEof
return False
self.cur_key = ReadToken(self.input.Stream(), self.input.IsBinary(),
False)
c = self.input.Stream().Peek(1)
# We expect a space ' ' after the key. We also allow tab, just so we
# can read archives generated by scripts that may not be fully aware
# of how this format works.
if c != ' ' and c != '\t' and c != '\n':
LogError(
'Invalid archive file format: expected space after key '
'\"%s\", got character \"%s\" when reading archive \"%s\".' %
(self.cur_key, c, self.archive_rxfilename))
if c != '\n': # Consume the space or tab.
self.input.Stream().Read(1)
binary = InitKaldiInputStream(self.input.Stream())
if not self.holder.Read(self.input.Stream(), binary):
self.holder.Clear()
LogError('Failed to read object from archive \"%s\"' %
self.archive_rxfilename)
self.state = RandomAccessTableReaderStateType.kHaveObject
return True
def Open(self, rxfilename, binary):
"""Open a KaldiInputStream for the given filename and offset.
Args:
rxfilename: The given filename including offset.
binary: The opening mode.
Returns:
A boolean variable indicating if the operation is successful.
"""
(filename, offset) = self.SplitFilename(rxfilename)
if self.stream:
if self.filename == filename and self.binary == binary:
if not self.Seek(offset):
LogError('Invalid offset = \"%d\"' % offset)
return True
else:
self.Close()
self.filename = filename
self.binary = binary
mode = 'rt'
if binary:
mode = 'rb'
self.stream = KaldiInputStream(open(self.filename, mode))
if not self.Seek(offset):
LogError('Invalid offset = \"%d\"' % offset)
return True
def Open(self, wspecifier):
"""Open a writer for the given wspecifier.
Args:
wspecifier: The given wspecifier.
Returns:
A boolean variable indicating if the operation is successful.
"""
if self.state == TableWriterStateType.kUninitialized:
pass
elif self.state == TableWriterStateType.kWriteError:
LogError('Already open with write error.')
elif self.state == TableWriterStateType.kOpen:
if not self.Close():
LogError('Failed closing previously open stream.')
else:
LogError('Invalid state \"%s\"' % self.state)
self.wspecifier = wspecifier
(wspecifier_type, archive_filename, script_filename, opts) = \
ClassifyWspecifier(wspecifier)
self.archive_wxfilename = archive_filename
self.opts = opts
if wspecifier_type != WspecifierType.kArchiveWspecifier:
LogError('Invalid wspecifier type \"%s\"' % wspecifier_type)
self.output = Output()
success = self.output.Open(self.archive_wxfilename, self.opts.binary,
False)
if not success:
self.state = TableWriterStateType.kUninitialized
LogError('Failed to open stream \"%s\"' % self.archive_wxfilename)
self.state = TableWriterStateType.kOpen
return True
def AddData(self, mat):
m_rows = mat.shape[0]
m_cols = mat.shape[1]
if self.data is None:
rows = self.conf.randomizer_size
cols = m_cols
self.data = numpy.empty([rows, cols], dtype=float, order='C')
if self.data_begin > 0:
if self.data_begin > self.data_end: # Sanity check.
LogError('Sanity check failed: self.data_begin \"%d\" > '
'self.data_end \"%d\"' %
(self.data_begin, self.data_end))
left_over = self.data_end - self.data_begin
if left_over >= self.data_begin: # No overlap.
LogError('Unexpected data overlap: left_over \"%d\" >= '
'self.data_begin \"%d\"' %
(left_over, self.data_begin))
if left_over > 0:
self.data[0:left_over, :] = \
self.data[self.data_begin:self.data_begin+left_over, :]
self.data_begin = 0
self.data_end = left_over
if self.data.shape[0] < self.data_end + m_rows:
self.data_aux = numpy.empty_like(self.data)
self.data_aux[:] = self.data
rows = self.data_end + m_rows + 1000 # Add extra 1000 rows, so we
# don't reallocate soon.
cols = self.data.shape[1]
self.data = numpy.empty([rows, cols], dtype=float, order='C')
self.data[0:self.data_aux.shape[0], :] = self.data_aux
self.data[self.data_end:self.data_end + m_rows, :] = mat
self.data_end += m_rows
def Write(self, key, value):
if self.state == TableWriterStateType.kOpen:
pass
elif self.state == TableWriterStateType.kWriteError:
LogWarning('Attempting to write to invalid stream.')
elif self.state == TableWriterStateType.kUninitialized:
LogError('Invalid state \"%s\"' % self.state)
else:
LogError('Invalid state \"%s\"' % self.state)
# state is now kOpen or kWriteError.
if not IsToken(key): # e.g. empty string or has spaces...
LogError('Using invalid key \"%s\"' % key)
self.output.Stream().Write('%s ' % key)
if not WriteHolderValueToStream(self.output.Stream(), self.type,
self.opts.binary, value):
LogWarning('Write failure to \"%s\"' % self.archive_wxfilename)
self.state = TableWriterStateType.kWriteError
return False
# Even if this Write seems to have succeeded, we fail because a previous
# Write failed and the archive may be corrupted and unreadable.
if self.state == TableWriterStateType.kWriteError:
return False
if self.opts.flush:
self.Flush()
return True
def Randomize(self, mask):
if self.data_begin != 0:
LogError('Unexpected self.data_begin %d vs. 0' % self.data_begin)
if self.data_end <= 0:
LogError('Unexpected self.data_end %d' % self.data_begin)
if self.data_end != len(mask):
LogError('Mismatched self.data_end \"%d\" vs. mask size \"%d\"' %
(self.data_end, len(mask)))
self.data = self.data[mask]
def Value(self):
if not self.EnsureObjectLoaded():
LogError('Failed to load object from \"%s\" to suppress this '
'error, add the permissive (p, ) option to the '
'rspecifier.' % self.data_rxfilename)
if self.state == SequentialTableReaderStateType.kHaveRange:
return self.range_holder.Value()
elif self.state == SequentialTableReaderStateType.kHaveObject:
return self.holder.Value()
else:
LogError('Invalid state \"%s\"' % self.state)
def WriteHolderValueToStream(stream, holder_type, binary, value):
if holder_type == HolderType.kNoHolder:
LogError('No holder type is specified.')
elif holder_type == HolderType.kFloatMatrixHolder:
WriteFloatMatrixToStream(stream, binary, value)
elif holder_type == HolderType.kFloatVectorHolder:
WriteFloatVectorToStream(stream, binary, value)
elif holder_type == HolderType.kPosteriorHolder:
WritePosteriorToStream(stream, binary, value)
elif holder_type == HolderType.kInt32VectorHolder:
WriteInt32VectorToStream(stream, binary, value)
else:
LogError('Unrecognized holder type \"%s\"' % holder_type)
return True
def Read(self, stream, binary):
if binary:
ExpectToken(stream, binary, 'SM')
num_rows = ReadInt32(stream, binary)
if num_rows < 0 or num_rows > 10000000:
LogError('Unexpected number of rows %s' % num_rows)
self.rows = []
for r in xrange(num_rows):
self.rows.append(SparseVector())
self.rows[r].Read(stream, binary)
else:
LogError('We do not supoort text mode Read(), as it could be '
'quite slow using python. If you would really like to '
'use text format, please use \'copy-feats\' in kaldi '
'to convert it to binary.')
def NewHolderByType(holder_type):
if holder_type == HolderType.kNoHolder:
LogError('No holder type is specified.')
elif holder_type == HolderType.kFloatMatrixHolder:
return FloatMatrixHolder()
elif holder_type == HolderType.kFloatVectorHolder:
return FloatVectorHolder()
elif holder_type == HolderType.kPosteriorHolder:
return PosteriorHolder()
elif holder_type == HolderType.kInt32VectorHolder:
return BasicVectorHolder(BasicType.cint32)
elif holder_type == HolderType.kNnetExampleHolder:
return NnetExampleHolder()
else:
LogError('Unrecognized holder type \"%s\"' % holder_type)
def Open(self, rspecifier):
"""Open a reader for the given rspecifier.
Args:
rspecifier: The given rspecifier.
Returns:
A boolean variable indicating if the operation is successful.
"""
# You may call Open from states kUninitialized and kError.
# It may leave the object in any of the states.
if self.state == RandomAccessTableReaderStateType.kNoObject or \
self.state == RandomAccessTableReaderStateType.kHaveObject:
# call Close() yourself to suppress this exception.
if not self.Close():
LogError(
'Error closing previous input, rspecifier was \"%s\"' %
self.rspecifier)
self.rspecifier = rspecifier
(rspecifier_type, rxfilename, opts) = ClassifyRspecifier(rspecifier)
self.script_rxfilename = rxfilename
self.opts = opts
if rspecifier_type != RspecifierType.kScriptRspecifier:
LogError('Invalid rspecifier type \"%s\"' % rspecifier_type)
script_input = Input()
if not script_input.Open(self.script_rxfilename):
LogError('Failed opening script file \"%s\"' %
self.script_rxfilename)
if script_input.IsBinary():
LogError('script file should not be in binary format.')
script = list()
while True:
line = script_input.Stream().Readline()
if not line:
break
token = line.rstrip().split()
if len(token) != 2:
LogError('Invalid line \"%s\"' % line)
script.append((token[0], token[1]))
self.script = sorted(script, key=itemgetter(0))
self.keys = [key for key, _ in self.script]
self.state = RandomAccessTableReaderStateType.kNoObject
self.key = None
return True
def SplitFilename(self, rxfilename):
pos = rxfilename.rfind(':')
if pos < 0:
LogError('Invalid rxfilename \"%s\"' % rxfilename)
filename = rxfilename[:pos]
offset = int(rxfilename[pos + 1:])
return (filename, offset)
def ReadComponent(self, stream, binary):
token = ReadToken(stream, binary)
if token == '<Nnet>': # Skip the optional initial token.
token = ReadToken(stream, binary)
if token == "</Nnet>": # Network ends after terminal token appears.
return None
input_dim = ReadInt32(stream, binary)
output_dim = ReadInt32(stream, binary)
component = None
if token == '<AffineTransform>':
component = AffineTransform(input_dim, output_dim)
elif token == '<Sigmoid>':
component = Sigmoid(input_dim, output_dim)
elif token == '<Softmax>':
component = Softmax(input_dim, output_dim)
else:
LogError('Unrecognized or not yet supported component \"%s\"' %
token)
component.Read(stream, binary)
if stream.Peek(2) == '<!':
ExpectToken(stream, binary, '<!EndOfComponent>')
return component
def GetNumpyMatrix(self):
rows = self.global_header.num_rows
cols = self.global_header.num_cols
mat = numpy.empty([rows, cols], dtype=float, order='C')
if self.global_header.format == 1:
for c in xrange(cols):
p0 = Uint16ToFloat(self.global_header,
self.percol_header[c].percentile_0)
p25 = Uint16ToFloat(self.global_header,
self.percol_header[c].percentile_25)
p75 = Uint16ToFloat(self.global_header,
self.percol_header[c].percentile_75)
p100 = Uint16ToFloat(self.global_header,
self.percol_header[c].percentile_100)
for r in xrange(rows):
mat[r, c] = CharToFloat(p0, p25, p75, p100, self.data[r,
c])
elif self.global_header.format == 2:
for r in xrange(rows):
for c in xrange(cols):
mat[r, c] = Uint16ToFloat(self.global_header, self.data[r,
c])
else:
LogError('Unrecognized format = %s' % self.global_header.format)
return mat
def Value(self, key):
self.HandlePendingDelete()
(success, value) = self.FindKeyInternal(key, True)
if not success:
LogError('No such key \"%s\" in archive \"%s\"' %
(key, self.archive_rxfilename))
return value
def Value(self):
if self.data_end - self.data_begin < self.conf.minibatch_size:
LogError(
'Insufficient data for mini batch: %d vs. %d' %
(self.data_end - self.data_begin, self.conf.minibatch_size))
self.minibatch = \
self.data[self.data_begin:self.data_begin+self.conf.minibatch_size]
return self.minibatch
def IsOpen(self):
if self.state == TableWriterStateType.kUninitialized:
return False
elif self.state == TableWriterStateType.kOpen or \
self.state == TableWriterStateType.kWriteError:
return True
else:
LogError('Invalid state \"%s\"' % self.state)
def Read(self, stream, binary):
if binary:
self.global_header = GlobalHeader()
token = ReadToken(stream, binary, False)
stream.Read(1) # comsume the space
if token == 'CM':
self.global_header.format = 1
elif token == 'CM2':
self.global_header.format = 2
else:
LogError('Unexpected token \"%s\", expecting CM or CM2.' %
token)
data = struct.unpack('f' * 2, stream.Read(4 * 2))
self.global_header.min_value = data[0]
self.global_header.range = data[1]
data = struct.unpack('i' * 2, stream.Read(4 * 2))
self.global_header.num_rows = data[0]
self.global_header.num_cols = data[1]
rows = self.global_header.num_rows
cols = self.global_header.num_cols
if self.global_header.format == 1: # num_rows > 8, in CM1 format.
self.percol_header = []
data = numpy.frombuffer(bytearray(stream.Read(2 * 4 * cols)),
numpy.uint16).reshape(cols, 4)
for c in xrange(cols):
percol_header = PerColHeader()
percol_header.percentile_0 = data[c, 0]
percol_header.percentile_25 = data[c, 1]
percol_header.percentile_75 = data[c, 2]
percol_header.percentile_100 = data[c, 3]
self.percol_header.append(percol_header)
self.data = numpy.frombuffer(
bytearray(stream.Read(rows * cols)),
numpy.uint8).reshape(cols, rows).transpose()
elif self.global_header.format == 2: # num_rows <= 8, in CM2 format.
self.data = numpy.frombuffer(
bytearray(stream.Read(2 * rows * cols)),
numpy.uint16).reshape(rows, cols)
else:
LogError('Unrecognized format = %s' %
self.global_header.format)
else:
LogError('We do not supoort text mode Read(), as it could be '
'quite slow using python. If you would really like to '
'use text format, please use \'copy-feats\' in kaldi '
'to convert it to binary.')
def Read(self, stream, binary):
"""Read posteriorgrams from the given stream.
Args:
stream: An opened KaldiInputStream.
binary: If the input stream is in binary.
Returns:
An boolean variable indicating if the operation is successful.
"""
if binary:
sz = ReadInt32(stream, binary)
if sz < 0 or sz > 10000000:
LogError('Got negative or improbably large size \"%d\"' % sz)
self.value = []
for i in xrange(sz):
self.value.append([])
sz2 = ReadInt32(stream, binary)
if sz2 < 0:
LogError('Got negative size \"%d\"' % sz2)
for j in xrange(sz2):
lab = ReadInt32(stream, binary)
val = ReadFloat(stream, binary)
self.value[i].append((lab, val))
else:
line = stream.Readline()
tokens = line.rstrip().split()
i = 0
self.value = []
while True:
if i == len(tokens):
break
if tokens[i] != '[':
LogError('Expecting \"[\", got \"%s\" instead.' %
tokens[i])
self.value.append([])
while True:
i += 1
if tokens[i] == ']':
break
lab = int(tokens[i])
i += 1
val = float(tokens[i])
self.value[-1].append((lab, val))
i += 1
return True
def __init__(self, rxfilename=None):
self.impl = None
self.binary = None
# Explicitly compare with None since rxfilename == '' is still a valid
# input (kStandardInput)
if rxfilename is not None:
if not self.Open(rxfilename):
LogError('Failed opening input stream')
def __init__(self, wxfilename=None, binary=None, write_header=False):
self.impl = None
self.binary = None
# Explicitly compare with None since wxfilename == '' is still a valid
# input (kStandardInput)
if wxfilename is not None and binary is not None:
if not self.Open(wxfilename, binary, write_header):
LogError('Failed opening output stream')
def Open(self, rspecifier):
"""Open a reader for the given rspecifier.
Args:
rspecifier: The given rspecifier.
Returns:
A boolean variable indicating if the operation is successful.
"""
if self.state != SequentialTableReaderStateType.kUninitialized:
# call Close() yourself to suppress this exception.
if not self.Close():
if self.opts.permissive:
LogWarning('Error closing previous input (only warning, '
'since permissive mode).')
else:
LogError('Error closing previous input, rspecifier was '
'\"%s\"' % self.rspecifier)
self.rspecifier = rspecifier
(rspecifier_type, rxfilename, opts) = ClassifyRspecifier(rspecifier)
self.archive_rxfilename = rxfilename
self.opts = opts
if rspecifier_type != RspecifierType.kArchiveRspecifier:
LogError('Invalid rspecifier type \"%s\"' % rspecifier_type)
self.input = Input()
if self.holder.IsReadInBinary():
success = self.input.Open(self.archive_rxfilename)
else:
success = self.input.OpenTextMode(self.archive_rxfilename)
if not success:
self.state = SequentialTableReaderStateType.kUninitialized
LogError('Failed to open stream \"%s\"' % self.archive_rxfilename)
self.state = SequentialTableReaderStateType.kFileStart
self.Next()
if self.state == SequentialTableReaderStateType.kError:
self.input.Close()
self.state = SequentialTableReaderStateType.kUninitialized
LogError('Error beginning to read archive file \"%s\" (wrong '
'filename?)' % self.archive_rxfilename)
if self.state != SequentialTableReaderStateType.kHaveObject and \
self.state != SequentialTableReaderStateType.kEof:
LogError('Invalid state \"%s\"' % self.state)
return True
def IsOpen(self):
if self.state == RandomAccessTableReaderStateType.kEof or \
self.state == RandomAccessTableReaderStateType.kError or \
self.state == RandomAccessTableReaderStateType.kHaveObject or \
self.state == RandomAccessTableReaderStateType.kNoObject:
return True
elif self.state == RandomAccessTableReaderStateType.kUninitialized:
return False
else:
LogError('Invalid state \"%s\"' % self.state)
def Close(self):
status = 0
if self.script_input.IsOpen():
status = self.script_input.Close()
if self.data_input.IsOpen():
self.data_input.Close()
self.range_holder.Clear()
self.holder.Clear()
if not self.IsOpen():
LogError('Called on input that was not open.')
def Done(self):
if self.state == SequentialTableReaderStateType.kHaveObject:
return False
elif self.state == SequentialTableReaderStateType.kEof or \
self.state == SequentialTableReaderStateType.kError:
# Error condition, like Eof, counts as Done(); the
# destructor/Close() will inform the user of the error.
return True
else:
LogError('Invalid state \"%s\"' % self.state)
def Open(self, wspecifier, holder_type):
if self.IsOpen():
if not self.Close():
LogError('Failed to close previously open writer.')
(wspecifier_type, archive_filename, script_filename, opts) = \
ClassifyWspecifier(wspecifier)
if wspecifier_type == WspecifierType.kBothWspecifier:
self.impl = TableWriterBothImpl(holder_type)
elif wspecifier_type == WspecifierType.kArchiveWspecifier:
self.impl = TableWriterArchiveImpl(holder_type)
elif wspecifier_type == WspecifierType.kScriptWspecifier:
self.impl = TableWriterScriptImpl(holder_type)
else:
LogError('Invalid wspecifier \"%s\"' % wspecifier)
if not self.impl.Open(wspecifier):
del self.impl
self.impl = None
return False
return True
def Read(self, stream, binary):
ExpectToken(stream, binary, '<NnetIo>')
self.name = ReadToken(stream, binary)
self.indexes = ReadIndexVector(stream, binary)
c = stream.Peek(1)
if binary:
# TODO(cfyeh): implement double matrix.
if c == 'F' or c == 'C':
self.features = FloatMatrix()
elif c == 'S':
self.features = SparseMatrix()
else:
LogError('Unrecognized identifier \"%s\"' % c)
else:
# TODO(cfyeh): implement text mode.
LogError('Text mode not implemented yet.')
self.features.Read(stream, binary)
ExpectToken(stream, binary, '</NnetIo>')
return True