def Open(self, command, binary, ignore_stderr=True): """Open a KaldiOutputStream for the given pipe command. Args: command: The given command for pipe output. binary: The opening mode, actually does not matter here. Returns: A boolean variable indicating if the operation is successful. """ if self.stream: LogError('Called on already open file.') if not command or command[0] != '|': LogError('Invalid command \"%s\"' % command) self.command = command[1:].split('|') # TODO(cfyeh): make sure the pipes are concatenated correctly stdin = sys.stdin stderr = DEVNULL if ignore_stderr else sys.stderr for i in xrange(len(self.command)): self.process.append( subprocess.Popen(self.command[i].split(), stdin=stdin, stdout=subprocess.PIPE, stderr=stderr)) stdin = self.process[-1].stdout self.stream = KaldiOutputStream(self.process[0]) return True
def AddData(self, data): if self.data_begin > 0: if self.data_begin > self.data_end: # Sanity check. LogError('Sanity check failed: self.data_begin \"%d\" > ' 'self.data_end \"%d\"' % (self.data_begin, self.data_end)) left_over = self.data_end - self.data_begin if left_over >= self.data_begin: # No overlap. LogError('Unexpected data overlap: left_over \"%d\" >= ' 'self.data_begin \"%d\"' % (left_over, self.data_begin)) if left_over > 0: self.data[0:left_over] = \ self.data[self.data_begin:self.data_begin+left_over] self.data_begin = 0 self.data_end = left_over if len(self.data) < self.data_end + len(data): self.data_aux = numpy.empty_like(self.data) self.data_aux[:] = self.data size = self.data_end + data.shape[ 0] + 1000 # Add extra 1000 elements, so we # don't reallocate soon. self.data = numpy.empty(size, self.data.dtype) self.data[0:self.data_aux.shape[0]] = self.data_aux self.data[self.data_end:self.data_end + data.shape[0]] = data self.data_end += data.shape[0]
def Read(self, stream, binary): if not binary: LogError('We do not supoort text mode Read(), as it could be ' 'quite slow using python. If you would really like to ' 'use text format, please use \'nnet-copy\' in kaldi ' 'to convert it to binary.') # Read all the '<Tokens>' in arbitrary order, # TODO(cfyeh): figure out whether to store these values. while stream.Peek(1) == '<': if stream.Peek(2) == '<L': ExpectToken(stream, binary, '<LearnRateCoef>') learn_rate_coef = ReadFloat(stream, binary) elif stream.Peek(2) == '<B': ExpectToken(stream, binary, '<BiasLearnRateCoef>') bias_learn_rate_coef = ReadFloat(stream, binary) elif stream.Peek(2) == '<M': ExpectToken(stream, binary, '<MaxNorm>') max_norm = ReadFloat(stream, binary) else: token = ReadToken(stream, binary) LogError('Unknown token \"%s\"' % token) matrix = FloatMatrix() matrix.Read(stream, binary) self.linearity = matrix.Value() vector = FloatVector() vector.Read(stream, binary) self.bias = vector.Value() return True
def ReadNextObject(self): if self.state != RandomAccessTableReaderStateType.kNoObject: LogError('Called from the wrong state \"%s\"' % self.state) if self.input.Stream().Eof(): self.state = RandomAccessTableReaderStateType.kEof return False self.cur_key = ReadToken(self.input.Stream(), self.input.IsBinary(), False) c = self.input.Stream().Peek(1) # We expect a space ' ' after the key. We also allow tab, just so we # can read archives generated by scripts that may not be fully aware # of how this format works. if c != ' ' and c != '\t' and c != '\n': LogError( 'Invalid archive file format: expected space after key ' '\"%s\", got character \"%s\" when reading archive \"%s\".' % (self.cur_key, c, self.archive_rxfilename)) if c != '\n': # Consume the space or tab. self.input.Stream().Read(1) binary = InitKaldiInputStream(self.input.Stream()) if not self.holder.Read(self.input.Stream(), binary): self.holder.Clear() LogError('Failed to read object from archive \"%s\"' % self.archive_rxfilename) self.state = RandomAccessTableReaderStateType.kHaveObject return True
def Open(self, rxfilename, binary): """Open a KaldiInputStream for the given filename and offset. Args: rxfilename: The given filename including offset. binary: The opening mode. Returns: A boolean variable indicating if the operation is successful. """ (filename, offset) = self.SplitFilename(rxfilename) if self.stream: if self.filename == filename and self.binary == binary: if not self.Seek(offset): LogError('Invalid offset = \"%d\"' % offset) return True else: self.Close() self.filename = filename self.binary = binary mode = 'rt' if binary: mode = 'rb' self.stream = KaldiInputStream(open(self.filename, mode)) if not self.Seek(offset): LogError('Invalid offset = \"%d\"' % offset) return True
def Open(self, wspecifier): """Open a writer for the given wspecifier. Args: wspecifier: The given wspecifier. Returns: A boolean variable indicating if the operation is successful. """ if self.state == TableWriterStateType.kUninitialized: pass elif self.state == TableWriterStateType.kWriteError: LogError('Already open with write error.') elif self.state == TableWriterStateType.kOpen: if not self.Close(): LogError('Failed closing previously open stream.') else: LogError('Invalid state \"%s\"' % self.state) self.wspecifier = wspecifier (wspecifier_type, archive_filename, script_filename, opts) = \ ClassifyWspecifier(wspecifier) self.archive_wxfilename = archive_filename self.opts = opts if wspecifier_type != WspecifierType.kArchiveWspecifier: LogError('Invalid wspecifier type \"%s\"' % wspecifier_type) self.output = Output() success = self.output.Open(self.archive_wxfilename, self.opts.binary, False) if not success: self.state = TableWriterStateType.kUninitialized LogError('Failed to open stream \"%s\"' % self.archive_wxfilename) self.state = TableWriterStateType.kOpen return True
def AddData(self, mat): m_rows = mat.shape[0] m_cols = mat.shape[1] if self.data is None: rows = self.conf.randomizer_size cols = m_cols self.data = numpy.empty([rows, cols], dtype=float, order='C') if self.data_begin > 0: if self.data_begin > self.data_end: # Sanity check. LogError('Sanity check failed: self.data_begin \"%d\" > ' 'self.data_end \"%d\"' % (self.data_begin, self.data_end)) left_over = self.data_end - self.data_begin if left_over >= self.data_begin: # No overlap. LogError('Unexpected data overlap: left_over \"%d\" >= ' 'self.data_begin \"%d\"' % (left_over, self.data_begin)) if left_over > 0: self.data[0:left_over, :] = \ self.data[self.data_begin:self.data_begin+left_over, :] self.data_begin = 0 self.data_end = left_over if self.data.shape[0] < self.data_end + m_rows: self.data_aux = numpy.empty_like(self.data) self.data_aux[:] = self.data rows = self.data_end + m_rows + 1000 # Add extra 1000 rows, so we # don't reallocate soon. cols = self.data.shape[1] self.data = numpy.empty([rows, cols], dtype=float, order='C') self.data[0:self.data_aux.shape[0], :] = self.data_aux self.data[self.data_end:self.data_end + m_rows, :] = mat self.data_end += m_rows
def Write(self, key, value): if self.state == TableWriterStateType.kOpen: pass elif self.state == TableWriterStateType.kWriteError: LogWarning('Attempting to write to invalid stream.') elif self.state == TableWriterStateType.kUninitialized: LogError('Invalid state \"%s\"' % self.state) else: LogError('Invalid state \"%s\"' % self.state) # state is now kOpen or kWriteError. if not IsToken(key): # e.g. empty string or has spaces... LogError('Using invalid key \"%s\"' % key) self.output.Stream().Write('%s ' % key) if not WriteHolderValueToStream(self.output.Stream(), self.type, self.opts.binary, value): LogWarning('Write failure to \"%s\"' % self.archive_wxfilename) self.state = TableWriterStateType.kWriteError return False # Even if this Write seems to have succeeded, we fail because a previous # Write failed and the archive may be corrupted and unreadable. if self.state == TableWriterStateType.kWriteError: return False if self.opts.flush: self.Flush() return True
def Randomize(self, mask): if self.data_begin != 0: LogError('Unexpected self.data_begin %d vs. 0' % self.data_begin) if self.data_end <= 0: LogError('Unexpected self.data_end %d' % self.data_begin) if self.data_end != len(mask): LogError('Mismatched self.data_end \"%d\" vs. mask size \"%d\"' % (self.data_end, len(mask))) self.data = self.data[mask]
def Value(self): if not self.EnsureObjectLoaded(): LogError('Failed to load object from \"%s\" to suppress this ' 'error, add the permissive (p, ) option to the ' 'rspecifier.' % self.data_rxfilename) if self.state == SequentialTableReaderStateType.kHaveRange: return self.range_holder.Value() elif self.state == SequentialTableReaderStateType.kHaveObject: return self.holder.Value() else: LogError('Invalid state \"%s\"' % self.state)
def WriteHolderValueToStream(stream, holder_type, binary, value): if holder_type == HolderType.kNoHolder: LogError('No holder type is specified.') elif holder_type == HolderType.kFloatMatrixHolder: WriteFloatMatrixToStream(stream, binary, value) elif holder_type == HolderType.kFloatVectorHolder: WriteFloatVectorToStream(stream, binary, value) elif holder_type == HolderType.kPosteriorHolder: WritePosteriorToStream(stream, binary, value) elif holder_type == HolderType.kInt32VectorHolder: WriteInt32VectorToStream(stream, binary, value) else: LogError('Unrecognized holder type \"%s\"' % holder_type) return True
def Read(self, stream, binary): if binary: ExpectToken(stream, binary, 'SM') num_rows = ReadInt32(stream, binary) if num_rows < 0 or num_rows > 10000000: LogError('Unexpected number of rows %s' % num_rows) self.rows = [] for r in xrange(num_rows): self.rows.append(SparseVector()) self.rows[r].Read(stream, binary) else: LogError('We do not supoort text mode Read(), as it could be ' 'quite slow using python. If you would really like to ' 'use text format, please use \'copy-feats\' in kaldi ' 'to convert it to binary.')
def NewHolderByType(holder_type): if holder_type == HolderType.kNoHolder: LogError('No holder type is specified.') elif holder_type == HolderType.kFloatMatrixHolder: return FloatMatrixHolder() elif holder_type == HolderType.kFloatVectorHolder: return FloatVectorHolder() elif holder_type == HolderType.kPosteriorHolder: return PosteriorHolder() elif holder_type == HolderType.kInt32VectorHolder: return BasicVectorHolder(BasicType.cint32) elif holder_type == HolderType.kNnetExampleHolder: return NnetExampleHolder() else: LogError('Unrecognized holder type \"%s\"' % holder_type)
def Open(self, rspecifier): """Open a reader for the given rspecifier. Args: rspecifier: The given rspecifier. Returns: A boolean variable indicating if the operation is successful. """ # You may call Open from states kUninitialized and kError. # It may leave the object in any of the states. if self.state == RandomAccessTableReaderStateType.kNoObject or \ self.state == RandomAccessTableReaderStateType.kHaveObject: # call Close() yourself to suppress this exception. if not self.Close(): LogError( 'Error closing previous input, rspecifier was \"%s\"' % self.rspecifier) self.rspecifier = rspecifier (rspecifier_type, rxfilename, opts) = ClassifyRspecifier(rspecifier) self.script_rxfilename = rxfilename self.opts = opts if rspecifier_type != RspecifierType.kScriptRspecifier: LogError('Invalid rspecifier type \"%s\"' % rspecifier_type) script_input = Input() if not script_input.Open(self.script_rxfilename): LogError('Failed opening script file \"%s\"' % self.script_rxfilename) if script_input.IsBinary(): LogError('script file should not be in binary format.') script = list() while True: line = script_input.Stream().Readline() if not line: break token = line.rstrip().split() if len(token) != 2: LogError('Invalid line \"%s\"' % line) script.append((token[0], token[1])) self.script = sorted(script, key=itemgetter(0)) self.keys = [key for key, _ in self.script] self.state = RandomAccessTableReaderStateType.kNoObject self.key = None return True
def SplitFilename(self, rxfilename): pos = rxfilename.rfind(':') if pos < 0: LogError('Invalid rxfilename \"%s\"' % rxfilename) filename = rxfilename[:pos] offset = int(rxfilename[pos + 1:]) return (filename, offset)
def ReadComponent(self, stream, binary): token = ReadToken(stream, binary) if token == '<Nnet>': # Skip the optional initial token. token = ReadToken(stream, binary) if token == "</Nnet>": # Network ends after terminal token appears. return None input_dim = ReadInt32(stream, binary) output_dim = ReadInt32(stream, binary) component = None if token == '<AffineTransform>': component = AffineTransform(input_dim, output_dim) elif token == '<Sigmoid>': component = Sigmoid(input_dim, output_dim) elif token == '<Softmax>': component = Softmax(input_dim, output_dim) else: LogError('Unrecognized or not yet supported component \"%s\"' % token) component.Read(stream, binary) if stream.Peek(2) == '<!': ExpectToken(stream, binary, '<!EndOfComponent>') return component
def GetNumpyMatrix(self): rows = self.global_header.num_rows cols = self.global_header.num_cols mat = numpy.empty([rows, cols], dtype=float, order='C') if self.global_header.format == 1: for c in xrange(cols): p0 = Uint16ToFloat(self.global_header, self.percol_header[c].percentile_0) p25 = Uint16ToFloat(self.global_header, self.percol_header[c].percentile_25) p75 = Uint16ToFloat(self.global_header, self.percol_header[c].percentile_75) p100 = Uint16ToFloat(self.global_header, self.percol_header[c].percentile_100) for r in xrange(rows): mat[r, c] = CharToFloat(p0, p25, p75, p100, self.data[r, c]) elif self.global_header.format == 2: for r in xrange(rows): for c in xrange(cols): mat[r, c] = Uint16ToFloat(self.global_header, self.data[r, c]) else: LogError('Unrecognized format = %s' % self.global_header.format) return mat
def Value(self, key): self.HandlePendingDelete() (success, value) = self.FindKeyInternal(key, True) if not success: LogError('No such key \"%s\" in archive \"%s\"' % (key, self.archive_rxfilename)) return value
def Value(self): if self.data_end - self.data_begin < self.conf.minibatch_size: LogError( 'Insufficient data for mini batch: %d vs. %d' % (self.data_end - self.data_begin, self.conf.minibatch_size)) self.minibatch = \ self.data[self.data_begin:self.data_begin+self.conf.minibatch_size] return self.minibatch
def IsOpen(self): if self.state == TableWriterStateType.kUninitialized: return False elif self.state == TableWriterStateType.kOpen or \ self.state == TableWriterStateType.kWriteError: return True else: LogError('Invalid state \"%s\"' % self.state)
def Read(self, stream, binary): if binary: self.global_header = GlobalHeader() token = ReadToken(stream, binary, False) stream.Read(1) # comsume the space if token == 'CM': self.global_header.format = 1 elif token == 'CM2': self.global_header.format = 2 else: LogError('Unexpected token \"%s\", expecting CM or CM2.' % token) data = struct.unpack('f' * 2, stream.Read(4 * 2)) self.global_header.min_value = data[0] self.global_header.range = data[1] data = struct.unpack('i' * 2, stream.Read(4 * 2)) self.global_header.num_rows = data[0] self.global_header.num_cols = data[1] rows = self.global_header.num_rows cols = self.global_header.num_cols if self.global_header.format == 1: # num_rows > 8, in CM1 format. self.percol_header = [] data = numpy.frombuffer(bytearray(stream.Read(2 * 4 * cols)), numpy.uint16).reshape(cols, 4) for c in xrange(cols): percol_header = PerColHeader() percol_header.percentile_0 = data[c, 0] percol_header.percentile_25 = data[c, 1] percol_header.percentile_75 = data[c, 2] percol_header.percentile_100 = data[c, 3] self.percol_header.append(percol_header) self.data = numpy.frombuffer( bytearray(stream.Read(rows * cols)), numpy.uint8).reshape(cols, rows).transpose() elif self.global_header.format == 2: # num_rows <= 8, in CM2 format. self.data = numpy.frombuffer( bytearray(stream.Read(2 * rows * cols)), numpy.uint16).reshape(rows, cols) else: LogError('Unrecognized format = %s' % self.global_header.format) else: LogError('We do not supoort text mode Read(), as it could be ' 'quite slow using python. If you would really like to ' 'use text format, please use \'copy-feats\' in kaldi ' 'to convert it to binary.')
def Read(self, stream, binary): """Read posteriorgrams from the given stream. Args: stream: An opened KaldiInputStream. binary: If the input stream is in binary. Returns: An boolean variable indicating if the operation is successful. """ if binary: sz = ReadInt32(stream, binary) if sz < 0 or sz > 10000000: LogError('Got negative or improbably large size \"%d\"' % sz) self.value = [] for i in xrange(sz): self.value.append([]) sz2 = ReadInt32(stream, binary) if sz2 < 0: LogError('Got negative size \"%d\"' % sz2) for j in xrange(sz2): lab = ReadInt32(stream, binary) val = ReadFloat(stream, binary) self.value[i].append((lab, val)) else: line = stream.Readline() tokens = line.rstrip().split() i = 0 self.value = [] while True: if i == len(tokens): break if tokens[i] != '[': LogError('Expecting \"[\", got \"%s\" instead.' % tokens[i]) self.value.append([]) while True: i += 1 if tokens[i] == ']': break lab = int(tokens[i]) i += 1 val = float(tokens[i]) self.value[-1].append((lab, val)) i += 1 return True
def __init__(self, rxfilename=None): self.impl = None self.binary = None # Explicitly compare with None since rxfilename == '' is still a valid # input (kStandardInput) if rxfilename is not None: if not self.Open(rxfilename): LogError('Failed opening input stream')
def __init__(self, wxfilename=None, binary=None, write_header=False): self.impl = None self.binary = None # Explicitly compare with None since wxfilename == '' is still a valid # input (kStandardInput) if wxfilename is not None and binary is not None: if not self.Open(wxfilename, binary, write_header): LogError('Failed opening output stream')
def Open(self, rspecifier): """Open a reader for the given rspecifier. Args: rspecifier: The given rspecifier. Returns: A boolean variable indicating if the operation is successful. """ if self.state != SequentialTableReaderStateType.kUninitialized: # call Close() yourself to suppress this exception. if not self.Close(): if self.opts.permissive: LogWarning('Error closing previous input (only warning, ' 'since permissive mode).') else: LogError('Error closing previous input, rspecifier was ' '\"%s\"' % self.rspecifier) self.rspecifier = rspecifier (rspecifier_type, rxfilename, opts) = ClassifyRspecifier(rspecifier) self.archive_rxfilename = rxfilename self.opts = opts if rspecifier_type != RspecifierType.kArchiveRspecifier: LogError('Invalid rspecifier type \"%s\"' % rspecifier_type) self.input = Input() if self.holder.IsReadInBinary(): success = self.input.Open(self.archive_rxfilename) else: success = self.input.OpenTextMode(self.archive_rxfilename) if not success: self.state = SequentialTableReaderStateType.kUninitialized LogError('Failed to open stream \"%s\"' % self.archive_rxfilename) self.state = SequentialTableReaderStateType.kFileStart self.Next() if self.state == SequentialTableReaderStateType.kError: self.input.Close() self.state = SequentialTableReaderStateType.kUninitialized LogError('Error beginning to read archive file \"%s\" (wrong ' 'filename?)' % self.archive_rxfilename) if self.state != SequentialTableReaderStateType.kHaveObject and \ self.state != SequentialTableReaderStateType.kEof: LogError('Invalid state \"%s\"' % self.state) return True
def IsOpen(self): if self.state == RandomAccessTableReaderStateType.kEof or \ self.state == RandomAccessTableReaderStateType.kError or \ self.state == RandomAccessTableReaderStateType.kHaveObject or \ self.state == RandomAccessTableReaderStateType.kNoObject: return True elif self.state == RandomAccessTableReaderStateType.kUninitialized: return False else: LogError('Invalid state \"%s\"' % self.state)
def Close(self): status = 0 if self.script_input.IsOpen(): status = self.script_input.Close() if self.data_input.IsOpen(): self.data_input.Close() self.range_holder.Clear() self.holder.Clear() if not self.IsOpen(): LogError('Called on input that was not open.')
def Done(self): if self.state == SequentialTableReaderStateType.kHaveObject: return False elif self.state == SequentialTableReaderStateType.kEof or \ self.state == SequentialTableReaderStateType.kError: # Error condition, like Eof, counts as Done(); the # destructor/Close() will inform the user of the error. return True else: LogError('Invalid state \"%s\"' % self.state)
def Open(self, wspecifier, holder_type): if self.IsOpen(): if not self.Close(): LogError('Failed to close previously open writer.') (wspecifier_type, archive_filename, script_filename, opts) = \ ClassifyWspecifier(wspecifier) if wspecifier_type == WspecifierType.kBothWspecifier: self.impl = TableWriterBothImpl(holder_type) elif wspecifier_type == WspecifierType.kArchiveWspecifier: self.impl = TableWriterArchiveImpl(holder_type) elif wspecifier_type == WspecifierType.kScriptWspecifier: self.impl = TableWriterScriptImpl(holder_type) else: LogError('Invalid wspecifier \"%s\"' % wspecifier) if not self.impl.Open(wspecifier): del self.impl self.impl = None return False return True
def Read(self, stream, binary): ExpectToken(stream, binary, '<NnetIo>') self.name = ReadToken(stream, binary) self.indexes = ReadIndexVector(stream, binary) c = stream.Peek(1) if binary: # TODO(cfyeh): implement double matrix. if c == 'F' or c == 'C': self.features = FloatMatrix() elif c == 'S': self.features = SparseMatrix() else: LogError('Unrecognized identifier \"%s\"' % c) else: # TODO(cfyeh): implement text mode. LogError('Text mode not implemented yet.') self.features.Read(stream, binary) ExpectToken(stream, binary, '</NnetIo>') return True