def load_from_file(filename,
parse_line,
skip=1,
encoding=None,
transport_params=None):
''' Loads a csv file.
This factory loads the provided file and returns its content as an
observable emitting one item per line.
Args:
filename: Path of the file to read
parse_line: A line parser, e.g. created with create_line_parser
skip: [Optional] Number of lines to skip before parsing
encoding [Optional] Encoding used to parse the text content
transport_params: [Optional] When smart-open is used, then this
parameter is used to provide additional configuration information
Returns:
An observable of namedtuple items, where each key is a csv column
'''
return file.read(filename,
size=64 * 1024,
encoding=encoding,
transport_params=transport_params).pipe(
line.unframe(),
load(parse_line, skip=skip),
)
def compute_features(config, file):
feature_utterance = file.pipe(
ops.flat_map(lambda file_path: read(file_path, mode='rb').pipe(
process_audio,
rs.with_latest_from(config),
ops.starmap(lambda data, config: Feature(
label=label_from_path(file_path),
data=data)),
)),
#trace_observable("utterance", trace_next_payload=False),
)
return feature_utterance,
def test_read_binary_with_size():
actual_data = []
def on_next(i):
actual_data.append(i)
with tempfile.NamedTemporaryFile() as f:
f.write(b'Hello world!')
f.flush()
data = file.read(f.name, mode='rb', size=5).subscribe(on_next=on_next)
assert len(actual_data) == 3
assert actual_data[0] == b'Hello'
assert actual_data[1] == b' worl'
assert actual_data[2] == b'd!'
def load_from_file(filename, parse_line, skip=1, encoding=None):
''' Loads a csv file.
This factory loads the provided file and returns its content as an
observable emitting one item per line.
Args:
filename: Path of the file to read or a file object
parse_line: A line parser, e.g. created with create_line_parser
skip: [Optional] Number of lines to skip before parsing
encoding [Optional] Encoding used to parse the text content
Returns:
An observable of namedtuple items, where each key is a csv column
'''
return file.read(filename, size=64 * 1024, encoding=encoding).pipe(
line.unframe(),
load(parse_line, skip=skip),
)
def test_read_text():
with tempfile.NamedTemporaryFile() as f:
f.write(b'Hello world!')
f.flush()
data = file.read(f.name).run()
assert data == 'Hello world!'
def test_read_binary():
with tempfile.NamedTemporaryFile() as f:
f.write(b'Hello world!')
f.flush()
data = file.read(f.name, mode='rb').run()
assert data == b'Hello world!'