def recordio(paths, buf_size=100):
"""
从给定的被","分开的RecordIO文件路径创建数据reader
Args:
path -- recordio文件的路径
buf_size -- buf大小设为100
Return:
dec.buffered(reader, buf_size) -- recordio文件的数据reader
"""
import recordio as rec
import paddle.v2.reader.decorator as dec
import cPickle as pickle
# 文件读取模块
def reader():
"""
定义一个reader
Args:
Return:
"""
data_file = rec.reader(paths)
while True:
data_read = data_file.read()
if data_read is None:
break
yield pickle.loads(data_read)
data_file.close()
return dec.buffered(reader, buf_size)
def recordio(paths, buf_size=100):
"""
Creates a data reader from given RecordIO file paths separated by ",",
glob pattern is supported.
:path: path of recordio files, can be a string or a string list.
:returns: data reader of recordio files.
"""
import recordio as rec
import paddle.v2.reader.decorator as dec
import cPickle as pickle
def reader():
if isinstance(paths, basestring):
path = paths
else:
path = ",".join(paths)
f = rec.reader(path)
while True:
r = f.read()
if r is None:
break
yield pickle.loads(r)
f.close()
return dec.buffered(reader, buf_size)
def recordio(paths, buf_size=100):
"""
Creates a data reader from given RecordIO file paths separated by ",",
glob pattern is supported.
:path: path of recordio files, can be a string or a string list.
:returns: data reader of recordio files.
"""
import recordio as rec
import paddle.v2.reader.decorator as dec
import cPickle as pickle
def reader():
if isinstance(paths, basestring):
path = paths
else:
path = ",".join(paths)
f = rec.reader(path)
while True:
r = f.read()
if r is None:
break
yield pickle.loads(r)
f.close()
return dec.buffered(reader, buf_size)
def recordio(paths, buf_size=100):
"""
Creates a data reader from given RecordIO file paths separated by ",",
glob pattern is supported.
:path: path of recordio files.
:returns: data reader of recordio files.
"""
import recordio as rec
import paddle.v2.reader.decorator as dec
import cPickle as pickle
def reader():
f = rec.reader(paths)
while True:
r = f.read()
if r is None:
break
yield pickle.loads(r)
f.close()
return dec.buffered(reader, buf_size)