def test_binary_image(self):
#write
w = recordio.writer("/tmp/image_binary")
with open("./images/10045_right_512", "rb") as f:
con = f.read()
d1 = {'img': con, 'md5': md5.new(con).hexdigest()}
#pickle
p1 = pickle.dumps(d1, pickle.HIGHEST_PROTOCOL)
print "in python before write:", md5.new(p1).hexdigest(), len(p1)
w.write(p1)
w.close()
#read
r = recordio.reader("/tmp/image_binary")
while True:
p2 = r.read()
if not p2:
break
print "in python after read:", md5.new(p2).hexdigest(), len(p2)
d2 = pickle.loads(p2)
self.assertEqual(md5.new(d2['img']).hexdigest(), d2['md5'])
r.close()
def test_convert():
record_num = 10
num_shards = 4
def test_reader():
def reader():
for x in xrange(record_num):
yield x
return reader
path = tempfile.mkdtemp()
#paddle.v2.dataset.common.convert(path,
convert(path,
test_reader(), num_shards, 'random_images')
files = glob.glob(path + '/random_images-*')
#self.assertEqual(len(files), num_shards)
print len(files) == num_shards
recs = []
for i in range(0, num_shards):
n = "%s/random_images-%05d-of-%05d" % (path, i, num_shards - 1)
r = recordio.reader(n)
while True:
d = r.read()
if d is None:
break
recs.append(d)
recs.sort()
#self.assertEqual(total, record_num)
print len(recs) == record_num
def test_binary_image(self):
#write
w = recordio.writer("d:\\tmp\\image_binary")
with open(".\\images\\10045_right_512", "rb") as f:
con = f.read()
d1 = {
'img': con,
'md5': get_md5_hexdigest(con)
}
#pickle
p1 = pickle.dumps(d1, pickle.HIGHEST_PROTOCOL)
print("in python before write:", get_md5_hexdigest(p1), len(p1))
w.write(p1)
w.close()
#read
r = recordio.reader("d:\\tmp\\image_binary")
while True:
p2 = r.read()
if not p2:
break
print("in python after read:", get_md5_hexdigest(p2), len(p2))
d2 = pickle.loads(p2)
self.assertEqual(get_md5_hexdigest(d2['img']), d2['md5'])
r.close()
def reader():
f = recordio.reader(path)
while True:
r = f.read()
if r is None:
break
yield r
f.close()
def reader():
f = rec.reader(paths)
while True:
r = f.read()
if r is None:
break
yield pickle.loads(r)
f.close()
def reader():
if isinstance(paths, basestring):
path = paths
else:
path = ",".join(paths)
f = rec.reader(path)
while True:
r = f.read()
if r is None:
break
yield pickle.loads(r)
f.close()
def reader():
if isinstance(paths, basestring):
path = paths
else:
path = ",".join(paths)
f = rec.reader(path)
while True:
r = f.read()
if r is None:
break
yield pickle.loads(r)
f.close()
def reader():
"""
定义一个reader
Args:
Return:
"""
data_file = rec.reader(paths)
while True:
data_read = data_file.read()
if data_read is None:
break
yield pickle.loads(data_read)
data_file.close()
def reader():
"""
定义一个reader
Args:
Return:
"""
f = rec.reader(paths)
while True:
r = f.read()
if r is None:
break
yield pickle.loads(r)
f.close()
def reader():
if isinstance(paths, six.string_types):
path = paths
elif isinstance(paths, six.binary_type):
path = paths.decode()
else:
path = ",".join(paths)
f = rec.reader(path)
while True:
r = f.read()
if r is None:
break
yield pickle.loads(r)
f.close()
def reader():
file_list = glob.glob(TRAIN_FILES_PATH)
file_list.sort()
my_file_list = []
# read files for current trainer_id
for idx, f in enumerate(file_list):
if idx % trainer_count == trainer_id:
my_file_list.append(f)
for f in my_file_list:
print "processing ", f
reader = recordio.reader(f)
record_raw = reader.read()
while record_raw:
yield pickle.loads(record_raw)
record_raw = reader.read()
reader.close()
def reader():
file_list = glob.glob(paths)
file_list.sort()
my_file_list = []
# collect a subset files according with the trainer_id
for idx, f in enumerate(file_list):
if idx % trainer_instances == trainer_id:
my_file_list.append(f)
for f in my_file_list:
print "processing", f
reader = rec.reader(f)
record_raw = reader.read()
while record_raw:
yield pickle.loads(record_raw)
record_raw = reader.read()
reader.close()
def test_write_read(self):
w = recordio.writer("/tmp/record_0")
w.write(pickle.dumps("1"))
w.write("2")
w.write("")
w.close()
w = recordio.writer("/tmp/record_1")
w.write("3")
w.write("4")
w.write("")
w.close()
r = recordio.reader("/tmp/record_*")
self.assertEqual(pickle.loads(r.read()), "1")
self.assertEqual(r.read(), "2")
self.assertEqual(r.read(), "")
self.assertEqual(r.read(), "3")
self.assertEqual(r.read(), "4")
self.assertEqual(r.read(), "")
self.assertEqual(r.read(), None)
self.assertEqual(r.read(), None)
r.close()
