def test_binary_image(self):
#write
w = recordio.writer("/tmp/image_binary")
with open("./images/10045_right_512", "rb") as f:
con = f.read()
d1 = {'img': con, 'md5': md5.new(con).hexdigest()}
#pickle
p1 = pickle.dumps(d1, pickle.HIGHEST_PROTOCOL)
print "in python before write:", md5.new(p1).hexdigest(), len(p1)
w.write(p1)
w.close()
#read
r = recordio.reader("/tmp/image_binary")
while True:
p2 = r.read()
if not p2:
break
print "in python after read:", md5.new(p2).hexdigest(), len(p2)
d2 = pickle.loads(p2)
self.assertEqual(md5.new(d2['img']).hexdigest(), d2['md5'])
r.close()
def open_writers():
w = []
for i in range(0, num_shards):
n = "%s/%s-%05d-of-%05d" % (output_path, name_prefix, i, num_shards - 1)
w.append(recordio.writer(n))
return w
def test_binary_image(self):
#write
w = recordio.writer("d:\\tmp\\image_binary")
with open(".\\images\\10045_right_512", "rb") as f:
con = f.read()
d1 = {
'img': con,
'md5': get_md5_hexdigest(con)
}
#pickle
p1 = pickle.dumps(d1, pickle.HIGHEST_PROTOCOL)
print("in python before write:", get_md5_hexdigest(p1), len(p1))
w.write(p1)
w.close()
#read
r = recordio.reader("d:\\tmp\\image_binary")
while True:
p2 = r.read()
if not p2:
break
print("in python after read:", get_md5_hexdigest(p2), len(p2))
d2 = pickle.loads(p2)
self.assertEqual(get_md5_hexdigest(d2['img']), d2['md5'])
r.close()
def write_data(indx_f, lines):
filename = "%s/%s-%05d" % (output_path, name_prefix, indx_f)
writer = recordio.writer(filename)
for l in lines:
# FIXME(Yancey1989):
# dumps with protocol: pickle.HIGHEST_PROTOCOL
writer.write(cPickle.dumps(l))
writer.close()
def write_data(indx_f, lines):
filename = "%s/%s-%05d" % (output_path, name_prefix, indx_f)
writer = recordio.writer(filename)
for l in lines:
# FIXME(Yancey1989):
# dumps with protocol: pickle.HIGHEST_PROTOCOL
writer.write(pickle.dumps(l))
writer.close()
def test_write_read(self):
w = recordio.writer("/tmp/record_0")
w.write(pickle.dumps("1"))
w.write("2")
w.write("")
w.close()
w = recordio.writer("/tmp/record_1")
w.write("3")
w.write("4")
w.write("")
w.close()
r = recordio.reader("/tmp/record_*")
self.assertEqual(pickle.loads(r.read()), "1")
self.assertEqual(r.read(), "2")
self.assertEqual(r.read(), "")
self.assertEqual(r.read(), "3")
self.assertEqual(r.read(), "4")
self.assertEqual(r.read(), "")
self.assertEqual(r.read(), None)
self.assertEqual(r.read(), None)
r.close()
import recordio
w = recordio.writer("/tmp/test_recordio_creator.dat")
for i in range(0, 10):
print i
w.write(str(i))
w.close()