def test_binary_image(self): #write w = recordio.writer("/tmp/image_binary") with open("./images/10045_right_512", "rb") as f: con = f.read() d1 = {'img': con, 'md5': md5.new(con).hexdigest()} #pickle p1 = pickle.dumps(d1, pickle.HIGHEST_PROTOCOL) print "in python before write:", md5.new(p1).hexdigest(), len(p1) w.write(p1) w.close() #read r = recordio.reader("/tmp/image_binary") while True: p2 = r.read() if not p2: break print "in python after read:", md5.new(p2).hexdigest(), len(p2) d2 = pickle.loads(p2) self.assertEqual(md5.new(d2['img']).hexdigest(), d2['md5']) r.close()
def test_convert(): record_num = 10 num_shards = 4 def test_reader(): def reader(): for x in xrange(record_num): yield x return reader path = tempfile.mkdtemp() #paddle.v2.dataset.common.convert(path, convert(path, test_reader(), num_shards, 'random_images') files = glob.glob(path + '/random_images-*') #self.assertEqual(len(files), num_shards) print len(files) == num_shards recs = [] for i in range(0, num_shards): n = "%s/random_images-%05d-of-%05d" % (path, i, num_shards - 1) r = recordio.reader(n) while True: d = r.read() if d is None: break recs.append(d) recs.sort() #self.assertEqual(total, record_num) print len(recs) == record_num
def test_binary_image(self): #write w = recordio.writer("d:\\tmp\\image_binary") with open(".\\images\\10045_right_512", "rb") as f: con = f.read() d1 = { 'img': con, 'md5': get_md5_hexdigest(con) } #pickle p1 = pickle.dumps(d1, pickle.HIGHEST_PROTOCOL) print("in python before write:", get_md5_hexdigest(p1), len(p1)) w.write(p1) w.close() #read r = recordio.reader("d:\\tmp\\image_binary") while True: p2 = r.read() if not p2: break print("in python after read:", get_md5_hexdigest(p2), len(p2)) d2 = pickle.loads(p2) self.assertEqual(get_md5_hexdigest(d2['img']), d2['md5']) r.close()
def reader(): f = recordio.reader(path) while True: r = f.read() if r is None: break yield r f.close()
def reader(): f = rec.reader(paths) while True: r = f.read() if r is None: break yield pickle.loads(r) f.close()
def reader(): if isinstance(paths, basestring): path = paths else: path = ",".join(paths) f = rec.reader(path) while True: r = f.read() if r is None: break yield pickle.loads(r) f.close()
def reader(): """ 定义一个reader Args: Return: """ data_file = rec.reader(paths) while True: data_read = data_file.read() if data_read is None: break yield pickle.loads(data_read) data_file.close()
def reader(): """ 定义一个reader Args: Return: """ f = rec.reader(paths) while True: r = f.read() if r is None: break yield pickle.loads(r) f.close()
def reader(): if isinstance(paths, six.string_types): path = paths elif isinstance(paths, six.binary_type): path = paths.decode() else: path = ",".join(paths) f = rec.reader(path) while True: r = f.read() if r is None: break yield pickle.loads(r) f.close()
def reader(): file_list = glob.glob(TRAIN_FILES_PATH) file_list.sort() my_file_list = [] # read files for current trainer_id for idx, f in enumerate(file_list): if idx % trainer_count == trainer_id: my_file_list.append(f) for f in my_file_list: print "processing ", f reader = recordio.reader(f) record_raw = reader.read() while record_raw: yield pickle.loads(record_raw) record_raw = reader.read() reader.close()
def reader(): file_list = glob.glob(paths) file_list.sort() my_file_list = [] # collect a subset files according with the trainer_id for idx, f in enumerate(file_list): if idx % trainer_instances == trainer_id: my_file_list.append(f) for f in my_file_list: print "processing", f reader = rec.reader(f) record_raw = reader.read() while record_raw: yield pickle.loads(record_raw) record_raw = reader.read() reader.close()
def test_write_read(self): w = recordio.writer("/tmp/record_0") w.write(pickle.dumps("1")) w.write("2") w.write("") w.close() w = recordio.writer("/tmp/record_1") w.write("3") w.write("4") w.write("") w.close() r = recordio.reader("/tmp/record_*") self.assertEqual(pickle.loads(r.read()), "1") self.assertEqual(r.read(), "2") self.assertEqual(r.read(), "") self.assertEqual(r.read(), "3") self.assertEqual(r.read(), "4") self.assertEqual(r.read(), "") self.assertEqual(r.read(), None) self.assertEqual(r.read(), None) r.close()