def _convert_dataset(split_name, filenames, class_names_to_ids, dataset_dir): assert split_name in ['train', 'validation'] num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS))) with tf.Graph().as_default(): image_reader = ImageReader() with tf.Session('') as sess: for shard_id in range(_NUM_SHARDS): output_filename = _get_dataset_filename( split_name, shard_id) with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer: start_ndx = shard_id * num_per_shard end_ndx = min((shard_id+1) * num_per_shard, len(filenames)) for i in range(start_ndx, end_ndx): sys.stdout.write('\r>> Converting image %d/%d shard %d' % ( i+1, len(filenames), shard_id)) sys.stdout.flush() # Read the filename: image_data = tf.gfile.FastGFile(filenames[i], 'rb').read() height, width = image_reader.read_image_dims(sess, image_data) class_name = os.path.basename(os.path.dirname(filenames[i])) class_id = class_names_to_ids[class_name] example = dataset_utils.image_to_tfexample( image_data, b'jpg', height, width, class_id) tfrecord_writer.write(example.SerializeToString()) sys.stdout.write('\n') sys.stdout.flush()
def _create_tfrecord_train(dataset_dir, tfrecord_writer, classes_train): count = 0 all_image_paths, class_labels = [], [] for cls_train in classes_train: images_dir = glob.glob(join(dataset_dir, 'train', cls_train, '*.JPG')) +\ glob.glob(join(dataset_dir, 'train', cls_train, '*.jpg')) all_image_paths.extend(images_dir) class_labels.extend( [classes_train[cls_train] for _ in range(len(images_dir))]) combined = list(zip(all_image_paths, class_labels)) shuffle(combined) all_image_paths, class_labels = zip(*combined) for i, im_path in enumerate(all_image_paths): with tf.gfile.Open(im_path, 'rb') as f: img = cv2.imread(im_path) img_shape = img.shape if img_shape != (256, 256, 3): print('Resizing!!!!!!!!!!!!!!') img = cv2.resize(img, (256, 256)) _, encoded_image = cv2.imencode('.jpg', img) label = class_labels[i] encoded_image = encoded_image.tobytes() example = dataset_utils.image_to_tfexample(encoded_image, b'jpg', 256, 256, label) output_shard_index = count % _NUM_TRAIN_FILES tfrecord_writer[output_shard_index].write( example.SerializeToString()) count += 1 print('Processed {} images'.format(count))
def _add_to_tfrecord(data_filename, labels_filename, num_images, tfrecord_writer): """Loads data from the binary MNIST files and writes files to a TFRecord. Args: data_filename: The filename of the MNIST images. labels_filename: The filename of the MNIST labels. num_images: The number of images in the dataset. tfrecord_writer: The TFRecord writer to use for writing. """ images = _extract_images(data_filename, num_images) labels = _extract_labels(labels_filename, num_images) shape = (_IMAGE_SIZE, _IMAGE_SIZE, _NUM_CHANNELS) with tf.Graph().as_default(): image = tf.placeholder(dtype=tf.uint8, shape=shape) encoded_png = tf.image.encode_png(image) with tf.Session('') as sess: for j in range(num_images): sys.stdout.write('\r>> Converting image %d/%d' % (j + 1, num_images)) sys.stdout.flush() png_string = sess.run(encoded_png, feed_dict={image: images[j]}) example = dataset_utils.image_to_tfexample( png_string, 'png'.encode(), _IMAGE_SIZE, _IMAGE_SIZE, labels[j]) tfrecord_writer.write(example.SerializeToString())
def tf_writer(phase): print 'phase: {}'.format(phase) pairs_npy_path = os.path.join(cfg.pairs_dir, 'leaders_{}_pairs.npy'.format(phase)) pairs_npy = np.load(pairs_npy_path) nofpairs = len(pairs_npy) print 'will write {} pairs'.format(nofpairs) tfrecords_name = os.path.join(cfg.tfrecord_dir, phase, '{}_0.tfrecords'.format(phase)) print 'creating tfrecords writer: {}'.format(tfrecords_name) writer = tf.python_io.TFRecordWriter(tfrecords_name) for ind in range(nofpairs): pair = pairs_npy[ind] if ind > 0 and ind % 1000 == 0: writer.close() tfrecords_name = os.path.join(cfg.tfrecord_dir, phase,'{}_{}.tfrecords'.format(phase, ind)) print 'creating tfrecords writer: {}'.format(tfrecords_name) writer = tf.python_io.TFRecordWriter(tfrecords_name) fn = pair[0] label = pair[1] try: im = Image.open(fn) except IOError: print 'cannot load file: {}'.format(fn) continue im = np.array(im) im = im.astype(np.uint8) # read file name image_data = tf.gfile.FastGFile(fn, 'rb').read() height , width = im.shape[0], im.shape[1] example = dataset_utils.image_to_tfexample(image_data, b'jpg', height, width, int(label)) writer.write(example.SerializeToString())
def tf_writer(phase): print 'phase: {}'.format(phase) pairs_npy_path = os.path.join(cfg.pairs_dir, 'leaders_{}_pairs.npy'.format(phase)) pairs_npy = np.load(pairs_npy_path) nofpairs = len(pairs_npy) print 'will write {} pairs'.format(nofpairs) tfrecords_name = os.path.join(cfg.tfrecord_dir, phase, '{}_0.tfrecords'.format(phase)) print 'creating tfrecords writer: {}'.format(tfrecords_name) writer = tf.python_io.TFRecordWriter(tfrecords_name) for ind in range(nofpairs): pair = pairs_npy[ind] if ind > 0 and ind % 1000 == 0: writer.close() tfrecords_name = os.path.join(cfg.tfrecord_dir, phase, '{}_{}.tfrecords'.format(phase, ind)) print 'creating tfrecords writer: {}'.format(tfrecords_name) writer = tf.python_io.TFRecordWriter(tfrecords_name) fn = pair[0] label = pair[1] try: im = Image.open(fn) except IOError: print 'cannot load file: {}'.format(fn) continue im = np.array(im) im = im.astype(np.uint8) # read file name image_data = tf.gfile.FastGFile(fn, 'rb').read() height, width = im.shape[0], im.shape[1] example = dataset_utils.image_to_tfexample(image_data, b'jpg', height, width, int(label)) writer.write(example.SerializeToString())
def _convert_dataset(split_name, filenames, dataset_dir, class_id): record_file_num = math.ceil(len(filenames) / _NUM_PER_RECORD) f = open('./log.txt', 'a') with tf.Graph().as_default(): image_reader = ImageReader() with tf.Session('') as sess: for record_id in range(record_file_num): output_filename = _get_dataset_filename( dataset_dir, split_name, class_id, record_id) with tf.python_io.TFRecordWriter( output_filename) as tfrecord_writer: for i in range(_NUM_PER_RECORD): if record_id * _NUM_PER_RECORD + i >= len(filenames): break # Read the filename: image_data = tf.gfile.FastGFile(filenames[i], 'rb').read() height, width = image_reader.read_image_dims( sess, image_data) example = dataset_utils.image_to_tfexample( image_data, b'jpg', height, width, class_id) tfrecord_writer.write(example.SerializeToString()) print('write ' + filenames[i] + ' to ' + output_filename) f.write('write ' + filenames[i] + ' to ' + output_filename + '\n') f.close()
def create_tf(split_name, filenames, class_names_to_ids, dataset_dir, num_shards, shard_id, num_per_shard, image_reader, sess): output_filename = _get_dataset_filename( OUTPUT_PATH, split_name, shard_id, num_shards) with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer: start_ndx = shard_id * num_per_shard end_ndx = min((shard_id + 1) * num_per_shard, len(filenames)) for i in range(start_ndx, end_ndx): # sys.stdout.write('\r>> Converting image %d/%d shard %d' % ( # i + 1, len(filenames), shard_id)) # sys.stdout.flush() # Read the filename: image_data = tf.gfile.FastGFile(filenames[i], 'rb').read() height, width = image_reader.read_image_dims(sess, image_data) class_name = os.path.basename(os.path.dirname(filenames[i])) class_id = class_names_to_ids[class_name] example = dataset_utils.image_to_tfexample( image_data, b'jpg', height, width, class_id) tfrecord_writer.write(example.SerializeToString()) g_process.add_finish(output_filename) sys.stdout.write('\r>> Converting image %d/%d of files: %d' % ( len(g_process.finished_file), g_process.all_step, len(filenames))) sys.stdout.flush()
def _convert_dataset(dataset_name, split_name, filenames, class_names_to_ids, dataset_dir, num_shards, num_channels=3): """Converts the given filenames to a TFRecord dataset. Args: split_name: The name of the dataset, either 'train' or 'validation'. filenames: A list of absolute paths to png or jpg images. class_names_to_ids: A dictionary from class names (strings) to ids (integers). dataset_dir: The directory where the converted datasets are stored. """ assert split_name in ['train', 'validation', 'test'] num_per_shard = int(math.ceil(len(filenames) / float(num_shards))) with tf.Graph().as_default(): image_reader = ImageReader(num_channels) with tf.Session('') as sess: for shard_id in range(num_shards): output_filename = _get_dataset_filename(dataset_name, dataset_dir, split_name, shard_id, num_shards) with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer: start_ndx = shard_id * num_per_shard end_ndx = min((shard_id + 1) * num_per_shard, len(filenames)) for i in range(start_ndx, end_ndx): sys.stdout.write('\r>> Converting image %d/%d shard %d, %s' % ( i + 1, len(filenames), shard_id, filenames[i])) sys.stdout.flush() # Read the filename: image_data = tf.gfile.FastGFile(filenames[i], 'rb').read() height, width = image_reader.read_image_dims(sess, image_data) class_name = os.path.basename(os.path.dirname(filenames[i])) class_id = class_names_to_ids[class_name] if sys.version_info[0] == 3: example = dataset_utils.image_to_tfexample( image_data, 'jpg'.encode(), height, width, class_id) else: example = dataset_utils.image_to_tfexample( image_data, 'jpg', height, width, class_id) tfrecord_writer.write(example.SerializeToString()) sys.stdout.write('\n') sys.stdout.flush()
def _build_shards(filenames, shard_name): """Loads data from the raw CIFAR10 files and writes to TFRecord files. Parameters ---------- filenames : list of str The name of the CIFAR10 pickle files. shard_name : str A string template for the shard name. """ images = [] labels = [] for filename in filenames: with open(filename, 'rb') as f: if six.PY3: data = cPickle.load(f, encoding='latin1') else: data = cPickle.load(f) images.append(data['data'].reshape((-1, 3, 32, 32))) labels.append(data['labels']) images = np.concatenate(images, 0) labels = np.concatenate(labels, 0) # num_images = images.shape[0] # num_shards = -(-num_images // _SHARD_SIZE) # ceiling function # sort images by class such that every shard only contains one class for c in range(len(_CLASS_NAMES)): c_labels = labels[labels == c] c_images = images[labels == c] c_num_images = c_images.shape[0] c_num_shards = -(-c_num_images // _SHARD_SIZE) # ceiling function with tf.Graph().as_default(), tf.Session('') as session: image_placeholder = tf.placeholder(dtype=tf.uint8) encoded_image = tf.image.encode_png(image_placeholder) for s in range(c_num_shards): shard = shard_name.format(c, s, c_num_shards) with tf.python_io.TFRecordWriter(shard) as tfrecord_writer: i_min, i_max = s * _SHARD_SIZE, (s + 1) * _SHARD_SIZE for i in range(i_min, min(c_num_images, i_max)): sys.stdout.write( '\r>> Building class {}/{}, shard {}/{}, image {}/{}' .format(c, len(_CLASS_NAMES), s, c_num_shards, i, c_num_images)) sys.stdout.flush() image = np.squeeze(c_images[i]).transpose((1, 2, 0)) label = c_labels[i] png_string = session.run( encoded_image, feed_dict={image_placeholder: image}) example = dataset_utils.image_to_tfexample( png_string, b'png', _IMAGE_SIZE, _IMAGE_SIZE, label) tfrecord_writer.write(example.SerializeToString())
def _add_to_tfrecord(filename, tfrecord_writer, offset=0): """Loads data from the cifar10 pickle files and writes files to a TFRecord. Args: filename: The filename of the cifar10 pickle file. tfrecord_writer: The TFRecord writer to use for writing. offset: An offset into the absolute number of images previously written. Returns: The new offset. """ # file_list = glob.glob(filename + '/*.jpg') file_list = glob.glob(filename + '/*.JPG') images = [] labels = [] count = 0 for file_name in file_list: count += 1 image = misc.imread(file_name) height, width, _ = image.shape # if height<200 or width<200: # os.remove(file_name) # continue # image_resize = misc.imresize(image, (299,299,3)) input_img = np.array(image, dtype='uint8') # label_name = int(os.path.basename(file_name).split('_')[0]) - 1 #start at 0 label_name = int(os.path.basename(file_name).split('.')[0]) images.append(input_img) labels.append(label_name) print(file_name) # if count>100: # break num_images = len(images) with tf.Graph().as_default(): image_placeholder = tf.placeholder(dtype=tf.uint8) encoded_image = tf.image.encode_png(image_placeholder) with tf.Session('') as sess: for j in range(num_images): sys.stdout.write('\r>> Reading file [%s] image %d/%d' % ( filename, offset + j + 1, offset + num_images)) sys.stdout.flush() image = np.squeeze(images[j]) label = labels[j] png_string = sess.run(encoded_image, feed_dict={image_placeholder: image}) example = dataset_utils.image_to_tfexample( png_string, b'png', height, width, label) tfrecord_writer.write(example.SerializeToString()) return offset + num_images
def _convert_dataset(split_name, filenames, class_names_to_ids, dataset_dir): """Converts the given filenames to a TFRecord dataset. Args: split_name: The name of the dataset, either 'train' or 'validation'. filenames: A list of absolute paths to png or jpg images. class_names_to_ids: A dictionary from class names (strings) to ids (integers). dataset_dir: The directory where the converted datasets are stored. """ assert split_name in ['train', 'validation'] num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS))) with tf.Graph().as_default(): image_reader = ImageReader() with tf.Session('') as sess: for shard_id in range(_NUM_SHARDS): output_filename = _get_dataset_filename( dataset_dir, split_name, shard_id) with tf.python_io.TFRecordWriter( output_filename) as tfrecord_writer: start_ndx = shard_id * num_per_shard end_ndx = min((shard_id + 1) * num_per_shard, len(filenames)) for i in range(start_ndx, end_ndx): sys.stdout.write( '\r>> Converting image %d/%d shard %d' % (i + 1, len(filenames), shard_id)) sys.stdout.flush() img = Image.open( filenames[i]) ########################### if img.mode != 'RGB': #test by penglong print(filenames[i] + ', ' + img.mode) ############################ img.close() os.remove(filenames[i]) continue # Read the filename: image_data = tf.gfile.FastGFile(filenames[i], 'rb').read() height, width = image_reader.read_image_dims( sess, image_data) class_name = os.path.basename( os.path.dirname(filenames[i])) class_id = class_names_to_ids[class_name] example = dataset_utils.image_to_tfexample( image_data, b'jpg', height, width, class_id) tfrecord_writer.write(example.SerializeToString()) sys.stdout.write('\n') sys.stdout.flush()
def _convert_dataset(split_name, filenames, filename_to_class_ids, dataset_dir): """ Convert the given list of files to a TFRecord dataset. Args: split_name: The name of the dataset split, either 'train' or 'test'. filenames: A list of paths to image files filename_to_class_ids: A dictionary from file name to its class id. dataset_dir: The path to the dataset directory. """ assert split_name in ['train', 'test'] num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS))) # shuffle file names np.random.shuffle(filenames) with tf.Graph().as_default(): image_reader = ImageReader() # just run on CPU config = tf.ConfigProto(device_count={'GPU': 0}) with tf.Session(config=config) as sess: for shard_id in range(_NUM_SHARDS): output_filename = _get_dataset_filename( dataset_dir, split_name, shard_id) with tf.python_io.TFRecordWriter( output_filename) as tfrecord_writer: start_idx = shard_id * num_per_shard end_idx = min((shard_id + 1) * num_per_shard, len(filenames)) for i in range(start_idx, end_idx): # write in a progress style sys.stdout.write( '\r>> Converting image %d/%d shard %d' % (i + 1, len(filenames), shard_id)) sys.stdout.flush() # read file image_data = tf.gfile.FastGFile(filenames[i], 'rb').read() height, width = image_reader.read_image_dims( sess, image_data) basename = os.path.basename(filenames[i]) class_id = filename_to_class_ids[basename] - 1 example = dataset_utils.image_to_tfexample( image_data, b'jpg', height, width, class_id) tfrecord_writer.write(example.SerializeToString()) sys.stdout.write('\n') sys.stdout.flush()
def get_data_images(ROOT, set_id): save_path = os.path.join(DATA_DIR, set_id) if os.path.isdir(save_path): pass #sys.exit('{} is already exist'.format(save_path)) else: os.makedirs(save_path) log_f = open(os.path.join(DATA_DIR, set_id+'_info'), 'a') pids = glob.glob(os.path.join(ROOT, '*')) pids = sorted(pids, key=lambda x: int(x.split('/')[-1])) #pids = pids[131:] for i, pname in enumerate(pids): tfrecord_filename = os.path.join(save_path, PRE+str(i)+'.tfrecord') tfrecord_writer = tf.python_io.TFRecordWriter(tfrecord_filename) # read images print(pname) filenames = glob.glob(os.path.join(pname, '*')) #filenames = sorted(filenames, key=lambda x: int(x.split('/')[-1].split('_')[0])) for j, imgname in enumerate(filenames): #print(imgname) if imgname.endswith('.png'): img = cv2.imread(imgname) img = img[...,0] oriimg = img[:,0:256] label = img[:,256*1:256*2] label_wall = (label==255*np.ones(label.shape)) label_endo = (label==127*np.ones(label.shape)) pred = img[:,256*2:256*3] pred_wall = (pred==255*np.ones(pred.shape)) pred_endo = (pred==127*np.ones(pred.shape)) masks_wall = [label_wall, pred_wall] masks_endo = [label_endo, pred_endo] masks = [label, pred] for ind in range(2): #Input = np.stack([oriimg*masks_wall[ind], oriimg*masks_endo[ind]], -1) #Input = np.stack([oriimg, masks[ind]], -1) Input = np.stack([oriimg, masks_wall[ind]*255, masks_endo[ind]*255], -1) Input = Input.astype(np.int8) label = ind #label = random.randint(0,1) data_point = Input.tostring() example = dataset_utils.image_to_tfexample(data_point, label, subject_id=i, index=j) tfrecord_writer.write(example.SerializeToString()) print('Finish writing data from {}'.format(i)) log_f.write('{}: {}\n'.format(i, len(imgname)))
def _add_to_tfrecord(data_filename, tfrecord_writer, label_filename=None, subset=None, test_fold=False): """Loads data from the stl10 files and writes files to a TFRecord. Args: data_filename: The filename of the stl10 file. tfrecord_writer: The TFRecord writer to use for writing. Returns: The new offset. """ images = read_all_images(data_filename) labels = None if label_filename: labels = read_labels(label_filename) if subset: if test_fold: images = images[subset] labels = labels[subset] else: images = np.delete(images, subset, axis=0) labels = np.delete(labels, subset, axis=0) num_images = images.shape[0] with tf.Graph().as_default(): coder = dataset_utils.ImageCoder() with tf.Session(''): for j in range(num_images): sys.stdout.write('\r>> Reading file [%s] image %d/%d' % (data_filename, j + 1, num_images)) sys.stdout.flush() # Get image, edge-map and cartooned image image = np.squeeze(images[j]) if label_filename: label = labels[j] - 1 # labels should be 0 indexed! else: label = -1 # Encode the images image_str = coder.encode_jpeg(image) # Build example example = dataset_utils.image_to_tfexample( image_str, 'jpg', HEIGHT, WIDTH, int(label)) tfrecord_writer.write(example.SerializeToString()) return num_images
def _convert_dataset(split_name, filenames, tps, Qs, classnames, classids, output_dir, NUM_SHARDS): """Converts the given filenames to a TFRecord dataset. Args: split_name: The name of the dataset, either 'train' or 'validation'. filenames: A list of absolute paths to png or jpg images. class_names_to_ids: A dictionary from class names (strings) to ids (integers). dataset_dir: The directory where the converted datasets are stored. """ assert split_name in ['train', 'validation'] num_per_shard = int(math.ceil(len(filenames) / float(NUM_SHARDS))) with tf.Graph().as_default(): image_reader = ImageReader() with tf.Session('') as sess: for shard_id in range(NUM_SHARDS): output_filename = _get_dataset_filename( output_dir, split_name, shard_id, NUM_SHARDS) print(output_filename) with tf.python_io.TFRecordWriter( output_filename) as tfrecord_writer: start_ndx = shard_id * num_per_shard end_ndx = min((shard_id + 1) * num_per_shard, len(filenames)) for i in range(start_ndx, end_ndx): sys.stdout.write( '\r>> Converting image %d/%d shard %d' % (i + 1, len(filenames), shard_id)) sys.stdout.flush() # Read the filename: print("reading file" + filenames[i]) image_data = tf.gfile.FastGFile(filenames[i], 'rb').read() height, width = image_reader.read_image_dims( sess, image_data) class_id = classids[i] tp = tps[i] filename = filenames[i] Q = Qs[i] example = dataset_utils.image_to_tfexample( image_data, b'jpg', height, width, int(class_id), int(tp), filename, Q) tfrecord_writer.write(example.SerializeToString()) sys.stdout.write('\n') sys.stdout.flush()
def _add_to_tfrecord(filename, tfrecord_writer, offset=0): """Loads data from the cifar10 pickle files and writes files to a TFRecord. Args: filename: The filename of the cifar10 pickle file. tfrecord_writer: The TFRecord writer to use for writing. offset: An offset into the absolute number of images previously written. Returns: The new offset. """ with tf.gfile.Open(filename, 'rb') as f: if sys.version_info < (3, ): data = cPickle.load(f) else: data = cPickle.load(f, encoding='bytes') images = data[b'data'] num_images = images.shape[0] images = images.reshape((num_images, 3, 32, 32)) labels = data[b'labels'] with tf.Graph().as_default(): image_placeholder = tf.placeholder(dtype=tf.uint8) encoded_image = tf.image.encode_png(image_placeholder) with tf.Session('') as sess: for j in range(num_images): sys.stdout.write( '\r>> Reading file [%s] image %d/%d' % (filename, offset + j + 1, offset + num_images)) sys.stdout.flush() image = np.squeeze(images[j]).transpose((1, 2, 0)) label = labels[j] png_string = sess.run(encoded_image, feed_dict={image_placeholder: image}) example = dataset_utils.image_to_tfexample( png_string, b'png', _IMAGE_SIZE, _IMAGE_SIZE, label) tfrecord_writer.write(example.SerializeToString()) return offset + num_images
def _convert_dataset(split_name, filenames, features, dataset_dir): """Converts the given filenames to a TFRecord dataset. Args: split_name: The name of the dataset, either 'train' or 'validation'. filenames: A list of absolute paths to png or jpg images. features: A lisr of absolute paths to feature binary file dataset_dir: The directory where the converted datasets are stored. """ assert split_name in ['train', 'validation'] num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS))) with tf.Graph().as_default(): image_reader = ImageReader() with tf.Session('') as sess: for shard_id in range(_NUM_SHARDS): output_filename = _get_dataset_filename( dataset_dir, split_name, shard_id) with tf.python_io.TFRecordWriter( output_filename) as tfrecord_writer: start_ndx = shard_id * num_per_shard end_ndx = min((shard_id + 1) * num_per_shard, len(filenames)) for i in range(start_ndx, end_ndx): sys.stdout.write( '\r>> Converting image %d/%d shard %d' % (i + 1, len(filenames), shard_id)) sys.stdout.flush() # Read the filename: image_data = load_image(filenames[i]) #read the feature binary files feature_data = np.fromfile(features[i], dtype=np.float32) example = dataset_utils.image_to_tfexample( image_data, feature_data) tfrecord_writer.write(example.SerializeToString()) sys.stdout.write('\n') sys.stdout.flush()
def _convert_dataset(split_name, filenames, class_names_to_ids, dataset_dir): """Converts the given filenames to a TFRecord dataset. Args: split_name: The name of the dataset, either 'train' or 'validation'. filenames: A list of absolute paths to png or jpg images. class_names_to_ids: A dictionary from class names (strings) to ids (integers). dataset_dir: The directory where the converted datasets are stored. """ assert split_name in ['train', 'validation'] num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS))) with tf.Graph().as_default(): image_reader = ImageReader() with tf.Session('') as sess: for shard_id in range(_NUM_SHARDS): output_filename = _get_dataset_filename( dataset_dir, split_name, shard_id) with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer: start_ndx = shard_id * num_per_shard end_ndx = min((shard_id+1) * num_per_shard, len(filenames)) for i in range(start_ndx, end_ndx): sys.stdout.write('\r>> Converting image %d/%d shard %d' % ( i+1, len(filenames), shard_id)) sys.stdout.flush() # Read the filename: image_data = tf.gfile.FastGFile(filenames[i], 'rb').read() height, width = image_reader.read_image_dims(sess, image_data) class_name = os.path.basename(os.path.dirname(filenames[i])) class_id = class_names_to_ids[class_name] example = dataset_utils.image_to_tfexample( image_data, b'jpg', height, width, class_id) tfrecord_writer.write(example.SerializeToString()) sys.stdout.write('\n') sys.stdout.flush()
def _convert_dataset(split_name, filenames, labels, bbox_dict, dataset_dir): """Converts the given filenames to a TFRecord dataset. Args: split_name: The name of the dataset, either 'train' or 'validation'. filenames: A list of absolute paths to png or jpg images. class_names_to_ids: A dictionary from class names (strings) to ids (integers). dataset_dir: The directory where the converted datasets are stored. """ assert split_name in ['train', 'validation', 'test'] with tf.Graph().as_default(): image_reader = ImageReader() with tf.Session('') as sess: output_filename = _get_dataset_filename(dataset_dir, split_name) with tf.python_io.TFRecordWriter( output_filename) as tfrecord_writer: for i in range(len(filenames)): sys.stdout.write('\r>> Converting image %d/%d' % (i + 1, len(filenames))) sys.stdout.flush() # Read the filename: image_data = tf.gfile.FastGFile(filenames[i], 'r').read() height, width = image_reader.read_image_dims( sess, image_data) x_min = bbox_dict[filenames[i]][0] / width y_min = bbox_dict[filenames[i]][1] / height x_max = bbox_dict[filenames[i]][2] / width y_max = bbox_dict[filenames[i]][3] / height example = dataset_utils.image_to_tfexample( image_data, 'jpeg', height, width, labels[i], [[x_min], [y_min], [x_max], [y_max]]) tfrecord_writer.write(example.SerializeToString()) sys.stdout.write('\n') sys.stdout.flush()
def _create_tfrecord_test(dataset_dir, tfrecord_writer, classes_train): count = 0 image_paths = sorted(glob.glob(join(dataset_dir, 'test_imgs', '*.JPG'))) +\ glob.glob(join(dataset_dir, 'test_imgs', '*.jpg')) num_images = len(image_paths) for i, im_path in enumerate(image_paths): with tf.gfile.Open(im_path, 'rb') as f: img = cv2.imread(im_path) img_shape = img.shape if img_shape != (256, 256, 3): print('Resizing!!!!!!!!!!!!!!!!') img = cv2.resize(img, (256, 256)) _, encoded_image = cv2.imencode('.jpg', img) encoded_image = encoded_image.tobytes() example = dataset_utils.image_to_tfexample(encoded_image, b'jpg', 256, 256, 0) output_shard_index = count % _NUM_TRAIN_FILES tfrecord_writer[output_shard_index].write( example.SerializeToString()) count += 1 print('Processed {} images'.format(count))
def _convert_dataset(specs_name, filenames, class_ids): with tf.Graph().as_default(): image_reader = ImageReader() config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: output_filename = specs_name + '.tfrecord' if os.path.exists(output_filename): os.remove(output_filename) with tf.python_io.TFRecordWriter( output_filename) as tfrecord_writer: for i in range(len(filenames)): # Read the filename: print str(i) + ' ' + filenames[i] img_path = filenames[i] image_data = tf.gfile.FastGFile(img_path, 'rb').read() height, width = image_reader.read_image_dims( sess, image_data) class_id = int(class_ids[i]) example = dataset_utils.image_to_tfexample( image_data, b'jpg', height, width, class_id) tfrecord_writer.write(example.SerializeToString())
def _convert_dataset(split_name, filenames, dataset_dir): assert split_name in ['train', 'test'] num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS))) with tf.Graph().as_default(): image_reader = ImageReader() with tf.Session('') as sess: for shard_id in range(_NUM_SHARDS): output_filename = _get_dataset_filename( dataset_dir, split_name, shard_id) with tf.python_io.TFRecordWriter( output_filename) as tfrecord_writer: start_ndx = shard_id * num_per_shard end_ndx = min((shard_id + 1) * num_per_shard, len(filenames)) for i in range(start_ndx, end_ndx): sys.stdout.write( '\r>> Converting image %d/%d shard %d' % (i + 1, len(filenames), shard_id)) sys.stdout.flush() for i in range(2): image_data = tf.gfile.FastGFile( dataset_dir + '/data/' + filenames[i] + '_' + str(i + 1) + '.png', 'r').read() height, width = image_reader.read_image_dims( sess, image_data) example = dataset_utils.image_to_tfexample( image_data, 'png', height, width, -1) tfrecord_writer.write(example.SerializeToString()) sys.stdout.write('\n') sys.stdout.flush()
def get_data_pickle(raw_data_path, set_id, seq_length, stride_frame, stride_seq, subject_index=[]): """ Convert data in pickle to tfrecord Args: raw_data_path: location of the raw data set_id: unique name for the created tfrecord dataset seq_length: the length of frames used for one input stride_frame: the step size of frames stride_seq: the step size of sequences subject_index: the subject id """ save_path = os.path.join(DATA_DIR, set_id+'_{}_{}_{}'.format(seq_length, stride_frame, stride_seq)) if os.path.isdir(save_path): pass #sys.exit('{} is already exist'.format(save_path)) else: os.makedirs(save_path) log_f = open(os.path.join(DATA_DIR, set_id+'_{}_{}_{}'.format(seq_length, stride_frame, stride_seq)+'_info'), 'a') dataset_stats = {} for each_class in CLASS: dataset_stats.setdefault(each_class,0) for pf in subject_index: tfrecord_filename = os.path.join(save_path, PRE+pf+'.tfrecord') tfrecord_writer = tf.python_io.TFRecordWriter(tfrecord_filename) subject_stats = {} for each_class in CLASS: subject_stats.setdefault(each_class,0) total = 0 files = glob.glob(os.path.join(raw_data_path, '*'+pf+'*')) for each_file in files: fname = each_file.split('/')[-1] with open(each_file, 'rb') as rf: [imgs, annots, eye, head, mouth]= pickle.load(rf) for index in range (0,len(imgs)-seq_length*stride_frame,stride_seq): # IF 2D input if seq_length == 1: data_point = imgs[index,:,:,:] label = int(annots[index]) subject_stats[label] += 1 dataset_stats[label] += 1 total += 1 # IF 3D input else: data_point = imgs[index:index+seq_length*stride_frame:stride_frame,:,:] # For drowsiness annots = annots.astype(np.int32) label_array = annots[index:index+seq_length*stride_frame:stride_frame] #if int(label[-1])==1: if sum(label_array)==seq_length: label = 1 #elif int(label[-1])==0: elif sum(label_array)==0: label = 0 else: continue subject_stats[label] += 1 dataset_stats[label] += 1 total += 1 data_point = data_point.tostring() example = dataset_utils.image_to_tfexample(data_point, label) tfrecord_writer.write(example.SerializeToString()) rf.close() print('Finish extracting data from %s'%(each_file)) print('Finish writing data from {}'.format(pf)) log_f.write('{}: {}\t {}\n'.format(pf, total, json.dumps(subject_stats))) log_f.write('{}'.format(json.dumps(dataset_stats)))
def _convert_dataset(split_name, filenames, dataset_dir): """Converts the given filenames to a TFRecord dataset. Args: split_name: The name of the dataset, either 'train' or 'validation'. filenames: A list of absolute paths to png or jpg images. class_names_to_ids: A dictionary from class names (strings) to ids (integers). dataset_dir: The directory where the converted datasets are stored. """ assert split_name in ['train', 'validation'] img_files_a = [] img_files_b = [] class_names = [] for filename in filenames: img_files_a.append( os.path.join(dataset_dir, parts[0], filename.strip().split()[0])) img_files_b.append( os.path.join(dataset_dir, parts[1], filename.strip().split()[0])) class_names.append(filename.strip().split()[1]) num_per_shard = int(math.ceil(len(filenames) / float(_NUM_SHARDS))) with tf.Graph().as_default(): image_reader = ImageReader() with tf.Session('') as sess: for shard_id in range(_NUM_SHARDS): output_filename = _get_dataset_filename( dataset_dir, split_name, shard_id) with tf.python_io.TFRecordWriter( output_filename) as tfrecord_writer: start_ndx = shard_id * num_per_shard end_ndx = min((shard_id + 1) * num_per_shard, len(filenames)) for i in range(start_ndx, end_ndx): sys.stdout.write( '\r>> Converting image %d/%d shard %d' % (i + 1, len(filenames), shard_id)) sys.stdout.flush() # Read the filename: image_a = tf.gfile.FastGFile(img_files_a[i], 'rb').read() image_b = tf.gfile.FastGFile(img_files_b[i], 'rb').read() class_id = int(class_names[i]) example = dataset_utils.image_to_tfexample( image_a, image_b, b'jpg', class_id) tfrecord_writer.write(example.SerializeToString()) sys.stdout.write('\n') sys.stdout.flush()