def test_cv_file_append_writer(): """tutorial for cv dataset append writer.""" writer = FileWriter(CV3_FILE_NAME, 4) data = get_data("../data/mindrecord/testImageNetData/") cv_schema_json = {"file_name": {"type": "string"}, "label": {"type": "int64"}, "data": {"type": "bytes"}} writer.add_schema(cv_schema_json, "img_schema") writer.add_index(["file_name", "label"]) writer.write_raw_data(data[0:5]) writer.commit() write_append = FileWriter.open_for_append(CV3_FILE_NAME + "0") write_append.write_raw_data(data[5:10]) write_append.commit() reader = FileReader(CV3_FILE_NAME + "0") count = 0 for index, x in enumerate(reader.get_next()): assert len(x) == 3 count = count + 1 logger.info("#item{}: {}".format(index, x)) assert count == 10 reader.close() paths = ["{}{}".format(CV3_FILE_NAME, str(x).rjust(1, '0')) for x in range(4)] for x in paths: os.remove("{}".format(x)) os.remove("{}.db".format(x))
def convert_data_to_mindrecord(): '''Covert data to mindrecord.''' print('Loading mindrecord...') writer = FileWriter.open_for_append(mindrecord_file_name) print('Loading train data...') total_data = [] with open(dataset_txt_file, 'r') as ft: lines = ft.readlines() for line in lines: sline = line.strip().split(" ") image_file = sline[0] labels = [] for item in sline[1:]: labels.append(int(item)) with open(image_file, 'rb') as f: img = f.read() data = {"image": img, "label": np.array(labels, dtype='int32')} total_data.append(data) print('Writing train data to mindrecord...') if total_data is None: raise ValueError("None needs writing to mindrecord.") writer.write_raw_data(total_data) writer.commit()
def convert_yolo_data_to_mindrecord(): '''convert_yolo_data_to_mindrecord''' print('Loading mindrecord...') writer = FileWriter.open_for_append(mindrecord_file_name, ) print('Loading train data...') image_files, anno_files = prepare_file_paths() dataset_size = len(anno_files) assert dataset_size == len(image_files) logger.info("#size of dataset: {}".format(dataset_size)) data = [] for i in range(dataset_size): data.append(get_data(image_files[i], anno_files[i])) print('Writing train data to mindrecord...') if data is None: raise ValueError("None needs writing to mindrecord.") writer.write_raw_data(data) writer.commit()