def parse_text_file(text_file): if not os.path.exists(text_file): print('No text file', text_file, file=sys.stderr) return num_lines = gezi.get_num_lines(text_file) pb = ProgressBar(num_lines, 'parse text file %s' % text_file) for line in open(text_file): pb.progress() l = line.split('\t') image = l[0] image = image[:image.index('#')] text = l[-1].strip() #why text and ori_text ? because fo cn corpus text will be \x01 seperated(segged text) #for en corpus text and ori_text is the same ori_text = l[FLAGS.ori_text_index].strip() if text == '': continue if image not in text_map: text_map_[image] = set([text]) text_map[image] = [(text, ori_text)] else: if text not in text_map_: text_map_[image].add(text) text_map[image].append((text, ori_text)) for image in text_map: text_map[image] = list(text_map[image])
def convert_to(feat_file, name): num_shards = FLAGS.shards num_threads = FLAGS.threads if FLAGS.threads > 1: assert(num_threads == num_shards) f = open(feat_file).readlines() num_lines = len(f) if FLAGS.debug: num_lines = NUM_DEBUG_LINES shard_ranges = np.linspace(0, num_lines, num_shards + 1).astype(int) record = [] for i in xrange(num_threads): args = (f, name, i, shard_ranges[i], shard_ranges[i + 1]) process = multiprocessing.Process(target=_convert_to,args=args) process.start() record.append(process) for process in record: process.join() return #--------------single thread num_lines = gezi.get_num_lines(feat_file) if FLAGS.debug: num_lines = NUM_DEBUG_LINES shard_ranges = np.linspace(0, num_lines, num_shards + 1).astype(int) pb = ProgressBar(num_lines, "convert") shard = 0 count = 0; output_filename = '%s-%.5d-of-%.5d' % (name, shard, num_shards) output_file = os.path.join(FLAGS.output_directory, output_filename) print('Writing', output_file, count) writer = tf.python_io.TFRecordWriter(output_file) for line in open(feat_file): pb.progress() if count >= shard_ranges[shard + 1]: shard += 1 writer.close() output_filename = '%s-%.5d-of-%.5d' % (name, shard, num_shards) output_file = os.path.join(FLAGS.output_directory, output_filename) print('Writing', output_file, count) writer = tf.python_io.TFRecordWriter(output_file) _parse_line(line, writer) count += 1 if FLAGS.debug and count >= NUM_DEBUG_LINES: break writer.close()