def main(data, vocab): data_config = Config(data) # datasets train_set = DataGenerator( path_formulas=data_config.path_formulas_train, dir_images=data_config.dir_images_train, path_matching=data_config.path_matching_train) test_set = DataGenerator( path_formulas=data_config.path_formulas_test, dir_images=data_config.dir_images_test, path_matching=data_config.path_matching_test) val_set = DataGenerator( path_formulas=data_config.path_formulas_val, dir_images=data_config.dir_images_val, path_matching=data_config.path_matching_val) # produce images and matching files train_set.build(buckets=data_config.buckets) test_set.build(buckets=data_config.buckets) val_set.build(buckets=data_config.buckets) # vocab vocab_config = Config(vocab) vocab = build_vocab([train_set], min_count=vocab_config.min_count_tok) write_vocab(vocab, vocab_config.path_vocab)
def main(data, vocab): data_config = Config(data) # datasets train_set = DataGenerator(path_formulas=data_config.path_formulas_train, dir_images=data_config.dir_images_train, path_matching=data_config.path_matching_train) test_set = DataGenerator(path_formulas=data_config.path_formulas_test, dir_images=data_config.dir_images_test, path_matching=data_config.path_matching_test) val_set = DataGenerator(path_formulas=data_config.path_formulas_val, dir_images=data_config.dir_images_val, path_matching=data_config.path_matching_val) # produce images and matching files # train_set.build(buckets=None, n_threads=1) train_set.build(buckets=None) test_set.build(buckets=None) val_set.build(buckets=None) # train_set.build(buckets=data_config.buckets) # test_set.build(buckets=data_config.buckets) # val_set.build(buckets=data_config.buckets) # p = Augmentor.Pipeline(data_config.dir_images_train) # p.zoom(probability=1, min_factor=1.1, max_factor=1.5) # # p.process() # augmented_images, labels = p.sample(3) # print labels # vocab vocab_config = Config(vocab) vocab = build_vocab([train_set], min_count=vocab_config.min_count_tok) write_vocab(vocab, vocab_config.path_vocab)