def main(_): # Train a word2vec model. if FLAGS.train_model: if not FLAGS.train_data or not FLAGS.save_path: print("--train_data and --save_path must be specified.") sys.exit(1) opts = Options(FLAGS) with tf.Graph().as_default(), tf.Session(config=tf.ConfigProto( allow_soft_placement=True)) as session: with tf.device("/gpu:2"): model = Word2Vec(opts, session) for i in range(opts.epochs_to_train): print("Beginning epoch {}".format(i)) model.train() # Process one epoch # Perform a final save. model.saver.save(session, os.path.join(opts.save_path, "model.ckpt"), global_step=model.global_step) else: opts = Options(FLAGS) with tf.Graph().as_default(), tf.Session(config=tf.ConfigProto( allow_soft_placement=True)) as session: with tf.device("/cpu:0"): model = Word2Vec(opts, session) model.get_embeddings_from_ckpt( './Results/wup_lch_nam/com_90_p/') model.get_eval_sims( "./Results/wup_lch_nam/com_90_p_report.csv", eval_ds=FLAGS.eval_ds)
def setup_datasets(flags_obj, shuffle=True): options_tr = Options() tr_dataset = MegaFaceDataset(options_tr) options_te = Options() options_te.data_mode = 'normal' te_dataset = MegaFaceDataset(options_te, read_ratio=0.1) if 'strip' in options_tr.data_mode: tr_dataset = strip_blend(tr_dataset, te_dataset, options_tr.strip_N) print('build tf dataset') ptr_class = MegaFaceImagePreprocessor(options_tr) tf_train = ptr_class.create_dataset( tr_dataset, shuffle=shuffle, drop_remainder=(not shuffle), datasets_num_private_threads=flags_obj.datasets_num_private_threads, tf_data_experimental_slack=flags_obj.tf_data_experimental_slack) print('tf_train done') pte_class = MegaFaceImagePreprocessor(options_te) tf_test = pte_class.create_dataset(te_dataset, shuffle=False) print('te_train done') print('dataset built done') return tf_train, tf_test, tr_dataset, te_dataset
def make_options_from_flags(FLAGS): if FLAGS.json_config is not None: options = read_options_from_file(FLAGS.json_config) else: options = Options() # the default value stored in config.Options if FLAGS.shuffle is not None: options.shuffle = FLAGS.shuffle if FLAGS.net_mode is not None: options.net_mode = FLAGS.net_mode if FLAGS.data_mode is not None: options.data_mode = FLAGS.data_mode if FLAGS.load_mode is not None: options.load_mode = FLAGS.load_mode if FLAGS.fix_level is not None: options.fix_level = FLAGS.fix_level if FLAGS.init_learning_rate is not None: options.base_lr = FLAGS.init_learning_rate if FLAGS.optimizer != 'sgd': options.optimizer = FLAGS.optimizer if FLAGS.weight_decay != 0.00004: options.weight_decay = FLAGS.weight_decay if FLAGS.global_label is not None: options.data_mode == 'global_label' options.global_label = FLAGS.global_label if options.load_mode != 'normal': if FLAGS.backbone_model_path is not None: options.backbone_model_path = FLAGS.backbone_model_path else: options.backbone_model_path = None return options
def prepare_opts(): opts = Options() opts.n_examples = int(1e3) opts.n_epoch = int(5e2) opts.rnn_type = 'tanh' # no gains from relu at a short distance opts.folder = opts.rnn_type opts.network_state_size = 100 opts.max_angle_change = 90 opts.batch_size = 50 opts.label_units = 'rad' # rad, deg - rad is imprecise, but deg is useless opts.direction = 'current' # current, home # home seems to work better, but it really should work both ways opts.separate_losses = False # useless - don't do this opts.zero_start = False # helps with 2-leg (obv) opts.angle_format = 'trig' opts.output_format = 'cartesian' # polar with gap can follow existence of turns, but is very imprecise # works well in cartesian opts.velocity = True # either form works opts.stopping_prob = .3 opts.network_loss = True if opts.network_loss: opts.name = opts.rnn_type + '_awloss' else: opts.name = opts.rnn_type + '_noloss' opts.step_gap = 3 opts.use_gap = False # use of gap improves performance for polar coords, not needed in cartesian if opts.use_gap: opts.name += '_gap' # opts.name += '_{}n'.format(opts.network_state_size) print(opts.name) return opts
def testtest(params): print(FLAGS.net_mode) print(FLAGS.batch_size) print(FLAGS.num_epochs) print(params.batch_size) print(params.num_epochs) options = Options() dataset = GTSRBDataset(options) model = Model_Builder('gtsrb', dataset.num_classes, options, params) p_class = dataset.get_input_preprocessor() preprocessor = p_class( options.batch_size, model.get_input_shapes('train'), options.batch_size, model.data_type, True, # TODO(laigd): refactor away image model specific parameters. distortions=params.distortions, resize_method='bilinear') ds = preprocessor.create_dataset(batch_size=options.batch_size, num_splits=1, batch_size_per_split=options.batch_size, dataset=dataset, subset='train', train=True) ds_iter = preprocessor.create_iterator(ds) input_list = ds_iter.get_next() with tf.variable_scope('v0'): bld_rst = model.build_network(input_list, phase_train=True, nclass=dataset.num_classes) # input_list = preprocessor.minibatch(dataset, subset='train', params=params) # img, lb = input_list # lb = input_list['img_path'] b = 0 show = False from scipy.special import softmax local_var_init_op = tf.local_variables_initializer() table_init_ops = tf.tables_initializer() # iterator_initilizor in here with tf.Session() as sess: sess.run(tf.global_variables_initializer()) sess.run(local_var_init_op) sess.run(table_init_ops) for i in range(330): print('%d: ' % i) lb, aux = sess.run([input_list[2], bld_rst.extra_info]) print(aux) print(softmax(aux, axis=1)) exit(0)
def gen_data_matrix(): from config import Options import dataset import builder import tensorflow as tf options = Options() my_set = dataset.MegafaceDataset(options) img_producer = dataset.ImageProducer(options, my_set) loader, img_op, lb_op, out_op = builder.build_model(img_producer,output_level=2) rst_image = [] rst_predict = [] t_label = 7707 sess_config = tf.ConfigProto() sess_config.gpu_options.allow_growth = True init_op = tf.global_variables_initializer() with tf.Session(config=sess_config) as sess: sess.run(init_op) loader.restore(sess, "/home/tdteach/data/benchmark/21-wedge") print(options.num_examples_per_epoch) for i in range(100): print(i) lgs, ims, lbs = sess.run([out_op,img_op,lb_op]) for j in range(ims.shape[0]): rst_predict.append(lgs[j][t_label]) rst_image.append(np.reshape(ims[j], 128*128*3)) img_producer.stop() ii = np.array(rst_image) ip = np.array(rst_predict) print(ii.shape) print(ip.shape) np.save('img_matrix',ii) np.save('prd_matrix',ip)
from config import Options from utils import save_options_to_file, read_options_from_file import os options = Options() for i in range(1, 50): options.pretrained_filepath = options.home_dir + 'workspace/benchmarks/gtsrb_models/haha_' + str( i) options.poison_number_limit = i + 1 #options.poison_number_limit = 0 options.out_npys_prefix = options.out_npys_folder + 'out_' + str(i + 1) options.checkpoint_folder = options.home_dir + 'data/checkpoint_' + str( i + 1) + '/' save_options_to_file(options, 'config.try') cmmd = 'python3 train_gtsrb.py --config=config.try' os.system(cmmd)
from official.utils.flags import core as flags_core from official.utils.misc import distribution_utils from official.utils.misc import model_helpers from official.vision.image_classification.resnet import common import numpy as np import cv2 import random from six.moves import xrange import csv import copy from config import Options GB_OPTIONS = Options() CROP_SIZE = 32 NUM_CLASSES = 43 IMAGE_RANGE = 'bilateral' #[-1,1] #IMAGE_RANGE = 'normal' #[0,1] for strip test #IMAGE_RANGE = 'raw' #[0,255] FLAGS = absl_flags.FLAGS class GTSRBImagePreprocessor(): def __init__(self, options): self.options = options if 'poison' in self.options.data_mode: self.poison_pattern, self.poison_mask = self.read_poison_pattern( self.options.poison_pattern_file)
# elif mode == "attended": # img = attend_crop_img(img, attend_size=opt.attend_size) else: print("Mode in image mode {} is not implemented.".format(mode)) # loop through models for model_name in model_names: print('\nComputing output for model {} and image {}.\n'.format( model_name, stimulus_name)) output = main(img_path, model_type=model_name) # loop through labels for model_region in opt.model_regions[model_name]: out = output[model_region].flatten() activation_data.append([ stimulus_name, img_mode, model_name, model_region, out ]) return pd.DataFrame(activation_data, columns=columns) opt = Options() # load pre-trained models model_names = ['ResNet50-ImageNet', 'ResNet50-HMR'] print("\t Model Data") # load activations of pre-trained models applied to stimuli model_data = models2df(model_names, opt) make_path(opt.model_data_dir) model_data.to_pickle(os.path.join(opt.model_data_dir, 'ResNets.pkl'))
def show_mask_norms(mask_folder, model_name = 'gtsrb', out_png=False): options = Options() options.model_name = model_name options = justify_options_for_model(options, model_name) options.data_subset = 'validation' options.batch_size = 1 options.num_epochs = 1 options.net_mode = 'backdoor_def' options.load_mode = 'all' options.fix_level = 'all' options.build_level = 'mask_only' options.selected_training_labels = None options.gen_ori_label = False ld_paths = dict() root_folder = mask_folder print(root_folder) dirs = os.listdir(root_folder) for d in dirs: tt = d.split('_')[0] if len(tt) == 0: continue try: tgt_id = int(tt) except: continue ld_paths[tgt_id] = get_last_checkpoint_in_folder(os.path.join(root_folder,d)) print(ld_paths) model, dataset, input_list, feed_list, out_op, aux_out_op = get_output(options, model_name=model_name) model.add_backbone_saver() mask_abs = dict() config = tf.ConfigProto() config.gpu_options.allow_growth = True import cv2 init_op = tf.global_variables_initializer() local_var_init_op = tf.local_variables_initializer() table_init_ops = tf.tables_initializer() # iterator_initilizor in here with tf.Session(config=config) as sess: sess.run(init_op) sess.run(local_var_init_op) sess.run(table_init_ops) for k, v in ld_paths.items(): print(v) model.load_backbone_model(sess, v) pattern, mask = sess.run([out_op, aux_out_op]) pattern = (pattern[0]+1)/2 mask = mask[0] mask_abs[k] = np.sum(np.abs(mask)) if out_png: show_name = '%d_pattern.png'%k out_pattern = pattern*255 cv2.imwrite(show_name, out_pattern.astype(np.uint8)) show_name = '%d_mask.png'%k out_mask = mask*255 cv2.imwrite(show_name, out_mask.astype(np.uint8)) show_name = '%d_color.png'%k out_color = pattern*mask*255 cv2.imwrite(show_name, out_color.astype(np.uint8)) #cv2.imshow(show_name,out_pattern) #cv2.waitKey() #break out_norms = np.zeros([len(mask_abs),2]) z = 0 for k,v in mask_abs.items(): out_norms[z][0] = k out_norms[z][1] = v z = z+1 print('===Results===') np.save('out_norms.npy', out_norms) print('write norms to out_norms.npy') #return vs = list(mask_abs.values()) import statistics me = statistics.median(vs) abvs = abs(vs - me) mad = statistics.median(abvs) rvs = abvs / (mad * 1.4826) print(mask_abs) print(rvs) x_arr = [i for i in range(len(mask_abs))] import matplotlib.pyplot as plt plt.figure() plt.boxplot(rvs) plt.show()
# Parse options Usage = 'Usage: python3 rsem.py -c config_file' parser = OptionParser(usage=Usage) parser.add_option('-c', '--config_file', action='store', type='string', dest='config_file', help='Path to configuration file') options, args = parser.parse_args() config_file = options.config_file opt = Options(config_file) csv_path = path.split(config_file)[0] + '/' + opt.name + '.csv' if opt.remove_resp: inventory = read_inventory(opt.inventory) else: inventory = None def pre_process(tr, inventory, freqmin, freqmax, order, factor): print('\nPre-processing') # Decimate if not opt.remove_resp:
def pipeline(): opts = Options() # get all present dirs opts = get_model_ix(opts) # opts.trial_time = {'no_lick': .5, 'sample': .5, 'delay': 1.5, 'test': .5, 'response': 1} opts.trial_time = { 'no_lick': .5, 'sample': .5, 'delay': 0, 'test': .5, 'response': .5 } opts.fixation = True opts.mask = True opts = get_loss_times(opts) opts.activation_fn = 'relu' opts.decay = True opts.epoch = 100 # 100-200 is enough # opts.epoch = 10 opts.batch_size = 20 opts.multilayer = False opts.layer_size = [80, 80] opts.noise = False opts.noise_intensity = .01 opts.noise_density = .5 opts.learning_rate = .001 opts.rnn_size = 100 opts.weight_alpha = .1 opts.activity_alpha = .1 run0 = opts run1 = copy.deepcopy(opts) run1.trial_time['delay'] = .5 run1 = get_loss_times(run1) run1.load_checkpoint = True run2 = copy.deepcopy(run1) run2.trial_time['delay'] = 1 run2 = get_loss_times(run2) run3 = copy.deepcopy(run1) run3.trial_time['delay'] = 1.5 run3 = get_loss_times(run3) for run in [run0, run1, run2, run3]: train.train(run) # opts.EI_in = False # opts.EI_h = False # opts.EI_out = False # opts.proportion_ex = .8 # train.eval(opts) # run_multiple(opts) return opts
from config import Options CMD_DATA1 = "/data/zhangyi/Noisy_dataset/CHJ14_cmd-CHJ/CHJ14_cmd-CHJ-2/CHJ14_cmd-CHJ/" CMD_DATA2 = "/data/zhangyi/Noisy_dataset/CHJ14_cmd-CHJ/CHJ14_cmd-CHJ-3/CHJ14_cmd-CHJ/hp/" AISHELL_DATA = "/data/tengxiang/dataset/aishell-chj-hp/" EXTRA_DATA = "/data/tengxiang/dataset/wenzhifan/4CH/" HISF_DATA = "/data/tengxiang/dataset/hisf/cmd_input/wav/" LOG_HOME = "/data/tengxiang/aishell_hp/" # tt_data_dir = os.path.join(AISHELL_DATA, "cv") tt_data_dir = EXTRA_DATA audio_log_dir = os.path.join(EXTRA_DATA, "../v1") ckpt = "/home/teng.xiang/projects/model-ckpt/aishell_hp-sir_005_020/TasNet-se-19" # ckpt = "/data/tengxiang/aishell_hp/v0/ckpt/TasNet-se-9" # ckpt = "/home/teng.xiang/projects/model-ckpt/aishell_time-delay_cln/TasNet-se-17" if __name__ == "__main__": opt = Options().parse() if opt.gpu_num == 1: os.environ['CUDA_VISIBLE_DEVICES'] = '1' if opt.tt_data_dir is None: opt.tt_data_dir = tt_data_dir if opt.audio_log_dir is None: opt.audio_log_dir = audio_log_dir tt_data = SeparatedDataset([opt.tt_data_dir]) tt_loader = DataLoader(tt_data, opt.batch_size * opt.gpu_num) tasnet = Model(opt) tasnet.inference(tt_loader, ckpt)
vec_dim = len(v) vec_list = [] for i, w in enumerate(w_list): if w in vec_dic: vec_list.append(vec_dic[w]) else: print(i, w) vec_list.append(np.random.rand(200)) return np.array(vec_list) if __name__ == '__main__': vocab_num = 100000 pubmed_w2v_path = 'pubmed_w2v.txt' emb_path = 'emb_cnn.pt' opt = Options(config_vocab=False) pubmedreader = PubMedReader(opt) print('loding text data') train_sents, train_labels, test_sents, test_labels, valid_sents, valid_labels = pubmedreader.get_data( ) print('read vocab') fixed_vocab_set = read_vocab(pubmed_w2v_path) print('fixed vocab set size {}'.format(len(fixed_vocab_set))) print('build vocab') vocab = Vocab.build_vocab(train_sents, fixed_vocab_set=fixed_vocab_set) # vocab.append_sents(valid_sents, fixed_vocab_set=fixed_vocab_set) vocab.append_sents(test_sents, fixed_vocab_set=fixed_vocab_set) # print('vocab size {} before shrink'.format(vocab.vocab_len))
def args_to_options(**kargs): options=Options() for k,v in kargs.items(): if hasattr(options,k): setattr(options,k,v) return options
def json_to_options(json_dict): options = Options() for k,v in json_dict.items(): setattr(options,k,v) return options
else: self.W_ah_trained = self.W_xh[:d_angle, :] self.W_sh_trained = self.W_xh[d_angle:, :] def scan_fn(self, hprev, input): state_update = tf.matmul(hprev, self.W_hh * self.W_hh_mask) input_update = tf.matmul(input, self.W_xh) # discrete time rnn h = self.fn(state_update + input_update + self.bias) # continuous time rnn if self.continuous: h = (1 - self.time_const) * hprev + self.time_const * h return h def batch_inputs(self, inputs, labels, opts): """Wrap the inputs in tensorflow batches.""" dataset = tf.data.Dataset.from_tensor_slices((inputs, labels)) if opts.test: dataset = dataset.batch(tf.cast(opts.batch_size, tf.int64)) else: dataset = dataset.shuffle(int(1E6)).batch( tf.cast(opts.batch_size, tf.int64)) # inferred repeat train_iter = dataset.make_initializable_iterator() next_element = train_iter.get_next() return train_iter, next_element if __name__ == '__main__': opts = Options() RNN(opts)
def main(): opt = Options() print('Use {}'.format(opt.pooling_type_str_dict[opt.pooling_type])) train_sents, train_labels = pickle.load(open(opt.train_path, 'rb')) valid_sents, valid_labels = pickle.load(open(opt.valid_path, 'rb')) test_sents, test_labels = pickle.load(open(opt.test_path, 'rb')) # np.set_printoptions(precision=3) np.set_printoptions(threshold=np.inf) # emb = Embedding(opt.vocab_size, 200, padding_idx=0, trainable=False) cnn = ML_CNN.CNN_Module(n_classes=opt.classifier_output_size) if opt.use_cuda: emb.cuda() cnn.cuda() param = [] param.extend(emb.parameters()) param.extend(cnn.parameters()) # optimizer = torch.optim.Adam(param, lr=opt.lr, weight_decay=0.01) # optimizer = torch.optim.Adam(param, lr=opt.lr, weight_decay=0.00001) optimizer = torch.optim.Adam(param, lr=opt.lr) criteron = torch.nn.CrossEntropyLoss() if opt.restore: if os.path.exists(opt.feature_net_path): print("Load pretrained embedding") emb.load_state_dict(torch.load(opt.feature_net_path)) else: print("No pretrained embedding") if os.path.exists(opt.classifier_net_path): print("Load pretrained cnn classifier") cnn.load_state_dict(torch.load(opt.classifier_net_path)) else: print("No pretrained cnn classifier") best_acc = -1 for epoch in range(opt.max_epochs): print("Starting epoch %d" % epoch) kf = get_minibatches_idx(len(train_sents), opt.batch_size, shuffle=True) epoch_losses = [] cnn.train() emb.train() for iteridx, train_index in kf: if len(train_index) <= 1: continue sents = [train_sents[t] for t in train_index] labels = [train_labels[t] for t in train_index] # X_batch, X_lengths, X_labels = prepare_data_for_rnn(sents, labels) X_batch, X_labels = prepare_data_for_cnn(sents, labels) X_batch = Variable(X_batch) X_labels = Variable(X_labels) if opt.use_cuda: X_batch = X_batch.cuda() X_labels = X_labels.cuda() optimizer.zero_grad() features = emb(X_batch) output = cnn(features) loss = criteron(output, X_labels) local_loss = loss.data[0] epoch_losses.append(local_loss) loss.backward() optimizer.step() if iteridx % opt.print_freq == 0: count = output.size(0) topK_correct = test_result(output.cpu().data, X_labels.cpu().data, topK=topK) topK_acc = [float(tmp) / count for tmp in topK_correct] topK_str = " , ".join(["acc@{}: {}".format(k, tmp_acc) for k, tmp_acc in zip(topK, topK_acc)]) print("Epoch {} Iteration {} loss: {} , {}".format(epoch + 1, iteridx + 1, local_loss, topK_str)) ave_loss = sum(epoch_losses) / len(epoch_losses) kf = get_minibatches_idx(len(valid_sents), opt.batch_size, shuffle=True) count = 0 all_topK_correct = np.zeros(len(topK), dtype=int) for _, valid_index in kf: emb.eval() cnn.eval() sents = [valid_sents[t] for t in valid_index] labels = [valid_labels[t] for t in valid_index] X_batch, X_labels = prepare_data_for_cnn(sents, labels) X_batch = Variable(X_batch) X_labels = Variable(X_labels) if opt.use_cuda: X_batch = X_batch.cuda() X_labels = X_labels.cuda() features = emb(X_batch) output = cnn(features) topK_correct = test_result(output.cpu().data, X_labels.cpu().data, topK=topK) topK_correct = np.array(topK_correct) all_topK_correct += topK_correct bsize = output.size(0) count += bsize all_topK_acc = all_topK_correct / float(count) all_topK_acc = all_topK_acc.tolist() all_topK_str = " , ".join(["val_acc@{}: {}".format(k, tmp_acc) for k, tmp_acc in zip(topK, all_topK_acc)]) print("Epoch {} Avg_loss: {}, {}".format(epoch+1, ave_loss, all_topK_str)) acc = all_topK_acc[important_K] if acc > best_acc: print('Dump current model due to current acc {} > past best acc {}'.format(acc, best_acc)) torch.save(cnn.state_dict(), opt.classifier_net_path) best_acc = acc fscore_records = [{k:FScore() for k in topK} for i in range(opt.classifier_output_size)] kf = get_minibatches_idx(len(test_sents), opt.batch_size, shuffle=True) emb.eval() cnn.eval() for _, test_index in kf: sents = [test_sents[t] for t in test_index] labels = [test_labels[t] for t in test_index] X_batch, X_labels = prepare_data_for_cnn(sents, labels) X_batch = Variable(X_batch) X_labels = Variable(X_labels) if opt.use_cuda: X_batch = X_batch.cuda() X_labels = X_labels.cuda() features = emb(X_batch) output = cnn(features) update_F1(output.cpu().data, X_labels.cpu().data, opt.classifier_output_size, topK, fscore_records) with open('F_score_dir/{}.pkl'.format(epoch+1),'w') as f: print('dumping fscore in epoch {}'.format(epoch+1)) pickle.dump(fscore_records, f) print('Loading best model') cnn.load_state_dict(torch.load(opt.classifier_net_path)) print('Testing Data') kf = get_minibatches_idx(len(test_sents), opt.batch_size, shuffle=True) count = 0 all_topK_correct = np.zeros(len(topK), dtype=int) fscore_records = [{k:FScore() for k in topK} for i in range(opt.classifier_output_size)] for _, test_index in kf: emb.eval() cnn.eval() sents = [test_sents[t] for t in test_index] labels = [test_labels[t] for t in test_index] X_batch, X_labels = prepare_data_for_cnn(sents, labels) X_batch = Variable(X_batch) X_labels = Variable(X_labels) if opt.use_cuda: X_batch = X_batch.cuda() X_labels = X_labels.cuda() features = emb(X_batch) output = cnn(features) update_F1(output.cpu().data, X_labels.cpu().data, opt.classifier_output_size, topK, fscore_records) topK_correct = test_result(output.cpu().data, X_labels.cpu().data, topK=topK) topK_correct = np.array(topK_correct) all_topK_correct += topK_correct bsize = output.size(0) count += bsize all_topK_acc = all_topK_correct / float(count) all_topK_acc = all_topK_acc.tolist() all_topK_str = " , ".join(["test_acc@{}: {}".format(k, tmp_acc) for k, tmp_acc in zip(topK, all_topK_acc)]) print("Training end {}".format(all_topK_str)) with open('F_score_dir/best.pkl','w') as f: print('dumping fscore in') pickle.dump(fscore_records, f)
def testtest(params): print(FLAGS.net_mode) print(FLAGS.batch_size) print(FLAGS.num_epochs) print(params.batch_size) print(params.num_epochs) options = Options() options.data_mode = 'normal' options.data_subset = 'train' dataset = CifarDataset(options) model = Model_Builder('cifar10', dataset.num_classes, options, params) labels, images = dataset.data images = np.asarray(images) data_dict = dict() data_dict['labels'] = labels data_dict['images'] = images save_to_mat('cifar-10.mat', data_dict) exit(0) p_class = dataset.get_input_preprocessor() preprocessor = p_class(options.batch_size, model.get_input_shapes('train'), options.batch_size, model.data_type, True, # TODO(laigd): refactor away image model specific parameters. distortions=params.distortions, resize_method='bilinear') ds = preprocessor.create_dataset(batch_size=options.batch_size, num_splits = 1, batch_size_per_split = options.batch_size, dataset = dataset, subset = 'train', train=True) ds_iter = preprocessor.create_iterator(ds) input_list = ds_iter.get_next() print(input_list) # input_list = preprocessor.minibatch(dataset, subset='train', params=params) # img, lb = input_list # lb = input_list['img_path'] lb = input_list print(lb) b = 0 show = False local_var_init_op = tf.local_variables_initializer() table_init_ops = tf.tables_initializer() # iterator_initilizor in here with tf.Session() as sess: sess.run(tf.global_variables_initializer()) sess.run(local_var_init_op) sess.run(table_init_ops) for i in range(330): print('%d: ' % i) if b == 0 or b+options.batch_size > dataset.num_examples_per_epoch('train'): show = True b = b+options.batch_size rst = sess.run(lb) # rst = rst.decode('utf-8') print(len(rst))