def make_options_from_flags(FLAGS): if FLAGS.json_config is not None: options = read_options_from_file(FLAGS.json_config) else: options = Options() # the default value stored in config.Options if FLAGS.shuffle is not None: options.shuffle = FLAGS.shuffle if FLAGS.net_mode is not None: options.net_mode = FLAGS.net_mode if FLAGS.data_mode is not None: options.data_mode = FLAGS.data_mode if FLAGS.load_mode is not None: options.load_mode = FLAGS.load_mode if FLAGS.fix_level is not None: options.fix_level = FLAGS.fix_level if FLAGS.init_learning_rate is not None: options.base_lr = FLAGS.init_learning_rate if FLAGS.optimizer != 'sgd': options.optimizer = FLAGS.optimizer if FLAGS.weight_decay != 0.00004: options.weight_decay = FLAGS.weight_decay if FLAGS.global_label is not None: options.data_mode == 'global_label' options.global_label = FLAGS.global_label if options.load_mode != 'normal': if FLAGS.backbone_model_path is not None: options.backbone_model_path = FLAGS.backbone_model_path else: options.backbone_model_path = None return options
def setup_datasets(flags_obj, shuffle=True): options_tr = Options() tr_dataset = MegaFaceDataset(options_tr) options_te = Options() options_te.data_mode = 'normal' te_dataset = MegaFaceDataset(options_te, read_ratio=0.1) if 'strip' in options_tr.data_mode: tr_dataset = strip_blend(tr_dataset, te_dataset, options_tr.strip_N) print('build tf dataset') ptr_class = MegaFaceImagePreprocessor(options_tr) tf_train = ptr_class.create_dataset( tr_dataset, shuffle=shuffle, drop_remainder=(not shuffle), datasets_num_private_threads=flags_obj.datasets_num_private_threads, tf_data_experimental_slack=flags_obj.tf_data_experimental_slack) print('tf_train done') pte_class = MegaFaceImagePreprocessor(options_te) tf_test = pte_class.create_dataset(te_dataset, shuffle=False) print('te_train done') print('dataset built done') return tf_train, tf_test, tr_dataset, te_dataset
# model_path = home_dir+'data/imagenet_models/benign_all' options.backbone_model_path = model_path options.net_mode = 'normal' # options.load_mode = 'bottom_affine' options.load_mode = 'all' options.num_epochs = 60 # options.data_mode = 'poison' options.data_mode = 'normal' #label_list = list(range(20)) options.poison_fraction = 1 options.cover_fraction = 1 #options.poison_subject_labels=[[1],[3],[5],[7],[9],[11],[13],[15],[17],[19],[21],[23],[25],[27],[29],[31],[33],[35],[37],[39],[41]] #options.poison_object_label=[0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40] #options.poison_cover_labels=[[11,12],[13,14]] #options.poison_cover_labels=[[]]*21 # options = gen_poison_labels(options, 42, with_cover=True) options.poison_subject_labels=[[1]] options.poison_object_label=[0] # options.poison_cover_labels=[[]] outfile_prefix = 'out_with_cover' options.poison_pattern_file = None
def testtest(params): print(FLAGS.net_mode) print(FLAGS.batch_size) print(FLAGS.num_epochs) print(params.batch_size) print(params.num_epochs) options = Options() options.data_mode = 'normal' options.data_subset = 'train' dataset = CifarDataset(options) model = Model_Builder('cifar10', dataset.num_classes, options, params) labels, images = dataset.data images = np.asarray(images) data_dict = dict() data_dict['labels'] = labels data_dict['images'] = images save_to_mat('cifar-10.mat', data_dict) exit(0) p_class = dataset.get_input_preprocessor() preprocessor = p_class(options.batch_size, model.get_input_shapes('train'), options.batch_size, model.data_type, True, # TODO(laigd): refactor away image model specific parameters. distortions=params.distortions, resize_method='bilinear') ds = preprocessor.create_dataset(batch_size=options.batch_size, num_splits = 1, batch_size_per_split = options.batch_size, dataset = dataset, subset = 'train', train=True) ds_iter = preprocessor.create_iterator(ds) input_list = ds_iter.get_next() print(input_list) # input_list = preprocessor.minibatch(dataset, subset='train', params=params) # img, lb = input_list # lb = input_list['img_path'] lb = input_list print(lb) b = 0 show = False local_var_init_op = tf.local_variables_initializer() table_init_ops = tf.tables_initializer() # iterator_initilizor in here with tf.Session() as sess: sess.run(tf.global_variables_initializer()) sess.run(local_var_init_op) sess.run(table_init_ops) for i in range(330): print('%d: ' % i) if b == 0 or b+options.batch_size > dataset.num_examples_per_epoch('train'): show = True b = b+options.batch_size rst = sess.run(lb) # rst = rst.decode('utf-8') print(len(rst))