def init_replaynet_data_provider(self): if self.output_method == 'disk': dp = data.get_by_name('intermediate') count = self.layer_output_dumper.get_count() self.train_dp = dp(self.layer_output_path, range(0, count), 'fc') elif self.output_method == 'memory': dp = data.get_by_name('memory') self.train_dp = dp(self.layer_output_dumper)
def set_category_range(self, r): dp = data.get_by_name(self.data_provider) self.train_dp = dp(self.data_dir, self.train_range, category_range=range(r)) self.test_dp = dp(self.data_dir, self.test_range, category_range=range(r))
def test_imagenet_loader(): df = data.get_by_name('imagenet')('/ssd/nn-data/imagenet/', batch_range=range(1000), test=True, batch_size=128) for i in range(32): st = time.time() batch = df.get_next_batch(8 * 10) print time.time() - st print batch.labels print batch.data.shape time.sleep(0.5) print batch.labels
def test_cifar_loader(): data_dir = '/ssd/nn-data/cifar-10.old/' dp = data.get_by_name('cifar10')(data_dir, [1]) batch_size = 128 data_list = [] for i in range(11000): batch = dp.get_next_batch(batch_size) batch = batch.data.get() data_list.append(batch) if batch.shape[1] != batch_size: break batch = np.concatenate(data_list, axis=1) print_matrix(batch, 'batch')
def test_imagenet_loader(): df = data.get_by_name('imagenet')( '/ssd/nn-data/imagenet/', batch_range=range(1000), test = True, batch_size=128) for i in range(32): st = time.time() batch = df.get_next_batch(8 * 10) print time.time() - st print batch.labels print batch.data.shape time.sleep(0.5) print batch.labels
def test_cifar_loader(): data_dir = '/ssd/nn-data/cifar-10.old/' dp = data.get_by_name('cifar10')(data_dir, [1]) batch_size = 128 data_list = [] for i in range(11000): batch = dp.get_next_batch(batch_size) batch = batch.data.get() data_list.append(batch) if batch.shape[1] != batch_size: break batch = np.concatenate(data_list, axis = 1) print_matrix(batch, 'batch')
# create a checkpoint dumper image_shape = (param_dict['image_color'], param_dict['image_size'], param_dict['image_size'], param_dict['batch_size']) param_dict['image_shape'] = image_shape cp_dumper = CheckpointDumper(param_dict['checkpoint_dir'], param_dict['test_id']) param_dict['checkpoint_dumper'] = cp_dumper # create the init_model init_model = cp_dumper.get_checkpoint() if init_model is None: init_model = parse_config_file(args.param_file) param_dict['init_model'] = init_model # create train dataprovider and test dataprovider dp_class = data.get_by_name(param_dict['data_provider']) train_dp = dp_class(param_dict['data_dir'], param_dict['train_range']) test_dp = dp_class(param_dict['data_dir'], param_dict['test_range']) param_dict['train_dp'] = train_dp param_dict['test_dp'] = test_dp # get all extra information num_batch = util.string_to_int_list(args.num_batch) if len(num_batch) == 1: param_dict['num_batch'] = num_batch[0] else: param_dict['num_batch'] = num_batch param_dict['num_group_list'] = util.string_to_int_list(args.num_group_list)
test_id = 'multiview' data_dir = '/ssd/nn-data/imagenet/' checkpoint_dir = '/home/justin/fastnet/fastnet/checkpoint/' param_file = '/home/justin/fastnet/config/imagenet.cfg' output_dir = '' output_method = 'disk' train_range = range(101, 102) #1,2,3,....,40 test_range = range(1, 2) #41, 42, ..., 48 data_provider = 'imagenet' multiview = True train_dp = data.get_by_name(data_provider)(data_dir, train_range) test_dp = data.get_by_name(data_provider)(data_dir, test_range, multiview=True) checkpoint_dumper = trainer.CheckpointDumper(checkpoint_dir, test_id) model = checkpoint_dumper.get_checkpoint() if model is None: model = parser.parse_config_file(param_file) save_freq = 100 test_freq = 4 adjust_freq = 100 factor = 1 num_epoch = 50 learning_rate = 0.1 batch_size = 128 image_color = 3
test_id = 'eye50-ann10' data_dir = '/ssd/fergusgroup/sainaa/imagenet/train/' data_dir_noisy = '/ssd/fergusgroup/sainaa/imagenet/noisy/' checkpoint_dir = '/ssd/fergusgroup/sainaa/imagenet/checkpoint/' param_file = '/home/ss7345/fastnet-noisy/config/imagenet-noisy.cfg' output_dir = '' output_method = 'disk' train_range = range(101, 1301) #1,2,3,....,40 train_range_noisy = range(1, 2000) #1,2,3,....,40 test_range = range(1, 101) #41, 42, ..., 48 data_provider = 'imagenet' train_dp_clear = data.get_by_name(data_provider)(data_dir,train_range) train_dp_noisy = data.get_by_name(data_provider)(data_dir_noisy,train_range_noisy) train_dp = data.NoisyDataProvider(train_dp_clear, train_dp_noisy) test_dp = data.get_by_name(data_provider)(data_dir, test_range) checkpoint_dumper = trainer.CheckpointDumper(checkpoint_dir, test_id) init_model = checkpoint_dumper.get_checkpoint() if init_model is None: init_model = parser.parse_config_file(param_file) save_freq = 100 test_freq = 100 adjust_freq = 100 factor = 1 num_epoch = 10 learning_rate = 0.1
def set_num_group(self, n): dp = data.get_by_name(self.data_provider) self.train_dp = dp(self.data_dir, self.train_range, n) self.test_dp = dp(self.data_dir, self.test_range, n)
# create a checkpoint dumper image_shape = (param_dict['image_color'], param_dict['image_size'], param_dict['image_size'], param_dict['batch_size']) param_dict['image_shape'] = image_shape cp_dumper = CheckpointDumper(param_dict['checkpoint_dir'], param_dict['test_id']) param_dict['checkpoint_dumper'] = cp_dumper # create the init_model init_model = cp_dumper.get_checkpoint() if init_model is None: init_model = parse_config_file(args.param_file) param_dict['init_model'] = init_model # create train dataprovider and test dataprovider dp_class = data.get_by_name(param_dict['data_provider']) train_dp = dp_class(param_dict['data_dir'], param_dict['train_range']) test_dp = dp_class(param_dict['data_dir'], param_dict['test_range'], multiview = param_dict['multiview']) param_dict['train_dp'] = train_dp param_dict['test_dp'] = test_dp # get all extra information num_batch = util.string_to_int_list(args.num_batch) if len(num_batch) == 1: param_dict['num_batch'] = num_batch[0] else: param_dict['num_batch'] = num_batch param_dict['num_group_list'] = util.string_to_int_list(args.num_group_list) param_dict['num_caterange_list'] = util.string_to_int_list(args.num_caterange_list)
from fastnet import data, trainer, net, parser test_id = 0 data_dir = '/ssd/nn-data/imagenet/' checkpoint_dir = '/home/justin/fastnet/fastnet/checkpoint/' param_file = '/home/justin/fastnet/config/imagenet_conv.cfg' output_dir = '' output_method = 'disk' train_range = range(101, 1301) #1,2,3,....,40 test_range = range(1, 101) #41, 42, ..., 48 data_provider = 'imagenet' train_dp = data.get_by_name(data_provider)(data_dir,train_range) test_dp = data.get_by_name(data_provider)(data_dir, test_range) checkpoint_dumper = trainer.CheckpointDumper(checkpoint_dir, test_id) save_freq = 100 test_freq = 100 adjust_freq = 100 factor = 1 num_epoch = 5 learning_rate = 0.1 batch_size = 128 image_color = 3 image_size = 224 image_shape = (image_color, image_size, image_size, batch_size)
num_epochs = int(sys.argv[1]) test_id = 'layerwise-%d' % num_epochs data_dir = '/ssd/nn-data/imagenet/' checkpoint_dir = '/big0/checkpoints/' param_file = './config/imagenet.cfg' output_dir = '' output_method = 'disk' train_range = range(101, 1301) #1,2,3,....,40 test_range = range(1, 101) #41, 42, ..., 48 data_provider = 'imagenet' train_dp = data.get_by_name(data_provider)(data_dir, train_range) test_dp = data.get_by_name(data_provider)(data_dir, test_range) checkpoint_dumper = trainer.CheckpointDumper(checkpoint_dir, test_id) model = checkpoint_dumper.get_checkpoint() if model is None: model = parser.parse_config_file(param_file) save_freq = 100000 test_freq = 1000 adjust_freq = 100000 factor = 1 num_batch = 1 learning_rate = 0.1 batch_size = 128 image_color = 3
test_id = 'multiview' data_dir = '/ssd/nn-data/imagenet/' checkpoint_dir = '/home/justin/fastnet/fastnet/checkpoint/' param_file = '/home/justin/fastnet/config/imagenet.cfg' output_dir = '' output_method = 'disk' train_range = range(101, 102) #1,2,3,....,40 test_range = range(1, 2) #41, 42, ..., 48 data_provider = 'imagenet' multiview = True train_dp = data.get_by_name(data_provider)(data_dir,train_range) test_dp = data.get_by_name(data_provider)(data_dir, test_range, multiview = True) checkpoint_dumper = trainer.CheckpointDumper(checkpoint_dir, test_id) model = checkpoint_dumper.get_checkpoint() if model is None: model = parser.parse_config_file(param_file) save_freq = 100 test_freq = 4 adjust_freq = 100 factor = 1 num_epoch = 50 learning_rate = 0.1 batch_size = 128 image_color = 3