def init_replaynet_data_provider(self):
if self.output_method == 'disk':
dp = data.get_by_name('intermediate')
count = self.layer_output_dumper.get_count()
self.train_dp = dp(self.layer_output_path, range(0, count), 'fc')
elif self.output_method == 'memory':
dp = data.get_by_name('memory')
self.train_dp = dp(self.layer_output_dumper)
def init_replaynet_data_provider(self):
if self.output_method == 'disk':
dp = data.get_by_name('intermediate')
count = self.layer_output_dumper.get_count()
self.train_dp = dp(self.layer_output_path, range(0, count), 'fc')
elif self.output_method == 'memory':
dp = data.get_by_name('memory')
self.train_dp = dp(self.layer_output_dumper)
def set_category_range(self, r):
dp = data.get_by_name(self.data_provider)
self.train_dp = dp(self.data_dir,
self.train_range,
category_range=range(r))
self.test_dp = dp(self.data_dir,
self.test_range,
category_range=range(r))
def test_imagenet_loader():
df = data.get_by_name('imagenet')('/ssd/nn-data/imagenet/',
batch_range=range(1000),
test=True,
batch_size=128)
for i in range(32):
st = time.time()
batch = df.get_next_batch(8 * 10)
print time.time() - st
print batch.labels
print batch.data.shape
time.sleep(0.5)
print batch.labels
def test_cifar_loader():
data_dir = '/ssd/nn-data/cifar-10.old/'
dp = data.get_by_name('cifar10')(data_dir, [1])
batch_size = 128
data_list = []
for i in range(11000):
batch = dp.get_next_batch(batch_size)
batch = batch.data.get()
data_list.append(batch)
if batch.shape[1] != batch_size:
break
batch = np.concatenate(data_list, axis=1)
print_matrix(batch, 'batch')
def test_imagenet_loader():
df = data.get_by_name('imagenet')(
'/ssd/nn-data/imagenet/',
batch_range=range(1000),
test = True,
batch_size=128)
for i in range(32):
st = time.time()
batch = df.get_next_batch(8 * 10)
print time.time() - st
print batch.labels
print batch.data.shape
time.sleep(0.5)
print batch.labels
def test_cifar_loader():
data_dir = '/ssd/nn-data/cifar-10.old/'
dp = data.get_by_name('cifar10')(data_dir, [1])
batch_size = 128
data_list = []
for i in range(11000):
batch = dp.get_next_batch(batch_size)
batch = batch.data.get()
data_list.append(batch)
if batch.shape[1] != batch_size:
break
batch = np.concatenate(data_list, axis = 1)
print_matrix(batch, 'batch')
def set_category_range(self, r): dp = data.get_by_name(self.data_provider) self.train_dp = dp(self.data_dir, self.train_range, category_range=range(r)) self.test_dp = dp(self.data_dir, self.test_range, category_range=range(r))
# create a checkpoint dumper
image_shape = (param_dict['image_color'], param_dict['image_size'], param_dict['image_size'], param_dict['batch_size'])
param_dict['image_shape'] = image_shape
cp_dumper = CheckpointDumper(param_dict['checkpoint_dir'], param_dict['test_id'])
param_dict['checkpoint_dumper'] = cp_dumper
# create the init_model
init_model = cp_dumper.get_checkpoint()
if init_model is None:
init_model = parse_config_file(args.param_file)
param_dict['init_model'] = init_model
# create train dataprovider and test dataprovider
dp_class = data.get_by_name(param_dict['data_provider'])
train_dp = dp_class(param_dict['data_dir'], param_dict['train_range'])
test_dp = dp_class(param_dict['data_dir'], param_dict['test_range'])
param_dict['train_dp'] = train_dp
param_dict['test_dp'] = test_dp
# get all extra information
num_batch = util.string_to_int_list(args.num_batch)
if len(num_batch) == 1:
param_dict['num_batch'] = num_batch[0]
else:
param_dict['num_batch'] = num_batch
param_dict['num_group_list'] = util.string_to_int_list(args.num_group_list)
test_id = 'multiview'
data_dir = '/ssd/nn-data/imagenet/'
checkpoint_dir = '/home/justin/fastnet/fastnet/checkpoint/'
param_file = '/home/justin/fastnet/config/imagenet.cfg'
output_dir = ''
output_method = 'disk'
train_range = range(101, 102) #1,2,3,....,40
test_range = range(1, 2) #41, 42, ..., 48
data_provider = 'imagenet'
multiview = True
train_dp = data.get_by_name(data_provider)(data_dir, train_range)
test_dp = data.get_by_name(data_provider)(data_dir, test_range, multiview=True)
checkpoint_dumper = trainer.CheckpointDumper(checkpoint_dir, test_id)
model = checkpoint_dumper.get_checkpoint()
if model is None:
model = parser.parse_config_file(param_file)
save_freq = 100
test_freq = 4
adjust_freq = 100
factor = 1
num_epoch = 50
learning_rate = 0.1
batch_size = 128
image_color = 3
test_id = 'eye50-ann10' data_dir = '/ssd/fergusgroup/sainaa/imagenet/train/' data_dir_noisy = '/ssd/fergusgroup/sainaa/imagenet/noisy/' checkpoint_dir = '/ssd/fergusgroup/sainaa/imagenet/checkpoint/' param_file = '/home/ss7345/fastnet-noisy/config/imagenet-noisy.cfg' output_dir = '' output_method = 'disk' train_range = range(101, 1301) #1,2,3,....,40 train_range_noisy = range(1, 2000) #1,2,3,....,40 test_range = range(1, 101) #41, 42, ..., 48 data_provider = 'imagenet' train_dp_clear = data.get_by_name(data_provider)(data_dir,train_range) train_dp_noisy = data.get_by_name(data_provider)(data_dir_noisy,train_range_noisy) train_dp = data.NoisyDataProvider(train_dp_clear, train_dp_noisy) test_dp = data.get_by_name(data_provider)(data_dir, test_range) checkpoint_dumper = trainer.CheckpointDumper(checkpoint_dir, test_id) init_model = checkpoint_dumper.get_checkpoint() if init_model is None: init_model = parser.parse_config_file(param_file) save_freq = 100 test_freq = 100 adjust_freq = 100 factor = 1 num_epoch = 10 learning_rate = 0.1
def set_num_group(self, n): dp = data.get_by_name(self.data_provider) self.train_dp = dp(self.data_dir, self.train_range, n) self.test_dp = dp(self.data_dir, self.test_range, n)
# create a checkpoint dumper
image_shape = (param_dict['image_color'], param_dict['image_size'], param_dict['image_size'], param_dict['batch_size'])
param_dict['image_shape'] = image_shape
cp_dumper = CheckpointDumper(param_dict['checkpoint_dir'], param_dict['test_id'])
param_dict['checkpoint_dumper'] = cp_dumper
# create the init_model
init_model = cp_dumper.get_checkpoint()
if init_model is None:
init_model = parse_config_file(args.param_file)
param_dict['init_model'] = init_model
# create train dataprovider and test dataprovider
dp_class = data.get_by_name(param_dict['data_provider'])
train_dp = dp_class(param_dict['data_dir'], param_dict['train_range'])
test_dp = dp_class(param_dict['data_dir'], param_dict['test_range'], multiview = param_dict['multiview'])
param_dict['train_dp'] = train_dp
param_dict['test_dp'] = test_dp
# get all extra information
num_batch = util.string_to_int_list(args.num_batch)
if len(num_batch) == 1:
param_dict['num_batch'] = num_batch[0]
else:
param_dict['num_batch'] = num_batch
param_dict['num_group_list'] = util.string_to_int_list(args.num_group_list)
param_dict['num_caterange_list'] = util.string_to_int_list(args.num_caterange_list)
def set_num_group(self, n): dp = data.get_by_name(self.data_provider) self.train_dp = dp(self.data_dir, self.train_range, n) self.test_dp = dp(self.data_dir, self.test_range, n)
from fastnet import data, trainer, net, parser test_id = 0 data_dir = '/ssd/nn-data/imagenet/' checkpoint_dir = '/home/justin/fastnet/fastnet/checkpoint/' param_file = '/home/justin/fastnet/config/imagenet_conv.cfg' output_dir = '' output_method = 'disk' train_range = range(101, 1301) #1,2,3,....,40 test_range = range(1, 101) #41, 42, ..., 48 data_provider = 'imagenet' train_dp = data.get_by_name(data_provider)(data_dir,train_range) test_dp = data.get_by_name(data_provider)(data_dir, test_range) checkpoint_dumper = trainer.CheckpointDumper(checkpoint_dir, test_id) save_freq = 100 test_freq = 100 adjust_freq = 100 factor = 1 num_epoch = 5 learning_rate = 0.1 batch_size = 128 image_color = 3 image_size = 224 image_shape = (image_color, image_size, image_size, batch_size)
num_epochs = int(sys.argv[1])
test_id = 'layerwise-%d' % num_epochs
data_dir = '/ssd/nn-data/imagenet/'
checkpoint_dir = '/big0/checkpoints/'
param_file = './config/imagenet.cfg'
output_dir = ''
output_method = 'disk'
train_range = range(101, 1301) #1,2,3,....,40
test_range = range(1, 101) #41, 42, ..., 48
data_provider = 'imagenet'
train_dp = data.get_by_name(data_provider)(data_dir, train_range)
test_dp = data.get_by_name(data_provider)(data_dir, test_range)
checkpoint_dumper = trainer.CheckpointDumper(checkpoint_dir, test_id)
model = checkpoint_dumper.get_checkpoint()
if model is None:
model = parser.parse_config_file(param_file)
save_freq = 100000
test_freq = 1000
adjust_freq = 100000
factor = 1
num_batch = 1
learning_rate = 0.1
batch_size = 128
image_color = 3
test_id = 'multiview' data_dir = '/ssd/nn-data/imagenet/' checkpoint_dir = '/home/justin/fastnet/fastnet/checkpoint/' param_file = '/home/justin/fastnet/config/imagenet.cfg' output_dir = '' output_method = 'disk' train_range = range(101, 102) #1,2,3,....,40 test_range = range(1, 2) #41, 42, ..., 48 data_provider = 'imagenet' multiview = True train_dp = data.get_by_name(data_provider)(data_dir,train_range) test_dp = data.get_by_name(data_provider)(data_dir, test_range, multiview = True) checkpoint_dumper = trainer.CheckpointDumper(checkpoint_dir, test_id) model = checkpoint_dumper.get_checkpoint() if model is None: model = parser.parse_config_file(param_file) save_freq = 100 test_freq = 4 adjust_freq = 100 factor = 1 num_epoch = 50 learning_rate = 0.1 batch_size = 128 image_color = 3
