def main():
database = RetrievalModel(args=args)
logging.info("db param size = %fMB", utils.count_parameters_in_MB(database))
logging.info(stat(database,(3,224,224)))
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, args.code_size, args.layers, args.auxiliary, genotype)
model.drop_path_prob = args.drop_path_prob
logging.info("query param size = %fMB", utils.count_parameters_in_MB(model))
logging.info(stat(model,(3,224,224)))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
args.seed = np.random.randint(0, 100)
logging.info("----------seed = {}-----------".format(args.seed))
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
logging.info("Arc = {}".format(genotype))
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs),eta_min=args.learning_rate_min)
Best_valid_acc = 0.0
for epoch in range(args.epochs):
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
scheduler.step()
if valid_acc >= Best_valid_acc:
Best_valid_acc = valid_acc
logging.info('Best_valid_acc %f', Best_valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
np.random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.benchmark = True
cudnn.enabled = True
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
# equal to: genotype = genotypes.DARTS_v2
genotype = eval("genotypes.%s" % args.arch)
print('Load genotype:', genotype)
model = Network(args.init_ch, 10, args.layers, args.auxiliary, genotype).cuda()
utils.load(model, args.exp_path)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss().cuda()
_, test_transform = utils._data_transforms_cifar10(args)
test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform)
test_queue = torch.utils.data.DataLoader(
test_data, batch_size=args.batchsz, shuffle=False, pin_memory=True, num_workers=2)
model.drop_path_prob = args.drop_path_prob
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
def main():
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
cudnn.enabled = True
torch.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_ch, 10, args.layers, args.auxiliary,
genotype).cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.wd)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batchsz,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batchsz,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc: %f', valid_acc)
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc: %f', train_acc)
utils.save(model, os.path.join(args.save, 'trained.pt'))
print('saved to: trained.pt')
def main():
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
cudnn.enabled = True
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
model = Network(args.init_ch, 10, args.layers, True, genotype).cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
checkpoint = torch.load(args.checkpoint + '/top1.pt')
model.load_state_dict(checkpoint['model_state_dict'])
criterion = nn.CrossEntropyLoss().cuda()
CIFAR_MEAN = [0.49139968, 0.48215827, 0.44653124]
CIFAR_STD = [0.24703233, 0.24348505, 0.26158768]
valid_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(CIFAR_MEAN, CIFAR_STD),
])
valid_queue = torch.utils.data.DataLoader(dset.CIFAR10(
root=args.data, train=False, transform=valid_transform),
batch_size=args.batch_size,
shuffle=True,
num_workers=2,
pin_memory=True)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc: %f', valid_acc)
def initialize_model(self):
"""
Initialize model, may change across different model.
:return:
"""
args = self.args
model = self.model_fn(args)
if args.gpus > 0:
if self.args.gpus == 1:
model = model.cuda()
self.parallel_model = model
else:
self.model = model
self.parallel_model = nn.DataParallel(self.model).cuda()
# IPython.embed(header='checking replicas and others.')
else:
self.parallel_model = model
# rewrite the pointer
model = self.parallel_model
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
# scheduler as Cosine.
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
return model, optimizer, scheduler
def main():
# logging.info('no gpu device available')
# sys.exit(1)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype)
utils.load(model, args.model_path, strict=False)
model = model.to_device() # to(torch._default_device)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.to_device() # to(torch._default_device)
_, test_transform = utils._data_transforms_cifar10(args)
test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform)
test_queue = torch.utils.data.DataLoader(
test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
model.drop_path_prob = args.drop_path_prob
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
def generate_pop(DNA_SIZE=args.DNA_SIZE, POP_SIZE=args.POP_SIZE):
logging.info('\n-----generating population------')
block_pop = []
size_pop = []
for idx in range(POP_SIZE):
param = 10.0
while param >= 8.0: # disgard network that larger than 3M
block_cfg, size_cfg, ds_cfg = make_cfg(DNA_SIZE)
model = nas.initial_network(block_cfg, size_cfg, ds_cfg)
param = utils.count_parameters_in_MB(nas.net)
logging.info('param size = {0:.2f}MB'.format(param))
block_pop.append(block_cfg)
size_pop.append(size_cfg)
logging.info('pop idx {} generated\n'.format(idx))
logging.info('block_pop shape: POP {}, DNA {}'.format(
len(block_pop), len(block_pop[0])))
logging.info(' size_pop shape: POP {}, DNA {}'.format(
len(size_pop), len(size_pop[0])))
logging.info('ds_cfg {}'.format(ds_cfg))
logging.info('Population block_pop {}'.format(block_pop))
logging.info('Population size_pop {}\n'.format(size_pop))
pop = {'block_pop': block_pop, 'size_pop': size_pop, 'ds_cfg': ds_cfg}
return pop
def main():
if not torch.cuda.is_available():
sys.exit(1)
## step 1 construct the selected network
genotype = eval("genotypes.%s" % args.selected_arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
## step 2 load pretrained model parameter
if args.cifar100:
model = torch.nn.DataParallel(model)
model = model.cuda()
model.load_state_dict(torch.load(args.model_path)['net'])
else:
utils.load(model, args.model_path)
model = torch.nn.DataParallel(model)
model = model.cuda()
model.module.drop_path_prob = 0
model.drop_path_prob = 0
print("param size = %fMB" % utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
## step 3 load test data
valid_queue = load_data_cifar(args)
## step 4. inference on test data
valid_acc, valid_obj = infer(valid_queue, model, criterion)
print('-----------------------------------------------')
print('Average Valid_acc: %f ' % valid_acc)
print('-----------------------------------------------')
def build_cifar10(model_state_dict=None, optimizer_state_dict=None, **kwargs):
epoch = kwargs.pop('epoch')
ratio = kwargs.pop('ratio')
train_ds, valid_ds = utils.load_cifar10(args.child_batch_size, args.child_cutout_size) # subset random sample
model = NASWSNetworkCIFAR(10, args.child_layers, args.child_nodes, args.child_channels, args.child_keep_prob,
args.child_drop_path_keep_prob,
args.child_use_aux_head, args.steps)
model = model.cuda()
train_criterion = nn.CrossEntropyLoss().cuda()
eval_criterion = nn.CrossEntropyLoss().cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(
model.parameters(),
args.child_lr_max,
momentum=0.9,
weight_decay=args.child_l2_reg,
)
if model_state_dict is not None:
model.load_state_dict(model_state_dict)
if optimizer_state_dict is not None:
optimizer.load_state_dict(optimizer_state_dict)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, args.child_epochs, args.child_lr_min, epoch)
return train_queue, valid_queue, model, train_criterion, eval_criterion, optimizer, scheduler
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
#损失函数
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, criterion)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
#优化器
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
torch.cuda.set_device(args.gpu)
cudnn.enabled = True
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
utils.load(model, args.model_path)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
_, test_transform = utils._data_transforms_cifar10(args)
test_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=test_transform)
test_queue = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=False,
num_workers=2)
model.drop_path_prob = 0.0
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('Test_acc %f', test_acc)
def build_cifar100(model_state_dict=None,
optimizer_state_dict=None,
args=None,
**kwargs):
epoch = kwargs.pop('epoch')
ratio = kwargs.pop('ratio')
train_transform, valid_transform = utils._data_transforms_cifar10(
args.cutout_size)
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=valid_transform)
num_train = len(train_data)
assert num_train == len(valid_data)
indices = list(range(num_train))
split = int(np.floor(ratio * num_train))
np.random.shuffle(indices)
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.child_batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=16)
valid_queue = torch.utils.data.DataLoader(
valid_data,
batch_size=args.child_eval_batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True,
num_workers=16)
model = NASWSNetworkCIFAR(100, args.child_layers, args.child_nodes,
args.child_channels, args.child_keep_prob,
args.child_drop_path_keep_prob,
args.child_use_aux_head, args.steps)
model = model.cuda()
train_criterion = nn.CrossEntropyLoss().cuda()
eval_criterion = nn.CrossEntropyLoss().cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(
model.parameters(),
args.child_lr_max,
momentum=0.9,
weight_decay=args.child_l2_reg,
)
if model_state_dict is not None:
model.load_state_dict(model_state_dict)
if optimizer_state_dict is not None:
optimizer.load_state_dict(optimizer_state_dict)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, args.child_epochs, args.child_lr_min, epoch)
return train_queue, valid_queue, model, train_criterion, eval_criterion, optimizer, scheduler
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
in_channels, num_classes, dataset_in_torch, stride_for_aux = utils.dataset_fields(
args, train=False) # new
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, in_channels, stride_for_aux,
num_classes, args.layers, args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_data, valid_data = utils.dataset_split_and_transform(
dataset_in_torch, args, train=False) # new
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
number_of_classes = class_dict[args.dataset]
in_channels = inp_channel_dict[args.dataset]
model = Network(args.init_channels, number_of_classes, args.layers,
criterion, in_channels)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
# Get transforms to apply on data
train_transform, valid_transform = utils.get_data_transforms(args)
# Get the training queue
train_queue, valid_queue = get_training_queues(args, train_transform)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(F.softmax(model.alphas_normal, dim=-1))
print(F.softmax(model.alphas_reduce, dim=-1))
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %s' % args.gpu)
logging.info('small regime or regular regime = %s' % args.regime)
logging.info("args = %s", args)
# read graph info
if args.regime:
graph_info_dict3 = read_graph_info(args.graph_txt + "3.txt")
graph_info_dict4 = read_graph_info(args.graph_txt + "4.txt")
graph_info_dict5 = read_graph_info(args.graph_txt + "5.txt")
graph_info_dicts = [
graph_info_dict3, graph_info_dict4, graph_info_dict5
]
else:
graph_info_dict2 = read_graph_info(args.graph_txt + "2.txt")
graph_info_dict3 = read_graph_info(args.graph_txt + "3.txt")
graph_info_dict4 = read_graph_info(args.graph_txt + "4.txt")
graph_info_dict5 = read_graph_info(args.graph_txt + "5.txt")
graph_info_dicts = [
graph_info_dict2, graph_info_dict3, graph_info_dict4,
graph_info_dict5
]
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = RandomlyWiredNN(args.base_channels, NUM_CLASSES,
args.output_channels, args.regime,
graph_info_dicts)
# When training with multi GPUs, first execute nn.DataParallel() and then load model in eval stage
if args.parallel:
model = nn.DataParallel(model).cuda()
print("multi GPUs")
else:
model = model.cuda()
print("single GPU")
utils.load_model(model, args.model_path)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
logging.info("FLOPs = %fGB",
utils.model_param_flops_in_MB(model, multiply_adds=False))
_, test_queue = preprocess_imagenet(args)
valid_acc, valid_obj, valid_speed = infer(test_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
logging.info('valid_obj %f', valid_obj)
logging.info('valid_speed_per_image %f', valid_speed)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
gpus = [int(i) for i in args.gpu.split(',')]
# torch.cuda.set_device(gpus)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %s' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = torch.nn.DataParallel(model)
model = model.cuda()
utils.load(model, args.model_path)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
test_transform = transforms.Compose([
ToTensor(),
])
DATASET_INDEX = args.DATASET_INDEX
STEGANOGRAPHY = args.STEGANOGRAPHY
EMBEDDING_RATE = args.EMBEDDING_RATE
BOSSBASE_COVER_DIR = args.BOSSBASE_COVER_DIR
BOSSBASE_STEGO_DIR = args.BOSSBASE_STEGO_DIR
BOWS_COVER_DIR = args.BOWS2_COVER_DIR
BOWS_STEGO_DIR = args.BOWS2_STEGO_DIR
TEST_INDEX_PATH = 'index_list{}/bossbase_test_index.npy'.format(
DATASET_INDEX)
test_data = MyDataset(TEST_INDEX_PATH, BOSSBASE_COVER_DIR,
BOSSBASE_STEGO_DIR, BOWS_COVER_DIR, BOWS_STEGO_DIR,
test_transform)
test_queue = DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
model.drop_path_prob = args.drop_path_prob
test_acc, test_obj = infer(test_queue, model, criterion)
print(test_acc)
logging.info('test_acc %f', test_acc)
def evaluate(individual, num_classes, num_epochs, batch_size, learning_rate):
train_transform, test_transform = utils._data_transforms_cifar10()
train_dataset = torchvision.datasets.CIFAR10(root='../../data',
train=True,
transform=train_transform)
test_dataset = torchvision.datasets.CIFAR10(root='../../data',
train=False,
transform=test_transform)
train_loader = torch.utils.data.DataLoader(
dataset=train_dataset,
batch_size=batch_size,
shuffle=True,
# pin_memory=True,
# num_workers=2
)
test_loader = torch.utils.data.DataLoader(
dataset=test_dataset,
batch_size=batch_size,
shuffle=False,
# pin_memory=True,
# num_workers=2
)
structure = Network(individual.structure, [(3, 32), (32, 128), (128, 128)],
num_classes, (32, 32)).to(device)
individual.size = utils.count_parameters_in_MB(structure)
parameters = filter(lambda p: p.requires_grad, structure.parameters())
cudnn.enabled = True
cudnn.benchmark = True
criterion = torch.nn.CrossEntropyLoss().to(device)
optimizer = torch.optim.SGD(parameters,
lr=learning_rate,
momentum=0.9,
weight_decay=3e-4)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,
num_epochs,
eta_min=0.0)
best_acc = 0
for epoch in range(num_epochs):
print('epoch[{}/{}]:'.format(epoch + 1, num_epochs))
train(train_loader, structure, criterion, optimizer)
scheduler.step()
valid_acc = test(test_loader, structure, criterion)
print()
if valid_acc > best_acc:
best_acc = valid_acc
individual.accuracy = best_acc
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
torch.cuda.set_device(args.gpu)
cudnn.enabled = True
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
if args.dataset in LARGE_DATASETS:
model = NetworkLarge(args.init_channels, CLASSES, args.layers,
args.auxiliary, genotype)
else:
model = Network(args.init_channels, CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
utils.load(model, args.model_path)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
_, test_transform = utils.data_transforms(args.dataset, args.cutout,
args.cutout_length)
if args.dataset == "CIFAR100":
test_data = dset.CIFAR100(root=args.data,
train=False,
download=True,
transform=test_transform)
elif args.dataset == "CIFAR10":
test_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=test_transform)
elif args.dataset == "sport8":
dset_cls = dset.ImageFolder
val_path = '%s/Sport8/test' % args.data
test_data = dset_cls(root=val_path, transform=test_transform)
elif args.dataset == "mit67":
dset_cls = dset.ImageFolder
val_path = '%s/MIT67/test' % args.data
test_data = dset_cls(root=val_path, transform=test_transform)
elif args.dataset == "flowers102":
dset_cls = dset.ImageFolder
val_path = '%s/flowers102/test' % args.tmp_data_dir
test_data = dset_cls(root=val_path, transform=test_transform)
test_queue = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=False,
num_workers=2)
model.drop_path_prob = 0.0
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('Test_acc %f', test_acc)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
args.seed += 1
logging.info("args = %s", args)
arch = utils.decode_arch(args.arch)
model = Network(arch, CIFAR_CLASSES)
model = model.cuda()
params = utils.count_parameters_in_MB(model)
logging.info("param size = %fMB", params / 1e6)
flops = utils.count_FLOPs(model)
logging.info("FLOPs = %d", flops)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
train_transform, valid_transform = utils._data_transforms_cifar10()
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, pin_memory=True, num_workers=8, shuffle=True)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, pin_memory=True, num_workers=8, shuffle=False)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
best_acc = 0
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
if valid_acc > best_acc:
best_acc = valid_acc
logging.info('valid_acc %f', valid_acc)
return params, flops, best_acc
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
in_channels = 3
num_classes = 10
stride_for_aux = 3
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, in_channels, stride_for_aux,
num_classes, args.layers, args.auxiliary, genotype)
model = model.cuda()
utils.load(model, args.model_path)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
_, test_transform = utils._data_transforms_cifar10(args)
#test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform)
#version = 'v6'
#images, labels = utils_test.load_new_test_data(version)
#num_images = images.shape[0]
data_10 = np.load(
"/home/sivan/darts/cnn/data_cifar_10_1/cifar10.1_v4_data.npy")
labels_10 = np.load(
'/home/sivan/darts/cnn/data_cifar_10_1/cifar10.1_v4_labels.npy')
#
tensor_x = torch.stack([torch.Tensor(i)
for i in data_10]) # transform to torch tensors
tensor_y = torch.stack([torch.Tensor(i) for i in labels_10])
test_data = utils_d.TensorDataset(tensor_x, tensor_y) # create your datset
test_queue = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
model.drop_path_prob = args.drop_path_prob
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype,output_height=args.img_cropped_height,output_width=args.img_cropped_width)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.MSELoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
train_transform, valid_transform = utils._data_trainsforms_denosining_dataset(args)
train_data = DENOISE_DATASET(root=args.data,train_folder=args.train_data,label_folder=args.label_data,train=True, transform=train_transform,target_transform=train_transform )
#valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]), pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]), pin_memory=True, num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_obj %f', train_obj)
valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_obj %f', valid_obj)
utils.save(model, os.path.join(args.save, './weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CLASSES, args.layers, args.auxiliary,
genotype)
if args.parallel:
model = nn.DataParallel(model).cuda()
else:
model = model.cuda()
model.load_state_dict(
torch.load(args.model_path, map_location='cuda:0')['state_dict'])
print("param size = {:.1f}MB".format(
floor(utils.count_parameters_in_MB(model), 1)))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
validdir = os.path.join(args.data, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
valid_data = dset.ImageFolder(
validdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
]))
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=4)
model.drop_path_prob = args.drop_path_prob
input = torch.randn(1, 3, 224, 224)
input = input.cuda()
valid_acc_top1, valid_acc_top5, valid_obj = infer(valid_queue, model,
criterion)
logging.info('valid_acc_top1 %f', valid_acc_top1)
logging.info('valid_acc_top5 %f', valid_acc_top5)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.rnn_cell, args.init_channels, CIFAR_CLASSES, args.layers, args.nhid, genotype, ds="svhn")
model = model.cuda()
print("params:", print_param_size(model))
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
train_transform, valid_transform = utils._data_transforms_svhn_train(args)
train_d = dset.SVHN(root=args.data, split="train", download=True, transform=train_transform)
extra_d = dset.SVHN(root=args.data, split="extra", download=True, transform=train_transform)
train_data = torch.utils.data.ConcatDataset([train_d, extra_d])
valid_data = dset.SVHN(root=args.data, split="test", download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=4)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=4)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, int(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights_train.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
# 得到train_search里学好的normal cell 和reduction cell，genotypes.DARTS就是选的学好的DARTS_V2
genotype = eval("genotypes.%s" % args.arch)#DARTS_V2 = Genotype(normal=[('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 1), ('skip_connect', 0), ('skip_connect', 0), ('dil_conv_3x3', 2)], normal_concat=[2, 3, 4, 5], reduce=[('max_pool_3x3', 0), ('max_pool_3x3', 1), ('skip_connect', 2), ('max_pool_3x3', 1), ('max_pool_3x3', 0), ('skip_connect', 2), ('skip_connect', 2), ('max_pool_3x3', 1)], reduce_concat=[2, 3, 4, 5])
# 这里的Network用的是model.py的NetworkCIFAR
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def build_cifar100(model_state_dict, optimizer_state_dict, **kwargs):
epoch = kwargs.pop('epoch')
train_transform, valid_transform = utils._data_transforms_cifar10(
args.cutout_size)
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR100(root=args.data,
train=False,
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=16)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.eval_batch_size,
shuffle=False,
pin_memory=True,
num_workers=16)
model = NASNetworkCIFAR(args, 100, args.layers, args.nodes, args.channels,
args.keep_prob, args.drop_path_keep_prob,
args.use_aux_head, args.steps, args.arch)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
logging.info("multi adds = %fM", model.multi_adds / 1000000)
if model_state_dict is not None:
model.load_state_dict(model_state_dict)
if torch.cuda.device_count() > 1:
logging.info("Use %d %s", torch.cuda.device_count(), "GPUs !")
model = nn.DataParallel(model)
model = model.cuda()
train_criterion = nn.CrossEntropyLoss().cuda()
eval_criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.lr_max,
momentum=0.9,
weight_decay=args.l2_reg,
)
if optimizer_state_dict is not None:
optimizer.load_state_dict(optimizer_state_dict)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), args.lr_min, epoch)
return train_queue, valid_queue, model, train_criterion, eval_criterion, optimizer, scheduler
def get_current_model_size(self):
model = ModelForModelSizeMeasure(self._C, self._num_classes,
self._layers, self._criterion,
self.alphas_normal,
self.alphas_reduce, self._steps,
self._multiplier,
self._stem_multiplier)
size = count_parameters_in_MB(model)
# This need to be further checked with cuda stuff
del model
return size
def calculate_batch_size(batch_model, memory, genotype: Genotype,
init_channels: int, classes: int, layers: int,
auxiliary: bool) -> int:
model = Network(init_channels, classes, layers, auxiliary, genotype)
model_size_mb = utils.count_parameters_in_MB(model)
batch_size = 0
consumption = 0
while consumption < (memory - 500):
batch_size += 1
consumption = batch_model.predict([[model_size_mb, batch_size]])
log.info("Predicted consumption is %d MB", consumption)
return batch_size
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
try:
genotype = eval("genotypes.%s" % args.arch)
except (AttributeError, SyntaxError):
genotype = genotypes.load_genotype_from_file(args.arch)
test_data, OUTPUT_DIM, IN_CHANNELS, is_regression = load_dataset(
args, train=False)
model = Network(args.init_channels,
OUTPUT_DIM,
args.layers,
args.auxiliary,
genotype,
num_channels=IN_CHANNELS)
model = model.cuda()
utils.load(model, args.model_path)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss() if not is_regression else nn.MSELoss()
criterion = criterion.cuda()
test_queue = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
model.drop_path_prob = args.drop_path_prob
test_acc, test_obj = infer(test_queue,
model,
criterion,
is_regression=is_regression)
logging.info('test_acc (R^2 for regression) %f', test_acc)
weights_foldername = os.path.dirname(args.model_path)
with open(os.path.join(weights_foldername, "test.txt"), "w") as f:
f.write(str(test_acc))
def __init__(self, context: PyTorchTrialContext) -> None:
self.context = context
self.data_config = context.get_data_config()
self.hparams = context.get_hparams()
self.criterion = torch.nn.functional.cross_entropy
# The last epoch is only used for logging.
self._last_epoch = -1
self.results = {
"loss": float("inf"),
"top1_accuracy": 0,
"top5_accuracy": 0
}
# Define the model
genotype = self.get_genotype_from_hps()
self.model = self.context.wrap_model(
Network(
self.hparams["init_channels"],
10, # num_classes
self.hparams["layers"],
self.hparams["auxiliary"],
genotype,
))
print("param size = {} MB".format(
utils.count_parameters_in_MB(self.model)))
size = 0
for p in self.model.parameters():
size += p.nelement()
print("param count: {}".format(size))
# Apply constraints if desired
if "use_constraints" in self.hparams and self.hparams[
"use_constraints"]:
apply_constraints(self.hparams, size)
# Define the optimizer
self.optimizer = self.context.wrap_optimizer(
torch.optim.SGD(
self.model.parameters(),
lr=self.context.get_hparam("learning_rate"),
momentum=self.context.get_hparam("momentum"),
weight_decay=self.context.get_hparam("weight_decay"),
))
# Define the LR scheduler
self.scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
self.optimizer,
self.context.get_hparam("train_epochs"),
)
step_mode = LRScheduler.StepMode.STEP_EVERY_EPOCH
self.wrapped_scheduler = self.context.wrap_lr_scheduler(
self.scheduler, step_mode=step_mode)
