def train():
try:
#check environment
env.config_env()
#check dataset
dt = dataset.dataset()
dt.check()
dt.generate_dataflow(image_size = config.configured_image_size)
dt.debug() #for debuging data set
#instance and initialize model
model = model_manager.get_model_instance(dataset = dt)
model.define()
model.generate()
model.train()
except Exception as e:
print(e)
traceback.print_exc()
finally:
print('over')
# ExpandBorder(size=(368,368),resize=True),
RandomResizedCrop(size=(200, 200)),
# RandomResizedCrop(size=(336, 336)),
RandomHflip(),
Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
]),
'val': Compose([
# ExpandBorder(size=(336,336),resize=True),
ExpandBorder(size=(200,200),resize=True),
Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
}
data_set = {}
data_set['train'] = dataset(imgroot=rawdata_root,anno_pd=train_pd,encoder=encoder,
transforms=data_transforms["train"],
)
data_set['val'] = dataset(imgroot=rawdata_root,anno_pd=val_pd,encoder=encoder,
transforms=data_transforms["val"],
)
dataloader = {}
dataloader['train']=torch.utils.data.DataLoader(data_set['train'], batch_size=16,
shuffle=True, num_workers=8,collate_fn=collate_fn)
dataloader['val']=torch.utils.data.DataLoader(data_set['val'], batch_size=16,
shuffle=True, num_workers=8,collate_fn=collate_fn)
'''model'''
model = se_resnet101_xuelang(num_classes=160)
random_state=43,
stratify=all_pd['label'])
true_test_pb = pd.read_csv("./datasets/test.txt",
sep=" ",
header=None,
names=['ImageName'])
"addFakeLabel"
true_test_pb['label'] = 1
test_pd = true_test_pb if mode == "test" else val_pd
print(test_pd.head())
data_set = {}
data_set['test'] = dataset(
imgroot=os.path.join(rawdata_root, mode),
anno_pd=test_pd,
transforms=test_transforms,
)
data_loader = {}
data_loader['test'] = torchdata.DataLoader(data_set['test'],
batch_size=4,
num_workers=4,
shuffle=False,
pin_memory=True,
collate_fn=collate_fn)
model_name = 'inceptionv4_nomal_label'
resume = './model/inceptionv4/weights-18-378-[0.9193].pth'
# model =resnet18(pretrained=True)
# model.avgpool = torch.nn.AdaptiveAvgPool2d(output_size=1)
def __init__(self, mode=config.configured_predict_modes[0]):
self.__mode = mode
self.__loaded_model = None
self.__dataset = dataset.dataset()
from core.net_sgd_corrector import SGD
from core.net_adam_corrector import Adam
from core.net_transfer_functions import functions
from dataset.dataset import dataset
from dataset.normalizer import calculate_normalize, normalize_dataset
net = Net(functions['tanh'], Adam)
# net = Net(functions['tanh'], NAG, corrector_param={'mu': 0.97})
# net = Net(functions['tanh'], SC)
net.initialize_from(
'/Users/nikon/PycharmProjects/laperseptron/data/iris.config.json',
0.001)
# net.load_from('/Users/nikon/PycharmProjects/laperseptron/data/iris.net.json')
train = dataset(
'/Users/nikon/PycharmProjects/laperseptron/data/iris.train.csv')
test = dataset(
'/Users/nikon/PycharmProjects/laperseptron/data/iris.test.csv')
net.set_normalization(calculate_normalize(train))
normalize_dataset(train, net)
normalize_dataset(test, net)
net.train(5, 0.1, train, test, 2)
# net.train(10, 0.1, train, test, 2)
# net.train(25, 0.1, train, test, 5)
net.save_to('/Users/nikon/PycharmProjects/laperseptron/data/iris.net.json')
print('\n%sTest Iris Setosa%s' % (Colors.OKGREEN if net.calculate(
test[0][0]) == 'Iris Setosa' else Colors.FAIL, Colors.ENDC))
tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_, logits=logits))
# loss += 0.0005 * tf.reduce_sum([tf.nn.l2_loss(var) for var in tf.trainable_variables()])
global_step = tf.Variable(0, trainable=False)
initial_learning_rate = 0.01
learning_rate = tf.train.exponential_decay(initial_learning_rate,
global_step=global_step,
decay_steps=1000,
decay_rate=0.9)
add_global = global_step.assign_add(1)
train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)
prediction = tf.to_int64(tf.argmax(logits, 1))
correct_prediction = tf.equal(prediction, tf.argmax(y_, 1))
acc = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# 训练和测试数据，可将n_epoch设置更大一些
x_train, y_train = dataset(rawdata_root, train_pd, image_size,
classes).images, dataset(rawdata_root, train_pd,
image_size, classes).labels
x_val, y_val = dataset(rawdata_root, val_pd, image_size,
classes).images, dataset(rawdata_root, val_pd,
image_size, classes).labels
n_epoch = 10
batch_size = 8
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
#saver = tf.tf.train.Saver()
for epoch in range(n_epoch):
start_time = time.time()
step = 0
# training
train_loss, train_acc, n_batch = 0, 0, 0
for x_train_a, y_train_a in minibatches(x_train,
AIM Lab
'''
import os
from other.parser import parse_input
from dataset.dataset import dataset
from preprocess.preprocess import MakeH5File
from train import Train
from test import Test
if __name__ == "__main__":
arg = parse_input()
# Download dataset
dataset(arg)
if arg.mode == 'train':
# PreProcess
root = os.path.dirname(os.path.realpath(__file__))
h5 = MakeH5File(root, arg)
h5.run()
# Train
train = Train(arg)
train.run()
if arg.mode == 'test':
test = Test(arg)
test.run()
save_path = os.path.join(save_dir, 'weights-%d-%d-[%.5f].pth' % (epoch, batch_cnt, loss.item()))
torch.save(model.state_dict(), save_path)
logging.info('saved model to %s' % (save_path))
logging.info('--' * 30)
if __name__ == '__main__':
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logfile = save_dir + '/trainlog.log'
trainlog(logfile)
'''data'''
data_set = {}
data_set['train'] = dataset(imgroot=rawdata_root, anno_pd=train_pd,
transforms=data_transforms["train"])
dataloader = {}
dataloader['train'] = torch.utils.data.DataLoader(data_set['train'], batch_size=4,
shuffle=True, num_workers=4, collate_fn=collate_fn)
'''model'''
# model = md.Modified_Densenet169(num_classs=100)
model = md.Modified_Resnet152(num_classs=100)
# model = md.Modified_SENet154(num_classs=100)
resume = None
if resume:
logging.info('Resuming finetune from %s' % resume)
model.load_state_dict(torch.load(resume))
model = model.cuda()
optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=1e-5)
predicted[predicted >= 2] = 1
predicted = predicted.view(-1)
accuracy = torch.sum(predicted == labels).item() / labels.size()[0]
return accuracy
BATCH_SIZE = 32
EPOCHS = 1
LEARNING_RATE = 0.001
np.random.seed(0)
torch.manual_seed(1)
cuda = torch.cuda.is_available()
train_set = dataset(train=True)
test_set = dataset(train=False)
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=BATCH_SIZE,
shuffle=True)
test_loader = torch.utils.data.DataLoader(test_set,
batch_size=2 * BATCH_SIZE,
shuffle=False)
model = net()
if cuda:
model = model.cuda()
criterion = loss()
optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE)
# optimizer = optim.SGD(model.parameters(), lr=LEARNING_RATE)
# PNASnet's transforms
# test_transforms = transforms.Compose([
# transforms.Resize(299),
# transforms.CenterCrop(299),
# transforms.ToTensor(),
# transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
# ])
if __name__ == '__main__':
test_pd = true_test_pb
# print(test_pd.head())
data_set = {}
data_set['test'] = dataset(imgroot=rawdata_root, anno_pd=test_pd,
transforms=test_transforms)
data_loader = {}
data_loader['test'] = torchdata.DataLoader(data_set['test'], batch_size=4, num_workers=4,
shuffle=False, pin_memory=True, collate_fn=collate_fn)
# model = md.Modified_Densenet169(num_classs=100)
model = md.Modified_Resnet152(num_classs=100)
# model = md.Modified_SENet154(num_classs=100)
print('Resuming finetune from %s' % resume)
model.load_state_dict(torch.load(resume))
model = model.cuda()
model.eval()
criterion = CrossEntropyLoss()
if not os.path.exists('./output'):
args=parser.parse_args()
log_dir=os.path.join(args.log_dir,args.model_name,args.patch_dataset,args.note)
ckpt_path = get_ckpt_path(log_dir)
print('load checkpoint file from', ckpt_path)
state_dict = torch.load(ckpt_path)
model=get_model(args)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.to(device)
if 'model_state' in state_dict.keys():
model.load_state_dict(state_dict['model_state'])
else:
model.load_state_dict(state_dict)
val_loader=TD.DataLoader(dataset=dataset(args,split='test'),
batch_size=args.batch_size,
shuffle=True,
drop_last=True,
num_workers=2)
model.eval()
for idx,(datas) in enumerate(val_loader):
imgs=datas['imgs'].to(device).float()
points_offset=32*datas['points_offset'].data.cpu().numpy()
points_perturb_true=datas['points_perturb'].data.cpu().numpy()
img_path=datas['img_path']
outputs=32*model.forward(imgs)
show_offset(imgs,img_path,points_offset,points_perturb_true,outputs)
rawdata_root = '/data/round2B/test'
true_test_pb = pd.read_csv("/data/round2B/semifinal_image_phase2/image.txt",
header=None,
names=['ImageName'])
print(true_test_pb.head(10))
true_test_pb['label'] = 'ZJL296'
test_pd = true_test_pb if mode == "test" else val_pd
print(test_pd.head())
data_set = {}
data_set['test'] = dataset(
imgroot=rawdata_root,
anno_pd=test_pd,
encoder=test_encoder,
transforms=test_transforms,
)
data_loader = {}
data_loader['test'] = torchdata.DataLoader(data_set['test'],
batch_size=8,
num_workers=4,
shuffle=False,
pin_memory=True,
collate_fn=collate_fn)
# model_name = 'resnet50-out'
resume = None
model = se_resnet101_xuelang(num_classes=160)
# model =resnet50(pretrained=False)
import pprint
if __name__ == '__main__':
from core.net_interface import Net
from core.net_standard_corrector import Standard
from core.net_potential_corrector import Potential
from core.net_distance_functions import distance_functions
from dataset.dataset import dataset
dataset = dataset('/Users/nikon/PycharmProjects/lakohonen/data/iris.train.csv')
classes = [j for j in range(3) for i in range(50)]
potential = Potential(nu=1, tau=3000, p_min=0.75)
net = Net('Iris', distance_functions['manhattan'], potential)
net.initialize(input=4, m=10, n=15, factor=1, negative=False)
# net.load_from('/Users/nikon/PycharmProjects/lakohonen/data/iris.cluster.net.json')
net.train(500, dataset, stop_error=10 ** -15, stop_delta=10 ** -15)
path = '/Users/nikon/PycharmProjects/lakohonen/data'
net.visualize_maps(dataset, 'avg', path)
net.visualize_u_matrix(path)
net.visualize_clusterization(dataset, cluster_number=3, classes=classes, path=path)
net.save_to('/Users/nikon/PycharmProjects/lakohonen/data/iris.net.json')
def DataTransform(cfg, rawdata_root, train_pd, test_pd):
if True:
data_transforms = {
'swap': transforms.Compose([
transforms.Resize((512,512)),
transforms.RandomRotation(degrees=15),
transforms.RandomCrop((448,448)),
transforms.RandomHorizontalFlip(),
transforms.Randomswap((cfg['swap_num'],cfg['swap_num'])),
]),
'unswap': transforms.Compose([
transforms.Resize((512,512)),
transforms.RandomRotation(degrees=15),
transforms.RandomCrop((448,448)),
transforms.RandomHorizontalFlip(),
]),
'totensor': transforms.Compose([
transforms.Resize((448,448)),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
]),
'None': transforms.Compose([
transforms.Resize((512,512)),
transforms.CenterCrop((448,448)),
]),
}
data_set = {}
data_set['train'] = dataset(
cfg,
imgroot=rawdata_root,
anno_pd=train_pd,
unswap=data_transforms["unswap"],
swap=data_transforms["swap"],
totensor=data_transforms["totensor"],
centercrop = data_transforms["None"],
train=True
)
data_set['val'] = dataset(
cfg,
imgroot=rawdata_root,
anno_pd=test_pd,
unswap=data_transforms["unswap"],
swap=data_transforms["swap"],
totensor=data_transforms["totensor"],
centercrop = data_transforms["None"],
train=False
)
dataloader = {}
dataloader['train']=torch.utils.data.DataLoader(
data_set['train'],
batch_size=cfg['batch_size'],
shuffle=True,
num_workers=cfg['batch_size'],
# select need origianl:swap ratio for training 1:0, 0:1, 1:1, 1:2, 1:3
collate_fn=collate_fn_for_train_11
)
dataloader['val']=torch.utils.data.DataLoader(
data_set['val'],
batch_size=cfg['batch_size'],
shuffle=True,
num_workers=cfg['batch_size'],
collate_fn=collate_fn_for_test
)
return data_set, dataloader
transforms.Resize(224),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
}
save_dir = 'resnet50'
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logfile = '%s/trainlog.log' % save_dir
trainlog(logfile)
data_set = {}
data_set['train'] = dataset(
imgroot=datapath + "/train/",
anno_pd=train_pd,
transforms=data_transforms["train"],
)
data_set['val'] = dataset(
imgroot=datapath + "/val/",
anno_pd=val_pd,
transforms=data_transforms["val"],
)
dataloader = {}
dataloader['train'] = torch.utils.data.DataLoader(data_set['train'],
batch_size=4,
shuffle=True,
num_workers=4,
collate_fn=collate_fn)
dataloader['val'] = torch.utils.data.DataLoader(data_set['val'],
batch_size=4,
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
}
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
save_dir = './BD_SEResNeXt101_32x4d/models/SEResNeXt101_32x4d'
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logfile = '%s/trainlog.log' % save_dir
trainlog(logfile)
data_set = {}
data_set['train'] = dataset(
imgroot=os.path.join(rawdata_root, "train"),
anno_pd=train_pd,
transforms=data_transforms["train"],
)
data_set['val'] = dataset(
imgroot=os.path.join(rawdata_root, "train"),
anno_pd=val_pd,
transforms=data_transforms["val"],
)
dataloader = {}
dataloader['train'] = torch.utils.data.DataLoader(data_set['train'],
batch_size=8,
shuffle=True,
num_workers=4,
collate_fn=collate_fn)
dataloader['val'] = torch.utils.data.DataLoader(data_set['val'],
batch_size=4,