def main():
args = get_argparser()
# our dataset has two class
classes = utils.parse_config(args.config_path)
print(len(classes), args.num_classes, classes)
assert len(classes) + 1 == args.num_classes, "Number of classes\
in config and argument is not same"
# use our dataset and defined transformations
dataset = loader.CellDataset(args.root_dir,
utils.get_transform(args.model, train=True),
args.labels_type, args.model, classes)
dataset_test = loader.CellDataset(args.root_dir,
utils.get_transform(args.model,
train=False),
args.labels_type,
args.model,
classes,
mode="Test")
indices = torch.arange(len(dataset)).tolist()
dataset = torch.utils.data.Subset(dataset,
indices[:int(len(indices) * 0.9)])
dataset_test = torch.utils.data.Subset(dataset_test,
indices[int(len(indices) * 0.9):])
print("Images in Test set", len(dataset_test), "Images in Train set ",
len(dataset))
# define training and validation data loaders
data_loader = torch.utils.data.DataLoader(dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4,
collate_fn=utils.collate_fn)
data_loader_test = torch.utils.data.DataLoader(dataset_test,
batch_size=1,
shuffle=True,
num_workers=4,
collate_fn=utils.collate_fn)
model = models.get_model(args.model, args.weight_path, args.num_classes,
args.max_instances, args.maskrcnn_backbone)
if args.cuda:
device = "cuda:0"
model.to(device)
else:
device = "cpu"
# optimizer = torch.optim.SGD(model.parameters(), lr=args.lr,
# momentum=0.9, weight_decay=0.0005)
print("\n\nStarting Training of ", args.model, "\n\n")
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
model_trainer = trainer.TrainModel(model, optimizer, args.model, device)
for epoch in range(args.epochs):
model_trainer.train(epoch, data_loader, data_loader_test)
print("That's it!")
def main(model_name):
model = choose_net(name=model_name,
num_classes=num_class,
weight_path='github')
model.to(device)
if torch.cuda.device_count() > 1:
net = nn.DataParallel(model)
optimizer = torch.optim.AdamW(model.parameters(),
lr=learning_rate,
amsgrad=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,
Epoches,
eta_min=1e-6)
# cheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.9, patience=2)
kf = KFold(n_splits=5, shuffle=True)
for fold, (train_idx, val_idx) in enumerate(kf.split(df)):
print(f'fold:{fold+1}...',
'train_size: %d, val_size: %d' % (len(train_idx), len(val_idx)))
df_train = df.values[train_idx]
df_val = df.values[val_idx]
train_dataset = MyData(root=Data_path,
df=df_train,
phase='train',
transform=get_transform(image_size, 'train'))
val_dataset = MyData(root=Data_path,
df=df_val,
phase='test',
transform=get_transform(image_size, 'test'))
train_loader = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=workers,
pin_memory=True)
val_loader = DataLoader(val_dataset,
batch_size=batch_size,
shuffle=False,
drop_last=True)
best_acc = 0.0
for epoch in range(Epoches):
print('Train {} / {}'.format(epoch + 1, Epoches))
train_loss = train(model, train_loader, optimizer)
if isinstance(scheduler,
torch.optim.lr_scheduler.ReduceLROnPlateau):
scheduler.step(train_loss)
else:
scheduler.step(epoch)
if epoch % 5 == 0:
acc = validate(model, val_loader)
if acc > best_acc:
if torch.cuda.device_count() > 1:
torch.save(
model.module.state_dict(), Model_path + '/' +
f"{model_name}_best_fold{fold + 1}.pth")
else:
torch.save(
model.state_dict(), Model_path + '/' +
f"{model_name}_best_fold{fold + 1}.pth")
def augmentBatch3(batch, config):
"""Shifts the polar radar image by a random amount, does NOT adjust ground truth transform.
The keypoints must be unrotated later using the T_aug transform stored in the batch dict.
"""
rot_max = config['augmentation']['rot_max']
data = batch['data'].numpy()
polar = batch['polar'].numpy()
azimuths = batch['azimuths'].numpy()
mask = batch['mask'].numpy()
azimuth_res = 0.9 * np.pi / 180
B, C, H, W = data.shape
T_aug = []
for i in range(B):
if np.mod(i, config['window_size']) == 0:
continue
plr = polar[i].squeeze()
azm = azimuths[i].squeeze()
rot = np.random.uniform(-rot_max, rot_max)
rot_azms = int(np.round(rot / azimuth_res))
rot = rot_azms * azimuth_res
plr = np.roll(plr, -1 * rot_azms, axis=0)
cart = radar_polar_to_cartesian(azm, plr, config['radar_resolution'],
config['cart_resolution'], config['cart_pixel_width']) # 1 x H x W
data[i] = cart[0]
polar_mask = mean_intensity_mask(plr)
msk = radar_polar_to_cartesian(azm, polar_mask, config['radar_resolution'],
config['cart_resolution'], config['cart_pixel_width']).astype(np.float32)
mask[i] = msk[0]
T_aug += [torch.from_numpy(get_transform(0, 0, -rot))]
polar[i] = plr
batch['data'] = torch.from_numpy(data)
batch['polar'] = torch.from_numpy(polar)
batch['mask'] = torch.from_numpy(mask > 0.5).type(batch['data'].dtype) # make into a binary mask
batch['T_aug'] = T_aug
return batch
def create_coco_loader(*paths):
transform = utils.get_transform(config.image_size, config.central_fraction)
datasets = [data.CocoImages(path, transform=transform) for path in paths]
dataset = data.Composite(*datasets)
data_loader = torch.utils.data.DataLoader(
dataset,
batch_size=config.preprocess_batch_size,
num_workers=config.data_workers,
shuffle=False,
pin_memory=True,
)
return data_loader
def main():
logging.basicConfig(
format=
'%(asctime)s: %(levelname)s: [%(filename)s:%(lineno)d]: %(message)s',
level=logging.INFO)
warnings.filterwarnings("ignore")
try:
ckpt = '../input/efefb2b2/weights/model_b2_386.ckpt'
except:
print('Set ckpt for evaluation in config.py')
return
test_dataset = FGVC7Data(
root=datasets,
phase='test',
transform=get_transform([config.image_size[0], config.image_size[1]],
'test'))
test_loader = DataLoader(test_dataset,
batch_size=1,
shuffle=False,
num_workers=2,
pin_memory=True)
num_classes = 5
net = efficientnet(net_name=net_name,
num_classes=num_classes,
weight_path='kaggle')
checkpoint = torch.load(ckpt)
state_dict = checkpoint['state_dict']
# Load weights
net.load_state_dict(state_dict)
print('Network loaded from {}'.format(ckpt))
net = net.to(device)
preds = []
image_name = []
with torch.no_grad():
net.eval()
pbar = tqdm(total=len(test_loader), unit=' batches')
pbar.set_description('Validation')
for i, input in enumerate(test_loader):
X, _, img_name = input
y_pred, y_metric = net(X.to(device))
pred = F.softmax(y_pred, dim=1).cpu().numpy()
preds.extend(pred.argmax(1))
image_name.append(img_name[0])
sub = pd.DataFrame({'image_id': image_name, 'label': preds})
print(sub)
sub.to_csv("submission.csv", index=False)
def main(file_name, log):
set_seed(cfg.SOLVER.SEED)
config_file = './configs/' + file_name
cfg.merge_from_file(config_file)
# os.environ["CUDA_VISIBLE_DEVICES"] = cfg.MODEL.DEVICE_ID
USE_CUDA = torch.cuda.is_available()
device = torch.device("cuda:0" if USE_CUDA else "cpu")
weight_path = cfg.MODEL.MODEL_PATH + cfg.MODEL.NAME + '.pth'
model = choose_net(name=cfg.MODEL.NAME, num_classes=cfg.MODEL.CLASSES, weight_path=cfg.MODEL.WEIGHT_FROM)
best_acc = 0.0
log.info('Train : {}'.format(cfg.MODEL.NAME))
if os.path.exists(weight_path):
checkpoint = torch.load(weight_path)
state_dict = checkpoint['state_dict']
best_acc = checkpoint['best_acc']
model.load_state_dict(state_dict)
log.info('Network loaded from {}'.format(weight_path))
model.to(device)
# model.cuda()
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model, device_ids=range(torch.cuda.device_count()))
optimizer = torch.optim.AdamW(model.parameters(), lr=cfg.SOLVER.BASE_LR, amsgrad=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, cfg.SOLVER.MAX_EPOCHS, eta_min=1e-6)
train_dataset = FGVC7Data(root=cfg.DATASETS.ROOT_DIR, phase='train', transform=get_transform(cfg.INPUT.SIZE_TRAIN, 'train'))
indices = range(len(train_dataset))
split = int(cfg.DATASETS.SPLIT * len(train_dataset))
train_indices = indices[split:]
test_indices = indices[:split]
train_sampler = SubsetRandomSampler(train_indices)
valid_sampler = SubsetRandomSampler(test_indices)
train_loader = DataLoader(train_dataset, batch_size=cfg.DATASETS.BATCH_SIZE, sampler=train_sampler, num_workers=cfg.DATASETS.WORKERS,
pin_memory=True)
val_loader = DataLoader(train_dataset, batch_size=cfg.DATASETS.BATCH_SIZE, sampler=valid_sampler, num_workers=cfg.DATASETS.WORKERS,
pin_memory=True)
for epoch in range(cfg.SOLVER.MAX_EPOCHS):
# pbar = tqdm(total=len(train_loader), unit='batches', ncols=150) # unit 表示迭代速度的单位
# pbar.set_description('Epoch {}/{}'.format(epoch + 1, cfg.SOLVER.MAX_EPOCHS))
train(model, optimizer, epoch, train_loader, log)
scheduler.step()
if (epoch+1) % 5 == 0:
acc = validate(model, val_loader, epoch, log)
if acc > best_acc:
if torch.cuda.device_count()>1:
torch.save({'best_acc':best_acc, 'state_dict':model.module.state_dict()}, weight_path)
else:
torch.save({'best_acc':best_acc, 'state_dict':model.state_dict()}, weight_path)
def augmentBatch2(batch, config):
"""Rotates the cartesian radar image by a random amount, does NOT adjust ground truth transform.
The keypoints must be unrotated later using the T_aug transform stored in the batch dict.
"""
rot_max = config['augmentation']['rot_max']
data = batch['data'].numpy() # this seems to return a reference, not a copy
mask = batch['mask'].numpy()
B, C, H, W = data.shape
T_aug = []
for i in range(B):
if np.mod(i, config['window_size']) == 0:
continue
img = data[i].squeeze()
mmg = mask[i].squeeze()
rot = np.random.uniform(-rot_max, rot_max)
M = cv2.getRotationMatrix2D((W / 2, H / 2), rot * 180 / np.pi, 1.0)
data[i] = cv2.warpAffine(img, M, (W, H), flags=cv2.INTER_CUBIC).reshape(C, H, W)
mask[i] = cv2.warpAffine(mmg, M, (W, H), flags=cv2.INTER_CUBIC).reshape(C, H, W)
T_aug += [torch.from_numpy(get_transform(0, 0, -rot))]
batch['data'] = torch.from_numpy(data)
batch['mask'] = torch.from_numpy(mask > 0.5).type(batch['data'].dtype) # make into a binary mask
batch['T_aug'] = T_aug
return batch
def get_groundtruth_odometry(self, radar_time, gt_path):
"""Retrieves the groundtruth 4x4 transform from current time to next
Args:
radar_time (int): UNIX INT64 time stamp that we want groundtruth for (also the filename for radar)
gt_path (AnyStr): path to the ground truth csv file
Returns:
np.ndarray: 4x4 transformation matrix from current time to next (T_2_1)
"""
def parse(gps_line):
out = [float(x) for x in gps_line.split(',')]
out[0] = int(gps_line.split(',')[0])
return out
gtfound = False
min_delta = 0.1
T_2_1 = np.identity(4, dtype=np.float32)
with open(gt_path, 'r') as f:
f.readline()
lines = f.readlines()
for i in range(len(lines) - 1):
gt1 = parse(lines[i])
delta = abs(float(gt1[0] - radar_time) / 1.0e9)
if delta < min_delta:
gt2 = parse(lines[i + 1])
T_enu_r1 = np.matmul(get_transform_boreas(gt1), T_prime)
T_enu_r2 = np.matmul(get_transform_boreas(gt2), T_prime)
T_r2_r1 = np.matmul(get_inverse_tf(T_enu_r2),
T_enu_r1) # 4x4 SE(3)
heading, _, _ = rotToYawPitchRoll(T_r2_r1[0:3, 0:3])
T_2_1 = get_transform(T_r2_r1[0, 3], T_r2_r1[1, 3],
heading) # 4x4 SE(2)
min_delta = delta
gtfound = True
assert (
gtfound), 'ground truth transform for {} not found in {}'.format(
radar_time, gt_path)
return T_2_1
def augmentBatch(batch, config):
"""Rotates the cartesian radar image by a random amount, adjusts the ground truth transform accordingly."""
rot_max = config['augmentation']['rot_max']
batch_size = config['batch_size']
window_size = config['window_size']
data = batch['data'].numpy()
mask = batch['mask'].numpy()
T_21 = batch['T_21'].numpy()
_, C, H, W = data.shape
for i in range(batch_size):
rot = np.random.uniform(-rot_max, rot_max)
T = get_transform(0, 0, rot)
for j in range(1, window_size):
k = j + i * window_size
img = data[k].squeeze()
mmg = mask[k].squeeze()
M = cv2.getRotationMatrix2D((W / 2, H / 2), rot * 180 * j / np.pi, 1.0)
data[k] = cv2.warpAffine(img, M, (W, H), flags=cv2.INTER_CUBIC).reshape(C, H, W)
mask[k] = cv2.warpAffine(mmg, M, (W, H), flags=cv2.INTER_CUBIC).reshape(1, H, W)
T_21[k - 1] = np.matmul(T, T_21[k - 1])
batch['data'] = torch.from_numpy(data)
batch['mask'] = torch.from_numpy(mask > 0.5).type(batch['data'].dtype) # make into a binary mask
batch['T_21'] = torch.from_numpy(T_21)
return batch
def main(model_name):
##################################
# Initialize saving directory
##################################
if not os.path.exists(config.save_dir):
os.makedirs(config.save_dir)
##################################
# Logging setting
##################################
logging.basicConfig(
stream=sys.stdout,
level=logging.INFO, # 设置输出级别
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
##################################
# Load dataset TODO: 10-fold cross validation
##################################
train_dataset = FGVC7Data(root=args.datasets,
phase='train',
transform=get_transform(config.image_size,
'train'))
indices = range(len(train_dataset))
split = int(0.3 * len(train_dataset))
train_indices = indices[split:]
test_indices = indices[:split]
#train_sampler = SubsetRandomSampler(train_indices)
valid_sampler = SubsetRandomSampler(test_indices)
train_loader = DataLoader(train_dataset,
batch_size=config.batch_size,
shuffle=True,
num_workers=config.workers,
pin_memory=True)
validate_loader = DataLoader(train_dataset,
batch_size=config.batch_size,
sampler=valid_sampler,
num_workers=config.workers,
pin_memory=True)
num_classes = 4
print('Train Size: {}'.format(len(train_indices)))
print('Valid Size: {}'.format(len(test_indices)))
##################################
# Initialize model
##################################
logs = {} # 有 lr, epoch, val_loss
start_epoch = 0
num_classes = 5
net = efficientnet(net_name=model_name,
num_classes=num_classes,
weight_path='github')
if config.ckpt:
# Load ckpt and get state_dict
checkpoint = torch.load(config.ckpt)
# Get epoch and some logs
logs = checkpoint['logs']
start_epoch = int(logs['epoch'])
# Load weights
state_dict = checkpoint['state_dict']
net.load_state_dict(state_dict)
logging.info('Network loaded from {}'.format(config.ckpt))
#net.re_init()
logging.info('Network weights save to {}'.format(config.save_dir))
##################################
# Use cuda
##################################
net.to(device)
if torch.cuda.device_count() > 1:
net = nn.DataParallel(net)
##################################
# Optimizer, LR Schedulerextract_features(img)
##################################
learning_rate = logs['lr'] if 'lr' in logs else config.learning_rate
#optimizer = torch.optim.SGD(net.parameters(), lr=learning_rate, momentum=0.9, weight_decay=1e-5)
optimizer = torch.optim.AdamW(net.parameters(),
lr=learning_rate,
amsgrad=True)
#scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.9, patience=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,
config.epochs,
eta_min=1e-6)
##################################
# ModelCheckpoint
##################################
callback_monitor = 'val_{}'.format(raw_metric.name) # topk_accuracy
callback = ModelCheckpoint(savepath=os.path.join(config.save_dir,
config.model_name),
monitor=callback_monitor,
mode='max')
if callback_monitor in logs:
callback.set_best_score(logs[callback_monitor])
else:
callback.reset()
##################################
# TRAINING
##################################
logging.info(
'Start training: Total epochs: {}, Batch size: {}, Training size: {}, Validation size: {}'
.format(config.epochs, config.batch_size, len(train_indices),
len(test_indices)))
logging.info('')
for epoch in range(start_epoch, config.epochs):
callback.on_epoch_begin()
logs['epoch'] = epoch + 1
logs['lr'] = optimizer.param_groups[0]['lr']
logging.info('Epoch {:03d}, LR {:g}'.format(
epoch + 1, optimizer.param_groups[0]['lr']))
# 每一个epoch都显示一个进度条
pbar = tqdm(total=len(train_loader), unit='batches') # unit 表示迭代速度的单位
pbar.set_description('Epoch {}/{}'.format(epoch + 1, config.epochs))
train(logs=logs,
data_loader=train_loader,
net=net,
optimizer=optimizer,
pbar=pbar)
validate(logs=logs, data_loader=validate_loader, net=net, pbar=pbar)
if isinstance(scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau):
scheduler.step(logs['val_loss'])
else:
scheduler.step(epoch)
callback.on_epoch_end(logs, net)
pbar.close()
self.transform = transform
def __getitem__(self, item):
id = self.ids[item]
img_name = self.images_name[item]
image = cv2.imread(id) # (C, H, W)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
label = self.labels[item] if self.phase == 'train' else 0
if self.transform != None:
image, _ = self.transform(image, label)
return image, label, img_name
if __name__ == '__main__':
import sys
sys.path.append(os.path.abspath('./'))
from utils.utils import get_transform
data_path = r'F:\competion\6\cassava-leaf-disease-classification'
# data_path = r'/home/gongke/data/cassava-leaf-disease-classification'
# data_path = r'../data/cassava-leaf-disease-classification'
dataset = FGVC7Data(data_path,
transform=get_transform((448, 448), 'train'),
phase='test')
from torch.utils.data import DataLoader
loader = DataLoader(dataset, batch_size=1)
d = []
for i, input in enumerate(loader):
x, _, y = input
d.append(y[0])
sub = pd.DataFrame({'image_id': d})
print(sub)