def set_train_augment(self, augment_cfg: CN, input_size, img_path_sampler):
self.train_augment = augment.Compose([
augment.RandomHFlip(p=augment_cfg.hflip_p),
augment.RandomVFlip(p=augment_cfg.vflip_p),
augment.RandomSafeCrop(p=augment_cfg.crop_p),
augment.ColorJitter(
brightness=[-0.1, 0.1],
contrast=[0.8, 1.2],
saturation=[0.1, 2],
p=augment_cfg.color_p,
),
augment.Resize(input_size),
augment.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
])
sampler = lambda: super(VOCSampleGetter, self).train(img_path_sampler(
))
self.compose_augment = augment.Compose([
augment.Mixup(sampler, p=augment_cfg.mixup_p, beta=1.5),
augment.ToTensor('cpu'),
])
# aug2 = augment.Compose([
# augment.Mosaic(sampler, p=1, size=input_size),
# augment.Mixup(sampler, p=0, beta=1.5),
# augment.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
# augment.ToTensor('cpu'),
# ])
return self
def eval_augment_voc(input_size, device):
return augment.Compose([
augment.Resize(input_size),
augment.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
augment.ToTensor(device),
])
def eval_augment_visdrone(_, device):
return augment.Compose([
augment.ResizeRatio(1.25),
augment.PadNearestDivisor(),
augment.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
augment.ToTensor(device),
])
def benchmark_onnx(config, args):
torch.backends.cudnn.benchmark = True
with open(config.dataset.eval_txt_file, 'r') as fr:
files = [
line.strip() for line in fr.readlines() if len(line.strip()) != 0
][:100]
model = onnx_model_for_benchmark(args.onnx, args.device == 'cuda')
size = size_fix(args.size)
# warm up
print('warmimg up')
for _ in range(50):
model(np.random.randn(1, 3, *size).astype(np.float32))
process = augment.Compose([
augment.Resize(size),
augment.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
augment.HWCtoCHW(),
])
print('loading images')
images = _prepare_images(files, process)
total_timer = tools.TicToc('TOTAL')
forward_timer = tools.TicToc('FORWARD')
convert_timer = tools.TicToc('CONVERT')
nms_timer = tools.TicToc('NMS')
input_shape = torch.FloatTensor(size).to(args.device, non_blocking=False)
threshold = args.threshold
nms_iou = args.nms_iou
for image, shape in tqdm(images):
image = image[None, ...]
shape = shape[None, ...].to(args.device)
total_timer.tic()
forward_timer.tic()
pred = model(image)
forward_timer.toc()
convert_timer.tic()
bboxes = RECOVER_BBOXES_REGISTER[args.dataset](
torch.from_numpy(pred).to(args.device), input_shape, shape)[0]
convert_timer.toc()
nms_timer.tic()
tools.torch_nms(bboxes, threshold, nms_iou)
nms_timer.toc()
total_timer.toc()
print('BENCHMARK: {} images in {} size'.format(len(images), size))
timers = [total_timer, forward_timer, convert_timer, nms_timer]
stats = [t.statistics() for t in timers]
for s in stats:
s['percent'] = s['mean'] / stats[0]['mean']
_print_statistics(s)
def process_sample(img, bboxes: np.ndarray):
process = augment.Compose([
augment.RandomHFlip(p=0.5),
augment.RandomSafeCrop(p=0.75),
augment.CutOut(p=0.8, size=30, n_holes=3),
augment.ColorJitter(
brightness=[-0.1, 0.1],
contrast=[0.8, 1.2],
saturation=[0.1, 2],
),
augment.Resize(lambda: [512, 512]),
augment.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
])
img, bboxes = process(img, bboxes)
return img, bboxes
def main(resume, n_epoch, learn_rate):
model_name = config['param']['model']
c = config['train']
batch_size = c.getint('n_batch')
n_worker = c.getint('n_worker')
n_ckpt_epoch = c.getint('n_ckpt_epoch')
iter_accum = c.getint('iter_accum')
cv_ratio = c.getfloat('cv_ratio')
cv_seed = c.getint('cv_seed')
data_src = json.loads(c.get('data_src'))
data_major = json.loads(c.get('data_major'))
data_sub = json.loads(c.get('data_sub'))
out_dir = TASK_OUTDIR
# initial_checkpoint = INITIAL_CP_FILE
# RESULTS_DIR + '/mask-rcnn-50-gray500-02/checkpoint/00014500_model.pth'
##
'''
# pretrain_file = PRETRAIN_FILE
# None #RESULTS_DIR + '/mask-single-shot-dummy-1a/checkpoint/00028000_model.pth'
skip = ['crop','mask']
'''
## setup -----------------
os.makedirs(out_dir, exist_ok=True, mode=0o777)
os.makedirs(out_dir + '/checkpoint', exist_ok=True)
os.makedirs(out_dir + '/train', exist_ok=True)
# os.makedirs(out_dir +'/backup', exist_ok=True)
# backup_project_as_zip(PROJECT_PATH, out_dir +'/backup/code.train.%s.zip'%IDENTIFIER)
log = Logger()
log.open(out_dir + '/log.train.txt', mode='a')
log.write('\n--- [START %s] %s\n\n' % (IDENTIFIER, '-' * 64))
log.write('** some experiment setting **\n')
log.write('\tSEED = %u\n' % SEED)
log.write('\tPROJECT_PATH = %s\n' % PROJECT_PATH)
log.write('\tout_dir = %s\n' % out_dir)
log.write('\n')
## net ----------------------
log.write('** net setting **\n')
cfg = Configuration()
net = MaskRcnnNet(cfg).cuda()
log.write('Classes map : %s\n' % (cfg.classes_map))
log.write('num_classes : %s\n' % (cfg.num_classes))
log.write('rpn single class : %s\n' % (cfg.rpn_single_class))
log.write('rpn using only P0 layer : %s\n' % (cfg.rpn_p0_pool))
log.write('Crop using only P0 layer : %s\n' % (cfg.crop_one_layer))
log.write('Increase feature resolution : %s\n' % (cfg.high_resolution))
## optimiser ----------------------------------
# LR = None #LR = StepLR([ (0, 0.01), (200, 0.001), (300, -1)])
optimizer = optim.SGD(filter(lambda p: p.requires_grad, net.parameters()),
lr=learn_rate / iter_accum,
momentum=0.9,
weight_decay=0.0001)
# resume checkpoint
start_iter, start_epoch = 0, 0
if resume:
rate = get_learning_rate(optimizer) #load all except learning rate
start_iter, start_epoch = load_ckpt(out_dir, net, optimizer)
'''
if os.path.isfile(initial_checkpoint):
log.write('\tinitial_checkpoint = %s\n' % initial_checkpoint)
net.load_state_dict(torch.load(initial_checkpoint, map_location=lambda storage, loc: storage))
#with open(out_dir +'/checkpoint/configuration.pkl','rb') as pickle_file:
# cfg = pickle.load(pickle_file)
checkpoint = torch.load(initial_checkpoint.replace('_model.pth','_optimizer.pth'))
start_iter = checkpoint['iter' ]
start_epoch = checkpoint['epoch']
rate = get_learning_rate(optimizer) #load all except learning rate
optimizer.load_state_dict(checkpoint['optimizer'])
adjust_learning_rate(optimizer, rate)
'''
if start_epoch == 0:
print('Grand new training ...')
else:
adjust_learning_rate(optimizer, rate)
log.write('%s\n\n' % (type(net)))
log.write('%s\n' % (net.version))
log.write('\n')
## dataset ----------------------------------------
log.write('** dataset setting **\n')
# Read Excel file
# input_path = EXCEL_PATH
# output_csv_path = COMMENT_CSV_PATH
# sheet_name = Train_CSV_SHEET
# csv_from_excel(input_path, output_csv_path, sheet_name)
# runs the csv_from_excel function:
comment_csv = pd.read_csv(COMMENT_CSV_PATH)
df = comment_csv
if 'all' not in data_src:
df = df[df['source'].isin(data_src)]
if 'all' not in data_major:
df = df[df['major_category'].isin(data_major)]
if 'all' not in data_sub:
df = df[df['sub_category'].isin(data_sub)]
comment_csv = df.reset_index()
# comment_csv = comment_csv[comment_csv['category']=='Cloud'].reset_index()
# split train/validation
comment_csv = comment_csv.sample(
frac=1, random_state=cv_seed).reset_index(drop=True)
split = int(np.floor(cv_ratio * comment_csv.shape[0]))
comment_train_csv = comment_csv[split:].reset_index(drop=True)
comment_valid_csv = comment_csv[:split]
train_dataset = ScienceDataset(
comment_train_csv,
COMMENT_CSV_PATH,
mode='train',
# comment_csv , mode='train',
# 'train1_ids_gray2_500', mode='train',
#'debug1_ids_gray_only_10', mode='train',
#'disk0_ids_dummy_9', mode='train', #12
#'train1_ids_purple_only1_101', mode='train', #12
#'merge1_1', mode='train',
transform=augment.Compose()) # augment.train_augment
train_loader = DataLoader(
train_dataset,
sampler=RandomSampler(train_dataset),
batch_size=batch_size,
drop_last=True,
num_workers=n_worker,
pin_memory=True, # torch.cuda.is_available()
collate_fn=train_collate)
valid_dataset = valid_loader = None
if len(comment_valid_csv) > 0:
valid_dataset = ScienceDataset(
comment_valid_csv,
COMMENT_CSV_PATH,
mode='train',
# 'valid1_ids_gray2_43', mode='train',
#'debug1_ids_gray_only_10', mode='train',
#'disk0_ids_dummy_9', mode='train',
#'train1_ids_purple_only1_101', mode='train', #12
#'merge1_1', mode='train',
transform=augment.Compose()) # augment.valid_augment
valid_loader = DataLoader(
valid_dataset,
sampler=SequentialSampler(valid_dataset),
batch_size=batch_size,
drop_last=False,
num_workers=n_worker,
pin_memory=True, # torch.cuda.is_available()
collate_fn=train_collate)
log.write('\tWIDTH, HEIGHT = %d, %d\n' % (augment.WIDTH, augment.HEIGHT))
log.write('\ttrain_dataset.split = %s\n' % (train_dataset.comment_path))
log.write('\tlen(train_dataset) = %d\n' % (len(train_dataset)))
log.write('\tlen(train_loader) = %d\n' % (len(train_loader)))
if valid_dataset is not None:
log.write('\tvalid_dataset.split = %s\n' %
(valid_dataset.comment_path))
log.write('\tlen(valid_dataset) = %d\n' % (len(valid_dataset)))
log.write('\tlen(valid_loader) = %d\n' % (len(valid_loader)))
else:
log.write('\t!!! No valid dataset !!!\n')
log.write('\tn_epoch = %d\n' % (n_epoch))
log.write('\tbatch_size = %d\n' % (batch_size))
log.write('\titer_accum = %d\n' % (iter_accum))
log.write('\tbatch_size*iter_accum = %d\n' % (batch_size * iter_accum))
log.write('\tn_ckpt_epoch = %d\n' % (n_ckpt_epoch))
log.write('\n')
# decide log directory name
txlog_dir = os.path.join(
out_dir, 'logs', '{}-{}'.format(model_name, augment.WIDTH),
'ep_{},{}-lr_{}'.format(
start_epoch,
n_epoch + start_epoch,
learn_rate,
))
with SummaryWriter(txlog_dir) as txwriter:
print('Training started...')
run_train((train_loader, valid_loader), net, optimizer,
(start_iter, iter_accum),
(start_epoch, n_epoch, n_ckpt_epoch),
(len(train_dataset),
len(valid_dataset) if valid_dataset is not None else 0,
batch_size), log, txwriter)
print('Training finished...')
def benchmark(config, args):
torch.backends.cudnn.benchmark = True
with open(config.dataset.eval_txt_file, 'r') as fr:
files = [
line.strip() for line in fr.readlines() if len(line.strip()) != 0
][:100]
model = tools.build_model(
config.model.cfg_path,
args.weight,
None,
device=args.device,
dataparallel=False,
qat=args.qat,
quantized=args.quant,
backend=args.backend,
)[0]
size = size_fix(args.size)
process = augment.Compose([
augment.Resize(size),
augment.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
augment.ToTensor(args.device),
])
print('loading images')
images = _prepare_images(files, process)
total_timer = tools.TicToc('TOTAL')
forward_timer = tools.TicToc('FORWARD')
convert_timer = tools.TicToc('CONVERT')
nms_timer = tools.TicToc('NMS')
input_shape = torch.FloatTensor(size).to(args.device, non_blocking=False)
threshold = args.threshold
nms_iou = args.nms_iou
# warm up
print('warmimg up')
with torch.no_grad():
for _ in range(5):
model(torch.rand(1, 3, *size).to(args.device))
torch.cuda.synchronize()
for image, shape in tqdm(images):
image = image.unsqueeze_(0).to(args.device, non_blocking=False)
shape = shape.unsqueeze_(0).to(args.device, non_blocking=False)
total_timer.tic()
forward_timer.tic()
pred = model(image)
torch.cuda.synchronize()
forward_timer.toc()
convert_timer.tic()
bboxes = RECOVER_BBOXES_REGISTER[args.dataset](pred, input_shape,
shape)[0]
torch.cuda.synchronize()
convert_timer.toc()
nms_timer.tic()
tools.torch_nms(bboxes, threshold, nms_iou)
torch.cuda.synchronize()
nms_timer.toc()
total_timer.toc()
print('BENCHMARK: {} images in {} size'.format(len(images), size))
timers = [total_timer, forward_timer, convert_timer, nms_timer]
stats = [t.statistics() for t in timers]
for s in stats:
s['percent'] = s['mean'] / stats[0]['mean']
_print_statistics(s)
