def read_train_data(cfg, train_indexes, train_steps, train_records, batch_size, _iter_id, train_dic, use_gpu, context, with_mixup, sample_transforms, batch_transforms, target_num): iter_id = _iter_id num_threads = cfg.train_cfg['num_threads'] while True: # 无限个epoch # 每个epoch之前洗乱 np.random.shuffle(train_indexes) for step in range(train_steps): iter_id += 1 key_list = list(train_dic.keys()) key_len = len(key_list) while key_len >= cfg.train_cfg['max_batch']: time.sleep(0.01) key_list = list(train_dic.keys()) key_len = len(key_list) # ==================== train ==================== sizes = cfg.randomShape['sizes'] shape = np.random.choice(sizes) images = [None] * batch_size gt_bbox = [None] * batch_size gt_score = [None] * batch_size gt_class = [None] * batch_size target0 = [None] * batch_size target1 = [None] * batch_size target2 = [None] * batch_size samples = get_samples(train_records, train_indexes, step, batch_size, with_mixup) # sample_transforms用多线程 threads = [] for i in range(num_threads): t = threading.Thread( target=multi_thread_op, args=(i, num_threads, batch_size, samples, context, with_mixup, sample_transforms, batch_transforms, shape, images, gt_bbox, gt_score, gt_class, target0, target1, target2, target_num)) threads.append(t) t.start() # 等待所有线程任务结束。 for t in threads: t.join() images = np.concatenate(images, 0) gt_bbox = np.concatenate(gt_bbox, 0) gt_score = np.concatenate(gt_score, 0) gt_class = np.concatenate(gt_class, 0) target0 = np.concatenate(target0, 0) target1 = np.concatenate(target1, 0) if target_num > 2: target2 = np.concatenate(target2, 0) images = torch.Tensor(images) gt_bbox = torch.Tensor(gt_bbox) gt_score = torch.Tensor(gt_score) gt_class = torch.Tensor(gt_class) target0 = torch.Tensor(target0) target1 = torch.Tensor(target1) if target_num > 2: target2 = torch.Tensor(target2) if use_gpu: images = images.cuda() gt_bbox = gt_bbox.cuda() gt_score = gt_score.cuda() gt_class = gt_class.cuda() target0 = target0.cuda() target1 = target1.cuda() if target_num > 2: target2 = target2.cuda() dic = {} dic['images'] = images dic['gt_bbox'] = gt_bbox dic['gt_score'] = gt_score dic['gt_class'] = gt_class dic['target0'] = target0 dic['target1'] = target1 if target_num > 2: dic['target2'] = target2 train_dic['%.8d' % iter_id] = dic # ==================== exit ==================== if iter_id == cfg.train_cfg['max_iters']: return 0
def read_train_data(cfg, train_indexes, train_steps, train_records, batch_size, _iter_id, train_dic, use_gpu, n_features, context, with_mixup, sample_transforms, batch_transforms): iter_id = _iter_id num_threads = cfg.train_cfg['num_threads'] while True: # 无限个epoch # 每个epoch之前洗乱 np.random.shuffle(train_indexes) for step in range(train_steps): iter_id += 1 key_list = list(train_dic.keys()) key_len = len(key_list) while key_len >= cfg.train_cfg['max_batch']: time.sleep(0.01) key_list = list(train_dic.keys()) key_len = len(key_list) # ==================== train ==================== batch_images = [None] * batch_size batch_labels0 = [None] * batch_size batch_reg_target0 = [None] * batch_size batch_centerness0 = [None] * batch_size batch_labels1 = [None] * batch_size batch_reg_target1 = [None] * batch_size batch_centerness1 = [None] * batch_size batch_labels2 = [None] * batch_size batch_reg_target2 = [None] * batch_size batch_centerness2 = [None] * batch_size batch_labels3 = [None] * batch_size batch_reg_target3 = [None] * batch_size batch_centerness3 = [None] * batch_size batch_labels4 = [None] * batch_size batch_reg_target4 = [None] * batch_size batch_centerness4 = [None] * batch_size samples = get_samples(train_records, train_indexes, step, batch_size, with_mixup) # sample_transforms用多线程 threads = [] for i in range(num_threads): t = threading.Thread(target=multi_thread_op, args=(i, num_threads, batch_size, samples, context, with_mixup, sample_transforms)) threads.append(t) t.start() # 等待所有线程任务结束。 for t in threads: t.join() # batch_transforms。需要先同步PadBatch coarsest_stride = cfg.padBatch['pad_to_stride'] max_shape = np.array([data['image'].shape for data in samples]).max( axis=0) # max_shape=[3, max_h, max_w] max_shape[1] = int( # max_h增加到最小的能被coarsest_stride=128整除的数 np.ceil(max_shape[1] / coarsest_stride) * coarsest_stride) max_shape[2] = int( # max_w增加到最小的能被coarsest_stride=128整除的数 np.ceil(max_shape[2] / coarsest_stride) * coarsest_stride) threads = [] for i in range(num_threads): t = threading.Thread(target=multi_thread_op_batch_transforms, args=(i, num_threads, batch_size, samples, context, batch_transforms, max_shape, batch_images, batch_labels0, batch_reg_target0, batch_centerness0, batch_labels1, batch_reg_target1, batch_centerness1, batch_labels2, batch_reg_target2, batch_centerness2, batch_labels3, batch_reg_target3, batch_centerness3, batch_labels4, batch_reg_target4, batch_centerness4, n_features)) threads.append(t) t.start() # 等待所有线程任务结束。 for t in threads: t.join() batch_images = np.concatenate(batch_images, 0) batch_labels0 = np.concatenate(batch_labels0, 0) batch_reg_target0 = np.concatenate(batch_reg_target0, 0) batch_centerness0 = np.concatenate(batch_centerness0, 0) batch_labels1 = np.concatenate(batch_labels1, 0) batch_reg_target1 = np.concatenate(batch_reg_target1, 0) batch_centerness1 = np.concatenate(batch_centerness1, 0) batch_labels2 = np.concatenate(batch_labels2, 0) batch_reg_target2 = np.concatenate(batch_reg_target2, 0) batch_centerness2 = np.concatenate(batch_centerness2, 0) if n_features == 5: batch_labels3 = np.concatenate(batch_labels3, 0) batch_reg_target3 = np.concatenate(batch_reg_target3, 0) batch_centerness3 = np.concatenate(batch_centerness3, 0) batch_labels4 = np.concatenate(batch_labels4, 0) batch_reg_target4 = np.concatenate(batch_reg_target4, 0) batch_centerness4 = np.concatenate(batch_centerness4, 0) batch_images = torch.Tensor(batch_images) batch_labels0 = torch.Tensor(batch_labels0) batch_reg_target0 = torch.Tensor(batch_reg_target0) batch_centerness0 = torch.Tensor(batch_centerness0) batch_labels1 = torch.Tensor(batch_labels1) batch_reg_target1 = torch.Tensor(batch_reg_target1) batch_centerness1 = torch.Tensor(batch_centerness1) batch_labels2 = torch.Tensor(batch_labels2) batch_reg_target2 = torch.Tensor(batch_reg_target2) batch_centerness2 = torch.Tensor(batch_centerness2) if n_features == 5: batch_labels3 = torch.Tensor(batch_labels3) batch_reg_target3 = torch.Tensor(batch_reg_target3) batch_centerness3 = torch.Tensor(batch_centerness3) batch_labels4 = torch.Tensor(batch_labels4) batch_reg_target4 = torch.Tensor(batch_reg_target4) batch_centerness4 = torch.Tensor(batch_centerness4) if use_gpu: batch_images = batch_images.cuda() batch_labels0 = batch_labels0.cuda() batch_reg_target0 = batch_reg_target0.cuda() batch_centerness0 = batch_centerness0.cuda() batch_labels1 = batch_labels1.cuda() batch_reg_target1 = batch_reg_target1.cuda() batch_centerness1 = batch_centerness1.cuda() batch_labels2 = batch_labels2.cuda() batch_reg_target2 = batch_reg_target2.cuda() batch_centerness2 = batch_centerness2.cuda() if n_features == 5: batch_labels3 = batch_labels3.cuda() batch_reg_target3 = batch_reg_target3.cuda() batch_centerness3 = batch_centerness3.cuda() batch_labels4 = batch_labels4.cuda() batch_reg_target4 = batch_reg_target4.cuda() batch_centerness4 = batch_centerness4.cuda() dic = {} dic['batch_images'] = batch_images dic['batch_labels0'] = batch_labels0 dic['batch_reg_target0'] = batch_reg_target0 dic['batch_centerness0'] = batch_centerness0 dic['batch_labels1'] = batch_labels1 dic['batch_reg_target1'] = batch_reg_target1 dic['batch_centerness1'] = batch_centerness1 dic['batch_labels2'] = batch_labels2 dic['batch_reg_target2'] = batch_reg_target2 dic['batch_centerness2'] = batch_centerness2 if n_features == 5: dic['batch_labels3'] = batch_labels3 dic['batch_reg_target3'] = batch_reg_target3 dic['batch_centerness3'] = batch_centerness3 dic['batch_labels4'] = batch_labels4 dic['batch_reg_target4'] = batch_reg_target4 dic['batch_centerness4'] = batch_centerness4 train_dic['%.8d'%iter_id] = dic # ==================== exit ==================== if iter_id == cfg.train_cfg['max_iters']: return 0
while True: # 无限个epoch # 每个epoch之前洗乱 np.random.shuffle(train_indexes) for step in range(train_steps): iter_id += 1 # 估计剩余时间 start_time = end_time end_time = time.time() time_stat.append(end_time - start_time) time_cost = np.mean(time_stat) eta_sec = (cfg.max_iters - iter_id) * time_cost eta = str(datetime.timedelta(seconds=int(eta_sec))) # ==================== train ==================== samples = get_samples(train_records, train_indexes, step, batch_size, with_mixup) # sample_transforms用多线程 threads = [] for i in range(batch_size): t = threading.Thread( target=multi_thread_op, args=(i, samples, decodeImage, context, train_dataset, with_mixup, mixupImage, photometricDistort, randomCrop, randomFlipImage, padBox)) threads.append(t) t.start() # 等待所有线程任务结束。 for t in threads: t.join() # debug 看数据增强后的图片。由于有随机裁剪,所以有的物体掩码不完整。
def read_train_data(cfg, train_indexes, train_steps, train_records, batch_size, _iter_id, train_dic, use_gpu, n_layers, context, with_mixup, with_cutmix, with_mosaic, mixup_steps, cutmix_steps, mosaic_steps, sample_transforms, batch_transforms): iter_id = _iter_id num_threads = cfg.train_cfg['num_threads'] while True: # 无限个epoch # 每个epoch之前洗乱 np.random.shuffle(train_indexes) for step in range(train_steps): iter_id += 1 key_list = list(train_dic.keys()) key_len = len(key_list) while key_len >= cfg.train_cfg['max_batch']: time.sleep(0.01) key_list = list(train_dic.keys()) key_len = len(key_list) # ==================== train ==================== images = [None] * batch_size fg_nums = [None] * batch_size ins_labels = [None] * n_layers cate_labels = [None] * n_layers grid_orders = [None] * n_layers for idx in range(n_layers): ins_labels[idx] = [None] * batch_size cate_labels[idx] = [None] * batch_size grid_orders[idx] = [None] * batch_size samples = get_samples(train_records, train_indexes, step, batch_size, iter_id, with_mixup, with_cutmix, with_mosaic, mixup_steps, cutmix_steps, mosaic_steps) # sample_transforms用多线程 threads = [] for i in range(num_threads): t = threading.Thread(target=multi_thread_op, args=(i, num_threads, batch_size, samples, context, with_mixup, sample_transforms)) threads.append(t) t.start() # 等待所有线程任务结束。 for t in threads: t.join() # batch_transforms for batch_transform in batch_transforms: samples = batch_transform(samples, context) # 整理成ndarray for k in range(batch_size): images[k] = np.expand_dims(samples[k]['image'].astype(np.float32), 0) fg_nums[k] = np.expand_dims(samples[k]['fg_num'].astype(np.int32), 0) for idx in range(n_layers): ins_labels[idx][k] = samples[k]['ins_label%d'%idx].astype(np.int32) cate_labels[idx][k] = samples[k]['cate_label%d'%idx].astype(np.int32) grid_orders[idx][k] = np.reshape(samples[k]['grid_order%d'%idx].astype(np.int32), (-1, )) # lod信息 # lods = [None] * n_layers # for idx in range(n_layers): # lod = [0] # for k in range(batch_size): # l = len(grid_orders[idx][k]) # lod.append(l) # lods[idx] = lod # lods = np.array(lods).astype(np.int32) # lods = np.cumsum(lods, axis=1) images = np.concatenate(images, 0) fg_nums = np.concatenate(fg_nums, 0) for idx in range(n_layers): ins_labels[idx] = np.concatenate(ins_labels[idx], 0) cate_labels[idx] = np.concatenate(cate_labels[idx], 0) grid_orders[idx] = np.concatenate(grid_orders[idx], 0) images = paddle.to_tensor(images, place=place) fg_nums = paddle.to_tensor(fg_nums, place=place) for idx in range(n_layers): ins_labels[idx] = paddle.to_tensor(ins_labels[idx], place=place) cate_labels[idx] = paddle.to_tensor(cate_labels[idx], place=place) grid_orders[idx] = paddle.to_tensor(grid_orders[idx], place=place) dic = {} dic['images'] = images dic['fg_nums'] = fg_nums for idx in range(n_layers): dic['ins_label%d'%idx] = ins_labels[idx] dic['cate_label%d'%idx] = cate_labels[idx] dic['grid_order%d'%idx] = grid_orders[idx] train_dic['%.8d'%iter_id] = dic # ==================== exit ==================== if iter_id == cfg.train_cfg['max_iters']: return 0