def __getitem__(self, seq_id):
# uniformly choosing video frames
seq_name = self._seqs[seq_id]['seq_name']
start_frame = self._seqs[seq_id]['start_frame']
end_frame = self._seqs[seq_id]['end_frame']
bboxes = self._seqs[seq_id]['gt_bboxes']
visible = self._seqs[seq_id]['visible']
visible_frame_idxes = np.arange(0, end_frame - start_frame + 1)[visible]
seq_len = len(visible_frame_idxes)
clip_len = config.look_ahead * config.time_step + 1
assert seq_len >= clip_len
start_idx = np.random.randint(seq_len - clip_len + 1) if self.is_train else 0
selected_idxes = [visible_frame_idxes[idx] for idx in range(start_idx, start_idx + clip_len)]
# build training examples for initial frame
patches, label_maps, gt_bboxes = [], [], []
img_path = self._get_img_path(seq_name, start_frame, selected_idxes[0])
init_image = default_loader(img_path)
init_bbox = np.array(bboxes[selected_idxes[0]])
for ratio in config.aug_init_ratios:
for scale in config.aug_init_scales:
# aspect ratio augmentation
height, width = init_image.shape[0: 2]
sw, sh = int(width * ratio), int(height / ratio)
image_resized = cv2.resize(init_image, (sw, sh))
bbox_reiszed = init_bbox * np.array([ratio, 1 / ratio, ratio, 1 / ratio])
# scale changes augmentation
search_scale = config.search_scale / scale
# generate training examples
patch, label_map, bbox_on_patch = self._generate_training_examples(image_resized, bbox_reiszed, search_scale)
patches.append(patch)
label_maps.append(label_map)
gt_bboxes.append(bbox_on_patch)
# build training examples for subsequent frames.
for i, idx in enumerate(selected_idxes[1:]):
img_path = self._get_img_path(seq_name, start_frame, idx)
image = default_loader(img_path)
bbox = np.array(bboxes[idx])
# aspect ratio augmentation
height, width = image.shape[0: 2]
ratio = np.random.uniform(config.aug_ratios_range[0], config.aug_ratios_range[1])
sw, sh = int(width * ratio), int(height / ratio)
image = cv2.resize(image, (sw, sh))
bbox = bbox * np.array([ratio, 1 / ratio, ratio, 1 / ratio])
# scale changes augmentation
obj_scale = np.random.uniform(config.aug_scales_range[0], config.aug_scales_range[1])
search_scale = config.search_scale/obj_scale
# generate training examples
patch, label_map, bbox_on_patch = self._generate_training_examples(image, bbox, search_scale)
patches.append(patch)
label_maps.append(label_map)
gt_bboxes.append(bbox_on_patch)
patches = torch.stack(patches, 0)
label_maps = torch.stack(label_maps, 0)
gt_bboxes = torch.stack(gt_bboxes, 0)
return patches, label_maps, gt_bboxes
def __getitem__(self, index):
data, target = self.data[index], self.targets[index]
if self.name == 'mnist':
data = Image.fromarray(data.numpy(), mode='L')
data = data.convert('RGB')
elif self.name == 'svhn':
data = Image.fromarray(np.transpose(data, (1, 2, 0)))
else:
data = utils.default_loader(self.data[index])
rand_aug_lst = [self.rand_aug_transforms(data) for _ in range(self.committee_size)]
return (self.transforms(data), self.base_transforms(data), rand_aug_lst), int(target), int(index)
def main(args):
model = resnet50
model = model.cuda()
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
if args.mode == 'train':
print('load train data')
file_list = os.listdir('./train/train_data/train_image')
file_list_a = glob.glob(f'{DATASET_PATH}/*')
print('file_list: ', file_list_a)
else:
print('load test data')
file_list = os.listdir('./test/test_data/test_image')
start_time = datetime.datetime.now()
print('start extracting...!')
y_pred_dict = {}
for fname in file_list:
img_name = os.path.join(DATASET_PATH, args.mode, args.mode + '_data',
args.mode + '_image', fname)
image = utils.default_loader(img_name)
data_transforms = utils.get_transforms(
'[transforms.Resize((456, 232))]', verbose=False)
image = data_transforms['train'](image)
image = image.unsqueeze(0)
image = image.cuda()
# forward
logits = model(image)
y_pred_dict[fname[:-4]] = logits.cpu().squeeze().numpy()
if len(y_pred_dict) % 100 == 0:
print('current stack size : ', len(y_pred_dict),
round(len(y_pred_dict) / len(file_list), 2) * 100, '%')
print('extraction is done')
dict_save_name = args.mode + '_image_features_0.pkl'
with open(dict_save_name, 'wb') as handle:
pickle.dump(y_pred_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
print('done')
def resnet_feature_extractor(phase):
resnet50 = models.resnet50(pretrained=True)
modules = list(resnet50.children())[:-1]
resnet50 = nn.Sequential(*modules)
model = resnet50
model = model.cuda()
#optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
for p in resnet50.parameters():
p.requires_grad = False
if phase == 'train':
print('load train data')
file_list = os.listdir(DATASET_PATH + '/train/train_data/train_image')
else:
print('load test data')
file_list = os.listdir(DATASET_PATH + '/test/test_data/test_image')
print('start extracting...!')
y_pred_dict = {}
for fname in file_list:
img_name = os.path.join(DATASET_PATH, phase, phase + '_data',
phase + '_image', fname)
image = utils.default_loader(img_name)
data_transforms = utils.get_transforms(
'[transforms.Resize((456, 232))]', verbose=False)
image = data_transforms['train'](image)
image = image.unsqueeze(0)
image = image.cuda()
# forward
logits = model(image)
y_pred_dict[fname[:-4]] = logits.cpu().squeeze().numpy()
if len(y_pred_dict) % 100 == 0:
print('current stack size : ', len(y_pred_dict),
round(len(y_pred_dict) / len(file_list), 2) * 100, '%')
print('extraction is done')
dict_save_name = phase + '_image_features_50.pkl'
with open(dict_save_name, 'wb') as handle:
pickle.dump(y_pred_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
print('done')
def __getitem__(self, idx):
article_id, hh, gender, age_range, read_article_ids = \
self.item.loc[idx, ['article_id', 'hh', 'gender', 'age_range', 'read_article_ids']]
if self.args['mode'] == 'train':
label = self.label.loc[idx, ['label']]
label = np.array(label, dtype=np.float32)
else:
# pseudo label for test mode
label = np.array(0, dtype=np.float32)
if nsml.IS_ON_NSML:
if self.args['mode'] == 'train':
img_name = os.path.join(self.root_dir, article_id + '.jpg')
else: # test, infer
img_name = os.path.join(self.root_dir, article_id + '.jpg')
else:
# on local machine
if self.args['mode'] == 'train':
img_name = os.path.join(DATASET_PATH, 'train/train_data/train_image/', article_id + '.jpg')
else:
img_name = os.path.join(DATASET_PATH, 'test/test_data/test_image/', article_id + '.jpg')
image = default_loader(img_name)
extracted_image_feature = self.image_feature_dict[article_id]
if self.transform:
image = self.transform['train'](image)
# Additional info for feeding FC layer
flat_features = []
if self.args['use_sex']:
sex = self.sex[gender]
label_onehot = np.zeros(2, dtype=np.float32)
label_onehot[sex - 1] = 1
flat_features.extend(label_onehot)
if self.args['use_age']:
age = self.age[age_range]
label_onehot = np.zeros(9, dtype=np.float32)
label_onehot[age - 1] = 1
flat_features.extend(label_onehot)
if self.args['use_exposed_time']:
time = hh
label_onehot = np.zeros((24), dtype=np.float32)
label_onehot[time - 1] = 1
flat_features.extend(label_onehot)
if self.args['use_read_history']:
history = str(read_article_ids)
#max length of train set is 2427, test set is 947...
max_length = 512
sequence = np.zeros((max_length), dtype=np.float32) # Max length of history = 2427
result = [x.strip() for x in history.split(',')]
# Slice the history (of max_length=2427 to max_length=512))
if len(result) >= max_length:
for i, data in enumerate(result[:max_length]):
sequence[i] = self.word2idx[data]
else: # len(result) < max_legnth
for i, data in enumerate(result):
sequence[i] = self.word2idx[data]
flat_features = np.array(flat_features).flatten()
# pytorch dataloader doesn't accept empty np array
if flat_features.shape[0] == 0:
raise NotImplementedError('no flat feature processed. is this on purpose? then delete this line')
flat_features = np.zeros(1, dtype=np.float32)
# hint: flat features are concatened into a Tensor, because I wanted to put them all into computational model,
# hint: if it is not what you wanted, then change the last return line
return image, extracted_image_feature, label, flat_features, sequence
def __getitem__(self, idx):
article_id, hh, gender, age_range, read_article_ids = \
self.item.loc[idx, ['article_id', 'hh', 'gender', 'age_range', 'read_article_ids']]
if self.args['mode'] == 'train':
label = self.label.loc[idx, ['label']]
label = np.array(label, dtype=np.float32)
else:
# pseudo label for test mode
label = np.array(0, dtype=np.float32)
if nsml.IS_ON_NSML:
if self.args['mode'] == 'train':
img_name = os.path.join(self.root_dir, article_id + '.jpg')
else: # test, infer
img_name = os.path.join(self.root_dir, article_id + '.jpg')
else:
# on local machine
if self.args['mode'] == 'train':
img_name = os.path.join(DATASET_PATH,
'train/train_data/train_image/',
article_id + '.jpg')
else:
img_name = os.path.join(DATASET_PATH,
'test/test_data/test_image/',
article_id + '.jpg')
image = default_loader(img_name)
extracted_image_feature = self.image_feature_dict[article_id]
if self.transform:
image = self.transform['train'](image)
# Additional info for feeding FC layer
flat_features = []
if self.args['use_sex']:
sex = self.sex[gender]
label_onehot = np.zeros(2, dtype=np.float32)
label_onehot[sex - 1] = 1
flat_features.extend(label_onehot)
if self.args['use_age']:
age = self.age[age_range]
label_onehot = np.zeros(9, dtype=np.float32)
label_onehot[age - 1] = 1
flat_features.extend(label_onehot)
if self.args['use_exposed_time']:
time = hh
label_onehot = np.zeros((24), dtype=np.float32)
label_onehot[time - 1] = 1
flat_features.extend(label_onehot)
if self.args['use_read_history']:
raise NotImplementedError(
'If you can handle "sequential" data, then.. hint: this helps a lot'
)
flat_features = np.array(flat_features).flatten()
# pytorch dataloader doesn't accept empty np array
if flat_features.shape[0] == 0:
raise NotImplementedError(
'no flat feature processed. is this on purpose? then delete this line'
)
flat_features = np.zeros(1, dtype=np.float32)
# hint: flat features are concatened into a Tensor, because I wanted to put them all into computational model,
# hint: if it is not what you wanted, then change the last return line
return image, extracted_image_feature, label, flat_features