def collate_train(data):
"""Creates mini-batch tensors from the list of quadruples
(image, questions, answers and qlengths).
We should build custom collate_fn rather than using default collate_fn,
because merging caption (including padding) is not supported in default.
Args:
data: list of tuple (image, caption).
- image: torch tensor of shape (3, 256, 256).
- caption: torch tensor of shape (?); variable length.
Returns:
images: torch tensor of shape (batch_size, 3, 256, 256).
targets: torch tensor of shape (batch_size, padded_length).
"""
# Sort a data list by caption length (descending order).
data.sort(key=lambda x: x[4], reverse=True)
feats, questions, answers, index, _, alengths = zip(*data)
feats = torch.stack(feats, 0)
questions = torch.stack(questions, 0).long()
answers = torch.stack(answers, 0).long()
aindices = np.flip(np.argsort(alengths), axis=0).copy()
aindices = torch.Tensor(aindices).long()
return feats, questions, answers, aindices, index
def collate_fn_use_topicFeat_test(data):
# for training set, it needs to
def merge(sequences, max_length=500):
lengths = [
max_length if len(seq) > max_length else len(seq)
for seq in sequences
]
padded_seqs = torch.ones(len(sequences), max(lengths)).long()
for i, seq in enumerate(sequences):
# seq = torch.tensor(seq)
end = lengths[i]
padded_seqs[i, :end] = seq[:end]
return padded_seqs, lengths
data.sort(key=lambda x: len(x[0]), reverse=True)
X, tit, y, id_ = zip(*data)
X, x_len = merge(X)
tit, tit_len = merge(tit)
X = torch.LongTensor(X)
id_topicFeat = constant.id_topicFeat_test
topic_feat = torch.Tensor(np.array([id_topicFeat[item] for item in id_]))
if (y[0] is None): pass
else: y = torch.LongTensor(y)
if constant.USE_CUDA:
tit = tit.cuda()
X = X.cuda() #X.cuda()
topic_feat = topic_feat.cuda()
if (y[0] is None): pass
else:
y = y.cuda() #y.cuda()
return X, x_len, tit, tit_len, y, "", topic_feat, id_
def collate_fun(data):
# sort a data list by caption length
data.sort(key=lambda x: len(x[1]), reverse=True)
zipped_data = list(zip(*data))
images, captions, image_texts, image_spans, spans, labels, ids = zipped_data
img_lengths = [len(x) for x in images]
max_img_len = max ([len(x) for x in images])
img_indices = torch.zeros(len(captions), max_img_len, 2).long()
for i, span in enumerate(image_spans):
img_len = len(images[i])
img_indices[i, :img_len - 1, :] = image_spans[i]
zero_img = torch.ones((1, 256, 256))
collate_images = []
for image in images:
image += [zero_img] * (max_img_len - len(image))
image = torch.stack(image)
collate_images.append(image)
images = torch.stack(collate_images)
max_lan_len = max([len(caption) for caption in captions])
indices = torch.zeros(len(captions), max_lan_len, 2).long()
targets = torch.zeros(len(captions), max_lan_len).long()
lengths = [len(cap) for cap in captions]
for i, cap in enumerate(captions):
cap_len = len(cap)
targets[i, : cap_len] = cap[: cap_len]
indices[i, : cap_len - 1, :] = spans[i]
return images, targets, lengths, img_lengths, image_texts, img_indices, indices, labels, ids
def collate_fn(data):
data.sort(key=lambda x: len(x[1]), reverse=True)
# print("len(data) = %s"%len(data))
_syllables, _lyrics, _melody, feature_size = zip(*data)
# print("len(_melody) = %s"%len(_melody))
# print("lyric size: ", torch.Size(_lyrics))
lengths = [len(_lyric) for _lyric in _lyrics
] # Creates an array of the lengths of each songs lyrics
max_length = lengths[0]
lyrics = torch.zeros(len(_lyrics), max_length).long() # Initialise tensors
syllables = torch.zeros(len(_syllables),
max_length).long() # Initialise tensors
melody = torch.zeros(len(_melody), max_length,
feature_size[0]).long() # Initialise tensors
for i, _lyric in enumerate(_lyrics):
end = lengths[i]
lyrics[i, :end] = _lyric[:end] # Create one long tensor for all songs
syllables[i, :end] = _syllables[
i][:end] # Create one long tensor for all songs
# print("_MELODY[i]: ", _melody[i])
# print("len(_MELODY[i]: ", len(_melody[i]))
# print("Tensor size: ", torch.Tensor(_melody[i]).long().size())
melody[i, :end].scatter_(1, torch.Tensor(_melody[i]).long(), 1)
lengths = torch.Tensor(lengths).long()
return syllables, lyrics, melody, lengths
def sequence_collate_fn(data):
"""
The `collate_fn` field of data.DataLoader requires a function that determines
how a list of `[(data_seq, target_seq)]` of length `batch_size` gets merged together
into a single batched tensor. Here, we perform dynamic padding to the maximum length
of the sequences in the batch.
Args:
* data: list of tuples of tensors of the form `(data_seq, target_seq)`.
Returns: four tensors of the form `(input_batch, input_lengths, target_batch, target_lengths)`, where
* input_batch: 3D FloatTensor of shape (batch_size, max_in_seq_length, in_dim).
* input_lengths: 1D IntTensor of shape (batch_size); the lengths of each input sequence.
* target_batch: 2D FloatTensor of shape (batch_size, max_target_seq_length).
* target_lengths: 1D IntTensor of shape (batch_size); the lengths of each target sequence.
Credits: the `collate_fn` implementation in Yunjey Choi's data loader provided helpful insight.
(https://github.com/yunjey/seq2seq-dataloader/blob/master/data_loader.py#L39)
"""
# sort dataset by decreasing signal length:
data.sort(key=lambda x: len(x[0]), reverse=True)
# get lengths:
input_lengths = torch.IntTensor([d[0].size(0) for d in data])
target_lengths = torch.IntTensor([d[1].size(0) for d in data])
# batch sequence tensors:
input_batch = pad_sequence([d[0] for d in data], batch_first=True)
target_batch = pad_sequence([d[1] for d in data],
batch_first=True,
pad_value=0)
return (input_batch, input_lengths, target_batch, target_lengths)
def collate_fn(data):
"""Creates mini-batch tensors from the list of tuples (image, attribute).
We should build custom collate_fn rather than using default collate_fn,
because merging attribute (including padding) is not supported in default.
Args:
data: list of tuple (image, attribute).
- image: torch tensor of shape (3, 256, 256).
- attribute: torch tensor of shape (?); variable length.
Returns:
images: torch tensor of shape (batch_size, 3, 256, 256).
targets: torch tensor of shape (batch_size, 24).
lengths: list; valid length for each attribute.
"""
# Sort a data list by attribute length (descending order).
data.sort(key=lambda x: len(x[3]), reverse=True)
images1, images2, images3, images4, attribs = zip(*data)
#print(images1.size())
attribs = torch.from_numpy(np.asarray(attribs)).float()
# Merge images (from tuple of 3D tensor to 4D tensor).
images1 = torch.stack(images1, 0)
images2 = torch.stack(images2, 0)
images3 = torch.stack(images3, 0)
images4 = torch.stack(images4, 0)
# Merge attributes (from tuple of 1D tensor to 2D tensor).
lengths_attrib = [len(cap) for cap in attribs]
targets_attrib = torch.zeros(len(attribs), max(lengths_attrib)).long()
for i, cap in enumerate(attribs):
end = lengths_attrib[i]
targets_attrib[i, :end] = cap[:end]
return images1, images2, images3, images4, targets_attrib, lengths_attrib
def collate_fn(data):
# mainly for padding, not sure if it works.
data.sort(key=lambda x: len(x[1]), reverse=True)
images, captions = zip(*data)
lengths = [len(cap) for cap in captions]
captions = pad_sequence(captions, batch_first=True)
return torch.stack(images, 0), captions, lengths
def collate_fn(data):
"""Creates mini-batch tensors from the list of tuples (image, caption)
by padding the captions to make them of equal length.
We can not use default collate_fn because variable length tensors can't be stacked vertically.
We need to pad the captions to make them of equal length so that they can be stacked for creating a mini-batch.
Read this for more information - https://pytorch.org/docs/stable/data.html#dataloader-collate-fn
Args:
data: list of tuple (image, caption).
- image: torch tensor of shape (3, 256, 256).
- caption: torch tensor of shape (?); variable length.
Returns:
images: torch tensor of shape (batch_size, 3, 256, 256).
targets: torch tensor of shape (batch_size, padded_length).
lengths: list; valid length for each padded caption.
"""
# Sort a data list by caption length (descending order).
data.sort(key=lambda x: len(x[1]), reverse=True)
images, captions, img_ids = zip(*data)
images = torch.stack(images, 0)
# Merge captions (from tuple of 1D tensor to 2D tensor).
lengths = [len(cap) for cap in captions]
targets = torch.zeros(len(captions), max(lengths)).long()
for i, cap in enumerate(captions):
end = lengths[i]
targets[i, :end] = cap[:end]
return images, targets, img_ids
def collate_fn(data):
# Sort a data list by caption length (descending order).
data.sort(key=lambda x: len(x[0]), reverse=True)
captions, fc_feats, att_feats, img_ids, img_names, img_captions, eng_gts = zip(*data)
lengths = [len(cap) for cap in captions]
targets = torch.zeros(len(captions), max(lengths)).long()
for i, cap in enumerate(captions):
end = lengths[i]
targets[i, :end] = cap[:end]
mask = targets > 0
fc_feats = torch.stack(fc_feats, 0)
if att_feats[0] is None:
att_feats = None
else:
att_feats = torch.stack(att_feats, 0)
data = {'fc_feats': fc_feats, 'att_feats': att_feats, 'targets':targets,
'lengths':lengths, 'mask':mask,
'img_ids':img_ids, 'img_names':img_names,
'img_captions': img_captions, 'eng_gts':eng_gts}
return data
'''
def collate_fn(data):
"""Creates mini-batch tensors from the list of tuples (image, caption).
We should build custom collate_fn rather than using default collate_fn,
because merging caption (including padding) is not supported in default.
Args:
data: list of tuple (image, caption).
- image: torch tensor of shape (3, 256, 256).
- caption: torch tensor of shape (?); variable length.
Returns:
images: torch tensor of shape (batch_size, 3, 256, 256).
targets: torch tensor of shape (batch_size, padded_length).
lengths: list; valid length for each padded caption.
"""
# Sort a data list by caption length (descending order).
data.sort(key=lambda x: len(x[1]), reverse=True)
images, captions = zip(*data)
# Merge images (from tuple of 3D tensor to 4D tensor).
images = torch.stack(images, 0)
# Merge captions (from tuple of 1D tensor to 2D tensor).
lengths = [len(cap) for cap in captions]
targets = torch.zeros(len(captions), max(lengths)).long()
for i, cap in enumerate(captions):
end = lengths[i]
targets[i, :end] = cap[:end]
return images, targets, lengths
def collate_fn(data):
"""Creates mini-batch tensors from the list of tuples ((image, question), target).
"""
# Sort a data list by qn length (descending order).
data.sort(key=lambda x: len(x[0][1]), reverse=True)
images = [item[0][0] for item in data]
questions = [item[0][1] for item in data]
targets = [item[1] for item in data]
# Merge images (from tuple of 3D tensor to 4D tensor).
images = torch.stack(images, 0)
# Merge questions (from tuple of 1D tensor to 2D tensor).
qn_lengths = [len(qn) for qn in questions]
qns = torch.zeros(len(questions), max(qn_lengths)).long()
for i, qn in enumerate(questions):
end = qn_lengths[i]
qns[i, :end] = qn[:end]
# Merge targets (from tuple of 1D tensor to 2D tensor).
ans_lengths = [len(ans) for ans in targets]
answers = torch.zeros(len(targets), max(ans_lengths))
for i, ans in enumerate(targets):
end = ans_lengths[i]
answers[i, :end] = ans[:end]
return (images, qns), answers, qn_lengths
def collate_fn(data):
def merge(sequences, max_len):
lengths = [len(seq) for seq in sequences]
if (max_len):
padded_seqs = torch.zeros(len(sequences), max(lengths)).long()
else:
padded_seqs = torch.zeros(len(sequences), max(lengths)).long()
for i, seq in enumerate(sequences):
end = lengths[i]
padded_seqs[i, :end] = seq[:end]
return padded_seqs, lengths
# sort a list by sequence length (descending order) to use pack_padded_sequence
data.sort(key=lambda x: len(x[0]), reverse=True)
# seperate source and target sequences
src_seqs, trg_seqs, ind_seqs, gete_s, max_len, src_plain, trg_plain, entity, entity_cal, entity_nav, entity_wet = zip(
*data)
# merge sequences (from tuple of 1D tensor to 2D tensor)
src_seqs, src_lengths = merge(src_seqs, max_len)
trg_seqs, trg_lengths = merge(trg_seqs, None)
ind_seqs, _ = merge(ind_seqs, None)
gete_s, _ = merge(gete_s, None)
src_seqs = Variable(src_seqs).transpose(0, 1)
trg_seqs = Variable(trg_seqs).transpose(0, 1)
ind_seqs = Variable(ind_seqs).transpose(0, 1)
gete_s = Variable(gete_s).transpose(0, 1)
if USE_CUDA:
src_seqs = src_seqs.cuda()
trg_seqs = trg_seqs.cuda()
ind_seqs = ind_seqs.cuda()
gete_s = gete_s.cuda()
return src_seqs, src_lengths, trg_seqs, trg_lengths, ind_seqs, gete_s, src_plain, trg_plain, entity, entity_cal, entity_nav, entity_wet
def collate_fn(data):
"""modify the default collate funtion coz it doesnt add padding ; """
""" Args:
data: list of tuple (image, caption).
- image: torch tensor of shape (3, 256, 256).
- caption: torch tensor of shape (?); variable length.
Returns:
images: torch tensor of shape (batch_size, 3, 256, 256).
targets: torch tensor of shape (batch_size, padded_length).
lengths: list; valid length for each padded caption.
# Sort a data list by caption length (descending order). """
data.sort(key=lambda x: len(x[1]), reverse=True)
images, captions = zip(
*data
) # images is the list of all tensor(3d) images; captions is the list of all tensor(1d) type captions
# Merge images (from tuple of 3D tensor to 4D tensor).
images = torch.stack(images, dim=0)
#merge captions (From 1d tensor to 2d tensor)
lengths = [len(cap) for cap in captions]
targets = torch.zeros(len(captions), max(lengths)).long()
for i, cap in enumerate(captions):
end = lengths[i]
targets[i, :end] = cap[:end]
return images, targets, lengths #the extra padded length has zeroes in it
def collate_fn(data):
"""Build mini-batch tensors from a list of (image, caption) tuples.
Args:
data: list of (image, caption, concept_label, concept_emb) tuple.
- image: torch tensor of shape (3, 256, 256).
- caption: torch tensor of shape (?); variable length.
- attribute_label: concept label, torch tensor of shape (concept_num);
- attribute_input_emb: initial concept embeddings, torch tensor of shape (concept_num, word_emb_dim);
Returns:
images: torch tensor of shape (batch_size, 3, 256, 256).
targets: torch tensor of shape (batch_size, padded_length).
attribute_label: torch tensor of shape (concept_num);
attribute_input_emb: torch tensor of shape (concept_num, word_emb_dim);
lengths: list; valid length for each padded caption.
ids: index
"""
# Sort a data list by caption length
data.sort(key=lambda x: len(x[1]), reverse=True)
images, captions, attribute_label, attribute_input_emb, ids, img_ids = zip(*data)
# Merge images (convert tuple of 3D tensor to 4D tensor)
images = torch.stack(images, 0)
# Merget captions (convert tuple of 1D tensor to 2D tensor)
lengths = [len(cap) for cap in captions]
targets = torch.zeros(len(captions), max(lengths)).long()
for i, cap in enumerate(captions):
end = lengths[i]
targets[i, :end] = cap[:end]
attribute_labels = torch.stack(attribute_label, 0)
attribute_input_embs = torch.stack(attribute_input_emb, 0)
return images, targets, attribute_labels, attribute_input_embs, lengths, ids
def collate_fn(data): """Build mini-batch tensors from a list of (image, caption) tuples. Args: data: list of (image, caption) tuple. - image: torch tensor of shape (3, 256, 256). - caption: torch tensor of shape (?); variable length. Returns: images: torch tensor of shape (batch_size, 3, 256, 256). targets: torch tensor of shape (batch_size, padded_length). lengths: list; valid length for each padded caption. """ # Sort a data list by caption length data.sort(key=lambda x: len(x[1]), reverse=True) # pdb.set_trace() images, captions, audios, ids, img_ids = zip(*data) # Merge images (convert tuple of 3D tensor to 4D tensor) images = torch.stack(images, 0) # Merget captions (convert tuple of 1D tensor to 2D tensor) lengths = [len(cap) for cap in captions] targets = torch.zeros(len(captions), max(lengths)).long() for i, cap in enumerate(captions): end = lengths[i] targets[i, :end] = cap[:end] audios = torch.stack(audios, 0) return images, targets, audios, lengths, ids
def read_composite_coco(vocab, coco_path, cand_type):
img_ids = []
caps_tensor = []
caps = []
evals = []
idxs = []
with open(coco_path) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=';')
line_count = 0
for row in csv_reader:
if line_count == 0 or row[0] == '':
line_count += 1
continue
else:
item = row[0].split('_')
coco_id = item[len(item) - 1].replace('.jpg', '').lstrip('0')
if cand_type == 'h':
coco_cap1 = row[1]
coco_eval1 = row[4]
target1 = cap_preprocessing(coco_cap1, coco_eval1, vocab)
caps_tensor.append(target1)
img_ids.append(coco_id)
caps.append(coco_cap1)
evals.append(coco_eval1)
idxs.append(caps.index(coco_cap1))
elif cand_type == 'm1':
coco_cap2 = row[2]
coco_eval2 = row[5]
target2 = cap_preprocessing(coco_cap2, coco_eval2, vocab)
caps_tensor.append(target2)
img_ids.append(coco_id)
caps.append(coco_cap2)
evals.append(coco_eval2)
idxs.append(caps.index(coco_cap2))
elif cand_type == 'm2':
coco_cap3 = row[3]
coco_eval3 = row[6]
target3 = cap_preprocessing(coco_cap3, coco_eval3, vocab)
caps_tensor.append(target3)
img_ids.append(coco_id)
caps.append(coco_cap3)
evals.append(coco_eval3)
idxs.append(caps.index(coco_cap3))
# Sort a data list by caption length
data = zip(img_ids, caps_tensor, caps, evals, idxs)
data.sort(key=lambda x: len(x[1]), reverse=True)
img_ids, caps_tensor, caps, evals, idxs = zip(*data)
# Merget captions (convert tuple of 1D tensor to 2D tensor)
lengths = [len(cap) for cap in caps_tensor]
targets = torch.zeros(len(caps_tensor), max(lengths)).long()
for i, cap in enumerate(caps_tensor):
end = lengths[i]
targets[i, :end] = cap[:end]
return img_ids, caps, targets, lengths, evals, idxs
def train_collate_fn(data): ''' :param data: -format:(image,caption,img_id) :return: images: tensor (batch_size,3,224,224) targets: tensor (batch_size,padded_length) lenghts: list,Every effective length of padding caption img_ids:list,id of image ''' # Sort by captions' length data.sort(key=lambda x: len(x[1]), reverse=True) images, captions, img_ids = zip(*data) images = torch.stack(images, 0) lengths = [len(cap) for cap in captions] targets = torch.zeros(len(captions), max(lengths)).long() for i, cap in enumerate(captions): end = lengths[i] targets[i, :end] = cap[:end] return images, targets, lengths, list(img_ids)
def collate_fn(data):
"""Creates mini-batch tensors from the list of tuples (image, caption).
We should build custom collate_fn rather than using default collate_fn,
because merging caption (including padding) is not supported in default.
Args:
data: list of tuple (image, caption).
- image: torch tensor of shape (3, 256, 256).
- caption: torch tensor of shape (?); variable length.
Returns:
images: torch tensor of shape (batch_size, 3, 256, 256).
targets: torch tensor of shape (batch_size, padded_length).
lengths: list; valid length for each padded caption.
"""
# Sort a data list by caption length (descending order).
data.sort(key=lambda x: len(x[1]), reverse=True)
images, captions = zip(*data)
# Merge images (from tuple of 3D tensor to 4D tensor).
images = torch.stack(images, 0)
# Merge captions (from tuple of 1D tensor to 2D tensor).
lengths = [len(cap) for cap in captions]
targets = torch.zeros(len(captions), max(lengths)).long()
for i, cap in enumerate(captions):
end = lengths[i]
targets[i, :end] = cap[:end]
return images, targets, lengths
def collate_fn(data):
"""
:src_seqs:Tensor
src_lengths:list
trg_seqs:Tensor
trg_lengths:list
ind_seqs:Tensor
gete_s:Tensor
src_plain:list
trg_plain:list
request_KB_arr:list
inform_KB_arr:list
"""
def merge(sequences, max_len):
lengths = [len(seq) for seq in sequences]
if (max_len):
# 填充batches
padded_seqs = torch.Tensor([PAD_token_index]).to(device=DEVICE).repeat(
len(sequences), max(lengths),
MEM_TOKEN_SIZE).long()
for i, seq in enumerate(sequences):
end = lengths[i]
padded_seqs[i, :end, :] = seq[:end]
else:
padded_seqs = torch.Tensor([
PAD_token_index
]).to(device=DEVICE).repeat(len(sequences), max(lengths)).long()
for i, seq in enumerate(sequences):
end = lengths[i]
padded_seqs[i, :end] = seq[:end]
return padded_seqs, lengths
# sort a list by sequence length (descending order) to use pack_padded_sequence
data.sort(key=lambda x: len(x[0]), reverse=True)
# seperate source and target sequences
# src_plain = user input splits words
# trg_plain = sys reply sentence
src_seqs, trg_seqs, ind_seqs, gete_s, max_len, src_plain, trg_plain, request_KB_arr, inform_KB_arr = zip(
*data)
# merge sequences (from tuple of 1D tensor to 2D tensor)
src_seqs, src_lengths = merge(src_seqs, max_len)
trg_seqs, trg_lengths = merge(trg_seqs, None)
ind_seqs, _ = merge(ind_seqs, None)
gete_s, _ = merge(gete_s, None)
# conv_seqs, conv_lengths = merge(conv_seq, max_len)
src_seqs = Variable(src_seqs).transpose(0, 1)
trg_seqs = Variable(trg_seqs).transpose(0, 1)
ind_seqs = Variable(ind_seqs).transpose(0, 1)
gete_s = Variable(gete_s).transpose(0, 1)
# conv_seqs = Variable(conv_seqs).transpose(0, 1)
# if USE_CUDA:
# src_seqs = src_seqs.cuda()
# trg_seqs = trg_seqs.cuda()
# ind_seqs = ind_seqs.cuda()
# gete_s = gete_s.cuda()
# conv_seqs = conv_seqs.cuda()
return src_seqs, src_lengths, trg_seqs, trg_lengths, ind_seqs, gete_s, src_plain, trg_plain, request_KB_arr, inform_KB_arr
def collate_fn(data):
data.sort(key=lambda x: len(x[2]), reverse=True)
diags, sides, meds = zip(*data)
diags, diags_length = padding_batch(diags)
meds, meds_length = padding_batch(meds)
sides = np.stack(sides, 0)
return diags, diags_length, sides, meds, meds_length
def get_data(data: list) -> tuple:
data.sort(key=lambda x: len(x[1]), reverse=True)
images, captions = zip(*data)
images = torch.stack(images, dim=0)
lengths = [len(x) for x in captions]
captions_arr = torch.zeros(size=(len(captions), max(lengths)), dtype=torch.long)
for i, cap in enumerate(captions):
captions_arr[i, :lengths[i]] = cap[:lengths[i]]
return images, captions_arr, lengths
def read_flickr8k(vocab, opt):
imgids = []
caps_id = []
caps_tensor = []
caps_string = []
caps_evals = []
caps_idxs = []
all_caps = {}
with open(
os.path.join(opt.candidate_path, 'flickr8k', 'Flickr8k.token.txt'),
'rb') as f:
for line in f:
item = line.split('\t')
all_caps[item[0]] = item[1].replace('\n', '')
with open(
os.path.join(opt.candidate_path, 'flickr8k',
'ExpertAnnotations.txt')) as f:
for line in f:
item = line.split('\t')
imgid = item[0]
cid = item[1]
##remove candidates that are actually belonged to the target image
if (cid.split('#')[0] == imgid):
continue
cstr = all_caps[cid]
ceva = []
ceva.append(item[2])
ceva.append(item[3])
ceva.append(item[4].replace('\n', ''))
target = cap_preprocessing(cstr, vocab)
#imgids.append(sudoid)
imgids.append(imgid)
caps_id.append(cid)
caps_tensor.append(target)
caps_string.append(cstr)
caps_evals.append(ceva)
caps_idxs.append(caps_string.index(cstr))
# Sort a data list by caption length
data = zip(imgids, caps_tensor, caps_string, caps_evals, caps_id,
caps_idxs)
data.sort(key=lambda x: len(x[1]), reverse=True)
imgids, caps_tensor, caps_string, caps_evals, caps_id, caps_idxs = zip(
*data)
# Merget captions (convert tuple of 1D tensor to 2D tensor)
caps_lengths = [len(cap) for cap in caps_tensor]
targets = torch.zeros(len(caps_tensor), max(caps_lengths)).long()
for i, cap in enumerate(caps_tensor):
end = caps_lengths[i]
targets[i, :end] = cap[:end]
return imgids, caps_string, targets, caps_lengths, caps_evals, caps_id, caps_idxs
def test_collate_fn(data):
# Sort a data list by caption length (descending order).
data.sort(key=lambda x: len(x[1]), reverse=True)
images, captions, img_ids = zip(*data)
# Merge images (from tuple of 3D tensor to 4D tensor).
images = torch.stack(images, 0)
return images, '', 0, img_ids
def collate_fn(data):
def merge(sequences):
lengths = [len(seq) for seq in sequences]
padded_seqs = torch.zeros(len(sequences), max(lengths)).long() + \
config.PAD_idx
for i, seq in enumerate(sequences):
end = lengths[i]
padded_seqs[i, :end] = seq[:end]
return padded_seqs, lengths
data.sort(key=lambda x: len(x["input_batch"]),
reverse=True) # sort by source seq
item_info = {}
for key in data[0].keys():
item_info[key] = [d[key] for d in data]
input_batch, input_lengths = merge(item_info['input_batch'])
target_batch, target_lengths = merge(item_info['target_batch'])
input_batch = input_batch.transpose(0, 1)
target_batch = target_batch.transpose(0, 1)
input_lengths = torch.LongTensor(input_lengths)
target_lengths = torch.LongTensor(target_lengths)
if config.USE_CUDA:
input_batch = input_batch.cuda()
target_batch = target_batch.cuda()
input_lengths = input_lengths.cuda()
target_lengths = target_lengths.cuda()
d = {}
d["input_batch"] = input_batch
d["target_batch"] = target_batch
d["input_lengths"] = input_lengths
d["target_lengths"] = target_lengths
d["input_txt"] = item_info["input_txt"]
d["target_txt"] = item_info["target_txt"]
d["cand_txt"] = item_info["cand_txt"]
d["cand_index"] = item_info["cand_index"]
d["persona_txt"] = item_info["persona_txt"]
if 'input_ext_vocab_batch' in item_info:
input_ext_vocab_batch, _ = merge(item_info['input_ext_vocab_batch'])
target_ext_vocab_batch, _ = merge(item_info['target_ext_vocab_batch'])
input_ext_vocab_batch = input_ext_vocab_batch.transpose(0, 1)
target_ext_vocab_batch = target_ext_vocab_batch.transpose(0, 1)
if config.USE_CUDA:
input_ext_vocab_batch = input_ext_vocab_batch.cuda()
target_ext_vocab_batch = target_ext_vocab_batch.cuda()
d["input_ext_vocab_batch"] = input_ext_vocab_batch
d["target_ext_vocab_batch"] = target_ext_vocab_batch
if "article_oovs" in item_info:
d["article_oovs"] = item_info["article_oovs"]
d["max_art_oovs"] = max(
len(art_oovs) for art_oovs in item_info["article_oovs"])
return d
def test_collate_fn(data):
# Sort a data list by caption length (descending order).
data.sort(key=lambda x: len(x[0]), reverse=True)
sentences, index = zip(*data)
lengths = [len(sent) for sent in sentences]
s = torch.zeros(len(sentences), max(lengths)+1).long()
for i, sent in enumerate(sentences):
end = lengths[i]
s[i, :end] = sent[:end]
return s, lengths, index
def eval_collate_fn(data):
data.sort(key=lambda x: x[-1], reverse=False)
videos, regions, spatials, video_ids = zip(*data)
videos = torch.stack(videos, 0)
regions = torch.stack(regions, 0)
spatials = torch.stack(spatials, 0)
return videos, regions, spatials, video_ids
def collate_fun_eval(data):
data.sort(key=lambda x: len(x[0]), reverse=True)
zipped_data = list(zip(*data))
captions, labels, spans, tags, ids = zipped_data
max_len = max([len(caption) for caption in captions])
targets = torch.zeros(len(captions), max_len).long()
lengths = [len(cap) for cap in captions]
for i, cap_len in enumerate(lengths):
targets[i, :cap_len] = captions[i][:cap_len]
return targets, lengths, spans, labels, tags, ids
def collate_fn(data):
data.sort(key=lambda x: len(x[2]), reverse=True)
video_feats, image_feats, captions, indexs, vids, lengths = zip(*data)
video_feats = torch.cat(video_feats, dim=0)
image_feats = torch.cat(image_feats, dim=0)
targets = torch.zeros(len(captions), min(max(lengths), 30)).long()
for i, cap in enumerate(captions):
end = lengths[i]
targets[i, :end] = cap
return video_feats, image_feats, targets, lengths, vids
def collate_fun(data):
data.sort(key=lambda x: len(x[1]), reverse=True)
images, captions = zip(*data)
images = torch.stack(images, 0)
caption_lengths = [len(caption) for caption in captions]
padded_captions = torch.zeros(len(captions), max(caption_lengths)).long()
for ix, caption in enumerate(captions):
end = caption_lengths[ix]
padded_captions[ix, :end] = caption[:end]
return images, padded_captions, caption_lengths
def make_batch(self, data):
# longest Y first
data.sort(key=lambda x: x[3], reverse=True)
articles, titles, article_lengths, title_lengths = zip(*data)
# dynamic padding
articles = self.make_square(articles)
titles = self.make_square(titles)
return articles, titles, torch.LongTensor(
article_lengths), torch.LongTensor(title_lengths)
def collate_gtr_fn(data):
"""Creates mini-batch tensors from the list of tuples (src_seq, trg_seq).
We should build a custom collate_fn rather than using default collate_fn,
because merging sequences (including padding) is not supported in default.
Seqeuences are padded to the maximum length of mini-batch sequences (dynamic padding).
Args:
data: list of tuple (src_seq, trg_seq).
- src_seq: torch tensor of shape (?); variable length.
- trg_seq: torch tensor of shape (?); variable length.
Returns:
src_seqs: torch tensor of shape (batch_size, padded_length).
src_lengths: list of length (batch_size); valid length for each padded source sequence.
trg_seqs: torch tensor of shape (batch_size, padded_length).
trg_lengths: list of length (batch_size); valid length for each padded target sequence.
"""
def merge(sequences):
lengths = [len(seq) for seq in sequences]
padded_seqs = torch.zeros(len(sequences), max(lengths)).long()
for i, seq in enumerate(sequences):
end = lengths[i]
padded_seqs[i, :end] = seq[:end]
return padded_seqs, lengths
def merge_src(sequences, pad_number=[0]):
lengths = [len(seq) for seq in sequences]
padded_seqs = []
mask = torch.zeros(len(sequences), max(lengths)).long()
for i, seq in enumerate(sequences):
end = lengths[i]
padded_seqs.append(seq[:end] + [pad_number] * (max(lengths) - end))
mask[i, :end] = torch.ones(end).long()
return padded_seqs, mask
# sort a list by sequence length (descending order) to use pack_padded_sequence
data.sort(key=lambda x: len(x[0][0]), reverse=True)
# seperate source and target sequences
src_inputs, trg_seqs = zip(*data)
src_gtr_seq_node, src_gtr_seq_rel, src_gtr_seq_father = zip(*src_inputs)
# merge sequences (from tuple of 1D tensor to 2D tensor)
src_gtr_seq_node, src_gtr_seq_node_mask = merge_src(src_gtr_seq_node)
src_gtr_seq_rel, _ = merge_src(src_gtr_seq_rel)
src_gtr_seq_father, _ = merge_src(src_gtr_seq_father, pad_number=-1)
trg_seqs, trg_lengths = merge(trg_seqs)
src_inputs = {
"src_gtr_seq_node": src_gtr_seq_node,
"src_gtr_seq_node_mask": src_gtr_seq_node_mask,
"src_gtr_seq_rel": src_gtr_seq_rel,
"src_gtr_seq_father": src_gtr_seq_father
}
return src_inputs, trg_seqs, trg_lengths
def eval_collate_fn(data):
'''
用来把多个数据样本合并成一个minibatch的函数
'''
data.sort(key=lambda x: x[-1], reverse=True)
videos, video_ids = zip(*data)
# 把视频合并在一起（把2D Tensor的序列变成3D Tensor）
videos = torch.stack(videos, 0)
return videos, video_ids
def train_collate_fn(data):
'''
用来把多个数据样本合并成一个minibatch的函数
'''
# 根据video的长度对数据进行排序
data.sort(key=lambda x: x[-1], reverse=True)
videos, captions, lengths, video_ids = zip(*data)
# 把视频合并在一起（把2D Tensor的序列变成3D Tensor）
videos = torch.stack(videos, 0)
# 把caption合并在一起（把1D Tensor的序列变成一个2D Tensor）
captions = torch.stack(captions, 0)
return videos, captions, lengths, video_ids
