def create_iterator(self):
spacy_de = spacy.load("de")
spacy_en = spacy.load("en")
def tokenize_de(text):
return [tok.text for tok in spacy_de.tokenizer(text)][::-1]
def tokenize_en(text):
return [tok.text for tok in spacy_en.tokenizer(text)][::-1]
if self.nopad:
self.source_field = Field(tokenize=tokenize_de,
init_token="<sos>",
eos_token="<eos>",
lower=True,
batch_first=True)
else:
self.source_field = Field(tokenize=tokenize_de,
init_token="<sos>",
eos_token="<eos>",
lower=True,
batch_first=True)
self.target_field = Field(tokenize=tokenize_en,
init_token="<sos>",
eos_token="<eos>",
lower=True,
batch_first=True)
train_data, valid_data, test_data = Multi30k.splits(
exts=(".de", ".en"), fields=(self.source_field, self.target_field))
print(f"Number of training examples = {len(train_data.examples)}")
print(f"Number of validation examples = {len(valid_data.examples)}")
print(f"Number of testing examples = {len(test_data.examples)}")
self.source_field.build_vocab(train_data, min_freq=2)
self.target_field.build_vocab(train_data, min_freq=2)
if self.nopad:
train_iterator, valid_iterator, test_iterator = BucketIterator.splits(
(train_data, valid_data, test_data),
batch_size=self.batch_size,
sort_within_batch=True,
sort_key=lambda x: len(x.src),
device=self.device)
else:
train_iterator, valid_iterator, test_iterator = BucketIterator.splits(
(train_data, valid_data, test_data),
batch_size=self.batch_size,
device=self.device)
return train_iterator, valid_iterator, test_iterator
def build_scan(split, batch_size, device):
# Get paths and filenames of each partition of split
if split == 'simple':
path = 'data/scan/simple/'
elif split == 'addjump':
path = 'data/scan/addjump/'
else:
assert split not in ['simple','addjump'], "Unknown split"
train_path = os.path.join(path,'train')
dev_path = os.path.join(path,'dev')
test_path = os.path.join(path,'test')
exts = ('.src','.trg')
# Fields for source (SRC) and target (TRG) sequences
SRC = Field(init_token='<sos>',eos_token='<eos>')
TRG = Field(init_token='<sos>',eos_token='<eos>')
fields = (SRC,TRG)
# Build datasets
train_ = TranslationDataset(train_path,exts,fields)
dev_ = TranslationDataset(dev_path,exts,fields)
test_ = TranslationDataset(test_path,exts,fields)
# Build vocabs: fields ensure same vocab used for each partition
SRC.build_vocab(train_)
TRG.build_vocab(train_)
# BucketIterator ensures similar sequence lengths to minimize padding
train, dev, test = BucketIterator.splits((train_, dev_, test_),
batch_size = batch_size, device = device)
return SRC, TRG, train, dev, test
def iterator_construction(
train: TabularDataset,
valid: TabularDataset,
test: TabularDataset,
feature: str, # the column name of the input text data
batch_sizes: Tuple[int, int, int] = (64, 64, 64), # order: train, valid, test
device: torch.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
) -> Tuple[BucketIterator, BucketIterator, BucketIterator]:
"""
This function takes torchtext.data.TabularDataset's as input and output the correspondent BucketIterator's,
splitting the datasets into batches. This iterator batches examples of similar lengths together, minimizing the
amount of padding needed while producing freshly shuffled batches for each new epoch.
:param train: a torchtest.data.TabularDataset representing the training dataset
:param valid: a torchtest.data.TabularDataset representing the validation dataset
:param test: a torchtest.data.TabularDataset representing the testing dataset
:param feature: a string represent the name of the input feature to the model. Multiple inputs are not supported.
:param batch_sizes: a tuple of 3 integers, each representing the batch size for the train, validation and test set
respectively. Default: (64, 64, 64)
:param device: the torch.device to be used, either 'cpu' or 'cuda' (gpu) if available
:return: train_iter: a torchtext.data.BucketIterator, the iterator for the training dataset
valid_iter: a torchtext.data.BucketIterator, the iterator for the validation dataset
test_iter: a torchtext.data.BucketIterator, the iterator for the testing dataset
"""
train_iter, valid_iter, test_iter = BucketIterator.splits(
(train, valid, test),
batch_sizes=(batch_sizes[0], batch_sizes[1], batch_sizes[2]),
device=device,
sort_key=lambda x: len(getattr(x, feature)),
sort_within_batch=True
)
return train_iter, valid_iter, test_iter
def load_dataset(batch_size):
'''
load data sets.
'''
Lang1 = Field(include_lengths=True, init_token='<sos>', eos_token='<eos>')
Lang2 = Field(include_lengths=True, init_token='<sos>', eos_token='<eos>')
train = TranslationDataset(path='data/40w/train',
exts=('.ch', '.en'),
fields=(Lang1, Lang2))
val = TranslationDataset(path='data/40w/valid',
exts=('.ch', '.en'),
fields=(Lang1, Lang2))
test = TranslationDataset(path='data/40w/test',
exts=('.ch', '.en'),
fields=(Lang1, Lang2))
Lang1.build_vocab(train.src, max_size=30000)
Lang2.build_vocab(train.trg, max_size=30000)
train_iter, val_iter, test_iter = BucketIterator.splits(
(train, val, test), batch_size=batch_size, repeat=False)
return train_iter, val_iter, test_iter, Lang1, Lang2
def task2_iterators(train, val, device, vectors, batch_size=256):
TEXT = Field(sequential=True,
tokenize=None,
init_token="<sos>",
eos_token="<eos>")
LABELS = Field(sequential=True,
init_token="<sos>",
eos_token="<eos>",
is_target=True)
tv_datafields = [("text", TEXT), ("labels", LABELS)]
trn, vld = TabularDataset.splits(
path=config.
TASK2['csv_files'], # the root directory where the data lies
train=train,
validation=val,
format='csv',
skip_header=True,
fields=tv_datafields)
TEXT.build_vocab(trn, vld, vectors=vectors)
LABELS.build_vocab(trn, vld)
train_iterator, val_iterator = BucketIterator.splits(
(trn, vld),
batch_sizes=(batch_size, batch_size),
device=device,
sort_within_batch=False,
sort_key=lambda x: len(x.text))
return train_iterator, val_iterator, TEXT, LABELS
def get_bucketized_iterators(path_to_data: PathOrStr, batch_size: int = 16,
len_context_vocab: int = 30000,
len_title_vocab: int = 30000,
len_aut_vocab: int = 30000) -> IteratorData:
"""
Gets path_to_data and delegates tasks to generate buckettized training iterators.
## Parameters:
- **path_to_data** *(PathOrStr)*: Path object or string to a .csv dataset.
- **batch_size** *(int=32)*: BucketIterator minibatch size.
- **len_context_vocab** *(int=30000)*: Maximum length of context vocab size before adding special tokens.
- **len_title_vocab** *(int=30000)*: Maximum length of context vocab size before adding special tokens.
- **len_aut_vocab** *(int=30000)*: Maximum length of context vocab size before adding special tokens.
## Output:
- **Training data** *(IteratorData)*: Container holding CNTXT (*Field*), TTL (*Field*), AUT (*Field*),
train_iterator (*BucketIterator*), valid_iterator (*BucketIterator*), test_iterator (*BucketIterator*) objects.
"""
data = get_datasets(path_to_data=path_to_data, len_context_vocab=len_context_vocab,
len_title_vocab=len_title_vocab, len_aut_vocab=len_aut_vocab)
# create bucketted iterators for each dataset
train_iterator, valid_iterator, test_iterator = BucketIterator.splits((data.train, data.valid, data.test),
batch_size = batch_size,
sort_within_batch = True,
sort_key = lambda x : len(x.title_cited))
return IteratorData(data.cntxt, data.ttl, data.aut, train_iterator, valid_iterator, test_iterator)
def __init__(self,
batch_size=64,
max_vocab=999999,
min_freq=1,
tokenizer=sp,
shuffle=True):
super().__init__()
self.TEXT = Field(sequential=True,
use_vocab=True,
tokenize=tokenizer,
lower=True,
batch_first=True,
fix_length=20)
self.LABEL = Field(sequential=False, use_vocab=False, is_target=True)
train_data, valid_data, test_data = TabularDataset.splits(
path='data/',
train='train.txt',
validation='validation.txt',
test='test.txt',
format='tsv',
fields=[('text', self.TEXT), ('label', self.LABEL)],
skip_header=True)
self.TEXT.build_vocab(train_data,
max_size=max_vocab,
min_freq=min_freq)
self.LABEL.build_vocab(train_data)
# self.train_loader = BucketIterator(dataset=train_data, batch_size=batch_size)
# self.test_loader = BucketIterator(dataset=valid_data, batch_size=batch_size)
self.tr_dl, self.val_dl, self.test_dl = BucketIterator.splits(
(train_data, valid_data, test_data),
sort_key=lambda x: len(x.text),
batch_size=batch_size,
shuffle=True)
def data_iter(data_path, vec_path, fix_length):
TEXT = data.Field(sequential=True,
lower=True,
fix_length=fix_length,
batch_first=True)
LABEL = data.Field(sequential=False, use_vocab=False)
train, test = TabularDataset.splits(path=data_path,
train='train.csv',
test='test.csv',
format='csv',
fields=[('label', LABEL),
('title', None),
('text', TEXT)],
skip_header=True)
train_iter, test_iter = BucketIterator.splits(
(train, test), # 构建数据集所需的数据集
batch_sizes=(8, 8),
sort_within_batch=False,
repeat=False)
cache = '.vector_cache'
if not os.path.exists(cache):
os.mkdir(cache)
vectors = Vectors(name=vec_path, cache=cache)
TEXT.build_vocab(train, vectors=vectors)
vocab = TEXT.vocab
return train_iter, test_iter, vocab
def get_dataset(args):
TEXT = Field(sequential=True,
tokenize=tokenize_line_en,
lower=True,
batch_first=True)
LABEL = Field(sequential=False, use_vocab=False, batch_first=True)
train, val, test = TabularDataset.splits(path='WikiQACorpus',
root='',
train='WikiQA-train.tsv',
validation='WikiQA-dev.tsv',
test='WikiQA-test.tsv',
format='tsv',
fields=[('question_id', None),
('question', TEXT),
('document_id', None),
('document_title', None),
('sentence_id', None),
('sentence', TEXT),
('label', LABEL)],
skip_header=True)
TEXT.build_vocab(train, vectors='glove.840B.300d')
device = torch.device(
'cuda', args.device) if args.device > -1 else torch.device('cpu')
train_iter, dev_iter, test_iter = BucketIterator.splits(
(train, val, test),
batch_size=args.batch_size,
sort=False,
shuffle=True,
repeat=False,
device=device)
return train_iter, dev_iter, test_iter, TEXT.vocab
def create_street_names_data_iterators(path,
char_max_size=50,
names_max_size=60000,
batch_size=32,
device='cuda'):
chars = Field(sequential=True,
use_vocab=True,
tokenize=lambda x: x.split(),
lower=True,
fix_length=18)
names = Field(sequential=False,
use_vocab=True,
tokenize=lambda x: x,
lower=True)
fields = {"chars": ("chars", chars), "street_name": ("names", names)}
train_data, test_data = TabularDataset.splits(
path="",
train=os.path.join(path, "train_.csv"),
test=os.path.join(path, "test_.csv"),
format="csv",
fields=fields)
chars.build_vocab(train_data, max_size=char_max_size, min_freq=2)
names.build_vocab(train_data, max_size=names_max_size, min_freq=1)
train_iterator, test_iterator = BucketIterator.splits(
(train_data, test_data), batch_size=batch_size, device=device)
return train_iterator, test_iterator, chars.vocab, names.vocab, train_data, test_data
def load_dataset(path="./data", train_csv="train.csv", val_csv="val.csv",
init_token='^', eos_token='$', batch_size=32):
INDEX = Field(sequential=False,
use_vocab=False,
pad_token=None,
unk_token=None)
EN = Field(tokenize=tokenize,
include_lengths=True,
init_token=init_token,
eos_token=eos_token)
JP = Field(tokenize=tokenize,
include_lengths=True,
init_token=init_token,
eos_token=eos_token,
is_target=True)
FREQ = Field(sequential=False,
use_vocab=False,
pad_token=None,
unk_token=None,
dtype=torch.float32)
data_fields = [('index', INDEX), ('english', EN),
('japanese', JP), ('frequency', FREQ)]
train, val = TabularDataset.splits(path=path,
train=train_csv,
validation=val_csv,
skip_header = True,
format='csv', fields=data_fields)
EN.build_vocab(train.english)
JP.build_vocab(train.japanese)
train_iter, val_iter = BucketIterator.splits((train, val),
batch_size=batch_size,
sort=False,
repeat=False)
return train_iter, val_iter, EN, JP
def iters(cls,
batch_size=64,
device=-1,
shuffle=True,
vectors='glove.840B.300d'):
cls.TEXT = Field(sequential=True,
tokenize='spacy',
lower=True,
batch_first=True)
cls.LABEL = Field(sequential=False,
use_vocab=False,
batch_first=True,
tensor_type=torch.FloatTensor,
postprocessing=Pipeline(get_class_probs))
cls.ID = RawField()
train, val, test = cls.splits(cls.TEXT, cls.LABEL, cls.ID)
cls.TEXT.build_vocab(train, vectors=vectors)
return BucketIterator.splits((train, val, test),
batch_size=batch_size,
shuffle=shuffle,
repeat=False,
device=device)
def get_train_valid_iter(self, batch_size):
train_data, valid_data = self._get_train_valid_data()
train_iter, valid_iter = BucketIterator.splits(
(train_data, valid_data),
batch_size=batch_size,
device=self.device)
return train_iter, valid_iter
def get_data():
SEED = 1234
random.seed(SEED)
torch.manual_seed(SEED)
torch.backends.cudnn.deterministic = True
SRC = Field(tokenize=tokenize_de,
init_token='<sos>',
eos_token='<eos>',
lower=True)
TRG = Field(tokenize=tokenize_en,
init_token='<sos>',
eos_token='<eos>',
lower=True)
train_data, valid_data, test_data = Multi30k.splits(exts=('.de', '.en'),
fields=(SRC, TRG))
SRC.build_vocab(train_data, min_freq=2)
TRG.build_vocab(train_data, min_freq=2)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
BATCH_SIZE = 128
INPUT_DIM = len(SRC.vocab)
OUTPUT_DIM = len(TRG.vocab)
PAD_IDX = TRG.vocab.stoi['<pad>']
train_iterator, valid_iterator, test_iterator = BucketIterator.splits(
(train_data, valid_data, test_data),
batch_size=BATCH_SIZE,
device=device)
return train_iterator, valid_iterator, test_iterator, INPUT_DIM, OUTPUT_DIM, PAD_IDX
def _get_iterators(self, train_data, valid_data, model_name):
return BucketIterator.splits((train_data, valid_data),
repeat=False,
batch_size=self.batch_size,
sort_within_batch=False,
sort_key=lambda x: len(x.src),
device=self.device)
def initialize_iterators(self):
''' build iterators for data (by batches) using the bucket iterator '''
self.train_iter, self.valid_iter, self.test_iter = BucketIterator.splits(
datasets=(self.train_set, self.valid_set, self.test_set),
batch_size=self.batch_size,
device=self.device,
random_state=seed)
def get_dataloader(self,
proc_id=0,
n_gpus=1,
device=torch.device('cpu'),
batch_size=64):
def _distribute_dataset(dataset):
n = len(dataset)
part = dataset[n * proc_id // n_gpus:n * (proc_id + 1) // n_gpus]
return torchtext.data.Dataset(part, dataset.fields)
train_ds = _distribute_dataset(self.train_ds)
self.verbose = self.verbose and (proc_id == 0)
train_iter, valid_iter = BucketIterator.splits(
(train_ds, self.valid_ds),
batch_sizes=(batch_size, batch_size),
sort_within_batch=True,
sort_key=lambda x: len(x.input),
device=device,
repeat=False,
)
test_iter = Iterator(
self.test_ds,
batch_size=1,
sort=False,
sort_within_batch=False,
device=device,
repeat=False,
)
train_dl = BatchWrapper(train_iter)
valid_dl = BatchWrapper(valid_iter)
test_dl = BatchWrapper(test_iter)
return train_dl, valid_dl, test_dl
def load_dataset(batch_size):
ASK = Field(tokenize=tokenizer,
include_lengths=True,
eos_token='<eos>',
init_token='<sos>')
ANS = Field(tokenize=tokenizer,
include_lengths=True,
eos_token='<eos>',
init_token='<sos>')
file_path = r"D:\pycharm_project\datasets\NLP\cn_chat/"
train, val, test = ttd.TabularDataset.splits(
path=file_path,
train='weibo_mini_train.csv',
test='weibo_mini_test.csv',
validation='weibo_mini_val.csv',
format='csv',
fields=[('Ask', ASK), ('Ans', ANS)])
ASK.build_vocab(train.Ask, min_freq=1)
ANS.build_vocab(train.Ans, max_size=20000)
torch.save(ASK, file_path + 'ASK')
torch.save(ANS, file_path + 'ANS')
# train_iter, val_iter, test_iter = BucketIterator.splits(
# (train, val, test), batch_size=batch_size, repeat=False)
train_iter, val_iter, test_iter = BucketIterator.splits(
(train, val, test), batch_size=batch_size, repeat=False, sort=False)
return train_iter, val_iter, test_iter, ASK, ANS
def get_splits(self, device, batch_size):
train_dataset = torchtext.data.Dataset(self.train_examples,
self.fields)
train_dataset.sort_key = lambda example: len(example.input)
dev_dataset = torchtext.data.Dataset(self.dev_examples, self.fields)
dev_dataset.sort_key = lambda example: len(example.input)
test_dataset = torchtext.data.Dataset(self.test_examples, self.fields)
test_dataset.sort_key = lambda example: len(example.input)
vectors = Vectors(name=WORD2VEC_EMBEDDING_FILE,
cache=WORD2VEC_EMBEDDING_DIR,
unk_init=torch.Tensor.zero_)
self.input_field.build_vocab(train_dataset,
dev_dataset,
test_dataset,
vectors=vectors)
self.query_field.build_vocab(train_dataset,
dev_dataset,
test_dataset,
vectors=vectors)
return BucketIterator.splits(
(train_dataset, dev_dataset, test_dataset),
batch_size=batch_size,
repeat=False,
shuffle=True,
sort_within_batch=True,
device=device)
def main():
train, test, field = dataset_reader(train=True, stop=900000)
evl, _ = dataset_reader(train=False, fields=field)
field.build_vocab(train, evl)
_, evl_iter = BucketIterator.splits((train, evl),
batch_sizes=(1024, 1024),
device=device,
sort_within_batch=False,
repeat=False,
sort=False)
with open('data/result.txt', 'w+') as f:
f.write('')
with open('data/2.txt', "r") as f1:
with open('data/3.txt', "r") as f2:
for i, data in tqdm.tqdm(enumerate(evl_iter),
total=evl_iter.__len__()):
with open('data/result.txt', 'a+') as f:
for b in range(data.create_time.size(0)):
start_day = field.vocab.itos[data.create_time[b]][:-2]
start_day = arrow.get('2019-' + start_day).timestamp
day = (float(f1.readline()) + float(f2.readline())) / 2
sign_day = int('%.0f' % day)
sign_hour = '15'
final = str(
arrow.get(start_day +
sign_day * 24 * 60 * 60))[:10]
final = final + ' ' + sign_hour
f.write(final + '\n')
def make_small_imdb(batch_size=8, device=-1, vectors=None):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# TEXT = data.Field(include_lengths=False, lower=True, batch_first=True)
TEXT = data.Field(tokenize=get_tokenizer("basic_english"),
init_token='<sos>',
eos_token='<eos>',
lower=True,
batch_first=False)
LABEL = data.LabelField()
datafields = [('text', TEXT), ('label', LABEL)]
train, test = TabularDataset.splits(path='.',
train='train.csv',
validation='cv.csv',
format='csv',
skip_header=True,
fields=datafields)
TEXT.build_vocab(train, test, vectors=vectors, max_size=30000)
LABEL.build_vocab(train, test)
train_iter, test_iter = BucketIterator.splits(
(train, test),
batch_sizes=(128, 128),
device=device,
sort_key=lambda x: len(x.text),
sort_within_batch=False,
repeat=False)
return train_iter, test_iter, TEXT, LABEL
def main_handler(cls, config, data_dir, shuffle=True):
# Getting Data Splits: train, dev, test
print("\n\n==>> Loading Data splits and tokenizing each document....")
train, val, test = cls.get_dataset_splits(data_dir)
# Build Vocabulary and obtain embeddings for each word in Vocabulary
print("\n==>> Building Vocabulary and obtaining embeddings....")
glove_embeds = torchtext.vocab.Vectors(name=config['glove_path'],
max_vectors=int(4e5))
cls.TEXT.build_vocab(train, val, test, vectors=glove_embeds)
# Setting 'unk' token as the average of all other embeddings
if config['model_name'] != 'han':
cls.TEXT.vocab.vectors[cls.TEXT.vocab.stoi['<unk>']] = torch.mean(
cls.TEXT.vocab.vectors, dim=0)
# Getting iterators for each set
print("\n==>> Preparing Iterators....")
train_iter, val_iter, test_iter = BucketIterator.splits(
(train, val, test),
batch_size=config['batch_size'],
repeat=False,
shuffle=shuffle,
sort_within_batch=False,
device=device)
return cls.TEXT, cls.LABEL, train_iter, val_iter, test_iter, train, val, test
def iterator_construction(
train: TranslationDataset, valid: TranslationDataset,
test: TranslationDataset, batch_sizes: Tuple[int, int, int],
device: Union[str, torch.device]
) -> Tuple[BucketIterator, BucketIterator, BucketIterator]:
"""
This function takes torchtext.data.TranslationDataset's as input and outputs the correspondent BucketIterator's,
splitting the datasets into batches. This iterator batches examples of similar lengths together, minimizing the
amount of padding needed while producing freshly shuffled batches for each new training epoch.
:param train: a torchtest.data.TranslationDataset representing the training dataset.
:param valid: a torchtest.data.TranslationDataset representing the validation dataset.
:param test: a torchtest.data.TranslationDataset representing the test dataset.
:param batch_sizes: a tuple of 3 integers, each representing the batch size for the train, validation and test set
respectively.
:param device: the device to be used for the calculations. Can be a str (e.g. 'cuda') or torch.device object.
:return: train_iter: a torchtext.data.BucketIterator, the iterator for the training dataset.
valid_iter: a torchtext.data.BucketIterator, the iterator for the validation dataset.
test_iter: a torchtext.data.BucketIterator, the iterator for the test dataset.
"""
train_iter, valid_iter, test_iter = BucketIterator.splits(
datasets=(train, valid, test),
batch_sizes=batch_sizes,
sort=True,
device=device)
return train_iter, valid_iter, test_iter
def get_data():
SRC = Field(tokenize=tokenize_de,
init_token='<sos>',
eos_token='<eos>',
pad_token='<pad>',
unk_token='<unk>',
lower=True)
TRG = Field(tokenize=tokenize_en,
init_token='<sos>',
eos_token='<eos>',
pad_token='<pad>',
unk_token='<unk>',
lower=True)
train_data, valid_data, test_data = Multi30k.splits(exts=('.de', '.en'),
fields=(SRC, TRG))
print("train: {}".format(len(train_data.examples)))
print("valid: {}".format(len(valid_data.examples)))
print("test: {}".format(len(test_data.examples)))
SRC.build_vocab(train_data, min_freq=params.MIN_FREQ)
TRG.build_vocab(train_data, min_freq=params.MIN_FREQ)
print("源语言词表大小: {}".format(len(SRC.vocab)))
print("目标语言词表大小: {}".format(len(TRG.vocab)))
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
train_iterator, valid_iterator, test_iterator = BucketIterator.splits(
(train_data, valid_data, test_data),
batch_size=params.BATCH_SIZE,
device=device)
return train_iterator, valid_iterator, test_iterator, SRC, TRG
def get_dataset(train_df, valid_df, batch_size, cache=None, gpus=1, vectors=None):
TEXT = data.Field(init_token='<START>', eos_token='<END>', tokenize=None, tokenizer_language='en',
batch_first=True, lower=True, stop_words=set(string.punctuation))
LABEL = data.Field(dtype=torch.float, is_target=True, unk_token=None, sequential=False, use_vocab=False)
train_dataset = DataFrameDataset(train_df, {
'text': TEXT,
'label': LABEL
})
val_dataset = DataFrameDataset(valid_df, {
'text': TEXT,
'label': LABEL
})
train_loader, val_loader = BucketIterator.splits(
(train_dataset, val_dataset),
batch_size=batch_size,
sort_key=lambda x: len(x.text),
device='cuda' if torch.cuda.is_available() and gpus else 'cpu'
)
embeddings = vectors if vectors is not None else GloVe('42B', cache=cache)
TEXT.build_vocab(train_dataset.text, vectors=embeddings)
return TEXT, LABEL, train_loader, val_loader
def build_iterator(self, train, val, test):
train_iter, valid_iter, test_iter = \
BucketIterator.splits((train, val, test), batch_size=32,
sort_key=lambda x: (len(x.orig), len(x.para)),
sort_within_batch=True, repeat=False,
device=self.device)
return train_iter, valid_iter, test_iter
def build_data(train_file, test_file):
TEXT = Field(sequential=True, tokenize=tokenize, lower=False)
LABELS = Field(sequential=False, use_vocab=True)
datafields = [('word', TEXT), ('label', LABELS), ('left', TEXT),
('right', TEXT)]
train_data, valid_data = TabularDataset.splits(
path='data',
train='sample_train.txt',
validation='sample_train.txt',
format='tsv',
skip_header=False,
fields=datafields)
TEXT.build_vocab(train_data)
LABELS.build_vocab(train_data)
train_iter, valid_iter = BucketIterator.splits(
(train_data, valid_data),
batch_size=(64, 64),
device=-1,
sort_key=lambda x: len(x.word),
sort_within_batch=False,
repeat=False)
# print(next(train_iter.__iter__()))
return train_iter, valid_iter
def create_dataset(path_to_dataset,batch_size,split_ratio=0.7,min_vocab_freq=10,max_vocab_size=4000):
text_field = Field(tokenize="spacy",tokenizer_language="en",batch_first=True,init_token="<sos>",eos_token="<eos>",lower=True)
def transform(caption):
caption = caption.strip().lower().split()
return caption
dataset = CocoCaptions(annFile=os.path.join(path_to_dataset,"captions_train2014.json"),text_field=text_field,transform=transform)
train,val = dataset.split(split_ratio=split_ratio)
test = CocoCaptions(annFile=os.path.join(path_to_dataset,"captions_val2014.json"),text_field=text_field,transform=transform)
print("Dataset loaded")
print("Train set size:",len(train))
text_field.build_vocab(dataset.text,min_freq=min_vocab_freq,max_size=max_vocab_size)
SOS_TOKEN = text_field.vocab.stoi['<sos>']
EOS_TOKEN = text_field.vocab.stoi['<eos>']
UNK_TOKEN = text_field.vocab.stoi['<unk>']
PAD_TOKEN = text_field.vocab.stoi['<pad>']
print("Vocabuly build")
print("Vocabuly statistics")
print("\nMost common words in the vocabulary:\n",text_field.vocab.freqs.most_common(10))
print("Size of the vocabulary:",len(text_field.vocab))
print("Max sequence lenght",dataset.max_seq_len)
train_iter,val_iter = BucketIterator.splits((train,val),repeat=False,batch_size=batch_size)
test_iter = BucketIterator(test,batch_size=batch_size,repeat=False,train=False)
vocab_dict = text_field.vocab.stoi
return {"data_iters":(train_iter,val_iter,test_iter),"fields":text_field,
"word_to_num_vocab":vocab_dict,"num_to_word_vocab":{y:x for x,y in vocab_dict.items()},
"num_classes":len(text_field.vocab),"tokens":(SOS_TOKEN,EOS_TOKEN,UNK_TOKEN,PAD_TOKEN),"max_seq_len":dataset.max_seq_len}
def get_iterator(dataset, device, batch_size, shuffle=True, repeat=False):
train, val, test = dataset
train_iter, val_iter = BucketIterator.splits(
(train, val),
batch_size=batch_size,
device=device,
sort_key=lambda x: len(x.comment_text),
sort_within_batch=False,
shuffle=shuffle,
repeat=repeat)
test_iter = Iterator(test,
batch_size=batch_size,
device=device,
sort_within_batch=False,
repeat=repeat,
sort=False)
train_dl = BatchWrapper(train_iter, "comment_text", [
"toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"
])
valid_dl = BatchWrapper(val_iter, "comment_text", [
"toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"
])
test_dl = BatchWrapper(test_iter, "comment_text", None)
return train_dl, valid_dl, test_dl
def load_dataset(batch_size):
spacy_de = spacy.load('de')
spacy_en = spacy.load('en')
url = re.compile('(<url>.*</url>)')
def tokenize_de(text):
return [tok.text for tok in spacy_de.tokenizer(url.sub('@URL@', text))]
def tokenize_en(text):
return [tok.text for tok in spacy_en.tokenizer(url.sub('@URL@', text))]
DE = Field(tokenize=tokenize_de,
include_lengths=True,
init_token='<sos>',
eos_token='<eos>')
EN = Field(tokenize=tokenize_en,
include_lengths=True,
init_token='<sos>',
eos_token='<eos>')
train, val, test = Multi30k.splits(exts=('.de', '.en'), fields=(DE, EN))
DE.build_vocab(train.src, min_freq=2)
EN.build_vocab(train.trg, max_size=10000)
train_iter, val_iter, test_iter = BucketIterator.splits(
(train, val, test), batch_size=batch_size, repeat=False)
return train_iter, val_iter, test_iter, DE, EN
def create_iterator(self, device):
# create iterators that outputs a batch of data samples
return BucketIterator.splits(
(self.train_data, self.valid_data, self.test_data),
batch_size=self.batch_size,
device=device)
def load_dataset(batch_size):
spacy_de = spacy.load('de')
spacy_en = spacy.load('en')
url = re.compile('(<url>.*</url>)')
def tokenize_de(text):
return [tok.text for tok in spacy_de.tokenizer(url.sub('@URL@', text))]
def tokenize_en(text):
return [tok.text for tok in spacy_en.tokenizer(url.sub('@URL@', text))]
DE = Field(tokenize=tokenize_de, include_lengths=True,
init_token='<sos>', eos_token='<eos>')
EN = Field(tokenize=tokenize_en, include_lengths=True,
init_token='<sos>', eos_token='<eos>')
train, val, test = Multi30k.splits(exts=('.de', '.en'), fields=(DE, EN))
DE.build_vocab(train.src, min_freq=2)
EN.build_vocab(train.trg, max_size=10000)
train_iter, val_iter, test_iter = BucketIterator.splits(
(train, val, test), batch_size=batch_size, repeat=False)
return train_iter, val_iter, test_iter, DE, EN
skip_header=True,
fields=[('id',None),('text',TEXT),('label',LABEL)],
filter_pred = lambda x: True if len(x.text) > 1 else False)
# 토큰 레벨 문장의 길이가 1 이상인 경우만 허용
TEXT.build_vocab(train_data,min_freq=2)
LABEL.build_vocab(train_data)
# print (TEXT.vocab)
# print (len(TEXT.vocab),len(LABEL.vocab))
# print (TEXT.vocab.itos[:5])
# print (LABEL.vocab.itos)
train_loader, test_loader = BucketIterator.splits((train_data,test_data),sort_key=lambda x:len(x.text), sort_within_batch=True,
repeat=False,shuffle=True,
batch_size=32,device=DEVICE)
for batch in train_loader:
break
EPOCH = 5
BATCH_SIZE = 32
EMBED = 300
KERNEL_SIZES = [3,4,5]
KERNEL_DIM = 100
LR = 0.001
# model = CNNClassifier(len(TEXT.vocab), EMBED, 1, KERNEL_DIM, KERNEL_SIZES)
model = RNN(len(TEXT.vocab), EMBED, KERNEL_DIM, 1, bidirec=False)
# multi30k dataloader
train,val,test = datasets.Multi30k.splits(exts=(".en",".de"),fields=(EN,DE),root=data_path)
# wmt14 dataloader (better than using datasets.WMT14.splits since it's slow)
#train,val,test = datasets.TranslationDataset.splits(exts=(".en",".de"),fields=[("src",EN),("trg",DE)],path=os.path.join(data_path,"wmt14"),
# train="train.tok.clean.bpe.32000",validation="newstest2013.tok.bpe.32000",test="newstest2014.tok.bpe.32000")
print("Dataset loaded")
EN.build_vocab(train.src,min_freq=3)
DE.build_vocab(train.trg,max_size=50000)
print("Vocabularies build")
train_iter,val_iter = BucketIterator.splits((train, val),batch_size=3)
test_iter = BucketIterator(test,batch_size=3)
print("Start iterating through data")
for i,batch in enumerate(train_iter):
print(batch.src) # the source language
print(batch.trg) # the target language
break
for i,batch in enumerate(val_iter):
print(batch.src) # the source language
print(batch.trg) # the target language
break
for i,batch in enumerate(test_iter):
