def load_dataset(transforms, batch_size, root='./data/'):
"""
Import data set
:param transforms:
:param batch_size:
:param root
:return:
"""
# Style change detection dataset, training set
pan18loader_train = torch.utils.data.DataLoader(
dataset.SCDSimpleDataset(root=root, download=True, transform=transforms, train=True),
batch_size=batch_size
)
# Style change detection dataset, validation set
pan18loader_valid = torch.utils.data.DataLoader(
dataset.SCDSimpleDataset(root=root, download=True, transform=transforms, train=False),
batch_size=batch_size
)
return pan18loader_train, pan18loader_valid
import torch.utils.data
import dataset
from echotorch.transforms import text
# Experience parameter
batch_size = 64
n_epoch = 1
window_size = 700
training_set_size = 10
test_set_size = 2
training_samples = training_set_size + test_set_size
stride = 100
# Style change detection dataset, training set
pan18loader_train = torch.utils.data.DataLoader(
dataset.SCDSimpleDataset(root='./data/', download=True, transform=text.Character(), train=True),
batch_size=1
)
# Style change detection dataset, validation set
pan18loader_valid = torch.utils.data.DataLoader(
dataset.SCDSimpleDataset(root='./data/', download=True, transform=text.Character(), train=False),
batch_size=1
)
# Get training data
for i, data in enumerate(pan18loader_train):
# Inputs and c
inputs, label = data
# TRAINING
# Experience parameter
n_epoch = 1
window_size = 700
training_set_size = 10
test_set_size = 2
training_samples = training_set_size + test_set_size
stride = 100
# Argument parser
args = functions.argument_parser_training_model()
# Get transforms
transforms = functions.text_transformer(args.n_gram, settings.window_size)
# Style change detection dataset, training set
pan18loader_train = torch.utils.data.DataLoader(dataset.SCDSimpleDataset(
root='./extended/', download=True, transform=transforms, train=True),
batch_size=1)
# Style change detection dataset, validation set
pan18loader_valid = torch.utils.data.DataLoader(dataset.SCDSimpleDataset(
root='./extended/', download=True, transform=transforms, train=False),
batch_size=1)
# Loss function
loss_function = nn.CrossEntropyLoss()
# Bi-directional Embedding GRU
model = models.BiEGRU(window_size=settings.window_size,
vocab_size=settings.voc_sizes[args.n_gram],
hidden_dim=settings.hidden_dim,
n_classes=2)
# Imports
import torch.utils.data
import dataset
from echotorch.transforms import text
# Experience parameter
batch_size = 64
n_epoch = 1
window_size = 700
training_set_size = 10
test_set_size = 2
training_samples = training_set_size + test_set_size
stride = 100
# Style change detection dataset, training set
pan18loader_train = torch.utils.data.DataLoader(dataset.SCDSimpleDataset(
root='./data/', download=True, transform=text.Character(), train=True),
batch_size=1)
# Style change detection dataset, validation set
pan18loader_valid = torch.utils.data.DataLoader(dataset.SCDSimpleDataset(
root='./data/', download=True, transform=text.Character(), train=False),
batch_size=1)
# For each epoch
for epoch in range(n_epoch):
# Training loss
training_loss = 0.0
# Get training data
for i, data in enumerate(pan18loader_train):
# Inputs and c
# Argument parser
args = functions.argument_parser_training_model()
# Get transforms
transforms = functions.text_transformer(args.n_gram, settings.window_size)
# Style change detection dataset, training set
pan18loader_train = torch.utils.data.DataLoader(
dataset.SCDPartsDataset(root='./data/', download=True, transform=transforms, train=True),
batch_size=1
)
# Style change detection dataset, validation set
pan18loader_valid = torch.utils.data.DataLoader(
dataset.SCDSimpleDataset(root='./data/', download=True, transform=transforms, train=False),
batch_size=1
)
# Samples
samples = list()
n_samples = 0
# Get training data
for i, data in enumerate(pan18loader_train):
# Parts and c
parts, _ = data
# Add to samples
samples.append(parts)
n_samples += 1
ltransforms.MaxIndex(max_id=settings.voc_sizes[args.n_gram])
])
else:
transforms = ltransforms.Compose([
ltransforms.ToLower(),
ltransforms.Character2Gram(),
ltransforms.ToIndex(start_ix=1),
ltransforms.ToLength(length=window_size),
ltransforms.Reshape((-1)),
ltransforms.MaxIndex(max_id=settings.voc_sizes[args.n_gram])
])
# end if
# Style change detection dataset, training set
pan18loader_train = torch.utils.data.DataLoader(
dataset.SCDSimpleDataset(root='./extended2/', download=True, transform=transforms, train=True),
batch_size=args.batch_size
)
# Style change detection dataset, validation set
pan18loader_valid = torch.utils.data.DataLoader(
dataset.SCDSimpleDataset(root='./extended2/', download=True, transform=transforms, train=False),
batch_size=args.batch_size
)
# Loss function
loss_function = nn.CrossEntropyLoss()
# CNN Distance learning
model = models.CNNSCD(
input_dim=window_size,
