def test_multiple_iterations(self):
nce_data = NCEData(self.dataset,
batch_size=16,
context_size=3,
num_noise_words=3,
max_size=1,
num_workers=1)
num_batches = len(nce_data)
nce_data.start()
nce_generator = nce_data.get_generator()
iter0_targets = []
for _ in range(num_batches):
batch = next(nce_generator)
iter0_targets.append([x[0] for x in batch.target_noise_ids])
iter1_targets = []
for _ in range(num_batches):
batch = next(nce_generator)
iter1_targets.append([x[0] for x in batch.target_noise_ids])
for ts0, ts1 in zip(iter0_targets, iter1_targets):
for t0, t1 in zip(ts0, ts0):
self.assertEqual(t0, t1)
nce_data.stop()
def test_parallel(self):
# serial version has max_size=3, because in the parallel version two
# processes advance the state before they are blocked by the queue.put()
nce_data = NCEData(self.dataset,
batch_size=32,
context_size=5,
num_noise_words=1,
max_size=3,
num_workers=1)
nce_data.start()
time.sleep(1)
nce_data.stop()
state_serial = nce_data._generator._state
nce_data = NCEData(self.dataset,
batch_size=32,
context_size=5,
num_noise_words=1,
max_size=2,
num_workers=2)
nce_data.start()
time.sleep(1)
nce_data.stop()
state_parallel = nce_data._generator._state
self.assertEqual(state_parallel._doc_id.value,
state_serial._doc_id.value)
self.assertEqual(state_parallel._in_doc_pos.value,
state_serial._in_doc_pos.value)
def test_multiple_iterations(self):
nce_data = NCEData(
self.dataset,
batch_size=16,
context_size=3,
num_noise_words=3,
max_size=1,
num_workers=1)
num_batches = len(nce_data)
nce_data.start()
nce_generator = nce_data.get_batch()
iter0_targets = []
for _ in range(num_batches):
batch = next(nce_generator)
iter0_targets.append([x[0] for x in batch.target_noise_ids])
iter1_targets = []
for _ in range(num_batches):
batch = next(nce_generator)
iter1_targets.append([x[0] for x in batch.target_noise_ids])
for ts0, ts1 in zip(iter0_targets, iter1_targets):
for t0, t1 in zip(ts0, ts0):
self.assertEqual(t0, t1)
nce_data.stop()
def test_parallel(self):
# serial version has max_size=3, because in the parallel version two
# processes advance the state before they are blocked by the queue.put()
nce_data = NCEData(
self.dataset,
batch_size=32,
context_size=5,
num_noise_words=1,
max_size=3,
num_workers=1)
nce_data.start()
time.sleep(1)
nce_data.stop()
state_serial = nce_data._generator._state
nce_data = NCEData(
self.dataset,
batch_size=32,
context_size=5,
num_noise_words=1,
max_size=2,
num_workers=2)
nce_data.start()
time.sleep(1)
nce_data.stop()
state_parallel = nce_data._generator._state
self.assertEqual(
state_parallel._doc_id.value,
state_serial._doc_id.value)
self.assertEqual(
state_parallel._in_doc_pos.value,
state_serial._in_doc_pos.value)
def test_no_context(self):
nce_data = NCEData(
self.dataset,
batch_size=16,
context_size=0,
num_noise_words=3,
max_size=1,
num_workers=1)
nce_data.start()
nce_generator = nce_data.get_generator()
batch = next(nce_generator)
nce_data.stop()
self.assertEqual(batch.context_ids, None)
def test_tensor_sizes(self):
nce_data = NCEData(self.dataset,
batch_size=32,
context_size=5,
num_noise_words=3,
max_size=1,
num_workers=1)
nce_data.start()
nce_generator = nce_data.get_generator()
batch = next(nce_generator)
nce_data.stop()
self.assertEqual(batch.context_ids.size()[0], 32)
self.assertEqual(batch.context_ids.size()[1], 10)
self.assertEqual(batch.doc_ids.size()[0], 32)
self.assertEqual(batch.target_noise_ids.size()[0], 32)
self.assertEqual(batch.target_noise_ids.size()[1], 4)
def _num_examples_with_batch_size(self, batch_size):
nce_data = NCEData(self.dataset,
batch_size=batch_size,
context_size=2,
num_noise_words=3,
max_size=1,
num_workers=1)
num_batches = len(nce_data)
nce_data.start()
nce_generator = nce_data.get_generator()
total = 0
for _ in range(num_batches):
batch = next(nce_generator)
total += len(batch)
nce_data.stop()
return total
def _num_examples_with_batch_size(self, batch_size):
nce_data = NCEData(
self.dataset,
batch_size=batch_size,
context_size=2,
num_noise_words=3,
max_size=1,
num_workers=1)
num_batches = len(nce_data)
nce_data.start()
nce_generator = nce_data.get_batch()
total = 0
for _ in range(num_batches):
batch = next(nce_generator)
total += len(batch)
nce_data.stop()
return total
def test_tensor_sizes(self):
nce_data = NCEData(
self.dataset,
batch_size=32,
context_size=5,
num_noise_words=3,
max_size=1,
num_workers=1)
nce_data.start()
nce_generator = nce_data.get_batch()
batch = next(nce_generator)
nce_data.stop()
self.assertEqual(batch.context_ids.size()[0], 32)
self.assertEqual(batch.context_ids.size()[1], 10)
self.assertEqual(batch.doc_ids.size()[0], 32)
self.assertEqual(batch.target_noise_ids.size()[0], 32)
self.assertEqual(batch.target_noise_ids.size()[1], 4)
def test_different_batch_sizes(self):
nce_data = NCEData(
self.dataset,
batch_size=16,
context_size=1,
num_noise_words=3,
max_size=1,
num_workers=1)
num_batches = len(nce_data)
nce_data.start()
nce_generator = nce_data.get_generator()
targets0 = []
for _ in range(num_batches):
batch = next(nce_generator)
for ts in batch.target_noise_ids:
targets0.append(ts[0])
nce_data.stop()
nce_data = NCEData(
self.dataset,
batch_size=19,
context_size=1,
num_noise_words=3,
max_size=1,
num_workers=1)
num_batches = len(nce_data)
nce_data.start()
nce_generator = nce_data.get_generator()
targets1 = []
for _ in range(num_batches):
batch = next(nce_generator)
for ts in batch.target_noise_ids:
targets1.append(ts[0])
nce_data.stop()
for t0, t1 in zip(targets0, targets1):
self.assertEqual(t0, t1)
def test_different_batch_sizes(self):
nce_data = NCEData(
self.dataset,
batch_size=16,
context_size=1,
num_noise_words=3,
max_size=1,
num_workers=1)
num_batches = len(nce_data)
nce_data.start()
nce_generator = nce_data.get_batch()
targets0 = []
for _ in range(num_batches):
batch = next(nce_generator)
for ts in batch.target_noise_ids:
targets0.append(ts[0])
nce_data.stop()
nce_data = NCEData(
self.dataset,
batch_size=19,
context_size=1,
num_noise_words=3,
max_size=1,
num_workers=1)
num_batches = len(nce_data)
nce_data.start()
nce_generator = nce_data.get_batch()
targets1 = []
for _ in range(num_batches):
batch = next(nce_generator)
for ts in batch.target_noise_ids:
targets1.append(ts[0])
nce_data.stop()
for t0, t1 in zip(targets0, targets1):
self.assertEqual(t0, t1)
def start(data_file_name,
context_size,
num_noise_words,
vec_dim,
num_epochs,
batch_size,
lr,
model_ver='dm',
vec_combine_method='sum',
save_all=False,
max_generated_batches=5,
num_workers=1):
"""Trains a new model. The latest checkpoint and the best performing
model are saved in the *models* directory.
Parameters
----------
data_file_name: str
Name of a file in the *data* directory.
context_size: int
Half the size of a neighbourhood of target words (i.e. how many
words left and right are regarded as context).
num_noise_words: int
Number of noise words to sample from the noise distribution.
vec_dim: int
Dimensionality of vectors to be learned (for paragraphs and words).
num_epochs: int
Number of iterations to train the model (i.e. number
of times every example is seen during training).
batch_size: int
Number of examples per single gradient update.
lr: float
Learning rate of the Adam optimizer.
model_ver: str, one of ('dm', 'dbow'), default='dm'
Version of the model as proposed by Q. V. Le et al., Distributed
Representations of Sentences and Documents. 'dm' stands for
Distributed Memory, 'dbow' stands for Distributed Bag Of Words.
Currently only the 'dm' version is implemented.
But according to [doc2vec paper](http://proceedings.mlr.press/v32/le14.pdf) and [empirical analysis](https://arxiv.org/pdf/1607.05368.pdf), 'dbow' is running better
vec_combine_method: str, one of ('sum', 'concat'), default='sum'
Method for combining paragraph and word vectors in the 'dm' model.
Currently only the 'sum' operation is implemented.
save_all: bool, default=False
Indicates whether a checkpoint is saved after each epoch.
If false, only the best performing model is saved.
max_generated_batches: int, default=5
Maximum number of pre-generated batches.
num_workers: int, default=1
Number of batch generator jobs to run in parallel. If value is set
to -1 number of machine cores are used.
"""
assert model_ver in ('dm', 'dbow')
assert vec_combine_method in ('sum', 'concat')
init_logging('../experiments/experiments.{0}.id={1}.log'.format(
'doc2vec', time.strftime('%Y%m%d-%H%M%S',
time.localtime(time.time()))))
dataset = load_dataset(data_file_name)
nce_data = NCEData(dataset, batch_size, context_size, num_noise_words,
max_generated_batches, num_workers)
nce_data.start()
try:
_run(nce_data, data_file_name, dataset, nce_data.get_batch(),
len(nce_data), nce_data.vocabulary_size(),
nce_data.number_examples, context_size, num_noise_words, vec_dim,
num_epochs, batch_size, lr, model_ver, vec_combine_method,
save_all)
except KeyboardInterrupt:
nce_data.stop()
def start(data_file_name,
context_size,
num_noise_words,
vec_dim,
num_epochs,
batch_size,
lr,
model_ver='dm',
vec_combine_method='sum',
save_all=False,
max_generated_batches=5,
num_workers=1):
"""Trains a new model. The latest checkpoint and the best performing
model are saved in the *models* directory.
Parameters
----------
data_file_name: str
Name of a file in the *data* directory.
context_size: int
Half the size of a neighbourhood of target words (i.e. how many
words left and right are regarded as context).
num_noise_words: int
Number of noise words to sample from the noise distribution.
vec_dim: int
Dimensionality of vectors to be learned (for paragraphs and words).
num_epochs: int
Number of iterations to train the model (i.e. number
of times every example is seen during training).
batch_size: int
Number of examples per single gradient update.
lr: float
Learning rate of the Adam optimizer.
model_ver: str, one of ('dm', 'dbow'), default='dm'
Version of the model as proposed by Q. V. Le et al., Distributed
Representations of Sentences and Documents. 'dm' stands for
Distributed Memory, 'dbow' stands for Distributed Bag Of Words.
Currently only the 'dm' version is implemented.
But according to [doc2vec paper](http://proceedings.mlr.press/v32/le14.pdf) and [empirical analysis](https://arxiv.org/pdf/1607.05368.pdf), 'dbow' is running better
vec_combine_method: str, one of ('sum', 'concat'), default='sum'
Method for combining paragraph and word vectors in the 'dm' model.
Currently only the 'sum' operation is implemented.
save_all: bool, default=False
Indicates whether a checkpoint is saved after each epoch.
If false, only the best performing model is saved.
max_generated_batches: int, default=5
Maximum number of pre-generated batches.
num_workers: int, default=1
Number of batch generator jobs to run in parallel. If value is set
to -1 number of machine cores are used.
"""
assert model_ver in ('dm', 'dbow')
assert vec_combine_method in ('sum', 'concat')
init_logging('../experiments/experiments.{0}.id={1}.log'.format('doc2vec', time.strftime('%Y%m%d-%H%M%S', time.localtime(time.time()))))
dataset = load_dataset(data_file_name)
nce_data = NCEData(
dataset,
batch_size,
context_size,
num_noise_words,
max_generated_batches,
num_workers)
nce_data.start()
try:
_run(nce_data, data_file_name, dataset, nce_data.get_batch(), len(nce_data),
nce_data.vocabulary_size(), nce_data.number_examples, context_size, num_noise_words, vec_dim,
num_epochs, batch_size, lr, model_ver, vec_combine_method,
save_all)
except KeyboardInterrupt:
nce_data.stop()
def start(data_file_name,
num_noise_words,
vec_dim,
num_epochs,
batch_size,
lr,
model_ver='dbow',
context_size=0,
vec_combine_method='sum',
save_all=False,
generate_plot=True,
max_generated_batches=5,
num_workers=1):
"""Trains a new model. The latest checkpoint and the best performing
model are saved in the *models* directory.
Parameters
----------
data_file_name: str
Name of a file in the *data* directory.
model_ver: str, one of ('dm', 'dbow'), default='dbow'
Version of the model as proposed by Q. V. Le et al., Distributed
Representations of Sentences and Documents. 'dbow' stands for
Distributed Bag Of Words, 'dm' stands for Distributed Memory.
vec_combine_method: str, one of ('sum', 'concat'), default='sum'
Method for combining paragraph and word vectors when model_ver='dm'.
Currently only the 'sum' operation is implemented.
context_size: int, default=0
Half the size of a neighbourhood of target words when model_ver='dm'
(i.e. how many words left and right are regarded as context). When
model_ver='dm' context_size has to greater than 0, when
model_ver='dbow' context_size has to be 0.
num_noise_words: int
Number of noise words to sample from the noise distribution.
vec_dim: int
Dimensionality of vectors to be learned (for paragraphs and words).
num_epochs: int
Number of iterations to train the model (i.e. number
of times every example is seen during training).
batch_size: int
Number of examples per single gradient update.
lr: float
Learning rate of the Adam optimizer.
save_all: bool, default=False
Indicates whether a checkpoint is saved after each epoch.
If false, only the best performing model is saved.
generate_plot: bool, default=True
Indicates whether a diagnostic plot displaying loss value over
epochs is generated after each epoch.
max_generated_batches: int, default=5
Maximum number of pre-generated batches.
num_workers: int, default=1
Number of batch generator jobs to run in parallel. If value is set
to -1 number of machine cores are used.
"""
if model_ver not in ('dm', 'dbow'):
raise ValueError("Invalid version of the model")
model_ver_is_dbow = model_ver == 'dbow'
if model_ver_is_dbow and context_size != 0:
raise ValueError("Context size has to be zero when using dbow")
if not model_ver_is_dbow:
if vec_combine_method not in ('sum', 'concat'):
raise ValueError("Invalid method for combining paragraph and word "
"vectors when using dm")
if context_size <= 0:
raise ValueError("Context size must be positive when using dm")
dataset = load_dataset(data_file_name)
nce_data = NCEData(dataset, batch_size, context_size, num_noise_words,
max_generated_batches, num_workers)
nce_data.start()
try:
_run(data_file_name, dataset, nce_data.get_generator(), len(nce_data),
nce_data.vocabulary_size(), context_size, num_noise_words,
vec_dim, num_epochs, batch_size, lr, model_ver,
vec_combine_method, save_all, generate_plot, model_ver_is_dbow)
except KeyboardInterrupt:
nce_data.stop()
def start(data_file_name,
context_size,
num_noise_words,
vec_dim,
num_epochs,
batch_size,
lr,
model_ver='dm',
vec_combine_method='sum',
save_all=False,
max_generated_batches=5,
num_workers=1):
"""Trains a new model. The latest checkpoint and the best performing
model are saved in the *models* directory.
Parameters
----------
data_file_name: str
Name of a file in the *data* directory.
context_size: int
Half the size of a neighbourhood of target words (i.e. how many
words left and right are regarded as context).
num_noise_words: int
Number of noise words to sample from the noise distribution.
vec_dim: int
Dimensionality of vectors to be learned (for paragraphs and words).
num_epochs: int
Number of iterations to train the model (i.e. number
of times every example is seen during training).
batch_size: int
Number of examples per single gradient update.
lr: float
Learning rate of the SGD optimizer (uses 0.9 nesterov momentum).
model_ver: str, one of ('dm', 'dbow'), default='dm'
Version of the model as proposed by Q. V. Le et al., Distributed
Representations of Sentences and Documents. 'dm' stands for
Distributed Memory, 'dbow' stands for Distributed Bag Of Words.
Currently only the 'dm' version is implemented.
vec_combine_method: str, one of ('sum', 'concat'), default='sum'
Method for combining paragraph and word vectors in the 'dm' model.
Currently only the 'sum' operation is implemented.
save_all: bool, default=False
Indicates whether a checkpoint is saved after each epoch.
If false, only the best performing model is saved.
max_generated_batches: int, default=5
Maximum number of pre-generated batches.
num_workers: int, default=1
Number of batch generator jobs to run in parallel. If value is set
to -1 number of machine cores are used.
"""
assert model_ver in ('dm', 'dbow')
assert vec_combine_method in ('sum', 'concat')
dataset = load_dataset(data_file_name)
nce_data = NCEData(
dataset,
batch_size,
context_size,
num_noise_words,
max_generated_batches,
num_workers)
nce_data.start()
try:
_run(data_file_name, dataset, nce_data.get_generator(), len(nce_data),
nce_data.vocabulary_size(), context_size, num_noise_words, vec_dim,
num_epochs, batch_size, lr, model_ver, vec_combine_method,
save_all)
except KeyboardInterrupt:
nce_data.stop()