示例#1
0
    def test_multiple_iterations(self):
        nce_data = NCEData(self.dataset,
                           batch_size=16,
                           context_size=3,
                           num_noise_words=3,
                           max_size=1,
                           num_workers=1)
        num_batches = len(nce_data)
        nce_data.start()
        nce_generator = nce_data.get_generator()

        iter0_targets = []
        for _ in range(num_batches):
            batch = next(nce_generator)
            iter0_targets.append([x[0] for x in batch.target_noise_ids])

        iter1_targets = []
        for _ in range(num_batches):
            batch = next(nce_generator)
            iter1_targets.append([x[0] for x in batch.target_noise_ids])

        for ts0, ts1 in zip(iter0_targets, iter1_targets):
            for t0, t1 in zip(ts0, ts0):
                self.assertEqual(t0, t1)
        nce_data.stop()
示例#2
0
    def test_parallel(self):
        # serial version has max_size=3, because in the parallel version two
        # processes advance the state before they are blocked by the queue.put()
        nce_data = NCEData(self.dataset,
                           batch_size=32,
                           context_size=5,
                           num_noise_words=1,
                           max_size=3,
                           num_workers=1)
        nce_data.start()
        time.sleep(1)
        nce_data.stop()
        state_serial = nce_data._generator._state

        nce_data = NCEData(self.dataset,
                           batch_size=32,
                           context_size=5,
                           num_noise_words=1,
                           max_size=2,
                           num_workers=2)
        nce_data.start()
        time.sleep(1)
        nce_data.stop()
        state_parallel = nce_data._generator._state

        self.assertEqual(state_parallel._doc_id.value,
                         state_serial._doc_id.value)
        self.assertEqual(state_parallel._in_doc_pos.value,
                         state_serial._in_doc_pos.value)
示例#3
0
    def test_multiple_iterations(self):
        nce_data = NCEData(
            self.dataset,
            batch_size=16,
            context_size=3,
            num_noise_words=3,
            max_size=1,
            num_workers=1)
        num_batches = len(nce_data)
        nce_data.start()
        nce_generator = nce_data.get_batch()

        iter0_targets = []
        for _ in range(num_batches):
            batch = next(nce_generator)
            iter0_targets.append([x[0] for x in batch.target_noise_ids])

        iter1_targets = []
        for _ in range(num_batches):
            batch = next(nce_generator)
            iter1_targets.append([x[0] for x in batch.target_noise_ids])

        for ts0, ts1 in zip(iter0_targets, iter1_targets):
            for t0, t1 in zip(ts0, ts0):
                self.assertEqual(t0, t1)
        nce_data.stop()
示例#4
0
    def test_parallel(self):
        # serial version has max_size=3, because in the parallel version two
        # processes advance the state before they are blocked by the queue.put()
        nce_data = NCEData(
            self.dataset,
            batch_size=32,
            context_size=5,
            num_noise_words=1,
            max_size=3,
            num_workers=1)
        nce_data.start()
        time.sleep(1)
        nce_data.stop()
        state_serial = nce_data._generator._state

        nce_data = NCEData(
            self.dataset,
            batch_size=32,
            context_size=5,
            num_noise_words=1,
            max_size=2,
            num_workers=2)
        nce_data.start()
        time.sleep(1)
        nce_data.stop()
        state_parallel = nce_data._generator._state

        self.assertEqual(
            state_parallel._doc_id.value,
            state_serial._doc_id.value)
        self.assertEqual(
            state_parallel._in_doc_pos.value,
            state_serial._in_doc_pos.value)
示例#5
0
    def test_no_context(self):
        nce_data = NCEData(
            self.dataset,
            batch_size=16,
            context_size=0,
            num_noise_words=3,
            max_size=1,
            num_workers=1)
        nce_data.start()
        nce_generator = nce_data.get_generator()
        batch = next(nce_generator)
        nce_data.stop()

        self.assertEqual(batch.context_ids, None)
示例#6
0
    def test_tensor_sizes(self):
        nce_data = NCEData(self.dataset,
                           batch_size=32,
                           context_size=5,
                           num_noise_words=3,
                           max_size=1,
                           num_workers=1)
        nce_data.start()
        nce_generator = nce_data.get_generator()
        batch = next(nce_generator)
        nce_data.stop()

        self.assertEqual(batch.context_ids.size()[0], 32)
        self.assertEqual(batch.context_ids.size()[1], 10)
        self.assertEqual(batch.doc_ids.size()[0], 32)
        self.assertEqual(batch.target_noise_ids.size()[0], 32)
        self.assertEqual(batch.target_noise_ids.size()[1], 4)
示例#7
0
    def _num_examples_with_batch_size(self, batch_size):
        nce_data = NCEData(self.dataset,
                           batch_size=batch_size,
                           context_size=2,
                           num_noise_words=3,
                           max_size=1,
                           num_workers=1)
        num_batches = len(nce_data)
        nce_data.start()
        nce_generator = nce_data.get_generator()

        total = 0
        for _ in range(num_batches):
            batch = next(nce_generator)
            total += len(batch)
        nce_data.stop()
        return total
示例#8
0
    def _num_examples_with_batch_size(self, batch_size):
        nce_data = NCEData(
            self.dataset,
            batch_size=batch_size,
            context_size=2,
            num_noise_words=3,
            max_size=1,
            num_workers=1)
        num_batches = len(nce_data)
        nce_data.start()
        nce_generator = nce_data.get_batch()

        total = 0
        for _ in range(num_batches):
            batch = next(nce_generator)
            total += len(batch)
        nce_data.stop()
        return total
示例#9
0
    def test_tensor_sizes(self):
        nce_data = NCEData(
            self.dataset,
            batch_size=32,
            context_size=5,
            num_noise_words=3,
            max_size=1,
            num_workers=1)
        nce_data.start()
        nce_generator = nce_data.get_batch()
        batch = next(nce_generator)
        nce_data.stop()

        self.assertEqual(batch.context_ids.size()[0], 32)
        self.assertEqual(batch.context_ids.size()[1], 10)
        self.assertEqual(batch.doc_ids.size()[0], 32)
        self.assertEqual(batch.target_noise_ids.size()[0], 32)
        self.assertEqual(batch.target_noise_ids.size()[1], 4)
示例#10
0
    def test_different_batch_sizes(self):
        nce_data = NCEData(
            self.dataset,
            batch_size=16,
            context_size=1,
            num_noise_words=3,
            max_size=1,
            num_workers=1)
        num_batches = len(nce_data)
        nce_data.start()
        nce_generator = nce_data.get_generator()

        targets0 = []
        for _ in range(num_batches):
            batch = next(nce_generator)
            for ts in batch.target_noise_ids:
                targets0.append(ts[0])
        nce_data.stop()

        nce_data = NCEData(
            self.dataset,
            batch_size=19,
            context_size=1,
            num_noise_words=3,
            max_size=1,
            num_workers=1)
        num_batches = len(nce_data)
        nce_data.start()
        nce_generator = nce_data.get_generator()

        targets1 = []
        for _ in range(num_batches):
            batch = next(nce_generator)
            for ts in batch.target_noise_ids:
                targets1.append(ts[0])
        nce_data.stop()

        for t0, t1 in zip(targets0, targets1):
            self.assertEqual(t0, t1)
示例#11
0
    def test_different_batch_sizes(self):
        nce_data = NCEData(
            self.dataset,
            batch_size=16,
            context_size=1,
            num_noise_words=3,
            max_size=1,
            num_workers=1)
        num_batches = len(nce_data)
        nce_data.start()
        nce_generator = nce_data.get_batch()

        targets0 = []
        for _ in range(num_batches):
            batch = next(nce_generator)
            for ts in batch.target_noise_ids:
                targets0.append(ts[0])
        nce_data.stop()

        nce_data = NCEData(
            self.dataset,
            batch_size=19,
            context_size=1,
            num_noise_words=3,
            max_size=1,
            num_workers=1)
        num_batches = len(nce_data)
        nce_data.start()
        nce_generator = nce_data.get_batch()

        targets1 = []
        for _ in range(num_batches):
            batch = next(nce_generator)
            for ts in batch.target_noise_ids:
                targets1.append(ts[0])
        nce_data.stop()

        for t0, t1 in zip(targets0, targets1):
            self.assertEqual(t0, t1)
示例#12
0
def start(data_file_name,
          context_size,
          num_noise_words,
          vec_dim,
          num_epochs,
          batch_size,
          lr,
          model_ver='dm',
          vec_combine_method='sum',
          save_all=False,
          max_generated_batches=5,
          num_workers=1):
    """Trains a new model. The latest checkpoint and the best performing
    model are saved in the *models* directory.

    Parameters
    ----------
    data_file_name: str
        Name of a file in the *data* directory.

    context_size: int
        Half the size of a neighbourhood of target words (i.e. how many
        words left and right are regarded as context).

    num_noise_words: int
        Number of noise words to sample from the noise distribution.

    vec_dim: int
        Dimensionality of vectors to be learned (for paragraphs and words).

    num_epochs: int
        Number of iterations to train the model (i.e. number
        of times every example is seen during training).

    batch_size: int
        Number of examples per single gradient update.

    lr: float
        Learning rate of the Adam optimizer.

    model_ver: str, one of ('dm', 'dbow'), default='dm'
        Version of the model as proposed by Q. V. Le et al., Distributed
        Representations of Sentences and Documents. 'dm' stands for
        Distributed Memory, 'dbow' stands for Distributed Bag Of Words.
        Currently only the 'dm' version is implemented.

        But according to [doc2vec paper](http://proceedings.mlr.press/v32/le14.pdf) and [empirical analysis](https://arxiv.org/pdf/1607.05368.pdf), 'dbow' is running better

    vec_combine_method: str, one of ('sum', 'concat'), default='sum'
        Method for combining paragraph and word vectors in the 'dm' model.
        Currently only the 'sum' operation is implemented.

    save_all: bool, default=False
        Indicates whether a checkpoint is saved after each epoch.
        If false, only the best performing model is saved.

    max_generated_batches: int, default=5
        Maximum number of pre-generated batches.

    num_workers: int, default=1
        Number of batch generator jobs to run in parallel. If value is set
        to -1 number of machine cores are used.
    """
    assert model_ver in ('dm', 'dbow')
    assert vec_combine_method in ('sum', 'concat')

    init_logging('../experiments/experiments.{0}.id={1}.log'.format(
        'doc2vec', time.strftime('%Y%m%d-%H%M%S',
                                 time.localtime(time.time()))))

    dataset = load_dataset(data_file_name)
    nce_data = NCEData(dataset, batch_size, context_size, num_noise_words,
                       max_generated_batches, num_workers)
    nce_data.start()

    try:
        _run(nce_data, data_file_name, dataset, nce_data.get_batch(),
             len(nce_data), nce_data.vocabulary_size(),
             nce_data.number_examples, context_size, num_noise_words, vec_dim,
             num_epochs, batch_size, lr, model_ver, vec_combine_method,
             save_all)
    except KeyboardInterrupt:
        nce_data.stop()
示例#13
0
def start(data_file_name,
          context_size,
          num_noise_words,
          vec_dim,
          num_epochs,
          batch_size,
          lr,
          model_ver='dm',
          vec_combine_method='sum',
          save_all=False,
          max_generated_batches=5,
          num_workers=1):
    """Trains a new model. The latest checkpoint and the best performing
    model are saved in the *models* directory.

    Parameters
    ----------
    data_file_name: str
        Name of a file in the *data* directory.

    context_size: int
        Half the size of a neighbourhood of target words (i.e. how many
        words left and right are regarded as context).

    num_noise_words: int
        Number of noise words to sample from the noise distribution.

    vec_dim: int
        Dimensionality of vectors to be learned (for paragraphs and words).

    num_epochs: int
        Number of iterations to train the model (i.e. number
        of times every example is seen during training).

    batch_size: int
        Number of examples per single gradient update.

    lr: float
        Learning rate of the Adam optimizer.

    model_ver: str, one of ('dm', 'dbow'), default='dm'
        Version of the model as proposed by Q. V. Le et al., Distributed
        Representations of Sentences and Documents. 'dm' stands for
        Distributed Memory, 'dbow' stands for Distributed Bag Of Words.
        Currently only the 'dm' version is implemented.

        But according to [doc2vec paper](http://proceedings.mlr.press/v32/le14.pdf) and [empirical analysis](https://arxiv.org/pdf/1607.05368.pdf), 'dbow' is running better

    vec_combine_method: str, one of ('sum', 'concat'), default='sum'
        Method for combining paragraph and word vectors in the 'dm' model.
        Currently only the 'sum' operation is implemented.

    save_all: bool, default=False
        Indicates whether a checkpoint is saved after each epoch.
        If false, only the best performing model is saved.

    max_generated_batches: int, default=5
        Maximum number of pre-generated batches.

    num_workers: int, default=1
        Number of batch generator jobs to run in parallel. If value is set
        to -1 number of machine cores are used.
    """
    assert model_ver in ('dm', 'dbow')
    assert vec_combine_method in ('sum', 'concat')

    init_logging('../experiments/experiments.{0}.id={1}.log'.format('doc2vec', time.strftime('%Y%m%d-%H%M%S', time.localtime(time.time()))))

    dataset = load_dataset(data_file_name)
    nce_data = NCEData(
        dataset,
        batch_size,
        context_size,
        num_noise_words,
        max_generated_batches,
        num_workers)
    nce_data.start()

    try:
        _run(nce_data, data_file_name, dataset, nce_data.get_batch(), len(nce_data),
             nce_data.vocabulary_size(), nce_data.number_examples, context_size, num_noise_words, vec_dim,
             num_epochs, batch_size, lr, model_ver, vec_combine_method,
             save_all)
    except KeyboardInterrupt:
        nce_data.stop()
def start(data_file_name,
          num_noise_words,
          vec_dim,
          num_epochs,
          batch_size,
          lr,
          model_ver='dbow',
          context_size=0,
          vec_combine_method='sum',
          save_all=False,
          generate_plot=True,
          max_generated_batches=5,
          num_workers=1):
    """Trains a new model. The latest checkpoint and the best performing
    model are saved in the *models* directory.

    Parameters
    ----------
    data_file_name: str
        Name of a file in the *data* directory.

    model_ver: str, one of ('dm', 'dbow'), default='dbow'
        Version of the model as proposed by Q. V. Le et al., Distributed
        Representations of Sentences and Documents. 'dbow' stands for
        Distributed Bag Of Words, 'dm' stands for Distributed Memory.

    vec_combine_method: str, one of ('sum', 'concat'), default='sum'
        Method for combining paragraph and word vectors when model_ver='dm'.
        Currently only the 'sum' operation is implemented.

    context_size: int, default=0
        Half the size of a neighbourhood of target words when model_ver='dm'
        (i.e. how many words left and right are regarded as context). When
        model_ver='dm' context_size has to greater than 0, when
        model_ver='dbow' context_size has to be 0.

    num_noise_words: int
        Number of noise words to sample from the noise distribution.

    vec_dim: int
        Dimensionality of vectors to be learned (for paragraphs and words).

    num_epochs: int
        Number of iterations to train the model (i.e. number
        of times every example is seen during training).

    batch_size: int
        Number of examples per single gradient update.

    lr: float
        Learning rate of the Adam optimizer.

    save_all: bool, default=False
        Indicates whether a checkpoint is saved after each epoch.
        If false, only the best performing model is saved.

    generate_plot: bool, default=True
        Indicates whether a diagnostic plot displaying loss value over
        epochs is generated after each epoch.

    max_generated_batches: int, default=5
        Maximum number of pre-generated batches.

    num_workers: int, default=1
        Number of batch generator jobs to run in parallel. If value is set
        to -1 number of machine cores are used.
    """
    if model_ver not in ('dm', 'dbow'):
        raise ValueError("Invalid version of the model")

    model_ver_is_dbow = model_ver == 'dbow'

    if model_ver_is_dbow and context_size != 0:
        raise ValueError("Context size has to be zero when using dbow")
    if not model_ver_is_dbow:
        if vec_combine_method not in ('sum', 'concat'):
            raise ValueError("Invalid method for combining paragraph and word "
                             "vectors when using dm")
        if context_size <= 0:
            raise ValueError("Context size must be positive when using dm")

    dataset = load_dataset(data_file_name)
    nce_data = NCEData(dataset, batch_size, context_size, num_noise_words,
                       max_generated_batches, num_workers)
    nce_data.start()

    try:
        _run(data_file_name, dataset, nce_data.get_generator(), len(nce_data),
             nce_data.vocabulary_size(), context_size, num_noise_words,
             vec_dim, num_epochs, batch_size, lr, model_ver,
             vec_combine_method, save_all, generate_plot, model_ver_is_dbow)
    except KeyboardInterrupt:
        nce_data.stop()
示例#15
0
def start(data_file_name,
          context_size,
          num_noise_words,
          vec_dim,
          num_epochs,
          batch_size,
          lr,
          model_ver='dm',
          vec_combine_method='sum',
          save_all=False,
          max_generated_batches=5,
          num_workers=1):
    """Trains a new model. The latest checkpoint and the best performing
    model are saved in the *models* directory.

    Parameters
    ----------
    data_file_name: str
        Name of a file in the *data* directory.

    context_size: int
        Half the size of a neighbourhood of target words (i.e. how many
        words left and right are regarded as context).

    num_noise_words: int
        Number of noise words to sample from the noise distribution.

    vec_dim: int
        Dimensionality of vectors to be learned (for paragraphs and words).

    num_epochs: int
        Number of iterations to train the model (i.e. number
        of times every example is seen during training).

    batch_size: int
        Number of examples per single gradient update.

    lr: float
        Learning rate of the SGD optimizer (uses 0.9 nesterov momentum).

    model_ver: str, one of ('dm', 'dbow'), default='dm'
        Version of the model as proposed by Q. V. Le et al., Distributed
        Representations of Sentences and Documents. 'dm' stands for
        Distributed Memory, 'dbow' stands for Distributed Bag Of Words.
        Currently only the 'dm' version is implemented.

    vec_combine_method: str, one of ('sum', 'concat'), default='sum'
        Method for combining paragraph and word vectors in the 'dm' model.
        Currently only the 'sum' operation is implemented.

    save_all: bool, default=False
        Indicates whether a checkpoint is saved after each epoch.
        If false, only the best performing model is saved.

    max_generated_batches: int, default=5
        Maximum number of pre-generated batches.

    num_workers: int, default=1
        Number of batch generator jobs to run in parallel. If value is set
        to -1 number of machine cores are used.
    """
    assert model_ver in ('dm', 'dbow')
    assert vec_combine_method in ('sum', 'concat')

    dataset = load_dataset(data_file_name)
    nce_data = NCEData(
        dataset,
        batch_size,
        context_size,
        num_noise_words,
        max_generated_batches,
        num_workers)
    nce_data.start()

    try:
        _run(data_file_name, dataset, nce_data.get_generator(), len(nce_data),
             nce_data.vocabulary_size(), context_size, num_noise_words, vec_dim,
             num_epochs, batch_size, lr, model_ver, vec_combine_method,
             save_all)
    except KeyboardInterrupt:
        nce_data.stop()