def get_slice_data_from_file(dataset, split, class_idx, gmm, nr_pos, nr_neg):
    samples = _get_samples(dataset, class_idx, data_type=split, nr_pos=nr_pos,
                           nr_neg=nr_neg)[0]
    len_descs = gmm.k + 2 * gmm.d * gmm.k
    sstats, labels, info = SstatsMap(
        os.path.join(dataset.SSTATS_DIR, 'stats.tmp')).get_merged(
            samples, len_descs)
    sstats = sstats.reshape((-1, len_descs))
    binary_labels = tuple_labels_to_list_labels(labels, class_idx)
    return SliceData(sstats, binary_labels, info)
Пример #2
0
def master(src_cfg, suffix_in, suffix_out, K, N, nr_processes, double_norm):
    D = 64

    dataset = Dataset(src_cfg, nr_clusters=K)
    samples = [
        str(sample) for sample in dataset.get_data('train')[0] +
        dataset.get_data('test')[0]
    ]

    if double_norm:
        worker = double_normalization
        suffix = '.double_norm'
        gmm = load_gmm(
            os.path.join(dataset.FEAT_DIR + suffix_in, 'gmm', 'gmm_%d' % K))
    else:
        worker = merge
        suffix = ''
        gmm = None

    path_in = os.path.join(dataset.FEAT_DIR + suffix_in,
                           'statistics_k_%d' % dataset.VOC_SIZE, 'stats.tmp')
    path_out = os.path.join(dataset.FEAT_DIR + suffix_out,
                            'statistics_k_%d' % dataset.VOC_SIZE,
                            'stats.tmp' + suffix)

    sstats_in = SstatsMap(path_in)
    sstats_out = SstatsMap(path_out)

    len_sstats = dataset.VOC_SIZE + 2 * D * dataset.VOC_SIZE

    kwargs = {
        'N': N,
        'sstats_in': sstats_in,
        'sstats_out': sstats_out,
        'len_sstats': len_sstats,
        'gmm': gmm
    }

    if nr_processes > 1:
        nr_samples_per_process = len(samples) / nr_processes + 1
        for ii in xrange(nr_processes):
            mp.Process(target=worker,
                       args=(samples[ii * nr_samples_per_process:(ii + 1) *
                                     nr_samples_per_process], ),
                       kwargs=kwargs).start()
    else:
        worker(samples, **kwargs)
Пример #3
0
 def setup(self):
     self.nr_dims = 10
     np.random.seed(1)
     self.data = SstatsMap('expected_results/test_data')
     # Generate some data.
     self.xx = np.random.randn(3, self.nr_dims)
     self.info = {
         'label': 4,
         'nr_slices': 3,
         'begin_frames': [1, 3, 10],
         'end_frames': [2, 9, 15],
         'nr_descs_per_slice': [10, 9, 13]}
Пример #4
0
class TestData():
    def setup(self):
        self.nr_dims = 10
        np.random.seed(1)
        self.data = SstatsMap('expected_results/test_data')
        # Generate some data.
        self.xx = np.random.randn(3, self.nr_dims)
        self.info = {
            'label': 4,
            'nr_slices': 3,
            'begin_frames': [1, 3, 10],
            'end_frames': [2, 9, 15],
            'nr_descs_per_slice': [10, 9, 13]}

    def test_write_data(self):
        self.data.write('test_1', self.xx[0])
        self.data.write('test_2', self.xx[1], info=self.info)
        self.data.write('test_3', self.xx[2], info=dict(label=10))

    def test_read_data(self):
        # Nose tests in random order.
        self.data.write('test_1', self.xx[0])
        self.data.write('test_2', self.xx[1], info=self.info)

        test_1_data = self.data.read('test_1')
        assert_array_almost_equal(self.xx[0], test_1_data)

        test_2_data = self.data.read('test_2')
        assert_array_almost_equal(self.xx[1], test_2_data)

        test_2_info = self.data.read_info('test_2')
        assert test_2_info == self.info

    @raises(Exception)
    def test_read_data_exception(self):
        self.data.read('file_that_does_not_exist')

    def test_check_data(self):
        self.data.write('test_1', self.xx)
        self.data.write('test_2', [])
        assert self.data.check(['test_1'], self.nr_dims, verbose=False) == True
        assert self.data.check(
            ['test_1'], self.nr_dims + 1, verbose=False) == False
        assert self.data.check(
            ['test_1', 'test_2'], self.nr_dims, verbose=False) == False