def test_exds_saving_and_loading(): folder = testutil.ensure_empty_tmp_subfolder( 'test_exds_repository__test_saving_and_loading') definition = default_exds_definition(folder) exds = definition.create_exds() # Due to the definition provided by default_exds_definition(), the # exds will be saved after building. exds.build() # Verify if the matrices have been finalized. assert exds.matrix.final is True assert exds.matrix_train.final is True assert exds.matrix_test.final is True # Verify if the matrices can be loaded individually from the saved # ModelBuildingExperimentalDataset. # - The original matrix: loadedMatrix_original = DatasetMatrix("dataset") loadedMatrix_original.load(exds.definition.path) assert exds.matrix == loadedMatrix_original # - The training matrix: loadedMatrix_train = DatasetMatrix("dataset_train") loadedMatrix_train.load(exds.definition.path) assert exds.matrix_train == loadedMatrix_train # - The test matrix: loadedMatrix_test = DatasetMatrix("dataset_test") loadedMatrix_test.load(exds.definition.path) assert exds.matrix_test == loadedMatrix_test
def load(self): super().load() if self.matrix_train is None: self.matrix_train = DatasetMatrix("dataset_train") self.matrix_train.load(self.definition.path) if self.matrix_test is None: self.matrix_test = DatasetMatrix("dataset_test") self.matrix_test.load(self.definition.path)
def test_removing_columns_Y(): # Set up a simple DatasetMatrix dm = DatasetMatrix('testmatrix') configure_default_datasetmatrix(dm) # Remove the first column from the Y matrix. dm.delete_column_Y(0) expected_X = default_matrix_X() expected_Y = scipy.sparse.csr_matrix(numpy.array([ [102], [202], [302], [402]])) assert DatasetMatrix.sparse_equal(expected_X, dm.X) is True assert DatasetMatrix.sparse_equal(expected_Y, dm.Y) is True assert default_row_labels() == dm.row_labels assert default_column_labels_X() == dm.column_labels_X assert ['coly1'] == dm.column_labels_Y # Remove the last remaining column from the Y matrix. dm.delete_column_Y(0) expected_X = default_matrix_X() expected_Y = scipy.sparse.csr_matrix((4, 0)) assert DatasetMatrix.sparse_equal(expected_X, dm.X) is True assert DatasetMatrix.sparse_equal(expected_Y, dm.Y) is True assert default_row_labels() == dm.row_labels assert default_column_labels_X() == dm.column_labels_X assert [] == dm.column_labels_Y folder = testutil.ensure_empty_tmp_subfolder('test_datasetmatrix__removing_columns_Y') check_saving_and_loading(dm, folder)
def test_getting_values_per_column(): dm = DatasetMatrix('testmatrix') dm.X = scipy.sparse.csr_matrix(numpy.array([ [0, 1, 1, 2, 0, 3], [1, 4, 1, 2, 0, 1], [1, 5, 1, 0, 0, 3], [2, 16, 1, 9, 0, 2], [2, -5, 1, 3, 0, 1] ])) dm.Y = dm.X.transpose() column_values_X = dm.get_values_per_column('X') column_values_Y = dm.get_values_per_column('Y') expected_col_values_X = [[0, 1, 2], [-5, 1, 4, 5, 16], [1], [0, 2, 3, 9], [0], [1, 2, 3]] assert column_values_X == expected_col_values_X expected_col_values_Y = [[0, 1, 2, 3], [0, 1, 2, 4], [0, 1, 3, 5], [0, 1, 2, 9, 16], [-5, 0, 1, 2, 3]] assert column_values_Y == expected_col_values_Y
def test_saving_and_loading(): # Set up a simple DatasetMatrix dm = DatasetMatrix('testmatrix') configure_default_datasetmatrix(dm) folder = testutil.ensure_empty_tmp_subfolder('test_datasetmatrix__save_load') check_saving_and_loading(dm, folder)
def create_dataset_matrix(self, label='binarydataset', other_random_seed=-1): if self.reset_random_seed or (other_random_seed != -1 and other_random_seed != self.configuration['random_seed']): if other_random_seed == -1: random.seed(self.configuration['random_seed']) else: random.seed(other_random_seed) self.reset_random_seed = False (X, col_labels_X) = self.create_random_binary_matrix( self.configuration['row_count'], self.configuration['features'] ) (Y, col_labels_Y) = self.create_random_binary_matrix( self.configuration['row_count'], self.configuration['objectives'] ) datasetmatrix = DatasetMatrix(label) datasetmatrix.X = X.tocsr() datasetmatrix.Y = Y.tocsr() datasetmatrix.row_labels = ['row{}'.format(i) for i in range(0, self.configuration['row_count'])] datasetmatrix.column_labels_X = col_labels_X datasetmatrix.column_labels_Y = col_labels_Y datasetmatrix.metadata['source'] = self return datasetmatrix
def default_datasetmatrix(label): sample_count = 8 feature_count = 8 datasetmatrix = DatasetMatrix(label) datasetmatrix.row_labels = [ 'row{}'.format(i) for i in range(0, sample_count) ] datasetmatrix.column_labels_X = [ 'feature{}'.format(i) for i in range(0, feature_count) ] datasetmatrix.column_labels_Y = ['objective'] datasetmatrix.Y = scipy.sparse.csr_matrix( numpy.array([ [1], # training sample [0], # training sample [1], # testing sample [0], # testing sample [1], # testing sample [0], # training sample [1], # testing sample [0] # testing sample ])) datasetmatrix.X = scipy.sparse.csr_matrix( numpy.array([ [1, 1, 1, 1, 0, 1, 0, 1], # training sample [0, 1, 1, 1, 1, 0, 0, 1], # training sample [1, 1, 1, 0, 0, 0, 1, 0], # testing sample [0, 0, 1, 0, 1, 1, 1, 0], # testing sample [1, 1, 0, 1, 0, 0, 1, 1], # testing sample [0, 0, 0, 1, 1, 1, 0, 1], # training sample [1, 1, 1, 1, 0, 0, 1, 0], # testing sample [0, 0, 0, 1, 1, 1, 1, 0] # testing sample ])) return datasetmatrix
def test_removing_rows(): # Set up a simple DatasetMatrix dm = DatasetMatrix('testmatrix') configure_default_datasetmatrix(dm) # Remove the third row. Affects X and Y at the same time. dm.delete_row(2) expected_X = scipy.sparse.csr_matrix(numpy.array([ [1, 2, 3, 4], [5, 6, 7, 8], [13, 14, 15, 16]])) expected_Y = scipy.sparse.csr_matrix(numpy.array([ [101, 102], [201, 202], [401, 402]])) assert DatasetMatrix.sparse_equal(expected_X, dm.X) is True assert DatasetMatrix.sparse_equal(expected_Y, dm.Y) is True assert ["row0", "row1", "row3"] == dm.row_labels assert default_column_labels_X() == dm.column_labels_X assert default_column_labels_Y() == dm.column_labels_Y # Remove the first row. dm.delete_row(0) expected_X = scipy.sparse.csr_matrix(numpy.array([ [5, 6, 7, 8], [13, 14, 15, 16]])) expected_Y = scipy.sparse.csr_matrix(numpy.array([ [201, 202], [401, 402]])) assert DatasetMatrix.sparse_equal(expected_X, dm.X) is True assert DatasetMatrix.sparse_equal(expected_Y, dm.Y) is True assert ["row1", "row3"] == dm.row_labels assert default_column_labels_X() == dm.column_labels_X assert default_column_labels_Y() == dm.column_labels_Y folder = testutil.ensure_empty_tmp_subfolder('test_datasetmatrix__removing_rows') check_saving_and_loading(dm, folder) dm.unfinalize() # Remove both remaining rows. dm.delete_row(0) dm.delete_row(0) expected_X = scipy.sparse.csr_matrix((0, 4)) expected_Y = scipy.sparse.csr_matrix((0, 2)) assert DatasetMatrix.sparse_equal(expected_X, dm.X) is True assert DatasetMatrix.sparse_equal(expected_Y, dm.Y) is True assert [] == dm.row_labels assert default_column_labels_X() == dm.column_labels_X assert default_column_labels_Y() == dm.column_labels_Y folder = testutil.ensure_empty_tmp_subfolder('test_datasetmatrix__removing_rows') check_saving_and_loading(dm, folder)
def load(self): """ Load the ExperimentalDataset from ``self.definition.path``. Simply calls :py:meth:`load() <mbtk.dataset.DatasetMatrix.DatasetMatrix.load>` on ``self.matrix``, ``self.matrix_train`` and ``self.matrix_test``. In case either of these three is ``None``, they are set to new :py:class:`DatasetMatrix <mbtk.dataset.DatasetMatrix.DatasetMatrix>` instances. :return: Nothing """ if self.matrix is None: self.matrix = DatasetMatrix("dataset") self.matrix.load(self.definition.path)
def make_test_datasetmatrix(configuration): folder = tmp_folder / 'mockdataset' label = configuration['label'] try: with Lock('dm-' + label, 'r'): datasetmatrix = DatasetMatrix(label) datasetmatrix.load(folder) except FileNotFoundError: with Lock('dm-' + label, 'w'): sbnds = SampledBayesianNetworkDatasetSource(configuration) sbnds.reset_random_seed = True datasetmatrix = sbnds.create_dataset_matrix(label) datasetmatrix.finalize() datasetmatrix.save(folder) return datasetmatrix
def check_saving_and_loading(dm, folder): # Saving must fail, because dm.finalize() has not yet been called. with pytest.raises(DatasetMatrixNotFinalizedError): dm.save(folder) check_no_datamatrix_folder(folder, dm.label) # Finalize the DatasetMatrix and save it. dm.finalize() dm.save(Path(folder)) check_datamatrix_files(folder, dm.label) # Load the saved data into a fresh DatasetMatrix with the same label # and compare with the old one. dm2 = DatasetMatrix(dm.label) dm2.load(Path(folder)) assert dm == dm2
def create_dataset_matrix(self, label='datasetsource'): """ The required method of a :py:class:`DatasetSource` class. This method reads an external source of data and produces a :py:class:`DatasetMatrix <mbtk.dataset.DatasetMatrix.DatasetMatrix>` instance based on ``configuration``. :param str label: The label of the ``DatasetMatrix`` """ datasetmatrix = DatasetMatrix(label) datasetmatrix.X = scipy.sparse.csr_matrix(numpy.identity(8)) datasetmatrix.Y = scipy.sparse.csr_matrix(numpy.identity(8)) datasetmatrix.row_labels = ["row{}".format(r) for r in range(8)] datasetmatrix.column_labels_X = ["colX{}".format(c) for c in range(8)] datasetmatrix.column_labels_Y = ["colY{}".format(c) for c in range(8)] datasetmatrix.metadata['source'] = self return datasetmatrix
def test_making_variables(): # Set up a simple DatasetMatrix dm = DatasetMatrix('testmatrix') configure_default_datasetmatrix(dm) variable = dm.get_variable('X', 1) assert 1 == variable.ID assert 'colx1' == variable.name assert variable.instances_list is None assert variable.values is None assert variable.lazy_instances_loader is not None variable.load_instances() assert variable.instances_list is not None assert 4 == len(variable) assert dm.get_column('X', 1).tolist() == variable.instances().tolist() variable.update_values() assert [2, 6, 10, 14] == variable.values
def create_dataset_matrix(self, label='bayesian_network', other_random_seed=-1): method = self.configuration.get('method', 'random') if method == 'random': instances_matrix = self.create_random_instances( label, other_random_seed) elif method == 'exact': instances_matrix = self.create_exact_instances(self, label) sample_count = self.configuration['sample_count'] numpy_datatype = self.configuration.get('numpy_datatype', numpy.int8) X = numpy.empty((sample_count, 0), dtype=numpy_datatype) Y = numpy.empty((sample_count, 0), dtype=numpy_datatype) objective_names = sorted(self.configuration.get('objectives', [])) feature_names = list( sorted( list( set(self.bayesian_network.variable_node_names()) - set(objective_names)))) for varname in feature_names: varindex = self.bayesian_network.variable_nodes_index(varname) feature = instances_matrix[:, varindex][numpy.newaxis].T X = numpy.hstack((X, feature)) for varname in objective_names: varindex = self.bayesian_network.variable_nodes_index(varname) objective = instances_matrix[:, varindex][numpy.newaxis].T Y = numpy.hstack((Y, objective)) datasetmatrix = DatasetMatrix(label) datasetmatrix.X = scipy.sparse.csr_matrix(X) datasetmatrix.Y = scipy.sparse.csr_matrix(Y) datasetmatrix.row_labels = [ 'row{}'.format(i) for i in range(0, sample_count) ] datasetmatrix.column_labels_X = feature_names datasetmatrix.column_labels_Y = objective_names datasetmatrix.metadata['source'] = self return datasetmatrix
def test_keeping_rows(): # Set up a simple DatasetMatrix dm = DatasetMatrix('testmatrix') configure_default_datasetmatrix(dm) # Empty lists are not allowed. with pytest.raises(ValueError): dm.keep_rows([]) # Keep rows 1 and 3. dm.keep_rows([1, 3]) expected_X = scipy.sparse.csr_matrix(numpy.array([ [5, 6, 7, 8], [13, 14, 15, 16]])) expected_Y = scipy.sparse.csr_matrix(numpy.array([ [201, 202], [401, 402]])) assert DatasetMatrix.sparse_equal(expected_X, dm.X) is True assert DatasetMatrix.sparse_equal(expected_Y, dm.Y) is True assert ['row1', 'row3'] == dm.row_labels assert default_column_labels_X() == dm.column_labels_X assert default_column_labels_Y() == dm.column_labels_Y # Keep row 0 of the remaining 2 (labeled 'row1'). dm.keep_rows([0]) expected_X = scipy.sparse.csr_matrix(numpy.array([ [5, 6, 7, 8]])) expected_Y = scipy.sparse.csr_matrix(numpy.array([ [201, 202]])) assert DatasetMatrix.sparse_equal(expected_X, dm.X) is True assert DatasetMatrix.sparse_equal(expected_Y, dm.Y) is True assert ['row1'] == dm.row_labels assert default_column_labels_X() == dm.column_labels_X assert default_column_labels_Y() == dm.column_labels_Y folder = testutil.ensure_empty_tmp_subfolder('test_datasetmatrix__keeping_rows') check_saving_and_loading(dm, folder)
def test_selecting_columns_X(): # Set up a simple DatasetMatrix dm = DatasetMatrix('testmatrix') configure_default_datasetmatrix(dm) # Empty lists are not allowed. with pytest.raises(ValueError): dm.select_columns_X([]) # Create new datasetmatrix where X has only columns 1 and 2. dm = dm.select_columns_X([1, 2], 'test_matrix_selected_colsX') expected_X = scipy.sparse.csr_matrix(numpy.array([ [2, 3], [6, 7], [10, 11], [14, 15]])) expected_Y = default_matrix_Y() assert DatasetMatrix.sparse_equal(expected_X, dm.X) is True assert DatasetMatrix.sparse_equal(expected_Y, dm.Y) is True assert default_row_labels() == dm.row_labels assert ['colx1', 'colx2'] == dm.column_labels_X assert default_column_labels_Y() == dm.column_labels_Y # Select X column 0 from the resulting datasetmatrix. dm = dm.select_columns_X([0], 'test_matrix_selected_colsX_2') expected_X = scipy.sparse.csr_matrix(numpy.array([ [2], [6], [10], [14]])) expected_Y = default_matrix_Y() assert DatasetMatrix.sparse_equal(expected_X, dm.X) is True assert DatasetMatrix.sparse_equal(expected_Y, dm.Y) is True assert default_row_labels() == dm.row_labels assert ['colx1'] == dm.column_labels_X assert default_column_labels_Y() == dm.column_labels_Y folder = testutil.ensure_empty_tmp_subfolder('test_datasetmatrix__selecting_colsX') check_saving_and_loading(dm, folder)
def test_removing_columns_X(): # Set up a simple DatasetMatrix dm = DatasetMatrix('testmatrix') configure_default_datasetmatrix(dm) # Remove the third column from the X matrix. dm.delete_column_X(2) expected_X = scipy.sparse.csr_matrix(numpy.array([ [1, 2, 4], [5, 6, 8], [9, 10, 12], [13, 14, 16]])) expected_Y = default_matrix_Y() assert DatasetMatrix.sparse_equal(expected_X, dm.X) is True assert DatasetMatrix.sparse_equal(expected_Y, dm.Y) is True assert default_row_labels() == dm.row_labels assert ['colx0', 'colx1', 'colx3'] == dm.column_labels_X assert default_column_labels_Y() == dm.column_labels_Y # Remove the last column from the X matrix. dm.delete_column_X(2) expected_X = scipy.sparse.csr_matrix(numpy.array([ [1, 2], [5, 6], [9, 10], [13, 14]])) expected_Y = default_matrix_Y() assert DatasetMatrix.sparse_equal(expected_X, dm.X) is True assert DatasetMatrix.sparse_equal(expected_Y, dm.Y) is True assert default_row_labels() == dm.row_labels assert ['colx0', 'colx1'] == dm.column_labels_X assert default_column_labels_Y() == dm.column_labels_Y folder = testutil.ensure_empty_tmp_subfolder('test_datasetmatrix__removing_columns_X') check_saving_and_loading(dm, folder)
def create_dataset_matrix(self, label='rcv1v2', feature_type='', filters=None): """ Create a :class:`DatasetMatrix <mbtk.dataset.DatasetMatrix.DatasetMatrix>` object containing a document-term matrix based on the documents in the RCV1v2 dataset (previously downloaded). If ``configuration['filters']`` has been defined, then only the documents that match the specified filters will be represented as rows of the returned ``DatasetMatrix`` object. Otherwise, all documents in RCV1v2 will be loaded. If ``configuration['feature_type'] == 'wordcount'``, then the ``X`` matrix of the returned ``DatasetMatrix`` object will contain the counts of each word in every document. If ``configuration['feature_type'] == 'binary'``, then the ``X`` matrix of the returned ``DatasetMatrix`` object will contain only values of ``0`` and ``1``, indicating the absence and presence, respectively, of a word in a document. See the `Wikipedia article on document-term matrices`_ for more details. .. _Wikipedia article on document-term matrices:\ https://en.wikipedia.org/wiki/Document-term_matrix :param str label: The label to be set on the returned ``DatasetMatrix`` instance. :return: A ``DatasetMatrix`` containing a document-term matrix in ``X`` and a class-assignment matrix in ``Y``. :rtype: mbtk.dataset.DatasetMatrix.DatasetMatrix """ if filters is None: filters = dict() documentIDs = list() if len(filters) == 0: filters = self.configuration['filters'] if 'industry' in filters.keys(): documentIDs = self.read_documentIDs_in_industry(filters['industry']) elif len(filters) == 0: documentIDs = self.read_all_documentIDs() else: raise ValueError("Unsupported RCV1v2 document filter specified. Either specify \ the 'industry' filter or no filter at all.") if feature_type == '': feature_type = self.configuration.get('feature_type', 'wordcount') documents = self.read_documents(documentIDs) words = self.gather_complete_word_list(documents) topics = self.gather_complete_topic_list(documents) dok_matrix_words, dok_matrix_topics = self.create_dok_matrices(documents, documentIDs, words, topics, feature_type) datasetmatrix = DatasetMatrix(label) datasetmatrix.X = dok_matrix_words.tocsr() datasetmatrix.Y = dok_matrix_topics.tocsr() datasetmatrix.row_labels = list(map(str, documentIDs)) datasetmatrix.column_labels_X = words datasetmatrix.column_labels_Y = topics datasetmatrix.metadata['source'] = self return datasetmatrix