示例#1
0
    def apply(self, transformation):
        """
        Applies a transformation on the current space.

        All transformations affect the data matrix. If the transformation
        reduces the dimensionality of the space, the column indexing
        structures are also updated. The operation applied is appended
        to the list of operations that the space holds.

        Args:
            transformation: of type Scaling, DimensionalityReduction or
              FeatureSelection

        Returns:
            A new space on which the transformation has been applied.

        """
        start = time.time()
        #TODO , FeatureSelection, DimReduction ..
        assert_is_instance(
            transformation,
            (Scaling, DimensionalityReduction, FeatureSelection))
        op = transformation.create_operation()
        new_matrix = op.apply(self.cooccurrence_matrix)

        new_operations = list(self.operations)
        new_operations.append(op)

        id2row, row2id = list(self.id2row), self.row2id.copy()

        if isinstance(op, DimensionalityReductionOperation):
            self.assert_1dim_element()
            id2column, column2id = [], {}
        elif isinstance(op, FeatureSelectionOperation):
            self.assert_1dim_element()
            op.original_columns = self.id2column

            if op.original_columns:
                id2column = list(
                    array(op.original_columns)[op.selected_columns])
                column2id = list2dict(id2column)
            else:
                id2column, column2id = [], {}
        else:
            id2column, column2id = list(self.id2column), self.column2id.copy()

        log.print_transformation_info(logger, transformation, 1,
                                      "\nApplied transformation:")
        log.print_matrix_info(logger, self.cooccurrence_matrix, 2,
                              "Original semantic space:")
        log.print_matrix_info(logger, new_matrix, 2,
                              "Resulted semantic space:")
        log.print_time_info(logger, time.time(), start, 2)

        return Space(new_matrix,
                     id2row,
                     id2column,
                     row2id,
                     column2id,
                     operations=new_operations)
示例#2
0
    def compose(self, data, arg_space):
        """
        Uses a composition model to compose elements.

        Args:
            data: data to be composed. List of tuples, each containing 3
            strings: (arg1, arg2, composed_phrase). arg1 and arg2 are the
            elements to be composed and composed_phrase is the string associated
            to their composition.

            arg_space: argument space(s). Space object or a tuple of two
            Space objects (e.g. my_space, or (my_space1, my_space2)).
            If two spaces are provided, arg1 elements of data are
            interpreted in space1, and arg2 in space2.

        Returns:
            composed space: a new object of type Space, containing the
            phrases obtained through composition.

        """
        start = time.time()

        arg1_space, arg2_space = self.extract_arg_spaces(arg_space)
        arg1_list, arg2_list, phrase_list = self.valid_data_to_lists(data,
                                                                     (arg1_space.row2id,
                                                                      arg2_space.row2id,
                                                                      None))
        
        # we try to achieve at most MAX_MEM_OVERHEAD*phrase_space memory overhead
        # the /3.0 is needed
        # because the composing data needs 3 * len(train_data) memory (arg1 vector, arg2 vector, phrase vector)
        chunk_size = int(max(arg1_space.cooccurrence_matrix.shape[0],arg2_space.cooccurrence_matrix.shape[0],len(phrase_list))
                          * self.MAX_MEM_OVERHEAD / 3.0) + 1
        
        composed_mats = []
        for i in range(int(math.ceil(len(arg1_list) / float(chunk_size)))):
            beg, end = i*chunk_size, min((i+1)*chunk_size, len(arg1_list))

            arg1_mat = arg1_space.get_rows(arg1_list[beg:end])
            arg2_mat = arg2_space.get_rows(arg2_list[beg:end])

            [arg1_mat, arg2_mat] = resolve_type_conflict([arg1_mat, arg2_mat],
                                                                    DenseMatrix)
            composed_mat = self._compose(arg1_mat, arg2_mat)
            composed_mats.append(composed_mat)
        
        composed_phrase_mat = composed_mat.nary_vstack(composed_mats)
        
        if self.composed_id2column is None:
            self.composed_id2column = self._build_id2column(arg1_space, arg2_space)

        log.print_name(logger, self, 1, "\nComposed with composition model:")
        log.print_info(logger, 3, "Composed total data points:%s" % arg1_mat.shape[0])
        log.print_matrix_info(logger, composed_phrase_mat, 4,
                              "Resulted (composed) semantic space::")
        log.print_time_info(logger, time.time(), start, 2)
        
        return Space(composed_phrase_mat, phrase_list, self.composed_id2column)
    def compose(self, data, arg_space):
        """
        Uses a lexical function composition model to compose elements.

        Args:
            data: data to be composed. List of tuples, each containing 3
            strings: (function_word, arg, composed_phrase). function_word and
            arg are the elements to be composed and composed_phrase is the
            string associated to their composition. function_word elements
            are interpreted in self.function_space.

            arg_space: argument space, of type Space. arg elements of data are
            interpreted in this space.

        Returns:
            composed space: a new object of type Space, containing the
            phrases obtained through composition.

        """
        start = time.time()

        assert_is_instance(arg_space, Space)
        arg1_list, arg2_list, phrase_list = self.valid_data_to_lists(
            data, (self._function_space.row2id, arg_space.row2id, None))

        composed_vec_list = []
        for i in range(len(arg1_list)):
            arg1_vec = self._function_space.get_row(arg1_list[i])
            arg2_vec = arg_space.get_row(arg2_list[i])

            matrix_type = get_type_of_largest([arg1_vec, arg2_vec])
            [arg1_vec, arg2_vec] = resolve_type_conflict([arg1_vec, arg2_vec],
                                                         matrix_type)

            composed_ph_vec = self._compose(arg1_vec, arg2_vec,
                                            self._function_space.element_shape)

            composed_vec_list.append(composed_ph_vec)

        result_element_shape = self._function_space.element_shape[0:-1]
        composed_ph_mat = composed_ph_vec.nary_vstack(composed_vec_list)

        log.print_name(logger, self, 1, "\nComposed with composition model:")
        log.print_info(logger, 3,
                       "Composed total data points:%s" % len(arg1_list))
        log.print_info(
            logger, 3,
            "Functional shape of the resulted (composed) elements:%s" %
            (result_element_shape, ))
        log.print_matrix_info(logger, composed_ph_mat, 4,
                              "Resulted (composed) semantic space:")
        log.print_time_info(logger, time.time(), start, 2)

        return Space(composed_ph_mat,
                     phrase_list,
                     self.composed_id2column,
                     element_shape=result_element_shape)
    def compose(self, data, arg_space):
        """
        Uses a lexical function composition model to compose elements.
        
        Args:
            data: data to be composed. List of tuples, each containing 3
            strings: (function_word, arg, composed_phrase). function_word and 
            arg are the elements to be composed and composed_phrase is the 
            string associated to their composition. function_word elements
            are interpreted in self.function_space. 
            
            arg_space: argument space, of type Space. arg elements of data are 
            interpreted in this space. 
        
        Returns:
            composed space: a new object of type Space, containing the 
            phrases obtained through composition.
            
        """
        start = time.time()
        
        assert_is_instance(arg_space, Space)
        arg1_list, arg2_list, phrase_list = self.valid_data_to_lists(data,
                                                                     (self._function_space.row2id,
                                                                      arg_space.row2id,
                                                                      None))

        composed_vec_list = []
        for i in xrange(len(arg1_list)):
            arg1_vec = self._function_space.get_row(arg1_list[i])
            arg2_vec = arg_space.get_row(arg2_list[i])
        
            matrix_type = get_type_of_largest([arg1_vec, arg2_vec])
            [arg1_vec, arg2_vec] = resolve_type_conflict([arg1_vec, arg2_vec],
                                                              matrix_type)
                
            composed_ph_vec = self._compose(arg1_vec, arg2_vec,
                                            self._function_space.element_shape)

            composed_vec_list.append(composed_ph_vec)
        
        result_element_shape = self._function_space.element_shape[0:-1]
        composed_ph_mat = composed_ph_vec.nary_vstack(composed_vec_list)
        
        log.print_name(logger, self, 1, "\nComposed with composition model:")
        log.print_info(logger, 3, "Composed total data points:%s" % len(arg1_list))
        log.print_info(logger, 3, "Functional shape of the resulted (composed) elements:%s" 
                       % (result_element_shape,))
        log.print_matrix_info(logger, composed_ph_mat, 4, 
                              "Resulted (composed) semantic space:")
        log.print_time_info(logger, time.time(), start, 2)
        
        return Space(composed_ph_mat, phrase_list, self.composed_id2column, 
                     element_shape = result_element_shape)
示例#5
0
    def apply(self, transformation):
        """
        Applies a transformation on the current space.
        
        All transformations affect the data matrix. If the transformation 
        reduces the dimensionality of the space, the column indexing
        structures are also updated. The operation applied is appended
        to the list of operations that the space holds. 
        
        Args:
            transformation: of type Scaling, DimensionalityReduction or 
              FeatureSelection  

        Returns:
            A new space on which the transformation has been applied.
            
        """
        start = time.time()
        #TODO , FeatureSelection, DimReduction ..
        assert_is_instance(transformation, (Scaling, DimensionalityReduction, 
                                            FeatureSelection))
        op = transformation.create_operation()
        new_matrix =  op.apply(self.cooccurrence_matrix)
        
        new_operations = list(self.operations)
        new_operations.append(op)

        id2row, row2id = list(self.id2row), self.row2id.copy() 
        
        
        if isinstance(op, DimensionalityReductionOperation):
            self.assert_1dim_element()
            id2column, column2id = [], {}
        elif isinstance(op, FeatureSelectionOperation):
            self.assert_1dim_element()
            op.original_columns = self.id2column
            
            if op.original_columns: 
                id2column = list(array(op.original_columns)[op.selected_columns])
                column2id = list2dict(id2column)
            else:
                id2column, column2id = [],{}
        else:
            id2column, column2id = list(self.id2column), self.column2id.copy()

        log.print_transformation_info(logger, transformation, 1, 
                                      "\nApplied transformation:")
        log.print_matrix_info(logger, self.cooccurrence_matrix, 2, 
                              "Original semantic space:")
        log.print_matrix_info(logger, new_matrix, 2, "Resulted semantic space:")
        log.print_time_info(logger, time.time(), start, 2)
                        
        return Space(new_matrix, id2row, id2column,
                     row2id, column2id, operations = new_operations)
示例#6
0
    def get_neighbours(self, word, no_neighbours, similarity, 
                       space2=None):            
        """
        Computes the neighbours of a word in the semantic space.

        Args:
            word: string, target word
            no_neighbours: int, the number of neighbours desired
            similarity: of type Similarity, the similarity measure to be used
            space2: Space type, Optional. If provided, the neighbours are 
                retrieved from this space, rather than the current space. 
                Default, neighbours are retrieved from the current space.
                
        Returns:
            list of (neighbour_string, similarity_value) tuples.
            
        Raises:
            KeyError: if the word is not found in the semantic space.
            
        """
        
        start = time.time()
        assert_is_instance(similarity, Similarity)       
        vector = self.get_row(word)
        
        if space2 is None:
            id2row = self.id2row
            sims_to_matrix = similarity.get_sims_to_matrix(vector, 
                                                          self.cooccurrence_matrix)
        else:
            mat_type = type(space2.cooccurrence_matrix)
            if not isinstance(vector, mat_type):
                vector = mat_type(vector)
            
            sims_to_matrix = similarity.get_sims_to_matrix(vector, 
                                         space2.cooccurrence_matrix)
            id2row = space2.id2row 
        
        sorted_perm = sims_to_matrix.sorted_permutation(sims_to_matrix.sum, 1)
        no_neighbours = min(no_neighbours, len(id2row))
        result = []
                
        for count in range(no_neighbours):
            i = sorted_perm[count]
            result.append((id2row[i], sims_to_matrix[i,0]))

        log.print_info(logger, 1, "\nGetting neighbours of:%s" % (word))
        log.print_name(logger, similarity, 1, "Similarity:")
        log.print_time_info(logger, time.time(), start, 2)
        return result    
示例#7
0
    def get_neighbours(self, word, no_neighbours, similarity, space2=None):
        """
        Computes the neighbours of a word in the semantic space.

        Args:
            word: string, target word
            no_neighbours: int, the number of neighbours desired
            similarity: of type Similarity, the similarity measure to be used
            space2: Space type, Optional. If provided, the neighbours are
                retrieved from this space, rather than the current space.
                Default, neighbours are retrieved from the current space.

        Returns:
            list of (neighbour_string, similarity_value) tuples.

        Raises:
            KeyError: if the word is not found in the semantic space.

        """

        start = time.time()
        assert_is_instance(similarity, Similarity)
        vector = self.get_row(word)

        if space2 is None:
            id2row = self.id2row
            sims_to_matrix = similarity.get_sims_to_matrix(
                vector, self.cooccurrence_matrix)
        else:
            mat_type = type(space2.cooccurrence_matrix)
            if not isinstance(vector, mat_type):
                vector = mat_type(vector)

            sims_to_matrix = similarity.get_sims_to_matrix(
                vector, space2.cooccurrence_matrix)
            id2row = space2.id2row

        sorted_perm = sims_to_matrix.sorted_permutation(sims_to_matrix.sum, 1)
        no_neighbours = min(no_neighbours, len(id2row))
        result = []

        for count in range(no_neighbours):
            i = sorted_perm[count]
            result.append((id2row[i], sims_to_matrix[i, 0]))

        log.print_info(logger, 1, "\nGetting neighbours of:%s" % (word))
        log.print_name(logger, similarity, 1, "Similarity:")
        log.print_time_info(logger, time.time(), start, 2)
        return result
示例#8
0
    def train(self, train_data, arg_space, phrase_space):
        """
        Trains a composition model and sets its learned parameters.

        Args:
            train_data: list of string tuples. Each tuple contains 3
            string elements: (arg1, arg2, phrase).

            arg_space: argument space(s). Space object or a tuple of two
            Space objects (e.g. my_space, or (my_space1, my_space2)).
            If two spaces are provided, arg1 elements of train data are
            interpreted in space1, and arg2 in space2.

            phrase space: phrase space, of type Space.

        Calls the specific training routine of the current composition
        model. Training tuples which contain strings not found in their
        respective spaces are ignored.

        The id2column attribute of the resulted composed space is set to
        be equal to that of the phrase space given as an input.
        """

        start = time.time()

        arg1_space, arg2_space = self.extract_arg_spaces(arg_space)
        arg1_list, arg2_list, phrase_list = self.valid_data_to_lists(train_data,
                                                                     (arg1_space.row2id,
                                                                      arg2_space.row2id,
                                                                      phrase_space.row2id)
                                                                     )


        self._train(arg1_space, arg2_space, phrase_space,
                 arg1_list, arg2_list, phrase_list)

        self.composed_id2column = phrase_space.id2column

        log.print_composition_model_info(logger, self, 1, "\nTrained composition model:")
        log.print_info(logger, 2, "With total data points:%s" % len(arg1_list))
        log.print_matrix_info(logger, arg1_space.cooccurrence_matrix, 3,
                              "Semantic space of argument 1:")
        log.print_matrix_info(logger, arg2_space.cooccurrence_matrix, 3,
                              "Semantic space of argument 2:")
        log.print_matrix_info(logger, phrase_space.cooccurrence_matrix, 3,
                              "Semantic space of phrases:")
        log.print_time_info(logger, time.time(), start, 2)
示例#9
0
    def compose(self, data, arg_space):
        """
        Uses a composition model to compose elements.

        Args:
            data: data to be composed. List of tuples, each containing 3
            strings: (arg1, arg2, composed_phrase). arg1 and arg2 are the
            elements to be composed and composed_phrase is the string associated
            to their composition.

            arg_space: argument space(s). Space object or a tuple of two
            Space objects (e.g. my_space, or (my_space1, my_space2)).
            If two spaces are provided, arg1 elements of data are
            interpreted in space1, and arg2 in space2.

        Returns:
            composed space: a new object of type Space, containing the
            phrases obtained through composition.

        """
        start = time.time()

        arg1_space, arg2_space = self.extract_arg_spaces(arg_space)
        arg1_list, arg2_list, phrase_list = self.valid_data_to_lists(
            data, (arg1_space.row2id, arg2_space.row2id, None))

        arg1_mat = arg1_space.get_rows(arg1_list)
        arg2_mat = arg2_space.get_rows(arg2_list)

        [arg1_mat, arg2_mat] = resolve_type_conflict([arg1_mat, arg2_mat],
                                                     DenseMatrix)

        composed_phrase_mat = self._compose(arg1_mat, arg2_mat)
        if self.composed_id2column is None:
            self.composed_id2column = self._build_id2column(
                arg1_space, arg2_space)

        log.print_name(logger, self, 1, "\nComposed with composition model:")
        log.print_info(logger, 3,
                       "Composed total data points:%s" % arg1_mat.shape[0])
        log.print_matrix_info(logger, composed_phrase_mat, 4,
                              "Resulted (composed) semantic space::")
        log.print_time_info(logger, time.time(), start, 2)

        return Space(composed_phrase_mat, phrase_list, self.composed_id2column)
示例#10
0
    def compose(self, data, arg_space):
        """
        Uses a composition model to compose elements.
        
        Args:
            data: data to be composed. List of tuples, each containing 3
            strings: (arg1, arg2, composed_phrase). arg1 and arg2 are the 
            elements to be composed and composed_phrase is the string associated
            to their composition.
            
            arg_space: argument space(s). Space object or a tuple of two 
            Space objects (e.g. my_space, or (my_space1, my_space2)). 
            If two spaces are provided, arg1 elements of data are 
            interpreted in space1, and arg2 in space2.
        
        Returns:
            composed space: a new object of type Space, containing the 
            phrases obtained through composition.
            
        """
        start = time.time()
         
        arg1_space, arg2_space = self.extract_arg_spaces(arg_space)
        arg1_list, arg2_list, phrase_list = self.valid_data_to_lists(data,
                                                                     (arg1_space.row2id,
                                                                      arg2_space.row2id,
                                                                      None))
                                                                     
        arg1_mat = arg1_space.get_rows(arg1_list)
        arg2_mat = arg2_space.get_rows(arg2_list)
        
        [arg1_mat, arg2_mat] = resolve_type_conflict([arg1_mat, arg2_mat], DenseMatrix) 
        
        composed_phrase_mat = self._compose(arg1_mat, arg2_mat)
        if self.composed_id2column is None:
            self.composed_id2column = self._build_id2column(arg1_space, arg2_space)

        log.print_name(logger, self, 1, "\nComposed with composition model:")
        log.print_info(logger, 3, "Composed total data points:%s" % arg1_mat.shape[0])
        log.print_matrix_info(logger, composed_phrase_mat, 4, 
                              "Resulted (composed) semantic space::")
        log.print_time_info(logger, time.time(), start, 2)
                
        return Space(composed_phrase_mat, phrase_list, self.composed_id2column)
示例#11
0
 def export(self, file_prefix, **kwargs):
     """
     Exports the current space to disk.
     If the space has no column information, it cannot be exported in 
     sparse format (sm).
     
     Args:
         file_prefix: string, prefix of the files to be exported
         format: string, one of dm/sm
    
     Prints: 
         - matrix in file_prefix.<format> 
         - row elements in file_prefix.<row>
         - col elements in file_prefix.<col>
     
     Raises:
         ValueError: if the space has no column info and "sm" exporting
             is attempted
         NotImplementedError: the space matrix is dense and "sm" exporting
             is attempted
     
     """
     
     start = time.time()
     create_parent_directories(file_prefix)
     format_ = "dm"
     if "format" in kwargs:
         format_ = kwargs["format"]
         if not format_ in ["dm","sm"]:
             raise ValueError("Unrecognized format: %s" %format_)
         elif format_ == "dm":
             print_cooc_mat_dense_format(self.cooccurrence_matrix,
                                         self.id2row, file_prefix)
         else:
             print_cooc_mat_sparse_format(self.cooccurrence_matrix,
                                          self.id2row,
                                          self.id2column, file_prefix)
     self._export_row_column(file_prefix)
     
     log.print_matrix_info(logger, self.cooccurrence_matrix, 1, 
                           "Printed semantic space:")
     log.print_time_info(logger, time.time(), start, 2) 
示例#12
0
    def export(self, file_prefix, **kwargs):
        """
        Exports the current space to disk.
        If the space has no column information, it cannot be exported in
        sparse format (sm).

        Args:
            file_prefix: string, prefix of the files to be exported
            format: string, one of dm/sm

        Prints:
            - matrix in file_prefix.<format>
            - row elements in file_prefix.<row>
            - col elements in file_prefix.<col>

        Raises:
            ValueError: if the space has no column info and "sm" exporting
                is attempted
            NotImplementedError: the space matrix is dense and "sm" exporting
                is attempted

        """

        start = time.time()
        create_parent_directories(file_prefix)
        format_ = "dm"
        if "format" in kwargs:
            format_ = kwargs["format"]
            if not format_ in ["dm", "sm"]:
                raise ValueError("Unrecognized format: %s" % format_)
            elif format_ == "dm":
                print_cooc_mat_dense_format(self.cooccurrence_matrix,
                                            self.id2row, file_prefix)
            else:
                print_cooc_mat_sparse_format(self.cooccurrence_matrix,
                                             self.id2row, self.id2column,
                                             file_prefix)
        self._export_row_column(file_prefix)

        log.print_matrix_info(logger, self.cooccurrence_matrix, 1,
                              "Printed semantic space:")
        log.print_time_info(logger, time.time(), start, 2)
示例#13
0
    def train(self, train_data, arg_space, phrase_space):
        """
        Trains a lexical function composition model to learn a function
        space and sets the function_space parameter. 
                
        Args:
            train_data: list of string tuples. Each tuple contains 3 
            string elements: (function_word, arg, phrase).
            
            arg_space: argument space, of type Space. arg elements of 
            train data are interpreted in this space.
        
            phrase space: phrase space, of type Space. phrase elements of 
            the train data are interpreted in this space.
            
        Training tuples which contain strings not found in their 
        respective spaces are ignored. Function words containing less than
        _MIN_SAMPLES training instances are ignored. For example, if
        _MIN_SAMPLES=2 and function word "red" occurs in only one phrase, "red"
        is ignored.
        
        The id2column attribute of the resulted composed space is set to
        be equal to that of the phrase space given as an input.
        """
        
        start = time.time()

        self._has_intercept = self._regression_learner.has_intercept()

        if not isinstance(arg_space, Space):
            raise ValueError("expected one input spaces!")  
               
        result_mats = []
               
        train_data = sorted(train_data, key=lambda tup: tup[0])
        function_word_list, arg_list, phrase_list = self.valid_data_to_lists(train_data,
                                                                             (None,
                                                                              arg_space.row2id,
                                                                              phrase_space.row2id))
        #partitions the sorted input data
        keys, key_ranges = get_partitions(function_word_list, self._MIN_SAMPLES)
        
        if not keys:
            raise ValueError("No valid training data found!")
                
        assert(len(arg_space.element_shape) == 1)
        
        if self._has_intercept:
            new_element_shape = phrase_space.element_shape + (arg_space.element_shape[0] + 1,)
        else:
            new_element_shape = phrase_space.element_shape + (arg_space.element_shape[0],)
            
        for i in xrange(len(key_ranges)):
            
            idx_beg, idx_end = key_ranges[i]
            
            print ("Training lexical function...%s with %d samples" 
                     % (keys[i], idx_end - idx_beg))
                            
            arg_mat = arg_space.get_rows(arg_list[idx_beg:idx_end]) 
            phrase_mat = phrase_space.get_rows(phrase_list[idx_beg:idx_end])
 
            #convert them to the same type
            matrix_type = get_type_of_largest([arg_mat, phrase_mat])
            [arg_mat, phrase_mat] = resolve_type_conflict([arg_mat, phrase_mat],
                                                          matrix_type)

            result_mat = self._regression_learner.train(arg_mat, phrase_mat).transpose()
            
            result_mat.reshape((1, np.prod(new_element_shape)))
            
            result_mats.append(result_mat)

        new_space_mat = arg_mat.nary_vstack(result_mats)
        
        self.composed_id2column = phrase_space.id2column
            
        self._function_space = Space(new_space_mat, keys, [], 
                                     element_shape=new_element_shape)
        
        log.print_composition_model_info(logger, self, 1, "\nTrained composition model:")
        log.print_info(logger, 3, "Trained: %s lexical functions" % len(keys))
        log.print_info(logger, 3, "With total data points:%s" % len(function_word_list))
        log.print_matrix_info(logger, arg_space.cooccurrence_matrix, 3, 
                              "Semantic space of arguments:")
        log.print_info(logger, 3, "Shape of lexical functions learned:%s" 
                       % (new_element_shape,))
        log.print_matrix_info(logger, new_space_mat, 3, 
                              "Semantic space of lexical functions:")
        log.print_time_info(logger, time.time(), start, 2)
示例#14
0
    def build(cls, **kwargs):
        """
        Reads in data files and extracts the data to construct a semantic space.
        
        If the data is read in dense format and no columns are provided, 
        the column indexing structures are set to empty.
         
        Args:
            data: file containing the counts
            format: format on the input data file: one of sm/dm
            rows: file containing the row elements. Optional, if not provided,
                extracted from the data file.
            cols: file containing the column elements
           
        Returns:
            A semantic space build from the input data files.
            
        Raises:
            ValueError: if one of data/format arguments is missing.
                        if cols is missing and format is "sm"
                        if the input columns provided are not consistent with
                        the shape of the matrix (for "dm" format)
            
        """
        start = time.time()
        id2row = None
        id2column = None
        
        if "data" in kwargs:
            data_file = kwargs["data"]
        else:
            raise ValueError("Space data file needs to be specified")
            
        if "format" in kwargs:
            format_ = kwargs["format"]
            if not format_ in ["dm","sm"]:
                raise ValueError("Unrecognized format: %s" % format_)
        else:
            raise ValueError("Format of input files needs to be specified")
        
        if "rows" in kwargs and not kwargs["rows"] is None:
            [id2row], [row2id] = extract_indexing_structs(kwargs["rows"], [0])
            
        if "cols" in kwargs and not kwargs["cols"] is None:
            [id2column], [column2id] = extract_indexing_structs(kwargs["cols"], [0])
        elif format_ == "sm":
            raise ValueError("Need to specify column file when input format is sm!")
            
        if format_ == "sm":
            if id2row is None:
                [id2row], [row2id] = extract_indexing_structs(data_file, [0])
            mat = read_sparse_space_data(data_file, row2id, column2id)

        else:
            if id2row is None:
                [id2row],[row2id] = extract_indexing_structs(data_file, [0])
            mat = read_dense_space_data(data_file, row2id)
                
        if id2column and len(id2column) != mat.shape[1]:
            raise ValueError("Columns provided inconsistent with shape of input matrix!")

        if id2column is None:
            id2column, column2id = [], {}
                            
        log.print_matrix_info(logger, mat, 1, "Built semantic space:")
        log.print_time_info(logger, time.time(), start, 2)    
        return Space(mat, id2row, id2column, row2id, column2id)
    def train(self, train_data, arg_space, phrase_space):
        """
        Trains a lexical function composition model to learn a function
        space and sets the function_space parameter.

        Args:
            train_data: list of string tuples. Each tuple contains 3
            string elements: (function_word, arg, phrase).

            arg_space: argument space, of type Space. arg elements of
            train data are interpreted in this space.

            phrase space: phrase space, of type Space. phrase elements of
            the train data are interpreted in this space.

        Training tuples which contain strings not found in their
        respective spaces are ignored. Function words containing less than
        _MIN_SAMPLES training instances are ignored. For example, if
        _MIN_SAMPLES=2 and function word "red" occurs in only one phrase, "red"
        is ignored.

        The id2column attribute of the resulted composed space is set to
        be equal to that of the phrase space given as an input.
        """

        start = time.time()

        self._has_intercept = self._regression_learner.has_intercept()

        if not isinstance(arg_space, Space):
            raise ValueError("expected one input spaces!")

        result_mats = []

        train_data = sorted(train_data, key=lambda tup: tup[0])
        function_word_list, arg_list, phrase_list = self.valid_data_to_lists(
            train_data, (None, arg_space.row2id, phrase_space.row2id))
        #partitions the sorted input data
        keys, key_ranges = get_partitions(function_word_list,
                                          self._MIN_SAMPLES)

        if not keys:
            raise ValueError("No valid training data found!")

        assert (len(arg_space.element_shape) == 1)

        if self._has_intercept:
            new_element_shape = phrase_space.element_shape + (
                arg_space.element_shape[0] + 1, )
        else:
            new_element_shape = phrase_space.element_shape + (
                arg_space.element_shape[0], )

        for i in range(len(key_ranges)):
            idx_beg, idx_end = key_ranges[i]

            print(("Training lexical function...%s with %d samples" %
                   (keys[i], idx_end - idx_beg)))

            arg_mat = arg_space.get_rows(arg_list[idx_beg:idx_end])
            phrase_mat = phrase_space.get_rows(phrase_list[idx_beg:idx_end])

            #convert them to the same type
            matrix_type = get_type_of_largest([arg_mat, phrase_mat])
            [arg_mat,
             phrase_mat] = resolve_type_conflict([arg_mat, phrase_mat],
                                                 matrix_type)

            result_mat = self._regression_learner.train(
                arg_mat, phrase_mat).transpose()

            result_mat.reshape((1, np.prod(new_element_shape)))

            result_mats.append(result_mat)

        new_space_mat = arg_mat.nary_vstack(result_mats)

        self.composed_id2column = phrase_space.id2column

        self._function_space = Space(new_space_mat,
                                     keys, [],
                                     element_shape=new_element_shape)

        log.print_composition_model_info(logger, self, 1,
                                         "\nTrained composition model:")
        log.print_info(logger, 3, "Trained: %s lexical functions" % len(keys))
        log.print_info(logger, 3,
                       "With total data points:%s" % len(function_word_list))
        log.print_matrix_info(logger, arg_space.cooccurrence_matrix, 3,
                              "Semantic space of arguments:")
        log.print_info(
            logger, 3,
            "Shape of lexical functions learned:%s" % (new_element_shape, ))
        log.print_matrix_info(logger, new_space_mat, 3,
                              "Semantic space of lexical functions:")
        log.print_time_info(logger, time.time(), start, 2)
示例#16
0
    def build(cls, **kwargs):
        """
        Reads in data files and extracts the data to construct a semantic space.

        If the data is read in dense format and no columns are provided,
        the column indexing structures are set to empty.

        Args:
            data: file containing the counts
            format: format on the input data file: one of sm/dm
            rows: file containing the row elements. Optional, if not provided,
                extracted from the data file.
            cols: file containing the column elements

        Returns:
            A semantic space build from the input data files.

        Raises:
            ValueError: if one of data/format arguments is missing.
                        if cols is missing and format is "sm"
                        if the input columns provided are not consistent with
                        the shape of the matrix (for "dm" format)

        """
        start = time.time()
        id2row = None
        id2column = None

        if "data" in kwargs:
            data_file = kwargs["data"]
        else:
            raise ValueError("Space data file needs to be specified")

        if "format" in kwargs:
            format_ = kwargs["format"]
            if not format_ in ["dm", "sm"]:
                raise ValueError("Unrecognized format: %s" % format_)
        else:
            raise ValueError("Format of input files needs to be specified")

        if "rows" in kwargs and not kwargs["rows"] is None:
            [id2row], [row2id] = extract_indexing_structs(kwargs["rows"], [0])

        if "cols" in kwargs and not kwargs["cols"] is None:
            [id2column], [column2id
                          ] = extract_indexing_structs(kwargs["cols"], [0])
        elif format_ == "sm":
            raise ValueError(
                "Need to specify column file when input format is sm!")

        if format_ == "sm":
            if id2row is None:
                [id2row], [row2id] = extract_indexing_structs(data_file, [0])
            mat = read_sparse_space_data(data_file, row2id, column2id)

        else:
            if id2row is None:
                [id2row], [row2id] = extract_indexing_structs(data_file, [0])
            mat = read_dense_space_data(data_file, row2id)

        if id2column and len(id2column) != mat.shape[1]:
            raise ValueError(
                "Columns provided inconsistent with shape of input matrix!")

        if id2column is None:
            id2column, column2id = [], {}

        log.print_matrix_info(logger, mat, 1, "Built semantic space:")
        log.print_time_info(logger, time.time(), start, 2)
        return Space(mat, id2row, id2column, row2id, column2id)