def load_integer_features(self, data): """Gives each POS tag in data a number.""" integer_features = [] pos_feature = np.array([]) for txt in data.textfiles: if self.annotations == "union": txt.compute_union_relations() elif self.annotations == "intersected": txt.compute_intersection_relations() for rel in txt.relations: f = Feature(rel) # Build arrays of integers with which we can fit the encoder # Standardize because f.get_pos_$x() doesn't have to be of length self.number_tags_per_feature/2 standardized_pos_target = self.standardize_sub_pos_feature(f.get_pos_target()) standardized_pos_source = self.standardize_sub_pos_feature(f.get_pos_source()) # Concatenate the two plain POS tag arrays from target and source event pos_feature = np.concatenate((standardized_pos_target, standardized_pos_source)) # Transform this array into the corresponding array of integers integer_feature = self.pos_tags_to_integers(pos_feature) integer_features.append(integer_feature) return integer_features
def load_integer_features(self, data): """Gives each POS tag in data a number.""" integer_features = [] pos_feature = np.array([]) for txt in data.textfiles: if self.annotations == "union": txt.compute_union_relations() elif self.annotations == "intersected": txt.compute_intersection_relations() for rel in txt.relations: f = Feature(rel) # Build arrays of integers with which we can fit the encoder # Standardize because f.get_pos_$x() doesn't have to be of length self.number_tags_per_feature/2 standardized_pos_target = self.standardize_sub_pos_feature( f.get_pos_target()) standardized_pos_source = self.standardize_sub_pos_feature( f.get_pos_source()) # Concatenate the two plain POS tag arrays from target and source event pos_feature = np.concatenate( (standardized_pos_target, standardized_pos_source)) # Transform this array into the corresponding array of integers integer_feature = self.pos_tags_to_integers(pos_feature) integer_features.append(integer_feature) return integer_features
def load_pos_tags(self, data): """Loads all POS tags used in the pos_surrounding area around an event.""" pos_tags = np.array([]) for txt in data.textfiles: if self.annotations == "union": txt.compute_union_relations() elif self.annotations == "intersected": txt.compute_intersection_relations() for rel in txt.relations: f = Feature(rel) # Collect all pos tags from the data pos_tags = np.concatenate((pos_tags, f.get_pos_target())) pos_tags = np.concatenate((pos_tags, f.get_pos_source())) pos_tags = np.unique(pos_tags) # Append a blank tag which will be used for filling up features which don't have enough elements pos_tags = np.append(pos_tags, 'BL') return pos_tags
def load_pos_tags(self, data): """Loads all POS tags used in the pos_surrounding area around an event.""" pos_tags = np.array([]) for txt in data.textfiles: if self.annotations == "union": txt.compute_union_relations() elif self.annotations == "intersected": txt.compute_intersection_relations() for rel in txt.relations: f = Feature(rel) # Collect all pos tags from the data pos_tags = np.concatenate((pos_tags, f.get_pos_target())) pos_tags = np.concatenate((pos_tags, f.get_pos_source())) pos_tags = np.unique(pos_tags) # Append a blank tag which will be used for filling up features which don't have enough elements pos_tags = np.append(pos_tags, 'BL') return pos_tags
def parse_Features(data, new=False, annotations="union", features=["pos", "stem", "aspect", "tense", "distance", "similarity", "polarity", "modality"], distance=False): """Extracts the features out of the dataset and returns a list of features with the corresponding classes. Args: data (list): The parsed data from fables-100-temporal-dependency.xml. new (bool): With new=True a new calculation of Pos() and Stem() can be enforced. Otherwise it will be loaded from a file. annotations (str): Looking on all relations ("union") or at all relations in common between the annotators ("intersected"). features (list): Determines which features should be activated. Possible values: "pos", "stem", "aspect", "tense", "distance", "similarity", "polarity", "modality". distance (bool): If set to True parse_Features() will return distance information for the data (needed for evaluation) """ # Only compute pos and stem if new flag is set if "pos" in features or "stem" in features: if new or not os.path.isfile("set.p"): pos = Pos(data, 6, annotations) stem = Stem(data, annotations) pickle.dump((pos, stem), open("save.p", "wb")) else: pos, stem = pickle.load(open("save.p", "rb")) if distance: distance_diff = [] X = [] y = np.array([], dtype=int) for txt in data.textfiles: # Union or intersected relations? if annotations == "union": txt.compute_union_relations() elif annotations == "intersected": txt.compute_intersection_relations() for rel in txt.relations: f = Feature(rel) feature = [] # Make polarity feature if "polarity" in features: feature = np.concatenate((feature, [f.get_polarity()])) # Make distance feature if "distance" in features: feature = np.concatenate((feature, f.get_distance())) # Make POS feature if "pos" in features: pos_feature = pos.transform(f.get_pos_target(), f.get_pos_source()) pos_feature = pos_feature.toarray()[0] feature = np.concatenate((feature, pos_feature)) # Make Stem feature if "stem" in features: stem_feature = stem.transform(f.get_stem_source(), f.get_stem_target()) stem_feature = stem_feature[0] feature = np.concatenate((feature, stem_feature)) # Make similarity feature if "similarity" in features: feature = np.concatenate((feature, [f.get_similarity_of_words()])) # Make modality feature if "modality" in features: feature = np.concatenate((feature, [f.get_modality()])) # Make aspect feature if "aspect" in features: feature = np.concatenate((feature, f.get_aspect())) # Make tense feature if "tense" in features: feature = np.concatenate((feature, f.get_tense())) # Append feature to X X.append(feature) y = np.append(y, [f.get_class()]) # Append distance information if needed if distance: distance_diff.append(f.get_distance_diff()) if distance: return (X, y, distance_diff) else: return (X, y)