def from_token(token: Token) -> 'MarkovWord': if CapitalizationMode.from_token( token, CAPITALIZATION_COMPOUND_RULES) == CapitalizationMode.COMPOUND: compound = True else: compound = False return MarkovWord(token.text, Pos.from_token(token), compound=compound, neighbors={})
def from_token(token: Token) -> 'MarkovNeighbor': key = token.text.lower() text = token.text if CapitalizationMode.from_token(token, CAPITALIZATION_COMPOUND_RULES) == CapitalizationMode.COMPOUND: compound = True else: compound = False pos = Pos.from_token(token) values = [0, 0] dist = [0] * (MARKOV_WINDOW_SIZE * 2 + 1) return MarkovNeighbor(key, text, pos, compound, values, dist)
def from_token(token: Token) -> 'MarkovNeighbor': key = token.text.lower() text = token.text if CapitalizationMode.from_token(token, CAPITALIZATION_COMPOUND_RULES) == CapitalizationMode.COMPOUND: compound = True else: compound = False pos = Pos.from_token(token) values = [0, 0] dist = [0] * (MARKOV_WINDOW_SIZE * 2 + 1) return MarkovNeighbor(key, text, pos, compound, values, dist)
def preprocess(self, doc: Doc) -> bool: if len(self.data) >= STRUCTURE_MODEL_TRAINING_MAX_SIZE: return False sequence = [] previous_item = None for sentence_idx, sentence in enumerate(doc.sents): if len(self.data) >= STRUCTURE_MODEL_TRAINING_MAX_SIZE: return False for token_idx, token in enumerate(sentence): item = StructureFeatureAnalyzer.analyze( token, CapitalizationMode.from_token( token, CAPITALIZATION_COMPOUND_RULES)) label = item if len(sequence) == 0: # Offset data by one, making label point to the next data item sequence.append( PoSCapitalizationMode( Pos.NONE, CapitalizationMode.NONE).to_embedding()) else: sequence.append(previous_item) # We only want the latest SEQUENCE_LENGTH items sequence = sequence[-StructureModel.SEQUENCE_LENGTH:] self.data.append(sequence.copy()) self.labels.append(label) previous_item = item # Handle EOS after each sentence item = PoSCapitalizationMode( Pos.EOS, CapitalizationMode.NONE).to_embedding() label = item sequence.append(previous_item) # We only want the latest SEQUENCE_LENGTH items sequence = sequence[-StructureModel.SEQUENCE_LENGTH:] self.data.append(sequence.copy()) self.labels.append(label) previous_item = item return True
def preprocess(self, doc: Doc) -> bool: if len(self.data) >= STRUCTURE_MODEL_TRAINING_MAX_SIZE: return False sequence = [] previous_item = None for sentence_idx, sentence in enumerate(doc.sents): if len(self.data) >= STRUCTURE_MODEL_TRAINING_MAX_SIZE: return False for token_idx, token in enumerate(sentence): item = StructureFeatureAnalyzer.analyze( token, CapitalizationMode.from_token(token, CAPITALIZATION_COMPOUND_RULES)) label = item if len(sequence) == 0: # Offset data by one, making label point to the next data item sequence.append(PoSCapitalizationMode(Pos.NONE, CapitalizationMode.NONE).to_embedding()) else: sequence.append(previous_item) # We only want the latest SEQUENCE_LENGTH items sequence = sequence[-StructureModel.SEQUENCE_LENGTH:] self.data.append(sequence.copy()) self.labels.append(label) previous_item = item # Handle EOS after each sentence item = PoSCapitalizationMode(Pos.EOS, CapitalizationMode.NONE).to_embedding() label = item sequence.append(previous_item) # We only want the latest SEQUENCE_LENGTH items sequence = sequence[-StructureModel.SEQUENCE_LENGTH:] self.data.append(sequence.copy()) self.labels.append(label) previous_item = item return True
def from_token(token: Token) -> 'MarkovWord': if CapitalizationMode.from_token(token, CAPITALIZATION_COMPOUND_RULES) == CapitalizationMode.COMPOUND: compound = True else: compound = False return MarkovWord(token.text, Pos.from_token(token), compound=compound, neighbors={})