def filter_tagged_frame( tagged_frame: pd.DataFrame, *, extract_opts: ExtractTaggedTokensOpts, token2id: Token2Id = None, pos_schema: PoS_Tag_Scheme = None, normalize_column_names: bool = True, transform_opts: TokensTransformOpts = None, ) -> pd.DataFrame: """Filters tagged frame (text or numeric). Returns tagged frame Args: tagged_frame ([pd.DataFrame]): Document frame to be filtered, can be text or numeric extract_opts (ExtractTaggedTokensOpts): PoS and lemma extract/filter opts token2id (Token2Id, optional): Vocabulary. Defaults to None. pos_schema (PoS_Tag_Scheme, optional): PoS schema. Defaults to None. transform_opts (TokensTransformOpts, optional): Filters and transforms. Defaults to None. normalize_column_names (bool, optional): If text, rename columns to `token` and `pos`. Defaults to True. Raises: Token2IdMissingError: Token2Id is mandatory if frame is numeric. PoSTagSchemaMissingError: PoS-schema is mandatory if frame is numeric. TaggedFrameColumnNameError: Missing target column (corrupt data) Returns: pd.DataFrame: Filtered and transformed document frame. """ if len(tagged_frame) == 0: return [] is_numeric_frame: bool = is_encoded_tagged_frame(tagged_frame) to_lower: bool = transform_opts and transform_opts.to_lower if is_numeric_frame: if token2id is None: raise Token2IdMissingError( "filter_tagged_frame: cannot filter tagged id frame without vocabulary" ) if pos_schema is None: raise PoSTagSchemaMissingError( "filter_tagged_frame: cannot filter tagged id frame without pos_schema" ) if to_lower: logger.warning( "lowercasing not implemented for numeric tagged frames") to_lower = False if not is_numeric_frame and extract_opts.lemmatize is None and extract_opts.target_override is None: raise ValueError("a valid target not supplied (no lemmatize or target") target_column: str = extract_opts.target_column pos_column: str = extract_opts.pos_column if target_column not in tagged_frame.columns: raise TaggedFrameColumnNameError( f"{target_column} is not valid target for given document (missing column)" ) if pos_column not in tagged_frame.columns: raise ValueError(f"configuration error: {pos_column} not in document") passthroughs: Set[str] = extract_opts.get_passthrough_tokens() blocks: Set[str] = extract_opts.get_block_tokens().union('') if is_numeric_frame: passthroughs = token2id.to_id_set(passthroughs) blocks = token2id.to_id_set(blocks) if not is_numeric_frame and (extract_opts.lemmatize or to_lower): tagged_frame[target_column] = tagged_frame[target_column].str.lower() # pd.Series([x.lower() for x in tagged_frame[target_column]]) passthroughs = {x.lower() for x in passthroughs} # if extract_opts.block_chars: # for char in extract_opts.block_chars: # doc[target] = doc[target].str.replace(char, '', regex=False) """ Phrase detection """ if extract_opts.phrases: if is_numeric_frame: logger.warning( "phrase detection not implemented for numeric tagged frames") extract_opts.phrases = None else: found_phrases = detect_phrases(tagged_frame[target_column], extract_opts.phrases, ignore_case=to_lower) if found_phrases: tagged_frame = merge_phrases(tagged_frame, found_phrases, target_column=target_column, pad=PHRASE_PAD) passthroughs = passthroughs.union( {'_'.join(x[1]) for x in found_phrases}) mask = np.repeat(True, len(tagged_frame.index)) if extract_opts.filter_opts and extract_opts.filter_opts.data: mask &= extract_opts.filter_opts.mask(tagged_frame) pos_includes: Set[str] = extract_opts.get_pos_includes() pos_excludes: Set[str] = extract_opts.get_pos_excludes() pos_paddings: Set[str] = extract_opts.get_pos_paddings() if is_numeric_frame: pg = pos_schema.pos_to_id.get pos_includes = {pg(x) for x in pos_includes} pos_excludes = {pg(x) for x in pos_excludes} pos_paddings = {pg(x) for x in pos_paddings} if pos_includes: """Don't filter if PoS-include is empty - and don't filter out PoS tokens that should be padded""" mask &= tagged_frame[pos_column].isin(pos_includes.union(pos_paddings)) if pos_excludes: mask &= ~(tagged_frame[pos_column].isin(pos_excludes)) if transform_opts and transform_opts.has_effect: mask &= transform_opts.mask(tagged_frame[target_column], token2id=token2id) if len(passthroughs) > 0: mask |= tagged_frame[target_column].isin(passthroughs) if len(blocks) > 0: mask &= ~tagged_frame[target_column].isin(blocks) filtered_data: pd.DataFrame = tagged_frame.loc[mask][[ target_column, pos_column ]] if extract_opts.global_tf_threshold > 1: if token2id is None or token2id.tf is None: logger.error( "Cannot apply TF filter since token2id has no term frequencies" ) extract_opts.global_tf_threshold = 1 else: filtered_data = filter_tagged_frame_by_term_frequency( tagged_frame=filtered_data, target_column=target_column, token2id=token2id, extract_opts=extract_opts, passthroughs=passthroughs, ) if not is_numeric_frame and normalize_column_names: filtered_data.rename(columns={ target_column: 'token', pos_column: 'pos' }, inplace=True) return filtered_data
def tagged_frame_to_tokens( # pylint: disable=too-many-arguments, too-many-statements doc: pd.DataFrame, extract_opts: ExtractTaggedTokensOpts, token2id: Token2Id = None, transform_opts: TokensTransformOpts = None, pos_schema: PoS_Tag_Scheme = None, ) -> Iterable[str | int]: """Extracts tokens from a tagged document represented as a Pandas data frame. Args: extract_opts (ExtractTaggedTokensOpts): Part-of-speech/lemma extract options (e.g. PoS-filter) Returns: Iterable[str]: Sequence of extracted tokens """ if len(doc) == 0: return [] is_numeric_frame: bool = is_encoded_tagged_frame(doc) if isinstance(extract_opts, str): passthrough_column: str = extract_opts """If extracts_opts is a string: return column with what name""" if transform_opts is not None: raise ValueError( "transform_opts must be None when passthrough is specified") if transform_opts is not None: raise ValueError( "transform_opts must be None when passthrough is specified") return doc[passthrough_column].tolist() # if is_numeric_frame: # raise NotImplementedError("tagged_frame_to_tokens cannot handle encoded frames yet") pad: str = "*" pos_paddings: Set[str] = extract_opts.get_pos_paddings() phrase_pad: str = PHRASE_PAD filtered_data = filter_tagged_frame( tagged_frame=doc, extract_opts=extract_opts, token2id=token2id, transform_opts=transform_opts, pos_schema=pos_schema, ) target_columns = ['token_id', 'pos_id' ] if is_numeric_frame else ['token', 'pos'] token_pos_tuples = filtered_data[target_columns].itertuples(index=False, name=None) if len(pos_paddings) > 0: passthroughs: Set[str] = extract_opts.get_passthrough_tokens() if is_numeric_frame: pos_paddings = {pos_schema.pos_to_id.get(x) for x in pos_paddings} passthroughs = token2id.to_id_set(passthroughs) token_pos_tuples = ((pad, x[1]) if x[1] in pos_paddings and x[0] not in passthroughs else x for x in token_pos_tuples) if extract_opts.append_pos: if is_numeric_frame: raise NotImplementedError( "tagged_frame_to_tokens: PoS tag on encoded frame") return [ pad if x[0] == pad else f"{x[0].replace(' ', '_')}@{x[1]}" for x in token_pos_tuples if x[0] != phrase_pad ] if not is_numeric_frame and extract_opts.phrases and len( extract_opts.phrases) > 0: return [ x[0].replace(' ', '_') for x in token_pos_tuples if x[0] != phrase_pad ] return [x[0] for x in token_pos_tuples]