def test_phrased_tagged_frame(): os.makedirs('./tests/output', exist_ok=True) tagged_corpus_source: str = "./tests/test_data/tranströmer_corpus_export.sparv4.csv.zip" checkpoint_opts: checkpoint.CheckpointOpts = None data = checkpoint.load_archive(source_name=tagged_corpus_source, checkpoint_opts=checkpoint_opts, reader_opts=None) payload = next(data.create_stream()) tokens = tagged_frame_to_tokens( payload.content, ExtractTaggedTokensOpts(lemmatize=False, **SPARV_TAGGED_COLUMNS), ) assert tokens is not None phrases = {'United Nations': 'United_Nations', 'United': 'United'} phrased_tokens = tagged_frame_to_tokens( payload.content, ExtractTaggedTokensOpts(lemmatize=False, phrases=phrases, **SPARV_TAGGED_COLUMNS), ) assert phrased_tokens[:9] == [ 'Constitution', 'of', 'the', 'United_Nations', 'Educational', ',', 'Scientific', 'and', 'Cultural', ]
def test_sparv_extract_and_store_when_only_nouns_and_source_is_sparv3_succeeds( ): os.makedirs(OUTPUT_FOLDER, exist_ok=True) target_filename = os.path.join(OUTPUT_FOLDER, f'{uuid.uuid1()}.zip') sparv_corpus.sparv_xml_extract_and_store( SPARV3_ZIPPED_XML_EXPORT_FILENAME, target_filename, version=3, extract_opts=ExtractTaggedTokensOpts(pos_includes='|NN|', pos_paddings=None, lemmatize=False), transform_opts=TokensTransformOpts(to_lower=True, min_len=2, stopwords=['<text>']), ) expected_document_start = "utredningar justitiedepartementet förslag utlänningslag angående om- händertagande förläggning års gere ide to lm \rstatens utredningar förteckning betänkande förslag utlänningslag lag omhändertagande utlänning anstalt förläggning tryckort tryckorten bokstäverna fetstil begynnelse- bokstäverna departement" test_filename = "sou_1945_1.txt" content = zip_utils.read_file_content(zip_or_filename=target_filename, filename=test_filename, as_binary=False) assert content.startswith(expected_document_start) os.remove(target_filename)
def test_reader_when_no_transforms_returns_source_tokens(): expected = [ 'Rödräven', 'är', 'ett', 'hunddjur', 'som', 'har', 'en', 'mycket', 'vidsträckt', 'utbredning', 'över', 'norra', 'halvklotet', '.', ] expected_name = "sparv_xml_export_small.txt" reader = readers.SparvXmlReader( SPARV_XML_EXPORT_FILENAME_SMALL, chunk_size=None, extract_opts=ExtractTaggedTokensOpts(pos_includes='', pos_paddings=None, lemmatize=False, pos_excludes=None), ) filename, tokens = next(iter(reader)) assert expected == list(tokens) assert expected_name == filename
def test_reader_when_ignore_puncts_returns_filter_outs_puncts(): expected = [ 'rödräv', 'vara', 'en', 'hunddjur', 'som', 'ha', 'en', 'mycken', 'vidsträckt', 'utbredning', 'över', 'norra', 'halvklot', ] expected_name = "sparv_xml_export_small.txt" reader = readers.SparvXmlReader( SPARV_XML_EXPORT_FILENAME_SMALL, chunk_size=None, extract_opts=ExtractTaggedTokensOpts(pos_includes='', pos_paddings=None, lemmatize=True, pos_excludes="|MAD|MID|PAD|"), ) filename, tokens = next(iter(reader)) assert expected == tokens assert expected_name == filename
def test_extract_tokens_when_punct_filter_is_disabled_succeeds( df_doc: pd.DataFrame): pytest.importorskip("spacy") df_doc = df_doc.copy() extract_opts = ExtractTaggedTokensOpts(lemmatize=True, **SPACY_TAGGED_COLUMNS, filter_opts=dict(is_punct=None)) tokens = tagged_frame_to_tokens(doc=df_doc, extract_opts=extract_opts) assert tokens == [ 'mars', 'be', 'once', 'home', 'to', 'sea', 'and', 'ocean', ',', 'and', 'perhaps', 'even', 'life', '.', ]
def test_reader_store_result(): os.makedirs(OUTPUT_FOLDER, exist_ok=True) expected_documents = [ ['rödräv', 'hunddjur', 'utbredning', 'halvklot'], [ 'fjällräv', 'fjällvärld', 'liv', 'fjällräv', 'vinter', 'men', 'variant', 'år' ], ] expected_names = ["document_001.txt", "document_002.txt"] target_filename = os.path.join(OUTPUT_FOLDER, 'test_reader_store_result.zip') sparv_corpus.sparv_xml_extract_and_store( SPARV_ZIPPED_XML_EXPORT_FILENAME, target_filename, version=4, extract_opts=ExtractTaggedTokensOpts(pos_includes='|NN|', pos_paddings=None, lemmatize=True), transform_opts=TokensTransformOpts(to_lower=True), ) for i in range(0, len(expected_names)): content = zip_utils.read_file_content(zip_or_filename=target_filename, filename=expected_names[i], as_binary=False) assert ' '.join(expected_documents[i]) == content os.remove(target_filename)
def test_extract_tokens_lemma_no_stops_succeeds(df_doc: pd.DataFrame): pytest.importorskip("spacy") df_doc = df_doc.copy() extract_opts = ExtractTaggedTokensOpts(lemmatize=True, **SPACY_TAGGED_COLUMNS, filter_opts=dict(is_stop=False, is_punct=False)) tokens = tagged_frame_to_tokens(doc=df_doc, extract_opts=extract_opts) assert tokens == ['mars', 'home', 'sea', 'ocean', 'life']
def create_test_corpus() -> SparvTokenizedCsvCorpus: corpus = SparvTokenizedCsvCorpus( SPARV_ZIPPED_CSV_EXPORT_FILENAME, reader_opts=TextReaderOpts(filename_fields="year:_:1", ), extract_opts=ExtractTaggedTokensOpts(lemmatize=True, **SPARV_TAGGED_COLUMNS), ) return corpus
def ComputeOptsSparvCSV( *, corpus_tag: str = 'TELLUS', corpus_source: str = './tests/test_data/tranströmer_corpus_export.sparv4.csv.zip', ) -> ComputeOpts: # pylint: disable=too-many-instance-attributes) return ComputeOpts( corpus_tag=corpus_tag, corpus_source=corpus_source, target_folder="./tests/output", corpus_type=CorpusType.SparvCSV, transform_opts=TokensTransformOpts( to_lower=True, min_len=1, remove_stopwords=None, keep_symbols=True, keep_numerals=True, only_alphabetic=False, only_any_alphanumeric=False, ), text_reader_opts=TextReaderOpts( filename_pattern='*.csv', filename_fields=('year:_:1', ), index_field=None, # use filename as_binary=False, ), extract_opts=ExtractTaggedTokensOpts( pos_includes=None, pos_excludes='|MAD|MID|PAD|', pos_paddings=None, lemmatize=False, **SPARV_TAGGED_COLUMNS, filter_opts=dict( is_alpha=False, is_punct=False, is_digit=None, is_stop=None, is_space=False, ), ), create_subfolder=False, persist=True, context_opts=ContextOpts( concept=('jag', ), context_width=2, partition_keys=['document_id'], ), tf_threshold=1, tf_threshold_mask=False, vectorize_opts=VectorizeOpts(already_tokenized=True, min_tf=1, max_tokens=None), )
def test_extract_tokens_target_text_succeeds(df_doc: pd.DataFrame): pytest.importorskip("spacy") df_doc = df_doc.copy() extract_opts = ExtractTaggedTokensOpts(lemmatize=False, **SPACY_TAGGED_COLUMNS, filter_opts=dict(is_punct=False)) tokens = tagged_frame_to_tokens(doc=df_doc, extract_opts=extract_opts) assert tokens == [ "Mars", "was", "once", "home", "to", "seas", "and", "oceans", "and", "perhaps", "even", "life" ]
def prepare_train_corpus( input_filename, pos_includes, pos_excludes, chunk_size, lemmatize, lower, remove_stopwords, min_word_length, keep_symbols, keep_numerals, version, ): """Prepares the a training corpus from Sparv XML archive""" transform_opts: TokensTransformOpts = TokensTransformOpts( to_lower=lower, remove_stopwords=remove_stopwords is not None, language=remove_stopwords, min_len=min_word_length, max_len=None, keep_numerals=keep_numerals, keep_symbols=keep_symbols, ) extract_opts = ExtractTaggedTokensOpts( pos_includes=pos_includes, pos_excludes=pos_excludes, lemmatize=lemmatize, ) output_filename = replace_extension( timestamp_filename(suffix_filename(input_filename, "text")), 'zip') reader_opts = { 'chunk_size': chunk_size, } sparv_corpus.sparv_xml_extract_and_store( source=input_filename, target=output_filename, version=version, extract_opts=extract_opts, reader_opts=reader_opts, transform_opts=transform_opts, ) store_options_to_json_file( input_filename, output_filename, transform_opts, dict(version=version, extract_tokens_opts=extract_opts, reader_opts=reader_opts), )
def test_extract_tokens_pos_verb_noun_text_succeeds(df_doc: pd.DataFrame): pytest.importorskip("spacy") df_doc = df_doc.copy() extract_opts = ExtractTaggedTokensOpts( lemmatize=False, pos_includes='|VERB|NOUN|', pos_paddings=None, **SPACY_TAGGED_COLUMNS, filter_opts=dict(is_punct=False), ) tokens = tagged_frame_to_tokens(doc=df_doc, extract_opts=extract_opts) assert tokens == ['seas', 'oceans', 'life']
def test_reader_when_lemmatized_nn_vb_returns_lemmatized_nn_vb(): reader = SparvCsvToText( extract_tokens_opts=ExtractTaggedTokensOpts( pos_includes='NN|VB', pos_paddings='JJ', pos_excludes='', lemmatize=True, **SPARV_TAGGED_COLUMNS ) ) expected = "rödräv vara hunddjur ha * utbredning * halvklot" result = reader.transform(TEST_DATA) assert expected == result
def test_reader_when_only_nn_returns_only_nn(): reader = SparvCsvToText( extract_tokens_opts=ExtractTaggedTokensOpts( pos_includes='NN', pos_paddings='VB', pos_excludes=None, lemmatize=False, **SPARV_TAGGED_COLUMNS ) ) expected = "Rödräven * hunddjur * utbredning halvklotet" result = reader.transform(TEST_DATA) assert expected == result
def test_reader_when_no_transforms_returns_source_tokens(): reader = SparvCsvToText( extract_tokens_opts=ExtractTaggedTokensOpts( pos_includes=None, pos_paddings=None, pos_excludes=None, lemmatize=False, **SPARV_TAGGED_COLUMNS ) ) expected = "Rödräven är ett hunddjur som har en mycket vidsträckt utbredning över norra halvklotet ." result = reader.transform(TEST_DATA) assert expected == result
def test_extract_tokens_pos_propn_succeeds(df_doc: pd.DataFrame): pytest.importorskip("spacy") df_doc = df_doc.copy() extract_opts = ExtractTaggedTokensOpts( lemmatize=True, pos_includes='|PROPN|', pos_paddings=None, **SPACY_TAGGED_COLUMNS, filter_opts=dict(is_punct=False), ) tokens = tagged_frame_to_tokens(doc=df_doc, extract_opts=extract_opts) assert tokens == ['mars']
def test_reader_when_source_is_sparv3_succeeds(): sparv_zipped_xml_export_v3_filename = './tests/test_data/sou_test_sparv3_xml.zip' reader = readers.Sparv3XmlReader( sparv_zipped_xml_export_v3_filename, chunk_size=None, extract_tokens_opts=ExtractTaggedTokensOpts(pos_includes='|NN|', pos_paddings=None, lemmatize=True), ) for _, (_, tokens) in enumerate(reader): assert len(list(tokens)) > 0
def test_extract_tokens_when_lemma_lacks_underscore_succeeds( df_doc: pd.DataFrame): pytest.importorskip("spacy") df_doc = df_doc.copy() extract_opts = ExtractTaggedTokensOpts( lemmatize=False, target_override="lemma_", **SPACY_TAGGED_COLUMNS, filter_opts=dict(is_punct=False), ) tokens = tagged_frame_to_tokens(doc=df_doc, extract_opts=extract_opts) assert tokens == [ 'Mars', 'be', 'once', 'home', 'to', 'sea', 'and', 'ocean', 'and', 'perhaps', 'even', 'life' ]
def test_tagged_frame_to_tokens_succeeds(): pipeline = Mock(spec=CorpusPipeline, payload=Mock(spec=PipelinePayload, tagged_columns_names={})) task = tasks.TaggedFrameToTokens( pipeline=pipeline, extract_opts=ExtractTaggedTokensOpts( lemmatize=True, **SPACY_TAGGED_COLUMNS, filter_opts=dict(is_punct=False), ), ).setup() current_payload = next(fake_data_frame_stream(1)) next_payload = task.process(current_payload) assert next_payload.content_type == ContentType.TOKENS
def test_extract_when_lemmatized_returns_baseform(): expected = 'rödräv vara en hunddjur som ha en mycken vidsträckt utbredning över norra halvklot . ' content = sparv_xml_test_file() parser = sparv.SparvXml2Text( delimiter=" ", extract_tokens_opts=ExtractTaggedTokensOpts(pos_includes='', pos_paddings=None, lemmatize=True, append_pos=False, pos_excludes=''), ) result = parser.transform(content) assert result == expected
def test_extract_when_ignore_punctuation_filters_out_punctuations(): expected = "Rödräven är ett hunddjur som har en mycket vidsträckt utbredning över norra halvklotet " content = sparv_xml_test_file() parser = sparv.SparvXml2Text( delimiter=" ", extract_tokens_opts=ExtractTaggedTokensOpts( pos_includes='', lemmatize=False, append_pos=False, pos_excludes="|MAD|MID|PAD|"), ) result = parser.transform(content) assert result == expected
def test_extract_when_no_filter_or_lemmatize_returns_original_text(): expected = "Rödräven är ett hunddjur som har en mycket vidsträckt utbredning över norra halvklotet . " content = sparv_xml_test_file() parser = sparv.SparvXml2Text( delimiter=" ", extract_tokens_opts=ExtractTaggedTokensOpts(pos_includes='', pos_paddings=None, lemmatize=False, append_pos=False, pos_excludes=''), ) result = parser.transform(content) assert result == expected
def test_extract_when_lemmatized_and_filter_nouns_returns_nouns_in_baseform(): expected = 'rödräv hunddjur utbredning halvklot ' content = sparv_xml_test_file() parser = sparv.SparvXml2Text( delimiter=" ", extract_tokens_opts=ExtractTaggedTokensOpts( pos_includes="|NN|", pos_paddings=None, lemmatize=True, append_pos=False, pos_excludes="|MAD|MID|PAD|"), ) result = parser.transform(content) assert result == expected
def test_reader_when_lemmatized_nn_vb_pos_appended_returns_lemmatized_nn_vb_pos(): reader = SparvCsvToText( extract_tokens_opts=ExtractTaggedTokensOpts( pos_includes='NN|VB', pos_paddings='JJ', pos_excludes='', lemmatize=True, append_pos=True, **SPARV_TAGGED_COLUMNS, ) ) expected = "rödräv@NN vara@VB hunddjur@NN ha@VB * utbredning@NN * halvklot@NN" result = reader.transform(TEST_DATA) assert expected == result
def test_reader_when_lemmatized_nn_vb_returns_lemmatized_nn_vb(): tokens_reader = readers.SparvCsvTokenizer( source=SPARV_CSV_EXPORT_FILENAME_SMALL, reader_opts=TextReaderOpts(), extract_opts=ExtractTaggedTokensOpts(pos_includes='NN|VB', pos_paddings=None, pos_excludes=None, lemmatize=True, **SPARV_TAGGED_COLUMNS), ) expected = "rödräv vara hunddjur ha utbredning halvklot".split() filename, tokens = next(tokens_reader) assert filename == os.path.split(filename)[1] assert expected == tokens
def test_reader_when_chunk_size_specified_returns_chunked_text(): expected_documents = [['rödräv', 'hunddjur'], ['utbredning', 'halvklot']] expected_names = [ "sparv_xml_export_small_001.txt", "sparv_xml_export_small_002.txt" ] reader = readers.SparvXmlReader( SPARV_XML_EXPORT_FILENAME_SMALL, chunk_size=2, extract_opts=ExtractTaggedTokensOpts(pos_includes='|NN|', lemmatize=True), ) for i, (filename, tokens) in enumerate(reader): assert expected_documents[i] == list(tokens) assert expected_names[i] == filename
def test_reader_when_no_transforms_returns_source_tokens(): tokens_reader = readers.SparvCsvTokenizer( source=SPARV_CSV_EXPORT_FILENAME_SMALL, reader_opts=TextReaderOpts(), extract_opts=ExtractTaggedTokensOpts(pos_includes=None, pos_paddings=None, pos_excludes=None, lemmatize=False, **SPARV_TAGGED_COLUMNS), ) expected = "Rödräven är ett hunddjur som har en mycket vidsträckt utbredning över norra halvklotet .".split( ) filename, tokens = next(tokens_reader) assert filename == os.path.split(filename)[1] assert expected == tokens
def test_reader_when_only_nouns_ignore_puncts_returns_filter_outs_puncts(): expected = ['rödräv', 'hunddjur', 'utbredning', 'halvklot'] expected_name = "sparv_xml_export_small.txt" reader = readers.SparvXmlReader( SPARV_XML_EXPORT_FILENAME_SMALL, chunk_size=None, extract_opts=ExtractTaggedTokensOpts( pos_includes='|NN|', pos_paddings=None, lemmatize=True, ), ) filename, tokens = next(iter(reader)) assert expected == list(tokens) assert expected_name == filename
def test_spacy_pipeline_load_text_to_spacy_to_dataframe_to_tokens_to_text_to_dtm( en_nlp): pytest.importorskip("spacy") reader_opts = TextReaderOpts(filename_pattern="*.txt", filename_fields="year:_:1") text_transform_opts = TextTransformOpts() reader = TextReader.create(MARY_TEST_CORPUS, reader_opts=reader_opts, transform_opts=text_transform_opts) attributes = ['text', 'lemma_', 'pos_', 'is_punct'] extract_opts = ExtractTaggedTokensOpts( lemmatize=True, pos_includes='|VERB|NOUN|', pos_paddings=None, **SPACY_TAGGED_COLUMNS, filter_opts=dict(is_punct=False), ) transform_opts = None vectorize_opts = VectorizeOpts() config = Mock( spec=CorpusConfig, pipeline_payload=PipelinePayload(source=reader).put2( **SPACY_TAGGED_COLUMNS), ) pipeline = (CorpusPipeline(config=config).load_text( reader_opts=reader_opts, transform_opts=text_transform_opts).set_spacy_model( en_nlp).text_to_spacy().spacy_to_tagged_frame( attributes=attributes).tagged_frame_to_tokens( extract_opts=extract_opts, transform_opts=transform_opts).tokens_to_text().to_dtm( vectorize_opts)) corpus = pipeline.value() assert corpus is not None assert isinstance(corpus, VectorizedCorpus)
def test_reader_when_source_is_zipped_archive_succeeds(): expected_documents = [ ['rödräv', 'hunddjur', 'utbredning', 'halvklot'], [ 'fjällräv', 'fjällvärld', 'liv', 'fjällräv', 'vinter', 'men', 'variant', 'år' ], ] expected_names = ["document_001.txt", "document_002.txt"] reader = readers.SparvXmlReader( SPARV_ZIPPED_XML_EXPORT_FILENAME, chunk_size=None, extract_opts=ExtractTaggedTokensOpts(pos_includes='|NN|', pos_paddings=None, lemmatize=True), ) for i, (filename, tokens) in enumerate(reader): assert expected_documents[i] == list(tokens) assert expected_names[i] == filename