def timespan_sql_with_dateinfo( corpus: Corpus = Corpus(), out: Export = Export("korp_timespan/timespan.sql"), docs: AllDocuments = AllDocuments(), token: AnnotationAllDocs = AnnotationAllDocs("<token>"), datefrom: AnnotationAllDocs = AnnotationAllDocs( "<text>:dateformat.datefrom"), dateto: AnnotationAllDocs = AnnotationAllDocs( "<text>:dateformat.dateto"), timefrom: AnnotationAllDocs = AnnotationAllDocs( "<text>:dateformat.timefrom"), timeto: AnnotationAllDocs = AnnotationAllDocs( "<text>:dateformat.timeto")): """Create timespan SQL data for use in Korp.""" corpus_name = corpus.upper() datespans = defaultdict(int) datetimespans = defaultdict(int) for doc in docs: text_tokens, orphans = Annotation(datefrom.name, doc=doc).get_children(token) if orphans: datespans[("0" * 8, "0" * 8)] += len(orphans) datetimespans[("0" * 14, "0" * 14)] += len(orphans) dateinfo = datefrom.read_attributes( doc, (datefrom, dateto, timefrom, timeto)) for text in text_tokens: d = next(dateinfo) datespans[(d[0].zfill(8), d[1].zfill(8))] += len(text) datetimespans[(d[0].zfill(8) + d[2].zfill(6), d[1].zfill(8) + d[3].zfill(6))] += len(text) rows_date = [] rows_datetime = [] for span in datespans: rows_date.append({ "corpus": corpus_name, "datefrom": span[0], "dateto": span[1], "tokens": datespans[span] }) for span in datetimespans: rows_datetime.append({ "corpus": corpus_name, "datefrom": span[0], "dateto": span[1], "tokens": datetimespans[span] }) create_sql(corpus_name, out, rows_date, rows_datetime)
def freq_list_simple(corpus: Corpus = Corpus(), docs: AllDocuments = AllDocuments(), word: AnnotationAllDocs = AnnotationAllDocs("<token:word>"), pos: AnnotationAllDocs = AnnotationAllDocs("<token:pos>"), baseform: AnnotationAllDocs = AnnotationAllDocs("<token:baseform>"), out: Export = Export("frequency_list/stats_[metadata.id].csv"), delimiter: str = Config("stats_export.delimiter"), cutoff: int = Config("stats_export.cutoff")): """Create a word frequency list for a corpus without sense, lemgram and complemgram annotations.""" freq_dict = defaultdict(int) for doc in docs: simple_tokens = word.read_attributes(doc, [word, pos, baseform]) # Add empty annotations for sense, lemgram and complemgram tokens = [] for w, p, b in simple_tokens: tokens.append((w, p, b, "|", "|", "|")) update_freqs(tokens, freq_dict) write_csv(out, freq_dict, delimiter, cutoff)
def freq_list(corpus: Corpus = Corpus(), docs: AllDocuments = AllDocuments(), word: AnnotationAllDocs = AnnotationAllDocs("<token:word>"), msd: AnnotationAllDocs = AnnotationAllDocs("<token:msd>"), baseform: AnnotationAllDocs = AnnotationAllDocs("<token:baseform>"), sense: AnnotationAllDocs = AnnotationAllDocs("<token:sense>"), lemgram: AnnotationAllDocs = AnnotationAllDocs("<token>:saldo.lemgram"), complemgram: AnnotationAllDocs = AnnotationAllDocs("<token>:saldo.complemgram"), out: Export = Export("frequency_list/stats_[metadata.id].csv"), delimiter: str = Config("stats_export.delimiter"), cutoff: int = Config("stats_export.cutoff"), include_all_compounds: bool = Config("stats_export.include_all_compounds")): """Create a word frequency list for the entire corpus. Args: corpus (str, optional): The corpus ID. Defaults to Corpus. docs (list, optional): The documents belonging to this corpus. Defaults to AllDocuments. word (str, optional): Word annotations. Defaults to AnnotationAllDocs("<token:word>"). msd (str, optional): MSD annotations. Defaults to AnnotationAllDocs("<token:msd>"). baseform (str, optional): Baseform annotations. Defaults to AnnotationAllDocs("<token:baseform>"). sense (str, optional): Sense annotations. Defaults to AnnotationAllDocs("<token:sense>"). lemgram (str, optional): Lemgram annotations. Defaults to AnnotationAllDocs("<token>:saldo.lemgram"). complemgram (str, optional): Compound lemgram annotations. Defaults to AnnotationAllDocs("<token>:saldo.complemgram"). out (str, optional): The output word frequency file. Defaults to Export("frequency_list/[metadata.id].csv"). delimiter (str, optional): Column delimiter to use in the csv. Defaults to Config("stats_export.delimiter"). cutoff (int, optional): The minimum frequency a word must have in order to be included in the result. Defaults to Config("stats_export.cutoff"). include_all_compounds (bool, optional): Whether to include compound analyses for every word or just for the words that are lacking a sense annotation. Defaults to Config("stats_export.include_all_compounds"). """ freq_dict = defaultdict(int) for doc in docs: tokens = word.read_attributes(doc, [word, msd, baseform, sense, lemgram, complemgram]) update_freqs(tokens, freq_dict, include_all_compounds) write_csv(out, freq_dict, delimiter, cutoff)