def get_char_counts(corpus_reader: CorpusReader) -> Dict[str, int]:
    """
    Get a frequency distribution of characters in a corpus.
    :param corpus_reader:
    :return:
    """
    char_counter = Counter()  # type: Dict[str, int]
    files = corpus_reader.fileids()
    for file in tqdm(files, total=len(files), unit="files"):
        for word in corpus_reader.words(file):
            if word.isalpha():
                for car in word:
                    char_counter.update({car: 1})
    return char_counter
def get_word_lengths(corpus_reader: CorpusReader, max_word_length: int = 100) -> Dict[int, int]:
    """
    Get the word length/frequency distribution
    :param corpus_reader:
    :param max_word_length:
    :return:
    """
    word_lengths = Counter()  # type: Dict[int, int]
    files = corpus_reader.fileids()
    for file in tqdm(files, total=len(files), unit='files'):
        for word in corpus_reader.words(file):
            word_length = len(word)
            if word.isalpha() and word_length <= max_word_length:
                word_lengths.update({word_length: 1})
    return word_lengths
Example #3
0
def assemble_corpus(
    corpus_reader: CorpusReader,
    types_requested: List[str],
    type_dirs: Dict[str, List[str]] = None,
    type_files: Dict[str, List[str]] = None
) -> Tuple[CorpusReader, List[str], Set[str]]:
    """
    Create a filtered corpus.
    :param corpus_reader: This get mutated
    :param types_requested: a list of string types, which are to be found in the type_dirs and
    type_files mappings
    :param type_dirs: a dict of corpus types to directories
    :param type_files: a dict of corpus types to files
    :return: a Tuple(CorpusReader object containing only the mappings desired,
    fileid_names: A list of file ids of the matching corpus files, and
    categories_found: a set of word categories used to build the reader
    """
    fileid_names = []  # type: List[str]
    categories_found = set()  # type: Set[str]
    try:
        ALL_FILE_IDS = list(corpus_reader.fileids())
        CLEAN_IDS_TYPES = []  # type: List[Tuple[str, str]]
        if type_files:
            for key, valuelist in type_files.items():
                if key in types_requested:
                    for value in valuelist:
                        if value in ALL_FILE_IDS:
                            if key:
                                CLEAN_IDS_TYPES.append((value, key))
        if type_dirs:
            for key, valuelist in type_dirs.items():
                if key in types_requested:
                    for value in valuelist:
                        corrected_dir = value.replace('./', '')
                        corrected_dir = '{}/'.format(corrected_dir)
                        for name in ALL_FILE_IDS:
                            if name and name.startswith(corrected_dir):
                                CLEAN_IDS_TYPES.append((name, key))
        CLEAN_IDS_TYPES.sort(key=lambda x: x[0])
        fileid_names, categories = zip(*CLEAN_IDS_TYPES)  # type: ignore
        categories_found = set(categories)  # type: Set[str]
        corpus_reader._fileids = fileid_names
    except Exception:
        LOG.exception('failure in corpus building')

    return (corpus_reader, fileid_names, categories_found)
def get_samples_for_lengths(corpus_reader: CorpusReader,
                            num_samples: int = 5) -> Dict[int, List[str]]:
    """
    Get a number of sample words for each word length; good for sanity checking.
    :param corpus_reader:
    :param num_samples:
    :return:
    """
    samples_lengths = defaultdict(list)  # type: Dict[int, List[str]]
    files = corpus_reader.fileids()
    for file in tqdm(files, total=len(files), unit="files"):
        for word in corpus_reader.words(file):
            if word.isalpha():
                word_length = len(word)
                samples_lengths[word_length].append(word)
                samples_lengths[word_length] = samples_lengths[
                    word_length][:num_samples]  # trim to num_samples size
    return samples_lengths
Example #5
0
 def fileids(self, channels=None, domains=None, categories=None):
     if channels is not None and domains is not None and categories is not None:
         raise ValueError('You can specify only one of channels, domains '
                          'and categories parameter at once')
     if channels is None and domains is None and categories is None:
         return CorpusReader.fileids(self)
     if isinstance(channels, string_types):
         channels = [channels]
     if isinstance(domains, string_types):
         domains = [domains]
     if isinstance(categories, string_types):
         categories = [categories]
     if channels:
         return self._list_morph_files_by('channel', channels)
     elif domains:
         return self._list_morph_files_by('domain', domains)
     else:
         return self._list_morph_files_by('keyTerm',
                                          categories,
                                          map=self._map_category)
Example #6
0
 def fileids(self, channels=None, domains=None, categories=None):
     if channels is not None and domains is not None and categories is not None:
         raise ValueError("You can specify only one of channels, domains "
                          "and categories parameter at once")
     if channels is None and domains is None and categories is None:
         return CorpusReader.fileids(self)
     if isinstance(channels, str):
         channels = [channels]
     if isinstance(domains, str):
         domains = [domains]
     if isinstance(categories, str):
         categories = [categories]
     if channels:
         return self._list_morph_files_by("channel", channels)
     elif domains:
         return self._list_morph_files_by("domain", domains)
     else:
         return self._list_morph_files_by("keyTerm",
                                          categories,
                                          map=self._map_category)
Example #7
0
def assemble_corpus(
    corpus_reader: CorpusReader,
    types_requested: List[str],
    type_dirs: Dict[str, List[str]] = None,
    type_files: Dict[str, List[str]] = None,
) -> CorpusReader:
    """
    Create a filtered corpus.
    :param corpus_reader: This get mutated
    :param types_requested: a list of string types, which are to be found in the type_dirs and
    type_files mappings
    :param type_dirs: a dict of corpus types to directories
    :param type_files: a dict of corpus types to files
    :return: a CorpusReader object containing only the mappings desired
    """
    fileid_names = []  # type: List[str]
    try:
        all_file_ids = list(corpus_reader.fileids())
        clean_ids_types = []  # type: List[Tuple[str, str]]
        if type_files:
            for key, valuelist in type_files.items():
                if key in types_requested:
                    for value in valuelist:
                        if value in all_file_ids:
                            if key:
                                clean_ids_types.append((value, key))
        if type_dirs:
            for key, valuelist in type_dirs.items():
                if key in types_requested:
                    for value in valuelist:
                        corrected_dir = value.replace("./", "")
                        corrected_dir = "{}/".format(corrected_dir)
                        for name in all_file_ids:
                            if name and name.startswith(corrected_dir):
                                clean_ids_types.append((name, key))
        clean_ids_types.sort(key=lambda x: x[0])
        fileid_names, categories = zip(*clean_ids_types)  # type: ignore
        corpus_reader._fileids = fileid_names
        return corpus_reader
    except Exception:
        LOG.exception("failure in corpus building")
def get_split_words(corpus_reader: CorpusReader,
                    word_trie: WordTrie,
                    max_word_length: int = 15) -> Dict[str, List[str]]:
    """
    Search a corpus for improperly joined words, defined by a discrete trie model.
    return a dictionary, keys are files, and values are lists of tuples of the split words.

    :param corpus_reader:
    :param word_trie:
    :param max_word_length:
    :return:
    """
    split_words = defaultdict(list)  # type: Dict[str, List[str]]
    files = corpus_reader.fileids()
    for file in tqdm(files, total=len(files), unit="files"):
        for word in corpus_reader.words(file):
            if len(word) > max_word_length and not word_trie.has_word(word):
                word_list = word_trie.extract_word_pair(word)
                if len(word_list) == 2:
                    split_words[file] += word_list
    return split_words
Example #9
0
 def fileids(self, channels=None, domains=None, categories=None):
     if channels is not None and domains is not None and \
             categories is not None:
         raise ValueError('You can specify only one of channels, domains '
                          'and categories parameter at once')
     if channels is None and domains is None and \
             categories is None:
         return CorpusReader.fileids(self)
     if isinstance(channels, basestring):
         channels = [channels]
     if isinstance(domains, basestring):
         domains = [domains]
     if isinstance(categories, basestring):
         categories = [categories]
     if channels:
         return self._list_morph_files_by('channel', channels)
     elif domains:
         return self._list_morph_files_by('domain', domains)
     else:
         return self._list_morph_files_by('keyTerm', categories,
                 map=self._map_category)
Example #10
0
def assemble_corpus(corpus_reader: CorpusReader,
                    types_requested: List[str],
                    type_dirs: Dict[str, List[str]] = None,
                    type_files: Dict[str, List[str]] = None) -> CorpusReader:
    """
    Create a filtered corpus.
    :param corpus_reader: This get mutated
    :param types_requested: a list of string types, which are to be found in the type_dirs and
    type_files mappings
    :param type_dirs: a dict of corpus types to directories
    :param type_files: a dict of corpus types to files
    :return: a CorpusReader object containing only the mappings desired
    """
    fileid_names = []  # type: List[str]
    try:
        all_file_ids = list(corpus_reader.fileids())
        clean_ids_types = []  # type: List[Tuple[str, str]]
        if type_files:
            for key, valuelist in type_files.items():
                if key in types_requested:
                    for value in valuelist:
                        if value in all_file_ids:
                            if key:
                                clean_ids_types.append((value, key))
        if type_dirs:
            for key, valuelist in type_dirs.items():
                if key in types_requested:
                    for value in valuelist:
                        corrected_dir = value.replace('./', '')
                        corrected_dir = '{}/'.format(corrected_dir)
                        for name in all_file_ids:
                            if name and name.startswith(corrected_dir):
                                clean_ids_types.append((name, key))
        clean_ids_types.sort(key=lambda x: x[0])
        fileid_names, categories = zip(*clean_ids_types)  # type: ignore
        corpus_reader._fileids = fileid_names
        return corpus_reader
    except Exception:
        LOG.exception('failure in corpus building')