def test_load_spelling_ignore(text_test_dir): spelling_path = os.path.join(text_test_dir, 'test_text_spelling.txt') a = inspect_discourse_spelling(spelling_path) a[0].ignored_characters = set(["'", '.']) c = load_discourse_spelling('test', spelling_path, a) assert (c.lexicon['ab'].frequency == 3) assert (c.lexicon['cabd'].frequency == 1)
def test_load_spelling_ignore(text_test_dir): spelling_path = os.path.join(text_test_dir, 'test_text_spelling.txt') a = inspect_discourse_spelling(spelling_path) a[0].ignored_characters = set(["'",'.']) c = load_discourse_spelling('test',spelling_path, a) assert(c.lexicon['ab'].frequency == 3) assert(c.lexicon['cabd'].frequency == 1)
def test_export_spelling(export_test_dir, unspecified_test_corpus): d = generate_discourse(unspecified_test_corpus) export_path = os.path.join(export_test_dir, 'test_export_spelling.txt') export_discourse_spelling(d, export_path, single_line = False) d2 = load_discourse_spelling('test', export_path) for k in unspecified_test_corpus.keys(): assert(d2.lexicon[k].spelling == unspecified_test_corpus[k].spelling) assert(d2.lexicon[k].frequency == unspecified_test_corpus[k].frequency)
def test_export_spelling(export_test_dir, unspecified_test_corpus): d = generate_discourse(unspecified_test_corpus) export_path = os.path.join(export_test_dir, 'test_export_spelling.txt') export_discourse_spelling(d, export_path, single_line=False) d2 = load_discourse_spelling('test', export_path) for k in unspecified_test_corpus.keys(): assert (d2.lexicon[k].spelling == unspecified_test_corpus[k].spelling) assert ( d2.lexicon[k].frequency == unspecified_test_corpus[k].frequency)
def run(self): time.sleep(0.1) textType = self.kwargs.pop('text_type') isDirectory = self.kwargs.pop('isDirectory') logging.info('Importing {} corpus named {}'.format( textType, self.kwargs['corpus_name'])) logging.info('Path: '.format(self.kwargs['path'])) log_annotation_types(self.kwargs['annotation_types']) try: if textType == 'spelling': if isDirectory: corpus = load_directory_spelling(**self.kwargs) else: corpus = load_discourse_spelling(**self.kwargs) elif textType == 'transcription': if isDirectory: corpus = load_directory_transcription(**self.kwargs) else: corpus = load_discourse_transcription(**self.kwargs) elif textType == 'ilg': if isDirectory: corpus = load_directory_ilg(**self.kwargs) else: corpus = load_discourse_ilg(**self.kwargs) elif textType == 'textgrid': if isDirectory: corpus = load_directory_textgrid(**self.kwargs) else: corpus = load_discourse_textgrid(**self.kwargs) elif textType == 'csv': corpus = load_corpus_csv(**self.kwargs) elif textType in ['buckeye', 'timit']: self.kwargs['dialect'] = textType if isDirectory: corpus = load_directory_multiple_files(**self.kwargs) else: corpus = load_discourse_multiple_files(**self.kwargs) except PCTError as e: self.errorEncountered.emit(e) return except Exception as e: e = PCTPythonError(e) self.errorEncountered.emit(e) return if self.stopped: time.sleep(0.1) self.finishedCancelling.emit() return self.dataReady.emit(corpus)
def run(self): time.sleep(0.1) textType = self.kwargs.pop('text_type') isDirectory = self.kwargs.pop('isDirectory') logging.info('Importing {} corpus named {}'.format(textType, self.kwargs['corpus_name'])) logging.info('Path: '.format(self.kwargs['path'])) log_annotation_types(self.kwargs['annotation_types']) try: if textType == 'spelling': if isDirectory: corpus = load_directory_spelling(**self.kwargs) else: corpus = load_discourse_spelling(**self.kwargs) elif textType == 'transcription': if isDirectory: corpus = load_directory_transcription(**self.kwargs) else: corpus = load_discourse_transcription(**self.kwargs) elif textType == 'ilg': if isDirectory: corpus = load_directory_ilg(**self.kwargs) else: corpus = load_discourse_ilg(**self.kwargs) elif textType == 'textgrid': if isDirectory: corpus = load_directory_textgrid(**self.kwargs) else: corpus = load_discourse_textgrid(**self.kwargs) elif textType == 'csv': corpus = load_corpus_csv(**self.kwargs) elif textType in ['buckeye', 'timit']: self.kwargs['dialect'] = textType if isDirectory: corpus = load_directory_multiple_files(**self.kwargs) else: corpus = load_discourse_multiple_files(**self.kwargs) except PCTError as e: self.errorEncountered.emit(e) return except Exception as e: e = PCTPythonError(e) self.errorEncountered.emit(e) return if self.stopped: time.sleep(0.1) self.finishedCancelling.emit() return self.dataReady.emit(corpus)
def test_load_spelling_no_ignore(text_test_dir): spelling_path = os.path.join(text_test_dir, 'test_text_spelling.txt') c = load_discourse_spelling('test',spelling_path) assert(c.lexicon['ab'].frequency == 2)
def test_load_spelling_no_ignore(text_test_dir): spelling_path = os.path.join(text_test_dir, 'test_text_spelling.txt') c = load_discourse_spelling('test', spelling_path) assert (c.lexicon['ab'].frequency == 2)