def line_handler(db: Database, line_queue: Queue, error_queue: Queue,
source_id: int) -> None:
session = get_session(db)
source = session.query(Source).filter(Source.id == source_id).one()
while True:
try:
line_pair = line_queue.get()
if line_pair == DONE_READING:
break
line_no, line = line_pair
stems = nlp.stem_sentence(line)
rhyme_sound = nlp.rhyme_sound(line)
syllables = nlp.count_syllables(line)
alliteration = nlp.has_alliteration(line)
phrase = Phrase(stems=stems,
raw=line,
alliteration=alliteration,
rhyme_sound=rhyme_sound,
syllables=syllables,
line_no=line_no,
source=source)
session.add(phrase)
except Exception as e:
error_queue.put(e)
log.error('Died while processing text, rolling back')
session.rollback()
session.close()
return
session.commit()
def line_handler(db: Database,
line_queue: Queue,
error_queue: Queue,
source_id: int) -> None:
session = get_session(db)
source = session.query(Source).filter(Source.id == source_id).one()
while True:
try:
line_pair = line_queue.get()
if line_pair == DONE_READING:
break
line_no, line = line_pair
stems = nlp.stem_sentence(line)
rhyme_sound = nlp.rhyme_sound(line)
syllables = nlp.count_syllables(line)
alliteration = nlp.has_alliteration(line)
phrase = Phrase(stems=stems, raw=line, alliteration=alliteration,
rhyme_sound=rhyme_sound,
syllables=syllables, line_no=line_no, source=source)
session.add(phrase)
except Exception as e:
error_queue.put(e)
log.error('Died while processing text, rolling back')
session.rollback()
session.close()
return
session.commit()
def process_sentence(sentence, source_name, line_no):
phonemes = list(map(nlp.word_to_phonemes, nlp.words(sentence)))
return {'stems': nlp.stem_sentence(sentence),
'source': source_name,
'tagged': nlp.tag(sentence),
'rhyme_sound': nlp.rhyme_sound(sentence),
'phonemes': phonemes,
'num_syllables': nlp.count_syllables(sentence),
'line_no': line_no,
'alliteration': nlp.has_alliteration(sentence),
'raw': sentence,
'blank': False,
}
def process_sentence(sentence, source_name, line_no):
phonemes = list(map(nlp.word_to_phonemes, nlp.words(sentence)))
return {
'stems': nlp.stem_sentence(sentence),
'source': source_name,
'tagged': nlp.tag(sentence),
'rhyme_sound': nlp.rhyme_sound(sentence),
'phonemes': phonemes,
'num_syllables': nlp.count_syllables(sentence),
'line_no': line_no,
'alliteration': nlp.has_alliteration(sentence),
'raw': sentence,
'blank': False,
}
def test_alliteration():
should = [("Give me the splendid silent sun", True),
("Peter Piper picked a peck of pickled peppers", True),
("And the silken sad uncertain rustling of each purple curtain", True),
("i ate the line along", True),
("Zany zebras need to leave", True),
("i eat green apples every day", False),
# TODO need better sound detection of unknown words to get this one:
#("cardossian cruiser, you are cleared to land.", True),
("power protip: touch plants and have feels", True),
("once upon ultimate fantasy", False),
("purely and fundamentally for analytical purposes", True),
("Cats chew on dead mice", False),
("Sad shanties line darkened blocks", False),]
for original, expected in should:
assert nlp.has_alliteration(original) == expected
def test_alliteration():
should = [
("Give me the splendid silent sun", True),
("Peter Piper picked a peck of pickled peppers", True),
("And the silken sad uncertain rustling of each purple curtain", True),
("i ate the line along", True),
("Zany zebras need to leave", True),
("i eat green apples every day", False),
# TODO need better sound detection of unknown words to get this one:
#("cardossian cruiser, you are cleared to land.", True),
("power protip: touch plants and have feels", True),
("once upon ultimate fantasy", False),
("purely and fundamentally for analytical purposes", True),
("Cats chew on dead mice", False),
("Sad shanties line darkened blocks", False),
]
for original, expected in should:
assert nlp.has_alliteration(original) == expected