def _store(self, en, pl):
phrase = Phrase(en, Handlers().resolve("en"))
length = phrase.length()
if length == 0:
return
self._cursor.execute(u"insert into phrases (en, pl, length) values (?, ?, ?)", (en, pl, length))
self._store_words(self._cursor.lastrowid, phrase)
def test_update_parsed(self):
scope = Scope()
phrase = 'changeable value is `changeable`'
phrase = Phrase(phrase, scope)
phrase._update_parsed()
self.assertEqual(phrase.latest, "changeable value is True")
# no exception
scope.change()
phrase._update_parsed()
self.assertEqual(phrase.latest, "changeable value is False")
# no exception
scope.change()
scope.raise_error()
self.assertRaises(KeyError, phrase._update_parsed)
# also test not changed, with exception
scope = Scope()
phrase = 'changeable value is `changeable`'
phrase = Phrase(phrase, scope)
phrase._update_parsed()
self.assertEqual(phrase.latest, "changeable value is True")
scope.raise_error()
self.assertRaises(KeyError, phrase._update_parsed)
def build_lattice(self, pt, sentence):
'''
Gets a phrase table and the tokenized sentence and outputs a lattice
file formatted as follows:
whole sentence
1-1:
<English translation> <Translation score>
<English translation> <Translation score>
...
1-2:
<English translation> <Translation score>
<English translation> <Translation score>
...
2-2:
The spans n-n refer to the tokens of the input Spanish sentence
'''
sentence = tokenize(sentence)
self.sentence = sentence
for start in xrange(len(sentence)):
self.phrases[start] = {}
for end in xrange(start+1, len(sentence)+1):
foreign = sentence[start:end]
p = Phrase(foreign, start, end)
if len(foreign) == 1 and foreign[0] == ',':
p.translations = [Translation(foreign, (',',), 0)]
else:
p.translations = pt.translate(foreign)
self.phrases[start][end] = p
def deconstructor(self, dynamics=90):
# Determine the content of each phrase
chords = self.chords
phrases = []
sum = 0 # Once sum hits 2, append the phrase to phrases
chordsInPhrase = []
prevPost = random.randint(60, 70)
for chord in chords:
chordsInPhrase.append(chord)
sum += chord.dur
if(numpy.isclose(sum, 1) and len(chordsInPhrase) >= 3):
phrases.append(Phrase(chordsInPhrase, self.banks, self.rhythmBank,
dynamics, self.genre, 1, prevPost))
prevPost = phrases[-1].lastEnd
chordsInPhrase = []
sum = 0
elif(numpy.isclose(sum, 2) and len(chordsInPhrase) >= 2):
phrases.append(Phrase(chordsInPhrase, self.banks, self.rhythmBank,
dynamics, self.genre, 2, prevPost))
prevPost = phrases[-1].lastEnd
chordsInPhrase = []
sum = 0
elif(numpy.isclose(sum, 4)):
phrases.append(Phrase(chordsInPhrase, self.banks, self.rhythmBank,
dynamics, self.genre, 4, prevPost))
prevPost = phrases[-1].lastEnd
chordsInPhrase = []
sum = 0
self.phrases = phrases
def testCommon(self):
db = Db()
db.dbFile = "testmetabrainwallet.db"
db.phrase.dropTable()
db.phrase.createTable()
db.phrase.addCommon('count_1w100.txt')
good = Phrase({'language': 'en', 'content': 'information'})
bad = Phrase({
'language': 'en',
'content': 'supercalifragilisticexpialidocious'
})
empty = Phrase({'language': 'en', 'content': ''})
self.assertTrue(db.phrase.contains(good))
memos = db.phrase.loadMemosByColumns({
'language': 'en',
'content': 'information'
})
self.assertTrue(int(memos[0]['frequency']) > 0)
self.assertFalse(db.phrase.contains(bad))
self.assertFalse(db.phrase.contains(empty))
db.close()
def create_copula_phrase(predicative_case="Nom"):
global grammar
structure = grammar["VP_COPULA"]
governance = {"predicative": {u"CASE": predicative_case}}
structure["governance"] = governance
vp = Phrase("olla", structure)
vp.morphology["VOICE"] = "ACT"
return vp
def create_copula_phrase(predicative_case="Nom"):
global grammar
structure = grammar["VP_COPULA"]
governance = { "predicative" : {u"CASE" : predicative_case}}
structure["governance"] = governance
vp = Phrase("olla", structure)
vp.morphology["VOICE"] = "ACT"
return vp
def __init__(self, phrases, lives=5, game_not_over=True):
self.raw_phrases = phrases
self.phrases = [phrase for phrase in phrases]
self.phrase = Phrase(self.phrases[random.randint(
0, (len(self.phrases) - 1))])
self.lives = lives
self.game_not_over = game_not_over
self.factor = lives
self.attempts = []
def test_erase_routine_requests(self):
phrase = self.fake_class('Hello world `my_var:32` `name~John`', Scope())
Phrase._erase_routine_requests(phrase)
self.assertEqual(phrase.phrase, 'Hello world `my_var:32` `name~John`')
phrase = self.fake_class('Hello world `routine1<"stop"` `my_var:32` `routine1<"stop"` `name~John`', Scope())
Phrase._erase_routine_requests(phrase)
self.assertEqual(phrase.phrase, 'Hello world `my_var:32` `name~John`')
def test_erase_flexible_setters(self):
phrase = self.fake_class('Hello world `my_var:32` `routine1<"stop"`', Scope())
Phrase._erase_flexible_setters(phrase)
self.assertEqual(phrase.phrase, 'Hello world `my_var:32` `routine1<"stop"`')
phrase = self.fake_class('Hello world `name~John` `my_var:32` `name~Mike` `routine1<"stop"`', Scope())
Phrase._erase_flexible_setters(phrase)
self.assertEqual(phrase.phrase, 'Hello world John `my_var:32` Mike `routine1<"stop"`')
def _store(self, en, pl):
phrase = Phrase(en, Handlers().resolve("en"))
length = phrase.length()
if length == 0:
return
self._cursor.execute(
u"insert into phrases (en, pl, length) values (?, ?, ?)",
(en, pl, length))
self._store_words(self._cursor.lastrowid, phrase)
def create_phrases(self):
phrases = [
Phrase("Misfit Love"),
Phrase("Battery Acid"),
Phrase("God is in the Radio"),
Phrase("Go With the Flow"),
Phrase("Make it wit Chu")
]
return phrases
def __init__(self):
self.missed = 0
self.phrases = [
Phrase('Is the juice worth the squeeze'),
Phrase('Green Lights'),
Phrase('An Arm and a Leg'),
Phrase('Dime a Dozen'),
Phrase('Down to the Wire')
]
self.active_phrase = self.get_random_phrase()
self.guesses = [" "]
def __init__(self):
self.misses = 0
self.phrases = [
Phrase('Hello world'),
Phrase('Just keep swimming'),
Phrase('Get to the chopper'),
Phrase('There is no place like home'),
Phrase('May the force be with you')
]
self.active_phrase = self.get_random_phrase()
self.guesses = [' ']
def test_erase_fixed_setters(self):
# origin = 'Hello world `my_var:32` `name~John` `routine1<"stop"`'
phrase = self.fake_class('Hello world `name~John` `routine1<"stop"`', Scope())
Phrase._erase_fixed_setters(phrase)
self.assertEqual(phrase.phrase, 'Hello world `name~John` `routine1<"stop"`')
phrase = self.fake_class('Hello world `my_var:32` `name~John` `my_var:32` `routine1<"stop"`', Scope())
Phrase._erase_fixed_setters(phrase)
self.assertEqual(phrase.phrase, 'Hello world `name~John` `routine1<"stop"`')
def __init__(self):
self.missed = 0
self.phrases = [
Phrase("how the turntables"),
Phrase("how you doing"),
Phrase("I am awesome"),
Phrase("we were on a break"),
Phrase("fish meat is practically a vegetable")
]
self.active_phrase = random.choice(self.phrases)
self.guesses = [" "]
def __init__(self):
self.phrases = [
Phrase("not my cup of tea"),
Phrase("have your cake and eat it too"),
Phrase("an apple a day keeps the doctor away"),
Phrase("no use crying over spilled milk"),
Phrase("two peas in a pod"),
]
self.active_phrase = None
self.guesses = []
self.missed = 0
self.replay = ""
def __init__(self, missed=0, phrases=[], active_phrase=None, guesses=[]):
self.missed = missed
self.phrases = [
Phrase('Hello world'),
Phrase('there is no trying'),
Phrase('May the force be with you'),
Phrase('you have to see the matrix for yourself'),
Phrase('life is like a box of chocolates')
]
self.active_phrase = self.get_random_phrase()
self.guesses = []
def __init__(self):
self.missed = 0
self.phrases = [
Phrase("simplicity becomes elegance"),
Phrase("i do not fear computers"),
Phrase("programming is fun"),
Phrase("imagination is more important than knowledge"),
]
self.active_phrase = self.random_phrase()
#this is the list that holds the letters that user has guessed
self.guesses = [" "]
def test_compare(self):
input_string = "my name is John"
input_parsed = link_parser.parse(input_string)
phrase = 'my name is `name~Mark`'
phrase = Phrase(phrase, Scope())
self.assertEqual(phrase.compare(input_parsed), 0.8) #4/5 simlar links
phrase = 'hello'
phrase = Phrase(phrase, Scope())
self.assertEqual(phrase.compare(input_parsed), 0)
def __init__(self):
self.phrases = [
Phrase("swings and roundabouts"),
Phrase("the ball is mightier than the keg"),
Phrase("the scythe is remorseless"),
Phrase("live by the sword die by the sword"),
Phrase("the whole of it calls for tears"),
]
self.missed = 0
self.active_phrase = self.get_random_phrase()
self.guesses = [" "]
self.replay = ""
def test_erase_routine_requests(self):
phrase = self.fake_class('Hello world `my_var:32` `name~John`',
Scope())
Phrase._erase_routine_requests(phrase)
self.assertEqual(phrase.phrase, 'Hello world `my_var:32` `name~John`')
phrase = self.fake_class(
'Hello world `routine1<"stop"` `my_var:32` `routine1<"stop"` `name~John`',
Scope())
Phrase._erase_routine_requests(phrase)
self.assertEqual(phrase.phrase,
'Hello world `my_var:32` `name~John`')
def __init__(self):
self.num_of_guesses = 5
self.maximum_guesses = 5
self.num_games_played = 0
self.phrases = [
Phrase("it takes one to know one"),
Phrase("the apple of my eye"),
Phrase("hair of the dog"),
Phrase("a frog in my throat"),
Phrase("fly by the seat of your pants")
]
self.active_phrase = self.get_random_phrase()
self.guesses = [" "]
def read(self, parsed_taxonomies):
"""
Read in taxonomies for a given code table.
params:
taxonomies (dict{id: Taxonomy})
"""
for key, taxonomy in parsed_taxonomies.iteritems():
synonym_phrases = [
Phrase(synonym) for synonym in taxonomy.synonyms
]
head_phrase = Phrase(taxonomy.head)
self.taxonomies[key] = Taxonomy(key, head_phrase, synonym_phrases)
def test_update_parsed(self):
scope = Scope()
phrase = 'changeable value is `changeable`'
phrase = Phrase(phrase, scope)
phrase._update_parsed()
self.assertEqual(phrase.latest, "changeable value is True")
# no exception
scope.change()
phrase._update_parsed()
self.assertEqual(phrase.latest, "changeable value is False")
# no exception
scope.change()
scope.raise_error()
self.assertRaises(KeyError, phrase._update_parsed)
# also test not changed, with exception
scope = Scope()
phrase = 'changeable value is `changeable`'
phrase = Phrase(phrase, scope)
phrase._update_parsed()
self.assertEqual(phrase.latest, "changeable value is True")
scope.raise_error()
self.assertRaises(KeyError, phrase._update_parsed)
def store_phrase(self, pid, lid, sentence, flags, lang):
phrase = Phrase(sentence, lang[:2])
length = phrase.length()
if length == 0 or len(sentence) < 2 or length > 10:
return
if flags:
flags = 1
else:
flags = 0
cursor.execute(u"""
insert into phrases_%s
(projectid, locationid, lang, length, phrase, flags)
values
(?, ?, ?, ?, ?, ?)
""" % lang, (pid, lid, lang, length, sentence.decode('utf-8'), flags))
def test_parse(self):
phrase = 'Hello `my_var` `my_var:32` `name~John` `routine1<"stop"`'
phrase = Phrase(phrase, Scope())
self.assertEqual(phrase.latest, 'Hello value John ')
self.assertEqual(phrase.phrase, 'Hello `my_var` John ')
self.assertEqual(phrase.substitute, [['my_var', 6, 14]])
self.assertEqual(len(phrase.setters), 1)
self.assertEqual(phrase.flexibles, [['name', 'John', 16, 20, None]])
self.assertEqual(len(phrase.requests), 1)
# exception raised by Scope module
phrase = 'Hello `undefined_var`'
phrase = Phrase(phrase, Scope())
self.assertEqual(phrase.latest, None)
def test_erase_flexible_setters(self):
phrase = self.fake_class('Hello world `my_var:32` `routine1<"stop"`',
Scope())
Phrase._erase_flexible_setters(phrase)
self.assertEqual(phrase.phrase,
'Hello world `my_var:32` `routine1<"stop"`')
phrase = self.fake_class(
'Hello world `name~John` `my_var:32` `name~Mike` `routine1<"stop"`',
Scope())
Phrase._erase_flexible_setters(phrase)
self.assertEqual(
phrase.phrase,
'Hello world John `my_var:32` Mike `routine1<"stop"`')
def __init__(self):
# used to track the number of incorrect guesses by the user. The initial value is 0 since no guesses have been made at the start of the game.
self.missed = 0
# a list of five Phrase objects to use with the game. A phrase should only include letters and spaces -- no numbers, puntuation or other special characters.
self.phrases = [
Phrase('cut to the chase'),
Phrase('the final countdown'),
Phrase('right on the nose'),
Phrase('possibly the worst'),
Phrase('could be better')
]
# This is the Phrase object that's currently in play. The initial value will be None. Within the start_game() method, this property will be set to the Phrase object returned from a call to the get_random_phrase() method
self.phrase = None
# This is a list that contains the letters guessed by the user.
self.guesses = []
def fa(self, arg1, arg2):
"""Functional application between Phrase objects. The heart of the theory."""
left = None
right = None
if arg1.mcat.combines_with(arg2.mcat):
left = arg1
right = arg2
elif arg2.mcat.combines_with(arg1.mcat):
left = arg2
right = arg1
else:
raise Exception("Categories uncombinable: %s, %s" %
(arg1.mcat, arg2.mcat))
# Output prem and hyp, as strings.
out_prem = "(%s %s)" % (left.prem, right.prem)
out_conc = "(%s %s)" % (left.conc, right.conc)
# Output projectivity.
out_proj_value = left.projs[0].get(right.lex, None)
# Increment the projectivity function.
left.projs.pop(0)
# Output lex according to Alex's rule of inference.
out_lex = JoinTable().join(left.lex, out_proj_value)
# Output category.
out_mcat = left.mcat.combine(right.mcat)
out_proj = {}
# For quantifiers and other things with multiple projectivity, pass on all remaining proj dictionaries.
if left.projs:
out_proj = left.projs
phrase = Phrase(out_prem, out_conc, out_mcat, out_lex, out_proj)
return phrase
def __init__(self, line):
self.line = line.strip()
self.phrases = list(
map(
lambda x:
apply_processors(x.strip().lower()),
re.findall(r"[^.;]+", self.line)
)
)
self.phrases += list(
filter(
lambda x: x is not None,
map(
lambda x: check_and_re_analyse(x),
self.phrases
)
)
)
self.phrases = list(
map(
lambda x: apply_postprocessing(x),
self.phrases
)
)
replace_subjects(self.phrases, find_subject(self.phrases))
self.phrases = [Phrase(phrase) for phrase in self.phrases]
predicates = list(set(find_predicates(self.phrases)))
self.predicates = {}
index = 0
for predicate in predicates:
self.predicates[predicate] = chr(ord("A") + index)
index += 1
def saveDb(self):
dbData = self.db.get_all()
modelData = self.getAllWords()
#That's for future optimization: update db instead of adding it all
dbKeysSet = set(dbData.keys())
dbValuesSet = set(dbData.values())
modelKeysSet = set(modelData.keys())
modelValuesSet = set(modelData.values())
newRecordsKeys = modelKeysSet - dbKeysSet
deletedRecordsKeys = dbKeysSet - modelKeysSet
if len(newRecordsKeys):
for newKey in newRecordsKeys:
self.db.add_phrase(Phrase(newKey, "pl", modelData[newKey]))
if len(deletedRecordsKeys):
for deletedKey in deletedRecordsKeys:
self.db.drop_record(deletedKey)
#Handle also value update
print("Saving database...")
def __init__(self, phrases):
#maximum no.of guesses allowed
self.lives = 5
#list of all possible phrases
self.phrases = [Phrase(phrase) for phrase in phrases]
#current phrase to be guessed by the user
self.current_phrase = random.choice(self.phrases)
def store_phrase(self, pid, lid, sentence, flags, lang):
phrase = Phrase(sentence, lang[:2])
length = phrase.length()
if length == 0 or len(sentence) < 2 or length > 10:
return
if flags:
flags = 1
else:
flags = 0
cursor.execute(
u"""
insert into phrases_%s
(projectid, locationid, lang, length, phrase, flags)
values
(?, ?, ?, ?, ?, ?)
""" % lang, (pid, lid, lang, length, sentence.decode('utf-8'), flags))
def test_erase_fixed_setters(self):
# origin = 'Hello world `my_var:32` `name~John` `routine1<"stop"`'
phrase = self.fake_class('Hello world `name~John` `routine1<"stop"`',
Scope())
Phrase._erase_fixed_setters(phrase)
self.assertEqual(phrase.phrase,
'Hello world `name~John` `routine1<"stop"`')
phrase = self.fake_class(
'Hello world `my_var:32` `name~John` `my_var:32` `routine1<"stop"`',
Scope())
Phrase._erase_fixed_setters(phrase)
self.assertEqual(phrase.phrase,
'Hello world `name~John` `routine1<"stop"`')
def create_phrase(name, head, morphology={}):
global grammar
structure = grammar[name]
if name == "NP":
for key in default_np_morphology.keys():
if key not in morphology:
morphology[key] = default_np_morphology[key]
return Phrase(head, structure, morphology)
def __init__(self):
"""
The constructor for the Game class.
"""
self.guessed_letters = []
self.lives = 5
self.phrases = [Phrase(phrase) for phrase in PHRASE_LIST]
self.correct_phrase = random.choice(self.phrases)
def translate(self,phrase):
translations={}
for target in self._languages:
if target == self.source:
translation=phrase
else:
original=Phrase({'language': self.source,'content':phrase})
self.db.phrase.save(original)
translateds=self.db.translation.find(original,target)
if len(translateds)>0:
translation=translateds[0].content
else:
response=self.retryingTranslate(phrase, target, None, self.source)
translation=response['translatedText']
translated = Phrase({'language':target, 'content': translation})
self.db.translation.addTranslation(original,translated)
translations[target]=translation
return translations
def start(self):
self.welcome()
while self.missed < 5 and self.active_phrase.check_complete(
self.guesses) == False:
print(f"Number missed: {self.missed}")
print("\n")
Phrase.display(self.active_phrase, self.guesses)
print("\n")
user_guess = self.get_guess()
self.guesses.append(user_guess)
if not self.active_phrase.check_guess(user_guess):
self.missed += 1
self.game_over()
self.play_again()
class Query:
def __init__(self, sentence):
self._phrase = Phrase(sentence, Handlers().resolve("en"))
self._candidates = {}
def _add_candidate(self, row, query_hits):
phraseid = row[0]
if phraseid not in self._candidates:
cursor = conn.cursor()
cursor.execute(u"select en, pl from phrases where id = ?", (phraseid,))
nrow = cursor.fetchone()
en_phrase = Phrase(nrow[0], Handlers().resolve("en"))
cand = Candidate(en_phrase, nrow[1])
self._candidates[phraseid] = cand
else:
cand = self._candidates[phraseid]
cand.query_hits += query_hits
cand.this_hits += row[1]
def _process_word(self, word, hits):
cur.execute(u"select phraseid, count from words where word = ?", (word,))
for row in cur:
self._add_candidate(row, hits)
def _get(self):
length = self._phrase.length()
if length == 0:
return
last_word = self._phrase.canonical_list()[0]
count = 1
for word in self._phrase.canonical_list()[1:]:
if word != last_word:
self._process_word(last_word, count)
last_word = word
count = 1
else:
count += 1
self._process_word(last_word, count)
def execute(self):
self._get()
return sorted(self._candidates.values(), Candidate.comparer)
def get_translations(self, text, srclang, dstlang):
srclang = sanitize_language(srclang)
dstlang = sanitize_language(dstlang)
result = []
phrase = Phrase(text, srclang, sort=False, stem=True)
if phrase.length() < 1:
return result
words = phrase.canonical_list()
qmarks = self.qmarks_string(words)
likes = self.required_likes(phrase.required())
where = self.required_where(likes)
conn = sqlite.connect(self.db + srclang + ".db")
cursor = conn.cursor ()
cursor.execute ("""
ATTACH ? AS dest
""", (self.db + dstlang + '.db',))
sql = """
SELECT dst.phrase, src.phrase, dstl.project, dstl.flags, val.value
FROM phrases src
JOIN locations srcl ON src.id = srcl.phraseid
JOIN (
SELECT p.id AS id, MAX(p.length) - SUM(wp.count) - COUNT(*) AS value
FROM words w JOIN wp ON w.id = wp.wordid
JOIN phrases p ON wp.phraseid = p.id
WHERE word IN %s
GROUP BY p.id
ORDER BY value
LIMIT 100
) val ON srcl.phraseid = val.id
JOIN dest.locations dstl ON srcl.locationid = dstl.locationid
JOIN dest.phrases dst ON dstl.phraseid = dst.id
%s
ORDER BY dstl.flags, val.value
""" % (qmarks, where)
cursor.execute (sql, tuple(words + likes))
rows = cursor.fetchall()
for (trans, orig, project, flags, value) in rows:
sug = TmpSug(trans, orig, project, value, flags)
result.append(sug)
cursor.close ()
conn.close()
return result
def create_verb_pharse(head):
global grammar
phrase_type = verb_valence.valency_count(head)
governance = {}
if phrase_type > 1:
#direct object case governancen
dir_obj = {}
dir_obj[u"CASE"] = verb_valence.most_frequent_case(verb_valence.verb_direct_objects(head))
governance["dir_object"] = dir_obj
if phrase_type > 2:
#indirect object case governance
indir_obj = {}
indir_obj[u"CASE"] = verb_valence.most_frequent_case(verb_valence.verb_indirect_objects(head))
governance["indir_object"] = indir_obj
phrase_structure = grammar["VP"+str(phrase_type)]
phrase_structure["governance"] = governance
vp = Phrase(head, phrase_structure)
vp.morphology["VOICE"] = "ACT"
return vp
def __init__(self, nlp_sentence, tagger):
# Can't deal with brackets
if "[" in nlp_sentence or "(" in nlp_sentence:
raise ValueError("Can't parse")
self.text = nlp_sentence
tagged = tagger.tag(nlp_sentence)
p = Phrase()
accum = []
for word, pos, norm in tagged:
try:
p.add_word(word, pos)
except ValueError:
if len(p) > 0:
accum.append(p)
p = Phrase()
accum.append((word, pos))
if len(p) > 0:
accum.append(p)
self.vector = accum
def move_phrases(oconn, ocur, lang):
global project_names
cnt = 0
lid = 0
phrase = ""
icur.execute("""
SELECT DISTINCT phrase, projectid, locationid, flags
FROM phrases_%s
ORDER BY phrase
""" % lang)
for (nphrase, projectid, nlid, flags) in icur.fetchall():
if cnt % 5000 == 0:
print ".",
sys.stdout.flush()
cnt += 1
if phrase != nphrase:
p = Phrase(nphrase, lang[:2], stem=True)
length = p.length()
if length < 1:
continue
lid = nlid
ocur.execute("""
INSERT INTO phrases (id, phrase, length)
VALUES (?, ?, ?)""", (nlid, nphrase, length))
store_words(oconn, ocur, nlid, p.canonical_list())
phrase = nphrase
if lang == "en":
proj = project_names[projectid]
else:
proj = ""
ocur.execute("""
INSERT INTO locations (locationid, phraseid, project, flags)
VALUES (?, ?, ?, ?)""", (nlid, lid, project_names[projectid], flags))
oconn.commit()
def load(f):
#FIXME
sentence = None
while True:
# sentence
l = f.readline().strip()
if not l:
break
sentence = tokenize(l)
assert sentence != None, "Can't find sentence in file!"
tl = TranslationLattice()
tl.set_sentence(sentence)
#read phrases
phrases = []
current_phrase = None
while True:
l = f.readline().strip()
if not l:
break
elif l.endswith(':'): #new phrase line
if current_phrase != None:
tl.add_phrase(current_phrase)
start, end = l[:-1].split('-')
current_phrase = Phrase(sentence[start : end], start, end)
else: # translation line:
t, prob = l.split()
current_phrase.add_translation(t, prob)
# add last phrase
if current_phrase != None:
tl.add_phrase(current_phrase)
return tl
def test_accept(self):
scope = Scope()
phrase = '`var:13` my name is `name~Jack` `surname~Daniels` `strvar:"string"` `routine<1` `routine2<"ok"`'
phrase = Phrase(phrase, scope)
phrase.accept('my name is John Walker')
self.assertEqual(scope.toset, {'var': 13, 'name': 'example', 'surname': 'example', 'strvar': 'string'})
self.assertEqual(scope.tosend, {'routine': 1, 'routine2': 'ok'})
scope = Scope()
phrase = Phrase("Hello world", scope)
phrase.accept()
self.assertEqual(scope.toset, {})
self.assertEqual(scope.tosend, {})
def test_evaluate(self):
phrase = 'no substitutions'
phrase = Phrase(phrase, Scope())
self.assertEqual(phrase.evaluate(), "no substitutions")
phrase = 'iteratable value is `iteratable`'
phrase = Phrase(phrase, Scope())
self.assertEqual(phrase.evaluate(), "iteratable value is 2") # 1 is passed by init call
self.assertEqual(phrase.evaluate(), "iteratable value is 3")
self.assertEqual(phrase.evaluate(), "iteratable value is 4")
phrase = 'undefined value is `undefined_var`'
phrase = Phrase(phrase, Scope())
self.assertRaises(KeyError, phrase.evaluate) # predicted behavior
def __init__(self):
Phrase.__init__(self, None)
def setUp(self):
self.phrase = Phrase("Another string for testing", Scope())
class ParsingTestShifters(unittest.TestCase):
"""
Tests for setters manipulations.
"""
def setUp(self):
self.phrase = Phrase("Another string for testing", Scope())
@unittest.skip("Pronouncable validation isn't yet implemented.")
def test_validate(self):
pass
def test_shift_other(self):
# Edge Coverage implemented
# this setters won't be shifted (before)
self.phrase._create_substition("my_var", 1, 7)
self.phrase._create_fixed_setter("my_var", "34", 9, 15)
self.phrase._create_flexible_setter("name", "John", 19, 25)
self.phrase._create_routine_request("routine1", "stop", 26, 30)
# shifter is not substitution!
# substitution is evaluated inside the eval method
shifter = ["my_var", "34", 32, 35]
length = 3 # shift all afters by 3
# this setters should be shifted (after)
self.phrase._create_substition("my_var", 36, 39)
self.phrase._create_fixed_setter("my_var", "34", 40, 45)
self.phrase._create_flexible_setter("name", "John", 52, 56)
self.phrase._create_routine_request("routine1", "stop", 60, 63)
self.phrase._shift_other(shifter, length)
self.assertEqual(self.phrase.substitute[0], ["my_var", 1, 7])
self.assertEqual(self.phrase.substitute[1], ["my_var", 33, 36])
self.assertEqual(self.phrase.setters[0], ["my_var", "34", 9, 15])
self.assertEqual(self.phrase.setters[1], ["my_var", "34", 37, 42])
self.assertEqual(self.phrase.flexibles[0], ["name", "John", 19, 25, None])
self.assertEqual(self.phrase.flexibles[1], ["name", "John", 49, 53, None])
self.assertEqual(self.phrase.requests[0], ["routine1", "stop", 26, 30])
self.assertEqual(self.phrase.requests[1], ["routine1", "stop", 57, 60])
from optparse import OptionParser
from phrase import Phrase, Handlers
conn = PgSQL.connect(database = 'trans3', client_encoding = 'utf-8', unicode_results = 1)
cursor = conn.cursor()
cursor.execute("set client_encoding to unicode")
parser = OptionParser()
parser.add_option("-s", "--source", dest="source", help="use LANG as source language", metavar="LANG", default="C")
parser.add_option("-d", "--destination", dest="destination", help="use LANG as destination language", metavar="LANG", default="pl")
parser.add_option("-l", "--limit", dest="limit", help="show top N results", metavar="N", default=10)
(options, args) = parser.parse_args()
sentence = ' '.join(args)
phrase = Phrase(sentence, Handlers().resolve(options.source))
query = u"""select dc.id as id, dc.phrase as phrase, min(sp.note) as note, count(*) as cnt
from
(
select sp.locationid, sp.phrase,
sc.wordcount - sw.occ - sw.cnt as note
from
(
select canonicalid, count(*) as cnt, sum(occurences) as occ
from words
where word in %s
group by canonicalid
) as sw join canonicalphrases sc on sw.canonicalid = sc.id join phrases sp on sp.canonicalid = sc.id
where sp.language = %s
) as sp join phrases dp on sp.locationid = dp.locationid
def test_construction(self):
phrase = Phrase('phrase type', { 'english' : 'some text' }, [ Atom('a', 'a'), Atom('b', 'b') ])
self.assertEqual(phrase.get_type(), 'phrase type')
self.assertEqual(phrase.get_snippet('english'), 'some text')
self.assertEqual(phrase.get_snippet('foo'), '')
self.assertEqual(phrase.get_atoms(), [ Atom('a', 'a'), Atom('b', 'b') ])
def create_initial(cls):
initial_phrase = Phrase(None, None, None)
initial_phrase.translations = [Translation(None, (u'<s>',), 0)]
return cls(None, initial_phrase, 0, (u'<s>',))
def __init__(self, sentence):
self._phrase = Phrase(sentence, Handlers().resolve("en"))
self._candidates = {}
def test_str(self):
phrase = 'Hello world `substitution` `my_var:32` `name~John` `routine1<"stop"`'
phrase = Phrase(phrase, Scope())
self.assertEqual(str(phrase), phrase.evaluate())
