def TreeAssign(self):
alphabets = [
Node(20, [';']),
Node(21,
alphabet.Alphabet().AlphabetNumber()),
Node(18, ['=']),
Node(19, [' ']),
Node(11,
alphabet.Alphabet().GenerateLetters())
]
tree = Tree.Tree()
for x in alphabets:
tree.root = tree.insert(tree.root, x.value, x.alphabet)
return tree
def CreateTree(self):
alphabets=[
Node(20,alphabet.Alphabet().AlphabetNumber()),
Node(10,['=']),
Node(22,[' ']),
Node(5,alphabet.Alphabet().GenerateLetters()),
Node(11,[' ']),
Node(4,['i','n','t']),
Node(8,[' '])
]
tree=Tree.Tree()
for x in alphabets:
tree.root=tree.insert(tree.root,x.value,x.alphabet)
return tree
def TreeAssign(self):
alphabets = [
Node(30, [';']),
Node(31, ['"', "'"]),
Node(25,
alphabet.Alphabet().LetterWithSimbols()),
Node(26, ['"', "'"]),
Node(23, ['=']),
Node(24, [' ']),
Node(1,
alphabet.Alphabet().GenerateLetters())
]
tree = Tree.Tree()
for x in alphabets:
tree.root = tree.insert(tree.root, x.value, x.alphabet)
return tree
def SearchVars(self,data,var):
simbols=['+','-','*','/','^']
comillas=['"',"'"]
cadana=''
caracter=''
comilla=False
c=''
numbers=[str(x) for x in range(0, 10)]
s=alphabet.Alphabet().AlphabetAllSimbols()
if self.HayComillas(data):
#aqui toca ver si hay comillas para diferencias entre variables o cadenas
for x in data:
caracter=x
if ('"' in x or "'" in x or x in simbols)==False:
valor=self.GetValue(var,x)
if valor!=None:
caracter=valor
else:
return None, 'Error la variable '+x +'no esta declarada..'
cadana+=caracter
else:
#aqui hay solo variables y numeros
for x in data:
print(x)
caracter=x
if (self.ConverNumber(x) or x in simbols)==False:
valor=self.GetValue(var,x)
if valor!=None:
caracter=valor
pass
else:
return None,'Error la variable '+x+' No esta declarada'
cadana+=caracter
pass
return cadana,''
def __init__(self, dbpath):
"""dbpath - полное имя файла БД sqlite3"""
self.db = None
self._subzero = 0.3
self._letters = alphabet.Alphabet()
self.dbpath = dbpath
try:
if os.path.exists(dbpath):
self.db = sqlite.connect(self.dbpath)
print('Подключен к БД ', self.dbpath)
else:
print('Не найден файл БД ', self.dbpath)
except:
print('Не удалось подключиться к БД ', self.dbpath)
def CreateTree(self):
alphabets = [
Node(21, [';']),
Node(22,
alphabet.Alphabet().GenerateLetters()),
Node(20, [' ']),
Node(18, ['t']),
Node(19, ['n']),
Node(17, ['i'])
]
tree = Tree.Tree()
for x in alphabets:
tree.root = tree.insert(tree.root, x.value, x.alphabet)
return tree
def CreateTree(self):
alphabets = [
Node(40, [';']),
Node(34,
alphabet.Alphabet().GenerateLetters()),
Node(36, [' ']),
Node(37, ['t']),
Node(35, ['a']),
Node(29, ['o']),
Node(30, ['l']),
Node(25, ['f'])
]
tree = Tree.Tree()
for x in alphabets:
tree.root = tree.insert(tree.root, x.value, x.alphabet)
return tree
def CreateTree(self):
alphabets = [
Node(31, [';']),
Node(40,
alphabet.Alphabet().GenerateLetters()),
Node(26, [' ']),
Node(29, ['g']),
Node(30, ['n']),
Node(28, ['i']),
Node(24, ['r']),
Node(25, ['t']),
Node(23, ['S'])
]
tree = Tree.Tree()
for x in alphabets:
tree.root = tree.insert(tree.root, x.value, x.alphabet)
return tree
def __init__(self, dbpath='', max_rank=1, word_min_len=2):
self._max_rank = max_rank
self._letters = alphabet.Alphabet()
self._current_id = self._letters.size + 1 # текущий id слова
# минимальная длина слова, всё что меньше - пропускается
self.word_min_len = word_min_len
if dbpath == '':
self._create_db(':memory:')
else:
if os.path.exists(dbpath):
self.db = sqlite.connect(dbpath)
c = self.db.cursor()
c.execute('select max(id) from words')
maxid = c.fetchone()
if maxid is not None:
self.current_id = maxid[0]
else:
self._create_db(dbpath)
def IsNumber(self, value):
for e in value:
if (e in alphabet.Alphabet().NumberDecimal()) == False:
return False
return True
def Is_there_letter(string):
for x in string:
if x in alphabet.Alphabet().GenerateLetters():
return True
return False
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import itertools
import math
import fst, operator
import alphabet
import sys
from functools import reduce
abc = alphabet.Alphabet()
syms = fst.SymbolTable()
semiring = 'tropical'
def Transducer(isyms=None, osyms=None, semiring=semiring):
global syms
if isyms is None:
isyms = syms
if osyms is None:
osyms = syms
return fst.Transducer(isyms=isyms, osyms=osyms, semiring=semiring)
def GetPaths(t, return_full_path_in_ostring=False):
if len(t) == 0:
raise StopIteration
seen_paths = set()
for path in t.paths():
def test_get_alpha(self):
test_alpha = alphabet.Alphabet()
self.assertEqual(test_alpha.get(),
list(string.ascii_letters + string.digits))
self.assertEqual(test_alpha.lookup(0), 'a')
# дерево букв.
# Например, слово ранга 1 - "АЗ", может быть представлено как [1,13]
# или как ['А','З'], а слово ранга 2 (предложение) "АЗ ЕСМЬ ЦАРЬ",
# как [65701, 65709, 65777] - идентификаторами слов "АЗ", "ЕСМЬ" и "ЦАРЬ",
# или как [['А','З'],['Е','С','М','Ь'],['Ц','А','Р','Ь']]
# #############################################################################
import scipy.sparse as sparse
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
from constants import WORD_SIZES
from scipy.spatial.distance import cdist
import alphabet
letters = alphabet.Alphabet()
def ischar(c):
return type(c) == str and len(c) == 1
def isstring(s):
return type(s) == str
def istoken(t):
return type(t) == int
def find_word(ids, scores, rank, wm):
# map labels to chars
res = ''
for l in bestLabeling:
res += classes[l]
return res
def _check_input_data(self, mat, classes):
#TODO mat must be numpy array!because use mat.shape
for one_timestep_probability in mat:
# len probs[i] must be equal to len-1 alphabet
assert len(one_timestep_probability) - 1 == len(classes)
# sum of probs on random timestep must be <=1,
# in test2 probs[1,2,3] we have many strings with sum(probs)>1(1.01 for example)
# assert sum(one_timestep_probability) <= 1
# print(sum(one_timestep_probability))
if __name__ == '__main__':
decoder = Decoder()
prob_list, alphabet_list = utils.parse_arguments()
probabilities_obj = probabilities_array.ProbabilitiesArray(prob_list)
alphabet_obj = alphabet.Alphabet(alphabet_list)
# print(f"prob list: {prob_list}, alphabet: {alphabet_obj.alphabet}")
res = decoder.beam_search(probabilities_obj.probabilities_np, alphabet_obj.alphabet)
print(f"result: {res}")
