class TestGitInterface(unittest.TestCase):
def setUp(self):
# repo = pygit2.init_repository(REPO_TEST_PATH)
self.git = GitInterface(folder=REPO_TEST_PATH)
# def tearDown(self):
# shutil.rmtree(REPO_TEST_PATH)
def test_checkout_at_commit(self):
versions = [
'148505dbea8ec25fa81162c7bdeed685986094a6',
'f8f3c2882db71e8b70c6f1b73566fc53c877ea6e'
]
# for version in versions:
# self.assertEqual(str(self.git.get_version()[1]), version)
for version in versions:
self.git.checkout(None, version)
self.assertEqual(str(self.git.get_version()[1]), version)
def test_reset(self):
with open(os.path.join(REPO_TEST_PATH, "test"), 'w') as fp:
fp.write("test\n")
self.assertIn('test', self.git.repo.status())
self.git.reset()
self.assertDictEqual(self.git.repo.status(), {})
def test_transaction(self):
trans = DBTransactions(self.git,
pygit2.Signature('git', 'git'),
REPO_TEST_PATH_CACHE,
use_cache=True)
trans.set_descriptors(script("E:"), "tags", {'fr': "test"})
def test_enumerate_variations_all_db(self):
gitdb = GitInterface(
origin='https://github.com/plevyieml/ieml-language.git')
gitdb.pull()
db = IEMLDatabase(folder=gitdb.folder)
usls = db.list(type=Word, parse=True) + db.list(
type=PolyMorpheme, parse=True) + db.list(type=Lexeme, parse=True)
for u in tqdm.tqdm(usls):
dim, partitions = enumerate_partitions(u)
def test_expand_compose_into_paths(self):
# parser = IEMLParser().parse
gitdb = GitInterface(
origin='https://github.com/plevyieml/ieml-language.git')
gitdb.pull()
db = IEMLDatabase(folder=gitdb.folder)
usls = db.list(type=Word, parse=True) + db.list(
type=PolyMorpheme, parse=True) + db.list(type=Lexeme, parse=True)
for u in tqdm.tqdm(usls):
p_u = list(u.iter_structure_path_by_script_ss())
res = usl_from_path_values(p_u)
self.assertEqual(str(u), str(res),
"expand_compose_into_paths failed on: " + str(u))
def test_all_scripts(self):
parser = ScriptParser()
dic = IEMLDatabase(folder=GitInterface().folder).get_dictionary()
terms = [str(script) for script in dic.scripts]
terms_ast = [str(parser.parse(term)) for term in terms]
self.assertListEqual(terms_ast, terms)
def migrate(function, _s_old, _s_new):
assert function(_s_old) == _s_new
folder = '/tmp/migrate_script_iemldb'
if os.path.isdir(folder):
shutil.rmtree(folder)
# os.mkdir(folder)
git_address = "https://github.com/IEMLdev/ieml-language.git"
credentials = pygit2.Keypair('ogrergo', '~/.ssh/id_rsa.pub',
'~/.ssh/id_rsa', None)
gitdb = GitInterface(origin=git_address,
credentials=credentials,
folder=folder)
signature = pygit2.Signature("Louis van Beurden",
"*****@*****.**")
db = IEMLDatabase(folder=folder, use_cache=False)
to_migrate = {}
desc = db.get_descriptors()
struct = db.get_structure()
for s in db.get_dictionary().scripts:
s2 = function(s)
if s2 != s:
to_migrate[s] = s2
print(to_migrate)
with gitdb.commit(
signature,
"[Translate script] Translate paradigm from '{}' to '{}".format(
str(_s_old), str(_s_new))):
for s_old, s_new in to_migrate.items():
db.remove_structure(s_old)
for (_, key), values in struct.get_values_partial(s_old).items():
for v in values:
db.add_structure(s_new, key, v)
db.remove_descriptor(s_old)
for (_, lang, d), values in desc.get_values_partial(s_old).items():
for v in values:
db.add_descriptor(s_new, lang, d, v)
def init_repo(folders):
if all(os.path.isdir(folder) for folder in folders):
commit_id = '7f3a0b96aba5cf299ecc4e2985ec49f9bb7559ba'
else:
commit_id = None
for folder in folders:
if os.path.isdir(folder):
shutil.rmtree(folder)
folder = '/tmp/iemldb_test/tmp'
if os.path.isdir(folder):
shutil.rmtree(folder)
print("Cloning IEML db : ", folder)
GitInterface(folder=folder)
gitdbs = []
for f in folders:
if not commit_id:
print("Copying IEML db: ", f)
shutil.copytree(folder, f)
git = GitInterface(folder=f)
else:
git = GitInterface(folder=f)
git.reset(commit_id)
gitdbs.append(git)
if not commit_id:
shutil.rmtree('/tmp/iemldb_test/tmp')
return gitdbs
def setUpClass(cls):
cls.git = GitInterface()
cls.git.pull()
cls.db = IEMLDatabase(folder=cls.git.folder)
cls.dictionary = cls.db.get_dictionary()
def setUp(self):
self.db = IEMLDatabase(folder=GitInterface().folder)
self.d = self.db.get_dictionary()
WORDS_FILENAME = "resources/words_sample.json"
DICTIONARY_FILENAME = "resources/dictionary.json"
resource_dir = "resources"
if not os.path.isdir(resource_dir):
os.mkdir(resource_dir)
def get_word_structure(w: Word):
return get(
"https://dev.intlekt.io/api/words/{}/?repository=IEMLdev".format(
str(w))).json()
gitdb = GitInterface(origin="https://github.com/plevyieml/ieml-language")
gitdb.pull() # download database in ~/.cache/ieml/ folder
# instanciate a ieml.ieml_database.IEMLDatabase from the downloaded git repository
db = IEMLDatabase(folder=gitdb.folder)
# usls = db.list(parse=True, type='word')
#
# parsed_usls = list()
# for e in tqdm(usls):
# parsed_usls.append(get_word_structure(e))
#
# with bz2.open(WORDS_FILENAME + ".bz2", "wt") as fout:
# json.dump(parsed_usls, fout, indent=2)
descriptors = db.get_descriptors()
"[E:S:. (E:.-wa.-t.o.-' E:.-'wu.-S:.-'t.o.-',)(u.A:.-) > ! E:.n.- (E:.wo.- E:S:.-d.u.-')(l.-T:.U:.-',n.-T:.A:.-',t.o.-f.o.-',_) > E:.t.- (E:.wo.- E:S:.-d.u.-')(S:.E:A:S:.-) > E:.l.- (E:.wo.- E:.-U:.d.-l.-' E:S:.-d.u.-')(n.-T:.U:.-') > E:.f.- (E:.wo.- E:S:.-d.u.-')(n.i.-d.i.-t.u.-')]",
"[E:S:. (E:.-wa.-t.o.-' E:.-'wu.-S:.-'t.o.-',)(u.A:.-) > ! E:.n.- (E:.wo.- E:S:.-d.u.-')(l.-T:.U:.-',n.-T:.A:.-',t.o.-f.o.-',_) > E:.l.- (E:.wo.- E:.-U:.d.-l.-' E:S:.-d.u.-')(n.-T:.U:.-') > E:.f.- (E:.wo.- E:S:.-d.u.-')(n.i.-d.i.-t.u.-') > E:.s.- (E:.wo.- E:T:.h.- E:S:.-d.u.-')(s.j.- d.-S:.U:.-')]"
]
if __name__ == '__main__':
folder = '/tmp/migrate_script_iemldb'
if os.path.isdir(folder):
shutil.rmtree(folder)
git_address = "https://github.com/IEMLdev/ieml-language.git"
credentials = pygit2.Keypair('git', '~/.ssh/id_rsa.pub', '~/.ssh/id_rsa',
None)
gitdb = GitInterface(origin=git_address,
credentials=credentials,
folder=folder)
#
gitdb.pull()
signature = pygit2.Signature("Louis van Beurden",
"*****@*****.**")
db = IEMLDatabase(folder=folder, use_cache=False)
desc = db.get_descriptors()
struct = db.get_structure()
to_migrate = {}
to_remove = []
return Word(sfun, role=w.role, context_type=w.syntagmatic_fun.__class__)
if __name__ == '__main__':
folder = '/tmp/migrate_script_iemldb'
if os.path.isdir(folder):
shutil.rmtree(folder)
git_address = "https://github.com/plevyieml/ieml-language.git"
credentials = pygit2.Keypair('git', '~/.ssh/id_rsa.pub', '~/.ssh/id_rsa',
None)
gitdb = GitInterface(origin=git_address,
credentials=credentials,
folder=folder)
#
# gitdb.pull()
signature = pygit2.Signature("Louis van Beurden",
"*****@*****.**")
db = IEMLDatabase(folder=folder, use_cache=False)
desc = db.get_descriptors()
struct = db.get_structure()
to_migrate = {}
to_remove = []
import pygit2
from ieml.ieml_database import GitInterface
from ieml.ieml_database.transactions.DBTransaction import DBTransactions
from ieml.usl import Word
from ieml.usl.lexeme import Lexeme
def migrate(u):
if not isinstance(u, (Lexeme, Word)):
return str(u)
return str(u) + ' asdasd'
if __name__ == '__main__':
gitdb = GitInterface(origin='ssh://[email protected]/ogrergo/ieml-language.git',
credentials=pygit2.Keypair('git', '/home/louis/.ssh/id_rsa.pub', '/home/louis/.ssh/id_rsa', ''),
folder='/tmp/gitdb')
gitdb.pull()
signature = pygit2.Signature("Louis van Beurden", "*****@*****.**")
transaction = DBTransactions(gitdb, signature)
transaction.update_all_ieml(migrate, "Test de migration")
def setUpClass(cls) -> None:
cls.dic = IEMLDatabase(folder=GitInterface().folder).get_dictionary()
from ieml.usl.word import simplify_word
# This module remove unnormalized USL and save theirs normalized versions
if __name__ == '__main__':
folder = '/tmp/migrate_script_iemldb'
if os.path.isdir(folder):
shutil.rmtree(folder)
# os.mkdir(folder)
git_address = "https://github.com/plevyieml/ieml-language.git"
credentials = pygit2.Keypair('git', '~/.ssh/id_rsa.pub', '~/.ssh/id_rsa',
None)
gitdb = GitInterface(origin=git_address,
credentials=credentials,
folder=folder)
#
# gitdb.pull()
signature = pygit2.Signature("Louis van Beurden",
"*****@*****.**")
db = IEMLDatabase(folder=folder, use_cache=False)
desc = db.get_descriptors()
struct = db.get_structure()
to_migrate = {}
to_remove = []
doc.packages.append(Package('xcolor', ['dvipsnames', 'table']))
try:
doc.generate_pdf(clean_tex=False, silent=False)
doc.generate_tex()
except subprocess.CalledProcessError as e:
os.chdir(
old_cwd) # because pylatex change it but doesnt restore it
raise e
with open(path + '.pdf', 'rb') as fp:
return fp.read()
def rendex_latex_word(w: Word, descriptors: Descriptors, language: LANGUAGES):
return compile_latex(word_to_latex(w, descriptors, language))
if __name__ == "__main__":
gitdb = GitInterface()
db = IEMLDatabase(gitdb.folder)
ieml = "[E:T:. (E:.b.wa.- E:.-wa.-t.o.-' E:.-'we.-S:.-'t.o.-',)(e.) > E:.n.- (E:.wo.- E:S:.-d.u.-') > E:.d.- (E:.wo.- E:S:.-d.u.-')(m.-S:.U:.-') > ! E:.n.- E:U:. ()]"
w = usl(ieml)
res = rendex_latex_word(w, db.get_descriptors(), 'en')
with open("output.pdf", 'wb') as fp:
fp.write(res)
def setUp(self):
# repo = pygit2.init_repository(REPO_TEST_PATH)
self.git = GitInterface(folder=REPO_TEST_PATH)