def test_enumerate_variations_all_db(self):
gitdb = GitInterface(
origin='https://github.com/plevyieml/ieml-language.git')
gitdb.pull()
db = IEMLDatabase(folder=gitdb.folder)
usls = db.list(type=Word, parse=True) + db.list(
type=PolyMorpheme, parse=True) + db.list(type=Lexeme, parse=True)
for u in tqdm.tqdm(usls):
dim, partitions = enumerate_partitions(u)
def __init__(self, input_database_folder_path, out_file_path=None):
from ieml.ieml_database import IEMLDatabase
# input file
self.database = IEMLDatabase(folder=input_database_folder_path)
# output file
if out_file_path is not None:
with open(out_file_path, "w") as output_file:
output_file.write("")
self.vocab_file_path = "{0}.vocab".format(
out_file_path.replace(".tsv", "").replace(".csv", ""))
self.vocab_file = open(self.vocab_file_path, "a")
self.out_file_path = out_file_path
self.out_file = open(out_file_path, "a")
def delete_morpheme_root_paradigm(self,
script: Script,
empty_descriptors=True):
db = IEMLDatabase(folder=self.gitdb.folder,
use_cache=self.use_cache,
cache_folder=self.cache_folder)
d = db.get_dictionary()
descriptors = db.get_descriptors()
script = _check_script(script)
if script not in d.tables.roots:
raise ValueError("Script {} is not a root paradigm".format(
str(script)))
message = "[dictionary] Remove root paradigm {} ({})"\
.format(str(script),
" / ".join("{}:{}".format(l, ', '.join(descriptors.get_values(script, l, 'translations'))) for l in
LANGUAGES))
with self.gitdb.commit(self.signature, message):
db.remove_structure(script)
if empty_descriptors:
for s in list(d.relations.object(script, 'contains')):
db.remove_descriptor(s)
def set_inhibitions(self, ieml, inhibitions):
db = IEMLDatabase(folder=self.gitdb.folder,
use_cache=self.use_cache,
cache_folder=self.cache_folder)
ieml = _check_ieml(ieml)
assert db.get_dictionary().tables.root(ieml) == ieml
inhibitions = _check_inhibitions(inhibitions)
ds = db.get_structure()
old_inhib = ds.get_values(ieml, 'inhibition')
if set(old_inhib) == set(inhibitions):
return
to_remove = [e for e in old_inhib if e not in inhibitions]
to_add = [e for e in inhibitions if e not in old_inhib]
with self.gitdb.commit(
self.signature,
'[inhibitions] Update inhibitions for {} to {}'.format(
str(ieml), json.dumps(inhibitions))):
for e in to_add:
db.add_structure(ieml, 'inhibition', e)
for e in to_remove:
db.remove_structure(ieml, 'inhibition', e)
def test_expand_compose_into_paths(self):
# parser = IEMLParser().parse
gitdb = GitInterface(
origin='https://github.com/plevyieml/ieml-language.git')
gitdb.pull()
db = IEMLDatabase(folder=gitdb.folder)
usls = db.list(type=Word, parse=True) + db.list(
type=PolyMorpheme, parse=True) + db.list(type=Lexeme, parse=True)
for u in tqdm.tqdm(usls):
p_u = list(u.iter_structure_path_by_script_ss())
res = usl_from_path_values(p_u)
self.assertEqual(str(u), str(res),
"expand_compose_into_paths failed on: " + str(u))
def test_all_scripts(self):
parser = ScriptParser()
dic = IEMLDatabase(folder=GitInterface().folder).get_dictionary()
terms = [str(script) for script in dic.scripts]
terms_ast = [str(parser.parse(term)) for term in terms]
self.assertListEqual(terms_ast, terms)
def test_remove(self):
git = init_repo(['/tmp/iemldb_test/A'])[0]
db = IEMLDatabase(git.folder)
self.assertTrue(os.path.isfile(db.path_of('E:')))
db.remove_descriptor('E:')
self.assertFalse(os.path.isfile(db.path_of('E:')))
def set_descriptors(self, ieml, descriptor, value):
db = IEMLDatabase(folder=self.gitdb.folder,
use_cache=self.use_cache,
cache_folder=self.cache_folder)
ieml = _check_ieml(ieml)
value = _check_descriptors(value)
desc = db.get_descriptors()
old_trans = {
l: desc.get_values(ieml=ieml, language=l, descriptor=descriptor)
for l in LANGUAGES
}
if all(sorted(value[l]) == sorted(old_trans[l]) for l in LANGUAGES):
error("No update needed, db already contains {}:{} for {}".format(
descriptor, json.dumps(value), str(ieml)))
return False
# test if after modification there is still at least a descriptor
if all(not (desc.get_values(ieml=ieml, language=l, descriptor=d
) if d != descriptor else value[l])
for l in LANGUAGES for d in DESCRIPTORS_CLASS):
error('[descriptors] Remove {}'.format(str(ieml)))
with self.gitdb.commit(self.signature,
'[descriptors] Remove {}'.format(
str(ieml))):
db.remove_descriptor(ieml)
return True
# to_add = {l: [e for e in value[l] if e not in old_trans[l]] for l in LANGUAGES}
# to_remove = {l: [e for e in old_trans[l] if e not in value[l]] for l in LANGUAGES}
with self.gitdb.commit(
self.signature, '[descriptors] Update {} for {} to {}'.format(
descriptor, str(ieml), json.dumps(value))):
db.remove_descriptor(ieml, None, descriptor)
for l in LANGUAGES:
for e in value[l]:
db.add_descriptor(ieml, l, descriptor, e)
return True
class TestTables(TestCase):
def setUp(self):
self.db = IEMLDatabase(folder=GitInterface().folder)
self.d = self.db.get_dictionary()
def test_iterable(self):
try:
all(self.assertIsInstance(t, Table) for t in self.d.tables)
except TypeError:
self.fail()
#
def test_shape(self):
for s in self.d.scripts:
self.assertIsInstance(s.cells, tuple)
for c in s.cells:
self.assertEqual(c.ndim, 3)
def migrate(database, out_folder):
descriptors = database.descriptors()
dictionary = database.dictionary_structure()
# 'root', 'paradigms', 'inhibitions'
shutil.rmtree(out_folder + '/descriptors')
shutil.rmtree(out_folder + '/structure')
# os.rmdir(out_folder)
# os.mkdir(out_folder)
db2 = IEMLDatabase(out_folder)
# db2.get_csv()
if not os.path.isdir(out_folder):
os.mkdir(out_folder)
for ieml, (paradigms,
inhibitions) in tqdm.tqdm(dictionary.structure.iterrows(),
'migrating structure'):
l = IEMLParser().parse(ieml, factorize_script=True)
db2.add_structure(str(l), 'is_root', True)
for i in inhibitions:
db2.add_structure(str(l), 'inhibition', i)
all_db = defaultdict(lambda: defaultdict(dict))
for (ieml, lang, desc), (v) in descriptors:
all_db[ieml][(lang, desc)] = v.values[0]
for ieml, dd in tqdm.tqdm(all_db.items(), 'migrating descriptors'):
l = IEMLParser().parse(ieml, factorize_script=True)
path = db2.path_of(l)
os.makedirs('/'.join(path.split('/')[:-1]), exist_ok=True)
with open(path, 'w') as fp:
for (lang, desc), v in dd.items():
for vv in v:
fp.write('"{}" {} {} "{}"\n'.format(
str(l), lang, desc, db2.escape_value(vv)))
def update_all_ieml(self, f, message: str):
db = IEMLDatabase(folder=self.gitdb.folder,
use_cache=self.use_cache,
cache_folder=self.cache_folder)
desc = db.get_descriptors()
with self.gitdb.commit(
self.signature,
'[IEML migration] Update all ieml in db: {}'.format(message)):
for old_ieml in tqdm.tqdm(db.list(parse=True), "Migrate all usls"):
new_ieml = f(old_ieml)
value = desc.get_values_partial(old_ieml)
db.remove_descriptor(old_ieml, None, None)
for l in LANGUAGES:
for d in value[l]:
for e in value[l][e]:
db.add_descriptor(new_ieml, l, d, e)
def test_merge_conflict(self):
# clone two id repository
gitA, gitB = init_repo(['/tmp/iemldb_test/A', '/tmp/iemldb_test/B'])
# gitA = gitdbs[0]
# gitB = gitdbs[1]
print("Building DB...")
dbA = IEMLDatabase(folder=gitA.folder)
dbB = IEMLDatabase(folder=gitB.folder)
# commit two differents values
ieml = "(a.)"
name = 'A'
valueA = {
'translations': {'fr': ['test' + name], 'en': ['test' + name]},
'comments': {'fr': ['test' + name], 'en': ['test' + name]},
'tags': {'fr': ['test' + name], 'en': ['test' + name]}
}
commit(gitA, dbA, ieml, 'A', valueA)
name = 'B'
valueB = {
'translations': {'fr': ['test' + name], 'en': ['test' + name]},
'comments': {'fr': ['test' + name], 'en': ['test' + name]},
'tags': {'fr': ['test' + name], 'en': ['test' + name]}
}
commit(gitB, dbB, ieml, 'B', valueB)
# then a same value commit
ieml2 = '(b.)'
commit(gitA, dbA, ieml2, 'A', valueB)
commit(gitB, dbB, ieml2, 'B', valueB)
# then a commit only on A
ieml3 = '(s.)'
commit(gitA, dbA, ieml3, 'A', valueA)
# set B as a remote for A
gitA.add_remote('B', os.path.join(gitB.folder, '.git'))
print("Pulling A from B")
conflicts = gitA.pull('B')
print(conflicts)
dbA = IEMLDatabase(folder=gitA.folder)
descA = dbA.get_descriptors()
self.assertDictEqual(descA.get_descriptor(ieml), valueB)
self.assertDictEqual(conflicts[ieml], valueA)
def delete_morpheme_paradigm(self, script: Script):
db = IEMLDatabase(folder=self.gitdb.folder,
use_cache=self.use_cache,
cache_folder=self.cache_folder)
d = db.get_dictionary()
descriptors = db.get_descriptors()
script = _check_script(script)
if script in d.scripts and len(script) == 1:
raise ValueError("Script {} is not a paradigm".format(str(script)))
root = d.tables.root(script)
message = "[dictionary] Remove paradigm {} ({})"\
.format(str(script),
" / ".join(
"{}:{}".format(l, ', '.join(descriptors.get_values(script, l, 'translations'))) for l in LANGUAGES),
str(root),
" / ".join(
"{}:{}".format(l, ', '.join(descriptors.get_values(root, l, 'translations'))) for l in LANGUAGES))
with self.gitdb.commit(self.signature, message):
db.remove_structure(script)
db.remove_descriptor(script)
shutil.rmtree(folder)
git_address = "https://github.com/IEMLdev/ieml-language.git"
credentials = pygit2.Keypair('git', '~/.ssh/id_rsa.pub', '~/.ssh/id_rsa',
None)
gitdb = GitInterface(origin=git_address,
credentials=credentials,
folder=folder)
#
gitdb.pull()
signature = pygit2.Signature("Louis van Beurden",
"*****@*****.**")
db = IEMLDatabase(folder=folder, use_cache=False)
desc = db.get_descriptors()
struct = db.get_structure()
to_migrate = {}
to_remove = []
parser = IEMLParser(dictionary=db.get_dictionary())
all_db = db.list()
# assert "[E:.b.E:B:.- E:S:. ()(a.T:.-) > ! E:.l.- ()(d.i.-l.i.-')]" in all_db
for s in TO_REMOVE:
to_pass = True
try:
resource_dir = "resources"
if not os.path.isdir(resource_dir):
os.mkdir(resource_dir)
def get_word_structure(w: Word):
return get(
"https://dev.intlekt.io/api/words/{}/?repository=IEMLdev".format(
str(w))).json()
gitdb = GitInterface(origin="https://github.com/plevyieml/ieml-language")
gitdb.pull() # download database in ~/.cache/ieml/ folder
# instanciate a ieml.ieml_database.IEMLDatabase from the downloaded git repository
db = IEMLDatabase(folder=gitdb.folder)
# usls = db.list(parse=True, type='word')
#
# parsed_usls = list()
# for e in tqdm(usls):
# parsed_usls.append(get_word_structure(e))
#
# with bz2.open(WORDS_FILENAME + ".bz2", "wt") as fout:
# json.dump(parsed_usls, fout, indent=2)
descriptors = db.get_descriptors()
usls = db.list()
translations = list()
def create_root_paradigm(self, root, inhibitions, translations, comments):
db = IEMLDatabase(folder=self.gitdb.folder,
use_cache=self.use_cache,
cache_folder=self.cache_folder)
root = _check_script(root)
if len(root) == 1:
raise ValueError(
"The script is not a paradigm {}, can't use it to define a root paradigm."
.format(str(root)))
translations = _check_descriptors(translations)
comments = _check_descriptors(comments)
# if not already exists (no descriptor no structures)
if db.get_descriptors().get_values_partial(root):
raise ValueError(
"Script {} already exists in dictionary".format(root))
dictionary = db.get_dictionary()
for ss in root.singular_sequences:
try:
r = dictionary.tables.root(ss)
raise ValueError(
"Root paradigms {} intersection with script {} ".format(
str(r), str(root)))
except KeyError:
pass
with self.gitdb.commit(
self.signature,
"[dictionary] Create root paradigm {} ({}), create {} singular sequences"
.format(
str(root), " / ".join("{}:{}".format(
l, ', '.join(db.get_descriptors().get_values(
str(root), l, 'translations')))
for l in LANGUAGES),
len(root.singular_sequences)),
):
db.remove_structure(root, 'is_root')
db.add_structure(root, 'is_root', True)
for i in _check_inhibitions(inhibitions):
db.add_structure(root, 'inhibition', i)
for l in LANGUAGES:
for v in translations[l]:
db.add_descriptor(root,
language=l,
descriptor='translations',
value=v)
for v in comments[l]:
db.add_descriptor(root,
language=l,
descriptor='comments',
value=v)
# add main tables header
for i, t in enumerate([tt for tt in root.tables_script if tt != root]):
self.add_morpheme_paradigm(
t,
translations=append_idx_to_dict(translations, i),
comments=append_idx_to_dict(comments, i))
# # d = Descriptors(db2.get_pandas())
# # print(d.get_values('wa.', 'en', 'translations'))
# # db2.remove_key('wa.', 'en', 'translations')
# # d = Descriptors(db2.get_pandas())
# # print(d.get_values('wa.', 'en', 'translations'))
gitdb = GitInterface(
origin='ssh://[email protected]/ogrergo/ieml-language.git',
credentials=pygit2.Keypair('git', '/home/louis/.ssh/id_rsa.pub',
'/home/louis/.ssh/id_rsa', ''),
folder=folder)
# gitdb = GitInterface(origin='https://github.com/ogrergo/ieml-language.git',
# credentials=pygit2.Username('git'),
# folder=folder)
db = IEMLDatabase(folder=folder)
# d = Descriptors(db2.get_pandas())
# # print(d.get_values('wa.', 'fr', 'translations'))
# # db2.remove_key('wa.', 'fr', 'translations', 'agir')
# # d = Descriptors(db2.get_pandas())
# # print(d.get_values('wa.', 'fr', 'translations'))
#
# # v = d.get_values_partial(None, 'fr', None)
#
# # v = d.get_values_partial(None, 'fr', None)
# # print(v)
signature = pygit2.Signature(args.author_name, args.author_email)
with gitdb.commit(signature, '[Migration] migrate from 0.3 to 0.4'):
def update_morpheme_paradigm(
self,
script_old: Script,
script_new: Script,
):
script_old = _check_script(script_old)
script_new = _check_script(script_new)
if script_old == script_new:
return
assert len(script_old) != 1 or len(
script_new) != 1, "Can't update singular sequences, only paradigms"
db = IEMLDatabase(folder=self.gitdb.folder,
use_cache=self.use_cache,
cache_folder=self.cache_folder)
d = db.get_dictionary()
desc = db.get_descriptors()
ds = db.get_structure()
assert script_old in d.scripts, "Source script not defined in dictionary"
assert script_new not in d.scripts, "Target script already defined in dictionary"
root_old = d.tables.root(script_old)
is_root = ds.get_values(script_old, 'is_root')
is_root = is_root and is_root[0][0].lower() == 't'
root_new_cand = set()
for ss in script_new.singular_sequences:
try:
root_new_cand.add(d.tables.root(ss))
except KeyError:
if not is_root:
raise ValueError(
"A non root paradigm is defined over singular sequences that are in no paradigms"
)
assert len(
root_new_cand
) == 1, "No root paradigms or too many for script {}".format(
str(script_new))
root_new = next(iter(root_new_cand))
message = "[dictionary] Update paradigm IEML from {} to {}"\
.format(str(script_old),
str(script_new),
" / ".join(
"{}:{}".format(l, desc.get_values(script_new, l, 'translations')) for l in LANGUAGES))
if is_root:
# 1st case: root paradigm
assert script_old in script_new, "Can only update a root paradigm to a bigger version of it"
# then we can update it to a bigger version of it
old_structure = ds.get_values_partial(script_old)
# transfers translations and structure
with self.gitdb.commit(self.signature, message):
if is_root:
db.remove_structure(script_old)
db.add_structure(script_old, 'is_root', 'False')
for (_, key), values in old_structure.items():
for v in values:
db.add_structure(script_new, key, v)
else:
db.remove_structure(script_old)
db.add_structure(script_new, 'is_root', 'False')
db.remove_descriptor(script_old)
for (_, l,
k), values in desc.get_values_partial(script_old).items():
for v in values:
db.add_descriptor(script_new, l, k, v)
if is_root:
db.add_descriptor(
script_old, l, k,
'(translation migrated to {}) '.format(
str(script_new)) + v)
doc.packages.append(Package('xcolor', ['dvipsnames', 'table']))
try:
doc.generate_pdf(clean_tex=False, silent=False)
doc.generate_tex()
except subprocess.CalledProcessError as e:
os.chdir(
old_cwd) # because pylatex change it but doesnt restore it
raise e
with open(path + '.pdf', 'rb') as fp:
return fp.read()
def rendex_latex_word(w: Word, descriptors: Descriptors, language: LANGUAGES):
return compile_latex(word_to_latex(w, descriptors, language))
if __name__ == "__main__":
gitdb = GitInterface()
db = IEMLDatabase(gitdb.folder)
ieml = "[E:T:. (E:.b.wa.- E:.-wa.-t.o.-' E:.-'we.-S:.-'t.o.-',)(e.) > E:.n.- (E:.wo.- E:S:.-d.u.-') > E:.d.- (E:.wo.- E:S:.-d.u.-')(m.-S:.U:.-') > ! E:.n.- E:U:. ()]"
w = usl(ieml)
res = rendex_latex_word(w, db.get_descriptors(), 'en')
with open("output.pdf", 'wb') as fp:
fp.write(res)
def add_morpheme_paradigm(self, script: Script, translations, comments):
db = IEMLDatabase(folder=self.gitdb.folder,
use_cache=self.use_cache,
cache_folder=self.cache_folder)
d = db.get_dictionary()
script = _check_script(script)
if len(script) == 1:
raise ValueError(
"The script is not a paradigm {}, can't use it to define a paradigm."
.format(str(script)))
if script in d.scripts:
raise ValueError(
"Script {} already defined in the dictionary".format(
str(script)))
r_cand = set()
for ss in script.singular_sequences:
try:
r_cand.add(d.tables.root(ss))
except KeyError:
raise ValueError(
"No root paradigms contains this script {}".format(
str(script)))
if len(r_cand) != 1:
raise ValueError(
"No root paradigms or too many for script {}".format(
str(script)))
root = next(iter(r_cand))
descriptors = db.get_descriptors()
message = "[dictionary] Create paradigm {} ({}) for root paradigm {} ({})"\
.format(str(script),
" / ".join(
"{}:{}".format(l, ', '.join(descriptors.get_values(script, l, 'translations'))) for l in LANGUAGES),
str(root),
" / ".join(
"{}:{}".format(l, ', '.join(descriptors.get_values(root, l, 'translations'))) for l in LANGUAGES))
with self.gitdb.commit(self.signature, message):
db.remove_descriptor(script)
db.remove_structure(script)
db.add_structure(script, 'is_root', False)
for l in LANGUAGES:
for v in translations[l]:
db.add_descriptor(script,
language=l,
descriptor='translations',
value=v)
for v in comments[l]:
db.add_descriptor(script,
language=l,
descriptor='comments',
value=v)
def setUp(self):
self.db = IEMLDatabase(folder=GitInterface().folder)
self.d = self.db.get_dictionary()
def setUpClass(cls) -> None:
cls.dic = IEMLDatabase(folder=GitInterface().folder).get_dictionary()
def migrate(function, _s_old, _s_new):
assert function(_s_old) == _s_new
folder = '/tmp/migrate_script_iemldb'
if os.path.isdir(folder):
shutil.rmtree(folder)
# os.mkdir(folder)
git_address = "https://github.com/IEMLdev/ieml-language.git"
credentials = pygit2.Keypair('ogrergo', '~/.ssh/id_rsa.pub',
'~/.ssh/id_rsa', None)
gitdb = GitInterface(origin=git_address,
credentials=credentials,
folder=folder)
signature = pygit2.Signature("Louis van Beurden",
"*****@*****.**")
db = IEMLDatabase(folder=folder, use_cache=False)
to_migrate = {}
desc = db.get_descriptors()
struct = db.get_structure()
for s in db.get_dictionary().scripts:
s2 = function(s)
if s2 != s:
to_migrate[s] = s2
print(to_migrate)
with gitdb.commit(
signature,
"[Translate script] Translate paradigm from '{}' to '{}".format(
str(_s_old), str(_s_new))):
for s_old, s_new in to_migrate.items():
db.remove_structure(s_old)
for (_, key), values in struct.get_values_partial(s_old).items():
for v in values:
db.add_structure(s_new, key, v)
db.remove_descriptor(s_old)
for (_, lang, d), values in desc.get_values_partial(s_old).items():
for v in values:
db.add_descriptor(s_new, lang, d, v)
def test_add_remove(self):
# clone two id repository
gitA, gitB = init_repo(['/tmp/iemldb_test/A', '/tmp/iemldb_test/B'])
# gitA = gitdbs[0]
# gitB = gitdbs[1]
print("Building DB...")
dbA = IEMLDatabase(folder=gitA.folder)
dbB = IEMLDatabase(folder=gitB.folder)
# commit two differents values
ieml = "(a.)"
name = 'A'
valueA = {
'translations': {'fr': ['test' + name], 'en': ['test' + name]},
'comments': {'fr': ['test' + name], 'en': ['test' + name]},
'tags': {'fr': ['test' + name], 'en': ['test' + name]}
}
commit(gitA, dbA, ieml, 'A', valueA)
commit(gitB, dbB, ieml, 'B', valueA)
valueB = {
'translations': {'fr': [], 'en': []},
'comments': {'fr': [], 'en': []},
'tags': {'fr': [], 'en': []}
}
# set B as a remote for A
gitA.add_remote('B', os.path.join(gitB.folder, '.git'))
gitB.add_remote('A', os.path.join(gitA.folder, '.git'))
print("Pulling A from B")
conflicts = gitA.pull('B')
print(conflicts)
dbA = IEMLDatabase(folder=gitA.folder)
descA = dbA.get_descriptors()
self.assertDictEqual(descA.get_descriptor(ieml), valueA)
self.assertDictEqual(conflicts, {})
# modify ieml in A, and remove it in B
valueA = {
'translations': {'fr': ['test2' + name], 'en': ['test' + name]},
'comments': {'fr': ['test2' + name], 'en': ['test' + name]},
'tags': {'fr': ['test2' + name], 'en': ['test' + name]}
}
commit(gitA, dbA, ieml, 'A', valueA)
commit(gitB, dbB, ieml, 'B', valueB)
print("Pulling A from B")
conflicts = gitA.pull('B')
print(conflicts)
dbA = IEMLDatabase(folder=gitA.folder)
descA = dbA.get_descriptors()
self.assertDictEqual(descA.get_descriptor(ieml), valueB)
self.assertDictEqual(conflicts[ieml], valueA)
print("Pulling B from A")
conflicts = gitB.pull('A')
print(conflicts)
dbB = IEMLDatabase(folder=gitB.folder)
descB = dbB.get_descriptors()
self.assertDictEqual(descB.get_descriptor(ieml), valueB)
self.assertDictEqual(conflicts, {})
def setUpClass(cls):
cls.git = GitInterface()
cls.git.pull()
cls.db = IEMLDatabase(folder=cls.git.folder)
cls.dictionary = cls.db.get_dictionary()
class IemlData:
"""
Modify the ieml data to output a vector representation
"""
def __init__(self, input_database_folder_path, out_file_path=None):
from ieml.ieml_database import IEMLDatabase
# input file
self.database = IEMLDatabase(folder=input_database_folder_path)
# output file
if out_file_path is not None:
with open(out_file_path, "w") as output_file:
output_file.write("")
self.vocab_file_path = "{0}.vocab".format(
out_file_path.replace(".tsv", "").replace(".csv", ""))
self.vocab_file = open(self.vocab_file_path, "a")
self.out_file_path = out_file_path
self.out_file = open(out_file_path, "a")
def close_all(self):
try:
self.vocab_file.close()
except AttributeError:
pass
try:
self.out_file.close()
except AttributeError:
pass
def get_word_objects(self):
return self.database.list(parse=False, type='word')
def list_polymorpheme_of_word(self, w):
########WORkAROUND############TO BE SOLVED THEN REMOVE########################################3
if w == "[! E:B:. ()(k.a.-k.a.-' l.o.-k.o.-') > E:.f.- ()(p.E:A:T:.-)] [>role>E:B:.>content>constant>k.a.-k.a.-'":
return []
##################################################################
w = usl(w)
assert isinstance(w, Word)
polyList = []
for sfun in w.syntagmatic_fun.actors.values():
if sfun.actor is not None:
polyList.append((sfun.actor.pm_content, sfun.actor.pm_flexion))
# return list(chain.from_iterable((sfun.actor.pm_content, sfun.actor.pm_flexion)
# for sfun in w.syntagmatic_fun.actors.values())) # encounteres AttributeError: 'NoneType' object has no attribute 'pm_content' since sfun.actor can be None
return polyList
def get_natural_lang_meanings(self, lang="en"):
nl_meanings = []
descriptors = self.database.get_descriptors()
for word in self.get_word_objects():
word_nl_meanings = []
# get meaning of word
desc_w_vals = descriptors.get_values_partial(word)
for (usl_w, language_w, label_w), tr_w_list in desc_w_vals.items():
if language_w == lang and label_w == "translations":
word_nl_meanings.append([" , ".join(tr_w_list)])
# divide the words form the polymorphemes
word_nl_meanings.append([" : "])
# get meaning of polymorpheme
polymorphemes = self.list_polymorpheme_of_word(word)
for polymorph in polymorphemes:
for poly in polymorph:
desc_p_vals = descriptors.get_values_partial(poly)
for (usl_p, language_p,
label_p), tr_p_list in desc_p_vals.items():
if language_p == lang and label_p == "translations":
word_nl_meanings.append(tr_p_list)
nl_meanings.append(word_nl_meanings)
return nl_meanings
def get_bert_emb(self, string, bert_class):
bert_class = bert_class if bert_class is not None else BertEmbedd()
return bert_class.bert([string])
def make_bert_emb_list(self, lang="en", bert_class=None, dump=False):
bert_class = bert_class if bert_class is not None else BertEmbedd()
bert_embeddings = []
for ieml_pm_in_nl in self.get_natural_lang_meanings(lang):
ieml_w_pm_sent = " ".join(
[" ".join(pm) for pm in ieml_pm_in_nl if len(pm) != 0])
# yield self.get_bert_emb(ieml_pm_in_nl, bert_class)
bert_embeddings.append(
self.get_bert_emb(ieml_w_pm_sent, bert_class))
# dump ieml the sentence embeddings
if dump is not False:
# dump the embeddings
for bert_emb in bert_embeddings:
for (bert_vocab, bert_vect) in bert_emb:
bert_vect = np.array(bert_vect)
self.vocab_file.write("{0}\n".format(
json.dumps(bert_vocab)))
try:
self.out_file.write("{0}\n".format(
json.dumps(bert_vect)))
except TypeError:
self.out_file.write("{0}\n".format(bert_vect.dumps()))
# numpy.save(self.out_file, sent_emb)
self.close_all()
return bert_embeddings