def main():
global encoding
args = parse_args()
# determine whether to use the aligned or unaligned data
assert args.aligned in [0, 1], "Too many instances of --aligned switch, should be 0 or 1"
aligned = bool(args.aligned)
# and decide between feature encodings and character embeddings
assert args.ortho in [0, 1], "Too many instances of --ortho switch, should be 0 or 1"
ortho = bool(args.ortho)
# load data
data_file = Path(args.data)
assert data_file.exists() and data_file.is_file(), "Data file {} does not exist".format(data_file)
# determine model
assert args.model in MODELS, "Model should be one of {}".format(MODELS)
# determine path to alphabet file & encoding
alphabet_file = None
if args.model == "ipa":
encoding = 'utf-16'
alphabet_file = Path("../data/alphabets/ipa.csv")
elif args.model == "asjp":
encoding = 'ascii'
alphabet_file = Path("../data/alphabets/asjp.csv")
elif args.model == 'latin':
encoding = 'utf-16'
alphabet_file = Path("../data/alphabets/latin.csv")
# load data from file
assert alphabet_file.exists() and alphabet_file.is_file(), "Alphabet file {} does not exist".format(alphabet_file)
alphabet = Alphabet(alphabet_file, encoding=encoding, ortho=ortho)
# number of epochs
assert isinstance(args.epochs, int), "Epochs not int, but {}".format(type(args.epochs))
assert args.epochs > 0, "Epochs out of range: {}".format(args.epochs)
epochs = args.epochs
# number of hidden layers
# assert args.n_hidden > 0, "Number of hidden layers should be at least 1 ;)"
# n_hidden = args.n_hidden
# determine output directories, create them if they do not exist
out_tag = "_{}".format(args.out_tag)
# and tag for files with train/test indices
indices_tag = args.out_tag
plots_dir = Path("../out/plots{}_many2one".format(out_tag))
if not plots_dir.exists():
plots_dir.mkdir(parents=True)
results_dir = Path("../out/results{}_many2one".format(out_tag))
if not results_dir.exists():
results_dir.mkdir(parents=True)
# create file for results
result_file_path = results_dir / "m2one_{}{}{}.txt".format(args.model,
"_aligned" if aligned else "",
"_ortho" if ortho else "")
result_file_path.touch()
result_file = result_file_path.open('w', encoding=encoding)
# determine ancestor
ancestor = args.ancestor
# create cognate sets
cognate_sets = []
data = data_file.open(encoding='utf-16').read().split("\n")
cols = data[HEADER_ROW].split(COLUMN_SEPARATOR)
langs = cols[2:]
# import tensorflow here to comply with the wiki entry https://wiki.lsv.uni-saarland.de/doku.php?id=cluster
import tensorflow as tf
# set random seed for weights
tf.random.set_seed(seed=42)
# start data extraction
for li, line in enumerate(data[HEADER_ROW:]):
# have to do that because the file with the latin characters doesn't contain aligned cognate sets
if args.model == 'latin':
if line == "":
continue
# but the other two do
elif aligned:
if line == "" or li % 2 == 0:
continue
# the unaligned case
else:
if line == "" or li % 2 != 0:
continue
row_split = line.split(COLUMN_SEPARATOR)
id = row_split[ID_COLUMN]
concept = row_split[CONCEPT_COLUMN]
words = row_split[CONCEPT_COLUMN + 1:]
cognate_dict = {}
assert len(langs) == len(words), "Langs / Words mismatch, expected {}, got {}".format(len(langs), len(words))
for lang, word in zip(langs, words):
cognate_dict[lang] = alphabet.translate(word)
cognate_set = CognateSet(id=id,
concept=concept,
ancestor=ancestor,
cognate_dict=cognate_dict,
alphabet=alphabet)
cognate_sets.append(cognate_set)
# prepare train_test_split
total_data = {str(i + 1): cognate_set for i, cognate_set in enumerate(cognate_sets)}
train_indices = set(total_data.keys())
runs = cross_validation_runs(5, train_indices)
# test_indices = Path("../data/{}_test_indices.txt".format(indices_tag)).open('r').read().split("\n")
# train_data = {i: cognate_set for i, cognate_set in data.items() if i in train_indices}
# test_data = {i: cognate_set for i, cognate_set in data.items() if i in test_indices}
# define model
model, optimizer, loss_object = create_many_to_one_model(lstm_dim=128,
timesteps=len(langs) - 1,
data_dim=alphabet.feature_dim,
fc_dim=100,
output_dim=alphabet.feature_dim)
model.summary()
# save model weights for reset
initital_weights = model.get_weights()
words_true = []
words_pred = []
wts = []
wps = []
epoch_losses = []
batch_losses = []
# Training with cross-validation
for i, run in enumerate(runs):
print("***** Cross-validation run [{}/{}] *****".format(i + 1, len(runs)))
# reload initial model weights
model.set_weights(initital_weights)
# get train & test folds
train_data = {i: cognate_set for i, cognate_set in total_data.items() if i in run['train']}
test_data = {i: cognate_set for i, cognate_set in total_data.items() if i in run['test']}
print("***** Start training *****")
for epoch in range(1, epochs + 1):
words_true.clear()
words_pred.clear()
batch_losses.clear()
for batch, cognate_set in train_data.items():
output_characters = []
for lang_array in cognate_set:
target = tf.keras.backend.expand_dims(lang_array.pop(ancestor).to_numpy(), axis=0)
target = tf.dtypes.cast(target, tf.float32)
data = []
for lang, vec in lang_array.items():
data.append(list(vec))
data = np.array(data)
data = tf.keras.backend.expand_dims(data, axis=0)
data = tf.dtypes.cast(data, tf.float32)
# data = tf.reshape(data, (1, -1))
with tf.GradientTape() as tape:
output = model(data)
loss = loss_object(target, output)
batch_losses.append(float(loss))
gradients = tape.gradient(loss, model.trainable_weights)
optimizer.apply_gradients(zip(gradients, model.trainable_weights))
output_characters.append(alphabet.get_char_by_vector(output))
words_pred.append("".join(output_characters))
words_true.append(str(cognate_set.ancestor_word))
# print("".join(output_characters), str(cognate_set.ancestor_word))
if int(batch) % 100 == 0:
print("Epoch [{}/{}], Batch [{}/{}]".format(epoch, epochs, batch, len(cognate_sets)))
# calculate mean epoch loss
mean_loss = np.mean(batch_losses)
epoch_losses.append(mean_loss)
print("Epoch[{}]/[{}], mean batch loss = {}".format(epoch, epochs, mean_loss))
# calculate levenshtein distance
ld = LevenshteinDistance(true=words_true, pred=words_pred)
ld.print_distances()
ld.print_percentiles()
words_pred.clear()
words_true.clear()
print("***** Training finished *****")
print()
# Testing
# Do the same thing as above with the test data, but don't collect the gradients
# and don't backpropagate
print("***** Start testing *****")
for i, cognate_set in test_data.items():
output_characters = []
for lang_array in cognate_set:
target = tf.keras.backend.expand_dims(lang_array.pop(ancestor).to_numpy(), axis=0)
target = tf.dtypes.cast(target, tf.float32)
data = []
for lang, vec in lang_array.items():
data.append(list(vec))
data = np.array(data)
data = tf.keras.backend.expand_dims(data, axis=0)
data = tf.dtypes.cast(data, tf.float32)
output = model(data)
# loss = loss_object(target, output)
output_characters.append(alphabet.get_char_by_vector(output))
# compile the reconstructed word
words_pred.append("".join(output_characters))
# save the true word for the distance calculation
words_true.append(str(cognate_set.ancestor_word))
wts.extend(words_true)
wps.extend(words_pred)
# create plots
ld = LevenshteinDistance(words_true, words_pred)
ld.print_distances()
ld.print_percentiles()
print("***** Testing finished *****")
# save results after last run
outfile = plots_dir / "many2one_test_{}{}{}.jpg".format(args.model, "_aligned" if aligned else "",
"_ortho" if ortho else "")
title = "Model [Test]: LSTM {}{}{}\n 5 cross-validation folds" \
.format(", " + args.model, ", aligned" if aligned else "", ", orthographic" if ortho else "")
ld = LevenshteinDistance(wts, wps)
plot_results(title=title,
distances={"=<" + str(d): count / 5 for d, count in ld.distances.items()},
percentiles={"=<" + str(d): perc for d, perc in ld.percentiles.items()},
mean_dist=ld.mean_distance,
mean_dist_norm=ld.mean_distance_normalized,
losses=[],
outfile=Path(outfile),
testing=True)
def main():
global encoding
args = parser_args()
# determine whether the model should use feature encodings or character embeddings
assert args.orthographic in [
0, 1
], "Too many instances of --orthographic switch, should be 0 or 1"
orthographic = bool(args.orthographic)
# determine whether to use the aligned or unaligned data
assert args.aligned in [
0, 1
], "Too many instances of --aligned switch, should be 0 or 1"
aligned = bool(args.aligned)
# load data
data_file = None
if args.data == "ipa":
encoding = 'utf-16'
data_file = Path("../data/romance_swadesh_ipa.csv")
elif args.data == "asjp":
encoding = 'ascii'
data_file = Path("../data/romance_swadesh_asjp.csv")
assert data_file.exists() and data_file.is_file(
), "Data file {} does not exist".format(data_file)
# determine model
assert args.model in MODELS, "Model should be one of {}".format(MODELS)
# determine path to alphabet file & encoding
alphabet_file = None
if args.model == "ipa":
encoding = 'utf-16'
alphabet_file = Path("../data/alphabets/ipa.csv")
elif args.model == "asjp":
encoding = 'ascii'
alphabet_file = Path("../data/alphabets/asjp.csv")
# load data from file
assert alphabet_file.exists() and alphabet_file.is_file(
), "Alphabet file {} does not exist".format(alphabet_file)
alphabet = Alphabet(alphabet_file,
encoding=encoding,
orthographic=orthographic)
assert isinstance(args.epochs,
int), "Epochs not int, but {}".format(type(args.epochs))
assert args.epochs > 0, "Epochs out of range: {}".format(args.epochs)
epochs = args.epochs
print("alphabet:")
print(alphabet)
# initialize model
model, optimizer, loss_object = create_model(
input_dim=alphabet.get_feature_dim(),
embedding_dim=28,
context_dim=128,
output_dim=alphabet.get_feature_dim())
model.summary()
print("data_file: {}".format(data_file.absolute()))
print("model: {}, orthographic={}, aligned={}".format(
args.model, orthographic, aligned))
print("alphabet: {}, read from {}".format(args.model,
alphabet_file.absolute()))
print("epochs: {}".format(epochs))
# create cognate sets
cognate_sets = []
data = data_file.open(encoding='utf-16').read().split("\n")
cols = data[HEADER_ROW].split(COLUMN_SEPARATOR)
langs = cols[2:]
for li, line in enumerate(data[HEADER_ROW:]):
if aligned:
if line == "" or li % 2 != 0:
continue
else:
if line == "" or li % 2 == 0:
continue
row_split = line.split(COLUMN_SEPARATOR)
id = row_split[ID_COLUMN]
concept = row_split[CONCEPT_COLUMN]
words = row_split[CONCEPT_COLUMN + 1:]
cognate_dict = {}
assert len(langs) == len(
words), "Langs / Words mismatch, expected {}, got {}".format(
len(langs), len(words))
for lang, word in zip(langs, words):
cognate_dict[lang] = alphabet.translate(word)
cs = CognateSet(id=id,
concept=concept,
ancestor='latin',
cognate_dict=cognate_dict,
alphabet=alphabet)
cognate_sets.append(cs)
# maybe we needn't do the evaluation, since we mainly want to know how
# the model behaves with the different inputs
#split_index = int(valid_size * len(cognate_sets))
#train_data = cognate_sets[:split_index]
#valid_data = cognate_sets[split_index:]
#print("train size: {}".format(len(train_data)))
#print("valid size: {}".format(len(valid_data)))
#cognate_sets = cognate_sets[10:30]
words_true = []
words_pred = []
epoch_losses = []
batch_losses = []
for epoch in range(epochs):
# reset lists
epoch_losses.clear()
words_true.clear()
words_pred.clear()
# iterate over the cognate sets
for i, cs in enumerate(cognate_sets):
# reset batch loss
batch_losses.clear()
# iterate over the character embeddings
for j, char_embeddings in enumerate(cs):
# add a dimension to the latin character embedding (ancestor embedding)
# we add a dimension because we use a batch size of 1 and TensorFlow does not
# automatically insert the batch size dimension
target = tf.keras.backend.expand_dims(char_embeddings.pop(
cs.ancestor).to_numpy(),
axis=0)
# convert the latin character embedding to float32 to match the dtype of the output (line 137)
target = tf.dtypes.cast(target, tf.float32)
# iterate through the embeddings
# initialize the GradientTape
with tf.GradientTape(persistent=True) as tape:
for lang, embedding in char_embeddings.items():
# add a dimension to the the embeddings
data = tf.keras.backend.expand_dims(
embedding.to_numpy(), axis=0)
output = model(data)
# calculate the loss
loss = loss_object(target, output)
epoch_losses.append(float(loss))
batch_losses.append(float(loss))
# calculate the gradients
gradients = tape.gradient(loss,
model.trainable_weights)
# backpropagate
optimizer.apply_gradients(
zip(gradients, model.trainable_weights))
# convert the character vector into a character
output_char = alphabet.get_char_by_feature_vector(output)
# append the converted vectors to a list so we can see the reconstructed word
output_characters.append(output_char)
# append the reconstructed word and the ancestor to the true/pred lists
words_pred.append("".join(output_characters))
words_true.append(str(cs.get_ancestor()))
# clear the list of output characters so we can create another word
output_characters.clear()
print("Batch {}, mean loss={}".format(i, np.mean(batch_losses)))
# calculate distances
ld = LevenshteinDistance(true=words_true, pred=words_pred)
print("Epoch {} finished".format(epoch + 1))
print("Mean loss={}".format(epoch, np.mean(epoch_losses)))
ld.print_distances()
ld.print_percentiles()
# do so again after training has finished, but now also save the plots
ld = LevenshteinDistance(true=words_true, pred=words_pred)
ld.print_distances()
ld.print_percentiles()
ld.plot_distances(Path("../data/out/distances.png"))
ld.plot_percentiles(Path("../data/out/percentiles.png"))
data = {}
for category, lines in {
'raw': romance_raw,
'aligned': romance_aligned
}.items():
data[category] = {}
for line in lines:
if line == "":
continue
cognate_set = {}
col_values = line.split(",")
id = col_values[id_col]
for col_name, col_value in zip(cols, col_values):
if col_name in langs:
word = ipa.translate(col_value)
cognate_set[col_name] = word
data[category][id] = cognate_set
errors = 0
for id, cognates in data['aligned'].items():
l_lat = len(cognates['latin'].feature_array)
for lang, word in cognates.items():
ls = [len(word.feature_array) for word in cognates.values()]
for l in ls:
if l != l_lat:
errors += 1
print(id, l_lat, ls, [char.char for char in word.chars])
if errors == 0:
print("Everything is fine!")
s = ""
if line != "":
row = line.split(",")
assert len(row) == len(col_names), "Expected {} fields, found {}"\
.format(len(col_names), row)
# create row data dict
row_data = {
col_name: row[col_names.index(col_name)]
for col_name in col_names
}
s += row_data['id']
s += "," + row_data['concept']
for lang in langs:
w = row_data[lang]
# We are not sure were some of these chars slip in (some are in the original data), but for our pipeline
# they have to be removed.
w = w.replace(":", "").replace("ː", "").replace("ʁ", "").replace("ɡ", "g").replace("ā", "a")\
.replace('ă', "").replace('ῑ', "i").replace("é", "").replace('ș', "").replace('ŭ', "").replace("í", "").replace("ý", "")\
.replace('ĭ', "").replace('š', "").replace("á", "a").replace("è", "").replace("â", "")\
.replace("̃", "").replace('̆', "").replace("́", "").replace('̄', "").replace("ʷ", "").replace('̈', "").replace('̂', "")\
.replace("ț", "").replace('͡', "").replace("ɬ", "").replace('̌', "").replace("<", "").replace(">", "").replace("2", "")
ipa_w = ipa.translate(w)
asjp_w = converter.convert(ipa_w.chars)
# construct string
s += "," + asjp_w
s += "\n"
print(s)
out_file.write(s)
out_file.close()
def train():
# Command line call I used:
# python ciobanu_rnn.py --data=ipa --model=ipa --epochs=10 --out_tag=test --model=ipa --ancestor=ancestor
global encoding
args = parser_args()
# determine whether the model should use feature encodings or character embeddings
assert args.ortho in [
0, 1
], "Too many instances of --orthographic switch, should be 0 or 1"
ortho = bool(args.ortho)
# determine whether to use the aligned or unaligned data
assert args.aligned in [
0, 1
], "Too many instances of --aligned switch, should be 0 or 1"
aligned = bool(args.aligned)
# load data
data_file = None
if args.data == "ipa":
encoding = 'utf-16'
data_file = Path("../data/romance_ciobanu_ipa.csv")
elif args.data == "asjp":
encoding = 'ascii'
data_file = Path("../data/romance_ciobanu_asjp.csv")
assert data_file.exists() and data_file.is_file(
), "Data file {} does not exist".format(data_file)
# determine model
assert args.model in MODELS, "Model should be one of {}".format(MODELS)
# determine path to alphabet file & encoding
alphabet_file = None
if args.model == "ipa":
encoding = 'utf-16'
alphabet_file = Path("../data/alphabets/ipa.csv")
elif args.model == "asjp":
encoding = 'ascii'
alphabet_file = Path("../data/alphabets/asjp.csv")
# load data from file
assert alphabet_file.exists() and alphabet_file.is_file(
), "Alphabet file {} does not exist".format(alphabet_file)
alphabet = Alphabet(alphabet_file, encoding=encoding, ortho=ortho)
assert isinstance(args.epochs,
int), "Epochs not int, but {}".format(type(args.epochs))
assert args.epochs > 0, "Epochs out of range: {}".format(args.epochs)
epochs = args.epochs
# ancestor
ancestor = args.ancestor
# determine output directories, create them if they do not exist
out_tag = "_{}".format(args.out_tag)
plots_dir = Path("../out/plots{}_deep".format(out_tag))
if not plots_dir.exists():
plots_dir.mkdir(parents=True)
results_dir = Path("../out/results{}_deep".format(out_tag))
if not results_dir.exists():
results_dir.mkdir(parents=True)
# create file for results
result_file_path = results_dir / "deep_{}{}{}.txt".format(
args.model, "_aligned" if aligned else "", "_ortho" if ortho else "")
result_file_path.touch()
result_file = result_file_path.open('w', encoding=encoding)
print("alphabet:")
print(alphabet)
# initialize model
model, optimizer, loss_object = create_model(
input_dim=alphabet.get_feature_dim(),
embedding_dim=28,
context_dim=128,
output_dim=alphabet.get_feature_dim())
model.summary()
print("data_file: {}".format(data_file.absolute()))
print("model: {}, orthographic={}, aligned={}".format(
args.model, ortho, aligned))
print("alphabet: {}, read from {}".format(args.model,
alphabet_file.absolute()))
print("epochs: {}".format(epochs))
# create cognate sets
cognate_sets = []
data = data_file.open(encoding='utf-16').read().split("\n")
cols = data[HEADER_ROW].split(COLUMN_SEPARATOR)
langs = cols[2:]
print("langs")
print(langs)
for li, line in enumerate(data[HEADER_ROW:]):
if aligned:
if line == "" or li % 2 != 0:
continue
else:
if line == "" or li % 2 == 0:
continue
row_split = line.split(COLUMN_SEPARATOR)
id = row_split[ID_COLUMN]
concept = row_split[CONCEPT_COLUMN]
words = row_split[CONCEPT_COLUMN + 1:]
# print("words")
# print(words)
cognate_dict = {}
assert len(langs) == len(
words), "Langs / Words mismatch, expected {}, got {}".format(
len(langs), len(words))
for lang, word in zip(langs, words):
# print("lang, word")
# print(lang, word)
cognate_dict[lang] = alphabet.translate(word)
cs = CognateSet(id=id,
concept=concept,
ancestor=ancestor,
cognate_dict=cognate_dict,
alphabet=alphabet)
cognate_sets.append(cs)
# maybe we needn't do the evaluation, since we mainly want to know how
# the model behaves with the different inputs
split_index = int(valid_size * len(cognate_sets))
train_data = cognate_sets[:split_index]
valid_data = cognate_sets[split_index:]
print("train size: {}".format(len(train_data)))
print("valid size: {}".format(len(valid_data)))
# cognate_sets = cognate_sets[10:30]
# print("cognate_sets in ral")
# print(cognate_sets)
words_true = []
words_pred = []
epoch_losses = []
batch_losses = []
for epoch in range(epochs):
# reset lists
epoch_losses.clear()
words_true.clear()
words_pred.clear()
# iterate over the cognate sets
for i, cs in enumerate(cognate_sets):
# reset batch loss
batch_losses.clear()
# iterate over the character embeddings
for j, char_embeddings in enumerate(cs):
# add a dimension to the latin character embedding (ancestor embedding)
# we add a dimension because we use a batch size of 1 and TensorFlow does not
# automatically insert the batch size dimension
target = tf.keras.backend.expand_dims(char_embeddings.pop(
cs.ancestor).to_numpy(),
axis=0)
# convert the latin character embedding to float32 to match the dtype of the output (line 137)
target = tf.dtypes.cast(target, tf.float32)
# iterate through the embeddings
# initialize the GradientTape
with tf.GradientTape(persistent=True) as tape:
for lang, embedding in char_embeddings.items():
# add a dimension to the the embeddings
data = tf.keras.backend.expand_dims(
embedding.to_numpy(), axis=0)
output = model(data)
# calculate the loss
loss = loss_object(target, output)
epoch_losses.append(float(loss))
batch_losses.append(float(loss))
# calculate the gradients
gradients = tape.gradient(loss,
model.trainable_weights)
# backpropagate
optimizer.apply_gradients(
zip(gradients, model.trainable_weights))
# convert the character vector into a character
output_char = alphabet.get_char_by_feature_vector(output)
# append the converted vectors to a list so we can see the reconstructed word
output_characters.append(output_char)
# append the reconstructed word and the ancestor to the true/pred lists
words_pred.append("".join(output_characters))
words_true.append(str(cs.ancestor))
# clear the list of output characters so we can create another word
output_characters.clear()
print("Batch {}, mean loss={}".format(i, np.mean(batch_losses)))
# calculate distances
ld = LevenshteinDistance(true=words_true, pred=words_pred)
print("Epoch {} finished".format(epoch + 1))
print("Mean loss={}".format(epoch, np.mean(epoch_losses)))
ld.print_distances()
ld.print_percentiles()
if epoch == epochs:
outfile = "../out/plots_swadesh_deep/deep_{}{}{}.jpg".format(
args.model, "_aligned" if aligned else "",
"_ortho" if ortho else "")
title = "Model: deep net{}{}{}".format(
", " + args.model, ", aligned" if aligned else "",
", orthographic" if ortho else "")
plot_results(title=title,
distances={
"=<" + str(d): count
for d, count in ld.distances.items()
},
percentiles={
"=<" + str(d): perc
for d, perc in ld.percentiles.items()
},
mean_dist=ld.mean_distance,
mean_dist_norm=ld.mean_distance_normalized,
losses=epoch_losses,
outfile=Path(outfile))
# save reconstructed words (but only if the edit distance is at least one)
import nltk
for t, p in zip(words_true, words_pred):
distance = nltk.edit_distance(t, p)
if distance > 0:
line = "{},{},distance={}\n".format(
t, p, nltk.edit_distance(t, p))
result_file.write(line)
result_file.close()