def get_languages(self):
"""
Find which languages are predicted for the WiLI dataset.
Returns
-------
languages : list of str
Each str is a ISO 369-3 code
"""
languages = set()
# Read data
data = wili.load_data()
logging.info("Finished loading data")
for set_name in ['test', 'train']:
x_set_name = 'x_{}'.format(set_name)
bar = progressbar.ProgressBar(redirect_stdout=True,
max_value=len(data[x_set_name]))
for i, el in enumerate(data[x_set_name]):
try:
predicted = self.predict(el)
except Exception as e:
predicted = 'UNK'
logging.error({'message': 'Exception in get_languages',
'error': e})
languages.add(predicted)
bar.update(i + 1)
bar.finish()
return sorted(list(languages))
def main(start, end):
"""Run."""
# Read data
data = wili.load_data()
logging.info("Finished loading data")
lang_amounts = {}
for paragraph, label in zip(data['x_train'], data['y_train']):
if label not in lang_amounts:
lang_amounts[label] = []
chars_in_range = 0
for char in paragraph:
if start <= ord(char) <= end:
chars_in_range += 1
amount = float(chars_in_range) / len(paragraph)
lang_amounts[label].append(amount)
for key in lang_amounts.keys():
lang_amounts[key] = np.array(lang_amounts[key]).mean() * 100
print('Label Chars in range [{} - {}]'.format(start, end))
print('-' * 80)
lang_a = sorted(lang_amounts.items(), key=lambda n: n[1], reverse=True)
for i, (label, chars_in_range) in enumerate(lang_a, start=1):
print('{:>3}. {:<10} {:>5.2f}%'.format(i, label, chars_in_range))
def main(start: int, end: int) -> None:
"""Run."""
# Read data
data = wili.load_data()
logger.info("Finished loading data")
lang_amounts = {} # type: Dict[str, List[Any]]
for paragraph, label in zip(data["x_train"], data["y_train"]):
if label not in lang_amounts:
lang_amounts[label] = []
chars_in_range = 0
for char in paragraph:
if start <= ord(char) <= end:
chars_in_range += 1
amount = float(chars_in_range) / len(paragraph)
lang_amounts[label].append(amount)
for key in lang_amounts.keys():
lang_amounts[key] = np.array(lang_amounts[key]).mean() * 100
print(f"Label Chars in range [{start} - {end}]")
print("-" * 80)
lang_a = sorted(lang_amounts.items(), key=lambda n: n[1], reverse=True)
for i, (label, chars_in_range_t) in enumerate(lang_a, start=1):
print(f"{i:>3}. {label:<10} {chars_in_range_t:>5.2f}%")
def get_languages(self) -> List[str]:
"""
Find which languages are predicted for the WiLI dataset.
Returns
-------
languages : List[str]
Each str is a ISO 369-3 code
"""
languages = set()
# Read data
data = wili.load_data()
logger.info("Finished loading data")
for set_name in ["test", "train"]:
x_set_name = f"x_{set_name}"
bar = progressbar.ProgressBar(redirect_stdout=True,
max_value=len(data[x_set_name]))
for i, el in enumerate(data[x_set_name]):
try:
predicted = self.predict(el)
except Exception as e:
predicted = "UNK"
logger.error({
"message": "Exception in get_languages",
"error": e
})
languages.add(predicted)
bar.update(i + 1)
bar.finish()
return sorted(languages)
def main(coverage: float, metric: int, unicode_cutoff: int, set_name: str = "train"):
"""
Train and test character distance models.
Parameters
----------
coverage : float
metric : int
Specify a function
unicode_cutoff : int
set_name : str
Define on which set to evaluate
"""
metrics = [
ido, # 0
distance.braycurtis, # 1
distance.canberra, # 2
distance.chebyshev, # 3 - l_infty
distance.cityblock, # 4
distance.correlation, # 5
distance.cosine, # 6
distance.euclidean, # 7
distance.sqeuclidean, # 8
scipy.stats.entropy, # 9
]
metric_function = metrics[metric]
# Read data
data = wili.load_data()
logger.info("Finished loading data")
# Train
trained = train(data, unicode_cutoff, coverage, metric_function)
# Create model for each language and store it
out_tmp = get_counts_by_lang(
trained["common_chars"], trained["char_counter_by_lang"]
)
language_models, chars = out_tmp
model_filename = "~/.lidtk/models/char_dist_{metric}_{cutoff}.pickle".format(
metric=metric_function.__name__, cutoff=unicode_cutoff
)
model_filename = os.path.expanduser(model_filename)
with open(model_filename, "wb") as handle:
model_info = {"language_models": language_models, "chars": chars}
pickle.dump(model_info, handle, protocol=pickle.HIGHEST_PROTOCOL)
# Evaluate
cm_filepath = "char_{metric}_{coverage}_{cutoff}_{set_name}.cm.csv".format(
metric=metric_function.__name__,
coverage=coverage,
cutoff=unicode_cutoff,
set_name=set_name,
)
cfg = lidtk.utils.load_cfg()
cm_filepath = os.path.join(cfg["artifacts_path"], cm_filepath)
def main(config_file):
config = load_cfg(config_file)
data = wili.load_data()
ret = get_features(config, data)
analyze_vocabulary(ret)
print("First 20 samplex of x_train:")
print(ret['xs']['x_train'][0])
filepath = config['feature-extraction']['serialization_path']
with open(filepath, 'wb') as handle:
logging.info('Store model to \'{}\''.format(filepath))
pickle.dump(ret['vectorizer'],
handle,
protocol=pickle.HIGHEST_PROTOCOL)
def main(config_file: str) -> None:
config = load_cfg(config_file)
data = wili.load_data()
ret = get_features(config, data)
analyze_vocabulary(ret)
print("First 20 samplex of x_train:")
print(ret["xs"]["x_train"][0])
filepath = config["feature-extraction"]["serialization_path"]
with open(filepath, "wb") as handle:
logger.info(f"Store model to '{filepath}'")
pickle.dump(ret["vectorizer"],
handle,
protocol=pickle.HIGHEST_PROTOCOL)
def train(config, data=None):
"""
Train a neural network.
Parameters
----------
config : dict
data : dict, optional (default: wili)
"""
cfg = lidtk.utils.load_cfg(config)
if data is None:
# Read data
data = wili.load_data()
logging.info("Finished loading data")
nn_module = imp.load_source('nn_module', cfg['classifier']['script_path'])
model = nn_module.create_model(lidtk.features.get_dim(cfg),
len(set(data['y_train'])))
print(model.summary())
def train(config: str, data: Optional[Dict[Any, Any]] = None) -> None:
"""
Train a neural network.
Parameters
----------
config : str
data : Optional[Dict[Any, Any]], optional (default: wili)
"""
assert config is not None, "Run lidtk.utils.load_cfg(config)"
cfg = lidtk.utils.load_cfg(config)
if data is None:
# Read data
data = wili.load_data()
logger.info("Finished loading data")
nn_module = imp.load_source("nn_module", cfg["classifier"]["script_path"])
model = nn_module.create_model( # type: ignore
lidtk.features.get_dim(cfg),
len(set(data["y_train"])) # type: ignore
)
print(model.summary())
def eval_wili(self, result_file, languages=None, eval_unk=False):
"""
Evaluate the classifier on WiLI.
Parameters
----------
result_file : str
Path to a file where the results will be stored
languages : list, optional (default: All languages)
Filter languages by this list
"""
# Read data
data = wili.load_data()
logging.info("Finished loading data")
times = []
bar = progressbar.ProgressBar(redirect_stdout=True,
max_value=len(data['x_test']))
result_filepath = os.path.abspath(result_file)
logging.info("Write results to {}".format(result_filepath))
results = {'meta': {}}
now = datetime.datetime.now()
results['meta']['experiment_start'] = ('{:%Y-%m-%d %H:%M:%S}'
.format(now))
cl_results = {}
if languages is None:
eval_unk = False
with open(result_filepath, 'w') as filepointer:
for i, (el, label_t) in enumerate(zip(data['x_test'],
data['y_test'])):
if languages is not None:
if label_t not in languages:
if eval_unk:
print('UNK')
else:
continue
else:
print(label_t)
try:
t0 = time.time()
predicted = self.predict(el)
t1 = time.time()
times.append(t1 - t0)
bar.update(i + 1)
if label_t != predicted:
if label_t not in cl_results:
cl_results[label_t] = {}
if predicted not in cl_results[label_t]:
cl_results[label_t][predicted] = []
identifier = 'test_{}'.format(i)
cl_results[label_t][predicted].append([identifier,
el])
except Exception as e: # catch them all
logging.error({'message': 'Exception in eval_wili',
'error': e})
predicted = 'UNK-exception'
filepointer.write(predicted + '\n')
bar.finish()
results['cl_results'] = cl_results
times = np.array(times)
print("Average time per 10**6 elements: {:.2f}s"
.format(times.mean() * 10**6))
results['time_per_10*6'] = times.mean() * 10**6
logfile = result_filepath + '.json'
results['meta']['hardware'] = lidtk.utils.get_hardware_info()
results['meta']['software'] = lidtk.utils.get_software_info()
with io.open(logfile, 'w', encoding='utf8') as f:
f.write(json.dumps(results,
indent=4,
sort_keys=True,
ensure_ascii=False))
def eval_wili(self,
result_file: str,
languages: List[str] = None,
eval_unk: bool = False) -> None:
"""
Evaluate the classifier on WiLI.
Parameters
----------
result_file : str
Path to a file where the results will be stored
languages : List[str], optional (default: All languages)
Filter languages by this list
eval_unk : bool, optional (default: False)
"""
# Read data
data = wili.load_data()
logger.info("Finished loading data")
times = []
bar = progressbar.ProgressBar(redirect_stdout=True,
max_value=len(data["x_test"]))
result_filepath = os.path.abspath(result_file)
logger.info(f"Write results to {result_filepath}")
results: Dict[str, Any] = {"meta": {}}
now = datetime.datetime.now()
results["meta"]["experiment_start"] = f"{now:%Y-%m-%d %H:%M:%S}"
cl_results = {} # type: Dict[str, Dict[str, List[Any]]]
if languages is None:
eval_unk = False
with open(result_filepath, "w") as filepointer:
for i, (el,
label_t) in enumerate(zip(data["x_test"], data["y_test"])):
if languages is not None:
if label_t not in languages:
if eval_unk:
print("UNK")
else:
continue
else:
print(label_t)
try:
t0 = time.time()
predicted = self.predict(el)
t1 = time.time()
times.append(t1 - t0)
bar.update(i + 1)
if label_t != predicted:
if label_t not in cl_results:
cl_results[label_t] = {}
if predicted not in cl_results[label_t]:
cl_results[label_t][predicted] = []
identifier = f"test_{i}"
cl_results[label_t][predicted].append([identifier, el])
except Exception as e: # catch them all
logger.error({
"message": "Exception in eval_wili",
"error": e
})
predicted = "UNK-exception"
filepointer.write(predicted + "\n")
bar.finish()
results["cl_results"] = cl_results
times_arr = np.array(times)
print(
f"Average time per 10**6 elements: {times_arr.mean() * 10 ** 6:.2f}s"
)
results["time_per_10*6"] = times_arr.mean() * 10**6
logfile = result_filepath + ".json"
results["meta"]["hardware"] = lidtk.utils.get_hardware_info()
results["meta"]["software"] = lidtk.utils.get_software_info()
with open(logfile, "w", encoding="utf8") as f:
f.write(
json.dumps(results,
indent=4,
sort_keys=True,
ensure_ascii=False))