from classification.ClassificationTrainer import ClassificationTrainer
# script for corresponding test case
# most test cases should be able to be executed without any further changes, if data is available
SAMPLE_SIZE = 100000
EPOCHS = 150
HIDDEN_SIZE = 10000
BATCH_SIZE = 128
# change device to "cpu" if cuda not available
DEVICE = "cuda"
stopwatch = StopWatch()
# pregenerated embedding and labels
file_words = open("../tests/embedding8.json")
file_labels = open("../data/unique_labels.json")
data_provider = DataProviderLight(file_words,
file_labels,
sample_size=SAMPLE_SIZE)
# embedding data, splitting up into train and test set
processor = PregeneratedProcessor(data_provider)
generator = DeterministicGenerator(data_provider, processor)
dataset = generator.generate_dataset()
# creating classifier, overwriting parameters
classifier = FFNClassifier(data_provider, dataset, DEVICE, BATCH_SIZE,
HIDDEN_SIZE)
# train classifier and output progress
trainer = ClassificationTrainer(classifier)
trainer.enable_loss(10, True)
trainer.enable_precision(10, True)
trainer.enable_trainset_precision(10, True)
trainer.train(EPOCHS)
trainer.show()
from utils.DatasetGeneration import DeterministicGenerator
from classification.ClassificationTest import ClassificationTest
from classification.KNNClassification import KNNClassifier
# script for corresponding test case
# most test cases should be able to be executed without any further changes, if data is available
SAMPLE_SIZE = [5000, 10000, 20000, 50000]
N_NEIGHBORS = 1
# change device to "cpu" if cuda not available
DEVICE = "cuda"
stopwatch = StopWatch()
# pregenerated embedding and labels
for sample_size in SAMPLE_SIZE:
file_words = open("../tests/embedding7.json")
file_labels = open("../data/unique_labels.json")
data_provider = DataProviderLight(file_words, file_labels, sample_size=sample_size)
# embedding data, splitting up into train and test set
processor = PregeneratedProcessor(data_provider)
generator = DeterministicGenerator(data_provider, processor)
dataset = generator.generate_dataset()
print("Sample Size: " + str(sample_size))
# creating classifier, overwriting parameters
classifier = KNNClassifier(data_provider, dataset, DEVICE)
classifier.n_neighbours = N_NEIGHBORS
# train classifier and output progress
classifier.train()
# testing
test = ClassificationTest(dataset, classifier)
print("Präzision: " + str(test.test()) + "%")
file_words.close()
file_labels.close()
from utils.DataProviderLight import DataProviderLight
import matplotlib.pyplot as plt
# analyze some aspects of the data
file_words = open("../data/unique_equations.json")
file_labels = open("../data/unique_labels.json")
data_provider = DataProviderLight(file_words, file_labels)
def count_word_lengths():
lengths = {}
for word in data_provider.words:
if len(word) in lengths.keys():
lengths[len(word)] = lengths[len(word)] + 1
else:
lengths[len(word)] = 1
x = [key for key in lengths.keys()]
y = [lengths[key] for key in lengths.keys()]
fig, ax = plt.subplots()
ax.scatter(x, y, color="tab:blue", s=10)
ax.set_xlabel("Länge der Formel")
ax.set_ylabel("Anzahl der Formeln")
plt.show()
print("done")
def count_class_sizes():
labels = {}
for label in data_provider.labels:
if label in labels:
labels[label] = labels[label] + 1