def start(self):
self.get_query_terms()
self.create_training_sets()
self.get_documents()
self.process()
for documents in self.data_dict:
E = Evaluator(documents)
E.startProcess()
class OnlineTester(object):
def __init__(self, name, sample):
self.name = name
self.evaluator = Evaluator()
self.evaluator.setSample(sample)
def __call__(self, log, context, model):
translator = Translator(model)
result = self.evaluator.evaluate(translator)
print >> log, 'ER %s: string errors %s symbol errors %s' % (
self.name, result.stringError, result.symbolError)
class OnlineTester(object):
def __init__(self, name, sample):
self.name = name
self.evaluator = Evaluator()
self.evaluator.setSample(sample)
def __call__(self, log, context, model):
translator = Translator(model)
result = self.evaluator.evaluate(translator)
print('ER %s: string errors %s symbol errors %s' % (
self.name, result.stringError, result.symbolError), file=log)
def procureModel(self):
if self.options.resume_from_checkpoint:
model = ModelTemplate.resume(self.options.resume_from_checkpoint)
self.sequitur = model.sequitur
elif self.options.modelFile:
model = pickle.load(open(self.options.modelFile))
self.sequitur = model.sequitur
else:
self.sequitur = Sequitur()
model = None
if self.options.shouldRampUp:
model.rampUp()
if self.options.trainSample:
model = self.trainModel(model)
if not model:
print >> self.log, 'failed to estimate or load model'
return
if not model:
raise UsageError
# model.sequenceModel.showMostProbable(sys.stdout, model.sequitur.symbol, limit=250)
if self.options.shouldTranspose:
model.transpose()
if self.options.newModelFile:
oldSize, newSize = model.strip()
print >> self.log, 'stripped number of multigrams from %d to %d' % (oldSize, newSize)
f = open(self.options.newModelFile, 'w')
pickle.dump(model, f, pickle.HIGHEST_PROTOCOL)
f.close()
del f
if self.options.shouldSelfTest:
print >> self.log, 'warning: --self-test does not treat pronunciation variants correctly'
if not self.develSample:
print >> self.log, 'error: cannot do --self-test without --devel sample'
else:
translator = Translator(model)
evaluator = Evaluator()
evaluator.setSample(self.develSample)
evaluator.verboseLog = self.log
result = evaluator.evaluate(translator)
print >> self.log, result
return model
def procureModel(self):
if self.options.resume_from_checkpoint:
model = ModelTemplate.resume(self.options.resume_from_checkpoint)
self.sequitur = model.sequitur
elif self.options.modelFile:
if sys.version_info[:2] >= (3, 0):
model = pickle.load(open(self.options.modelFile, 'rb'), encoding='latin1')
else:
try:
model = pickle.load(open(self.options.modelFile, 'rb'))
except ValueError:
print('This error most likely occured because the loaded model was created in python3.\n', file=sys.stderr)
raise
self.sequitur = model.sequitur
else:
self.sequitur = Sequitur()
model = None
if self.options.shouldRampUp:
model.rampUp()
if self.options.trainSample:
model = self.trainModel(model)
if not model:
print('failed to estimate or load model', file=self.log)
return
if not model:
raise UsageError
# model.sequenceModel.showMostProbable(sys.stdout, model.sequitur.symbol, limit=250)
if self.options.shouldTranspose:
model.transpose()
if self.options.newModelFile:
oldSize, newSize = model.strip()
print('stripped number of multigrams from %d to %d' % (oldSize, newSize), file=self.log)
f = open(self.options.newModelFile, 'wb')
pickle.dump(model, f, pickle.HIGHEST_PROTOCOL)
f.close()
del f
if self.options.shouldSelfTest:
print('warning: --self-test does not treat pronunciation variants correctly', file=self.log)
if not self.develSample:
print('error: cannot do --self-test without --devel sample', file=self.log)
else:
translator = Translator(model)
evaluator = Evaluator()
evaluator.setSample(self.develSample)
evaluator.verboseLog = self.log
result = evaluator.evaluate(translator)
print(result, file=self.log)
return model
def procureModel(self):
if self.options.resume_from_checkpoint:
model = ModelTemplate.resume(self.options.resume_from_checkpoint)
self.sequitur = model.sequitur
elif self.options.modelFile:
model = pickle.load(open(self.options.modelFile, "rb"))
self.sequitur = model.sequitur
else:
self.sequitur = Sequitur()
model = None
if self.options.shouldRampUp:
model.rampUp()
if self.options.trainSample:
model = self.trainModel(model)
if not model:
print('failed to estimate or load model', file=self.log)
return
if not model:
raise UsageError
# model.sequenceModel.showMostProbable(sys.stdout, model.sequitur.symbol, limit=250)
if self.options.shouldTranspose:
model.transpose()
if self.options.newModelFile:
oldSize, newSize = model.strip()
print('stripped number of multigrams from %d to %d' %
(oldSize, newSize),
file=self.log)
f = open(self.options.newModelFile, 'wb')
pickle.dump(model, f)
f.close()
del f
if self.options.shouldSelfTest:
print(
'warning: --self-test does not treat pronunciation variants correctly',
file=self.log)
if not self.develSample:
print('error: cannot do --self-test without --devel sample',
file=self.log)
else:
translator = Translator(model)
evaluator = Evaluator()
evaluator.setSample(self.develSample)
evaluator.verboseLog = self.log
result = evaluator.evaluate(translator)
print(result, file=self.log)
return model
def mainTest(translator, testSample, options):
if options.shouldTranspose:
testSample = SequiturTool.transposeSample(testSample)
if options.testResult:
resultFile = gOpenOut(options.testResult, defaultEncoding)
else:
resultFile = None
from Evaluation import Evaluator
evaluator = Evaluator()
evaluator.setSample(testSample)
evaluator.resultFile = resultFile
evaluator.verboseLog = stdout
if options.test_segmental:
supraSegmental = set(['.', "'", '"'])
def removeSupraSegmental(phon):
return filter(lambda p: p not in supraSegmental, phon)
evaluator.compareFilter = removeSupraSegmental
result = evaluator.evaluate(translator)
print >> stdout, result
def mainTest(translator, testSample, options):
if options.shouldTranspose:
testSample = transposeSample(testSample)
if options.testResult:
resultFile = gOpenOut(options.testResult, defaultEncoding)
else:
resultFile = None
from Evaluation import Evaluator
evaluator = Evaluator()
evaluator.setSample(testSample)
evaluator.resultFile = resultFile
evaluator.verboseLog = stdout
evaluator.has_tones = options.has_tones
if options.test_segmental:
supraSegmental = set(['.', "'", '"'])
def removeSupraSegmental(phon):
return filter(lambda p: p not in supraSegmental, phon)
evaluator.compareFilter = removeSupraSegmental
result = evaluator.evaluate(translator)
print >> stdout, result
def __init__(self, name, sample):
self.name = name
self.evaluator = Evaluator()
self.evaluator.setSample(sample)
start_time = time.time()
early.fit(dataset.X_train, dataset.y_train)
elapsed_time_training = time.time() - start_time
# Predicting
print('Predicting...')
start_time = time.time()
y_pred = early.estimator.predict(dataset.X_test)
elapsed_time_testing = time.time() - start_time
# Analytics
title = "RandomForest"
save_path = "C:/Users/thoma/source/repos/PythonMachineLearning/PythonMachineLearning/Library/Results"
print('Analyzing...')
evaluator = Evaluator(title, save_path)
evaluator.append_to_file(f'Best iteration: {early.best_iteration_}',
"info.txt")
evaluator.append_to_file(f'Training time (seconds): {elapsed_time_training}',
"info.txt")
evaluator.append_to_file(f'Testing time (seconds): {elapsed_time_testing}',
"info.txt")
evaluator.append_to_file(dataset_parameters, "dataset_parameters.txt")
evaluator.append_to_file(model_parameters, "model_parameters.txt")
evaluator.save_advanced_metrics(dataset.y_test, y_pred, dataset.class_labels,
dataset.class_descriptions)
evaluator.append_to_file(early.eval_results, "metric_results.txt")
evaluator.create_evaluation_metric_results(early.eval_results,
xlabel='number of trees',
ylabel='geometric mean')
evaluator.create_confusion_matrix(dataset.y_test,
from SentimentAnalysis import SentimentAnalyzer
from Evaluation import Evaluator
api_key = 's6J10zzMfa9Vm5q1AufUu4gTovcVJEmo'
'''
# perform Sentiment Analysis
sa = SentimentAnalyzer(api_key)
target_file = sa.perform_analysis('dow jones', 1980, 11, "../data/indices/DJI.csv",
"../data/indices/DJI_sentiments.csv")
'''
# model evaluation
ev = Evaluator(
model_path='../models/normalized_sentiments.h5',
eval_data_path='../data/dow_jones_stocks/sentiments/dataset_3/NKE.csv',
scaler_path='../models/normalized_sentiments_scaler.save')
ev.create_prediction_data(normalize=True)
print("Market Return: {}\nModel Return: {}".format(
ev.market_return(), ev.model_return(kapital=100)))
ev.plot_predictions()
def __init__(self, name, sample): self.name = name self.evaluator = Evaluator() self.evaluator.setSample(sample)
# Predicting
print('Predicting...')
start_time = time.time()
y_pred = model.predict(dataset.X_test.values)
elapsed_time_testing = time.time() - start_time
# Analytics
#eval_results = {
# 'multi_logloss': model.evals_result_['valid_0']['multi_logloss'],
# 'gmean': np.absolute(model.evals_result_['valid_0']['gmean'])}
title = "LinearSVC (longrun minmax)"
save_path = "C:/Users/thoma/source/repos/PythonMachineLearning/PythonMachineLearning/Library/Results"
print('Analyzing...')
evaluator = Evaluator(title, save_path)
#evaluator.append_to_file(f'Best iteration: {model.best_iteration_}', "info.txt")
evaluator.append_to_file(f'Training time (seconds): {elapsed_time_training}',
"info.txt")
evaluator.append_to_file(f'Testing time (seconds): {elapsed_time_testing}',
"info.txt")
evaluator.append_to_file(dataset_parameters, "dataset_parameters.txt")
evaluator.append_to_file(model_parameters, "model_parameters.txt")
evaluator.save_advanced_metrics(dataset.y_test, y_pred, dataset.class_labels,
dataset.class_descriptions)
#evaluator.append_to_file(eval_results, "metric_results.txt")
#evaluator.create_evaluation_metric_results(eval_results, xlabel='number of trees', ylabel='geometric mean')
evaluator.create_confusion_matrix(dataset.y_test,
y_pred,
dataset.class_labels,
normalize=True)
eval_results = {
'accuracy': callback[0].val_accuracy,
'gmean': callback[0].val_gmean,
'loss': model.history.history["val_loss"]
}
#eval_results = {
# 'accuracy': model.history.history["val_acc"],
# 'loss': model.history.history["val_loss"],
# }
title = "TensorFlow (4 layers weights smote)"
save_path = "C:/Users/thoma/source/repos/PythonMachineLearning/PythonMachineLearning/Library/Results"
print('Analyzing...')
evaluator = Evaluator(title, save_path)
evaluator.append_to_file(f'Best iteration: {callback[0].best_epoch}',
"info.txt")
#evaluator.append_to_file(f'Best iteration: {len(model.history.history["val_loss"])}', "info.txt")
evaluator.append_to_file(f'Training time (seconds): {elapsed_time_training}',
"info.txt")
evaluator.append_to_file(f'Testing time (seconds): {elapsed_time_testing}',
"info.txt")
evaluator.save_dict_to_file(dataset_parameters, "dataset_parameters.csv")
evaluator.save_dict_to_file(model_parameters, "model_parameters.csv")
evaluator.save_advanced_metrics(dataset.y_test, y_pred, dataset.class_labels,
dataset.class_descriptions)
evaluator.save_eval_scores_to_file(eval_results, "metric_results.csv")
evaluator.create_evaluation_metric_results(eval_results,
xlabel='epochs',
ylabel='metric score')
def procureModel(self):
#print self.options,type(self.options)
#print self.loadSample,type(self.loadSample)
#print self.log,type(self.log)
if self.options.resume_from_checkpoint:
model = ModelTemplate.resume(self.options.resume_from_checkpoint)
self.sequitur = model.sequitur
elif self.options.modelFile:
#print "loading",self.options.modelFile
f = open(self.options.modelFile)
#print "loaded",f
#print "type:",type(f)
#print pickle
class Model(object):
pass
model = pickle.load(f)
#print "loaded",self.options.modelFile
self.sequitur = model.sequitur
else:
self.sequitur = Sequitur()
model = None
if self.options.shouldRampUp:
model.rampUp()
if self.options.trainSample:
model = self.trainModel(model)
if not model:
print >> self.log, 'failed to estimate or load model'
return
if not model:
raise UsageError
# model.sequenceModel.showMostProbable(sys.stdout, model.sequitur.symbol, limit=250)
if self.options.shouldTranspose:
model.transpose()
if self.options.newModelFile:
oldSize, newSize = model.strip()
print >> self.log, 'stripped number of multigrams from %d to %d' % (
oldSize, newSize)
f = open(self.options.newModelFile, 'w')
pickle.dump(model, f, pickle.HIGHEST_PROTOCOL)
f.close()
del f
if self.options.shouldSelfTest:
print >> self.log, 'warning: --self-test does not treat pronunciation variants correctly'
if not self.develSample:
print >> self.log, 'error: cannot do --self-test without --devel sample'
else:
translator = Translator(model)
evaluator = Evaluator()
evaluator.setSample(self.develSample)
evaluator.verboseLog = self.log
result = evaluator.evaluate(translator)
print >> self.log, result
return model
"labels": classes,
"average": None
}],
"Recall_Class": [recall_score, {
"labels": classes,
"average": None
}],
}
for split in tqdm(splits.keys()):
#set-up evaluator
split_folder = os.path.join(results_folder, split)
evaluator = Evaluator(models,
prepare_batch_cnn,
splits[split]["dataset"],
splits[split]["loader"],
split_folder,
device,
metrics_dict=metrics_dict)
#compute report
report = get_split_report(evaluator, split)
#save report
save_results(report, split_folder)
print(f"Correctly process split {split}")
# if config["calibration"]:
# #calibrate model
# scaled_model = ModelWithTemperature(models,prepare_batch=prepare_batch_cnn)
# scaled_model.set_temperature(splits["val"]["loader"])
# for split in tqdm(splits.keys()):
# #set-up evaluator
