def start(self): self.get_query_terms() self.create_training_sets() self.get_documents() self.process() for documents in self.data_dict: E = Evaluator(documents) E.startProcess()
class OnlineTester(object): def __init__(self, name, sample): self.name = name self.evaluator = Evaluator() self.evaluator.setSample(sample) def __call__(self, log, context, model): translator = Translator(model) result = self.evaluator.evaluate(translator) print >> log, 'ER %s: string errors %s symbol errors %s' % ( self.name, result.stringError, result.symbolError)
class OnlineTester(object): def __init__(self, name, sample): self.name = name self.evaluator = Evaluator() self.evaluator.setSample(sample) def __call__(self, log, context, model): translator = Translator(model) result = self.evaluator.evaluate(translator) print('ER %s: string errors %s symbol errors %s' % ( self.name, result.stringError, result.symbolError), file=log)
def procureModel(self): if self.options.resume_from_checkpoint: model = ModelTemplate.resume(self.options.resume_from_checkpoint) self.sequitur = model.sequitur elif self.options.modelFile: model = pickle.load(open(self.options.modelFile)) self.sequitur = model.sequitur else: self.sequitur = Sequitur() model = None if self.options.shouldRampUp: model.rampUp() if self.options.trainSample: model = self.trainModel(model) if not model: print >> self.log, 'failed to estimate or load model' return if not model: raise UsageError # model.sequenceModel.showMostProbable(sys.stdout, model.sequitur.symbol, limit=250) if self.options.shouldTranspose: model.transpose() if self.options.newModelFile: oldSize, newSize = model.strip() print >> self.log, 'stripped number of multigrams from %d to %d' % (oldSize, newSize) f = open(self.options.newModelFile, 'w') pickle.dump(model, f, pickle.HIGHEST_PROTOCOL) f.close() del f if self.options.shouldSelfTest: print >> self.log, 'warning: --self-test does not treat pronunciation variants correctly' if not self.develSample: print >> self.log, 'error: cannot do --self-test without --devel sample' else: translator = Translator(model) evaluator = Evaluator() evaluator.setSample(self.develSample) evaluator.verboseLog = self.log result = evaluator.evaluate(translator) print >> self.log, result return model
def procureModel(self): if self.options.resume_from_checkpoint: model = ModelTemplate.resume(self.options.resume_from_checkpoint) self.sequitur = model.sequitur elif self.options.modelFile: if sys.version_info[:2] >= (3, 0): model = pickle.load(open(self.options.modelFile, 'rb'), encoding='latin1') else: try: model = pickle.load(open(self.options.modelFile, 'rb')) except ValueError: print('This error most likely occured because the loaded model was created in python3.\n', file=sys.stderr) raise self.sequitur = model.sequitur else: self.sequitur = Sequitur() model = None if self.options.shouldRampUp: model.rampUp() if self.options.trainSample: model = self.trainModel(model) if not model: print('failed to estimate or load model', file=self.log) return if not model: raise UsageError # model.sequenceModel.showMostProbable(sys.stdout, model.sequitur.symbol, limit=250) if self.options.shouldTranspose: model.transpose() if self.options.newModelFile: oldSize, newSize = model.strip() print('stripped number of multigrams from %d to %d' % (oldSize, newSize), file=self.log) f = open(self.options.newModelFile, 'wb') pickle.dump(model, f, pickle.HIGHEST_PROTOCOL) f.close() del f if self.options.shouldSelfTest: print('warning: --self-test does not treat pronunciation variants correctly', file=self.log) if not self.develSample: print('error: cannot do --self-test without --devel sample', file=self.log) else: translator = Translator(model) evaluator = Evaluator() evaluator.setSample(self.develSample) evaluator.verboseLog = self.log result = evaluator.evaluate(translator) print(result, file=self.log) return model
def procureModel(self): if self.options.resume_from_checkpoint: model = ModelTemplate.resume(self.options.resume_from_checkpoint) self.sequitur = model.sequitur elif self.options.modelFile: model = pickle.load(open(self.options.modelFile, "rb")) self.sequitur = model.sequitur else: self.sequitur = Sequitur() model = None if self.options.shouldRampUp: model.rampUp() if self.options.trainSample: model = self.trainModel(model) if not model: print('failed to estimate or load model', file=self.log) return if not model: raise UsageError # model.sequenceModel.showMostProbable(sys.stdout, model.sequitur.symbol, limit=250) if self.options.shouldTranspose: model.transpose() if self.options.newModelFile: oldSize, newSize = model.strip() print('stripped number of multigrams from %d to %d' % (oldSize, newSize), file=self.log) f = open(self.options.newModelFile, 'wb') pickle.dump(model, f) f.close() del f if self.options.shouldSelfTest: print( 'warning: --self-test does not treat pronunciation variants correctly', file=self.log) if not self.develSample: print('error: cannot do --self-test without --devel sample', file=self.log) else: translator = Translator(model) evaluator = Evaluator() evaluator.setSample(self.develSample) evaluator.verboseLog = self.log result = evaluator.evaluate(translator) print(result, file=self.log) return model
def mainTest(translator, testSample, options): if options.shouldTranspose: testSample = SequiturTool.transposeSample(testSample) if options.testResult: resultFile = gOpenOut(options.testResult, defaultEncoding) else: resultFile = None from Evaluation import Evaluator evaluator = Evaluator() evaluator.setSample(testSample) evaluator.resultFile = resultFile evaluator.verboseLog = stdout if options.test_segmental: supraSegmental = set(['.', "'", '"']) def removeSupraSegmental(phon): return filter(lambda p: p not in supraSegmental, phon) evaluator.compareFilter = removeSupraSegmental result = evaluator.evaluate(translator) print >> stdout, result
def mainTest(translator, testSample, options): if options.shouldTranspose: testSample = transposeSample(testSample) if options.testResult: resultFile = gOpenOut(options.testResult, defaultEncoding) else: resultFile = None from Evaluation import Evaluator evaluator = Evaluator() evaluator.setSample(testSample) evaluator.resultFile = resultFile evaluator.verboseLog = stdout evaluator.has_tones = options.has_tones if options.test_segmental: supraSegmental = set(['.', "'", '"']) def removeSupraSegmental(phon): return filter(lambda p: p not in supraSegmental, phon) evaluator.compareFilter = removeSupraSegmental result = evaluator.evaluate(translator) print >> stdout, result
def __init__(self, name, sample): self.name = name self.evaluator = Evaluator() self.evaluator.setSample(sample)
start_time = time.time() early.fit(dataset.X_train, dataset.y_train) elapsed_time_training = time.time() - start_time # Predicting print('Predicting...') start_time = time.time() y_pred = early.estimator.predict(dataset.X_test) elapsed_time_testing = time.time() - start_time # Analytics title = "RandomForest" save_path = "C:/Users/thoma/source/repos/PythonMachineLearning/PythonMachineLearning/Library/Results" print('Analyzing...') evaluator = Evaluator(title, save_path) evaluator.append_to_file(f'Best iteration: {early.best_iteration_}', "info.txt") evaluator.append_to_file(f'Training time (seconds): {elapsed_time_training}', "info.txt") evaluator.append_to_file(f'Testing time (seconds): {elapsed_time_testing}', "info.txt") evaluator.append_to_file(dataset_parameters, "dataset_parameters.txt") evaluator.append_to_file(model_parameters, "model_parameters.txt") evaluator.save_advanced_metrics(dataset.y_test, y_pred, dataset.class_labels, dataset.class_descriptions) evaluator.append_to_file(early.eval_results, "metric_results.txt") evaluator.create_evaluation_metric_results(early.eval_results, xlabel='number of trees', ylabel='geometric mean') evaluator.create_confusion_matrix(dataset.y_test,
from SentimentAnalysis import SentimentAnalyzer from Evaluation import Evaluator api_key = 's6J10zzMfa9Vm5q1AufUu4gTovcVJEmo' ''' # perform Sentiment Analysis sa = SentimentAnalyzer(api_key) target_file = sa.perform_analysis('dow jones', 1980, 11, "../data/indices/DJI.csv", "../data/indices/DJI_sentiments.csv") ''' # model evaluation ev = Evaluator( model_path='../models/normalized_sentiments.h5', eval_data_path='../data/dow_jones_stocks/sentiments/dataset_3/NKE.csv', scaler_path='../models/normalized_sentiments_scaler.save') ev.create_prediction_data(normalize=True) print("Market Return: {}\nModel Return: {}".format( ev.market_return(), ev.model_return(kapital=100))) ev.plot_predictions()
# Predicting print('Predicting...') start_time = time.time() y_pred = model.predict(dataset.X_test.values) elapsed_time_testing = time.time() - start_time # Analytics #eval_results = { # 'multi_logloss': model.evals_result_['valid_0']['multi_logloss'], # 'gmean': np.absolute(model.evals_result_['valid_0']['gmean'])} title = "LinearSVC (longrun minmax)" save_path = "C:/Users/thoma/source/repos/PythonMachineLearning/PythonMachineLearning/Library/Results" print('Analyzing...') evaluator = Evaluator(title, save_path) #evaluator.append_to_file(f'Best iteration: {model.best_iteration_}', "info.txt") evaluator.append_to_file(f'Training time (seconds): {elapsed_time_training}', "info.txt") evaluator.append_to_file(f'Testing time (seconds): {elapsed_time_testing}', "info.txt") evaluator.append_to_file(dataset_parameters, "dataset_parameters.txt") evaluator.append_to_file(model_parameters, "model_parameters.txt") evaluator.save_advanced_metrics(dataset.y_test, y_pred, dataset.class_labels, dataset.class_descriptions) #evaluator.append_to_file(eval_results, "metric_results.txt") #evaluator.create_evaluation_metric_results(eval_results, xlabel='number of trees', ylabel='geometric mean') evaluator.create_confusion_matrix(dataset.y_test, y_pred, dataset.class_labels, normalize=True)
eval_results = { 'accuracy': callback[0].val_accuracy, 'gmean': callback[0].val_gmean, 'loss': model.history.history["val_loss"] } #eval_results = { # 'accuracy': model.history.history["val_acc"], # 'loss': model.history.history["val_loss"], # } title = "TensorFlow (4 layers weights smote)" save_path = "C:/Users/thoma/source/repos/PythonMachineLearning/PythonMachineLearning/Library/Results" print('Analyzing...') evaluator = Evaluator(title, save_path) evaluator.append_to_file(f'Best iteration: {callback[0].best_epoch}', "info.txt") #evaluator.append_to_file(f'Best iteration: {len(model.history.history["val_loss"])}', "info.txt") evaluator.append_to_file(f'Training time (seconds): {elapsed_time_training}', "info.txt") evaluator.append_to_file(f'Testing time (seconds): {elapsed_time_testing}', "info.txt") evaluator.save_dict_to_file(dataset_parameters, "dataset_parameters.csv") evaluator.save_dict_to_file(model_parameters, "model_parameters.csv") evaluator.save_advanced_metrics(dataset.y_test, y_pred, dataset.class_labels, dataset.class_descriptions) evaluator.save_eval_scores_to_file(eval_results, "metric_results.csv") evaluator.create_evaluation_metric_results(eval_results, xlabel='epochs', ylabel='metric score')
def procureModel(self): #print self.options,type(self.options) #print self.loadSample,type(self.loadSample) #print self.log,type(self.log) if self.options.resume_from_checkpoint: model = ModelTemplate.resume(self.options.resume_from_checkpoint) self.sequitur = model.sequitur elif self.options.modelFile: #print "loading",self.options.modelFile f = open(self.options.modelFile) #print "loaded",f #print "type:",type(f) #print pickle class Model(object): pass model = pickle.load(f) #print "loaded",self.options.modelFile self.sequitur = model.sequitur else: self.sequitur = Sequitur() model = None if self.options.shouldRampUp: model.rampUp() if self.options.trainSample: model = self.trainModel(model) if not model: print >> self.log, 'failed to estimate or load model' return if not model: raise UsageError # model.sequenceModel.showMostProbable(sys.stdout, model.sequitur.symbol, limit=250) if self.options.shouldTranspose: model.transpose() if self.options.newModelFile: oldSize, newSize = model.strip() print >> self.log, 'stripped number of multigrams from %d to %d' % ( oldSize, newSize) f = open(self.options.newModelFile, 'w') pickle.dump(model, f, pickle.HIGHEST_PROTOCOL) f.close() del f if self.options.shouldSelfTest: print >> self.log, 'warning: --self-test does not treat pronunciation variants correctly' if not self.develSample: print >> self.log, 'error: cannot do --self-test without --devel sample' else: translator = Translator(model) evaluator = Evaluator() evaluator.setSample(self.develSample) evaluator.verboseLog = self.log result = evaluator.evaluate(translator) print >> self.log, result return model
"labels": classes, "average": None }], "Recall_Class": [recall_score, { "labels": classes, "average": None }], } for split in tqdm(splits.keys()): #set-up evaluator split_folder = os.path.join(results_folder, split) evaluator = Evaluator(models, prepare_batch_cnn, splits[split]["dataset"], splits[split]["loader"], split_folder, device, metrics_dict=metrics_dict) #compute report report = get_split_report(evaluator, split) #save report save_results(report, split_folder) print(f"Correctly process split {split}") # if config["calibration"]: # #calibrate model # scaled_model = ModelWithTemperature(models,prepare_batch=prepare_batch_cnn) # scaled_model.set_temperature(splits["val"]["loader"]) # for split in tqdm(splits.keys()): # #set-up evaluator