def run(self): folds_test = mask.compute_folds(self.I,self.J,self.K,self.M) folds_training = mask.compute_Ms(folds_test) for i,(train,test) in enumerate(zip(folds_training,folds_test)): print "Fold %s of nested cross-validation." % (i+1) # Run the cross-validation crossval = ParallelMatrixCrossValidation( #crossval = MatrixCrossValidation( method=self.method, X=self.X, M=train, K=self.K, parameter_search=self.parameter_search, train_config=self.train_config, file_performance=self.files_nested_performances[i], P=self.P ) crossval.run() (best_parameters,_) = crossval.find_best_parameters(evaluation_criterion='MSE',low_better=True) print "Best parameters for fold %s were %s." % (i+1,best_parameters) # Train the model and test the performance on the test set performance_dict = self.run_model(train,test,best_parameters) self.store_performances(performance_dict) print "Finished fold %s, with performances %s." % (i+1,performance_dict) self.log()
def run(self): for parameters in self.parameter_search: print "Trying parameters %s." % (parameters) try: folds_test = mask.compute_folds(self.I,self.J,self.K,self.M) folds_training = mask.compute_Ms(folds_test) # We need to put the parameter dict into json to hash it self.all_performances[self.JSON(parameters)] = {} # Create the threads for the folds, and run them pool = Pool(self.P) all_parameters = [ { 'parameters' : parameters, 'X' : numpy.copy(self.X), 'train' : train, 'test' : test, 'method' : self.method, 'train_config' : self.train_config } for (train,test) in zip(folds_training,folds_test) ] outputs = pool.map(run_fold,all_parameters) for performance_dict in outputs: self.store_performances(performance_dict,parameters) self.log(parameters) except Exception as e: self.fout.write("Tried parameters %s but got exception: %s. \n" % (parameters,e))
def run(self): folds_test = mask.compute_folds(self.I, self.J, self.K, self.M) folds_training = mask.compute_Ms(folds_test) for i, (train, test) in enumerate(zip(folds_training, folds_test)): print "Fold %s of nested cross-validation." % (i + 1) # Run the cross-validation crossval = ParallelMatrixCrossValidation( #crossval = MatrixCrossValidation( method=self.method, X=self.X, M=train, K=self.K, parameter_search=self.parameter_search, train_config=self.train_config, file_performance=self.files_nested_performances[i], P=self.P) crossval.run() (best_parameters, _) = crossval.find_best_parameters(evaluation_criterion='MSE', low_better=True) print "Best parameters for fold %s were %s." % (i + 1, best_parameters) # Train the model and test the performance on the test set performance_dict = self.run_model(train, test, best_parameters) self.store_performances(performance_dict) print "Finished fold %s, with performances %s." % ( i + 1, performance_dict) self.log()
def run(self): for parameters in self.parameter_search: print "Trying parameters %s." % (parameters) try: folds_test = mask.compute_folds(self.I, self.J, self.K, self.M) folds_training = mask.compute_Ms(folds_test) # We need to put the parameter dict into json to hash it self.all_performances[self.JSON(parameters)] = {} # Create the threads for the folds, and run them pool = Pool(self.P) all_parameters = [{ 'parameters': parameters, 'X': numpy.copy(self.X), 'train': train, 'test': test, 'method': self.method, 'train_config': self.train_config } for (train, test) in zip(folds_training, folds_test)] outputs = pool.map(run_fold, all_parameters) for performance_dict in outputs: self.store_performances(performance_dict, parameters) self.log(parameters) except Exception as e: self.fout.write( "Tried parameters %s but got exception: %s. \n" % (parameters, e))
def run(self,burn_in=None,thinning=None,minimum_TN=None): folds_test = mask.compute_folds(self.I,self.J,self.folds,self.M) folds_training = mask.compute_Ms(folds_test) for i,(train,test) in enumerate(zip(folds_training,folds_test)): print "Fold %s." % (i+1) # Run the line search line_search = LineSearch( classifier=self.classifier, values_K=self.values_K, R=self.R, M=self.M, priors=self.priors, initUV=self.init_UV, iterations=self.iterations, restarts=self.restarts) line_search.search(burn_in=burn_in,thinning=thinning,minimum_TN=minimum_TN) # Store the model fits, and find the best one according to the metric all_performances = line_search.all_values(metric=self.quality_metric) self.fout.write("All model fits for fold %s, metric %s: %s.\n" % (i+1,self.quality_metric,all_performances)) self.fout.flush() best_K = line_search.best_value(metric=self.quality_metric) self.fout.write("Best K for fold %s: %s.\n" % (i+1,best_K)) # Train a model with this K and measure performance on the test set performance = self.run_model(train,test,best_K,burn_in=burn_in,thinning=thinning,minimum_TN=minimum_TN) self.fout.write("Performance: %s.\n\n" % performance) self.fout.flush()
def run(self): for parameters in self.parameter_search: print "Trying parameters %s." % (parameters) try: folds_test = mask.compute_folds(self.I,self.J,self.K,self.M) folds_training = mask.compute_Ms(folds_test) # We need to put the parameter dict into json to hash it self.all_performances[self.JSON(parameters)] = {} for i,(train,test) in enumerate(zip(folds_training,folds_test)): print "Fold %s (parameters: %s)." % (i+1,parameters) performance_dict = self.run_model(train,test,parameters) self.store_performances(performance_dict,parameters) self.log(parameters) except Exception as e: self.fout.write("Tried parameters %s but got exception: %s. \n" % (parameters,e)) self.fout.flush()