def test_regress(self): model = ExpertModelAPI() model_id = model.create_model(model_type="KNN_regress") train = [permute_cmo(ClimateModelOutput(TEST_NC)) for _ in range(10)] labels = [0 for _ in range(5)] + [1 for _ in range(5)] performance = model.partial_fit(model_id=model_id, data=train, targets=labels) self.assertIsInstance(performance, ModelOutputs) self.assertIsInstance(performance.L1_loss, float) self.assertTrue(0 <= performance.L1_loss <= 1) prediction = model.predict(model_id=model_id, data=train)[0] self.assertIsInstance(prediction, float)
def updatePersonal(request): Helper.permissions(request) try: modelSpec = request.params["model_spec"] except: raise exc.HTTPBadRequest("Information missing") user = db.executeOne("User_Info", [request.session["username"]]) if True: # TODO # Change all the models currently implemented and re train expertsModels = db.execute_literal( "SELECT * FROM expertModels WHERE username = ?", [user["username"]]) # Update users Model Specification db.execute_literal("UPDATE users SET modelSpec = ? WHERE username = ?", [modelSpec, user["username"]]) for model in expertsModels: # Replace the old model with the new model in the database identifier = ExpertModelAPI().create_model(model_type=modelSpec) db.execute_literal( "UPDATE expertModels SET identifier = ? WHERE identifier = ?", [identifier, model["identifier"]]) trainingData = db.execute("collectModelLabels", [user["username"], model["qid"]]) CMOs, scores = [], [] for datapoint in trainingData: CMOs.append( ClimateModelOutput.load( os.path.join(CMOSTORAGE, str(datapoint["cmoid"])))) scores.append(datapoint["score"]) ProcessRunner().add_process(identifier, user["username"], CMOs, scores) # Delete old model ExpertModelAPI().delete_model(model_id=model["identifier"])
def confirmUser(request): if not Helper.HiddenPages.validate(request.path): return exc.HTTPNotFound() if not Helper.HiddenPages.validate(request.path): return exc.HTTPNotFound() try: username = request.params["username"] title = request.params["title"] firstname = request.params["firstname"] lastname = request.params["lastname"] organisation = request.params["organisation"] email = request.params["email"] password = request.params["password"] except: raise exc.HTTPBadRequest("Invalid request") # Check if username already exists if len(db.execute("User_Info", [username])): raise exc.HTTPBadRequest("Username Exists") salt, hashedPassword = Helper.hashPassword(password) db.execute("User_confirmTemp", [ username, email, salt, hashedPassword, title, firstname, lastname, organisation, request.path[-10:] ]) questions = db.execute("collectQuestions", []) api = ExpertModelAPI() for rows in questions: qid = rows["qid"] model_id = api.create_model(model_type=DEFAULT_TECH) db.execute("eModel_insertModel", [model_id, username, qid]) Helper.HiddenPages.remove(request.path)
def fit_model_and_write_db(model_id, username, CMOs, scores): try: recordModelMetrics( identifier=model_id) # Clear DB entries of records for this model _LOGGER.info("Metrics cleared for modelid={}, username={}".format( model_id, username)) ExpertModelAPI().fit_unsupervised(model_id=model_id, data=CMOStore.models()) _LOGGER.info( "Model fit unsupervised completed for modelid={}, username={}". format(model_id, username)) metrics = ExpertModelAPI().partial_fit(model_id=model_id, data=CMOs, targets=scores) _LOGGER.info( "Model fit supervised completed for modelid={}, username={}". format(model_id, username)) recordModelMetrics(identifier=model_id, metrics=metrics) _LOGGER.info( "Metrics written to DB for modelid={}, username={}".format( model_id, username)) except Exception as e: print(str(e)) # TODO
def test_semi_supervised(self): model = ExpertModelAPI() model_id = model.create_model(model_type="KNN_PCA") train = [permute_cmo(ClimateModelOutput(TEST_NC)) for _ in range(10)] for data in train: data.numpy_arrays += np.random.normal(size=data.numpy_arrays.shape) labels = [0 for _ in range(5)] + [1 for _ in range(5)] model.fit_unsupervised(model_id=model_id, data=train) performance = model.partial_fit(model_id=model_id, data=train, targets=labels) self.assertIsInstance(performance, ModelOutputs) self.assertIsInstance(performance.L1_loss, float) self.assertTrue(0 <= performance.L1_loss <= 1) prediction = model.predict(model_id=model_id, data=train)[0] self.assertIsInstance(prediction, float)
def test_initialise_create_and_train(self): model = ExpertModelAPI() model_id = model.create_model(model_type="KNN_classify") self.assertIn(model_id, model.models_that_exist) self.assertIn(model_id, model.open_models) model.close_model(model_id=model_id) self.assertIn(model_id, model.models_that_exist) self.assertNotIn(model_id, model.open_models) train = [permute_cmo(ClimateModelOutput(TEST_NC)) for _ in range(10)] labels = [0 for _ in range(5)] + [1 for _ in range(5)] performance = model.partial_fit(model_id=model_id, data=train, targets=labels) self.assertIsInstance(performance, ModelOutputs) self.assertIsInstance(performance.accuracy, float) self.assertTrue(0 <= performance.accuracy <= 1) predictions = np.array(model.predict(model_id=model_id, data=train)) model.close_model(model_id=model_id) predictions_2 = np.array(model.predict(model_id=model_id, data=train)) self.assertTrue(np.all(predictions == predictions_2)) model_id2 = model.create_model(model_type="KNN_classify") self.assertNotEqual(model_id, model_id2) model.delete_model(model_id=model_id) model = ExpertModelAPI() with self.assertRaisesRegex( ModelDoesNotExistException, r"The model with id: 0x[0-9]+ does not exist!!"): model.predict(model_id=model_id, data=train)
def test_all_models(self): for model_type in available_models(): train = [permute_cmo(ClimateModelOutput(TEST_NC)) for _ in range(10)] labels = [0 for _ in range(5)] + [1 for _ in range(5)] model = ExpertModelAPI() model_id = model.create_model(model_type=model_type) self.assertIn(model_id, model.models_that_exist) self.assertIn(model_id, model.open_models) model.close_model(model_id=model_id) self.assertIn(model_id, model.models_that_exist) self.assertNotIn(model_id, model.open_models) model.fit_unsupervised(model_id=model_id, data=train) performance = model.partial_fit(model_id=model_id, data=train, targets=labels) self.assertIsInstance(performance, ModelOutputs) predictions = np.array(model.predict(model_id=model_id, data=train)) model.close_model(model_id=model_id) predictions_2 = np.array(model.predict(model_id=model_id, data=train)) self.assertTrue(np.all(predictions == predictions_2)) model.delete_model(model_id=model_id)
def evalModels(request): Helper.permissions(request) # Collect information to evaluate try: data = request.json_body except: raise exc.HTTPBadRequest() questions = {} experts = set() for pair in data["experts"]: experts.add(pair["user"]) if pair["qid"] in questions: questions[pair["qid"]].append(pair["user"]) else: questions[pair["qid"]] = [pair["user"]] # Collect Expert information expertNames = { u: " ".join([i["title"], i["firstname"], i["lastname"]]) for u in experts for i in db.execute("User_Info", [u]) } # Collect uploaded models to evaluate modelDirectory = os.path.join(TEMPSTORAGE, request.session["username"]) CMOnames = os.listdir(modelDirectory) CMOpaths = [ os.path.join(modelDirectory, filename) for filename in CMOnames ] CMOs = [ClimateModelOutput.load(path) for path in CMOpaths] # Data type to return to the page data = [] for qid in questions.keys(): questionContents = { "text": db.executeOne("questionName", [qid])["text"], "models": [] } predictions = [] errored = [] for i, expert in enumerate(questions[qid]): # Collect the experts model identifier identifier = db.executeOne("collectEModel", [expert, qid])["identifier"] # Predict on the CMO's and storage the results try: predictions.append(ExpertModelAPI().predict( model_id=identifier, data=CMOs)) except ModelNotTrainedException: errored.append(i) for i in errored: del questions[qid][i] questionContents["experts"] = [expertNames[u] for u in questions[qid]] count = 0 for i in range(len(CMOs)): count += len(predictions) modelInfo = { "name": CMOnames[i], "values": [round(opinion[i] * 20, 5) for opinion in predictions] } questionContents["models"].append(modelInfo) if count != 0: data.append(questionContents) # Remove uploaded models that have not been evaluated. Helper.emptyDirectory(modelDirectory) return data