async def accuracy(self, sources: Sources): if not os.path.isfile( os.path.join(self.parent.config.output_dir, "tf_model.h5")): raise ModelNotTrained("Train model before assessing for accuracy.") config = self.parent.config._asdict() self.tokenizer = AutoTokenizer.from_pretrained( self.parent.config.output_dir) eval_features = await self._preprocess_data(sources) eval_dataset = await self.example_features_to_dataset(eval_features) def compute_metrics(p: EvalPrediction) -> Dict: preds = self.np.argmax(p.predictions, axis=1) return classification_compute_metrics(preds, p.label_ids) with self.parent.config.strategy.scope(): self.model = TFAutoModelForSequenceClassification.from_pretrained( config["directory"]) trainer = TFTrainer( model=self.model, args=self.parent.config, eval_dataset=eval_dataset, compute_metrics=compute_metrics, ) result = trainer.evaluate() return Accuracy(result["eval_acc"])
async def accuracy(self, sources: Sources): if not os.path.isfile( os.path.join(self.parent.config.output_dir, "tf_model.h5") ): raise ModelNotTrained("Train model before assessing for accuracy.") data_df = await self._preprocess_data(sources) eval_dataset = self.get_dataset(data_df, self.tokenizer, mode="eval",) with self.parent.config.strategy.scope(): self.model = TFAutoModelForTokenClassification.from_pretrained( self.parent.config.output_dir, config=self.config, cache_dir=self.parent.config.cache_dir, ) trainer = TFTrainer( model=self.model, args=self.parent.config, train_dataset=None, eval_dataset=eval_dataset.get_dataset(), compute_metrics=self.compute_metrics, ) result = trainer.evaluate() return Accuracy(result["eval_f1"])
async def accuracy(self, sources: Sources) -> Accuracy: try: with open("acc.txt", "r") as f: self.accuracy_value = float(f.read()) except: raise ModelNotTrained("Train model before assessing for accuracy.") return Accuracy(self.accuracy_value)
async def accuracy(self, sources: Sources) -> Accuracy: if not os.path.isfile(os.path.join(self.model_path)): raise ModelNotTrained("Train model before assessing for accuracy.") dataset, size = await self.dataset_generator(sources) dataloader = torch.utils.data.DataLoader( dataset, batch_size=self.parent.config.batch_size, shuffle=True, num_workers=4, ) self._model.eval() running_corrects = 0 for inputs, labels in dataloader: inputs = inputs.to(inputs) labels = labels.to(inputs) with torch.set_grad_enabled(False): outputs = self._model(inputs) _, preds = torch.max(outputs, 1) running_corrects += torch.sum(preds == labels.data) acc = running_corrects.double() / size return Accuracy(acc)
async def accuracy(self, sources: Sources) -> Accuracy: """ Evaluates the accuracy of our model after training using the input records as test data. """ if not os.path.isdir(self.model_dir_path): raise ModelNotTrained("Train model before assessing for accuracy.") input_fn = await self.accuracy_input_fn(sources) accuracy_score = self.model.evaluate(input_fn=input_fn) return Accuracy(accuracy_score["accuracy"])
async def accuracy(self, sources: Sources) -> Accuracy: """ Evaluates the accuracy of our model after training using the input records as test data. """ if not os.path.isfile( os.path.join(self.model_dir_path, "saved_model.pb")): raise ModelNotTrained("Train model before assessing for accuracy.") x, y = await self.train_data_generator(sources) accuracy_score = self._model.evaluate(x, y) return Accuracy(accuracy_score[1])
async def accuracy(self, sources: Sources) -> Accuracy: """ Evaluates the accuracy of our model after training using the input repos as test data. """ if not os.path.isdir(self.model_dir_path): raise NotADirectoryError("Model not trained") input_fn = await self.evaluate_input_fn(sources, batch_size=20, shuffle=False, epochs=1) metrics = self.model.evaluate(input_fn=input_fn) return Accuracy(1 - metrics["loss"]) # 1 - mse
async def accuracy(self, sources: Sources) -> Accuracy: if not self.model: raise ModelNotTrained("Train the model before assessing accuracy") test_data = [] async for record in sources.with_features( self.features + [self.parent.config.predict.name]): test_data.append(record.features()) df = pd.DataFrame(test_data) y_test = df[[self.parent.config.predict.name]] x_test = df.drop(columns=[self.parent.config.predict.name]) predictions = await self.get_predictions(x_test) accuracy = await self.accuracy_score(y_test, predictions) return Accuracy(accuracy)
async def accuracy(self, sources: Sources) -> Accuracy: """ Assess the accuracy of the network on the test data after training on records """ if not os.path.isfile(os.path.join(self.model_path)): raise ModelNotTrained("Train model before assessing for accuracy.") dataset, size = await self.dataset_generator(sources) dataloader = torch.utils.data.DataLoader( dataset, batch_size=self.parent.config.batch_size, shuffle=True, num_workers=4, ) self._model.eval() if self.classifications: running_corrects = 0 for inputs, labels in dataloader: inputs = inputs.to(self.device) labels = labels.to(self.device) with torch.set_grad_enabled(False): outputs = self._model(inputs) _, preds = torch.max(outputs, 1) running_corrects += torch.sum(preds == labels.data) acc = running_corrects.double() / size else: running_loss = 0.0 for inputs, labels in dataloader: inputs = inputs.to(inputs) labels = labels.to(inputs) with torch.set_grad_enabled(False): outputs = self._model(inputs) loss = self.criterion(inputs, outputs) running_loss += loss.item() * inputs.size(0) total_loss = running_loss / size acc = 1.0 - total_loss return Accuracy(acc)
async def accuracy(self, sources: Sources): if not os.path.isfile( os.path.join(self.parent.config.output_dir, "pytorch_model.bin")): raise ModelNotTrained("Train model before assessing for accuracy.") self.tokenizer = AutoTokenizer.from_pretrained( self.parent.config.output_dir, do_lower_case=self.parent.config.do_lower_case, ) eval_examples = await self._preprocess_data(sources) features, dataset = squad_convert_examples_to_features( examples=eval_examples, tokenizer=self.tokenizer, max_seq_length=self.parent.config.max_seq_length, doc_stride=self.parent.config.doc_stride, max_query_length=self.parent.config.max_query_length, is_training=False, return_dataset="pt", ) results = {} if self.parent.config.local_rank in [-1, 0]: logger.info( "Loading checkpoints saved during training for evaluation") self.model = AutoModelForQuestionAnswering.from_pretrained( self.parent.config.output_dir) self.model.to(self.parent.config.device) # Evaluate predictions = await self._custom_accuracy(eval_examples, features, dataset) results = squad_evaluate(eval_examples, predictions) logger.info("Results: {}".format(results)) # return results return Accuracy(results["f1"])
async def accuracy(self, sources: Sources): if not os.path.isfile( os.path.join(self.parent.config.output_dir, "tf_model.h5") ): raise ModelNotTrained("Train model before assessing for accuracy.") config = self.parent.config._asdict() config["strategy"] = self.parent.config.strategy config["n_device"] = self.parent.config.n_device self.tokenizer = self.tokenizer_class.from_pretrained( config["output_dir"], do_lower_case=config["do_lower_case"] ) eval_batch_size = ( config["per_device_eval_batch_size"] * config["n_device"] ) data_df = await self._preprocess_data(sources) eval_dataset, num_eval_examples = self.get_dataset( data_df, self.tokenizer, self.pad_token_label_id, eval_batch_size, mode="accuracy", ) eval_dataset = self.parent.config.strategy.experimental_distribute_dataset( eval_dataset ) checkpoints = [] results = [] if config["eval_all_checkpoints"]: checkpoints = list( os.path.dirname(c) for c in sorted( pathlib( config["output_dir"] + "/**/" + TF2_WEIGHTS_NAME ).glob(recursive=True), key=lambda f: int("".join(filter(str.isdigit, f)) or -1), ) ) if len(checkpoints) == 0: checkpoints.append(config["output_dir"]) self.logger.info("Evaluate the following checkpoints: %s", checkpoints) for checkpoint in checkpoints: global_step = ( checkpoint.split("-")[-1] if re.match(".*checkpoint-[0-9]", checkpoint) else "final" ) with self.parent.config.strategy.scope(): self.model = self.model_class.from_pretrained(checkpoint) y_true, y_pred, eval_loss = self._custom_accuracy( eval_dataset, self.tokenizer, self.model, num_eval_examples, eval_batch_size, ) report = classification_report(y_true, y_pred, digits=4) if global_step: results.append( { global_step + "_report": report, global_step + "_loss": eval_loss, } ) output_eval_file = os.path.join( config["output_dir"], "accuracy_results.txt" ) # create the report and save in output_dir with self.tf.io.gfile.GFile(output_eval_file, "w") as writer: for res in results: for key, val in res.items(): if "loss" in key: self.logger.debug(key + " = " + str(val)) writer.write(key + " = " + str(val)) writer.write("\n") else: self.logger.debug(key) self.logger.debug("\n" + report) writer.write(key + "\n") writer.write(report) writer.write("\n") # Return accuracy for the last checkpoint return Accuracy(f1_score(y_true, y_pred))
async def accuracy(self, sources: Sources) -> Accuracy: accuracy: int = 0 async for repo in sources.repos(): accuracy += int(repo.key) return Accuracy(accuracy)
async def accuracy(self, sources: Sources) -> Accuracy: if self.regression_line is None: raise ModelNotTrained("Train model before assessing for accuracy.") accuracy_value = self.regression_line[2] return Accuracy(accuracy_value)
def test_str(self): self.assertEqual(str(Accuracy(0.04242)), "4.24")