def test_update_new_result_existing_dataset(self):
new_result = metadata_eval_result(
model_pretty_name="RoBERTa fine-tuned on ReactionGIF",
task_pretty_name="Text Classification",
task_id="text-classification",
metrics_pretty_name="Recall",
metrics_id="recall",
metrics_value=0.7762102282047272,
dataset_pretty_name="ReactionGIF",
dataset_id="julien-c/reactiongif",
)
metadata_update(f"{USER}/{self.REPO_NAME}",
new_result,
token=self._token,
overwrite=False)
expected_metadata = copy.deepcopy(self.existing_metadata)
expected_metadata["model-index"][0]["results"][0]["metrics"].append(
new_result["model-index"][0]["results"][0]["metrics"][0])
self.repo.git_pull()
updated_metadata = metadata_load(self.repo_path / self.REPO_NAME /
"README.md")
self.assertDictEqual(updated_metadata, expected_metadata)
def test_update_dataset_name(self):
new_datasets_data = {"datasets": "['test/test_dataset']"}
metadata_update(f"{USER}/{self.REPO_NAME}",
new_datasets_data,
token=self._token)
self.repo.git_pull()
updated_metadata = metadata_load(self.repo_path / self.REPO_NAME /
"README.md")
expected_metadata = copy.deepcopy(self.existing_metadata)
expected_metadata.update(new_datasets_data)
self.assertDictEqual(updated_metadata, expected_metadata)
def test_update_existing_result_with_overwrite(self):
new_metadata = copy.deepcopy(self.existing_metadata)
new_metadata["model-index"][0]["results"][0]["metrics"][0][
"value"] = 0.2862102282047272
metadata_update(f"{USER}/{self.REPO_NAME}",
new_metadata,
token=self._token,
overwrite=True)
self.repo.git_pull()
updated_metadata = metadata_load(self.repo_path / self.REPO_NAME /
"README.md")
self.assertDictEqual(updated_metadata, new_metadata)
def test_update_existing_result_without_overwrite(self):
new_metadata = copy.deepcopy(self.existing_metadata)
new_metadata["model-index"][0]["results"][0]["metrics"][0][
"value"] = 0.2862102282047272
with pytest.raises(
ValueError,
match=
("You passed a new value for the existing metric 'name: Accuracy, type:"
" accuracy'. Set `overwrite=True` to overwrite existing metrics."
),
):
metadata_update(
f"{USER}/{self.REPO_NAME}",
new_metadata,
token=self._token,
overwrite=False,
)
def test_update_existing_field_without_overwrite(self):
new_datasets_data = {"datasets": "['test/test_dataset']"}
metadata_update(f"{USER}/{self.REPO_NAME}",
new_datasets_data,
token=self._token)
with pytest.raises(
ValueError,
match=
("You passed a new value for the existing meta data field 'datasets'."
" Set `overwrite=True` to overwrite existing metadata."),
):
new_datasets_data = {"datasets": "['test/test_dataset_2']"}
metadata_update(
f"{USER}/{self.REPO_NAME}",
new_datasets_data,
token=self._token,
overwrite=False,
)