def test_token_workflow(self):
"""
Test the whole token save/get/delete workflow,
with the desired behavior with respect to non-existent tokens.
"""
token = "token-{}".format(int(time.time()))
HfFolder.save_token(token)
self.assertEqual(HfFolder.get_token(), token)
HfFolder.delete_token()
HfFolder.delete_token()
# ^^ not an error, we test that the
# second call does not fail.
self.assertEqual(HfFolder.get_token(), None)
def run(self):
print(
ANSI.red(
"WARNING! Managing repositories through transformers-cli is deprecated. "
"Please use `huggingface-cli` instead."))
token = HfFolder.get_token()
if token is None:
print("Not logged in")
exit(1)
try:
objs = self._api.list_repos_objs(
token, organization=self.args.organization)
except HTTPError as e:
print(e)
print(ANSI.red(e.response.text))
exit(1)
if len(objs) == 0:
print("No shared file yet")
exit()
rows = [[obj.filename, obj.lastModified, obj.commit, obj.size]
for obj in objs]
print(
tabulate(
rows,
headers=["Filename", "LastModified", "Commit-Sha", "Size"]))
def run(self):
token = HfFolder.get_token()
if token is None:
print("Not logged in")
exit()
HfFolder.delete_token()
self._api.logout(token)
print("Successfully logged out.")
def test_token_workflow(self):
"""
Test the whole token save/get/delete workflow,
with the desired behavior with respect to non-existent tokens.
"""
token = "token-{}".format(int(time.time()))
HfFolder.save_token(token)
self.assertEqual(HfFolder.get_token(), token)
HfFolder.delete_token()
HfFolder.delete_token()
# ^^ not an error, we test that the
# second call does not fail.
self.assertEqual(HfFolder.get_token(), None)
# test TOKEN in env
self.assertEqual(HfFolder.get_token(), None)
with unittest.mock.patch.dict(os.environ,
{"HUGGING_FACE_HUB_TOKEN": token}):
self.assertEqual(HfFolder.get_token(), token)
def run(self):
print(
ANSI.red(
"WARNING! Managing repositories through transformers-cli is deprecated. "
"Please use `huggingface-cli` instead."))
token = HfFolder.get_token()
if token is None:
print("Not logged in")
exit(1)
try:
stdout = subprocess.check_output(["git",
"--version"]).decode("utf-8")
print(ANSI.gray(stdout.strip()))
except FileNotFoundError:
print("Looks like you do not have git installed, please install.")
try:
stdout = subprocess.check_output(["git-lfs",
"--version"]).decode("utf-8")
print(ANSI.gray(stdout.strip()))
except FileNotFoundError:
print(
ANSI.red(
"Looks like you do not have git-lfs installed, please install."
" You can install from https://git-lfs.github.com/."
" Then run `git lfs install` (you only have to do this once)."
))
print("")
user, _ = whoami(token)
namespace = self.args.organization if self.args.organization is not None else user
full_name = f"{namespace}/{self.args.name}"
print(f"You are about to create {ANSI.bold(full_name)}")
if not self.args.yes:
choice = input("Proceed? [Y/n] ").lower()
if not (choice == "" or choice == "y" or choice == "yes"):
print("Abort")
exit()
try:
url = create_repo(token,
name=self.args.name,
organization=self.args.organization)
except HTTPError as e:
print(e)
print(ANSI.red(e.response.text))
exit(1)
print("\nYour repo now lives at:")
print(f" {ANSI.bold(url)}")
print(
"\nYou can clone it locally with the command below, and commit/push as usual."
)
print(f"\n git clone {url}")
print("")
def run(self):
print(
ANSI.red(
"WARNING! `transformers-cli logout` is deprecated and will be removed in v5. Please use "
"`huggingface-cli logout` instead."))
token = HfFolder.get_token()
if token is None:
print("Not logged in")
exit()
HfFolder.delete_token()
self._api.logout(token)
print("Successfully logged out.")
def run(self):
token = HfFolder.get_token()
if token is None:
print("Not logged in")
exit()
HfFolder.delete_token()
HfApi.unset_access_token()
try:
self._api.logout(token)
except HTTPError as e:
# Logging out with an access token will return a client error.
if not e.response.status_code == 400:
raise e
print("Successfully logged out.")
def run(self):
token = HfFolder.get_token()
if token is None:
print("Not logged in")
exit()
try:
user, orgs = self._api.whoami(token)
print(user)
if orgs:
print(ANSI.bold("orgs: "), ",".join(orgs))
except HTTPError as e:
print(e)
print(ANSI.red(e.response.text))
exit(1)
def run(self):
token = HfFolder.get_token()
if token is None:
print("Not logged in")
exit()
try:
info = self._api.whoami(token)
print(info["name"])
orgs = [org["name"] for org in info["orgs"]]
if orgs:
print(ANSI.bold("orgs: "), ",".join(orgs))
except HTTPError as e:
print(e)
print(ANSI.red(e.response.text))
exit(1)
def run(self):
print(
ANSI.red(
"WARNING! `transformers-cli whoami` is deprecated and will be removed in v5. Please use "
"`huggingface-cli whoami` instead."))
token = HfFolder.get_token()
if token is None:
print("Not logged in")
exit()
try:
user, orgs = self._api.whoami(token)
print(user)
if orgs:
print(ANSI.bold("orgs: "), ",".join(orgs))
except HTTPError as e:
print(e)
print(ANSI.red(e.response.text))
exit(1)
def run(self):
token = HfFolder.get_token()
if token is None:
print("Not logged in")
exit(1)
try:
objs = self._api.list_repos_objs(token, organization=self.args.organization)
except HTTPError as e:
print(e)
print(ANSI.red(e.response.text))
exit(1)
if len(objs) == 0:
print("No shared file yet")
exit()
rows = [[obj.filename, obj.lastModified, obj.commit, obj.size] for obj in objs]
print(
tabulate(rows, headers=["Filename", "LastModified", "Commit-Sha", "Size"])
)
def run(self):
print(
ANSI.red(
"WARNING! Managing repositories through transformers-cli is deprecated. "
"Please use `huggingface-cli` instead."))
token = HfFolder.get_token()
if token is None:
print("Not logged in")
exit(1)
try:
self._api.delete_obj(token,
filename=self.args.filename,
organization=self.args.organization)
except HTTPError as e:
print(e)
print(ANSI.red(e.response.text))
exit(1)
print("Done")
def _get_hf_hub_pretrained_model_info(cls,
model_name: str,
refresh_cache: bool = False
) -> (type, str):
"""
Resolve the HuggingFace Hub model pretrained information given a model name.
The model name must be of general syntax ``{source_repo}/{model_name}``.
Note:
The ``{source_repo}`` need not be ``nvidia``, it can be any public repository, even external to Nvidia.
This allows public, externally contributed models to be run freely using Nvidia NeMo.
Args:
model_name: Str name of the model. Must be the original name or an alias of the model, without any '/'.
refresh_cache: Bool, determines whether cache must be refreshed (model is re-downloaded).
Returns:
A tuple of details describing :
- The resolved class of the model. Since the source is external to NeMo, always default to using
the calling class. Depend on target class resolution by restore_from() for calling the correct class.
- The path to the NeMo model (.nemo file) in some cached directory (managed by HF Hub).
"""
# Resolve the model name without origin for filename
resolved_model_filename = model_name.split("/")[-1] + '.nemo'
# Check if api token exists, use if it does
is_token_available = HfFolder.get_token() is not None
# Try to load the model from the Huggingface Hub
path = hf_hub_download(
repo_id=model_name,
filename=resolved_model_filename,
library_name="nemo",
library_version=nemo.__version__,
force_download=refresh_cache,
use_auth_token=is_token_available,
)
# Cannot pre-resolve the specific class without double instantiation (first for config, second for model params)
# Default to current class, and perform basic class path resolution (handled via restore_from() + target class)
class_ = cls
return class_, path
def push_to_hub(
save_directory: Optional[str],
model_id: Optional[str] = None,
repo_url: Optional[str] = None,
commit_message: Optional[str] = "add model",
organization: Optional[str] = None,
private: bool = None,
) -> str:
"""
Parameters:
save_directory (:obj:`Union[str, os.PathLike]`):
Directory having model weights & config.
model_id (:obj:`str`, `optional`, defaults to :obj:`save_directory`):
Repo name in huggingface_hub. If not specified, repo name will be same as `save_directory`
repo_url (:obj:`str`, `optional`):
Specify this in case you want to push to existing repo in hub.
organization (:obj:`str`, `optional`):
Organization in which you want to push your model.
private (:obj:`bool`, `optional`):
private: Whether the model repo should be private (requires a paid huggingface.co account)
commit_message (:obj:`str`, `optional`, defaults to :obj:`add model`):
Message to commit while pushing
Returns:
url to commit on remote repo.
"""
if model_id is None:
model_id = save_directory
token = HfFolder.get_token()
if repo_url is None:
repo_url = HfApi().create_repo(
token,
model_id,
organization=organization,
private=private,
repo_type=None,
exist_ok=True,
)
repo = Repository(save_directory, clone_from=repo_url, use_auth_token=token)
return repo.push_to_hub(commit_message=commit_message)
def run(self):
print(
ANSI.red(
"WARNING! Managing repositories through transformers-cli is deprecated. "
"Please use `huggingface-cli` instead."))
token = HfFolder.get_token()
if token is None:
print("Not logged in")
exit(1)
local_path = os.path.abspath(self.args.path)
if os.path.isdir(local_path):
if self.args.filename is not None:
raise ValueError(
"Cannot specify a filename override when uploading a folder."
)
rel_path = os.path.basename(local_path)
files = self.walk_dir(rel_path)
elif os.path.isfile(local_path):
filename = self.args.filename if self.args.filename is not None else os.path.basename(
local_path)
files = [(local_path, filename)]
else:
raise ValueError(f"Not a valid file or directory: {local_path}")
if sys.platform == "win32":
files = [(filepath, filename.replace(os.sep, "/"))
for filepath, filename in files]
if len(files) > UPLOAD_MAX_FILES:
print(
f"About to upload {ANSI.bold(len(files))} files to S3. This is probably wrong. Please filter files "
"before uploading.")
exit(1)
user, _ = self._api.whoami(token)
namespace = self.args.organization if self.args.organization is not None else user
for filepath, filename in files:
print(
f"About to upload file {ANSI.bold(filepath)} to S3 under filename {ANSI.bold(filename)} and namespace "
f"{ANSI.bold(namespace)}")
if not self.args.yes:
choice = input("Proceed? [Y/n] ").lower()
if not (choice == "" or choice == "y" or choice == "yes"):
print("Abort")
exit()
print(
ANSI.bold(
"Uploading... This might take a while if files are large"))
for filepath, filename in files:
try:
access_url = self._api.presign_and_upload(
token=token,
filename=filename,
filepath=filepath,
organization=self.args.organization)
except HTTPError as e:
print(e)
print(ANSI.red(e.response.text))
exit(1)
print("Your file now lives at:")
print(access_url)
def push_to_hub_fastai(
learner,
repo_id: str,
commit_message: Optional[str] = "Add model",
private: Optional[bool] = None,
token: Optional[str] = None,
config: Optional[dict] = None,
**kwargs,
):
"""
Upload learner checkpoint files to the Hub while synchronizing a local clone of the repo in
:obj:`repo_id`.
Args:
learner (`Learner`):
The `fastai.Learner' you'd like to push to the Hub.
repo_id (`str`):
The repository id for your model in Hub in the format of "namespace/repo_name". The namespace can be your individual account or an organization to which you have write access (for example, 'stanfordnlp/stanza-de').
commit_message (`str`, *optional*):
Message to commit while pushing. Will default to :obj:`"add model"`.
private (`bool`, *optional*):
Whether or not the repository created should be private.
token (`str`, *optional*):
The Hugging Face account token to use as HTTP bearer authorization for remote files. If :obj:`None`, the token will be asked by a prompt.
config (`dict`, *optional*):
Configuration object to be saved alongside the model weights.
Keyword Args:
api_endpoint (`str`, *optional*):
The API endpoint to use when pushing the model to the hub.
git_user (`str`, *optional*):
Will override the ``git config user.name`` for committing and pushing files to the hub.
git_email (`str`, *optional*):
Will override the ``git config user.email`` for committing and pushing files to the hub.
Returns:
The url of the commit of your model in the given repository.
<Tip>
Raises the following error:
- [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
if the user is not log on to the Hugging Face Hub.
</Tip>
"""
_check_fastai_fastcore_versions()
api_endpoint: str = kwargs.get("api_endpoint", None)
git_user: str = kwargs.get("git_user", None)
git_email: str = kwargs.get("git_email", None)
if token is None:
token = HfFolder.get_token()
if token is None:
raise ValueError(
"You must login to the Hugging Face Hub. There are two options: "
"(1) Type `huggingface-cli login` in your terminal and enter your token. "
"(2) Enter your token in the `token` argument. "
"Your token is available in the Settings of your Hugging Face account. "
)
# Create repo using `HfApi()`.
repo_url = HfApi(endpoint=api_endpoint).create_repo(
repo_id,
token=token,
private=private,
repo_type=None,
exist_ok=True,
)
# If repository exists in the Hugging Face Hub then clone it locally in `repo_id`.
repo = Repository(
repo_id,
clone_from=repo_url,
use_auth_token=token,
git_user=git_user,
git_email=git_email,
)
repo.git_pull(rebase=True)
_save_pretrained_fastai(learner, repo_id, config=config)
return repo.push_to_hub(commit_message=commit_message)
def run(self):
token = HfFolder.get_token()
if token is None:
print("Not logged in")
exit(1)
try:
stdout = subprocess.check_output(["git",
"--version"]).decode("utf-8")
print(ANSI.gray(stdout.strip()))
except FileNotFoundError:
print("Looks like you do not have git installed, please install.")
try:
stdout = subprocess.check_output(["git-lfs",
"--version"]).decode("utf-8")
print(ANSI.gray(stdout.strip()))
except FileNotFoundError:
print(
ANSI.red(
"Looks like you do not have git-lfs installed, please install."
" You can install from https://git-lfs.github.com/."
" Then run `git lfs install` (you only have to do this once)."
))
print("")
user = self._api.whoami(token)["name"]
namespace = (self.args.organization
if self.args.organization is not None else user)
repo_id = f"{namespace}/{self.args.name}"
if self.args.type not in REPO_TYPES:
print("Invalid repo --type")
exit(1)
if self.args.type in REPO_TYPES_URL_PREFIXES:
repo_id = REPO_TYPES_URL_PREFIXES[self.args.type] + repo_id
print(f"You are about to create {ANSI.bold(repo_id)}")
if not self.args.yes:
choice = input("Proceed? [Y/n] ").lower()
if not (choice == "" or choice == "y" or choice == "yes"):
print("Abort")
exit()
try:
url = self._api.create_repo(
self.args.name,
token=token,
organization=self.args.organization,
repo_type=self.args.type,
space_sdk=self.args.space_sdk,
)
except HTTPError as e:
print(e)
print(ANSI.red(e.response.text))
exit(1)
print("\nYour repo now lives at:")
print(f" {ANSI.bold(url)}")
print("\nYou can clone it locally with the command below,"
" and commit/push as usual.")
print(f"\n git clone {url}")
print("")