def test_set_notebook_slug_existing_entry(self):
filename = "Testing Jovian.ipynb"
slug = "46bd9a3f87e74de0baf8a6f0b60a8df9"
expected_result = data
set_notebook_slug(filename, slug)
self.assertEqual(get_rcdata(), expected_result)
def clone(slug,
version=None,
fresh=True,
include_outputs=True,
overwrite=False):
"""Download the files for a gist"""
# Print issues link
log(ISSUES_MSG)
# Download gist metadata
ver_str = '(version ' + str(version) + ')' if version else ''
log('Fetching ' + slug + " " + ver_str + "..")
gist = get_gist(slug, version, fresh)
if not gist:
return
title = gist['title']
# If fresh clone, create directory
if fresh and not os.path.exists(title):
os.makedirs(title)
os.chdir(title)
elif fresh and os.path.exists(title) and overwrite:
os.chdir(title)
elif fresh and os.path.exists(title) and not overwrite:
i = 1
while os.path.exists(title + '-' + str(i)):
i += 1
title = title + '-' + str(i)
os.makedirs(title)
os.chdir(title)
# Download the files
log('Downloading files..')
for f in gist['files']:
if not f['artifact'] or include_outputs:
if f['filename'].endswith('.ipynb'):
content = _sanitize_notebook(get(f['rawUrl']).content)
else:
content = get(f['rawUrl']).content
if f['folder'] and not os.path.exists(f['folder']):
os.makedirs(f['folder'])
filepath = os.path.join(f['folder'] or '', f['filename'])
with open(filepath, 'wb') as fp:
fp.write(content)
# Create .jovianrc for a fresh clone
if fresh and f['filename'].endswith('.ipynb'):
set_notebook_slug(f['filename'], slug)
# Print success message and instructions
if fresh:
post_clone_msg(title)
else:
log('Files dowloaded successfully in current directory')
def set_project(ctx, notebook, project):
"""Associate notebook (filename.ipynb) to Jovian project (username/title)
$ jovian set-project my_notebook.ipynb danb/keras-example
This will create or update the .jovianrc file in the current directory to ensure that commits
inside the Jupyter notebook "my_notebook.ipynb" add new versions to the project danb/keras-example
"""
set_notebook_slug(filename=notebook, slug=project)
def test_set_notebook_slug():
with fake_rc():
filename = "Testing Jovian 2.ipynb"
slug = "46bd9a3f87e74de0baf8a6f0b60a8df9"
expected_result = {
"notebooks": {
"Testing Jovian.ipynb": {
"slug": "46bd9a3f87e74de0baf8a6f0b60a8df9"
},
"Testing Jovian 2.ipynb": {
"slug": "46bd9a3f87e74de0baf8a6f0b60a8df9"
}
}
}
set_notebook_slug(filename, slug)
assert get_rcdata() == expected_result
def test_set_notebook_slug_new_entry(self):
filename = "Testing Jovian 2.ipynb"
slug = "46bd9a3f87e74de0baf8a6f0b60a8df9"
expected_result = {
"notebooks": {
"Testing Jovian.ipynb": {
"slug": "46bd9a3f87e74de0baf8a6f0b60a8df9"
},
"Testing Jovian 2.ipynb": {
"slug": "46bd9a3f87e74de0baf8a6f0b60a8df9"
}
}
}
set_notebook_slug(filename, slug)
self.assertEqual(get_rcdata(), expected_result)
def clone(slug, fresh=True):
"""Download the files for a gist"""
# Print issues link
log(ISSUES_MSG)
# Download gist metadata
log('Fetching ' + slug + "..")
gist = get_gist(slug, fresh)
title = gist['title']
# If fresh clone, create directory
if fresh:
if os.path.exists(title):
i = 1
while os.path.exists(title + '-' + str(i)):
i += 1
title = title + '-' + str(i)
if not os.path.exists(title):
os.makedirs(title)
os.chdir(title)
# Download the files
log('Downloading files..')
for f in gist['files']:
with open(f['filename'], 'wb') as fp:
fp.write(get(f['rawUrl']).content)
# Create .jovianrc for a fresh clone
if fresh and f['filename'].endswith('.ipynb'):
set_notebook_slug(f['filename'], slug)
# Print success message and instructions
if fresh:
log(post_clone_msg(title))
else:
log('Files dowloaded successfully in current directory')
def commit(message=None,
files=[],
outputs=[],
environment='auto',
privacy='auto',
filename=None,
project=None,
new_project=None,
git_commit=False,
git_message='auto',
**kwargs):
"""Uploads the current file (Jupyter notebook or python script) to |Jovian|
Saves the checkpoint of the notebook, captures the required dependencies from
the python environment and uploads the notebook, env file, additional files like scripts, csv etc.
to |Jovian|. Capturing the python environment ensures that the notebook can be reproduced.
Args:
message(string, optional): A short message to be used as the title for this version.
files(array, optional): Any additional scripts(.py files), CSVs etc. that are required to
run the notebook. These will be available in the files tab of the project page on Jovian.ml
outputs(array, optional): Any outputs files or artifacts generated from the modeling processing.
This can include model weights/checkpoints, generated CSVs, output images etc.
environment(string, optional): The type of Python environment to be captured. Allowed options are
'conda' , 'pip', 'auto' (for automatic detection) and None (to skip environment capture).
privacy(bool, optional): Privacy level of the project (if a new one is being created).
* 'auto' - use account level settings. Defaults to 'public'
* 'public' - visible on profile and publicly accessible/searchable
* 'secret' - not on profile only accessible via the direct link
* 'private' - only for the accessible to owner and collaborators
This argument has no effect on existing project. Change the privacy settings of a existing notebook
on the webapp.
filename(string, optional): The filename of the current Jupyter notebook or Python script. This is
detected automatically in most cases, but in certain environments like Jupyter Lab or password protected notebooks, the detection
may fail and the filename needs to be provided using this argument.
project(string, optional): Name of the |Jovian| project to which the current notebook/file should
be committed. Format: 'username/title' e.g. 'aakashns/jovian-example' or 'jovian-example'
(username of current user inferred automatically). If the project does not exist, a new one is
created. If it exists, the current notebook is added as a new version to the existing project, if
you are a owner/collaborator. If left empty, project name is picked up from the `.jovianrc` file in the
current directory, or a new project is created using the filename as the project name.
new_project(bool, optional): Whether to create a new project or update the existing one. Allowed option
are False (use the existing project, if a .jovianrc file exists, if available), True (create a new project)
git_commit(bool, optional): If True, also performs a Git commit and records the commit hash. This is
applicable only when the notebook is inside a Git repository.
git_message(string, optional): Commit message for git. If not provided, it uses the `message` argument
.. attention::
Pass notebook's name to `filename` argument, in certain environments like Jupyter Lab and password protected
notebooks sometimes it may fail to detect notebook automatically.
.. |Jovian| raw:: html
<a href="https://jovian.ml/?utm_source=docs" target="_blank"> Jovian.ml </a>
"""
global _current_slug
# Deprecated argument (secret)
if privacy == 'auto' and 'secret' in kwargs:
privacy = 'secret' if kwargs['secret'] else 'auto'
log('"secret" is deprecated. Use "privacy" instead (allowed options: ' +
'"public", "private", "secret", "auto")', error=True)
# Deprecated argument (nb_filename)
if filename is None and 'nb_filename' in kwargs:
filename = kwargs['nb_filename']
log('"nb_filename" is deprecated. Use "filename" instead', error=True)
# Deprecated argument (env_type)
if 'env_type' in kwargs:
environment = kwargs['environment']
log('"env_type" is deprecated. Use "environment" instead', error=True)
# Deprecated argument (capture_env)
if 'capture_env' in kwargs and not kwargs['capture_env']:
environment = None
log('"catpure_env" is deprecated. Use "environment=None" instead', error=True)
# Deprecated argument (notebook_id)
if 'notebook_id' in kwargs:
project = kwargs['notebook_id']
log('"notebook_id" is deprecated. Use "project" instead.', error=True)
# Deprecated argument (create_new)
if 'create_new' in kwargs:
new_project = kwargs['create_new']
log('"create_new" is deprecated. Use "new_project" instead.', error=True)
# Deprecated argument (artifacts)
if 'artifacts' in kwargs:
outputs = kwargs['artifacts']
log('"artifacts" is deprecated. Use "outputs" instead', error=True)
# Skip if unsupported environment
if not in_script() and not in_notebook():
log('Failed to detect Jupyter notebook or Python script. Skipping..', error=True)
return
# Attempt to save Jupyter notebook
if in_notebook():
save_notebook()
log('Attempting to save notebook..')
sleep(1)
# Extract notebook/script filename
filename = _parse_filename(filename)
if filename is None:
log(FILENAME_MSG)
return
# Ensure that the file exists
if not os.path.exists(filename):
log('The detected/provided file "' + filename +
'" does not exist. Please provide the correct notebook filename ' +
'as the "filename" argument to "jovian.commit".')
return
# Retrieve Gist ID & title
project_title, project_id = _parse_project(project, filename, new_project)
# Create or update gist (with title and )
res = api.create_gist_simple(filename, project_id, privacy, project_title, message)
slug, owner, version, title = res['slug'], res['owner'], res['version'], res['title']
username = owner['username']
# Cache slug for further commits
_current_slug = slug
set_notebook_slug(filename, slug)
# Attach environment, files and outputs
_capture_environment(environment, slug, version)
_attach_files(files, slug, version)
_attach_files(outputs, slug, version, output=True)
if not git_message or git_message == 'auto':
git_message = message or 'jovian commit ' + username + '/' + title + ' v' + str(version)
_perform_git_commit(filename, git_commit, git_message)
_attach_records(slug, version)
log('Committed successfully! ' + read_webapp_url() + username + "/" + title)
def commit(secret=False,
nb_filename=None,
files=[],
capture_env=True,
env_type='conda',
notebook_id=None,
create_new=None):
"""Save the notebook, capture environment and upload to the cloud for sharing.
In most cases, commit works well with the default arguments. It attempts to
1. Save the Jupyter notebook
2. Upload the notebook (and additional scripts, CSVs etc.) to https://jvn.io
3. Capture the python environment (using Anaconda or pip)
4. Upload the python environment to cloud
Capturing the python environment ensures that the notebook can be reproduced and
executed easily using the `jovian` command line tool. For more details, see
https://jvn.io/getting-started .
Issues and bugs can be reported here: https://github.com/swiftace-ai/jovian-py
Arguments:
secret (bool, optional): Create a secret notebook on https://jvn.io , which is only
accessible via the link, and is not visible on the owner's public profile. By default,
commited notebooks are public and visible on the owner's profile.
nb_filename (string, optional): The filename of the jupyter notebook (including
the .ipynb extension). This is detected automatically in most cases, but in
certain environments like Jupyter Lab, the detection may fail and the filename
needs to be provided using this argument.
files (array, optional): Any additional scripts (.py files), CSVs that are required to
run the notebook. These will be available in the files tab on https://jvn.io .
capture_env (bool, optional): If `True`, the Python environment (python version,
libraries etc.) are captured and uploaded along with the notebook.
env_type (string, optional): The type of environment to be captured. Allowed options are
'conda' and 'pip'.
notebook_id (string, optional): If you wish to update an existing notebook owned by you,
you can use this argument to provide the base64 ID (present in the URL) of an notebook
hosted on https://jvn.io . In most cases, this argument is not required, and the library
can automatically infer whether you are looking to update an existing notebook or create
a new one.
create_new (bool, optional): If set to True, doesn't update the existing notebook on
https://jvn.io (if one is detected). Instead, it creates a new notebook when commit is called.
"""
global _current_slug
# Check if we're in a Jupyter environment
if not in_notebook():
log('Failed to detect Juptyer notebook. Skipping..', error=True)
return
# Save the notebook (uses Javascript, doesn't work everywhere)
log('Saving notebook..')
save_notebook()
sleep(1)
# Get the filename of the notebook (if not provided)
if nb_filename is None:
nb_filename = get_notebook_name()
# Exit with help message if filename wasn't detected (or provided)
if nb_filename is None:
log(FILENAME_MSG)
return
# Check whether to create a new gist, or update an old one
if not create_new and notebook_id is None:
# First preference to the in-memory slug variable
if _current_slug is not None:
notebook_id = _current_slug
else:
notebook_id = get_notebook_slug(nb_filename)
# Check if the current user can push to this slug
if notebook_id is not None:
gist_meta = get_gist(notebook_id)
if not gist_meta['isOwner']:
notebook_id = None
# Log whether this is an update or creation
if notebook_id is None:
log('Creating a new notebook on https://jvn.io')
else:
log('Updating notebook "' + notebook_id + '" on https://jvn.io')
# Upload the notebook & create/update the gist
res = create_gist_simple(nb_filename, notebook_id, secret)
if res is None:
return
# Extract slug and owner from created gist
slug, owner = res['slug'], res['owner']
# Set/update the slug information
_current_slug = slug
set_notebook_slug(nb_filename, slug)
# Save & upload environment
if capture_env:
log('Capturing environment..')
if env_type == 'conda':
# Capture conda environment
try:
upload_conda_env(slug)
except CondaError as e:
log(str(e), error=True)
elif env_type == 'pip':
# Capture pip environment
try:
upload_pip_env(slug)
except Exception as e:
log(str(e), error=True)
# Upload additional files
if files and len(files) > 0:
log('Uploading additional files..')
# Upload each file
for fname in files:
if os.path.exists(fname) and not os.path.isdir(fname):
try:
upload_file(slug, fname)
except Exception as e:
log(str(e), error=True)
elif os.path.isdir(fname):
log('Ignoring directory "' + fname + '"', error=True)
else:
log('Ignoring "' + fname + '" (not found)', error=True)
# Print commit URL
log('Committed successfully! ' + WEBAPP_URL + "/" + owner['username'] +
"/" + slug)