def test_upload_file(self, mock_requests_post):
with fake_creds('.jovian', 'credentials.json'):
with open('jovian/tests/resources/creds/.jovian/credentials.json',
'rb') as f:
upload_file(gist_slug='fake_gist_slug',
file=('credentials.json', f),
folder='.jovian',
artifact=True,
version_title='fake_version_title')
mock_requests_post.assert_called_with(
'https://api-staging.jovian.ai/gist/fake_gist_slug/upload',
data={
'artifact': 'true',
'folder': '.jovian',
'version_title': 'fake_version_title'
},
files={'files': ('credentials.json', ANY)},
headers={
"Authorization": "Bearer fake_api_key",
"x-jovian-source": "library",
"x-jovian-library-version": __version__,
"x-jovian-guest": "b6538d4dfde04fcf993463a828a9cec6",
"x-jovian-org": "staging"
},
json=None)
def test_upload_file(mock_requests_post):
with fake_creds() as dir:
with open(
os.path.join(dir, ".jovian/credentials.json"), "rb"
) as f:
upload_file(
gist_slug="fake_gist_slug",
file=("credentials.json", f),
folder=".jovian",
artifact=True,
version_title="fake_version_title",
)
mock_requests_post.assert_called_with(
"https://api-staging.jovian.ai/gist/fake_gist_slug/upload",
data={
"artifact": "true",
"folder": ".jovian",
"version_title": "fake_version_title",
},
files={"files": ("credentials.json", ANY)},
headers={
"Authorization": "Bearer fake_api_key",
"x-jovian-source": "library",
"x-jovian-library-version": __version__,
"x-jovian-guest": "b6538d4dfde04fcf993463a828a9cec6",
"x-jovian-org": "staging",
},
json=None,
)
def test_upload_file_has_warning(mock_post_request, capsys):
with fake_creds() as dir:
with open(os.path.join(dir, ".jovian/credentials.json"), "rb") as f:
upload_file(gist_slug="fake_gist_slug",
file=("credentials.json", f))
assert capsys.readouterr().err.strip(
) == '[jovian] Error: Uploaded gist has a warning'.strip()
def _attach_file(path, gist_slug, version, output=False):
"""Helper function to attach a single file to a commit"""
try:
with open(path, 'rb') as f:
file_obj = os.path.basename(path), f
folder = os.path.dirname(path)
api.upload_file(gist_slug, file_obj, folder, version, output)
except Exception as e:
log(str(e) + " (" + path + ")", error=True)
def test_upload_file_raises_api_error(mock_requests_post, mock_get_api_key):
with fake_creds() as dir:
# setUp
creds = {
"WEBAPP_URL": "https://staging.jovian.ml/",
"GUEST_KEY": "b6538d4dfde04fcf993463a828a9cec6",
"API_URL": "https://api-staging.jovian.ai",
"API_KEY": "fake_invalid_api_key",
"ORG_ID": "staging",
}
write_creds(creds)
with pytest.raises(ApiError) as context:
with open(
os.path.join(dir, ".jovian/credentials.json"), "rb"
) as f:
upload_file(
gist_slug="fake_gist_slug",
file=("credentials.json", f),
folder=".jovian",
artifact=True,
version_title="fake_version_title",
)
mock_requests_post.assert_called_with(
"https://api-staging.jovian.ai/gist/fake_gist_slug/upload",
data={
"artifact": "true",
"folder": ".jovian",
"version_title": "fake_version_title",
},
files={"files": ("credentials.json", ANY)},
headers={
"Authorization": "Bearer fake_invalid_api_key",
"x-jovian-source": "library",
"x-jovian-library-version": __version__,
"x-jovian-guest": "b6538d4dfde04fcf993463a828a9cec6",
"x-jovian-org": "staging",
},
json=None,
)
assert (
str(context.value)
== "File upload failed: (HTTP 404) Gist not found"
)
def test_upload_file_raises_api_error(self, mock_requests_post,
mock_get_api_key):
with fake_creds('.jovian', 'credentials.json'):
# setUp
creds = {
"WEBAPP_URL": "https://staging.jovian.ml/",
"GUEST_KEY": "b6538d4dfde04fcf993463a828a9cec6",
"API_URL": "https://api-staging.jovian.ai",
"API_KEY": "fake_invalid_api_key",
"ORG_ID": "staging"
}
write_creds(creds)
with self.assertRaises(ApiError) as context:
with open(
'jovian/tests/resources/creds/.jovian/credentials.json',
'rb') as f:
upload_file(gist_slug='fake_gist_slug',
file=('credentials.json', f),
folder='.jovian',
artifact=True,
version_title='fake_version_title')
mock_requests_post.assert_called_with(
'https://api-staging.jovian.ai/gist/fake_gist_slug/upload',
data={
'artifact': 'true',
'folder': '.jovian',
'version_title': 'fake_version_title'
},
files={'files': ('credentials.json', ANY)},
headers={
"Authorization": "Bearer fake_invalid_api_key",
"x-jovian-source": "library",
"x-jovian-library-version": __version__,
"x-jovian-guest": "b6538d4dfde04fcf993463a828a9cec6",
"x-jovian-org": "staging"
},
json=None)
assert context.exception.args[
0] == 'File upload failed: (HTTP 404) Gist not found'
def upload_conda_env(gist_slug):
"""Read and save the current Anaconda environment to server"""
# Export environment to YML string
env_str = read_conda_env(get_conda_env_name())
# Upload environment.yml
upload_file(gist_slug, ('environment.yml', env_str))
# Check and include existing os-specific files
platform = get_platform()
for p in PLATFORMS:
pfname = 'environment-' + p + '.yml'
if p == platform:
# Use the new environment for current platform
upload_file(gist_slug, (pfname, env_str))
elif os.path.exists(pfname):
# Reuse old environments for other platforms
upload_file(gist_slug, pfname)
def upload_pip_env(gist_slug, version=None):
"""Read and upload the current virtual environment to server"""
return upload_file(gist_slug=gist_slug,
file=('requirements.txt', read_pip_env()),
version=version)
def upload_pip_env(gist_slug):
"""Read and upload the current virtual environment to server"""
return upload_file(gist_slug, ('requirements.txt', read_pip_env()))
def commit(secret=False,
nb_filename=None,
files=[],
capture_env=True,
env_type='conda',
notebook_id=None,
create_new=None):
"""Save the notebook, capture environment and upload to the cloud for sharing.
In most cases, commit works well with the default arguments. It attempts to
1. Save the Jupyter notebook
2. Upload the notebook (and additional scripts, CSVs etc.) to https://jvn.io
3. Capture the python environment (using Anaconda or pip)
4. Upload the python environment to cloud
Capturing the python environment ensures that the notebook can be reproduced and
executed easily using the `jovian` command line tool. For more details, see
https://jvn.io/getting-started .
Issues and bugs can be reported here: https://github.com/swiftace-ai/jovian-py
Arguments:
secret (bool, optional): Create a secret notebook on https://jvn.io , which is only
accessible via the link, and is not visible on the owner's public profile. By default,
commited notebooks are public and visible on the owner's profile.
nb_filename (string, optional): The filename of the jupyter notebook (including
the .ipynb extension). This is detected automatically in most cases, but in
certain environments like Jupyter Lab, the detection may fail and the filename
needs to be provided using this argument.
files (array, optional): Any additional scripts (.py files), CSVs that are required to
run the notebook. These will be available in the files tab on https://jvn.io .
capture_env (bool, optional): If `True`, the Python environment (python version,
libraries etc.) are captured and uploaded along with the notebook.
env_type (string, optional): The type of environment to be captured. Allowed options are
'conda' and 'pip'.
notebook_id (string, optional): If you wish to update an existing notebook owned by you,
you can use this argument to provide the base64 ID (present in the URL) of an notebook
hosted on https://jvn.io . In most cases, this argument is not required, and the library
can automatically infer whether you are looking to update an existing notebook or create
a new one.
create_new (bool, optional): If set to True, doesn't update the existing notebook on
https://jvn.io (if one is detected). Instead, it creates a new notebook when commit is called.
"""
global _current_slug
# Check if we're in a Jupyter environment
if not in_notebook():
log('Failed to detect Juptyer notebook. Skipping..', error=True)
return
# Save the notebook (uses Javascript, doesn't work everywhere)
log('Saving notebook..')
save_notebook()
sleep(1)
# Get the filename of the notebook (if not provided)
if nb_filename is None:
nb_filename = get_notebook_name()
# Exit with help message if filename wasn't detected (or provided)
if nb_filename is None:
log(FILENAME_MSG)
return
# Check whether to create a new gist, or update an old one
if not create_new and notebook_id is None:
# First preference to the in-memory slug variable
if _current_slug is not None:
notebook_id = _current_slug
else:
notebook_id = get_notebook_slug(nb_filename)
# Check if the current user can push to this slug
if notebook_id is not None:
gist_meta = get_gist(notebook_id)
if not gist_meta['isOwner']:
notebook_id = None
# Log whether this is an update or creation
if notebook_id is None:
log('Creating a new notebook on https://jvn.io')
else:
log('Updating notebook "' + notebook_id + '" on https://jvn.io')
# Upload the notebook & create/update the gist
res = create_gist_simple(nb_filename, notebook_id, secret)
if res is None:
return
# Extract slug and owner from created gist
slug, owner = res['slug'], res['owner']
# Set/update the slug information
_current_slug = slug
set_notebook_slug(nb_filename, slug)
# Save & upload environment
if capture_env:
log('Capturing environment..')
if env_type == 'conda':
# Capture conda environment
try:
upload_conda_env(slug)
except CondaError as e:
log(str(e), error=True)
elif env_type == 'pip':
# Capture pip environment
try:
upload_pip_env(slug)
except Exception as e:
log(str(e), error=True)
# Upload additional files
if files and len(files) > 0:
log('Uploading additional files..')
# Upload each file
for fname in files:
if os.path.exists(fname) and not os.path.isdir(fname):
try:
upload_file(slug, fname)
except Exception as e:
log(str(e), error=True)
elif os.path.isdir(fname):
log('Ignoring directory "' + fname + '"', error=True)
else:
log('Ignoring "' + fname + '" (not found)', error=True)
# Print commit URL
log('Committed successfully! ' + WEBAPP_URL + "/" + owner['username'] +
"/" + slug)