def perform_colab_commit(project, privacy):
if '/' not in project:
project = get_current_user()['username'] + '/' + project
data = {
'project': project,
'file_id': get_colab_file_id(),
'visibility': privacy
}
if privacy == 'auto':
data['public'] = True
elif privacy == 'secret' or privacy == 'private':
data['public'] = False
auth_headers = _h()
log("Uploading colab notebook to Jovian...")
res = post(url=_u('/gist/colab-commit'), data=data, headers=auth_headers)
if res.status_code == 200:
data, warning = parse_success_response(res)
if warning:
log(warning, error=True)
return data
raise ApiError('Colab commit failed: ' + pretty(res))
def perform_colab_commit(project, privacy):
# file_id, project, privacy api key
file_id = get_colab_file_id()
if file_id is None:
log("Colab File Id is not provided", error=True)
# /gist/colab-commit data = {file_id, project}, return status
if '/' not in project:
project = get_current_user()['username'] + '/' + project
data = {'project': project, 'file_id': file_id, 'visibility': privacy}
if privacy == 'auto':
data['public'] = True
elif privacy == 'secret' or privacy == 'private':
data['public'] = False
auth_headers = _h()
log("Uploading colab notebook to Jovian...")
res = post(url=_u('/gist/colab-commit'),
data=data,
headers=auth_headers)
if res.status_code == 200:
return res.json()['data']
raise ApiError('Colab commit failed: ' + pretty(res))
def create_gist_simple(filename=None, gist_slug=None, privacy='auto', title=None, version_title=None):
"""Upload the current notebook to create/update a gist"""
auth_headers = _h()
with open(filename, 'rb') as f:
nb_file = (filename, f)
log('Uploading notebook..')
if gist_slug:
return upload_file(gist_slug=gist_slug, file=nb_file, version_title=version_title)
else:
data = {'visibility': privacy}
# For compatibility with old version of API endpoint
if privacy == 'auto':
data['public'] = True
elif privacy == 'secret' or privacy == 'private':
data['public'] = False
if title:
data['title'] = title
if version_title:
data['version_title'] = version_title
res = post(url=_u('/gist/create'),
data=data,
files={'files': nb_file},
headers=auth_headers)
if res.status_code == 200:
return res.json()['data']
raise ApiError('File upload failed: ' + pretty(res))
def notify(data, verbose=True, safe=False):
"""Sends the data to the `Slack`_ workspace connected with your `Jovian`_ account.
Args:
data(dict|string): A dict or string to be pushed to Slack
verbose(bool, optional): By default it prints the acknowledgement, you can remove this by setting the argument to False.
safe(bool, optional): To avoid raising ApiError exception. Defaults to False.
Example
.. code-block::
import jovian
data = "Hello from the Integration!"
jovian.notify(data)
.. important::
This feature requires for your Jovian account to be connected to a Slack workspace, visit `Jovian Integrations`_ to integrate them and to control the type of notifications.
.. _Slack: https://slack.com
.. _Jovian: https://jovian.ml?utm_source=docs
.. _Jovian Integrations: https://jovian.ml/settings/integrations?utm_source=docs
"""
res = post_slack_message(data=data, safe=safe)
if verbose:
if not res.get('errors'):
log('message_sent:' + str(res.get('data').get('messageSent')))
else:
log(str(res.get('errors')[0].get('message')), error=True)
def request_org_id():
"""Ask the user to provide the organization ID"""
log("If you're a jovian-pro user please enter your company's organization ID on Jovian (otherwise leave it blank)."
)
msg = "Organization ID"
return click.prompt(msg, default='', show_default=False)
def exec_commit(ctx, notebook):
"""Create a new notebook on Jovian
$ jovian commit my_notebook.ipynb
"""
if is_py2():
log("Committing is not supported for Python 2.x. Please install and run Jovian from Python 3.6 and above.",
warn=True)
commit_path(path=notebook, environment=None, is_cli=True)
def _attach_file(path, gist_slug, version, output=False):
"""Helper function to attach a single file to a commit"""
try:
with open(path, 'rb') as f:
file_obj = os.path.basename(path), f
folder = os.path.dirname(path)
api.upload_file(gist_slug, file_obj, folder, version, output)
except Exception as e:
log(str(e) + " (" + path + ")", error=True)
def get_api_key():
"""Retrieve and validate the API Key (from memory, config or user input)"""
if API_KEY not in CREDS:
key, source = read_or_request_api_key()
if not validate_api_key(key):
log('The current API key is invalid or expired.', error=True)
key, source = request_api_key(), 'request'
if not validate_api_key(key):
raise ApiError('The API key provided is invalid or expired.')
write_api_key(key)
return key
return CREDS[API_KEY]
def print_conda_message(env_name):
if env_name:
message = ("""
#
# To activate this environment, use
#
# $ conda activate %s
#
# To deactivate an active environment, use
#
# $ conda deactivate
""") % env_name
log(message)
def configure():
"""Configure Jovian for first time usage"""
# Check if already exists
if creds_exist():
log('It looks like Jovian is already configured ( check ~/.jovian/credentials.json ).')
msg = 'Do you want to overwrite the existing configuration?'
confirm = click.confirm(msg)
if confirm:
log('Removing existing configuration..')
else:
log('Skipping..')
return
# Remove existing credentials
purge_creds()
# Capture and save organization ID
ensure_org(check_pro=False)
# Ask for API Key
get_guest_key()
get_api_key()
log('Configuration complete!')
def _request_wrapper(*args, **kwargs):
for i in range(2):
res = request(*args, **kwargs)
if res.status_code == 401:
log('The current API key is invalid or expired.', error=True)
purge_api_key()
# This will ensure that fresh api token is requested
if 'headers' in kwargs:
kwargs['headers'][
'Authorization'] = "Bearer " + get_api_key()
else:
return res
return res
def install(env_fname=None, env_name=None):
"""Install packages for a cloned gist"""
# Check for conda and get the binary path
conda_bin = get_conda_bin()
# Identify the right environment file, and exit if absent
env_fname = identify_env_file(env_fname=env_fname)
if env_fname is None:
log('Failed to detect a conda environment YML file. Skipping..', error=True)
return
else:
log('Detected conda environment file: ' + env_fname + "\n")
# Get the environment name from user input
env_name = request_env_name(env_name=env_name, env_fname=env_fname)
if env_name is None:
log('Environment name not provided/detected. Skipping..')
return
# Construct the command
command = conda_bin + ' env update --file "' + \
env_fname + '" --name "' + env_name + '"'
packages = extract_env_packages(env_fname=env_fname)
if len(packages) > 0:
success = run_command(command=command, env_fname=env_fname, packages=packages, run=1)
if not success:
log('Some pip packages failed to install.')
print_conda_message(env_name=env_name)
def run_command(command, env_fname, packages, run=1):
# Run the command
if run > 3:
return
log('Executing:\n' + command + "\n")
install_task = subprocess.Popen(command, shell=True, stderr=subprocess.PIPE)
# Extract the error (if any)
_, error_string = install_task.communicate()
error_string = error_string.decode('utf8', errors='ignore')
if error_string:
print(error_string, file=stderr)
# Check for errors
error, pkgs = check_error(error_string, packages=packages)
pip_failed = check_pip_failed(error_string)
if error:
log('Installation failed!', error=True)
log('Ignoring ' + error + ' dependencies and trying again...\n')
sleep(1)
sanitize_envfile(env_fname=env_fname, pkgs=pkgs)
return run_command(command=command, env_fname=env_fname, packages=packages, run=run+1)
elif pip_failed:
# TODO: Extract env details and run pip sub-command.
# pip_packages = extract_pip_packages(env_fname=env_fname)
return False
else:
# Print beta warning and github link
log(ISSUES_MSG)
return True
def log_record(record_type, data=None, verbose=True, **data_args):
"""Create records with the given data & type"""
global _data_blocks
# Create the combined data dictionary
data = _parse_data(data, data_args)
if data is None and verbose:
log('Nothing to record. Skipping..', error=True)
return
# Send to API endpoint
res = api.post_block(data, record_type)
tracking_slug = res['tracking']['trackingSlug']
# Save to data block
_data_blocks.append((tracking_slug, record_type, data))
if verbose:
log(record_type.capitalize() + ' logged.')
def _perform_git_commit(filename, git_commit, git_message):
if git_commit and git.is_git():
reset('git') # resets git commit info
git.commit(git_message)
log('Git repository identified. Performing git commit...')
git_info = {
'repository': git.get_remote(),
'commit': git.get_current_commit(),
'filename': filename,
'path': git.get_relative_path(),
'branch': git.get_branch()
}
log_git(git_info, verbose=False)
def add_slack():
"""prints instructions for connecting Slack, if Slack connection is not already present.
if Slack is already connected, prints details about the workspace and the channel"""
url = _u('/slack/integration_details')
res = get(url, headers=_h())
if res.status_code == 200:
res = res.json()
if not res.get('errors'):
slack_account = res.get('data').get('slackAccount')
log('Slack already connected. \nWorkspace: {}\nConnected Channel: {}'
.format(slack_account.get('workspace'), slack_account.get('channel')))
else:
log(str(res.get('errors')[0].get('message')))
else:
raise ApiError('Slack trigger failed: ' + pretty(res))
def _attach_files(paths, gist_slug, version, output=False, exclude_files=None):
"""Helper functions to attach files & folders to a commit"""
config = read_creds().get("DEFAULT_CONFIG", {})
whitelist = config.get("EXTENSION_WHITELIST")
upload_wd = config.get("UPLOAD_WORKING_DIRECTORY", False)
if not isinstance(whitelist, list):
whitelist = DEFAULT_EXTENSION_WHITELIST
if not paths:
if output or not upload_wd:
return
paths = [
f for f in glob.glob('**/*', recursive=True)
if get_file_extension(f) in whitelist
]
if exclude_files:
if not isinstance(exclude_files, list):
exclude_files = [exclude_files]
for filename in exclude_files:
try:
paths.remove(filename)
except ValueError:
pass
log('Uploading additional ' + ('outputs' if output else 'files') + '...')
# Convert single path to list
if type(paths) == str:
paths = [paths]
for path in paths:
if os.path.isdir(path):
files = [
f
for f in glob.glob(os.path.join(path, '**/*'), recursive=True)
if get_file_extension(f) in whitelist
]
for file in files:
_attach_file(file, gist_slug, version, output)
elif os.path.exists(path):
_attach_file(path, gist_slug, version, output)
else:
log('Ignoring "' + path + '" (not found)', error=True)
def clone(slug,
version=None,
fresh=True,
include_outputs=True,
overwrite=False):
"""Download the files for a gist"""
# Print issues link
log(ISSUES_MSG)
# Download gist metadata
ver_str = '(version ' + str(version) + ')' if version else ''
log('Fetching ' + slug + " " + ver_str + "..")
gist = get_gist(slug, version, fresh)
if not gist:
return
title = gist['title']
# If fresh clone, create directory
if fresh and not os.path.exists(title):
os.makedirs(title)
os.chdir(title)
elif fresh and os.path.exists(title) and overwrite:
os.chdir(title)
elif fresh and os.path.exists(title) and not overwrite:
i = 1
while os.path.exists(title + '-' + str(i)):
i += 1
title = title + '-' + str(i)
os.makedirs(title)
os.chdir(title)
# Download the files
log('Downloading files..')
for f in gist['files']:
if not f['artifact'] or include_outputs:
if f['filename'].endswith('.ipynb'):
content = _sanitize_notebook(get(f['rawUrl']).content)
else:
content = get(f['rawUrl']).content
if f['folder'] and not os.path.exists(f['folder']):
os.makedirs(f['folder'])
filepath = os.path.join(f['folder'] or '', f['filename'])
with open(filepath, 'wb') as fp:
fp.write(content)
# Create .jovianrc for a fresh clone
if fresh and f['filename'].endswith('.ipynb'):
set_notebook_slug(f['filename'], slug)
# Print success message and instructions
if fresh:
post_clone_msg(title)
else:
log('Files dowloaded successfully in current directory')
def create_gist_simple(filename=None, gist_slug=None, secret=False):
"""Upload the current notebook to create a gist"""
auth_headers = _h()
nb_file = (filename, open(filename, 'rb'))
log('Uploading notebook..')
if gist_slug:
return upload_file(gist_slug, nb_file)
else:
res = post(url=_u('/gist/create'),
data={'public': 0 if secret else 1},
files={'files': nb_file},
headers=auth_headers)
if res.status_code == 200:
return res.json()['data']
raise ApiError('File upload failed: ' + _pretty(res))
def pull(slug=None):
"""Get the latest files associated with the current gist"""
# If a slug is provided, just use that
if slug:
clone(slug, fresh=False)
return
# Check if .jovianrc exists
if not rcfile_exists():
log(RCFILE_NOTFOUND, error=True)
return
# Get list of notebooks
nbs = get_rcdata()['notebooks']
for fname in nbs:
# Get the latest files for each notebook
clone(nbs[fname]['slug'], fresh=False)
def get_gist(slug, version, fresh):
"""Download a gist"""
if '/' in slug:
parts = slug.split('/')
username, title = parts[0], parts[1]
url = _u('user/' + username + '/gist/' + title + _v(version))
else:
url = _u('gist/' + slug + _v(version))
res = get(url, headers=_h(fresh))
if res.status_code == 200:
return res.json()['data']
elif res.status_code == 401:
log('This notebook does not exist or is private. Please provide the API key')
get_api_key()
return get_gist(slug, version, fresh)
else:
log('Failed to retrieve notebook: ' + pretty(res), error=True)
def check_error(error_str, packages=None):
"""Check if the error output contains ResolvePackageNotFound or UnsatisfiableError"""
if not packages:
packages = []
error_lines = error_str.split('\n')
error = None
pkgs = []
for line in error_lines:
if 'ResolvePackageNotFound:' in line:
error = 'unresolved'
elif 'UnsatisfiableError:' in line:
error = 'unsatisfiable'
log(MISSING_MSG)
if error:
pkg = extract_package_from_line(line, packages)
if pkg and pkg not in pkgs:
pkgs.append(pkg)
return error, pkgs
def _capture_environment(environment, gist_slug, version):
"""Capture the python environment and attach it to the commit"""
if environment is not None:
# Check credentials if environment config exists
creds = read_creds()
if 'DEFAULT_CONFIG' in creds and 'environment' in creds['DEFAULT_CONFIG']:
environment_config = creds['DEFAULT_CONFIG']['environment']
if not environment_config:
# Disable environment capture
return
if environment == 'auto' and (environment_config == 'conda' or environment_config == 'pip'):
environment = environment_config
log('Capturing environment..')
captured = False
if environment == 'auto' or environment == 'conda':
# Capture conda environment
try:
upload_conda_env(gist_slug, version)
captured = True
except CondaError as e:
log(str(e), error=True)
if not captured and (environment == 'pip' or environment == 'auto'):
# Capture pip environment
try:
upload_pip_env(gist_slug, version)
except Exception as e:
log(str(e), error=True)
def _attach_files(paths, gist_slug, version, output=False):
"""Helper functions to attach files & folders to a commit"""
# Skip if empty
if not paths or len(paths) == 0:
return
log('Uploading additional ' + ('outputs' if output else 'files') + '...')
# Convert single path to list
if type(paths) == str:
paths = [paths]
for path in paths:
if os.path.isdir(path):
for folder, _, files in os.walk(path):
for fname in files:
fpath = os.path.join(folder, fname)
_attach_file(fpath, gist_slug, version, output)
elif os.path.exists(path):
_attach_file(path, gist_slug, version, output)
else:
log('Ignoring "' + path + '" (not found)', error=True)
def _print_update_message(current_version, latest_version):
log('Update Available: {0} --> {1}'.format(current_version,
latest_version))
if in_notebook():
log('Run `!pip install jovian --upgrade` to upgrade')
else:
log('Run `pip install jovian --upgrade` to upgrade\n')
def submit(assignment=None, notebook_url=None, **kwargs):
""" Performs jovian.commit and makes a assignment submission with the uploaded notebook.
"""
if not assignment:
log("Please provide assignment name", error=True)
return
filename = _parse_filename(kwargs.get('filename'))
if filename == '__notebook_source__.ipynb':
log("""jovian.submit does not support kaggle notebooks directly.
Please make a commit first, copy the notebook URL and pass it to jovian.submit.
eg. jovian.submit(assignment="zero-to-pandas-a1",
notebook_url="https://jovian.ai/PrajwalPrashanth/assignment")""", error=True)
return
post_url = POST_API.format(assignment)
nb_url = notebook_url if notebook_url else commit(**kwargs)
if nb_url:
data = {
'assignment_url': nb_url
}
auth_headers = _h()
log('Submitting assignment..')
res = post(url=_u(post_url),
json=data,
headers=auth_headers)
if res.status_code == 200:
data = res.json()['data']
course_slug = data.get('course_slug')
assignment_slug = data.get('section_slug')
assignment_page_url = ASSIGNMENT_PAGE_URL.format(course_slug, assignment_slug)
log('Verify your submission at {}'.format(urljoin(read_webapp_url(), assignment_page_url)))
else:
log('Jovian submit failed. {}'.format(pretty(res)), error=True)
def upload_file(gist_slug,
file,
folder=None,
version=None,
artifact=False,
version_title=None):
"""Upload an additional file to a gist"""
data = {'artifact': 'true'} if artifact else {}
if folder:
data['folder'] = folder
if version_title:
data['version_title'] = version_title
res = post(url=_u('/gist/' + gist_slug + '/upload' + _v(version)),
files={'files': file},
data=data,
headers=_h())
if res.status_code == 200:
data, warning = parse_success_response(res)
if warning:
log(warning, error=True)
return data
raise ApiError('File upload failed: ' + pretty(res))
def clone(slug, fresh=True):
"""Download the files for a gist"""
# Print issues link
log(ISSUES_MSG)
# Download gist metadata
log('Fetching ' + slug + "..")
gist = get_gist(slug, fresh)
title = gist['title']
# If fresh clone, create directory
if fresh:
if os.path.exists(title):
i = 1
while os.path.exists(title + '-' + str(i)):
i += 1
title = title + '-' + str(i)
if not os.path.exists(title):
os.makedirs(title)
os.chdir(title)
# Download the files
log('Downloading files..')
for f in gist['files']:
with open(f['filename'], 'wb') as fp:
fp.write(get(f['rawUrl']).content)
# Create .jovianrc for a fresh clone
if fresh and f['filename'].endswith('.ipynb'):
set_notebook_slug(f['filename'], slug)
# Print success message and instructions
if fresh:
log(post_clone_msg(title))
else:
log('Files dowloaded successfully in current directory')
def _parse_project(project, filename, new_project):
"""Perform the required checks and get the final project name"""
current_slug = get_cached_slug()
# Check for existing project in-memory or in .jovianrc
if not new_project and project is None:
# From in-memory variable
if current_slug is not None:
project = current_slug
# From .jovianrc file
else:
project = get_notebook_slug(filename)
# Skip if project is not provided & can't be read
if project is None:
return None, None
# Get project metadata for UUID & username/title
if is_uuid(project):
project_title = None
metadata = api.get_gist(project)
elif '/' in project:
project_title = project.split('/')[1]
username = api.get_current_user()['username']
metadata = api.get_gist(project)
# Attach username to the title
else:
project_title = project
username = api.get_current_user()['username']
metadata = api.get_gist(username + '/' + project)
# Skip if metadata could not be found
if not metadata:
log('Creating a new project "' + username + '/' + project_title + '"')
return project_title, None
# Extract information from metadata
username = metadata['owner']['username']
project_title = metadata['title']
project_id = metadata['slug']
# Check if the current user can commit to this project
permissions = api.get_gist_access(project_id)
if not permissions['write']:
return project_title, None
# Log whether this is an update or creation
if project_id is None:
log('Creating a new notebook on ' + read_webapp_url())
else:
log('Updating notebook "' + username + "/" + project_title + '" on ' +
read_webapp_url())
return project_title, project_id
def reset_config(confirm=True):
"""Remove the existing configuration by purging credentials"""
if creds_exist():
if confirm:
msg = 'Do you want to remove the existing configuration?'
confirmed = click.confirm(msg)
else:
confirmed = True
if confirmed:
log('Removing existing configuration. Run "jovian configure" to set up Jovian')
purge_creds()
else:
log('Skipping..')
return
else:
log('Jovian is not configured yet. Run "jovian configure" to set it up.')
