def perform_colab_commit(project, privacy): if '/' not in project: project = get_current_user()['username'] + '/' + project data = { 'project': project, 'file_id': get_colab_file_id(), 'visibility': privacy } if privacy == 'auto': data['public'] = True elif privacy == 'secret' or privacy == 'private': data['public'] = False auth_headers = _h() log("Uploading colab notebook to Jovian...") res = post(url=_u('/gist/colab-commit'), data=data, headers=auth_headers) if res.status_code == 200: data, warning = parse_success_response(res) if warning: log(warning, error=True) return data raise ApiError('Colab commit failed: ' + pretty(res))
def perform_colab_commit(project, privacy): # file_id, project, privacy api key file_id = get_colab_file_id() if file_id is None: log("Colab File Id is not provided", error=True) # /gist/colab-commit data = {file_id, project}, return status if '/' not in project: project = get_current_user()['username'] + '/' + project data = {'project': project, 'file_id': file_id, 'visibility': privacy} if privacy == 'auto': data['public'] = True elif privacy == 'secret' or privacy == 'private': data['public'] = False auth_headers = _h() log("Uploading colab notebook to Jovian...") res = post(url=_u('/gist/colab-commit'), data=data, headers=auth_headers) if res.status_code == 200: return res.json()['data'] raise ApiError('Colab commit failed: ' + pretty(res))
def create_gist_simple(filename=None, gist_slug=None, privacy='auto', title=None, version_title=None): """Upload the current notebook to create/update a gist""" auth_headers = _h() with open(filename, 'rb') as f: nb_file = (filename, f) log('Uploading notebook..') if gist_slug: return upload_file(gist_slug=gist_slug, file=nb_file, version_title=version_title) else: data = {'visibility': privacy} # For compatibility with old version of API endpoint if privacy == 'auto': data['public'] = True elif privacy == 'secret' or privacy == 'private': data['public'] = False if title: data['title'] = title if version_title: data['version_title'] = version_title res = post(url=_u('/gist/create'), data=data, files={'files': nb_file}, headers=auth_headers) if res.status_code == 200: return res.json()['data'] raise ApiError('File upload failed: ' + pretty(res))
def notify(data, verbose=True, safe=False): """Sends the data to the `Slack`_ workspace connected with your `Jovian`_ account. Args: data(dict|string): A dict or string to be pushed to Slack verbose(bool, optional): By default it prints the acknowledgement, you can remove this by setting the argument to False. safe(bool, optional): To avoid raising ApiError exception. Defaults to False. Example .. code-block:: import jovian data = "Hello from the Integration!" jovian.notify(data) .. important:: This feature requires for your Jovian account to be connected to a Slack workspace, visit `Jovian Integrations`_ to integrate them and to control the type of notifications. .. _Slack: https://slack.com .. _Jovian: https://jovian.ml?utm_source=docs .. _Jovian Integrations: https://jovian.ml/settings/integrations?utm_source=docs """ res = post_slack_message(data=data, safe=safe) if verbose: if not res.get('errors'): log('message_sent:' + str(res.get('data').get('messageSent'))) else: log(str(res.get('errors')[0].get('message')), error=True)
def request_org_id(): """Ask the user to provide the organization ID""" log("If you're a jovian-pro user please enter your company's organization ID on Jovian (otherwise leave it blank)." ) msg = "Organization ID" return click.prompt(msg, default='', show_default=False)
def exec_commit(ctx, notebook): """Create a new notebook on Jovian $ jovian commit my_notebook.ipynb """ if is_py2(): log("Committing is not supported for Python 2.x. Please install and run Jovian from Python 3.6 and above.", warn=True) commit_path(path=notebook, environment=None, is_cli=True)
def _attach_file(path, gist_slug, version, output=False): """Helper function to attach a single file to a commit""" try: with open(path, 'rb') as f: file_obj = os.path.basename(path), f folder = os.path.dirname(path) api.upload_file(gist_slug, file_obj, folder, version, output) except Exception as e: log(str(e) + " (" + path + ")", error=True)
def get_api_key(): """Retrieve and validate the API Key (from memory, config or user input)""" if API_KEY not in CREDS: key, source = read_or_request_api_key() if not validate_api_key(key): log('The current API key is invalid or expired.', error=True) key, source = request_api_key(), 'request' if not validate_api_key(key): raise ApiError('The API key provided is invalid or expired.') write_api_key(key) return key return CREDS[API_KEY]
def print_conda_message(env_name): if env_name: message = (""" # # To activate this environment, use # # $ conda activate %s # # To deactivate an active environment, use # # $ conda deactivate """) % env_name log(message)
def configure(): """Configure Jovian for first time usage""" # Check if already exists if creds_exist(): log('It looks like Jovian is already configured ( check ~/.jovian/credentials.json ).') msg = 'Do you want to overwrite the existing configuration?' confirm = click.confirm(msg) if confirm: log('Removing existing configuration..') else: log('Skipping..') return # Remove existing credentials purge_creds() # Capture and save organization ID ensure_org(check_pro=False) # Ask for API Key get_guest_key() get_api_key() log('Configuration complete!')
def _request_wrapper(*args, **kwargs): for i in range(2): res = request(*args, **kwargs) if res.status_code == 401: log('The current API key is invalid or expired.', error=True) purge_api_key() # This will ensure that fresh api token is requested if 'headers' in kwargs: kwargs['headers'][ 'Authorization'] = "Bearer " + get_api_key() else: return res return res
def install(env_fname=None, env_name=None): """Install packages for a cloned gist""" # Check for conda and get the binary path conda_bin = get_conda_bin() # Identify the right environment file, and exit if absent env_fname = identify_env_file(env_fname=env_fname) if env_fname is None: log('Failed to detect a conda environment YML file. Skipping..', error=True) return else: log('Detected conda environment file: ' + env_fname + "\n") # Get the environment name from user input env_name = request_env_name(env_name=env_name, env_fname=env_fname) if env_name is None: log('Environment name not provided/detected. Skipping..') return # Construct the command command = conda_bin + ' env update --file "' + \ env_fname + '" --name "' + env_name + '"' packages = extract_env_packages(env_fname=env_fname) if len(packages) > 0: success = run_command(command=command, env_fname=env_fname, packages=packages, run=1) if not success: log('Some pip packages failed to install.') print_conda_message(env_name=env_name)
def run_command(command, env_fname, packages, run=1): # Run the command if run > 3: return log('Executing:\n' + command + "\n") install_task = subprocess.Popen(command, shell=True, stderr=subprocess.PIPE) # Extract the error (if any) _, error_string = install_task.communicate() error_string = error_string.decode('utf8', errors='ignore') if error_string: print(error_string, file=stderr) # Check for errors error, pkgs = check_error(error_string, packages=packages) pip_failed = check_pip_failed(error_string) if error: log('Installation failed!', error=True) log('Ignoring ' + error + ' dependencies and trying again...\n') sleep(1) sanitize_envfile(env_fname=env_fname, pkgs=pkgs) return run_command(command=command, env_fname=env_fname, packages=packages, run=run+1) elif pip_failed: # TODO: Extract env details and run pip sub-command. # pip_packages = extract_pip_packages(env_fname=env_fname) return False else: # Print beta warning and github link log(ISSUES_MSG) return True
def log_record(record_type, data=None, verbose=True, **data_args): """Create records with the given data & type""" global _data_blocks # Create the combined data dictionary data = _parse_data(data, data_args) if data is None and verbose: log('Nothing to record. Skipping..', error=True) return # Send to API endpoint res = api.post_block(data, record_type) tracking_slug = res['tracking']['trackingSlug'] # Save to data block _data_blocks.append((tracking_slug, record_type, data)) if verbose: log(record_type.capitalize() + ' logged.')
def _perform_git_commit(filename, git_commit, git_message): if git_commit and git.is_git(): reset('git') # resets git commit info git.commit(git_message) log('Git repository identified. Performing git commit...') git_info = { 'repository': git.get_remote(), 'commit': git.get_current_commit(), 'filename': filename, 'path': git.get_relative_path(), 'branch': git.get_branch() } log_git(git_info, verbose=False)
def add_slack(): """prints instructions for connecting Slack, if Slack connection is not already present. if Slack is already connected, prints details about the workspace and the channel""" url = _u('/slack/integration_details') res = get(url, headers=_h()) if res.status_code == 200: res = res.json() if not res.get('errors'): slack_account = res.get('data').get('slackAccount') log('Slack already connected. \nWorkspace: {}\nConnected Channel: {}' .format(slack_account.get('workspace'), slack_account.get('channel'))) else: log(str(res.get('errors')[0].get('message'))) else: raise ApiError('Slack trigger failed: ' + pretty(res))
def _attach_files(paths, gist_slug, version, output=False, exclude_files=None): """Helper functions to attach files & folders to a commit""" config = read_creds().get("DEFAULT_CONFIG", {}) whitelist = config.get("EXTENSION_WHITELIST") upload_wd = config.get("UPLOAD_WORKING_DIRECTORY", False) if not isinstance(whitelist, list): whitelist = DEFAULT_EXTENSION_WHITELIST if not paths: if output or not upload_wd: return paths = [ f for f in glob.glob('**/*', recursive=True) if get_file_extension(f) in whitelist ] if exclude_files: if not isinstance(exclude_files, list): exclude_files = [exclude_files] for filename in exclude_files: try: paths.remove(filename) except ValueError: pass log('Uploading additional ' + ('outputs' if output else 'files') + '...') # Convert single path to list if type(paths) == str: paths = [paths] for path in paths: if os.path.isdir(path): files = [ f for f in glob.glob(os.path.join(path, '**/*'), recursive=True) if get_file_extension(f) in whitelist ] for file in files: _attach_file(file, gist_slug, version, output) elif os.path.exists(path): _attach_file(path, gist_slug, version, output) else: log('Ignoring "' + path + '" (not found)', error=True)
def clone(slug, version=None, fresh=True, include_outputs=True, overwrite=False): """Download the files for a gist""" # Print issues link log(ISSUES_MSG) # Download gist metadata ver_str = '(version ' + str(version) + ')' if version else '' log('Fetching ' + slug + " " + ver_str + "..") gist = get_gist(slug, version, fresh) if not gist: return title = gist['title'] # If fresh clone, create directory if fresh and not os.path.exists(title): os.makedirs(title) os.chdir(title) elif fresh and os.path.exists(title) and overwrite: os.chdir(title) elif fresh and os.path.exists(title) and not overwrite: i = 1 while os.path.exists(title + '-' + str(i)): i += 1 title = title + '-' + str(i) os.makedirs(title) os.chdir(title) # Download the files log('Downloading files..') for f in gist['files']: if not f['artifact'] or include_outputs: if f['filename'].endswith('.ipynb'): content = _sanitize_notebook(get(f['rawUrl']).content) else: content = get(f['rawUrl']).content if f['folder'] and not os.path.exists(f['folder']): os.makedirs(f['folder']) filepath = os.path.join(f['folder'] or '', f['filename']) with open(filepath, 'wb') as fp: fp.write(content) # Create .jovianrc for a fresh clone if fresh and f['filename'].endswith('.ipynb'): set_notebook_slug(f['filename'], slug) # Print success message and instructions if fresh: post_clone_msg(title) else: log('Files dowloaded successfully in current directory')
def create_gist_simple(filename=None, gist_slug=None, secret=False): """Upload the current notebook to create a gist""" auth_headers = _h() nb_file = (filename, open(filename, 'rb')) log('Uploading notebook..') if gist_slug: return upload_file(gist_slug, nb_file) else: res = post(url=_u('/gist/create'), data={'public': 0 if secret else 1}, files={'files': nb_file}, headers=auth_headers) if res.status_code == 200: return res.json()['data'] raise ApiError('File upload failed: ' + _pretty(res))
def pull(slug=None): """Get the latest files associated with the current gist""" # If a slug is provided, just use that if slug: clone(slug, fresh=False) return # Check if .jovianrc exists if not rcfile_exists(): log(RCFILE_NOTFOUND, error=True) return # Get list of notebooks nbs = get_rcdata()['notebooks'] for fname in nbs: # Get the latest files for each notebook clone(nbs[fname]['slug'], fresh=False)
def get_gist(slug, version, fresh): """Download a gist""" if '/' in slug: parts = slug.split('/') username, title = parts[0], parts[1] url = _u('user/' + username + '/gist/' + title + _v(version)) else: url = _u('gist/' + slug + _v(version)) res = get(url, headers=_h(fresh)) if res.status_code == 200: return res.json()['data'] elif res.status_code == 401: log('This notebook does not exist or is private. Please provide the API key') get_api_key() return get_gist(slug, version, fresh) else: log('Failed to retrieve notebook: ' + pretty(res), error=True)
def check_error(error_str, packages=None): """Check if the error output contains ResolvePackageNotFound or UnsatisfiableError""" if not packages: packages = [] error_lines = error_str.split('\n') error = None pkgs = [] for line in error_lines: if 'ResolvePackageNotFound:' in line: error = 'unresolved' elif 'UnsatisfiableError:' in line: error = 'unsatisfiable' log(MISSING_MSG) if error: pkg = extract_package_from_line(line, packages) if pkg and pkg not in pkgs: pkgs.append(pkg) return error, pkgs
def _capture_environment(environment, gist_slug, version): """Capture the python environment and attach it to the commit""" if environment is not None: # Check credentials if environment config exists creds = read_creds() if 'DEFAULT_CONFIG' in creds and 'environment' in creds['DEFAULT_CONFIG']: environment_config = creds['DEFAULT_CONFIG']['environment'] if not environment_config: # Disable environment capture return if environment == 'auto' and (environment_config == 'conda' or environment_config == 'pip'): environment = environment_config log('Capturing environment..') captured = False if environment == 'auto' or environment == 'conda': # Capture conda environment try: upload_conda_env(gist_slug, version) captured = True except CondaError as e: log(str(e), error=True) if not captured and (environment == 'pip' or environment == 'auto'): # Capture pip environment try: upload_pip_env(gist_slug, version) except Exception as e: log(str(e), error=True)
def _attach_files(paths, gist_slug, version, output=False): """Helper functions to attach files & folders to a commit""" # Skip if empty if not paths or len(paths) == 0: return log('Uploading additional ' + ('outputs' if output else 'files') + '...') # Convert single path to list if type(paths) == str: paths = [paths] for path in paths: if os.path.isdir(path): for folder, _, files in os.walk(path): for fname in files: fpath = os.path.join(folder, fname) _attach_file(fpath, gist_slug, version, output) elif os.path.exists(path): _attach_file(path, gist_slug, version, output) else: log('Ignoring "' + path + '" (not found)', error=True)
def _print_update_message(current_version, latest_version): log('Update Available: {0} --> {1}'.format(current_version, latest_version)) if in_notebook(): log('Run `!pip install jovian --upgrade` to upgrade') else: log('Run `pip install jovian --upgrade` to upgrade\n')
def submit(assignment=None, notebook_url=None, **kwargs): """ Performs jovian.commit and makes a assignment submission with the uploaded notebook. """ if not assignment: log("Please provide assignment name", error=True) return filename = _parse_filename(kwargs.get('filename')) if filename == '__notebook_source__.ipynb': log("""jovian.submit does not support kaggle notebooks directly. Please make a commit first, copy the notebook URL and pass it to jovian.submit. eg. jovian.submit(assignment="zero-to-pandas-a1", notebook_url="https://jovian.ai/PrajwalPrashanth/assignment")""", error=True) return post_url = POST_API.format(assignment) nb_url = notebook_url if notebook_url else commit(**kwargs) if nb_url: data = { 'assignment_url': nb_url } auth_headers = _h() log('Submitting assignment..') res = post(url=_u(post_url), json=data, headers=auth_headers) if res.status_code == 200: data = res.json()['data'] course_slug = data.get('course_slug') assignment_slug = data.get('section_slug') assignment_page_url = ASSIGNMENT_PAGE_URL.format(course_slug, assignment_slug) log('Verify your submission at {}'.format(urljoin(read_webapp_url(), assignment_page_url))) else: log('Jovian submit failed. {}'.format(pretty(res)), error=True)
def upload_file(gist_slug, file, folder=None, version=None, artifact=False, version_title=None): """Upload an additional file to a gist""" data = {'artifact': 'true'} if artifact else {} if folder: data['folder'] = folder if version_title: data['version_title'] = version_title res = post(url=_u('/gist/' + gist_slug + '/upload' + _v(version)), files={'files': file}, data=data, headers=_h()) if res.status_code == 200: data, warning = parse_success_response(res) if warning: log(warning, error=True) return data raise ApiError('File upload failed: ' + pretty(res))
def clone(slug, fresh=True): """Download the files for a gist""" # Print issues link log(ISSUES_MSG) # Download gist metadata log('Fetching ' + slug + "..") gist = get_gist(slug, fresh) title = gist['title'] # If fresh clone, create directory if fresh: if os.path.exists(title): i = 1 while os.path.exists(title + '-' + str(i)): i += 1 title = title + '-' + str(i) if not os.path.exists(title): os.makedirs(title) os.chdir(title) # Download the files log('Downloading files..') for f in gist['files']: with open(f['filename'], 'wb') as fp: fp.write(get(f['rawUrl']).content) # Create .jovianrc for a fresh clone if fresh and f['filename'].endswith('.ipynb'): set_notebook_slug(f['filename'], slug) # Print success message and instructions if fresh: log(post_clone_msg(title)) else: log('Files dowloaded successfully in current directory')
def _parse_project(project, filename, new_project): """Perform the required checks and get the final project name""" current_slug = get_cached_slug() # Check for existing project in-memory or in .jovianrc if not new_project and project is None: # From in-memory variable if current_slug is not None: project = current_slug # From .jovianrc file else: project = get_notebook_slug(filename) # Skip if project is not provided & can't be read if project is None: return None, None # Get project metadata for UUID & username/title if is_uuid(project): project_title = None metadata = api.get_gist(project) elif '/' in project: project_title = project.split('/')[1] username = api.get_current_user()['username'] metadata = api.get_gist(project) # Attach username to the title else: project_title = project username = api.get_current_user()['username'] metadata = api.get_gist(username + '/' + project) # Skip if metadata could not be found if not metadata: log('Creating a new project "' + username + '/' + project_title + '"') return project_title, None # Extract information from metadata username = metadata['owner']['username'] project_title = metadata['title'] project_id = metadata['slug'] # Check if the current user can commit to this project permissions = api.get_gist_access(project_id) if not permissions['write']: return project_title, None # Log whether this is an update or creation if project_id is None: log('Creating a new notebook on ' + read_webapp_url()) else: log('Updating notebook "' + username + "/" + project_title + '" on ' + read_webapp_url()) return project_title, project_id
def reset_config(confirm=True): """Remove the existing configuration by purging credentials""" if creds_exist(): if confirm: msg = 'Do you want to remove the existing configuration?' confirmed = click.confirm(msg) else: confirmed = True if confirmed: log('Removing existing configuration. Run "jovian configure" to set up Jovian') purge_creds() else: log('Skipping..') return else: log('Jovian is not configured yet. Run "jovian configure" to set it up.')