def create_notebook_deployment_bundle(file_name, extra_files, app_mode, python, environment,
extra_files_need_validating=True):
"""
Create an in-memory bundle, ready to deploy.
:param file_name: the Jupyter notebook being deployed.
:param extra_files: a sequence of any extra files to include in the bundle.
:param app_mode: the mode of the app being deployed.
:param python: information about the version of Python being used.
:param environment: environmental information.
:param extra_files_need_validating: a flag indicating whether the list of extra
files should be validated or not. Part of validating includes qualifying each
with the parent directory of the notebook file. If you provide False here, make
sure the names are properly qualified first.
:return: the bundle.
"""
validate_file_is_notebook(file_name)
if extra_files_need_validating:
extra_files = validate_extra_files(dirname(file_name), extra_files)
if app_mode == AppModes.STATIC:
try:
return make_notebook_html_bundle(file_name, python)
except subprocess.CalledProcessError as exc:
# Jupyter rendering failures are often due to
# user code failing, vs. an internal failure of rsconnect-python.
raise api.RSConnectException(str(exc))
else:
return make_notebook_source_bundle(file_name, environment, extra_files)
def check_server_capabilities(connect_server, capability_functions, details_source=gather_server_details):
"""
Uses a sequence of functions that check for capabilities in a Connect server. The
server settings data is retrieved by the gather_server_details() function.
Each function provided must accept one dictionary argument which will be the server
settings data returned by the gather_server_details() function. That function must
return a boolean value. It must also contain a docstring which itself must contain
an ":error:" tag as the last thing in the docstring. If the function returns False,
an exception is raised with the function's ":error:" text as its message.
:param connect_server: the information needed to interact with the Connect server.
:param capability_functions: a sequence of functions that will be called.
:param details_source: the source for obtaining server details, gather_server_details(),
by default.
"""
details = details_source(connect_server)
for function in capability_functions:
if not function(details):
index = function.__doc__.find(':error:') if function.__doc__ else -1
if index >= 0:
message = function.__doc__[index + 7:].strip()
else:
message = 'The server does not satisfy the %s capability check.' % function.__name__
raise api.RSConnectException(message)
def test_server(connect_server):
"""
Test whether the given server can be reached and is running Connect. The server
may be provided with or without a scheme. If a scheme is omitted, the server will
be tested with both `https` and `http` until one of them works.
:param connect_server: the Connect server information.
:return: a second server object with any scheme expansions applied and the server
settings from the server.
"""
url = connect_server.url
key = connect_server.api_key
insecure = connect_server.insecure
ca_data = connect_server.ca_data
failures = ['Invalid server URL: %s' % url]
for test in _to_server_check_list(url):
try:
connect_server = api.RSConnectServer(test, key, insecure, ca_data)
result = _verify_server(connect_server)
return connect_server, result
except api.RSConnectException:
failures.append(' %s - failed to verify as RStudio Connect.' % test)
# In case the user may need https instead of http...
if len(failures) == 2 and url.startswith('http://'):
failures.append(' Do you need to use "https://%s?"' % url[7:])
# If we're here, nothing worked.
raise api.RSConnectException('\n'.join(failures))
def gather_basic_deployment_info_for_notebook(connect_server, app_store, file_name, new, app_id, title, static):
"""
Helps to gather the necessary info for performing a deployment.
:param connect_server: the Connect server information.
:param app_store: the store for the specified file
:param file_name: the primary file being deployed.
:param new: a flag noting whether we should force a new deployment.
:param app_id: the ID of the app to redeploy.
:param title: an optional title. If this isn't specified, a default title will
be generated.
:param static: a flag to note whether a static document should be deployed.
:return: the app ID, name, title information and mode for the deployment.
"""
validate_file_is_notebook(file_name)
_validate_title(title)
if new and app_id:
raise api.RSConnectException('Specify either a new deploy or an app ID but not both.')
if app_id is not None:
# Don't read app metadata if app-id is specified. Instead, we need
# to get this from Connect.
app = api.get_app_info(connect_server, app_id)
app_mode = AppModes.get_by_ordinal(app.get('app_mode', 0), True)
logger.debug('Using app mode from app %s: %s' % (app_id, app_mode))
elif static:
app_mode = AppModes.STATIC
else:
app_mode = AppModes.JUPYTER_NOTEBOOK
if not new and app_id is None:
# Possible redeployment - check for saved metadata.
# Use the saved app information unless overridden by the user.
app_id, app_mode = app_store.resolve(connect_server.url, app_id, app_mode)
if static and app_mode != AppModes.STATIC:
raise api.RSConnectException('Cannot change app mode to "static" once deployed. '
'Use --new to create a new deployment.')
default_title = not bool(title)
title = title or _default_title(file_name)
return app_id, _make_deployment_name(connect_server, title, app_id is None), title, default_title, app_mode
def validate_file_is_notebook(file_name):
"""
Validate that the given file is a Jupyter Notebook. If it isn't, an exception is
thrown. A file must exist and have the '.ipynb' extension.
:param file_name: the name of the file to validate.
"""
file_suffix = splitext(file_name)[1].lower()
if file_suffix != '.ipynb' or not exists(file_name):
raise api.RSConnectException('A Jupyter notebook (.ipynb) file is required here.')
def _validate_title(title):
"""
If the user specified a title, validate that it meets Connect's length requirements.
If the validation fails, an exception is raised. Otherwise,
:param title: the title to validate.
"""
if title:
if not (3 <= len(title) <= 1024):
raise api.RSConnectException('A title must be between 3-1024 characters long.')
def _verify_server(connect_server):
"""
Test whether the server identified by the given full URL can be reached and is
running Connect.
:param connect_server: the Connect server information.
:return: the server settings from the Connect server.
"""
uri = urlparse(connect_server.url)
if not uri.netloc:
raise api.RSConnectException('Invalid server URL: "%s"' % connect_server.url)
return api.verify_server(connect_server)
def validate_manifest_file(file_or_directory):
"""
Validates that the name given represents either an existing manifest.json file or
a directory that contains one. If not, an exception is raised.
:param file_or_directory: the name of the manifest file or directory that contains it.
:return: the real path to the manifest file.
"""
if isdir(file_or_directory):
file_or_directory = join(file_or_directory, 'manifest.json')
if basename(file_or_directory) != 'manifest.json' or not exists(file_or_directory):
raise api.RSConnectException('A manifest.json file or a directory containing one is required here.')
return file_or_directory
def validate_extra_files(directory, extra_files):
"""
If the user specified a list of extra files, validate that they all exist and are
beneath the given directory and, if so, return a list of them made relative to that
directory.
:param directory: the directory that the extra files must be relative to.
:param extra_files: the list of extra files to qualify and validate.
:return: the extra files qualified by the directory.
"""
result = []
if extra_files:
for extra in extra_files:
extra_file = relpath(extra, directory)
# It's an error if we have to leave the given dir to get to the extra
# file.
if extra_file.startswith('../'):
raise api.RSConnectException('%s must be under %s.' % (extra_file, directory))
if not exists(join(directory, extra_file)):
raise api.RSConnectException('Could not find file %s under %s' % (extra, directory))
result.append(extra_file)
return result
def resolve(self, name, url, api_key, insecure, ca_data):
"""
This function will resolve the given inputs into a set of server information.
It assumes that either `name` or `url` is provided.
If `name` is provided, the server information is looked up by its nickname
and an error is produced if the nickname is not known.
If `url` is provided, the server information is looked up by its URL. If
that is found, the stored information is returned. Otherwise the corresponding
arguments are returned as-is.
If neither 'name' nor 'url' is provided and there is only one stored server,
that information is returned. In this case, the last value in the tuple returned
notes this situation. It is `False` in all other cases.
:param name: the nickname to look for.
:param url: the Connect server URL to look for.
:param api_key: the API key provided on the command line.
:param insecure: the insecure flag provided on the command line.
:param ca_data: the CA certification data provided on the command line.
:return: the information needed to interact with the resolved server and whether
it came from the store or the arguments.
"""
if name:
entry = self.get_by_name(name)
if not entry:
raise api.RSConnectException(
'The nickname, "%s", does not exist.' % name)
elif url:
entry = self.get_by_url(url)
else:
# if there is a single server, default to it
if self.count() == 1:
entry = self._get_first_value()
else:
entry = None
if entry:
return (
entry["url"],
entry["api_key"],
entry["insecure"],
entry["ca_cert"],
True,
)
else:
return url, api_key, insecure, ca_data, False
def which_python(python, env=os.environ):
"""Determine which python binary should be used.
In priority order:
* --python specified on the command line
* RETICULATE_PYTHON defined in the environment
* the python binary running this script
"""
if python:
if not (exists(python) and os.access(python, os.X_OK)):
raise api.RSConnectException('The file, "%s", does not exist or is not executable.' % python)
return python
if 'RETICULATE_PYTHON' in env:
return env['RETICULATE_PYTHON']
return sys.executable
def _gather_basic_deployment_info_for_framework(connect_server, app_store, directory, entry_point, new, app_id,
app_mode, title):
"""
Helps to gather the necessary info for performing a deployment.
:param connect_server: the Connect server information.
:param app_store: the store for the specified directory.
:param directory: the primary file being deployed.
:param entry_point: the entry point for the API in '<module>:<object> format. if
the object name is omitted, it defaults to the module name. If nothing is specified,
it defaults to 'app'.
:param new: a flag noting whether we should force a new deployment.
:param app_id: the ID of the app to redeploy.
:param app_mode: the app mode to use.
:param title: an optional title. If this isn't specified, a default title will
be generated.
:return: the entry point, app ID, name, title and mode for the deployment.
"""
entry_point = validate_entry_point(entry_point)
_validate_title(title)
if new and app_id:
raise api.RSConnectException('Specify either a new deploy or an app ID but not both.')
if app_id is not None:
# Don't read app metadata if app-id is specified. Instead, we need
# to get this from Connect.
app = api.get_app_info(connect_server, app_id)
app_mode = AppModes.get_by_ordinal(app.get('app_mode', 0), True)
logger.debug('Using app mode from app %s: %s' % (app_id, app_mode))
if not new and app_id is None:
# Possible redeployment - check for saved metadata.
# Use the saved app information unless overridden by the user.
app_id, app_mode = app_store.resolve(connect_server.url, app_id, app_mode)
if directory[-1] == '/':
directory = directory[:-1]
default_title = not bool(title)
title = title or _default_title(directory)
return entry_point, app_id, _make_deployment_name(connect_server, title, app_id is None), title, default_title,\
app_mode
def validate_entry_point(entry_point):
"""
Validates the entry point specified by the user, expanding as necessary. If the
user specifies nothing, a module of "app" is assumed. If the user specifies a
module only, the object is assumed to be the same name as the module.
:param entry_point: the entry point as specified by the user.
:return: the fully expanded and validated entry point and the module file name..
"""
if not entry_point:
entry_point = 'app'
parts = entry_point.split(':')
if len(parts) > 2:
raise api.RSConnectException('Entry point is not in "module:object" format.')
return entry_point
def inspect_environment(python, directory, compatibility_mode=False, force_generate=False,
check_output=subprocess.check_output):
"""Run the environment inspector using the specified python binary.
Returns a dictionary of information about the environment,
or containing an "error" field if an error occurred.
"""
flags = []
if compatibility_mode:
flags.append('c')
if force_generate:
flags.append('f')
args = [python, '-m', 'rsconnect.environment']
if len(flags) > 0:
args.append('-'+''.join(flags))
args.append(directory)
try:
environment_json = check_output(args, universal_newlines=True)
except subprocess.CalledProcessError as e:
raise api.RSConnectException("Error inspecting environment: %s" % e.output)
environment = json.loads(environment_json)
return environment
def write_notebook_manifest_json(entry_point_file, environment, app_mode=None, extra_files=None):
"""
Creates and writes a manifest.json file for the given entry point file. If
the application mode is not provided, an attempt will be made to resolve one
based on the extension portion of the entry point file.
:param entry_point_file: the entry point file (Jupyter notebook, etc.) to build
the manifest for.
:param environment: the Python environment to start with. This should be what's
returned by the inspect_environment() function.
:param app_mode: the application mode to assume. If this is None, the extension
portion of the entry point file name will be used to derive one.
:param extra_files: any extra files that should be included in the manifest.
:return: whether or not the environment file (requirements.txt, environment.yml,
etc.) that goes along with the manifest exists.
"""
extra_files = validate_extra_files(dirname(entry_point_file), extra_files)
directory = dirname(entry_point_file)
file_name = basename(entry_point_file)
manifest_path = join(directory, 'manifest.json')
if app_mode is None:
_, extension = splitext(file_name)
app_mode = AppModes.get_by_extension(extension, True)
if app_mode == AppModes.UNKNOWN:
raise api.RSConnectException('Could not determine the app mode from "%s"; please specify one.' % extension)
manifest_data = make_source_manifest(file_name, environment, app_mode)
manifest_add_file(manifest_data, file_name, directory)
manifest_add_buffer(manifest_data, environment['filename'], environment['contents'])
for rel_path in extra_files:
manifest_add_file(manifest_data, rel_path, directory)
with open(manifest_path, 'w') as f:
json.dump(manifest_data, f, indent=2)
return exists(join(directory, environment['filename']))
def get_python_env_info(file_name, python, compatibility_mode, force_generate):
"""
Gathers the python and environment information relating to the specified file
with an eye to deploy it.
:param file_name: the primary file being deployed.
:param python: the optional name of a Python executable.
:param compatibility_mode: force freezing the current environment using pip
instead of conda, when conda is not supported on RStudio Connect (version<=1.8.0).
:param force_generate: force generating "requirements.txt" or "environment.yml",
even if it already exists.
:return: information about the version of Python in use plus some environmental
stuff.
"""
python = which_python(python)
logger.debug('Python: %s' % python)
environment = inspect_environment(python, dirname(file_name), compatibility_mode=compatibility_mode,
force_generate=force_generate)
if 'error' in environment:
raise api.RSConnectException(environment['error'])
logger.debug('Python: %s' % python)
logger.debug('Environment: %s' % pformat(environment))
return python, environment
def gather_basic_deployment_info_from_manifest(connect_server, app_store, file_name, new, app_id, title):
"""
Helps to gather the necessary info for performing a deployment.
:param connect_server: the Connect server information.
:param app_store: the store for the specified file
:param file_name: the manifest file being deployed.
:param new: a flag noting whether we should force a new deployment.
:param app_id: the ID of the app to redeploy.
:param title: an optional title. If this isn't specified, a default title will
be generated.
:return: the app ID, name, title information, mode, and package manager for the
deployment.
"""
file_name = validate_manifest_file(file_name)
_validate_title(title)
if new and app_id:
raise api.RSConnectException('Specify either a new deploy or an app ID but not both.')
source_manifest, _ = read_manifest_file(file_name)
# noinspection SpellCheckingInspection
app_mode = AppModes.get_by_name(source_manifest['metadata']['appmode'])
if not new and app_id is None:
# Possible redeployment - check for saved metadata.
# Use the saved app information unless overridden by the user.
app_id, app_mode = app_store.resolve(connect_server.url, app_id, app_mode)
package_manager = source_manifest.get('python', {}).get('package_manager', {}).get('name', None)
default_title = not bool(title)
title = title or _default_title_from_manifest(source_manifest, file_name)
return app_id, _make_deployment_name(connect_server, title, app_id is None), title, default_title, app_mode,\
package_manager
