def bundle_add_file(bundle, rel_path, base_dir):
"""Add the specified file to the tarball.
The file path is relative to the notebook directory.
"""
logger.debug("adding file: %s", rel_path)
path = join(base_dir, rel_path)
bundle.add(path, arcname=rel_path)
def get_cookie_header_value(self):
result = "; ".join([
"%s=%s" %
(key, self._reference.value_encode(self._content[key])[1])
for key in self._keys
])
logger.debug("Cookie: %s" % result)
return result
def bundle_add_buffer(bundle, filename, contents):
"""Add an in-memory buffer to the tarball.
`contents` may be a string or bytes object
"""
logger.debug("adding file: %s", filename)
buf = io.BytesIO(to_bytes(contents))
file_info = tarfile.TarInfo(filename)
file_info.size = len(buf.getvalue())
bundle.addfile(file_info, buf)
def store_cookies(self, response):
headers = filter(lambda h: h[0].lower() == "set-cookie", response.getheaders())
for header in headers:
cookie = SimpleCookie(header[1])
for morsel in cookie.values():
if morsel.key not in self._keys:
self._keys.append(morsel.key)
self._content[morsel.key] = morsel.value
logger.debug("--> Set cookie %s: %s" % (morsel.key, morsel.value))
logger.debug("CookieJar contents: %s\n%s" % (self._keys, self._content))
def make_notebook_source_bundle(
file, # type: str
environment, # type: Environment
extra_files=None, # type: typing.Optional[typing.List[str]]
):
# type: (...) -> typing.IO[bytes]
"""Create a bundle containing the specified notebook and python environment.
Returns a file-like object containing the bundle tarball.
"""
if extra_files is None:
extra_files = []
base_dir = dirname(file)
nb_name = basename(file)
manifest = make_source_manifest(nb_name, environment,
AppModes.JUPYTER_NOTEBOOK)
manifest_add_file(manifest, nb_name, base_dir)
manifest_add_buffer(manifest, environment.filename, environment.contents)
if extra_files:
skip = [nb_name, environment.filename, "manifest.json"]
extra_files = sorted(list(set(extra_files) - set(skip)))
for rel_path in extra_files:
manifest_add_file(manifest, rel_path, base_dir)
logger.debug("manifest: %r", manifest)
bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle")
with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle:
# add the manifest first in case we want to partially untar the bundle for inspection
bundle_add_buffer(bundle, "manifest.json",
json.dumps(manifest, indent=2))
bundle_add_buffer(bundle, environment.filename, environment.contents)
bundle_add_file(bundle, nb_name, base_dir)
for rel_path in extra_files:
bundle_add_file(bundle, rel_path, base_dir)
bundle_file.seek(0)
return bundle_file
def make_notebook_source_bundle(file, environment, extra_files=None):
"""Create a bundle containing the specified notebook and python environment.
Returns a file-like object containing the bundle tarball.
"""
if extra_files is None:
extra_files = []
base_dir = dirname(file)
nb_name = basename(file)
manifest = make_source_manifest(nb_name, environment,
AppModes.JUPYTER_NOTEBOOK)
manifest_add_file(manifest, nb_name, base_dir)
manifest_add_buffer(manifest, environment['filename'],
environment['contents'])
if extra_files:
skip = [nb_name, environment['filename'], 'manifest.json']
extra_files = sorted(list(set(extra_files) - set(skip)))
for rel_path in extra_files:
manifest_add_file(manifest, rel_path, base_dir)
logger.debug('manifest: %r', manifest)
bundle_file = tempfile.TemporaryFile(prefix='rsc_bundle')
with tarfile.open(mode='w:gz', fileobj=bundle_file) as bundle:
# add the manifest first in case we want to partially untar the bundle for inspection
bundle_add_buffer(bundle, 'manifest.json',
json.dumps(manifest, indent=2))
bundle_add_buffer(bundle, environment['filename'],
environment['contents'])
bundle_add_file(bundle, nb_name, base_dir)
for rel_path in extra_files:
bundle_add_file(bundle, rel_path, base_dir)
bundle_file.seek(0)
return bundle_file
def write_manifest(relative_dir, nb_name, environment, output_dir):
# type: (str, str, Environment, str) -> typing.Tuple[list, list]
"""Create a manifest for source publishing the specified notebook.
The manifest will be written to `manifest.json` in the output directory..
A requirements.txt file will be created if one does not exist.
Returns the list of filenames written.
"""
manifest_filename = "manifest.json"
manifest = make_source_manifest(nb_name, environment,
AppModes.JUPYTER_NOTEBOOK)
manifest_file = join(output_dir, manifest_filename)
created = []
skipped = []
manifest_relative_path = join(relative_dir, manifest_filename)
if exists(manifest_file):
skipped.append(manifest_relative_path)
else:
with open(manifest_file, "w") as f:
f.write(json.dumps(manifest, indent=2))
created.append(manifest_relative_path)
logger.debug("wrote manifest file: %s", manifest_file)
environment_filename = environment.filename
environment_file = join(output_dir, environment_filename)
environment_relative_path = join(relative_dir, environment_filename)
if environment.source == "file":
skipped.append(environment_relative_path)
else:
with open(environment_file, "w") as f:
f.write(environment.contents)
created.append(environment_relative_path)
logger.debug("wrote environment file: %s", environment_file)
return created, skipped
def write_manifest(relative_dir, nb_name, environment, output_dir):
"""Create a manifest for source publishing the specified notebook.
The manifest will be written to `manifest.json` in the output directory..
A requirements.txt file will be created if one does not exist.
Returns the list of filenames written.
"""
manifest_filename = 'manifest.json'
manifest = make_source_manifest(nb_name, environment,
AppModes.JUPYTER_NOTEBOOK)
manifest_file = join(output_dir, manifest_filename)
created = []
skipped = []
manifest_relative_path = join(relative_dir, manifest_filename)
if exists(manifest_file):
skipped.append(manifest_relative_path)
else:
with open(manifest_file, 'w') as f:
f.write(json.dumps(manifest, indent=2))
created.append(manifest_relative_path)
logger.debug('wrote manifest file: %s', manifest_file)
environment_filename = environment['filename']
environment_file = join(output_dir, environment_filename)
environment_relative_path = join(relative_dir, environment_filename)
if environment['source'] == 'file':
skipped.append(environment_relative_path)
else:
with open(environment_file, 'w') as f:
f.write(environment['contents'])
created.append(environment_relative_path)
logger.debug('wrote environment file: %s', environment_file)
return created, skipped
def resolve(self, server, app_id, app_mode):
metadata = self.get(server)
if metadata is None:
logger.debug(
"No previous deployment to this server was found; this will be a new deployment."
)
return app_id, app_mode
logger.debug("Found previous deployment data in %s" % self.get_path())
if app_id is None:
app_id = metadata.get("app_guid") or metadata.get("app_id")
logger.debug("Using saved app ID: %s" % app_id)
# app mode cannot be changed on redeployment
app_mode = AppModes.get_by_name(metadata.get("app_mode"))
return app_id, app_mode
def _do_request(self,
method,
path,
query_params,
body,
maximum_redirects,
extra_headers=None):
full_uri = path
if query_params is not None:
full_uri = "%s?%s" % (path, urlencode(query_params))
headers = self._headers.copy()
if extra_headers is not None:
headers.update(extra_headers)
local_connection = False
try:
if logger.is_debugging():
logger.debug("Request: %s %s" % (method, full_uri))
logger.debug("Headers:")
for key, value in headers.items():
logger.debug("--> %s: %s" % (key, value))
# if we weren't called under a `with` statement, we'll need to manage the
# connection here.
if self._conn is None:
self.__enter__()
local_connection = True
try:
self._conn.request(method, full_uri, body, headers)
response = self._conn.getresponse()
response_body = response.read().decode("utf-8").strip()
if logger.is_debugging():
logger.debug("Response: %s %s" %
(response.status, response.reason))
logger.debug("Headers:")
for key, value in response.getheaders():
logger.debug("--> %s: %s" % (key, value))
logger.debug("--> %s" % response_body)
finally:
if local_connection:
self.__exit__()
# Handle any redirects.
if 300 <= response.status < 400:
if maximum_redirects == 0:
raise http.CannotSendRequest("Too many redirects")
location = response.getheader("Location")
next_url = urljoin(self._url.geturl(), location)
logger.debug("--> Redirected to: %s" % next_url)
return self._do_request(
method,
next_url,
query_params,
body,
maximum_redirects - 1,
extra_headers,
)
self._handle_set_cookie(response)
return self._tweak_response(
HTTPResponse(full_uri, response=response, body=response_body))
except (
http.HTTPException,
ssl.CertificateError,
IOError,
OSError,
socket.error,
socket.herror,
socket.gaierror,
socket.timeout,
) as exception:
logger.debug("An exception occurred processing the HTTP request.",
exc_info=True)
return HTTPResponse(full_uri, exception=exception)
def override_title_search(connect_server, app_id, app_title):
"""
Returns a list of abbreviated app data that contains apps with a title
that matches the given one and/or the specific app noted by its ID.
:param connect_server: the Connect server information.
:param app_id: the ID of a specific app to look for, if any.
:param app_title: the title to search for.
:return: the list of matching apps, each trimmed to ID, name, title, mode
URL and dashboard URL.
"""
def map_app(app, config):
"""
Creates the abbreviated data dictionary for the specified app and config
information.
:param app: the raw app data to start with.
:param config: the configuration data to use.
:return: the abbreviated app data dictionary.
"""
return {
"id": app["id"],
"name": app["name"],
"title": app["title"],
"app_mode": AppModes.get_by_ordinal(app["app_mode"]).name(),
"url": app["url"],
"config_url": config["config_url"],
}
def mapping_filter(client, app):
"""
Mapping/filter function for retrieving apps. We only keep apps
that have an app mode of static or Jupyter notebook. The data
for the apps we keep is an abbreviated subset.
:param client: the client object to use for RStudio Connect calls.
:param app: the current app from Connect.
:return: the abbreviated data for the app or None.
"""
# Only keep apps that match our app modes.
app_mode = AppModes.get_by_ordinal(app["app_mode"])
if app_mode not in (AppModes.STATIC, AppModes.JUPYTER_NOTEBOOK):
return None
config = client.app_config(app["id"])
connect_server.handle_bad_response(config)
return map_app(app, config)
apps = retrieve_matching_apps(
connect_server,
filters={
"filter": "min_role:editor",
"search": app_title
},
mapping_function=mapping_filter,
limit=5,
)
if app_id:
found = next((app for app in apps if app["id"] == app_id), None)
if not found:
try:
app = get_app_info(connect_server, app_id)
mode = AppModes.get_by_ordinal(app["app_mode"])
if mode in (AppModes.STATIC, AppModes.JUPYTER_NOTEBOOK):
apps.append(
map_app(app, get_app_config(connect_server, app_id)))
except RSConnectException:
logger.debug(
'Error getting info for previous app_id "%s", skipping.',
app_id)
return apps
