def bundle_add_file(bundle, rel_path, base_dir): """Add the specified file to the tarball. The file path is relative to the notebook directory. """ logger.debug("adding file: %s", rel_path) path = join(base_dir, rel_path) bundle.add(path, arcname=rel_path)
def get_cookie_header_value(self): result = "; ".join([ "%s=%s" % (key, self._reference.value_encode(self._content[key])[1]) for key in self._keys ]) logger.debug("Cookie: %s" % result) return result
def bundle_add_buffer(bundle, filename, contents): """Add an in-memory buffer to the tarball. `contents` may be a string or bytes object """ logger.debug("adding file: %s", filename) buf = io.BytesIO(to_bytes(contents)) file_info = tarfile.TarInfo(filename) file_info.size = len(buf.getvalue()) bundle.addfile(file_info, buf)
def store_cookies(self, response): headers = filter(lambda h: h[0].lower() == "set-cookie", response.getheaders()) for header in headers: cookie = SimpleCookie(header[1]) for morsel in cookie.values(): if morsel.key not in self._keys: self._keys.append(morsel.key) self._content[morsel.key] = morsel.value logger.debug("--> Set cookie %s: %s" % (morsel.key, morsel.value)) logger.debug("CookieJar contents: %s\n%s" % (self._keys, self._content))
def make_notebook_source_bundle( file, # type: str environment, # type: Environment extra_files=None, # type: typing.Optional[typing.List[str]] ): # type: (...) -> typing.IO[bytes] """Create a bundle containing the specified notebook and python environment. Returns a file-like object containing the bundle tarball. """ if extra_files is None: extra_files = [] base_dir = dirname(file) nb_name = basename(file) manifest = make_source_manifest(nb_name, environment, AppModes.JUPYTER_NOTEBOOK) manifest_add_file(manifest, nb_name, base_dir) manifest_add_buffer(manifest, environment.filename, environment.contents) if extra_files: skip = [nb_name, environment.filename, "manifest.json"] extra_files = sorted(list(set(extra_files) - set(skip))) for rel_path in extra_files: manifest_add_file(manifest, rel_path, base_dir) logger.debug("manifest: %r", manifest) bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle") with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle: # add the manifest first in case we want to partially untar the bundle for inspection bundle_add_buffer(bundle, "manifest.json", json.dumps(manifest, indent=2)) bundle_add_buffer(bundle, environment.filename, environment.contents) bundle_add_file(bundle, nb_name, base_dir) for rel_path in extra_files: bundle_add_file(bundle, rel_path, base_dir) bundle_file.seek(0) return bundle_file
def make_notebook_source_bundle(file, environment, extra_files=None): """Create a bundle containing the specified notebook and python environment. Returns a file-like object containing the bundle tarball. """ if extra_files is None: extra_files = [] base_dir = dirname(file) nb_name = basename(file) manifest = make_source_manifest(nb_name, environment, AppModes.JUPYTER_NOTEBOOK) manifest_add_file(manifest, nb_name, base_dir) manifest_add_buffer(manifest, environment['filename'], environment['contents']) if extra_files: skip = [nb_name, environment['filename'], 'manifest.json'] extra_files = sorted(list(set(extra_files) - set(skip))) for rel_path in extra_files: manifest_add_file(manifest, rel_path, base_dir) logger.debug('manifest: %r', manifest) bundle_file = tempfile.TemporaryFile(prefix='rsc_bundle') with tarfile.open(mode='w:gz', fileobj=bundle_file) as bundle: # add the manifest first in case we want to partially untar the bundle for inspection bundle_add_buffer(bundle, 'manifest.json', json.dumps(manifest, indent=2)) bundle_add_buffer(bundle, environment['filename'], environment['contents']) bundle_add_file(bundle, nb_name, base_dir) for rel_path in extra_files: bundle_add_file(bundle, rel_path, base_dir) bundle_file.seek(0) return bundle_file
def write_manifest(relative_dir, nb_name, environment, output_dir): # type: (str, str, Environment, str) -> typing.Tuple[list, list] """Create a manifest for source publishing the specified notebook. The manifest will be written to `manifest.json` in the output directory.. A requirements.txt file will be created if one does not exist. Returns the list of filenames written. """ manifest_filename = "manifest.json" manifest = make_source_manifest(nb_name, environment, AppModes.JUPYTER_NOTEBOOK) manifest_file = join(output_dir, manifest_filename) created = [] skipped = [] manifest_relative_path = join(relative_dir, manifest_filename) if exists(manifest_file): skipped.append(manifest_relative_path) else: with open(manifest_file, "w") as f: f.write(json.dumps(manifest, indent=2)) created.append(manifest_relative_path) logger.debug("wrote manifest file: %s", manifest_file) environment_filename = environment.filename environment_file = join(output_dir, environment_filename) environment_relative_path = join(relative_dir, environment_filename) if environment.source == "file": skipped.append(environment_relative_path) else: with open(environment_file, "w") as f: f.write(environment.contents) created.append(environment_relative_path) logger.debug("wrote environment file: %s", environment_file) return created, skipped
def write_manifest(relative_dir, nb_name, environment, output_dir): """Create a manifest for source publishing the specified notebook. The manifest will be written to `manifest.json` in the output directory.. A requirements.txt file will be created if one does not exist. Returns the list of filenames written. """ manifest_filename = 'manifest.json' manifest = make_source_manifest(nb_name, environment, AppModes.JUPYTER_NOTEBOOK) manifest_file = join(output_dir, manifest_filename) created = [] skipped = [] manifest_relative_path = join(relative_dir, manifest_filename) if exists(manifest_file): skipped.append(manifest_relative_path) else: with open(manifest_file, 'w') as f: f.write(json.dumps(manifest, indent=2)) created.append(manifest_relative_path) logger.debug('wrote manifest file: %s', manifest_file) environment_filename = environment['filename'] environment_file = join(output_dir, environment_filename) environment_relative_path = join(relative_dir, environment_filename) if environment['source'] == 'file': skipped.append(environment_relative_path) else: with open(environment_file, 'w') as f: f.write(environment['contents']) created.append(environment_relative_path) logger.debug('wrote environment file: %s', environment_file) return created, skipped
def resolve(self, server, app_id, app_mode): metadata = self.get(server) if metadata is None: logger.debug( "No previous deployment to this server was found; this will be a new deployment." ) return app_id, app_mode logger.debug("Found previous deployment data in %s" % self.get_path()) if app_id is None: app_id = metadata.get("app_guid") or metadata.get("app_id") logger.debug("Using saved app ID: %s" % app_id) # app mode cannot be changed on redeployment app_mode = AppModes.get_by_name(metadata.get("app_mode")) return app_id, app_mode
def _do_request(self, method, path, query_params, body, maximum_redirects, extra_headers=None): full_uri = path if query_params is not None: full_uri = "%s?%s" % (path, urlencode(query_params)) headers = self._headers.copy() if extra_headers is not None: headers.update(extra_headers) local_connection = False try: if logger.is_debugging(): logger.debug("Request: %s %s" % (method, full_uri)) logger.debug("Headers:") for key, value in headers.items(): logger.debug("--> %s: %s" % (key, value)) # if we weren't called under a `with` statement, we'll need to manage the # connection here. if self._conn is None: self.__enter__() local_connection = True try: self._conn.request(method, full_uri, body, headers) response = self._conn.getresponse() response_body = response.read().decode("utf-8").strip() if logger.is_debugging(): logger.debug("Response: %s %s" % (response.status, response.reason)) logger.debug("Headers:") for key, value in response.getheaders(): logger.debug("--> %s: %s" % (key, value)) logger.debug("--> %s" % response_body) finally: if local_connection: self.__exit__() # Handle any redirects. if 300 <= response.status < 400: if maximum_redirects == 0: raise http.CannotSendRequest("Too many redirects") location = response.getheader("Location") next_url = urljoin(self._url.geturl(), location) logger.debug("--> Redirected to: %s" % next_url) return self._do_request( method, next_url, query_params, body, maximum_redirects - 1, extra_headers, ) self._handle_set_cookie(response) return self._tweak_response( HTTPResponse(full_uri, response=response, body=response_body)) except ( http.HTTPException, ssl.CertificateError, IOError, OSError, socket.error, socket.herror, socket.gaierror, socket.timeout, ) as exception: logger.debug("An exception occurred processing the HTTP request.", exc_info=True) return HTTPResponse(full_uri, exception=exception)
def override_title_search(connect_server, app_id, app_title): """ Returns a list of abbreviated app data that contains apps with a title that matches the given one and/or the specific app noted by its ID. :param connect_server: the Connect server information. :param app_id: the ID of a specific app to look for, if any. :param app_title: the title to search for. :return: the list of matching apps, each trimmed to ID, name, title, mode URL and dashboard URL. """ def map_app(app, config): """ Creates the abbreviated data dictionary for the specified app and config information. :param app: the raw app data to start with. :param config: the configuration data to use. :return: the abbreviated app data dictionary. """ return { "id": app["id"], "name": app["name"], "title": app["title"], "app_mode": AppModes.get_by_ordinal(app["app_mode"]).name(), "url": app["url"], "config_url": config["config_url"], } def mapping_filter(client, app): """ Mapping/filter function for retrieving apps. We only keep apps that have an app mode of static or Jupyter notebook. The data for the apps we keep is an abbreviated subset. :param client: the client object to use for RStudio Connect calls. :param app: the current app from Connect. :return: the abbreviated data for the app or None. """ # Only keep apps that match our app modes. app_mode = AppModes.get_by_ordinal(app["app_mode"]) if app_mode not in (AppModes.STATIC, AppModes.JUPYTER_NOTEBOOK): return None config = client.app_config(app["id"]) connect_server.handle_bad_response(config) return map_app(app, config) apps = retrieve_matching_apps( connect_server, filters={ "filter": "min_role:editor", "search": app_title }, mapping_function=mapping_filter, limit=5, ) if app_id: found = next((app for app in apps if app["id"] == app_id), None) if not found: try: app = get_app_info(connect_server, app_id) mode = AppModes.get_by_ordinal(app["app_mode"]) if mode in (AppModes.STATIC, AppModes.JUPYTER_NOTEBOOK): apps.append( map_app(app, get_app_config(connect_server, app_id))) except RSConnectException: logger.debug( 'Error getting info for previous app_id "%s", skipping.', app_id) return apps