def get_notebook_path():
"""Returns the absolute path of the Notebook or None if it cannot be determined
NOTE: works only when the security is token-based or there is also no password
"""
connection_file = os.path.basename(ipykernel.get_connection_file())
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
# Assumes you've already run `jupyter notebook --generate-config` to generate
# `jupyter_notebook_config.py` and have edited and/or uncommented the line
# containing `c.FileContentsManager.root_dir =`:
c = Config()
file_path = os.path.join(jupyter_config_dir(),
'jupyter_notebook_config.py')
exec(open(file_path).read())
root_dir = c['FileContentsManager']['root_dir']
for srv in notebookapp.list_running_servers():
try:
if srv['token'] == '' and not srv[
'password']: # No token and no password, ahem...
req = urllib.request.urlopen(srv['url'] + 'api/sessions')
else:
req = urllib.request.urlopen(srv['url'] +
'api/sessions?token=' +
srv['token'])
sessions = json.load(req)
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
return os.path.abspath(
os.path.join(root_dir, sess['notebook']['path']))
except:
pass # There may be stale entries in the runtime directory
return None
def get_current_notebook_path():
from notebook import notebookapp
import ipykernel
"""Returns the absolute path of the Notebook or None if it cannot be determined
NOTE: works only when the security is token-based or there is also no password
"""
connection_file = os.path.basename(ipykernel.get_connection_file())
kernel_id = connection_file.split("-", 1)[1].split(".")[0]
for srv in notebookapp.list_running_servers():
try:
if (srv["token"] == "" and
not srv["password"]): # No token and no password, ahem...
req = urllib.request.urlopen(srv["url"] + "api/sessions")
else:
req = urllib.request.urlopen(srv["url"] +
"api/sessions?token=" +
srv["token"])
sessions = json.load(req)
for sess in sessions:
if sess["kernel"]["id"] == kernel_id:
return os.path.join(srv["notebook_dir"],
sess["notebook"]["path"])
except:
pass # There may be stale entries in the runtime directory
return None
def get_notebook_path():
"""Returns the asb path of the Notebook or None if it cannot be determined.
NOTE: works only when the security is token-based or there is no password
"""
log.info("Retrieving absolute path of the active notebook")
connection_file = os.path.basename(ipykernel.get_connection_file())
try:
kernel_id = re.search(r"kernel-(.+?)\.json", connection_file).group(1)
except AttributeError:
log.error("Could not load kernel id.")
return None
for srv in notebookapp.list_running_servers():
sess_url = "%sapi/sessions" % srv["url"]
# No token and no password
if srv["token"] == "" and not srv['password']:
req = requests.get(sess_url)
else:
token_query_param = "?token=%s" % srv["token"]
req = requests.get(sess_url + token_query_param)
if req.status_code != 200:
log.error("Session request failed with status code %s" %
req.status_code)
return None
try:
sessions = json.loads(req.text)
except json.JSONDecodeError as e:
log.error("Could not parse session response: %s" % e)
return None
for sess in sessions:
if sess["kernel"]["id"] == kernel_id:
return os.path.join(srv["notebook_dir"],
sess["notebook"]["path"])
return None
def connect_info(self, arg_s):
"""Print information for connecting other clients to this kernel
It will print the contents of this session's connection file, as well as
shortcuts for local clients.
In the simplest case, when called from the most recently launched kernel,
secondary clients can be connected, simply with:
$> jupyter <app> --existing
"""
try:
connection_file = get_connection_file()
info = get_connection_info(unpack=False)
except Exception as e:
warnings.warn("Could not get connection info: %r" % e)
return
# if it's in the default dir, truncate to basename
if jupyter_runtime_dir() == os.path.dirname(connection_file):
connection_file = os.path.basename(connection_file)
print(info + '\n')
print("Paste the above JSON into a file, and connect with:\n"
" $> jupyter <app> --existing <file>\n"
"or, if you are local, you can connect with just:\n"
" $> jupyter <app> --existing {0}\n"
"or even just:\n"
" $> jupyter <app> --existing\n"
"if this is the most recent Jupyter kernel you have started.".
format(connection_file))
def notebook_path(self):
"""Returns the absolute path of the Notebook or None if it cannot be determined
NOTE: works only when the security is token-based or there is also no password
"""
try:
connection_file = os.path.basename(ipykernel.get_connection_file())
kernel_id = connection_file.split("-", 1)[1].split(".")[0]
for srv in notebookapp.list_running_servers():
try:
base_url = (
f"{self.__public_url}/user/{self.__jup_user}/api/sessions"
)
req = urllib.request.urlopen(
f"{base_url}?token={self.__jup_token}")
sessions = json.load(req)
for sess in sessions:
if sess["kernel"]["id"] == kernel_id:
return os.path.join(srv["notebook_dir"],
sess["notebook"]["path"])
except urllib.error.HTTPError:
pass
except IndexError:
pass
except RuntimeError:
pass
return None
def get_full_notebook_information() -> Optional[dict]:
"""
Javascriptless method to get information about the current Jupyter sessions and the one matching
this kernel.
:return:
"""
try: # Respect those that opt not to use IPython
from notebook import notebookapp
import ipykernel
except ImportError:
return None
connection_file = os.path.basename(ipykernel.get_connection_file())
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
servers = notebookapp.list_running_servers()
for server in servers:
try:
passwordless = not server['token'] and not server['password']
url = server['url'] + 'api/sessions' + (
'' if passwordless else '?token={}'.format(server['token']))
sessions = json.load(urllib.request.urlopen(url))
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
return {
'server': server,
'session': sess,
}
except:
pass
return None
def notebook_path():
from notebook import notebookapp
import urllib
import json
import os
import ipykernel
"""Returns the absolute path of the Notebook or None if it cannot be determined
NOTE: works only when the security is token-based or there is also no password
"""
connection_file = os.path.basename(ipykernel.get_connection_file())
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
for srv in notebookapp.list_running_servers():
try:
if srv['token'] == '' and not srv[
'password']: # No token and no password, ahem...
req = urllib.request.urlopen(srv['url'] + 'api/sessions')
else:
req = urllib.request.urlopen(srv['url'] +
'api/sessions?token=' +
srv['token'])
sessions = json.load(req)
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
return os.path.join(srv['notebook_dir'],
sess['notebook']['path'])
except:
pass # There may be stale entries in the runtime directory
return None
def get_notebook_id():
"""Returns the absolute path of the Notebook or None if it cannot be determined
NOTE: works only when the security is token-based or there is also no password
"""
connection_file = os.path.basename(ipykernel.get_connection_file())
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
for srv in notebookapp.list_running_servers():
try:
if srv['token'] == '' and not srv[
'password']: # No token and no password, ahem...
req = urllib.request.urlopen(srv['url'] + 'api/sessions')
else:
req = urllib.request.urlopen(srv['url'] +
'api/sessions?token=' +
srv['token'])
sessions = json.load(req)
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
file_name = os.path.basename(
os.path.join(srv['notebook_dir'],
sess['notebook']['path']))
m = re.match(r'\d\d', file_name)
if m:
return m.group()
else:
print(f"Couldn't parse notebook name: {file_name}")
except:
pass # There may be stale entries in the runtime directory
return None
def this_notebook():
"""Returns the absolute path of the Notebook or None if it cannot be determined
NOTE: works only when the security is token-based or there is also no password
"""
connection_file = os.path.basename(ipykernel.get_connection_file())
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
for srv in notebookapp.list_running_servers():
try:
if srv['token'] == '' and not srv[
'password']: # No token and no password, ahem...
req = urllib.request.urlopen(srv['url'] + 'api/sessions')
else:
req = urllib.request.urlopen(srv['url'] +
'api/sessions?token=' +
srv['token'])
sessions = json.load(req)
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
np = sess['notebook']['path']
if np[0] == '/': np = np[1:]
return os.path.join(srv['notebook_dir'], np)
# except Exception as exc:
# print(traceback.format_exc())
except:
pass # There may be stale entries in the runtime directory
return None
def jupyter_session_info() -> dict or None:
"""
Returns information about the running jupyter sessions, such as the name and content of the notebook
"""
# https://stackoverflow.com/questions/12544056/how-do-i-get-the-current-ipython-notebook-name
# https://github.com/jupyter/notebook/issues/1000
if not in_jupyter_environment():
raise Exception('Not in a Jupyter environment.')
try:
import ipykernel
except ImportError:
raise Exception('Failed to import Jupyter kernel.')
try:
from urllib.request import urlopen, pathname2url
except ImportError:
from urllib import urlopen, pathname2url
try: # Python 3 (see Edit2 below for why this may not work in Python 2)
from notebook.notebookapp import list_running_servers
except ImportError: # Python 2
import warnings
from IPython.utils.shimmodule import ShimWarning
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=ShimWarning)
from IPython.html.notebookapp import list_running_servers
connection_file_path = ipykernel.get_connection_file()
connection_file = os.path.basename(connection_file_path)
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
for server in list(list_running_servers()):
api_url = server['url']
try:
token_param = ""
if server['token'] == '' and not server[
'password']: # No token and no password
token_param = ""
else:
token_param = '?token=' + server['token']
response = urlopen(api_url + 'api/sessions' + token_param)
sessions = json.loads(response.read().decode())
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
notebook_path = sess['notebook']['path']
content_url = api_url + 'api/contents/' + pathname2url(
notebook_path).lstrip("/") + token_param
return json.loads(urlopen(content_url).read().decode())
except:
pass # There may be stale entries in the runtime directory
return None
def __find_notebook_path(self):
connection_file = os.path.basename(ipykernel.get_connection_file())
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
token = os.environ.get("JUPYTERHUB_API_TOKEN")
netid = self.__netid
response = requests.get(f'http://127.0.0.1:8888/user/{netid}/api/sessions?token={token}')
response.raise_for_status()
sessions = response.json()
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
return sess['notebook']['path']
def get_layer_vrt_path(gLayer):
kernel_id = os.path.basename(
ipykernel.get_connection_file()).split('-', 1)[1].split('.')[0]
request_url = "{}/{}/{}".format(gLayer.config.vis_server.base_url,
kernel_id, gLayer.name)
r = requests.get(request_url)
response = r.json()
return response['provider']['vrt_path']
def get_layer_vrt_path(gLayer):
kernel_id = os.path.basename(ipykernel.get_connection_file()).split(
'-', 1)[1].split('.')[0]
request_url = "{}/{}/{}".format(gLayer.config.vis_server.base_url,
kernel_id, gLayer.name)
r = requests.get(request_url)
response = r.json()
return response['provider']['vrt_path']
def connect_info(self, arg_s):
"""Print information for connecting other clients to this kernel
It will print the contents of this session's connection file, as well as
shortcuts for local clients.
In the simplest case, when called from the most recently launched kernel,
secondary clients can be connected, simply with:
$> ipython <app> --existing
"""
from IPython.core.application import BaseIPythonApplication as BaseIPApp
if BaseIPApp.initialized():
app = BaseIPApp.instance()
security_dir = app.profile_dir.security_dir
profile = app.profile
else:
profile = 'default'
security_dir = ''
try:
connection_file = get_connection_file()
info = get_connection_info(unpack=False)
except Exception as e:
error("Could not get connection info: %r" % e)
return
# add profile flag for non-default profile
profile_flag = "--profile %s" % profile if profile != 'default' else ""
# if it's in the security dir, truncate to basename
if security_dir == os.path.dirname(connection_file):
connection_file = os.path.basename(connection_file)
print (info + '\n')
print ("Paste the above JSON into a file, and connect with:\n"
" $> ipython <app> --existing <file>\n"
"or, if you are local, you can connect with just:\n"
" $> ipython <app> --existing {0} {1}\n"
"or even just:\n"
" $> ipython <app> --existing {1}\n"
"if this is the most recent IPython session you have started.".format(
connection_file, profile_flag
)
)
def _handle_magics(self, magic_code, code):
"""
Handle cell magics
"""
_exec_status = False
_content = None
err_content = None
if(magic_code == 'connect_info'):
try:
connection_file = get_connection_file()
_content = get_connection_info(unpack=False)
except Exception as e:
error("Could not get connection info: %r" % e)
return
if(magic_code == 'history'):
_args = re.search(r'^%(\S+)(?:\s*)(\d*)', code)
self._shell.run_raw('history(' + _args.group(2) + ')')
_content = self._shell.output[:-1]
if(magic_code == 'help'):
_args = re.search(r'^%(\S+)(?:\s*)(\S*)', code)
_content = self._shell.get_info(_args.group(2))
if(magic_code == 'image'):
_args = re.search(r'^%(\S+)(?:\s*)(\S+)', code)
if _args is not None:
return self._show_image_inline(_args.group(2))
if(magic_code == 'flush'):
self._shell.flush()
_content = ''
if(_content is not None):
execute_content = {'execution_count': self.execution_count,
'data': {'text/plain': _content},
'metadata': {}}
self.send_response(self.iopub_socket, 'execute_result',
execute_content)
_exec_status = True
else:
err_content = {'execution_count': self.execution_count,
'ename': str('CellMagicError'),
'evalue': str(1),
'traceback': ['Invalid cell magic']}
self.send_response(self.iopub_socket, 'error', err_content)
return _exec_status, err_content
def _handle_magics(self, magic_code, code):
"""
Handle cell magics
"""
_exec_status = False
_content = None
err_content = None
if(magic_code == 'connect_info'):
try:
connection_file = get_connection_file()
_content = get_connection_info(unpack=False)
except Exception as e:
error("Could not get connection info: %r" % e)
return
if(magic_code == 'history'):
_args = re.search(r'^%(\S+)(?:\s*)(\d*)', code)
self._shell.run_raw('history(' + _args.group(2) + ')')
_content = self._shell.output[:-1]
if(magic_code == 'help'):
_args = re.search(r'^%(\S+)(?:\s*)(\S*)', code)
_content = self._shell.get_info(_args.group(2))
if(magic_code == 'image'):
_args = re.search(r'^%(\S+)(?:\s*)(\S+)', code)
if _args is not None:
return self._show_image_inline(_args.group(2))
if(magic_code == 'flush'):
_content = ''
if(_content is not None):
execute_content = {'execution_count': self.execution_count,
'data': {'text/plain': _content},
'metadata': {}}
self.send_response(self.iopub_socket, 'execute_result',
execute_content)
_exec_status = True
else:
err_content = {'execution_count': self.execution_count,
'ename': str('CellMagicError'),
'evalue': str(1),
'traceback': ['Invalid cell magic']}
self.send_response(self.iopub_socket, 'error', err_content)
return _exec_status, err_content
def get_notebook_kernel_id(cls)->str:
if cls.kernel_id is None:
try:
import re
try:
import ipykernel as kernel
except:
from IPython.lib import kernel
connection_file = kernel.get_connection_file()
cls.kernel_id = re.search('kernel-(.*).json', connection_file).group(1)
except:
import uuid
cls.kernel_id = f"{uuid.uuid4()}"
return cls.kernel_id
def notebook_word():
"""Convert a Jupyter notebook runtime connection file to a simple word.
Must be called within a Jupyter notebook. Combined with 'notebook_file',
these functions allow you to connect to an active Jupyter notebook at the
command line.
notebook_file(notebook_word()) will return back the name of the connection
file.
Returns
-------
word : str
"""
from ipykernel import get_connection_file
return wordhash.word(str(os.path.basename(get_connection_file())))
def _get_nbname(self):
from notebook import notebookapp
import urllib
import json
import os
import ipykernel
connection_file = os.path.basename(ipykernel.get_connection_file())
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
import urllib.request
opener = urllib.request.build_opener()
import os
user_id = get_user_id(debug=self.debug)
opener.addheaders = [('Authorization', 'Token '+os.environ.get('JPY_API_TOKEN'))]
req = opener.open('http://localhost:8888/sdr/user/'+user_id[0]+'/api/sessions')
raw_data = req.read()
data = json.loads(raw_data.decode('utf-8'))
for each in data:
if each['kernel']['id'] == kernel_id:
nbname = each['notebook']['name']
return nbname
def notebook_path():
connection_file = os.path.basename(ipykernel.get_connection_file())
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
for srv in notebookapp.list_running_servers():
try:
if srv['token'] == '' and not srv[
'password']: # No token and no password, ahem...
req = urllib.request.urlopen(srv['url'] + 'api/sessions')
else:
req = urllib.request.urlopen(srv['url'] +
'api/sessions?token=' +
srv['token'])
sessions = json.load(req)
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
return os.path.join(srv['notebook_dir'],
sess['notebook']['path'])
except:
pass
return None
def connect_info(self, arg_s):
"""Print information for connecting other clients to this kernel
It will print the contents of this session's connection file, as well as
shortcuts for local clients.
In the simplest case, when called from the most recently launched kernel,
secondary clients can be connected, simply with:
$> jupyter <app> --existing
"""
try:
connection_file = get_connection_file()
info = get_connection_info(unpack=False)
except Exception as e:
warnings.warn("Could not get connection info: %r" % e)
return
# if it's in the default dir, truncate to basename
if jupyter_runtime_dir() == os.path.dirname(connection_file):
connection_file = os.path.basename(connection_file)
print (info + '\n')
print ("Paste the above JSON into a file, and connect with:\n"
" $> jupyter <app> --existing <file>\n"
"or, if you are local, you can connect with just:\n"
" $> jupyter <app> --existing {0}\n"
"or even just:\n"
" $> jupyter <app> --existing\n"
"if this is the most recent Jupyter kernel you have started.".format(
connection_file
)
)
def _get_jupyter_notebook_filename(cls):
if not (sys.argv[0].endswith(os.path.sep + 'ipykernel_launcher.py') or
sys.argv[0].endswith(os.path.join(os.path.sep, 'ipykernel', '__main__.py'))) \
or len(sys.argv) < 3 or not sys.argv[2].endswith('.json'):
return None
# we can safely assume that we can import the notebook package here
# noinspection PyBroadException
try:
# noinspection PyPackageRequirements
from notebook.notebookapp import list_running_servers
import requests
current_kernel = sys.argv[2].split(os.path.sep)[-1].replace(
'kernel-', '').replace('.json', '')
# noinspection PyBroadException
try:
server_info = next(list_running_servers())
except Exception:
# on some jupyter notebook versions this function can crash on parsing the json file,
# we will parse it manually here
# noinspection PyPackageRequirements
import ipykernel
from glob import glob
import json
for f in glob(
os.path.join(
os.path.dirname(ipykernel.get_connection_file()),
'nbserver-*.json')):
# noinspection PyBroadException
try:
with open(f, 'r') as json_data:
server_info = json.load(json_data)
except Exception:
server_info = None
if server_info:
break
try:
r = requests.get(url=server_info['url'] + 'api/sessions',
headers={
'Authorization':
'token {}'.format(
server_info.get('token', '')),
})
except requests.exceptions.SSLError:
# disable SSL check warning
from urllib3.exceptions import InsecureRequestWarning
# noinspection PyUnresolvedReferences
requests.packages.urllib3.disable_warnings(
category=InsecureRequestWarning)
# fire request
r = requests.get(url=server_info['url'] + 'api/sessions',
headers={
'Authorization':
'token {}'.format(
server_info.get('token', '')),
},
verify=False)
# enable SSL check warning
import warnings
warnings.simplefilter('default', InsecureRequestWarning)
r.raise_for_status()
notebooks = r.json()
cur_notebook = None
for n in notebooks:
if n['kernel']['id'] == current_kernel:
cur_notebook = n
break
notebook_path = cur_notebook['notebook'].get('path', '')
notebook_name = cur_notebook['notebook'].get('name', '')
is_google_colab = False
# check if this is google.colab, then there is no local file
# noinspection PyBroadException
try:
# noinspection PyPackageRequirements
from IPython import get_ipython
if get_ipython() and 'google.colab' in get_ipython(
).extension_manager.loaded:
is_google_colab = True
except Exception:
pass
if is_google_colab:
script_entry_point = str(notebook_name or 'notebook').replace(
'>', '_').replace('<', '_').replace('.ipynb', '.py')
if not script_entry_point.lower().endswith('.py'):
script_entry_point += '.py'
local_ipynb_file = None
else:
# always slash, because this is from uri (so never backslash not even oon windows)
entry_point_filename = notebook_path.split('/')[-1]
# now we should try to find the actual file
entry_point = (Path.cwd() / entry_point_filename).absolute()
if not entry_point.is_file():
entry_point = (Path.cwd() / notebook_path).absolute()
# get local ipynb for observer
local_ipynb_file = entry_point.as_posix()
# now replace the .ipynb with .py
# we assume we will have that file available with the Jupyter notebook plugin
entry_point = entry_point.with_suffix('.py')
script_entry_point = entry_point.as_posix()
# install the post store hook,
# notice that if we do not have a local file we serialize/write every time the entire notebook
cls._jupyter_install_post_store_hook(local_ipynb_file,
is_google_colab)
return script_entry_point
except Exception:
return None
def _get_kernel_id():
""" Returns the kernel ID of the ipykernel.
"""
connection_file = os.path.basename(ipykernel.get_connection_file())
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
return kernel_id
def _get_jupyter_notebook_filename(cls):
if not (sys.argv[0].endswith(os.path.sep + 'ipykernel_launcher.py') or
sys.argv[0].endswith(os.path.join(os.path.sep, 'ipykernel', '__main__.py'))) \
or len(sys.argv) < 3 or not sys.argv[2].endswith('.json'):
return None
# we can safely assume that we can import the notebook package here
# noinspection PyBroadException
try:
# noinspection PyPackageRequirements
from notebook.notebookapp import list_running_servers
import requests
current_kernel = sys.argv[2].split(os.path.sep)[-1].replace('kernel-', '').replace('.json', '')
# noinspection PyBroadException
try:
server_info = next(list_running_servers())
except Exception:
# on some jupyter notebook versions this function can crash on parsing the json file,
# we will parse it manually here
# noinspection PyPackageRequirements
import ipykernel
from glob import glob
import json
for f in glob(os.path.join(os.path.dirname(ipykernel.get_connection_file()), '??server-*.json')):
# noinspection PyBroadException
try:
with open(f, 'r') as json_data:
server_info = json.load(json_data)
except Exception:
server_info = None
if server_info:
break
cookies = None
password = None
if server_info and server_info.get('password'):
# we need to get the password
from ....config import config
password = config.get('development.jupyter_server_password', '')
if not password:
_logger.warning(
'Password protected Jupyter Notebook server was found! '
'Add `sdk.development.jupyter_server_password=<jupyter_password>` to ~/clearml.conf')
return os.path.join(os.getcwd(), 'error_notebook_not_found.py')
r = requests.get(url=server_info['url'] + 'login')
cookies = {'_xsrf': r.cookies.get('_xsrf', '')}
r = requests.post(server_info['url'] + 'login?next', cookies=cookies,
data={'_xsrf': cookies['_xsrf'], 'password': password})
cookies.update(r.cookies)
try:
r = requests.get(
url=server_info['url'] + 'api/sessions', cookies=cookies,
headers={'Authorization': 'token {}'.format(server_info.get('token', '')), })
except requests.exceptions.SSLError:
# disable SSL check warning
from urllib3.exceptions import InsecureRequestWarning
# noinspection PyUnresolvedReferences
requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
# fire request
r = requests.get(
url=server_info['url'] + 'api/sessions', cookies=cookies,
headers={'Authorization': 'token {}'.format(server_info.get('token', '')), }, verify=False)
# enable SSL check warning
import warnings
warnings.simplefilter('default', InsecureRequestWarning)
# send request to the jupyter server
try:
r.raise_for_status()
except Exception as ex:
_logger.warning('Failed accessing the jupyter server{}: {}'.format(
' [password={}]'.format(password) if server_info.get('password') else '', ex))
return os.path.join(os.getcwd(), 'error_notebook_not_found.py')
notebooks = r.json()
cur_notebook = None
for n in notebooks:
if n['kernel']['id'] == current_kernel:
cur_notebook = n
break
notebook_path = cur_notebook['notebook'].get('path', '')
notebook_name = cur_notebook['notebook'].get('name', '')
is_google_colab = False
# check if this is google.colab, then there is no local file
# noinspection PyBroadException
try:
# noinspection PyPackageRequirements
from IPython import get_ipython
if get_ipython() and 'google.colab' in get_ipython().extension_manager.loaded:
is_google_colab = True
except Exception:
pass
if is_google_colab:
script_entry_point = str(notebook_name or 'notebook').replace(
'>', '_').replace('<', '_').replace('.ipynb', '.py')
if not script_entry_point.lower().endswith('.py'):
script_entry_point += '.py'
local_ipynb_file = None
else:
# always slash, because this is from uri (so never backslash not even on windows)
entry_point_filename = notebook_path.split('/')[-1]
# now we should try to find the actual file
entry_point = (Path.cwd() / entry_point_filename).absolute()
if not entry_point.is_file():
entry_point = (Path.cwd() / notebook_path).absolute()
# fix for VSCode pushing uuid at the end of the notebook name.
if not entry_point.exists():
# noinspection PyBroadException
try:
alternative_entry_point = '-'.join(entry_point_filename.split('-')[:-5])+'.ipynb'
# now we should try to find the actual file
entry_point_alternative = (Path.cwd() / alternative_entry_point).absolute()
if not entry_point_alternative.is_file():
entry_point_alternative = (Path.cwd() / alternative_entry_point).absolute()
# If we found it replace it
if entry_point_alternative.exists():
entry_point = entry_point_alternative
except Exception as ex:
_logger.warning('Failed accessing jupyter notebook {}: {}'.format(notebook_path, ex))
# get local ipynb for observer
local_ipynb_file = entry_point.as_posix()
# now replace the .ipynb with .py
# we assume we will have that file available with the Jupyter notebook plugin
entry_point = entry_point.with_suffix('.py')
script_entry_point = entry_point.as_posix()
# install the post store hook,
# notice that if we do not have a local file we serialize/write every time the entire notebook
cls._jupyter_install_post_store_hook(local_ipynb_file, is_google_colab)
return script_entry_point
except Exception:
return None
def get_kernel_id(kernel):
connection_file_path = ipykernel.get_connection_file()
connection_file = os.path.basename(connection_file_path)
return connection_file.split('-', 1)[1].split('.')[0]
# solution for:
# https://stackoverflow.com/questions/12544056/how-to-i-get-the-current-ipython-notebook-name
#
# aimed at:
# IPython 4.2.0 and Python 3.5
import json
import os
import urllib.request
import ipykernel
connection_file_path = ipykernel.get_connection_file()
connection_file = os.path.basename(connection_file_path)
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
response = urllib.request.urlopen('http://127.0.0.1:8888/api/sessions')
sessions = json.loads(response.read().decode())
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
print(sess['notebook']['path'])
break
import json
import os
import requests
import IPython
import ipykernel as kernel
connection_file_path = kernel.get_connection_file()
connection_file = os.path.basename(connection_file_path)
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
def executeCell(x=0):
''' executes the code in cell no 'x' (zero-based indexing)
'''
sessions = requests.get('http://127.0.0.1:8888/api/sessions').json()
ipynbFileName = ""
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
ipynbFileName = sess['notebook']['path']
ipynbFileName = ipynbFileName.split(os.sep)[-1]
break
# read this notebook's file
if ipynbFileName != "":
with open(ipynbFileName) as f:
nb = json.load(f)
# locate cell's code
if type(nb) == dict:
try:
code = ""
if nb[u'cells'][x][u'cell_type'] == u'code':
for s in nb[u'cells'][x]['source']:
def load_ipython_extension(ipython):
"""
Any module file that define a function named `load_ipython_extension`
can be loaded via `%load_ext module.path` or be configured to be
autoloaded by IPython at startup time.
"""
# You can register the class itself without instantiating it.
# IPython will call the default constructor on it.
# https://gist.github.com/mbdevpl/f97205b73610dd30254652e7817f99cb
connection_file_path = get_connection_file()
connection_file = os.path.basename(connection_file_path)
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
config_files = {'linux': "/.local/share/nb-agent/runtime.json",
'darwin': "/Library/nb-agent/runtime.json",
'windows': "/.local/share/nb-agent/runtime.json"} # I am guessing!
config_file = str(Path.home()) + config_files[sys.platform]
with open(config_file) as json_file:
data = json.load(json_file)
port = data['magic-server-port']
jupyter_port = data['jupyter-port']
ipynb_filename = 'unknown'
last_active = 'unknown'
servers = serverapp.list_running_servers() # for 3.0.9 2021-03-04
prefix = 'http://127.0.0.1:%s/api/sessions?token=' % (jupyter_port,)
for svr in servers:
response = urllib.request.urlopen(prefix + svr['token'])
sessions = json.loads(response.read().decode())
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
session_id = sess['id']
ipynb_filename = (sess['notebook']['path'])
last_active = (sess['kernel']['last_activity'])
break
gw = NBAgateway()
gw.start_server()
ipython.push({'nba_gateway': gw}) # Not sure that this is needed!
ctx = zmq.Context()
sock = ctx.socket(zmq.REQ)
sock.connect('tcp://localhost:' + str(port))
try:
msg = {'action': 'notify',
'session-id': session_id,
'nba-gateway-port': gw.port,
'dir': os.getcwd(),
'ipynb-file': ipynb_filename,
'last-active': last_active}
msg_str = json.dumps(msg)
sock.send_string(msg_str)
print('MiniZinc Notebook Agent Communicator version %s'
% (__version__,))
print('This session (%s) connected to nb-agent at port %s.' % (session_id, port))
print(json.loads(sock.recv()))
except KeyError:
print('''Not able to communicate with nb-agent.''')
magics = MznbMagics(ipython, port, session_id)
ipython.register_magics(magics)
def _get_kernel_id() -> str:
""" Returns the kernel ID of the ipykernel.
"""
connection_file = Path(ipykernel.get_connection_file()).stem
kernel_id = connection_file.split('-', 1)[1]
return kernel_id
def setup_console_widget(self, kernel=None):
"""
Create and return console QWidget. If Jupyter / IPython is installed
this widget will be a full-featured IPython console. If Jupyter is unavailable
it will fallback to a pyqtgraph.console.ConsoleWidget.
If the app is started in an Jupyter notebook, the console will be
connected to the notebook's IPython kernel.
the returned console_widget will also be accessible as self.console_widget
In order to see the console widget, remember to insert it into an existing
window or call self.console_widget.show() to create a new window
"""
if CONSOLE_TYPE == 'pyqtgraph.console':
self.console_widget = pyqtgraph.console.ConsoleWidget(
namespace={
'app': self,
'pg': pg,
'np': np
},
text="ScopeFoundry Console")
elif CONSOLE_TYPE == 'qtconsole':
if kernel == None:
try: # try to find an existing kernel
#https://github.com/jupyter/notebook/blob/master/docs/source/examples/Notebook/Connecting%20with%20the%20Qt%20Console.ipynb
import ipykernel as kernel
conn_file = kernel.get_connection_file()
import qtconsole.qtconsoleapp
self.qtconsole_app = qtconsole.qtconsoleapp.JupyterQtConsoleApp(
)
self.console_widget = self.qtconsole_app.new_frontend_connection(
conn_file)
self.console_widget.setWindowTitle(
"ScopeFoundry IPython Console")
except: # make your own new in-process kernel
# https://github.com/ipython/ipython-in-depth/blob/master/examples/Embedding/inprocess_qtconsole.py
self.kernel_manager = QtInProcessKernelManager()
self.kernel_manager.start_kernel()
self.kernel = self.kernel_manager.kernel
self.kernel.shell.banner1 += """
ScopeFoundry Console
Variables:
* np: numpy package
* app: the ScopeFoundry App object
"""
self.kernel.gui = 'qt4'
self.kernel.shell.push({'np': np, 'app': self})
self.kernel_client = self.kernel_manager.client()
self.kernel_client.start_channels()
#self.console_widget = RichIPythonWidget()
self.console_widget = RichJupyterWidget()
self.console_widget.setWindowTitle(
"ScopeFoundry IPython Console")
self.console_widget.kernel_manager = self.kernel_manager
self.console_widget.kernel_client = self.kernel_client
else:
import qtconsole.qtconsoleapp
self.qtconsole_app = qtconsole.qtconsoleapp.JupyterQtConsoleApp(
)
self.console_widget = self.qtconsole_app.new_frontend_connection(
kernel.get_connection_file())
self.console_widget.setWindowTitle(
"ScopeFoundry IPython Console")
else:
raise ValueError("CONSOLE_TYPE undefined")
return self.console_widget
def start(self):
"""Start the session against actual livy server."""
self._spark_events.emit_session_creation_start_event(
self.guid, self.kind)
self._printed_resource_warning = False
try:
connection_file = os.path.basename(ipykernel.get_connection_file())
if 'kernel' in connection_file:
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
self.properties['conf'][
'spark.yarn.appMasterEnv.HOPSWORKS_KERNEL_ID'] = kernel_id
self.properties['conf'][
'spark.executorEnv.HOPSWORKS_KERNEL_ID'] = kernel_id
if 'hops.util' in sys.modules:
util.attach_jupyter_configuration_to_notebook(kernel_id)
r = self._http_client.post_session(self.properties)
self.id = r[u"id"]
self.status = str(r[u"state"])
self.ipython_display.writeln(u"Starting Spark application")
# Start heartbeat thread to keep Livy interactive session alive.
self._start_heartbeat_thread()
# We wait for livy_session_startup_timeout_seconds() for the session to start up.
try:
self.wait_for_idle(conf.livy_session_startup_timeout_seconds())
except LivyClientTimeoutException:
raise LivyClientTimeoutException(
u"Session {} did not start up in {} seconds.".format(
self.id, conf.livy_session_startup_timeout_seconds()))
html = get_sessions_info_html([self], self.id)
self.ipython_display.html(html)
command = Command("spark")
(success, out, mimetype) = command.execute(self)
if success:
self.ipython_display.writeln(
u"SparkSession available as 'spark'.")
self.sql_context_variable_name = "spark"
else:
command = Command("sqlContext")
(success, out, mimetype) = command.execute(self)
if success:
self.ipython_display.writeln(
u"SparkContext available as 'sc'.")
if ("hive" in out.lower()):
self.ipython_display.writeln(
u"HiveContext available as 'sqlContext'.")
else:
self.ipython_display.writeln(
u"SqlContext available as 'sqlContext'.")
self.sql_context_variable_name = "sqlContext"
else:
raise SqlContextNotFoundException(
u"Neither SparkSession nor HiveContext/SqlContext is available."
)
except Exception as e:
self._spark_events.emit_session_creation_end_event(
self.guid, self.kind, self.id, self.status, False,
e.__class__.__name__, str(e))
raise
else:
self._spark_events.emit_session_creation_end_event(
self.guid, self.kind, self.id, self.status, True, "", "")
def get_kernel_id(kernel):
connection_file_path = ipykernel.get_connection_file()
connection_file = os.path.basename(connection_file_path)
return connection_file.split('-', 1)[1].split('.')[0]
def get_kernel_id():
"""Return the kernel id."""
import ipykernel
connection_file = os.path.basename(ipykernel.get_connection_file())
return connection_file.split('-', 1)[1].split('.')[0]
