def launch_console(self):
# If IPython failed to load, bail out.
if not IPYTHON_LOADED:
print "Unable to load IPython"
return
#if self.kernel is None:
try:
# This will throw if the kernel is not yet running
get_connection_file()
InternalIPKernel().new_qt_console()
except RuntimeError:
self.init_kernel()
self.kernel.new_qt_console()
def print_kernel_connection():
import json
import os
import urllib
import IPython
import re
from IPython.lib import kernel
connection_file_path = kernel.get_connection_file()
connection_file = os.path.basename(connection_file_path)
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
r = urllib.request.urlopen('http://127.0.0.1:8888/api/sessions')
sessions = json.loads(r.read().decode(r.info().get_param('charset')
or 'utf-8'))
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
jpath = sess['notebook']['path']
break
name = re.search('.*/(.*).ipynb', jpath).group(1)
print('PROMPT_COMMAND=\'echo -ne "\\033]0; %s \\007"\'' % name)
print('ssh [email protected] -t \'ipython console --existing %s \'' %
(connection_file_path))
print('')
def get_current_notebook_path():
import json
import os
import urllib2
import IPython
from IPython.lib import kernel
connection_file_path = kernel.get_connection_file()
connection_file = os.path.basename(connection_file_path)
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
# Updated answer with semi-solutions for both IPython 2.x and IPython < 2.x
if IPython.version_info[0] < 2:
## Not sure if it's even possible to get the port for the
## notebook app; so just using the default...
notebooks = json.load(urllib2.urlopen('http://127.0.0.1:8888/notebooks'))
for nb in notebooks:
if nb['kernel_id'] == kernel_id:
path = nb.get('name')
break
else:
sessions = json.load(urllib2.urlopen('http://127.0.0.1:8888/api/sessions'))
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
path = sess.get('notebook').get('path')
break
return path
def get_ipython_notebook_path():
"""Return the path to the current notebook file.
This is a horrible hack, but there doesn't seem to be another way to
do it.
"""
# Partly from: http://stackoverflow.com/a/13055551/1144479
from IPython.core.getipython import get_ipython
import requests
import os
from IPython.lib import kernel
cf = kernel.get_connection_file()
kernel_id = cf.split('-', 1)[1].split('.')[0]
r = requests.get('http://127.0.0.1:8888/api/sessions')
r.json()
notebook_path = None
for s in requests.get('http://127.0.0.1:8888/api/sessions').json():
if s['kernel']['id'] == kernel_id:
notebook_path = os.path.join(get_ipython().starting_dir,
s['notebook']['path'],
s['notebook']['name'])
break
return notebook_path
def get_ipython_server_info():
"""Return the notebook server info as a dict, using an egregious hack.
This will be correct only if there is only one IPython notebook
instance running for each get_ipython().starting_dir
"""
import json
import os
from IPython.core.getipython import get_ipython
from IPython.lib import kernel
cf = kernel.get_connection_file()
root_dir = get_ipython().starting_dir
security_dir = os.path.dirname(cf)
for file in os.listdir(security_dir):
if file.startswith("nbserver"):
with open(os.path.join(security_dir, file)) as f:
d = json.loads(f.read())
print d
if d and d.get('notebook_dir') == root_dir:
return d
def get_ipython_notebook_path():
"""Return the path to the current notebook file
This is a horrible hack, but there doesn't seem to be another way to do it.
"""
# Partly from: http://stackoverflow.com/a/13055551/1144479
from IPython.core.getipython import get_ipython
import requests
import os
from IPython.lib import kernel
cf = kernel.get_connection_file()
kernel_id = cf.split('-', 1)[1].split('.')[0]
r = requests.get('http://127.0.0.1:8888/api/sessions')
r.json()
notebook_path = None
for s in requests.get('http://127.0.0.1:8888/api/sessions').json():
if s['kernel']['id'] == kernel_id:
notebook_path = os.path.join(get_ipython().starting_dir, s['notebook']['path'], s['notebook']['name'])
break
return notebook_path
def NotebookName():
"""
.. doctest::
>>> import _ShapelyChipDesigns as SD
>>> print SD.NotebookName()
"""
connection_file_path = kernel.get_connection_file()
connection_file = os.path.basename(connection_file_path)
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
# Updated answer with semi-solutions for both IPython 2.x and IPython < 2.x
if IPython.version_info[0] < 2:
## Not sure if it's even possible to get the port for the
## notebook app; so just using the default...
notebooks = json.load(urllib2.urlopen('http://127.0.0.1:8888/notebooks'))
for nb in notebooks:
if nb['kernel_id'] == kernel_id:
return str(nb['name'])
break
else:
sessions = json.load(urllib2.urlopen('http://127.0.0.1:8888/api/sessions'))
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
return str(sess['notebook']['name'])
break
def get_notebook_info():
connection_file_path = kernel.get_connection_file()
connection_file = os.path.basename(connection_file_path)
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
servers = list(list_running_servers())
nb_name = None
student_id = None
metadata = None
for s in servers:
if s['hostname'] == '0.0.0.0' and s['base_url'].startswith('/user/'):
# Hub
s['url'] = s['url'].replace('0.0.0.0', )
try:
sessions = requests.get('%sapi/sessions?token=%s' %
(s['url'], s['token'])).json()
except:
continue
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
metadata = json.load(open(os.path.basename(
sess['path'])))['metadata']
if 'uva_student_id' in metadata:
student_id = metadata['uva_student_id']
nb_name = os.path.basename(sess['notebook']['path'])
break
return (nb_name, student_id, metadata)
def setNotebookPath():
import os
import sys
import json
import IPython
import requests
from IPython.lib import kernel
project_path = '/home/jovyan/work/'
try:
connection_file = os.path.basename(kernel.get_connection_file())
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
r = requests.get('http://0.0.0.0:8888/api/sessions',
headers={
'Authorization':
'token {}'.format(os.environ['JUPYTER_TOKEN']),
'Content-type':
'application/json'
})
relative_notebook_path = next(session['path']
for session in json.loads(r.text)
if session['kernel']['id'] == kernel_id)
dirPath = os.path.dirname(
os.path.abspath(project_path + relative_notebook_path))
sys.path = list(
filter(lambda path: not path.startswith(project_path), sys.path))
if dirPath not in sys.path: sys.path.append(dirPath)
os.chdir(dirPath)
except:
pass
def get_ipython_server_info():
"""Return the notebook server info as a dict, using an egregious hack.
This will be correct only if there is only one IPython notebook
instance running for each get_ipython().starting_dir
"""
import json
import os
from IPython.core.getipython import get_ipython
from IPython.lib import kernel
cf = kernel.get_connection_file()
root_dir = get_ipython().starting_dir
security_dir = os.path.dirname(cf)
for file in os.listdir(security_dir):
if file.startswith("nbserver"):
with open(os.path.join(security_dir, file)) as f:
d = json.loads(f.read())
print d
if d and d.get('notebook_dir') == root_dir:
return d
def lookup_file(girder_client, jupyterhub_base_url):
"""
Utility function to lookup the Girder file that the current notebook is running
from.
"""
connection_file_path = kernel.get_connection_file()
me = girder_client.get('user/me')
login = me['login']
kernel_id = re.search('kernel-(.*).json', connection_file_path).group(1)
# Authenticate with jupyterhub so we can get the cookie
r = requests.post('%s/hub/login' % jupyterhub_base_url,
data={'Girder-Token': girder_client.token},
allow_redirects=False)
r.raise_for_status()
cookies = r.cookies
url = "%s/user/%s/api/sessions" % (jupyterhub_base_url, login)
r = requests.get(url, cookies=cookies)
r.raise_for_status()
sessions = r.json()
matching = [s for s in sessions if s['kernel']['id'] == kernel_id]
path = matching[0]['path']
name = os.path.basename(path)
# Logout
url = "%s/hub/logout" % jupyterhub_base_url
r = requests.get(url, cookies=cookies)
return girder_client.resourceLookup('user/%s/Private/oc/notebooks/%s/%s' %
(login, path, name))
def magic_connect_info(self, arg_s):
"""Print information for connecting other clients to this kernel
It will print the contents of this session's connection file, as well as
shortcuts for local clients.
In the simplest case, when called from the most recently launched kernel,
secondary clients can be connected, simply with:
$> ipython <app> --existing
"""
try:
connection_file = get_connection_file()
info = get_connection_info(unpack=False)
except Exception as e:
error("Could not get connection info: %r" % e)
return
print (info + '\n')
print ("Paste the above JSON into a file, and connect with:\n"
" $> ipython <app> --existing <file>\n"
"or, if you are local, you can connect with just:\n"
" $> ipython <app> --existing %s\n"
"or even just:\n"
" $> ipython <app> --existing\n"
"if this is the most recent IPython session you have started."
% os.path.basename(connection_file)
)
def _find_file_path():
"""
Find the local path to the current notebook.
"""
# fetch into the socket file to find the kernel id
connection_file = os.path.basename(ip_ker.get_connection_file())
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
# from the kernel id, retrieve the PID of the kernel
app_dict = IPython.Application.instance().session.__dict__
pid = app_dict['_trait_values']['pid']
# from the pid, retrive the running port on the local machine
address = [
a for a in psutil.Process(pid).parent().connections()
if a.status == psutil.CONN_LISTEN
].pop().laddr
address = ':'.join([address[0], str(address[1])])
# build the url of the client
url = 'http://' + address + '/api/sessions'
url = url.replace('http://::1', 'http://localhost')
# fetch the current session of the client
sessions = requests.get(url).json()
ntb_name_list = [
sess['notebook']['path'] for sess in sessions
if sess['kernel']['id'] == kernel_id
]
assert len(ntb_name_list) == 1
ntb_name = '.'.join(ntb_name_list[0].split('.')[:-1]).split('/')[-1]
return {'top_dir': os.getcwd(), 'file_name': ntb_name}
def get_current_notebook_path(
notebook_url='http://127.0.0.1:8888/api/sessions'):
connection_file_path = kernel.get_connection_file()
connection_file = os.path.basename(connection_file_path)
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
sessions = json.load(urllib2.urlopen(notebook_url))
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
notebook_path = str(sess["notebook"]["path"])
break
else:
notebook_path = None
return notebook_path
def magic_connect_info(self, arg_s):
"""Print information for connecting other clients to this kernel
It will print the contents of this session's connection file, as well as
shortcuts for local clients.
In the simplest case, when called from the most recently launched kernel,
secondary clients can be connected, simply with:
$> ipython <app> --existing
"""
from IPython.core.application import BaseIPythonApplication as BaseIPApp
if BaseIPApp.initialized():
app = BaseIPApp.instance()
security_dir = app.profile_dir.security_dir
profile = app.profile
else:
profile = 'default'
security_dir = ''
try:
connection_file = get_connection_file()
info = get_connection_info(unpack=False)
except Exception as e:
error("Could not get connection info: %r" % e)
return
# add profile flag for non-default profile
profile_flag = "--profile %s" % profile if profile != 'default' else ""
# if it's in the security dir, truncate to basename
if security_dir == os.path.dirname(connection_file):
connection_file = os.path.basename(connection_file)
print (info + '\n')
print ("Paste the above JSON into a file, and connect with:\n"
" $> ipython <app> --existing <file>\n"
"or, if you are local, you can connect with just:\n"
" $> ipython <app> --existing {0} {1}\n"
"or even just:\n"
" $> ipython <app> --existing {1}\n"
"if this is the most recent IPython session you have started.".format(
connection_file, profile_flag
)
)
def magic_connect_info(self, arg_s):
"""Print information for connecting other clients to this kernel
It will print the contents of this session's connection file, as well as
shortcuts for local clients.
In the simplest case, when called from the most recently launched kernel,
secondary clients can be connected, simply with:
$> ipython <app> --existing
"""
from IPython.core.application import BaseIPythonApplication as BaseIPApp
if BaseIPApp.initialized():
app = BaseIPApp.instance()
security_dir = app.profile_dir.security_dir
profile = app.profile
else:
profile = 'default'
security_dir = ''
try:
connection_file = get_connection_file()
info = get_connection_info(unpack=False)
except Exception as e:
error("Could not get connection info: %r" % e)
return
# add profile flag for non-default profile
profile_flag = "--profile %s" % profile if profile != 'default' else ""
# if it's in the security dir, truncate to basename
if security_dir == os.path.dirname(connection_file):
connection_file = os.path.basename(connection_file)
print (info + '\n')
print ("Paste the above JSON into a file, and connect with:\n"
" $> ipython <app> --existing <file>\n"
"or, if you are local, you can connect with just:\n"
" $> ipython <app> --existing {0} {1}\n"
"or even just:\n"
" $> ipython <app> --existing {1}\n"
"if this is the most recent IPython session you have started.".format(
connection_file, profile_flag
)
)
def get_notebook_name():
"""
Fetch current notebook's name through IPython
Parameters
----------
None
Return
------
notebook_name : string
name of currently running notebook
Notes
-----
The python in this function could be replaced with Javascript
"""
def sess_open(url):
try:
urllib2.urlopen(url + 'api/sessions')
return True
except:
return False
load = lambda url: json.loads(
urllib2.urlopen(url + 'api/sessions').fp.read())
base = lambda path: os.path.basename(path)
nbpath = lambda session: base(session['notebook']['path'])
nbid = lambda session: session['kernel']['id']
connection_file_path = kernel.get_connection_file()
connection_file = base(connection_file_path)
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
sessions = [
load(data['url']) for data in list_running_servers()
if sess_open(data['url'])
]
sessions = reduce(list.__add__, sessions) if isinstance(sessions[0],
list) else sessions
notebook_name = [
nbpath(sess) for sess in sessions if nbid(sess) == kernel_id
]
if notebook_name:
return notebook_name[0]
else:
sys.stderr.write('No notebook name was found. Export manually.')
sys.stderr.flush()
def new_qt_console(self, evt=None):
"""start a new qtconsole connected to our kernel"""
import sys
# We have to step in and cannibalise connect_qtconsole if we're on windows because
# it launches sys.executable assuming it'll be python, when in fact it's MantidPlot
if sys.platform == 'win32':
argv = []
cf = get_connection_file()
cmd = ';'.join([
"from IPython.frontend.qt.console import qtconsoleapp",
"qtconsoleapp.main()"
])
from subprocess import Popen, PIPE
return Popen([get_executable(), '-c', cmd, '--existing', cf] + argv, stdout=PIPE, stderr=PIPE)
return connect_qtconsole()
def __init__(self, **kwargs):
super().__init__(**kwargs)
# Set flag to check for first call to 'do_execute'
self.is_first_call = True
# Source: https://stackoverflow.com/a/13055551/2926226
# Fetch kernel id
connection_file_path = kernel.get_connection_file()
connection_file = os.path.basename(connection_file_path)
self.kernel_id = connection_file.split('-', 1)[1].split('.')[0]
# Fetch and create sessions url for server
# Used to fetch the notebook name
server = next(notebookapp.list_running_servers(
)) # WARNING: Uses the first server found in the generator
self.sessions_url = f"{server['url']}api/sessions?token={server['token']}"
def convert_to_py():
connection_file_path = kernel.get_connection_file()
connection_file = os.path.basename(connection_file_path)
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
sessions = json.load(urllib2.urlopen('http://127.0.0.1:8888/api/sessions'))
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
file_out=sess['notebook']
break
file_out1=file_out["path"]
pattern=u"/\w*.ipynb"
ipython_name=re.findall(pattern,file_out1)[0][1:]
notebookPath = ipython_name
modulePath= notebookPath[:-6]+".py"
convertNotebook(notebookPath,modulePath)
def get_notebook_name():
"""
Fetch current notebook's name through IPython
Parameters
----------
None
Return
------
notebook_name : string
name of currently running notebook
Notes
-----
The python in this function could be replaced with Javascript
"""
def sess_open(url):
try:
urllib2.urlopen(url+'api/sessions')
return True
except:
return False
load = lambda url: json.loads(urllib2.urlopen(url+'api/sessions').fp.read())
base = lambda path: os.path.basename(path)
nbpath = lambda session: base(session['notebook']['path'])
nbid = lambda session: session['kernel']['id']
connection_file_path = kernel.get_connection_file()
connection_file = base(connection_file_path)
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
sessions = [load(data['url']) for data in list_running_servers() if sess_open(data['url'])]
sessions = reduce(list.__add__, sessions) if isinstance(sessions[0], list) else sessions
notebook_name = [nbpath(sess) for sess in sessions if nbid(sess) == kernel_id]
if notebook_name:
return notebook_name[0]
else:
sys.stderr.write('No notebook name was found. Export manually.')
sys.stderr.flush()
def get_current_notebook_name():
# http://stackoverflow.com/questions/12544056/how-to-i-get-the-current-ipython-notebook-name
connection_file_path = kernel.get_connection_file()
connection_file = os.path.basename(connection_file_path)
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
# Updated answer with semi-solutions for both IPython 2.x and IPython < 2.x
if IPython.version_info[0] < 2:
## Not sure if it's even possible to get the port for the
## notebook app; so just using the default...
notebooks = json.load(urllib2.urlopen('http://' + config['ipython-url'] + 'notebooks'))
for nb in notebooks:
if nb['kernel_id'] == kernel_id:
return nb['name']
else:
sessions = json.load(urllib2.urlopen('http://' + config['ipython-url'] + '/api/sessions'))
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
name = sess['notebook']['name']
if name[-6:] == '.ipynb':
return name[:-6]
else:
return name
def print_kernel_connection():
import json
import os
import urllib
import IPython
import re
from IPython.lib import kernel
connection_file_path = kernel.get_connection_file()
connection_file = os.path.basename(connection_file_path)
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
r = urllib.request.urlopen('http://127.0.0.1:8888/api/sessions')
sessions = json.loads(r.read().decode(r.info().get_param('charset') or 'utf-8'))
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
jpath = sess['notebook']['path']
break
name = re.search('.*/(.*).ipynb', jpath).group(1)
print('PROMPT_COMMAND=\'echo -ne "\\033]0; %s \\007"\''%name)
print('ssh [email protected] -t \'ipython console --existing %s \''%(connection_file_path))
print('')
conf.set("spark.executor.uri", os.environ["SPARK_BINARY"])
# locate current notebook server
api = 'http://127.0.0.1:8888/'
with open(
glob.glob("{}/nbserver-*.json".format(
jupyter_core.paths.jupyter_runtime_dir()))[0]) as infile:
cfg = json.loads(infile.read())
if cfg['url']:
api = cfg['url']
# search all sessions in notebook server
sessions = json.load(urllib2.urlopen('{}api/sessions'.format(api)))
# locate current notebook kernel
connection_file_path = kernel.get_connection_file()
connection_file = os.path.basename(connection_file_path)
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
# get name of notebook matching current notebook kernel
notebook_name = "pyspark"
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
notebook_name = os.path.basename(sess['notebook']['path'])
break
# establish config-based context
sc = SparkContext(appName=notebook_name, pyFiles=add_files, conf=conf)
atexit.register(lambda: sc.stop())
try:
def process(df_train,
param,
df_test=None,
trial=None,
remark=None,
is_output_feature_importance=False):
columns = param['columns']
assert 'y' in df_train.columns.tolist(), 'y is not in df_train'
assert 'index' in df_train.columns.tolist(), 'index is not in df_train'
assert 'index' not in param['columns'], 'index is in features'
assert 'y' not in param['columns'], 'y is in features'
assert 'label' not in param['columns'], 'label is in features'
assert 'group' not in param['columns'], 'group is in features'
assert EP.check_param(param), 'param format is not right '
assert (type(trial)
== list) | (trial == None), 'trial is neither list nor none'
assert len(columns) != 0, 'columns size is 0'
df_test_pred = None
if type(df_test) == pd.DataFrame:
assert 'index' in df_test.columns.tolist(
), 'index is not in df_test'
df_test_pred = pd.concat([df_test_pred, df_test[['index']]],
axis=1)
history = []
df_valid_pred = pd.DataFrame()
df_feature_importances_i_list = []
# stratified,group,timeseries
if 'splits' in param['kfold']:
splits = param['kfold']['splits']
else:
if param['kfold']['type'] == 'stratified':
assert 'label' in df_train.columns.tolist(
), 'label is not in df_train'
folds = StratifiedKFold(
n_splits=param['kfold']['n_splits'],
shuffle=param['kfold']['shuffle'],
random_state=param['kfold']['random_state'])
splits = list(folds.split(df_train, df_train['label']))
elif param['kfold']['type'] == 'group':
assert 'group' in df_train.columns.tolist(
), 'group is not in df_train'
folds = GroupKFold(n_splits=param['kfold']['n_splits'])
splits = list(folds.split(df_train, groups=df_train['group']))
elif param['kfold']['type'] == 'timeseries':
folds = TimeSeriesSplit(n_splits=param['kfold']['n_splits'])
splits = list(folds.split(df_train))
else:
folds = KFold(n_splits=param['kfold']['n_splits'],
shuffle=param['kfold']['shuffle'],
random_state=param['kfold']['random_state'])
splits = list(folds.split(df_train))
if type(param['scaler']) == type(None):
scaler_cls = None
else:
scaler_cls = EP.str2class(param['scaler']['cls'])
regressor_cls = EP.str2class(param['algorithm']['cls'])
permutation_random_state = 42
if scaler_cls != None:
scaler = scaler_cls(**param['scaler']['init'])
if type(df_test) == pd.DataFrame:
scaler.fit(
np.concatenate(
[df_train[columns].values, df_test[columns].values],
axis=0))
else:
scaler.fit(df_train[columns].values)
for fold_n, (train_index, valid_index) in enumerate(splits):
if (len(columns) == 1) & (columns[0] == 'X'):
X = np.array(df_train['X'].values.tolist())
X_train, X_valid = X[train_index, :], X[valid_index, :]
y_train, y_valid = df_train['y'].values[train_index], df_train[
'y'].values[valid_index]
else:
X_train, X_valid = df_train[columns].values[
train_index, :], df_train[columns].values[valid_index, :]
y_train, y_valid = df_train['y'].values[train_index], df_train[
'y'].values[valid_index]
if scaler_cls != None:
X_train = scaler.transform(X_train)
X_valid = scaler.transform(X_valid)
algorithm_init_param = param['algorithm']['init'].copy()
if 'alias' in list(algorithm_init_param.keys()):
algorithm_init_param['alias'] = algorithm_init_param[
'alias'] + '_{}'.format(fold_n)
model = regressor_cls(**algorithm_init_param)
fit_param = param['algorithm']['fit'].copy()
if 'eval_set' in fit_param:
fit_param['eval_set'] = [(X_valid, y_valid)]
if 'FMRegression' in param['algorithm']['cls']:
X_train = sparse.csc_matrix(X_train)
X_valid = sparse.csc_matrix(X_valid)
model.fit(X_train, y_train, **fit_param)
y_valid_pred = model.predict(X_valid)
y_train_pred = model.predict(X_train)
original_index = df_train['index'].values[valid_index]
df_valid_pred_i = pd.DataFrame({
'index':
original_index,
'predict':
y_valid_pred,
'fold_n':
np.zeros(y_valid_pred.shape[0]) + fold_n
})
df_valid_pred = pd.concat([df_valid_pred, df_valid_pred_i], axis=0)
if is_output_feature_importance:
df_feature_importances_i = pd.DataFrame({
'feature':
columns,
'model_weight':
model.feature_importances_
})
df_feature_importances_i = df_feature_importances_i.sort_values(
by=['feature'])
df_feature_importances_i = df_feature_importances_i.reset_index(
drop=True)
perm = PermutationImportance(
model, random_state=permutation_random_state).fit(
X_valid, y_valid)
df_feature_importances_i2 = eli5.explain_weights_dfs(
perm, feature_names=columns,
top=len(columns))['feature_importances']
df_feature_importances_i2 = df_feature_importances_i2.sort_values(
by=['feature'])
df_feature_importances_i2 = df_feature_importances_i2.reset_index(
drop=True)
df_feature_importances_i = pd.merge(df_feature_importances_i,
df_feature_importances_i2,
on='feature')
df_feature_importances_i_list.append(df_feature_importances_i)
if type(df_test) == pd.DataFrame:
if (len(columns) == 1) & (columns[0] == 'X'):
X_test = np.array(df_test['X'].values.tolist())
else:
X_test = df_test[columns].values
if scaler_cls != None:
X_test = scaler.transform(X_test)
if 'FMRegression' in param['algorithm']['cls']:
X_test = sparse.csc_matrix(X_test)
y_test_pred = model.predict(X_test)
df_test_pred_i = pd.DataFrame({fold_n: y_test_pred})
df_test_pred = pd.concat([df_test_pred, df_test_pred_i],
axis=1)
history.append({
'fold_n': fold_n,
'train': mean_absolute_error(y_train, y_train_pred),
'valid': mean_absolute_error(y_valid, y_valid_pred)
})
df_his = pd.DataFrame(history)
df_feature_importances = None
if is_output_feature_importance:
df_feature_importances = df_feature_importances_i_list[0]
for idx, df_feature_importances_i in enumerate(
df_feature_importances_i_list[1:]):
df_feature_importances = pd.merge(df_feature_importances,
df_feature_importances_i,
on='feature',
suffixes=('', idx + 1))
df_valid_pred = df_valid_pred.sort_values(by=['index'])
df_valid_pred = df_valid_pred.reset_index(drop=True)
if type(df_test) == pd.DataFrame:
df_test_pred = df_test_pred.sort_values(by=['index'])
df_test_pred = df_test_pred.reset_index(drop=True)
if type(trial) == list:
pid_ = os.getpid()
datetime_ = datetime.datetime.now()
connection_file = os.path.basename(kernel.get_connection_file())
val_mae_mean = np.mean(df_his.valid)
val_mae_var = np.var(df_his.valid)
train_mae_mean = np.mean(df_his.train)
train_mae_var = np.var(df_his.train)
trial.append({
'datetime': datetime_,
'kernel': connection_file,
'remark': remark,
'val_mae': val_mae_mean,
'train_mae': train_mae_mean,
'val_mae_var': val_mae_var,
'train_mae_var': train_mae_var,
'mae_diff': val_mae_mean - train_mae_mean,
'df_his': df_his,
'df_feature_importances': df_feature_importances,
'df_valid_pred': df_valid_pred,
'df_test_pred': df_test_pred,
'param': param.copy(),
'nfeatures': len(columns)
})
return df_his, df_feature_importances, df_valid_pred, df_test_pred
import os
import urllib2
import IPython
from IPython.lib import kernel
# nbconvert related imports
from IPython.nbconvert import get_export_names, export_by_name
from IPython.nbconvert.writers import FilesWriter
from IPython.nbformat import read, NO_CONVERT
from IPython.nbconvert.utils.exceptions import ConversionException
#exporter_names = widgets.Dropdown(options=get_export_names(), value='html')
#export_button = widgets.Button(description="Export")
#download_link = widgets.HTML(visible=False)
connection_file_path = kernel.get_connection_file()
connection_file = os.path.basename(connection_file_path)
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
def get_name():
sessions = json.load(urllib2.urlopen('http://127.0.0.1:8888/api/sessions'))
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
return sess['notebook']['path']
break
class publish():
filename = os.getcwd()+"/"+get_name()#notebook_name.value
epochs=5, shuffle = True, verbose = 1, validation_data = validation_generator)
with tf.device("/device:GPU:0"):
history_vanilla = inception_transfer_vanilla.fit_generator(
train_generator,
epochs=5, shuffle = True, verbose = 1, validation_data = validation_generator)
import matplotlib.pyplot as plt
# summarize history for accuracy
plt.plot(history_pretrained.history['val_acc'])
plt.plot(history_vanilla.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['Pretrained', 'Vanilla'], loc='upper left')
plt.show()
# summarize history for loss
plt.plot(history_pretrained.history['val_loss'])
plt.plot(history_vanilla.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['Pretrained', 'Vanilla'], loc='upper left')
plt.show()
from IPython.lib import kernel
print(dir(kernel))
print(kernel.get_connection_info())
print(kernel.get_connection_file())
