def launch_console(self): # If IPython failed to load, bail out. if not IPYTHON_LOADED: print "Unable to load IPython" return #if self.kernel is None: try: # This will throw if the kernel is not yet running get_connection_file() InternalIPKernel().new_qt_console() except RuntimeError: self.init_kernel() self.kernel.new_qt_console()
def print_kernel_connection(): import json import os import urllib import IPython import re from IPython.lib import kernel connection_file_path = kernel.get_connection_file() connection_file = os.path.basename(connection_file_path) kernel_id = connection_file.split('-', 1)[1].split('.')[0] r = urllib.request.urlopen('http://127.0.0.1:8888/api/sessions') sessions = json.loads(r.read().decode(r.info().get_param('charset') or 'utf-8')) for sess in sessions: if sess['kernel']['id'] == kernel_id: jpath = sess['notebook']['path'] break name = re.search('.*/(.*).ipynb', jpath).group(1) print('PROMPT_COMMAND=\'echo -ne "\\033]0; %s \\007"\'' % name) print('ssh [email protected] -t \'ipython console --existing %s \'' % (connection_file_path)) print('')
def get_current_notebook_path(): import json import os import urllib2 import IPython from IPython.lib import kernel connection_file_path = kernel.get_connection_file() connection_file = os.path.basename(connection_file_path) kernel_id = connection_file.split('-', 1)[1].split('.')[0] # Updated answer with semi-solutions for both IPython 2.x and IPython < 2.x if IPython.version_info[0] < 2: ## Not sure if it's even possible to get the port for the ## notebook app; so just using the default... notebooks = json.load(urllib2.urlopen('http://127.0.0.1:8888/notebooks')) for nb in notebooks: if nb['kernel_id'] == kernel_id: path = nb.get('name') break else: sessions = json.load(urllib2.urlopen('http://127.0.0.1:8888/api/sessions')) for sess in sessions: if sess['kernel']['id'] == kernel_id: path = sess.get('notebook').get('path') break return path
def get_ipython_notebook_path(): """Return the path to the current notebook file. This is a horrible hack, but there doesn't seem to be another way to do it. """ # Partly from: http://stackoverflow.com/a/13055551/1144479 from IPython.core.getipython import get_ipython import requests import os from IPython.lib import kernel cf = kernel.get_connection_file() kernel_id = cf.split('-', 1)[1].split('.')[0] r = requests.get('http://127.0.0.1:8888/api/sessions') r.json() notebook_path = None for s in requests.get('http://127.0.0.1:8888/api/sessions').json(): if s['kernel']['id'] == kernel_id: notebook_path = os.path.join(get_ipython().starting_dir, s['notebook']['path'], s['notebook']['name']) break return notebook_path
def get_ipython_server_info(): """Return the notebook server info as a dict, using an egregious hack. This will be correct only if there is only one IPython notebook instance running for each get_ipython().starting_dir """ import json import os from IPython.core.getipython import get_ipython from IPython.lib import kernel cf = kernel.get_connection_file() root_dir = get_ipython().starting_dir security_dir = os.path.dirname(cf) for file in os.listdir(security_dir): if file.startswith("nbserver"): with open(os.path.join(security_dir, file)) as f: d = json.loads(f.read()) print d if d and d.get('notebook_dir') == root_dir: return d
def get_ipython_notebook_path(): """Return the path to the current notebook file This is a horrible hack, but there doesn't seem to be another way to do it. """ # Partly from: http://stackoverflow.com/a/13055551/1144479 from IPython.core.getipython import get_ipython import requests import os from IPython.lib import kernel cf = kernel.get_connection_file() kernel_id = cf.split('-', 1)[1].split('.')[0] r = requests.get('http://127.0.0.1:8888/api/sessions') r.json() notebook_path = None for s in requests.get('http://127.0.0.1:8888/api/sessions').json(): if s['kernel']['id'] == kernel_id: notebook_path = os.path.join(get_ipython().starting_dir, s['notebook']['path'], s['notebook']['name']) break return notebook_path
def NotebookName(): """ .. doctest:: >>> import _ShapelyChipDesigns as SD >>> print SD.NotebookName() """ connection_file_path = kernel.get_connection_file() connection_file = os.path.basename(connection_file_path) kernel_id = connection_file.split('-', 1)[1].split('.')[0] # Updated answer with semi-solutions for both IPython 2.x and IPython < 2.x if IPython.version_info[0] < 2: ## Not sure if it's even possible to get the port for the ## notebook app; so just using the default... notebooks = json.load(urllib2.urlopen('http://127.0.0.1:8888/notebooks')) for nb in notebooks: if nb['kernel_id'] == kernel_id: return str(nb['name']) break else: sessions = json.load(urllib2.urlopen('http://127.0.0.1:8888/api/sessions')) for sess in sessions: if sess['kernel']['id'] == kernel_id: return str(sess['notebook']['name']) break
def get_notebook_info(): connection_file_path = kernel.get_connection_file() connection_file = os.path.basename(connection_file_path) kernel_id = connection_file.split('-', 1)[1].split('.')[0] servers = list(list_running_servers()) nb_name = None student_id = None metadata = None for s in servers: if s['hostname'] == '0.0.0.0' and s['base_url'].startswith('/user/'): # Hub s['url'] = s['url'].replace('0.0.0.0', ) try: sessions = requests.get('%sapi/sessions?token=%s' % (s['url'], s['token'])).json() except: continue for sess in sessions: if sess['kernel']['id'] == kernel_id: metadata = json.load(open(os.path.basename( sess['path'])))['metadata'] if 'uva_student_id' in metadata: student_id = metadata['uva_student_id'] nb_name = os.path.basename(sess['notebook']['path']) break return (nb_name, student_id, metadata)
def setNotebookPath(): import os import sys import json import IPython import requests from IPython.lib import kernel project_path = '/home/jovyan/work/' try: connection_file = os.path.basename(kernel.get_connection_file()) kernel_id = connection_file.split('-', 1)[1].split('.')[0] r = requests.get('http://0.0.0.0:8888/api/sessions', headers={ 'Authorization': 'token {}'.format(os.environ['JUPYTER_TOKEN']), 'Content-type': 'application/json' }) relative_notebook_path = next(session['path'] for session in json.loads(r.text) if session['kernel']['id'] == kernel_id) dirPath = os.path.dirname( os.path.abspath(project_path + relative_notebook_path)) sys.path = list( filter(lambda path: not path.startswith(project_path), sys.path)) if dirPath not in sys.path: sys.path.append(dirPath) os.chdir(dirPath) except: pass
def get_ipython_server_info(): """Return the notebook server info as a dict, using an egregious hack. This will be correct only if there is only one IPython notebook instance running for each get_ipython().starting_dir """ import json import os from IPython.core.getipython import get_ipython from IPython.lib import kernel cf = kernel.get_connection_file() root_dir = get_ipython().starting_dir security_dir = os.path.dirname(cf) for file in os.listdir(security_dir): if file.startswith("nbserver"): with open(os.path.join(security_dir, file)) as f: d = json.loads(f.read()) print d if d and d.get('notebook_dir') == root_dir: return d
def lookup_file(girder_client, jupyterhub_base_url): """ Utility function to lookup the Girder file that the current notebook is running from. """ connection_file_path = kernel.get_connection_file() me = girder_client.get('user/me') login = me['login'] kernel_id = re.search('kernel-(.*).json', connection_file_path).group(1) # Authenticate with jupyterhub so we can get the cookie r = requests.post('%s/hub/login' % jupyterhub_base_url, data={'Girder-Token': girder_client.token}, allow_redirects=False) r.raise_for_status() cookies = r.cookies url = "%s/user/%s/api/sessions" % (jupyterhub_base_url, login) r = requests.get(url, cookies=cookies) r.raise_for_status() sessions = r.json() matching = [s for s in sessions if s['kernel']['id'] == kernel_id] path = matching[0]['path'] name = os.path.basename(path) # Logout url = "%s/hub/logout" % jupyterhub_base_url r = requests.get(url, cookies=cookies) return girder_client.resourceLookup('user/%s/Private/oc/notebooks/%s/%s' % (login, path, name))
def magic_connect_info(self, arg_s): """Print information for connecting other clients to this kernel It will print the contents of this session's connection file, as well as shortcuts for local clients. In the simplest case, when called from the most recently launched kernel, secondary clients can be connected, simply with: $> ipython <app> --existing """ try: connection_file = get_connection_file() info = get_connection_info(unpack=False) except Exception as e: error("Could not get connection info: %r" % e) return print (info + '\n') print ("Paste the above JSON into a file, and connect with:\n" " $> ipython <app> --existing <file>\n" "or, if you are local, you can connect with just:\n" " $> ipython <app> --existing %s\n" "or even just:\n" " $> ipython <app> --existing\n" "if this is the most recent IPython session you have started." % os.path.basename(connection_file) )
def _find_file_path(): """ Find the local path to the current notebook. """ # fetch into the socket file to find the kernel id connection_file = os.path.basename(ip_ker.get_connection_file()) kernel_id = connection_file.split('-', 1)[1].split('.')[0] # from the kernel id, retrieve the PID of the kernel app_dict = IPython.Application.instance().session.__dict__ pid = app_dict['_trait_values']['pid'] # from the pid, retrive the running port on the local machine address = [ a for a in psutil.Process(pid).parent().connections() if a.status == psutil.CONN_LISTEN ].pop().laddr address = ':'.join([address[0], str(address[1])]) # build the url of the client url = 'http://' + address + '/api/sessions' url = url.replace('http://::1', 'http://localhost') # fetch the current session of the client sessions = requests.get(url).json() ntb_name_list = [ sess['notebook']['path'] for sess in sessions if sess['kernel']['id'] == kernel_id ] assert len(ntb_name_list) == 1 ntb_name = '.'.join(ntb_name_list[0].split('.')[:-1]).split('/')[-1] return {'top_dir': os.getcwd(), 'file_name': ntb_name}
def get_current_notebook_path( notebook_url='http://127.0.0.1:8888/api/sessions'): connection_file_path = kernel.get_connection_file() connection_file = os.path.basename(connection_file_path) kernel_id = connection_file.split('-', 1)[1].split('.')[0] sessions = json.load(urllib2.urlopen(notebook_url)) for sess in sessions: if sess['kernel']['id'] == kernel_id: notebook_path = str(sess["notebook"]["path"]) break else: notebook_path = None return notebook_path
def magic_connect_info(self, arg_s): """Print information for connecting other clients to this kernel It will print the contents of this session's connection file, as well as shortcuts for local clients. In the simplest case, when called from the most recently launched kernel, secondary clients can be connected, simply with: $> ipython <app> --existing """ from IPython.core.application import BaseIPythonApplication as BaseIPApp if BaseIPApp.initialized(): app = BaseIPApp.instance() security_dir = app.profile_dir.security_dir profile = app.profile else: profile = 'default' security_dir = '' try: connection_file = get_connection_file() info = get_connection_info(unpack=False) except Exception as e: error("Could not get connection info: %r" % e) return # add profile flag for non-default profile profile_flag = "--profile %s" % profile if profile != 'default' else "" # if it's in the security dir, truncate to basename if security_dir == os.path.dirname(connection_file): connection_file = os.path.basename(connection_file) print (info + '\n') print ("Paste the above JSON into a file, and connect with:\n" " $> ipython <app> --existing <file>\n" "or, if you are local, you can connect with just:\n" " $> ipython <app> --existing {0} {1}\n" "or even just:\n" " $> ipython <app> --existing {1}\n" "if this is the most recent IPython session you have started.".format( connection_file, profile_flag ) )
def magic_connect_info(self, arg_s): """Print information for connecting other clients to this kernel It will print the contents of this session's connection file, as well as shortcuts for local clients. In the simplest case, when called from the most recently launched kernel, secondary clients can be connected, simply with: $> ipython <app> --existing """ from IPython.core.application import BaseIPythonApplication as BaseIPApp if BaseIPApp.initialized(): app = BaseIPApp.instance() security_dir = app.profile_dir.security_dir profile = app.profile else: profile = 'default' security_dir = '' try: connection_file = get_connection_file() info = get_connection_info(unpack=False) except Exception as e: error("Could not get connection info: %r" % e) return # add profile flag for non-default profile profile_flag = "--profile %s" % profile if profile != 'default' else "" # if it's in the security dir, truncate to basename if security_dir == os.path.dirname(connection_file): connection_file = os.path.basename(connection_file) print (info + '\n') print ("Paste the above JSON into a file, and connect with:\n" " $> ipython <app> --existing <file>\n" "or, if you are local, you can connect with just:\n" " $> ipython <app> --existing {0} {1}\n" "or even just:\n" " $> ipython <app> --existing {1}\n" "if this is the most recent IPython session you have started.".format( connection_file, profile_flag ) )
def get_notebook_name(): """ Fetch current notebook's name through IPython Parameters ---------- None Return ------ notebook_name : string name of currently running notebook Notes ----- The python in this function could be replaced with Javascript """ def sess_open(url): try: urllib2.urlopen(url + 'api/sessions') return True except: return False load = lambda url: json.loads( urllib2.urlopen(url + 'api/sessions').fp.read()) base = lambda path: os.path.basename(path) nbpath = lambda session: base(session['notebook']['path']) nbid = lambda session: session['kernel']['id'] connection_file_path = kernel.get_connection_file() connection_file = base(connection_file_path) kernel_id = connection_file.split('-', 1)[1].split('.')[0] sessions = [ load(data['url']) for data in list_running_servers() if sess_open(data['url']) ] sessions = reduce(list.__add__, sessions) if isinstance(sessions[0], list) else sessions notebook_name = [ nbpath(sess) for sess in sessions if nbid(sess) == kernel_id ] if notebook_name: return notebook_name[0] else: sys.stderr.write('No notebook name was found. Export manually.') sys.stderr.flush()
def new_qt_console(self, evt=None): """start a new qtconsole connected to our kernel""" import sys # We have to step in and cannibalise connect_qtconsole if we're on windows because # it launches sys.executable assuming it'll be python, when in fact it's MantidPlot if sys.platform == 'win32': argv = [] cf = get_connection_file() cmd = ';'.join([ "from IPython.frontend.qt.console import qtconsoleapp", "qtconsoleapp.main()" ]) from subprocess import Popen, PIPE return Popen([get_executable(), '-c', cmd, '--existing', cf] + argv, stdout=PIPE, stderr=PIPE) return connect_qtconsole()
def __init__(self, **kwargs): super().__init__(**kwargs) # Set flag to check for first call to 'do_execute' self.is_first_call = True # Source: https://stackoverflow.com/a/13055551/2926226 # Fetch kernel id connection_file_path = kernel.get_connection_file() connection_file = os.path.basename(connection_file_path) self.kernel_id = connection_file.split('-', 1)[1].split('.')[0] # Fetch and create sessions url for server # Used to fetch the notebook name server = next(notebookapp.list_running_servers( )) # WARNING: Uses the first server found in the generator self.sessions_url = f"{server['url']}api/sessions?token={server['token']}"
def convert_to_py(): connection_file_path = kernel.get_connection_file() connection_file = os.path.basename(connection_file_path) kernel_id = connection_file.split('-', 1)[1].split('.')[0] sessions = json.load(urllib2.urlopen('http://127.0.0.1:8888/api/sessions')) for sess in sessions: if sess['kernel']['id'] == kernel_id: file_out=sess['notebook'] break file_out1=file_out["path"] pattern=u"/\w*.ipynb" ipython_name=re.findall(pattern,file_out1)[0][1:] notebookPath = ipython_name modulePath= notebookPath[:-6]+".py" convertNotebook(notebookPath,modulePath)
def get_notebook_name(): """ Fetch current notebook's name through IPython Parameters ---------- None Return ------ notebook_name : string name of currently running notebook Notes ----- The python in this function could be replaced with Javascript """ def sess_open(url): try: urllib2.urlopen(url+'api/sessions') return True except: return False load = lambda url: json.loads(urllib2.urlopen(url+'api/sessions').fp.read()) base = lambda path: os.path.basename(path) nbpath = lambda session: base(session['notebook']['path']) nbid = lambda session: session['kernel']['id'] connection_file_path = kernel.get_connection_file() connection_file = base(connection_file_path) kernel_id = connection_file.split('-', 1)[1].split('.')[0] sessions = [load(data['url']) for data in list_running_servers() if sess_open(data['url'])] sessions = reduce(list.__add__, sessions) if isinstance(sessions[0], list) else sessions notebook_name = [nbpath(sess) for sess in sessions if nbid(sess) == kernel_id] if notebook_name: return notebook_name[0] else: sys.stderr.write('No notebook name was found. Export manually.') sys.stderr.flush()
def get_current_notebook_name(): # http://stackoverflow.com/questions/12544056/how-to-i-get-the-current-ipython-notebook-name connection_file_path = kernel.get_connection_file() connection_file = os.path.basename(connection_file_path) kernel_id = connection_file.split('-', 1)[1].split('.')[0] # Updated answer with semi-solutions for both IPython 2.x and IPython < 2.x if IPython.version_info[0] < 2: ## Not sure if it's even possible to get the port for the ## notebook app; so just using the default... notebooks = json.load(urllib2.urlopen('http://' + config['ipython-url'] + 'notebooks')) for nb in notebooks: if nb['kernel_id'] == kernel_id: return nb['name'] else: sessions = json.load(urllib2.urlopen('http://' + config['ipython-url'] + '/api/sessions')) for sess in sessions: if sess['kernel']['id'] == kernel_id: name = sess['notebook']['name'] if name[-6:] == '.ipynb': return name[:-6] else: return name
def print_kernel_connection(): import json import os import urllib import IPython import re from IPython.lib import kernel connection_file_path = kernel.get_connection_file() connection_file = os.path.basename(connection_file_path) kernel_id = connection_file.split('-', 1)[1].split('.')[0] r = urllib.request.urlopen('http://127.0.0.1:8888/api/sessions') sessions = json.loads(r.read().decode(r.info().get_param('charset') or 'utf-8')) for sess in sessions: if sess['kernel']['id'] == kernel_id: jpath = sess['notebook']['path'] break name = re.search('.*/(.*).ipynb', jpath).group(1) print('PROMPT_COMMAND=\'echo -ne "\\033]0; %s \\007"\''%name) print('ssh [email protected] -t \'ipython console --existing %s \''%(connection_file_path)) print('')
conf.set("spark.executor.uri", os.environ["SPARK_BINARY"]) # locate current notebook server api = 'http://127.0.0.1:8888/' with open( glob.glob("{}/nbserver-*.json".format( jupyter_core.paths.jupyter_runtime_dir()))[0]) as infile: cfg = json.loads(infile.read()) if cfg['url']: api = cfg['url'] # search all sessions in notebook server sessions = json.load(urllib2.urlopen('{}api/sessions'.format(api))) # locate current notebook kernel connection_file_path = kernel.get_connection_file() connection_file = os.path.basename(connection_file_path) kernel_id = connection_file.split('-', 1)[1].split('.')[0] # get name of notebook matching current notebook kernel notebook_name = "pyspark" for sess in sessions: if sess['kernel']['id'] == kernel_id: notebook_name = os.path.basename(sess['notebook']['path']) break # establish config-based context sc = SparkContext(appName=notebook_name, pyFiles=add_files, conf=conf) atexit.register(lambda: sc.stop()) try:
def process(df_train, param, df_test=None, trial=None, remark=None, is_output_feature_importance=False): columns = param['columns'] assert 'y' in df_train.columns.tolist(), 'y is not in df_train' assert 'index' in df_train.columns.tolist(), 'index is not in df_train' assert 'index' not in param['columns'], 'index is in features' assert 'y' not in param['columns'], 'y is in features' assert 'label' not in param['columns'], 'label is in features' assert 'group' not in param['columns'], 'group is in features' assert EP.check_param(param), 'param format is not right ' assert (type(trial) == list) | (trial == None), 'trial is neither list nor none' assert len(columns) != 0, 'columns size is 0' df_test_pred = None if type(df_test) == pd.DataFrame: assert 'index' in df_test.columns.tolist( ), 'index is not in df_test' df_test_pred = pd.concat([df_test_pred, df_test[['index']]], axis=1) history = [] df_valid_pred = pd.DataFrame() df_feature_importances_i_list = [] # stratified,group,timeseries if 'splits' in param['kfold']: splits = param['kfold']['splits'] else: if param['kfold']['type'] == 'stratified': assert 'label' in df_train.columns.tolist( ), 'label is not in df_train' folds = StratifiedKFold( n_splits=param['kfold']['n_splits'], shuffle=param['kfold']['shuffle'], random_state=param['kfold']['random_state']) splits = list(folds.split(df_train, df_train['label'])) elif param['kfold']['type'] == 'group': assert 'group' in df_train.columns.tolist( ), 'group is not in df_train' folds = GroupKFold(n_splits=param['kfold']['n_splits']) splits = list(folds.split(df_train, groups=df_train['group'])) elif param['kfold']['type'] == 'timeseries': folds = TimeSeriesSplit(n_splits=param['kfold']['n_splits']) splits = list(folds.split(df_train)) else: folds = KFold(n_splits=param['kfold']['n_splits'], shuffle=param['kfold']['shuffle'], random_state=param['kfold']['random_state']) splits = list(folds.split(df_train)) if type(param['scaler']) == type(None): scaler_cls = None else: scaler_cls = EP.str2class(param['scaler']['cls']) regressor_cls = EP.str2class(param['algorithm']['cls']) permutation_random_state = 42 if scaler_cls != None: scaler = scaler_cls(**param['scaler']['init']) if type(df_test) == pd.DataFrame: scaler.fit( np.concatenate( [df_train[columns].values, df_test[columns].values], axis=0)) else: scaler.fit(df_train[columns].values) for fold_n, (train_index, valid_index) in enumerate(splits): if (len(columns) == 1) & (columns[0] == 'X'): X = np.array(df_train['X'].values.tolist()) X_train, X_valid = X[train_index, :], X[valid_index, :] y_train, y_valid = df_train['y'].values[train_index], df_train[ 'y'].values[valid_index] else: X_train, X_valid = df_train[columns].values[ train_index, :], df_train[columns].values[valid_index, :] y_train, y_valid = df_train['y'].values[train_index], df_train[ 'y'].values[valid_index] if scaler_cls != None: X_train = scaler.transform(X_train) X_valid = scaler.transform(X_valid) algorithm_init_param = param['algorithm']['init'].copy() if 'alias' in list(algorithm_init_param.keys()): algorithm_init_param['alias'] = algorithm_init_param[ 'alias'] + '_{}'.format(fold_n) model = regressor_cls(**algorithm_init_param) fit_param = param['algorithm']['fit'].copy() if 'eval_set' in fit_param: fit_param['eval_set'] = [(X_valid, y_valid)] if 'FMRegression' in param['algorithm']['cls']: X_train = sparse.csc_matrix(X_train) X_valid = sparse.csc_matrix(X_valid) model.fit(X_train, y_train, **fit_param) y_valid_pred = model.predict(X_valid) y_train_pred = model.predict(X_train) original_index = df_train['index'].values[valid_index] df_valid_pred_i = pd.DataFrame({ 'index': original_index, 'predict': y_valid_pred, 'fold_n': np.zeros(y_valid_pred.shape[0]) + fold_n }) df_valid_pred = pd.concat([df_valid_pred, df_valid_pred_i], axis=0) if is_output_feature_importance: df_feature_importances_i = pd.DataFrame({ 'feature': columns, 'model_weight': model.feature_importances_ }) df_feature_importances_i = df_feature_importances_i.sort_values( by=['feature']) df_feature_importances_i = df_feature_importances_i.reset_index( drop=True) perm = PermutationImportance( model, random_state=permutation_random_state).fit( X_valid, y_valid) df_feature_importances_i2 = eli5.explain_weights_dfs( perm, feature_names=columns, top=len(columns))['feature_importances'] df_feature_importances_i2 = df_feature_importances_i2.sort_values( by=['feature']) df_feature_importances_i2 = df_feature_importances_i2.reset_index( drop=True) df_feature_importances_i = pd.merge(df_feature_importances_i, df_feature_importances_i2, on='feature') df_feature_importances_i_list.append(df_feature_importances_i) if type(df_test) == pd.DataFrame: if (len(columns) == 1) & (columns[0] == 'X'): X_test = np.array(df_test['X'].values.tolist()) else: X_test = df_test[columns].values if scaler_cls != None: X_test = scaler.transform(X_test) if 'FMRegression' in param['algorithm']['cls']: X_test = sparse.csc_matrix(X_test) y_test_pred = model.predict(X_test) df_test_pred_i = pd.DataFrame({fold_n: y_test_pred}) df_test_pred = pd.concat([df_test_pred, df_test_pred_i], axis=1) history.append({ 'fold_n': fold_n, 'train': mean_absolute_error(y_train, y_train_pred), 'valid': mean_absolute_error(y_valid, y_valid_pred) }) df_his = pd.DataFrame(history) df_feature_importances = None if is_output_feature_importance: df_feature_importances = df_feature_importances_i_list[0] for idx, df_feature_importances_i in enumerate( df_feature_importances_i_list[1:]): df_feature_importances = pd.merge(df_feature_importances, df_feature_importances_i, on='feature', suffixes=('', idx + 1)) df_valid_pred = df_valid_pred.sort_values(by=['index']) df_valid_pred = df_valid_pred.reset_index(drop=True) if type(df_test) == pd.DataFrame: df_test_pred = df_test_pred.sort_values(by=['index']) df_test_pred = df_test_pred.reset_index(drop=True) if type(trial) == list: pid_ = os.getpid() datetime_ = datetime.datetime.now() connection_file = os.path.basename(kernel.get_connection_file()) val_mae_mean = np.mean(df_his.valid) val_mae_var = np.var(df_his.valid) train_mae_mean = np.mean(df_his.train) train_mae_var = np.var(df_his.train) trial.append({ 'datetime': datetime_, 'kernel': connection_file, 'remark': remark, 'val_mae': val_mae_mean, 'train_mae': train_mae_mean, 'val_mae_var': val_mae_var, 'train_mae_var': train_mae_var, 'mae_diff': val_mae_mean - train_mae_mean, 'df_his': df_his, 'df_feature_importances': df_feature_importances, 'df_valid_pred': df_valid_pred, 'df_test_pred': df_test_pred, 'param': param.copy(), 'nfeatures': len(columns) }) return df_his, df_feature_importances, df_valid_pred, df_test_pred
import os import urllib2 import IPython from IPython.lib import kernel # nbconvert related imports from IPython.nbconvert import get_export_names, export_by_name from IPython.nbconvert.writers import FilesWriter from IPython.nbformat import read, NO_CONVERT from IPython.nbconvert.utils.exceptions import ConversionException #exporter_names = widgets.Dropdown(options=get_export_names(), value='html') #export_button = widgets.Button(description="Export") #download_link = widgets.HTML(visible=False) connection_file_path = kernel.get_connection_file() connection_file = os.path.basename(connection_file_path) kernel_id = connection_file.split('-', 1)[1].split('.')[0] def get_name(): sessions = json.load(urllib2.urlopen('http://127.0.0.1:8888/api/sessions')) for sess in sessions: if sess['kernel']['id'] == kernel_id: return sess['notebook']['path'] break class publish(): filename = os.getcwd()+"/"+get_name()#notebook_name.value
epochs=5, shuffle = True, verbose = 1, validation_data = validation_generator) with tf.device("/device:GPU:0"): history_vanilla = inception_transfer_vanilla.fit_generator( train_generator, epochs=5, shuffle = True, verbose = 1, validation_data = validation_generator) import matplotlib.pyplot as plt # summarize history for accuracy plt.plot(history_pretrained.history['val_acc']) plt.plot(history_vanilla.history['val_acc']) plt.title('model accuracy') plt.ylabel('accuracy') plt.xlabel('epoch') plt.legend(['Pretrained', 'Vanilla'], loc='upper left') plt.show() # summarize history for loss plt.plot(history_pretrained.history['val_loss']) plt.plot(history_vanilla.history['val_loss']) plt.title('model loss') plt.ylabel('loss') plt.xlabel('epoch') plt.legend(['Pretrained', 'Vanilla'], loc='upper left') plt.show() from IPython.lib import kernel print(dir(kernel)) print(kernel.get_connection_info()) print(kernel.get_connection_file())