def predict(self, filename, input): dill.load_session(filename) self.batch_size = 1 self.forward_pass(input) a = self.layers[self.num_layers-1].activations a[np.where(a==np.max(a))] = 1 a[np.where(a!=np.max(a))] = 0 return a
def dump_session(file_path): """Pickle the current python session to be used in the worker. Note: Due to the inconsistency in the first dump of dill dump_session we create and load the dump twice to have consistent results in the worker and the running session. Check: https://github.com/uqfoundation/dill/issues/195 """ dill.dump_session(file_path) dill.load_session(file_path) return dill.dump_session(file_path)
def load_dill(self): fname = fdialog.askopenfilename(filetypes=['dill {.pkl}'], title='Open dill PKL file', defaultextension='PKL') if fname: dill.load_session(fname) self.set_dataset_name(fname) curdir = Path(fname).parent os.chdir(curdir) messagebox.showinfo("Success", "Data loaded.")
def load_game(): if os.path.exists('savefile.pkl') == True: dill.load_session('savefile.pkl') print('Game succesfully loaded!') input('<Continue (Press Enter)>') print_location() else: os.system('cls') print("There\'s no saved game to load!") input('<Return (Press Enter)>') title_screen()
def restore_session(file): """Resore a session from file. :return: Returns True if session is restored :rtype: bool """ if not os.path.isfile(file): print(file + " doesn't exist") return False #load the session dill.load_session(file) print("Session restored.") return True
def dump_session(file_path): """For internal use only; no backwards-compatibility guarantees. Pickle the current python session to be used in the worker. Note: Due to the inconsistency in the first dump of dill dump_session we create and load the dump twice to have consistent results in the worker and the running session. Check: https://github.com/uqfoundation/dill/issues/195 """ with _pickle_lock_unless_py2: dill.dump_session(file_path) dill.load_session(file_path) return dill.dump_session(file_path)
def check_accuracy(self, filename, inputs, labels): dill.load_session(filename) self.batch_size = len(inputs) self.forward_pass(inputs) a = self.layers[self.num_layers - 1].activations a[np.where(a == np.max(a))] = 1 a[np.where(a != np.max(a))] = 0 total = 0 correct = 0 for i in range(len(a)): total += 1 if np.equal(a[i], labels[i]).all(): correct += 1 print("Accuracy: ", correct * 100 / total)
def checkAccuracy(self, dataFile, data, flags): dill.load_session(dataFile) self.batchSize = len(data) self.passForward(data) a = self.layers[self.numLayers - 1].activations a[np.where(a == np.max(a))] = 1 a[np.where(a != np.max(a))] = 0 total = 0 correct = 0 for i in range(len(a)): total += 1 if np.equal(a[i], flags[i]).all(): correct += 1 print("Accuracy: ", correct * 100 / total)
def predict(self, filename, input): ''' It takes 2 inputs: 1. filename: The file from which trained model is to be loaded. 2. input: The input for which we want the prediction. It does a forward pass, and then converts the output into one-hot encoding i.e. the maximum element of array is 1 and all others are 0. ''' dill.load_session(filename) self.batch_size = 1 self.forward_pass(input) a = self.layers[self.num_layers - 1].activations a[np.where(a == np.max(a))] = 1 a[np.where(a != np.max(a))] = 0 return a
def check_accuracy(self, filename, inputs, labels): dill.load_session(filename) self.batch_size = len(inputs) self.forward_pass(inputs) a = self.layers[self.num_layers-1].activations num_classes = 10 targets = np.array([a]).reshape(-1) a = np.asarray(a) one_hot_labels = np.eye(num_classes)[a.astype(int)] total=0 correct=0 for i in range(len(a)): total += 1 if np.equal(one_hot_labels[i], labels[i]).all(): correct += 1 print("Accuracy: ", correct*100/total)
def WorkspaceBasedCheckPt(CheckPtPosition = 0, AccessRecentOrNewWorkSpaceImage = False, Force = {'access':False, 'task':'load'}, CheckPoint_Dir = "NotebookCheckpoints/"): if not os.path.exists(CheckPoint_Dir): os.makedirs(CheckPoint_Dir) file= open((CheckPoint_Dir + '0__CheckPt.db'),"w+") file.close() LastCheckPt = max([int(CheckPoint.split('/')[len(CheckPoint.split('/')) -1].split('__')[0]) for CheckPoint in glob.glob(CheckPoint_Dir + '*{}'.format('CheckPt.db'))]) ## Force can be used when proceding in non serialized manner if Force['access']: ## could have been compressed with chunk below using boolean algebra to make code more smaller AccessRecentOrNewWorkSpaceImage = False if Force['task'] == 'load': print("Checkpoint ", CheckPtPosition, "is to be loaded") if os.path.exists(CheckPoint_Dir + str(CheckPtPosition) + '__CheckPt.db'): dill.load_session(CheckPoint_Dir + str(CheckPtPosition) + '__CheckPt.db') # To restore a session else: print("This checkpoint doesn't exist, hence won't be loaded.") elif Force['task'] == 'save': print("Checkpoint ", CheckPtPosition, "is to be Saved") dill.dump_session(CheckPoint_Dir + str(CheckPtPosition) + '__CheckPt.db') # To Save a session return "Force used to {} workspace checkpoint_{}.".format(Force['task'], CheckPtPosition) ## exit here only ## This Code below is used to handle the actions on returning the value to run/not run the cell ### = is set so that the current check point cell code are able to run if ((AccessRecentOrNewWorkSpaceImage == False) and (CheckPtPosition <= LastCheckPt)): print('Most Recent Checkpoint is : {} \nHence, cells won\'t be running content untill most recent checkpoint is crossed.'.format(LastCheckPt)) return False elif ((AccessRecentOrNewWorkSpaceImage == False) and (CheckPtPosition == (LastCheckPt +1))): print('Running this cell') return True elif ((AccessRecentOrNewWorkSpaceImage == False) and (CheckPtPosition > (LastCheckPt +1))): print("You have skipped over a checkpoint. Still running this cell") return True ## This Code below is used to handle the actions on saving/loading the workspace images if (AccessRecentOrNewWorkSpaceImage and (CheckPtPosition == 0)): print("Initial Phase, hence not saving workspace.") elif (AccessRecentOrNewWorkSpaceImage and (LastCheckPt > CheckPtPosition)): print("This is not the most recent checkpoint, hence not loading it. [Use Force to force load a checkpoint]") elif (AccessRecentOrNewWorkSpaceImage and (LastCheckPt == CheckPtPosition)): dill.load_session(CheckPoint_Dir + str(CheckPtPosition) + '__CheckPt.db') # To restore a session print("This is the most recent checkpoint, hence loading it.") elif (AccessRecentOrNewWorkSpaceImage and ((LastCheckPt +1) == CheckPtPosition)): dill.dump_session(CheckPoint_Dir + str(CheckPtPosition) + '__CheckPt.db') # To Save a session print("Congrats, on reaching a new checkpoint, saving it.") elif (AccessRecentOrNewWorkSpaceImage and ((LastCheckPt +1) < CheckPtPosition)): print("You have skipped over a checkpoint. Hence not Saving anything.")
def main(): plt.close('all') file_path = './result' if not os.path.isdir(file_path): os.makedirs(file_path) dill.load_session('result/compare.pkl') S_list = main.S_list err = main.err err_pca = main.err_pca err_svd = main.err_svd err_mds = main.err_mds pr.plot_compare_mean_std(S_list, err, err_pca, err_svd, err_mds, False) pr.plot_compare_mean_std(S_list, err, err_pca, err_svd, err_mds, True) pr.plot_compare_scatter(S_list, err, err_pca, err_svd, err_mds, False) pr.plot_compare_scatter(S_list, err, err_pca, err_svd, err_mds, True) plt.close('all')
def check_accuracy(self, filename, inputs, targets): dill.load_session(filename) # self.batch_size = len(inputs) self.forward_pass(inputs) a = self.layers[self.num_layers - 1].activations # a[np.where(a == np.max(a))] = 1 # a[np.where(a != np.max(a))] = 0 a = np.rint(a) total = 0 correct = 0 for i in range(len(a)): total = total + 1 # print(self.layers[self.num_layers - 1].activations[i], a[i], labels[i]) # print(a[i], targets[i]) if np.equal(a[i], targets[i]).all(): correct = correct + 1 print("Accuracy on test: " + str(correct) + "/" + str(total))
def _run(func_fn): """Executes dill-ed function/the calculator of an ASE Atoms object using args/kwargs from dill-ed args/kwargs files""" session_fn = func_fn.replace('.pkl', '_session.pkl') try: dill.load_session(session_fn) except IOError: pass with open(func_fn) as func_f: func, args, kwargs = dill.load(func_f) if func.__class__ == gausspy.gaussian.Gaussian: func.start(*args, **kwargs) elif func.__class__ == ase.atoms.Atoms: func.calc.start(*args, **kwargs) else: func(*args, **kwargs)
def plot_alpha(filename=(), prename=(), name=()): dill.load_session(filename) k = 0 it_alpha_fixed_ratio = 0 p.z = p.isolver.save_alpha[:, it_alpha_fixed_ratio] p.plot_pre() fig = plt.figure() plt.contour(p.x, p.y, p.z.reshape(p.y.size, p.x.size), 30) p.ld.plot(lw=0.8, ms=1) p.so.plot(lw=0.8, ms=1) plt.colorbar() plt.title('alpha') plt.axis('equal') plt.axis('square') plt.show(block=False) if savefig: plt.savefig('runs/fig-thesis/%s_lsm_%s%s_alpha_it_alpha_%s.eps' % (prename, name, k, it_alpha_fixed_ratio), bbox_inches='tight')
def test_loadRuntimeDataAndRun(self): """ Tests the pickled function and data can be loaded and run correctly. """ # ensure the 'add' function from the test's __main__ session is removed and # not available to this test case. del globals()['add'] self.assertFalse('add' in globals().keys()) # load session data and test if the 'add' function is again avaliable after # data is loaded. dill.load_session(self.__class__.sesPathPkl) self.assertTrue('add' in globals().keys()) # load data for the 'add' function, and test if we get expected result after # running the 'add' function with the loaded data. with open(self.__class__.jobPathPkl, 'rb') as f: data = dill.load(f) self.assertTrue(add(*data) == 3)
def show_num_feat_stat(): axs = {} ds = ['NSL-KDD', 'UCI-BCW'] r = [] for t in [1, 2]: app.mText.insert('end-1c', 'type: %s\n' % t) with os.scandir() as li: for entry in li: if entry.is_file() and ".pkl" in entry.name: dill.load_session(entry.name) s = 1 if (t == 1 and ld[0][0] in ['src_bytes', '\'src_bytes\''])\ or (t == 2 and ld[0][0] not in ['src_bytes', '\'src_bytes\'']): app.mText.insert('end-1c', '%s\n' % entry.name) for l in ld: if ld[0][0][0] == '\'': r.append( [ds[t - 1], entry.name, s, l[1], l[0]]) elif isinstance(ld[0][2], type({})): r.append([ ds[t - 1], entry.name, s, l[2][l[0]], l[0] ]) else: r.append( [ds[t - 1], entry.name, s, l[2], l[0]]) s += 1 rd = pd.DataFrame(r, columns=['dataset', 'name', 'step', 'acc', 'feature']) rds = rd.groupby(by=['dataset', 'name']).agg('max')['step'] rd2 = pd.DataFrame({ 'dataset': np.array(rds.index.get_level_values(0).values), 'step': np.array(rds.values) }) ax: plt.Axes = rd2.boxplot(column='step', by='dataset') ax.get_figure().suptitle('') ax.set_title('Number of selected features') ax.get_figure().canvas.set_window_title('num_feat') plt.show()
def show_growth_stat(): axs = {} for t in [1, 2]: app.mText.insert('end-1c', 'type: %s\n' % t) r = [] with os.scandir() as li: for entry in li: if entry.is_file() and ".pkl" in entry.name: dill.load_session(entry.name) s = 1 if (t == 1 and ld[0][0] in ['src_bytes', '\'src_bytes\''])\ or (t == 2 and ld[0][0] not in ['src_bytes', '\'src_bytes\'']): app.mText.insert('end-1c', '%s\n' % entry.name) for l in ld: if ld[0][0][0] == '\'': r.append([entry.name, s, l[1], l[0]]) elif isinstance(ld[0][2], type({})): r.append([entry.name, s, l[2][l[0]], l[0]]) else: r.append([entry.name, s, l[2], l[0]]) s += 1 rd = pd.DataFrame(r, columns=['name', 'step', 'acc', 'feature']) ax: plt.Axes = rd.boxplot(column='acc', by='step') ax.get_figure().suptitle('') ax.set_title('Accuracy by step') ax.get_figure().canvas.set_window_title('growth') ''' ax = rd.boxplot(column='acc', by='step') ax.get_figure().suptitle('') ax.set_title('Accuracy by step') ax.set_yscale('log') ax.get_figure().canvas.set_window_title('growth_log') ''' axs[t] = ax xlim = axs[1].get_xlim() ylim = axs[1].get_ylim() axs[2].set_xlim(xlim[0], xlim[1]) axs[2].set_ylim(ylim[0], ylim[1]) plt.show()
def show_important_features_builtin(): axs = {} global top_n, FI for t in [1, 2]: app.mText.insert('end-1c', 'type: %s\n' % t) r = [] with os.scandir() as li: for entry in li: if entry.is_file() and ".pkl" in entry.name: top_n = [] FI = [] dill.load_session(entry.name) if (t == 1 and ld[0][0] in ['src_bytes', '\'src_bytes\''])\ or (t == 2 and ld[0][0] not in ['src_bytes', '\'src_bytes\'']): app.mText.insert('end-1c', '%s\n' % entry.name) if len(top_n): app.mText.insert( 'end-1c', '%s;%s;%s;%s;%s;%s\n' % (X_train.columns[top_n[0]], locale.format("%g", FI[top_n[0]]), X_train.columns[top_n[1]], locale.format("%g", FI[top_n[1]]), X_train.columns[top_n[2]], locale.format("%g", FI[top_n[2]])))
def load_session(dir): """ The method to load a session #------------------ Parameters: *argv: the objects that wanted to be stored in the session #------------------ Returns: Loads the session #------------------ """ return dill.load_session(dir)
def load_data_KDD(): dill.load_session('NSL-KDD.pkl')
# In[1]: import numpy as np import pandas as pd from sklearn.model_selection import train_test_split from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import r2_score, mean_squared_error from sklearn import metrics from sklearn import preprocessing from sklearn.preprocessing import LabelEncoder from sklearn.preprocessing import StandardScaler import seaborn as sns import matplotlib.pyplot as plt import dill dill.load_session("tsne.db") # In[ ]: from sklearn.manifold import TSNE import seaborn as sns def run_tsne(X, y, Perplexity, Iterations, noprogresstolerance): tsneModel = TSNE(perplexity=Perplexity, n_iter=Iterations, n_iter_without_progress=noprogresstolerance, n_jobs=100) X_embedded = tsneModel.fit_transform(X) return sns.scatterplot(X_embedded[:, 0], X_embedded[:, 1],
# 2018 Oct 9 - Continuing to analyze import dill in_filename = 'Analyzing_Kepler-76b_Using_Emcee.pkl' dill.load_session(in_filename) # Recover last position pos = sampler.chain[:, -1, :] nsteps = 5000 for i, result in enumerate(sampler.sample(pos, iterations=nsteps)): if (i + 1) % 50 == 0: print("{0:5.1%}".format(float(i) / nsteps)) out_filename = 'Continuing_to_Analyze_Kepler-76b_Using_Emcee.pkl' dill.dump_session(out_filename) print(np.mean(sampler.chain[:, :, 0]), np.std(sampler.chain[:, :, 0]))
def load_data_cancer(): dill.load_session('UCI-BCW.pkl')
from __future__ import absolute_import import os import dill import __main__ as _main_module import six if six.PY2: PYCALC_SESSION_NAME = os.path.expanduser("~/.pycalc_session") elif six.PY3: PYCALC_SESSION_NAME = os.path.expanduser("~/.pycalc3_session") else: print("Python version not supported") try: dill.load_session(PYCALC_SESSION_NAME) except IOError: print("Re-loading pycalc modules...") import numpy as np _main_module.np = np import scipy as sc _main_module.sc = sc import sympy as sp import sympy.physics _main_module.sp = sp _main_module.u = sp.physics.units _main_module.u.B = sympy.physics.units.Unit('byte', 'B') _main_module.u.b = sympy.Rational(1, 8) * sympy.physics.units.B _main_module.u.KiB = 1024 * sympy.physics.units.B
# Broadcasting: https://docs.scipy.org/doc/numpy-1.13.0/reference/ufuncs.html#ufuncs-broadcasting # 当多个array做element-wize function时,按下面的规则broadcasting # - 对齐维度: 确定最大的dimension, 先prepend 长度为1的dimension来对齐所有dimension # - 对齐维度长度: 所有维度要么长度相等,要么长度为1. # 存储整个环境变量 # https://stackoverflow.com/a/35296032 import dill # pip install dill filename = 'globalsave.pkl' dill.dump_session(filename) # and to load the session again: dill.load_session(filename) # Jupyter notebook 相关 -------------------------- jupyter nbconvert --to=python [YOUR_NOTEBOOK].ipynb # 会生成 [YOUR_NOTEBOOK].py 文件 jupyter nbconvert --to notebook --output OUT.ipynb --execute [YOUR_NOTEBOOK].ipynb # 如果没有指定 --to nbtebook,默认输出类型是html # 不加output会生成 [YOUR_NOTEBOOK].html 或者 [YOUR_NOTEBOOK].nbconvert.ipynb # Seems below method does not work # Configuring https://nbconvert.readthedocs.io/en/latest/config_options.html # add below into ~/.jupyter/jupyter_nbconvert_config.json "ExecutePreprocessor": {
uct_visit_ave_cnt_list, ocba_visit_ave_cnt_list = [], [] uct_ave_Q_list, ocba_ave_Q_list = [], [] uct_ave_std_list, ocba_ave_std_list = [], [] if opp_first_move == 0: optimal_set = {4} setup = 1 elif opp_first_move == 4: optimal_set = {0, 2, 6, 8} setup = 2 else: raise ValueError('opp_first_move should either be 0 or 4.') ckpt = args.checkpoint if ckpt != '': dill.load_session(ckpt) # resume experiment from the last finished budget budget_range = range(budget + step, budget_end + 1, step) for budget in budget_range: PCS_uct = 0 PCS_ocba = 0 uct_selection.append([]) ocba_selection.append([]) uct_visit_cnt, ocba_visit_cnt = defaultdict(int), defaultdict(int) uct_ave_Q, ocba_ave_Q = defaultdict(int), defaultdict(int) uct_ave_std, ocba_ave_std = defaultdict(int), defaultdict(int) for i in range(rep): uct_mcts, uct_root_node, uct_cur_node = play_game_uct( budget=budget, exploration_weight=exploration_weight,
def load_model(self, filename): dill.load_session(filename)
d_tpp = datetime.strptime(d, '%Y-%m-%d') df['date'][df['date']==d] = d_tpp # Maybe then better to store session from here # Will review tomorrow # Save session dnm = r'C:\Users\rghiglia\Documents\ML_ND\Yelp' fnmO = (dnm + '\\' + 'yelp_data.pkl') import time import dill #pip install dill --user start_time = time.time() dill.dump_session(fnmO) print("--- %s seconds ---" % (time.time() - start_time)) # It seems prohibitively long :( # Load session dnm = r'C:\Users\rghiglia\Documents\ML_ND\Yelp' import time import dill #pip install dill --user start_time = time.time() dill.load_session(fnmO) print("--- %s seconds ---" % (time.time() - start_time))
def load_session(file_path): return dill.load_session(file_path)
import sys import os import dill #----- Global variables ---------- path_globals = str(sys.argv[1]) dill.load_session(path_globals) #-------------------------------- prefix = "abs_" dir_plots = dir_main + "plots/" dir_chunks = dir_main + "chunks/" file_abs = dir_main + "Absolute_magnitudes.csv" bands = [] errors = [] covariates = [] waves = [] for instrument in instruments: bands.append(instrument["observables"]) errors.append(instrument["uncertainties"]) covariates.append(instrument["covariates"]) waves.append(instrument["wavelengths"]) bands = sum(bands, []) errors = sum(errors, []) covariates = sum(covariates, []) waves = sum(waves, []) observables = [prefix + band for band in bands] uncertainties = [prefix + error for error in errors]
def load_session(file_path): with _pickle_lock_unless_py2: return dill.load_session(file_path)
# -*- coding: utf-8 -*- """ Created on Thu Nov 13 16:53:46 2014 @author: s362khan """ import dill import matplotlib.pyplot as plt import numpy as np import scipy.stats as ss plt.ion() dill.load_session('positionToleranceData.pkl') # --------------------------------------------------------- # Code to plot Zoccolan Data onto a figure zoc_pos_tol = yScatterRaw * np.pi / 180.0 # in Radians zoc_sel_af = xScatterRaw plt.scatter(zoc_sel_af, zoc_pos_tol, label='Original Data', marker='o', s=60, color='blue') plt.plot(xScatterRaw, (yLineFit[0]* np.pi / 180.0 * xScatterRaw + yLineFit[1] * np.pi / 180.0), color='blue', linewidth=2) #---------------------------------------------------------- plt.figure('Original Data')
# -*- coding: utf-8 -*- """ Created on Sat Apr 7 13:31:16 2018 @author: msval """ import dill filename = 'globalsave.pkl' dill.load_session(filename)
a_const = 1 a_lst = [1, 2, 3] a_dict = {'a': 1, 'b': [1, 2]} a_df = pandas.DataFrame({'col': [2, 4]}) a_lambda_func = lambda x: x**2 def a_func(x): return x**2 class a_class(object): def class_func(self, n): return n + 1 a_class_instance = a_class() # to save session dill.dump_session('./backup_workspace/backup/backup_dill.db') # ====== to load your session ====== import dill bk_restore = dill.load_session('./backup_workspace/backup/backup_dill.db') restored_variables = [el for el in dir() if el.startswith('a_')] print(f'{len(restored_variables)} restored variables: ', restored_variables)
reload(msabpy) ######################################## # Play sound when a program finishes ######################################## from IPython.display import Audio, display def allDone(): display(Audio(url="https://sound.peal.io/ps/audios/000/000/537/original/woo_vu_luvub_dub_dub.wav', autoplay=True)) ######################################## # Clear notebook output ######################################## from IPython.display import clear_output clear_output(wait=True) ######################################## # Save (& restore) variables ######################################## import dill dill.dump_session("notebook.db") dill.load_session("notebook.db") ######################################## # Reusing variables across notebooks ######################################## %store # List of all variables and their current values %store var1 # Store variable `var1` %store -r var1 # Restore variable `var1` %store -z # Remove all variables from storage # Reference: https://ipython.org/ipython-doc/3/config/extensions/storemagic.html