def save_session(direc,name="isocompy_saved_session", *argv):
"""
The method to save a session
#------------------
Parameters:
name: str default="isocompy_saved_object"
The output name string
*argv: the objects that wanted to be stored in the session
#------------------
Returns:
filename string
Directory of the saved session
#------------------
"""
dateTimeObj = datetime.now().strftime("_%d_%b_%Y_%H_%M")
filename=os.path.join(direc,name+dateTimeObj+".pkl")
for arg in argv:
arg
dill.dump_session(filename)
print ("\n\n pkl saved session directory: \n\n",filename)
def save_progress(data=None, folder_name=None, fig=None):
"""Saves session to the project data folder.
Path can be specified, if not, an auto generated folder based on the
current date-time is used. May include a plot.
:param data: A data object to save. If None, whole session is saved.
:type data: object
:param folder_name: A path-like string containing the output path.
:type folder_name: str
:param fig: A figure object.
:type fig: matplotlib.figure.Figure
"""
if folder_name is None:
folder_name = time.strftime("%Y%m%d-%H%M%S")
folder_name = "data/local/" + str(folder_name)
if not os.path.exists(os.path.join(str(folder_name), "plots")):
os.makedirs(os.path.join(str(folder_name), "plots"))
if fig:
plot_path = os.path.join(str(folder_name), "plots/plot.pdf")
fig.savefig(plot_path)
if data is None:
try:
import dill
except ImportError:
print("Couldn't import package dill. Aborting save progress.")
return None
sess_path = os.path.join(str(folder_name), "session.pkl")
dill.dump_session(sess_path)
else:
data_path = os.path.join(str(folder_name), "data.pt")
th.save(data, data_path, pickle_protocol=-1)
return folder_name
def persist_env(file):
"""
Persist an entire environment on file. This function depends on Dill package
:param file: file name to store the environment
"""
dill.dump_session(file)
def train(self, batch_size, inputs, labels, num_epochs, learning_rate,
filename):
'''
It takes 6 inputs:
1. batch_size: Mini batch size for gradient descent.
2. inputs: Inputs to be given to the network.
3. labels: Target values.
4. num_epochs: Number of epochs i.e. how many times the program should iterate over all training .
5. learning_rate: Learning rate for the algorithm, as discussed in DL01.
6. filename: The name of the file that will finally store all variables after training. (filename must have the extension .pkl).
'''
self.batch_size = batch_size
self.learning_rate = learning_rate
self.check_training_data(self.batch_size, inputs, labels)
for j in range(num_epochs):
i = 0
print("== EPOCH: ", j + 1, "/", num_epochs, " ==")
while i + batch_size != len(inputs):
print("Training with ",
i + batch_size + 1,
"/",
len(inputs),
end="\r")
self.error = 0
self.forward_pass(inputs[i:i + batch_size])
self.calculate_error(labels[i:i + batch_size])
self.back_pass(labels[i:i + batch_size])
i += batch_size
self.error /= batch_size
print("\nError: ", self.error)
print("Saving...")
dill.dump_session(filename)
def save_session(filename, add_timestamp=True, out_dir=""):
"""Save current workspace using dill.
Returns True is save is successful
"""
#timestamp format YYYYMMDDHHMISE
timestamp = getTimestamp(True)
if not out_dir:
out_dir = os.getcwd()
else:
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
outFile = os.path.join(out_dir, filename)
if add_timestamp:
outFile = outFile + "_" + timestamp + ".pyrpipe"
else:
outFile = outFile + ".pyrpipe"
#save workspace
try:
dill.dump_session(outFile)
print("Session saved to: " + outFile)
return True
except Exception:
raise OSError("Failed to save session")
def save_game() -> list:
messages = []
try:
dill.dump_session(filename='game.pkl')
messages.append(Message('Game saved'))
except:
messages.append(Message('There was an error saving the game. '))
return messages
def save():
now = time.strftime('%Y-%m-%d_%H:%M')
try:
os.mkdir('jupyterData')
except FileExistsError:
pass
filename = 'jupyterData/%s.jupyterData' % now
dill.dump_session(filename)
print('data saved')
def dump_session(file_path):
"""Pickle the current python session to be used in the worker.
Note: Due to the inconsistency in the first dump of dill dump_session we
create and load the dump twice to have consistent results in the worker and
the running session. Check: https://github.com/uqfoundation/dill/issues/195
"""
dill.dump_session(file_path)
dill.load_session(file_path)
return dill.dump_session(file_path)
def dump_session(file_path): """Pickle the current python session to be used in the worker. Note: Due to the inconsistency in the first dump of dill dump_session we create and load the dump twice to have consistent results in the worker and the running session. Check: https://github.com/uqfoundation/dill/issues/195 """ dill.dump_session(file_path) dill.load_session(file_path) return dill.dump_session(file_path)
def train(self, inputs, targets, num_epochs, learning_rate, filename):
self.learning_rate = learning_rate
for i in range(num_epochs):
print("== EPOCH: ", i, " ==")
# self.error = 0
self.forward_pass(inputs)
self.calculate_error(targets)
self.back_pass(targets)
print("Error: ", self.error)
dill.dump_session(filename)
def _dumpSession(namePrefix=None):
""" Dump session variables to a pickle file """
sessId = ''
if namePrefix:
sessId = '%s_%s_b%d' % (namePrefix, os.getpid(), _batchId)
else:
sessId = '%s_%s_%s_b%d' % (getpass.getuser(), socket.gethostname(),
os.getpid(), _batchId)
pathPkl = os.path.join(os.getcwd(), '%s.pkl' % sessId)
pickle.dump_session(filename=pathPkl)
return sessId, pathPkl
def session_to_s3(prefix, bucket_name, timestamp=True, **kwargs):
"""Save session to S3 bucket. Login via ~/.aws/credentials as per boto3."""
if timestamp:
now_str = str(datetime.now())
date_time_str = re.sub('[^0-9a-zA-Z]+', '_', now_str)
filename = prefix + "_" + date_time_str + ".pkl"
else:
filename = prefix + ".pkl"
dill.dump_session(filename)
s3 = boto3.resource('s3')
s3.meta.client.upload_file(filename, bucket_name, filename, **kwargs)
return filename
def deploy(self, model, sample_inputs=[], ignore=[]):
assert is_iterable(sample_inputs)
pickled_model = codecs.encode(dill.dumps(model),
"base64_codec").decode()
with io.BytesIO() as buffer:
dill.dump_session(buffer)
pickled_session = codecs.encode(buffer.getvalue(),
"base64_codec").decode()
pickled_session = exclude_ignores_and_redump(pickled_session, ignore)
large_vars = list_large_vars(pickled_session)
if len(large_vars) > 0:
error_msg = f'''
Convect cannot deploy a session containing variables holding more than 0.5MB of data.
If these variables are not required to host the model, you can use the "ignore" argument
to exclude them from the deployment. E.g.
convect_sdk.deploy(model, sample_inputs={sample_inputs}, ignore=[{", ".join([f'"{v}"' for v in (ignore + large_vars)])}])
'''
print(error_msg)
return
model_submission_data = {
"pickled_model": pickled_model,
"pickled_session": pickled_session,
"enable_endpoint": False
}
response = requests.post(
f"{self.create_host}/submitted-models/",
json=model_submission_data,
headers={'Authorization': f'Token {self.api_token}'})
if response.status_code != 201:
response.raise_for_status()
else:
response_json = response.json()
sample_input_data = [{
"submitted_model": response_json['pk'],
"json_payload": sample_input,
} for sample_input in sample_inputs]
requests.post(f"{self.create_host}/sample-inputs/",
json=sample_input_data,
headers={'Authorization': f'Token {self.api_token}'})
submission_data = {
"model_id": response_json['pk'],
}
print(
f"Your model ({response_json['pk'][:6]}...) is now deployed! Try it out with this shell command:\n\n"
+
f"curl -H 'Content-Type: application/json' -d '{json.dumps(sample_inputs)}' -X POST {self.predict_host}/{response_json['pk']}/"
)
return submission_data
def save_Session(name='temp'):
import shelve
from pathlib import Path
import dill
home = str(Path.home())
import os
cwd = os.getcwd()
filename = cwd + "\\tmp\\" + name + '.pk1'
# filename = home + r'\University Of Cambridge\OneDrive - University Of Cambridge\Lab Computer\UtilityRangeAdaptation\tmp' + "\\" + name + '.pk1'
# dill.settings['recurse'] = True
dill.dump_session(filename)
def dump_session(file_path):
"""For internal use only; no backwards-compatibility guarantees.
Pickle the current python session to be used in the worker.
Note: Due to the inconsistency in the first dump of dill dump_session we
create and load the dump twice to have consistent results in the worker and
the running session. Check: https://github.com/uqfoundation/dill/issues/195
"""
with _pickle_lock_unless_py2:
dill.dump_session(file_path)
dill.load_session(file_path)
return dill.dump_session(file_path)
def store_session(cls,
name: str,
type: str = 'pickle',
sign: str = 'SHA256',
main=None,
byref=False,
**kwargs):
'''
In Memory Storage
'''
this = cls(name, type, sign, **kwargs)
dill.dump_session(this._stream, main, byref, **kwargs)
this._stream.seek(0)
return this
def train(self, batch_size, inputs, labels, num_epochs, learning_rate, filename):
self.batch_size = batch_size
self.learning_rate = learning_rate
for j in range(num_epochs):
i = 0
print("== EPOCH: ", j, " ==")
while i+batch_size != len(inputs):
self.error = 0
self.forward_pass(inputs[i:i+batch_size])
self.calculate_error(labels[i:i+batch_size])
self.back_pass(labels[i:i+batch_size])
i += batch_size
print("Error: ", self.error)
dill.dump_session(filename)
def train(self, batch_size, inputs, labels, num_epochs, learning_rate, filename):
self.batch_size = batch_size
self.learning_rate = learning_rate
for j in range(num_epochs):
i = 0
print("== EPOCH: ", j, " ==")
while i+batch_size != len(inputs):
self.error = 0
self.forward_pass(inputs[i:i+batch_size])
self.calculate_error(labels[i:i+batch_size])
self.back_pass(labels[i:i+batch_size])
i += batch_size
print("Error: ", self.error)
dill.dump_session(filename)
def WorkspaceBasedCheckPt(CheckPtPosition = 0, AccessRecentOrNewWorkSpaceImage = False, Force = {'access':False, 'task':'load'}, CheckPoint_Dir = "NotebookCheckpoints/"):
if not os.path.exists(CheckPoint_Dir):
os.makedirs(CheckPoint_Dir)
file= open((CheckPoint_Dir + '0__CheckPt.db'),"w+")
file.close()
LastCheckPt = max([int(CheckPoint.split('/')[len(CheckPoint.split('/')) -1].split('__')[0]) for CheckPoint in glob.glob(CheckPoint_Dir + '*{}'.format('CheckPt.db'))])
## Force can be used when proceding in non serialized manner
if Force['access']: ## could have been compressed with chunk below using boolean algebra to make code more smaller
AccessRecentOrNewWorkSpaceImage = False
if Force['task'] == 'load':
print("Checkpoint ", CheckPtPosition, "is to be loaded")
if os.path.exists(CheckPoint_Dir + str(CheckPtPosition) + '__CheckPt.db'):
dill.load_session(CheckPoint_Dir + str(CheckPtPosition) + '__CheckPt.db') # To restore a session
else:
print("This checkpoint doesn't exist, hence won't be loaded.")
elif Force['task'] == 'save':
print("Checkpoint ", CheckPtPosition, "is to be Saved")
dill.dump_session(CheckPoint_Dir + str(CheckPtPosition) + '__CheckPt.db') # To Save a session
return "Force used to {} workspace checkpoint_{}.".format(Force['task'], CheckPtPosition) ## exit here only
## This Code below is used to handle the actions on returning the value to run/not run the cell
### = is set so that the current check point cell code are able to run
if ((AccessRecentOrNewWorkSpaceImage == False) and (CheckPtPosition <= LastCheckPt)):
print('Most Recent Checkpoint is : {} \nHence, cells won\'t be running content untill most recent checkpoint is crossed.'.format(LastCheckPt))
return False
elif ((AccessRecentOrNewWorkSpaceImage == False) and (CheckPtPosition == (LastCheckPt +1))):
print('Running this cell')
return True
elif ((AccessRecentOrNewWorkSpaceImage == False) and (CheckPtPosition > (LastCheckPt +1))):
print("You have skipped over a checkpoint. Still running this cell")
return True
## This Code below is used to handle the actions on saving/loading the workspace images
if (AccessRecentOrNewWorkSpaceImage and (CheckPtPosition == 0)):
print("Initial Phase, hence not saving workspace.")
elif (AccessRecentOrNewWorkSpaceImage and (LastCheckPt > CheckPtPosition)):
print("This is not the most recent checkpoint, hence not loading it. [Use Force to force load a checkpoint]")
elif (AccessRecentOrNewWorkSpaceImage and (LastCheckPt == CheckPtPosition)):
dill.load_session(CheckPoint_Dir + str(CheckPtPosition) + '__CheckPt.db') # To restore a session
print("This is the most recent checkpoint, hence loading it.")
elif (AccessRecentOrNewWorkSpaceImage and ((LastCheckPt +1) == CheckPtPosition)):
dill.dump_session(CheckPoint_Dir + str(CheckPtPosition) + '__CheckPt.db') # To Save a session
print("Congrats, on reaching a new checkpoint, saving it.")
elif (AccessRecentOrNewWorkSpaceImage and ((LastCheckPt +1) < CheckPtPosition)):
print("You have skipped over a checkpoint. Hence not Saving anything.")
def run(self,
nprocesses=4,
print_errors=False,
save_session=False,
autokill=None):
"""Execute all tasks in the `{directory}/todo}` directory.
All tasks are executed in their own processes, and `run` makes
sure that no more than `nprocesses` are active at any time.
If `print_errors=True`, processes will print full stack traces
of failing tasks. Since these errors happen on another
process, this will not be caught by the debugger, and will not
stop the `run`.
Use `save_session` to recreate all current globals in each
process.
"""
if save_session:
dill.dump_session(self._directory / 'session.pkl')
class TaskIterator:
def __init__(self, parent, todos, save_session):
self.parent = parent
self.todos = todos
self.save_session = save_session
def __iter__(self):
for todo in self.todos:
yield from self.parent._finish_tasks(nprocesses,
autokill=autokill)
self.parent._start_task(todo.name, print_errors,
save_session)
# wait for running jobs to finish:
yield from self.parent._finish_tasks(1, autokill=autokill)
def __len__(self):
return len(self.todos)
return TaskIterator(self, list((self._directory / 'todo').iterdir()),
save_session)
def train(self, batch_size, inputs, labels, num_epochs, learning_rate,
filename):
self.batch_size = batch_size
self.learning_rate = learning_rate
for j in range(num_epochs):
i = 0
print("== EPOCH: ", j, " ==")
while i + batch_size != len(inputs):
self.error = 0
input_batch = []
label_batch = []
# print(i)
for i in range(i, i + batch_size):
input_batch.append(inputs[i])
label_batch.append(labels[i])
self.forward_pass(input_batch)
self.calculate_error(label_batch)
self.back_pass(label_batch)
i += 1
print("Error: ", self.error)
dill.dump_session(filename)
def train(self, batch_size, training_inputs, labels, n_epochs,
learning_rate, filename):
self.batch_size = batch_size
self.learning_rate = learning_rate
for j in range(n_epochs):
i = 0
print("/// EPOCH: ", j + 1, "/", n_epochs, " ///")
output_layer_1, output_layer_2, output_layer_3 = self.forward_pass(
training_inputs)
layer3_error = self.calculate_error(labels, output_layer_3)
self.backward_pass(learning_rate, labels, training_inputs,
output_layer_1, output_layer_2, output_layer_3)
error_result = layer3_error / len(output_layer_3)
print("\nError: ", error_result)
costs.append(error_result)
print("Saving...")
dill.dump_session(filename)
def trainer(self, batchSize, data, flags, numPasses, learningRate,
dataFile):
self.batchSize = batchSize
self.learningRate = learningRate
self.compareTrainingSet(self.batchSize, data, flags)
for j in range(numPasses):
i = 0
print("== ITERATIONS: ", j + 1, "/", numPasses, " ==")
while i + batchSize != len(data):
print("Training with ",
i + batchSize + 1,
"/",
len(data),
end="\r")
self.error = 0
self.passForward(data[i:i + batchSize])
self.errorCalc(flags[i:i + batchSize])
self.passBackward(flags[i:i + batchSize])
i += batchSize
self.error /= batchSize
print("\nError: ", self.error)
print("Saving...")
dill.dump_session(dataFile)
def train(self, batch_size, inputs, labels, num_epochs, learning_rate,
filename):
self.batch_size = batch_size
self.learning_rate = learning_rate
self.check_training_data(self.batch_size, inputs, labels)
for j in range(num_epochs):
i = 0
print("== EPOCH: ", j + 1, "/", num_epochs, " ==")
while i + batch_size != len(inputs):
print("Training with ",
i + batch_size + 1,
"/",
len(inputs),
end="\r")
self.error = 0
self.forward_pass(inputs[i:i + batch_size])
self.calculate_error(labels[i:i + batch_size])
self.back_pass(labels[i:i + batch_size])
i += batch_size
self.error /= batch_size
print("\nError: ", self.error)
print("Saving...")
dill.dump_session(filename)
def save_session(filename, add_timestamp=True, out_dir=""):
"""Save current workspace using dill.
Returns True is save is successful
"""
#timestamp format YYYYMMDDHHMISE
timestamp = getTimestamp(True)
if not out_dir:
out_dir = os.getcwd()
else:
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
outFile = os.path.join(out_dir, filename)
if add_timestamp:
outFile = outFile + "_" + timestamp + ".pyrpipe"
else:
outFile = outFile + ".pyrpipe"
"""
Do not pickle logger. This causes problems when restoring session with python < 3.7
Delete all logger instances.
del pre.pyrpipeLoggerObject
del pyrpipe.pyrpipe_engine.pyrpipeLoggerObject
"""
"""
creating a logger class fixed this issue
"""
#save workspace
try:
dill.dump_session(outFile)
print("Session saved to: " + outFile)
return True
except Exception:
raise Exception("Failed to save session")
return False
# Updating the param grid
param_grid = dict(activation=['identity', 'logistic', 'tanh', 'relu'],
alpha=[0.0005, 0.0001, 0.005, 0.001, 0.01, 0.1],
batch_size=[32, 64, 96, 128, 256],
hidden_layer_sizes=[(50, 100, 50), (50, 100), (100, 50)],
learning_rate=['constant', 'invscaling', 'adaptive'],
learning_rate_init=[0.0001, 0.001, 0.01, 0.1],
max_iter=[250, 500, 1000],
solver=['lbfgs', 'sgd', 'adam'])
print('Multi-layer Perceptron Classifier')
mlp_grid_search = GridSearchCV(mlp,
param_grid,
scoring='accuracy',
n_jobs=-1,
cv=kfold,
verbose=1)
mlp_grid_result = mlp_grid_search.fit(X, Y)
mlp_predict = mlp_grid_search.predict(X)
mlp_predict_proba = pd.DataFrame(mlp_grid_search.predict_proba(X))
# Store metrics
mlp_accuracy = metrics.accuracy_score(Y, mlp_predict)
mlp_precision = metrics.precision_score(Y, mlp_predict, pos_label=1)
mlp_recall = metrics.recall_score(Y, mlp_predict, pos_label=1)
mlp_f1 = metrics.f1_score(Y, mlp_predict, pos_label=1)
mlp_auroc = metrics.roc_auc_score(Y, mlp_predict_proba[1])
mlp_aurpc = metrics.average_precision_score(Y, mlp_predict, pos_label=1)
dill.dump_session('MLPClassifier_Parameter_Tuning.db')
df[['mpg', 'displacement', 'horsepower', 'weight', 'acceleration']],
diagonal='kde')
#%% [markdown]
# Plot the scatterplot, with custom variable values on mouse hover (using Plotly)
# Marker color and size can also be set to variables.
import plotly.graph_objects as go
fig = go.Figure(data=go.Scatter(
x=df['horsepower'], y=df['mpg'], mode='markers', text=df['name']))
fig.update_layout(
xaxis=go.layout.XAxis(title=go.layout.xaxis.Title(text='Horsepower')),
yaxis=go.layout.YAxis(title=go.layout.yaxis.Title(text='MPG')))
fig.show()
#%% [markdown]
#### Descriptive statistics
# Include all variables
df.describe()
#%% [markdown]
# Include only a subset of variables
df['mpg'].describe()
#%% [markdown]
#### Serialization - save or load the current session
import dill
filename = 'globalsave.pkl'
dill.dump_session(filename)
# and to load the session again:
dill.load_session(filename)
print ret, mtx, dist, rvecs, tvecs
##img = cv2.imread('C:/Users/d7rob/thesis/chess/rgb/img1_2017-09-25_23-18-52.001_1.jpg')
####img = cv2.imread('chess/left12.jpg')
##h, w = img.shape[:2]
##newcameramtx, roi=cv2.getOptimalNewCameraMatrix(mtx, dist, (w,h), 1, (w,h))
mean_error = 0
for i in xrange(len(objpoints)):
imgpoints2, _ = cv2.projectPoints(objpoints[i], rvecs[i], tvecs[i], mtx,
dist)
error = cv2.norm(imgpoints[i], imgpoints2, cv2.NORM_L2) / len(imgpoints2)
mean_error += error
total_error = mean_error / len(objpoints)
print("total error: {}".format(total_error))
dill.dump_session(savefile)
### undistort
##mapx, mapy = cv2.initUndistortRectifyMap(mtx, dist, None, newcameramtx, (w,h), 5)
##dst = cv2.remap(img, mapx, mapy, cv2.INTER_LINEAR)
### crop the image
##x, y, w, h = roi
##dst = dst[y:y+h, x:x+w]
##cv2.imwrite('calibresult.png', dst)
### undistort
##dst = cv2.undistort(img, mtx, dist, None, newcameramtx)
### crop the image
##x, y, w, h = roi
##dst = dst[y:y+h, x:x+w]
##cv2.imwrite('calibresult.png', dst)
# - 但是如果是 a[<高级index>] += XXX， 会对原来影响 # Broadcasting: https://docs.scipy.org/doc/numpy-1.13.0/reference/ufuncs.html#ufuncs-broadcasting # 当多个array做element-wize function时，按下面的规则broadcasting # - 对齐维度: 确定最大的dimension， 先prepend 长度为1的dimension来对齐所有dimension # - 对齐维度长度： 所有维度要么长度相等，要么长度为1. # 存储整个环境变量 # https://stackoverflow.com/a/35296032 import dill # pip install dill filename = 'globalsave.pkl' dill.dump_session(filename) # and to load the session again: dill.load_session(filename) # Jupyter notebook 相关 -------------------------- jupyter nbconvert --to=python [YOUR_NOTEBOOK].ipynb # 会生成 [YOUR_NOTEBOOK].py 文件 jupyter nbconvert --to notebook --output OUT.ipynb --execute [YOUR_NOTEBOOK].ipynb # 如果没有指定 --to nbtebook，默认输出类型是html # 不加output会生成 [YOUR_NOTEBOOK].html 或者 [YOUR_NOTEBOOK].nbconvert.ipynb # Seems below method does not work # Configuring https://nbconvert.readthedocs.io/en/latest/config_options.html
# In[ ]:
from sklearn.manifold import TSNE
import seaborn as sns
def run_tsne(X, y, Perplexity, Iterations, noprogresstolerance):
tsneModel = TSNE(perplexity=Perplexity,
n_iter=Iterations,
n_iter_without_progress=noprogresstolerance,
n_jobs=100)
X_embedded = tsneModel.fit_transform(X)
return sns.scatterplot(X_embedded[:, 0],
X_embedded[:, 1],
hue=y.ravel(),
legend='full')
# In[ ]:
TSNEres = run_tsne(X=X,
y=ywardD2,
Perplexity=100,
Iterations=6500,
noprogresstolerance=2500)
# In[ ]:
dill.dump_session("tsne_done.env")
_main_module.u = sp.physics.units
_main_module.u.B = sympy.physics.units.Unit('byte', 'B')
_main_module.u.b = sympy.Rational(1, 8) * sympy.physics.units.B
_main_module.u.KiB = 1024 * sympy.physics.units.B
_main_module.u.MiB = 1024**2 * sympy.physics.units.B
import pandas as pd
_main_module.pd = pd
from matplotlib import pyplot as plt
_main_module.plt = plt
import seaborn as sns
_main_module.sns = sns
dill.dump_session(PYCALC_SESSION_NAME, main=_main_module)
def setup_plotting():
"""Better defaults for plotting
"""
import matplotlib
import seaborn as sns
font = {'family': 'normal',
'weight': 'normal',
'size': 14}
matplotlib.rc('font', **font)
matplotlib.rc('text', usetex=True)
def dump_session(file_path): return dill.dump_session(file_path)
print "%i %i" % (i, nU)
d_tpp = datetime.strptime(d, '%Y-%m-%d')
df['date'][df['date']==d] = d_tpp
# Maybe then better to store session from here
# Will review tomorrow
# Save session
dnm = r'C:\Users\rghiglia\Documents\ML_ND\Yelp'
fnmO = (dnm + '\\' + 'yelp_data.pkl')
import time
import dill #pip install dill --user
start_time = time.time()
dill.dump_session(fnmO)
print("--- %s seconds ---" % (time.time() - start_time))
# It seems prohibitively long :(
# Load session
dnm = r'C:\Users\rghiglia\Documents\ML_ND\Yelp'
import time
import dill #pip install dill --user
start_time = time.time()
dill.load_session(fnmO)
print("--- %s seconds ---" % (time.time() - start_time))
# 2018 Oct 9 - Continuing to analyze
import dill
in_filename = 'Analyzing_Kepler-76b_Using_Emcee.pkl'
dill.load_session(in_filename)
# Recover last position
pos = sampler.chain[:, -1, :]
nsteps = 5000
for i, result in enumerate(sampler.sample(pos, iterations=nsteps)):
if (i + 1) % 50 == 0:
print("{0:5.1%}".format(float(i) / nsteps))
out_filename = 'Continuing_to_Analyze_Kepler-76b_Using_Emcee.pkl'
dill.dump_session(out_filename)
print(np.mean(sampler.chain[:, :, 0]), np.std(sampler.chain[:, :, 0]))
def run_on_server(func_master, *args, **kwargs):
"""Runs the given object - a function or class with a .start() method - on the pbs server
args/kwargs are passed to the function"""
from gausspy.gaussian_job_manager import Job
#we can pass in job_obj details with the function/class if we wish
try:
func_obj, job_obj = func_master
except TypeError:
func_obj = func_master
job_obj = None
try:
name = func_obj.calc.label
except AttributeError:
try:
name = func_obj.func_name
except AttributeError:
name = 'unknown'
name += '_' + str(uuid.uuid1())
#this means functions that actually have an inc_session keyword argument
#will have the value stripped out (I'm hoping we don't come across any)
try:
inc_session = kwargs.pop('inc_session')
except KeyError:
inc_session=False
try:
savefiles = kwargs.pop('savefiles')
except KeyError:
savefiles = ''
try:
namefile_f = kwargs.pop('namefile_f')
except KeyError:
namefile_f = lambda e: name
try:
compress = kwargs.pop('compress')
except KeyError:
compress = False
if inc_session:
dill.dump_session(name + '_session.pkl')
with open(name + '.pkl', 'w') as f:
dill.dump([func_obj, args, kwargs], f)
serv_home = config.get('gaussian', 'gauss_home')
path = os.path.join(serv_home + get_active_path(), '')
serv_work = config.get('gaussian', 'gauss_scratch')
scratch_path = os.path.join(serv_work + get_active_path(), '')
#if we have set the ASE and Gaussian home/work directories to nothing. I.e. we are running on the node
#then the only way of recovering the original directories is to use the PBS shell variables that contain the directory
#that the job was submitted from (which is the correct home directory).
#scratch_path = os.environ['PBS_O_WORKDIR '].replace('/home','/work')
exec_command = 'execute_calc {f_pckl};'.format(
pth=path,
f_pckl=path + name + '.pkl')
if compress and savefiles:
exec_command += 'mkdir -p {scratch};'.format(scratch=scratch_path)
exec_command += 'tar -cvjf {n}.tar.bz2 {f};'.format(n=name, f=savefiles)
exec_command += 'cp {n}.tar.bz2 {scratch};'.format(n=name, scratch=scratch_path)
elif savefiles:
exec_command += 'mkdir -p {scratch};'.format(scratch=scratch_path)
#exec_command += 'find . -maxdepth 1 ! -iregex "{r}" -exec cp -r {} {scratch} \;'.format(r=savefiles, scratch=scratch_path)
exec_command += 'cp {f} {scratch};'.format(f=savefiles, scratch=scratch_path)
if not job_obj:
try:
nodes = func_obj.calc.job_params['nodes']
mem = func_obj.calc.job_params['memory'] + nodes * 150
time = func_obj.calc.job_params['time']
queue = func_obj.calc.job_params['queue']
job_obj = Job(procs=nodes, memory=mem, walltime=time, queue=queue)
except AttributeError:
job_obj = Job()
script = job_obj.gen_header() + exec_command
with open(name + '_job_script.sh', 'w') as f:
f.write(script)
if inc_session:
extra_files = [name + '.pkl', name + '_session.pkl']
else:
extra_files = [name + '.pkl']
submission = remote.qsub(os.getcwd() + '/' + name + '_job_script.sh', extra_files=extra_files)
os.remove(name + '.pkl')
try:
os.remove(name + '_session.pkl')
except OSError:
pass
return submission
def save_data(fname):
dill.dump_session('%s.pkl' % fname)
def testJson(self):
wl = WinnerLoser('one','two')
ww = dill.dump_session('dill.pkl')
print ww
a_const = 1
a_lst = [1, 2, 3]
a_dict = {'a': 1, 'b': [1, 2]}
a_df = pandas.DataFrame({'col': [2, 4]})
a_lambda_func = lambda x: x**2
def a_func(x):
return x**2
class a_class(object):
def class_func(self, n):
return n + 1
a_class_instance = a_class()
# to save session
dill.dump_session('./backup_workspace/backup/backup_dill.db')
# ====== to load your session ======
import dill
bk_restore = dill.load_session('./backup_workspace/backup/backup_dill.db')
restored_variables = [el for el in dir() if el.startswith('a_')]
print(f'{len(restored_variables)} restored variables: ', restored_variables)
dump_str = pickle.dumps(OA())
f = open("/tmp/dump_str", "wb")
f.write(dump_str)
f.close()
f = open("/tmp/dump_str", "wb")
pickle.dump(OA(), f)
f.close()
#Open by another python
#======================
import dill as pickle
f = open("/tmp/dump_str", "rb")
dump_str = f.read()
f.close()
obj = pickle.loads(dump_str)
obj.test()
f = open("/tmp/dump_str", "rb")
obj = pickle.load(f)
f.close()
obj.test()
#=========Test sessions
import dill
dill.dump_session()
