def capture_print(state=True,
to_file=False,
log_file_path='dump/%T-log',
**print_and_store_kwargs):
"""
:param state: True to caputure print, False to not capture print
:param to_file: True to print to file
:param log_file_path: Path of file to print to, if (state and to_file)
:param print_and_store_kwargs: Passed to the PrintAndStoreLogger constructor.
:return: The relative path to the logger.
"""
if state:
rel_log_file_path = format_filename(log_file_path,
current_time=datetime.now(),
directory='logs',
ext='txt')
local_log_file_path = get_local_path(rel_log_file_path)
logger = PrintAndStoreLogger(log_file_path=local_log_file_path,
**print_and_store_kwargs)
if to_file:
relative_link = get_relative_link_from_relative_path(
rel_log_file_path)
log_folder_link = get_relative_link_from_relative_path('logs')
display(
HTML(
"Writing to <a href='%s' target='_blank'>this log file</a>. See <a href='%s' target='_blank'>all logs</a>"
% (relative_link, log_folder_link)))
sys.stdout = logger
sys.stderr = logger
return rel_log_file_path
else:
sys.stdout = _ORIGINAL_STDOUT
sys.stderr = _ORIGINAL_STDERR
def get_file(relative_name, url=None, data_transformation=None):
relative_folder, file_name = os.path.split(relative_name)
local_folder = get_local_path(relative_folder)
try: # Best way to see if folder exists already - avoids race condition between processes
os.makedirs(local_folder)
except OSError:
pass
full_filename = os.path.join(local_folder, file_name)
if not os.path.exists(full_filename):
assert url is not None, "No local copy of '%s' was found, and you didn't provide a URL to fetch it from" % (
full_filename, )
print 'Downloading file from url: "%s"...' % (url, )
response = urllib2.urlopen(url)
data = response.read()
print '...Done.'
if data_transformation is not None:
data = data_transformation(data)
with open(full_filename, 'w') as f:
f.write(data)
return full_filename
def get_file(relative_name, url = None, data_transformation = None):
relative_folder, file_name = os.path.split(relative_name)
local_folder = get_local_path(relative_folder)
try: # Best way to see if folder exists already - avoids race condition between processes
os.makedirs(local_folder)
except OSError:
pass
full_filename = os.path.join(local_folder, file_name)
if not os.path.exists(full_filename):
assert url is not None, "No local copy of '%s' was found, and you didn't provide a URL to fetch it from" % (full_filename, )
print 'Downloading file from url: "%s"...' % (url, )
response = urllib2.urlopen(url)
data = response.read()
print '...Done.'
if data_transformation is not None:
data = data_transformation(data)
with open(full_filename, 'w') as f:
f.write(data)
return full_filename
def capture_print(state = True, to_file = False, log_file_path = 'dump/%T-log', **print_and_store_kwargs):
"""
:param state: True to caputure print, False to not capture print
:param to_file: True to print to file
:param log_file_path: Path of file to print to, if (state and to_file)
:param print_and_store_kwargs: Passed to the PrintAndStoreLogger constructor.
:return: The relative path to the logger.
"""
if state:
rel_log_file_path = format_filename(log_file_path, current_time = datetime.now(), directory='logs', ext = 'txt')
local_log_file_path = get_local_path(rel_log_file_path)
logger = PrintAndStoreLogger(log_file_path=local_log_file_path, **print_and_store_kwargs)
if to_file:
relative_link = get_relative_link_from_relative_path(rel_log_file_path)
log_folder_link = get_relative_link_from_relative_path('logs')
display(HTML("Writing to <a href='%s' target='_blank'>this log file</a>. See <a href='%s' target='_blank'>all logs</a>"
% (relative_link, log_folder_link)))
sys.stdout = logger
sys.stderr = logger
return rel_log_file_path
else:
sys.stdout = _ORIGINAL_STDOUT
sys.stderr = _ORIGINAL_STDERR
def save_and_show(fig=None,
name='%T-%N',
ext='pdf',
base_dir='figures',
subdir='dump',
block=None,
print_loc=True,
show=True):
"""
Save and show a figure.
:param fig: The figure in question (or None to just get the current figure)
:param name: The base-name to save it under. If you put %T in, it will be replaced with the date/time. E.g.
'%T-test_fig.pdf' becomes '2015.03.20T13.35.00.068260-test_fig.pdf'
:param ext: The file format, if not specified in the name\
:param base_dir: The root directory for figures
:param subdir: The subdirectory in which to save this figure. You can also put %T in this to replace with current time.
:param block: Should the desiplay block execution? (If None, just go with current interactivemode setting)
:param print_loc: Print the save location?
:param show: Actually show the figure?
:return: The local file-path of the saved figure.
"""
if fig is None:
fig = plt.gcf()
fig_name = fig.get_label() if fig.get_label() is not '' else 'unnamed'
now = datetime.now().isoformat().replace(':', '.').replace('-', '.')
subdir = subdir.replace('%T', now)
name = name.replace('%T', now) + '.' + ext
name = name.replace('%N', fig_name)
is_interactive = plt.isinteractive()
if block is None:
block = not is_interactive
rel_figure_loc = os.path.join(base_dir, subdir, name)
local_figure_loc = get_local_path(rel_figure_loc)
make_file_dir(local_figure_loc)
fig.savefig(local_figure_loc)
if print_loc:
print 'Saved figure to "%s"' % (local_figure_loc, )
_SAVED_FIGURES.append(
rel_figure_loc
) # Which is technically a memory leak, but you'd have to make a lot of figures.
if show:
plt.interactive(not block)
_ORIGINAL_SHOW_FCN(
) # There's an argument block, but it's not supported for all backends
plt.interactive(is_interactive)
else:
plt.close()
return rel_figure_loc
def get_relative_link_from_relative_path(relative_path):
"""
Given a local path to a file in the data folder, return the relative link that will access it from
the server.
To do this, we make a soft-link from the server directory to the data folder - this way we can
browse our data folder from Jupyter, which is nice.
:param relative_path: Relative path (from within Data folder)
:return: A string representing the relative link to get to that file.
"""
true_local_data_dir = get_local_path()
local_path = get_local_path(relative_path)
launcher = 'tree' if os.path.isdir(local_path) else 'files'
if not os.path.lexists(SERVER_RELATIVE_DATA_DIR):
os.symlink(true_local_data_dir, SERVER_RELATIVE_DATA_DIR)
return os.path.join('/', launcher, DATA_FOLDER_NAME, relative_path)
def get_all_experiment_ids(expr = None):
"""
:param expr: A regexp for matching experiments
None if you just want all of them
:return: A list of experiment identifiers.
"""
expdir = get_local_path('experiments')
experiments = [e[:-len('.exp.pkl')] for e in os.listdir(expdir) if e.endswith('.exp.pkl')]
if expr is not None:
experiments = [e for e in experiments if re.match(expr, e)]
return experiments
def test_new_log_file():
# Just a shorthand for persistent print.
log_file_loc = new_log_file('dump/test_file')
print 'eee'
print 'fff'
capture_print(False)
local_log_loc = get_local_path(log_file_loc)
with open(local_log_loc) as f:
text = f.read()
assert text == 'eee\nfff\n'
os.remove(local_log_loc)
def test_new_log_file():
# Just a shorthand for persistent print.
log_file_loc = new_log_file('dump/test_file')
print 'eee'
print 'fff'
capture_print(False)
local_log_loc = get_local_path(log_file_loc)
with open(local_log_loc) as f:
text = f.read()
assert text == 'eee\nfff\n'
os.remove(local_log_loc)
def save_and_show(fig = None, name = '%T-%N', ext = 'pdf', base_dir = 'figures',
subdir = 'dump', block = None, print_loc = True, show = True):
"""
Save and show a figure.
:param fig: The figure in question (or None to just get the current figure)
:param name: The base-name to save it under. If you put %T in, it will be replaced with the date/time. E.g.
'%T-test_fig.pdf' becomes '2015.03.20T13.35.00.068260-test_fig.pdf'
:param ext: The file format, if not specified in the name\
:param base_dir: The root directory for figures
:param subdir: The subdirectory in which to save this figure. You can also put %T in this to replace with current time.
:param block: Should the desiplay block execution? (If None, just go with current interactivemode setting)
:param print_loc: Print the save location?
:param show: Actually show the figure?
:return: The local file-path of the saved figure.
"""
if fig is None:
fig = plt.gcf()
fig_name = fig.get_label() if fig.get_label() is not '' else 'unnamed'
now = datetime.now().isoformat().replace(':', '.').replace('-', '.')
subdir = subdir.replace('%T', now)
name = name.replace('%T', now) + '.'+ext
name = name.replace('%N', fig_name)
fig.canvas.set_window_title(name)
is_interactive = plt.isinteractive()
if block is None:
block = not is_interactive
rel_figure_loc = os.path.join(base_dir, subdir, name)
local_figure_loc = get_local_path(rel_figure_loc)
make_file_dir(local_figure_loc)
fig.savefig(local_figure_loc)
if print_loc:
print 'Saved figure to "%s"' % (local_figure_loc, )
_SAVED_FIGURES.append(rel_figure_loc) # Which is technically a memory leak, but you'd have to make a lot of figures.
if show:
plt.interactive(not block)
_ORIGINAL_SHOW_FCN() # There's an argument block, but it's not supported for all backends
plt.interactive(is_interactive)
else:
plt.close()
return rel_figure_loc
def get_all_experiment_ids(expr=None):
"""
:param expr: A regexp for matching experiments
None if you just want all of them
:return: A list of experiment identifiers.
"""
expdir = get_local_path('experiments')
experiments = [
e[:-len('.exp.pkl')] for e in os.listdir(expdir)
if e.endswith('.exp.pkl')
]
if expr is not None:
experiments = [e for e in experiments if re.match(expr, e)]
return experiments
def test_persistent_print():
test_log_path = capture_print(to_file=True)
print 'aaa'
print 'bbb'
assert read_print() == 'aaa\nbbb\n'
capture_print(False)
capture_print(True)
assert read_print() == ''
print 'ccc'
print 'ddd'
assert read_print() == 'ccc\nddd\n'
os.remove(get_local_path(test_log_path))
def test_persistent_print():
test_log_path = capture_print(to_file=True)
print 'aaa'
print 'bbb'
assert read_print() == 'aaa\nbbb\n'
capture_print(False)
capture_print(True)
assert read_print() == ''
print 'ccc'
print 'ddd'
assert read_print() == 'ccc\nddd\n'
os.remove(get_local_path(test_log_path))
def __exit__(self, *args):
# On exit, we read the log file. After this, the log file is no longer associated with the experiment.
capture_print(False)
with open(get_local_path(self._log_file_path)) as f:
self._captured_logs = f.read()
set_show_callback(None)
self._captured_figure_locs = get_saved_figure_locs()
self._has_run = True
if self._save_result:
file_path = get_local_experiment_path(self._experiment_identifier)
make_file_dir(file_path)
with open(file_path, 'w') as f:
pickle.dump(self, f)
print 'Saving Experiment "%s"' % (self._experiment_identifier, )
def __exit__(self, *args):
# On exit, we read the log file. After this, the log file is no longer associated with the experiment.
capture_print(False)
with open(get_local_path(self._log_file_path)) as f:
self._captured_logs = f.read()
set_show_callback(None)
self._captured_figure_locs = get_saved_figure_locs()
self._has_run = True
if self._save_result:
file_path = get_local_experiment_path(self._experiment_identifier)
make_file_dir(file_path)
with open(file_path, 'w') as f:
pickle.dump(self, f)
print 'Saving Experiment "%s"' % (
self._experiment_identifier, )
def test_save_figures():
clear_saved_figure_locs()
test_dir = os.path.join(get_local_figures_dir(), 'testing')
try: # Remove dir if not already removed
shutil.rmtree(test_dir)
except OSError:
pass
always_save_figures(subdir = 'testing', block = False, name = 'test_fig')
plt.plot(np.random.randn(100, 3))
plt.show()
figures = get_saved_figure_locs()
assert len(figures) == 1 and os.path.exists(get_local_path(figures[0])) and figures[0].endswith('testing/test_fig.pdf')
try: # Always good to clean up after yourself.
shutil.rmtree(test_dir)
except OSError:
pass
def get_local_figures_dir(subdir = None):
figures_dir = get_local_path('figures')
if subdir is not None:
figures_dir = os.path.join(figures_dir, subdir)
return figures_dir
def get_local_experiment_path(identifier):
return format_filename(identifier,
directory=get_local_path('experiments'),
ext='exp.pkl')
def get_local_experiment_path(identifier):
return format_filename(identifier, directory = get_local_path('experiments'), ext = 'exp.pkl')
def get_local_figures_dir(subdir=None):
figures_dir = get_local_path('figures')
if subdir is not None:
figures_dir = os.path.join(figures_dir, subdir)
return figures_dir
def show_saved_figure(relative_loc):
_, ext = os.path.splitext(relative_loc)
abs_loc = get_local_path(relative_loc)
assert os.path.exists(abs_loc), '"%s" did not exist. That is odd.' % (abs_loc, )
subprocess.call('open "%s"' % abs_loc, shell = True)
import hashlib
from collections import OrderedDict
import logging
from fileman.local_dir import get_local_path, make_file_dir
import numpy as np
import pickle
import os
__author__ = 'peter'
MEMO_WRITE_ENABLED = True
MEMO_READ_ENABLED = True
MEMO_DIR = get_local_path('memoize_to_disk')
def memoize_to_disk(fcn, local_cache = False):
"""
Save (memoize) computed results to disk, so that the same function, called with the
same arguments, does not need to be recomputed. This is useful if you have a long-running
function that is often being given the same arguments. Note: this does NOT check for the state
of Global variables/time/whatever else the function may use, so you need to make sure your
function is truly a function in that outputs only depend on inputs. Otherwise, this will
give you misleading results.
e.g.
@memoize_to_disk
def fcn(a, b, c = None):
results = ...
return results
You can also use this without the decorator.
e.g.
