def convergence(results, xvar, yvar, title, xlabel, ylabel, ymin, ymax):
"""
Plots a variable over e.g. time, iterations, evaluations.
"""
fig = plt.figure(figsize=(11, 4.5))
plt.suptitle(title + ' (' + pfunk.date() + ')')
xs, ys = results[xvar, yvar]
if len(xs) == 0:
plt.text(0.5, 0.5, 'No data')
return fig
# Left plot: All data, full view
plt.subplot(1, 2, 1)
plt.xlabel(xlabel)
plt.ylabel(ylabel)
for i, x in enumerate(xs):
plt.plot(x, ys[i])
# Right plot: Same data, but zoomed in
plt.subplot(1, 2, 2)
plt.xlabel(xlabel)
plt.ylabel(ylabel)
for i, x in enumerate(xs):
plt.plot(x, ys[i])
ymin = max(ymin, min([np.min(y) for y in ys]))
ymax = min(ymax, max([np.max(y) for y in ys]))
plt.ylim(ymin, ymax)
if ymax > 1000 or ymax < 1.5:
plt.subplots_adjust(0.1, 0.1, 0.99, 0.92, 0.2, 0)
else:
plt.subplots_adjust(0.07, 0.1, 0.99, 0.92, 0.15, 0)
return fig
def commit_results():
"""
Commits any new results.
"""
message = 'New results (' + socket.gethostname() + ' ' + pfunk.date() + ')'
log = logging.getLogger(__name__)
log.info('Loading results repo')
repo = git.Repo(pfunk.DIR_RES_REPO)
log.info('Checkout out master')
repo.git.checkout('master')
log.info('Checking for changes')
if not (repo.is_dirty() or repo.untracked_files):
log.info('No changes found')
return
log.info('Perfoming git pull')
log.info(repo.git.pull())
log.info('Perfoming git add')
log.info(repo.git.add(pfunk.DIR_RES_REPO))
log.info(repo.git.status())
log.info('Performing git commit')
log.info(repo.git.commit('-m', message))
log.info('Performing git push')
log.info(repo.git.push())
def list_tests(args):
"""
Shows all available tests and the date they were last run.
"""
dates = pfunk.find_test_dates()
w = max(4, max([len(k) for k in dates.keys()]))
print('| Name' + ' ' * (w - 4) + ' | Last run |')
print('-' * (w + 26))
for test in sorted(dates.items(), key=lambda x: x[1]):
name, date = test
print('| ' + name + ' ' * (w - len(name)) + ' | ' + pfunk.date(date) +
' |')
def run(self, path, run_number):
"""
Runs this test and logs the output.
"""
# Log status
log = logging.getLogger(__name__)
log.info(f'Running test: {self._name} run {run_number}')
# Seed numpy random generator, so that we know the value
max_uint32 = np.iinfo(np.uint32).max
seed = int(
np.mod(np.random.randint(max_uint32) + run_number, max_uint32))
np.random.seed(seed)
# Create test name
date = pfunk.date()
name = self.name()
# Store an identifier to the result writer's output, so we don't have
# to hold onto it while running the (potentially very long) test
results_id = None
# Create result writer
with self._writer_generator(name, date, path) as w:
w['status'] = 'uninitialised'
w['date'] = date
w['name'] = name
w['python'] = pfunk.PYTHON_VERSION
w['pints'] = pfunk.PINTS_VERSION
w['pints_commit'] = pfunk.PINTS_COMMIT
w['pints_authored_date'] = pfunk.PINTS_COMMIT_AUTHORED
w['pints_committed_date'] = pfunk.PINTS_COMMIT_COMMITTED
w['pints_commit_msg'] = pfunk.PINTS_COMMIT_MESSAGE
w['pfunk_commit'] = pfunk.PFUNK_COMMIT
w['pfunk_authored_date'] = pfunk.PFUNK_COMMIT_AUTHORED
w['pfunk_committed_date'] = pfunk.PFUNK_COMMIT_COMMITTED
w['pfunk_commit_msg'] = pfunk.PFUNK_COMMIT_MESSAGE
w['seed'] = seed
results_id = w.row_id()
# Run test
results = {}
try:
self._run(results)
except Exception:
log.error('Exception in test: ' + self.name())
results['status'] = 'failed'
raise
finally:
log.info('Writing result to ' + path)
with self._writer_generator(name, date, path, results_id) as w:
for k in results.keys():
w[k] = results[k]
def info():
"""
Returns a multi-line string with information about the currently selected
pints commit.
"""
c = git.Repo(pfunk.DIR_PINTS_REPO).head.commit
lines = []
lines.append('commit ' + str(c.hexsha))
lines.append('Author: ' + c.author.name)
lines.append('Date: ' + pfunk.date(time.gmtime(c.authored_date)))
lines.append('')
w = textwrap.TextWrapper()
w.initial_indent = w.subsequent_indent = ' '
lines.extend(w.wrap(c.message))
lines.append('')
return '\n'.join(lines)
def list_tests(args):
"""
Shows all available tests and the date they were last run.
"""
if args.next:
# Show next test only
print(pfunk.find_next_test(args.database))
return
# Show table of tests
dates = pfunk.find_test_dates(args.database)
w = max(4, max([len(k) for k in dates.keys()]))
print('| Name' + ' ' * (w - 4) + ' | Last run |')
print('-' * (w + 26))
for test in sorted(dates.items(), key=lambda x: x[1]):
name, date = test
print('| ' + name + ' ' * (w - len(name)) + ' | ' + pfunk.date(date) +
' |')
def run(self):
"""
Runs this test and logs the output.
"""
# Create logger for _global_ console/file output
log = logging.getLogger(__name__)
log.info('Running test: ' + self.name())
# Seed numpy random generator, so that we know the value
seed = np.random.randint(2**32) # Numpy says max seed is 2**32 - 1
np.random.seed(seed)
# Create test name
date = pfunk.date()
name = self.name()
# Get path to log and result files
base = name + '-' + date + '.txt'
log_path = pfunk.unique_path(os.path.join(pfunk.DIR_LOG, base))
res_path = pfunk.unique_path(os.path.join(pfunk.DIR_RESULT, base))
# Create result writer
w = pfunk.ResultWriter(res_path)
w['status'] = 'unitialised'
w['date'] = date
w['name'] = name
w['python'] = pfunk.PYTHON_VERSION
w['pints'] = pfunk.PINTS_VERSION
w['pints_commit'] = pfunk.PINTS_COMMIT
w['pfunk_commit'] = pfunk.PFUNK_COMMIT
w['commit'] = pfunk.PFUNK_COMMIT + '/' + pfunk.PINTS_COMMIT
w['seed'] = seed
# Run test
try:
self._run(w, log_path)
except Exception:
log.error('Exception in test: ' + self.name())
w['status'] = 'failed'
raise
finally:
log.info('Writing result to ' + w.filename())
w.write()
def variable(results, variable, title, ylabel, threshold=None):
"""
Creates and returns a default plot for a variable vs commits.
Parameters
----------
results : ResultsDatabaseResultsSet
Results for this plot
variable : str
The variable to plot
title : str
A title for this plot
ylabel : str
A y-axis label
threshold : float
An optional pass/fail threshold: if given, a horizontal line will be
drawn at this value.
"""
fig = plt.figure(figsize=(11, 4.5))
plt.suptitle(title + ' : ' + pfunk.date())
r = 0.3
# Left plot: Variable per commit, all data
plt.subplot(1, 2, 1)
plt.ylabel(ylabel)
plt.xlabel('Commit')
fig.autofmt_xdate()
x, y, u, m, s = pfunk.gather_statistics_per_commit(results, variable)
if len(x) == 0:
plt.text(0.5, 0.5, 'No data')
else:
xlookup = dict(zip(u, np.arange(len(u))))
x = np.array([xlookup[i] for i in x], dtype=float)
x += np.random.uniform(-r, r, x.shape)
plt.plot(u, m, 'ko-', alpha=0.5)
plt.plot(x, y, 'x', alpha=0.75)
if threshold:
plt.axhline(threshold)
try:
y = np.array(y)
ymax = np.max(y[np.isfinite(y)])
except ValueError:
ymax = 1
# Right plot: Same, but with outliers removed, to achieve a "zoom" on the
# most common, recent data.
plt.subplot(1, 2, 2)
plt.ylabel(ylabel)
plt.xlabel('Commit')
fig.autofmt_xdate()
x, y, u, m, s = pfunk.gather_statistics_per_commit(
results, variable, remove_outliers=True, n=10)
if len(x) == 0:
plt.text(0.5, 0.5, 'No data')
else:
xlookup = dict(zip(u, np.arange(len(u))))
x = np.array([xlookup[i] for i in x], dtype=float)
x += np.random.uniform(-r, r, x.shape)
plt.plot(u, m, 'ko-', alpha=0.5)
plt.plot(x, y, 'x', alpha=0.75)
if threshold:
# Show threshold line only if it doesn't change the zoom
if threshold <= np.max(y) and threshold >= np.min(y):
plt.axhline(threshold)
y = np.array(y)
try:
y = np.array(y)
ymax = max(ymax, np.max(y[np.isfinite(y)]))
except ValueError:
pass
if ymax > 1000:
plt.subplots_adjust(0.1, 0.16, 0.99, 0.92, 0.2, 0)
else:
plt.subplots_adjust(0.07, 0.16, 0.99, 0.92, 0.17, 0)
return fig
def plot(self, show=False):
"""
Generates the plots defined for this test
All plots returned by the test are written out to a filename defined
by the test name and current date. If ``show==True`` then the figures
are also shown on the current display.
"""
# Create logger for _global_ console/file output
log = logging.getLogger(__name__)
log.info('Running plot: ' + self.name())
# Load test results
results = pfunk.find_test_results(self._name)
# Plot
try:
figs = self._plot(results)
except Exception:
log.error('Exception in plot: ' + self.name())
raise
# Create path (assuming 1 figure; will fix with unique_path if more)
date = pfunk.date()
name = self.name()
path = name + '-' + date + '.png'
# Store names of generated files and glob mask of pathname to delete
# old figures later
generated = []
mask = name + '-' + '*.png'
try:
# Assume that the user returns an iterable object containing
# figures
for i, fig in enumerate(figs):
plot_path = pfunk.unique_path(
os.path.join(pfunk.DIR_PLOT, path))
log.info('Storing plot: ' + plot_path)
fig.savefig(plot_path)
generated.append(plot_path)
except TypeError:
# If not, then assume that the user returns a single figure
plot_path = pfunk.unique_path(os.path.join(pfunk.DIR_PLOT, path))
log.info('Storing plot: ' + plot_path)
figs.savefig(plot_path)
generated.append(plot_path)
# Close all figures
import matplotlib.pyplot as plt
if show:
plt.show()
plt.close('all')
# Delete old figures
for path in glob.glob(os.path.join(pfunk.DIR_PLOT, mask)):
path = os.path.realpath(path)
if not path.startswith(pfunk.DIR_PLOT):
continue
if path in generated:
continue
try:
os.remove(path)
log.info('Removed old plot: ' + path)
except IOError:
log.info('Removal of old plot failed: ' + path)