def plot_points(dfmed, dfmin, dfmax, subtitle, ideal_dist='numpy'):
"""Plot the timing results.
Plot an ideal scaling line from the origin through the first `ideal_idst`
point.
"""
styles = iter(STYLES)
for col in dfmed.columns:
if col == 'ideal ' + ideal_dist:
fmt='--'
else:
fmt = next(styles)
pyplot.errorbar(x=dfmed.index, y=dfmed[col],
yerr=[dfmin[col], dfmax[col]],
lolims=True, uplims=True,
fmt=fmt)
xmin = dfmed.index.min() - 1
xmax = dfmed.index.max() + 1
pyplot.xlim((xmin, xmax))
pyplot.suptitle("Julia Set Benchmark")
pyplot.title(subtitle, fontsize=12)
pyplot.xlabel('Engine Count')
pyplot.ylabel('mpoints / s')
pyplot.legend(dfmed.columns, loc='upper left')
pyplot.grid(axis='y')
pyplot.show()
def plot_points(dfmed, dfmin, dfmax, subtitle, ideal_dist='numpy'):
"""Plot the timing results.
Plot an ideal scaling line from the origin through the first `ideal_idst`
point.
"""
styles = iter(STYLES)
for col in dfmed.columns:
if col == 'ideal ' + ideal_dist:
fmt = '--'
else:
fmt = next(styles)
pyplot.errorbar(x=dfmed.index,
y=dfmed[col],
yerr=[dfmin[col], dfmax[col]],
lolims=True,
uplims=True,
fmt=fmt)
xmin = dfmed.index.min() - 1
xmax = dfmed.index.max() + 1
pyplot.xlim((xmin, xmax))
pyplot.suptitle("Julia Set Benchmark")
pyplot.title(subtitle, fontsize=12)
pyplot.xlabel('Engine Count')
pyplot.ylabel('mpoints / s')
pyplot.legend(dfmed.columns, loc='upper left')
pyplot.grid(axis='y')
pyplot.show()
def all_equal(iterable):
"""Return True if all elements in `iterable` are equal.
Also returns True if iterable is empty.
"""
iterator = iter(iterable)
try:
first = next(iterator)
except StopIteration:
return True # vacuously True
return all(element == first for element in iterator)
def all_equal(iterable):
"""Return True if all elements in `iterable` are equal.
Also returns True if iterable is empty.
"""
iterator = iter(iterable)
try:
first = next(iterator)
except StopIteration:
return True # vacuously True
return all(element == first for element in iterator)
def plot_ratios(dfmed, dfmin, dfmax, subtitle, ideal_dist='numpy'):
"""Plot the timing results.
Plot each line divided by ideal.
"""
ideal = 'ideal ' + ideal_dist
ratios = []
for df in (dfmed, dfmin, dfmax):
if df is None:
ratios.append(None)
continue
ratio = df.div(df[ideal], axis='index')
del ratio[ideal]
ratios.append(ratio)
dfmed, dfmin, dfmax = ratios
styles = iter(STYLES)
for col in dfmed.columns:
fmt = next(styles)
if dfmin and dfmax:
yerr = [dfmin[col], dfmax[col]]
else:
yerr = None
pyplot.errorbar(x=dfmed.index,
y=dfmed[col],
yerr=yerr,
lolims=True,
uplims=True,
fmt=fmt)
xmin = dfmed.index.min() - 1
xmax = dfmed.index.max() + 1
pyplot.xlim((xmin, xmax))
pyplot.suptitle("Julia Set Benchmark (ratios)")
pyplot.title(subtitle, fontsize=12)
pyplot.xlabel('Engine Count')
pyplot.ylabel('mpoints / ideal mpoints')
pyplot.legend(dfmed.columns, loc='lower left')
pyplot.grid(axis='y')
pyplot.show()
def plot_ratios(dfmed, dfmin, dfmax, subtitle, ideal_dist='numpy'):
"""Plot the timing results.
Plot each line divided by ideal.
"""
ideal = 'ideal ' + ideal_dist
ratios = []
for df in (dfmed, dfmin, dfmax):
if df is None:
ratios.append(None)
continue
ratio = df.div(df[ideal], axis='index')
del ratio[ideal]
ratios.append(ratio)
dfmed, dfmin, dfmax = ratios
styles = iter(STYLES)
for col in dfmed.columns:
fmt = next(styles)
if dfmin and dfmax:
yerr = [dfmin[col], dfmax[col]]
else:
yerr = None
pyplot.errorbar(x=dfmed.index, y=dfmed[col],
yerr=yerr,
lolims=True, uplims=True,
fmt=fmt)
xmin = dfmed.index.min() - 1
xmax = dfmed.index.max() + 1
pyplot.xlim((xmin, xmax))
pyplot.suptitle("Julia Set Benchmark (ratios)")
pyplot.title(subtitle, fontsize=12)
pyplot.xlabel('Engine Count')
pyplot.ylabel('mpoints / ideal mpoints')
pyplot.legend(dfmed.columns, loc='lower left')
pyplot.grid(axis='y')
pyplot.show()
def process_return_value(subcontext, result_key, targets):
"""Figure out what to return on the Client.
Parameters
----------
key : string
Key corresponding to wrapped function's return value.
Returns
-------
A DistArray (if locally all values are DistArray), a None (if
locally all values are None), or else, pull the result back to the
client and return it. If all but one of the pulled values is None,
return that non-None value only.
"""
type_key = subcontext._generate_key()
type_statement = "{} = str(type({}))".format(type_key, result_key)
subcontext._execute(type_statement, targets=targets)
result_type_str = subcontext._pull(type_key, targets=targets)
def is_NoneType(typestring):
return (typestring == "<type 'NoneType'>" or
typestring == "<class 'NoneType'>")
def is_LocalArray(typestring):
return typestring == "<class 'distarray.local.localarray.LocalArray'>"
if all(is_LocalArray(r) for r in result_type_str):
result = DistArray.from_localarrays(result_key, context=subcontext)
elif all(is_NoneType(r) for r in result_type_str):
result = None
else:
result = subcontext._pull(result_key, targets=targets)
if has_exactly_one(result):
result = next(x for x in result if x is not None)
return result
def make_grid_shape(shape, dist, comm_size):
""" Generate a `grid_shape` from `shape` tuple and `dist` tuple.
Does not assume that `dim_data` has `proc_grid_size` set for each
dimension.
Attempts to allocate processes optimally for distributed dimensions.
Parameters
----------
shape : tuple of int
The global shape of the array.
dist: tuple of str
dist_type character per dimension.
comm_size : int
Total number of processes to distribute.
Returns
-------
dist_grid_shape : tuple of int
Raises
------
GridShapeError
if not possible to distribute `comm_size` processes over number of
dimensions.
"""
check_grid_shape_preconditions(shape, dist, comm_size)
distdims = tuple(i for (i, v) in enumerate(dist) if v != 'n')
ndistdim = len(distdims)
if ndistdim == 0:
dist_grid_shape = ()
elif ndistdim == 1:
# Trivial case: all processes used for the one distributed dimension.
if comm_size >= shape[distdims[0]]:
dist_grid_shape = (shape[distdims[0]],)
else:
dist_grid_shape = (comm_size,)
elif comm_size == 1:
# Trivial case: only one process to distribute over!
dist_grid_shape = (1,) * ndistdim
else: # Main case: comm_size > 1, ndistdim > 1.
factors = utils.mult_partitions(comm_size, ndistdim)
if not factors: # Can't factorize appropriately.
raise GridShapeError("Cannot distribute array over processors.")
reduced_shape = [shape[i] for i in distdims]
# Reorder factors so they match the relative ordering in reduced_shape
factors = [utils.mirror_sort(f, reduced_shape) for f in factors]
# Pick the "best" factoring from `factors` according to which matches
# the ratios among the dimensions in `shape`.
rs_ratio = _compute_grid_ratios(reduced_shape)
f_ratios = [_compute_grid_ratios(f) for f in factors]
distances = [rs_ratio - f_ratio for f_ratio in f_ratios]
norms = numpy.array([numpy.linalg.norm(d, 2) for d in distances])
index = norms.argmin()
# we now have the grid shape for the distributed dimensions.
dist_grid_shape = tuple(int(i) for i in factors[index])
# Create the grid_shape, all 1's for now.
grid_shape = [1] * len(shape)
# Fill grid_shape in the distdim slots using dist_grid_shape
it = iter(dist_grid_shape)
for distdim in distdims:
grid_shape[distdim] = next(it)
out_grid_shape = tuple(grid_shape)
check_grid_shape_postconditions(out_grid_shape, shape, dist, comm_size)
return out_grid_shape
def grid_shape(self, grid_shape):
grid_size = iter(grid_shape)
for dist, dd in zip(self.dist, self.dim_data):
if dist != 'n':
dd['proc_grid_size'] = next(grid_size)
def make_grid_shape(shape, dist, comm_size):
""" Generate a `grid_shape` from `shape` tuple and `dist` tuple.
Does not assume that `dim_data` has `proc_grid_size` set for each
dimension.
Attempts to allocate processes optimally for distributed dimensions.
Parameters
----------
shape : tuple of int
The global shape of the array.
dist: tuple of str
dist_type character per dimension.
comm_size : int
Total number of processes to distribute.
Returns
-------
dist_grid_shape : tuple of int
Raises
------
GridShapeError
if not possible to distribute `comm_size` processes over number of
dimensions.
"""
check_grid_shape_preconditions(shape, dist, comm_size)
distdims = tuple(i for (i, v) in enumerate(dist) if v != 'n')
ndistdim = len(distdims)
if ndistdim == 0:
dist_grid_shape = ()
elif ndistdim == 1:
# Trivial case: all processes used for the one distributed dimension.
if comm_size >= shape[distdims[0]]:
dist_grid_shape = (shape[distdims[0]], )
else:
dist_grid_shape = (comm_size, )
elif comm_size == 1:
# Trivial case: only one process to distribute over!
dist_grid_shape = (1, ) * ndistdim
else: # Main case: comm_size > 1, ndistdim > 1.
factors = utils.mult_partitions(comm_size, ndistdim)
if not factors: # Can't factorize appropriately.
raise GridShapeError("Cannot distribute array over processors.")
reduced_shape = [shape[i] for i in distdims]
# Reorder factors so they match the relative ordering in reduced_shape
factors = [utils.mirror_sort(f, reduced_shape) for f in factors]
# Pick the "best" factoring from `factors` according to which matches
# the ratios among the dimensions in `shape`.
rs_ratio = _compute_grid_ratios(reduced_shape)
f_ratios = [_compute_grid_ratios(f) for f in factors]
distances = [rs_ratio - f_ratio for f_ratio in f_ratios]
norms = numpy.array([numpy.linalg.norm(d, 2) for d in distances])
index = norms.argmin()
# we now have the grid shape for the distributed dimensions.
dist_grid_shape = tuple(int(i) for i in factors[index])
# Create the grid_shape, all 1's for now.
grid_shape = [1] * len(shape)
# Fill grid_shape in the distdim slots using dist_grid_shape
it = iter(dist_grid_shape)
for distdim in distdims:
grid_shape[distdim] = next(it)
out_grid_shape = tuple(grid_shape)
check_grid_shape_postconditions(out_grid_shape, shape, dist, comm_size)
return out_grid_shape