def CreateSubstring(df, inCol, outCol, strLen, delim, startPos, endPos, makeList=False):
if endPos <= startPos:
df = df.withColumn(outCol, lit(''))
#here we create a substring of a string column
startPos = builtin.min(builtin.max(0, startPos), strLen)
endPos = builtin.min(builtin.max(startPos, endPos), strLen)
#if one end of string coincides with beginning
if startPos == 0:
df = df.withColumn(outCol, substring_index(inCol, delim, endPos))
#if one end of string coincides with end
elif endPos == strLen:
df = df.withColumn(outCol, substring_index(inCol, delim, startPos - endPos))
#if string is in middle
else:
#extract string from beginning upto position and then extract right end
df = df.withColumn(outCol, substring_index(inCol, delim, endPos)) \
.withColumn(outCol, substring_index(outCol, delim, startPos - endPos))
#if string should be broken into list
if makeList == True:
df = df.withColumn(outCol, split(outCol, delim))
return df
def show_count_stats(
counts,
group_size=10,
label_0="None",
out=None,
prefix=""):
assert counts.size() != 0
if (out is None): out = sys.stdout
from __builtin__ import int, max
counts_sorted = sorted(counts, reverse=True)
threshold = max(1, int(counts_sorted[0] / group_size) * group_size)
n = counts_sorted.size()
wt = max(len(label_0), len(str(threshold)))
wc = len(str(n))
fmt_val = prefix + ">= %%%dd: %%%dd %%7.5f" % (wt, wc)
fmt_zero = prefix + " %s: %%%dd %%7.5f" % (("%%%ds" % wt) % label_0, wc)
for i,count in enumerate(counts_sorted):
if (count >= threshold): continue
assert count >= 0
if (i > 0):
print(fmt_val % (threshold, i, i/n), file=out)
if (count == 0):
print(fmt_zero % (n-i, 1-i/n), file=out)
break
threshold = max(1, threshold-group_size)
else:
print(fmt_val % (threshold, n, 1), file=out)
def __init__(self, initChars, bodyChars=None, min=1, max=0, exact=0, excludeChars=None):
"""not implemented kwargs: asKeyword """
if max != 0 and min > max:
raise RuntimeError("min <= max needed")
if excludeChars:
initChars = initChars + "--" + excludeChars
if bodyChars:
bodyChars = bodyChars + "--" + excludeChars
if exact == 1 or max == 1:
pattern = r"[%s]{1}"%(initChars)
elif exact > 1:
if bodyChars:
pattern = r"[%s]{1}[%s]{%s}"%(initChars, bodyChars, exact-1)
else:
pattern = r"[%s]{%s}"%(initChars, exact)
elif max > 1:
if bodyChars:
pattern = r"[%s]{1}[%s]{%s,%s}"%(initChars, bodyChars, __builtin__.max(min-1,0), max-1)
else:
pattern = r"[%s]{%s,%s}"%(initChars, min, max)
else: # arbitrary upper bound
if bodyChars:
pattern = r"[%s]{1}[%s]{%s,}"%(initChars, bodyChars, __builtin__.max(min-1,0))
else:
pattern = r"[%s]{%s,}"%(initChars, min)
super(Word, self).__init__(pattern)
def show_count_stats(
counts,
group_size=10,
label_0="None",
out=None,
prefix=""):
assert counts.size() != 0
if (out is None): out = sys.stdout
from __builtin__ import int, max
counts_sorted = sorted(counts, reverse=True)
threshold = max(1, int(counts_sorted[0] / group_size) * group_size)
n = counts_sorted.size()
wt = max(len(label_0), len(str(threshold)))
wc = len(str(n))
fmt_val = prefix + ">= %%%dd: %%%dd %%7.5f" % (wt, wc)
fmt_zero = prefix + " %s: %%%dd %%7.5f" % (("%%%ds" % wt) % label_0, wc)
for i,count in enumerate(counts_sorted):
if (count >= threshold): continue
assert count >= 0
if (i > 0):
print >> out, fmt_val % (threshold, i, i/n)
if (count == 0):
print >> out, fmt_zero % (n-i, 1-i/n)
break
threshold = max(1, threshold-group_size)
else:
print >> out, fmt_val % (threshold, n, 1)
def __call__(self,*args):
while type(args[0]) is list: args = args[0]
assert len(args) == 1
x = args[0]
if isinstance(x, expr):
return expr(self,[x])
else:
return __builtin__.max(x,0)*__builtin__.max(x,0)
def numpy_max_pool_2d_stride_padding(x,
ds,
ignore_border=True,
st=None,
padding=(0, 0),
mode='max'):
pad_h = padding[0]
pad_w = padding[1]
h = x.shape[-2]
w = x.shape[-1]
assert ds[0] > pad_h
assert ds[1] > pad_w
def pad_img(x):
y = numpy.zeros((x.shape[0], x.shape[1], x.shape[2] + pad_h * 2,
x.shape[3] + pad_w * 2),
dtype=x.dtype)
y[:, :, pad_h:(x.shape[2] + pad_h), pad_w:(x.shape[3] + pad_w)] = x
return y
img_rows = h + 2 * pad_h
img_cols = w + 2 * pad_w
out_r = (img_rows - ds[0]) // st[0] + 1
out_c = (img_cols - ds[1]) // st[1] + 1
out_shp = list(x.shape[:-2])
out_shp.append(out_r)
out_shp.append(out_c)
ds0, ds1 = ds
st0, st1 = st
output_val = numpy.zeros(out_shp)
tt = []
y = pad_img(x)
func = numpy.max
if mode == 'sum':
func = numpy.sum
elif mode != 'max':
func = numpy.average
inc_pad = mode == 'average_inc_pad'
for k in numpy.ndindex(*x.shape[:-2]):
for i in range(output_val.shape[-2]):
ii_st = i * st[0]
ii_end = __builtin__.min(ii_st + ds[0], img_rows)
if not inc_pad:
ii_st = __builtin__.max(ii_st, pad_h)
ii_end = __builtin__.min(ii_end, h + pad_h)
for j in range(output_val.shape[-1]):
jj_st = j * st[1]
jj_end = __builtin__.min(jj_st + ds[1], img_cols)
if not inc_pad:
jj_st = __builtin__.max(jj_st, pad_w)
jj_end = __builtin__.min(jj_end, w + pad_w)
patch = y[k][ii_st:ii_end, jj_st:jj_end]
output_val[k][i, j] = func(patch)
return output_val
def __call__(self, *args):
while type(args[0]) is list:
args = args[0]
assert len(args) == 1
x = args[0]
if isinstance(x, expr):
return expr(self, [x])
else:
return __builtin__.max(x, 0) * __builtin__.max(x, 0)
def perform(self, node, inp, out):
x, = inp
z, = out
if len(x.shape) != 4:
raise NotImplementedError(
'DownsampleFactorMax requires 4D input for now')
z_shape = self.out_shape(x.shape, self.ds, self.ignore_border, self.st,
self.padding)
if (z[0] is None) or (z[0].shape != z_shape):
z[0] = numpy.empty(z_shape, dtype=x.dtype)
zz = z[0]
# number of pooling output rows
pr = zz.shape[-2]
# number of pooling output cols
pc = zz.shape[-1]
ds0, ds1 = self.ds
st0, st1 = self.st
pad_h = self.padding[0]
pad_w = self.padding[1]
img_rows = x.shape[-2] + 2 * pad_h
img_cols = x.shape[-1] + 2 * pad_w
inc_pad = self.mode == 'average_inc_pad'
# pad the image
if self.padding != (0, 0):
y = numpy.zeros(
(x.shape[0], x.shape[1], img_rows, img_cols),
dtype=x.dtype)
y[:, :, pad_h:(img_rows-pad_h), pad_w:(img_cols-pad_w)] = x
else:
y = x
func = numpy.max
if self.mode == 'sum':
func = numpy.sum
elif self.mode != 'max':
func = numpy.average
for n in xrange(x.shape[0]):
for k in xrange(x.shape[1]):
for r in xrange(pr):
row_st = r * st0
row_end = __builtin__.min(row_st + ds0, img_rows)
if not inc_pad:
row_st = __builtin__.max(row_st, self.padding[0])
row_end = __builtin__.min(row_end, x.shape[-2] + pad_h)
for c in xrange(pc):
col_st = c * st1
col_end = __builtin__.min(col_st + ds1, img_cols)
if not inc_pad:
col_st = __builtin__.max(col_st, self.padding[1])
col_end = __builtin__.min(col_end,
x.shape[-1] + pad_w)
zz[n, k, r, c] = func(y[
n, k, row_st:row_end, col_st:col_end])
def perform(self, node, inp, out):
x, = inp
z, = out
if len(x.shape) != 4:
raise NotImplementedError(
'DownsampleFactorMax requires 4D input for now')
z_shape = self.out_shape(x.shape, self.ds, self.ignore_border, self.st,
self.padding)
if (z[0] is None) or (z[0].shape != z_shape):
z[0] = numpy.empty(z_shape, dtype=x.dtype)
zz = z[0]
# number of pooling output rows
pr = zz.shape[-2]
# number of pooling output cols
pc = zz.shape[-1]
ds0, ds1 = self.ds
st0, st1 = self.st
pad_h = self.padding[0]
pad_w = self.padding[1]
img_rows = x.shape[-2] + 2 * pad_h
img_cols = x.shape[-1] + 2 * pad_w
inc_pad = self.mode == 'average_inc_pad'
# pad the image
if self.padding != (0, 0):
y = numpy.zeros((x.shape[0], x.shape[1], img_rows, img_cols),
dtype=x.dtype)
y[:, :, pad_h:(img_rows - pad_h), pad_w:(img_cols - pad_w)] = x
else:
y = x
func = numpy.max
if self.mode == 'sum':
func = numpy.sum
elif self.mode != 'max':
func = numpy.average
for n in xrange(x.shape[0]):
for k in xrange(x.shape[1]):
for r in xrange(pr):
row_st = r * st0
row_end = __builtin__.min(row_st + ds0, img_rows)
if not inc_pad:
row_st = __builtin__.max(row_st, self.padding[0])
row_end = __builtin__.min(row_end, x.shape[-2] + pad_h)
for c in xrange(pc):
col_st = c * st1
col_end = __builtin__.min(col_st + ds1, img_cols)
if not inc_pad:
col_st = __builtin__.max(col_st, self.padding[1])
col_end = __builtin__.min(col_end,
x.shape[-1] + pad_w)
zz[n, k, r, c] = func(y[n, k, row_st:row_end,
col_st:col_end])
def numpy_max_pool_2d_stride_padding(
x, ds, ignore_border=True, st=None, padding=(0, 0), mode='max'):
pad_h = padding[0]
pad_w = padding[1]
h = x.shape[-2]
w = x.shape[-1]
assert ds[0] > pad_h
assert ds[1] > pad_w
def pad_img(x):
y = numpy.zeros(
(x.shape[0], x.shape[1],
x.shape[2]+pad_h*2, x.shape[3]+pad_w*2),
dtype=x.dtype)
y[:, :, pad_h:(x.shape[2]+pad_h), pad_w:(x.shape[3]+pad_w)] = x
return y
img_rows = h + 2 * pad_h
img_cols = w + 2 * pad_w
out_r = (img_rows - ds[0]) // st[0] + 1
out_c = (img_cols - ds[1]) // st[1] + 1
out_shp = list(x.shape[:-2])
out_shp.append(out_r)
out_shp.append(out_c)
ds0, ds1 = ds
st0, st1 = st
output_val = numpy.zeros(out_shp)
tt = []
y = pad_img(x)
func = numpy.max
if mode == 'sum':
func = numpy.sum
elif mode != 'max':
func = numpy.average
inc_pad = mode == 'average_inc_pad'
for k in numpy.ndindex(*x.shape[:-2]):
for i in range(output_val.shape[-2]):
ii_st = i * st[0]
ii_end = __builtin__.min(ii_st + ds[0], img_rows)
if not inc_pad:
ii_st = __builtin__.max(ii_st, pad_h)
ii_end = __builtin__.min(ii_end, h + pad_h)
for j in range(output_val.shape[-1]):
jj_st = j * st[1]
jj_end = __builtin__.min(jj_st + ds[1], img_cols)
if not inc_pad:
jj_st = __builtin__.max(jj_st, pad_w)
jj_end = __builtin__.min(jj_end, w + pad_w)
patch = y[k][ii_st:ii_end, jj_st:jj_end]
output_val[k][i, j] = func(patch)
return output_val
def max(*args, **kwargs):
"""Symbolic maximum."""
if len(args) > 1:
return max(args, **kwargs)
# 1 argument: iterable
if isinstance(args[0], Range):
return args[0].max()
if isgenerator(args[0]):
# don't process geneartors
return __builtin__.max(*args, **kwargs)
if any(isinstance(arg, Expression) for arg in args[0]):
return Max(*args[0])
return __builtin__.max(*args, **kwargs)
def FindZeroAngle(self):
from __builtin__ import max, min, sorted
# input
x = self.Angle
y = self.DetCtr
# limits
y_max = max(y)
y_low = 0.01 * y_max # find suitable x range
x_min = max(x)
x_max = min(x)
for xi, yi in zip(x, y):
if yi > y_low:
if x_min > xi:
x_min = xi
if x_max < xi:
x_max = xi
# sampling
x_sam = self.linspace(x_min, x_max, num=500)
y_sam = self.sample(x, y, x_sam)
# normalized cross-correlation
y_cnv = self.normxcorr(y_sam, y_sam)
x_cnv = self.linspace(x_min, x_max, num=len(y_cnv))
# find suitable maximum of y_cnv
yLevel = 0.5 * y_max
maxima = self.localmaxima(x_cnv, y_cnv)
maxima = [m for m in maxima if m[1] > 0.0] # ignore negative matches
maxima = [m for m in maxima if self.sample(x, y, m[0]) > yLevel
] # only consider high y values
maxima = sorted(maxima, key=lambda m: m[1],
reverse=True) # best fit first
if not maxima:
self.PeakAng = x[y.index(y_max)]
self.PeakVal = y_max
else:
x_cnv_max, y_cnv_max, i_cnv_max = maxima[0]
self.PeakAng = self.maximumX(x_cnv, y_cnv, i_cnv_max)
self.PeakVal = y_max
print "Peak Angle:", self.PeakAng
print "I(rock):", self.PeakVal
return self.PeakAng
def pos(obj):
if isinstance(obj, (int, float)):
return __builtin__.max(obj, 0)
elif isinstance(obj, (matrix, spmatrix)):
(r, c) = size(obj)
z = zeros(r, c, full=True)
for i in xrange(r * c):
z[i] = __builtin__.max(obj[i], 0)
return z
elif isneg(obj):
return zeros(size(obj))
elif ispos(obj):
return obj
else:
return posfunction(obj)
def pos(obj):
if isinstance(obj, (int, float)):
return __builtin__.max(obj, 0)
elif isinstance(obj, (matrix, spmatrix)):
(r, c) = size(obj)
z = zeros(r, c, full=True)
for i in xrange(r*c):
z[i] = __builtin__.max(obj[i], 0)
return z
elif isneg(obj):
return zeros(size(obj))
elif ispos(obj):
return obj
else:
return posfunction(obj)
def FindZeroAngle(self):
from __builtin__ import max, min, sorted
# input
x = self.Angle
y = self.DetCtr
# limits
y_max = max(y)
y_low = 0.01 * y_max # find suitable x range
x_min = max(x)
x_max = min(x)
for xi, yi in zip(x, y):
if yi > y_low:
if x_min > xi:
x_min = xi
if x_max < xi:
x_max = xi
# sampling
x_sam = self.linspace(x_min, x_max, num=500)
y_sam = self.sample(x, y, x_sam)
# normalized cross-correlation
y_cnv = self.normxcorr(y_sam, y_sam)
x_cnv = self.linspace(x_min, x_max, num=len(y_cnv))
# find suitable maximum of y_cnv
yLevel = 0.5 * y_max
maxima = self.localmaxima(x_cnv, y_cnv)
maxima = [m for m in maxima if m[1] > 0.0] # ignore negative matches
maxima = [m for m in maxima if self.sample(x, y, m[0]) > yLevel] # only consider high y values
maxima = sorted(maxima, key=lambda m: m[1], reverse=True) # best fit first
if not maxima:
self.PeakAng = x[y.index(y_max)]
self.PeakVal = y_max
else:
x_cnv_max, y_cnv_max, i_cnv_max = maxima[0]
self.PeakAng = self.maximumX(x_cnv, y_cnv, i_cnv_max)
self.PeakVal = y_max
print "Peak Angle:", self.PeakAng
print "I(rock):", self.PeakVal
return self.PeakAng
def _nanmax(values, axis=None, skipna=True):
mask = isnull(values)
if skipna and not issubclass(values.dtype.type,
(np.integer, np.datetime64)):
values = values.copy()
np.putmask(values, mask, -np.inf)
# numpy 1.6.1 workaround in Python 3.x
if (values.dtype == np.object_
and sys.version_info[0] >= 3): # pragma: no cover
import __builtin__
if values.ndim > 1:
apply_ax = axis if axis is not None else 0
result = np.apply_along_axis(__builtin__.max, apply_ax, values)
else:
result = __builtin__.max(values)
else:
if ((axis is not None and values.shape[axis] == 0)
or values.size == 0):
result = values.sum(axis)
result.fill(np.nan)
else:
result = values.max(axis)
return _maybe_null_out(result, axis, mask)
def _get_range(sfunc, min, max):
" Truncate PDFs with long tails"
num_tails = int(sfunc.ppf(0) == np.NINF) + int(sfunc.ppf(1) == np.PINF)
_range = options['pdf']['range']
if num_tails:
if num_tails == 2:
range = [(1.0 - _range)/2, (1.0 + _range)/2]
else:
range = [1.0 - _range, _range]
mmin = sfunc.ppf(0)
if mmin == np.NINF:
mmin = sfunc.ppf(range[0])
mmax = sfunc.ppf(1)
if mmax == np.PINF:
mmax = sfunc.ppf(range[1])
if min is not None:
min = __builtin__.max(min, mmin)
else:
min = mmin
if max is not None:
max = __builtin__.min(max, mmax)
else:
max = mmax
return min, max
def HPDF(data, min=None, max=None):
"""
Histogram PDF - initialized with points from a histogram.
This function creates a PDF from a histogram. This is useful when some other software has
generated a PDF from your data.
:param data: A two dimensional array. The first column is the histogram interval mean,
and the second column is the probability. The probability values do not need to be
normalized.
:param min: A minimum value for the PDF range. If your histogram has values very close
to 0, and you know values of 0 are impossible, then you should set the ***min*** parameter.
:param max: A maximum value for the PDF range.
:type data: 2D numpy array
:returns: A PDF object.
"""
x = data[:, 0]
y = data[:, 1]
sp = interpolate.splrep(x, y)
dx = (x[1] - x[0]) / 2.0
mmin = x[0] - dx
mmax = x[-1] + dx
if min is not None:
mmin = __builtin__.max(min, mmin)
if max is not None:
mmax = __builtin__.min(max, mmax)
x = np.linspace(mmin, mmax, options['pdf']['numpart'])
y = interpolate.splev(x, sp)
y[y < 0] = 0 # if the extrapolation goes negative...
return PDF(x, y)
def _nanmax(values, axis=None, skipna=True):
mask = isnull(values)
dtype = values.dtype
if skipna and _na_ok_dtype(dtype):
values = values.copy()
np.putmask(values, mask, -np.inf)
values = _view_if_needed(values)
# numpy 1.6.1 workaround in Python 3.x
if (values.dtype == np.object_
and sys.version_info[0] >= 3): # pragma: no cover
import __builtin__
if values.ndim > 1:
apply_ax = axis if axis is not None else 0
result = np.apply_along_axis(__builtin__.max, apply_ax, values)
else:
result = __builtin__.max(values)
else:
if ((axis is not None and values.shape[axis] == 0)
or values.size == 0):
result = com.ensure_float(values.sum(axis))
result.fill(np.nan)
else:
result = values.max(axis)
result = _wrap_results(result,dtype)
return _maybe_null_out(result, axis, mask)
def SetupBasisPairs(self):
distr = self.Representation.GetDistributedModel()
rank = self.Representation.GetBaseRank()
localRange = distr.GetLocalIndexRange(self.GetGlobalBasisPairCount(),
rank)
count = self.GetGlobalBasisPairCount()
N = self.BSplineObject.NumberOfBSplines
k = self.BSplineObject.MaxSplineOrder
pairs = zeros((self.GetGlobalBasisPairCount(), 2), dtype=int32)
pairs[:, 0] = 0
pairs[:, 1] = N - 1
index = 0
for i in xrange(N):
for j in xrange(max(0, i - k + 1), min(i + k, N)):
pairs[index, 0] = i
pairs[index, 1] = j
index += 1
if index != pairs.shape[0]:
raise Execption()
#store pairs
self.GlobalIndexPairs = pairs
self.LocalBasisPairIndices = r_[localRange]
self.LocalIndexPairs = pairs[localRange]
return pairs
def test_subgrad(self):
x = var('x')
ex = square_pos(x)
x_vals = [-3.14, 0.0, 123.456]
for val in x_vals:
g = ex.subgrad({'x': val})
self.assertAlmostEqual(g['x'], 2 * __builtin__.max(val, 0))
def max(*args, **kargs):
if len(args) == 0:
raise TypeError("max expected at least 1 arguments, got 0")
key = kargs.pop("key", None)
#default implementation
if key is None:
return __builtin__.max(*args, **kargs)
for karg in kargs:
raise TypeError(
"unexpected keyword argument %s for function max") % karg
if len(args) == 1:
items = iter(args[0])
else:
items = iter(args)
try:
best_item = items.next()
best_value = key(best_item)
except StopIteration:
raise ValueError("max() arg is an empty sequence")
for item in items:
value = key(item)
if value > best_value:
best_item = item
best_value = value
return best_item
def _nanmax(values, axis=None, skipna=True):
mask = isnull(values)
if skipna and not issubclass(values.dtype.type,
(np.integer, np.datetime64)):
values = values.copy()
np.putmask(values, mask, -np.inf)
# numpy 1.6.1 workaround in Python 3.x
if (values.dtype == np.object_
and sys.version_info[0] >= 3): # pragma: no cover
import __builtin__
if values.ndim > 1:
apply_ax = axis if axis is not None else 0
result = np.apply_along_axis(__builtin__.max, apply_ax, values)
else:
result = __builtin__.max(values)
else:
if ((axis is not None and values.shape[axis] == 0)
or values.size == 0):
result = values.sum(axis)
result.fill(np.nan)
else:
result = values.max(axis)
return _maybe_null_out(result, axis, mask)
def test_subgrad(self):
x = var('x')
ex = square_pos(x)
x_vals = [ -3.14, 0.0, 123.456]
for val in x_vals:
g = ex.subgrad({'x':val})
self.assertAlmostEqual(g['x'],2*__builtin__.max(val,0))
def set_range(self, min=None, max=None, relative=False):
'''
:param min: The lower limit of the range
:param max: The upper limit of the range
:param relative: If True then min and max are provided as 0 to 1 values
otherwise are absolute values.
'''
if min is None and max is None:
raise ValueError('You must set at least the min or the max')
if min is not None:
if not relative:
self._range['min'] = min
else:
self._range['min'] = self._to_absolute(__builtin__.max(min, 0))
if max is not None:
if not relative:
self._range['max'] = max
else:
self._range['max'] = self._to_absolute(__builtin__.min(max, 1))
self._cache_clear()
old_index = self._sieve.index
self._sieve.query = self._generate_query()
included = self._sieve.index - old_index
excluded = old_index - self._sieve.index
return dict(included=list(included), excluded=list(excluded))
def perform(self, node, inp, out):
"""
"""
x, = inp
z, = out
if len(x.shape) != 4:
raise NotImplementedError("DownsampleFactorMax requires 4D input for now")
z_shape = self.out_shape(x.shape, self.ds, self.ignore_border, self.st)
if (z[0] is None) or (z[0].shape != z_shape):
z[0] = numpy.zeros(self.out_shape(x.shape, self.ds, self.ignore_border, self.st))
z[0] = theano._asarray(z[0], dtype=x.dtype)
zz = z[0]
## zz needs to be initialized with -inf for the following to work
zz -= numpy.inf
# number of pooling output rows
pr = zz.shape[-2]
# number of pooling output cols
pc = zz.shape[-1]
ds0, ds1 = self.ds
st0, st1 = self.st
img_rows = x.shape[-2]
img_cols = x.shape[-1]
for n in xrange(x.shape[0]):
for k in xrange(x.shape[1]):
for r in xrange(pr):
row_st = r * st0
row_end = __builtin__.min(row_st + ds0, img_rows)
for c in xrange(pc):
col_st = c * st1
col_end = __builtin__.min(col_st + ds1, img_cols)
for row_ind in xrange(row_st, row_end):
for col_ind in xrange(col_st, col_end):
zz[n, k, r, c] = __builtin__.max(zz[n, k, r, c], x[n, k, row_ind, col_ind])
def SetupBasisPairs(self): distr = self.Representation.GetDistributedModel() rank = self.Representation.GetBaseRank() localRange = distr.GetLocalIndexRange(self.GetGlobalBasisPairCount(), rank) count = self.GetGlobalBasisPairCount() N = self.BSplineObject.NumberOfBSplines k = self.BSplineObject.MaxSplineOrder pairs = zeros((self.GetGlobalBasisPairCount(), 2), dtype=int32) pairs[:,0] = 0 pairs[:,1] = N-1 index = 0 for i in xrange(N): for j in xrange(max(0, i-k+1), min(i+k, N)): pairs[index, 0] = i pairs[index, 1] = j index+=1 if index != pairs.shape[0]: raise Execption() #store pairs self.GlobalIndexPairs = pairs self.LocalBasisPairIndices = r_[localRange] self.LocalIndexPairs = pairs[localRange] return pairs
def HPDF(data, min=None, max=None):
"""
Histogram PDF - initialized with points from a histogram.
This function creates a PDF from a histogram. This is useful when some other software has
generated a PDF from your data.
:param data: A two dimensional array. The first column is the histogram interval mean,
and the second column is the probability. The probability values do not need to be
normalized.
:param min: A minimum value for the PDF range. If your histogram has values very close
to 0, and you know values of 0 are impossible, then you should set the ***min*** parameter.
:param max: A maximum value for the PDF range.
:type data: 2D numpy array
:returns: A PDF object.
"""
x = data[:, 0]
y = data[:, 1]
sp = interpolate.splrep(x, y)
dx = (x[1] - x[0]) / 2.0
mmin = x[0] - dx
mmax = x[-1] + dx
if min is not None:
mmin = __builtin__.max(min, mmin)
if max is not None:
mmax = __builtin__.min(max, mmax)
x = np.linspace(mmin, mmax, options['pdf']['numpart'])
y = interpolate.splev(x, sp)
y[y < 0] = 0 # if the extrapolation goes negative...
return PDF(x, y)
def implementation(groupByRecord, resultColumnName, state, record=None):
if (state == AGGREGATION_STATE__PROCESS_RECORD):
v = record[columnName] if (columnName in record) else None
if (v is not None):
groupByRecord[resultColumnName] = __builtin__.max(
v, groupByRecord[resultColumnName]) if (
resultColumnName in groupByRecord) else v
def __show_sizes(f):
typename_n_size = f()
from __builtin__ import max
l = max([ len(typename) for typename, size in typename_n_size ])
fmt = "%%%is : %%i" % l
for typename, size in typename_n_size:
print(fmt % (typename, size))
def perform(self, node, inp, out):
"""
"""
x, = inp
z, = out
if len(x.shape) != 4:
raise NotImplementedError("DownsampleFactorMax requires 4D input for now")
z_shape = self.out_shape(x.shape, self.ds, self.ignore_border)
if (z[0] is None) or (z[0].shape != z_shape):
z[0] = numpy.zeros(self.out_shape(x.shape, self.ds, self.ignore_border))
z[0] = theano._asarray(z[0], dtype=x.dtype)
zz = z[0]
## zz needs to be initialized with -inf for the following to work
zz -= numpy.inf
ds0, ds1 = self.ds
if self.ignore_border:
x_usable2 = x.shape[2] / ds0 * ds0
else:
x_usable2 = x.shape[2]
if self.ignore_border:
x_usable3 = x.shape[3] / ds1 * ds1
else:
x_usable3 = x.shape[3]
for n in xrange(x.shape[0]):
for k in xrange(x.shape[1]):
for i in xrange(x_usable2):
zi = i / ds0
for j in xrange(x_usable3):
zj = j / ds1
zz[n, k, zi, zj] = __builtin__.max(zz[n, k, zi, zj], x[n, k, i, j])
def perform(self, node, inp, out):
"""
"""
x, = inp
z, = out
if len(x.shape) != 4:
raise NotImplementedError(
'DownsampleFactorMax requires 4D input for now')
z_shape = self.out_shape(x.shape, self.ds, self.ignore_border)
if (z[0] is None) or (z[0].shape != z_shape):
z[0] = numpy.zeros(
self.out_shape(x.shape, self.ds, self.ignore_border))
z[0] = theano._asarray(z[0], dtype=x.dtype)
zz = z[0]
## zz needs to be initialized with -inf for the following to work
zz -= numpy.inf
ds0, ds1 = self.ds
if self.ignore_border:
x_usable2 = (x.shape[2] // ds0 * ds0)
else:
x_usable2 = x.shape[2]
if self.ignore_border:
x_usable3 = (x.shape[3] // ds1 * ds1)
else:
x_usable3 = x.shape[3]
for n in xrange(x.shape[0]):
for k in xrange(x.shape[1]):
for i in xrange(x_usable2):
zi = i / ds0
for j in xrange(x_usable3):
zj = j / ds1
zz[n, k, zi,
zj] = __builtin__.max(zz[n, k, zi, zj], x[n, k, i,
j])
def _scan_mapper(array, ex, scan_fn=None, axis=None, scan_base=None):
if scan_fn is None:
yield (ex, array.fetch(ex))
else:
local_data = array.fetch(ex)
if sp.issparse(local_data):
local_data = local_data.todense()
if axis is None:
exts = sorted(array.tiles.keys(), key=lambda x: x.ul)
id = exts.index(ex)
if id > 0:
local_data[tuple(np.zeros(len(ex.shape), dtype=int))] += scan_base[id-1]
else:
max_axis_shape = __builtin__.max([ext.shape[axis] for ext in array.tiles.keys()])
id = ex.ul[axis] / max_axis_shape
if id > 0:
base_slice = list(ex.to_slice())
base_slice[axis] = slice(id-1, id, None)
new_slice = [slice(0, ex.shape[i], None) for i in range(len(ex.shape))]
new_slice[axis] = slice(0,1,None)
local_data[new_slice] += scan_base[base_slice]
#util.log_info('local_data type:%s data:%s', type(local_data), local_data)
yield (ex, np.asarray(scan_fn(local_data, axis=axis)).reshape(ex.shape))
def bincount(v, weights=None, minlength=None):
'''
Count unique values in ``v``.
See `numpy.bincount` for more information.
Arguments:
v (Expr): Array of non-negative integers
Returns:
Expr: Integer array of counts.
'''
minval = min(v).glom()
maxval = max(v).glom()
assert minval > 0
if minlength is not None:
minlength = __builtin__.max(maxval + 1, minlength)
else:
minlength = maxval + 1
if weights is not None:
return map2((v, weights),
fn=_bincount_mapper,
fn_kw={'minlength': minlength},
shape=(minlength, ),
reducer=np.add)
else:
return map2(v,
fn=_bincount_mapper,
fn_kw={'minlength': minlength},
shape=(minlength, ),
reducer=np.add)
def max( *args, **kargs):
if len(args) == 0:
raise TypeError("max expected at least 1 arguments, got 0")
key= kargs.pop("key", None)
#default implementation
if key is None:
return __builtin__.max(*args,**kargs)
for karg in kargs:
raise TypeError("unexpected keyword argument %s for function max") % karg
if len(args) == 1:
items = iter(args[0])
else:
items = iter(args)
try:
best_item = items.next()
best_value = key(best_item)
except StopIteration:
raise ValueError("max() arg is an empty sequence")
for item in items:
value = key(item)
if value > best_value:
best_item = item
best_value = value
return best_item
def max(cp, size):
_check_params(len(cp), size)
if len(cp) == 0:
return 0
return __builtin__.max(abs(sample) for sample in _get_samples(cp, size))
def _get_range(sfunc, min, max):
" Truncate PDFs with long tails"
num_tails = int(sfunc.ppf(0) == np.NINF) + int(sfunc.ppf(1) == np.PINF)
_range = options['pdf']['range']
if num_tails:
if num_tails == 2:
range = [(1.0 - _range)/2, (1.0 + _range)/2]
else:
range = [1.0 - _range, _range]
mmin = sfunc.ppf(0)
if mmin == np.NINF:
mmin = sfunc.ppf(range[0])
mmax = sfunc.ppf(1)
if mmax == np.PINF:
mmax = sfunc.ppf(range[1])
if min is not None:
min = __builtin__.max(min, mmin)
else:
min = mmin
if max is not None:
max = __builtin__.min(max, mmax)
else:
max = mmax
return min, max
def max(cp, size):
_check_params(len(cp), size)
if len(cp) == 0:
return 0
return builtins.max(abs(sample) for sample in _get_samples(cp, size))
def restricted(self, min=None, max=None):
"""
Returns the power series restricted to the coefficients starting at
min and going up to, but not including max. If min is not
specified, then it is assumed to be zero. If max is not specified,
then it is assumed to be infinity.
EXAMPLES::
sage: L = LazyPowerSeriesRing(QQ)
sage: a = L([1])
sage: a.restricted().coefficients(10)
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
sage: a.restricted(min=2).coefficients(10)
[0, 0, 1, 1, 1, 1, 1, 1, 1, 1]
sage: a.restricted(max=5).coefficients(10)
[1, 1, 1, 1, 1, 0, 0, 0, 0, 0]
sage: a.restricted(min=2, max=6).coefficients(10)
[0, 0, 1, 1, 1, 1, 0, 0, 0, 0]
"""
import __builtin__
if ((min is None and max is None) or
(max is None and self.get_aorder() >= min)):
return self
return self._new(partial(self._restricted_gen, min, max),
lambda ao: __builtin__.max(ao, min), self)
def __show_sizes(f):
typename_n_size = f()
from __builtin__ import max
l = max([ len(typename) for typename, size in typename_n_size ])
fmt = "%%%is : %%i" % l
for typename, size in typename_n_size:
print fmt % (typename, size)
def max(l):
"Return the maximum of the elements in l, or nan if any element is nan."
import __builtin__
try:
return __builtin__.max(ensure_nonan(x) for x in l)
except NanException:
return nan
def manhattan_distance(x, xp=None, **kwds):
"""1-D manhattan distance between points
d[ij] = | x[i] - x'[j] |
Notes:
manhattan matrix where x'==x will be symmetric with zeros on the diagonal
use dmin=2 for forced upconversion of 1-D arrays
"""
# dmin: force upconvert to x,x' to dimension >= dmin
dmin = kwds['dmin'] if 'dmin' in kwds else 0 # default dimension
from numpy import abs, asarray, newaxis as nwxs, zeros_like
from __builtin__ import max
# cast as arrays of the same dimension
x = asarray(x)
xp = zeros_like(x) if xp is None else asarray(xp)
xsize = max(len(x.shape), len(xp.shape), dmin)
while len(x.shape) < xsize: x = x[nwxs]
while len(xp.shape) < xsize: xp = xp[nwxs]
# prep for build manhattan matrix in single operation
if xsize >= 2:
x = x.T[:,:,nwxs]; xp = xp.T[:,nwxs]
elif xsize == 1:
x = x[:,nwxs]# x.T[:,nwxs]; xp = xp.T
else:#xsize == 0:
x = x[nwxs]
return abs(x - xp)#.T
def compare_derivatives(more_reliable, less_reliable, eps=1e-6):
from __builtin__ import max
scale = max(1, ext.max(ext.abs(more_reliable)))
if (not (more_reliable/scale).all_approx_equal( # fast
other=less_reliable/scale, tolerance=eps)):
from libtbx.test_utils import approx_equal
assert approx_equal( # slow but helpful output
more_reliable/scale, less_reliable/scale, eps=eps)
def compare_derivatives(more_reliable, less_reliable, eps=1e-6):
from __builtin__ import max
scale = max(1, ext.max(ext.abs(more_reliable)))
if (not (more_reliable/scale).all_approx_equal( # fast
other=less_reliable/scale, tolerance=eps)):
from libtbx.test_utils import approx_equal
assert approx_equal( # slow but helpful output
more_reliable/scale, less_reliable/scale, eps=eps)
def find_max_ratio(self, resources, max_resources):
'''
helper method for finding the max ratio for a resoruces
'''
cpu_ratio = (float(resources.clock_ticks)
/ float(max_resources.clock_ticks))
dtcm_ratio = (float(resources.dtcm) / float(max_resources.dtcm))
sdram_ratio = (float(resources.sdram) / float(max_resources.sdram))
return max((cpu_ratio, dtcm_ratio, sdram_ratio))
def minmax(cp, size):
_check_params(len(cp), size)
min_sample, max_sample = 0x7fffffff, -0x80000000
for sample in _get_samples(cp, size):
max_sample = builtins.max(sample, max_sample)
min_sample = builtins.min(sample, min_sample)
return min_sample, max_sample
def minmax(cp, size):
_check_params(len(cp), size)
max_sample, min_sample = 0, 0
for sample in _get_samples(cp, size):
max_sample = __builtin__.max(sample, max_sample)
min_sample = __builtin__.min(sample, min_sample)
return min_sample, max_sample
def handle_input(self):
while True:
key = game_input.readkey()
if key == '+' or key == libtcod.KEY_DOWN:
self.offset = min(self.offset + 1, len(self.content) - 1)
elif key == '-' or key == libtcod.KEY_UP:
self.offset = max(self.offset - 1, 0)
elif key == libtcod.KEY_PAGEUP:
self.offset = max(self.offset - self.height, 0)
elif key == libtcod.KEY_PAGEDOWN or key == ' ':
self.offset = min(self.offset + self.height, len(self.content) - 1)
elif key == libtcod.KEY_ESCAPE:
break
elif key == libtcod.KEY_ENTER:
return libtcod.KEY_ENTER
else:
return key
self.render()
def minmax(cp, size):
_check_params(len(cp), size)
max_sample, min_sample = 0, 0
for sample in _get_samples(cp, size):
max_sample = __builtin__.max(sample, max_sample)
min_sample = __builtin__.min(sample, min_sample)
return min_sample, max_sample
def test_simplify(self):
"""Test for simplify()."""
A, B, C, n1, n2, n3 = self.A, self.B, self.C, self.n1, self.n2, self.n3
self.assertEqual(max(max(A, B), C)(), max(A, max(B, C))())
self.assertEqual(Max(A)(), A)
self.assertEqual(Max()(), float("inf"))
self.assertEqual(Max(n1, n2)(), __builtin__.max(n1, n2))
def minmax(cp, size):
_check_params(len(cp), size)
min_sample, max_sample = 0x7FFFFFFF, -0x80000000
for sample in _get_samples(cp, size):
max_sample = builtins.max(sample, max_sample)
min_sample = builtins.min(sample, min_sample)
return min_sample, max_sample
def Lnorm(weights, p=1):
"calculate L-p norm of weights"
# weights is a numpy array
# p is an int
from numpy import asarray, seterr, inf
weights = asarray(weights).flatten()
if not p:
w = float(len(weights[weights != 0.0])) # total number of nonzero elements
elif p == inf:
w = float(max(abs(weights)))
else:
orig = seterr(over='raise', invalid='raise')
try:
w = float(sum(abs(weights**p)))**(1./p)
except FloatingPointError: # use the infinity norm
w = float(max(abs(weights)))
seterr(**orig)
return w
def max(*args, **kwargs):
"""Symbolic maximum."""
if len(args) > 1:
return max(args, **kwargs)
if isinstance(args[0], Range):
return args[0].max()
if any(isinstance(arg, Expression) for arg in args[0]):
return Max(*args[0])
return __builtin__.max(*args, **kwargs)
def test_simplify(self):
"""Test for simplify()."""
A, B, C, n1, n2, n3 = self.A, self.B, self.C, self.n1, self.n2, self.n3
self.assertEqual(max(max(A, B), C)(), max(A, max(B, C))())
self.assertEqual(Max(A)(), A)
self.assertEqual(Max()(), None)
self.assertEqual(Max(n1, n2)(), __builtin__.max(n1, n2))