def lstm(concat, c, forget_bias=None):
batches = concat.shape[0]
vec_len = concat.shape[1]/4
assert c.shape[0] == concat.shape[0]
assert c.shape[1] == vec_len
assert c.dtype == concat.dtype
pos = ovl.position_in([batches, vec_len])
cur_batch = pos[0]
cur_elem = pos[1]
i = concat[cur_batch, cur_elem]
j = concat[cur_batch, cur_elem + vec_len]
f = concat[cur_batch, cur_elem + 2*vec_len]
o = concat[cur_batch, cur_elem + 3*vec_len]
c_cur = c[cur_batch, cur_elem]
new_c = ovl.output_like(c)
new_h = ovl.output_like(c)
new_c_cur = c_cur*sig(f + forget_bias) + sig(i) * ovl.tanh(j)
new_c[pos] = new_c_cur
new_h[pos] = ovl.tanh(new_c_cur) * sig(o)
return new_c, new_h
def op(self, input0, input1, input2):
pos = ops.position_in(input0.shape)
out0 = ops.output_like(input0)
a = input0[pos]
b = input1[pos]
c = input2[pos]
d = a*a + b*b + c*c
out0[pos] = d
return out0
def accumulate(x, inner_fcn=None, axis=None):
"""
Define the operator function.
:param x: The input tensor
:param inner_fcn: a lambda function to be applied for accumulation
:param axis: The axis across which accumulation will be applied
:return: The accumulated result
"""
# assert that the axis parameter makes sense
assert isinstance(axis, int)
assert axis >= 0
assert axis < x.rank
# Define the workgroup shape. Here we use a single worker to perform the accumulation across the
# accumulation axis. The workgroup shape is then the size of all other axes with accumulation axis removed.
if x.rank is 1:
workgroup_shape = [1]
else:
workgroup_shape = []
for cur_dim, num_elements in enumerate(x.shape):
if cur_dim == axis:
pass
else:
workgroup_shape.append(num_elements)
pos = ovl.position_in(workgroup_shape)
# Define the accumulated output to be the same type as the input
out = ovl.output_like(x)
# Define a function for determining the index of the input tensor as a function of accumulation axis position
# and the current worker position. This is equal to the worker position with the accumulation axis position
# inserted where it should be in the indexing order.
def resolve_position(axis_n):
cur_pos = []
offset = 0
for cur_dim in range(x.rank):
if cur_dim == axis:
cur_pos.append(axis_n)
offset = 1
else:
cur_pos.append(pos[cur_dim - offset])
return cur_pos
# initialize accumulator to be the first element along the accumulation axis
initial_value = x[resolve_position(0)]
accum = ovl.variable(initial_value, x.dtype)
out[resolve_position(0)] = accum
# use this worker to iterate over and accumulate the rest of the elements in the accumulation axis
for i in ovl.arange(1, x.shape[axis]):
accum <<= inner_fcn(accum, x[resolve_position(i)])
out[resolve_position(i)] = accum
return out
def accumulate(x, inner_fcn=None, axis=None):
"""
Define the operator function.
:param x: The input tensor
:param inner_fcn: a lambda function to be applied for accumulation
:param axis: The axis across which accumulation will be applied
:return: The accumulated result
"""
# assert that the axis parameter makes sense
assert isinstance(axis, int)
assert axis >= 0
assert axis < x.rank
# Define the workgroup shape. Here we use a single worker to perform the accumulation across the
# accumulation axis. The workgroup shape is then the size of all other axes with accumulation axis removed.
if x.rank is 1:
workgroup_shape = [1]
else:
workgroup_shape = []
for cur_dim, num_elements in enumerate(x.shape):
if cur_dim == axis:
pass
else:
workgroup_shape.append(num_elements)
pos = ovl.position_in(workgroup_shape)
# Define the accumulated output to be the same type as the input
out = ovl.output_like(x)
# Define a function for determining the index of the input tensor as a function of accumulation axis position
# and the current worker position. This is equal to the worker position with the accumulation axis position
# inserted where it should be in the indexing order.
def resolve_position(axis_n):
cur_pos = []
offset = 0
for cur_dim in range(x.rank):
if cur_dim == axis:
cur_pos.append(axis_n)
offset = 1
else:
cur_pos.append(pos[cur_dim-offset])
return cur_pos
# initialize accumulator to be the first element along the accumulation axis
initial_value = x[resolve_position(0)]
accum = ovl.variable(initial_value, x.dtype)
out[resolve_position(0)] = accum
# use this worker to iterate over and accumulate the rest of the elements in the accumulation axis
for i in ovl.arange(1, x.shape[axis]):
accum <<= inner_fcn(accum, x[resolve_position(i)])
out[resolve_position(i)] = accum
return out
def op(self, input0, input1, input2):
sum = ops.output_like(input0)
pos = ops.position_in(input0.shape)
a = input0[pos]
b = input1[pos]
c = input2[pos]
sum[pos] = ops.sqrt(a*a + b*b + c*c)
return sum
def lstm_grad(concat, c, d_new_c, d_new_h, forget_bias=None):
batches = concat.shape[0]
vec_len = concat.shape[1]/4
assert c.shape[0] == concat.shape[0]
assert c.shape[1] == vec_len
assert c.dtype == concat.dtype
assert d_new_c.tensor_type == c.tensor_type
assert d_new_h.tensor_type == c.tensor_type
pos = ovl.position_in([batches, vec_len])
cur_batch = pos[0]
cur_elem = pos[1]
i = concat[cur_batch, cur_elem]
j = concat[cur_batch, cur_elem + vec_len]
f = concat[cur_batch, cur_elem + 2*vec_len]
o = concat[cur_batch, cur_elem + 3*vec_len]
c_cur = c[cur_batch, cur_elem]
new_c_cur = c_cur*sig(f + forget_bias) + sig(i) * ovl.tanh(j)
d_new_c_cur = d_new_c[cur_batch, cur_elem]
d_new_h_cur = d_new_h[cur_batch, cur_elem]
d_concat = ovl.output_like(concat)
d_c = ovl.output_like(c)
back_ch = d_new_c_cur + tanh_grad(new_c_cur)*sig(o)*d_new_h_cur
d_i = ovl.tanh(j)*sig_grad(i)*back_ch
d_j = sig(i)*tanh_grad(j)*back_ch
d_f = c_cur*sig_grad(f+forget_bias)*back_ch
d_c_cur = sig(f+forget_bias)*back_ch
d_o = ovl.tanh(new_c_cur)*sig_grad(o)*d_new_h_cur
d_concat[cur_batch, cur_elem] = d_i
d_concat[cur_batch, cur_elem+vec_len] = d_j
d_concat[cur_batch, cur_elem+2*vec_len] = d_f
d_concat[cur_batch, cur_elem+3*vec_len] = d_o
d_c[pos] = d_c_cur
return d_concat, d_c
def expm1(x):
"""
Define the expm1 operator by defining the its operator function to be
.. math::
out_{i} = exp(x_{i}) - 1.0
:param x: The input tensor
:return: Element-wise exp(x) - 1
:Examples:
.. doctest::
>>> import numpy as np
>>> from opveclib import evaluate
>>> from opveclib.examples import expm1
>>> a = np.array([1e-10, -1e-10])
>>> evaluate(expm1(a))
array([ 1.00000000e-10, -1.00000000e-10])
>>> np.expm1(a)
array([ 1.00000000e-10, -1.00000000e-10])
>>> ones = np.ones_like(a)
>>> np.exp(a) - ones
array([ 1.00000008e-10, -1.00000008e-10])
"""
output = ovl.output_like(x)
pos = ovl.position_in(x.shape)
e = ovl.exp(x[pos])
# note, this is an example of the use of the OVL conditional operators
with ovl.if_(ovl.logical_and(ovl.isinf(x[pos]), x[pos] > 0.0)):
output[pos] = x[pos]
with ovl.elif_(e == 1.0):
output[pos] = x[pos]
with ovl.elif_((e - 1.0) == -1.0):
output[pos] = -1.0
with ovl.else_():
output[pos] = (e - 1.0) * x[pos] / ovl.log(e)
return output
def expm1(x):
"""
Define the expm1 operator by defining the its operator function to be
.. math::
out_{i} = exp(x_{i}) - 1.0
:param x: The input tensor
:return: Element-wise exp(x) - 1
:Examples:
.. doctest::
>>> import numpy as np
>>> from opveclib import evaluate
>>> from opveclib.examples import expm1
>>> a = np.array([1e-10, -1e-10])
>>> evaluate(expm1(a))
array([ 1.00000000e-10, -1.00000000e-10])
>>> np.expm1(a)
array([ 1.00000000e-10, -1.00000000e-10])
>>> ones = np.ones_like(a)
>>> np.exp(a) - ones
array([ 1.00000008e-10, -1.00000008e-10])
"""
output = ovl.output_like(x)
pos = ovl.position_in(x.shape)
e = ovl.exp(x[pos])
# note, this is an example of the use of the OVL conditional operators
with ovl.if_(ovl.logical_and(ovl.isinf(x[pos]), x[pos] > 0.0)):
output[pos] = x[pos]
with ovl.elif_(e == 1.0):
output[pos] = x[pos]
with ovl.elif_ ((e - 1.0) == -1.0):
output[pos] = -1.0
with ovl.else_():
output[pos] = (e - 1.0) * x[pos] / ovl.log(e)
return output
def log1p(x):
"""
Define the log1p operator by defining the its operator function to be
.. math::
out_{i} = log(1.0 + x_{i})
:param x: The input tensor
:return: Element-wise log(1 + x)
:Examples:
.. doctest::
>>> import numpy as np
>>> from opveclib import evaluate
>>> from opveclib.examples import log1p
>>> a = np.array([1e-99, -1e-99])
>>> evaluate(log1p(a))
array([ 1.00000000e-99, -1.00000000e-99])
>>> np.log1p(a)
array([ 1.00000000e-99, -1.00000000e-99])
>>> ones = np.ones_like(a)
>>> np.log(ones + a)
array([ 0., 0.])
"""
output = ovl.output_like(x)
pos = ovl.position_in(x.shape)
u = 1.0 + x[pos]
d = u - 1.0
# note, this is an example of the use of the OVL conditional operators
with ovl.if_(ovl.logical_and(ovl.isinf(x[pos]), x[pos] > 0.0)):
output[pos] = x[pos]
with ovl.elif_(d == 0):
output[pos] = x[pos]
with ovl.else_():
output[pos] = ovl.log(u) * x[pos] / d
return output
def op(self, input0, input1, input2, input3, input4):
sum = ops.output_like(input0)
pos = ops.position_in(input0.shape)
sum[pos] = input0[pos] + input1[pos] + input2[pos] + input3[pos] + input4[pos]
return sum