def random_projection(indices, num_projections, projection_dim):
# Expand input indices to get an index for each vector entry
# Note: proj_indices(i) = index*projection_dim + i
proj_indices = lbann.WeightedSum(
indices,
scaling_factors=utils.str_list(projection_dim),
)
iota = lbann.WeightsLayer(
dims=utils.str_list(projection_dim),
weights=lbann.Weights(
initializer=lbann.ValueInitializer(
values=utils.str_list(range(projection_dim))),
optimizer=lbann.NoOptimizer(),
),
)
proj_indices = lbann.Sum(
lbann.Tessellate(
lbann.Reshape(proj_indices,
dims=utils.str_list([num_projections, 1])),
dims=utils.str_list([num_projections, projection_dim]),
),
lbann.Tessellate(
lbann.Reshape(iota, dims=utils.str_list([1, projection_dim])),
dims=utils.str_list([num_projections, projection_dim]),
),
)
# Apply hash function and convert to Gaussian distribution
proj = lbann.UniformHash(proj_indices)
ones = lbann.Constant(
value=1,
num_neurons=utils.str_list([num_projections, projection_dim]),
)
eps = 0.001
proj = lbann.ErfInv(
lbann.WeightedSum(
proj,
ones,
scaling_factors=utils.str_list([2 * (1 - eps), -(1 - eps)]),
))
proj = lbann.InstanceNorm(proj)
proj = lbann.WeightedSum(
proj,
scaling_factors=utils.str_list(1 / projection_dim),
)
return proj
def solve(n):
s, _, *rules = str_list(14)
rules = parse_rules(rules)
sc = Counter(map(lambda x: ''.join(x), zip(s, s[1:])))
for _ in range(n):
sc = run(sc, rules)
count(sc, s[0], s[1])
def mean_squared_error(
data_dim,
sequence_length,
source_sequence,
target_sequence,
scale_decay=0.8,
):
# Compute inner product between source and target vectors
# Note: Inner products are computed for each (x,y) pair and a
# weighted sum is computed. The scaling factors sum to 1 and decay
# exponentially as x and y get further apart in the sequence.
prods = lbann.MatMul(
source_sequence,
target_sequence,
transpose_b=True,
)
scale_dims = (sequence_length, sequence_length)
scales = np.zeros(scale_dims)
for i in range(sequence_length):
for j in range(sequence_length):
if i != j:
scales[i, j] = ((1 - scale_decay) / (2 * scale_decay) *
scale_decay**np.abs(j - i))
scales = lbann.Weights(
initializer=lbann.ValueInitializer(
values=utils.str_list(np.nditer(scales))),
optimizer=lbann.NoOptimizer(),
)
scales = lbann.WeightsLayer(dims=utils.str_list(scale_dims),
weights=scales)
prods = lbann.MatMul(
lbann.Reshape(prods, dims='1 -1'),
lbann.Reshape(scales, dims='1 -1'),
transpose_b=True,
)
prods = lbann.Reshape(prods, dims='1')
# MSE(x,y) = ( norm(x)^2 + norm(y)^T - 2*prod(x,y) ) / dim(x)
scale = 1 / (data_dim * sequence_length)
return lbann.WeightedSum(lbann.L2Norm2(source_sequence),
lbann.L2Norm2(target_sequence),
prods,
scaling_factors=utils.str_list(
[scale, scale, -2 * scale]))
def positive_samples_loss(
sequence_length,
encoder_embeddings,
decoder_embeddings,
scale_decay=0.8,
):
# Compute similarity scores between encoder and decoder embeddings
scores = lbann.MatMul(
encoder_embeddings,
decoder_embeddings,
transpose_b=True,
)
scores = lbann.LogSigmoid(scores)
# Scale similarity scores and add together
# Note: The scaling factor decays exponentially as embeddings get
# futher apart in the sequence.
# Note: The sum of all the scaling factors is approximately -1.
scale_dims = (sequence_length,sequence_length)
scales = np.zeros(scale_dims)
for i in range(sequence_length):
for j in range(sequence_length):
if i != j:
scales[i,j] = (
-(1-scale_decay)/(2*scale_decay*sequence_length)
* scale_decay**np.abs(j-i)
)
scales = lbann.Weights(
initializer=lbann.ValueInitializer(values=utils.str_list(np.nditer(scales))),
optimizer=lbann.NoOptimizer(),
)
scales = lbann.WeightsLayer(dims=utils.str_list(scale_dims), weights=scales)
loss = lbann.MatMul(
lbann.Reshape(scores, dims='1 -1'),
lbann.Reshape(scales, dims='1 -1'),
transpose_b=True,
)
loss = lbann.Reshape(loss, dims='1')
return loss
def encode(self, x):
x = lbann.Reshape(x, dims=utils.str_list([-1, self.input_dim]))
for i, dim in enumerate(self.hidden_dims):
x = lbann.ChannelwiseFullyConnected(
x,
weights=self.weights[i],
output_channel_dims=dim,
bias=False,
)
x = lbann.Relu(x)
x = lbann.ChannelwiseFullyConnected(
x,
weights=self.weights[-1],
output_channel_dims=self.output_dim,
bias=False,
)
return x
def decode(self, x):
x = lbann.Reshape(x, dims=utils.str_list([-1, self.output_dim]))
for i in range(len(self.hidden_dims)):
x = lbann.ChannelwiseFullyConnected(
x,
weights=self.weights[-i - 1],
output_channel_dims=self.hidden_dims[-i - 1],
transpose=True,
bias=False,
)
x = lbann.Relu(x)
x = lbann.ChannelwiseFullyConnected(
x,
weights=self.weights[0],
output_channel_dims=self.input_dim,
transpose=True,
bias=False,
)
return x
),
name='embeddings',
)
embeddings = lbann.Embedding(
input_,
weights=embeddings_weights,
num_embeddings=num_vertices,
embedding_dim=args.latent_dim,
)
else:
raise RuntimeError(
f'unknown method to get embedding vectors ({args.embeddings})')
embeddings_slice = lbann.Slice(
embeddings,
axis=0,
slice_points=utils.str_list(
[0, num_negative_samples, num_negative_samples + walk_length]),
)
negative_samples_embeddings = lbann.Identity(embeddings_slice)
walk_embeddings = lbann.Identity(embeddings_slice)
# Skip-Gram objective function
positive_loss = model.skip_gram.positive_samples_loss(
walk_length,
lbann.Identity(walk_embeddings),
lbann.Identity(walk_embeddings),
scale_decay=0.8,
)
negative_loss = model.skip_gram.negative_samples_loss(
walk_embeddings,
negative_samples_embeddings,
)
def move_down(g):
o = [l[:] for l in g]
for i, l in enumerate(g):
for j, x in enumerate(l):
if x == 'v':
ll = len(g)
ii = (i + 1) % ll
if g[ii][j] == '.':
o[ii][j] = x
o[i][j] = '.'
return o
def move(g):
r = move_right(g)
r = move_down(r)
return r
g = [list(s) for s in str_list(25)]
n = 0
while True:
n += 1
r = move(g)
if r == g:
break
g = r
print(n)
def get_values(fix, vs):
return [points(l, fix, vs) for l in str_list(10)]
def test_str_list():
assert ['a', 'b/c', 'd'] == utils.str_list(['a', 'b/c', 'd'])
assert ['a/b/c/d'] == utils.str_list('a/b/c/d')
assert ['a', 'b', 'c/d'] == utils.str_list(['a', ['b', 'c/d']])
help='latent space dimensions (default: 128)',
metavar='NUM')
args = parser.parse_args()
# ----------------------------------
# Construct layer graph
# ----------------------------------
# Dataset properties
vocab_size = dataset.corpus.vocab_size
sequence_length = dataset.sample_dims()[0]
# Input is a sequence of token IDs
input_ = lbann.Identity(lbann.Input())
input_slice = lbann.Slice(input_,
slice_points=str_list(range(sequence_length + 1)))
tokens_list = [lbann.Identity(input_slice) for _ in range(sequence_length)]
# Get sequence of embedding vectors
embeddings = lbann.Embedding(input_,
num_embeddings=vocab_size,
embedding_dim=args.latent_dim)
embeddings_slice = lbann.Slice(embeddings,
axis=0,
slice_points=str_list(range(sequence_length +
1)))
embeddings_list = [
lbann.Reshape(embeddings_slice, dims='-1') for _ in range(sequence_length)
]
# Layer modules
def solve(fn):
print(fn(build_map(str_list(12)), 'start'))
),
name='embeddings',
)
embeddings = lbann.Embedding(
input_,
weights=embeddings_weights,
num_embeddings=num_vertices,
embedding_dim=args.latent_dim,
)
else:
raise RuntimeError(
f'unknown method to get embedding vectors ({args.embeddings})')
embeddings_slice = lbann.Slice(
embeddings,
axis=0,
slice_points=utils.str_list([0, num_negative_samples, sample_size]),
)
negative_samples_embeddings = lbann.Identity(embeddings_slice)
walk_embeddings = lbann.Identity(embeddings_slice)
# Skip-Gram objective function
positive_loss = model.skip_gram.positive_samples_loss(
walk_length,
lbann.Identity(walk_embeddings),
lbann.Identity(walk_embeddings),
scale_decay=0.8,
)
negative_loss = model.skip_gram.negative_samples_loss(
walk_embeddings,
negative_samples_embeddings,
)
from utils import str_list
from collections import Counter
g = [Counter(n).most_common(1)[0][0] for n in zip(*str_list(3))]
e = ''.join(['1' if i == '0' else '0' for i in g])
print(int(''.join(g), 2) * int(e, 2))
from utils import str_list
from collections import Counter
l = str_list(3)
def r(ty=0, o=l, i=0):
if len(o) == 1:
return int(o[0], 2)
m = Counter([x for x in zip(*o)][i]).most_common(2)
s = '1' if ty == 0 else '0'
e = s if m[0][1] == m[1][1] else m[ty][0]
return r(ty, [f for f in o if f[i] == e], i + 1)
print(r() * r(1))
