def linear_regression():
x_train = np.asarray([1, 2, 3, 4, 5, 6, 7, 8, 9, 11])
y_train = np.asarray([0.1, 0.2, 0.32, 0.43, 0.54, 0.65, 0.77, 0.88, 0.94, 1])
n_sample = x_train.shape[0]
x_ = tf.placeholder(tf.float32, name="x")
y_ = tf.placeholder(tf.float32, name="y")
w = tf.get_variable("weights", initializer=tf.constant(0.0))
b = tf.get_variable("bias", initializer=tf.constant(0.0))
y_predict = w * x_ + b
loss = tf.square(y_ - y_predict, name='loss')
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001).minimize(loss)
writer = tf.summary.FileWriter("./graphs", tf.get_default_graph())
writer.close()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(100):
total_loss = 0
for x, y in zip(x_train, y_train):
_, _loss = sess.run([optimizer, loss], feed_dict={x_: x, y_: y})
total_loss += _loss
print(f"Epoch {i}: {total_loss / n_sample}")
w_out, b_out = sess.run([w, b])
y_predict = x_train * w_out + b_out
for i, j in zip(y_predict, y_train):
print(f"{i} : {j}")
plt.plot(x_train, y_predict, "r-", label="predict")
plt.plot(x_train, y_train, "go", label="data")
plt.title("ABC")
plt.xlabel("x")
plt.ylabel("y")
plt.show()
def model(hparams, X, past=None, scope='model', reuse=tf.AUTO_REUSE):
with tf.variable_scope(scope, reuse=reuse):
results = {}
batch, sequence = shape_list(X)
wpe = tf.get_variable('wpe', [hparams.n_ctx, hparams.n_embd],
initializer=tf.random_normal_initializer(stddev=0.01))
wte = tf.get_variable('wte', [hparams.n_vocab, hparams.n_embd],
initializer=tf.random_normal_initializer(stddev=0.02))
past_length = 0 if past is None else tf.shape(past)[-2]
h = tf.gather(wte, X) + tf.gather(wpe, positions_for(X, past_length))
# Transformer
presents = []
pasts = tf.unstack(past, axis=1) if past is not None else [None] * hparams.n_layer
assert len(pasts) == hparams.n_layer
for layer, past in enumerate(pasts):
h, present = block(h, 'h%d' % layer, past=past, hparams=hparams)
if layer == 10:
tf.add_to_collection('checkpoints', h)
presents.append(present)
results['present'] = tf.stack(presents, axis=1)
h = norm(h, 'ln_f')
# Language model loss. Do tokens <n predict token n?
h_flat = tf.reshape(h, [batch * sequence, hparams.n_embd])
logits = tf.matmul(h_flat, wte, transpose_b=True)
logits = tf.reshape(logits, [batch, sequence, hparams.n_vocab])
results['logits'] = logits
return results
def conv1d(x, scope, nf, *, w_init_stdev=0.02):
with tf.variable_scope(scope):
*start, nx = shape_list(x)
w = tf.get_variable('w', [1, nx, nf],
initializer=tf.random_normal_initializer(stddev=w_init_stdev))
b = tf.get_variable('b', [nf], initializer=tf.constant_initializer(0))
c = tf.reshape(tf.matmul(tf.reshape(x, [-1, nx]), tf.reshape(w, [-1, nf])) + b,
start + [nf])
return c
def norm(x, scope, *, axis=-1, epsilon=1e-5):
"""Normalize to mean = 0, std = 1, then do a diagonal affine transform."""
with tf.variable_scope(scope):
n_state = x.shape[-1]#.value
g = tf.get_variable('g', [n_state], initializer=tf.constant_initializer(1))
b = tf.get_variable('b', [n_state], initializer=tf.constant_initializer(0))
u = tf.reduce_mean(x, axis=axis, keepdims=True)
s = tf.reduce_mean(tf.square(x - u), axis=axis, keepdims=True)
x = (x - u) * tf.rsqrt(s + epsilon)
x = x * g + b
return x
def build_layer(s, c_names, n_l1, n_l2, w_initializer, b_initializer):
with tf.variable_scope('l1'):
w1 = tf.get_variable('w1', [self.n_features, n_l1],
initializer=w_initializer,
collections=c_names)
b1 = tf.get_variable('b1', [1, n_l1],
initializer=b_initializer,
collections=c_names)
l1 = tf.nn.relu(tf.matmul(s, w1) + b1)
with tf.variable_scope('l2'):
w2 = tf.get_variable('w2', [n_l1, n_l2],
initializer=w_initializer,
collections=c_names)
b2 = tf.get_variable('b2', [1, n_l2],
initializer=b_initializer,
collections=c_names)
l2 = tf.nn.relu(tf.matmul(l1, w2) + b2)
with tf.variable_scope('l3'):
w3 = tf.get_variable('w3', [n_l2, self.n_actions],
initializer=w_initializer,
collections=c_names)
b3 = tf.get_variable('b3', [1, self.n_actions],
initializer=b_initializer,
collections=c_names)
l3 = tf.nn.relu(tf.matmul(l2, w3) + b3)
return l3
def _build_layer(input_dim,
output_dim,
input_data,
c_name,
layer_name,
w_name='w',
bias_name='b',
has_activate=True):
with tf.variable_scope(layer_name):
w = tf.get_variable(w_name, [input_dim, output_dim],
dtype=tf.float32,
initializer=tf.truncated_normal_initializer(),
collections=c_name)
b = tf.get_variable(bias_name, [output_dim],
dtype=tf.float32,
initializer=tf.truncated_normal_initializer(),
collections=c_name)
if has_activate:
l = tf.nn.relu(tf.add(tf.matmul(input_data, w), b))
else:
l = tf.add(tf.matmul(input_data, w), b)
return l