def lasso_test():
"""Runs sample"""
(A, b, l, partition) = dg.data_gen()
(x, hist) = group_lasso.lasso(A, b, l, partition, 1.0, 1.0)
K = len(hist['objval'])
x = np.arange(K)
# Plot the objective values
fig = plt.figure()
y = hist['objval']
ax = fig.add_subplot(111)
ax.plot(x, y)
plt.xlabel('iter(k)')
plt.ylabel('f(x^k) + g(z^k)')
plt.show()
# Plot norms
fig2 = plt.figure()
bx = fig2.add_subplot(211)
bx.semilogy(x, np.maximum(hist['r_norm'], 1e-8))
bx.semilogy(x, hist['eps_pri'], 'r--')
plt.ylabel('||r||_2')
cx = fig2.add_subplot(212)
cx.semilogy(x, np.maximum(hist['s_norm'], 1e-8))
cx.semilogy(x, hist['eps_dual'], 'r--')
plt.ylabel('||s||_2')
plt.xlabel('iter(k)')
plt.show()
def lasso_test():
"""Runs sample"""
(A, b, l, partition) = dg.data_gen()
(x, hist) = group_lasso.lasso(A, b, l, partition, 1.0, 1.0)
K = len(hist['objval'])
x = np.arange(K)
# Plot the objective values
fig = plt.figure()
y = hist['objval']
ax = fig.add_subplot(111)
ax.plot(x, y)
plt.xlabel('iter(k)')
plt.ylabel('f(x^k) + g(z^k)')
plt.show()
# Plot norms
fig2 = plt.figure()
bx = fig2.add_subplot(211)
bx.semilogy(x, np.maximum(hist['r_norm'], 1e-8))
bx.semilogy(x, hist['eps_pri'], 'r--')
plt.ylabel('||r||_2')
cx = fig2.add_subplot(212)
cx.semilogy(x, np.maximum(hist['s_norm'], 1e-8))
cx.semilogy(x, hist['eps_dual'], 'r--')
plt.ylabel('||s||_2')
plt.xlabel('iter(k)')
plt.show()
def dump_data(n_records):
r = redis.Redis(host='localhost', port=6379)
data = data_gen(n_records)
redis_graph = Graph('file_activity1', r)
nodes = {}
edges = {}
pprint(data)
for rec in data:
_node = Node(label='file',
properties={
'fid': rec['_id'],
'name': rec['name'],
'date_added': rec['date_added'],
'platform': rec['platform']
})
r.set(rec['_id'], _node.alias)
redis_graph.add_node(_node)
nodes[rec['_id']] = _node
for rec in data:
for fileid, time_stamp in rec['downloaded']:
edge = Edge(nodes[rec['_id']],
'DOWNLOADED',
nodes[fileid],
properties={
'time': time_stamp,
'activity': 'downloaded'
})
redis_graph.add_edge(edge)
for fileid, time_stamp in rec['executed']:
edge = Edge(nodes[rec['_id']],
'EXECUTED',
nodes[fileid],
properties={
'time': time_stamp,
'activity': 'executed'
})
redis_graph.add_edge(edge)
for fileid, time_stamp in rec['removed']:
edge = Edge(nodes[rec['_id']],
'REMOVED',
nodes[fileid],
properties={
'time': time_stamp,
'activity': 'removed'
})
redis_graph.add_edge(edge)
redis_graph.commit()
print("Graph created")
def main():
m = 100000
ab = 0
num1 = 1
if ab == 1:
x11 = np.array([[1,2,1],[1,1,20],[1,1,5],[1,4,1],[1,1,40],[1,3,30]])
y1 = np.array([-1,-1,-1,1,1,1])
x22 = np.array([[1,2,0.1],[1,1,2],[1,1,0.5],[1,4,0.1],[1,1,4],[1,3,3]])
if num1 == 1:
w, loss = logistic_gradient(x11,y1,m)
else:
w, loss = logistic_gradient(x22,y1,m)
fig = plt.figure(0)
ax = fig.add_subplot(111)
ax.set_xlabel('x_1')
ax.set_ylabel('x_2')
if num1 == 1:
ax.plot(x11[:,1][:3],x11[:,2][:3],'o')
ax.plot(x11[:,1][3:],x11[:,2][3:],'ro')
plt.axis([0, 6, 0, 50])
else:
ax.plot(x22[:,1][:3],x22[:,2][:3],'o')
ax.plot(x22[:,1][3:],x22[:,2][3:],'ro')
plt.axis([0, 6, 0, 5])
ax.plot([-w[0]/w[1],0],[0,-w[0]/w[2]])
plt.savefig('hw5-3a.pdf')
fig1 = plt.figure(1)
ax1 = fig1.add_subplot(111)
ax1.set_xlabel('t')
ax1.set_ylabel('Loss')
ax1.semilogy(range(m), loss)
plt.savefig('hw5-3b.pdf')
else:
x3, y3 = data_generator.data_gen()
w,loss = logistic_gradient(x3,y3,m)
fig = plt.figure(0)
ax = fig.add_subplot(111)
ax.set_xlabel('x_1')
ax.set_ylabel('x_2')
ax.plot(x3[:,1][:50],x3[:,2][:50],'o')
ax.plot(x3[:,1][50:],x3[:,2][50:],'ro')
ax.plot([-10,-w[0]/w[1],0,10],[(10.0*w[1]-w[0])/w[2],0,-w[0]/w[2],(-10.0*w[1]-w[0])/w[2]])
plt.axis([x3[:,1].min(), x3[:,1].max(), x3[:,2].min(), x3[:,2].max()])
plt.savefig('hw5-3d.pdf')
fig1 = plt.figure(1)
ax1 = fig1.add_subplot(111)
ax1.set_xlabel('t')
ax1.set_ylabel('Loss')
ax1.semilogy(range(m), loss)
plt.savefig('hw5-3dl.pdf')
return
import data_generator
import keras
from keras import backend as K
from keras.models import Sequential
from keras.layers import Activation
from keras.layers.core import Dense
from keras.optimizers import Adam
from keras.metrics import categorical_crossentropy
from keras.models import load_model
import test_set
from pprint import pprint
import os
import sys
# Training data
scaled_train_sampels, train_labels = data_generator.data_gen()
# Testing data
obj = test_set.Testing_Data()
test_sampels = obj.test_data_generator()
def _neural_net_config():
# NEURAL NET
model = Sequential([])
model.add(Dense(16, input_shape=(1, ), activation="relu"))
model.add(Dense(32, activation="relu"))
model.add(Dense(2, activation="softmax"))
#model.summary()
model.compile(Adam(lr=.0001),
default=10,
type=int,
help='Number of epochs to train')
args = parser.parse_args()
df = pd.read_csv(os.path.join(Config.DATA_DIR, Config.CLEANED_TRG_FILE))
# Only take geohashes with at least 600 occurences to train
df = df.groupby('geohash6').filter(lambda x: len(x) > 600)
train_df, val_df = train_test_split(df,
test_size=0.2,
random_state=0,
stratify=df[['geohash6']])
train_df = train_df.reset_index(drop=True, inplace=False)
val_df = val_df.reset_index(drop=True, inplace=False)
train_gen = data_gen(train_df, batch_size=args.batch)
val_gen = data_gen(val_df, batch_size=args.batch)
checkpoint_path = os.path.join(Config.MODEL_LOGS_DIR, Config.WEIGHTS)
checkpoint_dir = os.path.dirname(checkpoint_path)
cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path,
save_weights_only=True,
save_best_only=True,
verbose=1)
base_lr = 0.001
def lr_linear_decay(epoch):
return (base_lr * (1 - (epoch / args.epochs)))
model = traffic_demand_model()
try:
import config
import data_generator
import paths
input_paths, context_paths, label_paths = paths.get_paths(
config.DATA_PATH, config.CONTEXT)
train_paths, valid_paths, test_paths = paths.split_paths(
input_paths, config.RATIO)
train_gen = data_generator.data_gen(
train_paths,
context_paths,
label_paths,
context=config.CONTEXT,
batch_size=config.BATCH_SIZE,
structure_names=config.STRUCTURE_NAMES,
resize=config.GRID_SIZE,
)
valid_gen = data_generator.data_gen(
valid_paths,
context_paths,
label_paths,
context=config.CONTEXT,
batch_size=config.BATCH_SIZE,
structure_names=config.STRUCTURE_NAMES,
resize=config.GRID_SIZE,
)
import tensorflow as tf config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) from tensorflow.contrib import rnn import numpy as np import data_generator as da import time data_train, data_valid, data_test, word2id, id2word, tag2id, id2tag, labels = da.data_gen() ''' For Chinese word segmentation. ''' # ##################### config ###################### decay = 0.85 max_epoch = 5 max_max_epoch = 10 timestep_size = max_len = 32 # 句子长度 vocab_size = 5159 # 样本中不同字的个数+1(padding 0)，根据处理数据的时候得到 input_size = embedding_size = 64 # 字向量长度 class_num = 5 hidden_size = 128 # 隐含层节点数 layer_num = 2 # bi-lstm 层数 max_grad_norm = 5.0 # 最大梯度（超过此值的梯度将被裁剪） lr = tf.placeholder(tf.float32, []) keep_prob = tf.placeholder(tf.float32, []) batch_size = tf.placeholder(tf.int32, []) # 注意类型必须为 tf.int32 model_save_path = 'ckpt/bi-lstm.ckpt' # 模型保存位置
