def train_rnn(self):
loss, train_op = self._compute_loss()
with tf.name_scope('accuracy'):
predict = tf.transpose(self.predict, [1, 0, 2])
correct_pred = tf.equal(tf.argmax(self.label, 2),
tf.argmax(predict, axis=2))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
dataset = DataLoad('./dataset/kalman/',
time_step=self.rnn_net.time_step,
class_num=self.rnn_net.class_num)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for step in range(1, self.train_iterior + 1):
x, y = dataset.get_batch(self.rnn_net.batch_size)
if step == 1:
feed_dict = {self.rnn_net.input_tensor: x, self.label: y}
else:
feed_dict = {
self.rnn_net.input_tensor: x,
self.label: y,
self.rnn_net.cell_state: state
}
_, compute_loss, state = sess.run(
[train_op, loss, self.rnn_net.cell_state],
feed_dict=feed_dict)
if step % 10 == 0:
compute_accuracy = sess.run(accuracy, feed_dict=feed_dict)
self.train_logger.info(
'train step = %d,loss = %f,accuracy = %f' %
(step, compute_loss, compute_accuracy))
if step % 1000 == 0:
save_path = saver.save(sess, './model/model.ckpt')
self.train_logger.info(
"train step = %d ,model save to =%s" %
(step, save_path))
l2_reg_lambda_m=0.02)
return model, '{}_{}_{}_{}'.format(model_name, hp.DATA,
model.l2_reg_lambda_u,
model.l2_reg_lambda_m)
if __name__ == '__main__':
with tf.Graph().as_default():
session_conf = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
sess = tf.Session(config=session_conf)
with sess.as_default():
data = DataLoad(data_path=hp.DATA_PATH,
fnames=hp.FNAMES,
forced_seq_len=hp.FORCED_SEQ_LEN,
vocab_size=hp.VOCAB_SIZE,
paly_times=hp.PLAY_TIMES,
num_main_actors=hp.NUM_MAIN_ACTORS,
batch_size=hp.BATCH_SIZE,
num_epochs=hp.NUM_EPOCHS)
model, model_name = load_model(data, 'cdmf')
# define graph
model.construct_netword()
global_step = tf.Variable(0, name='global_step', trainable=False)
optimizer = tf.train.AdamOptimizer(learning_rate=hp.LEARNING_RATE)
train_op = optimizer.minimize(model.loss_op,
global_step=global_step)
# tensorboard graph visualizing
graph_writer = tf.summary.FileWriter(
db_locations = db_connection.connect(DB_CREDENTIALS,
DB_LOCATIONS_COLLECTION)
with open(TWITTER_CREDENTIALS, 'r', encoding='utf-8') as f:
[access_key, access_secret, consumer_key,
consumer_secret] = f.read().splitlines()
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
twitter_api = tweepy.API(auth)
print("twitter connection successful\n")
de = DataExtraction(twitter_api)
dl = DataLoad(DB_CREDENTIALS)
for location in db_locations.find():
print("Processing data for ", location["name"])
# extraction: download tweets from Twitter API
data = de.collect(location)
# transformation: process/analyze downloaded tweets
data_processed = data_transformation.analyze(data)
# load: load processed tweets into the DB
dl.load(data_processed)
print("Completed\n")
def create_object_test(self):
db = mock.Mock()
loader = DataLoad(db)
loader.execute()
db.execute.assert_called_once_with('SELECT * FROM users')
2])
# plt.cla()
# plt.plot(x, ax)
# plt.plot(x, ay)
# plt.plot(x, az)
#
# plt.title(title)
# plt.draw()
# plt.pause(0.001)
during = str(time.time() - start_time)
print(sum / num)
print('检测耗时=', during)
if __name__ == '__main__':
net_config = parser_cfg_file('./config/rnn_net.cfg')
time_step = 50
class_num = int(net_config['class_num'])
run = Run_AFD_RNN('./model/', time_step=time_step)
data_load = DataLoad('./dataset/test/FKL/',
time_step=time_step,
class_num=class_num)
test_data, test_label = data_load.get_test_data()
run.draw_flow(test_data, test_label)
run.run_stop()
layer.weights = np.array(layer_data['weights'])
layer.biases = np.array(layer_data['biases'])
def save(self, filename = 'layers_data.txt'):
layers_data = self.get_layers_data()
with open(filename, 'w') as f:
json.dump(layers_data, f)
def load(self, filename = 'layers_data.txt'):
layers_data = json.load(open(filename, 'r'))
self.apply_layers_data(layers_data)
if __name__ == '__main__':
from data_load import DataLoad
data_load = DataLoad()
option = 4
if option == 1: # all full connected network
# 50000, 10000, 10000
training_data, validation_data, test_data = data_load.load_mnist()
fc_layer1 = FullConnectedLayer((28*28, 30))
fc_layer2 = FullConnectedLayer((30, 10))
net = Network([fc_layer1, fc_layer2],
cost = CrossEntropy(Sigmoid),
optimizer = Sgd(0.1),
regularization = RegularL2(0.1))
net.train(training_data, 30, test_data = test_data)
elif option == 2: # CNN
training_data, validation_data, test_data = data_load.load_mnist(shape = ((1,28,28), -1))
fpaths.append(fpath)
return fpaths
if __name__ == '__main__':
graph = tf.Graph()
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=False)
sess = tf.Session(config=session_conf)
with sess.as_default():
data = DataLoad(data_path=hp.DATA_PATH,
fnames=hp.FNAMES,
forced_seq_len=hp.FORCED_SEQ_LEN,
vocab_size=hp.VOCAB_SIZE,
paly_times=hp.PLAY_TIMES,
num_main_actors=hp.NUM_MAIN_ACTORS,
batch_size=hp.BATCH_SIZE,
num_epochs=hp.NUM_EPOCHS,
noise_rate=hp.NOISE_RATE)
for fpath in load_ckpt_paths('cdmf'):
saver = tf.train.import_meta_graph(fpath+'.meta')
saver.restore(sess, fpath)
# Get the placeholders from the graph by name
m_oids = graph.get_tensor_by_name('movie_order_ids:0')
info = graph.get_tensor_by_name('info:0')
actors = graph.get_tensor_by_name('actors:0')
descriptions = graph.get_tensor_by_name('descriptions:0')
u_oids = graph.get_tensor_by_name('user_order_ids:0')
def train(FLAGS):
# training
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
# loading training data
data = DataLoad(data_path=FLAGS.train_data_path,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.num_epochs,
dev_sample_rate=FLAGS.dev_sample_rate)
train_batches = data.train_batch_iter()
# loading devolopment data
dev_x, dev_y = data.get_dev_data()
# declare model
model = TextCNN(forced_seq_len=data.forced_seq_len,
vocab_size=data.vocab_size,
embedding_size=FLAGS.embedding_size,
filters_size_list=FLAGS.filters_size_list,
num_classes=FLAGS.num_classes,
num_filters=FLAGS.num_filters)
# define training procedure
global_step = tf.Variable(0, name='global_step', trainable=False)
optimizer = tf.train.AdamOptimizer(
learning_rate=FLAGS.learning_rate)
train_op = optimizer.minimize(loss=model.loss_op,
global_step=global_step)
# tensorboard graph visualizing
graph_writer = tf.summary.FileWriter(logdir='../graph/model/')
graph_writer.add_graph(sess.graph)
# summaries director
summary_dir = '../summaries/'
loss_summary = tf.summary.scalar('loss_summaries', model.loss_op)
acc_summary = tf.summary.scalar('accuracy', model.accuracy_op)
train_summary_dir = os.path.join(summary_dir, 'train')
dev_summary_dir = os.path.join(summary_dir, 'dev')
train_merged_summaries = tf.summary.merge(
[loss_summary, acc_summary])
dev_merged_summaries = tf.summary.merge(
[loss_summary, acc_summary])
train_summaries_writer = tf.summary.FileWriter(
train_summary_dir, sess.graph)
dev_summaries_writer = tf.summary.FileWriter(
dev_summary_dir, sess.graph)
# initialize all variables
sess.run(tf.global_variables_initializer())
# training and eval into dev set
while 1:
batch_x, batch_y = next(train_batches)
_, step, loss, acc, summaries = sess.run(
[
train_op, global_step, model.loss_op,
model.accuracy_op, train_merged_summaries
],
feed_dict={
model.inputs: batch_x,
model.labels: batch_y,
model.dropout_keep_prob: FLAGS.dropout_keep_prob
})
train_summaries_writer.add_summary(summaries, step)
if step % FLAGS.evaluate_every == 0:
loss, acc, summaries = sess.run(
[
model.loss_op, model.accuracy_op,
dev_merged_summaries
],
feed_dict={
model.inputs: dev_x,
model.labels: dev_y,
model.dropout_keep_prob: FLAGS.dropout_keep_prob
})
print('setp:{} | loss:{} | acc:{}'.format(step, loss, acc))
dev_summaries_writer.add_summary(summaries, step)