import networkx as nx
import matplotlib
import pandas as pd
from networkx import *
import graphlab as gl
import pandas as pd
import pymonetdb
from subprocess import call
import subprocess
import time
import util
log = util.setup_log('mdb_wcc')
class MDB_WCC:
def __init__(self, node_f, edge_f, db, node_tbl, edge_tbl):
self.edge_f = edge_f
self.node_f = node_f
self.db = db
self.edge_tbl = edge_tbl
self.node_tbl = node_tbl
self.steps = -1
self.metrics = None
try:
self.connection = pymonetdb.connect(username="******",
password="******",
hostname="localhost",
database=db,
def train():
logger = util.setup_log("ZERO")
train_set, valid_set = load_data(params)
params["input_size"] = train_set["data"].shape[1]
logger.info("Shape of data: %s.", train_set["data"].shape)
train_bm = BatchManager(train_set, 1, params)
valid_bm = BatchManager(valid_set, 1, params)
model = Model(params, logger)
merge_summary = tf.summary.merge_all()
def mean_absolute_percentage_error(y_true, y_pred):
y_true, y_pred = np.array(y_true), np.array(y_pred)
return np.mean(np.abs((y_true - y_pred) / y_true)) * 100
def metrics(targets, preds, key="train"):
# if key == "val":
# print('mae:', mean_absolute_error(targets, preds))
# print('mape:', mean_absolute_percentage_error(targets, preds))
# print('rmse:', math.sqrt(mean_squared_error(targets, preds)))
return mean_squared_error(targets, preds), ""
def get_time_dif(start_time):
end_time = time.time()
time_dif = end_time - start_time
return timedelta(seconds=int(round(time_dif)))
summary_path = os.path.join(params["train_summary_path"], expname)
_ = os.system("rm -rf %s" % summary_path)
model_path = os.path.join(params["model_path"], expname)
_ = os.system("rm -rf %s" % model_path)
eval_summary_path = os.path.join(params["eval_summary_path"], expname)
_ = os.system("rm -rf %s" % eval_summary_path)
_ = os.system("mkdir -p %s" % eval_summary_path)
train_writer = tf.summary.FileWriter(summary_path, tf.get_default_graph())
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
steps_per_run = params["steps_per_run"]
global_step = 0
best_mse = sys.maxsize
start_time = time.time()
valid_step = 0
for epoch in range(params["train_epoch"]):
train_bm.init()
while True:
global_step, loss, n_steps, (mse, _) = model.train(sess, train_bm, steps_per_run, metrics, merge_summary=merge_summary,
train_writer=train_writer)
logger.info("TRAIN %d steps[%d]: loss %.4f mse %.4f" % (global_step, epoch, loss, mse))
if train_bm.is_finished:
break
valid_step += 1
if valid_step % 10 == 0:
valid_bm.init()
loss, (mse, _) = model.eval(sess, valid_bm, metrics)
if mse < best_mse:
best_mse = mse
model.save(sess, save_path=model_path)
best_flag = '*'
else:
best_flag = ''
time_dif = get_time_dif(start_time)
logger.info("EVALUATION: %d steps: loss %.4f mse %.4f cost_time %s %s" % (global_step, loss, mse, str(time_dif), best_flag))
import networkx as nx
import matplotlib
import pandas as pd
from networkx import *
import graphlab as gl
import pandas as pd
import pymonetdb
import mdb_bfs
import mdb_wcc
import util
log = util.setup_log('mdb_test')
def test_mdb_bfs():
node_f = 'nodes.csv'
edge_f = 'edges.csv'
# test with one connected graph with equal distances
g = util.create_star(10)
util.save_graph(g, node_f, edge_f)
util.load_mdb_graph(node_f, edge_f)
bfs = mdb_bfs.MDB_BFS(node_f,
edge_f,
db='graph_data',
node_tbl='node',
edge_tbl='edge',
src_node=0)
bfs.compute_bfs()
import torch
from torch import nn
from torch.autograd import Variable
from torch import optim
import torch.nn.functional as F
from torchviz import make_dot, make_dot_from_trace
import torch.utils.data as Data
import datetime as dt, itertools, pandas as pd, matplotlib.pyplot as plt, numpy as np
import util
import glob
import os
#os.environ["CUDA_VISIBLE_DEVICES"] = "0，1"
util.setup_log()
use_cuda = torch.cuda.is_available()
# In[2]:
#DATA
X_raw = np.zeros((86400, 11))
dat1 = pd.read_csv('/home/liuchen/wind_data/K111+340_2017-09-30.txt',
header=None)
dat2 = pd.read_csv('/home/liuchen/wind_data/K117+378_2017-09-30.txt',
header=None)
dat3 = pd.read_csv('/home/liuchen/wind_data/K127+995_2017-09-30.txt',
header=None)
dat4 = pd.read_csv('/home/liuchen/wind_data/K135+913_2017-09-30.txt',
header=None)
dat5 = pd.read_csv('/home/liuchen/wind_data/K145+688_2017-09-30.txt',
import networkx as nx
import matplotlib
import pandas as pd
from networkx import *
import graphlab as gl
import pandas as pd
import pymonetdb
import util
log = util.setup_log('mdb_bfs')
class MDB_BFS:
def __init__(self, node_f, edge_f, db, node_tbl, edge_tbl, src_node=0):
self.edge_f = edge_f
self.node_f = node_f
self.db = db
self.edge_tbl = edge_tbl
self.node_tbl = node_tbl
self.steps = -1
self.metrics = None
self.n_updates = []
try:
self.connection = pymonetdb.connect(username="******",
password="******",
hostname="localhost",
database=db,
autocommit=True)
except Exception as e: